diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f787509..cc79d4a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: go-version-file: go.mod cache: false - - uses: goreleaser/goreleaser-action@ec59f474b9834571250b370d4735c50f8e2d1e29 # v7.0.0 + - uses: goreleaser/goreleaser-action@e24998b8b67b290c2fa8b7c14fcfa7de2c5c9b8c # v7.1.0 with: version: "~> v2" args: release --clean diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 03ea882..4acd19b 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -26,4 +26,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 + uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3 diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b6256de..e5881db 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -36,11 +36,10 @@ checksum: signs: - cmd: cosign - certificate: "${artifact}.pem" + signature: "${artifact}.cosign.bundle" args: - sign-blob - - "--output-certificate=${certificate}" - - "--output-signature=${signature}" + - "--bundle=${signature}" - "${artifact}" - "--yes" artifacts: checksum diff --git a/config.example.yaml b/config.example.yaml index ea17d15..cb4881e 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -29,6 +29,23 @@ storage: # Empty or "0" means unlimited max_size: "" + # Redirect cached artifact downloads to presigned storage URLs (HTTP 302) + # instead of streaming through the proxy. Only effective for S3 and Azure. + # Leave disabled if clients reach the proxy through an authenticating gateway, + # since presigned URLs bypass it. + direct_serve: false + + # How long presigned URLs remain valid (e.g. "5m", "1h"). Default: "15m". + direct_serve_ttl: "15m" + + # Public base URL to substitute into presigned URLs. Set this when the + # proxy reaches storage at an internal address (127.0.0.1, a Docker + # service name) but clients must use a public hostname. Only scheme and + # host are used; the signed path and query are preserved. For S3/MinIO + # the reverse proxy at this address must forward requests with the + # internal Host header or the SigV4 signature will not validate. + # direct_serve_base_url: "https://minio.example.com" + # Database configuration database: # Database driver: "sqlite" (default) or "postgres" @@ -49,6 +66,145 @@ log: # Log format: "text" or "json" format: "text" +# Ecosystem support - routes and upstream repositories +# +# This section is optional, since 'include_default' in each section +# defaults to 'true' and the route map will be populated with all of +# the default routes if no configuration is provided. +ecosystem: + cargo: + include_default: true + # the default route for crates.io + # route: + # - path: /cargo + # upstream: + # - name: crates.io + # index: https://index.crates.io + # crates: https://static.crates.io/crates + composer: + include_default: true + # the default route for packagist.org + # route: + # - path: /composer + # upstream: + # - name: packagist.org + # upstream: https://packagist.org + # repository: https://repo.packagist.org + conan: + include_default: true + # the default route for conan.io + # route: + # - path: /conan + # upstream: + # - name: conan.io + # upstream: https://center.conan.io + conda: + include_default: true + # the default route for anaconda.org + # route: + # - path: /conda + # upstream: + # - name: anaconda.org + # upstream: https://conda.anaconda.org + cran: + include_default: true + # the default route for r-project.org + # route: + # - path: /cran + # upstream: + # - name: r-project.org + # upstream: https://cloud.r-project.org + debian: + include_default: true + # the default route for debian.org + # route: + # - path: /debian + # upstream: + # - name: debian.org + # upstream: http://deb.debian.org/debian + gem: + include_default: true + # the default route for rubygems.org + # route: + # - path: /gem + # upstream: + # - name: rubygems.org + # upstream: https://rubygems.org + go: + include_default: true + # the default route for golang.org + # route: + # - path: /go + # upstream: + # - name: golang.org + # upstream: https://proxy.golang.org + hex: + include_default: true + # the default route for hex.pm + # route: + # - path: /hex + # upstream: + # - name: hex.pm + # upstream: https://repo.hex.pm + maven: + include_default: true + # the default route for maven.org + # route: + # - path: /maven + # upstream: + # - name: maven.org + # upstream: https://repo1.maven.org/maven2 + npm: + include_default: true + # the default route for npmjs.org + # route: + # - path: /npm + # upstream: + # - name: npmjs.org + # upstream: https://registry.npmjs.org + nuget: + include_default: true + # the default route for nuget.org + # route: + # - path: /nuget + # upstream: + # - name: nuget.org + # upstream: https://api.nuget.org + oci: + include_default: true + # the default route for docker.io + # route: + # - path: /v2 + # upstream: + # - name: docker.io + # registry: https://registry-1.docker.io + # auth: https://auth.docker.io + pub: + include_default: true + # the default route for pub.dev + # route: + # - path: /pub + # upstream: + # - name: pub.dev + # upstream: https://pub.dev + pypi: + include_default: true + # the default route for pypi.org + # route: + # - path: /pypi + # upstream: + # - name: pypi.org + # index: https://pypi.org + # files_host: files.pythonhosted.org + rpm: + include_default: true + # the default route for fedoraproject.org + # route: + # - path: /rpm + # upstream: + # - name: fedoraproject.org + # upstream: https://dl.fedoraproject.org/pub/fedora/linux + # Upstream registry URLs and authentication upstream: # npm registry URL diff --git a/go.mod b/go.mod index 7ae8518..2619970 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/git-pkgs/archives v0.2.2 github.com/git-pkgs/enrichment v0.2.2 github.com/git-pkgs/purl v0.1.10 - github.com/git-pkgs/registries v0.4.0 + github.com/git-pkgs/registries v0.4.1 github.com/git-pkgs/spdx v0.1.2 github.com/git-pkgs/vers v0.2.4 github.com/git-pkgs/vulns v0.1.4 @@ -22,12 +22,14 @@ require ( golang.org/x/sync v0.20.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.48.2 + modernc.org/sqlite v1.49.1 ) require ( 4d63.com/gocheckcompilerdirectives v1.3.0 // indirect 4d63.com/gochecknoglobals v0.2.2 // indirect + cloud.google.com/go/auth v0.18.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect codeberg.org/chavacava/garif v0.2.0 // indirect codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect @@ -40,6 +42,13 @@ require ( github.com/Antonboom/errname v1.1.1 // indirect github.com/Antonboom/nilnil v1.1.1 // indirect github.com/Antonboom/testifylint v1.6.4 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/BurntSushi/toml v1.6.0 // indirect github.com/Djarvur/go-err113 v0.1.1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect @@ -138,6 +147,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/godoc-lint/godoc-lint v0.11.2 // indirect github.com/gofrs/flock v0.13.0 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golangci/asciicheck v0.5.0 // indirect github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 // indirect github.com/golangci/go-printf-func-name v0.1.1 // indirect @@ -150,8 +160,10 @@ require ( github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect github.com/google/go-cmp v0.7.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect github.com/google/wire v0.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.12 // indirect github.com/googleapis/gax-go/v2 v2.17.0 // indirect github.com/gordonklaus/ineffassign v0.2.0 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect @@ -174,6 +186,7 @@ require ( github.com/kkHAIKE/contextcheck v1.1.6 // indirect github.com/kulti/thelper v0.7.1 // indirect github.com/kunwardeep/paralleltest v1.0.15 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lasiar/canonicalheader v1.1.2 // indirect github.com/ldez/exptostd v0.4.5 // indirect github.com/ldez/gomoddirectives v0.8.0 // indirect @@ -209,6 +222,7 @@ require ( github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect @@ -277,10 +291,12 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.48.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.51.0 // indirect + golang.org/x/oauth2 v0.35.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/tools v0.42.0 // indirect @@ -291,7 +307,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.7.0 // indirect - modernc.org/libc v1.70.0 // indirect + modernc.org/libc v1.72.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect mvdan.cc/gofumpt v0.9.2 // indirect diff --git a/go.sum b/go.sum index 9bdf6f9..f8028ee 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,26 @@ github.com/Antonboom/nilnil v1.1.1 h1:9Mdr6BYd8WHCDngQnNVV0b554xyisFioEKi30sksuf github.com/Antonboom/nilnil v1.1.1/go.mod h1:yCyAmSw3doopbOWhJlVci+HuyNRuHJKIv6V2oYQa8II= github.com/Antonboom/testifylint v1.6.4 h1:gs9fUEy+egzxkEbq9P4cpcMB6/G0DYdMeiFS87UiqmQ= github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/to v0.4.1 h1:CxNHBqdzTr7rLtdrtb5CMjJcDut+WNGCVv7OmS5+lTc= +github.com/Azure/go-autorest/autorest/to v0.4.1/go.mod h1:EtaofgU4zmtvn1zT2ARsjRFdq9vXx0YWtmElwL+GZ9M= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= @@ -238,8 +258,8 @@ github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6 github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/purl v0.1.10 h1:NMjeF10nzFn3tdQlz6rbmHB+i+YkyrFQxho3e33ePTQ= github.com/git-pkgs/purl v0.1.10/go.mod h1:C5Vp/kyZ/wGckCLexx4wPVfUxEiToRkdsOPh5Z7ig/I= -github.com/git-pkgs/registries v0.4.0 h1:GO7fQ8/jot0ulSQHBdxLSNSX/p8eB3gEXWO+98fmoEo= -github.com/git-pkgs/registries v0.4.0/go.mod h1:49UCPFWQmwNV7rBEr9TrTDWKR7vYxFcxp3VfdkeFbdE= +github.com/git-pkgs/registries v0.4.1 h1:4qlKVNhC/6x6Bt87t3wrGJtF3EFrUpHQt9/zKsa5IvU= +github.com/git-pkgs/registries v0.4.1/go.mod h1:49UCPFWQmwNV7rBEr9TrTDWKR7vYxFcxp3VfdkeFbdE= github.com/git-pkgs/spdx v0.1.2 h1:wHSK+CqFsO5N7yDTPvxDmer5LgNEa7vAsiZhi5Aci0A= github.com/git-pkgs/spdx v0.1.2/go.mod h1:V98MgZapNgYw54/pdGR82d7RU93qzJoybahbpZqTfw8= github.com/git-pkgs/vers v0.2.4 h1:Zr3jR/Xf1i/6cvBaJKPxhCwjzqz7uvYHE0Fhid/GPBk= @@ -305,6 +325,8 @@ github.com/godoc-lint/godoc-lint v0.11.2 h1:Bp0FkJWoSdNsBikdNgIcgtaoo+xz6I/Y9s5W github.com/godoc-lint/godoc-lint v0.11.2/go.mod h1:iVpGdL1JCikNH2gGeAn3Hh+AgN5Gx/I/cxV+91L41jo= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golangci/asciicheck v0.5.0 h1:jczN/BorERZwK8oiFBOGvlGPknhvq0bjnysTj4nUfo0= @@ -396,6 +418,8 @@ github.com/julz/importas v0.2.0 h1:y+MJN/UdL63QbFJHws9BVC5RpA2iq0kpjrFajTGivjQ= github.com/julz/importas v0.2.0/go.mod h1:pThlt589EnCYtMnmhmRYY/qn9lCf/frPOK+WMx3xiJY= github.com/karamaru-alpha/copyloopvar v1.2.2 h1:yfNQvP9YaGQR7VaWLYcfZUlRP2eo2vhExWKxD/fP6q0= github.com/karamaru-alpha/copyloopvar v1.2.2/go.mod h1:oY4rGZqZ879JkJMtX3RRkcXRkmUvH0x35ykgaKgsgJY= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3M= github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= @@ -509,6 +533,8 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea h1:sKwxy1H95npauwu8vtF95vG/syrL0p8fSZo/XlDg5gk= github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea/go.mod h1:1VcHEd3ro4QMoHfiNl/j7Jkln9+KQuorp0PItHMJYNg= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -771,6 +797,7 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -851,10 +878,10 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= -modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= -modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.32.0 h1:hjG66bI/kqIPX1b2yT6fr/jt+QedtP2fqojG2VrFuVw= -modernc.org/ccgo/v4 v4.32.0/go.mod h1:6F08EBCx5uQc38kMGl+0Nm0oWczoo1c7cgpzEry7Uc0= +modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U= +modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8= +modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU= +modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= @@ -863,8 +890,8 @@ modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.70.0 h1:U58NawXqXbgpZ/dcdS9kMshu08aiA6b7gusEusqzNkw= -modernc.org/libc v1.70.0/go.mod h1:OVmxFGP1CI/Z4L3E0Q3Mf1PDE0BucwMkcXjjLntvHJo= +modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= +modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -873,8 +900,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.48.2 h1:5CnW4uP8joZtA0LedVqLbZV5GD7F/0x91AXeSyjoh5c= -modernc.org/sqlite v1.48.2/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig= +modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U= +modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/internal/config/cargo/cargo.go b/internal/config/cargo/cargo.go new file mode 100644 index 0000000..3e9c3cf --- /dev/null +++ b/internal/config/cargo/cargo.go @@ -0,0 +1,78 @@ +package cargo + +import ( + "fmt" + "net/url" +) + +// Config configures routes +type Config struct { + IncludeDefault bool `json:"include_default" yaml:"include_default"` + Route []RouteConfig `json:"route" yaml:"route"` +} + +// RouteConfig configures a route +type RouteConfig struct { + Path string `json:"path" yaml:"path"` + Upstream []UpstreamConfig `json:"upstream" yaml:"upstream"` +} + +// UpstreamConfig configures an upstream (source) +type UpstreamConfig struct { + Name string `json:"name" yaml:"name"` + Index string `json:"index" yaml:"index"` + Crates string `json:"crates" yaml:"crates"` +} + +// RouteDefault is the default route +var RouteDefault = RouteConfig{ + Path: "/cargo", + Upstream: []UpstreamConfig{ + { + Name: "crates.io", + Index: "https://index.crates.io", + Crates: "https://static.crates.io/crates", + }, + }, +} + +func (c *Config) Validate() error { + for _, route := range c.Route { + if err := route.Validate(); err != nil { + return err + } + } + + return nil +} + +func (r *RouteConfig) Validate() error { + // TODO: validate Path + + if len(r.Upstream) == 0 { + return fmt.Errorf("cargo route %q does not have any upstreams", r.Path) + } + if len(r.Upstream) > 1 { + return fmt.Errorf("cargo route %q has multiple upstreams; this is not yet supported", r.Path) + } + + for _, upstream := range r.Upstream { + if err := upstream.Validate(); err != nil { + return err + } + } + + return nil +} + +func (u *UpstreamConfig) Validate() error { + if _, err := url.Parse(u.Index); err != nil { + return fmt.Errorf("cargo upstream index %q is not a valid URL", u.Index) + } + + if _, err := url.Parse(u.Crates); err != nil { + return fmt.Errorf("cargo upstream crates %q is not a valid URL", u.Crates) + } + + return nil +} diff --git a/internal/config/config.go b/internal/config/config.go index ad0acc0..3a47347 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,12 +51,15 @@ import ( "encoding/base64" "encoding/json" "fmt" + "net/url" "os" "path/filepath" "strconv" "strings" "time" + "github.com/git-pkgs/proxy/internal/config/cargo" + "github.com/git-pkgs/proxy/internal/config/debian" "gopkg.in/yaml.v3" ) @@ -82,6 +85,9 @@ type Config struct { // Upstream configures upstream registry URLs (optional overrides). Upstream UpstreamConfig `json:"upstream" yaml:"upstream"` + // Ecosystem configures ecosystem routes and upstreams + Ecosystem EcosystemConfig `json:"ecosystem" yaml:"ecosystem"` + // Cooldown configures version age filtering to mitigate supply chain attacks. Cooldown CooldownConfig `json:"cooldown" yaml:"cooldown"` @@ -133,6 +139,21 @@ type StorageConfig struct { // When exceeded, least recently used artifacts are evicted. // Empty or "0" means unlimited. MaxSize string `json:"max_size" yaml:"max_size"` + + // DirectServe enables redirecting cached artifact downloads to presigned + // storage URLs (HTTP 302) instead of streaming bytes through the proxy. + // Only effective for backends that support URL signing (S3, Azure). + DirectServe bool `json:"direct_serve" yaml:"direct_serve"` + + // DirectServeTTL is how long presigned URLs remain valid. + // Uses Go duration syntax (e.g. "5m", "1h"). Default: "15m". + DirectServeTTL string `json:"direct_serve_ttl" yaml:"direct_serve_ttl"` + + // DirectServeBaseURL overrides the scheme and host of presigned URLs + // before returning them to clients. Useful when the proxy reaches + // storage at an internal address (e.g. 127.0.0.1 or a Docker hostname) + // but clients must use a public one. + DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"` } // DatabaseConfig configures the cache database. @@ -239,6 +260,14 @@ func Default() *Config { Level: "info", Format: "text", }, + Ecosystem: EcosystemConfig{ + Cargo: cargo.Config{ + IncludeDefault: true, + }, + Debian: debian.Config{ + IncludeDefault: true, + }, + }, Upstream: UpstreamConfig{ NPM: "https://registry.npmjs.org", Cargo: "https://index.crates.io", @@ -303,6 +332,15 @@ func (c *Config) LoadFromEnv() { if v := os.Getenv("PROXY_STORAGE_MAX_SIZE"); v != "" { c.Storage.MaxSize = v } + if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE"); v != "" { + c.Storage.DirectServe = envBool(v) + } + if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_TTL"); v != "" { + c.Storage.DirectServeTTL = v + } + if v := os.Getenv("PROXY_STORAGE_DIRECT_SERVE_BASE_URL"); v != "" { + c.Storage.DirectServeBaseURL = v + } if v := os.Getenv("PROXY_DATABASE_DRIVER"); v != "" { c.Database.Driver = v } @@ -322,10 +360,10 @@ func (c *Config) LoadFromEnv() { c.Cooldown.Default = v } if v := os.Getenv("PROXY_CACHE_METADATA"); v != "" { - c.CacheMetadata = v == "true" || v == "1" + c.CacheMetadata = envBool(v) } if v := os.Getenv("PROXY_MIRROR_API"); v != "" { - c.MirrorAPI = v == "true" || v == "1" + c.MirrorAPI = envBool(v) } if v := os.Getenv("PROXY_METADATA_TTL"); v != "" { c.MetadataTTL = v @@ -334,6 +372,14 @@ func (c *Config) LoadFromEnv() { // Validate checks the configuration for errors. func (c *Config) Validate() error { + // finalize the configuration by injecting default routes if requested + if c.Ecosystem.Cargo.IncludeDefault { + c.Ecosystem.Cargo.Route = append(c.Ecosystem.Cargo.Route, cargo.RouteDefault) + } + if c.Ecosystem.Debian.IncludeDefault { + c.Ecosystem.Debian.Route = append(c.Ecosystem.Debian.Route, debian.RouteDefault) + } + if c.Listen == "" { return fmt.Errorf("listen address is required") } @@ -379,6 +425,21 @@ func (c *Config) Validate() error { } } + // Validate direct serve TTL if specified + if c.Storage.DirectServeTTL != "" { + if _, err := time.ParseDuration(c.Storage.DirectServeTTL); err != nil { + return fmt.Errorf("invalid storage.direct_serve_ttl %q: %w", c.Storage.DirectServeTTL, err) + } + } + + // Validate direct serve base URL if specified + if c.Storage.DirectServeBaseURL != "" { + u, err := url.Parse(c.Storage.DirectServeBaseURL) + if err != nil || u.Scheme == "" || u.Host == "" { + return fmt.Errorf("invalid storage.direct_serve_base_url %q: must be an absolute URL", c.Storage.DirectServeBaseURL) + } + } + // Validate metadata TTL if specified if c.MetadataTTL != "" && c.MetadataTTL != "0" { if _, err := time.ParseDuration(c.MetadataTTL); err != nil { @@ -386,10 +447,33 @@ func (c *Config) Validate() error { } } + if err := c.Ecosystem.Cargo.Validate(); err != nil { + return err + } + if err := c.Ecosystem.Debian.Validate(); err != nil { + return err + } + return nil } -const defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default +const ( + defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default + defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default +) + +// ParseMaxSize returns the maximum cache size in bytes. +// Returns 0 if unset or explicitly disabled (meaning unlimited). +func (c *Config) ParseMaxSize() int64 { + if c.Storage.MaxSize == "" || c.Storage.MaxSize == "0" { + return 0 + } + size, err := ParseSize(c.Storage.MaxSize) + if err != nil { + return 0 + } + return size +} // ParseMetadataTTL returns the metadata TTL duration. // Returns 5 minutes if unset, 0 if explicitly disabled. @@ -407,6 +491,19 @@ func (c *Config) ParseMetadataTTL() time.Duration { return d } +// ParseDirectServeTTL returns the presigned URL expiry duration. +// Returns 15 minutes if unset. +func (c *Config) ParseDirectServeTTL() time.Duration { + if c.Storage.DirectServeTTL == "" { + return defaultDirectServeTTL + } + d, err := time.ParseDuration(c.Storage.DirectServeTTL) + if err != nil { + return defaultDirectServeTTL + } + return d +} + // ParseSize parses a human-readable size string (e.g., "10GB", "500MB"). // Returns the size in bytes. func ParseSize(s string) (int64, error) { @@ -487,3 +584,7 @@ func (a *AuthConfig) Header() (name, value string) { func expandEnv(s string) string { return os.Expand(s, os.Getenv) } + +func envBool(v string) bool { + return v == "true" || v == "1" +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6e8c3a0..de10191 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -303,6 +303,31 @@ func TestLoadFileNotFound(t *testing.T) { } } +func TestParseMaxSize(t *testing.T) { + tests := []struct { + name string + maxSize string + want int64 + }{ + {"empty means unlimited", "", 0}, + {"zero means unlimited", "0", 0}, + {"10GB", "10GB", 10 * 1024 * 1024 * 1024}, + {"500MB", "500MB", 500 * 1024 * 1024}, + {"invalid returns 0", "invalid", 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.Storage.MaxSize = tt.maxSize + got := cfg.ParseMaxSize() + if got != tt.want { + t.Errorf("ParseMaxSize() = %d, want %d", got, tt.want) + } + }) + } +} + func TestParseMetadataTTL(t *testing.T) { tests := []struct { name string @@ -355,3 +380,77 @@ func TestLoadMetadataTTLFromEnv(t *testing.T) { t.Errorf("MetadataTTL = %q, want %q", cfg.MetadataTTL, "10m") } } + +func TestParseDirectServeTTL(t *testing.T) { + tests := []struct { + name string + ttl string + want time.Duration + }{ + {"empty defaults to 15m", "", 15 * time.Minute}, + {"5 minutes", "5m", 5 * time.Minute}, + {"1 hour", "1h", 1 * time.Hour}, + {"invalid defaults to 15m", "not-a-duration", 15 * time.Minute}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.Storage.DirectServeTTL = tt.ttl + got := cfg.ParseDirectServeTTL() + if got != tt.want { + t.Errorf("ParseDirectServeTTL() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestValidateDirectServeTTL(t *testing.T) { + cfg := Default() + cfg.Storage.DirectServeTTL = "invalid" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for invalid storage.direct_serve_ttl") + } + + cfg.Storage.DirectServeTTL = "5m" + if err := cfg.Validate(); err != nil { + t.Errorf("unexpected error for valid storage.direct_serve_ttl: %v", err) + } +} + +func TestLoadDirectServeFromEnv(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_STORAGE_DIRECT_SERVE", "true") + t.Setenv("PROXY_STORAGE_DIRECT_SERVE_TTL", "30m") + t.Setenv("PROXY_STORAGE_DIRECT_SERVE_BASE_URL", "https://cdn.example.com") + cfg.LoadFromEnv() + + if !cfg.Storage.DirectServe { + t.Error("Storage.DirectServe should be true") + } + if cfg.Storage.DirectServeTTL != "30m" { + t.Errorf("Storage.DirectServeTTL = %q, want %q", cfg.Storage.DirectServeTTL, "30m") + } + if cfg.Storage.DirectServeBaseURL != "https://cdn.example.com" { + t.Errorf("Storage.DirectServeBaseURL = %q, want %q", cfg.Storage.DirectServeBaseURL, "https://cdn.example.com") + } +} + +func TestValidateDirectServeBaseURL(t *testing.T) { + cfg := Default() + + cfg.Storage.DirectServeBaseURL = "not a url" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for relative direct_serve_base_url") + } + + cfg.Storage.DirectServeBaseURL = "://bad" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for unparseable direct_serve_base_url") + } + + cfg.Storage.DirectServeBaseURL = "https://cdn.example.com" + if err := cfg.Validate(); err != nil { + t.Errorf("unexpected error for valid direct_serve_base_url: %v", err) + } +} diff --git a/internal/config/debian/debian.go b/internal/config/debian/debian.go new file mode 100644 index 0000000..06c5f97 --- /dev/null +++ b/internal/config/debian/debian.go @@ -0,0 +1,72 @@ +package debian + +import ( + "fmt" + "net/url" +) + +// Config configures routes +type Config struct { + IncludeDefault bool `json:"include_default" yaml:"include_default"` + Route []RouteConfig `json:"route" yaml:"route"` +} + +// RouteConfig configures a route +type RouteConfig struct { + Path string `json:"path" yaml:"path"` + Upstream []UpstreamConfig `json:"upstream" yaml:"upstream"` +} + +// UpstreamConfig configures an upstream (source) +type UpstreamConfig struct { + Name string `json:"name" yaml:"name"` + Upstream string `json:"upstream" yaml:"upstream"` +} + +// RouteDefault is the default route +var RouteDefault = RouteConfig{ + Path: "/debian", + Upstream: []UpstreamConfig{ + { + Name: "debian.org", + Upstream: "http://deb.debian.org/debian", + }, + }, +} + +func (c *Config) Validate() error { + for _, route := range c.Route { + if err := route.Validate(); err != nil { + return err + } + } + + return nil +} + +func (r *RouteConfig) Validate() error { + // TODO: validate Path + + if len(r.Upstream) == 0 { + return fmt.Errorf("debian route %q does not have any upstreams", r.Path) + } + if len(r.Upstream) > 1 { + return fmt.Errorf("debian route %q has multiple upstreams; this is not yet supported", r.Path) + } + + for _, upstream := range r.Upstream { + if err := upstream.Validate(); err != nil { + return err + } + } + + return nil +} + +func (u *UpstreamConfig) Validate() error { + if _, err := url.Parse(u.Upstream); err != nil { + return fmt.Errorf("debian upstream upstream %q is not a valid URL", u.Upstream) + } + + return nil +} diff --git a/internal/config/ecosystem.go b/internal/config/ecosystem.go new file mode 100644 index 0000000..bc0bc62 --- /dev/null +++ b/internal/config/ecosystem.go @@ -0,0 +1,12 @@ +package config + +import ( + "github.com/git-pkgs/proxy/internal/config/cargo" + "github.com/git-pkgs/proxy/internal/config/debian" +) + +// Ecosystem configuration (routes and upstreams) +type EcosystemConfig struct { + Cargo cargo.Config `json:"cargo" yaml:"cargo"` + Debian debian.Config `json:"debian" yaml:"debian"` +} diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 5d7810c..4c6f195 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -9,13 +9,11 @@ import ( "strings" "time" + "github.com/git-pkgs/proxy/internal/config/cargo" "github.com/git-pkgs/purl" ) const ( - cargoUpstream = "https://index.crates.io" - cargoDownloadBase = "https://static.crates.io/crates" - cargoIndexLen1 = 1 cargoIndexLen2 = 2 cargoIndexLen3 = 3 @@ -24,21 +22,27 @@ const ( // CargoHandler handles cargo registry protocol requests. type CargoHandler struct { proxy *Proxy + path string indexURL string downloadURL string proxyURL string } // NewCargoHandler creates a new cargo protocol handler. -func NewCargoHandler(proxy *Proxy, proxyURL string) *CargoHandler { +func NewCargoHandler(proxy *Proxy, proxyURL string, cfg cargo.RouteConfig) *CargoHandler { return &CargoHandler{ proxy: proxy, - indexURL: cargoUpstream, - downloadURL: cargoDownloadBase, + path: cfg.Path, + indexURL: cfg.Upstream[0].Index, + downloadURL: cfg.Upstream[0].Crates, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } +func (h *CargoHandler) Path() string { + return h.path +} + // Routes returns the HTTP handler for cargo requests. // Mount this at /cargo on your router. func (h *CargoHandler) Routes() http.Handler { @@ -71,7 +75,7 @@ type CargoConfig struct { // handleConfig returns the registry configuration. func (h *CargoHandler) handleConfig(w http.ResponseWriter, r *http.Request) { config := CargoConfig{ - DL: h.proxyURL + "/cargo/crates/{crate}/{version}/download", + DL: h.proxyURL + h.path + "/crates/{crate}/{version}/download", } w.Header().Set("Content-Type", "application/json") diff --git a/internal/handler/cargo_test.go b/internal/handler/cargo_test.go index 5ce81b6..9f76a9c 100644 --- a/internal/handler/cargo_test.go +++ b/internal/handler/cargo_test.go @@ -48,6 +48,7 @@ func TestCargoBuildIndexPath(t *testing.T) { func TestCargoConfigEndpoint(t *testing.T) { h := &CargoHandler{ proxyURL: "http://localhost:8080", + path: "/xyzzy", } req := httptest.NewRequest(http.MethodGet, "/config.json", nil) @@ -64,7 +65,7 @@ func TestCargoConfigEndpoint(t *testing.T) { t.Fatalf("failed to parse config: %v", err) } - expectedDL := "http://localhost:8080/cargo/crates/{crate}/{version}/download" + expectedDL := "http://localhost:8080/xyzzy/crates/{crate}/{version}/download" if config.DL != expectedDL { t.Errorf("DL = %q, want %q", config.DL, expectedDL) } diff --git a/internal/handler/debian.go b/internal/handler/debian.go index b767f6d..8b313dd 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -2,33 +2,39 @@ package handler import ( "fmt" + "github.com/git-pkgs/proxy/internal/config/debian" "net/http" "regexp" "strings" ) const ( - debianUpstream = "http://deb.debian.org/debian" - debMatchCount = 4 // full match + name + version + arch + debMatchCount = 4 // full match + name + version + arch ) // DebianHandler handles APT/Debian repository protocol requests. // It proxies requests to upstream Debian/Ubuntu repositories and caches .deb packages. type DebianHandler struct { proxy *Proxy + path string upstreamURL string proxyURL string } // NewDebianHandler creates a new Debian/APT protocol handler. -func NewDebianHandler(proxy *Proxy, proxyURL string) *DebianHandler { +func NewDebianHandler(proxy *Proxy, proxyURL string, cfg debian.RouteConfig) *DebianHandler { return &DebianHandler{ proxy: proxy, - upstreamURL: debianUpstream, + path: cfg.Path, + upstreamURL: cfg.Upstream[0].Upstream, proxyURL: strings.TrimSuffix(proxyURL, "/"), } } +func (h *DebianHandler) Path() string { + return h.path +} + // Routes returns the HTTP handler for Debian requests. // Mount this at /debian on your router. func (h *DebianHandler) Routes() http.Handler { diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index dfdd326..6a26573 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -2,6 +2,7 @@ package handler import ( "testing" + "github.com/git-pkgs/proxy/internal/config/debian" ) func TestDebianHandler_parsePoolPath(t *testing.T) { @@ -18,6 +19,6 @@ func TestDebianHandler_parsePoolPath(t *testing.T) { } func TestDebianHandler_Routes(t *testing.T) { - h := NewDebianHandler(nil, "http://localhost:8080") + h := NewDebianHandler(nil, "http://localhost:8080", debian.RouteDefault) assertRoutesBasics(t, h.Routes(), "/dists/stable/Release", "/pool/../../../etc/passwd") } diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 639e976..620b022 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/git-pkgs/proxy/internal/config/debian" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/storage" "github.com/git-pkgs/purl" @@ -897,7 +898,7 @@ func TestDebianHandler_DownloadCacheMiss(t *testing.T) { ContentType: "application/vnd.debian.binary-package", } - h := NewDebianHandler(proxy, "http://localhost") + h := NewDebianHandler(proxy, "http://localhost", debian.RouteDefault) srv := httptest.NewServer(h.Routes()) defer srv.Close() diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d7d79c9..746b9e8 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -10,6 +10,7 @@ import ( "io" "log/slog" "net/http" + "net/url" "strconv" "strings" "time" @@ -61,15 +62,21 @@ func ReadMetadata(r io.Reader) ([]byte, error) { // Proxy provides shared functionality for protocol handlers. type Proxy struct { - DB *database.DB - Storage storage.Storage - Fetcher fetch.FetcherInterface - Resolver *fetch.Resolver - Logger *slog.Logger - Cooldown *cooldown.Config - CacheMetadata bool - MetadataTTL time.Duration - HTTPClient *http.Client + DB *database.DB + Storage storage.Storage + Fetcher fetch.FetcherInterface + Resolver *fetch.Resolver + Logger *slog.Logger + Cooldown *cooldown.Config + CacheMetadata bool + MetadataTTL time.Duration + DirectServe bool + DirectServeTTL time.Duration + // DirectServeBaseURL, if set, replaces the scheme and host of presigned + // URLs so clients receive a public address even when the proxy reaches + // storage at an internal one. + DirectServeBaseURL string + HTTPClient *http.Client } // NewProxy creates a new Proxy with the given dependencies. @@ -92,6 +99,7 @@ func NewProxy(db *database.DB, store storage.Storage, fetcher fetch.FetcherInter // CacheResult contains information about a cached or fetched artifact. type CacheResult struct { Reader io.ReadCloser + RedirectURL string Size int64 ContentType string Hash string @@ -138,6 +146,26 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s return nil, nil } + result := &CacheResult{ + Size: artifact.Size.Int64, + ContentType: artifact.ContentType.String, + Hash: artifact.ContentHash.String, + Cached: true, + } + + if p.DirectServe { + signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath.String, p.DirectServeTTL) + if err == nil { + result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL) + p.recordCacheHit(pkgPURL, versionPURL, filename) + return result, nil + } + if !errors.Is(err, storage.ErrSignedURLUnsupported) { + p.Logger.Warn("failed to sign storage URL, falling back to streaming", + "path", artifact.StoragePath.String, "error", err) + } + } + start := time.Now() reader, err := p.Storage.Open(ctx, artifact.StoragePath.String) metrics.RecordStorageOperation("read", time.Since(start)) @@ -148,20 +176,36 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s return nil, nil } - _ = p.DB.RecordArtifactHit(versionPURL, filename) + result.Reader = reader + p.recordCacheHit(pkgPURL, versionPURL, filename) + return result, nil +} - // Extract ecosystem from pkgPURL for metrics - if p, err := purl.Parse(pkgPURL); err == nil { - metrics.RecordCacheHit(purl.PURLTypeToEcosystem(p.Type)) +// rewriteSignedURLHost replaces the scheme and host of a signed URL with those +// from baseURL, preserving the path and query (which carry the signature). +// Returns signed unchanged if baseURL is empty or either URL fails to parse. +func rewriteSignedURLHost(signed, baseURL string) string { + if baseURL == "" { + return signed } + s, err := url.Parse(signed) + if err != nil { + return signed + } + b, err := url.Parse(baseURL) + if err != nil || b.Scheme == "" || b.Host == "" { + return signed + } + s.Scheme = b.Scheme + s.Host = b.Host + return s.String() +} - return &CacheResult{ - Reader: reader, - Size: artifact.Size.Int64, - ContentType: artifact.ContentType.String, - Hash: artifact.ContentHash.String, - Cached: true, - }, nil +func (p *Proxy) recordCacheHit(pkgPURL, versionPURL, filename string) { + _ = p.DB.RecordArtifactHit(versionPURL, filename) + if parsed, err := purl.Parse(pkgPURL); err == nil { + metrics.RecordCacheHit(purl.PURLTypeToEcosystem(parsed.Type)) + } } func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string) (*CacheResult, error) { @@ -276,6 +320,15 @@ func (p *Proxy) updateCacheDB(ecosystem, name, filename, pkgPURL, versionPURL, u // ServeArtifact writes a CacheResult to an HTTP response. func ServeArtifact(w http.ResponseWriter, result *CacheResult) { + if result.RedirectURL != "" { + if result.Hash != "" { + w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.Hash)) + } + w.Header().Set("Location", result.RedirectURL) + w.WriteHeader(http.StatusFound) + return + } + defer func() { _ = result.Reader.Close() }() if result.ContentType != "" { diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 78ed415..3a1d2ab 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -21,9 +21,11 @@ import ( // mockStorage implements storage.Storage for testing. type mockStorage struct { - files map[string][]byte - storeErr error - openErr error + files map[string][]byte + storeErr error + openErr error + signedURL string + signErr error } func newMockStorage() *mockStorage { @@ -79,6 +81,16 @@ func (s *mockStorage) UsedSpace(_ context.Context) (int64, error) { return total, nil } +func (s *mockStorage) SignedURL(_ context.Context, _ string, _ time.Duration) (string, error) { + if s.signErr != nil { + return "", s.signErr + } + if s.signedURL == "" { + return "", storage.ErrSignedURLUnsupported + } + return s.signedURL, nil +} + func (s *mockStorage) URL() string { return "mem://" } func (s *mockStorage) Close() error { return nil } @@ -311,6 +323,213 @@ func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) { } } +func TestGetOrFetchArtifact_DirectServe_Redirect(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") + + proxy.DirectServe = true + proxy.DirectServeTTL = 15 * time.Minute + store.signedURL = "https://bucket.s3.amazonaws.com/npm/lodash?X-Amz-Signature=abc" + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !result.Cached { + t.Error("expected result to be cached") + } + if result.RedirectURL != store.signedURL { + t.Errorf("RedirectURL = %q, want %q", result.RedirectURL, store.signedURL) + } + if result.Reader != nil { + t.Error("Reader should be nil when redirecting") + } + if fetcher.fetchCalled { + t.Error("fetcher should not be called on cache hit") + } + + // Hit count should still be recorded on the redirect path. + art, _ := db.GetArtifact("pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz") + if art == nil || art.HitCount != 1 { + t.Errorf("artifact hit count not recorded: %+v", art) + } +} + +func TestGetOrFetchArtifact_DirectServe_BaseURLRewrite(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") + + proxy.DirectServe = true + proxy.DirectServeBaseURL = "https://cdn.example.com" + store.signedURL = "http://127.0.0.1:9000/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900" + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + want := "https://cdn.example.com/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900" + if result.RedirectURL != want { + t.Errorf("RedirectURL = %q, want %q", result.RedirectURL, want) + } +} + +func TestRewriteSignedURLHost(t *testing.T) { + tests := []struct { + name string + signed string + baseURL string + want string + }{ + { + "empty base url is no-op", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "", + "http://127.0.0.1:9000/bucket/key?sig=abc", + }, + { + "replaces scheme and host", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "https://cdn.example.com", + "https://cdn.example.com/bucket/key?sig=abc", + }, + { + "preserves path and query", + "http://minio:9000/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101", + "https://files.example.com", + "https://files.example.com/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101", + }, + { + "ignores base url path", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "https://cdn.example.com/ignored", + "https://cdn.example.com/bucket/key?sig=abc", + }, + { + "invalid base url is no-op", + "http://127.0.0.1:9000/bucket/key?sig=abc", + "://bad", + "http://127.0.0.1:9000/bucket/key?sig=abc", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := rewriteSignedURLHost(tt.signed, tt.baseURL) + if got != tt.want { + t.Errorf("rewriteSignedURLHost(%q, %q) = %q, want %q", tt.signed, tt.baseURL, got, tt.want) + } + }) + } +} + +func TestGetOrFetchArtifact_DirectServe_FallbackOnUnsupported(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") + + proxy.DirectServe = true + // store.signedURL is empty so SignedURL returns ErrSignedURLUnsupported. + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + if result.RedirectURL != "" { + t.Errorf("RedirectURL should be empty, got %q", result.RedirectURL) + } + if result.Reader == nil { + t.Fatal("Reader should be set when signing is unsupported") + } + body, _ := io.ReadAll(result.Reader) + if string(body) != "cached content" { + t.Errorf("got body %q, want %q", body, "cached content") + } +} + +func TestGetOrFetchArtifact_DirectServe_FallbackOnError(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") + + proxy.DirectServe = true + store.signErr = errors.New("signing failed") + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + if result.RedirectURL != "" { + t.Errorf("RedirectURL should be empty on signing error, got %q", result.RedirectURL) + } + if result.Reader == nil { + t.Fatal("Reader should be set when signing fails") + } +} + +func TestGetOrFetchArtifact_DirectServe_DisabledIgnoresSigning(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content") + + proxy.DirectServe = false + store.signedURL = "https://bucket.example/should-not-be-used" + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + if result.RedirectURL != "" { + t.Errorf("RedirectURL should be empty when DirectServe is off, got %q", result.RedirectURL) + } +} + +func TestServeArtifact_Redirect(t *testing.T) { + w := httptest.NewRecorder() + ServeArtifact(w, &CacheResult{ + RedirectURL: "https://bucket.s3.amazonaws.com/file?sig=abc", + Hash: "abc123", + Cached: true, + }) + + if w.Code != http.StatusFound { + t.Errorf("status = %d, want %d", w.Code, http.StatusFound) + } + if loc := w.Header().Get("Location"); loc != "https://bucket.s3.amazonaws.com/file?sig=abc" { + t.Errorf("Location = %q", loc) + } + if etag := w.Header().Get("ETag"); etag != `"abc123"` { + t.Errorf("ETag = %q, want %q", etag, `"abc123"`) + } + if cl := w.Header().Get("Content-Length"); cl != "" { + t.Errorf("Content-Length should not be set on redirect, got %q", cl) + } +} + +func TestServeArtifact_Stream(t *testing.T) { + w := httptest.NewRecorder() + ServeArtifact(w, &CacheResult{ + Reader: io.NopCloser(strings.NewReader("payload")), + Size: 7, + ContentType: "application/octet-stream", + Hash: "abc123", + }) + + if w.Code != http.StatusOK { + t.Errorf("status = %d, want %d", w.Code, http.StatusOK) + } + if w.Body.String() != "payload" { + t.Errorf("body = %q, want %q", w.Body.String(), "payload") + } + if ct := w.Header().Get("Content-Type"); ct != "application/octet-stream" { + t.Errorf("Content-Type = %q", ct) + } +} + func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) seedPackage(t, db, store, "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "pypi content") diff --git a/internal/server/eviction.go b/internal/server/eviction.go new file mode 100644 index 0000000..4173bd5 --- /dev/null +++ b/internal/server/eviction.go @@ -0,0 +1,105 @@ +package server + +import ( + "context" + "log/slog" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" +) + +const ( + evictionInterval = 1 * time.Minute + evictionBatch = 50 +) + +func (s *Server) startEvictionLoop(ctx context.Context) { + maxSize := s.cfg.ParseMaxSize() + if maxSize <= 0 { + return + } + + s.logger.Info("cache eviction enabled", "max_size", s.cfg.Storage.MaxSize) + + ticker := time.NewTicker(evictionInterval) + defer ticker.Stop() + + s.runEviction(ctx, maxSize) + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.runEviction(ctx, maxSize) + } + } +} + +func (s *Server) runEviction(ctx context.Context, maxSize int64) { + evictLRU(ctx, s.db, s.storage, s.logger, maxSize) +} + +func evictLRU(ctx context.Context, db *database.DB, store storage.Storage, logger *slog.Logger, maxSize int64) { + totalSize, err := db.GetTotalCacheSize() + if err != nil { + logger.Warn("eviction: failed to get cache size", "error", err) + return + } + + if totalSize <= maxSize { + return + } + + logger.Info("eviction: cache size exceeds limit, evicting", + "current_size", totalSize, "max_size", maxSize) + + evicted := 0 + freedBytes := int64(0) + + for totalSize-freedBytes > maxSize { + artifacts, err := db.GetLeastRecentlyUsedArtifacts(evictionBatch) + if err != nil { + logger.Warn("eviction: failed to get LRU artifacts", "error", err) + return + } + if len(artifacts) == 0 { + break + } + + for _, art := range artifacts { + if totalSize-freedBytes <= maxSize { + break + } + + if !art.StoragePath.Valid { + continue + } + + if err := store.Delete(ctx, art.StoragePath.String); err != nil { + logger.Warn("eviction: failed to delete from storage", + "path", art.StoragePath.String, "error", err) + continue + } + + if err := db.ClearArtifactCache(art.VersionPURL, art.Filename); err != nil { + logger.Warn("eviction: failed to clear artifact record", + "version_purl", art.VersionPURL, "filename", art.Filename, "error", err) + continue + } + + size := int64(0) + if art.Size.Valid { + size = art.Size.Int64 + } + freedBytes += size + evicted++ + } + } + + if evicted > 0 { + logger.Info("eviction: completed", + "evicted", evicted, "freed_bytes", freedBytes) + } +} diff --git a/internal/server/eviction_test.go b/internal/server/eviction_test.go new file mode 100644 index 0000000..9fa9e6b --- /dev/null +++ b/internal/server/eviction_test.go @@ -0,0 +1,290 @@ +package server + +import ( + "context" + "database/sql" + "io" + "log/slog" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" +) + +func setupEvictionTest(t *testing.T) (*database.DB, *storage.Filesystem) { + t.Helper() + + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "test.db") + storagePath := filepath.Join(tempDir, "artifacts") + + db, err := database.Create(dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + + store, err := storage.NewFilesystem(storagePath) + if err != nil { + _ = db.Close() + t.Fatalf("failed to create storage: %v", err) + } + + t.Cleanup(func() { + _ = db.Close() + }) + + return db, store +} + +func seedArtifact(t *testing.T, ctx context.Context, db *database.DB, store storage.Storage, name string, dataSize int, accessedAt time.Time) { + t.Helper() + + pkgPURL := "pkg:npm/" + name + versionPURL := pkgPURL + "@1.0.0" + filename := name + "-1.0.0.tgz" + + if err := db.UpsertPackage(&database.Package{ + PURL: pkgPURL, + Ecosystem: "npm", + Name: name, + }); err != nil { + t.Fatalf("failed to upsert package: %v", err) + } + + if err := db.UpsertVersion(&database.Version{ + PURL: versionPURL, + PackagePURL: pkgPURL, + }); err != nil { + t.Fatalf("failed to upsert version: %v", err) + } + + storagePath := storage.ArtifactPath("npm", "", name, "1.0.0", filename) + data := strings.NewReader(strings.Repeat("x", dataSize)) + size, hash, err := store.Store(ctx, storagePath, data) + if err != nil { + t.Fatalf("failed to store artifact: %v", err) + } + + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: filename, + UpstreamURL: "https://example.com/" + filename, + StoragePath: sql.NullString{String: storagePath, Valid: true}, + ContentHash: sql.NullString{String: hash, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + ContentType: sql.NullString{String: "application/gzip", Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + LastAccessedAt: sql.NullTime{Time: accessedAt, Valid: true}, + }); err != nil { + t.Fatalf("failed to upsert artifact: %v", err) + } +} + +func TestEvictLRU_NoEvictionWhenUnderLimit(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + seedArtifact(t, ctx, db, store, "pkg-a", 100, time.Now()) + + evictLRU(ctx, db, store, logger, 1024) + + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 1 { + t.Errorf("expected 1 cached artifact, got %d", count) + } +} + +func TestEvictLRU_EvictsOldestFirst(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + now := time.Now() + seedArtifact(t, ctx, db, store, "old-pkg", 500, now.Add(-3*time.Hour)) + seedArtifact(t, ctx, db, store, "mid-pkg", 500, now.Add(-1*time.Hour)) + seedArtifact(t, ctx, db, store, "new-pkg", 500, now) + + // Total is 1500 bytes, limit to 1100 so only the oldest gets evicted + evictLRU(ctx, db, store, logger, 1100) + + // old-pkg should be evicted + art, err := db.GetArtifact("pkg:npm/old-pkg@1.0.0", "old-pkg-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if art.StoragePath.Valid { + t.Error("expected old-pkg to be evicted (storage_path should be NULL)") + } + + // mid-pkg and new-pkg should remain + art, err = db.GetArtifact("pkg:npm/mid-pkg@1.0.0", "mid-pkg-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if !art.StoragePath.Valid { + t.Error("expected mid-pkg to remain cached") + } + + art, err = db.GetArtifact("pkg:npm/new-pkg@1.0.0", "new-pkg-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if !art.StoragePath.Valid { + t.Error("expected new-pkg to remain cached") + } + + // Storage file should be removed for old-pkg + storagePath := storage.ArtifactPath("npm", "", "old-pkg", "1.0.0", "old-pkg-1.0.0.tgz") + exists, _ := store.Exists(ctx, storagePath) + if exists { + t.Error("expected old-pkg file to be deleted from storage") + } +} + +func TestEvictLRU_EvictsMultipleToGetUnderLimit(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + now := time.Now() + seedArtifact(t, ctx, db, store, "pkg-1", 400, now.Add(-4*time.Hour)) + seedArtifact(t, ctx, db, store, "pkg-2", 400, now.Add(-3*time.Hour)) + seedArtifact(t, ctx, db, store, "pkg-3", 400, now.Add(-2*time.Hour)) + seedArtifact(t, ctx, db, store, "pkg-4", 400, now) + + // Total is 1600 bytes, limit to 900 so pkg-1 and pkg-2 get evicted + evictLRU(ctx, db, store, logger, 900) + + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 2 { + t.Errorf("expected 2 cached artifacts remaining, got %d", count) + } + + // Verify the right ones remain + for _, name := range []string{"pkg-3", "pkg-4"} { + art, err := db.GetArtifact("pkg:npm/"+name+"@1.0.0", name+"-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact %s: %v", name, err) + } + if !art.StoragePath.Valid { + t.Errorf("expected %s to remain cached", name) + } + } +} + +func TestEvictLRU_NothingToEvictWhenEmpty(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + // Should not panic or error with no artifacts + evictLRU(ctx, db, store, logger, 1024) + + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 0 { + t.Errorf("expected 0 cached artifacts, got %d", count) + } +} + +func TestEvictLRU_StorageFileDeleted(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + seedArtifact(t, ctx, db, store, "delete-me", 1000, time.Now().Add(-1*time.Hour)) + + storagePath := storage.ArtifactPath("npm", "", "delete-me", "1.0.0", "delete-me-1.0.0.tgz") + exists, _ := store.Exists(ctx, storagePath) + if !exists { + t.Fatal("expected artifact file to exist before eviction") + } + + evictLRU(ctx, db, store, logger, 500) + + exists, _ = store.Exists(ctx, storagePath) + if exists { + t.Error("expected artifact file to be deleted after eviction") + } + + art, err := db.GetArtifact("pkg:npm/delete-me@1.0.0", "delete-me-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if art.StoragePath.Valid { + t.Error("expected storage_path to be NULL after eviction") + } + if art.Size.Valid { + t.Error("expected size to be NULL after eviction") + } +} + +func TestStartEvictionLoop_UnlimitedSkips(t *testing.T) { + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "test.db") + storagePath := filepath.Join(tempDir, "artifacts") + + db, err := database.Create(dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + defer func() { _ = db.Close() }() + + store, err := storage.NewFilesystem(storagePath) + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + + cfg := defaultTestConfig(storagePath, dbPath) + + s := &Server{ + cfg: cfg, + db: db, + storage: store, + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + // Should return immediately since max_size is empty (unlimited) + done := make(chan struct{}) + go func() { + s.startEvictionLoop(ctx) + close(done) + }() + + select { + case <-done: + // Good, returned immediately + case <-time.After(1 * time.Second): + t.Error("startEvictionLoop should return immediately when max_size is unlimited") + cancel() + } +} + +func defaultTestConfig(storagePath, dbPath string) *config.Config { + return &config.Config{ + Listen: ":8080", + BaseURL: "http://localhost:8080", + Storage: config.StorageConfig{Path: storagePath, MaxSize: ""}, + Database: config.DatabaseConfig{ + Driver: "sqlite", + Path: dbPath, + }, + Log: config.LogConfig{Level: "info", Format: "text"}, + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 5d544a2..e3b13af 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -148,6 +148,9 @@ func (s *Server) Start() error { proxy.Cooldown = cd proxy.CacheMetadata = s.cfg.CacheMetadata proxy.MetadataTTL = s.cfg.ParseMetadataTTL() + proxy.DirectServe = s.cfg.Storage.DirectServe + proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL() + proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL // Create router with Chi r := chi.NewRouter() @@ -170,7 +173,6 @@ func (s *Server) Start() error { // Mount protocol handlers npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL) - cargoHandler := handler.NewCargoHandler(proxy, s.cfg.BaseURL) gemHandler := handler.NewGemHandler(proxy, s.cfg.BaseURL) goHandler := handler.NewGoHandler(proxy, s.cfg.BaseURL) hexHandler := handler.NewHexHandler(proxy, s.cfg.BaseURL) @@ -183,11 +185,21 @@ func (s *Server) Start() error { condaHandler := handler.NewCondaHandler(proxy, s.cfg.BaseURL) cranHandler := handler.NewCRANHandler(proxy, s.cfg.BaseURL) containerHandler := handler.NewContainerHandler(proxy, s.cfg.BaseURL) - debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL) rpmHandler := handler.NewRPMHandler(proxy, s.cfg.BaseURL) + for _, route := range s.cfg.Ecosystem.Cargo.Route { + routeHandler := handler.NewCargoHandler(proxy, s.cfg.BaseURL, route) + r.Mount(routeHandler.Path(), http.StripPrefix(routeHandler.Path(), routeHandler.Routes())) + s.logger.Info("mounted handler", "ecosystem", "cargo", "path", routeHandler.Path()) + } + + for _, route := range s.cfg.Ecosystem.Debian.Route { + routeHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL, route) + r.Mount(routeHandler.Path(), http.StripPrefix(routeHandler.Path(), routeHandler.Routes())) + s.logger.Info("mounted handler", "ecosystem", "debian", "path", routeHandler.Path()) + } + r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) - r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes())) r.Mount("/gem", http.StripPrefix("/gem", gemHandler.Routes())) r.Mount("/go", http.StripPrefix("/go", goHandler.Routes())) r.Mount("/hex", http.StripPrefix("/hex", hexHandler.Routes())) @@ -200,7 +212,6 @@ func (s *Server) Start() error { r.Mount("/conda", http.StripPrefix("/conda", condaHandler.Routes())) r.Mount("/cran", http.StripPrefix("/cran", cranHandler.Routes())) r.Mount("/v2", http.StripPrefix("/v2", containerHandler.Routes())) - r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes())) r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) // Health, stats, and static endpoints @@ -261,6 +272,7 @@ func (s *Server) Start() error { "storage", s.storage.URL(), "database", s.cfg.Database.Path) go s.updateCacheStatsMetrics() + go s.startEvictionLoop(bgCtx) return s.http.ListenAndServe() } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index be88bf6..0155437 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/config/cargo" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/storage" @@ -68,13 +69,13 @@ func newTestServer(t *testing.T) *testServer { // Mount handlers npmHandler := handler.NewNPMHandler(proxy, cfg.BaseURL) - cargoHandler := handler.NewCargoHandler(proxy, cfg.BaseURL) + cargoHandler := handler.NewCargoHandler(proxy, cfg.BaseURL, cargo.RouteDefault) gemHandler := handler.NewGemHandler(proxy, cfg.BaseURL) goHandler := handler.NewGoHandler(proxy, cfg.BaseURL) pypiHandler := handler.NewPyPIHandler(proxy, cfg.BaseURL) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes())) - r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes())) + r.Mount(cargoHandler.Path(), http.StripPrefix(cargoHandler.Path(), cargoHandler.Routes())) r.Mount("/gem", http.StripPrefix("/gem", gemHandler.Routes())) r.Mount("/go", http.StripPrefix("/go", goHandler.Routes())) r.Mount("/pypi", http.StripPrefix("/pypi", pypiHandler.Routes())) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 2d6af46..dc41668 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -6,12 +6,15 @@ import ( "encoding/hex" "fmt" "io" + "net/http" "os" "path/filepath" "runtime" "strings" + "time" "gocloud.dev/blob" + _ "gocloud.dev/blob/azureblob" _ "gocloud.dev/blob/fileblob" _ "gocloud.dev/blob/s3blob" "gocloud.dev/gcerrors" @@ -138,6 +141,20 @@ func (b *Blob) Delete(ctx context.Context, path string) error { return nil } +func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { + url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ + Method: http.MethodGet, + Expiry: expiry, + }) + if err != nil { + if gcerrors.Code(err) == gcerrors.Unimplemented { + return "", ErrSignedURLUnsupported + } + return "", fmt.Errorf("signing URL: %w", err) + } + return url, nil +} + func (b *Blob) Size(ctx context.Context, path string) (int64, error) { attrs, err := b.bucket.Attributes(ctx, path) if err != nil { diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index bb2d089..d80290b 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -10,6 +10,7 @@ import ( "runtime" "strings" "testing" + "time" ) func TestOpenBucket(t *testing.T) { @@ -188,6 +189,18 @@ func TestBlobLargeFile(t *testing.T) { assertLargeFileRoundTrip(t, createTestBlob(t)) } +func TestBlobSignedURLUnsupported(t *testing.T) { + b := createTestBlob(t) + ctx := context.Background() + + // fileblob has no URL signer configured, so this must surface as + // ErrSignedURLUnsupported rather than a generic error. + _, err := b.SignedURL(ctx, "test/file.txt", time.Minute) + if !errors.Is(err, ErrSignedURLUnsupported) { + t.Errorf("SignedURL on fileblob = %v, want ErrSignedURLUnsupported", err) + } +} + func TestBlobOverwrite(t *testing.T) { b := createTestBlob(t) ctx := context.Background() diff --git a/internal/storage/filesystem.go b/internal/storage/filesystem.go index cf6a1fe..7a4147a 100644 --- a/internal/storage/filesystem.go +++ b/internal/storage/filesystem.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "time" ) // Filesystem implements Storage using the local filesystem. @@ -129,6 +130,10 @@ func (fs *Filesystem) Delete(ctx context.Context, path string) error { return nil } +func (fs *Filesystem) SignedURL(_ context.Context, _ string, _ time.Duration) (string, error) { + return "", ErrSignedURLUnsupported +} + func (fs *Filesystem) Size(ctx context.Context, path string) (int64, error) { fullPath := fs.fullPath(path) diff --git a/internal/storage/filesystem_test.go b/internal/storage/filesystem_test.go index 7b7828d..cdd7f6b 100644 --- a/internal/storage/filesystem_test.go +++ b/internal/storage/filesystem_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "testing" + "time" ) func TestNewFilesystem(t *testing.T) { @@ -236,6 +237,15 @@ func TestFilesystemLargeFile(t *testing.T) { assertLargeFileRoundTrip(t, createTestFilesystem(t)) } +func TestFilesystemSignedURLUnsupported(t *testing.T) { + fs := createTestFilesystem(t) + + _, err := fs.SignedURL(context.Background(), "test/file.txt", time.Minute) + if !errors.Is(err, ErrSignedURLUnsupported) { + t.Errorf("SignedURL = %v, want ErrSignedURLUnsupported", err) + } +} + func createTestFilesystem(t *testing.T) *Filesystem { t.Helper() dir := t.TempDir() diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 8a9026c..0dba46a 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -15,12 +15,17 @@ import ( "encoding/hex" "errors" "io" + "time" ) const dirPermissions = 0755 var ( ErrNotFound = errors.New("artifact not found") + + // ErrSignedURLUnsupported is returned by SignedURL when the backend + // cannot generate presigned URLs (e.g. local filesystem). + ErrSignedURLUnsupported = errors.New("signed URLs not supported by storage backend") ) // Storage defines the interface for artifact storage backends. @@ -45,6 +50,10 @@ type Storage interface { // Returns ErrNotFound if the path does not exist. Size(ctx context.Context, path string) (int64, error) + // SignedURL returns a presigned URL granting time-limited GET access to path. + // Returns ErrSignedURLUnsupported if the backend cannot generate presigned URLs. + SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) + // UsedSpace returns the total bytes used by all stored content. UsedSpace(ctx context.Context) (int64, error)