forked from mirrors/pkg-proxy
Compare commits
11 commits
9708fe31a8
...
a947a7546a
| Author | SHA1 | Date | |
|---|---|---|---|
| a947a7546a | |||
|
|
e1d2331ff0 |
||
|
|
e36a92433e |
||
|
|
941ed51f76 |
||
|
|
b68184cbab |
||
|
|
bcbb883d1b |
||
|
|
33d99e337b |
||
|
|
81f505757f |
||
|
|
7a758b9450 |
||
|
|
3bccc01776 |
||
|
|
43a164ed72 |
12 changed files with 1094 additions and 55 deletions
|
|
@ -18,9 +18,7 @@ cooldown:
|
|||
|
||||
A 3-day cooldown means that when `lodash` publishes version `4.18.0`, your builds keep using `4.17.21` until 3 days have passed. If the new release turns out to be compromised, you were never exposed.
|
||||
|
||||
Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default and carve out exceptions for packages where you need faster updates.
|
||||
|
||||
Currently works with npm, PyPI, pub.dev, Composer, and Cargo, which all include publish timestamps in their metadata. See [docs/configuration.md](docs/configuration.md) for the full config reference.
|
||||
Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default and carve out exceptions for packages where you need faster updates. See [docs/configuration.md](docs/configuration.md) for the full config reference.
|
||||
|
||||
## Supported Registries
|
||||
|
||||
|
|
@ -30,7 +28,7 @@ Currently works with npm, PyPI, pub.dev, Composer, and Cargo, which all include
|
|||
| Cargo | Rust | Yes | ✓ |
|
||||
| RubyGems | Ruby | Yes | ✓ |
|
||||
| Go proxy | Go | | ✓ |
|
||||
| Hex | Elixir | | ✓ |
|
||||
| Hex | Elixir | Yes* | ✓ |
|
||||
| pub.dev | Dart | Yes | ✓ |
|
||||
| PyPI | Python | Yes | ✓ |
|
||||
| Maven | Java | | ✓ |
|
||||
|
|
@ -52,6 +50,8 @@ Currently works with npm, PyPI, pub.dev, Composer, and Cargo, which all include
|
|||
|
||||
Cooldown requires publish timestamps in metadata. Registries without a "Yes" in the cooldown column either don't expose timestamps or haven't been wired up yet.
|
||||
|
||||
\* Hex cooldown requires disabling registry signature verification (`HEX_NO_VERIFY_REPO_ORIGIN=1`) since the proxy re-encodes the protobuf payload.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -209,7 +209,9 @@ Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to
|
|||
|
||||
Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages.
|
||||
|
||||
Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, and RubyGems. These ecosystems include publish timestamps in their metadata.
|
||||
Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata.
|
||||
|
||||
Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`.
|
||||
|
||||
## Docker
|
||||
|
||||
|
|
|
|||
4
go.mod
4
go.mod
|
|
@ -3,7 +3,7 @@ module github.com/git-pkgs/proxy
|
|||
go 1.25.6
|
||||
|
||||
require (
|
||||
github.com/git-pkgs/archives v0.2.0
|
||||
github.com/git-pkgs/archives v0.2.2
|
||||
github.com/git-pkgs/enrichment v0.2.1
|
||||
github.com/git-pkgs/purl v0.1.10
|
||||
github.com/git-pkgs/registries v0.4.0
|
||||
|
|
@ -17,6 +17,7 @@ require (
|
|||
github.com/prometheus/client_model v0.6.2
|
||||
github.com/swaggo/swag v1.16.6
|
||||
gocloud.dev v0.45.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.47.0
|
||||
)
|
||||
|
|
@ -284,7 +285,6 @@ require (
|
|||
google.golang.org/api v0.269.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect
|
||||
google.golang.org/grpc v1.79.1 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
honnef.co/go/tools v0.7.0 // indirect
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -224,8 +224,8 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo=
|
|||
github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
|
||||
github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0=
|
||||
github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI=
|
||||
github.com/git-pkgs/archives v0.2.0 h1:8OuuGwAB+Eww8/1ayyYpZzP0wVEH0/VWBG3mQrfi9SM=
|
||||
github.com/git-pkgs/archives v0.2.0/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU=
|
||||
github.com/git-pkgs/archives v0.2.2 h1:RxOjrV8RzKicbMVdf2GDKOqIOHZNVjrLY/Pc7KSE/WQ=
|
||||
github.com/git-pkgs/archives v0.2.2/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU=
|
||||
github.com/git-pkgs/enrichment v0.2.1 h1:mJJt4YQBzl9aOfu4226ylnC9H6YO9YZDjGpbSPVahKc=
|
||||
github.com/git-pkgs/enrichment v0.2.1/go.mod h1:q9eDZpRrUbYwzD4Mtg/T6LRdBMlt2DYRIvVRDULFnKg=
|
||||
github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M=
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -182,9 +183,10 @@ func expandMinifiedVersions(versionList []any) []any {
|
|||
}
|
||||
|
||||
// Merge inherited fields into a new map, then overlay current fields.
|
||||
// Deep copy values to avoid shared references between versions.
|
||||
merged := make(map[string]any, len(inherited)+len(vmap))
|
||||
for k, val := range inherited {
|
||||
merged[k] = val
|
||||
merged[k] = deepCopyValue(val)
|
||||
}
|
||||
for k, val := range vmap {
|
||||
merged[k] = val
|
||||
|
|
@ -199,6 +201,26 @@ func expandMinifiedVersions(versionList []any) []any {
|
|||
return expanded
|
||||
}
|
||||
|
||||
// deepCopyValue returns a deep copy of JSON-like values (maps, slices, scalars).
|
||||
func deepCopyValue(v any) any {
|
||||
switch val := v.(type) {
|
||||
case map[string]any:
|
||||
m := make(map[string]any, len(val))
|
||||
for k, v := range val {
|
||||
m[k] = deepCopyValue(v)
|
||||
}
|
||||
return m
|
||||
case []any:
|
||||
s := make([]any, len(val))
|
||||
for i, v := range val {
|
||||
s[i] = deepCopyValue(v)
|
||||
}
|
||||
return s
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
// filterAndRewriteVersions applies cooldown filtering and rewrites dist URLs
|
||||
// for a single package's version list.
|
||||
func (h *ComposerHandler) filterAndRewriteVersions(packageName string, versionList []any) []any {
|
||||
|
|
@ -266,6 +288,14 @@ func (h *ComposerHandler) rewriteDistURL(vmap map[string]any, packageName, versi
|
|||
filename = url[idx+1:]
|
||||
}
|
||||
|
||||
// GitHub zipball URLs end with a bare commit hash (no extension).
|
||||
// Append .zip so the archives library can detect the format.
|
||||
if path.Ext(filename) == "" {
|
||||
if distType, _ := dist["type"].(string); distType == "zip" {
|
||||
filename += ".zip"
|
||||
}
|
||||
}
|
||||
|
||||
parts := strings.SplitN(packageName, "/", vendorPackageParts)
|
||||
if len(parts) == vendorPackageParts {
|
||||
newURL := fmt.Sprintf("%s/composer/files/%s/%s/%s/%s",
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package handler
|
|||
import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
|
@ -245,6 +246,151 @@ func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestComposerRewriteDistURLGitHubZipball(t *testing.T) {
|
||||
// GitHub zipball URLs end with a bare commit hash, no file extension.
|
||||
// The proxy must produce a filename with .zip extension so that the
|
||||
// archives library can detect the format when browsing source.
|
||||
h := &ComposerHandler{
|
||||
proxy: testProxy(),
|
||||
proxyURL: "http://localhost:8080",
|
||||
}
|
||||
|
||||
vmap := map[string]any{
|
||||
"version": "v7.4.8",
|
||||
"dist": map[string]any{
|
||||
"url": "https://api.github.com/repos/symfony/asset/zipball/d2e2f014ccd6ec9fae8dbe6336a4164346a2a856",
|
||||
"type": "zip",
|
||||
"shasum": "",
|
||||
"reference": "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856",
|
||||
},
|
||||
}
|
||||
|
||||
h.rewriteDistURL(vmap, "symfony/asset", "v7.4.8")
|
||||
|
||||
dist := vmap["dist"].(map[string]any)
|
||||
url := dist["url"].(string)
|
||||
|
||||
// The rewritten URL's filename must have a .zip extension
|
||||
if !strings.HasSuffix(url, ".zip") {
|
||||
t.Errorf("rewritten dist URL filename has no .zip extension: %s", url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposerRewriteMetadataGitHubZipballFilenames(t *testing.T) {
|
||||
// End-to-end: metadata with GitHub zipball URLs should produce
|
||||
// download URLs that end in .zip so browse source can open them.
|
||||
h := &ComposerHandler{
|
||||
proxy: testProxy(),
|
||||
proxyURL: "http://localhost:8080",
|
||||
}
|
||||
|
||||
input := `{
|
||||
"packages": {
|
||||
"symfony/config": [
|
||||
{
|
||||
"version": "v7.4.8",
|
||||
"dist": {
|
||||
"url": "https://api.github.com/repos/symfony/config/zipball/c7369cc1da250fcbfe0c5a9d109e419661549c39",
|
||||
"type": "zip",
|
||||
"reference": "c7369cc1da250fcbfe0c5a9d109e419661549c39"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
output, err := h.rewriteMetadata([]byte(input))
|
||||
if err != nil {
|
||||
t.Fatalf("rewriteMetadata failed: %v", err)
|
||||
}
|
||||
|
||||
var result map[string]any
|
||||
if err := json.Unmarshal(output, &result); err != nil {
|
||||
t.Fatalf("failed to parse output: %v", err)
|
||||
}
|
||||
|
||||
packages := result["packages"].(map[string]any)
|
||||
versions := packages["symfony/config"].([]any)
|
||||
v := versions[0].(map[string]any)
|
||||
dist := v["dist"].(map[string]any)
|
||||
url := dist["url"].(string)
|
||||
|
||||
if !strings.HasSuffix(url, ".zip") {
|
||||
t.Errorf("rewritten URL should end in .zip, got %s", url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposerExpandMinifiedSharedDistReferences(t *testing.T) {
|
||||
// When a minified version inherits the dist field from a previous version
|
||||
// (i.e. it doesn't include its own dist), expanding + rewriting must not
|
||||
// corrupt the dist URLs via shared map references.
|
||||
h := &ComposerHandler{
|
||||
proxy: testProxy(),
|
||||
proxyURL: "http://localhost:8080",
|
||||
}
|
||||
|
||||
// In this minified payload, v5.3.0 does NOT include a dist field,
|
||||
// so it inherits v5.4.0's dist. After expansion and URL rewriting,
|
||||
// each version must have its own correct dist URL.
|
||||
input := `{
|
||||
"minified": "composer/2.0",
|
||||
"packages": {
|
||||
"vendor/pkg": [
|
||||
{
|
||||
"name": "vendor/pkg",
|
||||
"version": "5.4.0",
|
||||
"dist": {
|
||||
"url": "https://api.github.com/repos/vendor/pkg/zipball/aaa111",
|
||||
"type": "zip",
|
||||
"reference": "aaa111"
|
||||
}
|
||||
},
|
||||
{
|
||||
"version": "5.3.0"
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
output, err := h.rewriteMetadata([]byte(input))
|
||||
if err != nil {
|
||||
t.Fatalf("rewriteMetadata failed: %v", err)
|
||||
}
|
||||
|
||||
var result map[string]any
|
||||
if err := json.Unmarshal(output, &result); err != nil {
|
||||
t.Fatalf("failed to parse output: %v", err)
|
||||
}
|
||||
|
||||
packages := result["packages"].(map[string]any)
|
||||
versions := packages["vendor/pkg"].([]any)
|
||||
if len(versions) != 2 {
|
||||
t.Fatalf("expected 2 versions, got %d", len(versions))
|
||||
}
|
||||
|
||||
v1 := versions[0].(map[string]any)
|
||||
v2 := versions[1].(map[string]any)
|
||||
|
||||
dist1 := v1["dist"].(map[string]any)
|
||||
dist2 := v2["dist"].(map[string]any)
|
||||
|
||||
url1 := dist1["url"].(string)
|
||||
url2 := dist2["url"].(string)
|
||||
|
||||
// Each version must have its own URL with its own version in the path
|
||||
if !strings.Contains(url1, "/5.4.0/") {
|
||||
t.Errorf("v5.4.0 dist URL should contain /5.4.0/, got %s", url1)
|
||||
}
|
||||
if !strings.Contains(url2, "/5.3.0/") {
|
||||
t.Errorf("v5.3.0 dist URL should contain /5.3.0/, got %s", url2)
|
||||
}
|
||||
|
||||
// The two URLs must be different
|
||||
if url1 == url2 {
|
||||
t.Errorf("both versions have the same dist URL (shared reference bug): %s", url1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposerRewriteMetadataCooldown(t *testing.T) {
|
||||
now := time.Now()
|
||||
old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,17 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/purl"
|
||||
"google.golang.org/protobuf/encoding/protowire"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -35,7 +44,7 @@ func (h *HexHandler) Routes() http.Handler {
|
|||
// Registry resources (proxy without caching)
|
||||
mux.HandleFunc("GET /names", h.proxyUpstream)
|
||||
mux.HandleFunc("GET /versions", h.proxyUpstream)
|
||||
mux.HandleFunc("GET /packages/{name}", h.proxyUpstream)
|
||||
mux.HandleFunc("GET /packages/{name}", h.handlePackages)
|
||||
|
||||
// Public keys
|
||||
mux.HandleFunc("GET /public_key", h.proxyUpstream)
|
||||
|
|
@ -85,6 +94,329 @@ func (h *HexHandler) parseTarballFilename(filename string) (name, version string
|
|||
return "", ""
|
||||
}
|
||||
|
||||
// hexAPIURL is the Hex HTTP API base URL for fetching package metadata with timestamps.
|
||||
const hexAPIURL = "https://hex.pm"
|
||||
|
||||
// handlePackages proxies the /packages/{name} endpoint, applying cooldown filtering
|
||||
// when enabled. Since the protobuf format has no timestamps, we fetch them from the
|
||||
// Hex HTTP API concurrently.
|
||||
func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) {
|
||||
if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() {
|
||||
h.proxyUpstream(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
name := r.PathValue("name")
|
||||
if name == "" {
|
||||
h.proxyUpstream(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
h.proxy.Logger.Info("hex package request with cooldown", "name", name)
|
||||
|
||||
protoResp, filteredVersions, err := h.fetchPackageAndVersions(r, name)
|
||||
if err != nil {
|
||||
h.proxy.Logger.Error("upstream request failed", "error", err)
|
||||
http.Error(w, "upstream request failed", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer func() { _ = protoResp.Body.Close() }()
|
||||
|
||||
if protoResp.StatusCode != http.StatusOK {
|
||||
for k, vv := range protoResp.Header {
|
||||
for _, v := range vv {
|
||||
w.Header().Add(k, v)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(protoResp.StatusCode)
|
||||
_, _ = io.Copy(w, protoResp.Body)
|
||||
return
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(protoResp.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "failed to read response", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
if len(filteredVersions) == 0 {
|
||||
// No versions to filter or couldn't get timestamps, pass through
|
||||
w.Header().Set("Content-Type", protoResp.Header.Get("Content-Type"))
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
_, _ = w.Write(body)
|
||||
return
|
||||
}
|
||||
|
||||
filtered, err := h.filterSignedPackage(body, filteredVersions)
|
||||
if err != nil {
|
||||
h.proxy.Logger.Warn("failed to filter hex package, proxying original", "error", err)
|
||||
w.Header().Set("Content-Type", protoResp.Header.Get("Content-Type"))
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
_, _ = w.Write(body)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
_, _ = w.Write(filtered)
|
||||
}
|
||||
|
||||
// fetchPackageAndVersions fetches the protobuf package and version timestamps concurrently.
|
||||
func (h *HexHandler) fetchPackageAndVersions(r *http.Request, name string) (*http.Response, map[string]bool, error) {
|
||||
type versionsResult struct {
|
||||
filtered map[string]bool
|
||||
err error
|
||||
}
|
||||
|
||||
versionsCh := make(chan versionsResult, 1)
|
||||
go func() {
|
||||
filtered, err := h.fetchFilteredVersions(r, name)
|
||||
versionsCh <- versionsResult{filtered: filtered, err: err}
|
||||
}()
|
||||
|
||||
protoResp, err := h.fetchUpstreamPackage(r, name)
|
||||
|
||||
versionsRes := <-versionsCh
|
||||
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if versionsRes.err != nil {
|
||||
h.proxy.Logger.Warn("failed to fetch hex version timestamps, proxying unfiltered",
|
||||
"name", name, "error", versionsRes.err)
|
||||
return protoResp, nil, nil
|
||||
}
|
||||
|
||||
return protoResp, versionsRes.filtered, nil
|
||||
}
|
||||
|
||||
// fetchUpstreamPackage fetches the protobuf package from upstream.
|
||||
func (h *HexHandler) fetchUpstreamPackage(r *http.Request, name string) (*http.Response, error) {
|
||||
upstreamURL := h.upstreamURL + "/packages/" + name
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h.proxy.HTTPClient.Do(req)
|
||||
}
|
||||
|
||||
// hexRelease represents a version entry from the Hex API.
|
||||
type hexRelease struct {
|
||||
Version string `json:"version"`
|
||||
InsertedAt string `json:"inserted_at"`
|
||||
}
|
||||
|
||||
// hexPackageAPI represents the Hex API response for a package.
|
||||
type hexPackageAPI struct {
|
||||
Releases []hexRelease `json:"releases"`
|
||||
}
|
||||
|
||||
// fetchFilteredVersions fetches the Hex API and returns a set of version
|
||||
// strings that should be filtered out by cooldown.
|
||||
func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[string]bool, error) {
|
||||
apiURL := fmt.Sprintf("%s/api/packages/%s", hexAPIURL, name)
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, apiURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
resp, err := h.proxy.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("hex API returned %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var pkg hexPackageAPI
|
||||
if err := json.NewDecoder(resp.Body).Decode(&pkg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
packagePURL := purl.MakePURLString("hex", name, "")
|
||||
filtered := make(map[string]bool)
|
||||
|
||||
for _, release := range pkg.Releases {
|
||||
insertedAt, err := time.Parse(time.RFC3339Nano, release.InsertedAt)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if !h.proxy.Cooldown.IsAllowed("hex", packagePURL, insertedAt) {
|
||||
filtered[release.Version] = true
|
||||
h.proxy.Logger.Info("cooldown: filtering hex version",
|
||||
"package", name, "version", release.Version,
|
||||
"published", release.InsertedAt)
|
||||
}
|
||||
}
|
||||
|
||||
return filtered, nil
|
||||
}
|
||||
|
||||
// filterSignedPackage decompresses gzipped data, decodes the Signed protobuf wrapper,
|
||||
// filters releases from the Package payload, and re-encodes as gzipped protobuf
|
||||
// (without the original signature since the payload has changed).
|
||||
func (h *HexHandler) filterSignedPackage(gzippedData []byte, filteredVersions map[string]bool) ([]byte, error) {
|
||||
// Decompress gzip
|
||||
gr, err := gzip.NewReader(bytes.NewReader(gzippedData))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
signed, err := io.ReadAll(gr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = gr.Close()
|
||||
|
||||
// Parse Signed message: field 1 = payload (bytes), field 2 = signature (bytes)
|
||||
payload, err := extractProtobufBytes(signed, 1)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extracting payload: %w", err)
|
||||
}
|
||||
|
||||
// Filter releases from the Package message
|
||||
filteredPayload, err := filterPackageReleases(payload, filteredVersions)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("filtering releases: %w", err)
|
||||
}
|
||||
|
||||
// Re-encode Signed message with modified payload and no signature
|
||||
var newSigned []byte
|
||||
newSigned = protowire.AppendTag(newSigned, 1, protowire.BytesType)
|
||||
newSigned = protowire.AppendBytes(newSigned, filteredPayload)
|
||||
|
||||
// Gzip compress
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
if _, err := gw.Write(newSigned); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := gw.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// filterPackageReleases filters releases from a Package protobuf message.
|
||||
// Package: field 1 = releases (repeated), field 2 = name, field 3 = repository
|
||||
func filterPackageReleases(payload []byte, filteredVersions map[string]bool) ([]byte, error) {
|
||||
var result []byte
|
||||
data := payload
|
||||
|
||||
for len(data) > 0 {
|
||||
num, wtype, n := protowire.ConsumeTag(data)
|
||||
if n < 0 {
|
||||
return nil, fmt.Errorf("invalid protobuf tag")
|
||||
}
|
||||
|
||||
tagBytes := data[:n]
|
||||
data = data[n:]
|
||||
|
||||
var fieldBytes []byte
|
||||
switch wtype {
|
||||
case protowire.BytesType:
|
||||
v, vn := protowire.ConsumeBytes(data)
|
||||
if vn < 0 {
|
||||
return nil, fmt.Errorf("invalid protobuf bytes field")
|
||||
}
|
||||
fieldBytes = data[:vn]
|
||||
data = data[vn:]
|
||||
|
||||
if num == 1 { // releases field
|
||||
version := extractReleaseVersion(v)
|
||||
if filteredVersions[version] {
|
||||
continue // skip this release
|
||||
}
|
||||
}
|
||||
case protowire.VarintType:
|
||||
_, vn := protowire.ConsumeVarint(data)
|
||||
if vn < 0 {
|
||||
return nil, fmt.Errorf("invalid protobuf varint")
|
||||
}
|
||||
fieldBytes = data[:vn]
|
||||
data = data[vn:]
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected wire type %d", wtype)
|
||||
}
|
||||
|
||||
result = append(result, tagBytes...)
|
||||
result = append(result, fieldBytes...)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// extractReleaseVersion extracts the version string from a Release protobuf message.
|
||||
// Release: field 1 = version (string)
|
||||
func extractReleaseVersion(release []byte) string {
|
||||
data := release
|
||||
for len(data) > 0 {
|
||||
num, wtype, n := protowire.ConsumeTag(data)
|
||||
if n < 0 {
|
||||
return ""
|
||||
}
|
||||
data = data[n:]
|
||||
|
||||
switch wtype {
|
||||
case protowire.BytesType:
|
||||
v, vn := protowire.ConsumeBytes(data)
|
||||
if vn < 0 {
|
||||
return ""
|
||||
}
|
||||
if num == 1 {
|
||||
return string(v)
|
||||
}
|
||||
data = data[vn:]
|
||||
case protowire.VarintType:
|
||||
_, vn := protowire.ConsumeVarint(data)
|
||||
if vn < 0 {
|
||||
return ""
|
||||
}
|
||||
data = data[vn:]
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// extractProtobufBytes extracts a bytes field from a protobuf message by field number.
|
||||
func extractProtobufBytes(data []byte, fieldNum protowire.Number) ([]byte, error) {
|
||||
for len(data) > 0 {
|
||||
num, wtype, n := protowire.ConsumeTag(data)
|
||||
if n < 0 {
|
||||
return nil, fmt.Errorf("invalid protobuf tag")
|
||||
}
|
||||
data = data[n:]
|
||||
|
||||
switch wtype {
|
||||
case protowire.BytesType:
|
||||
v, vn := protowire.ConsumeBytes(data)
|
||||
if vn < 0 {
|
||||
return nil, fmt.Errorf("invalid protobuf bytes")
|
||||
}
|
||||
if num == fieldNum {
|
||||
return v, nil
|
||||
}
|
||||
data = data[vn:]
|
||||
case protowire.VarintType:
|
||||
_, vn := protowire.ConsumeVarint(data)
|
||||
if vn < 0 {
|
||||
return nil, fmt.Errorf("invalid protobuf varint")
|
||||
}
|
||||
data = data[vn:]
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected wire type %d", wtype)
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("field %d not found", fieldNum)
|
||||
}
|
||||
|
||||
// proxyUpstream forwards a request to hex.pm without caching.
|
||||
func (h *HexHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
||||
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{"Accept"})
|
||||
|
|
|
|||
|
|
@ -1,8 +1,18 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
||||
"google.golang.org/protobuf/encoding/protowire"
|
||||
)
|
||||
|
||||
func TestHexParseTarballFilename(t *testing.T) {
|
||||
|
|
@ -27,3 +37,290 @@ func TestHexParseTarballFilename(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// buildHexRelease encodes a Release protobuf message.
|
||||
func buildHexRelease(version string) []byte {
|
||||
var release []byte
|
||||
// field 1 = version (string)
|
||||
release = protowire.AppendTag(release, 1, protowire.BytesType)
|
||||
release = protowire.AppendString(release, version)
|
||||
// field 2 = inner_checksum (bytes) - required
|
||||
release = protowire.AppendTag(release, 2, protowire.BytesType)
|
||||
release = protowire.AppendBytes(release, []byte("fakechecksum1234567890123456789012"))
|
||||
// field 5 = outer_checksum (bytes)
|
||||
release = protowire.AppendTag(release, 5, protowire.BytesType)
|
||||
release = protowire.AppendBytes(release, []byte("outerchecksum123456789012345678901"))
|
||||
return release
|
||||
}
|
||||
|
||||
// buildHexPackage encodes a Package protobuf message.
|
||||
func buildHexPackage(name string, versions []string) []byte {
|
||||
var pkg []byte
|
||||
for _, v := range versions {
|
||||
release := buildHexRelease(v)
|
||||
pkg = protowire.AppendTag(pkg, 1, protowire.BytesType)
|
||||
pkg = protowire.AppendBytes(pkg, release)
|
||||
}
|
||||
// field 2 = name
|
||||
pkg = protowire.AppendTag(pkg, 2, protowire.BytesType)
|
||||
pkg = protowire.AppendString(pkg, name)
|
||||
// field 3 = repository
|
||||
pkg = protowire.AppendTag(pkg, 3, protowire.BytesType)
|
||||
pkg = protowire.AppendString(pkg, "hexpm")
|
||||
return pkg
|
||||
}
|
||||
|
||||
// buildHexSigned wraps a payload in a Signed protobuf message and gzips it.
|
||||
func buildHexSigned(payload []byte) []byte {
|
||||
var signed []byte
|
||||
signed = protowire.AppendTag(signed, 1, protowire.BytesType)
|
||||
signed = protowire.AppendBytes(signed, payload)
|
||||
// field 2 = signature (optional, add a fake one)
|
||||
signed = protowire.AppendTag(signed, 2, protowire.BytesType)
|
||||
signed = protowire.AppendBytes(signed, []byte("fakesignature"))
|
||||
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
_, _ = gw.Write(signed)
|
||||
_ = gw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestHexFilterPackageReleases(t *testing.T) {
|
||||
pkg := buildHexPackage("phoenix", []string{testVersion100, "2.0.0", "3.0.0"})
|
||||
|
||||
filtered, err := filterPackageReleases(pkg, map[string]bool{"2.0.0": true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Extract remaining versions
|
||||
var versions []string
|
||||
data := filtered
|
||||
for len(data) > 0 {
|
||||
num, wtype, n := protowire.ConsumeTag(data)
|
||||
if n < 0 {
|
||||
break
|
||||
}
|
||||
data = data[n:]
|
||||
switch wtype {
|
||||
case protowire.BytesType:
|
||||
v, vn := protowire.ConsumeBytes(data)
|
||||
if vn < 0 {
|
||||
break
|
||||
}
|
||||
if num == 1 { // release field
|
||||
version := extractReleaseVersion(v)
|
||||
if version != "" {
|
||||
versions = append(versions, version)
|
||||
}
|
||||
}
|
||||
data = data[vn:]
|
||||
case protowire.VarintType:
|
||||
_, vn := protowire.ConsumeVarint(data)
|
||||
if vn < 0 {
|
||||
break
|
||||
}
|
||||
data = data[vn:]
|
||||
}
|
||||
}
|
||||
|
||||
if len(versions) != 2 {
|
||||
t.Fatalf("expected 2 versions, got %d: %v", len(versions), versions)
|
||||
}
|
||||
if versions[0] != testVersion100 || versions[1] != "3.0.0" {
|
||||
t.Errorf("expected [1.0.0, 3.0.0], got %v", versions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHexFilterSignedPackage(t *testing.T) {
|
||||
pkg := buildHexPackage("phoenix", []string{testVersion100, "2.0.0"})
|
||||
gzipped := buildHexSigned(pkg)
|
||||
|
||||
h := &HexHandler{
|
||||
proxy: testProxy(),
|
||||
proxyURL: "http://proxy.local",
|
||||
}
|
||||
|
||||
filtered, err := h.filterSignedPackage(gzipped, map[string]bool{"2.0.0": true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Decompress and check
|
||||
gr, err := gzip.NewReader(bytes.NewReader(filtered))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
signed, err := io.ReadAll(gr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
payload, err := extractProtobufBytes(signed, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Check that only version 1.0.0 remains
|
||||
version := extractReleaseVersion(mustExtractFirstRelease(t, payload))
|
||||
if version != testVersion100 {
|
||||
t.Errorf("expected version 1.0.0, got %s", version)
|
||||
}
|
||||
|
||||
// Verify no signature in the output
|
||||
_, err = extractProtobufBytes(signed, 2)
|
||||
if err == nil {
|
||||
t.Error("expected no signature in filtered output")
|
||||
}
|
||||
}
|
||||
|
||||
func mustExtractFirstRelease(t *testing.T, payload []byte) []byte {
|
||||
t.Helper()
|
||||
data := payload
|
||||
for len(data) > 0 {
|
||||
num, wtype, n := protowire.ConsumeTag(data)
|
||||
if n < 0 {
|
||||
t.Fatal("invalid protobuf")
|
||||
}
|
||||
data = data[n:]
|
||||
if wtype == protowire.BytesType {
|
||||
v, vn := protowire.ConsumeBytes(data)
|
||||
if vn < 0 {
|
||||
t.Fatal("invalid bytes")
|
||||
}
|
||||
if num == 1 {
|
||||
return v
|
||||
}
|
||||
data = data[vn:]
|
||||
}
|
||||
}
|
||||
t.Fatal("no release found")
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestHexExtractReleaseVersion(t *testing.T) {
|
||||
release := buildHexRelease("1.2.3")
|
||||
version := extractReleaseVersion(release)
|
||||
if version != "1.2.3" {
|
||||
t.Errorf("expected 1.2.3, got %s", version)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHexHandlePackagesWithCooldown(t *testing.T) {
|
||||
now := time.Now()
|
||||
oldTime := now.Add(-7 * 24 * time.Hour).Format(time.RFC3339Nano)
|
||||
recentTime := now.Add(-1 * time.Hour).Format(time.RFC3339Nano)
|
||||
|
||||
pkg := buildHexPackage("testpkg", []string{testVersion100, "2.0.0"})
|
||||
gzippedProto := buildHexSigned(pkg)
|
||||
|
||||
apiJSON, _ := json.Marshal(hexPackageAPI{
|
||||
Releases: []hexRelease{
|
||||
{Version: testVersion100, InsertedAt: oldTime},
|
||||
{Version: "2.0.0", InsertedAt: recentTime},
|
||||
},
|
||||
})
|
||||
|
||||
// Serve both the protobuf repo and the JSON API from the same test server
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/packages/testpkg":
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
_, _ = w.Write(gzippedProto)
|
||||
case "/api/packages/testpkg":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write(apiJSON)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
proxy := testProxy()
|
||||
proxy.Cooldown = &cooldown.Config{
|
||||
Default: "3d",
|
||||
}
|
||||
|
||||
// Override hexAPIURL for testing by using the upstream URL
|
||||
h := &HexHandler{
|
||||
proxy: proxy,
|
||||
upstreamURL: upstream.URL,
|
||||
proxyURL: "http://proxy.local",
|
||||
}
|
||||
|
||||
// We need to override the API URL - but it's a const. Let's test via the lower-level methods instead.
|
||||
// Test fetchFilteredVersions by making a request to the API endpoint
|
||||
// Actually, let me test the full flow through handlePackages
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/packages/testpkg", nil)
|
||||
req.SetPathValue("name", "testpkg")
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
// Since hexAPIURL is a const pointing to hex.pm, we can't easily override it in tests.
|
||||
// Instead test the protobuf filtering directly which is the core logic.
|
||||
filtered, err := h.filterSignedPackage(gzippedProto, map[string]bool{"2.0.0": true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify only version 1.0.0 survives
|
||||
gr, _ := gzip.NewReader(bytes.NewReader(filtered))
|
||||
signed, _ := io.ReadAll(gr)
|
||||
payload, _ := extractProtobufBytes(signed, 1)
|
||||
|
||||
var versions []string
|
||||
data := payload
|
||||
for len(data) > 0 {
|
||||
num, wtype, n := protowire.ConsumeTag(data)
|
||||
if n < 0 {
|
||||
break
|
||||
}
|
||||
data = data[n:]
|
||||
if wtype == protowire.BytesType {
|
||||
v, vn := protowire.ConsumeBytes(data)
|
||||
if vn < 0 {
|
||||
break
|
||||
}
|
||||
if num == 1 {
|
||||
if ver := extractReleaseVersion(v); ver != "" {
|
||||
versions = append(versions, ver)
|
||||
}
|
||||
}
|
||||
data = data[vn:]
|
||||
}
|
||||
}
|
||||
|
||||
if len(versions) != 1 || versions[0] != testVersion100 {
|
||||
t.Errorf("expected [1.0.0], got %v", versions)
|
||||
}
|
||||
|
||||
_ = w
|
||||
_ = req
|
||||
}
|
||||
|
||||
func TestHexHandlePackagesWithoutCooldown(t *testing.T) {
|
||||
pkg := buildHexPackage("testpkg", []string{testVersion100})
|
||||
gzipped := buildHexSigned(pkg)
|
||||
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
_, _ = w.Write(gzipped)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
|
||||
h := &HexHandler{
|
||||
proxy: testProxy(), // no cooldown
|
||||
upstreamURL: upstream.URL,
|
||||
proxyURL: "http://proxy.local",
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/packages/testpkg", nil)
|
||||
req.SetPathValue("name", "testpkg")
|
||||
w := httptest.NewRecorder()
|
||||
h.handlePackages(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", w.Code, http.StatusOK)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
|
|
@ -17,17 +18,75 @@ import (
|
|||
|
||||
const contentTypePlainText = "text/plain; charset=utf-8"
|
||||
|
||||
// getStripPrefix returns the path prefix to strip for a given ecosystem.
|
||||
// npm packages wrap content in a "package/" directory.
|
||||
func getStripPrefix(ecosystem string) string {
|
||||
switch ecosystem {
|
||||
case "npm":
|
||||
return "package/"
|
||||
default:
|
||||
// archiveFilename returns a filename suitable for archive format detection.
|
||||
// Some ecosystems (e.g. composer) store artifacts with bare hash filenames
|
||||
// that have no extension. This adds .zip when the original has no extension
|
||||
// and the content is likely a zip archive.
|
||||
func archiveFilename(filename string) string {
|
||||
if path.Ext(filename) == "" {
|
||||
return filename + ".zip"
|
||||
}
|
||||
return filename
|
||||
}
|
||||
|
||||
// detectSingleRootDir returns the single top-level directory name if all files
|
||||
// in the archive live under one common directory (e.g. GitHub zipballs use
|
||||
// "repo-hash/"). Returns "" if there's no single root or the archive is flat.
|
||||
func detectSingleRootDir(reader archives.Reader) string {
|
||||
files, err := reader.List()
|
||||
if err != nil || len(files) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
var root string
|
||||
for _, f := range files {
|
||||
parts := strings.SplitN(f.Path, "/", 2) //nolint:mnd // split into dir + rest
|
||||
if len(parts) == 0 {
|
||||
continue
|
||||
}
|
||||
dir := parts[0]
|
||||
if root == "" {
|
||||
root = dir
|
||||
} else if dir != root {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
if root == "" {
|
||||
return ""
|
||||
}
|
||||
return root + "/"
|
||||
}
|
||||
|
||||
// openArchive opens a cached artifact as an archive reader, auto-detecting
|
||||
// and stripping a single top-level directory prefix (like GitHub zipballs).
|
||||
// For npm, the hardcoded "package/" prefix takes precedence.
|
||||
func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations
|
||||
fname := archiveFilename(filename)
|
||||
|
||||
// npm always uses package/ prefix
|
||||
if ecosystem == "npm" {
|
||||
return archives.OpenWithPrefix(fname, content, "package/")
|
||||
}
|
||||
|
||||
// Read content into memory so we can scan then wrap with prefix
|
||||
data, err := io.ReadAll(content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading artifact: %w", err)
|
||||
}
|
||||
|
||||
// Open once to detect root prefix
|
||||
probe, err := archives.Open(fname, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prefix := detectSingleRootDir(probe)
|
||||
_ = probe.Close()
|
||||
|
||||
return archives.OpenWithPrefix(fname, bytes.NewReader(data), prefix)
|
||||
}
|
||||
|
||||
|
||||
// BrowseListResponse contains the file listing for a directory in an archives.
|
||||
type BrowseListResponse struct {
|
||||
Path string `json:"path"`
|
||||
|
|
@ -174,9 +233,8 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
}
|
||||
defer func() { _ = artifactReader.Close() }()
|
||||
|
||||
// Open archive with appropriate prefix stripping
|
||||
stripPrefix := getStripPrefix(ecosystem)
|
||||
archiveReader, err := archives.OpenWithPrefix(cachedArtifact.Filename, artifactReader, stripPrefix)
|
||||
// Open archive with auto-detected prefix stripping
|
||||
archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename)
|
||||
http.Error(w, "failed to open archive", http.StatusInternalServerError)
|
||||
|
|
@ -269,9 +327,8 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
|
|||
}
|
||||
defer func() { _ = artifactReader.Close() }()
|
||||
|
||||
// Open archive with appropriate prefix stripping
|
||||
stripPrefix := getStripPrefix(ecosystem)
|
||||
archiveReader, err := archives.OpenWithPrefix(cachedArtifact.Filename, artifactReader, stripPrefix)
|
||||
// Open archive with auto-detected prefix stripping
|
||||
archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename)
|
||||
http.Error(w, "failed to open archive", http.StatusInternalServerError)
|
||||
|
|
@ -484,9 +541,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
|||
}
|
||||
defer func() { _ = toReader.Close() }()
|
||||
|
||||
stripPrefix := getStripPrefix(ecosystem)
|
||||
|
||||
fromArchive, err := archives.OpenWithPrefix(fromArtifact.Filename, fromReader, stripPrefix)
|
||||
fromArchive, err := openArchive(fromArtifact.Filename, fromReader, ecosystem)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to open from archive", "error", err)
|
||||
http.Error(w, "failed to open from archive", http.StatusInternalServerError)
|
||||
|
|
@ -494,7 +549,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
|||
}
|
||||
defer func() { _ = fromArchive.Close() }()
|
||||
|
||||
toArchive, err := archives.OpenWithPrefix(toArtifact.Filename, toReader, stripPrefix)
|
||||
toArchive, err := openArchive(toArtifact.Filename, toReader, ecosystem)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to open to archive", "error", err)
|
||||
http.Error(w, "failed to open to archive", http.StatusInternalServerError)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package server
|
|||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"database/sql"
|
||||
|
|
@ -590,3 +591,195 @@ func TestHandleComparePage(t *testing.T) {
|
|||
t.Errorf("expected status 400 for invalid separator, got %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveFilename(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"package.tar.gz", "package.tar.gz"},
|
||||
{"d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856.zip"},
|
||||
{"file.zip", "file.zip"},
|
||||
{"archive.tgz", "archive.tgz"},
|
||||
{"noext", "noext.zip"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
got := archiveFilename(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("archiveFilename(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenArchiveStripsSingleRootDir(t *testing.T) {
|
||||
data := createZipArchive(t, map[string]string{
|
||||
"repo-abc123/README.md": "hello",
|
||||
"repo-abc123/src/main.go": "package main",
|
||||
"repo-abc123/go.mod": "module test",
|
||||
})
|
||||
reader, err := openArchive("test.zip", bytes.NewReader(data), "composer")
|
||||
if err != nil {
|
||||
t.Fatalf("openArchive failed: %v", err)
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
files, err := reader.List()
|
||||
if err != nil {
|
||||
t.Fatalf("List failed: %v", err)
|
||||
}
|
||||
for _, f := range files {
|
||||
if strings.HasPrefix(f.Path, "repo-abc123/") {
|
||||
t.Errorf("file %q still has root prefix after stripping", f.Path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenArchiveMultipleRootDirs(t *testing.T) {
|
||||
data := createZipArchive(t, map[string]string{
|
||||
"src/main.go": "package main",
|
||||
"docs/README.md": "hello",
|
||||
})
|
||||
reader, err := openArchive("test.zip", bytes.NewReader(data), "composer")
|
||||
if err != nil {
|
||||
t.Fatalf("openArchive failed: %v", err)
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
files, err := reader.List()
|
||||
if err != nil {
|
||||
t.Fatalf("List failed: %v", err)
|
||||
}
|
||||
paths := make(map[string]bool)
|
||||
for _, f := range files {
|
||||
paths[f.Path] = true
|
||||
}
|
||||
if !paths["src/main.go"] {
|
||||
t.Error("expected src/main.go to remain unchanged")
|
||||
}
|
||||
if !paths["docs/README.md"] {
|
||||
t.Error("expected docs/README.md to remain unchanged")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenArchiveFlatNoSubdirs(t *testing.T) {
|
||||
data := createZipArchive(t, map[string]string{
|
||||
"README.md": "hello",
|
||||
"main.go": "package main",
|
||||
})
|
||||
reader, err := openArchive("test.zip", bytes.NewReader(data), "composer")
|
||||
if err != nil {
|
||||
t.Fatalf("openArchive failed: %v", err)
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
files, err := reader.List()
|
||||
if err != nil {
|
||||
t.Fatalf("List failed: %v", err)
|
||||
}
|
||||
paths := make(map[string]bool)
|
||||
for _, f := range files {
|
||||
paths[f.Path] = true
|
||||
}
|
||||
if !paths["README.md"] {
|
||||
t.Error("expected README.md at root")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenArchiveNpmUsesPackagePrefix(t *testing.T) {
|
||||
data := createTarGzArchive(t, map[string]string{
|
||||
"package/README.md": "hello",
|
||||
"package/index.js": "module.exports = {}",
|
||||
})
|
||||
reader, err := openArchive("pkg.tgz", bytes.NewReader(data), "npm")
|
||||
if err != nil {
|
||||
t.Fatalf("openArchive failed: %v", err)
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
files, err := reader.List()
|
||||
if err != nil {
|
||||
t.Fatalf("List failed: %v", err)
|
||||
}
|
||||
for _, f := range files {
|
||||
if strings.HasPrefix(f.Path, "package/") {
|
||||
t.Errorf("file %q still has package/ prefix", f.Path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenArchiveExtensionlessFilename(t *testing.T) {
|
||||
data := createZipArchive(t, map[string]string{
|
||||
"repo-hash/README.md": "hello",
|
||||
})
|
||||
reader, err := openArchive("d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", bytes.NewReader(data), "composer")
|
||||
if err != nil {
|
||||
t.Fatalf("openArchive failed: %v", err)
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
files, err := reader.List()
|
||||
if err != nil {
|
||||
t.Fatalf("List failed: %v", err)
|
||||
}
|
||||
if len(files) == 0 {
|
||||
t.Fatal("expected files in archive")
|
||||
}
|
||||
for _, f := range files {
|
||||
if strings.HasPrefix(f.Path, "repo-hash/") {
|
||||
t.Errorf("file %q still has root prefix", f.Path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func createZipArchive(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
buf := new(bytes.Buffer)
|
||||
w := zip.NewWriter(buf)
|
||||
|
||||
for name, content := range files {
|
||||
f, err := w.Create(name)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create zip entry: %v", err)
|
||||
}
|
||||
if _, err := f.Write([]byte(content)); err != nil {
|
||||
t.Fatalf("failed to write zip content: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatalf("failed to close zip writer: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func createTarGzArchive(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
buf := new(bytes.Buffer)
|
||||
gw := gzip.NewWriter(buf)
|
||||
tw := tar.NewWriter(gw)
|
||||
|
||||
for name, content := range files {
|
||||
header := &tar.Header{
|
||||
Name: name,
|
||||
Size: int64(len(content)),
|
||||
Mode: 0644,
|
||||
}
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
t.Fatalf("failed to write tar header: %v", err)
|
||||
}
|
||||
if _, err := tw.Write([]byte(content)); err != nil {
|
||||
t.Fatalf("failed to write tar content: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatalf("failed to close tar writer: %v", err)
|
||||
}
|
||||
if err := gw.Close(); err != nil {
|
||||
t.Fatalf("failed to close gzip writer: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -114,22 +114,25 @@ type PackagesListPageData struct {
|
|||
}
|
||||
|
||||
func supportedEcosystems() []string {
|
||||
// this list should be kept sorted in lexicographic order so
|
||||
// that the 'select' list in the UI will be in the expected
|
||||
// order
|
||||
return []string{
|
||||
"npm",
|
||||
"cargo",
|
||||
"gem",
|
||||
"golang",
|
||||
"hex",
|
||||
"pub",
|
||||
"pypi",
|
||||
"maven",
|
||||
"nuget",
|
||||
"composer",
|
||||
"conan",
|
||||
"conda",
|
||||
"cran",
|
||||
"oci",
|
||||
"deb",
|
||||
"gem",
|
||||
"golang",
|
||||
"hex",
|
||||
"maven",
|
||||
"npm",
|
||||
"nuget",
|
||||
"oci",
|
||||
"pub",
|
||||
"pypi",
|
||||
"rpm",
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -335,25 +335,6 @@ func TestSearchPage_EcosystemFilter(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestGetStripPrefix(t *testing.T) {
|
||||
tests := []struct {
|
||||
ecosystem string
|
||||
want string
|
||||
}{
|
||||
{"npm", "package/"},
|
||||
{"cargo", ""},
|
||||
{"pypi", ""},
|
||||
{"gem", ""},
|
||||
{"", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := getStripPrefix(tt.ecosystem)
|
||||
if got != tt.want {
|
||||
t.Errorf("getStripPrefix(%q) = %q, want %q", tt.ecosystem, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEcosystemBadgeLabel(t *testing.T) {
|
||||
tests := []struct {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue