diff --git a/README.md b/README.md index 8da1165..242f378 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,9 @@ cooldown: A 3-day cooldown means that when `lodash` publishes version `4.18.0`, your builds keep using `4.17.21` until 3 days have passed. If the new release turns out to be compromised, you were never exposed. -Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default and carve out exceptions for packages where you need faster updates. See [docs/configuration.md](docs/configuration.md) for the full config reference. +Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default and carve out exceptions for packages where you need faster updates. + +Currently works with npm, PyPI, pub.dev, Composer, and Cargo, which all include publish timestamps in their metadata. See [docs/configuration.md](docs/configuration.md) for the full config reference. ## Supported Registries @@ -28,7 +30,7 @@ Resolution order: package override, then ecosystem override, then global default | Cargo | Rust | Yes | ✓ | | RubyGems | Ruby | Yes | ✓ | | Go proxy | Go | | ✓ | -| Hex | Elixir | Yes* | ✓ | +| Hex | Elixir | | ✓ | | pub.dev | Dart | Yes | ✓ | | PyPI | Python | Yes | ✓ | | Maven | Java | | ✓ | @@ -50,8 +52,6 @@ Resolution order: package override, then ecosystem override, then global default Cooldown requires publish timestamps in metadata. Registries without a "Yes" in the cooldown column either don't expose timestamps or haven't been wired up yet. -\* Hex cooldown requires disabling registry signature verification (`HEX_NO_VERIFY_REPO_ORIGIN=1`) since the proxy re-encodes the protobuf payload. - ## Quick Start ```bash diff --git a/docs/configuration.md b/docs/configuration.md index 68ace5f..7e1ef4b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -209,9 +209,7 @@ Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages. -Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata. - -Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. +Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, and RubyGems. These ecosystems include publish timestamps in their metadata. ## Docker diff --git a/go.mod b/go.mod index 805edf0..0f5f271 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/git-pkgs/proxy go 1.25.6 require ( - github.com/git-pkgs/archives v0.2.2 + github.com/git-pkgs/archives v0.2.0 github.com/git-pkgs/enrichment v0.2.1 github.com/git-pkgs/purl v0.1.10 github.com/git-pkgs/registries v0.4.0 @@ -17,7 +17,6 @@ require ( github.com/prometheus/client_model v0.6.2 github.com/swaggo/swag v1.16.6 gocloud.dev v0.45.0 - google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.47.0 ) @@ -285,6 +284,7 @@ require ( google.golang.org/api v0.269.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect google.golang.org/grpc v1.79.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect honnef.co/go/tools v0.7.0 // indirect diff --git a/go.sum b/go.sum index 6e68fb7..63f37e1 100644 --- a/go.sum +++ b/go.sum @@ -224,8 +224,8 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0= github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI= -github.com/git-pkgs/archives v0.2.2 h1:RxOjrV8RzKicbMVdf2GDKOqIOHZNVjrLY/Pc7KSE/WQ= -github.com/git-pkgs/archives v0.2.2/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU= +github.com/git-pkgs/archives v0.2.0 h1:8OuuGwAB+Eww8/1ayyYpZzP0wVEH0/VWBG3mQrfi9SM= +github.com/git-pkgs/archives v0.2.0/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU= github.com/git-pkgs/enrichment v0.2.1 h1:mJJt4YQBzl9aOfu4226ylnC9H6YO9YZDjGpbSPVahKc= github.com/git-pkgs/enrichment v0.2.1/go.mod h1:q9eDZpRrUbYwzD4Mtg/T6LRdBMlt2DYRIvVRDULFnKg= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= diff --git a/internal/handler/composer.go b/internal/handler/composer.go index d47a0f2..b9edbdd 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "net/http" - "path" "strings" "time" @@ -183,10 +182,9 @@ func expandMinifiedVersions(versionList []any) []any { } // Merge inherited fields into a new map, then overlay current fields. - // Deep copy values to avoid shared references between versions. merged := make(map[string]any, len(inherited)+len(vmap)) for k, val := range inherited { - merged[k] = deepCopyValue(val) + merged[k] = val } for k, val := range vmap { merged[k] = val @@ -201,26 +199,6 @@ func expandMinifiedVersions(versionList []any) []any { return expanded } -// deepCopyValue returns a deep copy of JSON-like values (maps, slices, scalars). -func deepCopyValue(v any) any { - switch val := v.(type) { - case map[string]any: - m := make(map[string]any, len(val)) - for k, v := range val { - m[k] = deepCopyValue(v) - } - return m - case []any: - s := make([]any, len(val)) - for i, v := range val { - s[i] = deepCopyValue(v) - } - return s - default: - return v - } -} - // filterAndRewriteVersions applies cooldown filtering and rewrites dist URLs // for a single package's version list. func (h *ComposerHandler) filterAndRewriteVersions(packageName string, versionList []any) []any { @@ -288,14 +266,6 @@ func (h *ComposerHandler) rewriteDistURL(vmap map[string]any, packageName, versi filename = url[idx+1:] } - // GitHub zipball URLs end with a bare commit hash (no extension). - // Append .zip so the archives library can detect the format. - if path.Ext(filename) == "" { - if distType, _ := dist["type"].(string); distType == "zip" { - filename += ".zip" - } - } - parts := strings.SplitN(packageName, "/", vendorPackageParts) if len(parts) == vendorPackageParts { newURL := fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", diff --git a/internal/handler/composer_test.go b/internal/handler/composer_test.go index 94ff8cb..89a4c33 100644 --- a/internal/handler/composer_test.go +++ b/internal/handler/composer_test.go @@ -3,7 +3,6 @@ package handler import ( "encoding/json" "log/slog" - "strings" "testing" "time" @@ -246,151 +245,6 @@ func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) { } } -func TestComposerRewriteDistURLGitHubZipball(t *testing.T) { - // GitHub zipball URLs end with a bare commit hash, no file extension. - // The proxy must produce a filename with .zip extension so that the - // archives library can detect the format when browsing source. - h := &ComposerHandler{ - proxy: testProxy(), - proxyURL: "http://localhost:8080", - } - - vmap := map[string]any{ - "version": "v7.4.8", - "dist": map[string]any{ - "url": "https://api.github.com/repos/symfony/asset/zipball/d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", - "type": "zip", - "shasum": "", - "reference": "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", - }, - } - - h.rewriteDistURL(vmap, "symfony/asset", "v7.4.8") - - dist := vmap["dist"].(map[string]any) - url := dist["url"].(string) - - // The rewritten URL's filename must have a .zip extension - if !strings.HasSuffix(url, ".zip") { - t.Errorf("rewritten dist URL filename has no .zip extension: %s", url) - } -} - -func TestComposerRewriteMetadataGitHubZipballFilenames(t *testing.T) { - // End-to-end: metadata with GitHub zipball URLs should produce - // download URLs that end in .zip so browse source can open them. - h := &ComposerHandler{ - proxy: testProxy(), - proxyURL: "http://localhost:8080", - } - - input := `{ - "packages": { - "symfony/config": [ - { - "version": "v7.4.8", - "dist": { - "url": "https://api.github.com/repos/symfony/config/zipball/c7369cc1da250fcbfe0c5a9d109e419661549c39", - "type": "zip", - "reference": "c7369cc1da250fcbfe0c5a9d109e419661549c39" - } - } - ] - } - }` - - output, err := h.rewriteMetadata([]byte(input)) - if err != nil { - t.Fatalf("rewriteMetadata failed: %v", err) - } - - var result map[string]any - if err := json.Unmarshal(output, &result); err != nil { - t.Fatalf("failed to parse output: %v", err) - } - - packages := result["packages"].(map[string]any) - versions := packages["symfony/config"].([]any) - v := versions[0].(map[string]any) - dist := v["dist"].(map[string]any) - url := dist["url"].(string) - - if !strings.HasSuffix(url, ".zip") { - t.Errorf("rewritten URL should end in .zip, got %s", url) - } -} - -func TestComposerExpandMinifiedSharedDistReferences(t *testing.T) { - // When a minified version inherits the dist field from a previous version - // (i.e. it doesn't include its own dist), expanding + rewriting must not - // corrupt the dist URLs via shared map references. - h := &ComposerHandler{ - proxy: testProxy(), - proxyURL: "http://localhost:8080", - } - - // In this minified payload, v5.3.0 does NOT include a dist field, - // so it inherits v5.4.0's dist. After expansion and URL rewriting, - // each version must have its own correct dist URL. - input := `{ - "minified": "composer/2.0", - "packages": { - "vendor/pkg": [ - { - "name": "vendor/pkg", - "version": "5.4.0", - "dist": { - "url": "https://api.github.com/repos/vendor/pkg/zipball/aaa111", - "type": "zip", - "reference": "aaa111" - } - }, - { - "version": "5.3.0" - } - ] - } - }` - - output, err := h.rewriteMetadata([]byte(input)) - if err != nil { - t.Fatalf("rewriteMetadata failed: %v", err) - } - - var result map[string]any - if err := json.Unmarshal(output, &result); err != nil { - t.Fatalf("failed to parse output: %v", err) - } - - packages := result["packages"].(map[string]any) - versions := packages["vendor/pkg"].([]any) - if len(versions) != 2 { - t.Fatalf("expected 2 versions, got %d", len(versions)) - } - - v1 := versions[0].(map[string]any) - v2 := versions[1].(map[string]any) - - dist1 := v1["dist"].(map[string]any) - dist2 := v2["dist"].(map[string]any) - - url1 := dist1["url"].(string) - url2 := dist2["url"].(string) - - // Each version must have its own URL with its own version in the path - if !strings.Contains(url1, "/5.4.0/") { - t.Errorf("v5.4.0 dist URL should contain /5.4.0/, got %s", url1) - } - if !strings.Contains(url2, "/5.3.0/") { - t.Errorf("v5.3.0 dist URL should contain /5.3.0/, got %s", url2) - } - - // The two URLs must be different - if url1 == url2 { - t.Errorf("both versions have the same dist URL (shared reference bug): %s", url1) - } -} - func TestComposerRewriteMetadataCooldown(t *testing.T) { now := time.Now() old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339) diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 990fb55..4e0f2a2 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -1,17 +1,8 @@ package handler import ( - "bytes" - "compress/gzip" - "encoding/json" - "fmt" - "io" "net/http" "strings" - "time" - - "github.com/git-pkgs/purl" - "google.golang.org/protobuf/encoding/protowire" ) const ( @@ -44,7 +35,7 @@ func (h *HexHandler) Routes() http.Handler { // Registry resources (proxy without caching) mux.HandleFunc("GET /names", h.proxyUpstream) mux.HandleFunc("GET /versions", h.proxyUpstream) - mux.HandleFunc("GET /packages/{name}", h.handlePackages) + mux.HandleFunc("GET /packages/{name}", h.proxyUpstream) // Public keys mux.HandleFunc("GET /public_key", h.proxyUpstream) @@ -94,329 +85,6 @@ func (h *HexHandler) parseTarballFilename(filename string) (name, version string return "", "" } -// hexAPIURL is the Hex HTTP API base URL for fetching package metadata with timestamps. -const hexAPIURL = "https://hex.pm" - -// handlePackages proxies the /packages/{name} endpoint, applying cooldown filtering -// when enabled. Since the protobuf format has no timestamps, we fetch them from the -// Hex HTTP API concurrently. -func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - h.proxyUpstream(w, r) - return - } - - name := r.PathValue("name") - if name == "" { - h.proxyUpstream(w, r) - return - } - - h.proxy.Logger.Info("hex package request with cooldown", "name", name) - - protoResp, filteredVersions, err := h.fetchPackageAndVersions(r, name) - if err != nil { - h.proxy.Logger.Error("upstream request failed", "error", err) - http.Error(w, "upstream request failed", http.StatusBadGateway) - return - } - defer func() { _ = protoResp.Body.Close() }() - - if protoResp.StatusCode != http.StatusOK { - for k, vv := range protoResp.Header { - for _, v := range vv { - w.Header().Add(k, v) - } - } - w.WriteHeader(protoResp.StatusCode) - _, _ = io.Copy(w, protoResp.Body) - return - } - - body, err := io.ReadAll(protoResp.Body) - if err != nil { - http.Error(w, "failed to read response", http.StatusInternalServerError) - return - } - - if len(filteredVersions) == 0 { - // No versions to filter or couldn't get timestamps, pass through - w.Header().Set("Content-Type", protoResp.Header.Get("Content-Type")) - w.Header().Set("Content-Encoding", "gzip") - _, _ = w.Write(body) - return - } - - filtered, err := h.filterSignedPackage(body, filteredVersions) - if err != nil { - h.proxy.Logger.Warn("failed to filter hex package, proxying original", "error", err) - w.Header().Set("Content-Type", protoResp.Header.Get("Content-Type")) - w.Header().Set("Content-Encoding", "gzip") - _, _ = w.Write(body) - return - } - - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("Content-Encoding", "gzip") - _, _ = w.Write(filtered) -} - -// fetchPackageAndVersions fetches the protobuf package and version timestamps concurrently. -func (h *HexHandler) fetchPackageAndVersions(r *http.Request, name string) (*http.Response, map[string]bool, error) { - type versionsResult struct { - filtered map[string]bool - err error - } - - versionsCh := make(chan versionsResult, 1) - go func() { - filtered, err := h.fetchFilteredVersions(r, name) - versionsCh <- versionsResult{filtered: filtered, err: err} - }() - - protoResp, err := h.fetchUpstreamPackage(r, name) - - versionsRes := <-versionsCh - - if err != nil { - return nil, nil, err - } - - if versionsRes.err != nil { - h.proxy.Logger.Warn("failed to fetch hex version timestamps, proxying unfiltered", - "name", name, "error", versionsRes.err) - return protoResp, nil, nil - } - - return protoResp, versionsRes.filtered, nil -} - -// fetchUpstreamPackage fetches the protobuf package from upstream. -func (h *HexHandler) fetchUpstreamPackage(r *http.Request, name string) (*http.Response, error) { - upstreamURL := h.upstreamURL + "/packages/" + name - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) - if err != nil { - return nil, err - } - return h.proxy.HTTPClient.Do(req) -} - -// hexRelease represents a version entry from the Hex API. -type hexRelease struct { - Version string `json:"version"` - InsertedAt string `json:"inserted_at"` -} - -// hexPackageAPI represents the Hex API response for a package. -type hexPackageAPI struct { - Releases []hexRelease `json:"releases"` -} - -// fetchFilteredVersions fetches the Hex API and returns a set of version -// strings that should be filtered out by cooldown. -func (h *HexHandler) fetchFilteredVersions(r *http.Request, name string) (map[string]bool, error) { - apiURL := fmt.Sprintf("%s/api/packages/%s", hexAPIURL, name) - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, apiURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("Accept", "application/json") - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - return nil, err - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("hex API returned %d", resp.StatusCode) - } - - var pkg hexPackageAPI - if err := json.NewDecoder(resp.Body).Decode(&pkg); err != nil { - return nil, err - } - - packagePURL := purl.MakePURLString("hex", name, "") - filtered := make(map[string]bool) - - for _, release := range pkg.Releases { - insertedAt, err := time.Parse(time.RFC3339Nano, release.InsertedAt) - if err != nil { - continue - } - - if !h.proxy.Cooldown.IsAllowed("hex", packagePURL, insertedAt) { - filtered[release.Version] = true - h.proxy.Logger.Info("cooldown: filtering hex version", - "package", name, "version", release.Version, - "published", release.InsertedAt) - } - } - - return filtered, nil -} - -// filterSignedPackage decompresses gzipped data, decodes the Signed protobuf wrapper, -// filters releases from the Package payload, and re-encodes as gzipped protobuf -// (without the original signature since the payload has changed). -func (h *HexHandler) filterSignedPackage(gzippedData []byte, filteredVersions map[string]bool) ([]byte, error) { - // Decompress gzip - gr, err := gzip.NewReader(bytes.NewReader(gzippedData)) - if err != nil { - return nil, err - } - signed, err := io.ReadAll(gr) - if err != nil { - return nil, err - } - _ = gr.Close() - - // Parse Signed message: field 1 = payload (bytes), field 2 = signature (bytes) - payload, err := extractProtobufBytes(signed, 1) - if err != nil { - return nil, fmt.Errorf("extracting payload: %w", err) - } - - // Filter releases from the Package message - filteredPayload, err := filterPackageReleases(payload, filteredVersions) - if err != nil { - return nil, fmt.Errorf("filtering releases: %w", err) - } - - // Re-encode Signed message with modified payload and no signature - var newSigned []byte - newSigned = protowire.AppendTag(newSigned, 1, protowire.BytesType) - newSigned = protowire.AppendBytes(newSigned, filteredPayload) - - // Gzip compress - var buf bytes.Buffer - gw := gzip.NewWriter(&buf) - if _, err := gw.Write(newSigned); err != nil { - return nil, err - } - if err := gw.Close(); err != nil { - return nil, err - } - - return buf.Bytes(), nil -} - -// filterPackageReleases filters releases from a Package protobuf message. -// Package: field 1 = releases (repeated), field 2 = name, field 3 = repository -func filterPackageReleases(payload []byte, filteredVersions map[string]bool) ([]byte, error) { - var result []byte - data := payload - - for len(data) > 0 { - num, wtype, n := protowire.ConsumeTag(data) - if n < 0 { - return nil, fmt.Errorf("invalid protobuf tag") - } - - tagBytes := data[:n] - data = data[n:] - - var fieldBytes []byte - switch wtype { - case protowire.BytesType: - v, vn := protowire.ConsumeBytes(data) - if vn < 0 { - return nil, fmt.Errorf("invalid protobuf bytes field") - } - fieldBytes = data[:vn] - data = data[vn:] - - if num == 1 { // releases field - version := extractReleaseVersion(v) - if filteredVersions[version] { - continue // skip this release - } - } - case protowire.VarintType: - _, vn := protowire.ConsumeVarint(data) - if vn < 0 { - return nil, fmt.Errorf("invalid protobuf varint") - } - fieldBytes = data[:vn] - data = data[vn:] - default: - return nil, fmt.Errorf("unexpected wire type %d", wtype) - } - - result = append(result, tagBytes...) - result = append(result, fieldBytes...) - } - - return result, nil -} - -// extractReleaseVersion extracts the version string from a Release protobuf message. -// Release: field 1 = version (string) -func extractReleaseVersion(release []byte) string { - data := release - for len(data) > 0 { - num, wtype, n := protowire.ConsumeTag(data) - if n < 0 { - return "" - } - data = data[n:] - - switch wtype { - case protowire.BytesType: - v, vn := protowire.ConsumeBytes(data) - if vn < 0 { - return "" - } - if num == 1 { - return string(v) - } - data = data[vn:] - case protowire.VarintType: - _, vn := protowire.ConsumeVarint(data) - if vn < 0 { - return "" - } - data = data[vn:] - default: - return "" - } - } - return "" -} - -// extractProtobufBytes extracts a bytes field from a protobuf message by field number. -func extractProtobufBytes(data []byte, fieldNum protowire.Number) ([]byte, error) { - for len(data) > 0 { - num, wtype, n := protowire.ConsumeTag(data) - if n < 0 { - return nil, fmt.Errorf("invalid protobuf tag") - } - data = data[n:] - - switch wtype { - case protowire.BytesType: - v, vn := protowire.ConsumeBytes(data) - if vn < 0 { - return nil, fmt.Errorf("invalid protobuf bytes") - } - if num == fieldNum { - return v, nil - } - data = data[vn:] - case protowire.VarintType: - _, vn := protowire.ConsumeVarint(data) - if vn < 0 { - return nil, fmt.Errorf("invalid protobuf varint") - } - data = data[vn:] - default: - return nil, fmt.Errorf("unexpected wire type %d", wtype) - } - } - return nil, fmt.Errorf("field %d not found", fieldNum) -} - // proxyUpstream forwards a request to hex.pm without caching. func (h *HexHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{"Accept"}) diff --git a/internal/handler/hex_test.go b/internal/handler/hex_test.go index 19d34b4..f8516bd 100644 --- a/internal/handler/hex_test.go +++ b/internal/handler/hex_test.go @@ -1,18 +1,8 @@ package handler import ( - "bytes" - "compress/gzip" - "encoding/json" - "io" "log/slog" - "net/http" - "net/http/httptest" "testing" - "time" - - "github.com/git-pkgs/proxy/internal/cooldown" - "google.golang.org/protobuf/encoding/protowire" ) func TestHexParseTarballFilename(t *testing.T) { @@ -37,290 +27,3 @@ func TestHexParseTarballFilename(t *testing.T) { } } } - -// buildHexRelease encodes a Release protobuf message. -func buildHexRelease(version string) []byte { - var release []byte - // field 1 = version (string) - release = protowire.AppendTag(release, 1, protowire.BytesType) - release = protowire.AppendString(release, version) - // field 2 = inner_checksum (bytes) - required - release = protowire.AppendTag(release, 2, protowire.BytesType) - release = protowire.AppendBytes(release, []byte("fakechecksum1234567890123456789012")) - // field 5 = outer_checksum (bytes) - release = protowire.AppendTag(release, 5, protowire.BytesType) - release = protowire.AppendBytes(release, []byte("outerchecksum123456789012345678901")) - return release -} - -// buildHexPackage encodes a Package protobuf message. -func buildHexPackage(name string, versions []string) []byte { - var pkg []byte - for _, v := range versions { - release := buildHexRelease(v) - pkg = protowire.AppendTag(pkg, 1, protowire.BytesType) - pkg = protowire.AppendBytes(pkg, release) - } - // field 2 = name - pkg = protowire.AppendTag(pkg, 2, protowire.BytesType) - pkg = protowire.AppendString(pkg, name) - // field 3 = repository - pkg = protowire.AppendTag(pkg, 3, protowire.BytesType) - pkg = protowire.AppendString(pkg, "hexpm") - return pkg -} - -// buildHexSigned wraps a payload in a Signed protobuf message and gzips it. -func buildHexSigned(payload []byte) []byte { - var signed []byte - signed = protowire.AppendTag(signed, 1, protowire.BytesType) - signed = protowire.AppendBytes(signed, payload) - // field 2 = signature (optional, add a fake one) - signed = protowire.AppendTag(signed, 2, protowire.BytesType) - signed = protowire.AppendBytes(signed, []byte("fakesignature")) - - var buf bytes.Buffer - gw := gzip.NewWriter(&buf) - _, _ = gw.Write(signed) - _ = gw.Close() - return buf.Bytes() -} - -func TestHexFilterPackageReleases(t *testing.T) { - pkg := buildHexPackage("phoenix", []string{testVersion100, "2.0.0", "3.0.0"}) - - filtered, err := filterPackageReleases(pkg, map[string]bool{"2.0.0": true}) - if err != nil { - t.Fatal(err) - } - - // Extract remaining versions - var versions []string - data := filtered - for len(data) > 0 { - num, wtype, n := protowire.ConsumeTag(data) - if n < 0 { - break - } - data = data[n:] - switch wtype { - case protowire.BytesType: - v, vn := protowire.ConsumeBytes(data) - if vn < 0 { - break - } - if num == 1 { // release field - version := extractReleaseVersion(v) - if version != "" { - versions = append(versions, version) - } - } - data = data[vn:] - case protowire.VarintType: - _, vn := protowire.ConsumeVarint(data) - if vn < 0 { - break - } - data = data[vn:] - } - } - - if len(versions) != 2 { - t.Fatalf("expected 2 versions, got %d: %v", len(versions), versions) - } - if versions[0] != testVersion100 || versions[1] != "3.0.0" { - t.Errorf("expected [1.0.0, 3.0.0], got %v", versions) - } -} - -func TestHexFilterSignedPackage(t *testing.T) { - pkg := buildHexPackage("phoenix", []string{testVersion100, "2.0.0"}) - gzipped := buildHexSigned(pkg) - - h := &HexHandler{ - proxy: testProxy(), - proxyURL: "http://proxy.local", - } - - filtered, err := h.filterSignedPackage(gzipped, map[string]bool{"2.0.0": true}) - if err != nil { - t.Fatal(err) - } - - // Decompress and check - gr, err := gzip.NewReader(bytes.NewReader(filtered)) - if err != nil { - t.Fatal(err) - } - signed, err := io.ReadAll(gr) - if err != nil { - t.Fatal(err) - } - - payload, err := extractProtobufBytes(signed, 1) - if err != nil { - t.Fatal(err) - } - - // Check that only version 1.0.0 remains - version := extractReleaseVersion(mustExtractFirstRelease(t, payload)) - if version != testVersion100 { - t.Errorf("expected version 1.0.0, got %s", version) - } - - // Verify no signature in the output - _, err = extractProtobufBytes(signed, 2) - if err == nil { - t.Error("expected no signature in filtered output") - } -} - -func mustExtractFirstRelease(t *testing.T, payload []byte) []byte { - t.Helper() - data := payload - for len(data) > 0 { - num, wtype, n := protowire.ConsumeTag(data) - if n < 0 { - t.Fatal("invalid protobuf") - } - data = data[n:] - if wtype == protowire.BytesType { - v, vn := protowire.ConsumeBytes(data) - if vn < 0 { - t.Fatal("invalid bytes") - } - if num == 1 { - return v - } - data = data[vn:] - } - } - t.Fatal("no release found") - return nil -} - -func TestHexExtractReleaseVersion(t *testing.T) { - release := buildHexRelease("1.2.3") - version := extractReleaseVersion(release) - if version != "1.2.3" { - t.Errorf("expected 1.2.3, got %s", version) - } -} - -func TestHexHandlePackagesWithCooldown(t *testing.T) { - now := time.Now() - oldTime := now.Add(-7 * 24 * time.Hour).Format(time.RFC3339Nano) - recentTime := now.Add(-1 * time.Hour).Format(time.RFC3339Nano) - - pkg := buildHexPackage("testpkg", []string{testVersion100, "2.0.0"}) - gzippedProto := buildHexSigned(pkg) - - apiJSON, _ := json.Marshal(hexPackageAPI{ - Releases: []hexRelease{ - {Version: testVersion100, InsertedAt: oldTime}, - {Version: "2.0.0", InsertedAt: recentTime}, - }, - }) - - // Serve both the protobuf repo and the JSON API from the same test server - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.URL.Path { - case "/packages/testpkg": - w.Header().Set("Content-Encoding", "gzip") - _, _ = w.Write(gzippedProto) - case "/api/packages/testpkg": - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(apiJSON) - default: - w.WriteHeader(http.StatusNotFound) - } - })) - defer upstream.Close() - - proxy := testProxy() - proxy.Cooldown = &cooldown.Config{ - Default: "3d", - } - - // Override hexAPIURL for testing by using the upstream URL - h := &HexHandler{ - proxy: proxy, - upstreamURL: upstream.URL, - proxyURL: "http://proxy.local", - } - - // We need to override the API URL - but it's a const. Let's test via the lower-level methods instead. - // Test fetchFilteredVersions by making a request to the API endpoint - // Actually, let me test the full flow through handlePackages - - req := httptest.NewRequest(http.MethodGet, "/packages/testpkg", nil) - req.SetPathValue("name", "testpkg") - w := httptest.NewRecorder() - - // Since hexAPIURL is a const pointing to hex.pm, we can't easily override it in tests. - // Instead test the protobuf filtering directly which is the core logic. - filtered, err := h.filterSignedPackage(gzippedProto, map[string]bool{"2.0.0": true}) - if err != nil { - t.Fatal(err) - } - - // Verify only version 1.0.0 survives - gr, _ := gzip.NewReader(bytes.NewReader(filtered)) - signed, _ := io.ReadAll(gr) - payload, _ := extractProtobufBytes(signed, 1) - - var versions []string - data := payload - for len(data) > 0 { - num, wtype, n := protowire.ConsumeTag(data) - if n < 0 { - break - } - data = data[n:] - if wtype == protowire.BytesType { - v, vn := protowire.ConsumeBytes(data) - if vn < 0 { - break - } - if num == 1 { - if ver := extractReleaseVersion(v); ver != "" { - versions = append(versions, ver) - } - } - data = data[vn:] - } - } - - if len(versions) != 1 || versions[0] != testVersion100 { - t.Errorf("expected [1.0.0], got %v", versions) - } - - _ = w - _ = req -} - -func TestHexHandlePackagesWithoutCooldown(t *testing.T) { - pkg := buildHexPackage("testpkg", []string{testVersion100}) - gzipped := buildHexSigned(pkg) - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Encoding", "gzip") - _, _ = w.Write(gzipped) - })) - defer upstream.Close() - - h := &HexHandler{ - proxy: testProxy(), // no cooldown - upstreamURL: upstream.URL, - proxyURL: "http://proxy.local", - } - - req := httptest.NewRequest(http.MethodGet, "/packages/testpkg", nil) - req.SetPathValue("name", "testpkg") - w := httptest.NewRecorder() - h.handlePackages(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) - } -} diff --git a/internal/server/browse.go b/internal/server/browse.go index 7e035d2..c3ec9f7 100644 --- a/internal/server/browse.go +++ b/internal/server/browse.go @@ -1,7 +1,6 @@ package server import ( - "bytes" "encoding/json" "fmt" "io" @@ -18,75 +17,17 @@ import ( const contentTypePlainText = "text/plain; charset=utf-8" -// archiveFilename returns a filename suitable for archive format detection. -// Some ecosystems (e.g. composer) store artifacts with bare hash filenames -// that have no extension. This adds .zip when the original has no extension -// and the content is likely a zip archive. -func archiveFilename(filename string) string { - if path.Ext(filename) == "" { - return filename + ".zip" - } - return filename -} - -// detectSingleRootDir returns the single top-level directory name if all files -// in the archive live under one common directory (e.g. GitHub zipballs use -// "repo-hash/"). Returns "" if there's no single root or the archive is flat. -func detectSingleRootDir(reader archives.Reader) string { - files, err := reader.List() - if err != nil || len(files) == 0 { +// getStripPrefix returns the path prefix to strip for a given ecosystem. +// npm packages wrap content in a "package/" directory. +func getStripPrefix(ecosystem string) string { + switch ecosystem { + case "npm": + return "package/" + default: return "" } - - var root string - for _, f := range files { - parts := strings.SplitN(f.Path, "/", 2) //nolint:mnd // split into dir + rest - if len(parts) == 0 { - continue - } - dir := parts[0] - if root == "" { - root = dir - } else if dir != root { - return "" - } - } - - if root == "" { - return "" - } - return root + "/" } -// openArchive opens a cached artifact as an archive reader, auto-detecting -// and stripping a single top-level directory prefix (like GitHub zipballs). -// For npm, the hardcoded "package/" prefix takes precedence. -func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations - fname := archiveFilename(filename) - - // npm always uses package/ prefix - if ecosystem == "npm" { - return archives.OpenWithPrefix(fname, content, "package/") - } - - // Read content into memory so we can scan then wrap with prefix - data, err := io.ReadAll(content) - if err != nil { - return nil, fmt.Errorf("reading artifact: %w", err) - } - - // Open once to detect root prefix - probe, err := archives.Open(fname, bytes.NewReader(data)) - if err != nil { - return nil, err - } - prefix := detectSingleRootDir(probe) - _ = probe.Close() - - return archives.OpenWithPrefix(fname, bytes.NewReader(data), prefix) -} - - // BrowseListResponse contains the file listing for a directory in an archives. type BrowseListResponse struct { Path string `json:"path"` @@ -233,8 +174,9 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n } defer func() { _ = artifactReader.Close() }() - // Open archive with auto-detected prefix stripping - archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem) + // Open archive with appropriate prefix stripping + stripPrefix := getStripPrefix(ecosystem) + archiveReader, err := archives.OpenWithPrefix(cachedArtifact.Filename, artifactReader, stripPrefix) if err != nil { s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename) http.Error(w, "failed to open archive", http.StatusInternalServerError) @@ -327,8 +269,9 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n } defer func() { _ = artifactReader.Close() }() - // Open archive with auto-detected prefix stripping - archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem) + // Open archive with appropriate prefix stripping + stripPrefix := getStripPrefix(ecosystem) + archiveReader, err := archives.OpenWithPrefix(cachedArtifact.Filename, artifactReader, stripPrefix) if err != nil { s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename) http.Error(w, "failed to open archive", http.StatusInternalServerError) @@ -541,7 +484,9 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, } defer func() { _ = toReader.Close() }() - fromArchive, err := openArchive(fromArtifact.Filename, fromReader, ecosystem) + stripPrefix := getStripPrefix(ecosystem) + + fromArchive, err := archives.OpenWithPrefix(fromArtifact.Filename, fromReader, stripPrefix) if err != nil { s.logger.Error("failed to open from archive", "error", err) http.Error(w, "failed to open from archive", http.StatusInternalServerError) @@ -549,7 +494,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, } defer func() { _ = fromArchive.Close() }() - toArchive, err := openArchive(toArtifact.Filename, toReader, ecosystem) + toArchive, err := archives.OpenWithPrefix(toArtifact.Filename, toReader, stripPrefix) if err != nil { s.logger.Error("failed to open to archive", "error", err) http.Error(w, "failed to open to archive", http.StatusInternalServerError) diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go index 1deaf5b..13680a5 100644 --- a/internal/server/browse_test.go +++ b/internal/server/browse_test.go @@ -2,7 +2,6 @@ package server import ( "archive/tar" - "archive/zip" "bytes" "compress/gzip" "database/sql" @@ -591,195 +590,3 @@ func TestHandleComparePage(t *testing.T) { t.Errorf("expected status 400 for invalid separator, got %d", w.Code) } } - -func TestArchiveFilename(t *testing.T) { - tests := []struct { - input string - want string - }{ - {"package.tar.gz", "package.tar.gz"}, - {"d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", "d2e2f014ccd6ec9fae8dbe6336a4164346a2a856.zip"}, - {"file.zip", "file.zip"}, - {"archive.tgz", "archive.tgz"}, - {"noext", "noext.zip"}, - } - - for _, tt := range tests { - t.Run(tt.input, func(t *testing.T) { - got := archiveFilename(tt.input) - if got != tt.want { - t.Errorf("archiveFilename(%q) = %q, want %q", tt.input, got, tt.want) - } - }) - } -} - -func TestOpenArchiveStripsSingleRootDir(t *testing.T) { - data := createZipArchive(t, map[string]string{ - "repo-abc123/README.md": "hello", - "repo-abc123/src/main.go": "package main", - "repo-abc123/go.mod": "module test", - }) - reader, err := openArchive("test.zip", bytes.NewReader(data), "composer") - if err != nil { - t.Fatalf("openArchive failed: %v", err) - } - defer func() { _ = reader.Close() }() - - files, err := reader.List() - if err != nil { - t.Fatalf("List failed: %v", err) - } - for _, f := range files { - if strings.HasPrefix(f.Path, "repo-abc123/") { - t.Errorf("file %q still has root prefix after stripping", f.Path) - } - } -} - -func TestOpenArchiveMultipleRootDirs(t *testing.T) { - data := createZipArchive(t, map[string]string{ - "src/main.go": "package main", - "docs/README.md": "hello", - }) - reader, err := openArchive("test.zip", bytes.NewReader(data), "composer") - if err != nil { - t.Fatalf("openArchive failed: %v", err) - } - defer func() { _ = reader.Close() }() - - files, err := reader.List() - if err != nil { - t.Fatalf("List failed: %v", err) - } - paths := make(map[string]bool) - for _, f := range files { - paths[f.Path] = true - } - if !paths["src/main.go"] { - t.Error("expected src/main.go to remain unchanged") - } - if !paths["docs/README.md"] { - t.Error("expected docs/README.md to remain unchanged") - } -} - -func TestOpenArchiveFlatNoSubdirs(t *testing.T) { - data := createZipArchive(t, map[string]string{ - "README.md": "hello", - "main.go": "package main", - }) - reader, err := openArchive("test.zip", bytes.NewReader(data), "composer") - if err != nil { - t.Fatalf("openArchive failed: %v", err) - } - defer func() { _ = reader.Close() }() - - files, err := reader.List() - if err != nil { - t.Fatalf("List failed: %v", err) - } - paths := make(map[string]bool) - for _, f := range files { - paths[f.Path] = true - } - if !paths["README.md"] { - t.Error("expected README.md at root") - } -} - -func TestOpenArchiveNpmUsesPackagePrefix(t *testing.T) { - data := createTarGzArchive(t, map[string]string{ - "package/README.md": "hello", - "package/index.js": "module.exports = {}", - }) - reader, err := openArchive("pkg.tgz", bytes.NewReader(data), "npm") - if err != nil { - t.Fatalf("openArchive failed: %v", err) - } - defer func() { _ = reader.Close() }() - - files, err := reader.List() - if err != nil { - t.Fatalf("List failed: %v", err) - } - for _, f := range files { - if strings.HasPrefix(f.Path, "package/") { - t.Errorf("file %q still has package/ prefix", f.Path) - } - } -} - -func TestOpenArchiveExtensionlessFilename(t *testing.T) { - data := createZipArchive(t, map[string]string{ - "repo-hash/README.md": "hello", - }) - reader, err := openArchive("d2e2f014ccd6ec9fae8dbe6336a4164346a2a856", bytes.NewReader(data), "composer") - if err != nil { - t.Fatalf("openArchive failed: %v", err) - } - defer func() { _ = reader.Close() }() - - files, err := reader.List() - if err != nil { - t.Fatalf("List failed: %v", err) - } - if len(files) == 0 { - t.Fatal("expected files in archive") - } - for _, f := range files { - if strings.HasPrefix(f.Path, "repo-hash/") { - t.Errorf("file %q still has root prefix", f.Path) - } - } -} - -func createZipArchive(t *testing.T, files map[string]string) []byte { - t.Helper() - buf := new(bytes.Buffer) - w := zip.NewWriter(buf) - - for name, content := range files { - f, err := w.Create(name) - if err != nil { - t.Fatalf("failed to create zip entry: %v", err) - } - if _, err := f.Write([]byte(content)); err != nil { - t.Fatalf("failed to write zip content: %v", err) - } - } - - if err := w.Close(); err != nil { - t.Fatalf("failed to close zip writer: %v", err) - } - return buf.Bytes() -} - -func createTarGzArchive(t *testing.T, files map[string]string) []byte { - t.Helper() - buf := new(bytes.Buffer) - gw := gzip.NewWriter(buf) - tw := tar.NewWriter(gw) - - for name, content := range files { - header := &tar.Header{ - Name: name, - Size: int64(len(content)), - Mode: 0644, - } - if err := tw.WriteHeader(header); err != nil { - t.Fatalf("failed to write tar header: %v", err) - } - if _, err := tw.Write([]byte(content)); err != nil { - t.Fatalf("failed to write tar content: %v", err) - } - } - - if err := tw.Close(); err != nil { - t.Fatalf("failed to close tar writer: %v", err) - } - if err := gw.Close(); err != nil { - t.Fatalf("failed to close gzip writer: %v", err) - } - return buf.Bytes() -} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index b935628..7c15048 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -114,25 +114,22 @@ type PackagesListPageData struct { } func supportedEcosystems() []string { - // this list should be kept sorted in lexicographic order so - // that the 'select' list in the UI will be in the expected - // order return []string{ + "npm", "cargo", + "gem", + "golang", + "hex", + "pub", + "pypi", + "maven", + "nuget", "composer", "conan", "conda", "cran", - "deb", - "gem", - "golang", - "hex", - "maven", - "npm", - "nuget", "oci", - "pub", - "pypi", + "deb", "rpm", } } diff --git a/internal/server/templates_test.go b/internal/server/templates_test.go index e19244e..7fe5cf5 100644 --- a/internal/server/templates_test.go +++ b/internal/server/templates_test.go @@ -335,6 +335,25 @@ func TestSearchPage_EcosystemFilter(t *testing.T) { } } +func TestGetStripPrefix(t *testing.T) { + tests := []struct { + ecosystem string + want string + }{ + {"npm", "package/"}, + {"cargo", ""}, + {"pypi", ""}, + {"gem", ""}, + {"", ""}, + } + + for _, tt := range tests { + got := getStripPrefix(tt.ecosystem) + if got != tt.want { + t.Errorf("getStripPrefix(%q) = %q, want %q", tt.ecosystem, got, tt.want) + } + } +} func TestEcosystemBadgeLabel(t *testing.T) { tests := []struct {