; rel="canonical"`)
- _, _ = io.WriteString(w, "// swift-tools-version: 5.9\n")
- }))
- defer upstream.Close()
-
- proxy, _, _, _ := setupTestProxy(t)
- handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes()
- req := httptest.NewRequest(http.MethodGet, "/APPLE/EXAMPLE/1.2.3/Package.swift?swift-version=5.9", nil)
- req.Header.Set("Accept", swiftAcceptManifest)
- w := httptest.NewRecorder()
- handler.ServeHTTP(w, req)
-
- if w.Code != http.StatusOK {
- t.Fatalf("status = %d, want 200", w.Code)
- }
- if gotAccept != swiftAcceptManifest {
- t.Errorf("upstream Accept = %q, want %q", gotAccept, swiftAcceptManifest)
- }
- links := strings.Join(w.Header().Values("Link"), ",")
- if !strings.Contains(links, "https://proxy.example/swift/apple/example/1.2.3/Package.swift?swift-version=5.8") {
- t.Errorf("internal manifest Link was not rewritten: %q", links)
- }
- if !strings.Contains(links, "https://github.com/apple/example") {
- t.Errorf("external canonical Link was changed: %q", links)
- }
- if got := w.Header().Get("Content-Version"); got != "1" {
- t.Errorf("Content-Version = %q, want 1", got)
- }
-}
-
-func TestSwiftSourceArchiveCachesAndPreservesSecurityMetadata(t *testing.T) {
- archive := []byte("swift source archive")
- checksumBytes := sha256.Sum256(archive)
- checksum := hex.EncodeToString(checksumBytes[:])
- signature := base64.StdEncoding.EncodeToString([]byte("signature"))
-
- upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path != "/registry/apple/example/1.2.3" {
- t.Errorf("metadata path = %q", r.URL.Path)
- }
- w.Header().Set("Content-Type", "application/json")
- _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q,"signing":{"signatureBase64Encoded":%q,"signatureFormat":"cms-1.0.0"}}]}`, checksum, signature)
- }))
- defer upstream.Close()
-
- proxy, db, _, fetcher := setupTestProxy(t)
- fetcher.artifact = &fetch.Artifact{
- Body: io.NopCloser(strings.NewReader(string(archive))),
- Size: int64(len(archive)),
- ContentType: "application/zip",
- }
- handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes()
-
- requestArchive := func(method string) *httptest.ResponseRecorder {
- req := httptest.NewRequest(method, "/apple/example/1.2.3.zip", nil)
- req.Header.Set("Accept", swiftAcceptArchive)
- w := httptest.NewRecorder()
- handler.ServeHTTP(w, req)
- return w
- }
-
- w := requestArchive(http.MethodGet)
- if w.Code != http.StatusOK {
- t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
- }
- if got := w.Body.Bytes(); string(got) != string(archive) {
- t.Errorf("archive body = %q", got)
- }
- if !fetcher.fetchCalled {
- t.Fatal("archive fetcher was not called")
- }
- if got := fetcher.fetchedURL; got != upstream.URL+"/registry/apple/example/1.2.3.zip" {
- t.Errorf("fetched URL = %q", got)
- }
- if got := fetcher.fetchedHeader.Get("Accept"); got != swiftAcceptArchive {
- t.Errorf("archive Accept = %q, want %q", got, swiftAcceptArchive)
- }
- if got := w.Header().Get("Digest"); got != "sha-256="+base64.StdEncoding.EncodeToString(checksumBytes[:]) {
- t.Errorf("Digest = %q", got)
- }
- if got := w.Header().Get("X-Swift-Package-Signature"); got != signature {
- t.Errorf("signature = %q", got)
- }
- if got := w.Header().Get("X-Swift-Package-Signature-Format"); got != "cms-1.0.0" {
- t.Errorf("signature format = %q", got)
- }
- if got := w.Header().Get("Content-Disposition"); got != `attachment; filename="example-1.2.3.zip"` {
- t.Errorf("Content-Disposition = %q", got)
- }
-
- packagePURL, versionPURL := packageurl.MakeCacheStrings("swift", "apple/example", "1.2.3")
- if strings.HasPrefix(packagePURL, "pkg:swift/") {
- t.Fatalf("registry identity produced source PURL %q", packagePURL)
- }
- versionRecord, err := db.GetVersionByPURL(versionPURL)
- if err != nil {
- t.Fatalf("cached Swift version %q not found: %v", versionPURL, err)
- }
- if versionRecord == nil {
- t.Fatalf("cached Swift version %q not found", versionPURL)
- }
- if versionRecord.PackagePURL != packagePURL {
- t.Errorf("cached package PURL = %q, want %q", versionRecord.PackagePURL, packagePURL)
- }
-
- fetcher.fetchCalled = false
- w = requestArchive(http.MethodHead)
- if w.Code != http.StatusOK {
- t.Fatalf("HEAD status = %d, want 200", w.Code)
- }
- if w.Body.Len() != 0 {
- t.Errorf("HEAD body length = %d, want 0", w.Body.Len())
- }
- if got := w.Header().Get("Content-Length"); got != fmt.Sprint(len(archive)) {
- t.Errorf("HEAD Content-Length = %q", got)
- }
-
- w = requestArchive(http.MethodGet)
- if w.Code != http.StatusOK || w.Body.String() != string(archive) {
- t.Fatalf("cached response = %d %q", w.Code, w.Body.Bytes())
- }
- if fetcher.fetchCalled {
- t.Error("cached archive contacted artifact upstream")
- }
-}
-
-func TestSwiftSourceArchiveRejectsChecksumMismatch(t *testing.T) {
- archive := []byte("unexpected archive")
- expectedChecksum := sha256.Sum256([]byte("expected archive"))
-
- upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- w.Header().Set("Content-Type", "application/json")
- _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(expectedChecksum[:]))
- }))
- defer upstream.Close()
-
- proxy, db, store, fetcher := setupTestProxy(t)
- fetcher.artifact = &fetch.Artifact{
- Body: io.NopCloser(strings.NewReader(string(archive))),
- Size: int64(len(archive)),
- ContentType: "application/zip",
- }
- handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes()
-
- w := httptest.NewRecorder()
- handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil))
-
- if w.Code != http.StatusBadGateway {
- t.Fatalf("status = %d, want 502; body: %s", w.Code, w.Body.String())
- }
- if len(store.files) != 0 {
- t.Errorf("mismatched archive remained in storage: %v", store.files)
- }
- packagePURL, versionPURL := packageurl.MakeCacheStrings("swift", "apple/example", "1.2.3")
- cached, err := db.GetCachedArtifact(packagePURL, versionPURL, "example-1.2.3.zip")
- if err != nil {
- t.Fatalf("checking cache: %v", err)
- }
- if cached != nil {
- t.Error("mismatched archive gained a cache record")
- }
-}
-
-func TestSwiftSourceArchiveCanonicalizesPackageIdentity(t *testing.T) {
- archive := []byte("swift source archive")
- checksumBytes := sha256.Sum256(archive)
- checksum := hex.EncodeToString(checksumBytes[:])
- var metadataPaths []string
-
- upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- metadataPaths = append(metadataPaths, r.URL.Path)
- w.Header().Set("Content-Type", "application/json")
- _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, checksum)
- }))
- defer upstream.Close()
-
- proxy, db, store, fetcher := setupTestProxy(t)
- handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes()
- requestArchive := func(path string) {
- fetcher.artifact = &fetch.Artifact{
- Body: io.NopCloser(strings.NewReader(string(archive))),
- Size: int64(len(archive)),
- ContentType: "application/zip",
- }
- w := httptest.NewRecorder()
- handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil))
- if w.Code != http.StatusOK {
- t.Fatalf("GET %s status = %d, want 200; body: %s", path, w.Code, w.Body.String())
- }
- }
-
- requestArchive("/apple/example/1.2.3.zip")
- requestArchive("/APPLE/EXAMPLE/1.2.3.zip")
-
- if len(store.files) != 1 {
- t.Errorf("cached files = %d, want 1", len(store.files))
- }
- for _, path := range metadataPaths {
- if path != "/apple/example/1.2.3" {
- t.Errorf("metadata path = %q, want canonical lowercase path", path)
- }
- }
-
- canonicalPURL, _ := packageurl.MakeCacheStrings("swift", "apple/example", "1.2.3")
- canonical, err := db.GetPackageByPURL(canonicalPURL)
- if err != nil {
- t.Fatalf("getting canonical package: %v", err)
- }
- if canonical == nil {
- t.Fatalf("canonical package %q not found", canonicalPURL)
- }
-
- nonCanonicalPURL, _ := packageurl.MakeCacheStrings("swift", "APPLE/EXAMPLE", "1.2.3")
- if nonCanonicalPURL != canonicalPURL {
- t.Errorf("uppercase cache PURL = %q, want %q", nonCanonicalPURL, canonicalPURL)
- }
-}
-
-func TestSwiftSourceArchiveHeadLeavesStaleCacheForTheFetch(t *testing.T) {
- archive := []byte("cached archive")
- upstreamChecksum := sha256.Sum256([]byte("upstream archive"))
-
- var probed bool
- upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if strings.HasSuffix(r.URL.Path, ".zip") {
- probed = true
- w.Header().Set("Content-Range", "bytes 0-0/456")
- w.WriteHeader(http.StatusPartialContent)
- _, _ = w.Write([]byte("x"))
- return
- }
- w.Header().Set("Content-Type", "application/json")
- _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(upstreamChecksum[:]))
- }))
- defer upstream.Close()
-
- proxy, db, store, fetcher := setupTestProxy(t)
- fetcher.artifact = &fetch.Artifact{
- Body: io.NopCloser(strings.NewReader(string(archive))),
- Size: int64(len(archive)),
- ContentType: "application/zip",
- }
- packagePURL, versionPURL := packageurl.MakeCacheStrings("swift", "apple/example", "1.2.3")
- cached, err := proxy.getOrFetchArtifactFromURLWithCachePURLs(
- context.Background(), "swift", "apple/example", "1.2.3", "example-1.2.3.zip",
- packagePURL, versionPURL, upstream.URL+"/apple/example/1.2.3.zip", nil, "",
- )
- if err != nil {
- t.Fatalf("seeding cache: %v", err)
- }
- _ = cached.Reader.Close()
-
- handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes()
- w := httptest.NewRecorder()
- handler.ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/apple/example/1.2.3.zip", nil))
-
- if w.Code != http.StatusOK {
- t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
- }
- if !probed {
- t.Error("stale cache entry was not replaced by an upstream probe")
- }
- if got := w.Header().Get("Content-Length"); got != "456" {
- t.Errorf("Content-Length = %q, want 456 from upstream probe", got)
- }
- // HEAD leaves the stale entry alone; the next GET replaces it under the
- // coalescing key.
- if len(store.files) != 1 {
- t.Errorf("HEAD must leave the stale archive in storage, got %v", store.files)
- }
- if rec, _ := db.GetCachedArtifact(packagePURL, versionPURL, "example-1.2.3.zip"); rec == nil {
- t.Error("HEAD must leave the stale cache record in place")
- }
-
- fetcher.artifact = artifactBody("upstream archive")
- w = httptest.NewRecorder()
- handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil))
- if w.Code != http.StatusOK {
- t.Fatalf("GET status = %d, want 200; body: %s", w.Code, w.Body.String())
- }
- if w.Body.String() != "upstream archive" {
- t.Errorf("GET body = %q, want the refreshed archive", w.Body.String())
- }
- rec, _ := db.GetCachedArtifact(packagePURL, versionPURL, "example-1.2.3.zip")
- if rec == nil || rec.Artifact.Digest.Encoded() != hex.EncodeToString(upstreamChecksum[:]) {
- t.Errorf("cache record after GET = %+v, want the upstream checksum", rec)
- }
-}
-
-func TestSwiftSourceArchiveRequiresReleaseMetadata(t *testing.T) {
- upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- http.Error(w, "unavailable", http.StatusServiceUnavailable)
- }))
- defer upstream.Close()
-
- proxy, _, store, fetcher := setupTestProxy(t)
- fetcher.artifact = &fetch.Artifact{
- Body: io.NopCloser(strings.NewReader("signed archive")),
- ContentType: "application/zip",
- }
- handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes()
-
- w := httptest.NewRecorder()
- handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil))
-
- if w.Code != http.StatusBadGateway {
- t.Fatalf("status = %d, want 502; body: %s", w.Code, w.Body.String())
- }
- if fetcher.fetchCalled {
- t.Error("archive was fetched without release security metadata")
- }
- if len(store.files) != 0 {
- t.Errorf("archive was cached without release security metadata: %v", store.files)
- }
-}
-
-func TestSwiftSourceArchiveColdHeadUsesRangeGetAcrossRedirect(t *testing.T) {
- checksum := strings.Repeat("a", sha256.Size*2)
- var archiveAccept string
- var archiveMethod string
- var archiveRange string
- download := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- archiveMethod = r.Method
- archiveRange = r.Header.Get("Range")
- w.Header().Set("Content-Range", "bytes 0-0/123")
- w.WriteHeader(http.StatusPartialContent)
- _, _ = w.Write([]byte("x"))
- }))
- defer download.Close()
-
- upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- switch r.URL.Path {
- case "/apple/example/1.2.3":
- w.Header().Set("Content-Type", "application/json")
- _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, checksum)
- case "/apple/example/1.2.3.zip":
- archiveAccept = r.Header.Get("Accept")
- http.Redirect(w, r, download.URL, http.StatusSeeOther)
- default:
- http.NotFound(w, r)
- }
- }))
- defer upstream.Close()
-
- proxy, _, _, _ := setupTestProxy(t)
- proxy.HTTPClient = upstream.Client()
- handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes()
-
- w := httptest.NewRecorder()
- handler.ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/apple/example/1.2.3.zip", nil))
-
- if w.Code != http.StatusOK {
- t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
- }
- if archiveMethod != http.MethodGet {
- t.Errorf("download method = %q, want GET", archiveMethod)
- }
- if archiveRange != "bytes=0-0" {
- t.Errorf("download Range = %q, want bytes=0-0", archiveRange)
- }
- if archiveAccept != swiftAcceptArchive {
- t.Errorf("upstream Accept = %q, want %q", archiveAccept, swiftAcceptArchive)
- }
- if got := w.Header().Get("Content-Length"); got != "123" {
- t.Errorf("Content-Length = %q, want 123", got)
- }
-}
-
-func TestSwiftIdentifiersAndPublishingUnsupported(t *testing.T) {
- upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path != "/registry/identifiers" {
- t.Errorf("upstream path = %q", r.URL.Path)
- }
- if got := r.URL.Query().Get("url"); got != "https://github.com/apple/example" {
- t.Errorf("lookup URL = %q", got)
- }
- w.Header().Set("Content-Type", "application/json")
- _, _ = io.WriteString(w, `{"identifiers":["apple.example"]}`)
- }))
- defer upstream.Close()
-
- proxy, _, _, _ := setupTestProxy(t)
- handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL+"/registry").Routes()
-
- req := httptest.NewRequest(http.MethodGet, "/identifiers?url=https%3A%2F%2Fgithub.com%2Fapple%2Fexample", nil)
- w := httptest.NewRecorder()
- handler.ServeHTTP(w, req)
- if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), "apple.example") {
- t.Fatalf("identifier response = %d %q", w.Code, w.Body.String())
- }
-
- req = httptest.NewRequest(http.MethodGet, "/identifiers", nil)
- w = httptest.NewRecorder()
- handler.ServeHTTP(w, req)
- if w.Code != http.StatusBadRequest {
- t.Errorf("missing URL status = %d, want 400", w.Code)
- }
-
- req = httptest.NewRequest(http.MethodPut, "/apple/example/1.2.3", strings.NewReader("ignored"))
- w = httptest.NewRecorder()
- handler.ServeHTTP(w, req)
- if w.Code != http.StatusMethodNotAllowed {
- t.Errorf("publish status = %d, want 405", w.Code)
- }
- if got := w.Header().Get("Allow"); got != "GET, HEAD" {
- t.Errorf("Allow = %q", got)
- }
-}
-
-func TestSwiftIdentifierValidation(t *testing.T) {
- tests := []struct {
- name string
- value string
- valid func(string) bool
- want bool
- }{
- {"scope", "apple", validSwiftScope, true},
- {"scope hyphen", "swift-server", validSwiftScope, true},
- {"scope underscore", "swift_server", validSwiftScope, false},
- {"scope repeated separator", "swift--server", validSwiftScope, false},
- {"package", "swift-argument_parser", validSwiftPackageName, true},
- {"package repeated separators", "swift-_argument", validSwiftPackageName, false},
- {"package trailing separator", "example-", validSwiftPackageName, false},
- {"package non-ASCII", "café", validSwiftPackageName, false},
- }
-
- for _, test := range tests {
- t.Run(test.name, func(t *testing.T) {
- if got := test.valid(test.value); got != test.want {
- t.Errorf("validation of %q = %v, want %v", test.value, got, test.want)
- }
- })
- }
-}
diff --git a/internal/handler/upstream_test.go b/internal/handler/upstream_test.go
deleted file mode 100644
index 7839eaa..0000000
--- a/internal/handler/upstream_test.go
+++ /dev/null
@@ -1,130 +0,0 @@
-package handler
-
-import (
- "io"
- "net/http"
- "net/http/httptest"
- "testing"
-)
-
-func TestHandlerUpstreamConfiguration(t *testing.T) {
- const (
- proxyURL = "https://proxy.example.com/"
- baseURL = "https://upstream.example.com"
- )
- hex := NewHexHandlerWithUpstreams(nil, proxyURL, baseURL+"/hex/", baseURL+"/hex-api/")
- pypi := NewPyPIHandlerWithUpstreams(nil, proxyURL, baseURL+"/pypi/", baseURL+"/pypi-download/")
- nuget := NewNuGetHandlerWithUpstreams(nil, proxyURL, baseURL+"/nuget/", baseURL+"/nuget-search/")
- composer := NewComposerHandlerWithUpstreams(
- nil, proxyURL, baseURL+"/composer/", baseURL+"/composer-repository/",
- )
-
- got := map[string]string{
- "gem": NewGemHandlerWithUpstream(nil, proxyURL, baseURL+"/gem/").upstreamURL,
- "go": NewGoHandlerWithUpstream(nil, proxyURL, baseURL+"/go/").upstreamURL,
- "hex": hex.upstreamURL,
- "hex_api": hex.apiURL,
- "pub": NewPubHandlerWithUpstream(nil, proxyURL, baseURL+"/pub/").upstreamURL,
- "pypi": pypi.upstreamURL,
- "pypi_download": pypi.downloadURL,
- "nuget": nuget.upstreamURL,
- "nuget_search": nuget.searchURL,
- "composer": composer.upstreamURL,
- "composer_repository": composer.repoURL,
- "conan": NewConanHandlerWithUpstream(nil, proxyURL, baseURL+"/conan/").upstreamURL,
- "conda": NewCondaHandlerWithUpstream(nil, proxyURL, baseURL+"/conda/").upstreamURL,
- "cran": NewCRANHandlerWithUpstream(nil, proxyURL, baseURL+"/cran/").upstreamURL,
- "julia": NewJuliaHandlerWithUpstream(nil, baseURL+"/julia/").upstreamURL,
- "oci_default": NewContainerHandlerWithRegistry(nil, proxyURL, baseURL+"/oci/").registryURL,
- "rpm": NewRPMHandlerWithUpstream(nil, proxyURL, baseURL+"/rpm/").upstreamURL,
- }
-
- want := map[string]string{
- "gem": baseURL + "/gem",
- "go": baseURL + "/go",
- "hex": baseURL + "/hex",
- "hex_api": baseURL + "/hex-api",
- "pub": baseURL + "/pub",
- "pypi": baseURL + "/pypi",
- "pypi_download": baseURL + "/pypi-download",
- "nuget": baseURL + "/nuget",
- "nuget_search": baseURL + "/nuget-search",
- "composer": baseURL + "/composer",
- "composer_repository": baseURL + "/composer-repository",
- "conan": baseURL + "/conan",
- "conda": baseURL + "/conda",
- "cran": baseURL + "/cran",
- "julia": baseURL + "/julia",
- "oci_default": baseURL + "/oci",
- "rpm": baseURL + "/rpm",
- }
-
- for name, wantURL := range want {
- if gotURL := got[name]; gotURL != wantURL {
- t.Errorf("%s upstream = %q, want %q", name, gotURL, wantURL)
- }
- }
-}
-
-func TestConfiguredUpstreamURL(t *testing.T) {
- if got := configuredUpstreamURL("", "https://default.example.com/"); got != "https://default.example.com" {
- t.Errorf("empty configured URL = %q, want default", got)
- }
- if got := configuredUpstreamURL("https://custom.example.com/", "https://default.example.com"); got != "https://custom.example.com" {
- t.Errorf("configured URL = %q, want trimmed custom URL", got)
- }
- if got := configuredUpstreamURL("https://custom.example.com///", "https://default.example.com"); got != "https://custom.example.com" {
- t.Errorf("configured URL with trailing slashes = %q, want trimmed custom URL", got)
- }
-}
-
-func TestHexHandlerUsesConfiguredAPIUpstream(t *testing.T) {
- var requestedPath string
- upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- requestedPath = r.URL.Path
- _, _ = io.WriteString(w, `{"releases":[]}`)
- }))
- defer upstream.Close()
-
- h := NewHexHandlerWithUpstreams(
- &Proxy{HTTPClient: upstream.Client()},
- "https://proxy.example.com",
- upstream.URL+"/hex",
- upstream.URL+"/hex-api",
- )
- _, err := h.fetchFilteredVersions(httptest.NewRequest(http.MethodGet, "/", nil), "demo")
- if err != nil {
- t.Fatalf("fetchFilteredVersions failed: %v", err)
- }
- if requestedPath != "/hex-api/api/packages/demo" {
- t.Errorf("API path = %q, want %q", requestedPath, "/hex-api/api/packages/demo")
- }
-}
-
-func TestPyPIHandlerRewritesConfiguredDownloadUpstream(t *testing.T) {
- h := NewPyPIHandlerWithUpstreams(
- nil,
- "https://proxy.example.com",
- "https://upstream.example.com/pypi",
- "https://upstream.example.com/pypi",
- )
- body := []byte(`demo `)
- want := `demo `
- if got := string(h.rewriteSimpleHTML(body, nil)); got != want {
- t.Errorf("rewritten HTML = %q, want %q", got, want)
- }
-}
-
-func TestNuGetHandlerUsesConfiguredSearchUpstream(t *testing.T) {
- h := NewNuGetHandlerWithUpstreams(
- nil,
- "https://proxy.example.com",
- "https://upstream.example.com/nuget",
- "https://upstream.example.com/nuget-search",
- )
- req := httptest.NewRequest(http.MethodGet, "/query?q=demo", nil)
- want := "https://upstream.example.com/nuget-search/query?q=demo"
- if got := h.buildUpstreamURL(req); got != want {
- t.Errorf("search URL = %q, want %q", got, want)
- }
-}
diff --git a/internal/httpclient/access_log.go b/internal/httpclient/access_log.go
deleted file mode 100644
index 8e13f23..0000000
--- a/internal/httpclient/access_log.go
+++ /dev/null
@@ -1,74 +0,0 @@
-package httpclient
-
-import (
- "log/slog"
- "net/http"
- "net/url"
- "strings"
- "time"
-
- "github.com/git-pkgs/proxy/internal/accesslog"
-)
-
-type accessLogTransport struct {
- base http.RoundTripper
- accessLog *accesslog.Logger
- logger *slog.Logger
-}
-
-// NewAccessLogTransport records each upstream HTTP exchange around base.
-func NewAccessLogTransport(base http.RoundTripper, log *accesslog.Logger, logger *slog.Logger) http.RoundTripper {
- if base == nil {
- base = http.DefaultTransport
- }
- if logger == nil {
- logger = slog.Default()
- }
- if log == nil {
- return base
- }
- return &accessLogTransport{
- base: base,
- accessLog: log,
- logger: logger,
- }
-}
-
-func (t *accessLogTransport) RoundTrip(req *http.Request) (*http.Response, error) {
- start := time.Now()
- resp, err := t.base.RoundTrip(req)
-
- entry := accesslog.Entry{
- Event: accesslog.EventUpstream,
- RequestID: accesslog.RequestID(req.Context()),
- Method: req.Method,
- URL: accesslog.URLWithoutSecrets(req.URL),
- DurationMS: time.Since(start).Milliseconds(),
- }
- if resp != nil {
- entry.StatusCode = resp.StatusCode
- }
- if err != nil {
- entry.Error = errorWithoutSecrets(err, req.URL)
- }
- if writeErr := t.accessLog.Write(entry); writeErr != nil {
- t.logger.Error("failed to write access log", "error", writeErr)
- }
-
- return resp, err
-}
-
-func errorWithoutSecrets(err error, requestURL *url.URL) string {
- message := err.Error()
- if requestURL == nil {
- return message
- }
-
- cleanURL := accesslog.URLWithoutSecrets(requestURL)
- for _, value := range []string{requestURL.String(), requestURL.Redacted()} {
- if value != "" {
- message = strings.ReplaceAll(message, value, cleanURL)
- }
- }
- return message
-}
diff --git a/internal/httpclient/access_log_test.go b/internal/httpclient/access_log_test.go
deleted file mode 100644
index aa3a43c..0000000
--- a/internal/httpclient/access_log_test.go
+++ /dev/null
@@ -1,121 +0,0 @@
-package httpclient
-
-import (
- "bufio"
- "encoding/json"
- "errors"
- "io"
- "log/slog"
- "net/http"
- "os"
- "path/filepath"
- "strings"
- "testing"
-
- "github.com/git-pkgs/proxy/internal/accesslog"
-)
-
-type roundTripFunc func(*http.Request) (*http.Response, error)
-
-func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
- return f(req)
-}
-
-func TestAccessLogTransportRecordsUpstreamStatus(t *testing.T) {
- path := filepath.Join(t.TempDir(), "access.jsonl")
- accessLogger, err := accesslog.Open(path)
- if err != nil {
- t.Fatal(err)
- }
-
- base := roundTripFunc(func(req *http.Request) (*http.Response, error) {
- return &http.Response{
- StatusCode: http.StatusTooManyRequests,
- Body: io.NopCloser(strings.NewReader("rate limited")),
- Request: req,
- }, nil
- })
- client := &http.Client{Transport: NewAccessLogTransport(base, accessLogger, slog.Default())}
- req, err := http.NewRequest(http.MethodGet, "https://user:password@registry.example/package.tgz?token=secret", nil)
- if err != nil {
- t.Fatal(err)
- }
- req = req.WithContext(accesslog.WithRequestID(req.Context(), "request-123"))
-
- resp, err := client.Do(req)
- if err != nil {
- t.Fatal(err)
- }
- _ = resp.Body.Close()
- if err := accessLogger.Close(); err != nil {
- t.Fatal(err)
- }
-
- entry := readAccessLogEntry(t, path)
- if entry.Event != accesslog.EventUpstream {
- t.Errorf("event = %q, want %q", entry.Event, accesslog.EventUpstream)
- }
- if entry.RequestID != "request-123" {
- t.Errorf("request_id = %q, want %q", entry.RequestID, "request-123")
- }
- if entry.StatusCode != http.StatusTooManyRequests {
- t.Errorf("status_code = %d, want %d", entry.StatusCode, http.StatusTooManyRequests)
- }
- if entry.URL != "https://registry.example/package.tgz" {
- t.Errorf("url = %q, want URL without credentials or query", entry.URL)
- }
-}
-
-func TestAccessLogTransportRecordsUpstreamError(t *testing.T) {
- path := filepath.Join(t.TempDir(), "access.jsonl")
- accessLogger, err := accesslog.Open(path)
- if err != nil {
- t.Fatal(err)
- }
-
- wantErr := errors.New("GET https://user:password@registry.example/package.tgz?token=secret: connection refused")
- base := roundTripFunc(func(*http.Request) (*http.Response, error) {
- return nil, wantErr
- })
- client := &http.Client{Transport: NewAccessLogTransport(base, accessLogger, slog.Default())}
-
- _, err = client.Get("https://user:password@registry.example/package.tgz?token=secret")
- if !errors.Is(err, wantErr) {
- t.Fatalf("GET error = %v, want %v", err, wantErr)
- }
- if err := accessLogger.Close(); err != nil {
- t.Fatal(err)
- }
-
- entry := readAccessLogEntry(t, path)
- if entry.StatusCode != 0 {
- t.Errorf("status_code = %d, want 0", entry.StatusCode)
- }
- if strings.Contains(entry.Error, "password") || strings.Contains(entry.Error, "secret") {
- t.Errorf("error contains URL credentials or query: %q", entry.Error)
- }
- if !strings.Contains(entry.Error, "connection refused") {
- t.Errorf("error = %q, want connection failure", entry.Error)
- }
-}
-
-func readAccessLogEntry(t *testing.T, path string) accesslog.Entry {
- t.Helper()
-
- file, err := os.Open(path)
- if err != nil {
- t.Fatal(err)
- }
- defer func() { _ = file.Close() }()
-
- scanner := bufio.NewScanner(file)
- if !scanner.Scan() {
- t.Fatalf("access log is empty: %v", scanner.Err())
- }
-
- var entry accesslog.Entry
- if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {
- t.Fatalf("decoding access log: %v", err)
- }
- return entry
-}
diff --git a/internal/httpclient/transport.go b/internal/httpclient/transport.go
deleted file mode 100644
index 1ee9597..0000000
--- a/internal/httpclient/transport.go
+++ /dev/null
@@ -1,507 +0,0 @@
-// Package httpclient provides authentication-aware HTTP transports for upstream requests.
-package httpclient
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "net"
- "net/http"
- "net/url"
- "strings"
- "sync"
- "time"
-)
-
-const (
- defaultTokenLifetime = 60 * time.Second
- tokenExpirySkew = 5 * time.Second
- maxTokenResponseSize = 1 << 20
- shortTokenSkewDivisor = 10
- tokenMaxRetries = 3
- tokenRetryBaseDelay = 500 * time.Millisecond
-)
-
-// AuthFunc returns a configured authentication header for a URL.
-type AuthFunc func(url string) (headerName, headerValue string)
-
-// Transport adds configured authentication and follows OCI Bearer challenges.
-type Transport struct {
- base http.RoundTripper
- authForURL AuthFunc
- retryWait func(context.Context, time.Duration) error
-
- mu sync.Mutex
- tokens map[string]cachedToken
- challenges map[string]bearerChallenge
-}
-
-type cachedToken struct {
- value string
- expiresAt time.Time
-}
-
-type bearerChallenge struct {
- realm string
- service string
- scopes []string
-}
-
-type tokenResponse struct {
- Token string `json:"token"`
- AccessToken string `json:"access_token"`
- ExpiresIn int64 `json:"expires_in"`
- IssuedAt string `json:"issued_at"`
-}
-
-// NewTransport creates an authentication-aware transport around base.
-func NewTransport(base http.RoundTripper, authForURL AuthFunc) *Transport {
- if base == nil {
- base = http.DefaultTransport
- }
- return &Transport{
- base: base,
- authForURL: authForURL,
- retryWait: waitForRetry,
- tokens: make(map[string]cachedToken),
- challenges: make(map[string]bearerChallenge),
- }
-}
-
-// RoundTrip implements http.RoundTripper.
-func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
- hasExplicitAuthorization := req.Header.Get("Authorization") != ""
- outbound := cloneRequest(req)
- t.applyAuthentication(outbound, hasExplicitAuthorization)
-
- resp, err := t.base.RoundTrip(outbound)
- if err != nil || resp.StatusCode != http.StatusUnauthorized {
- return resp, err
- }
- if hasExplicitAuthorization {
- return resp, nil
- }
- if registryProtectionSpace(req.URL) == "" {
- return resp, nil
- }
-
- challenge, ok := parseBearerChallenge(resp.Header.Values("WWW-Authenticate"))
- if !ok || !canReplay(req) {
- return resp, nil
- }
-
- drainAndClose(resp.Body)
- token, err := t.token(req.Context(), challenge)
- if err != nil {
- return nil, fmt.Errorf("registry authentication: %w", err)
- }
- t.rememberChallenge(req.URL, challenge)
-
- retry, err := cloneRequestForRetry(req)
- if err != nil {
- return nil, err
- }
- t.applyConfiguredAuthentication(retry)
- retry.Header.Set("Authorization", "Bearer "+token)
- return t.base.RoundTrip(retry)
-}
-
-func (t *Transport) applyAuthentication(req *http.Request, hasExplicitAuthorization bool) {
- t.applyConfiguredAuthentication(req)
- if hasExplicitAuthorization {
- return
- }
- if token := t.cachedTokenForRequest(req.URL); token != "" {
- req.Header.Set("Authorization", "Bearer "+token)
- }
-}
-
-func (t *Transport) applyConfiguredAuthentication(req *http.Request) {
- if t.authForURL == nil {
- return
- }
- name, value := t.authForURL(req.URL.String())
- if name != "" && value != "" && req.Header.Get(name) == "" {
- req.Header.Set(name, value)
- }
-}
-
-func (t *Transport) token(ctx context.Context, challenge bearerChallenge) (string, error) {
- key := challenge.key()
- if token := t.cachedToken(key); token != "" {
- return token, nil
- }
-
- token, expiresAt, err := t.fetchToken(ctx, challenge)
- if err != nil {
- return "", err
- }
-
- t.cacheToken(key, cachedToken{value: token, expiresAt: expiresAt})
- return token, nil
-}
-
-func (t *Transport) cacheToken(key string, token cachedToken) {
- now := time.Now()
- t.mu.Lock()
- defer t.mu.Unlock()
-
- for cachedKey, cached := range t.tokens {
- if !now.Before(cached.expiresAt) {
- delete(t.tokens, cachedKey)
- }
- }
- t.tokens[key] = token
-}
-
-func (t *Transport) fetchToken(ctx context.Context, challenge bearerChallenge) (string, time.Time, error) {
- tokenURL, err := url.Parse(challenge.realm)
- if err != nil || !tokenURL.IsAbs() || (tokenURL.Scheme != "https" && tokenURL.Scheme != "http") {
- return "", time.Time{}, fmt.Errorf("invalid token realm %q", challenge.realm)
- }
-
- query := tokenURL.Query()
- if challenge.service != "" {
- query.Set("service", challenge.service)
- }
- for _, scope := range challenge.scopes {
- query.Add("scope", scope)
- }
- query.Set("client_id", "git-pkgs-proxy")
- tokenURL.RawQuery = query.Encode()
-
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL.String(), nil)
- if err != nil {
- return "", time.Time{}, err
- }
-
- client := &http.Client{Transport: configuredTransport{parent: t}}
- for attempt := 0; attempt <= tokenMaxRetries; attempt++ {
- resp, err := client.Do(req.Clone(ctx))
- if err != nil {
- requestErr := fmt.Errorf("requesting token: %w", err)
- if !shouldRetryTokenRequest(ctx, err) || attempt == tokenMaxRetries {
- return "", time.Time{}, requestErr
- }
- if err := t.waitForTokenRetry(ctx, attempt); err != nil {
- return "", time.Time{}, err
- }
- continue
- }
-
- if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices {
- return decodeTokenResponse(resp)
- }
-
- responseErr := tokenResponseError(resp)
- if !shouldRetryTokenStatus(resp.StatusCode) || attempt == tokenMaxRetries {
- return "", time.Time{}, responseErr
- }
- if err := t.waitForTokenRetry(ctx, attempt); err != nil {
- return "", time.Time{}, err
- }
- }
-
- return "", time.Time{}, errors.New("token request retries exhausted")
-}
-
-func decodeTokenResponse(resp *http.Response) (string, time.Time, error) {
- defer func() { _ = resp.Body.Close() }()
-
- var payload tokenResponse
- if err := json.NewDecoder(io.LimitReader(resp.Body, maxTokenResponseSize)).Decode(&payload); err != nil {
- return "", time.Time{}, fmt.Errorf("decoding token response: %w", err)
- }
- token := payload.Token
- if token == "" {
- token = payload.AccessToken
- }
- if token == "" {
- return "", time.Time{}, fmt.Errorf("token response did not contain a token")
- }
-
- issuedAt := time.Now()
- if payload.IssuedAt != "" {
- if parsed, parseErr := time.Parse(time.RFC3339, payload.IssuedAt); parseErr == nil {
- issuedAt = parsed
- }
- }
- lifetime := time.Duration(payload.ExpiresIn) * time.Second
- if lifetime <= 0 {
- lifetime = defaultTokenLifetime
- }
- expiresAt := issuedAt.Add(lifetime).Add(-expirySkew(lifetime))
- return token, expiresAt, nil
-}
-
-func tokenResponseError(resp *http.Response) error {
- defer func() { _ = resp.Body.Close() }()
- body, _ := io.ReadAll(io.LimitReader(resp.Body, maxTokenResponseSize))
- return fmt.Errorf("token service returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
-}
-
-func shouldRetryTokenRequest(ctx context.Context, err error) bool {
- if ctx.Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
- return false
- }
-
- var networkErr net.Error
- if !errors.As(err, &networkErr) {
- return false
- }
-
- var dnsErr *net.DNSError
- if errors.As(err, &dnsErr) {
- return dnsErr.IsTemporary || dnsErr.IsTimeout
- }
- return networkErr.Timeout()
-}
-
-func shouldRetryTokenStatus(status int) bool {
- return status == http.StatusTooManyRequests || status >= http.StatusInternalServerError
-}
-
-func (t *Transport) waitForTokenRetry(ctx context.Context, attempt int) error {
- delay := tokenRetryBaseDelay << attempt
- if t.retryWait != nil {
- return t.retryWait(ctx, delay)
- }
- return waitForRetry(ctx, delay)
-}
-
-func waitForRetry(ctx context.Context, delay time.Duration) error {
- timer := time.NewTimer(delay)
- defer timer.Stop()
-
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-timer.C:
- return nil
- }
-}
-
-type configuredTransport struct {
- parent *Transport
-}
-
-func (t configuredTransport) RoundTrip(req *http.Request) (*http.Response, error) {
- outbound := cloneRequest(req)
- t.parent.applyConfiguredAuthentication(outbound)
- return t.parent.base.RoundTrip(outbound)
-}
-
-func (t *Transport) cachedTokenForRequest(requestURL *url.URL) string {
- space := registryProtectionSpace(requestURL)
- if space == "" {
- return ""
- }
-
- t.mu.Lock()
- challenge, ok := t.challenges[space]
- t.mu.Unlock()
- if !ok {
- return ""
- }
- return t.cachedToken(challenge.key())
-}
-
-func (t *Transport) cachedToken(key string) string {
- now := time.Now()
- t.mu.Lock()
- defer t.mu.Unlock()
-
- token, ok := t.tokens[key]
- if !ok {
- return ""
- }
- if !now.Before(token.expiresAt) {
- delete(t.tokens, key)
- return ""
- }
- return token.value
-}
-
-func (t *Transport) rememberChallenge(requestURL *url.URL, challenge bearerChallenge) {
- space := registryProtectionSpace(requestURL)
- if space == "" {
- return
- }
- t.mu.Lock()
- t.challenges[space] = challenge
- t.mu.Unlock()
-}
-
-func (c bearerChallenge) key() string {
- return c.realm + "\x00" + c.service + "\x00" + strings.Join(c.scopes, "\x00")
-}
-
-func registryProtectionSpace(u *url.URL) string {
- const registryPrefix = "/v2/"
- if u == nil || !strings.HasPrefix(u.Path, registryPrefix) {
- return ""
- }
- rest := strings.TrimPrefix(u.Path, registryPrefix)
- end := len(rest)
- for _, marker := range []string{"/blobs/", "/manifests/", "/tags/", "/referrers/"} {
- if index := strings.Index(rest, marker); index >= 0 && index < end {
- end = index
- }
- }
- if end == len(rest) || end == 0 {
- return ""
- }
- return u.Scheme + "://" + u.Host + registryPrefix + rest[:end]
-}
-
-func parseBearerChallenge(values []string) (bearerChallenge, bool) {
- for _, value := range values {
- params, ok := bearerParameters(value)
- if !ok || params["realm"] == "" {
- continue
- }
- challenge := bearerChallenge{
- realm: params["realm"],
- service: params["service"],
- }
- if scope := params["scope"]; scope != "" {
- challenge.scopes = append(challenge.scopes, scope)
- }
- return challenge, true
- }
- return bearerChallenge{}, false
-}
-
-func bearerParameters(value string) (map[string]string, bool) {
- start := findAuthScheme(value, "Bearer")
- if start < 0 {
- return nil, false
- }
- rest := value[start+len("Bearer"):]
- params := make(map[string]string)
- for {
- rest = strings.TrimLeft(rest, " \t,")
- if rest == "" {
- break
- }
-
- keyEnd := strings.IndexAny(rest, "= \t,")
- if keyEnd <= 0 {
- break
- }
- key := strings.ToLower(rest[:keyEnd])
- rest = strings.TrimLeft(rest[keyEnd:], " \t")
- if rest == "" || rest[0] != '=' {
- break
- }
- rest = strings.TrimLeft(rest[1:], " \t")
-
- parsed, remaining, ok := parseAuthValue(rest)
- if !ok {
- return nil, false
- }
- params[key] = parsed
- rest = remaining
- }
- return params, true
-}
-
-func findAuthScheme(value, scheme string) int {
- inQuote := false
- escaped := false
- for index := 0; index+len(scheme) <= len(value); index++ {
- char := value[index]
- if escaped {
- escaped = false
- continue
- }
- if char == '\\' && inQuote {
- escaped = true
- continue
- }
- if char == '"' {
- inQuote = !inQuote
- continue
- }
- if inQuote || !strings.EqualFold(value[index:index+len(scheme)], scheme) {
- continue
- }
- beforeOK := index == 0 || value[index-1] == ',' || value[index-1] == ' ' || value[index-1] == '\t'
- after := index + len(scheme)
- afterOK := after < len(value) && (value[after] == ' ' || value[after] == '\t')
- if beforeOK && afterOK {
- return index
- }
- }
- return -1
-}
-
-func parseAuthValue(value string) (parsed, remaining string, ok bool) {
- if value == "" {
- return "", "", false
- }
- if value[0] != '"' {
- end := strings.IndexAny(value, " \t,")
- if end < 0 {
- return value, "", true
- }
- return value[:end], value[end:], end > 0
- }
-
- var builder strings.Builder
- escaped := false
- for index := 1; index < len(value); index++ {
- char := value[index]
- if escaped {
- builder.WriteByte(char)
- escaped = false
- continue
- }
- if char == '\\' {
- escaped = true
- continue
- }
- if char == '"' {
- return builder.String(), value[index+1:], true
- }
- builder.WriteByte(char)
- }
- return "", "", false
-}
-
-func cloneRequest(req *http.Request) *http.Request {
- clone := req.Clone(req.Context())
- clone.Header = req.Header.Clone()
- return clone
-}
-
-func canReplay(req *http.Request) bool {
- return req.Body == nil || req.GetBody != nil
-}
-
-func cloneRequestForRetry(req *http.Request) (*http.Request, error) {
- clone := cloneRequest(req)
- if req.Body == nil {
- return clone, nil
- }
- body, err := req.GetBody()
- if err != nil {
- return nil, fmt.Errorf("replaying authenticated request: %w", err)
- }
- clone.Body = body
- return clone, nil
-}
-
-func expirySkew(lifetime time.Duration) time.Duration {
- if lifetime < tokenExpirySkew*2 {
- return lifetime / shortTokenSkewDivisor
- }
- return tokenExpirySkew
-}
-
-func drainAndClose(body io.ReadCloser) {
- _, _ = io.Copy(io.Discard, io.LimitReader(body, maxTokenResponseSize))
- _ = body.Close()
-}
diff --git a/internal/httpclient/transport_test.go b/internal/httpclient/transport_test.go
deleted file mode 100644
index 01f3e8b..0000000
--- a/internal/httpclient/transport_test.go
+++ /dev/null
@@ -1,414 +0,0 @@
-package httpclient
-
-import (
- "context"
- "errors"
- "io"
- "net"
- "net/http"
- "net/http/httptest"
- "strings"
- "testing"
- "time"
-)
-
-type roundTripperFunc func(*http.Request) (*http.Response, error)
-
-func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) {
- return fn(req)
-}
-
-func TestTransportFollowsBearerChallengeAndCachesToken(t *testing.T) {
- var registryRequests int
- var tokenRequests int
- var server *httptest.Server
-
- server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- switch r.URL.Path {
- case "/token":
- tokenRequests++
- if got := r.URL.Query().Get("service"); got != "registry.test" {
- t.Errorf("service = %q, want %q", got, "registry.test")
- }
- if got := r.URL.Query().Get("scope"); got != "repository:library/test:pull" {
- t.Errorf("scope = %q, want %q", got, "repository:library/test:pull")
- }
- w.Header().Set("Content-Type", "application/json")
- _, _ = io.WriteString(w, `{"token":"registry-token","expires_in":3600}`)
- case "/v2/library/test/blobs/sha256:first", "/v2/library/test/blobs/sha256:second":
- registryRequests++
- if r.Header.Get("Authorization") != "Bearer registry-token" {
- w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token",service="registry.test",scope="repository:library/test:pull"`)
- http.Error(w, "authentication required", http.StatusUnauthorized)
- return
- }
- _, _ = io.WriteString(w, "blob")
- default:
- http.NotFound(w, r)
- }
- }))
- defer server.Close()
-
- client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)}
- for _, digest := range []string{"sha256:first", "sha256:second"} {
- resp, err := client.Get(server.URL + "/v2/library/test/blobs/" + digest)
- if err != nil {
- t.Fatalf("GET %s: %v", digest, err)
- }
- body, readErr := io.ReadAll(resp.Body)
- _ = resp.Body.Close()
- if readErr != nil {
- t.Fatalf("read %s response: %v", digest, readErr)
- }
- if resp.StatusCode != http.StatusOK {
- t.Fatalf("GET %s status = %d, want %d", digest, resp.StatusCode, http.StatusOK)
- }
- if string(body) != "blob" {
- t.Errorf("GET %s body = %q, want %q", digest, body, "blob")
- }
- }
-
- if tokenRequests != 1 {
- t.Errorf("token requests = %d, want 1", tokenRequests)
- }
- if registryRequests != 3 {
- t.Errorf("registry requests = %d, want 3", registryRequests)
- }
-}
-
-func TestTransportRetriesTemporaryTokenLookupFailures(t *testing.T) {
- var registryRequests int
- var tokenRequests int
- var tokenLookupFailures int
- var server *httptest.Server
-
- server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- switch r.URL.Path {
- case "/token":
- tokenRequests++
- _, _ = io.WriteString(w, `{"token":"registry-token"}`)
- case "/v2/library/test/blobs/sha256:test":
- registryRequests++
- if r.Header.Get("Authorization") != "Bearer registry-token" {
- w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token",service="registry.test",scope="repository:library/test:pull"`)
- http.Error(w, "authentication required", http.StatusUnauthorized)
- return
- }
- _, _ = io.WriteString(w, "blob")
- default:
- http.NotFound(w, r)
- }
- }))
- defer server.Close()
-
- base := roundTripperFunc(func(req *http.Request) (*http.Response, error) {
- if req.URL.Path == "/token" && tokenLookupFailures < 2 {
- tokenLookupFailures++
- return nil, &net.DNSError{Err: "server misbehaving", IsTemporary: true}
- }
- return http.DefaultTransport.RoundTrip(req)
- })
- transport := NewTransport(base, nil)
- transport.retryWait = func(context.Context, time.Duration) error { return nil }
- client := &http.Client{Transport: transport}
-
- resp, err := client.Get(server.URL + "/v2/library/test/blobs/sha256:test")
- if err != nil {
- t.Fatalf("GET blob: %v", err)
- }
- defer func() { _ = resp.Body.Close() }()
-
- if resp.StatusCode != http.StatusOK {
- t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK)
- }
- if tokenLookupFailures != 2 {
- t.Errorf("token lookup failures = %d, want 2", tokenLookupFailures)
- }
- if tokenRequests != 1 {
- t.Errorf("token requests = %d, want 1", tokenRequests)
- }
- if registryRequests != 2 {
- t.Errorf("registry requests = %d, want 2", registryRequests)
- }
-}
-
-func TestTransportDoesNotRetryPermanentTokenLookupFailures(t *testing.T) {
- var tokenRequests int
- base := roundTripperFunc(func(*http.Request) (*http.Response, error) {
- tokenRequests++
- return nil, &net.DNSError{Err: "no such host"}
- })
- transport := NewTransport(base, nil)
- transport.retryWait = func(context.Context, time.Duration) error { return nil }
-
- _, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: "https://auth.example.test/token"})
- if err == nil {
- t.Fatal("fetchToken succeeded, want error")
- }
- if tokenRequests != 1 {
- t.Errorf("token requests = %d, want 1", tokenRequests)
- }
-}
-
-func TestTransportDoesNotRetryPermanentTokenFailures(t *testing.T) {
- var tokenRequests int
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- tokenRequests++
- http.Error(w, "invalid credentials", http.StatusUnauthorized)
- }))
- defer server.Close()
-
- transport := NewTransport(http.DefaultTransport, nil)
- _, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: server.URL + "/token"})
- if err == nil {
- t.Fatal("fetchToken succeeded, want error")
- }
- if tokenRequests != 1 {
- t.Errorf("token requests = %d, want 1", tokenRequests)
- }
-}
-
-func TestTransportRetriesTokenServiceFailures(t *testing.T) {
- for _, status := range []int{http.StatusTooManyRequests, http.StatusServiceUnavailable} {
- t.Run(http.StatusText(status), func(t *testing.T) {
- var tokenRequests int
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- tokenRequests++
- http.Error(w, "temporary token service failure", status)
- }))
- defer server.Close()
-
- var delays []time.Duration
- transport := NewTransport(http.DefaultTransport, nil)
- transport.retryWait = func(_ context.Context, delay time.Duration) error {
- delays = append(delays, delay)
- return nil
- }
-
- _, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: server.URL + "/token"})
- if err == nil {
- t.Fatal("fetchToken succeeded, want error")
- }
- if tokenRequests != tokenMaxRetries+1 {
- t.Errorf("token requests = %d, want %d", tokenRequests, tokenMaxRetries+1)
- }
- wantDelays := []time.Duration{500 * time.Millisecond, time.Second, 2 * time.Second}
- if len(delays) != len(wantDelays) {
- t.Fatalf("retry delays = %v, want %v", delays, wantDelays)
- }
- for index, want := range wantDelays {
- if delays[index] != want {
- t.Errorf("retry delay %d = %s, want %s", index, delays[index], want)
- }
- }
- })
- }
-}
-
-func TestTransportStopsTokenRetriesWhenWaitingIsCancelled(t *testing.T) {
- var tokenRequests int
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- tokenRequests++
- http.Error(w, "temporary token service failure", http.StatusServiceUnavailable)
- }))
- defer server.Close()
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
- transport := NewTransport(http.DefaultTransport, nil)
- transport.retryWait = func(ctx context.Context, _ time.Duration) error {
- cancel()
- <-ctx.Done()
- return ctx.Err()
- }
-
- _, _, err := transport.fetchToken(ctx, bearerChallenge{realm: server.URL + "/token"})
- if !errors.Is(err, context.Canceled) {
- t.Errorf("fetchToken error = %v, want context canceled", err)
- }
- if tokenRequests != 1 {
- t.Errorf("token requests = %d, want 1", tokenRequests)
- }
-}
-
-func TestTransportAddsConfiguredAuthentication(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if got := r.Header.Get("X-Registry-Token"); got != "configured-token" {
- t.Errorf("X-Registry-Token = %q, want %q", got, "configured-token")
- }
- w.WriteHeader(http.StatusNoContent)
- }))
- defer server.Close()
-
- authForURL := func(url string) (string, string) {
- if strings.HasPrefix(url, server.URL) {
- return "X-Registry-Token", "configured-token"
- }
- return "", ""
- }
- client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)}
-
- resp, err := client.Get(server.URL + "/metadata")
- if err != nil {
- t.Fatalf("GET metadata: %v", err)
- }
- _ = resp.Body.Close()
- if resp.StatusCode != http.StatusNoContent {
- t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent)
- }
-}
-
-func TestTransportPreservesExplicitAuthentication(t *testing.T) {
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" {
- t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token")
- }
- w.WriteHeader(http.StatusNoContent)
- }))
- defer server.Close()
-
- authForURL := func(string) (string, string) {
- return "Authorization", "Bearer configured-token"
- }
- client := &http.Client{Transport: NewTransport(http.DefaultTransport, authForURL)}
- req, err := http.NewRequest(http.MethodGet, server.URL+"/artifact", nil)
- if err != nil {
- t.Fatal(err)
- }
- req.Header.Set("Authorization", "Bearer explicit-token")
-
- resp, err := client.Do(req)
- if err != nil {
- t.Fatalf("GET artifact: %v", err)
- }
- _ = resp.Body.Close()
- if resp.StatusCode != http.StatusNoContent {
- t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent)
- }
-}
-
-func TestTransportDoesNotReplaceExplicitAuthenticationAfterBearerChallenge(t *testing.T) {
- var registryRequests int
- var tokenRequests int
- var server *httptest.Server
-
- server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- switch r.URL.Path {
- case "/token":
- tokenRequests++
- _, _ = io.WriteString(w, `{"token":"registry-token"}`)
- case "/v2/library/test/blobs/sha256:test":
- registryRequests++
- if got := r.Header.Get("Authorization"); got != "Bearer explicit-token" {
- t.Errorf("Authorization = %q, want %q", got, "Bearer explicit-token")
- }
- w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`)
- http.Error(w, "authentication required", http.StatusUnauthorized)
- default:
- http.NotFound(w, r)
- }
- }))
- defer server.Close()
-
- client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)}
- req, err := http.NewRequest(http.MethodGet, server.URL+"/v2/library/test/blobs/sha256:test", nil)
- if err != nil {
- t.Fatal(err)
- }
- req.Header.Set("Authorization", "Bearer explicit-token")
-
- resp, err := client.Do(req)
- if err != nil {
- t.Fatalf("GET blob: %v", err)
- }
- _ = resp.Body.Close()
- if resp.StatusCode != http.StatusUnauthorized {
- t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized)
- }
- if registryRequests != 1 {
- t.Errorf("registry requests = %d, want 1", registryRequests)
- }
- if tokenRequests != 0 {
- t.Errorf("token requests = %d, want 0", tokenRequests)
- }
-}
-
-func TestTransportDoesNotForwardConfiguredAuthenticationOnTokenRedirect(t *testing.T) {
- destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if got := r.Header.Get("X-Registry-Token"); got != "" {
- t.Errorf("redirected X-Registry-Token = %q, want empty", got)
- }
- _, _ = io.WriteString(w, `{"token":"registry-token"}`)
- }))
- defer destination.Close()
-
- source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if got := r.Header.Get("X-Registry-Token"); got != "configured-token" {
- t.Errorf("source X-Registry-Token = %q, want %q", got, "configured-token")
- }
- http.Redirect(w, r, destination.URL+"/token", http.StatusFound)
- }))
- defer source.Close()
-
- authForURL := func(rawURL string) (string, string) {
- if strings.HasPrefix(rawURL, source.URL) {
- return "X-Registry-Token", "configured-token"
- }
- return "", ""
- }
- transport := NewTransport(http.DefaultTransport, authForURL)
- token, _, err := transport.fetchToken(context.Background(), bearerChallenge{realm: source.URL + "/token"})
- if err != nil {
- t.Fatalf("fetchToken: %v", err)
- }
- if token != "registry-token" {
- t.Errorf("token = %q, want %q", token, "registry-token")
- }
-}
-
-func TestTransportPrunesExpiredTokens(t *testing.T) {
- transport := NewTransport(http.DefaultTransport, nil)
- transport.tokens["expired-unused"] = cachedToken{
- value: "expired-token",
- expiresAt: time.Now().Add(-time.Minute),
- }
- transport.cacheToken("current", cachedToken{
- value: "current-token",
- expiresAt: time.Now().Add(time.Minute),
- })
-
- if got := transport.cachedToken("current"); got != "current-token" {
- t.Errorf("cachedToken(current) = %q, want %q", got, "current-token")
- }
- if _, ok := transport.tokens["expired-unused"]; ok {
- t.Error("expired unused token was not pruned")
- }
-}
-
-func TestTransportDoesNotFollowBearerChallengeOutsideOCIRegistry(t *testing.T) {
- tokenRequests := 0
- var server *httptest.Server
- server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path == "/token" {
- tokenRequests++
- _, _ = io.WriteString(w, `{"token":"unexpected"}`)
- return
- }
- w.Header().Set("WWW-Authenticate", `Bearer realm="`+server.URL+`/token"`)
- http.Error(w, "authentication required", http.StatusUnauthorized)
- }))
- defer server.Close()
-
- client := &http.Client{Transport: NewTransport(http.DefaultTransport, nil)}
- resp, err := client.Get(server.URL + "/api/packages")
- if err != nil {
- t.Fatalf("GET API: %v", err)
- }
- _ = resp.Body.Close()
- if resp.StatusCode != http.StatusUnauthorized {
- t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusUnauthorized)
- }
- if tokenRequests != 0 {
- t.Errorf("token requests = %d, want 0", tokenRequests)
- }
-}
diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
index aff5768..da8bde6 100644
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@@ -6,7 +6,6 @@ import (
"strconv"
"time"
- "github.com/git-pkgs/purl"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
@@ -121,48 +120,6 @@ var (
Help: "Number of currently active requests",
},
)
-
- IntegrityFailures = prometheus.NewCounterVec(
- prometheus.CounterOpts{
- Name: "proxy_integrity_failures_total",
- Help: "Cached artifacts that failed hash verification on read",
- },
- []string{"ecosystem"},
- )
-
- HealthProbeFailures = prometheus.NewCounterVec(
- prometheus.CounterOpts{
- Name: "proxy_health_probe_failures_total",
- Help: "Total number of storage health probe failures, by step (write|size|read|verify|delete).",
- },
- []string{"step"},
- )
-
- // Scanning metrics
- ScanDuration = prometheus.NewHistogramVec(
- prometheus.HistogramOpts{
- Name: "proxy_scan_duration_seconds",
- Help: "Pre-cache artifact scan duration in seconds, by ecosystem and scanner",
- Buckets: prometheus.DefBuckets,
- },
- []string{"ecosystem", "scanner"},
- )
-
- ScanBlocked = prometheus.NewCounterVec(
- prometheus.CounterOpts{
- Name: "proxy_scan_blocked_total",
- Help: "Total number of artifacts blocked by a pre-cache scan, by ecosystem and scanner",
- },
- []string{"ecosystem", "scanner"},
- )
-
- ScanErrors = prometheus.NewCounterVec(
- prometheus.CounterOpts{
- Name: "proxy_scan_errors_total",
- Help: "Total number of pre-cache scan errors, by ecosystem, scanner, and error type",
- },
- []string{"ecosystem", "scanner", "error_type"},
- )
)
func init() {
@@ -181,11 +138,6 @@ func init() {
StorageOperationDuration,
StorageErrors,
ActiveRequests,
- IntegrityFailures,
- HealthProbeFailures,
- ScanDuration,
- ScanBlocked,
- ScanErrors,
)
}
@@ -203,12 +155,12 @@ func RecordRequest(ecosystem string, status int, duration time.Duration) {
// RecordCacheHit increments cache hit counter.
func RecordCacheHit(ecosystem string) {
- CacheHits.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc()
+ CacheHits.WithLabelValues(ecosystem).Inc()
}
// RecordCacheMiss increments cache miss counter.
func RecordCacheMiss(ecosystem string) {
- CacheMisses.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc()
+ CacheMisses.WithLabelValues(ecosystem).Inc()
}
// RecordUpstreamFetch tracks upstream fetch duration.
@@ -226,37 +178,11 @@ func RecordStorageOperation(operation string, duration time.Duration) {
StorageOperationDuration.WithLabelValues(operation).Observe(duration.Seconds())
}
-// RecordIntegrityFailure increments the integrity failure counter.
-func RecordIntegrityFailure(ecosystem string) {
- IntegrityFailures.WithLabelValues(ecosystem).Inc()
-}
-
-// RecordHealthProbeFailure increments the health probe failure counter.
-// step is one of: "write", "size", "read", "verify", "delete".
-func RecordHealthProbeFailure(step string) {
- HealthProbeFailures.WithLabelValues(step).Inc()
-}
-
// RecordStorageError increments storage error counter.
func RecordStorageError(operation string) {
StorageErrors.WithLabelValues(operation).Inc()
}
-// RecordScanResult tracks a completed pre-cache scan call.
-func RecordScanResult(ecosystem, scannerName string, allowed bool, duration time.Duration) {
- ecosystem = purl.NormalizeEcosystem(ecosystem)
- ScanDuration.WithLabelValues(ecosystem, scannerName).Observe(duration.Seconds())
- if !allowed {
- ScanBlocked.WithLabelValues(ecosystem, scannerName).Inc()
- }
-}
-
-// RecordScanError increments the scan error counter.
-// errorType is one of: "error" (scanner call failed), "timeout", "cancelled".
-func RecordScanError(ecosystem, scannerName, errorType string) {
- ScanErrors.WithLabelValues(purl.NormalizeEcosystem(ecosystem), scannerName, errorType).Inc()
-}
-
// UpdateCacheStats updates cache size and artifact count gauges.
func UpdateCacheStats(sizeBytes, artifactCount int64) {
CacheSize.Set(float64(sizeBytes))
diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go
index a445467..db7b097 100644
--- a/internal/metrics/metrics_test.go
+++ b/internal/metrics/metrics_test.go
@@ -6,7 +6,6 @@ import (
"time"
"github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/client_golang/prometheus/testutil"
dto "github.com/prometheus/client_model/go"
)
@@ -49,7 +48,7 @@ func TestRecordStorageOperations(t *testing.T) {
func TestUpdateCacheStats(t *testing.T) {
UpdateCacheStats(1024*1024*1024, 100) // 1GB, 100 artifacts
- UpdateCacheStats(0, 0) // Empty cache
+ UpdateCacheStats(0, 0) // Empty cache
// No panics = success
}
@@ -192,45 +191,22 @@ func TestMetricsEndpointOutput(t *testing.T) {
func TestMetricsLabeling(t *testing.T) {
// Test that different ecosystems are properly labeled
- ecosystems := []struct {
- input string
- label string
- }{
- {input: "npm", label: "npm"},
- {input: "pypi", label: "pypi"},
- {input: "cargo", label: "cargo"},
- {input: "gem", label: "rubygems"},
- }
+ ecosystems := []string{"npm", "pypi", "cargo", "gem"}
for _, eco := range ecosystems {
- RecordRequest(eco.input, 200, 10*time.Millisecond)
- RecordCacheHit(eco.input)
+ RecordRequest(eco, 200, 10*time.Millisecond)
+ RecordCacheHit(eco)
}
// Verify each ecosystem has metrics
for _, eco := range ecosystems {
- val := getMetricValue(t, CacheHits, eco.label)
+ val := getMetricValue(t, CacheHits, eco)
if val == 0 {
- t.Errorf("no cache hits recorded for %s", eco.label)
+ t.Errorf("no cache hits recorded for %s", eco)
}
}
}
-func TestCacheMetricLabelsAreNormalized(t *testing.T) {
- rubyHitsBefore := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems"))
- composerMissesBefore := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist"))
-
- RecordCacheHit("gem")
- RecordCacheMiss("composer")
-
- if diff := testutil.ToFloat64(CacheHits.WithLabelValues("rubygems")) - rubyHitsBefore; diff != 1 {
- t.Errorf("rubygems cache hits delta = %.0f, want 1", diff)
- }
- if diff := testutil.ToFloat64(CacheMisses.WithLabelValues("packagist")) - composerMissesBefore; diff != 1 {
- t.Errorf("packagist cache misses delta = %.0f, want 1", diff)
- }
-}
-
func TestMetricNames(t *testing.T) {
// Verify metric names follow Prometheus naming conventions
expectedMetrics := []string{
diff --git a/internal/mirror/job.go b/internal/mirror/job.go
index 70d7003..8915d4a 100644
--- a/internal/mirror/job.go
+++ b/internal/mirror/job.go
@@ -3,7 +3,6 @@ package mirror
import (
"context"
"crypto/rand"
- "encoding/json"
"fmt"
"sync"
"time"
@@ -36,16 +35,15 @@ type Job struct {
// JobRequest is the JSON body for starting a mirror job via the API.
type JobRequest struct {
- PURLs []string `json:"purls,omitempty"`
- SBOM json.RawMessage `json:"sbom,omitempty"`
- Registry string `json:"registry,omitempty"`
+ PURLs []string `json:"purls,omitempty"`
+ Registry string `json:"registry,omitempty"`
}
// JobStore manages in-memory mirror jobs.
type JobStore struct {
- mu sync.RWMutex
- jobs map[string]*Job
- mirror *Mirror
+ mu sync.RWMutex
+ jobs map[string]*Job
+ mirror *Mirror
parentCtx context.Context
}
@@ -192,16 +190,12 @@ func (js *JobStore) runJob(ctx context.Context, cancel context.CancelFunc, job *
func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint:ireturn // interface return is the design
switch {
- case len(req.PURLs) > 0 && len(req.SBOM) > 0:
- return nil, fmt.Errorf("request must include only one of purls or sbom")
case len(req.PURLs) > 0:
return &PURLSource{PURLs: req.PURLs}, nil
- case len(req.SBOM) > 0:
- return &SBOMSource{Data: req.SBOM}, nil
case req.Registry != "":
return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead")
default:
- return nil, fmt.Errorf("request must include purls or sbom")
+ return nil, fmt.Errorf("request must include purls")
}
}
diff --git a/internal/mirror/job_test.go b/internal/mirror/job_test.go
index c0ac3f1..f7f2f1c 100644
--- a/internal/mirror/job_test.go
+++ b/internal/mirror/job_test.go
@@ -2,7 +2,6 @@ package mirror
import (
"context"
- "encoding/json"
"testing"
"time"
)
@@ -101,37 +100,6 @@ func TestSourceFromRequestPURLs(t *testing.T) {
}
}
-func TestSourceFromRequestSBOM(t *testing.T) {
- m := setupTestMirror(t, 1)
- js := NewJobStore(context.Background(), m)
- sbom := json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`)
-
- source, err := js.sourceFromRequest(JobRequest{SBOM: sbom})
- if err != nil {
- t.Fatalf("sourceFromRequest() error = %v", err)
- }
- sbomSource, ok := source.(*SBOMSource)
- if !ok {
- t.Fatalf("expected *SBOMSource, got %T", source)
- }
- if got := string(sbomSource.Data); got != string(sbom) {
- t.Errorf("SBOM data = %q, want %q", got, sbom)
- }
-}
-
-func TestSourceFromRequestRejectsPURLsAndSBOM(t *testing.T) {
- m := setupTestMirror(t, 1)
- js := NewJobStore(context.Background(), m)
-
- _, err := js.sourceFromRequest(JobRequest{
- PURLs: []string{"pkg:npm/lodash@4.17.21"},
- SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`),
- })
- if err == nil {
- t.Fatal("expected error when both purls and sbom are provided")
- }
-}
-
func TestSourceFromRequestRegistryRejected(t *testing.T) {
m := setupTestMirror(t, 1)
js := NewJobStore(context.Background(), m)
diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go
index b52c4f6..06b496f 100644
--- a/internal/mirror/mirror.go
+++ b/internal/mirror/mirror.go
@@ -40,14 +40,14 @@ func New(proxy *handler.Proxy, db *database.DB, store storage.Storage, logger *s
// Progress tracks the state of a mirror operation.
type Progress struct {
- Total int64 `json:"total"`
- Completed int64 `json:"completed"`
- Skipped int64 `json:"skipped"`
- Failed int64 `json:"failed"`
- Bytes int64 `json:"bytes"`
+ Total int64 `json:"total"`
+ Completed int64 `json:"completed"`
+ Skipped int64 `json:"skipped"`
+ Failed int64 `json:"failed"`
+ Bytes int64 `json:"bytes"`
Errors []MirrorError `json:"errors,omitempty"`
- StartedAt time.Time `json:"started_at"`
- Phase string `json:"phase"`
+ StartedAt time.Time `json:"started_at"`
+ Phase string `json:"phase"`
}
// MirrorError records a single failed mirror attempt.
@@ -212,9 +212,7 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog
return
}
- if result.Reader != nil {
- _ = result.Reader.Close()
- }
+ _ = result.Reader.Close()
if result.Cached {
tracker.skipped.Add(1)
@@ -222,9 +220,9 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog
"ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version)
} else {
tracker.completed.Add(1)
- tracker.bytes.Add(result.Artifact.Size)
+ tracker.bytes.Add(result.Size)
m.logger.Info("mirrored",
"ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version,
- "size", result.Artifact.Size)
+ "size", result.Size)
}
}
diff --git a/internal/mirror/mirror_test.go b/internal/mirror/mirror_test.go
index 3a6420f..1d7d30d 100644
--- a/internal/mirror/mirror_test.go
+++ b/internal/mirror/mirror_test.go
@@ -2,11 +2,8 @@ package mirror
import (
"context"
- "crypto/sha256"
- "database/sql"
"log/slog"
"os"
- "strings"
"testing"
"time"
@@ -46,14 +43,6 @@ func setupTestMirror(t *testing.T, workers int) *Mirror {
const testPackageLodash = "lodash"
-type signedURLStorage struct {
- storage.Storage
-}
-
-func (signedURLStorage) SignedURL(context.Context, string, time.Duration) (string, error) {
- return "https://storage.example/artifact", nil
-}
-
func TestMirrorRunEmptySource(t *testing.T) {
m := setupTestMirror(t, 2)
@@ -122,54 +111,6 @@ func TestMirrorRunCanceled(t *testing.T) {
}
}
-func TestMirrorOneDirectServeCacheHit(t *testing.T) {
- m := setupTestMirror(t, 1)
- m.proxy.DirectServe = true
- m.proxy.Storage = signedURLStorage{Storage: m.storage}
-
- packagePURL := "pkg:npm/example"
- versionPURL := packagePURL + "@1.0.0"
- if err := m.db.UpsertPackage(&database.Package{
- PURL: packagePURL,
- Ecosystem: "npm",
- Name: "example",
- }); err != nil {
- t.Fatalf("UpsertPackage() error = %v", err)
- }
- if err := m.db.UpsertVersion(&database.Version{
- PURL: versionPURL,
- PackagePURL: packagePURL,
- }); err != nil {
- t.Fatalf("UpsertVersion() error = %v", err)
- }
- if err := m.db.UpsertArtifact(&database.Artifact{
- VersionPURL: versionPURL,
- Filename: "",
- UpstreamURL: "https://registry.example/artifact",
- StoragePath: sql.NullString{String: "npm/example/1.0.0/artifact", Valid: true},
- ContentHash: sql.NullString{String: strings.Repeat("a", sha256.Size*2), Valid: true},
- Size: sql.NullInt64{Int64: 1, Valid: true},
- FetchedAt: sql.NullTime{Time: time.Now(), Valid: true},
- }); err != nil {
- t.Fatalf("UpsertArtifact() error = %v", err)
- }
-
- tracker := newProgressTracker()
- m.mirrorOne(context.Background(), PackageVersion{
- Ecosystem: "npm",
- Name: "example",
- Version: "1.0.0",
- }, tracker)
-
- progress := tracker.snapshot()
- if progress.Skipped != 1 {
- t.Errorf("skipped = %d, want 1", progress.Skipped)
- }
- if progress.Failed != 0 {
- t.Errorf("failed = %d, want 0", progress.Failed)
- }
-}
-
func TestProgressTrackerSnapshot(t *testing.T) {
pt := newProgressTracker()
pt.total.Store(10)
diff --git a/internal/mirror/registry.go b/internal/mirror/registry.go
new file mode 100644
index 0000000..6b2c449
--- /dev/null
+++ b/internal/mirror/registry.go
@@ -0,0 +1,16 @@
+package mirror
+
+import (
+ "context"
+ "fmt"
+)
+
+// RegistrySource enumerates all packages in a registry for full mirroring.
+// Registry enumeration is not yet implemented for any ecosystem.
+type RegistrySource struct {
+ Ecosystem string
+}
+
+func (s *RegistrySource) Enumerate(_ context.Context, _ func(PackageVersion) error) error {
+ return fmt.Errorf("registry enumeration is not yet implemented for ecosystem %q", s.Ecosystem)
+}
diff --git a/internal/mirror/registry_test.go b/internal/mirror/registry_test.go
new file mode 100644
index 0000000..363bfea
--- /dev/null
+++ b/internal/mirror/registry_test.go
@@ -0,0 +1,46 @@
+package mirror
+
+import (
+ "context"
+ "testing"
+)
+
+func TestRegistrySourceUnsupported(t *testing.T) {
+ source := &RegistrySource{Ecosystem: "golang"}
+ err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
+ return nil
+ })
+ if err == nil {
+ t.Fatal("expected error for unsupported ecosystem")
+ }
+}
+
+func TestRegistrySourceNPMNotImplemented(t *testing.T) {
+ source := &RegistrySource{Ecosystem: "npm"}
+ err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
+ return nil
+ })
+ if err == nil {
+ t.Fatal("expected not-implemented error")
+ }
+}
+
+func TestRegistrySourcePyPINotImplemented(t *testing.T) {
+ source := &RegistrySource{Ecosystem: "pypi"}
+ err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
+ return nil
+ })
+ if err == nil {
+ t.Fatal("expected not-implemented error")
+ }
+}
+
+func TestRegistrySourceCargoNotImplemented(t *testing.T) {
+ source := &RegistrySource{Ecosystem: "cargo"}
+ err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
+ return nil
+ })
+ if err == nil {
+ t.Fatal("expected not-implemented error")
+ }
+}
diff --git a/internal/mirror/source.go b/internal/mirror/source.go
index 4269264..a6fa364 100644
--- a/internal/mirror/source.go
+++ b/internal/mirror/source.go
@@ -5,13 +5,14 @@ import (
"context"
"encoding/json"
"fmt"
+ "os"
cdx "github.com/CycloneDX/cyclonedx-go"
"github.com/git-pkgs/purl"
"github.com/git-pkgs/registries"
_ "github.com/git-pkgs/registries/all"
- spdxjson "github.com/spdx/tools-golang/json"
"github.com/spdx/tools-golang/spdx"
+ spdxjson "github.com/spdx/tools-golang/json"
spdxtv "github.com/spdx/tools-golang/tagvalue"
)
@@ -93,20 +94,16 @@ func (s *PURLSource) fetchVersions(ctx context.Context, client *registries.Clien
return result, nil
}
-// SBOMSource extracts package versions from CycloneDX or SPDX SBOM data.
+// SBOMSource extracts package versions from a CycloneDX or SPDX SBOM file.
type SBOMSource struct {
- Data []byte
- Name string
+ Path string
RegClient *registries.Client
}
func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) error) error {
purls, err := s.extractPURLs()
if err != nil {
- if s.Name != "" {
- return fmt.Errorf("parsing SBOM %s: %w", s.Name, err)
- }
- return fmt.Errorf("parsing SBOM: %w", err)
+ return fmt.Errorf("reading SBOM %s: %w", s.Path, err)
}
inner := &PURLSource{PURLs: purls, RegClient: s.RegClient}
@@ -114,18 +111,23 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
}
func (s *SBOMSource) extractPURLs() ([]string, error) {
+ data, err := os.ReadFile(s.Path)
+ if err != nil {
+ return nil, err
+ }
+
// Try CycloneDX first
- if purls, err := extractCycloneDXPURLs(s.Data); err == nil && len(purls) > 0 {
+ if purls, err := extractCycloneDXPURLs(data); err == nil && len(purls) > 0 {
return purls, nil
}
// Try SPDX JSON
- if purls, err := extractSPDXJSONPURLs(s.Data); err == nil && len(purls) > 0 {
+ if purls, err := extractSPDXJSONPURLs(data); err == nil && len(purls) > 0 {
return purls, nil
}
// Try SPDX tag-value
- if purls, err := extractSPDXTVPURLs(s.Data); err == nil && len(purls) > 0 {
+ if purls, err := extractSPDXTVPURLs(data); err == nil && len(purls) > 0 {
return purls, nil
}
diff --git a/internal/mirror/source_test.go b/internal/mirror/source_test.go
index fe56ca4..ce53acf 100644
--- a/internal/mirror/source_test.go
+++ b/internal/mirror/source_test.go
@@ -3,7 +3,8 @@ package mirror
import (
"context"
"encoding/json"
- "strings"
+ "os"
+ "path/filepath"
"testing"
)
@@ -115,7 +116,8 @@ func TestSBOMSourceCycloneDXJSON(t *testing.T) {
},
}
- source := &SBOMSource{Data: marshalJSON(t, bom)}
+ path := writeTempJSON(t, bom)
+ source := &SBOMSource{Path: path}
var items []PackageVersion
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
@@ -147,9 +149,9 @@ func TestSBOMSourceSPDXJSON(t *testing.T) {
"documentNamespace": "https://example.com/test",
"packages": []map[string]any{
{
- "SPDXID": "SPDXRef-Package",
- "name": "lodash",
- "version": "4.17.21",
+ "SPDXID": "SPDXRef-Package",
+ "name": "lodash",
+ "version": "4.17.21",
"downloadLocation": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"externalRefs": []map[string]any{
{
@@ -162,7 +164,8 @@ func TestSBOMSourceSPDXJSON(t *testing.T) {
},
}
- source := &SBOMSource{Data: marshalJSON(t, doc)}
+ path := writeTempJSON(t, doc)
+ source := &SBOMSource{Path: path}
var items []PackageVersion
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
@@ -182,28 +185,30 @@ func TestSBOMSourceSPDXJSON(t *testing.T) {
}
}
-func TestSBOMSourceEmptyData(t *testing.T) {
- source := &SBOMSource{}
+func TestSBOMSourceNonexistentFile(t *testing.T) {
+ source := &SBOMSource{Path: "/nonexistent/sbom.json"}
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
return nil
})
if err == nil {
- t.Fatal("expected error for empty SBOM")
+ t.Fatal("expected error for nonexistent file")
}
}
func TestSBOMSourceInvalidFormat(t *testing.T) {
- source := &SBOMSource{Data: []byte("this is not an SBOM"), Name: "invalid.sbom"}
+ path := filepath.Join(t.TempDir(), "invalid.txt")
+ if err := os.WriteFile(path, []byte("this is not an SBOM"), 0644); err != nil {
+ t.Fatal(err)
+ }
+
+ source := &SBOMSource{Path: path}
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
return nil
})
if err == nil {
t.Fatal("expected error for invalid SBOM")
}
- if !strings.Contains(err.Error(), "invalid.sbom") {
- t.Errorf("error = %q, want source name", err)
- }
}
func TestSBOMSourceEmptyCycloneDX(t *testing.T) {
@@ -211,9 +216,11 @@ func TestSBOMSourceEmptyCycloneDX(t *testing.T) {
"bomFormat": "CycloneDX",
"specVersion": "1.4",
}
+ path := writeTempJSON(t, bom)
+
// This should fall through to SPDX parsing, which will also fail,
// resulting in an error about not being able to parse
- source := &SBOMSource{Data: marshalJSON(t, bom)}
+ source := &SBOMSource{Path: path}
err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
return nil
})
@@ -222,11 +229,15 @@ func TestSBOMSourceEmptyCycloneDX(t *testing.T) {
}
}
-func marshalJSON(t *testing.T, v any) []byte {
+func writeTempJSON(t *testing.T, v any) string {
t.Helper()
data, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
- return data
+ path := filepath.Join(t.TempDir(), "sbom.json")
+ if err := os.WriteFile(path, data, 0644); err != nil {
+ t.Fatal(err)
+ }
+ return path
}
diff --git a/internal/packageurl/packageurl.go b/internal/packageurl/packageurl.go
deleted file mode 100644
index 7110942..0000000
--- a/internal/packageurl/packageurl.go
+++ /dev/null
@@ -1,62 +0,0 @@
-// Package packageurl builds package URLs from ecosystem-native package names.
-package packageurl
-
-import (
- "strings"
-
- "github.com/git-pkgs/purl"
-)
-
-// Make constructs a package URL from an ecosystem-native package name.
-func Make(ecosystem, name, version string) *purl.PURL {
- return purl.MakePURL(ecosystem, name, version)
-}
-
-// MakeString constructs a package URL string. It returns an empty string when
-// the package identity cannot be represented as a PURL.
-func MakeString(ecosystem, name, version string) string {
- return purl.MakePURLString(ecosystem, name, version)
-}
-
-// WithVersionString returns a package PURL with its version replaced. It
-// returns an empty string when packagePURL is invalid.
-func WithVersionString(packagePURL, version string) string {
- pkg, err := purl.Parse(packagePURL)
- if err != nil {
- return ""
- }
- return pkg.WithVersion(version).String()
-}
-
-// MakeCacheStrings returns package and version PURLs suitable for artifact
-// cache records. Swift registry identities use an explicit generic PURL until
-// their source repository has been resolved. The result is independent of the
-// configured upstream so cache entries survive an upstream.swift change,
-// matching every other ecosystem.
-func MakeCacheStrings(ecosystem, name, version string) (packagePURL, versionPURL string) {
- if pkg := Make(ecosystem, name, ""); pkg != nil {
- return pkg.String(), pkg.WithVersion(version).String()
- }
- if purl.NormalizeEcosystem(ecosystem) != "swift" {
- return "", ""
- }
-
- identity, ok := swiftRegistryIdentity(name)
- if !ok {
- return "", ""
- }
-
- pkg := purl.New("generic", "swift-registry", identity, "", nil)
- return pkg.String(), pkg.WithVersion(version).String()
-}
-
-func swiftRegistryIdentity(name string) (string, bool) {
- scope, packageName, found := strings.Cut(name, "/")
- if !found {
- scope, packageName, found = strings.Cut(name, ".")
- }
- if !found || scope == "" || packageName == "" || strings.ContainsAny(packageName, "/.") {
- return "", false
- }
- return strings.ToLower(scope) + "." + strings.ToLower(packageName), true
-}
diff --git a/internal/packageurl/packageurl_test.go b/internal/packageurl/packageurl_test.go
deleted file mode 100644
index 8982288..0000000
--- a/internal/packageurl/packageurl_test.go
+++ /dev/null
@@ -1,75 +0,0 @@
-package packageurl
-
-import "testing"
-
-func TestMakeSwiftRegistryIdentityUnsupported(t *testing.T) {
- identities := []string{"apple.swift-argument-parser", "apple/swift-argument-parser"}
- for _, identity := range identities {
- t.Run(identity, func(t *testing.T) {
- if got := Make("swift", identity, "1.8.2"); got != nil {
- t.Errorf("Make() = %q, want nil", got.String())
- }
- if got := MakeString("swift", identity, "1.8.2"); got != "" {
- t.Errorf("MakeString() = %q, want empty string", got)
- }
- })
- }
-}
-
-func TestMakeStringSwiftSourceCoordinate(t *testing.T) {
- got := MakeString("swift", "github.com/apple/swift-package-manager", "1.7.0")
- want := "pkg:swift/github.com/apple/swift-package-manager@1.7.0"
- if got != want {
- t.Errorf("MakeString() = %q, want %q", got, want)
- }
-}
-
-func TestWithVersionStringPreservesQualifiers(t *testing.T) {
- packagePURL := "pkg:generic/swift-registry/apple.example?repository_url=https:%2F%2Fold.example%2Fswift"
- got := WithVersionString(packagePURL, "1.2.3")
- want := "pkg:generic/swift-registry/apple.example@1.2.3?repository_url=https:%2F%2Fold.example%2Fswift"
- if got != want {
- t.Errorf("WithVersionString() = %q, want %q", got, want)
- }
-
- if got := WithVersionString("not a purl", "1.2.3"); got != "" {
- t.Errorf("WithVersionString() = %q for invalid PURL, want empty string", got)
- }
-}
-
-func TestMakeCacheStringsSwiftRegistryIdentity(t *testing.T) {
- packagePURL, versionPURL := MakeCacheStrings("swift", "APPLE/EXAMPLE", "1.2.3")
-
- wantPackage := "pkg:generic/swift-registry/apple.example"
- if packagePURL != wantPackage {
- t.Errorf("package PURL = %q, want %q", packagePURL, wantPackage)
- }
- wantVersion := "pkg:generic/swift-registry/apple.example@1.2.3"
- if versionPURL != wantVersion {
- t.Errorf("version PURL = %q, want %q", versionPURL, wantVersion)
- }
-
- dottedPackage, dottedVersion := MakeCacheStrings("swift", "apple.example", "1.2.3")
- if dottedPackage != packagePURL || dottedVersion != versionPURL {
- t.Errorf("dotted identity cache PURLs = %q, %q; want %q, %q", dottedPackage, dottedVersion, packagePURL, versionPURL)
- }
-}
-
-func TestMakeCacheStringsUsesSourcePURLWhenAvailable(t *testing.T) {
- packagePURL, versionPURL := MakeCacheStrings("swift", "github.com/apple/swift-package-manager", "1.7.0")
-
- if packagePURL != "pkg:swift/github.com/apple/swift-package-manager" {
- t.Errorf("package PURL = %q", packagePURL)
- }
- if versionPURL != "pkg:swift/github.com/apple/swift-package-manager@1.7.0" {
- t.Errorf("version PURL = %q", versionPURL)
- }
-}
-
-func TestMakeStringDelegatesOtherEcosystems(t *testing.T) {
- got := MakeString("npm", "@babel/core", "7.23.0")
- want := "pkg:npm/%40babel/core@7.23.0"
- if got != want {
- t.Errorf("MakeString() = %q, want %q", got, want)
- }
-}
diff --git a/internal/scanner/group.go b/internal/scanner/group.go
deleted file mode 100644
index ed0be57..0000000
--- a/internal/scanner/group.go
+++ /dev/null
@@ -1,226 +0,0 @@
-package scanner
-
-import (
- "context"
- "errors"
- "fmt"
- "log/slog"
- "net/http"
- "sync"
- "time"
-
- "github.com/git-pkgs/proxy/internal/config"
- "github.com/git-pkgs/proxy/internal/metrics"
-)
-
-const (
- modeBlock = "block"
- modeMonitor = "monitor"
-
- defaultTimeout = 30 * time.Second
-)
-
-type entry struct {
- scanner Scanner
- mode string
- ecosystems map[string]struct{} // nil/empty means all ecosystems
-}
-
-// Group runs a set of configured Scanners concurrently and turns their
-// individual verdicts into a single decision.
-type Group struct {
- entries []entry
- timeout time.Duration
- failOpen bool
- logger *slog.Logger
-}
-
-// NewGroup builds a Group from cfg. If cfg.Enabled is false, the returned
-// Group has no entries and Enabled() reports false, so callers can skip
-// the scan path entirely.
-func NewGroup(cfg config.ScanningConfig, logger *slog.Logger) (*Group, error) {
- g := &Group{
- timeout: defaultTimeout,
- failOpen: cfg.FailOpen,
- logger: logger,
- }
- if !cfg.Enabled {
- return g, nil
- }
- if cfg.SigningKeyExpanded() == "" {
- return nil, fmt.Errorf("scanning.signing_key is required when scanning.enabled is true")
- }
- if d, err := time.ParseDuration(cfg.Timeout); err == nil && d > 0 {
- g.timeout = d
- }
-
- for _, sc := range cfg.Scanners {
- mode := sc.Mode
- if mode == "" {
- mode = modeBlock
- }
- if mode != modeBlock && mode != modeMonitor {
- return nil, fmt.Errorf("scanner %q: invalid mode %q (must be %q or %q)", sc.Name, mode, modeBlock, modeMonitor)
- }
-
- var ecosystems map[string]struct{}
- if len(sc.Ecosystems) > 0 {
- ecosystems = make(map[string]struct{}, len(sc.Ecosystems))
- for _, eco := range sc.Ecosystems {
- ecosystems[eco] = struct{}{}
- }
- }
-
- g.entries = append(g.entries, entry{
- scanner: NewHTTPScanner(sc.Name, sc.URL, sc.HeadersExpanded(), http.DefaultClient),
- mode: mode,
- ecosystems: ecosystems,
- })
- }
-
- return g, nil
-}
-
-// Enabled reports whether any scanner is configured.
-func (g *Group) Enabled() bool {
- return g != nil && len(g.entries) > 0
-}
-
-// Timeout returns the per-scan-call timeout used to bound the signed fetch
-// URL's validity.
-func (g *Group) Timeout() time.Duration {
- return g.timeout
-}
-
-func (g *Group) applicable(ecosystem string) []entry {
- var out []entry
- for _, e := range g.entries {
- if len(e.ecosystems) == 0 {
- out = append(out, e)
- continue
- }
- if _, ok := e.ecosystems[ecosystem]; ok {
- out = append(out, e)
- }
- }
- return out
-}
-
-// Scan runs every scanner applicable to req.Ecosystem concurrently, never
-// sequentially, and returns a single decision.
-//
-// The moment any "block" mode scanner reports Allowed: false (or errors,
-// unless FailOpen is set), Scan cancels a context shared by every
-// goroutine: in-flight calls to the other scanners are aborted rather than
-// waited out, since a single block already decides the outcome. Scan still
-// waits for all goroutines to observe that cancellation and return before
-// it itself returns, so no scan call outlives this method call.
-//
-// If nothing blocks, Scan waits for every "block" mode scanner to finish
-// before reporting Allowed: true — an allow decision can't be finalized
-// until all of them have answered. "monitor" mode scanners never gate the
-// wait or trigger cancellation: a monitor verdict of Allowed: false is
-// logged and folded into Result.Findings, but never blocks.
-func (g *Group) Scan(ctx context.Context, req Request) Result {
- entries := g.applicable(req.Ecosystem)
- if len(entries) == 0 {
- return Result{Allowed: true}
- }
-
- scanCtx, cancel := context.WithTimeout(ctx, g.timeout)
- defer cancel()
-
- var (
- mu sync.Mutex
- findings []Finding
- blocked *Result
- )
-
- var wg sync.WaitGroup
- for _, e := range entries {
- wg.Add(1)
- go func(e entry) {
- defer wg.Done()
- entryFindings, entryBlock := g.evaluate(scanCtx, req, e)
-
- mu.Lock()
- findings = append(findings, entryFindings...)
- if entryBlock != nil && blocked == nil {
- blocked = entryBlock
- cancel()
- }
- mu.Unlock()
- }(e)
- }
-
- wg.Wait()
-
- if blocked != nil {
- blocked.Findings = findings
- return *blocked
- }
-
- return Result{Allowed: true, Findings: findings}
-}
-
-// evaluate runs a single scanner and reports its findings plus, if this
-// scanner's verdict should block the artifact, the Result to block with
-// (nil otherwise). It never sets Result.Findings on a returned block
-// Result — the caller assembles Findings from every entry once all of them
-// have finished.
-func (g *Group) evaluate(scanCtx context.Context, req Request, e entry) (findings []Finding, block *Result) {
- start := time.Now()
- res, err := e.scanner.Scan(scanCtx, req)
- duration := time.Since(start)
-
- if err != nil {
- errType := "error"
- switch {
- case errors.Is(scanCtx.Err(), context.DeadlineExceeded):
- errType = "timeout"
- case errors.Is(scanCtx.Err(), context.Canceled):
- // scanCtx was cancelled because another scanner in the group
- // already decided the verdict, not because this call itself
- // timed out or failed on its own.
- errType = "cancelled"
- }
- metrics.RecordScanError(req.Ecosystem, e.scanner.Name(), errType)
-
- if errType == "cancelled" {
- // Another scanner already decided the verdict and cancelled
- // scanCtx; this call didn't fail on its own, so don't log it
- // as if it did.
- return nil, nil
- }
-
- if e.mode == modeMonitor || g.failOpen {
- g.logger.Warn("scanner call failed, treating as allowed",
- "scanner", e.scanner.Name(), "mode", e.mode, "error", err)
- return nil, nil
- }
-
- g.logger.Warn("scanner call failed, blocking artifact",
- "scanner", e.scanner.Name(), "mode", e.mode, "error", err)
- return nil, &Result{
- Allowed: false,
- Reason: fmt.Sprintf("scanner %q failed: %v", e.scanner.Name(), err),
- ScannerName: e.scanner.Name(),
- InfraError: true,
- }
- }
-
- metrics.RecordScanResult(req.Ecosystem, e.scanner.Name(), res.Allowed, duration)
-
- if e.mode == modeMonitor {
- if !res.Allowed {
- g.logger.Warn("monitor scanner flagged artifact",
- "scanner", e.scanner.Name(), "reason", res.Reason)
- }
- return res.Findings, nil
- }
-
- if !res.Allowed {
- return res.Findings, &Result{Allowed: false, Reason: res.Reason, ScannerName: e.scanner.Name()}
- }
- return res.Findings, nil
-}
diff --git a/internal/scanner/group_test.go b/internal/scanner/group_test.go
deleted file mode 100644
index e5b40ee..0000000
--- a/internal/scanner/group_test.go
+++ /dev/null
@@ -1,245 +0,0 @@
-package scanner
-
-import (
- "context"
- "errors"
- "io"
- "log/slog"
- "strings"
- "testing"
- "time"
-
- "github.com/git-pkgs/proxy/internal/config"
- "github.com/git-pkgs/proxy/internal/metrics"
- "github.com/prometheus/client_golang/prometheus/testutil"
-)
-
-func discardLogger() *slog.Logger {
- return slog.New(slog.NewTextHandler(io.Discard, nil))
-}
-
-// fakeScanner is a Scanner test double that can simulate a delay, a fixed
-// result or error, and report whether its context was cancelled before it
-// returned.
-type fakeScanner struct {
- name string
- delay time.Duration
- result Result
- err error
- cancelled *bool
-}
-
-func (f *fakeScanner) Name() string { return f.name }
-
-func (f *fakeScanner) Scan(ctx context.Context, _ Request) (Result, error) {
- select {
- case <-time.After(f.delay):
- case <-ctx.Done():
- if f.cancelled != nil {
- *f.cancelled = true
- }
- return Result{}, ctx.Err()
- }
- return f.result, f.err
-}
-
-func newTestGroup(entries []entry, failOpen bool) *Group {
- return &Group{
- entries: entries,
- timeout: time.Second,
- failOpen: failOpen,
- logger: discardLogger(),
- }
-}
-
-func TestGroup_Enabled(t *testing.T) {
- disabled, err := NewGroup(config.ScanningConfig{Enabled: false}, discardLogger())
- if err != nil {
- t.Fatalf("NewGroup() error: %v", err)
- }
- if disabled.Enabled() {
- t.Error("Enabled() = true for disabled config, want false")
- }
-
- enabled, err := NewGroup(config.ScanningConfig{
- Enabled: true,
- Timeout: "10s",
- SigningKey: "test-signing-key",
- Scanners: []config.ScannerConfig{
- {Name: "clamav", URL: "http://clamav.invalid", Mode: "block"},
- },
- }, discardLogger())
- if err != nil {
- t.Fatalf("NewGroup() error: %v", err)
- }
- if !enabled.Enabled() {
- t.Error("Enabled() = false for configured scanner, want true")
- }
-}
-
-func TestGroup_NewGroup_InvalidMode(t *testing.T) {
- _, err := NewGroup(config.ScanningConfig{
- Enabled: true,
- SigningKey: "test-signing-key",
- Scanners: []config.ScannerConfig{
- {Name: "bad", URL: "http://bad.invalid", Mode: "quarantine"},
- },
- }, discardLogger())
- if err == nil {
- t.Fatal("NewGroup() error = nil, want error for invalid mode")
- }
-}
-
-func TestGroup_NewGroup_MissingSigningKey(t *testing.T) {
- _, err := NewGroup(config.ScanningConfig{
- Enabled: true,
- Scanners: []config.ScannerConfig{
- {Name: "clamav", URL: "http://clamav.invalid", Mode: "block"},
- },
- }, discardLogger())
- if err == nil {
- t.Fatal("NewGroup() error = nil, want error for missing signing key")
- }
-}
-
-func TestGroup_Scan_NoApplicableScanners(t *testing.T) {
- g := newTestGroup([]entry{
- {
- scanner: &fakeScanner{name: "npm-only", result: Result{Allowed: false}},
- mode: modeBlock,
- ecosystems: map[string]struct{}{"npm": {}},
- },
- }, false)
-
- result := g.Scan(context.Background(), Request{Ecosystem: "pypi"})
- if !result.Allowed {
- t.Error("Allowed = false, want true when no scanner applies to the ecosystem")
- }
-}
-
-func TestGroup_Scan_Allowed(t *testing.T) {
- g := newTestGroup([]entry{
- {scanner: &fakeScanner{name: "clamav", result: Result{Allowed: true}}, mode: modeBlock},
- }, false)
-
- result := g.Scan(context.Background(), Request{Ecosystem: "npm"})
- if !result.Allowed {
- t.Error("Allowed = false, want true")
- }
-}
-
-func TestGroup_Scan_Blocked(t *testing.T) {
- g := newTestGroup([]entry{
- {scanner: &fakeScanner{name: "clamav", result: Result{Allowed: false, Reason: "malware"}}, mode: modeBlock},
- }, false)
-
- result := g.Scan(context.Background(), Request{Ecosystem: "npm"})
- if result.Allowed {
- t.Error("Allowed = true, want false")
- }
- if result.Reason != "malware" {
- t.Errorf("Reason = %q, want %q", result.Reason, "malware")
- }
- if result.ScannerName != "clamav" {
- t.Errorf("ScannerName = %q, want %q", result.ScannerName, "clamav")
- }
- if result.InfraError {
- t.Error("InfraError = true, want false — this is a genuine scanner verdict, not an infrastructure failure")
- }
-}
-
-func TestGroup_Scan_MonitorNeverBlocks(t *testing.T) {
- g := newTestGroup([]entry{
- {
- scanner: &fakeScanner{name: "trivy", result: Result{
- Allowed: false,
- Reason: "cve found",
- Findings: []Finding{{Severity: "medium", Title: "CVE-1234"}},
- }},
- mode: modeMonitor,
- },
- }, false)
-
- result := g.Scan(context.Background(), Request{Ecosystem: "npm"})
- if !result.Allowed {
- t.Error("Allowed = false, want true — monitor mode must never block")
- }
- if len(result.Findings) != 1 || result.Findings[0].Title != "CVE-1234" {
- t.Errorf("Findings = %+v, want the monitor scanner's finding folded in", result.Findings)
- }
-}
-
-func TestGroup_Scan_FirstBlockCancelsOthers(t *testing.T) {
- var slowSawCancel bool
- g := newTestGroup([]entry{
- {scanner: &fakeScanner{name: "fast-block", result: Result{Allowed: false, Reason: "blocked"}}, mode: modeBlock},
- {scanner: &fakeScanner{name: "slow", delay: 2 * time.Second, cancelled: &slowSawCancel}, mode: modeBlock},
- }, false)
-
- start := time.Now()
- result := g.Scan(context.Background(), Request{Ecosystem: "npm"})
- elapsed := time.Since(start)
-
- if result.Allowed {
- t.Error("Allowed = true, want false")
- }
- if elapsed >= 2*time.Second {
- t.Errorf("Scan() took %v, want it to return promptly once the slow scanner's context was cancelled", elapsed)
- }
- if !slowSawCancel {
- t.Error("slow scanner never observed context cancellation")
- }
-
- if got := testutil.ToFloat64(metrics.ScanErrors.WithLabelValues("npm", "slow", "cancelled")); got != 1 {
- t.Errorf("scan_errors{error_type=cancelled} = %v, want 1 — the slow scanner was aborted by a sibling's block, not by its own timeout", got)
- }
- if got := testutil.ToFloat64(metrics.ScanErrors.WithLabelValues("npm", "slow", "timeout")); got != 0 {
- t.Errorf("scan_errors{error_type=timeout} = %v, want 0 — intra-group cancellation must not be mislabelled as a timeout", got)
- }
-}
-
-func TestGroup_Scan_WaitsForAllBlockScannersBeforeAllowing(t *testing.T) {
- g := newTestGroup([]entry{
- {scanner: &fakeScanner{name: "fast", result: Result{Allowed: true}}, mode: modeBlock},
- {scanner: &fakeScanner{name: "slow", delay: 30 * time.Millisecond, result: Result{Allowed: true}}, mode: modeBlock},
- }, false)
-
- start := time.Now()
- result := g.Scan(context.Background(), Request{Ecosystem: "npm"})
- elapsed := time.Since(start)
-
- if !result.Allowed {
- t.Error("Allowed = false, want true")
- }
- if elapsed < 30*time.Millisecond {
- t.Errorf("Scan() returned after %v, want it to wait for the slower block scanner", elapsed)
- }
-}
-
-func TestGroup_Scan_ErrorFailClosedByDefault(t *testing.T) {
- g := newTestGroup([]entry{
- {scanner: &fakeScanner{name: "flaky", err: errors.New("connection refused")}, mode: modeBlock},
- }, false)
-
- result := g.Scan(context.Background(), Request{Ecosystem: "npm"})
- if result.Allowed {
- t.Error("Allowed = true, want false — default posture is fail-closed on scanner error")
- }
- if !result.InfraError {
- t.Error("InfraError = false, want true — the block came from a scanner call failure, not a verdict")
- }
- if !strings.Contains(result.Reason, "connection refused") {
- t.Errorf("Reason = %q, want it to include the underlying error for server-side logging", result.Reason)
- }
-}
-
-func TestGroup_Scan_ErrorFailOpen(t *testing.T) {
- g := newTestGroup([]entry{
- {scanner: &fakeScanner{name: "flaky", err: errors.New("connection refused")}, mode: modeBlock},
- }, true)
-
- result := g.Scan(context.Background(), Request{Ecosystem: "npm"})
- if !result.Allowed {
- t.Error("Allowed = false, want true — FailOpen must treat scanner errors as allowed")
- }
-}
diff --git a/internal/scanner/http.go b/internal/scanner/http.go
deleted file mode 100644
index 90a4e5e..0000000
--- a/internal/scanner/http.go
+++ /dev/null
@@ -1,97 +0,0 @@
-package scanner
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "net/http"
-)
-
-// HTTPScanner adapts an external HTTP scanning service to the Scanner
-// interface. It POSTs a small JSON notification describing the staged
-// artifact, including a signed fetch URL; the external service is
-// responsible for GETting that URL itself, running the real scan against
-// those bytes, and replying with a verdict before the request's deadline.
-//
-// Request body:
-//
-// {
-// "ecosystem": "npm", "name": "left-pad", "version": "1.0.0",
-// "filename": "left-pad-1.0.0.tgz", "purl": "pkg:npm/left-pad@1.0.0",
-// "content_type": "application/octet-stream", "size": 1234,
-// "fetch_url": "https://proxy.internal/_internal/scan-fetch?..."
-// }
-//
-// Response body:
-//
-// {
-// "allowed": true, "reason": "",
-// "findings": [{"severity": "high", "title": "...", "description": "..."}]
-// }
-//
-// Any compliant adapter — a trivy wrapper, a clamav-rest bridge, a Wiz
-// connector, or an in-house service — need only implement this contract.
-type HTTPScanner struct {
- name string
- url string
- headers map[string]string
- client *http.Client
-}
-
-// NewHTTPScanner creates an HTTPScanner named name that notifies url of
-// staged artifacts, attaching headers to every request (e.g. for auth).
-// If client is nil, http.DefaultClient is used.
-func NewHTTPScanner(name, url string, headers map[string]string, client *http.Client) *HTTPScanner {
- if client == nil {
- client = http.DefaultClient
- }
- return &HTTPScanner{name: name, url: url, headers: headers, client: client}
-}
-
-// Name returns the scanner's configured name.
-func (s *HTTPScanner) Name() string { return s.name }
-
-type httpScanResponse struct {
- Allowed bool `json:"allowed"`
- Reason string `json:"reason"`
- Findings []Finding `json:"findings"`
-}
-
-// Scan notifies the configured URL of req and waits for a verdict.
-func (s *HTTPScanner) Scan(ctx context.Context, req Request) (Result, error) {
- body, err := json.Marshal(req)
- if err != nil {
- return Result{}, fmt.Errorf("marshal scan request: %w", err)
- }
-
- httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, s.url, bytes.NewReader(body))
- if err != nil {
- return Result{}, fmt.Errorf("build scan request: %w", err)
- }
- httpReq.Header.Set("Content-Type", "application/json")
- for k, v := range s.headers {
- httpReq.Header.Set(k, v)
- }
-
- resp, err := s.client.Do(httpReq)
- if err != nil {
- return Result{}, fmt.Errorf("calling scanner %q: %w", s.name, err)
- }
- defer func() { _ = resp.Body.Close() }()
-
- if resp.StatusCode != http.StatusOK {
- return Result{}, fmt.Errorf("scanner %q returned status %d", s.name, resp.StatusCode)
- }
-
- var out httpScanResponse
- if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
- return Result{}, fmt.Errorf("decoding scanner %q response: %w", s.name, err)
- }
-
- return Result{
- Allowed: out.Allowed,
- Reason: out.Reason,
- Findings: out.Findings,
- }, nil
-}
diff --git a/internal/scanner/http_test.go b/internal/scanner/http_test.go
deleted file mode 100644
index 7597826..0000000
--- a/internal/scanner/http_test.go
+++ /dev/null
@@ -1,119 +0,0 @@
-package scanner
-
-import (
- "context"
- "encoding/json"
- "net/http"
- "net/http/httptest"
- "testing"
- "time"
-)
-
-func TestHTTPScanner_Scan(t *testing.T) {
- tests := []struct {
- name string
- respStatus int
- respBody string
- wantAllowed bool
- wantReason string
- wantErr bool
- }{
- {
- name: "allowed",
- respStatus: http.StatusOK,
- respBody: `{"allowed": true}`,
- wantAllowed: true,
- },
- {
- name: "blocked with reason and findings",
- respStatus: http.StatusOK,
- respBody: `{"allowed": false, "reason": "malware detected", "findings": [{"severity": "high", "title": "EICAR", "description": "test signature"}]}`,
- wantAllowed: false,
- wantReason: "malware detected",
- },
- {
- name: "non-200 status is an error",
- respStatus: http.StatusInternalServerError,
- respBody: `{}`,
- wantErr: true,
- },
- {
- name: "malformed JSON is an error",
- respStatus: http.StatusOK,
- respBody: `not json`,
- wantErr: true,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- var got Request
- if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
- t.Errorf("decode request body: %v", err)
- }
- if got.Ecosystem != "npm" || got.Name != "left-pad" || got.FetchURL == "" {
- t.Errorf("unexpected request body: %+v", got)
- }
- if r.Header.Get("Authorization") != "Bearer secret" {
- t.Errorf("missing/incorrect Authorization header: %q", r.Header.Get("Authorization"))
- }
-
- w.WriteHeader(tt.respStatus)
- _, _ = w.Write([]byte(tt.respBody))
- }))
- defer srv.Close()
-
- s := NewHTTPScanner("test", srv.URL, map[string]string{"Authorization": "Bearer secret"}, nil)
-
- result, err := s.Scan(context.Background(), Request{
- Ecosystem: "npm",
- Name: "left-pad",
- Version: "1.0.0",
- FetchURL: srv.URL + "/fetch",
- })
-
- if tt.wantErr {
- if err == nil {
- t.Fatalf("Scan() error = nil, want error")
- }
- return
- }
- if err != nil {
- t.Fatalf("Scan() unexpected error: %v", err)
- }
- if result.Allowed != tt.wantAllowed {
- t.Errorf("Allowed = %v, want %v", result.Allowed, tt.wantAllowed)
- }
- if result.Reason != tt.wantReason {
- t.Errorf("Reason = %q, want %q", result.Reason, tt.wantReason)
- }
- })
- }
-}
-
-func TestHTTPScanner_Scan_ContextTimeout(t *testing.T) {
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- time.Sleep(50 * time.Millisecond)
- w.WriteHeader(http.StatusOK)
- _, _ = w.Write([]byte(`{"allowed": true}`))
- }))
- defer srv.Close()
-
- s := NewHTTPScanner("slow", srv.URL, nil, nil)
-
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond)
- defer cancel()
-
- _, err := s.Scan(ctx, Request{Ecosystem: "npm", Name: "left-pad"})
- if err == nil {
- t.Fatal("Scan() error = nil, want timeout error")
- }
-}
-
-func TestHTTPScanner_Name(t *testing.T) {
- s := NewHTTPScanner("clamav", "http://example.invalid", nil, nil)
- if got := s.Name(); got != "clamav" {
- t.Errorf("Name() = %q, want %q", got, "clamav")
- }
-}
diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go
deleted file mode 100644
index 380bb4d..0000000
--- a/internal/scanner/scanner.go
+++ /dev/null
@@ -1,65 +0,0 @@
-// Package scanner provides pluggable pre-cache artifact scanning.
-//
-// A Scanner inspects an artifact staged in the proxy's own storage before
-// it becomes visible to clients, and returns a verdict on whether it may
-// be cached. The proxy never uploads artifact bytes to a scanner directly:
-// it hands the scanner a short-lived signed URL and the scanner pulls the
-// bytes itself. See HTTPScanner for the built-in adapter that implements
-// this over a small HTTP/JSON contract, letting trivy, ClamAV, Wiz, or any
-// custom service integrate without the proxy needing built-in knowledge of
-// any specific tool.
-package scanner
-
-import "context"
-
-// Request describes a staged artifact awaiting a scan verdict.
-type Request struct {
- Ecosystem string `json:"ecosystem"`
- Name string `json:"name"`
- Version string `json:"version"`
- Filename string `json:"filename"`
- PURL string `json:"purl"`
- ContentType string `json:"content_type"`
- Size int64 `json:"size"`
-
- // FetchURL is a short-lived signed URL the scanner must GET itself to
- // retrieve the exact bytes staged in the proxy's storage.
- FetchURL string `json:"fetch_url"`
-}
-
-// Finding describes a single issue reported by a scanner.
-type Finding struct {
- Severity string
- Title string
- Description string
-}
-
-// Result is a scanner's verdict for a Request.
-type Result struct {
- Allowed bool
- Reason string
- Findings []Finding
-
- // ScannerName identifies which scanner produced this result. Set by
- // Group, not by individual Scanner implementations.
- ScannerName string
-
- // InfraError reports whether Allowed: false was forced by a scanner
- // call failing (network error, timeout, bad response) rather than an
- // actual verdict from the scanner. Set by Group. Callers that surface
- // Reason to untrusted clients must not do so when this is true: it may
- // contain raw connection errors (internal hostnames, ports) instead of
- // a verdict meant to be shown outside the proxy.
- InfraError bool
-}
-
-// Scanner is the extension point for pluggable pre-cache scanning.
-type Scanner interface {
- // Name identifies this scanner in logs and metrics.
- Name() string
-
- // Scan requests a verdict for req. Implementations must respect ctx
- // cancellation: Group cancels in-flight scans once a blocking verdict
- // has already been decided by another scanner.
- Scan(ctx context.Context, req Request) (Result, error)
-}
diff --git a/internal/server/api.go b/internal/server/api.go
index 992d736..e687f6d 100644
--- a/internal/server/api.go
+++ b/internal/server/api.go
@@ -139,14 +139,11 @@ type BulkResponse struct {
// Resolves namespaced package names (Composer vendor/name, npm @scope/name) from the path.
func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) {
ecosystem := chi.URLParam(r, "ecosystem")
- segments, err := packagePathSegments(r)
- if err != nil {
- badRequest(w, err.Error())
- return
- }
+ wildcard := chi.URLParam(r, "*")
+ segments := splitWildcardPath(wildcard)
if ecosystem == "" || len(segments) == 0 {
- badRequest(w, "ecosystem and name are required")
+ http.Error(w, "ecosystem and name are required", http.StatusBadRequest)
return
}
@@ -193,12 +190,12 @@ func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) {
func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosystem, name string) {
info, err := h.enrichment.EnrichPackage(r.Context(), ecosystem, name)
if err != nil {
- writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to enrich package")
+ http.Error(w, "failed to enrich package", http.StatusInternalServerError)
return
}
if info == nil {
- notFound(w, "package not found")
+ http.Error(w, "package not found", http.StatusNotFound)
return
}
@@ -220,7 +217,7 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste
func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) {
result, err := h.enrichment.EnrichFull(r.Context(), ecosystem, name, version)
if err != nil {
- writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to enrich version")
+ http.Error(w, "failed to enrich version", http.StatusInternalServerError)
return
}
@@ -276,14 +273,11 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste
// Supports both {name} and {name}/{version} paths with namespaced package names.
func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
ecosystem := chi.URLParam(r, "ecosystem")
- segments, err := packagePathSegments(r)
- if err != nil {
- badRequest(w, err.Error())
- return
- }
+ wildcard := chi.URLParam(r, "*")
+ segments := splitWildcardPath(wildcard)
if ecosystem == "" || len(segments) == 0 {
- badRequest(w, "ecosystem and name are required")
+ http.Error(w, "ecosystem and name are required", http.StatusBadRequest)
return
}
@@ -304,7 +298,7 @@ func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
vulns, err := h.enrichment.CheckVulnerabilities(r.Context(), ecosystem, name, version)
if err != nil {
- writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to check vulnerabilities")
+ http.Error(w, "failed to check vulnerabilities", http.StatusInternalServerError)
return
}
@@ -336,19 +330,19 @@ func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
// @Produce json
// @Param request body OutdatedRequest true "Packages to check"
// @Success 200 {object} OutdatedResponse
-// @Failure 400 {object} ErrorResponse
-// @Failure 500 {object} ErrorResponse
+// @Failure 400 {string} string
+// @Failure 500 {string} string
// @Router /api/outdated [post]
func (h *APIHandler) HandleOutdated(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
var req OutdatedRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
- badRequest(w, "invalid request body")
+ http.Error(w, "invalid request body", http.StatusBadRequest)
return
}
if len(req.Packages) == 0 {
- badRequest(w, "packages list is required")
+ http.Error(w, "packages list is required", http.StatusBadRequest)
return
}
@@ -382,19 +376,19 @@ func (h *APIHandler) HandleOutdated(w http.ResponseWriter, r *http.Request) {
// @Produce json
// @Param request body BulkRequest true "PURLs"
// @Success 200 {object} BulkResponse
-// @Failure 400 {object} ErrorResponse
-// @Failure 500 {object} ErrorResponse
+// @Failure 400 {string} string
+// @Failure 500 {string} string
// @Router /api/bulk [post]
func (h *APIHandler) HandleBulkLookup(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
var req BulkRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
- badRequest(w, "invalid request body")
+ http.Error(w, "invalid request body", http.StatusBadRequest)
return
}
if len(req.PURLs) == 0 {
- badRequest(w, "purls list is required")
+ http.Error(w, "purls list is required", http.StatusBadRequest)
return
}
@@ -482,15 +476,15 @@ type SearchPackageResult struct {
// @Param q query string true "Query"
// @Param ecosystem query string false "Ecosystem"
// @Success 200 {object} SearchResponse
-// @Failure 400 {object} ErrorResponse
-// @Failure 500 {object} ErrorResponse
+// @Failure 400 {string} string
+// @Failure 500 {string} string
// @Router /api/search [get]
func (h *APIHandler) HandleSearch(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query().Get("q")
ecosystem := r.URL.Query().Get("ecosystem")
if query == "" {
- badRequest(w, "query parameter 'q' is required")
+ http.Error(w, "query parameter 'q' is required", http.StatusBadRequest)
return
}
@@ -500,7 +494,7 @@ func (h *APIHandler) HandleSearch(w http.ResponseWriter, r *http.Request) {
// Search in database
results, err := h.db.SearchPackages(query, ecosystem, limit, (page-1)*limit)
if err != nil {
- internalError(w, "search failed")
+ http.Error(w, "search failed", http.StatusInternalServerError)
return
}
@@ -544,7 +538,7 @@ func (h *APIHandler) HandleSearch(w http.ResponseWriter, r *http.Request) {
func writeJSON(w http.ResponseWriter, v any) {
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(v); err != nil {
- internalError(w, "failed to encode response")
+ http.Error(w, "failed to encode response", http.StatusInternalServerError)
}
}
@@ -579,8 +573,8 @@ type PackageListResult struct {
// @Param ecosystem query string false "Ecosystem"
// @Param sort query string false "Sort" Enums(hits,name,size,cached_at,ecosystem,vulns)
// @Success 200 {object} PackagesListResponse
-// @Failure 400 {object} ErrorResponse
-// @Failure 500 {object} ErrorResponse
+// @Failure 400 {string} string
+// @Failure 500 {string} string
// @Router /api/packages [get]
func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request) {
ecosystem := r.URL.Query().Get("ecosystem")
@@ -591,14 +585,14 @@ func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request)
validSorts := map[string]bool{
defaultSortBy: true,
- "name": true,
- "size": true,
- "cached_at": true,
- "ecosystem": true,
- "vulns": true,
+ "name": true,
+ "size": true,
+ "cached_at": true,
+ "ecosystem": true,
+ "vulns": true,
}
if !validSorts[sortBy] {
- badRequest(w, "invalid sort parameter")
+ http.Error(w, "invalid sort parameter", http.StatusBadRequest)
return
}
@@ -607,7 +601,7 @@ func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request)
packages, err := h.db.ListCachedPackages(ecosystem, sortBy, limit, (page-1)*limit)
if err != nil {
- internalError(w, "failed to list packages")
+ http.Error(w, "failed to list packages", http.StatusInternalServerError)
return
}
diff --git a/internal/server/api_test.go b/internal/server/api_test.go
index 0494b2f..548f324 100644
--- a/internal/server/api_test.go
+++ b/internal/server/api_test.go
@@ -9,7 +9,6 @@ import (
"net/http/httptest"
"os"
"path/filepath"
- "strings"
"testing"
"github.com/git-pkgs/proxy/internal/database"
@@ -49,35 +48,6 @@ func TestHandlePackagePath_MissingParams(t *testing.T) {
}
}
-func TestHandlePackagePath_InvalidName(t *testing.T) {
- logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
- svc := enrichment.New(logger)
- h := NewAPIHandler(svc, nil)
-
- r := chi.NewRouter()
- r.Get("/api/package/{ecosystem}/*", h.HandlePackagePath)
-
- tests := []struct {
- name string
- path string
- }{
- {"null byte", "/api/package/npm/lodash%00"},
- {"too long", "/api/package/npm/" + strings.Repeat("a", maxPackagePathLen+1)},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- req := httptest.NewRequest("GET", tt.path, nil)
- w := httptest.NewRecorder()
- r.ServeHTTP(w, req)
-
- if w.Code != http.StatusBadRequest {
- t.Errorf("expected status 400, got %d", w.Code)
- }
- })
- }
-}
-
func TestHandleVulnsPath_MissingParams(t *testing.T) {
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
svc := enrichment.New(logger)
diff --git a/internal/server/breakers.go b/internal/server/breakers.go
deleted file mode 100644
index 383e0db..0000000
--- a/internal/server/breakers.go
+++ /dev/null
@@ -1,130 +0,0 @@
-package server
-
-import (
- "log/slog"
- "sync"
-
- "github.com/git-pkgs/proxy/internal/metrics"
-)
-
-// Gauge values for proxy_circuit_breaker_state. The fetcher reports only open
-// or closed, so half-open (1) is never published.
-const (
- breakerGaugeClosed = 0
- breakerGaugeOpen = 2
-)
-
-const (
- breakerStateOpen = "open"
- breakerStateClosed = "closed"
-)
-
-// breakerStateSource reports circuit breaker state per upstream registry, keyed
-// by the identifier the fetcher derives from the fetch URL, with values
-// breakerStateOpen or breakerStateClosed. Implemented by
-// fetch.CircuitBreakerFetcher.
-type breakerStateSource interface {
- GetBreakerState() map[string]string
-}
-
-// breakerMonitor mirrors the artifact fetcher's per-registry circuit breaker
-// state into Prometheus metrics, the health report, and the log.
-//
-// Breaker state lives only in the fetcher's memory. While a breaker is open
-// every artifact fetch it covers that misses the cache fails without reaching
-// the upstream, bar the one probe per backoff interval the breaker admits to
-// test recovery. That looks identical to an upstream outage from the outside:
-// metadata still serves (it does not go through the fetcher), other registries
-// still serve, and /health reports the database and storage as fine. Publishing
-// the state makes that distinguishable.
-type breakerMonitor struct {
- source breakerStateSource
- logger *slog.Logger
-
- // mu serializes snapshots. It guards seen, which holds one entry per
- // registry that has tripped at least once in this process — the only
- // registries published as metrics — and keeps each state read paired with
- // the updates it produces.
- mu sync.Mutex
- seen map[string]string
-}
-
-func newBreakerMonitor(source breakerStateSource, logger *slog.Logger) *breakerMonitor {
- if logger == nil {
- logger = slog.Default()
- }
- return &breakerMonitor{
- source: source,
- logger: logger,
- seen: map[string]string{},
- }
-}
-
-// snapshot returns the current state of every breaker the fetcher has created,
-// keyed by registry identifier, and mirrors it into the breaker metrics as a
-// side effect. It returns nil for a nil monitor so callers that build a Server
-// without a fetcher (tests) need no special case.
-//
-// The keys pass through unaltered into Prometheus labels and the /health body,
-// neither of which is authenticated, so they are only as safe to publish as the
-// fetcher makes them. It keys by the fetch URL's host, and where it cannot take
-// a host from that URL — a signed composer dist.url that fails to parse, say —
-// by an opaque keyed digest of the URL rather than the URL itself, so a
-// credential carried in one does not reach either endpoint
-// (github.com/git-pkgs/registries v0.9.0 and later).
-//
-// Only registries that have tripped at least once are published as metrics.
-// The fetcher creates a breaker per identifier it fetches under, and for some
-// ecosystems that identifier comes from upstream metadata rather than
-// configuration (composer takes it from a package's dist.url, helm from the
-// chart URLs in index.yaml), so publishing every one would let upstream content
-// grow the series count for the life of the process — the more so for URLs with
-// no host, which get an identifier apiece rather than sharing one. An
-// identifier that has never tripped carries no information a series could
-// convey; once it trips it keeps reporting, including the 0 that marks its
-// recovery. /health is a per-request response rather than a persistent series,
-// so it reports every breaker.
-//
-// Trips are counted on the closed→open transitions observed between calls,
-// because the fetcher exposes current state rather than trip events: a breaker
-// that opens and recovers entirely between two calls is not counted.
-func (m *breakerMonitor) snapshot() map[string]string {
- if m == nil || m.source == nil {
- return nil
- }
-
- m.mu.Lock()
- defer m.mu.Unlock()
-
- // Read under the lock. Two concurrent snapshots — a /health request and a
- // /metrics scrape landing during a transition — can otherwise apply their
- // reads to seen in the opposite order, counting one trip twice, logging a
- // close for a breaker that is still open, and leaving the gauge at 0 until
- // the next call.
- states := m.source.GetBreakerState()
-
- for registry, state := range states {
- previous, published := m.seen[registry]
-
- switch {
- case state == breakerStateOpen && previous != breakerStateOpen:
- metrics.RecordCircuitBreakerTrip(registry)
- m.logger.Error("circuit breaker open, artifact fetches for this registry "+
- "fail without contacting it", "registry", registry)
- case state == breakerStateClosed && previous == breakerStateOpen:
- m.logger.Info("circuit breaker closed", "registry", registry)
- case state == breakerStateClosed && !published:
- // Never tripped: nothing to publish.
- continue
- }
-
- gauge := breakerGaugeClosed
- if state == breakerStateOpen {
- gauge = breakerGaugeOpen
- }
- metrics.UpdateCircuitBreakerState(registry, gauge)
- m.seen[registry] = state
- }
-
- return states
-}
diff --git a/internal/server/breakers_test.go b/internal/server/breakers_test.go
deleted file mode 100644
index ae39759..0000000
--- a/internal/server/breakers_test.go
+++ /dev/null
@@ -1,348 +0,0 @@
-package server
-
-import (
- "context"
- "encoding/json"
- "io"
- "log/slog"
- "net/http"
- "net/http/httptest"
- "strings"
- "sync/atomic"
- "testing"
- "time"
-
- "github.com/git-pkgs/proxy/internal/metrics"
- "github.com/git-pkgs/registries/fetch"
- "github.com/prometheus/client_golang/prometheus"
- dto "github.com/prometheus/client_model/go"
-)
-
-// cbThreshold in registries/fetch: failures inside the breaker's rolling window
-// needed to trip it. The library counts the window, not a consecutive run, so
-// these fetches only have to land close together, which they do.
-const breakerTripFailures = 5
-
-// fakeBreakerSource reports breaker state without a real fetcher, so tests can
-// drive transitions that would otherwise need to wait out a 30s backoff.
-type fakeBreakerSource struct {
- states map[string]string
-}
-
-func (f *fakeBreakerSource) GetBreakerState() map[string]string {
- states := make(map[string]string, len(f.states))
- for registry, state := range f.states {
- states[registry] = state
- }
- return states
-}
-
-// newTrippedMonitor returns a monitor over a real circuit-breaker fetcher whose
-// breaker for the test server's host has been tripped by repeated 5xx
-// responses, plus that host.
-func newTrippedMonitor(t *testing.T) (*breakerMonitor, string) {
- t.Helper()
-
- upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- w.WriteHeader(http.StatusBadGateway)
- }))
- t.Cleanup(upstream.Close)
-
- monitor, fetcher, host := newMonitorFor(t, upstream)
- for range breakerTripFailures {
- if _, err := fetcher.Fetch(context.Background(), upstream.URL+"/artifact.tgz"); err == nil {
- t.Fatal("fetch against a 502 upstream should fail")
- }
- }
- return monitor, host
-}
-
-// newMonitorFor builds a monitor over a circuit-breaker fetcher that talks to
-// srv through its own client, bypassing the SSRF dial gate that would otherwise
-// refuse the loopback address.
-func newMonitorFor(t *testing.T, srv *httptest.Server) (
- monitor *breakerMonitor, fetcher *fetch.CircuitBreakerFetcher, host string,
-) {
- t.Helper()
-
- base := fetch.NewFetcher(
- fetch.WithHTTPClient(srv.Client()),
- fetch.WithMaxRetries(0),
- )
- t.Cleanup(func() { _ = base.Close() })
-
- fetcher = fetch.NewCircuitBreakerFetcher(base)
- return newBreakerMonitor(fetcher, slog.New(slog.DiscardHandler)),
- fetcher,
- strings.TrimPrefix(srv.URL, "http://")
-}
-
-// metricValue returns the value of the series carrying registry=want, and
-// whether such a series exists at all. It collects rather than calling
-// WithLabelValues, which would create the series it is looking for.
-func metricValue(t *testing.T, collector prometheus.Collector, want string) (value float64, found bool) {
- t.Helper()
-
- ch := make(chan prometheus.Metric, 64)
- go func() {
- collector.Collect(ch)
- close(ch)
- }()
-
- for metric := range ch {
- var parsed dto.Metric
- if err := metric.Write(&parsed); err != nil {
- t.Fatalf("writing metric: %v", err)
- }
- for _, label := range parsed.GetLabel() {
- if label.GetName() != "registry" || label.GetValue() != want {
- continue
- }
- if gauge := parsed.GetGauge(); gauge != nil {
- return gauge.GetValue(), true
- }
- return parsed.GetCounter().GetValue(), true
- }
- }
- return 0, false
-}
-
-// resetSeries drops any series for host left behind by an earlier test, since
-// httptest ports can be reused within a process and the metrics registry is
-// global. Absolute trip counts stay meaningful after it.
-func resetSeries(host string) {
- metrics.CircuitBreakerState.DeleteLabelValues(host)
- metrics.CircuitBreakerTrips.DeleteLabelValues(host)
-}
-
-func TestBreakerMonitor_OpenBreakerReportedAndCounted(t *testing.T) {
- monitor, host := newTrippedMonitor(t)
- resetSeries(host)
-
- if state := monitor.snapshot()[host]; state != breakerStateOpen {
- t.Fatalf("state for %s = %q, want open", host, state)
- }
-
- gauge, found := metricValue(t, metrics.CircuitBreakerState, host)
- if !found {
- t.Fatalf("no state gauge published for %s", host)
- }
- if gauge != breakerGaugeOpen {
- t.Errorf("state gauge = %v, want %v", gauge, breakerGaugeOpen)
- }
- if trips, _ := metricValue(t, metrics.CircuitBreakerTrips, host); trips != 1 {
- t.Errorf("trips = %v, want 1", trips)
- }
-
- // A breaker that stays open is one trip, not one per scrape.
- monitor.snapshot()
- monitor.snapshot()
- if trips, _ := metricValue(t, metrics.CircuitBreakerTrips, host); trips != 1 {
- t.Errorf("trips after further snapshots = %v, want 1", trips)
- }
-}
-
-// A breaker per fetched host is created even for hosts that come from upstream
-// metadata (composer dist.url, helm chart URLs), so publishing a series for
-// every one of them would let upstream content grow the series count without
-// bound. Only registries that have actually tripped are published.
-func TestBreakerMonitor_HealthyRegistryPublishesNoSeries(t *testing.T) {
- upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- _, _ = w.Write([]byte("artifact"))
- }))
- defer upstream.Close()
-
- monitor, fetcher, host := newMonitorFor(t, upstream)
- resetSeries(host)
-
- artifact, err := fetcher.Fetch(context.Background(), upstream.URL+"/artifact.tgz")
- if err != nil {
- t.Fatalf("fetch: %v", err)
- }
- _, _ = io.Copy(io.Discard, artifact.Body)
- _ = artifact.Body.Close()
-
- // The breaker exists and is reported to /health...
- if state := monitor.snapshot()[host]; state != breakerStateClosed {
- t.Fatalf("state for %s = %q, want closed", host, state)
- }
- // ...but carries no metric series.
- if _, found := metricValue(t, metrics.CircuitBreakerState, host); found {
- t.Errorf("state gauge published for %s, want none until it trips", host)
- }
- if _, found := metricValue(t, metrics.CircuitBreakerTrips, host); found {
- t.Errorf("trip counter published for %s, want none until it trips", host)
- }
-}
-
-// Once a registry has tripped it keeps reporting, so recovery is visible as a
-// transition to 0 rather than as a series that disappears.
-func TestBreakerMonitor_RecoveryReportedAfterTrip(t *testing.T) {
- const host = "recovering.example.com"
-
- resetSeries(host)
-
- source := &fakeBreakerSource{states: map[string]string{host: breakerStateOpen}}
- monitor := newBreakerMonitor(source, slog.New(slog.DiscardHandler))
-
- monitor.snapshot()
- if gauge, _ := metricValue(t, metrics.CircuitBreakerState, host); gauge != breakerGaugeOpen {
- t.Fatalf("state gauge = %v, want %v", gauge, breakerGaugeOpen)
- }
-
- source.states[host] = breakerStateClosed
- if state := monitor.snapshot()[host]; state != breakerStateClosed {
- t.Fatalf("state for %s = %q, want closed", host, state)
- }
- gauge, found := metricValue(t, metrics.CircuitBreakerState, host)
- if !found {
- t.Fatal("state gauge disappeared after recovery, want 0")
- }
- if gauge != breakerGaugeClosed {
- t.Errorf("state gauge = %v, want %v", gauge, breakerGaugeClosed)
- }
- if trips, _ := metricValue(t, metrics.CircuitBreakerTrips, host); trips != 1 {
- t.Errorf("trips = %v, want 1", trips)
- }
-
- // Tripping again after recovery counts a second trip.
- source.states[host] = breakerStateOpen
- monitor.snapshot()
- if trips, _ := metricValue(t, metrics.CircuitBreakerTrips, host); trips != 2 {
- t.Errorf("trips after re-trip = %v, want 2", trips)
- }
-}
-
-func TestBreakerMonitor_NilSafe(t *testing.T) {
- var nilMonitor *breakerMonitor
- if states := nilMonitor.snapshot(); states != nil {
- t.Errorf("nil monitor snapshot = %v, want nil", states)
- }
- if states := newBreakerMonitor(nil, nil).snapshot(); states != nil {
- t.Errorf("snapshot without a state source = %v, want nil", states)
- }
-}
-
-func TestHealthEndpoint_ReportsOpenBreaker(t *testing.T) {
- ts := newTestServer(t)
- defer ts.close()
-
- monitor, host := newTrippedMonitor(t)
- ts.server.breakers = monitor
-
- req := httptest.NewRequest("GET", "/health", nil)
- w := httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
-
- // An unreachable upstream is not this proxy being unhealthy: the breaker
- // state is reported, but the probe still passes.
- if w.Code != http.StatusOK {
- t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
- }
-
- var resp HealthResponse
- if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
- t.Fatalf("decoding response: %v", err)
- }
- if resp.Status != "ok" {
- t.Errorf("status = %q, want ok", resp.Status)
- }
- if got := resp.CircuitBreakers[host]; got != breakerStateOpen {
- t.Errorf("circuit_breakers[%s] = %q, want open", host, got)
- }
-}
-
-func TestHealthEndpoint_OmitsBreakersWhenNoneExist(t *testing.T) {
- ts := newTestServer(t)
- defer ts.close()
-
- req := httptest.NewRequest("GET", "/health", nil)
- w := httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
-
- if strings.Contains(w.Body.String(), "circuit_breakers") {
- t.Errorf("body should omit circuit_breakers when no breaker exists: %s", w.Body.String())
- }
-}
-
-// blockingBreakerSource holds each state read open until the test releases it,
-// and reports every entry, so a read that escaped the monitor lock announces
-// itself. inFlight is a second net: it catches an overlap anywhere in the test,
-// not just while the release channel is held.
-type blockingBreakerSource struct {
- states map[string]string
- entered chan struct{}
- release chan struct{}
- inFlight atomic.Int32
- overlap atomic.Bool
-}
-
-func (b *blockingBreakerSource) GetBreakerState() map[string]string {
- if b.inFlight.Add(1) > 1 {
- b.overlap.Store(true)
- }
- defer b.inFlight.Add(-1)
-
- b.entered <- struct{}{}
- <-b.release
-
- states := make(map[string]string, len(b.states))
- for registry, state := range b.states {
- states[registry] = state
- }
- return states
-}
-
-// /health requests and /metrics scrapes call snapshot concurrently. The state
-// read has to happen under the same lock as the seen updates it feeds: read
-// outside it, two snapshots straddling a transition can apply their reads to
-// seen in the opposite order and count one trip twice.
-func TestBreakerMonitor_SnapshotSerializesStateReads(t *testing.T) {
- const host = "serialized.example.com"
-
- resetSeries(host)
-
- source := &blockingBreakerSource{
- states: map[string]string{host: breakerStateOpen},
- entered: make(chan struct{}, 2),
- release: make(chan struct{}),
- }
- monitor := newBreakerMonitor(source, slog.New(slog.DiscardHandler))
-
- first := make(chan map[string]string, 1)
- go func() { first <- monitor.snapshot() }()
-
- // The first snapshot is inside GetBreakerState now, holding the lock.
- <-source.entered
-
- second := make(chan map[string]string, 1)
- go func() { second <- monitor.snapshot() }()
-
- // Nothing can release the first read, so the second cannot get past the
- // lock: an unlocked read would have queued an entry on the channel.
- select {
- case <-source.entered:
- t.Fatal("second snapshot read breaker state while the first still held the lock")
- case <-time.After(100 * time.Millisecond):
- }
-
- close(source.release)
-
- if state := (<-first)[host]; state != breakerStateOpen {
- t.Errorf("first snapshot state for %s = %q, want open", host, state)
- }
- // Only reached once the first snapshot has returned the lock.
- <-source.entered
- if state := (<-second)[host]; state != breakerStateOpen {
- t.Errorf("second snapshot state for %s = %q, want open", host, state)
- }
-
- if source.overlap.Load() {
- t.Error("two snapshots read breaker state concurrently")
- }
- // The same open breaker seen twice is one trip, which is what the paired
- // read and update buys.
- if trips, _ := metricValue(t, metrics.CircuitBreakerTrips, host); trips != 1 {
- t.Errorf("trips = %v, want 1", trips)
- }
-}
diff --git a/internal/server/browse.go b/internal/server/browse.go
index fc5e658..7e035d2 100644
--- a/internal/server/browse.go
+++ b/internal/server/browse.go
@@ -1,7 +1,7 @@
package server
import (
- "bufio"
+ "bytes"
"encoding/json"
"fmt"
"io"
@@ -11,45 +11,22 @@ import (
"github.com/git-pkgs/archives"
"github.com/git-pkgs/archives/diff"
- "github.com/git-pkgs/magic"
"github.com/git-pkgs/proxy/internal/database"
- "github.com/git-pkgs/proxy/internal/handler"
+ "github.com/git-pkgs/purl"
"github.com/go-chi/chi/v5"
)
-const (
- contentTypePlainText = "text/plain; charset=utf-8"
- browseSniffSize = 512
-)
+const contentTypePlainText = "text/plain; charset=utf-8"
-// maxBrowseArchiveSize caps how much data openArchive will buffer for
-// prefix detection. Artifacts larger than this are rejected to prevent
-// memory exhaustion from a single request.
-const maxBrowseArchiveSize = 512 << 20 // 512 MB
-
-// firstBrowsableArtifact returns the first cached artifact that can be opened as
-// an archive, or nil if the version has none.
-//
-// A version's artifact list is not all archives: a PEP 658 core-metadata sidecar
-// resolves to the same name and version as the distribution it describes, so it
-// is cached under that version too. Sidecars are plain text, and because '-'
-// sorts before '.' one can even precede the real distribution in the
-// filename-ordered list, so selecting blindly would hand openArchive a file it
-// cannot parse.
-func firstBrowsableArtifact(artifacts []database.Artifact) *database.Artifact {
- for i := range artifacts {
- if artifacts[i].StoragePath.Valid && !isMetadataSidecar(artifacts[i].Filename) {
- return &artifacts[i]
- }
+// archiveFilename returns a filename suitable for archive format detection.
+// Some ecosystems (e.g. composer) store artifacts with bare hash filenames
+// that have no extension. This adds .zip when the original has no extension
+// and the content is likely a zip archive.
+func archiveFilename(filename string) string {
+ if path.Ext(filename) == "" {
+ return filename + ".zip"
}
-
- return nil
-}
-
-// isMetadataSidecar reports whether filename is a core-metadata sidecar rather
-// than a distribution archive.
-func isMetadataSidecar(filename string) bool {
- return strings.HasSuffix(filename, handler.PyPIMetadataSuffix)
+ return filename
}
// detectSingleRootDir returns the single top-level directory name if all files
@@ -85,29 +62,31 @@ func detectSingleRootDir(reader archives.Reader) string {
// and stripping a single top-level directory prefix (like GitHub zipballs).
// For npm, the hardcoded "package/" prefix takes precedence.
func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations
- limited := io.LimitReader(content, maxBrowseArchiveSize+1)
- data, err := io.ReadAll(limited)
+ fname := archiveFilename(filename)
+
+ // npm always uses package/ prefix
+ if ecosystem == "npm" {
+ return archives.OpenWithPrefix(fname, content, "package/")
+ }
+
+ // Read content into memory so we can scan then wrap with prefix
+ data, err := io.ReadAll(content)
if err != nil {
return nil, fmt.Errorf("reading artifact: %w", err)
}
- if int64(len(data)) > maxBrowseArchiveSize {
- return nil, fmt.Errorf("artifact too large for browsing (%d bytes)", len(data))
- }
- if ecosystem == "npm" {
- return archives.OpenBytesWithPrefix(filename, data, "package/")
- }
-
- probe, err := archives.OpenBytes(filename, data)
+ // Open once to detect root prefix
+ probe, err := archives.Open(fname, bytes.NewReader(data))
if err != nil {
return nil, err
}
prefix := detectSingleRootDir(probe)
_ = probe.Close()
- return archives.OpenBytesWithPrefix(filename, data, prefix)
+ return archives.OpenWithPrefix(fname, bytes.NewReader(data), prefix)
}
+
// BrowseListResponse contains the file listing for a directory in an archives.
type BrowseListResponse struct {
Path string `json:"path"`
@@ -134,9 +113,9 @@ type BrowseFileInfo struct {
// @Param version path string true "Version"
// @Param path query string false "Directory path inside the archive"
// @Success 200 {object} BrowseListResponse
-// @Failure 404 {object} ErrorResponse
-// @Failure 500 {object} ErrorResponse
-// @Router /ui/api/browse/{ecosystem}/{name}/{version} [get]
+// @Failure 404 {string} string
+// @Failure 500 {string} string
+// @Router /api/browse/{ecosystem}/{name}/{version} [get]
// handleBrowsePath dispatches /api/browse/{ecosystem}/* to the appropriate browse handler.
// It resolves namespaced package names by consulting the database.
//
@@ -146,14 +125,11 @@ type BrowseFileInfo struct {
// {name}/{version}/file/{path} -> browse file
func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
ecosystem := chi.URLParam(r, "ecosystem")
- segments, err := packagePathSegments(r)
- if err != nil {
- badRequest(w, err.Error())
- return
- }
+ wildcard := chi.URLParam(r, "*")
+ segments := splitWildcardPath(wildcard)
if ecosystem == "" || len(segments) < 2 {
- badRequest(w, "ecosystem, name, and version required")
+ http.Error(w, "ecosystem, name, and version required", http.StatusBadRequest)
return
}
@@ -177,7 +153,7 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
rest = nameVersionSegments[len(nameVersionSegments)-1:]
}
if len(rest) != 1 {
- notFound(w, "not found")
+ http.Error(w, "not found", http.StatusNotFound)
return
}
s.browseFile(w, r, ecosystem, name, rest[0], filePath)
@@ -191,7 +167,7 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
rest = segments[len(segments)-1:]
}
if len(rest) != 1 {
- notFound(w, "not found")
+ http.Error(w, "not found", http.StatusNotFound)
return
}
s.browseList(w, r, ecosystem, name, rest[0])
@@ -201,14 +177,11 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
// Supported paths: {name}/{fromVersion}/{toVersion}
func (s *Server) handleComparePath(w http.ResponseWriter, r *http.Request) {
ecosystem := chi.URLParam(r, "ecosystem")
- segments, err := packagePathSegments(r)
- if err != nil {
- badRequest(w, err.Error())
- return
- }
+ wildcard := chi.URLParam(r, "*")
+ segments := splitWildcardPath(wildcard)
if ecosystem == "" || len(segments) < 3 {
- badRequest(w, "ecosystem, name, fromVersion, and toVersion required")
+ http.Error(w, "ecosystem, name, fromVersion, and toVersion required", http.StatusBadRequest)
return
}
@@ -225,22 +198,29 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
dirPath := r.URL.Query().Get("path")
// Get the artifact for this version
- versionPURL := s.cachedVersionPURL(ecosystem, name, version)
+ versionPURL := purl.MakePURLString(ecosystem, name, version)
artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL)
if err != nil {
- notFound(w, "version not found")
+ http.Error(w, "version not found", http.StatusNotFound)
return
}
if len(artifacts) == 0 {
- notFound(w, "no artifacts cached")
+ http.Error(w, "no artifacts cached", http.StatusNotFound)
return
}
- cachedArtifact := firstBrowsableArtifact(artifacts)
+ // Find the first cached artifact
+ var cachedArtifact *database.Artifact
+ for i := range artifacts {
+ if artifacts[i].StoragePath.Valid {
+ cachedArtifact = &artifacts[i]
+ break
+ }
+ }
if cachedArtifact == nil {
- notFound(w, "artifact not cached")
+ http.Error(w, "artifact not cached", http.StatusNotFound)
return
}
@@ -248,7 +228,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
artifactReader, err := s.storage.Open(r.Context(), cachedArtifact.StoragePath.String)
if err != nil {
s.logger.Error("failed to read artifact from storage", "error", err)
- internalError(w, "failed to read artifact")
+ http.Error(w, "failed to read artifact", http.StatusInternalServerError)
return
}
defer func() { _ = artifactReader.Close() }()
@@ -257,7 +237,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem)
if err != nil {
s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename)
- internalError(w, "failed to open archive")
+ http.Error(w, "failed to open archive", http.StatusInternalServerError)
return
}
defer func() { _ = archiveReader.Close() }()
@@ -266,7 +246,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
files, err := archiveReader.ListDir(dirPath)
if err != nil {
s.logger.Error("failed to list directory", "error", err, "path", dirPath)
- internalError(w, "failed to list directory")
+ http.Error(w, "failed to list directory", http.StatusInternalServerError)
return
}
@@ -301,33 +281,40 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
// @Param version path string true "Version"
// @Param filepath path string true "File path inside the archive"
// @Success 200 {file} file
-// @Failure 400 {object} ErrorResponse
-// @Failure 404 {object} ErrorResponse
-// @Failure 500 {object} ErrorResponse
-// @Router /ui/api/browse/{ecosystem}/{name}/{version}/file/{filepath} [get]
+// @Failure 400 {string} string
+// @Failure 404 {string} string
+// @Failure 500 {string} string
+// @Router /api/browse/{ecosystem}/{name}/{version}/file/{filepath} [get]
func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, name, version, filePath string) {
if filePath == "" {
- badRequest(w, "file path required")
+ http.Error(w, "file path required", http.StatusBadRequest)
return
}
// Get the artifact for this version
- versionPURL := s.cachedVersionPURL(ecosystem, name, version)
+ versionPURL := purl.MakePURLString(ecosystem, name, version)
artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL)
if err != nil {
- notFound(w, "version not found")
+ http.Error(w, "version not found", http.StatusNotFound)
return
}
if len(artifacts) == 0 {
- notFound(w, "no artifacts cached")
+ http.Error(w, "no artifacts cached", http.StatusNotFound)
return
}
- cachedArtifact := firstBrowsableArtifact(artifacts)
+ // Find the first cached artifact
+ var cachedArtifact *database.Artifact
+ for i := range artifacts {
+ if artifacts[i].StoragePath.Valid {
+ cachedArtifact = &artifacts[i]
+ break
+ }
+ }
if cachedArtifact == nil {
- notFound(w, "artifact not cached")
+ http.Error(w, "artifact not cached", http.StatusNotFound)
return
}
@@ -335,7 +322,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
artifactReader, err := s.storage.Open(r.Context(), cachedArtifact.StoragePath.String)
if err != nil {
s.logger.Error("failed to read artifact from storage", "error", err)
- internalError(w, "failed to read artifact")
+ http.Error(w, "failed to read artifact", http.StatusInternalServerError)
return
}
defer func() { _ = artifactReader.Close() }()
@@ -344,7 +331,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem)
if err != nil {
s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename)
- internalError(w, "failed to open archive")
+ http.Error(w, "failed to open archive", http.StatusInternalServerError)
return
}
defer func() { _ = archiveReader.Close() }()
@@ -353,128 +340,103 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
fileReader, err := archiveReader.Extract(filePath)
if err != nil {
if strings.Contains(err.Error(), "not found") {
- notFound(w, "file not found")
+ http.Error(w, "file not found", http.StatusNotFound)
return
}
s.logger.Error("failed to extract file", "error", err, "path", filePath)
- internalError(w, "failed to extract file")
+ http.Error(w, "failed to extract file", http.StatusInternalServerError)
return
}
defer func() { _ = fileReader.Close() }()
- contentType, knownPath := detectContentTypeFromPath(filePath)
- var content io.Reader = fileReader
- if !knownPath {
- bufferedFile := bufio.NewReaderSize(fileReader, browseSniffSize)
- prefix, _ := bufferedFile.Peek(browseSniffSize)
- contentType = detectContentTypeFromPrefix(prefix)
- content = bufferedFile
- }
+ // Set content type based on file extension
+ contentType := detectContentType(filePath)
w.Header().Set("Content-Type", contentType)
- w.Header().Set("Content-Security-Policy", "sandbox")
- w.Header().Set("X-Content-Type-Options", "nosniff")
+ // Set filename for download
_, filename := path.Split(filePath)
w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", filename))
// Stream the file
- _, _ = io.Copy(w, content)
+ _, _ = io.Copy(w, fileReader)
}
-func detectContentTypeFromPath(filename string) (string, bool) {
+// detectContentType returns an appropriate content type based on file extension.
+func detectContentType(filename string) string {
ext := strings.ToLower(path.Ext(filename))
switch ext {
// Text formats
case ".txt", ".md", ".markdown":
- return contentTypePlainText, true
- case ".html", ".htm", ".xhtml":
- return contentTypePlainText, true
+ return contentTypePlainText
+ case ".html", ".htm":
+ return "text/html; charset=utf-8"
case ".css":
- return "text/css; charset=utf-8", true
+ return "text/css; charset=utf-8"
case ".js", ".mjs":
- return "application/javascript; charset=utf-8", true
+ return "application/javascript; charset=utf-8"
case ".json":
- return "application/json; charset=utf-8", true
+ return "application/json; charset=utf-8"
case ".xml":
- return "application/xml; charset=utf-8", true
+ return "application/xml; charset=utf-8"
case ".yaml", ".yml":
- return "text/yaml; charset=utf-8", true
+ return "text/yaml; charset=utf-8"
case ".toml":
- return "text/toml; charset=utf-8", true
+ return "text/toml; charset=utf-8"
// Programming languages
case ".go":
- return "text/x-go; charset=utf-8", true
+ return "text/x-go; charset=utf-8"
case ".rs":
- return "text/x-rust; charset=utf-8", true
+ return "text/x-rust; charset=utf-8"
case ".py":
- return "text/x-python; charset=utf-8", true
+ return "text/x-python; charset=utf-8"
case ".rb":
- return "text/x-ruby; charset=utf-8", true
+ return "text/x-ruby; charset=utf-8"
case ".java":
- return "text/x-java; charset=utf-8", true
+ return "text/x-java; charset=utf-8"
case ".c", ".h":
- return "text/x-c; charset=utf-8", true
+ return "text/x-c; charset=utf-8"
case ".cpp", ".cc", ".cxx", ".hpp":
- return "text/x-c++; charset=utf-8", true
+ return "text/x-c++; charset=utf-8"
case ".ts":
- return "text/typescript; charset=utf-8", true
+ return "text/typescript; charset=utf-8"
case ".tsx":
- return "text/tsx; charset=utf-8", true
+ return "text/tsx; charset=utf-8"
case ".jsx":
- return "text/jsx; charset=utf-8", true
+ return "text/jsx; charset=utf-8"
case ".php":
- return "text/x-php; charset=utf-8", true
+ return "text/x-php; charset=utf-8"
// Config files
case ".conf", ".config", ".ini":
- return contentTypePlainText, true
+ return contentTypePlainText
case ".sh", ".bash":
- return "text/x-shellscript; charset=utf-8", true
+ return "text/x-shellscript; charset=utf-8"
case ".dockerfile":
- return "text/x-dockerfile; charset=utf-8", true
+ return "text/x-dockerfile; charset=utf-8"
// Images
case ".png":
- return "image/png", true
+ return "image/png"
case ".jpg", ".jpeg":
- return "image/jpeg", true
+ return "image/jpeg"
case ".gif":
- return "image/gif", true
+ return "image/gif"
case ".svg":
- return contentTypePlainText, true
+ return "image/svg+xml"
case ".ico":
- return "image/x-icon", true
+ return "image/x-icon"
// Archives
case ".zip", ".tar", ".gz", ".bz2", ".xz":
- return "application/octet-stream", true
+ return "application/octet-stream"
default:
+ // Try to detect if it looks like text
if isLikelyText(filename) {
- return contentTypePlainText, true
+ return contentTypePlainText
}
- return "", false
- }
-}
-
-func detectContentTypeFromPrefix(prefix []byte) string {
- result := magic.DetectPrefix(prefix)
- if result.Kind == magic.KindText {
- return contentTypePlainText
- }
-
- switch result.Format {
- case "png":
- return "image/png"
- case "jpeg":
- return "image/jpeg"
- case "gif":
- return "image/gif"
- case "pdf":
- return "application/pdf"
- default:
return "application/octet-stream"
}
}
@@ -503,16 +465,10 @@ func isLikelyText(filename string) bool {
}
// BrowseSourceData contains data for the browse source page.
-//
-// Version is the decoded version, for display. EscapedVersion is the same value
-// escaped as a single URL path segment and is what the links and the browse API
-// calls must use; see database.Version.EscapedVersion.
type BrowseSourceData struct {
- Layout
- Ecosystem string
- PackageName string
- Version string
- EscapedVersion string
+ Ecosystem string
+ PackageName string
+ Version string
}
// handleBrowseSource is now showBrowseSource in server.go, dispatched via handlePackagePath.
@@ -528,32 +484,43 @@ type BrowseSourceData struct {
// @Param fromVersion path string true "From version"
// @Param toVersion path string true "To version"
// @Success 200 {object} map[string]any
-// @Failure 404 {object} ErrorResponse
-// @Failure 500 {object} ErrorResponse
-// @Router /ui/api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion} [get]
+// @Failure 404 {string} string
+// @Failure 500 {string} string
+// @Router /api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion} [get]
func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, name, fromVersion, toVersion string) {
// Get artifacts for both versions
- fromPURL := s.cachedVersionPURL(ecosystem, name, fromVersion)
- toPURL := s.cachedVersionPURL(ecosystem, name, toVersion)
+ fromPURL := purl.MakePURLString(ecosystem, name, fromVersion)
+ toPURL := purl.MakePURLString(ecosystem, name, toVersion)
fromArtifacts, err := s.db.GetArtifactsByVersionPURL(fromPURL)
if err != nil || len(fromArtifacts) == 0 {
- notFound(w, "from version not found or not cached")
+ http.Error(w, "from version not found or not cached", http.StatusNotFound)
return
}
toArtifacts, err := s.db.GetArtifactsByVersionPURL(toPURL)
if err != nil || len(toArtifacts) == 0 {
- notFound(w, "to version not found or not cached")
+ http.Error(w, "to version not found or not cached", http.StatusNotFound)
return
}
// Find cached artifacts
- fromArtifact := firstBrowsableArtifact(fromArtifacts)
- toArtifact := firstBrowsableArtifact(toArtifacts)
+ var fromArtifact, toArtifact *database.Artifact
+ for i := range fromArtifacts {
+ if fromArtifacts[i].StoragePath.Valid {
+ fromArtifact = &fromArtifacts[i]
+ break
+ }
+ }
+ for i := range toArtifacts {
+ if toArtifacts[i].StoragePath.Valid {
+ toArtifact = &toArtifacts[i]
+ break
+ }
+ }
if fromArtifact == nil || toArtifact == nil {
- notFound(w, "one or both versions not cached")
+ http.Error(w, "one or both versions not cached", http.StatusNotFound)
return
}
@@ -561,7 +528,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
fromReader, err := s.storage.Open(r.Context(), fromArtifact.StoragePath.String)
if err != nil {
s.logger.Error("failed to open from artifact", "error", err)
- internalError(w, "failed to read from version")
+ http.Error(w, "failed to read from version", http.StatusInternalServerError)
return
}
defer func() { _ = fromReader.Close() }()
@@ -569,7 +536,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
toReader, err := s.storage.Open(r.Context(), toArtifact.StoragePath.String)
if err != nil {
s.logger.Error("failed to open to artifact", "error", err)
- internalError(w, "failed to read to version")
+ http.Error(w, "failed to read to version", http.StatusInternalServerError)
return
}
defer func() { _ = toReader.Close() }()
@@ -577,7 +544,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
fromArchive, err := openArchive(fromArtifact.Filename, fromReader, ecosystem)
if err != nil {
s.logger.Error("failed to open from archive", "error", err)
- internalError(w, "failed to open from archive")
+ http.Error(w, "failed to open from archive", http.StatusInternalServerError)
return
}
defer func() { _ = fromArchive.Close() }()
@@ -585,7 +552,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
toArchive, err := openArchive(toArtifact.Filename, toReader, ecosystem)
if err != nil {
s.logger.Error("failed to open to archive", "error", err)
- internalError(w, "failed to open to archive")
+ http.Error(w, "failed to open to archive", http.StatusInternalServerError)
return
}
defer func() { _ = toArchive.Close() }()
@@ -594,7 +561,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
result, err := diff.Compare(fromArchive, toArchive)
if err != nil {
s.logger.Error("failed to generate diff", "error", err)
- internalError(w, "failed to generate diff")
+ http.Error(w, "failed to generate diff", http.StatusInternalServerError)
return
}
@@ -603,17 +570,11 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
}
// ComparePageData contains data for the version comparison page.
-//
-// FromVersion and ToVersion are decoded, for display; the Escaped variants are
-// the path-segment form used to build the compare API URL.
type ComparePageData struct {
- Layout
- Ecosystem string
- PackageName string
- FromVersion string
- ToVersion string
- EscapedFromVersion string
- EscapedToVersion string
+ Ecosystem string
+ PackageName string
+ FromVersion string
+ ToVersion string
}
// handleComparePage is now showComparePage in server.go, dispatched via handlePackagePath.
diff --git a/internal/server/browse_bench_test.go b/internal/server/browse_bench_test.go
deleted file mode 100644
index 840bc75..0000000
--- a/internal/server/browse_bench_test.go
+++ /dev/null
@@ -1,89 +0,0 @@
-package server
-
-import (
- "archive/tar"
- "bytes"
- "compress/gzip"
- "fmt"
- "math/rand"
- "testing"
-)
-
-func createBenchTarGz(prefix string, fileCount, fileSize int) []byte {
- rnd := rand.New(rand.NewSource(1)) //nolint:gosec
- buf := new(bytes.Buffer)
- gw := gzip.NewWriter(buf)
- tw := tar.NewWriter(gw)
-
- payload := make([]byte, fileSize)
- for i := range fileCount {
- rnd.Read(payload)
- _ = tw.WriteHeader(&tar.Header{
- Name: fmt.Sprintf("%sfile%04d.dat", prefix, i),
- Size: int64(fileSize),
- Mode: 0644,
- })
- _, _ = tw.Write(payload)
- }
- _ = tw.Close()
- _ = gw.Close()
- return buf.Bytes()
-}
-
-func BenchmarkOpenArchive(b *testing.B) {
- cases := []struct {
- name string
- ecosystem string
- filename string
- data []byte
- }{
- {"npm", "npm", "pkg.tgz", createBenchTarGz("package/", 64, 16*1024)},
- {"go", "go", "v1.2.3.tar.gz", createBenchTarGz("repo-abc123/", 64, 16*1024)},
- }
-
- for _, tc := range cases {
- b.Run(tc.name, func(b *testing.B) {
- b.SetBytes(int64(len(tc.data)))
- b.ReportAllocs()
- for b.Loop() {
- r, err := openArchive(tc.filename, bytes.NewReader(tc.data), tc.ecosystem)
- if err != nil {
- b.Fatal(err)
- }
- _ = r.Close()
- }
- })
- }
-}
-
-func BenchmarkDetectContentType(b *testing.B) {
- cases := []struct {
- name string
- filename string
- prefix []byte
- knownPath bool
- }{
- {"known-path", "README.md", nil, true},
- {"text-prefix", "artifact", bytes.Repeat([]byte("a"), browseSniffSize), false},
- {"png-prefix", "artifact", append([]byte("\x89PNG\r\n\x1a\n"), make([]byte, browseSniffSize-8)...), false},
- }
-
- for _, tc := range cases {
- b.Run(tc.name, func(b *testing.B) {
- b.ReportAllocs()
- var contentType string
- if tc.knownPath {
- for b.Loop() {
- contentType, _ = detectContentTypeFromPath(tc.filename)
- }
- } else {
- for b.Loop() {
- contentType = detectContentTypeFromPrefix(tc.prefix)
- }
- }
- if contentType == "" {
- b.Fatal("empty content type")
- }
- })
- }
-}
diff --git a/internal/server/browse_test.go b/internal/server/browse_test.go
index f4f2f9a..1deaf5b 100644
--- a/internal/server/browse_test.go
+++ b/internal/server/browse_test.go
@@ -65,7 +65,7 @@ func TestHandleBrowseList(t *testing.T) {
}
// Test listing root directory
- req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0", nil)
+ req := httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0", nil)
w := httptest.NewRecorder()
ts.handler.ServeHTTP(w, req)
@@ -83,7 +83,7 @@ func TestHandleBrowseList(t *testing.T) {
}
// Test listing subdirectory
- req = httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0?path=lib", nil)
+ req = httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0?path=lib", nil)
w = httptest.NewRecorder()
ts.handler.ServeHTTP(w, req)
@@ -137,44 +137,29 @@ func TestHandleBrowseFile(t *testing.T) {
t.Fatalf("failed to upsert artifact: %v", err)
}
- files := []struct {
- path string
- content string
- contentType string
- }{
- {"README.md", "# Test Package\n", contentTypePlainText},
- {"notes.data", "short text\n", contentTypePlainText},
- {"logo", "\x89PNG\r\n\x1a\nimage data", "image/png"},
- {"page", "", contentTypePlainText},
- {"misleading.txt", "\x89PNG\r\n\x1a\nimage data", contentTypePlainText},
- }
- for _, file := range files {
- t.Run(file.path, func(t *testing.T) {
- req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/"+file.path, nil)
- w := httptest.NewRecorder()
- ts.handler.ServeHTTP(w, req)
+ // Test fetching a file
+ req := httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0/file/README.md", nil)
+ w := httptest.NewRecorder()
+ ts.handler.ServeHTTP(w, req)
- if w.Code != http.StatusOK {
- t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String())
- }
- if w.Body.String() != file.content {
- t.Errorf("unexpected file content: %q", w.Body.String())
- }
- if got := w.Header().Get("Content-Type"); got != file.contentType {
- t.Errorf("Content-Type = %q, want %q", got, file.contentType)
- }
- if got := w.Header().Get("Content-Security-Policy"); got != "sandbox" {
- t.Errorf("Content-Security-Policy = %q, want sandbox", got)
- }
- if got := w.Header().Get("X-Content-Type-Options"); got != "nosniff" {
- t.Errorf("X-Content-Type-Options = %q, want nosniff", got)
- }
- })
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String())
+ }
+
+ body := w.Body.String()
+ if body != "# Test Package\n" {
+ t.Errorf("unexpected file content: %q", body)
+ }
+
+ // Check content type
+ contentType := w.Header().Get("Content-Type")
+ if contentType != contentTypePlainText {
+ t.Errorf("expected text/plain content type, got %q", contentType)
}
// Test fetching non-existent file
- req := httptest.NewRequest("GET", "/ui/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil)
- w := httptest.NewRecorder()
+ req = httptest.NewRequest("GET", "/api/browse/npm/test-browse/1.0.0/file/nonexistent.txt", nil)
+ w = httptest.NewRecorder()
ts.handler.ServeHTTP(w, req)
if w.Code != http.StatusNotFound {
@@ -182,70 +167,37 @@ func TestHandleBrowseFile(t *testing.T) {
}
}
-func TestBrowseContentTypePolicy(t *testing.T) {
+func TestDetectContentType(t *testing.T) {
tests := []struct {
- name string
- filename string
- prefix []byte
- expectedCT string
+ filename string
+ expectedCT string
}{
- {"text extension", "file.txt", nil, contentTypePlainText},
- {"markdown extension", "file.md", nil, contentTypePlainText},
- {"JSON extension", "file.json", nil, "application/json; charset=utf-8"},
- {"JavaScript extension", "file.js", nil, "application/javascript; charset=utf-8"},
- {"Go extension", "file.go", nil, "text/x-go; charset=utf-8"},
- {"Python extension", "file.py", nil, "text/x-python; charset=utf-8"},
- {"Rust extension", "file.rs", nil, "text/x-rust; charset=utf-8"},
- {"HTML extension", "file.html", nil, contentTypePlainText},
- {"HTM extension", "file.htm", nil, contentTypePlainText},
- {"XHTML extension", "file.xhtml", nil, contentTypePlainText},
- {"SVG extension", "file.svg", nil, contentTypePlainText},
- {"PNG extension", "file.png", nil, "image/png"},
- {"JPEG extension", "file.jpg", nil, "image/jpeg"},
- {"README", "README", nil, contentTypePlainText},
- {"LICENSE", "LICENSE", nil, contentTypePlainText},
- {"Makefile", "Makefile", nil, contentTypePlainText},
- {"gitignore", ".gitignore", nil, contentTypePlainText},
- {"unknown empty", "file.bin", nil, "application/octet-stream"},
- {"extensionless PNG", "asset", []byte("\x89PNG\r\n\x1a\n"), "image/png"},
- {"extensionless JPEG", "asset", []byte("\xff\xd8\xff"), "image/jpeg"},
- {"extensionless GIF", "asset", []byte("GIF89a"), "image/gif"},
- {"extensionless PDF", "asset", []byte("%PDF-1.7"), "application/pdf"},
- {"extensionless text", "asset", []byte("plain text\n"), contentTypePlainText},
- {"extensionless HTML", "asset", []byte(""), contentTypePlainText},
- {"extensionless XML", "asset", []byte("Configure Gradle to use the proxy for HttpBuildCache:
-// In settings.gradle(.kts)
-buildCache {
- remote<HttpBuildCache> {
- url = uri("` + baseURL + `/gradle/")
- push = true
- }
-}Set the Pkg server before starting Julia:
-export JULIA_PKG_SERVER=` + baseURL + `/juliaOr inside a running session:
-ENV["JULIA_PKG_SERVER"] = "` + baseURL + `/julia"
-using Pkg; Pkg.update()Configure SwiftPM to use the proxy for this project:
-swift package-registry set ` + swiftInsecureFlag + baseURL + `/swiftUse scoped package identifiers in Package.swift, for example apple.swift-argument-parser.
`),
},
{
ID: "oci",
@@ -434,7 +379,7 @@ using Pkg; Pkg.update()`),
sudo systemctl restart docker
# Or pull directly
-docker pull ` + dockerHost + `/library/nginx:latest`),
+docker pull ` + baseURL[8:] + `/library/nginx:latest`),
},
{
ID: "deb",
@@ -466,18 +411,5 @@ gpgcheck=0
sudo dnf clean all
sudo dnf update`),
},
- {
- ID: "apk",
- Name: "Alpine APK",
- Language: "Alpine Linux",
- Endpoint: "/apk/",
- Instructions: template.HTML(`Configure apk to use the proxy:
-# In /etc/apk/repositories
-` + baseURL + `/apk/alpine/v3.22/main
-` + baseURL + `/apk/alpine/v3.22/community
-
-# Then run:
-apk update{{block "title" .}}git-pkgs proxy{{end}}
- {{if .UIBaseURL}}
-
+
{{block "content" .}}{{end}}
diff --git a/internal/server/templates/layout/footer.html b/internal/server/templates/layout/footer.html
index 33daddf..3245d1d 100644
--- a/internal/server/templates/layout/footer.html
+++ b/internal/server/templates/layout/footer.html
@@ -5,23 +5,13 @@
About
- git-pkgs proxy is a caching proxy for package registries supporting 17+ ecosystems.
+ git-pkgs proxy is a caching proxy for package registries supporting 16+ ecosystems.
-
-
- github.com/git-pkgs/proxy
-
-
- {{if .BuildInfo.Version}}
-
- proxy {{.BuildInfo.Version}}{{if .BuildInfo.Commit}} ({{.BuildInfo.Commit}}){{end}}
-
- {{end}}
+
{{end}}
diff --git a/internal/server/templates/layout/header.html b/internal/server/templates/layout/header.html
index b3103f1..d893565 100644
--- a/internal/server/templates/layout/header.html
+++ b/internal/server/templates/layout/header.html
@@ -1,21 +1,31 @@
{{define "header"}}
-
-
+
+
-
git-pkgs proxy
-
-
-
- {{template "nav_links" .}}
-
-
-
-
- {{template "search_form" .}}
-
- {{template "nav_links" .}}
Install
-
Health
-
API
-{{end}}
diff --git a/internal/server/templates/layout/styles.html b/internal/server/templates/layout/styles.html
index 76e9d9e..7fecaff 100644
--- a/internal/server/templates/layout/styles.html
+++ b/internal/server/templates/layout/styles.html
@@ -9,18 +9,5 @@
localStorage.theme = 'dark';
}
});
-
- (function() {
- const toggle = document.getElementById('nav-toggle');
- const menu = document.getElementById('mobile-nav');
- if (!toggle || !menu) return;
- toggle.addEventListener('click', function() {
- const open = toggle.getAttribute('aria-expanded') === 'true';
- toggle.setAttribute('aria-expanded', String(!open));
- menu.classList.toggle('hidden');
- toggle.querySelector('.nav-icon-open').classList.toggle('hidden');
- toggle.querySelector('.nav-icon-close').classList.toggle('hidden');
- });
- })();
{{end}}
diff --git a/internal/server/templates/pages/browse_source.html b/internal/server/templates/pages/browse_source.html
index a949111..710da1c 100644
--- a/internal/server/templates/pages/browse_source.html
+++ b/internal/server/templates/pages/browse_source.html
@@ -3,11 +3,11 @@
{{define "content"}}
- Home
+ Home
/
- {{.PackageName}}
+ {{.PackageName}}
/
- {{.Version}}
+ {{.Version}}
/
Browse Source
@@ -51,10 +51,7 @@
{{end}}
diff --git a/internal/server/templates/pages/search.html b/internal/server/templates/pages/search.html
index 42e82c5..a3e76b1 100644
--- a/internal/server/templates/pages/search.html
+++ b/internal/server/templates/pages/search.html
@@ -28,7 +28,7 @@
{{template "ecosystem_badge" .Ecosystem}}
-
{{.Name}}
+
{{.Name}}
{{if .LatestVersion}}
@{{.LatestVersion}}
{{end}}
@@ -50,7 +50,7 @@
{{else}}
No packages found matching "{{.Query}}"
-
Return to dashboard
+
Return to dashboard
{{end}}
diff --git a/internal/server/templates/pages/version_show.html b/internal/server/templates/pages/version_show.html
index 81bc067..dede127 100644
--- a/internal/server/templates/pages/version_show.html
+++ b/internal/server/templates/pages/version_show.html
@@ -1,15 +1,15 @@
-{{define "title"}}{{.Version.DisplayPURL}} - git-pkgs proxy{{end}}
+{{define "title"}}{{.Package.Name}}@{{.Version.PURL}} - git-pkgs proxy{{end}}
{{define "content"}}
- Home
+ Home
/
- {{.Package.Name}}
+ {{.Package.Name}}
{{template "ecosystem_badge" .Package.Ecosystem}}
-
{{.Version.DisplayPURL}}
+
{{.Version.PURL}}
{{if .IsOutdated}}
outdated
{{end}}
@@ -22,7 +22,7 @@
{{end}}
{{if .HasCachedArtifact}}
-
__". Slashes stay as "/" for filepath.Localize to convert.
-func escapeKey(key string) string {
- runes := []rune(key)
- var out strings.Builder
- for i, r := range runes {
- if escapeRune(runes, i) {
- fmt.Fprintf(&out, "__%#x__", r)
- } else {
- out.WriteRune(r)
- }
- }
- return out.String()
-}
-
-// escapeRune is fileblob's rule for which runes of a key to escape: control
-// characters, a raw path separator, a slash that would form "../", "//" or
-// end the key, and on Windows the characters its filesystem reserves.
-func escapeRune(r []rune, i int) bool {
- c := r[i]
- switch {
- case c < ' ':
- return true
- case os.PathSeparator != '/' && c == os.PathSeparator:
- return true
- case i > 1 && c == '/' && r[i-1] == '.' && r[i-2] == '.':
- return true
- case i > 0 && c == '/' && r[i-1] == '/':
- return true
- case c == '/' && i == len(r)-1:
- return true
- case os.PathSeparator == '\\' && strings.ContainsRune(`<>:"|?*`, c):
- return true
- }
- return false
-}
-
-// clearLegacySidecar removes the ".attrs" file an earlier version wrote for
-// key. Nothing rewrites one now, so a sidecar left partial by an interrupted
-// write would fail every read of that key for good. Removing is atomic where
-// the rewrite was not, so a concurrent reader gets the whole old file or
-// nothing.
-//
-// Failure is deliberately not fatal. Usually the key never had a sidecar and
-// os.Remove reports not-exist. A real failure leaves exactly the state this
-// change inherited, while failing the write would turn a cleanup miss into a
-// failed request. Windows makes that concrete: Go opens files without
-// FILE_SHARE_DELETE, so a reader holding the sidecar open blocks deletion, and
-// that reader is the very workload this change protects. The next store of the
-// key retries.
-func (b *Blob) clearLegacySidecar(key string) {
- if sidecar := b.legacySidecarPath(key); sidecar != "" {
- _ = os.Remove(sidecar)
- }
+ return &Blob{bucket: bucket, url: urlStr}, nil
}
func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
- b.clearLegacySidecar(path)
-
// Compute hash while writing
h := sha256.New()
tee := io.TeeReader(r, h)
@@ -246,20 +138,6 @@ func (b *Blob) Delete(ctx context.Context, path string) error {
return nil
}
-func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
- url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{
- Method: http.MethodGet,
- Expiry: expiry,
- })
- if err != nil {
- if gcerrors.Code(err) == gcerrors.Unimplemented {
- return "", ErrSignedURLUnsupported
- }
- return "", fmt.Errorf("signing URL: %w", err)
- }
- return url, nil
-}
-
func (b *Blob) Size(ctx context.Context, path string) (int64, error) {
attrs, err := b.bucket.Attributes(ctx, path)
if err != nil {
@@ -289,35 +167,6 @@ func (b *Blob) UsedSpace(ctx context.Context) (int64, error) {
return total, nil
}
-// ListPrefix returns object metadata for keys under a prefix.
-func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) {
- iter := b.bucket.List(&blob.ListOptions{Prefix: prefix})
- objects := make([]ObjectInfo, 0)
-
- for {
- obj, err := iter.Next(ctx)
- if err == io.EOF {
- break
- }
- if err != nil {
- return nil, fmt.Errorf("listing objects: %w", err)
- }
- if obj.IsDir {
- continue
- }
-
- info := ObjectInfo{
- Path: obj.Key,
- Size: obj.Size,
- ModTime: obj.ModTime,
- }
-
- objects = append(objects, info)
- }
-
- return objects, nil
-}
-
func (b *Blob) Close() error {
return b.bucket.Close()
}
diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go
index 3a97704..bb2d089 100644
--- a/internal/storage/blob_test.go
+++ b/internal/storage/blob_test.go
@@ -6,17 +6,10 @@ import (
"encoding/hex"
"errors"
"io"
- "io/fs"
- "os"
"path/filepath"
"runtime"
"strings"
- "sync"
- "sync/atomic"
"testing"
- "time"
-
- "gocloud.dev/blob"
)
func TestOpenBucket(t *testing.T) {
@@ -195,18 +188,6 @@ func TestBlobLargeFile(t *testing.T) {
assertLargeFileRoundTrip(t, createTestBlob(t))
}
-func TestBlobSignedURLUnsupported(t *testing.T) {
- b := createTestBlob(t)
- ctx := context.Background()
-
- // fileblob has no URL signer configured, so this must surface as
- // ErrSignedURLUnsupported rather than a generic error.
- _, err := b.SignedURL(ctx, "test/file.txt", time.Minute)
- if !errors.Is(err, ErrSignedURLUnsupported) {
- t.Errorf("SignedURL on fileblob = %v, want ErrSignedURLUnsupported", err)
- }
-}
-
func TestBlobOverwrite(t *testing.T) {
b := createTestBlob(t)
ctx := context.Background()
@@ -284,11 +265,7 @@ func createTestBlob(t *testing.T) *Blob {
t.Fatalf("OpenBucket failed: %v", err)
}
t.Cleanup(func() { _ = b.Close() })
- blob, ok := b.(*Blob)
- if !ok {
- t.Fatalf("OpenBucket returned %T, want *Blob", b)
- }
- return blob
+ return b
}
func fileURLFromPath(path string) string {
@@ -299,317 +276,3 @@ func fileURLFromPath(path string) string {
}
return "file://" + path
}
-
-func TestOpenBucketWritesNoAttrsSidecar(t *testing.T) {
- dir := t.TempDir()
- ctx := context.Background()
-
- b, err := OpenBucket(ctx, fileURLFromPath(dir))
- if err != nil {
- t.Fatalf("OpenBucket failed: %v", err)
- }
- defer func() { _ = b.Close() }()
-
- if _, _, err := b.Store(ctx, "pkg/thing-1.0.0.tgz", strings.NewReader("content")); err != nil {
- t.Fatalf("Store failed: %v", err)
- }
-
- sidecars, err := filepath.Glob(filepath.Join(dir, "*", "*.attrs"))
- if err != nil {
- t.Fatalf("Glob failed: %v", err)
- }
- if len(sidecars) != 0 {
- t.Errorf("got sidecar files %v, want none: a truncated sidecar fails reads that overlap a write", sidecars)
- }
-}
-
-// A read overlapping a write to the same key must not fail. fileblob rewrote
-// its ".attrs" sidecar in place, so a reader decoding it mid-write saw a
-// partial file, which the proxy served as a 502 on an artifact it held.
-func TestConcurrentReadsSurviveWritesToSameKey(t *testing.T) {
- if runtime.GOOS == "windows" {
- // Go opens files without FILE_SHARE_DELETE, so a writer cannot replace
- // a file a reader holds open: its rename fails with access denied
- // instead of contending. The other platforms exercise this race.
- t.Skip("Windows refuses to replace a file readers hold open")
- }
- const (
- key = "pkg/thing-1.0.0.tgz"
- readers = 4
- readsPerRead = 500
- )
- dir := t.TempDir()
- ctx := context.Background()
-
- b, err := OpenBucket(ctx, fileURLFromPath(dir))
- if err != nil {
- t.Fatalf("OpenBucket failed: %v", err)
- }
- defer func() { _ = b.Close() }()
-
- payload := strings.Repeat("x", 4096)
- if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil {
- t.Fatalf("seeding Store failed: %v", err)
- }
-
- // The writer reports how it ended: a Store failure would otherwise stop
- // the writes silently and let zero read failures pass for a test that
- // never contended anything.
- done := make(chan struct{})
- var writers sync.WaitGroup
- var writes int
- var writeErr error
- writers.Add(1)
- go func() {
- defer writers.Done()
- for {
- select {
- case <-done:
- return
- default:
- }
- if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil {
- writeErr = err
- return
- }
- writes++
- }
- }()
-
- var failures atomic.Int64
- var reading sync.WaitGroup
- for range readers {
- reading.Add(1)
- go func() {
- defer reading.Done()
- for range readsPerRead {
- r, err := b.Open(ctx, key)
- if err != nil {
- failures.Add(1)
- continue
- }
- if _, err := io.Copy(io.Discard, r); err != nil {
- failures.Add(1)
- }
- _ = r.Close()
- }
- }()
- }
- reading.Wait()
- close(done)
- writers.Wait()
-
- if writeErr != nil {
- t.Fatalf("writer stopped early: %v", writeErr)
- }
- if writes == 0 {
- t.Fatal("no write completed, so the reads were never contended")
- }
- if got := failures.Load(); got != 0 {
- t.Errorf("%d of %d reads failed while one writer rewrote the same key, want 0", got, readers*readsPerRead)
- }
-}
-
-// seedLegacySidecar stores key through a bucket that still writes sidecars, as
-// an earlier version did, and returns the path fileblob actually used. It is
-// discovered rather than assumed, so callers test the real mapping.
-func seedLegacySidecar(t *testing.T, dir, key, payload string) string {
- t.Helper()
- ctx := context.Background()
-
- legacy, err := blob.OpenBucket(ctx, fileURLFromPath(dir)+"?no_tmp_dir=true")
- if err != nil {
- t.Fatalf("opening legacy bucket: %v", err)
- }
- if err := legacy.WriteAll(ctx, key, []byte(payload), nil); err != nil {
- t.Fatalf("legacy WriteAll: %v", err)
- }
- if err := legacy.Close(); err != nil {
- t.Fatalf("closing legacy bucket: %v", err)
- }
-
- var found []string
- walkErr := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
- if err != nil {
- return err
- }
- if !d.IsDir() && strings.HasSuffix(path, ".attrs") {
- found = append(found, path)
- }
- return nil
- })
- if walkErr != nil {
- t.Fatalf("walking %s: %v", dir, walkErr)
- }
- if len(found) != 1 {
- t.Fatalf("got sidecars %v, want exactly one", found)
- }
- return found[0]
-}
-
-// An interrupted setAttrs leaves a partial sidecar that fails every read of the
-// key, and nothing rewrites one now, so a store has to clear it.
-//
-// One key per storage path the proxy builds: ArtifactPath across ecosystems,
-// metadata blobs, and the Gradle build cache. Scoped npm names, Go's "!" case
-// escaping and the ":" in OCI digests and Debian epochs are the characters
-// most likely to part fileblob's mapping from a plain path join.
-func TestStoreClearsLegacyAttrsSidecar(t *testing.T) {
- keys := []string{
- "npm/@babel/core/7.24.0/core-7.24.0.tgz",
- "maven/org.apache.commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar",
- "golang/github.com/!burnt!sushi/toml/v1.3.2/v1.3.2.zip",
- "oci/library/nginx/sha256:abc123def456/manifest",
- "debian/tzdata/1:2024a-1/tzdata_2024a-1_all.deb",
- "pypi/requests/2.31.0/requests-2.31.0-py3-none-any.whl",
- "cargo/serde/1.0.197/serde-1.0.197.crate",
- "julia/Example/a1b2c3/a1b2c3.tar.gz",
- "conda/numpy/1.26.4/numpy-1.26.4-py311.conda",
- "_metadata/npm/@babel/core/metadata",
- "_gradle/http-build-cache/0a1b2c3d4e5f",
- // Keys fileblob escapes on every platform.
- "npm/pkg//1.0.0/x.tgz",
- "npm/pkg/../1.0.0/x.tgz",
- }
-
- for _, key := range keys {
- t.Run(key, func(t *testing.T) {
- assertStoreClearsSidecar(t, key)
- })
- }
-}
-
-func assertStoreClearsSidecar(t *testing.T, key string) {
- t.Helper()
- const payload = "payload"
- ctx := context.Background()
- dir := t.TempDir()
-
- sidecar := seedLegacySidecar(t, dir, key, payload)
- if err := os.WriteFile(sidecar, []byte(`{"user.content_type":"appl`), 0o600); err != nil {
- t.Fatalf("corrupting sidecar: %v", err)
- }
-
- b := openFileBlob(t, dir)
- if _, err := b.Open(ctx, key); err == nil {
- t.Fatal("corrupt sidecar did not fail the read, so it is not the file fileblob reads for this key")
- }
-
- derived := b.legacySidecarPath(key)
- if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil {
- t.Fatalf("Store failed: %v", err)
- }
-
- if derived == "" {
- t.Fatalf("legacySidecarPath declined %q, but fileblob wrote %q", key, sidecar)
- }
- if derived != sidecar {
- t.Fatalf("derived %q, but fileblob wrote %q", derived, sidecar)
- }
- if _, err := os.Stat(sidecar); !os.IsNotExist(err) {
- t.Errorf("sidecar still present after Store, stat err = %v", err)
- }
- assertReadsBack(t, b, key, payload)
-}
-
-func assertReadsBack(t *testing.T, b *Blob, key, want string) {
- t.Helper()
- r, err := b.Open(context.Background(), key)
- if err != nil {
- t.Fatalf("read still failing after Store cleared the sidecar: %v", err)
- }
- defer func() { _ = r.Close() }()
- got, err := io.ReadAll(r)
- if err != nil {
- t.Fatalf("ReadAll failed: %v", err)
- }
- if string(got) != want {
- t.Errorf("got %q, want %q", got, want)
- }
-}
-
-func openFileBlob(t *testing.T, dir string) *Blob {
- t.Helper()
- s, err := OpenBucket(context.Background(), fileURLFromPath(dir))
- if err != nil {
- t.Fatalf("OpenBucket failed: %v", err)
- }
- t.Cleanup(func() { _ = s.Close() })
- b, ok := s.(*Blob)
- if !ok {
- t.Fatalf("got %T, want *Blob", s)
- }
- return b
-}
-
-// fileblob escapes a non-local key in a way this cannot reproduce, and one
-// holding ".." resolves outside the cache directory. Removal declines both
-// rather than delete the wrong file.
-func TestLegacySidecarPathEscapesLikeFileblob(t *testing.T) {
- root := filepath.FromSlash("/var/cache/proxy")
- b := &Blob{fileRoot: root}
- windows := runtime.GOOS == osWindows
-
- for _, tc := range []struct{ key, unix, windows string }{
- {"npm/pkg/1.0.0/x.tgz", "npm/pkg/1.0.0/x.tgz", "npm/pkg/1.0.0/x.tgz"},
- {"npm/pkg//1.0.0/x.tgz", "npm/pkg/__0x2f__1.0.0/x.tgz", "npm/pkg/__0x2f__1.0.0/x.tgz"},
- {"npm/pkg/../../etc/passwd", "npm/pkg/..__0x2f__..__0x2f__etc/passwd", "npm/pkg/..__0x2f__..__0x2f__etc/passwd"},
- {"npm/pkg/1.0.0/", "npm/pkg/1.0.0__0x2f__", "npm/pkg/1.0.0__0x2f__"},
- {"npm/a\x01b", "npm/a__0x1__b", "npm/a__0x1__b"},
- {"oci/nginx/sha256:abc/manifest", "oci/nginx/sha256:abc/manifest", "oci/nginx/sha256__0x3a__abc/manifest"},
- {"debian/tzdata/1:2024a-1/x.deb", "debian/tzdata/1:2024a-1/x.deb", "debian/tzdata/1__0x3a__2024a-1/x.deb"},
- {`npm/a\b`, `npm/a\b`, "npm/a__0x5c__b"},
- } {
- want := tc.unix
- if windows {
- want = tc.windows
- }
- want = filepath.Join(root, filepath.FromSlash(want)) + attrsExt
- if got := b.legacySidecarPath(tc.key); got != want {
- t.Errorf("legacySidecarPath(%q) = %q, want %q", tc.key, got, want)
- }
- }
-}
-
-func TestLegacySidecarPathDeclinesNonLocalKeys(t *testing.T) {
- b := &Blob{fileRoot: filepath.FromSlash("/var/cache/proxy")}
-
- for _, key := range []string{"", ".", "..", "/etc/passwd"} {
- if got := b.legacySidecarPath(key); got != "" {
- t.Errorf("legacySidecarPath(%q) = %q, want \"\"", key, got)
- }
- }
-}
-
-// Cloud backends have no local directory, so nothing is removed for them.
-func TestLegacySidecarPathEmptyForCloudBackends(t *testing.T) {
- b := &Blob{}
- if got := b.legacySidecarPath("npm/pkg/1.0.0/x.tgz"); got != "" {
- t.Errorf("legacySidecarPath = %q, want \"\" when there is no file root", got)
- }
-}
-
-// Cleanup that cannot complete must not fail the write. A non-empty directory
-// at the sidecar path makes os.Remove fail with something other than not-exist
-// on every platform, which is what a Windows sharing violation would look like
-// here.
-func TestStoreSucceedsWhenSidecarCannotBeRemoved(t *testing.T) {
- const key = "npm/pkg/1.0.0/pkg-1.0.0.tgz"
- const payload = "payload"
- dir := t.TempDir()
- ctx := context.Background()
-
- b := openFileBlob(t, dir)
-
- sidecar := filepath.Join(dir, filepath.FromSlash(key)) + ".attrs"
- if err := os.MkdirAll(filepath.Join(sidecar, "blocker"), 0o750); err != nil {
- t.Fatalf("seeding an unremovable sidecar: %v", err)
- }
- if err := os.Remove(sidecar); err == nil {
- t.Fatal("sidecar path was removable, so the test proves nothing")
- }
-
- if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil {
- t.Errorf("Store failed because cleanup could not complete: %v", err)
- }
-}
diff --git a/internal/storage/filesystem.go b/internal/storage/filesystem.go
new file mode 100644
index 0000000..cf6a1fe
--- /dev/null
+++ b/internal/storage/filesystem.go
@@ -0,0 +1,182 @@
+package storage
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+)
+
+// Filesystem implements Storage using the local filesystem.
+type Filesystem struct {
+ root string
+}
+
+// NewFilesystem creates a new filesystem storage rooted at the given directory.
+// The directory will be created if it does not exist.
+func NewFilesystem(root string) (*Filesystem, error) {
+ absRoot, err := filepath.Abs(root)
+ if err != nil {
+ return nil, fmt.Errorf("resolving root path: %w", err)
+ }
+
+ if err := os.MkdirAll(absRoot, dirPermissions); err != nil {
+ return nil, fmt.Errorf("creating root directory: %w", err)
+ }
+
+ return &Filesystem{root: absRoot}, nil
+}
+
+func (fs *Filesystem) fullPath(path string) string {
+ return filepath.Join(fs.root, filepath.FromSlash(path))
+}
+
+func (fs *Filesystem) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
+ fullPath := fs.fullPath(path)
+
+ dir := filepath.Dir(fullPath)
+ if err := os.MkdirAll(dir, dirPermissions); err != nil {
+ return 0, "", fmt.Errorf("creating directory: %w", err)
+ }
+
+ // Write to temp file first for atomic operation
+ tmpFile, err := os.CreateTemp(dir, ".tmp-*")
+ if err != nil {
+ return 0, "", fmt.Errorf("creating temp file: %w", err)
+ }
+ tmpPath := tmpFile.Name()
+
+ // Clean up temp file on error
+ success := false
+ defer func() {
+ if !success {
+ _ = tmpFile.Close()
+ _ = os.Remove(tmpPath)
+ }
+ }()
+
+ // Write content and compute hash
+ h := sha256.New()
+ w := io.MultiWriter(tmpFile, h)
+
+ size, err := io.Copy(w, r)
+ if err != nil {
+ return 0, "", fmt.Errorf("writing content: %w", err)
+ }
+
+ if err := tmpFile.Close(); err != nil {
+ return 0, "", fmt.Errorf("closing temp file: %w", err)
+ }
+
+ // Atomic rename
+ if err := os.Rename(tmpPath, fullPath); err != nil {
+ return 0, "", fmt.Errorf("renaming temp file: %w", err)
+ }
+
+ success = true
+ hash := hex.EncodeToString(h.Sum(nil))
+ return size, hash, nil
+}
+
+func (fs *Filesystem) Open(ctx context.Context, path string) (io.ReadCloser, error) {
+ fullPath := fs.fullPath(path)
+
+ f, err := os.Open(fullPath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, ErrNotFound
+ }
+ return nil, fmt.Errorf("opening file: %w", err)
+ }
+
+ return f, nil
+}
+
+func (fs *Filesystem) Exists(ctx context.Context, path string) (bool, error) {
+ fullPath := fs.fullPath(path)
+
+ _, err := os.Stat(fullPath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return false, nil
+ }
+ return false, fmt.Errorf("checking file: %w", err)
+ }
+
+ return true, nil
+}
+
+func (fs *Filesystem) Delete(ctx context.Context, path string) error {
+ fullPath := fs.fullPath(path)
+
+ err := os.Remove(fullPath)
+ if err != nil && !os.IsNotExist(err) {
+ return fmt.Errorf("removing file: %w", err)
+ }
+
+ // Try to clean up empty parent directories
+ dir := filepath.Dir(fullPath)
+ for dir != fs.root {
+ if err := os.Remove(dir); err != nil {
+ break // Directory not empty or other error
+ }
+ dir = filepath.Dir(dir)
+ }
+
+ return nil
+}
+
+func (fs *Filesystem) Size(ctx context.Context, path string) (int64, error) {
+ fullPath := fs.fullPath(path)
+
+ info, err := os.Stat(fullPath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return 0, ErrNotFound
+ }
+ return 0, fmt.Errorf("stat file: %w", err)
+ }
+
+ return info.Size(), nil
+}
+
+func (fs *Filesystem) UsedSpace(ctx context.Context) (int64, error) {
+ var total int64
+
+ err := filepath.Walk(fs.root, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if !info.IsDir() {
+ total += info.Size()
+ }
+ return nil
+ })
+ if err != nil {
+ return 0, fmt.Errorf("walking directory: %w", err)
+ }
+
+ return total, nil
+}
+
+// Root returns the root directory of the storage.
+func (fs *Filesystem) Root() string {
+ return fs.root
+}
+
+// FullPath returns the full filesystem path for a storage path.
+// Useful for serving files directly or debugging.
+func (fs *Filesystem) FullPath(path string) string {
+ return fs.fullPath(path)
+}
+
+func (fs *Filesystem) URL() string {
+ return "file://" + filepath.ToSlash(fs.root)
+}
+
+func (fs *Filesystem) Close() error {
+ return nil
+}
diff --git a/internal/storage/filesystem_test.go b/internal/storage/filesystem_test.go
new file mode 100644
index 0000000..7fbba10
--- /dev/null
+++ b/internal/storage/filesystem_test.go
@@ -0,0 +1,248 @@
+package storage
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestNewFilesystem(t *testing.T) {
+ dir := t.TempDir()
+ root := filepath.Join(dir, "cache")
+
+ fs, err := NewFilesystem(root)
+ if err != nil {
+ t.Fatalf("NewFilesystem failed: %v", err)
+ }
+
+ if _, err := os.Stat(root); err != nil {
+ t.Errorf("root directory not created: %v", err)
+ }
+
+ if fs.Root() != root {
+ t.Errorf("Root() = %q, want %q", fs.Root(), root)
+ }
+}
+
+func TestFilesystemStore(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+ content := "test content for storage"
+
+ size, hash, err := fs.Store(ctx, "npm/lodash/4.17.21/lodash.tgz", strings.NewReader(content))
+ if err != nil {
+ t.Fatalf("Store failed: %v", err)
+ }
+
+ if size != int64(len(content)) {
+ t.Errorf("size = %d, want %d", size, len(content))
+ }
+
+ h := sha256.Sum256([]byte(content))
+ wantHash := hex.EncodeToString(h[:])
+ if hash != wantHash {
+ t.Errorf("hash = %s, want %s", hash, wantHash)
+ }
+
+ // Verify file exists on disk
+ fullPath := fs.FullPath("npm/lodash/4.17.21/lodash.tgz")
+ data, err := os.ReadFile(fullPath)
+ if err != nil {
+ t.Fatalf("reading stored file: %v", err)
+ }
+ if string(data) != content {
+ t.Errorf("stored content = %q, want %q", string(data), content)
+ }
+}
+
+func TestFilesystemStoreAtomic(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ // Store initial content
+ _, _, err := fs.Store(ctx, "test/file.txt", strings.NewReader("initial"))
+ if err != nil {
+ t.Fatalf("initial Store failed: %v", err)
+ }
+
+ // Overwrite with new content
+ _, _, err = fs.Store(ctx, "test/file.txt", strings.NewReader("updated"))
+ if err != nil {
+ t.Fatalf("update Store failed: %v", err)
+ }
+
+ // Verify updated content
+ r, err := fs.Open(ctx, "test/file.txt")
+ if err != nil {
+ t.Fatalf("Open failed: %v", err)
+ }
+ defer func() { _ = r.Close() }()
+
+ data, _ := io.ReadAll(r)
+ if string(data) != "updated" {
+ t.Errorf("content = %q, want %q", string(data), "updated")
+ }
+}
+
+func TestFilesystemOpen(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+ content := "readable content"
+
+ _, _, _ = fs.Store(ctx, "test/read.txt", strings.NewReader(content))
+
+ r, err := fs.Open(ctx, "test/read.txt")
+ if err != nil {
+ t.Fatalf("Open failed: %v", err)
+ }
+ defer func() { _ = r.Close() }()
+
+ data, err := io.ReadAll(r)
+ if err != nil {
+ t.Fatalf("ReadAll failed: %v", err)
+ }
+ if string(data) != content {
+ t.Errorf("content = %q, want %q", string(data), content)
+ }
+}
+
+func TestFilesystemOpenNotFound(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ _, err := fs.Open(ctx, "does/not/exist.txt")
+ if !errors.Is(err, ErrNotFound) {
+ t.Errorf("Open non-existent = %v, want ErrNotFound", err)
+ }
+}
+
+func TestFilesystemExists(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ exists, err := fs.Exists(ctx, "test/exists.txt")
+ if err != nil {
+ t.Fatalf("Exists failed: %v", err)
+ }
+ if exists {
+ t.Error("Exists returned true for non-existent file")
+ }
+
+ _, _, _ = fs.Store(ctx, "test/exists.txt", strings.NewReader("content"))
+
+ exists, err = fs.Exists(ctx, "test/exists.txt")
+ if err != nil {
+ t.Fatalf("Exists after store failed: %v", err)
+ }
+ if !exists {
+ t.Error("Exists returned false for existing file")
+ }
+}
+
+func TestFilesystemDelete(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ _, _, _ = fs.Store(ctx, "test/delete/nested/file.txt", strings.NewReader("content"))
+
+ err := fs.Delete(ctx, "test/delete/nested/file.txt")
+ if err != nil {
+ t.Fatalf("Delete failed: %v", err)
+ }
+
+ exists, _ := fs.Exists(ctx, "test/delete/nested/file.txt")
+ if exists {
+ t.Error("file still exists after delete")
+ }
+
+ // Empty parent directories should be cleaned up
+ nestedDir := fs.FullPath("test/delete/nested")
+ if _, err := os.Stat(nestedDir); !os.IsNotExist(err) {
+ t.Error("empty nested directory not cleaned up")
+ }
+}
+
+func TestFilesystemDeleteNotFound(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ // Delete non-existent file should not error
+ err := fs.Delete(ctx, "does/not/exist.txt")
+ if err != nil {
+ t.Errorf("Delete non-existent = %v, want nil", err)
+ }
+}
+
+func TestFilesystemSize(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+ content := "size test content"
+
+ _, _, _ = fs.Store(ctx, "test/size.txt", strings.NewReader(content))
+
+ size, err := fs.Size(ctx, "test/size.txt")
+ if err != nil {
+ t.Fatalf("Size failed: %v", err)
+ }
+ if size != int64(len(content)) {
+ t.Errorf("Size = %d, want %d", size, len(content))
+ }
+}
+
+func TestFilesystemSizeNotFound(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ _, err := fs.Size(ctx, "does/not/exist.txt")
+ if !errors.Is(err, ErrNotFound) {
+ t.Errorf("Size non-existent = %v, want ErrNotFound", err)
+ }
+}
+
+func TestFilesystemUsedSpace(t *testing.T) {
+ fs := createTestFilesystem(t)
+ ctx := context.Background()
+
+ // Empty storage
+ used, err := fs.UsedSpace(ctx)
+ if err != nil {
+ t.Fatalf("UsedSpace failed: %v", err)
+ }
+ if used != 0 {
+ t.Errorf("UsedSpace empty = %d, want 0", used)
+ }
+
+ // Add some files
+ _, _, _ = fs.Store(ctx, "a.txt", strings.NewReader("aaaa")) // 4 bytes
+ _, _, _ = fs.Store(ctx, "b.txt", strings.NewReader("bbbbbb")) // 6 bytes
+ _, _, _ = fs.Store(ctx, "c/d.txt", strings.NewReader("ccccc")) // 5 bytes
+
+ used, err = fs.UsedSpace(ctx)
+ if err != nil {
+ t.Fatalf("UsedSpace failed: %v", err)
+ }
+ if used != 15 {
+ t.Errorf("UsedSpace = %d, want 15", used)
+ }
+}
+
+func TestFilesystemLargeFile(t *testing.T) {
+ assertLargeFileRoundTrip(t, createTestFilesystem(t))
+}
+
+func createTestFilesystem(t *testing.T) *Filesystem {
+ t.Helper()
+ dir := t.TempDir()
+
+ fs, err := NewFilesystem(dir)
+ if err != nil {
+ t.Fatalf("NewFilesystem failed: %v", err)
+ }
+ return fs
+}
diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go
deleted file mode 100644
index a096c18..0000000
--- a/internal/storage/gcs.go
+++ /dev/null
@@ -1,97 +0,0 @@
-package storage
-
-import (
- "context"
- "crypto/sha256"
- "encoding/hex"
- "errors"
- "io"
- "time"
-
- gcstorage "github.com/git-pkgs/gcs"
-)
-
-// GCS adapts a Google Cloud Storage bucket to Storage.
-type GCS struct {
- bucket *gcstorage.Bucket
- url string
-}
-
-// OpenGCS opens a Google Cloud Storage bucket from a gs:// URL.
-func OpenGCS(ctx context.Context, urlStr string) (*GCS, error) {
- bucket, err := gcstorage.OpenBucket(ctx, urlStr)
- if err != nil {
- return nil, err
- }
- return &GCS{bucket: bucket, url: urlStr}, nil
-}
-
-func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
- h := sha256.New()
- size, err := g.bucket.Write(ctx, path, io.TeeReader(r, h))
- if err != nil {
- return 0, "", err
- }
- return size, hex.EncodeToString(h.Sum(nil)), nil
-}
-
-func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) {
- r, err := g.bucket.Open(ctx, path)
- if errors.Is(err, gcstorage.ErrNotFound) {
- return nil, ErrNotFound
- }
- return r, err
-}
-
-func (g *GCS) Exists(ctx context.Context, path string) (bool, error) {
- return g.bucket.Exists(ctx, path)
-}
-
-func (g *GCS) Delete(ctx context.Context, path string) error {
- return g.bucket.Delete(ctx, path)
-}
-
-func (g *GCS) Size(ctx context.Context, path string) (int64, error) {
- size, err := g.bucket.Size(ctx, path)
- if errors.Is(err, gcstorage.ErrNotFound) {
- return 0, ErrNotFound
- }
- return size, err
-}
-
-func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
- u, err := g.bucket.SignedURL(ctx, path, expiry)
- if errors.Is(err, gcstorage.ErrSignedURLUnsupported) {
- return "", ErrSignedURLUnsupported
- }
- return u, err
-}
-
-func (g *GCS) UsedSpace(ctx context.Context) (int64, error) {
- return g.bucket.UsedSpace(ctx)
-}
-
-func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) {
- objects, err := g.bucket.ListPrefix(ctx, prefix)
- if err != nil {
- return nil, err
- }
-
- result := make([]ObjectInfo, 0, len(objects))
- for _, object := range objects {
- result = append(result, ObjectInfo{
- Path: object.Name,
- Size: object.Size,
- ModTime: object.ModTime,
- })
- }
- return result, nil
-}
-
-func (g *GCS) Close() error {
- return nil
-}
-
-func (g *GCS) URL() string {
- return g.url
-}
diff --git a/internal/storage/gcs_test.go b/internal/storage/gcs_test.go
deleted file mode 100644
index 1dfd0a2..0000000
--- a/internal/storage/gcs_test.go
+++ /dev/null
@@ -1,152 +0,0 @@
-package storage
-
-import (
- "context"
- "crypto/sha256"
- "encoding/hex"
- "encoding/json"
- "errors"
- "io"
- "net/http"
- "net/http/httptest"
- "net/url"
- "sort"
- "strconv"
- "strings"
- "testing"
- "time"
-)
-
-func TestOpenBucketGCSRoundTripWithEmulator(t *testing.T) {
- server := httptest.NewServer(&fakeGCSServer{t: t, objects: map[string]string{}})
- defer server.Close()
- t.Setenv("STORAGE_EMULATOR_HOST", server.URL)
-
- ctx := context.Background()
- store, err := OpenBucket(ctx, "gs://test-bucket")
- if err != nil {
- t.Fatalf("OpenBucket failed: %v", err)
- }
-
- size, hash, err := store.Store(ctx, "npm/pkg/file.tgz", strings.NewReader("content"))
- if err != nil {
- t.Fatalf("Store failed: %v", err)
- }
- wantHash := sha256.Sum256([]byte("content"))
- if size != int64(len("content")) || hash != hex.EncodeToString(wantHash[:]) {
- t.Fatalf("Store returned size=%d hash=%q", size, hash)
- }
-
- exists, err := store.Exists(ctx, "npm/pkg/file.tgz")
- if err != nil || !exists {
- t.Fatalf("Exists = %v, %v; want true, nil", exists, err)
- }
-
- r, err := store.Open(ctx, "npm/pkg/file.tgz")
- if err != nil {
- t.Fatalf("Open failed: %v", err)
- }
- data, _ := io.ReadAll(r)
- _ = r.Close()
- if string(data) != "content" {
- t.Fatalf("Open content = %q, want content", data)
- }
-
- lister, ok := store.(interface {
- ListPrefix(context.Context, string) ([]ObjectInfo, error)
- })
- if !ok {
- t.Fatal("GCS storage does not support prefix listing")
- }
- list, err := lister.ListPrefix(ctx, "npm/")
- if err != nil {
- t.Fatalf("ListPrefix failed: %v", err)
- }
- if len(list) != 1 || list[0].Path != "npm/pkg/file.tgz" {
- t.Fatalf("ListPrefix = %#v", list)
- }
-
- if err := store.Delete(ctx, "npm/pkg/file.tgz"); err != nil {
- t.Fatalf("Delete failed: %v", err)
- }
- exists, err = store.Exists(ctx, "npm/pkg/file.tgz")
- if err != nil || exists {
- t.Fatalf("Exists after delete = %v, %v; want false, nil", exists, err)
- }
-
- reader, err := store.Open(ctx, "npm/pkg/file.tgz")
- if reader != nil || !errors.Is(err, ErrNotFound) {
- t.Fatalf("Open missing object = %v, %v; want nil, ErrNotFound", reader, err)
- }
- if _, err := store.Size(ctx, "npm/pkg/file.tgz"); !errors.Is(err, ErrNotFound) {
- t.Fatalf("Size missing object = %v, want ErrNotFound", err)
- }
- if _, err := store.SignedURL(ctx, "npm/pkg/file.tgz", time.Minute); !errors.Is(err, ErrSignedURLUnsupported) {
- t.Fatalf("SignedURL with emulator = %v, want ErrSignedURLUnsupported", err)
- }
-}
-
-type fakeGCSServer struct {
- t *testing.T
- objects map[string]string
-}
-
-func (f *fakeGCSServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- switch {
- case r.Method == http.MethodPost && r.URL.Path == "/upload/storage/v1/b/test-bucket/o":
- name := r.URL.Query().Get("name")
- data, _ := io.ReadAll(r.Body)
- f.objects[name] = string(data)
- writeJSON(w, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)})
- case r.Method == http.MethodGet && r.URL.Path == "/storage/v1/b/test-bucket/o":
- prefix := r.URL.Query().Get("prefix")
- page := fakeGCSListResponse{}
- for name, data := range f.objects {
- if strings.HasPrefix(name, prefix) {
- page.Items = append(page.Items, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)})
- }
- }
- sort.Slice(page.Items, func(i, j int) bool { return page.Items[i].Name < page.Items[j].Name })
- writeJSON(w, page)
- case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"):
- name := objectNameFromPath(r.URL.Path)
- data, ok := f.objects[name]
- if !ok {
- http.NotFound(w, r)
- return
- }
- if r.URL.Query().Get("alt") == "media" {
- _, _ = io.WriteString(w, data)
- return
- }
- writeJSON(w, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)})
- case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"):
- delete(f.objects, objectNameFromPath(r.URL.Path))
- w.WriteHeader(http.StatusNoContent)
- default:
- f.t.Errorf("unexpected request: %s %s", r.Method, r.URL.String())
- http.Error(w, "unexpected request", http.StatusInternalServerError)
- }
-}
-
-type fakeGCSObject struct {
- Name string `json:"name"`
- Size string `json:"size"`
- Updated string `json:"updated"`
-}
-
-type fakeGCSListResponse struct {
- NextPageToken string `json:"nextPageToken"`
- Items []fakeGCSObject `json:"items"`
-}
-
-func writeJSON(w http.ResponseWriter, v any) {
- w.Header().Set("Content-Type", "application/json")
- _ = json.NewEncoder(w).Encode(v)
-}
-
-func objectNameFromPath(p string) string {
- escaped := strings.TrimPrefix(p, "/storage/v1/b/test-bucket/o/")
- name, _ := url.PathUnescape(escaped)
- return name
-}
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
index 5ff86f2..8a9026c 100644
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -5,37 +5,24 @@
// - file:///path/to/dir - Local filesystem storage
// - s3://bucket-name - Amazon S3
// - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
-// - gs://bucket-name - Google Cloud Storage (supports GKE Workload Identity
-// via Application Default Credentials)
-// - azblob://container-name - Azure Blob Storage
//
// Use OpenBucket to create a storage backend from a URL.
package storage
import (
"context"
+ "crypto/sha256"
+ "encoding/hex"
"errors"
"io"
- "time"
)
const dirPermissions = 0755
var (
ErrNotFound = errors.New("artifact not found")
-
- // ErrSignedURLUnsupported is returned by SignedURL when the backend
- // cannot generate presigned URLs (e.g. local filesystem).
- ErrSignedURLUnsupported = errors.New("signed URLs not supported by storage backend")
)
-// ObjectInfo contains metadata for a stored object.
-type ObjectInfo struct {
- Path string
- Size int64
- ModTime time.Time
-}
-
// Storage defines the interface for artifact storage backends.
type Storage interface {
// Store writes content from r to the given path.
@@ -58,10 +45,6 @@ type Storage interface {
// Returns ErrNotFound if the path does not exist.
Size(ctx context.Context, path string) (int64, error)
- // SignedURL returns a presigned URL granting time-limited GET access to path.
- // Returns ErrSignedURLUnsupported if the backend cannot generate presigned URLs.
- SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error)
-
// UsedSpace returns the total bytes used by all stored content.
UsedSpace(ctx context.Context) (int64, error)
@@ -81,3 +64,42 @@ func ArtifactPath(ecosystem, namespace, name, version, filename string) string {
}
return ecosystem + "/" + name + "/" + version + "/" + filename
}
+
+// HashingReader wraps a reader and computes SHA256 hash as content is read.
+type HashingReader struct {
+ r io.Reader
+ hash []byte
+ h interface{ Sum([]byte) []byte }
+ size int64
+ done bool
+}
+
+func NewHashingReader(r io.Reader) *HashingReader {
+ h := sha256.New()
+ return &HashingReader{
+ r: io.TeeReader(r, h),
+ h: h,
+ }
+}
+
+func (hr *HashingReader) Read(p []byte) (n int, err error) {
+ n, err = hr.r.Read(p)
+ hr.size += int64(n)
+ if err == io.EOF {
+ hr.done = true
+ hr.hash = hr.h.Sum(nil)
+ }
+ return
+}
+
+func (hr *HashingReader) Sum() string {
+ if !hr.done {
+ hr.hash = hr.h.Sum(nil)
+ hr.done = true
+ }
+ return hex.EncodeToString(hr.hash)
+}
+
+func (hr *HashingReader) Size() int64 {
+ return hr.size
+}
diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go
index 65f5a23..97800f0 100644
--- a/internal/storage/storage_test.go
+++ b/internal/storage/storage_test.go
@@ -6,6 +6,7 @@ import (
"crypto/sha256"
"encoding/hex"
"io"
+ "strings"
"testing"
)
@@ -34,6 +35,30 @@ func TestArtifactPath(t *testing.T) {
}
}
+func TestHashingReader(t *testing.T) {
+ content := "hello world"
+ r := NewHashingReader(strings.NewReader(content))
+
+ data, err := io.ReadAll(r)
+ if err != nil {
+ t.Fatalf("ReadAll failed: %v", err)
+ }
+
+ if string(data) != content {
+ t.Errorf("got content %q, want %q", string(data), content)
+ }
+
+ if r.Size() != int64(len(content)) {
+ t.Errorf("got size %d, want %d", r.Size(), len(content))
+ }
+
+ h := sha256.Sum256([]byte(content))
+ wantHash := hex.EncodeToString(h[:])
+ if r.Sum() != wantHash {
+ t.Errorf("got hash %s, want %s", r.Sum(), wantHash)
+ }
+}
+
// assertLargeFileRoundTrip stores a 1MB file in the given storage, verifies size and
// hash, then reads it back and confirms the content matches.
func assertLargeFileRoundTrip(t *testing.T, s Storage) {