From b2553ddd49187fa9f3f6c243ced07b86121bd105 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 16:15:31 +0100 Subject: [PATCH 001/198] Pass Composer security advisories and filter lists through The proxy's packages.json listed only metadata, search and notify URLs, so a client using the proxy as its only Composer repository lost `composer audit`, `audit.block-insecure` and Packagist's malware filter list without any warning. The service index now carries the security-advisories and filter entries of the upstream repository's own packages.json, cached for the metadata TTL, with URLs pointing at the proxy. An upstream that does not offer them gets nothing advertised, so a custom upstream.composer_repository without the API keeps working as before. Advisories and filter entries embedded in p2 metadata were already passed on unchanged. Advisories queries and the filter summary are relayed uncached. Names of routed packages and hosted-only vendors are removed from advisories queries first, so private package names are not sent to the public API. --- README.md | 11 + docs/configuration.md | 3 + internal/handler/composer.go | 14 +- internal/handler/composer_advisories.go | 268 ++++++++++++++++ internal/handler/composer_advisories_test.go | 308 +++++++++++++++++++ 5 files changed, 602 insertions(+), 2 deletions(-) create mode 100644 internal/handler/composer_advisories.go create mode 100644 internal/handler/composer_advisories_test.go diff --git a/README.md b/README.md index 20c6d5d..eb0db0e 100644 --- a/README.md +++ b/README.md @@ -387,6 +387,15 @@ Or set globally: composer config -g repositories.proxy composer http://localhost:8080/composer ``` +`composer audit`, `audit.block-insecure` and Packagist's malware filter list +work through the proxy when `upstream.composer_repository` offers them in its +`packages.json`, as repo.packagist.org does. The proxy advertises them in its +own `packages.json` and passes the advisories and filter list requests through +to the upstream uncached. Names of packages in `upstream.composer_routes` or +`hosted.composer_vendors` are removed from advisories requests before they are +sent. Versions withheld by [cooldown](#version-cooldown) are not excluded from +the report. + To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). ### Conan (C/C++) @@ -1263,6 +1272,8 @@ Recently cached: | `GET /maven/*` | Maven repository protocol | | `GET /nuget/*` | NuGet V3 API | | `GET /composer/*` | Composer/Packagist protocol | +| `GET`, `POST /composer/api/security-advisories/` | Composer security advisories, passed through to upstream without private package names | +| `GET /composer/lists/summary.json` | Composer filter list summary, passed through to upstream uncached | | `GET /conan/*` | Conan C/C++ protocol | | `GET /chef/*` | Chef Supermarket protocol (universe, cookbook API and downloads) | | `GET /conda/*` | Conda/Anaconda protocol | diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b2..5497b91 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -339,6 +339,8 @@ upstream: composer_repository: "https://upstream-proxy.example.com/composer" ``` +The proxy reads `packages.json` from `upstream.composer_repository`, cached for the metadata TTL, and passes on its `security-advisories` and `filter` entries (Packagist's malware list) with URLs pointing at the proxy. An upstream that offers neither is still usable; `composer audit` then reports nothing from it. A filter `api-url` is not passed on; Composer reads filter entries from package metadata instead. + `upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint. `upstream.chef` is the Supermarket served at `/chef/`, for Berkshelf (`source "https://proxy.example.com/chef"`) and Policyfiles (`default_source :supermarket, "https://proxy.example.com/chef"`). The upstream has to serve the Supermarket API: `/universe`, `/api/v1/cookbooks/{name}`, its versions and their downloads. A private Supermarket that needs credentials takes them from `upstream.auth`, scoped to its URL. A download redirect to another host, such as the storage bucket behind Supermarket, does not receive them: @@ -379,6 +381,7 @@ upstream: - Composer metadata of routed packages carries an empty `notification-url`, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. - Use `upstream.auth` for the registry's credentials. The proxy has no authentication of its own, so every client that can reach it can download routed packages. - Search (`/composer/search.json`) and the package list (`/composer/packages/list.json`) still come from the default upstream only. +- Composer security advisories and filter lists come from the default upstream. Routed package names, and those in `hosted.composer_vendors`, are removed from advisories requests before they are sent, so they only get advisories that their route's own metadata carries. - npm audit, signing-key, search and ping requests still go to `upstream.npm`. An audit request lists every package in the dependency tree, routed ones included. - Routes can only be set in the configuration file, not with environment variables. diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 116d106..bf04da0 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -100,12 +100,19 @@ func (h *ComposerHandler) Routes() http.Handler { mux.HandleFunc("GET /search.json", h.proxyUpstream) mux.HandleFunc("GET /packages/list.json", h.proxyUpstream) + // Security advisories and filter lists, as the default repository offers them + mux.HandleFunc("GET "+composerAdvisoriesPath, h.handleSecurityAdvisories) + mux.HandleFunc("POST "+composerAdvisoriesPath, h.handleSecurityAdvisories) + mux.HandleFunc("GET "+composerFilterSummaryPath, h.handleFilterSummary) + return mux } -// handleServiceIndex returns the Composer repository service index. +// handleServiceIndex returns the Composer repository service index. It +// carries the security advisories and filter list entries of the default +// repository's own index when that has them; if the repository cannot be +// reached and nothing is cached, the index goes out without them. func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request) { - // Return a minimal service index pointing to our proxy index := map[string]any{ "packages": map[string]any{}, "metadata-url": h.proxyURL + "/composer/p2/%package%.json", @@ -113,6 +120,9 @@ func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Requ "search": h.proxyURL + "/composer/search.json?q=%query%&type=%type%", "providers-lazy-url": h.proxyURL + "/composer/p2/%package%.json", } + if upstream, ok := h.upstreamIndex(r.Context()); ok { + upstream.advertise(index, h.proxyURL) + } w.Header().Set(headerContentType, "application/json") _ = json.NewEncoder(w).Encode(index) diff --git a/internal/handler/composer_advisories.go b/internal/handler/composer_advisories.go new file mode 100644 index 0000000..e77da37 --- /dev/null +++ b/internal/handler/composer_advisories.go @@ -0,0 +1,268 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "strings" +) + +const ( + composerAdvisoriesPath = "/api/security-advisories/" + composerFilterSummaryPath = "/lists/summary.json" + composerPackagesParam = "packages" + + // composerAdvisoriesMaxBody caps an advisories query. Composer sends at + // most 500 package names per request. + composerAdvisoriesMaxBody = 1 << 20 +) + +// composerUpstreamIndex holds the parts of the default repository's +// packages.json that the proxy advertises in its own: the security +// advisories API and the filter lists (Packagist's malware list). Composer +// reads advisories and filter entries for each package from its p2 metadata, +// which the proxy passes on unchanged, and asks these URLs for the rest. +type composerUpstreamIndex struct { + advisoriesURL string + advisoriesMetadata bool + filterLists json.RawMessage + filterSummaryURL string +} + +// upstreamIndex fetches the default repository's packages.json, cached for +// the metadata TTL. ok is false when the repository could not be reached and +// nothing is cached; a document without the fields yields an empty index. +func (h *ComposerHandler) upstreamIndex(ctx context.Context) (composerUpstreamIndex, bool) { + indexURL := h.repoURL + "/packages.json" + cacheKey := hashedCacheKey(h.repoURL, "/packages.json") + res := h.proxy.fetchMetadataDocument(ctx, "composer", cacheKey, indexURL, contentTypeJSON, nil) + if res.err != nil { + h.proxy.Logger.Warn("failed to fetch upstream composer packages.json", "url", indexURL, "error", res.err) + return composerUpstreamIndex{}, false + } + return parseComposerUpstreamIndex(indexURL, res.body), true +} + +func parseComposerUpstreamIndex(indexURL string, body []byte) composerUpstreamIndex { + var doc struct { + SecurityAdvisories struct { + Metadata bool `json:"metadata"` + APIURL string `json:"api-url"` + } `json:"security-advisories"` + Filter struct { + Metadata bool `json:"metadata"` + Lists json.RawMessage `json:"lists"` + SummaryURL string `json:"summary-url"` + } `json:"filter"` + } + var index composerUpstreamIndex + if err := json.Unmarshal(body, &doc); err != nil { + return index + } + // Composer refuses advisories configured without an api-url unless the + // repository lists its packages, which the proxy does not. + if apiURL, ok := resolveComposerURL(indexURL, doc.SecurityAdvisories.APIURL); ok { + index.advisoriesURL = apiURL + index.advisoriesMetadata = doc.SecurityAdvisories.Metadata + } + // Composer only applies filter lists a repository marks with metadata. + // A filter api-url is not passed on: without it Composer reads the + // entries from package metadata, narrowed by the summary. + if doc.Filter.Metadata && len(doc.Filter.Lists) > 0 && doc.Filter.Lists[0] == '{' { + index.filterLists = doc.Filter.Lists + index.filterSummaryURL, _ = resolveComposerURL(indexURL, doc.Filter.SummaryURL) + } + return index +} + +// resolveComposerURL resolves a URL from packages.json against the document's +// own, as Composer does, and accepts only http and https. +func resolveComposerURL(base, ref string) (string, bool) { + if ref == "" { + return "", false + } + b, err := url.Parse(base) + if err != nil { + return "", false + } + u, err := b.Parse(ref) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return "", false + } + return u.String(), true +} + +// advertise adds the advisories and filter entries to the proxy's +// packages.json, pointing at the proxy. +func (index composerUpstreamIndex) advertise(packages map[string]any, proxyURL string) { + if index.advisoriesURL != "" { + packages["security-advisories"] = map[string]any{ + "metadata": index.advisoriesMetadata, + "api-url": proxyURL + "/composer" + composerAdvisoriesPath, + } + } + if index.filterLists != nil { + filter := map[string]any{"metadata": true, "lists": index.filterLists} + if index.filterSummaryURL != "" { + filter["summary-url"] = proxyURL + "/composer" + composerFilterSummaryPath + } + packages["filter"] = filter + } +} + +// handleSecurityAdvisories relays a security advisories query to the URL the +// default repository advertises. Composer sends the package names as +// packages[] in a POST form; a GET query is accepted too. Names served by a +// package route or a hosted-only vendor are removed first, so private package +// names are never sent to the public API. +// +// Responses are not cached: each query names a different set of packages. +func (h *ComposerHandler) handleSecurityAdvisories(w http.ResponseWriter, r *http.Request) { + query, err := h.advisoriesQuery(w, r) + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + JSONError(w, http.StatusRequestEntityTooLarge, "advisories request too large") + return + } + JSONError(w, http.StatusBadRequest, "could not read advisories request") + return + } + + asked := len(query[composerPackagesParam+"[]"]) + public := h.publicPackageNames(query[composerPackagesParam+"[]"]) + if asked > 0 && len(public) == 0 { + w.Header().Set(headerContentType, contentTypeJSON) + _, _ = io.WriteString(w, `{"advisories":{}}`) + return + } + if asked > 0 { + query[composerPackagesParam+"[]"] = public + } + + index, ok := h.upstreamIndex(r.Context()) + if !ok { + JSONError(w, http.StatusBadGateway, "upstream repository unavailable") + return + } + if index.advisoriesURL == "" { + JSONError(w, http.StatusNotFound, "upstream repository has no security advisories API") + return + } + + req, err := h.advisoriesRequest(r, index.advisoriesURL, query) + if err != nil { + JSONError(w, http.StatusInternalServerError, "failed to create request") + return + } + h.proxy.Logger.Debug("composer advisories request", "packages", len(public), "withheld", asked-len(public)) + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("composer advisories request failed", "error", err) + JSONError(w, http.StatusBadGateway, "failed to reach upstream repository") + return + } + defer func() { _ = resp.Body.Close() }() + + h.proxy.relayResponse(w, r, resp, nil) +} + +// advisoriesQuery reads the query of an advisories request, from the form +// body of a POST or the URL of a GET. Package names come back under +// "packages[]" whichever spelling the client used: PHP's http_build_query, +// which Composer uses, numbers them as packages[0], packages[1], ... +func (h *ComposerHandler) advisoriesQuery(w http.ResponseWriter, r *http.Request) (url.Values, error) { + raw := r.URL.RawQuery + if r.Method == http.MethodPost { + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, composerAdvisoriesMaxBody)) + if err != nil { + return nil, err + } + raw = string(body) + } + values, err := url.ParseQuery(raw) + if err != nil { + return nil, err + } + query := url.Values{} + for key, vals := range values { + if key == composerPackagesParam || strings.HasPrefix(key, composerPackagesParam+"[") { + key = composerPackagesParam + "[]" + } + query[key] = append(query[key], vals...) + } + return query, nil +} + +// publicPackageNames drops the names the default repository does not serve. +func (h *ComposerHandler) publicPackageNames(names []string) []string { + public := make([]string, 0, len(names)) + for _, name := range names { + if _, routed := h.routes.match(name); routed || h.proxy.hostedOnly("composer", name) { + continue + } + public = append(public, name) + } + return public +} + +// advisoriesRequest builds the upstream request, sending the query the way +// the client did. +func (h *ComposerHandler) advisoriesRequest(r *http.Request, apiURL string, query url.Values) (*http.Request, error) { + u, err := url.Parse(apiURL) + if err != nil { + return nil, err + } + var body io.Reader + if r.Method == http.MethodPost { + body = strings.NewReader(query.Encode()) + } else { + u.RawQuery = joinQuery(u.RawQuery, query.Encode()) + } + req, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), body) + if err != nil { + return nil, err + } + if r.Method == http.MethodPost { + req.Header.Set(headerContentType, "application/x-www-form-urlencoded") + } + for _, header := range []string{headerAccept, headerAcceptEncoding} { + if v := r.Header.Get(header); v != "" { + req.Header.Set(header, v) + } + } + return req, nil +} + +func joinQuery(a, b string) string { + if a == "" || b == "" { + return a + b + } + return a + "&" + b +} + +// handleFilterSummary relays the filter list summary the default repository +// advertises. It lists every package on a filter list, so the request names +// no packages. It is not cached, so a newly listed package is blocked as soon +// as the upstream lists it; Composer revalidates its own copy with +// If-Modified-Since, which is passed on. +func (h *ComposerHandler) handleFilterSummary(w http.ResponseWriter, r *http.Request) { + index, ok := h.upstreamIndex(r.Context()) + if !ok { + JSONError(w, http.StatusBadGateway, "upstream repository unavailable") + return + } + if index.filterSummaryURL == "" { + JSONError(w, http.StatusNotFound, "upstream repository has no filter list summary") + return + } + h.proxy.ProxyUpstream(w, r, index.filterSummaryURL, + []string{headerAccept, headerAcceptEncoding, "If-Modified-Since", "If-None-Match"}) +} diff --git a/internal/handler/composer_advisories_test.go b/internal/handler/composer_advisories_test.go new file mode 100644 index 0000000..6adc7d2 --- /dev/null +++ b/internal/handler/composer_advisories_test.go @@ -0,0 +1,308 @@ +package handler + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync" + "testing" +) + +// composerAdvisoryUpstream serves a packages.json advertising security +// advisories and a malware filter list, and records the advisories and +// summary requests it gets. +type composerAdvisoryUpstream struct { + *httptest.Server + index string + + mu sync.Mutex + requests []*recordedRequest +} + +type recordedRequest struct { + method string + path string + query url.Values + body string + header http.Header +} + +const composerAdvisoriesResponse = `{"advisories":{"public/library":[{"advisoryId":"PKSA-1","affectedVersions":"<1.0.1"}]}}` + +const composerFilterSummary = `{"filter":{"malware":{"evil/package":"1.0.0"}}}` + +func newComposerAdvisoryUpstream(t *testing.T) *composerAdvisoryUpstream { + t.Helper() + u := &composerAdvisoryUpstream{} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + u.mu.Lock() + u.requests = append(u.requests, &recordedRequest{r.Method, r.URL.Path, r.URL.Query(), string(body), r.Header.Clone()}) + u.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/packages.json": + _, _ = io.WriteString(w, u.index) + case "/api/security-advisories/": + _, _ = io.WriteString(w, composerAdvisoriesResponse) + case "/lists/all/summary.json": + if r.Header.Get("If-Modified-Since") != "" { + w.WriteHeader(http.StatusNotModified) + return + } + _, _ = io.WriteString(w, composerFilterSummary) + default: + http.NotFound(w, r) + } + })) + // Like Packagist's: the API on one host, the summary as a relative URL. + u.index = `{ + "packages": [], + "metadata-url": "/p2/%package%.json", + "security-advisories": {"metadata": true, "api-url": "` + u.URL + `/api/security-advisories/"}, + "filter": {"metadata": true, "lists": {"malware": {"enabled": true}}, "summary-url": "/lists/all/summary.json"} + }` + t.Cleanup(u.Close) + return u +} + +// requestsTo returns the recorded requests for path. +func (u *composerAdvisoryUpstream) requestsTo(path string) []*recordedRequest { + u.mu.Lock() + defer u.mu.Unlock() + var out []*recordedRequest + for _, r := range u.requests { + if r.path == path { + out = append(out, r) + } + } + return out +} + +func newAdvisoryComposerHandler(t *testing.T, upstream *composerAdvisoryUpstream) *ComposerHandler { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HostedEnabled = true + proxy.HostedComposerVendors = []string{"acme"} + return NewComposerHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL). + WithPackageRoutes(map[string]string{"example/*": "https://packages.example.com/composer"}) +} + +func serveComposerIndex(t *testing.T, h *ComposerHandler) map[string]any { + t.Helper() + w := serveComposer(h, "/packages.json") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body) + } + var index map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &index); err != nil { + t.Fatalf("parsing packages.json: %v", err) + } + return index +} + +func TestComposerServiceIndexAdvertisesAdvisoriesAndFilter(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + index := serveComposerIndex(t, newAdvisoryComposerHandler(t, upstream)) + + advisories, _ := index["security-advisories"].(map[string]any) + if advisories["api-url"] != "http://proxy.test/composer/api/security-advisories/" || advisories["metadata"] != true { + t.Errorf("security-advisories = %v, want the proxy's api-url with metadata", index["security-advisories"]) + } + filter, _ := index["filter"].(map[string]any) + if filter["summary-url"] != "http://proxy.test/composer/lists/summary.json" || filter["metadata"] != true { + t.Errorf("filter = %v, want the proxy's summary-url with metadata", index["filter"]) + } + lists, _ := filter["lists"].(map[string]any) + if _, ok := lists["malware"]; !ok { + t.Errorf("filter lists = %v, want upstream's malware list", filter["lists"]) + } + if index["metadata-url"] != "http://proxy.test/composer/p2/%package%.json" { + t.Errorf("metadata-url = %v, want the proxy's", index["metadata-url"]) + } +} + +func TestComposerServiceIndexWithoutUpstreamSupport(t *testing.T) { + tests := []struct { + name string + index string + }{ + {"not advertised", `{"packages": [], "metadata-url": "/p2/%package%.json"}`}, + {"advisories without api-url", `{"security-advisories": {"metadata": true}}`}, + {"filter without metadata", `{"filter": {"lists": {"malware": {"enabled": true}}, "api-url": "/filter"}}`}, + {"not json", ``}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + upstream.index = tt.index + index := serveComposerIndex(t, newAdvisoryComposerHandler(t, upstream)) + for _, key := range []string{"security-advisories", "filter"} { + if _, ok := index[key]; ok { + t.Errorf("%s advertised although upstream does not offer it: %v", key, index[key]) + } + } + }) + } +} + +func TestComposerServiceIndexUpstreamDown(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + upstream.Close() + + index := serveComposerIndex(t, h) + if _, ok := index["security-advisories"]; ok { + t.Errorf("security-advisories advertised without an upstream index: %v", index["security-advisories"]) + } + if index["metadata-url"] == nil { + t.Error("packages.json lacks metadata-url while the upstream is down") + } +} + +func serveAdvisories(h *ComposerHandler, method, target, body string) *httptest.ResponseRecorder { + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req := httptest.NewRequest(method, target, reader) + if method == http.MethodPost { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + return w +} + +func TestComposerAdvisoriesWithholdPrivateNames(t *testing.T) { + // The form Composer sends: PHP's http_build_query numbers the entries. + form := url.Values{ + "packages[0]": {"public/library"}, + "packages[1]": {"example/private"}, + "packages[2]": {"Acme/hosted"}, + "packages[3]": {"other/library"}, + } + tests := []struct { + name, method, target, body string + }{ + {"POST form", http.MethodPost, "/api/security-advisories/", form.Encode()}, + {"GET query", http.MethodGet, "/api/security-advisories/?" + form.Encode(), ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveAdvisories(h, tt.method, tt.target, tt.body) + + if w.Code != http.StatusOK || w.Body.String() != composerAdvisoriesResponse { + t.Fatalf("got %d %q, want upstream's advisories", w.Code, w.Body) + } + sent := upstream.requestsTo("/api/security-advisories/") + if len(sent) != 1 { + t.Fatalf("upstream got %d advisories requests, want 1", len(sent)) + } + if sent[0].method != tt.method { + t.Errorf("upstream method = %s, want %s", sent[0].method, tt.method) + } + query := sent[0].query + if tt.method == http.MethodPost { + query, _ = url.ParseQuery(sent[0].body) + } + got := query["packages[]"] + slices.Sort(got) + if want := []string{"other/library", "public/library"}; !slices.Equal(got, want) { + t.Errorf("upstream was asked about %v, want only the public packages %v", got, want) + } + }) + } +} + +func TestComposerAdvisoriesOnlyPrivateNames(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B0%5D=example%2Fprivate") + + if w.Code != http.StatusOK || w.Body.String() != `{"advisories":{}}` { + t.Errorf("got %d %q, want an empty advisories answer", w.Code, w.Body) + } + if sent := upstream.requestsTo("/api/security-advisories/"); len(sent) != 0 { + t.Errorf("upstream was asked although every package is private: %+v", sent[0]) + } +} + +func TestComposerAdvisoriesUpdatedSince(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveAdvisories(h, http.MethodGet, "/api/security-advisories/?updatedSince=1700000000", "") + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + sent := upstream.requestsTo("/api/security-advisories/") + if len(sent) != 1 || sent[0].query.Get("updatedSince") != "1700000000" { + t.Errorf("updatedSince was not passed on: %+v", sent) + } +} + +func TestComposerAdvisoriesNotOffered(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + upstream.index = `{"packages": []}` + h := newAdvisoryComposerHandler(t, upstream) + + if w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B%5D=public%2Flibrary"); w.Code != http.StatusNotFound { + t.Errorf("advisories status = %d, want 404 when upstream has no API", w.Code) + } + if w := serveComposer(h, "/lists/summary.json"); w.Code != http.StatusNotFound { + t.Errorf("summary status = %d, want 404 when upstream has no filter", w.Code) + } +} + +func TestComposerFilterSummaryRelayed(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + w := serveComposer(h, "/lists/summary.json") + if w.Code != http.StatusOK || w.Body.String() != composerFilterSummary { + t.Fatalf("got %d %q, want upstream's summary", w.Code, w.Body) + } + + req := httptest.NewRequest(http.MethodGet, "/lists/summary.json", nil) + req.Header.Set("If-Modified-Since", "Wed, 21 Oct 2026 15:45:01 GMT") + w = httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusNotModified { + t.Errorf("revalidation status = %d, want 304 passed on from upstream", w.Code) + } +} + +func TestComposerMetadataKeepsAdvisoriesAndFilter(t *testing.T) { + h := NewComposerHandler(testProxy(), "http://proxy.test") + body := []byte(`{ + "minified": "composer/2.0", + "packages": {"evil/package": [{"name": "evil/package", "version": "1.0.0", "dist": {"url": "https://example.com/evil.zip", "type": "zip"}}]}, + "security-advisories": [{"advisoryId": "PKSA-1", "affectedVersions": "<2.0.0"}], + "filter": {"malware": [{"constraint": "1.0.0", "reason": "malware", "id": "PKFE-1"}]} + }`) + + out, _, err := h.rewriteMetadataKeeping(body) + if err != nil { + t.Fatalf("rewrite: %v", err) + } + var doc map[string]json.RawMessage + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("parsing rewritten metadata: %v", err) + } + if !strings.Contains(string(doc["security-advisories"]), "PKSA-1") { + t.Errorf("security-advisories lost in rewrite: %s", out) + } + if !strings.Contains(string(doc["filter"]), "PKFE-1") { + t.Errorf("filter lost in rewrite: %s", out) + } +} From 1abb8c029a3783197f5856831dc0a5f5b921f658 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 16:18:26 +0100 Subject: [PATCH 002/198] Serve dated snapshots of Alpine APK indexes Alpine mirrors keep only the current build of each package, so once a package is upgraded `apk add curl=8.5.0-r0` fails: the current index no longer lists the old build, even when the proxy still has it cached. With upstream.apk_snapshots set (e.g. "90d"), each version of an index or detached signature the metadata cache stores is copied to _snapshots/alpine/ and recorded in a new metadata_snapshots table with the time it was first served. /apk/{name}@{YYYY-MM-DD}/ serves the version that was current at the end of that day; packages below it are the plain path's requests. The kept bytes are the signed upstream bytes, so apk verifies them unchanged. An hourly job deletes versions replaced longer ago than the configured duration and queues their blobs on the pending-delete queue; the newest version of each index stays. Packages listed by kept snapshots are not protected from eviction or retention; the docs say so. --- README.md | 5 + docs/configuration.md | 46 +++++ internal/config/config.go | 26 +++ internal/config/config_test.go | 35 ++++ internal/database/database_test.go | 2 +- internal/database/metadata_snapshots.go | 136 ++++++++++++++ internal/database/metadata_snapshots_test.go | 108 +++++++++++ internal/database/queries.go | 1 + internal/database/schema.go | 59 ++++++ internal/handler/apk.go | 20 +- internal/handler/apk_snapshots.go | 187 +++++++++++++++++++ internal/handler/apk_snapshots_test.go | 168 +++++++++++++++++ internal/server/apk_snapshots.go | 48 +++++ internal/server/apk_snapshots_test.go | 63 +++++++ internal/server/server.go | 4 +- 15 files changed, 902 insertions(+), 6 deletions(-) create mode 100644 internal/database/metadata_snapshots.go create mode 100644 internal/database/metadata_snapshots_test.go create mode 100644 internal/handler/apk_snapshots.go create mode 100644 internal/handler/apk_snapshots_test.go create mode 100644 internal/server/apk_snapshots.go create mode 100644 internal/server/apk_snapshots_test.go diff --git a/README.md b/README.md index 20c6d5d..c360866 100644 --- a/README.md +++ b/README.md @@ -722,6 +722,11 @@ signature verification keeps working. Indexes use the metadata cache (`metadata_ttl`, stale fallback); `.apk` packages are stored in the shared artifact cache and remain available when the upstream is unreachable. +Set `upstream.apk_snapshots` (e.g. `"90d"`) to keep replaced indexes and serve +them by date at `/apk/alpine@2026-10-01/v3.22/main`, so pinned package versions +stay installable after the mirror moves on. See +[Alpine APK snapshots](docs/configuration.md#alpine-apk-snapshots). + To proxy other mirrors or private repositories, configure named upstreams under `upstream.apk` (this replaces the built-in default; re-add `alpine` if you still want it): diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b2..f2faf46 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -314,6 +314,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.rpm` | `PROXY_UPSTREAM_RPM` | `https://dl.fedoraproject.org/pub/fedora/linux` | | `upstream.homebrew_api` | `PROXY_UPSTREAM_HOMEBREW_API` | `https://formulae.brew.sh/api` | | `upstream.homebrew_artifact` | `PROXY_UPSTREAM_HOMEBREW_ARTIFACT` | `https://ghcr.io` | +| `upstream.apk_snapshots` | `PROXY_UPSTREAM_APK_SNAPSHOTS` | disabled; see [Alpine APK snapshots](#alpine-apk-snapshots) | Private, ULA, CGNAT, and loopback addresses are rejected by default. Add each private upstream hostname or IP address to `upstream.allow_private_hosts`. The matching environment variable accepts a comma-separated list. Loopback upstreams also require `upstream.allow_loopback: true`. That setting permits upstream requests and redirects to reach any loopback address. @@ -566,6 +567,8 @@ with the metadata TTL and served byte-for-byte unchanged so apk signature verification keeps working; `.apk` packages use the shared artifact cache. When `upstream.apk` is empty, a single repository named `alpine` pointing at the official mirror is available; configuring any entry replaces that default. +Earlier index versions can be kept and served by date; see +[Alpine APK snapshots](#alpine-apk-snapshots). ```yaml upstream: @@ -613,6 +616,49 @@ as signatures and keys, is passed through uncached. When `upstream.freebsd` is e repository named `freebsd` pointing at `https://pkg.freebsd.org` is available; configuring any entry replaces that default. +### Alpine APK snapshots + +```yaml +upstream: + # Keep replaced APK indexes for 90 days. + apk_snapshots: "90d" +``` + +Alpine mirrors only keep the current build of each package, so after an +upgrade `apk add curl=8.5.0-r0` fails because the index no longer lists that +build. With `upstream.apk_snapshots` set, the proxy keeps every version of an +index (and of a detached signature) it stores, and serves them under a dated +repository name: + +``` +http://localhost:8080/apk/alpine@2026-10-01/v3.22/main +``` + +`/apk/{name}@{YYYY-MM-DD}/…` returns the index that was current at the end of +that day (UTC). The kept files are the signed upstream bytes, so apk verifies +them as usual. A day before the first stored version of an index, or a day +that has not started yet, is `404`. Today's snapshot can still change until the +day ends. Packages under a snapshot path are the same as under the plain path: +they come from the artifact cache, or from upstream if not cached; a build the +upstream has dropped and the proxy never cached is `404`. A configured +repository whose name contains `@` takes precedence over a snapshot path. + +Snapshots are recorded when an index is requested through the plain path, so +they only exist for repositories clients use. A version counts from the first +time the proxy served it, which can be up to `metadata_ttl` after upstream +published it. A version is deleted once it was replaced more than the +configured duration ago; the newest version of each index is always kept. +Durations take a `d` suffix; empty or `"0"` disables snapshots. Snapshots need +`cache_metadata`. Their files live under `_snapshots/` in storage and do not +count towards `storage.max_size`. + +Snapshots keep indexes, not packages. A package listed by an old snapshot can +still be evicted by `storage.max_size` or [retention](#retention), and is then +only available while upstream still has it. Retention counts from a package's +last download, so a pinned build nobody installed for longer than the `alpine` +retention is evicted even while a snapshot lists it. Use package rules with +`"0"` for builds that must stay installable. + ### Go checksum database ```yaml diff --git a/internal/config/config.go b/internal/config/config.go index a660699..6b90541 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -744,6 +744,12 @@ type UpstreamConfig struct { // Default when empty: {"alpine": "https://dl-cdn.alpinelinux.org/alpine"}. APK map[string]string `json:"apk" yaml:"apk"` + // APKSnapshots keeps earlier versions of Alpine APK indexes for this + // long after upstream replaced them, and serves them at + // /apk/{name}@{YYYY-MM-DD}/. Accepts a "d" suffix, e.g. "90d". + // Empty or "0" disables snapshots. + APKSnapshots string `json:"apk_snapshots" yaml:"apk_snapshots"` + // Arch maps repository names to Arch Linux pacman mirror base URLs, // served at /arch/{name}/. The remaining request path mirrors the // upstream layout, e.g. /arch/archlinux/core/os/x86_64/core.db. @@ -852,6 +858,11 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.apk", u.APK); err != nil { return err } + if u.APKSnapshots != "" { + if _, err := parseRetentionDuration("upstream.apk_snapshots", u.APKSnapshots); err != nil { + return err + } + } if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil { return err } @@ -1431,6 +1442,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.Swift, "PROXY_UPSTREAM_SWIFT") setEnvString(&c.Upstream.OCIDefault, "PROXY_UPSTREAM_OCI_DEFAULT") setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN") + setEnvString(&c.Upstream.APKSnapshots, "PROXY_UPSTREAM_APK_SNAPSHOTS") setEnvString(&c.Upstream.RPM, "PROXY_UPSTREAM_RPM") setEnvString(&c.Upstream.HomebrewAPI, "PROXY_UPSTREAM_HOMEBREW_API") setEnvString(&c.Upstream.HomebrewArtifact, "PROXY_UPSTREAM_HOMEBREW_ARTIFACT") @@ -1928,6 +1940,20 @@ func (c *Config) ParseGradleBuildCacheMaxUploadSize() int64 { return size } +// ParseAPKSnapshots returns how long earlier Alpine APK indexes are kept +// after upstream replaced them. Returns 0 when snapshots are disabled or the +// value is invalid. +func (c *Config) ParseAPKSnapshots() time.Duration { + if c.Upstream.APKSnapshots == "" { + return 0 + } + d, err := parseDayDuration(c.Upstream.APKSnapshots) + if err != nil || d < 0 { + return 0 + } + return d +} + // ParseGradleBuildCacheMaxAge returns age-based eviction threshold. // Returns 0 when disabled or invalid. func (c *Config) ParseGradleBuildCacheMaxAge() time.Duration { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7590a64..e3f2dc2 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2052,3 +2052,38 @@ upstream: t.Errorf("generic_paths = %v", cfg.Upstream.GenericPaths) } } + +func TestAPKSnapshots(t *testing.T) { + cfg := Default() + if got := cfg.ParseAPKSnapshots(); got != 0 { + t.Errorf("default ParseAPKSnapshots = %v, want 0", got) + } + + for value, want := range map[string]time.Duration{ + "90d": 90 * 24 * time.Hour, + "12h": 12 * time.Hour, + "0": 0, + } { + cfg.Upstream.APKSnapshots = value + if err := cfg.Validate(); err != nil { + t.Errorf("Validate(%q): %v", value, err) + } + if got := cfg.ParseAPKSnapshots(); got != want { + t.Errorf("ParseAPKSnapshots(%q) = %v, want %v", value, got, want) + } + } + + for _, value := range []string{"forever", "-1d"} { + cfg.Upstream.APKSnapshots = value + if err := cfg.Validate(); err == nil { + t.Errorf("Validate(%q) succeeded, want an error", value) + } + } + + cfg = Default() + t.Setenv("PROXY_UPSTREAM_APK_SNAPSHOTS", "30d") + cfg.LoadFromEnv() + if cfg.Upstream.APKSnapshots != "30d" { + t.Errorf("APKSnapshots from env = %q, want 30d", cfg.Upstream.APKSnapshots) + } +} diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a6..a61f91d 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB { // Drop and recreate every table CreateSchema creates for clean test state; // leftover migration records make the next CreateSchema fail on the // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "migrations", "schema_info"} + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "metadata_snapshots", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go new file mode 100644 index 0000000..b174aab --- /dev/null +++ b/internal/database/metadata_snapshots.go @@ -0,0 +1,136 @@ +package database + +import ( + "database/sql" + "fmt" + "math" + "time" +) + +// Metadata snapshots keep earlier versions of a metadata document after +// upstream replaced it, so a request for a past date can be answered with the +// document that was current then. Each row is one version, valid from +// valid_from_ms (Unix milliseconds, the time the proxy first stored it) until +// the next row for the same ecosystem and name. + +// MetadataSnapshot is one kept version of a metadata document. +type MetadataSnapshot struct { + ID int64 `db:"id"` + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + StoragePath string `db:"storage_path"` + ContentType sql.NullString `db:"content_type"` + ContentEncoding sql.NullString `db:"content_encoding"` + ContentDigest string `db:"content_digest"` + Size int64 `db:"size"` + ValidFromMS int64 `db:"valid_from_ms"` +} + +// ValidFrom returns the time the snapshot became current. +func (s *MetadataSnapshot) ValidFrom() time.Time { + return time.UnixMilli(s.ValidFromMS) +} + +const metadataSnapshotColumns = `id, ecosystem, name, storage_path, content_type, content_encoding, + content_digest, size, valid_from_ms` + +// MetadataSnapshotAt returns the snapshot of ecosystem and name that was +// current at at: the newest one that became current before it. It returns nil +// when there is none. +func (db *DB) MetadataSnapshotAt(ecosystem, name string, at time.Time) (*MetadataSnapshot, error) { + return db.metadataSnapshotBefore(ecosystem, name, at.UnixMilli()) +} + +// LatestMetadataSnapshot returns the newest snapshot of ecosystem and name, +// or nil when there is none. +func (db *DB) LatestMetadataSnapshot(ecosystem, name string) (*MetadataSnapshot, error) { + return db.metadataSnapshotBefore(ecosystem, name, math.MaxInt64) +} + +func (db *DB) metadataSnapshotBefore(ecosystem, name string, beforeMS int64) (*MetadataSnapshot, error) { + var snap MetadataSnapshot + err := db.Get(&snap, db.Rebind(`SELECT `+metadataSnapshotColumns+` + FROM metadata_snapshots + WHERE ecosystem = ? AND name = ? AND valid_from_ms < ? + ORDER BY valid_from_ms DESC, id DESC + LIMIT 1`), ecosystem, name, beforeMS) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("reading metadata snapshot: %w", err) + } + return &snap, nil +} + +// AddMetadataSnapshot records snap as the current version of its document, +// unless the newest snapshot already has its digest, and reports whether it +// did. The check and the insert are one statement, so concurrent callers +// recording the same document add it once. The casts let Postgres type the +// parameters of the INSERT ... SELECT. +func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { + res, err := db.Exec(db.Rebind(` + INSERT INTO metadata_snapshots (ecosystem, name, storage_path, content_type, content_encoding, + content_digest, size, valid_from_ms) + SELECT CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), + CAST(? AS TEXT), CAST(? AS BIGINT), CAST(? AS BIGINT) + WHERE COALESCE(( + SELECT content_digest FROM metadata_snapshots + WHERE ecosystem = ? AND name = ? + ORDER BY valid_from_ms DESC, id DESC + LIMIT 1 + ), '') <> CAST(? AS TEXT) + `), snap.Ecosystem, snap.Name, snap.StoragePath, snap.ContentType, snap.ContentEncoding, + snap.ContentDigest, snap.Size, snap.ValidFromMS, + snap.Ecosystem, snap.Name, snap.ContentDigest) + if err != nil { + return false, fmt.Errorf("recording metadata snapshot: %w", err) + } + n, err := res.RowsAffected() + return n > 0, err +} + +// PruneMetadataSnapshots deletes the snapshots of ecosystem that stopped +// being current before cutoff, that is those replaced by a newer snapshot +// that became current before cutoff, and queues their blobs for deletion. The +// newest snapshot of each document is always kept. It returns the number of +// snapshots deleted. +func (db *DB) PruneMetadataSnapshots(ecosystem string, cutoff time.Time) (int, error) { + tx, err := db.Beginx() + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + + var expired []struct { + ID int64 `db:"id"` + StoragePath string `db:"storage_path"` + } + err = tx.Select(&expired, db.Rebind(` + SELECT s.id, s.storage_path FROM metadata_snapshots s + WHERE s.ecosystem = ? + AND EXISTS ( + SELECT 1 FROM metadata_snapshots n + WHERE n.ecosystem = s.ecosystem AND n.name = s.name + AND (n.valid_from_ms > s.valid_from_ms OR (n.valid_from_ms = s.valid_from_ms AND n.id > s.id)) + AND n.valid_from_ms < ? + ) + `), ecosystem, cutoff.UnixMilli()) + if err != nil { + return 0, fmt.Errorf("listing expired metadata snapshots: %w", err) + } + + queuedAt := time.Now().UTC() + for _, row := range expired { + if _, err := tx.Exec(db.Rebind(`DELETE FROM metadata_snapshots WHERE id = ?`), row.ID); err != nil { + return 0, fmt.Errorf("deleting metadata snapshot: %w", err) + } + if _, err := tx.Exec(db.Rebind(queuePendingDeleteQuery), row.StoragePath, queuedAt); err != nil { + return 0, fmt.Errorf("queueing metadata snapshot blob: %w", err) + } + } + if err := tx.Commit(); err != nil { + return 0, err + } + return len(expired), nil +} diff --git a/internal/database/metadata_snapshots_test.go b/internal/database/metadata_snapshots_test.go new file mode 100644 index 0000000..ae859fd --- /dev/null +++ b/internal/database/metadata_snapshots_test.go @@ -0,0 +1,108 @@ +package database + +import ( + "testing" + "time" +) + +func addSnapshot(t *testing.T, db *DB, name, digest string, validFrom time.Time) bool { + t.Helper() + added, err := db.AddMetadataSnapshot(&MetadataSnapshot{ + Ecosystem: "alpine", + Name: name, + StoragePath: "_snapshots/alpine/" + name + "/" + digest, + ContentDigest: digest, + Size: 1, + ValidFromMS: validFrom.UnixMilli(), + }) + if err != nil { + t.Fatalf("AddMetadataSnapshot: %v", err) + } + return added +} + +func snapshotDigestAt(t *testing.T, db *DB, name string, at time.Time) string { + t.Helper() + snap, err := db.MetadataSnapshotAt("alpine", name, at) + if err != nil { + t.Fatalf("MetadataSnapshotAt: %v", err) + } + if snap == nil { + return "" + } + return snap.ContentDigest +} + +func TestMetadataSnapshots_AddAndLookup(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + day := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + if !addSnapshot(t, db, "idx", "sha256:a", day) { + t.Fatal("first snapshot not added") + } + if addSnapshot(t, db, "idx", "sha256:a", day.Add(time.Hour)) { + t.Error("unchanged digest added again") + } + if !addSnapshot(t, db, "idx", "sha256:b", day.Add(48*time.Hour)) { + t.Fatal("changed digest not added") + } + // A rollback to an earlier version is a new snapshot. + if !addSnapshot(t, db, "idx", "sha256:a", day.Add(96*time.Hour)) { + t.Fatal("rolled back digest not added") + } + + for _, tc := range []struct { + at time.Time + want string + }{ + {day, ""}, + {day.Add(time.Minute), "sha256:a"}, + {day.Add(72 * time.Hour), "sha256:b"}, + {day.Add(100 * time.Hour), "sha256:a"}, + } { + if got := snapshotDigestAt(t, db, "idx", tc.at); got != tc.want { + t.Errorf("snapshot at %s = %q, want %q", tc.at, got, tc.want) + } + } + latest, err := db.LatestMetadataSnapshot("alpine", "idx") + if err != nil || latest == nil || !latest.ValidFrom().Equal(day.Add(96*time.Hour)) { + t.Errorf("LatestMetadataSnapshot = %+v, %v", latest, err) + } + }) +} + +func TestMetadataSnapshots_Prune(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + addSnapshot(t, db, "idx", "sha256:a", now.AddDate(0, 0, -100)) + addSnapshot(t, db, "idx", "sha256:b", now.AddDate(0, 0, -60)) + addSnapshot(t, db, "idx", "sha256:c", now.AddDate(0, 0, -10)) + // The only version of another index stays however old it is. + addSnapshot(t, db, "other", "sha256:x", now.AddDate(0, 0, -200)) + + pruned, err := db.PruneMetadataSnapshots("alpine", now.AddDate(0, 0, -30)) + if err != nil { + t.Fatalf("PruneMetadataSnapshots: %v", err) + } + // a was replaced 60 days ago, b only 10 days ago. + if pruned != 1 { + t.Errorf("pruned = %d, want 1", pruned) + } + if got := snapshotDigestAt(t, db, "idx", now.AddDate(0, 0, -90)); got != "" { + t.Errorf("snapshot of a pruned day = %q, want none", got) + } + if got := snapshotDigestAt(t, db, "idx", now.AddDate(0, 0, -50)); got != "sha256:b" { + t.Errorf("snapshot 50 days ago = %q, want sha256:b", got) + } + if got := snapshotDigestAt(t, db, "other", now); got != "sha256:x" { + t.Errorf("only snapshot of other = %q, want sha256:x", got) + } + + due, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if len(due) != 1 || due[0] != "_snapshots/alpine/idx/sha256:a" { + t.Errorf("queued deletes = %v, want the pruned blob", due) + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e8..973bd08 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -645,6 +645,7 @@ func (db *DB) GetDuePendingDeletes(cutoff time.Time, limit int) ([]string, error WHERE queued_at < ? AND NOT EXISTS (SELECT 1 FROM artifacts WHERE artifacts.storage_path = pending_deletes.path) AND NOT EXISTS (SELECT 1 FROM metadata_cache WHERE metadata_cache.storage_path = pending_deletes.path) + AND NOT EXISTS (SELECT 1 FROM metadata_snapshots WHERE metadata_snapshots.storage_path = pending_deletes.path) ORDER BY queued_at LIMIT ? `) diff --git a/internal/database/schema.go b/internal/database/schema.go index f0ec651..6d31ff4 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -154,6 +154,20 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS metadata_snapshots ( + id INTEGER PRIMARY KEY, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + storage_path TEXT NOT NULL, + content_type TEXT, + content_encoding TEXT, + content_digest TEXT NOT NULL, + size INTEGER NOT NULL, + valid_from_ms INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at DATETIME NOT NULL @@ -297,6 +311,20 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms BIGINT NOT NULL ); +CREATE TABLE IF NOT EXISTS metadata_snapshots ( + id SERIAL PRIMARY KEY, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + storage_path TEXT NOT NULL, + content_type TEXT, + content_encoding TEXT, + content_digest TEXT NOT NULL, + size BIGINT NOT NULL, + valid_from_ms BIGINT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms); +CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at TIMESTAMP NOT NULL @@ -468,6 +496,7 @@ var migrations = []migration{ {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, + {"020_add_metadata_snapshots", migrateAddMetadataSnapshots}, } // isTableNotFound returns true if the error indicates a missing table. @@ -850,6 +879,36 @@ func migrateAddArtifactFilenameIndex(db *DB) error { return nil } +// migrateAddMetadataSnapshots creates the table of kept earlier versions of +// metadata documents, which dated snapshot requests are served from. +func migrateAddMetadataSnapshots(db *DB) error { + pk, integer := "id INTEGER PRIMARY KEY", "INTEGER" + if db.dialect == DialectPostgres { + pk, integer = "id SERIAL PRIMARY KEY", "BIGINT" + } + statements := []string{ + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS metadata_snapshots ( + %s, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + storage_path TEXT NOT NULL, + content_type TEXT, + content_encoding TEXT, + content_digest TEXT NOT NULL, + size %s NOT NULL, + valid_from_ms %s NOT NULL + )`, pk, integer, integer), + `CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_key ON metadata_snapshots(ecosystem, name, valid_from_ms)`, + `CREATE INDEX IF NOT EXISTS idx_metadata_snapshots_path ON metadata_snapshots(storage_path)`, + } + for _, query := range statements { + if _, err := db.Exec(query); err != nil { + return fmt.Errorf("creating metadata_snapshots: %w", err) + } + } + return nil +} + // migrateAddMetadataDates creates the table of Last-Modified dates for // rewritten metadata, which replicas sharing the database agree on. func migrateAddMetadataDates(db *DB) error { diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 9024fe0..071ae3a 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -1,11 +1,13 @@ package handler import ( + "context" "crypto/sha256" "encoding/hex" "net/http" "regexp" "strings" + "time" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" @@ -51,6 +53,9 @@ type APKHandler struct { proxy *Proxy proxyURL string repositories map[string]string + // snapshots is how long replaced index versions are kept; 0 disables + // snapshots. See WithSnapshots. + snapshots time.Duration } // NewAPKHandler creates an Alpine APK repository protocol handler. @@ -87,14 +92,18 @@ func (h *APKHandler) Routes() http.Handler { return } - repository, rest, ok := strings.Cut(path, "/") - upstreamURL, found := h.repositories[repository] + segment, rest, ok := strings.Cut(path, "/") + repository, upstreamURL, day, snapshot, found := h.resolveRepository(segment) if !ok || rest == "" || !found { http.NotFound(w, r) return } switch { + case snapshot && (isAPKIndex(rest) || isAPKSignature(rest)): + // A dated snapshot serves the kept version of the index; packages + // and other files below are the same as under the plain path. + h.serveSnapshot(w, r, repository, upstreamURL, rest, day) case isAPKIndex(rest) || isAPKSignature(rest): // Indexes and detached signatures are signed upstream metadata. // Cache them with the metadata TTL and serve the stored bytes @@ -160,8 +169,11 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques // metadata cache. Stored bytes are re-served verbatim, which keeps embedded // and detached signatures valid. func (h *APKHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { - h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, apkEcosystem, - h.metadataCacheKey(repository, upstreamURL, path), "*/*") + cacheKey := h.metadataCacheKey(repository, upstreamURL, path) + h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, apkEcosystem, cacheKey, "*/*") + if h.snapshotsEnabled() { + h.recordSnapshot(context.WithoutCancel(r.Context()), cacheKey) + } } // metadataCacheKey derives the metadata cache key from the repository name, diff --git a/internal/handler/apk_snapshots.go b/internal/handler/apk_snapshots.go new file mode 100644 index 0000000..cca04d6 --- /dev/null +++ b/internal/handler/apk_snapshots.go @@ -0,0 +1,187 @@ +package handler + +import ( + "context" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// Dated snapshots of APK indexes. +// +// Alpine mirrors keep only the current build of each package, so once a +// package is upgraded the index no longer lists the old build and +// `apk add name=old-version` fails. With snapshots enabled, every version of an +// index (and of a detached signature) the proxy stores is kept, and +// /apk/{name}@{YYYY-MM-DD}/... serves the version that was current at the end +// of that day (UTC). The kept bytes are the signed upstream bytes, so apk +// verifies them as usual. Packages under a snapshot path are the same +// requests as under the plain path. + +// apkSnapshotDateLayout is the date format of a snapshot path segment. +const apkSnapshotDateLayout = "2006-01-02" + +// WithSnapshots keeps earlier versions of the indexes this handler serves and +// makes them available under /apk/{name}@{YYYY-MM-DD}/. keep is how long a +// version is kept after it was replaced; 0 disables snapshots. +func (h *APKHandler) WithSnapshots(keep time.Duration) *APKHandler { + h.snapshots = keep + return h +} + +// snapshotsEnabled reports whether snapshots are recorded and served. They +// need the metadata cache, which stores the documents they are copied from. +func (h *APKHandler) snapshotsEnabled() bool { + return h.snapshots > 0 && h.proxy.CacheMetadata && h.proxy.DB != nil && h.proxy.Storage != nil +} + +// resolveRepository maps the first path segment to a configured repository. +// A segment of the form {name}@{YYYY-MM-DD} names a snapshot of repository +// name for that day, when snapshots are enabled and no repository is +// configured under the full segment. ok is false for anything else. +func (h *APKHandler) resolveRepository(segment string) (repository, upstreamURL string, day time.Time, snapshot, ok bool) { + if upstreamURL, found := h.repositories[segment]; found { + return segment, upstreamURL, time.Time{}, false, true + } + if !h.snapshotsEnabled() { + return "", "", time.Time{}, false, false + } + at := strings.LastIndexByte(segment, '@') + if at < 0 { + return "", "", time.Time{}, false, false + } + upstreamURL, found := h.repositories[segment[:at]] + if !found { + return "", "", time.Time{}, false, false + } + day, err := time.Parse(apkSnapshotDateLayout, segment[at+1:]) + if err != nil { + return "", "", time.Time{}, false, false + } + return segment[:at], upstreamURL, day, true, true +} + +// serveSnapshot answers a request for an index or signature under a snapshot +// path with the version that was current at the end of day. A day for which +// no version was stored, or one that has not begun yet, is 404. +func (h *APKHandler) serveSnapshot(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, day time.Time) { + end := day.AddDate(0, 0, 1) + if day.After(time.Now()) { + http.NotFound(w, r) + return + } + snap, err := h.proxy.DB.MetadataSnapshotAt(apkEcosystem, h.metadataCacheKey(repository, upstreamURL, path), end) + if err != nil { + h.proxy.Logger.Error("apk snapshot lookup failed", "repository", repository, "path", path, "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + if snap == nil { + http.NotFound(w, r) + return + } + + if writeMetadataValidators(w, r, cachedMeta{etag: `"` + snap.ContentDigest + `"`, lastModified: snap.ValidFrom()}) { + return + } + body, err := h.proxy.Storage.Open(r.Context(), snap.StoragePath) + if err != nil { + h.proxy.Logger.Error("apk snapshot unreadable", "repository", repository, "path", path, + "storage_path", snap.StoragePath, "error", err) + http.Error(w, "snapshot unavailable", http.StatusInternalServerError) + return + } + defer func() { _ = body.Close() }() + + if snap.ContentType.String != "" { + w.Header().Set(headerContentType, snap.ContentType.String) + } + w.Header().Set(headerContentLength, strconv.FormatInt(snap.Size, 10)) + if snap.ContentEncoding.String != "" { + w.Header().Set(headerContentEncoding, snap.ContentEncoding.String) + } + w.WriteHeader(http.StatusOK) + if r.Method != http.MethodHead { + _, _ = io.Copy(w, body) + } +} + +// recordSnapshot keeps the document now cached under cacheKey as a snapshot +// when it differs from the newest one kept. It runs after the document was +// served, so a failure only means this version is recorded on a later +// request instead. +func (h *APKHandler) recordSnapshot(ctx context.Context, cacheKey string) { + entry, err := h.proxy.DB.GetMetadataCache(apkEcosystem, cacheKey) + if err != nil || entry == nil || !entry.ContentDigest.Valid { + return + } + digest := entry.ContentDigest.String + latest, err := h.proxy.DB.LatestMetadataSnapshot(apkEcosystem, cacheKey) + if err != nil || (latest != nil && latest.ContentDigest == digest) { + return + } + + validFrom := time.Now() + storagePath := apkSnapshotStoragePath(cacheKey, digest, validFrom) + size, ok := h.copySnapshotBlob(ctx, entry.StoragePath, storagePath, digest) + if !ok { + return + } + added, err := h.proxy.DB.AddMetadataSnapshot(&database.MetadataSnapshot{ + Ecosystem: apkEcosystem, + Name: cacheKey, + StoragePath: storagePath, + ContentType: entry.ContentType, + ContentEncoding: entry.ContentEncoding, + ContentDigest: digest, + Size: size, + ValidFromMS: validFrom.UnixMilli(), + }) + if err != nil { + h.proxy.Logger.Warn("failed to record apk snapshot", "key", cacheKey, "error", err) + } + if err != nil || !added { + // Another request recorded this version first. + h.discardSnapshotBlob(ctx, storagePath) + } +} + +// copySnapshotBlob copies the cached document at from to to and reports its +// size, provided the copied bytes still have digest. A concurrent fetch can +// replace the cached document after its row was read; that version is +// recorded by the request that stored it. +func (h *APKHandler) copySnapshotBlob(ctx context.Context, from, to, digest string) (int64, bool) { + src, err := h.proxy.Storage.Open(ctx, from) + if err != nil { + return 0, false + } + defer func() { _ = src.Close() }() + size, hash, err := h.proxy.Storage.Store(ctx, to, src) + if err != nil { + h.proxy.Logger.Warn("failed to store apk snapshot", "path", to, "error", err) + return 0, false + } + if "sha256:"+hash != digest { + h.discardSnapshotBlob(ctx, to) + return 0, false + } + return size, true +} + +func (h *APKHandler) discardSnapshotBlob(ctx context.Context, path string) { + if err := h.proxy.Storage.Delete(ctx, path); err != nil { + h.proxy.Logger.Warn("failed to discard apk snapshot blob", "path", path, "error", err) + } +} + +// apkSnapshotStoragePath is where one kept version of the document cached +// under cacheKey is stored. The time it became current keeps a version that +// returns after an upgrade was rolled back apart from its earlier copy. +func apkSnapshotStoragePath(cacheKey, digest string, validFrom time.Time) string { + return "_snapshots/" + apkEcosystem + "/" + cacheKey + "/" + + strconv.FormatInt(validFrom.UnixMilli(), 10) + "-" + strings.TrimPrefix(digest, "sha256:") +} diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go new file mode 100644 index 0000000..50291c1 --- /dev/null +++ b/internal/handler/apk_snapshots_test.go @@ -0,0 +1,168 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/registries/fetch" +) + +const apkSnapshotIndexPath = "/v3.22/main/x86_64/APKINDEX.tar.gz" + +// apkSnapshotUpstream serves the index held in index, initially "index v1", +// and counts package downloads. +type apkSnapshotUpstream struct { + index atomic.Value + downloads atomic.Int32 + server *httptest.Server +} + +func newAPKSnapshotUpstream(t *testing.T) *apkSnapshotUpstream { + t.Helper() + u := &apkSnapshotUpstream{} + u.index.Store("index v1") + u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case apkSnapshotIndexPath: + _, _ = w.Write([]byte(u.index.Load().(string))) + case "/v3.22/main/x86_64/curl-8.5.0-r0.apk": + u.downloads.Add(1) + _, _ = w.Write([]byte("curl 8.5.0")) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.server.Close) + return u +} + +// newSnapshotAPKHandler returns an APK handler with snapshots enabled whose +// metadata TTL is 0, so every index request asks upstream. +func newSnapshotAPKHandler(t *testing.T, upstreamURL string) (*APKHandler, *database.DB) { + t.Helper() + proxy, db, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 + proxy.HTTPClient = http.DefaultClient + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(http.DefaultClient), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewAPKHandler(proxy, "http://proxy.example", map[string]string{"alpine": upstreamURL}). + WithSnapshots(30 * 24 * time.Hour) + return h, db +} + +func snapshotCount(t *testing.T, db *database.DB) int { + t.Helper() + var n int + if err := db.Get(&n, `SELECT COUNT(*) FROM metadata_snapshots`); err != nil { + t.Fatal(err) + } + return n +} + +func expectAPKResponse(t *testing.T, h *APKHandler, target string, status int, body string) { + t.Helper() + w := serveAPKRequest(h, target) + if w.Code != status { + t.Fatalf("%s: status = %d, want %d: %s", target, w.Code, status, w.Body.String()) + } + if status == http.StatusOK && w.Body.String() != body { + t.Errorf("%s: body = %q, want %q", target, w.Body.String(), body) + } +} + +// TestAPKSnapshots_ServeIndexCurrentAtDate records two versions of an index +// through the plain path and checks that each dated path serves the version +// that was current at the end of that day. +func TestAPKSnapshots_ServeIndexCurrentAtDate(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, upstream.server.URL) + + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + // Pretend v1 was first stored on 2026-01-10. + recorded := time.Date(2026, 1, 10, 12, 0, 0, 0, time.UTC) + if _, err := db.Exec(`UPDATE metadata_snapshots SET valid_from_ms = ?`, recorded.UnixMilli()); err != nil { + t.Fatal(err) + } + + upstream.index.Store("index v2") + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v2") + // An unchanged index adds no snapshot. + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v2") + if got := snapshotCount(t, db); got != 2 { + t.Fatalf("snapshots = %d, want 2", got) + } + + today := time.Now().UTC().Format(apkSnapshotDateLayout) + future := time.Now().UTC().AddDate(0, 0, 2).Format(apkSnapshotDateLayout) + expectAPKResponse(t, h, "/alpine@2026-01-09"+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/alpine@2026-01-10"+apkSnapshotIndexPath, http.StatusOK, "index v1") + expectAPKResponse(t, h, "/alpine@"+today+apkSnapshotIndexPath, http.StatusOK, "index v2") + expectAPKResponse(t, h, "/alpine@"+future+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/alpine@2026-13-01"+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/other@2026-01-10"+apkSnapshotIndexPath, http.StatusNotFound, "") + expectAPKResponse(t, h, "/alpine@2026-01-10/v3.22/community/x86_64/APKINDEX.tar.gz", http.StatusNotFound, "") +} + +func TestAPKSnapshots_ConditionalRequest(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, _ := newSnapshotAPKHandler(t, upstream.server.URL) + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + + target := "/alpine@" + time.Now().UTC().Format(apkSnapshotDateLayout) + apkSnapshotIndexPath + first := serveAPKRequest(h, target) + etag := first.Header().Get("ETag") + if first.Code != http.StatusOK || etag == "" { + t.Fatalf("status = %d, ETag = %q, want 200 with an ETag", first.Code, etag) + } + req := httptest.NewRequest(http.MethodGet, target, nil) + req.Header.Set("If-None-Match", etag) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + if w.Code != http.StatusNotModified { + t.Errorf("conditional request: status = %d, want 304", w.Code) + } +} + +// TestAPKSnapshots_PackagesShareTheCache checks that a package requested +// under a snapshot path is the cached artifact of the plain path. +func TestAPKSnapshots_PackagesShareTheCache(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, _ := newSnapshotAPKHandler(t, upstream.server.URL) + + pkg := "/v3.22/main/x86_64/curl-8.5.0-r0.apk" + expectAPKResponse(t, h, "/alpine"+pkg, http.StatusOK, "curl 8.5.0") + expectAPKResponse(t, h, "/alpine@2026-01-10"+pkg, http.StatusOK, "curl 8.5.0") + if got := upstream.downloads.Load(); got != 1 { + t.Errorf("upstream downloads = %d, want 1", got) + } + expectAPKResponse(t, h, "/alpine@2026-01-10/v3.22/main/x86_64/gone-1.0-r0.apk", http.StatusNotFound, "") +} + +func TestAPKSnapshots_DisabledByDefault(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, upstream.server.URL) + h.WithSnapshots(0) + + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if got := snapshotCount(t, db); got != 0 { + t.Errorf("snapshots = %d, want 0", got) + } + today := time.Now().UTC().Format(apkSnapshotDateLayout) + expectAPKResponse(t, h, "/alpine@"+today+apkSnapshotIndexPath, http.StatusNotFound, "") +} + +// TestAPKSnapshots_ConfiguredNameWins checks that a repository whose +// configured name contains '@' is served as configured, not as a snapshot. +func TestAPKSnapshots_ConfiguredNameWins(t *testing.T) { + upstream := newAPKSnapshotUpstream(t) + h, _ := newSnapshotAPKHandler(t, upstream.server.URL) + h.repositories["alpine@2026-01-10"] = upstream.server.URL + + expectAPKResponse(t, h, "/alpine@2026-01-10"+apkSnapshotIndexPath, http.StatusOK, "index v1") +} diff --git a/internal/server/apk_snapshots.go b/internal/server/apk_snapshots.go new file mode 100644 index 0000000..ee8e3c9 --- /dev/null +++ b/internal/server/apk_snapshots.go @@ -0,0 +1,48 @@ +package server + +import ( + "context" + "time" +) + +// apkSnapshotPruneInterval is how often snapshots kept past +// upstream.apk_snapshots are deleted. +const apkSnapshotPruneInterval = time.Hour + +// startAPKSnapshotPruning deletes APK index snapshots that were replaced more +// than upstream.apk_snapshots ago, once at startup and then every hour. Their +// blobs go through the pending-delete queue like evicted cache entries. +func (s *Server) startAPKSnapshotPruning(ctx context.Context) { + keep := s.cfg.ParseAPKSnapshots() + if keep <= 0 { + return + } + if !s.cfg.CacheMetadata { + s.logger.Warn("upstream.apk_snapshots needs cache_metadata, snapshots disabled") + return + } + + ticker := time.NewTicker(apkSnapshotPruneInterval) + defer ticker.Stop() + for { + s.pruneAPKSnapshots(ctx, keep, time.Now()) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (s *Server) pruneAPKSnapshots(ctx context.Context, keep time.Duration, now time.Time) { + s.runExclusive(ctx, "apk-snapshot-prune", func() { + n, err := s.db.PruneMetadataSnapshots("alpine", now.Add(-keep)) + if err != nil { + s.logger.Warn("apk snapshots: prune failed", "error", err) + return + } + if n > 0 { + s.logger.Info("apk snapshots: pruned", "deleted", n) + } + }) +} diff --git a/internal/server/apk_snapshots_test.go b/internal/server/apk_snapshots_test.go new file mode 100644 index 0000000..191e073 --- /dev/null +++ b/internal/server/apk_snapshots_test.go @@ -0,0 +1,63 @@ +package server + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" + "github.com/go-chi/chi/v5" +) + +// TestAPKSnapshotsConfigured checks that upstream.apk_snapshots enables the +// dated APK paths, with the digest the file storage records for the cached +// index matching the copy kept as a snapshot. +func TestAPKSnapshotsConfigured(t *testing.T) { + const index = "signed index" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, index) + })) + defer upstream.Close() + + db, err := database.Create(t.TempDir() + "/test.db") + if err != nil { + t.Fatalf("creating database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) + if err != nil { + t.Fatalf("opening storage: %v", err) + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy := handler.NewProxy(db, store, fetch.NewFetcher(), fetch.NewResolver(), logger) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + + cfg := config.Default() + cfg.BaseURL = "http://proxy.test" + cfg.Upstream.APK = map[string]string{"alpine": upstream.URL} + cfg.Upstream.APKSnapshots = "30d" + s := &Server{cfg: cfg, logger: logger} + r := chi.NewRouter() + s.mountProtocolHandlers(r, proxy) + + today := time.Now().UTC().Format("2006-01-02") + for _, target := range []string{ + "/apk/alpine/v3.22/main/x86_64/APKINDEX.tar.gz", + "/apk/alpine@" + today + "/v3.22/main/x86_64/APKINDEX.tar.gz", + } { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + if w.Code != http.StatusOK || w.Body.String() != index { + t.Fatalf("%s: status = %d, body = %q, want 200 %q", target, w.Code, w.Body.String(), index) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 12619a6..fb378ac 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -425,6 +425,7 @@ func (s *Server) serve(listener net.Listener) error { s.background.Go(func() { backfillMetadataAccess(bgCtx, s.db, s.logger, metadataBackfillBatch) }) s.background.Go(func() { s.startEvictionLoop(bgCtx) }) s.background.Go(func() { s.startRetentionLoop(bgCtx) }) + s.background.Go(func() { s.startAPKSnapshotPruning(bgCtx) }) s.background.Go(func() { s.startReclaimLoop(bgCtx) }) if listener != nil { @@ -489,7 +490,8 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { containerHandler := s.newContainerHandler(proxy) helmHandler := handler.NewHelmHandlerWithOCIRegistries( proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI) - apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK) + apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK). + WithSnapshots(s.cfg.ParseAPKSnapshots()) archHandler := handler.NewArchHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Arch) freebsdHandler := handler.NewFreeBSDHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.FreeBSD) debianHandler := handler.NewDebianHandler( From 92b1fb20a6d451693d4d5347092f505a9a398d10 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 16:31:47 +0100 Subject: [PATCH 003/198] Follow npm and Packagist change feeds to revalidate changed metadata Within metadata_ttl cached npm and Composer metadata is served without asking upstream, so a new release can take that long to show up. Both registries publish a feed of the packages that changed. upstream.npm_change_feed and upstream.composer_change_feed (off by default, require cache_metadata) start a reader per feed that runs every minute and clears fetched_at on the cached rows of listed packages that were fetched before the read. The rows keep their document and ETag, so the next request revalidates them upstream. Routed packages are cached under other keys and never match. A Packagist resync expires all cached Composer metadata. Failed reads back off up to 15 minutes. The feed position is kept in memory, so after a restart the reader starts from the feed's current position. --- README.md | 7 +- config.example.yaml | 9 + docs/configuration.md | 22 ++ internal/changefeed/changefeed.go | 261 +++++++++++++++++ internal/changefeed/changefeed_test.go | 198 +++++++++++++ internal/config/config.go | 20 ++ internal/config/config_test.go | 35 +++ internal/database/metadata_cache_test.go | 90 ++++++ internal/database/queries.go | 46 +++ internal/metrics/metrics.go | 16 ++ internal/server/analytics_coverage_test.go | 40 +-- internal/server/analytics_runtime.go | 2 + internal/server/analytics_runtime_test.go | 18 ++ internal/server/change_feeds.go | 94 ++++++ internal/server/change_feeds_test.go | 272 ++++++++++++++++++ internal/server/server.go | 1 + internal/server/server_test.go | 7 - .../templates/components/runtime_metrics.html | 1 + 18 files changed, 1110 insertions(+), 29 deletions(-) create mode 100644 internal/changefeed/changefeed.go create mode 100644 internal/changefeed/changefeed_test.go create mode 100644 internal/server/change_feeds.go create mode 100644 internal/server/change_feeds_test.go diff --git a/README.md b/README.md index 20c6d5d..4f0c033 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ therefore come from upstream's database, not from the proxy's own vulnerability data, and versions withheld by [cooldown](#version-cooldown) are not excluded from the report. -To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). +To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds). ### Cargo @@ -387,7 +387,7 @@ Or set globally: composer config -g repositories.proxy composer http://localhost:8080/composer ``` -To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). +To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds). ### Conan (C/C++) @@ -1521,6 +1521,7 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre | `proxy_metadata_cache_entries` | gauge | `ecosystem` | Number of cached metadata documents per ecosystem. | | `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) | | `proxy_metadata_evicted_total` | counter | `ecosystem` | Cached metadata documents evicted by the size limit | +| `proxy_metadata_expired_by_change_feed_total` | counter | `ecosystem` | Cached metadata documents set to revalidate because an upstream [change feed](docs/configuration.md#change-feeds) listed them | | `proxy_fetch_lease_waits_total` | counter | `kind` | Cache misses (`artifact` or `metadata`) that waited for another instance sharing the database to fetch them | | `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream | | `proxy_upstream_errors_total` | counter | `ecosystem`, `error_type` | Upstream fetch failures | @@ -1591,7 +1592,7 @@ The `kind` label on `proxy_cache_hits_total` and `proxy_cache_misses_total` keep `ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which. -**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. +**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_metadata_expired_by_change_feed_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. **From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all. diff --git a/config.example.yaml b/config.example.yaml index d3b6da4..8ad3759 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -189,6 +189,10 @@ upstream: # npm_routes: # "@example/*": "https://npm.example.com/registry" + # Follow the npm registry's change feed, so a new release is fetched on the + # next request instead of after metadata_ttl. Requires cache_metadata. + # npm_change_feed: true + # Cargo sparse index URL cargo: "https://index.crates.io" @@ -245,6 +249,11 @@ upstream: # composer_routes: # "example/*": "https://composer.example.com/packages" + # Follow the change feed of the Packagist at composer, so a new release is + # fetched on the next request instead of after metadata_ttl. Requires + # cache_metadata. + # composer_change_feed: true + # Conan registry URL conan: "https://center.conan.io" diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b2..e18263f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -287,6 +287,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.allow_loopback` | `PROXY_UPSTREAM_ALLOW_LOOPBACK` | `false` | | `upstream.npm` | `PROXY_UPSTREAM_NPM` | `https://registry.npmjs.org` | | `upstream.npm_full_metadata` | `PROXY_UPSTREAM_NPM_FULL_METADATA` | `false` | +| `upstream.npm_change_feed` | `PROXY_UPSTREAM_NPM_CHANGE_FEED` | `false` (see [change feeds](#change-feeds)) | | `upstream.cargo` | `PROXY_UPSTREAM_CARGO` | `https://index.crates.io` | | `upstream.cargo_download` | `PROXY_UPSTREAM_CARGO_DOWNLOAD` | `https://static.crates.io/crates` | | `upstream.gem` | `PROXY_UPSTREAM_GEM` | `https://rubygems.org` | @@ -303,6 +304,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.nuget_search` | `PROXY_UPSTREAM_NUGET_SEARCH` | `https://azuresearch-usnc.nuget.org` | | `upstream.composer` | `PROXY_UPSTREAM_COMPOSER` | `https://packagist.org` | | `upstream.composer_repository` | `PROXY_UPSTREAM_COMPOSER_REPOSITORY` | `https://repo.packagist.org` | +| `upstream.composer_change_feed` | `PROXY_UPSTREAM_COMPOSER_CHANGE_FEED` | `false` (see [change feeds](#change-feeds)) | | `upstream.conan` | `PROXY_UPSTREAM_CONAN` | `https://center.conan.io` | | `upstream.chef` | `PROXY_UPSTREAM_CHEF` | `https://supermarket.chef.io` | | `upstream.conda` | `PROXY_UPSTREAM_CONDA` | `https://conda.anaconda.org` | @@ -1346,6 +1348,26 @@ Set to `"0"` to always revalidate with upstream (ETag-based conditional requests When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. +### Change feeds + +Within `metadata_ttl` a new release isn't seen. For npm and Composer the proxy can instead follow the registry's change feed, which lists the packages that changed, and revalidate just those: + +```yaml +cache_metadata: true +upstream: + npm_change_feed: true + composer_change_feed: true +``` + +Or via environment variables: `PROXY_UPSTREAM_NPM_CHANGE_FEED=true`, `PROXY_UPSTREAM_COMPOSER_CHANGE_FEED=true`. Both require `cache_metadata`, and do nothing with a `metadata_ttl` of `"0"`. + +Every minute the proxy reads what changed since its last read. Cached metadata of a listed package that was fetched before the read is marked as needing revalidation, and the next request for it asks the upstream with its `ETag` (a `304` when nothing changed). Artifacts are not touched. Listed packages the proxy hasn't cached cost nothing beyond the read. The feed is read in full, so the upstream doesn't learn which packages the proxy has cached. Packages in `upstream.npm_routes` or `upstream.composer_routes` are cached under other keys and never matched. + +- Composer reads `{upstream.composer}/metadata/changes.json`, which packagist.org advertises as `metadata-changes-url`. Both `vendor/package` and its `vendor/package~dev` file are covered. Packagist keeps 24 hours of changes; if the proxy's position is older than that, Packagist asks for a resync and all cached Composer metadata is revalidated. repo.packagist.org sends `Cache-Control: max-age=900`, but Packagist purges its CDN when a package changes, so revalidation gets the new file within seconds; the proxy doesn't apply that header either way. +- npm reads `https://replicate.npmjs.com/registry/_changes`, the public registry's feed, whatever `upstream.npm` is. It is only useful when `upstream.npm` is registry.npmjs.org or serves the same packages. Packuments (`/npm/{name}`) are covered; version and dist-tag documents (`/npm/{name}/latest`) still wait out `metadata_ttl`. A busy minute on npm lists a few thousand packages; the proxy reads up to 100,000 per pass and continues where it stopped on the next. + +A new release therefore shows up about a minute after the registry lists it rather than up to `metadata_ttl` later. The feed position is kept in memory: after a restart the proxy starts from the feed's current position, and changes made while it was down are covered by `metadata_ttl` as before. While a feed can't be read the proxy logs a warning and retries with a growing delay, up to 15 minutes, and `metadata_ttl` applies meanwhile. A request whose upstream fetch overlaps the change and the feed read can still store the old document for one TTL. Each instance sharing a database reads the feeds itself. `proxy_metadata_expired_by_change_feed_total` counts the cached documents marked, by ecosystem, and debug logging names each one. + ### Metadata size limit Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. diff --git a/internal/changefeed/changefeed.go b/internal/changefeed/changefeed.go new file mode 100644 index 0000000..2e2d48f --- /dev/null +++ b/internal/changefeed/changefeed.go @@ -0,0 +1,261 @@ +// Package changefeed reads the feeds in which public registries list the +// packages that changed since a given point, so the proxy can revalidate its +// cached metadata for them before metadata_ttl runs out. +package changefeed + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "slices" + "strconv" + "strings" +) + +// Feed is one registry's change feed. It remembers how far it has read, in +// memory only: after a restart it starts again from the registry's current +// position. +type Feed interface { + // Ecosystem is the metadata cache ecosystem the changed names belong to. + Ecosystem() string + // Changes returns what changed since the previous call. The first call + // only finds the feed's current position and reports no changes. On error + // the position stays where it was, so the next call asks again. + Changes(ctx context.Context) (Changes, error) +} + +// Changes is one read of a feed. +type Changes struct { + // Names are the cache keys of the documents that changed. + Names []string + // All is set when the feed can't say what changed, so every document of + // the ecosystem has to be treated as changed. + All bool +} + +// maxFeedResponse bounds a feed response read into memory. A full page of +// the npm feed is about 1.5 MB. +const maxFeedResponse = 64 << 20 + +// getJSON fetches u and decodes its JSON body into v. A status other than 200 +// is an error unless allowStatus also accepts it. +func getJSON(ctx context.Context, client *http.Client, u string, v any, allowStatus ...int) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return fmt.Errorf("creating request: %w", err) + } + req.Header.Set("Accept", "application/json") + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK && !slices.Contains(allowStatus, resp.StatusCode) { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxFeedResponse)) + return fmt.Errorf("%s: unexpected status %d", redact(u), resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxFeedResponse+1)) + if err != nil { + return fmt.Errorf("%s: reading response: %w", redact(u), err) + } + if len(body) > maxFeedResponse { + return fmt.Errorf("%s: response larger than %d bytes", redact(u), maxFeedResponse) + } + if err := json.Unmarshal(body, v); err != nil { + return fmt.Errorf("%s: decoding response: %w", redact(u), err) + } + return nil +} + +// redact drops credentials and the query from u for error messages. +func redact(u string) string { + parsed, err := url.Parse(u) + if err != nil { + return "feed" + } + parsed.User = nil + parsed.RawQuery = "" + return parsed.String() +} + +// withQuery returns base with the query parameters in q added. +func withQuery(base string, q url.Values) string { + sep := "?" + if strings.Contains(base, "?") { + sep = "&" + } + return base + sep + q.Encode() +} + +// ComposerFeedURL is where a Packagist instance at apiURL, such as +// https://packagist.org, publishes its metadata change feed. Packagist +// advertises it as "metadata-changes-url" in its packages.json. +func ComposerFeedURL(apiURL string) string { + return strings.TrimSuffix(apiURL, "/") + "/metadata/changes.json" +} + +// ComposerFeed reads Packagist's metadata change feed +// (https://packagist.org/apidoc#track-package-updates). Its position is a +// timestamp in units of 1/10000 s. Asked without one it answers 400 with the +// current timestamp; asked with one older than its 24 hours of history it +// answers with a single "resync" action. Each update names the file that +// changed, "vendor/package" or "vendor/package~dev", which is also the key +// the Composer handler caches it under; a delete names the package, and +// covers both files. +type ComposerFeed struct { + client *http.Client + url string + since int64 +} + +// NewComposerFeed returns the feed at feedURL, see ComposerFeedURL. +func NewComposerFeed(client *http.Client, feedURL string) *ComposerFeed { + return &ComposerFeed{client: client, url: feedURL} +} + +func (f *ComposerFeed) Ecosystem() string { return "composer" } + +type composerChanges struct { + Actions []struct { + Type string `json:"type"` + Package string `json:"package"` + } `json:"actions"` + Timestamp int64 `json:"timestamp"` + Error string `json:"error"` +} + +func (f *ComposerFeed) Changes(ctx context.Context) (Changes, error) { + if f.since == 0 { + var start composerChanges + if err := getJSON(ctx, f.client, f.url, &start, http.StatusBadRequest); err != nil { + return Changes{}, err + } + if start.Timestamp <= 0 { + return Changes{}, errors.New("composer change feed: no timestamp to start from") + } + f.since = start.Timestamp + return Changes{}, nil + } + + var page composerChanges + u := withQuery(f.url, url.Values{"since": {strconv.FormatInt(f.since, 10)}}) + if err := getJSON(ctx, f.client, u, &page); err != nil { + return Changes{}, err + } + if page.Error != "" || page.Timestamp <= 0 { + return Changes{}, fmt.Errorf("composer change feed: %q", page.Error) + } + var changes Changes + for _, a := range page.Actions { + switch a.Type { + case "update": + changes.Names = append(changes.Names, a.Package) + case "delete": + // A deleted package takes its dev versions with it. + changes.Names = append(changes.Names, a.Package, a.Package+"~dev") + case "resync": + // The position is too old for the feed to list what changed. + changes.All = true + } + } + f.since = page.Timestamp + return changes, nil +} + +// NPMFeedURL is the change feed of the public npm registry. +const NPMFeedURL = "https://replicate.npmjs.com/registry/_changes" + +// npmPageLimit is the most changes the npm feed returns in one response. +const npmPageLimit = 10000 + +// npmMaxPages bounds how many pages one Changes call reads. The rest is read +// on the next call. +const npmMaxPages = 10 + +// NPMFeed reads the npm registry's CouchDB-style changes feed at +// replicate.npmjs.com. Its position is a sequence, which npm asks followers +// to treat as opaque; the database document at the feed's parent URL, with +// its trailing slash (without it the URL is 404), gives the current one as +// update_seq. A page holds at most 10000 changes (1000 when no limit is +// given). Each result's id is a package name, which is also the key the npm +// handler caches its packument under. +type NPMFeed struct { + client *http.Client + url string + // since is the sequence exactly as the feed returned it. + since json.RawMessage +} + +// NewNPMFeed returns the feed at feedURL, see NPMFeedURL. +func NewNPMFeed(client *http.Client, feedURL string) *NPMFeed { + return &NPMFeed{client: client, url: feedURL} +} + +func (f *NPMFeed) Ecosystem() string { return "npm" } + +type npmChanges struct { + Results []struct { + ID string `json:"id"` + } `json:"results"` + LastSeq json.RawMessage `json:"last_seq"` +} + +func (f *NPMFeed) Changes(ctx context.Context) (Changes, error) { + if f.since == nil { + var info struct { + UpdateSeq json.RawMessage `json:"update_seq"` + } + if err := getJSON(ctx, f.client, strings.TrimSuffix(f.url, "_changes"), &info); err != nil { + return Changes{}, err + } + if len(info.UpdateSeq) == 0 { + return Changes{}, errors.New("npm change feed: no update_seq to start from") + } + f.since = info.UpdateSeq + return Changes{}, nil + } + + var changes Changes + for range npmMaxPages { + var page npmChanges + u := withQuery(f.url, url.Values{ + "since": {seqParam(f.since)}, + "limit": {strconv.Itoa(npmPageLimit)}, + }) + if err := getJSON(ctx, f.client, u, &page); err != nil { + // Changes already read are reported, and the position kept + // after them, so they aren't lost to a later page's failure. + if len(changes.Names) > 0 { + return changes, nil + } + return Changes{}, err + } + if len(page.LastSeq) == 0 { + return changes, errors.New("npm change feed: response without last_seq") + } + for _, r := range page.Results { + if r.ID != "" && !strings.HasPrefix(r.ID, "_design/") { + changes.Names = append(changes.Names, r.ID) + } + } + f.since = page.LastSeq + if len(page.Results) < npmPageLimit { + break + } + } + return changes, nil +} + +// seqParam returns a sequence as the since parameter takes it: a number as +// is, a string without its quotes. +func seqParam(seq json.RawMessage) string { + var s string + if json.Unmarshal(seq, &s) == nil { + return s + } + return string(seq) +} diff --git a/internal/changefeed/changefeed_test.go b/internal/changefeed/changefeed_test.go new file mode 100644 index 0000000..21a4765 --- /dev/null +++ b/internal/changefeed/changefeed_test.go @@ -0,0 +1,198 @@ +package changefeed + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" +) + +// fakePackagist serves a change feed the way packagist.org does: without +// since, 400 with the current timestamp; with an old since, a resync; with a +// known one, the actions recorded after it. +type fakePackagist struct { + mu sync.Mutex + now int64 + actions map[int64]string // since -> JSON actions + asked []string +} + +func (p *fakePackagist) ServeHTTP(w http.ResponseWriter, r *http.Request) { + p.mu.Lock() + defer p.mu.Unlock() + since := r.URL.Query().Get("since") + p.asked = append(p.asked, since) + w.Header().Set("Content-Type", "application/json") + if since == "" { + w.WriteHeader(http.StatusBadRequest) + _, _ = fmt.Fprintf(w, `{"error":"Invalid or missing since query parameter","timestamp":%d}`, p.now) + return + } + n, _ := strconv.ParseInt(since, 10, 64) + actions, ok := p.actions[n] + if !ok { + actions = `[{"type":"resync","package":"*","time":1}]` + } + _, _ = fmt.Fprintf(w, `{"actions":%s,"timestamp":%d}`, actions, p.now) +} + +func TestComposerFeed(t *testing.T) { + p := &fakePackagist{now: 100, actions: map[int64]string{ + 100: `[{"type":"update","package":"acme/lib","time":1},{"type":"update","package":"acme/lib~dev","time":1},{"type":"delete","package":"acme/gone","time":1}]`, + 200: `[]`, + }} + srv := httptest.NewServer(p) + defer srv.Close() + feed := NewComposerFeed(srv.Client(), ComposerFeedURL(srv.URL+"/")) + ctx := context.Background() + + if c, err := feed.Changes(ctx); err != nil || c.Names != nil || c.All { + t.Fatalf("first read = %+v, %v; want the start position only", c, err) + } + p.mu.Lock() + p.now = 200 + p.mu.Unlock() + c, err := feed.Changes(ctx) + if err != nil { + t.Fatal(err) + } + if want := []string{"acme/lib", "acme/lib~dev", "acme/gone", "acme/gone~dev"}; !slices.Equal(c.Names, want) || c.All { + t.Errorf("changes = %+v, want %v", c, want) + } + if c, err := feed.Changes(ctx); err != nil || len(c.Names) != 0 || c.All { + t.Errorf("changes without news = %+v, %v", c, err) + } + + // A position the feed no longer has history for gets a resync. + feed.since = 50 + if c, err := feed.Changes(ctx); err != nil || !c.All { + t.Errorf("changes after a resync = %+v, %v, want All", c, err) + } + if want := []string{"", "100", "200", "50"}; !slices.Equal(p.asked, want) { + t.Errorf("asked since %q, want %q", p.asked, want) + } +} + +func TestComposerFeedErrorKeepsPosition(t *testing.T) { + var fail atomic.Bool + fail.Store(true) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if fail.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + if r.URL.Query().Get("since") != "7" { + t.Errorf("since = %q, want 7", r.URL.Query().Get("since")) + } + _, _ = w.Write([]byte(`{"actions":[{"type":"update","package":"acme/lib","time":1}],"timestamp":8}`)) + })) + defer srv.Close() + feed := &ComposerFeed{client: srv.Client(), url: srv.URL, since: 7} + if _, err := feed.Changes(context.Background()); err == nil { + t.Fatal("want an error from a failing feed") + } + if feed.since != 7 { + t.Fatalf("since = %d after an error, want 7", feed.since) + } + fail.Store(false) + if c, err := feed.Changes(context.Background()); err != nil || !slices.Equal(c.Names, []string{"acme/lib"}) { + t.Errorf("changes after recovery = %+v, %v", c, err) + } +} + +// fakeReplicate serves a CouchDB-style changes feed over seqs 1..len(ids). +type fakeReplicate struct { + mu sync.Mutex + ids []string + limit int + asked []string +} + +func (f *fakeReplicate) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/registry/" { + _, _ = fmt.Fprintf(w, `{"db_name":"registry","update_seq":%d}`, len(f.ids)) + return + } + if r.URL.Path != "/registry/_changes" { + w.WriteHeader(http.StatusNotFound) + return + } + since, _ := strconv.Atoi(r.URL.Query().Get("since")) + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + f.asked = append(f.asked, r.URL.Query().Get("since")) + if f.limit > 0 { + limit = min(limit, f.limit) + } + var results []string + last := since + for seq := since + 1; seq <= len(f.ids) && len(results) < limit; seq++ { + results = append(results, fmt.Sprintf(`{"seq":%d,"id":%q,"changes":[{"rev":"1-a"}]}`, seq, f.ids[seq-1])) + last = seq + } + _, _ = fmt.Fprintf(w, `{"results":[%s],"last_seq":%d}`, strings.Join(results, ","), last) +} + +func TestNPMFeed(t *testing.T) { + f := &fakeReplicate{ids: []string{"old"}} + srv := httptest.NewServer(f) + defer srv.Close() + feed := NewNPMFeed(srv.Client(), srv.URL+"/registry/_changes") + ctx := context.Background() + + if c, err := feed.Changes(ctx); err != nil || c.Names != nil { + t.Fatalf("first read = %+v, %v; want the start position only", c, err) + } + f.mu.Lock() + f.ids = append(f.ids, "lodash", "@scope/pkg", "_design/app") + f.mu.Unlock() + c, err := feed.Changes(ctx) + if err != nil { + t.Fatal(err) + } + if want := []string{"lodash", "@scope/pkg"}; !slices.Equal(c.Names, want) { + t.Errorf("changes = %v, want %v", c.Names, want) + } + if c, err := feed.Changes(ctx); err != nil || len(c.Names) != 0 { + t.Errorf("changes without news = %+v, %v", c, err) + } + if want := []string{"1", "4"}; !slices.Equal(f.asked, want) { + t.Errorf("asked since %q, want %q", f.asked, want) + } +} + +// TestNPMFeedPages checks that a backlog longer than a page is read page by +// page, and that one read stops after npmMaxPages. +func TestNPMFeedPages(t *testing.T) { + f := &fakeReplicate{limit: npmPageLimit} + srv := httptest.NewServer(f) + defer srv.Close() + feed := &NPMFeed{client: srv.Client(), url: srv.URL + "/registry/_changes", since: []byte("0")} + f.ids = make([]string, npmPageLimit*(npmMaxPages+1)+5) + for i := range f.ids { + f.ids[i] = "pkg-" + strconv.Itoa(i) + } + + c, err := feed.Changes(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(c.Names) != npmPageLimit*npmMaxPages { + t.Errorf("first read returned %d names, want %d", len(c.Names), npmPageLimit*npmMaxPages) + } + c, err = feed.Changes(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(c.Names) != npmPageLimit+5 { + t.Errorf("second read returned %d names, want %d", len(c.Names), npmPageLimit+5) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index a660699..1787dc2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -590,6 +590,12 @@ type UpstreamConfig struct { // wins. NPMRoutes map[string]string `json:"npm_routes" yaml:"npm_routes"` + // NPMChangeFeed follows the public npm registry's change feed and + // revalidates cached packuments of packages listed in it on their next + // request, rather than serving them until metadata_ttl runs out. + // Requires cache_metadata. Default: false. + NPMChangeFeed bool `json:"npm_change_feed" yaml:"npm_change_feed"` + // Cargo is the upstream cargo index URL. // Default: https://index.crates.io Cargo string `json:"cargo" yaml:"cargo"` @@ -682,6 +688,12 @@ type UpstreamConfig struct { // matching pattern wins. ComposerRoutes map[string]string `json:"composer_routes" yaml:"composer_routes"` + // ComposerChangeFeed follows the metadata change feed of + // upstream.composer and revalidates cached metadata of packages listed + // in it on their next request, rather than serving it until + // metadata_ttl runs out. Requires cache_metadata. Default: false. + ComposerChangeFeed bool `json:"composer_change_feed" yaml:"composer_change_feed"` + // Conan is the upstream Conan registry URL. // Default: https://center.conan.io Conan string `json:"conan" yaml:"conan"` @@ -969,6 +981,12 @@ func (c *Config) validateUpstream() error { if c.Upstream.GoSumDBsOffline && !c.CacheMetadata { return fmt.Errorf("upstream.go_sumdbs_offline requires cache_metadata") } + if c.Upstream.NPMChangeFeed && !c.CacheMetadata { + return fmt.Errorf("upstream.npm_change_feed requires cache_metadata") + } + if c.Upstream.ComposerChangeFeed && !c.CacheMetadata { + return fmt.Errorf("upstream.composer_change_feed requires cache_metadata") + } return nil } @@ -1407,6 +1425,7 @@ func (c *Config) LoadFromEnv() { setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK") setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM") setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA") + setEnvBool(&c.Upstream.NPMChangeFeed, "PROXY_UPSTREAM_NPM_CHANGE_FEED") setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE") setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO") setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD") @@ -1423,6 +1442,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.NuGetSearch, "PROXY_UPSTREAM_NUGET_SEARCH") setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER") setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY") + setEnvBool(&c.Upstream.ComposerChangeFeed, "PROXY_UPSTREAM_COMPOSER_CHANGE_FEED") setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN") setEnvString(&c.Upstream.Chef, "PROXY_UPSTREAM_CHEF") setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7590a64..0730e01 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1137,6 +1137,17 @@ func TestLoadMetadataTTLFromEnv(t *testing.T) { } } +func TestLoadChangeFeedsFromEnv(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_UPSTREAM_NPM_CHANGE_FEED", "true") + t.Setenv("PROXY_UPSTREAM_COMPOSER_CHANGE_FEED", "true") + cfg.LoadFromEnv() + + if !cfg.Upstream.NPMChangeFeed || !cfg.Upstream.ComposerChangeFeed { + t.Errorf("change feeds = npm %t, composer %t, want both on", cfg.Upstream.NPMChangeFeed, cfg.Upstream.ComposerChangeFeed) + } +} + func TestParseGradleBuildCacheConfig(t *testing.T) { cfg := Default() @@ -1725,6 +1736,30 @@ func TestValidateGoSumDBs(t *testing.T) { cfg.Upstream.GoSumDBsOffline = true }, }, + { + name: "npm change feed requires metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = false + cfg.Upstream.NPMChangeFeed = true + }, + wantErr: true, + }, + { + name: "Composer change feed requires metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = false + cfg.Upstream.ComposerChangeFeed = true + }, + wantErr: true, + }, + { + name: "change feeds with metadata caching", + modify: func(cfg *Config) { + cfg.CacheMetadata = true + cfg.Upstream.NPMChangeFeed = true + cfg.Upstream.ComposerChangeFeed = true + }, + }, } for _, tt := range tests { diff --git a/internal/database/metadata_cache_test.go b/internal/database/metadata_cache_test.go index 0e19f6c..ee16c25 100644 --- a/internal/database/metadata_cache_test.go +++ b/internal/database/metadata_cache_test.go @@ -2,7 +2,9 @@ package database import ( "database/sql" + "fmt" "path/filepath" + "slices" "testing" "time" ) @@ -448,3 +450,91 @@ func TestRefreshMetadataCache(t *testing.T) { t.Errorf("refresh of a missing row = %v, %v, want false", refreshed, err) } } + +// TestExpireMetadataCache checks that expiring clears fetched_at only on the +// named rows of the ecosystem fetched before the cutoff, and keeps their +// documents and validators. +func TestExpireMetadataCache(t *testing.T) { + db := setupMetadataCacheDB(t) + cutoff := time.Now().UTC().Truncate(time.Second) + rows := []struct { + ecosystem, name string + fetched time.Time + }{ + {"composer", "acme/lib", cutoff.Add(-time.Minute)}, + {"composer", "acme/lib~dev", cutoff.Add(-time.Minute)}, + {"composer", "acme/new", cutoff.Add(time.Second)}, + {"composer", "acme/other", cutoff.Add(-time.Minute)}, + {"composer", "routes/abc/acme/lib", cutoff.Add(-time.Minute)}, + {testEcosystemNPM, "acme/lib", cutoff.Add(-time.Minute)}, + } + for _, r := range rows { + err := db.UpsertMetadataCache(&MetadataCacheEntry{ + Ecosystem: r.ecosystem, + Name: r.name, + StoragePath: "_metadata/" + r.ecosystem + "/" + r.name, + ETag: sql.NullString{String: `"abc"`, Valid: true}, + ContentDigest: sql.NullString{String: "sha256:aaaa", Valid: true}, + FetchedAt: sql.NullTime{Time: r.fetched, Valid: true}, + }) + if err != nil { + t.Fatal(err) + } + } + + expired, err := db.ExpireMetadataCache("composer", []string{"acme/lib", "acme/lib~dev", "acme/new", "acme/missing"}, cutoff) + if err != nil { + t.Fatal(err) + } + slices.Sort(expired) + if want := []string{"acme/lib", "acme/lib~dev"}; !slices.Equal(expired, want) { + t.Errorf("expired = %v, want %v", expired, want) + } + got, err := db.GetMetadataCache("composer", "acme/lib") + if err != nil { + t.Fatal(err) + } + if got.FetchedAt.Valid || got.ETag.String != `"abc"` || got.ContentDigest.String != "sha256:aaaa" { + t.Errorf("expired row = %+v, want no fetched_at and the same validators", got) + } + for _, r := range []struct{ ecosystem, name string }{{"composer", "acme/new"}, {"composer", "acme/other"}, {testEcosystemNPM, "acme/lib"}} { + if got, _ := db.GetMetadataCache(r.ecosystem, r.name); !got.FetchedAt.Valid { + t.Errorf("%s %s expired, want it left alone", r.ecosystem, r.name) + } + } + + expired, err = db.ExpireAllMetadataCache("composer", cutoff) + if err != nil { + t.Fatal(err) + } + if want := []string{"acme/other"}; !slices.Equal(expired, want) { + t.Errorf("expired all = %v, want %v (routed and newer rows kept)", expired, want) + } +} + +// TestExpireMetadataCacheBatches checks that more names than one statement +// takes are all expired. +func TestExpireMetadataCacheBatches(t *testing.T) { + db := setupMetadataCacheDB(t) + cutoff := time.Now().UTC() + names := make([]string, expireMetadataBatch+2) + for i := range names { + names[i] = fmt.Sprintf("pkg-%d", i) + } + for _, name := range []string{names[0], names[len(names)-1]} { + err := db.UpsertMetadataCache(&MetadataCacheEntry{ + Ecosystem: testEcosystemNPM, Name: name, StoragePath: "_metadata/npm/" + name, + FetchedAt: sql.NullTime{Time: cutoff.Add(-time.Minute), Valid: true}, + }) + if err != nil { + t.Fatal(err) + } + } + expired, err := db.ExpireMetadataCache(testEcosystemNPM, names, cutoff) + if err != nil { + t.Fatal(err) + } + if len(expired) != 2 { + t.Errorf("expired = %v, want both stored names", expired) + } +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e8..d97ef25 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -4,7 +4,9 @@ import ( "database/sql" "errors" "fmt" + "slices" "sort" + "strings" "time" "github.com/git-pkgs/artifacts" @@ -1357,6 +1359,50 @@ func (db *DB) ForgetMetadataDocument(ecosystem, name, from, to string) error { return err } +// expireMetadataBatch is how many names one ExpireMetadataCache statement +// covers, well below the bind parameter limits of SQLite and Postgres. +const expireMetadataBatch = 500 + +// ExpireMetadataCache clears fetched_at on the metadata cache rows for +// ecosystem and names that were fetched before cutoff, and returns the names +// of the rows it changed. The rows keep their documents and validators, so +// the next lookup of each revalidates it upstream instead of serving it as +// fresh. It is for an upstream that reports which documents changed. +func (db *DB) ExpireMetadataCache(ecosystem string, names []string, cutoff time.Time) ([]string, error) { + var expired []string + for batch := range slices.Chunk(names, expireMetadataBatch) { + args := []any{time.Now(), ecosystem, cutoff.UTC()} + for _, name := range batch { + args = append(args, name) + } + var got []string + err := db.DB.Select(&got, db.Rebind(` + UPDATE metadata_cache SET fetched_at = NULL, updated_at = ? + WHERE ecosystem = ? AND fetched_at < ? + AND name IN (`+strings.TrimSuffix(strings.Repeat("?, ", len(batch)), ", ")+`) + RETURNING name + `), args...) + if err != nil { + return expired, err + } + expired = append(expired, got...) + } + return expired, nil +} + +// ExpireAllMetadataCache is ExpireMetadataCache for every row of ecosystem, +// except those of packages routed to another registry (named "routes/..."), +// for an upstream that can't say which of its documents changed. +func (db *DB) ExpireAllMetadataCache(ecosystem string, cutoff time.Time) ([]string, error) { + var expired []string + err := db.DB.Select(&expired, db.Rebind(` + UPDATE metadata_cache SET fetched_at = NULL, updated_at = ? + WHERE ecosystem = ? AND fetched_at < ? AND name NOT LIKE 'routes/%' + RETURNING name + `), time.Now(), ecosystem, cutoff.UTC()) + return expired, err +} + // UpsertMetadataCache writes entry and marks it accessed now, since a // document is stored because a request asked for it. func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 708b5b3..984a607 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -86,6 +86,14 @@ var ( []string{"ecosystem"}, ) + MetadataExpiredByChangeFeed = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_metadata_expired_by_change_feed_total", + Help: "Cached metadata documents set to revalidate because an upstream change feed listed them, by ecosystem", + }, + []string{"ecosystem"}, + ) + FetchLeaseWaits = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "proxy_fetch_lease_waits_total", @@ -311,6 +319,7 @@ func init() { CachedArtifacts, ArtifactsEvicted, MetadataEvicted, + MetadataExpiredByChangeFeed, FetchLeaseWaits, UpstreamFetchDuration, UpstreamErrors, @@ -409,6 +418,13 @@ func RecordMetadataEvicted(ecosystem string) { MetadataEvicted.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() } +// RecordMetadataExpiredByChangeFeed counts n metadata documents an upstream +// change feed listed as changed. ecosystem is the metadata cache's, +// normalized like the cache hit counter. +func RecordMetadataExpiredByChangeFeed(ecosystem string, n int) { + MetadataExpiredByChangeFeed.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Add(float64(n)) +} + // RecordFetchLeaseWait counts a cache miss that found another instance // sharing the database already fetching it. func RecordFetchLeaseWait(kind string) { diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go index 82e5ca2..02e67e2 100644 --- a/internal/server/analytics_coverage_test.go +++ b/internal/server/analytics_coverage_test.go @@ -34,25 +34,26 @@ var metricSurface = map[string]string{ "proxy_client_response_bytes_total": "Runtime: Request sources -- By client", // Registry-derived, shown in the Runtime card. - "proxy_requests_total": "Runtime: Requests + Responses by status", - "proxy_request_duration_seconds": "Runtime: Mean latency", - "proxy_active_requests": "Runtime: In flight", - "proxy_cache_hits_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", - "proxy_cache_misses_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", - "proxy_metadata_cache_served_bytes_total": "Runtime: Metadata cache lookups -- served from cache", - "proxy_upstream_fetch_duration_seconds": "Runtime: Upstream fetches + mean fetch", - "proxy_upstream_errors_total": "Runtime: Upstream errors", - "proxy_storage_operation_duration_seconds": "Runtime: Storage operations", - "proxy_storage_errors_total": "Runtime: Storage errors", - "proxy_integrity_failures_total": "Runtime: Integrity failures", - "proxy_health_probe_failures_total": "Runtime: Health probe failures", - "proxy_circuit_breaker_state": "Runtime: Circuit breakers", - "proxy_circuit_breaker_trips_total": "Runtime: Circuit breaker trips", - "proxy_artifacts_evicted_total": "Runtime: Evicted artifacts", - "proxy_metadata_evicted_total": "Runtime: Evicted metadata", - "proxy_scan_duration_seconds": "Runtime: Pre-cache scanning -- Scans", - "proxy_scan_blocked_total": "Runtime: Pre-cache scanning -- Artifacts blocked", - "proxy_scan_errors_total": "Runtime: Pre-cache scanning -- Scan errors", + "proxy_requests_total": "Runtime: Requests + Responses by status", + "proxy_request_duration_seconds": "Runtime: Mean latency", + "proxy_active_requests": "Runtime: In flight", + "proxy_cache_hits_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", + "proxy_cache_misses_total": "Runtime: Artifact and metadata cache lookups + artifact hit rate", + "proxy_metadata_cache_served_bytes_total": "Runtime: Metadata cache lookups -- served from cache", + "proxy_upstream_fetch_duration_seconds": "Runtime: Upstream fetches + mean fetch", + "proxy_upstream_errors_total": "Runtime: Upstream errors", + "proxy_storage_operation_duration_seconds": "Runtime: Storage operations", + "proxy_storage_errors_total": "Runtime: Storage errors", + "proxy_integrity_failures_total": "Runtime: Integrity failures", + "proxy_health_probe_failures_total": "Runtime: Health probe failures", + "proxy_circuit_breaker_state": "Runtime: Circuit breakers", + "proxy_circuit_breaker_trips_total": "Runtime: Circuit breaker trips", + "proxy_artifacts_evicted_total": "Runtime: Evicted artifacts", + "proxy_metadata_evicted_total": "Runtime: Evicted metadata", + "proxy_metadata_expired_by_change_feed_total": "Runtime: Metadata changed upstream", + "proxy_scan_duration_seconds": "Runtime: Pre-cache scanning -- Scans", + "proxy_scan_blocked_total": "Runtime: Pre-cache scanning -- Artifacts blocked", + "proxy_scan_errors_total": "Runtime: Pre-cache scanning -- Scan errors", } // TestEveryMetricIsSurfaced fails when a metric is registered but has no home @@ -80,6 +81,7 @@ func TestEveryMetricIsSurfaced(t *testing.T) { metrics.UpdateCacheStats(1, 1) metrics.RecordArtifactEvicted("retention", "npm") metrics.RecordMetadataEvicted("npm") + metrics.RecordMetadataExpiredByChangeFeed("npm", 1) metrics.UpdateEcosystemStats([]metrics.EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 1}}) metrics.UpdateMetadataCacheStats([]metrics.MetadataCacheStats{{Ecosystem: "npm", Entries: 1, Size: 1}}) diff --git a/internal/server/analytics_runtime.go b/internal/server/analytics_runtime.go index 5773879..103e33e 100644 --- a/internal/server/analytics_runtime.go +++ b/internal/server/analytics_runtime.go @@ -35,6 +35,7 @@ type RuntimeView struct { ProbeFailures []LabelledCount Evictions []LabelledCount MetadataEvictions []LabelledCount + ChangeFeedExpired []LabelledCount Breakers []BreakerRow BreakerTrips []LabelledCount @@ -161,6 +162,7 @@ func runtimeView(snap *metrics.Snapshot, failureDetails bool) RuntimeView { v.ProbeFailures = failures("proxy_health_probe_failures_total", "step") v.Evictions = countRows(snap, "proxy_artifacts_evicted_total", "reason", "ecosystem") v.MetadataEvictions = countRows(snap, "proxy_metadata_evicted_total", "ecosystem") + v.ChangeFeedExpired = countRows(snap, "proxy_metadata_expired_by_change_feed_total", "ecosystem") v.Breakers = breakerRows(snap) v.BreakerTrips = failures("proxy_circuit_breaker_trips_total", "registry") diff --git a/internal/server/analytics_runtime_test.go b/internal/server/analytics_runtime_test.go index 66c154e..03bc236 100644 --- a/internal/server/analytics_runtime_test.go +++ b/internal/server/analytics_runtime_test.go @@ -170,6 +170,24 @@ func TestRuntimeViewShowsMetadataEvictions(t *testing.T) { } } +func TestRuntimeViewShowsChangeFeedExpiries(t *testing.T) { + reg := prometheus.NewRegistry() + expired := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_metadata_expired_by_change_feed_total", Help: "t"}, []string{"ecosystem"}) + reg.MustRegister(expired) + expired.WithLabelValues("packagist").Add(2) + + snap, err := metrics.GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + v := runtimeView(snap, false) + + if len(v.ChangeFeedExpired) != 1 || v.ChangeFeedExpired[0].Label != "packagist" || v.ChangeFeedExpired[0].Count != "2" { + t.Errorf("ChangeFeedExpired = %+v, want packagist = 2", v.ChangeFeedExpired) + } +} + // A counter sitting at zero carries no information and would bury the rows // that do, so it is left out. func TestRuntimeViewDropsZeroCounters(t *testing.T) { diff --git a/internal/server/change_feeds.go b/internal/server/change_feeds.go new file mode 100644 index 0000000..059bd2f --- /dev/null +++ b/internal/server/change_feeds.go @@ -0,0 +1,94 @@ +package server + +import ( + "context" + "log/slog" + "net/http" + "time" + + "github.com/git-pkgs/proxy/internal/changefeed" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" +) + +// changeFeedInterval is how often an upstream change feed is read. +const changeFeedInterval = time.Minute + +// changeFeedMaxBackoff caps the wait between reads of a failing feed. +const changeFeedMaxBackoff = 15 * time.Minute + +// startChangeFeeds follows the change feeds the configuration turns on. Each +// instance sharing a database follows them itself: the reads are small, and +// expiring a row twice does no harm. +func (s *Server) startChangeFeeds(ctx context.Context, client *http.Client, metadataTTL time.Duration) { + var feeds []changefeed.Feed + if s.cfg.Upstream.NPMChangeFeed { + feeds = append(feeds, changefeed.NewNPMFeed(client, changefeed.NPMFeedURL)) + } + if s.cfg.Upstream.ComposerChangeFeed { + feeds = append(feeds, changefeed.NewComposerFeed(client, changefeed.ComposerFeedURL(s.cfg.Upstream.Composer))) + } + if len(feeds) == 0 { + return + } + if !s.cfg.CacheMetadata || metadataTTL <= 0 { + // Every request already revalidates, so there is nothing to expire. + s.logger.Info("change feeds not followed: metadata is not served from cache without revalidating") + return + } + for _, feed := range feeds { + s.background.Go(func() { followChangeFeed(ctx, s.db, s.logger, feed, changeFeedInterval) }) + } +} + +// followChangeFeed reads feed every interval until ctx ends and expires the +// cached metadata it lists. After a failed read it waits twice as long as +// before, up to changeFeedMaxBackoff. +func followChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changefeed.Feed, interval time.Duration) { + wait := interval + for { + if err := readChangeFeed(ctx, db, logger, feed); err != nil { + if ctx.Err() != nil { + return + } + wait = min(wait+wait, changeFeedMaxBackoff) + logger.Warn("change feed: read failed", "ecosystem", feed.Ecosystem(), "error", err, "retry_in", wait) + } else { + wait = interval + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + } + } +} + +// readChangeFeed reads feed once and expires the cached metadata of what it +// lists, so the next request for it revalidates upstream. Only rows fetched +// before the read finished are expired: a row fetched after it was fetched +// after every change the read returned. If expiring fails, the feed has +// already moved past the changes, and those rows wait out metadata_ttl. +func readChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changefeed.Feed) error { + changes, err := feed.Changes(ctx) + if err != nil { + return err + } + cutoff := time.Now() + ecosystem := feed.Ecosystem() + var expired []string + switch { + case changes.All: + logger.Info("change feed: upstream asked for a full resync, revalidating all cached metadata", "ecosystem", ecosystem) + expired, err = db.ExpireAllMetadataCache(ecosystem, cutoff) + case len(changes.Names) > 0: + expired, err = db.ExpireMetadataCache(ecosystem, changes.Names, cutoff) + } + for _, name := range expired { + logger.Debug("change feed: cached metadata changed upstream", "ecosystem", ecosystem, "key", name) + } + metrics.RecordMetadataExpiredByChangeFeed(ecosystem, len(expired)) + return err +} diff --git a/internal/server/change_feeds_test.go b/internal/server/change_feeds_test.go new file mode 100644 index 0000000..90441ac --- /dev/null +++ b/internal/server/change_feeds_test.go @@ -0,0 +1,272 @@ +package server + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/changefeed" + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" + "github.com/go-chi/chi/v5" +) + +// changingRegistry serves npm packuments, Composer metadata, and both +// registries' change feeds, all from one set of documents that a test can +// change. The feeds list every document changed since the previous read. +type changingRegistry struct { + *httptest.Server + mu sync.Mutex + docs map[string]string // path -> body + changed []string // paths changed since the feed was last read + feedSeq int + fetches map[string]int +} + +func newChangingRegistry(t *testing.T) *changingRegistry { + t.Helper() + reg := &changingRegistry{docs: map[string]string{}, fetches: map[string]int{}} + reg.Server = httptest.NewServer(http.HandlerFunc(reg.serve)) + t.Cleanup(reg.Close) + return reg +} + +func (reg *changingRegistry) set(path, body string) { + reg.mu.Lock() + defer reg.mu.Unlock() + if _, ok := reg.docs[path]; ok { + reg.changed = append(reg.changed, path) + } + reg.docs[path] = body +} + +func (reg *changingRegistry) fetched(path string) int { + reg.mu.Lock() + defer reg.mu.Unlock() + return reg.fetches[path] +} + +// takeChanged returns the changed paths with prefix and forgets them. +func (reg *changingRegistry) takeChanged(prefix string) []string { + var taken, kept []string + for _, p := range reg.changed { + if name, ok := strings.CutPrefix(p, prefix); ok { + taken = append(taken, name) + } else { + kept = append(kept, p) + } + } + reg.changed = kept + return taken +} + +func (reg *changingRegistry) serve(w http.ResponseWriter, r *http.Request) { + reg.mu.Lock() + defer reg.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/metadata/changes.json": + if r.URL.Query().Get("since") == "" { + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"error":"missing since","timestamp":1}`) + return + } + var actions []string + for _, name := range reg.takeChanged("/p2/") { + actions = append(actions, fmt.Sprintf(`{"type":"update","package":%q,"time":1}`, strings.TrimSuffix(name, ".json"))) + } + _, _ = fmt.Fprintf(w, `{"actions":[%s],"timestamp":2}`, strings.Join(actions, ",")) + return + case "/replicate/registry/": + _, _ = fmt.Fprintf(w, `{"update_seq":%d}`, reg.feedSeq) + return + case "/replicate/registry/_changes": + var results []string + for _, name := range reg.takeChanged("/npm/") { + reg.feedSeq++ + results = append(results, fmt.Sprintf(`{"seq":%d,"id":%q}`, reg.feedSeq, name)) + } + _, _ = fmt.Fprintf(w, `{"results":[%s],"last_seq":%d}`, strings.Join(results, ","), reg.feedSeq) + return + } + reg.fetches[r.URL.Path]++ + body, ok := reg.docs[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + etag := fmt.Sprintf(`"%x"`, sha256.Sum256([]byte(body))) + if r.Header.Get("If-None-Match") == etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", etag) + _, _ = io.WriteString(w, body) +} + +func composerDoc(versions ...string) string { + var entries []string + for _, v := range versions { + entries = append(entries, fmt.Sprintf(`{"name":"acme/lib","version":%q,"version_normalized":"%s.0","dist":{"type":"zip","url":"https://example.test/acme/lib/%s.zip","reference":"r%s"}}`, v, v, v, v)) + } + return `{"packages":{"acme/lib":[` + strings.Join(entries, ",") + `]},"minified":"composer/2.0"}` +} + +func npmDoc(versions ...string) string { + var entries []string + for _, v := range versions { + entries = append(entries, fmt.Sprintf(`%q:{"name":"left-pad","version":%q,"dist":{"tarball":"https://registry.example.test/left-pad/-/left-pad-%s.tgz"}}`, v, v, v)) + } + return `{"name":"left-pad","dist-tags":{"latest":"` + versions[len(versions)-1] + `"},"versions":{` + strings.Join(entries, ",") + `}}` +} + +// changeFeedRouter mounts the npm and Composer handlers on reg, with +// metadata cached for an hour. +func changeFeedRouter(t *testing.T, reg *changingRegistry) (http.Handler, *database.DB) { + t.Helper() + db, err := database.Create(t.TempDir() + "/test.db") + if err != nil { + t.Fatalf("creating database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := storage.OpenBucket(context.Background(), "file://"+t.TempDir()) + if err != nil { + t.Fatalf("opening storage: %v", err) + } + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(reg.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), slog.New(slog.NewTextHandler(io.Discard, nil))) + proxy.HTTPClient = reg.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(1 << 20) + + cfg := config.Default() + cfg.BaseURL = "http://proxy.example" + cfg.Upstream.NPM = reg.URL + "/npm" + cfg.Upstream.Composer = reg.URL + cfg.Upstream.ComposerRepository = reg.URL + s := &Server{cfg: cfg} + r := chi.NewRouter() + s.mountProtocolHandlers(r, proxy) + return r, db +} + +func getBody(t *testing.T, h http.Handler, path string) string { + t.Helper() + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusOK { + t.Fatalf("GET %s = %d: %s", path, w.Code, w.Body.String()) + } + return w.Body.String() +} + +// TestChangeFeedRevalidatesChangedMetadata checks the whole path: cached +// metadata is served within its TTL even after upstream changed, until the +// change feed lists the package, after which the next request revalidates +// and serves the new release. Packages the feed doesn't list stay cached. +func TestChangeFeedRevalidatesChangedMetadata(t *testing.T) { + reg := newChangingRegistry(t) + reg.set("/p2/acme/lib.json", composerDoc("1.0.0")) + reg.set("/p2/acme/other.json", `{"packages":{"acme/other":[]},"minified":"composer/2.0"}`) + reg.set("/npm/left-pad", npmDoc("1.0.0")) + h, db := changeFeedRouter(t, reg) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + feeds := []changefeed.Feed{ + changefeed.NewComposerFeed(reg.Client(), changefeed.ComposerFeedURL(reg.URL)), + changefeed.NewNPMFeed(reg.Client(), reg.URL+"/replicate/registry/_changes"), + } + for _, feed := range feeds { + if err := readChangeFeed(context.Background(), db, logger, feed); err != nil { + t.Fatalf("starting %s feed: %v", feed.Ecosystem(), err) + } + } + + getBody(t, h, "/composer/p2/acme/lib.json") + getBody(t, h, "/composer/p2/acme/other.json") + getBody(t, h, "/npm/left-pad") + + reg.set("/p2/acme/lib.json", composerDoc("1.0.0", "1.1.0")) + reg.set("/npm/left-pad", npmDoc("1.0.0", "1.1.0")) + if body := getBody(t, h, "/composer/p2/acme/lib.json"); strings.Contains(body, "1.1.0") { + t.Fatal("composer metadata revalidated before the feed listed it; the test proves nothing") + } + if body := getBody(t, h, "/npm/left-pad"); strings.Contains(body, "1.1.0") { + t.Fatal("npm packument revalidated before the feed listed it; the test proves nothing") + } + + for _, feed := range feeds { + if err := readChangeFeed(context.Background(), db, logger, feed); err != nil { + t.Fatalf("reading %s feed: %v", feed.Ecosystem(), err) + } + } + + if body := getBody(t, h, "/composer/p2/acme/lib.json"); !strings.Contains(body, "1.1.0") { + t.Errorf("composer metadata after the feed listed it = %s, want 1.1.0", body) + } + if body := getBody(t, h, "/npm/left-pad"); !strings.Contains(body, "1.1.0") { + t.Errorf("npm packument after the feed listed it = %s, want 1.1.0", body) + } + getBody(t, h, "/composer/p2/acme/other.json") + if n := reg.fetched("/p2/acme/other.json"); n != 1 { + t.Errorf("unlisted package fetched %d times, want once", n) + } + if n := reg.fetched("/p2/acme/lib.json"); n != 2 { + t.Errorf("listed package fetched %d times, want twice", n) + } +} + +// countingFeed fails its first reads and then reports no changes. +type countingFeed struct { + reads atomic.Int32 + failures int32 +} + +func (f *countingFeed) Ecosystem() string { return "npm" } + +func (f *countingFeed) Changes(context.Context) (changefeed.Changes, error) { + if f.reads.Add(1) <= f.failures { + return changefeed.Changes{}, errors.New("feed down") + } + return changefeed.Changes{}, nil +} + +// TestFollowChangeFeedStopsWithContext checks that a failing feed is retried +// rather than given up on, and that following stops when ctx ends. +func TestFollowChangeFeedStopsWithContext(t *testing.T) { + feed := &countingFeed{failures: 2} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + followChangeFeed(ctx, nil, slog.New(slog.NewTextHandler(io.Discard, nil)), feed, time.Millisecond) + }() + deadline := time.After(5 * time.Second) + for feed.reads.Load() < 4 { + select { + case <-deadline: + t.Fatalf("feed read %d times, want it read again after failing", feed.reads.Load()) + case <-time.After(time.Millisecond): + } + } + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("followChangeFeed did not return after ctx ended") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 12619a6..8d3a746 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -397,6 +397,7 @@ func (s *Server) serve(listener net.Listener) error { }) } s.startMirrorSchedules(bgCtx, proxy) + s.startChangeFeeds(bgCtx, proxy.HTTPClient, proxy.MetadataTTL) // Hosted package endpoints (opt-in via hosted.enabled or PROXY_HOSTED_ENABLED) if s.cfg.Hosted.Enabled { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 292643c..06f6b23 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -633,13 +633,6 @@ func TestSwiftCachedVersionPURLUsesStoredPackagePURL(t *testing.T) { } } -func min(a, b int) int { - if a < b { - return a - } - return b -} - func TestNPMPackageMetadata(t *testing.T) { ts := newTestServer(t) defer ts.close() diff --git a/internal/server/templates/components/runtime_metrics.html b/internal/server/templates/components/runtime_metrics.html index 18a3140..b5427b7 100644 --- a/internal/server/templates/components/runtime_metrics.html +++ b/internal/server/templates/components/runtime_metrics.html @@ -60,6 +60,7 @@ {{template "runtime_counts" dict "Title" "Health probe failures" "Rows" .ProbeFailures "Empty" "None"}} {{template "runtime_counts" dict "Title" "Evicted artifacts" "Rows" .Evictions "Empty" "None"}} {{template "runtime_counts" dict "Title" "Evicted metadata" "Rows" .MetadataEvictions "Empty" "None"}} + {{template "runtime_counts" dict "Title" "Metadata changed upstream" "Rows" .ChangeFeedExpired "Empty" "None"}} {{template "runtime_counts" dict "Title" "Circuit breaker trips" "Rows" .BreakerTrips "Empty" "None"}}
From 0bc0e903f872f5b49328811e6d8f30c20820dcd2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 19:30:34 +0100 Subject: [PATCH 004/198] Add a project inventory built from uploaded SBOMs Repositories upload an SBOM of their default branch, for example from syft in CI, to PUT /api/projects/{project}/sbom. The proxy records the package versions it lists, keyed by the same unversioned PURL the handlers record packages under, so the inventory joins with what the proxy has cached and the vulnerabilities it holds. That answers which repository uses which version of a package: the new /ui/projects pages list each project's package versions with outdated and vulnerability markers, package pages show the projects per version, the dashboard lists vulnerable versions in use, and GET /api/projects?purl= does the same for scripts. The feature is off by default. Uploads and deletes need projects.token; an upload replaces the project's components in one transaction. projects.max_age removes projects that stopped uploading. --- README.md | 12 + cmd/proxy/main.go | 5 +- config.example.yaml | 9 + docs/architecture.md | 4 + docs/configuration.md | 82 ++++ docs/swagger/docs.go | 288 +++++++++++++- docs/swagger/swagger.json | 288 +++++++++++++- go.mod | 2 +- internal/config/config.go | 10 + internal/config/projects.go | 53 +++ internal/config/projects_test.go | 69 ++++ internal/database/database_test.go | 2 +- internal/database/projects.go | 257 +++++++++++++ internal/database/projects_test.go | 228 +++++++++++ internal/database/schema.go | 76 ++++ internal/inventory/inventory.go | 185 +++++++++ internal/inventory/inventory_test.go | 133 +++++++ .../inventory/testdata/syft-cyclonedx.json | 1 + internal/server/dashboard.go | 6 + internal/server/layout.go | 3 + internal/server/projects_api.go | 335 +++++++++++++++++ internal/server/projects_api_test.go | 354 ++++++++++++++++++ internal/server/projects_prune.go | 43 +++ internal/server/projects_prune_test.go | 48 +++ internal/server/projects_ui.go | 287 ++++++++++++++ internal/server/projects_ui_test.go | 137 +++++++ internal/server/server.go | 14 + .../templates/components/severity_badge.html | 3 + internal/server/templates/layout/header.html | 1 + .../server/templates/pages/dashboard.html | 27 ++ .../server/templates/pages/package_show.html | 28 ++ .../server/templates/pages/project_show.html | 53 +++ .../server/templates/pages/projects_list.html | 33 ++ 33 files changed, 3071 insertions(+), 5 deletions(-) create mode 100644 internal/config/projects.go create mode 100644 internal/config/projects_test.go create mode 100644 internal/database/projects.go create mode 100644 internal/database/projects_test.go create mode 100644 internal/inventory/inventory.go create mode 100644 internal/inventory/inventory_test.go create mode 100644 internal/inventory/testdata/syft-cyclonedx.json create mode 100644 internal/server/projects_api.go create mode 100644 internal/server/projects_api_test.go create mode 100644 internal/server/projects_prune.go create mode 100644 internal/server/projects_prune_test.go create mode 100644 internal/server/projects_ui.go create mode 100644 internal/server/projects_ui_test.go create mode 100644 internal/server/templates/components/severity_badge.html create mode 100644 internal/server/templates/pages/project_show.html create mode 100644 internal/server/templates/pages/projects_list.html diff --git a/README.md b/README.md index 20c6d5d..61edf9d 100644 --- a/README.md +++ b/README.md @@ -1300,6 +1300,17 @@ Registered when `hosted.enabled` is set. Pushing and deleting require `Authoriza | `GET /api/hosted/{ecosystem}/{name}` | List the archives pushed for a package | | `DELETE /api/hosted/{ecosystem}/{name}/{version}` | Delete a pushed version | +### Projects API + +Registered when `projects.enabled` is set. Uploading and deleting require `Authorization: Bearer `. See [Project inventory](docs/configuration.md#project-inventory). + +| Endpoint | Description | +|----------|-------------| +| `PUT /api/projects/{project}/sbom` | Replace a project's recorded package versions with the ones the SBOM in the body lists | +| `GET /api/projects` | List projects, or with `?purl=pkg:npm/lodash@4.17.20` the projects using a package or version | +| `GET /api/projects/{project}` | A project's recorded package versions | +| `DELETE /api/projects/{project}` | Delete a project | + ### Cache API Registered when `cache_delete` is set. It has no authentication of its own. @@ -1503,6 +1514,7 @@ The proxy serves a web UI under `/ui`. No separate frontend build is needed -- t - **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts. - **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews. - **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files. +- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated or vulnerable. Package pages then show which projects use each version, and the dashboard lists vulnerable versions in use. ## Monitoring diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index b19f689..728a579 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -8,7 +8,7 @@ // @securityDefinitions.apikey BearerAuth // @in header // @name Authorization -// @description "Bearer " followed by hosted.token +// @description "Bearer " followed by hosted.token, or projects.token for /api/projects // // The proxy caches package artifacts from upstream registries (npm, cargo, etc.) // providing faster, more reliable access for development teams. @@ -93,6 +93,9 @@ // PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB") // PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only // PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only +// PROXY_PROJECTS_ENABLED - Enable the project inventory (/api/projects, /ui/projects) +// PROXY_PROJECTS_TOKEN - Bearer token for uploading and deleting project SBOMs +// PROXY_PROJECTS_MAX_AGE - Remove projects not uploaded for this long (e.g. "90d") // PROXY_URL - Proxy URL for proxy hosted // // Example: diff --git a/config.example.yaml b/config.example.yaml index d3b6da4..a875322 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -442,6 +442,15 @@ gradle: # npm_scopes: ["@acme"] # composer_vendors: ["acme"] +# Project inventory (disabled by default). Repositories upload an SBOM to +# PUT /api/projects/{project}/sbom, and the UI shows which project uses +# which package versions. Project names are visible on the unauthenticated UI. +# projects: +# enabled: true +# token: "${PROXY_PROJECTS_TOKEN}" +# # Drop projects that have not uploaded for this long ("" keeps them) +# max_age: "90d" + # Re-mirror container image tags on a schedule, so a moved tag is picked up # even when nobody pulls it. An unchanged tag costs one HEAD request. # mirror: diff --git a/docs/architecture.md b/docs/architecture.md index 660d810..0105d3f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -363,6 +363,10 @@ Selective package mirroring for pre-populating the proxy cache. Supports multipl The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. +### `internal/inventory` + +Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated and vulnerability markers on the project pages. + ### `internal/config` Configuration loading. diff --git a/docs/configuration.md b/docs/configuration.md index f10e4b2..c5c6f7a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1504,6 +1504,88 @@ Generic files never come from an upstream: anything not pushed answers 404. Whil Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones. +## Project inventory + +The project inventory answers "which repository uses which version of a package?". Each repository uploads an SBOM of its default branch, and the proxy records the package versions it lists. The UI then shows: + +- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version or vulnerabilities for that version. +- On a package page: which projects use each version, including versions the proxy hasn't cached. +- On the dashboard: vulnerable versions in use and the projects using them. + +It is disabled by default. Uploads and deletes require a token: + +```yaml +projects: + enabled: true + token: "${PROXY_PROJECTS_TOKEN}" + max_age: "90d" +``` + +| Config | Environment | Description | +|--------|-------------|-------------| +| `projects.enabled` | `PROXY_PROJECTS_ENABLED` | Register the `/api/projects` endpoints and the UI pages (default `false`) | +| `projects.token` | `PROXY_PROJECTS_TOKEN` | Bearer token required to upload and delete; required when enabled, supports `${VAR}` | +| `projects.max_age` | `PROXY_PROJECTS_MAX_AGE` | Remove projects that haven't uploaded for this long, e.g. `90d` or `720h`; empty or `0` keeps them (default) | + +Removing old projects runs hourly. With several instances on one database, one of them does it. + +### Uploading an SBOM + +`PUT /api/projects/{project}/sbom` takes a CycloneDX (JSON or XML) or SPDX (JSON or tag-value) document of up to 32 MiB and replaces everything recorded for the project with what it lists. The optional `ref` and `branch` query parameters are stored and shown with the project. + +The project name is a path such as `group/sub/repo`: segments of letters, digits, `.`, `_` and `-`, at most 255 characters. It can be sent as path segments (`/api/projects/group/sub/repo/sbom`) or URL-encoded as one segment (`/api/projects/group%2Fsub%2Frepo/sbom`). + +```bash +syft dir:. -o cyclonedx-json > sbom.json +curl -X PUT --data-binary @sbom.json \ + -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ + "https://proxy.example.com/api/projects/acme/shop/web/sbom?ref=$(git rev-parse HEAD)&branch=main" +``` + +The response counts the recorded package versions and the skipped entries: + +```json +{"project":"acme/shop/web","components":10,"skipped":5} +``` + +Entries are recorded by their package URL. Entries without one (files, the operating system) and package types the proxy doesn't serve (such as `pkg:github`) are skipped. Duplicates are recorded once. + +A GitLab CI job on the default branch could look like this: + +```yaml +sbom: + image: alpine:3.20 + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + script: + - apk add --no-cache curl + - curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin + - syft dir:. -o cyclonedx-json > sbom.json + - > + curl -sSf -X PUT --data-binary @sbom.json + -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" + "https://proxy.example.com/api/projects/$CI_PROJECT_PATH/sbom?ref=$CI_COMMIT_SHA&branch=$CI_COMMIT_BRANCH" +``` + +### Querying + +```bash +curl https://proxy.example.com/api/projects # all projects +curl https://proxy.example.com/api/projects/acme/shop/web # one project's package versions +curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash' # projects using lodash +curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash@4.17.20' # projects using lodash 4.17.20 +curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ + https://proxy.example.com/api/projects/acme/shop/web +``` + +Reading needs no token, like the rest of `/api`. + +### What it shows + +The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. Outdated and vulnerability markers use what the proxy knows about a package: its recorded latest version and the vulnerabilities recorded for it, matched against the version a project uses by the vulnerability's fixed version. A package the proxy has no record of has neither. + +Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy. + ## Mirror API The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP: diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index e35a86f..7ddf630 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -460,6 +460,192 @@ const docTemplate = `{ } } }, + "/api/projects": { + "get": { + "description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "List projects", + "parameters": [ + { + "type": "string", + "description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20", + "name": "purl", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}": { + "get": { + "description": "Returns the project with the package versions recorded for it.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Get a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the project and the package versions recorded for it.", + "tags": [ + "projects" + ], + "summary": "Delete a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}/sbom": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Upload a project's SBOM", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Commit the SBOM was generated from", + "name": "ref", + "in": "query" + }, + { + "type": "string", + "description": "Branch the SBOM was generated from", + "name": "branch", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectUploadResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/search": { "get": { "produces": [ @@ -1120,6 +1306,106 @@ const docTemplate = `{ } } }, + "server.ProjectComponent": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + }, + "purl": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "server.ProjectListResponse": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectSummary" + } + } + } + }, + "server.ProjectResponse": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectComponent" + } + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectSummary": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectUploadResponse": { + "type": "object", + "properties": { + "components": { + "description": "Components is the number of package versions recorded.", + "type": "integer" + }, + "project": { + "type": "string" + }, + "skipped": { + "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "type": "integer" + } + } + }, "server.SearchPackageResult": { "type": "object", "properties": { @@ -1206,7 +1492,7 @@ const docTemplate = `{ }, "securityDefinitions": { "BearerAuth": { - "description": "\"Bearer \" followed by hosted.token", + "description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects", "type": "apiKey", "name": "Authorization", "in": "header" diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 09e4f11..eb6f3b9 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -453,6 +453,192 @@ } } }, + "/api/projects": { + "get": { + "description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "List projects", + "parameters": [ + { + "type": "string", + "description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20", + "name": "purl", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}": { + "get": { + "description": "Returns the project with the package versions recorded for it.", + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Get a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + }, + "delete": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Removes the project and the package versions recorded for it.", + "tags": [ + "projects" + ], + "summary": "Delete a project", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, + "/api/projects/{project}/sbom": { + "put": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "projects" + ], + "summary": "Upload a project's SBOM", + "parameters": [ + { + "type": "string", + "description": "Project path, e.g. group/sub/repo", + "name": "project", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Commit the SBOM was generated from", + "name": "ref", + "in": "query" + }, + { + "type": "string", + "description": "Branch the SBOM was generated from", + "name": "branch", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ProjectUploadResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/search": { "get": { "produces": [ @@ -1113,6 +1299,106 @@ } } }, + "server.ProjectComponent": { + "type": "object", + "properties": { + "ecosystem": { + "type": "string" + }, + "name": { + "type": "string" + }, + "purl": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "server.ProjectListResponse": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectSummary" + } + } + } + }, + "server.ProjectResponse": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "components": { + "type": "array", + "items": { + "$ref": "#/definitions/server.ProjectComponent" + } + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectSummary": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "component_count": { + "description": "ComponentCount is the number of package versions recorded.", + "type": "integer" + }, + "name": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "uploaded_at": { + "type": "string" + }, + "version": { + "description": "Version is set when listing the projects using a package: the version\nof it this project uses.", + "type": "string" + } + } + }, + "server.ProjectUploadResponse": { + "type": "object", + "properties": { + "components": { + "description": "Components is the number of package versions recorded.", + "type": "integer" + }, + "project": { + "type": "string" + }, + "skipped": { + "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "type": "integer" + } + } + }, "server.SearchPackageResult": { "type": "object", "properties": { @@ -1199,7 +1485,7 @@ }, "securityDefinitions": { "BearerAuth": { - "description": "\"Bearer \" followed by hosted.token", + "description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects", "type": "apiKey", "name": "Authorization", "in": "header" diff --git a/go.mod b/go.mod index 94fbe6c..2213d68 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/git-pkgs/registries v0.11.1 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 + github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 github.com/klauspost/compress v1.20.1 @@ -76,7 +77,6 @@ require ( github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/pom v0.1.8 // indirect - github.com/git-pkgs/vers v0.7.2 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index a660699..42a4b9e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -211,6 +211,9 @@ type Config struct { // Hosted configures pushing packages to the proxy (/api/hosted). Hosted HostedConfig `json:"hosted" yaml:"hosted"` + // Projects configures the project inventory (/api/projects). + Projects ProjectsConfig `json:"projects" yaml:"projects"` + // Gradle configures Gradle HttpBuildCache behavior. Gradle GradleConfig `json:"gradle" yaml:"gradle"` @@ -1448,6 +1451,9 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES") setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS") + setEnvBool(&c.Projects.Enabled, "PROXY_PROJECTS_ENABLED") + setEnvString(&c.Projects.Token, "PROXY_PROJECTS_TOKEN") + setEnvString(&c.Projects.MaxAge, "PROXY_PROJECTS_MAX_AGE") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") @@ -1622,6 +1628,10 @@ func (c *Config) validateComponents() error { HostedGenericRepository, HostedGenericRepository) } + if err := c.Projects.Validate(); err != nil { + return err + } + if err := c.Mirror.Validate(); err != nil { return err } diff --git a/internal/config/projects.go b/internal/config/projects.go new file mode 100644 index 0000000..f416007 --- /dev/null +++ b/internal/config/projects.go @@ -0,0 +1,53 @@ +package config + +import ( + "fmt" + "time" +) + +// ProjectsConfig configures the project inventory: repositories upload an +// SBOM to /api/projects, and the proxy shows which of them use which package +// versions. +type ProjectsConfig struct { + // Enabled registers the /api/projects endpoints and the UI pages. Off by + // default. + Enabled bool `json:"enabled" yaml:"enabled"` + + // Token is the bearer token required to upload and delete projects. + // Required when Enabled. Supports ${VAR_NAME} expansion like AuthConfig + // fields. + Token string `json:"token" yaml:"token"` + + // MaxAge removes projects that have not uploaded an SBOM for this long + // (e.g., "90d"). Empty or "0" keeps them until they are deleted. + MaxAge string `json:"max_age" yaml:"max_age"` + + maxAge time.Duration +} + +// TokenExpanded returns Token with ${VAR_NAME} references expanded. +func (p *ProjectsConfig) TokenExpanded() string { + return expandEnv(p.Token) +} + +// MaxAgeDuration returns the parsed MaxAge, 0 to keep projects. Validate +// must run first. +func (p *ProjectsConfig) MaxAgeDuration() time.Duration { + return p.maxAge +} + +// Validate checks the project inventory settings. +func (p *ProjectsConfig) Validate() error { + p.maxAge = 0 + if p.MaxAge != "" { + d, err := parseRetentionDuration("projects.max_age", p.MaxAge) + if err != nil { + return err + } + p.maxAge = d + } + if p.Enabled && p.TokenExpanded() == "" { + return fmt.Errorf("projects.token is required when projects.enabled is true") + } + return nil +} diff --git a/internal/config/projects_test.go b/internal/config/projects_test.go new file mode 100644 index 0000000..ce3c27c --- /dev/null +++ b/internal/config/projects_test.go @@ -0,0 +1,69 @@ +package config + +import ( + "testing" + "time" +) + +func TestProjectsConfigValidate(t *testing.T) { + t.Setenv("PROJECTS_TEST_TOKEN", "from-env") + + tests := []struct { + name string + projects ProjectsConfig + wantErr bool + maxAge time.Duration + }{ + {name: "disabled without token", projects: ProjectsConfig{}}, + {name: "enabled without token", projects: ProjectsConfig{Enabled: true}, wantErr: true}, + {name: "enabled with token", projects: ProjectsConfig{Enabled: true, Token: "s3cret"}}, + {name: "token from env reference", projects: ProjectsConfig{Enabled: true, Token: "${PROJECTS_TEST_TOKEN}"}}, + {name: "token env reference unset", projects: ProjectsConfig{Enabled: true, Token: "${PROJECTS_TEST_UNSET}"}, wantErr: true}, + {name: "max age in days", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "90d"}, maxAge: 90 * 24 * time.Hour}, + {name: "max age in hours", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "36h"}, maxAge: 36 * time.Hour}, + {name: "max age zero keeps projects", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "0"}}, + {name: "invalid max age", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "soon"}, wantErr: true}, + {name: "negative max age", projects: ProjectsConfig{Enabled: true, Token: "s3cret", MaxAge: "-1d"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.projects.Validate() + if (err != nil) != tt.wantErr { + t.Fatalf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + if got := tt.projects.MaxAgeDuration(); err == nil && got != tt.maxAge { + t.Errorf("MaxAgeDuration() = %v, want %v", got, tt.maxAge) + } + }) + } +} + +func TestLoadFromEnvProjects(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_PROJECTS_ENABLED", "true") + t.Setenv("PROXY_PROJECTS_TOKEN", "env-token") + t.Setenv("PROXY_PROJECTS_MAX_AGE", "30d") + + cfg.LoadFromEnv() + + if !cfg.Projects.Enabled { + t.Error("Projects.Enabled not loaded from PROXY_PROJECTS_ENABLED") + } + if cfg.Projects.TokenExpanded() != "env-token" { + t.Errorf("Projects.Token = %q, want env-token", cfg.Projects.Token) + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := cfg.Projects.MaxAgeDuration(); got != 30*24*time.Hour { + t.Errorf("MaxAgeDuration() = %v, want 30 days", got) + } +} + +func TestValidateRejectsProjectsWithoutToken(t *testing.T) { + cfg := Default() + cfg.Projects = ProjectsConfig{Enabled: true} + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted projects.enabled without a token") + } +} diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a6..ea3b349 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -613,7 +613,7 @@ func createTestPostgresDB(t *testing.T) *DB { // Drop and recreate every table CreateSchema creates for clean test state; // leftover migration records make the next CreateSchema fail on the // migrations primary key. - tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "migrations", "schema_info"} + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "metadata_dates", "job_runs", "fetch_leases", "upstream_holds", "projects", "project_components", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } diff --git a/internal/database/projects.go b/internal/database/projects.go new file mode 100644 index 0000000..b00596b --- /dev/null +++ b/internal/database/projects.go @@ -0,0 +1,257 @@ +package database + +import ( + "database/sql" + "errors" + "fmt" + "time" +) + +// Project is a repository that uploaded an SBOM to the project inventory. +type Project struct { + ID int64 `db:"id"` + Name string `db:"name"` + Ref sql.NullString `db:"ref"` + Branch sql.NullString `db:"branch"` + UploadedAt time.Time `db:"uploaded_at"` + // Components is the number of package versions recorded for it. + Components int64 `db:"components"` +} + +// ProjectComponent is a package version a project uses. PackagePURL is the +// unversioned package URL, as packages.purl records it. +type ProjectComponent struct { + PackagePURL string `db:"package_purl"` + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + Version string `db:"version"` +} + +// ProjectComponentInfo is a project's component together with the package +// record the proxy holds for it, if any. +type ProjectComponentInfo struct { + ProjectComponent + CachedEcosystem sql.NullString `db:"cached_ecosystem"` + CachedName sql.NullString `db:"cached_name"` + LatestVersion sql.NullString `db:"latest_version"` +} + +// ProjectUsage is a project using a version of a package. +type ProjectUsage struct { + Project + Version string `db:"version"` +} + +// ComponentVulnerability is a vulnerability recorded for the package of a +// component a project uses. It is not yet filtered by version: whether it +// affects Version depends on FixedVersion. +type ComponentVulnerability struct { + Project string `db:"project"` + ProjectComponent + CachedEcosystem string `db:"cached_ecosystem"` + CachedName string `db:"cached_name"` + VulnID string `db:"vuln_id"` + Severity sql.NullString `db:"severity"` + FixedVersion sql.NullString `db:"fixed_version"` + CVSSScore sql.NullFloat64 `db:"cvss_score"` +} + +const projectColumns = ` + SELECT pr.id, pr.name, pr.ref, pr.branch, pr.uploaded_at, + (SELECT COUNT(*) FROM project_components pc WHERE pc.project_id = pr.id) AS components` + +// SaveProject records the components of a project, replacing everything +// recorded for it before, in one transaction. Readers see either the old or +// the new set, never a mix. +func (db *DB) SaveProject(name, ref, branch string, uploadedAt time.Time, components []ProjectComponent) error { + tx, err := db.Beginx() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + // The upsert locks the project row, so concurrent uploads of the same + // project and pruning take turns. + if _, err := tx.Exec(db.Rebind(` + INSERT INTO projects (name, ref, branch, uploaded_at) VALUES (?, ?, ?, ?) + ON CONFLICT(name) DO UPDATE SET ref = excluded.ref, branch = excluded.branch, uploaded_at = excluded.uploaded_at + `), name, nullString(ref), nullString(branch), uploadedAt.UTC()); err != nil { + return fmt.Errorf("recording project: %w", err) + } + var id int64 + if err := tx.Get(&id, db.Rebind(`SELECT id FROM projects WHERE name = ?`), name); err != nil { + return fmt.Errorf("reading project id: %w", err) + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return fmt.Errorf("removing old components: %w", err) + } + + insert, err := tx.Preparex(db.Rebind(` + INSERT INTO project_components (project_id, package_purl, ecosystem, name, version) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(project_id, package_purl, version) DO NOTHING + `)) + if err != nil { + return err + } + defer func() { _ = insert.Close() }() + for _, c := range components { + if _, err := insert.Exec(id, c.PackagePURL, c.Ecosystem, c.Name, c.Version); err != nil { + return fmt.Errorf("recording component %s@%s: %w", c.PackagePURL, c.Version, err) + } + } + return tx.Commit() +} + +// DeleteProject removes a project and its components. It reports whether the +// project existed. +func (db *DB) DeleteProject(name string) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + var id int64 + err = tx.Get(&id, db.Rebind(`DELETE FROM projects WHERE name = ? RETURNING id`), name) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("deleting project: %w", err) + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return false, fmt.Errorf("deleting project components: %w", err) + } + return true, tx.Commit() +} + +// PruneProjects removes the projects last uploaded before cutoff, with their +// components, and returns how many it removed. A project uploaded again while +// the prune runs is kept. +func (db *DB) PruneProjects(cutoff time.Time) (int, error) { + var ids []int64 + if err := db.Select(&ids, db.Rebind(`SELECT id FROM projects WHERE uploaded_at < ?`), cutoff.UTC()); err != nil { + return 0, fmt.Errorf("listing expired projects: %w", err) + } + pruned := 0 + for _, id := range ids { + ok, err := db.pruneProject(id, cutoff) + if err != nil { + return pruned, err + } + if ok { + pruned++ + } + } + return pruned, nil +} + +func (db *DB) pruneProject(id int64, cutoff time.Time) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + + res, err := tx.Exec(db.Rebind(`DELETE FROM projects WHERE id = ? AND uploaded_at < ?`), id, cutoff.UTC()) + if err != nil { + return false, fmt.Errorf("pruning project: %w", err) + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return false, err + } + if _, err := tx.Exec(db.Rebind(`DELETE FROM project_components WHERE project_id = ?`), id); err != nil { + return false, fmt.Errorf("pruning project components: %w", err) + } + return true, tx.Commit() +} + +// ListProjects returns all projects ordered by name. +func (db *DB) ListProjects() ([]Project, error) { + projects := []Project{} + if err := db.Select(&projects, projectColumns+` FROM projects pr ORDER BY pr.name`); err != nil { + return nil, err + } + return projects, nil +} + +// GetProject returns the named project, or nil if there is none. +func (db *DB) GetProject(name string) (*Project, error) { + var p Project + err := db.Get(&p, db.Rebind(projectColumns+` FROM projects pr WHERE pr.name = ?`), name) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &p, nil +} + +// GetProjectComponents returns the components of a project, ordered by +// ecosystem, name and version, with the package record the proxy holds for +// each. +func (db *DB) GetProjectComponents(projectID int64) ([]ProjectComponentInfo, error) { + components := []ProjectComponentInfo{} + err := db.Select(&components, db.Rebind(` + SELECT c.package_purl, c.ecosystem, c.name, c.version, + p.ecosystem AS cached_ecosystem, p.name AS cached_name, p.latest_version + FROM project_components c + LEFT JOIN packages p ON p.purl = c.package_purl + WHERE c.project_id = ? + ORDER BY c.ecosystem, c.name, c.version + `), projectID) + if err != nil { + return nil, err + } + return components, nil +} + +// GetProjectsUsingPackage returns the projects using a package, or only +// those using version when it is not empty, ordered by project and version. +func (db *DB) GetProjectsUsingPackage(packagePURL, version string) ([]ProjectUsage, error) { + query := projectColumns + `, c.version + FROM project_components c + JOIN projects pr ON pr.id = c.project_id + WHERE c.package_purl = ?` + args := []any{packagePURL} + if version != "" { + query += ` AND c.version = ?` + args = append(args, version) + } + usages := []ProjectUsage{} + if err := db.Select(&usages, db.Rebind(query+` ORDER BY pr.name, c.version`), args...); err != nil { + return nil, err + } + return usages, nil +} + +// GetComponentVulnerabilities returns the vulnerabilities the proxy holds for +// packages that projects use, for one project or, with an empty name, for all +// of them. Vulnerabilities are recorded per package, so only packages the +// proxy has a record of are matched. +func (db *DB) GetComponentVulnerabilities(project string) ([]ComponentVulnerability, error) { + query := ` + SELECT pr.name AS project, c.package_purl, c.ecosystem, c.name, c.version, + p.ecosystem AS cached_ecosystem, p.name AS cached_name, + v.vuln_id, v.severity, v.fixed_version, v.cvss_score + FROM project_components c + JOIN projects pr ON pr.id = c.project_id + JOIN packages p ON p.purl = c.package_purl + JOIN vulnerabilities v ON v.ecosystem = p.ecosystem AND v.package_name = p.name` + var args []any + if project != "" { + query += ` WHERE pr.name = ?` + args = append(args, project) + } + vulns := []ComponentVulnerability{} + if err := db.Select(&vulns, db.Rebind(query+` ORDER BY c.package_purl, c.version, pr.name, v.vuln_id`), args...); err != nil { + return nil, err + } + return vulns, nil +} + +func nullString(s string) sql.NullString { + return sql.NullString{String: s, Valid: s != ""} +} diff --git a/internal/database/projects_test.go b/internal/database/projects_test.go new file mode 100644 index 0000000..4eaaf50 --- /dev/null +++ b/internal/database/projects_test.go @@ -0,0 +1,228 @@ +package database + +import ( + "database/sql" + "slices" + "testing" + "time" +) + +var ( + lodash20 = ProjectComponent{PackagePURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", Version: "4.17.20"} + lodash21 = ProjectComponent{PackagePURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", Version: "4.17.21"} + console = ProjectComponent{PackagePURL: "pkg:composer/symfony/console", Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1"} +) + +func saveProject(t *testing.T, db *DB, name string, at time.Time, components ...ProjectComponent) { + t.Helper() + if err := db.SaveProject(name, "", "", at, components); err != nil { + t.Fatalf("SaveProject(%s): %v", name, err) + } +} + +func usageNames(usages []ProjectUsage) []string { + var names []string + for _, u := range usages { + names = append(names, u.Name+"@"+u.Version) + } + return names +} + +func TestSaveProjectReplacesComponents(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now().UTC().Truncate(time.Second) + if err := db.SaveProject("acme/web", "abc123", "main", now, []ProjectComponent{lodash20, console, lodash20}); err != nil { + t.Fatal(err) + } + saveProject(t, db, "acme/web", now.Add(time.Minute), lodash21) + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + if project.Components != 1 || project.Ref.Valid || project.Branch.Valid { + t.Errorf("project = %+v, want one component and ref and branch cleared", project) + } + if !project.UploadedAt.Equal(now.Add(time.Minute)) { + t.Errorf("uploaded_at = %v, want %v", project.UploadedAt, now.Add(time.Minute)) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].ProjectComponent != lodash21 { + t.Errorf("components = %+v, want only lodash 4.17.21", components) + } + + missing, err := db.GetProject("acme/other") + if err != nil || missing != nil { + t.Errorf("GetProject(missing) = %v, %v", missing, err) + } + }) +} + +func TestGetProjectsUsingPackage(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/web", now, lodash20, console) + saveProject(t, db, "acme/api", now, lodash21) + saveProject(t, db, "acme/cli", now, console) + + all, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(all), []string{"acme/api@4.17.21", "acme/web@4.17.20"}; !slices.Equal(got, want) { + t.Errorf("any version = %v, want %v", got, want) + } + one, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "4.17.20") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(one), []string{"acme/web@4.17.20"}; !slices.Equal(got, want) { + t.Errorf("4.17.20 = %v, want %v", got, want) + } + none, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "1.0.0") + if err != nil || len(none) != 0 { + t.Errorf("unused version = %v, %v", none, err) + } + }) +} + +func TestListAndDeleteProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/web", now, lodash20, console) + saveProject(t, db, "acme/api", now) + + projects, err := db.ListProjects() + if err != nil { + t.Fatal(err) + } + if len(projects) != 2 || projects[0].Name != "acme/api" || projects[0].Components != 0 || projects[1].Components != 2 { + t.Fatalf("projects = %+v", projects) + } + + deleted, err := db.DeleteProject("acme/web") + if err != nil || !deleted { + t.Fatalf("DeleteProject = %v, %v", deleted, err) + } + deleted, err = db.DeleteProject("acme/web") + if err != nil || deleted { + t.Errorf("second DeleteProject = %v, %v, want false", deleted, err) + } + var rows int + if err := db.Get(&rows, `SELECT COUNT(*) FROM project_components`); err != nil { + t.Fatal(err) + } + if rows != 0 { + t.Errorf("%d component rows left after the delete", rows) + } + }) +} + +func TestPruneProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + saveProject(t, db, "acme/old", now.Add(-100*24*time.Hour), lodash20) + saveProject(t, db, "acme/new", now.Add(-time.Hour), lodash21) + + pruned, err := db.PruneProjects(now.Add(-90 * 24 * time.Hour)) + if err != nil { + t.Fatal(err) + } + if pruned != 1 { + t.Errorf("pruned = %d, want 1", pruned) + } + projects, err := db.ListProjects() + if err != nil { + t.Fatal(err) + } + if len(projects) != 1 || projects[0].Name != "acme/new" { + t.Errorf("projects after prune = %+v", projects) + } + usages, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "") + if err != nil { + t.Fatal(err) + } + if got, want := usageNames(usages), []string{"acme/new@4.17.21"}; !slices.Equal(got, want) { + t.Errorf("usages after prune = %v, want %v", got, want) + } + }) +} + +func TestProjectComponentsJoinPackages(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + if err := db.UpsertPackage(&Package{ + PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", + LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } + if err := db.UpsertVulnerability(&Vulnerability{ + VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", + Severity: sql.NullString{String: "high", Valid: true}, + FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } + saveProject(t, db, "acme/web", time.Now(), lodash20, console) + saveProject(t, db, "acme/api", time.Now(), lodash21) + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) != 2 { + t.Fatalf("components = %+v", components) + } + // Ordered by ecosystem: composer first, which the proxy has no record of. + if components[0].CachedName.Valid { + t.Errorf("uncached component has a package record: %+v", components[0]) + } + if components[1].CachedName.String != "lodash" || components[1].LatestVersion.String != "4.17.21" { + t.Errorf("cached component = %+v", components[1]) + } + + vulns, err := db.GetComponentVulnerabilities("acme/web") + if err != nil { + t.Fatal(err) + } + if len(vulns) != 1 || vulns[0].Version != "4.17.20" || vulns[0].VulnID != "GHSA-35jh-r3h4-6jhm" { + t.Errorf("acme/web vulnerabilities = %+v", vulns) + } + all, err := db.GetComponentVulnerabilities("") + if err != nil { + t.Fatal(err) + } + if len(all) != 2 { + t.Errorf("all vulnerabilities = %+v, want one row per project", all) + } + }) +} + +func TestMigrateAddsProjects(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + // A database from before the project inventory. + for _, table := range []string{"project_components", "projects"} { + if _, err := db.Exec("DROP TABLE " + table); err != nil { + t.Fatalf("dropping %s: %v", table, err) + } + } + if _, err := db.Exec(db.Rebind("DELETE FROM migrations WHERE name = ?"), "020_add_projects"); err != nil { + t.Fatalf("forgetting the migration: %v", err) + } + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + saveProject(t, db, "acme/web", time.Now(), lodash20) + usages, err := db.GetProjectsUsingPackage("pkg:npm/lodash", "4.17.20") + if err != nil || len(usages) != 1 { + t.Fatalf("GetProjectsUsingPackage after migration = %v, %v", usages, err) + } + }) +} diff --git a/internal/database/schema.go b/internal/database/schema.go index f0ec651..861429e 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -154,6 +154,26 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms INTEGER NOT NULL ); +CREATE TABLE IF NOT EXISTS projects ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at DATETIME NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name); +CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at); + +CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) +); +CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at DATETIME NOT NULL @@ -297,6 +317,26 @@ CREATE TABLE IF NOT EXISTS upstream_holds ( until_ms BIGINT NOT NULL ); +CREATE TABLE IF NOT EXISTS projects ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at TIMESTAMP NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name); +CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at); + +CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) +); +CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version); + CREATE TABLE IF NOT EXISTS migrations ( name TEXT NOT NULL PRIMARY KEY, applied_at TIMESTAMP NOT NULL @@ -468,6 +508,7 @@ var migrations = []migration{ {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, + {"020_add_projects", migrateAddProjects}, } // isTableNotFound returns true if the error indicates a missing table. @@ -820,6 +861,41 @@ func migrateAddUpstreamHolds(db *DB) error { return nil } +// migrateAddProjects creates the tables of the project inventory: the +// projects that uploaded an SBOM and the package versions each one uses. +func migrateAddProjects(db *DB) error { + id, ts := "INTEGER PRIMARY KEY", sqliteDatetime + if db.dialect == DialectPostgres { + id, ts = "SERIAL PRIMARY KEY", postgresTimestamp + } + statements := []string{ + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS projects ( + id %s, + name TEXT NOT NULL, + ref TEXT, + branch TEXT, + uploaded_at %s NOT NULL + )`, id, ts), + `CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_name ON projects(name)`, + `CREATE INDEX IF NOT EXISTS idx_projects_uploaded_at ON projects(uploaded_at)`, + `CREATE TABLE IF NOT EXISTS project_components ( + project_id INTEGER NOT NULL, + package_purl TEXT NOT NULL, + ecosystem TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + PRIMARY KEY (project_id, package_purl, version) + )`, + `CREATE INDEX IF NOT EXISTS idx_project_components_package ON project_components(package_purl, version)`, + } + for _, stmt := range statements { + if _, err := db.Exec(stmt); err != nil { + return fmt.Errorf("creating project tables: %w", err) + } + } + return nil +} + // migrateAddPendingDeletesQueuedAtIndex indexes the column reclaim orders // the queue by, which grows large after a retention sweep. func migrateAddPendingDeletesQueuedAtIndex(db *DB) error { diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go new file mode 100644 index 0000000..954d46e --- /dev/null +++ b/internal/inventory/inventory.go @@ -0,0 +1,185 @@ +// Package inventory turns SBOMs uploaded for a project into the package +// versions the proxy records for it, named the way the proxy's handlers name +// the packages they cache, so the two can be joined. +package inventory + +import ( + "errors" + "fmt" + "strings" + + "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/purl" + "github.com/git-pkgs/sbom" + "github.com/git-pkgs/vers" +) + +// MaxProjectNameLength bounds a project name, such as a GitLab project path. +const MaxProjectNameLength = 255 + +// Component is one package version a project uses. +type Component struct { + // Ecosystem is the name the proxy's handlers store the package under, + // e.g. "npm", "composer", "golang" or "alpine". + Ecosystem string + // Name is the ecosystem-native package name, e.g. "@babel/core", + // "symfony/console" or "org.slf4j:slf4j-api". + Name string + // Version is the version as the SBOM gives it; it may be empty. + Version string + // PackagePURL is the unversioned package URL the proxy records the + // package under, so it matches packages.purl. + PackagePURL string +} + +// ecosystems maps the PURL types the proxy serves to the ecosystem name its +// handlers store packages under. +var ecosystems = map[string]string{ + "alpm": "arch", + "apk": "alpine", + "bsd": "freebsd", + "cargo": "cargo", + "chef": "chef", + "composer": "composer", + "conan": "conan", + "conda": "conda", + "cran": "cran", + "deb": "deb", + "gem": "gem", + "golang": "golang", + "hex": "hex", + "julia": "julia", + "maven": "maven", + "npm": "npm", + "nuget": "nuget", + "pub": "pub", + "pypi": "pypi", + "rpm": "rpm", + "swift": "swift", +} + +// distroTypes are PURL types whose namespace names the distribution +// ("debian", "alpine"), not part of the package name the proxy stores. +var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} + +// ErrUnsupportedPURL is returned for a package URL whose type the proxy +// doesn't serve. +var ErrUnsupportedPURL = errors.New("package type is not served by the proxy") + +// FromPURL maps a package URL to a component. Qualifiers and subpath are +// dropped; the version is kept as given. +func FromPURL(s string) (Component, error) { + p, err := purl.Parse(s) + if err != nil { + return Component{}, err + } + ecosystem, ok := ecosystems[p.Type] + if !ok { + return Component{}, fmt.Errorf("%w: %s", ErrUnsupportedPURL, p.Type) + } + name := p.FullName() + switch { + case distroTypes[p.Type]: + name = p.Name + case p.Type == "conda" && p.Namespace == "" && p.Qualifier("channel") != "": + name = p.Qualifier("channel") + "/" + p.Name + } + packagePURL, _ := packageurl.MakeCacheStrings(ecosystem, name, "") + if packagePURL == "" { + return Component{}, fmt.Errorf("package URL %q has no valid package name", s) + } + return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil +} + +// Parse reads a CycloneDX (JSON or XML) or SPDX SBOM and returns the package +// versions it lists, without duplicates. Packages without a package URL, or +// of a type the proxy doesn't serve, are counted in skipped. +func Parse(data []byte) (components []Component, skipped int, err error) { + doc, err := sbom.Parse(data) + if err != nil { + return nil, 0, err + } + seen := make(map[[2]string]bool, len(doc.Packages)) + for _, pkg := range doc.Packages { + c, ok := componentOf(pkg) + if !ok { + skipped++ + continue + } + key := [2]string{c.PackagePURL, c.Version} + if seen[key] { + continue + } + seen[key] = true + components = append(components, c) + } + return components, skipped, nil +} + +func componentOf(pkg sbom.Package) (Component, bool) { + locator := pkg.PURL() + if locator == "" { + return Component{}, false + } + c, err := FromPURL(locator) + if err != nil { + return Component{}, false + } + if c.Version == "" { + c.Version = pkg.Version + } + return c, true +} + +// ValidateProjectName checks a project name such as "group/sub/repo": one or +// more segments of letters, digits, ".", "_" and "-", none of them "." or +// "..". +func ValidateProjectName(name string) error { + if name == "" { + return errors.New("project name is empty") + } + if len(name) > MaxProjectNameLength { + return fmt.Errorf("project name is longer than %d characters", MaxProjectNameLength) + } + for segment := range strings.SplitSeq(name, "/") { + if segment == "" || segment == "." || segment == ".." { + return fmt.Errorf("project name %q has an empty, \".\" or \"..\" segment", name) + } + for _, r := range segment { + if !validNameRune(r) { + return fmt.Errorf("project name %q may only contain letters, digits, '.', '_', '-' and '/'", name) + } + } + } + return nil +} + +func validNameRune(r rune) bool { + return r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '.' || r == '_' || r == '-' +} + +// CompareVersions compares two versions of a package in ecosystem with the +// ecosystem's version rules, returning -1, 0 or 1. +func CompareVersions(ecosystem, a, b string) int { + return vers.CompareWithScheme(a, b, purl.EcosystemToPURLType(ecosystem)) +} + +// Outdated reports whether version is lower than latest. +func Outdated(ecosystem, version, latest string) bool { + if version == "" || latest == "" || version == latest { + return false + } + return CompareVersions(ecosystem, version, latest) < 0 +} + +// Affected reports whether a vulnerability fixed in fixedVersion affects +// version. A vulnerability without a fixed version affects every version. +func Affected(ecosystem, version, fixedVersion string) bool { + if fixedVersion == "" { + return true + } + if version == "" { + return false + } + return CompareVersions(ecosystem, version, fixedVersion) < 0 +} diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go new file mode 100644 index 0000000..3d55d50 --- /dev/null +++ b/internal/inventory/inventory_test.go @@ -0,0 +1,133 @@ +package inventory + +import ( + "errors" + "os" + "slices" + "strings" + "testing" +) + +func TestParseSyftCycloneDX(t *testing.T) { + data, err := os.ReadFile("testdata/syft-cyclonedx.json") + if err != nil { + t.Fatal(err) + } + components, skipped, err := Parse(data) + if err != nil { + t.Fatalf("Parse: %v", err) + } + want := []Component{ + {Ecosystem: "npm", Name: "@babel/core", Version: "7.23.0", PackagePURL: "pkg:npm/%40babel/core"}, + {Ecosystem: "alpine", Name: "busybox", Version: "1.36.1-r15", PackagePURL: "pkg:apk/alpine/busybox"}, + {Ecosystem: "golang", Name: "github.com/inconshreveable/mousetrap", Version: "v1.1.0", PackagePURL: "pkg:golang/github.com/inconshreveable/mousetrap"}, + {Ecosystem: "golang", Name: "github.com/spf13/cobra", Version: "v1.8.0", PackagePURL: "pkg:golang/github.com/spf13/cobra"}, + {Ecosystem: "golang", Name: "github.com/spf13/pflag", Version: "v1.0.5", PackagePURL: "pkg:golang/github.com/spf13/pflag"}, + {Ecosystem: "npm", Name: "lodash", Version: "4.17.20", PackagePURL: "pkg:npm/lodash"}, + {Ecosystem: "composer", Name: "monolog/monolog", Version: "3.5.0", PackagePURL: "pkg:composer/monolog/monolog"}, + {Ecosystem: "alpine", Name: "musl", Version: "1.2.4-r2", PackagePURL: "pkg:apk/alpine/musl"}, + {Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1", PackagePURL: "pkg:composer/symfony/console"}, + {Ecosystem: "npm", Name: "webapp", Version: "1.0.0", PackagePURL: "pkg:npm/webapp"}, + } + if !slices.Equal(components, want) { + t.Errorf("components =\n%v\nwant\n%v", components, want) + } + // The operating system and the four lock files syft lists carry no PURL. + if skipped != 5 { + t.Errorf("skipped = %d, want 5", skipped) + } +} + +func TestParseDeduplicatesAndSkipsUnknownTypes(t *testing.T) { + doc := `{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}, + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}, + {"type":"library","name":"lodash","version":"4.17.20","purl":"pkg:npm/lodash@4.17.20"}, + {"type":"library","name":"checkout","version":"v4","purl":"pkg:github/actions/checkout@v4"}, + {"type":"library","name":"Django","version":"4.2.0","purl":"pkg:pypi/Django"}, + {"type":"library","name":"guava","version":"32.1.3-jre","purl":"pkg:maven/com.google.guava/guava@32.1.3-jre?type=jar"}, + {"type":"library","name":"libc6","version":"2.36-9","purl":"pkg:deb/debian/libc6@2.36-9?arch=amd64&distro=debian-12"}, + {"type":"library","name":"broken","purl":"not a purl"} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + want := []Component{ + {Ecosystem: "npm", Name: "lodash", Version: "4.17.21", PackagePURL: "pkg:npm/lodash"}, + {Ecosystem: "npm", Name: "lodash", Version: "4.17.20", PackagePURL: "pkg:npm/lodash"}, + {Ecosystem: "pypi", Name: "django", Version: "4.2.0", PackagePURL: "pkg:pypi/django"}, + {Ecosystem: "maven", Name: "com.google.guava:guava", Version: "32.1.3-jre", PackagePURL: "pkg:maven/com.google.guava/guava"}, + {Ecosystem: "deb", Name: "libc6", Version: "2.36-9", PackagePURL: "pkg:deb/libc6"}, + } + if !slices.Equal(components, want) { + t.Errorf("components =\n%v\nwant\n%v", components, want) + } + if skipped != 2 { + t.Errorf("skipped = %d, want 2", skipped) + } +} + +func TestParseRejectsNonSBOM(t *testing.T) { + if _, _, err := Parse([]byte(`{"hello":"world"}`)); err == nil { + t.Error("Parse accepted a document that is not an SBOM") + } +} + +func TestFromPURLUnsupported(t *testing.T) { + _, err := FromPURL("pkg:github/actions/checkout") + if !errors.Is(err, ErrUnsupportedPURL) { + t.Errorf("err = %v, want ErrUnsupportedPURL", err) + } +} + +func TestValidateProjectName(t *testing.T) { + valid := []string{"repo", "group/repo", "group/sub/my_repo-2.0", strings.Repeat("a", MaxProjectNameLength)} + for _, name := range valid { + if err := ValidateProjectName(name); err != nil { + t.Errorf("ValidateProjectName(%q) = %v, want nil", name, err) + } + } + invalid := []string{ + "", "/repo", "repo/", "group//repo", ".", "..", "group/../repo", "group/./repo", + "group/re po", "group/repo?x", "group\\repo", "grüppe/repo", "group/repo%2F", + strings.Repeat("a", MaxProjectNameLength+1), + } + for _, name := range invalid { + if err := ValidateProjectName(name); err == nil { + t.Errorf("ValidateProjectName(%q) = nil, want an error", name) + } + } +} + +func TestOutdatedAndAffected(t *testing.T) { + tests := []struct { + name string + ecosystem string + version string + other string + outdated bool + affected bool + }{ + {"npm older", "npm", "4.17.20", "4.17.21", true, true}, + {"npm same", "npm", "4.17.21", "4.17.21", false, false}, + {"npm newer", "npm", "4.18.0", "4.17.21", false, false}, + {"composer v prefix", "composer", "v6.4.1", "v6.4.2", true, true}, + {"golang", "golang", "v1.8.0", "v1.10.0", true, true}, + {"alpine revisions", "alpine", "1.36.1-r15", "1.36.1-r2", false, false}, + {"empty version", "npm", "", "1.0.0", false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Outdated(tt.ecosystem, tt.version, tt.other); got != tt.outdated { + t.Errorf("Outdated = %v, want %v", got, tt.outdated) + } + if got := Affected(tt.ecosystem, tt.version, tt.other); got != tt.affected { + t.Errorf("Affected = %v, want %v", got, tt.affected) + } + }) + } + if !Affected("npm", "1.0.0", "") { + t.Error("a vulnerability without a fix should affect every version") + } +} diff --git a/internal/inventory/testdata/syft-cyclonedx.json b/internal/inventory/testdata/syft-cyclonedx.json new file mode 100644 index 0000000..7647d68 --- /dev/null +++ b/internal/inventory/testdata/syft-cyclonedx.json @@ -0,0 +1 @@ +{"$schema":"http://cyclonedx.org/schema/bom-1.7.schema.json","bomFormat":"CycloneDX","specVersion":"1.7","serialNumber":"urn:uuid:d69cf664-a43e-4d65-93e3-e43e24d33c0e","version":1,"metadata":{"timestamp":"2026-10-27T19:17:53+01:00","tools":{"components":[{"type":"application","author":"anchore","name":"syft","version":"1.55.0"}]},"component":{"bom-ref":"af63bd4c8601b7f1","type":"file","name":"."}},"components":[{"bom-ref":"pkg:npm/%40babel/core@7.23.0?package-id=7b01fd7a9ce63128","type":"library","name":"@babel/core","version":"7.23.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:\\@babel\\/core:\\@babel\\/core:7.23.0:*:*:*:*:*:*:*","purl":"pkg:npm/%40babel/core@7.23.0","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64&distro=alpine-3.19.1&package-id=464f552f6c89c192","type":"library","publisher":"Sören Tempel ","name":"busybox","version":"1.36.1-r15","description":"Size optimized toolbox of many common UNIX utilities","licenses":[{"license":{"id":"GPL-2.0-only"}}],"cpe":"cpe:2.3:a:busybox:busybox:1.36.1-r15:*:*:*:*:*:*:*","purl":"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64&distro=alpine-3.19.1","externalReferences":[{"url":"https://busybox.net/","type":"distribution"}],"properties":[{"name":"syft:package:foundBy","value":"apk-db-cataloger"},{"name":"syft:package:type","value":"apk"},{"name":"syft:package:metadataType","value":"apk-db-entry"},{"name":"syft:location:0:path","value":"/lib/apk/db/installed"},{"name":"syft:metadata:gitCommitOfApkPort","value":"1b0b2f6e7d0e6a5b4c3d2e1f0a9b8c7d6e5f4a3b"},{"name":"syft:metadata:installedSize","value":"950272"},{"name":"syft:metadata:originPackage","value":"busybox"},{"name":"syft:metadata:pullChecksum","value":"Q1Xy0qB0j7Y7uQ2kGmWbN8vT6Ff0A4E="},{"name":"syft:metadata:pullDependencies:0","value":"so:libc.musl-x86_64.so.1"},{"name":"syft:metadata:size","value":"509268"}]},{"bom-ref":"pkg:golang/github.com/inconshreveable/mousetrap@v1.1.0?package-id=f6300927a9f6b106","type":"library","name":"github.com/inconshreveable/mousetrap","version":"v1.1.0","cpe":"cpe:2.3:a:inconshreveable:mousetrap:v1.1.0:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/inconshreveable/mousetrap@v1.1.0","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8="}]},{"bom-ref":"pkg:golang/github.com/spf13/cobra@v1.8.0?package-id=81b06fbcbe87f414","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","cpe":"cpe:2.3:a:spf13:cobra:v1.8.0:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0="}]},{"bom-ref":"pkg:golang/github.com/spf13/pflag@v1.0.5?package-id=163f67459b76e534","type":"library","name":"github.com/spf13/pflag","version":"v1.0.5","cpe":"cpe:2.3:a:spf13:pflag:v1.0.5:*:*:*:*:*:*:*","purl":"pkg:golang/github.com/spf13/pflag@v1.0.5","properties":[{"name":"syft:package:foundBy","value":"go-module-file-cataloger"},{"name":"syft:package:language","value":"go"},{"name":"syft:package:type","value":"go-module"},{"name":"syft:package:metadataType","value":"go-module-entry"},{"name":"syft:location:0:path","value":"/go.mod"},{"name":"syft:metadata:h1Digest","value":"h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRPuqXzS0rkr6qZw="}]},{"bom-ref":"pkg:npm/lodash@4.17.20?package-id=ece3658907628a6b","type":"library","name":"lodash","version":"4.17.20","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:lodash:lodash:4.17.20:*:*:*:*:node.js:*:*","purl":"pkg:npm/lodash@4.17.20","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"pkg:composer/monolog/monolog@3.5.0?package-id=3abfdcdd00fd8d2f","type":"library","name":"monolog/monolog","version":"3.5.0","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:monolog\\/monolog:monolog\\/monolog:3.5.0:*:*:*:*:*:*:*","purl":"pkg:composer/monolog/monolog@3.5.0","properties":[{"name":"syft:package:foundBy","value":"php-composer-lock-cataloger"},{"name":"syft:package:language","value":"php"},{"name":"syft:package:type","value":"php-composer"},{"name":"syft:package:metadataType","value":"php-composer-lock-entry"},{"name":"syft:location:0:path","value":"/composer.lock"}]},{"bom-ref":"pkg:apk/alpine/musl@1.2.4-r2?arch=x86_64&distro=alpine-3.19.1&package-id=9f0e3d80c35c10bc","type":"library","publisher":"Timo Teräs ","name":"musl","version":"1.2.4-r2","description":"the musl c library (libc) implementation","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:musl-libc:musl:1.2.4-r2:*:*:*:*:*:*:*","purl":"pkg:apk/alpine/musl@1.2.4-r2?arch=x86_64&distro=alpine-3.19.1","externalReferences":[{"url":"https://musl.libc.org/","type":"distribution"}],"properties":[{"name":"syft:package:foundBy","value":"apk-db-cataloger"},{"name":"syft:package:type","value":"apk"},{"name":"syft:package:metadataType","value":"apk-db-entry"},{"name":"syft:cpe23","value":"cpe:2.3:a:musl_libc:musl:1.2.4-r2:*:*:*:*:*:*:*"},{"name":"syft:cpe23","value":"cpe:2.3:a:musl:musl:1.2.4-r2:*:*:*:*:*:*:*"},{"name":"syft:location:0:path","value":"/lib/apk/db/installed"},{"name":"syft:metadata:gitCommitOfApkPort","value":"4b2f0d7b4e0e8a4b0e1b1b4d2b8a7a3d1a2f8e0c"},{"name":"syft:metadata:installedSize","value":"647168"},{"name":"syft:metadata:originPackage","value":"musl"},{"name":"syft:metadata:pullChecksum","value":"Q1N9CgYbNRm8SgkqbXqJhCAc6iqXo="},{"name":"syft:metadata:size","value":"407383"}]},{"bom-ref":"pkg:composer/symfony/console@v6.4.1?package-id=767524bf09da7da7","type":"library","name":"symfony/console","version":"v6.4.1","licenses":[{"license":{"id":"MIT"}}],"cpe":"cpe:2.3:a:symfony\\/console:symfony\\/console:v6.4.1:*:*:*:*:*:*:*","purl":"pkg:composer/symfony/console@v6.4.1","properties":[{"name":"syft:package:foundBy","value":"php-composer-lock-cataloger"},{"name":"syft:package:language","value":"php"},{"name":"syft:package:type","value":"php-composer"},{"name":"syft:package:metadataType","value":"php-composer-lock-entry"},{"name":"syft:location:0:path","value":"/composer.lock"}]},{"bom-ref":"pkg:npm/webapp@1.0.0?package-id=982a05090388c92d","type":"library","name":"webapp","version":"1.0.0","cpe":"cpe:2.3:a:webapp:webapp:1.0.0:*:*:*:*:*:*:*","purl":"pkg:npm/webapp@1.0.0","properties":[{"name":"syft:package:foundBy","value":"javascript-lock-cataloger"},{"name":"syft:package:language","value":"javascript"},{"name":"syft:package:type","value":"npm"},{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]},{"bom-ref":"os:alpine@3.19.1","type":"operating-system","name":"alpine","version":"3.19.1","description":"Alpine Linux v3.19","swid":{"tagId":"alpine","name":"alpine","version":"3.19.1"},"properties":[{"name":"syft:distro:extendedSupport","value":"false"},{"name":"syft:distro:id","value":"alpine"},{"name":"syft:distro:prettyName","value":"Alpine Linux v3.19"},{"name":"syft:distro:versionID","value":"3.19.1"}]},{"bom-ref":"c8bb62e6bded633e","type":"file","name":"/builds/acme/webapp/composer.lock","hashes":[{"alg":"SHA-1","content":"53c59ca459a8ff42918b5ed4fdcea1d8ba98432f"},{"alg":"SHA-256","content":"63738301cc7186a6645f00ae0a63ae6347e04552489f3cc70e92200463f5b4ac"}]},{"bom-ref":"3fc5a8d3d86e9790","type":"file","name":"/builds/acme/webapp/go.mod","hashes":[{"alg":"SHA-1","content":"c67c41265c415c41049b31f850b4497898eff1fd"},{"alg":"SHA-256","content":"f2303d4139e491238d8cb9a6c448e4c366b82bae3a646164465387df76e44f32"}]},{"bom-ref":"9f5aca292136191a","type":"file","name":"/builds/acme/webapp/lib/apk/db/installed","hashes":[{"alg":"SHA-1","content":"b2ebf78ea604d3a76359a262afd401ab966a36a1"},{"alg":"SHA-256","content":"d363cb96812dc6eb1d6f4c00879c2564cdbff238b6c1d0a37701d2adedd326e0"}]},{"bom-ref":"fd71c2238fc07657","type":"file","name":"/builds/acme/webapp/package-lock.json","hashes":[{"alg":"SHA-1","content":"fc26eadd11405354163eb0f53ffd2bb8cb56c5e7"},{"alg":"SHA-256","content":"69094786dd997053d1c7a81d4349bb5b608c08b576e67360f07cdc2726a041ea"}]}],"dependencies":[{"ref":"pkg:npm/webapp@1.0.0?package-id=982a05090388c92d","dependsOn":["pkg:npm/%40babel/core@7.23.0?package-id=7b01fd7a9ce63128","pkg:npm/lodash@4.17.20?package-id=ece3658907628a6b"]}]} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index da866a6..ebcc333 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -15,6 +15,9 @@ type DashboardData struct { EnrichmentStats EnrichmentStatsView RecentPackages []PackageInfo PopularPackages []PackageInfo + // VulnerableInUse lists vulnerable package versions projects use, when + // the project inventory is enabled. + VulnerableInUse []VulnerableVersion } // DashboardStats contains cache statistics for the dashboard. @@ -68,6 +71,9 @@ type PackageShowData struct { Versions []database.Version Vulnerabilities []database.Vulnerability LicenseCategory string + // UsedBy lists the projects using each version, when the project + // inventory is enabled. + UsedBy []VersionUsage } // VersionShowData contains data for rendering the version show page. diff --git a/internal/server/layout.go b/internal/server/layout.go index 3a51876..1a63d2f 100644 --- a/internal/server/layout.go +++ b/internal/server/layout.go @@ -17,6 +17,8 @@ type Layout struct { CanonicalPath string // CacheDelete shows the controls for deleting cached artifacts. CacheDelete bool + // Projects shows the project inventory pages. + Projects bool } func (s *Server) layoutFor(r *http.Request) Layout { @@ -25,5 +27,6 @@ func (s *Server) layoutFor(r *http.Request) Layout { UIBaseURL: s.cfg.UIBaseURL, CanonicalPath: r.URL.Path, CacheDelete: s.cfg.CacheDelete, + Projects: s.cfg.Projects.Enabled, } } diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go new file mode 100644 index 0000000..a007319 --- /dev/null +++ b/internal/server/projects_api.go @@ -0,0 +1,335 @@ +package server + +import ( + "errors" + "io" + "log/slog" + "net/http" + "net/url" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/inventory" + "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/go-chi/chi/v5" +) + +// maxSBOMBytes caps an uploaded SBOM. syft's CycloneDX output for a large +// monorepo is a few MiB. +const maxSBOMBytes = 32 << 20 + +// maxRefLength caps the ref and branch query parameters of an upload. +const maxRefLength = 255 + +// sbomPathSuffix ends the upload path, /api/projects/{project}/sbom. +const sbomPathSuffix = "/sbom" + +// ProjectsAPIHandler serves /api/projects, where repositories upload the SBOM +// of their default branch so the proxy can show which of them use which +// package versions. +type ProjectsAPIHandler struct { + db *database.DB + logger *slog.Logger +} + +// NewProjectsAPIHandler creates the /api/projects handler. +func NewProjectsAPIHandler(db *database.DB, logger *slog.Logger) *ProjectsAPIHandler { + return &ProjectsAPIHandler{db: db, logger: logger} +} + +// ProjectUploadResponse summarizes an uploaded SBOM. +type ProjectUploadResponse struct { + Project string `json:"project"` + // Components is the number of package versions recorded. + Components int `json:"components"` + // Skipped is the number of SBOM entries without a package URL or of a + // package type the proxy doesn't serve. + Skipped int `json:"skipped"` +} + +// ProjectSummary describes a project in the inventory. +type ProjectSummary struct { + Name string `json:"name"` + Ref string `json:"ref,omitempty"` + Branch string `json:"branch,omitempty"` + UploadedAt time.Time `json:"uploaded_at"` + // ComponentCount is the number of package versions recorded. + ComponentCount int64 `json:"component_count"` + // Version is set when listing the projects using a package: the version + // of it this project uses. + Version string `json:"version,omitempty"` +} + +// ProjectListResponse lists projects. +type ProjectListResponse struct { + Projects []ProjectSummary `json:"projects"` +} + +// ProjectComponent is a package version a project uses. +type ProjectComponent struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + Version string `json:"version"` + PURL string `json:"purl"` +} + +// ProjectResponse is a project with its components. +type ProjectResponse struct { + ProjectSummary + Components []ProjectComponent `json:"components"` +} + +// mountProjectAPI registers /api/projects when projects.enabled is set. +func (s *Server) mountProjectAPI(r chi.Router) { + if s.cfg.Projects.Enabled { + registerProjectRoutes(r, NewProjectsAPIHandler(s.db, s.logger), s.cfg.Projects.TokenExpanded()) + } +} + +// registerProjectRoutes mounts /api/projects. Reading is open like the rest +// of the API; uploading and deleting require the bearer token. +// +// A project name such as "group/sub/repo" is accepted both as path segments, +// /api/projects/group/sub/repo, and URL-encoded as one segment, +// /api/projects/group%2Fsub%2Frepo, which is how GitLab's CI_PROJECT_PATH +// usually ends up when a client escapes it. +func registerProjectRoutes(r chi.Router, h *ProjectsAPIHandler, token string) { + r.Get("/api/projects", h.HandleList) + r.Get("/api/projects/*", h.HandleGet) + r.Group(func(w chi.Router) { + w.Use(requireBearerToken(token)) + w.Put("/api/projects/*", h.HandleUpload) + w.Delete("/api/projects/*", h.HandleDelete) + }) +} + +// HandleUpload handles PUT /api/projects/{project}/sbom. +// @Summary Upload a project's SBOM +// @Description Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment. +// @Tags projects +// @Accept json +// @Produce json +// @Security BearerAuth +// @Param project path string true "Project path, e.g. group/sub/repo" +// @Param ref query string false "Commit the SBOM was generated from" +// @Param branch query string false "Branch the SBOM was generated from" +// @Success 200 {object} ProjectUploadResponse +// @Failure 400 {object} ErrorResponse +// @Failure 401 {object} ErrorResponse +// @Failure 413 {object} ErrorResponse +// @Router /api/projects/{project}/sbom [put] +func (h *ProjectsAPIHandler) HandleUpload(w http.ResponseWriter, r *http.Request) { + path, err := projectPath(r) + if err != nil { + badRequest(w, err.Error()) + return + } + name, ok := strings.CutSuffix(path, sbomPathSuffix) + if !ok { + badRequest(w, "path must be /api/projects/{project}/sbom") + return + } + if err := inventory.ValidateProjectName(name); err != nil { + badRequest(w, err.Error()) + return + } + ref, branch := r.URL.Query().Get("ref"), r.URL.Query().Get("branch") + if len(ref) > maxRefLength || len(branch) > maxRefLength { + badRequest(w, "ref and branch must not be longer than 255 characters") + return + } + + data, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxSBOMBytes)) + var tooLarge *http.MaxBytesError + switch { + case errors.As(err, &tooLarge): + writeError(w, http.StatusRequestEntityTooLarge, ErrCodeBadRequest, "SBOM is larger than 32 MiB") + return + case err != nil: + badRequest(w, "failed to read request body") + return + case len(data) == 0: + badRequest(w, "request body is empty") + return + } + + components, skipped, err := inventory.Parse(data) + if err != nil { + badRequest(w, "request body is not a CycloneDX or SPDX SBOM: "+err.Error()) + return + } + records := make([]database.ProjectComponent, len(components)) + for i, c := range components { + records[i] = database.ProjectComponent{ + PackagePURL: c.PackagePURL, Ecosystem: c.Ecosystem, Name: c.Name, Version: c.Version, + } + } + if err := h.db.SaveProject(name, ref, branch, time.Now(), records); err != nil { + h.logger.Error("failed to save project", "project", name, "error", err) + internalError(w, "failed to save project") + return + } + h.logger.Info("recorded project SBOM", "project", name, "ref", ref, + "components", len(records), "skipped", skipped) + writeJSON(w, ProjectUploadResponse{Project: name, Components: len(records), Skipped: skipped}) +} + +// HandleDelete handles DELETE /api/projects/{project}. +// @Summary Delete a project +// @Description Removes the project and the package versions recorded for it. +// @Tags projects +// @Security BearerAuth +// @Param project path string true "Project path, e.g. group/sub/repo" +// @Success 204 +// @Failure 400 {object} ErrorResponse +// @Failure 401 {object} ErrorResponse +// @Failure 404 {object} ErrorResponse +// @Router /api/projects/{project} [delete] +func (h *ProjectsAPIHandler) HandleDelete(w http.ResponseWriter, r *http.Request) { + name, ok := validProjectPath(w, r) + if !ok { + return + } + deleted, err := h.db.DeleteProject(name) + switch { + case err != nil: + h.logger.Error("failed to delete project", "project", name, "error", err) + internalError(w, "failed to delete project") + case !deleted: + notFound(w, "project not found") + default: + h.logger.Info("deleted project", "project", name) + w.WriteHeader(http.StatusNoContent) + } +} + +// HandleList handles GET /api/projects. +// @Summary List projects +// @Description Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses. +// @Tags projects +// @Produce json +// @Param purl query string false "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20" +// @Success 200 {object} ProjectListResponse +// @Failure 400 {object} ErrorResponse +// @Router /api/projects [get] +func (h *ProjectsAPIHandler) HandleList(w http.ResponseWriter, r *http.Request) { + if query := r.URL.Query().Get("purl"); query != "" { + h.listUsages(w, query) + return + } + projects, err := h.db.ListProjects() + if err != nil { + h.logger.Error("failed to list projects", "error", err) + internalError(w, "failed to list projects") + return + } + resp := ProjectListResponse{Projects: make([]ProjectSummary, len(projects))} + for i, p := range projects { + resp.Projects[i] = projectSummary(p) + } + writeJSON(w, resp) +} + +func (h *ProjectsAPIHandler) listUsages(w http.ResponseWriter, query string) { + c, err := inventory.FromPURL(query) + if err != nil { + badRequest(w, "invalid purl: "+err.Error()) + return + } + usages, err := h.db.GetProjectsUsingPackage(c.PackagePURL, c.Version) + if err != nil { + h.logger.Error("failed to look up package usage", "purl", query, "error", err) + internalError(w, "failed to look up package usage") + return + } + resp := ProjectListResponse{Projects: make([]ProjectSummary, len(usages))} + for i, u := range usages { + resp.Projects[i] = projectSummary(u.Project) + resp.Projects[i].Version = u.Version + } + writeJSON(w, resp) +} + +// HandleGet handles GET /api/projects/{project}. +// @Summary Get a project +// @Description Returns the project with the package versions recorded for it. +// @Tags projects +// @Produce json +// @Param project path string true "Project path, e.g. group/sub/repo" +// @Success 200 {object} ProjectResponse +// @Failure 400 {object} ErrorResponse +// @Failure 404 {object} ErrorResponse +// @Router /api/projects/{project} [get] +func (h *ProjectsAPIHandler) HandleGet(w http.ResponseWriter, r *http.Request) { + name, ok := validProjectPath(w, r) + if !ok { + return + } + project, err := h.db.GetProject(name) + if err != nil { + h.logger.Error("failed to get project", "project", name, "error", err) + internalError(w, "failed to get project") + return + } + if project == nil { + notFound(w, "project not found") + return + } + components, err := h.db.GetProjectComponents(project.ID) + if err != nil { + h.logger.Error("failed to get project components", "project", name, "error", err) + internalError(w, "failed to get project components") + return + } + resp := ProjectResponse{ProjectSummary: projectSummary(*project), Components: make([]ProjectComponent, len(components))} + for i, c := range components { + resp.Components[i] = ProjectComponent{ + Ecosystem: c.Ecosystem, + Name: c.Name, + Version: c.Version, + PURL: packageurl.WithVersionString(c.PackagePURL, c.Version), + } + } + writeJSON(w, resp) +} + +func projectSummary(p database.Project) ProjectSummary { + return ProjectSummary{ + Name: p.Name, + Ref: p.Ref.String, + Branch: p.Branch.String, + UploadedAt: p.UploadedAt, + ComponentCount: p.Components, + } +} + +// projectPath returns the decoded wildcard of a /api/projects or +// /ui/projects route. chi hands it over still encoded when the request path +// had escapes its canonical form doesn't (wildcardIsEncoded), e.g. a project +// name sent as one segment with %2F. +func projectPath(r *http.Request) (string, error) { + wildcard := chi.URLParam(r, "*") + if !wildcardIsEncoded(r) { + return wildcard, nil + } + decoded, err := url.PathUnescape(wildcard) + if err != nil { + return "", errors.New("invalid escape in project path") + } + return decoded, nil +} + +// validProjectPath returns the project named by the route, answering 400 if +// it is not a valid project name. +func validProjectPath(w http.ResponseWriter, r *http.Request) (string, bool) { + name, err := projectPath(r) + if err == nil { + err = inventory.ValidateProjectName(name) + } + if err != nil { + badRequest(w, err.Error()) + return "", false + } + return name, true +} diff --git a/internal/server/projects_api_test.go b/internal/server/projects_api_test.go new file mode 100644 index 0000000..880a4cf --- /dev/null +++ b/internal/server/projects_api_test.go @@ -0,0 +1,354 @@ +package server + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/go-chi/chi/v5" +) + +const testProjectsToken = "sbom-token" + +// syftSBOM is a CycloneDX document syft produced for a directory with an +// npm, Composer and Go lock file and an Alpine package database. It lists +// ten packages; five entries (the OS and the lock files) have no PURL. +func syftSBOM(t *testing.T) []byte { + t.Helper() + data, err := os.ReadFile("../inventory/testdata/syft-cyclonedx.json") + if err != nil { + t.Fatal(err) + } + return data +} + +type projectsTestServer struct { + server *Server + router http.Handler + db *database.DB +} + +// newProjectsTestServer routes the project API and the UI pages that show +// the inventory, with projects.enabled set. +func newProjectsTestServer(t *testing.T) projectsTestServer { + t.Helper() + db, err := database.Create(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("database.Create: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + cfg := &config.Config{Projects: config.ProjectsConfig{Enabled: true, Token: testProjectsToken}} + s := &Server{cfg: cfg, db: db, logger: logger, templates: &Templates{}} + + r := chi.NewRouter() + registerProjectRoutes(r, NewProjectsAPIHandler(db, logger), testProjectsToken) + r.Route("/ui", func(ui chi.Router) { + ui.Get("/", s.handleRoot) + ui.Get("/package/{ecosystem}/*", s.handlePackagePath) + s.mountProjectUI(ui) + }) + return projectsTestServer{server: s, router: r, db: db} +} + +func (ts projectsTestServer) do(t *testing.T, method, target, token string, body []byte) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(method, target, bytes.NewReader(body)) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + ts.router.ServeHTTP(rec, req) + return rec +} + +func (ts projectsTestServer) upload(t *testing.T, target string, body []byte) ProjectUploadResponse { + t.Helper() + rec := ts.do(t, http.MethodPut, target, testProjectsToken, body) + if rec.Code != http.StatusOK { + t.Fatalf("PUT %s = %d: %s", target, rec.Code, rec.Body) + } + var resp ProjectUploadResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decoding upload response: %v", err) + } + return resp +} + +func decodeJSON(t *testing.T, rec *httptest.ResponseRecorder, v any) { + t.Helper() + if rec.Code != http.StatusOK { + t.Fatalf("status = %d: %s", rec.Code, rec.Body) + } + if err := json.Unmarshal(rec.Body.Bytes(), v); err != nil { + t.Fatalf("decoding response: %v", err) + } +} + +func TestProjectUploadAndGet(t *testing.T) { + ts := newProjectsTestServer(t) + + resp := ts.upload(t, "/api/projects/acme/shop/web/sbom?ref=0123abcd&branch=main", syftSBOM(t)) + if resp != (ProjectUploadResponse{Project: "acme/shop/web", Components: 10, Skipped: 5}) { + t.Errorf("upload response = %+v", resp) + } + + // The same project, URL-encoded as one segment. + var project ProjectResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme%2Fshop%2Fweb", "", nil), &project) + if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 10 { + t.Errorf("project = %+v", project.ProjectSummary) + } + var purls []string + for _, c := range project.Components { + purls = append(purls, c.PURL) + } + for _, want := range []string{"pkg:npm/%40babel/core@7.23.0", "pkg:composer/symfony/console@v6.4.1", "pkg:golang/github.com/spf13/cobra@v1.8.0", "pkg:apk/alpine/busybox@1.36.1-r15"} { + if !strings.Contains(strings.Join(purls, " "), want) { + t.Errorf("components %v do not include %s", purls, want) + } + } + + var list ProjectListResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects", "", nil), &list) + if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 10 { + t.Errorf("list = %+v", list) + } +} + +func TestProjectUploadReplacesComponents(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme%2Fweb/sbom?ref=one", syftSBOM(t)) + resp := ts.upload(t, "/api/projects/acme%2Fweb/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + if resp.Components != 1 || resp.Skipped != 0 { + t.Errorf("upload response = %+v", resp) + } + var project ProjectResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil), &project) + if len(project.Components) != 1 || project.Components[0].PURL != "pkg:npm/lodash@4.17.21" || project.Ref != "" { + t.Errorf("project after second upload = %+v", project) + } +} + +func TestProjectLookupByPURL(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + ts.upload(t, "/api/projects/acme/api/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + + tests := []struct { + purl string + want []string + }{ + {"pkg:npm/lodash", []string{"acme/api@4.17.21", "acme/web@4.17.20"}}, + {"pkg:npm/lodash@4.17.20", []string{"acme/web@4.17.20"}}, + {"pkg:npm/lodash@1.0.0", nil}, + {"pkg:npm/%40babel/core", []string{"acme/web@7.23.0"}}, + {"pkg:apk/alpine/musl@1.2.4-r2?arch=x86_64", []string{"acme/web@1.2.4-r2"}}, + } + for _, tt := range tests { + t.Run(tt.purl, func(t *testing.T) { + var list ProjectListResponse + decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects?purl="+url.QueryEscape(tt.purl), "", nil), &list) + var got []string + for _, p := range list.Projects { + got = append(got, p.Name+"@"+p.Version) + } + if strings.Join(got, ",") != strings.Join(tt.want, ",") { + t.Errorf("projects = %v, want %v", got, tt.want) + } + }) + } + + for _, bad := range []string{"lodash", "pkg:github/actions/checkout@v4"} { + if rec := ts.do(t, http.MethodGet, "/api/projects?purl="+url.QueryEscape(bad), "", nil); rec.Code != http.StatusBadRequest { + t.Errorf("purl %q status = %d, want 400", bad, rec.Code) + } + } +} + +func TestProjectWritesRequireToken(t *testing.T) { + ts := newProjectsTestServer(t) + for _, token := range []string{"", "wrong-token"} { + if rec := ts.do(t, http.MethodPut, "/api/projects/acme/web/sbom", token, syftSBOM(t)); rec.Code != http.StatusUnauthorized { + t.Errorf("upload with token %q status = %d, want 401", token, rec.Code) + } + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme/web", token, nil); rec.Code != http.StatusUnauthorized { + t.Errorf("delete with token %q status = %d, want 401", token, rec.Code) + } + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("project exists after unauthorized upload: status %d", rec.Code) + } +} + +func TestProjectDelete(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme%2Fweb", testProjectsToken, nil); rec.Code != http.StatusNoContent { + t.Fatalf("delete status = %d: %s", rec.Code, rec.Body) + } + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme/web", testProjectsToken, nil); rec.Code != http.StatusNotFound { + t.Errorf("second delete status = %d, want 404", rec.Code) + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("get after delete status = %d, want 404", rec.Code) + } +} + +func TestProjectPathsAreValidated(t *testing.T) { + ts := newProjectsTestServer(t) + uploads := []string{ + "/api/projects/sbom", + "/api/projects/acme/web", + "/api/projects/acme/../web/sbom", + "/api/projects/acme%2F..%2Fweb/sbom", + "/api/projects/..%2Fweb/sbom", + "/api/projects/acme//web/sbom", + "/api/projects/acme/./web/sbom", + "/api/projects/acme%20web/sbom", + "/api/projects/acme%2Fweb%3F/sbom", + "/api/projects/" + strings.Repeat("a", 256) + "/sbom", + } + for _, target := range uploads { + if rec := ts.do(t, http.MethodPut, target, testProjectsToken, syftSBOM(t)); rec.Code != http.StatusBadRequest { + t.Errorf("PUT %s status = %d, want 400", target, rec.Code) + } + } + for _, target := range []string{"/api/projects/", "/api/projects/acme/../web", "/api/projects/acme%2F..%2Fweb"} { + if rec := ts.do(t, http.MethodGet, target, "", nil); rec.Code != http.StatusBadRequest { + t.Errorf("GET %s status = %d, want 400", target, rec.Code) + } + if rec := ts.do(t, http.MethodDelete, target, testProjectsToken, nil); rec.Code != http.StatusBadRequest { + t.Errorf("DELETE %s status = %d, want 400", target, rec.Code) + } + } + if projects, err := ts.db.ListProjects(); err != nil || len(projects) != 0 { + t.Errorf("projects after invalid uploads = %v, %v", projects, err) + } +} + +func TestProjectUploadRejectsBadBodies(t *testing.T) { + ts := newProjectsTestServer(t) + tests := []struct { + name string + target string + body []byte + want int + }{ + {"empty", "/api/projects/acme/web/sbom", nil, http.StatusBadRequest}, + {"not an SBOM", "/api/projects/acme/web/sbom", []byte(`{"hello":"world"}`), http.StatusBadRequest}, + {"too large", "/api/projects/acme/web/sbom", make([]byte, maxSBOMBytes+1), http.StatusRequestEntityTooLarge}, + {"long ref", "/api/projects/acme/web/sbom?ref=" + strings.Repeat("a", maxRefLength+1), syftSBOM(t), http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if rec := ts.do(t, http.MethodPut, tt.target, testProjectsToken, tt.body); rec.Code != tt.want { + t.Errorf("status = %d, want %d: %s", rec.Code, tt.want, rec.Body) + } + }) + } +} + +// startProjectsServer runs the real server with the given projects config. +func startProjectsServer(t *testing.T, projects config.ProjectsConfig) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("reserving proxy address: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + + tempDir := t.TempDir() + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(tempDir, "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") + cfg.Projects = projects + if err := cfg.Validate(); err != nil { + t.Fatalf("validating config: %v", err) + } + + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{Version: "test", Commit: "test"}) + if err != nil { + t.Fatalf("creating server: %v", err) + } + startErr := make(chan error, 1) + go func() { startErr <- proxyServer.Start(listener) }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = proxyServer.Shutdown(ctx) + if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("server stopped with %v", err) + } + }) + return cfg.BaseURL +} + +func request(t *testing.T, method, url, token string, body []byte) (int, string) { + t.Helper() + req, err := http.NewRequest(method, url, bytes.NewReader(body)) + if err != nil { + t.Fatalf("building request: %v", err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, url, err) + } + defer func() { _ = resp.Body.Close() }() + data, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(data) +} + +func TestProjectRoutesFollowConfig(t *testing.T) { + disabled := startProjectsServer(t, config.ProjectsConfig{}) + for _, path := range []string{"/api/projects", "/api/projects/acme/web", "/ui/projects"} { + if status, _ := request(t, http.MethodGet, disabled+path, "", nil); status != http.StatusNotFound { + t.Errorf("GET %s with projects disabled = %d, want 404", path, status) + } + } + if status, _ := request(t, http.MethodPut, disabled+"/api/projects/acme/web/sbom", testProjectsToken, syftSBOM(t)); status != http.StatusNotFound { + t.Errorf("upload with projects disabled = %d, want 404", status) + } + if _, body := request(t, http.MethodGet, disabled+"/ui/", "", nil); strings.Contains(body, `href="/ui/projects"`) { + t.Error("navigation links to projects while they are disabled") + } + + enabled := startProjectsServer(t, config.ProjectsConfig{Enabled: true, Token: testProjectsToken, MaxAge: "90d"}) + if status, _ := request(t, http.MethodPut, enabled+"/api/projects/acme%2Fweb/sbom", "", syftSBOM(t)); status != http.StatusUnauthorized { + t.Errorf("upload without token = %d, want 401", status) + } + if status, body := request(t, http.MethodPut, enabled+"/api/projects/acme%2Fweb/sbom", testProjectsToken, syftSBOM(t)); status != http.StatusOK { + t.Fatalf("upload = %d: %s", status, body) + } + if status, body := request(t, http.MethodGet, enabled+"/api/projects/acme/web", "", nil); status != http.StatusOK || !strings.Contains(body, `"name":"acme/web"`) { + t.Errorf("GET project = %d: %s", status, body) + } + if status, body := request(t, http.MethodGet, enabled+"/ui/projects", "", nil); status != http.StatusOK || !strings.Contains(body, `href="/ui/projects/acme/web"`) { + t.Errorf("projects page = %d, missing the project link", status) + } + if _, body := request(t, http.MethodGet, enabled+"/ui/", "", nil); !strings.Contains(body, `href="/ui/projects"`) { + t.Error("navigation does not link to projects") + } +} diff --git a/internal/server/projects_prune.go b/internal/server/projects_prune.go new file mode 100644 index 0000000..b9ba936 --- /dev/null +++ b/internal/server/projects_prune.go @@ -0,0 +1,43 @@ +package server + +import ( + "context" + "time" +) + +// projectPruneInterval is how often projects older than projects.max_age +// are removed. +const projectPruneInterval = time.Hour + +// startProjectPruning removes projects that have not uploaded an SBOM for +// projects.max_age, once at startup and then every projectPruneInterval. +func (s *Server) startProjectPruning(ctx context.Context) { + maxAge := s.cfg.Projects.MaxAgeDuration() + if !s.cfg.Projects.Enabled || maxAge <= 0 { + return + } + prune := func() { + s.runExclusive(ctx, "project-pruning", func() { + pruned, err := s.db.PruneProjects(time.Now().Add(-maxAge)) + if err != nil { + s.logger.Warn("pruning projects failed", "error", err) + } + if pruned > 0 { + s.logger.Info("pruned projects", "count", pruned, "max_age", maxAge) + } + }) + } + s.background.Go(func() { + prune() + ticker := time.NewTicker(projectPruneInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + prune() + } + } + }) +} diff --git a/internal/server/projects_prune_test.go b/internal/server/projects_prune_test.go new file mode 100644 index 0000000..0f74d55 --- /dev/null +++ b/internal/server/projects_prune_test.go @@ -0,0 +1,48 @@ +package server + +import ( + "context" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" +) + +func TestProjectPruningRemovesStaleProjects(t *testing.T) { + ts := newProjectsTestServer(t) + ts.server.cfg.Projects = config.ProjectsConfig{Enabled: true, Token: testProjectsToken, MaxAge: "30d"} + if err := ts.server.cfg.Projects.Validate(); err != nil { + t.Fatal(err) + } + if err := ts.db.SaveProject("acme/old", "", "", time.Now().Add(-31*24*time.Hour), nil); err != nil { + t.Fatal(err) + } + if err := ts.db.SaveProject("acme/new", "", "", time.Now(), nil); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + ts.server.startProjectPruning(ctx) + defer func() { + cancel() + ts.server.background.Wait() + }() + + deadline := time.Now().Add(5 * time.Second) + for { + old, err := ts.db.GetProject("acme/old") + if err != nil { + t.Fatal(err) + } + if old == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("acme/old was not pruned") + } + time.Sleep(10 * time.Millisecond) + } + if current, err := ts.db.GetProject("acme/new"); err != nil || current == nil { + t.Errorf("acme/new was pruned: %v, %v", current, err) + } +} diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go new file mode 100644 index 0000000..76ae8bd --- /dev/null +++ b/internal/server/projects_ui.go @@ -0,0 +1,287 @@ +package server + +import ( + "cmp" + "net/http" + "net/url" + "slices" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/inventory" + "github.com/go-chi/chi/v5" +) + +// ProjectsListData contains data for rendering the projects list page. +type ProjectsListData struct { + Layout + Projects []ProjectRow +} + +// ProjectRow is a project in the UI. +type ProjectRow struct { + Name string + Ref string + Branch string + UploadedAt string + Components int64 +} + +// ProjectShowData contains data for rendering a project's page. +type ProjectShowData struct { + Layout + Project ProjectRow + Components []ProjectComponentRow + Vulnerable int + Outdated int +} + +// ProjectComponentRow is a package version a project uses, with what the +// proxy knows about it. +type ProjectComponentRow struct { + Ecosystem string + Name string + Version string + // Cached is set when the proxy has a package record, so the package + // page exists; CachedEcosystem and CachedName address it. + Cached bool + CachedEcosystem string + CachedName string + LatestVersion string + Outdated bool + VulnCount int + Severity string +} + +// VersionUsage lists the projects using one version of a package. +type VersionUsage struct { + Version string + EscapedVersion string + // Cached is set when the proxy has a record of the version. + Cached bool + Projects []string +} + +// VulnerableVersion is a package version with known vulnerabilities that +// projects use. +type VulnerableVersion struct { + Ecosystem string + Name string + Version string + Severity string + VulnCount int + Projects []string +} + +// mountProjectUI registers the project inventory pages when +// projects.enabled is set. +func (s *Server) mountProjectUI(ui chi.Router) { + if !s.cfg.Projects.Enabled { + return + } + ui.Get("/projects", s.handleProjectsList) + ui.Get("/projects/*", s.handleProjectShow) +} + +func (s *Server) handleProjectsList(w http.ResponseWriter, r *http.Request) { + projects, err := s.db.ListProjects() + if err != nil { + s.logger.Error("failed to list projects", "error", err) + http.Error(w, "failed to list projects", http.StatusInternalServerError) + return + } + data := ProjectsListData{Layout: s.layoutFor(r), Projects: make([]ProjectRow, len(projects))} + for i, p := range projects { + data.Projects[i] = projectRow(p) + } + if err := s.templates.Render(w, "projects_list", data); err != nil { + s.logger.Error("failed to render projects list", "error", err) + } +} + +func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { + name, err := projectPath(r) + if err == nil { + err = inventory.ValidateProjectName(name) + } + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + project, err := s.db.GetProject(name) + if err != nil { + s.logger.Error("failed to get project", "project", name, "error", err) + http.Error(w, "failed to get project", http.StatusInternalServerError) + return + } + if project == nil { + http.Error(w, "project not found", http.StatusNotFound) + return + } + components, err := s.db.GetProjectComponents(project.ID) + if err != nil { + s.logger.Error("failed to get project components", "project", name, "error", err) + components = nil + } + vulns, err := s.db.GetComponentVulnerabilities(name) + if err != nil { + s.logger.Error("failed to get project vulnerabilities", "project", name, "error", err) + } + affecting := affectingVulnerabilities(vulns) + + data := ProjectShowData{Layout: s.layoutFor(r), Project: projectRow(*project)} + for _, c := range components { + row := ProjectComponentRow{ + Ecosystem: c.Ecosystem, + Name: c.Name, + Version: c.Version, + Cached: c.CachedName.Valid, + CachedEcosystem: c.CachedEcosystem.String, + CachedName: c.CachedName.String, + LatestVersion: c.LatestVersion.String, + Outdated: inventory.Outdated(c.Ecosystem, c.Version, c.LatestVersion.String), + } + if v, ok := affecting[componentKey{c.PackagePURL, c.Version}]; ok { + row.VulnCount, row.Severity = len(v.ids), v.severity + data.Vulnerable++ + } + if row.Outdated { + data.Outdated++ + } + data.Components = append(data.Components, row) + } + if err := s.templates.Render(w, "project_show", data); err != nil { + s.logger.Error("failed to render project page", "error", err) + } +} + +func projectRow(p database.Project) ProjectRow { + return ProjectRow{ + Name: p.Name, + Ref: p.Ref.String, + Branch: p.Branch.String, + UploadedAt: formatTimeAgo(p.UploadedAt), + Components: p.Components, + } +} + +// packageUsedBy lists the projects using each version of pkg, highest +// version first, or nothing when the project inventory is disabled. versions +// are the versions the proxy has records of; a version only projects know of +// is listed too. +func (s *Server) packageUsedBy(pkg *database.Package, versions []database.Version) []VersionUsage { + if !s.cfg.Projects.Enabled { + return nil + } + usages, err := s.db.GetProjectsUsingPackage(pkg.PURL, "") + if err != nil { + s.logger.Error("failed to get projects using package", "purl", pkg.PURL, "error", err) + return nil + } + known := make(map[string]bool, len(versions)) + for i := range versions { + known[versions[i].Version()] = true + } + byVersion := map[string]*VersionUsage{} + var result []*VersionUsage + for _, u := range usages { + vu, ok := byVersion[u.Version] + if !ok { + vu = &VersionUsage{Version: u.Version, EscapedVersion: url.PathEscape(u.Version), Cached: known[u.Version]} + byVersion[u.Version] = vu + result = append(result, vu) + } + vu.Projects = append(vu.Projects, u.Name) + } + slices.SortFunc(result, func(a, b *VersionUsage) int { + return inventory.CompareVersions(pkg.Ecosystem, b.Version, a.Version) + }) + out := make([]VersionUsage, len(result)) + for i, vu := range result { + out[i] = *vu + } + return out +} + +// vulnerableVersionsInUse returns the package versions projects use that +// have vulnerabilities, most severe and most used first, at most limit. It +// returns nothing when the project inventory is disabled. +func (s *Server) vulnerableVersionsInUse(limit int) []VulnerableVersion { + if !s.cfg.Projects.Enabled { + return nil + } + vulns, err := s.db.GetComponentVulnerabilities("") + if err != nil { + s.logger.Error("failed to get vulnerable versions in use", "error", err) + return nil + } + affecting := affectingVulnerabilities(vulns) + result := make([]VulnerableVersion, 0, len(affecting)) + for _, v := range affecting { + projects := make([]string, 0, len(v.projects)) + for p := range v.projects { + projects = append(projects, p) + } + slices.Sort(projects) + result = append(result, VulnerableVersion{ + Ecosystem: v.ecosystem, Name: v.name, Version: v.version, + Severity: v.severity, VulnCount: len(v.ids), Projects: projects, + }) + } + slices.SortFunc(result, func(a, b VulnerableVersion) int { + return cmp.Or( + cmp.Compare(severityRank(b.Severity), severityRank(a.Severity)), + cmp.Compare(len(b.Projects), len(a.Projects)), + cmp.Compare(a.Name, b.Name), + cmp.Compare(a.Version, b.Version), + ) + }) + if len(result) > limit { + result = result[:limit] + } + return result +} + +type componentKey struct{ packagePURL, version string } + +// affectingVersion collects the vulnerabilities affecting one package +// version, and the projects using it. +type affectingVersion struct { + ecosystem, name, version string + severity string + ids map[string]bool + projects map[string]bool +} + +// affectingVulnerabilities keeps the vulnerabilities that affect the version +// each component uses, grouped by package version. +func affectingVulnerabilities(vulns []database.ComponentVulnerability) map[componentKey]*affectingVersion { + out := map[componentKey]*affectingVersion{} + for _, v := range vulns { + if !inventory.Affected(v.Ecosystem, v.Version, v.FixedVersion.String) { + continue + } + key := componentKey{v.PackagePURL, v.Version} + a, ok := out[key] + if !ok { + a = &affectingVersion{ + ecosystem: v.CachedEcosystem, name: v.CachedName, version: v.Version, + ids: map[string]bool{}, projects: map[string]bool{}, + } + out[key] = a + } + a.ids[v.VulnID] = true + a.projects[v.Project] = true + if severityRank(v.Severity.String) > severityRank(a.severity) { + a.severity = v.Severity.String + } + } + return out +} + +// severityOrder lists severities from least to most severe. +var severityOrder = []string{"low", "medium", "high", "critical"} + +// severityRank orders severities; unknown ones rank below "low". +func severityRank(severity string) int { + return slices.Index(severityOrder, severity) + 1 +} diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go new file mode 100644 index 0000000..18fc5c1 --- /dev/null +++ b/internal/server/projects_ui_test.go @@ -0,0 +1,137 @@ +package server + +import ( + "database/sql" + "net/http" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/database" +) + +// seedLodash records lodash with 4.17.21 as its latest and only cached +// version, and a vulnerability fixed in 4.17.21. The syft fixture uses +// 4.17.20. +func seedLodash(t *testing.T, db *database.DB, withVuln bool) { + t.Helper() + if err := db.UpsertPackage(&database.Package{ + PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", + LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } + if err := db.UpsertVersion(&database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}); err != nil { + t.Fatal(err) + } + if !withVuln { + return + } + if err := db.UpsertVulnerability(&database.Vulnerability{ + VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", + Severity: sql.NullString{String: "high", Valid: true}, + FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, + }); err != nil { + t.Fatal(err) + } +} + +func (ts projectsTestServer) page(t *testing.T, target string) string { + t.Helper() + rec := ts.do(t, http.MethodGet, target, "", nil) + if rec.Code != http.StatusOK { + t.Fatalf("GET %s = %d: %s", target, rec.Code, rec.Body) + } + return rec.Body.String() +} + +func assertContains(t *testing.T, page string, want ...string) { + t.Helper() + for _, w := range want { + if !strings.Contains(page, w) { + t.Errorf("page does not contain %q", w) + } + } +} + +func TestProjectPageShowsComponents(t *testing.T) { + ts := newProjectsTestServer(t) + seedLodash(t, ts.db, true) + ts.upload(t, "/api/projects/acme/web/sbom?branch=main", syftSBOM(t)) + + page := ts.page(t, "/ui/projects/acme/web") + assertContains(t, page, + "acme/web", + `href="/ui/package/npm/lodash"`, // cached, so linked + "outdated, latest 4.17.21", + "1 vulns", + "1 vulnerable", + "symfony/console", // not cached: listed without a link + "not cached", + ) + if strings.Contains(page, `href="/ui/package/composer/symfony/console"`) { + t.Error("links to the package page of a package the proxy has no record of") + } + + // Encoded as one segment, too. + assertContains(t, ts.page(t, "/ui/projects/acme%2Fweb"), "symfony/console") + + if rec := ts.do(t, http.MethodGet, "/ui/projects/acme/other", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("unknown project status = %d, want 404", rec.Code) + } + if rec := ts.do(t, http.MethodGet, "/ui/projects/acme/../web", "", nil); rec.Code != http.StatusBadRequest { + t.Errorf("invalid project status = %d, want 400", rec.Code) + } + + list := ts.page(t, "/ui/projects") + assertContains(t, list, `href="/ui/projects/acme/web"`, "10 packages") +} + +func TestPackagePageShowsUsedBy(t *testing.T) { + ts := newProjectsTestServer(t) + seedLodash(t, ts.db, false) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + ts.upload(t, "/api/projects/acme/api/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + + page := ts.page(t, "/ui/package/npm/lodash") + assertContains(t, page, + "Used by", + `href="/ui/package/npm/lodash/4.17.21"`, + `href="/ui/projects/acme/api"`, + `href="/ui/projects/acme/web"`, + ) + // 4.17.20 is used but not cached. + start := strings.Index(page, "Used by") + if start < 0 { + t.Fatal("no Used by section") + } + used := page[start:] + if i := strings.Index(used, "4.17.20"); i < 0 || !strings.Contains(used[i:], "not cached") { + t.Error("the uncached version 4.17.20 is not listed as not cached") + } + if strings.Index(used, "4.17.21") > strings.Index(used, "4.17.20") { + t.Error("versions are not listed highest first") + } +} + +func TestDashboardShowsVulnerableVersionsInUse(t *testing.T) { + ts := newProjectsTestServer(t) + seedLodash(t, ts.db, false) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + + if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { + t.Error("dashboard shows the vulnerable versions card without vulnerabilities") + } + assertContains(t, ts.page(t, "/ui/"), `href="/ui/projects"`) + + seedLodash(t, ts.db, true) + page := ts.page(t, "/ui/") + assertContains(t, page, "Vulnerable versions in use", "@4.17.20", `href="/ui/projects/acme/web"`, ">high<") + + // A project on the fixed version doesn't count. + ts.upload(t, "/api/projects/acme/web/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) + if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { + t.Error("dashboard lists a version the vulnerability is fixed in") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 12619a6..d96b9dc 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -39,6 +39,7 @@ // - /ui/packages - List all cached packages // - /ui/search - Search packages // - /ui/package/... - Package and version detail pages +// - /ui/projects/... - Project inventory (when projects.enabled) // - /ui/api/browse/... - Archive browsing (used by the UI) // - /ui/api/compare/... - Archive diffing (used by the UI) // @@ -50,6 +51,12 @@ // - POST /api/outdated - Check outdated packages // - POST /api/bulk - Bulk package lookup // - GET /api/packages - List cached packages (JSON) +// +// Project inventory endpoints (when projects.enabled): +// - PUT /api/projects/{project}/sbom - Upload a project's SBOM (token) +// - DELETE /api/projects/{project} - Delete a project (token) +// - GET /api/projects - List projects, or those using ?purl= +// - GET /api/projects/{project} - A project's package versions package server import ( @@ -357,6 +364,7 @@ func (s *Server) serve(listener net.Listener) error { if s.cfg.CacheDelete { ui.Delete("/api/cache/{ecosystem}/*", s.handleCacheDelete) } + s.mountProjectUI(ui) }) r.Get("/", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/ui/", http.StatusFound) @@ -404,6 +412,10 @@ func (s *Server) serve(listener net.Listener) error { registerHostedRoutes(r, hostedAPI, s.cfg.Hosted.TokenExpanded()) } + // Project inventory endpoints (opt-in via projects.enabled or PROXY_PROJECTS_ENABLED) + s.mountProjectAPI(r) + s.startProjectPruning(bgCtx) + s.http = &http.Server{ Addr: s.cfg.Listen, Handler: r, @@ -833,6 +845,7 @@ func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) { }, EnrichmentStats: enrichmentStatsView(enrichStats), } + data.VulnerableInUse = s.vulnerableVersionsInUse(dashboardTopN) for _, p := range popular { pkgInfo := PackageInfo{ @@ -1170,6 +1183,7 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, Vulnerabilities: vulns, LicenseCategory: categorizeLicense(pkg.License), } + data.UsedBy = s.packageUsedBy(pkg, versions) if err := s.templates.Render(w, "package_show", data); err != nil { s.logger.Error("failed to render package show", "error", err) diff --git a/internal/server/templates/components/severity_badge.html b/internal/server/templates/components/severity_badge.html new file mode 100644 index 0000000..a060ed2 --- /dev/null +++ b/internal/server/templates/components/severity_badge.html @@ -0,0 +1,3 @@ +{{define "severity_badge"}} +{{if .}}{{.}}{{else}}unknown{{end}} +{{end}} diff --git a/internal/server/templates/layout/header.html b/internal/server/templates/layout/header.html index 560fe2b..8b4d94f 100644 --- a/internal/server/templates/layout/header.html +++ b/internal/server/templates/layout/header.html @@ -54,6 +54,7 @@ {{end}} {{define "nav_links"}} +{{if .Projects}}Projects{{end}} Analytics Install Health diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index e48e205..ff9bc1b 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -25,6 +25,33 @@ {{template "security_overview" .EnrichmentStats}} {{end}} +{{if .VulnerableInUse}} +
+
+

Vulnerable versions in use

+
+
+ {{range .VulnerableInUse}} +
+
+ {{template "ecosystem_badge" .Ecosystem}} + {{.Name}} + @{{.Version}} + {{template "severity_badge" .Severity}} + {{.VulnCount}} vulns +
+
+ {{range .Projects}}{{.}}{{end}} +
+
+ {{end}} +
+
+ See all projects → +
+
+{{end}} +
diff --git a/internal/server/templates/pages/package_show.html b/internal/server/templates/pages/package_show.html index a2724a8..4c5a0fc 100644 --- a/internal/server/templates/pages/package_show.html +++ b/internal/server/templates/pages/package_show.html @@ -49,6 +49,34 @@ {{end}}
+{{if .UsedBy}} +
+
+

Used by

+

Projects whose latest SBOM lists this package

+
+
+ {{range .UsedBy}} +
+
+ {{if .Cached}} + {{if .Version}}{{.Version}}{{else}}unknown version{{end}} + {{else}} + {{if .Version}}{{.Version}}{{else}}unknown version{{end}} + not cached + {{end}} +
+
+ {{range .Projects}} + {{.}} + {{end}} +
+
+ {{end}} +
+
+{{end}} + {{if .Versions}}
diff --git a/internal/server/templates/pages/project_show.html b/internal/server/templates/pages/project_show.html new file mode 100644 index 0000000..0125f9c --- /dev/null +++ b/internal/server/templates/pages/project_show.html @@ -0,0 +1,53 @@ +{{define "title"}}{{.Project.Name}} - git-pkgs proxy{{end}} + +{{define "content"}} +
+ ← Projects +

{{.Project.Name}}

+

+ {{.Project.Components}} packages, uploaded {{.Project.UploadedAt}}{{if .Project.Branch}} from {{.Project.Branch}}{{end}}{{if .Project.Ref}} at {{.Project.Ref}}{{end}} +

+ {{if or .Vulnerable .Outdated}} +
+ {{if .Vulnerable}}{{.Vulnerable}} vulnerable{{end}} + {{if .Outdated}}{{.Outdated}} outdated{{end}} +
+ {{end}} +
+ +{{if .Components}} +
+
+ {{range .Components}} +
+
+ {{template "ecosystem_badge" .Ecosystem}} + {{if .Cached}} + {{.Name}} + {{else}} + {{.Name}} + {{end}} + {{.Version}} +
+
+ {{if .VulnCount}} + {{template "severity_badge" .Severity}} + {{.VulnCount}} vulns + {{end}} + {{if .Outdated}} + outdated, latest {{.LatestVersion}} + {{end}} + {{if not .Cached}} + not cached + {{end}} +
+
+ {{end}} +
+
+{{else}} +
+

The latest SBOM lists no packages the proxy serves

+
+{{end}} +{{end}} diff --git a/internal/server/templates/pages/projects_list.html b/internal/server/templates/pages/projects_list.html new file mode 100644 index 0000000..f5ce62d --- /dev/null +++ b/internal/server/templates/pages/projects_list.html @@ -0,0 +1,33 @@ +{{define "title"}}Projects - git-pkgs proxy{{end}} + +{{define "content"}} +
+

Projects

+

{{len .Projects}} projects, as of their latest SBOM upload

+
+ +{{if .Projects}} +
+
+ {{range .Projects}} +
+
+ {{.Name}} + {{if or .Branch .Ref}} +
{{.Branch}}{{if and .Branch .Ref}} @ {{end}}{{.Ref}}
+ {{end}} +
+
+ {{.Components}} packages + {{.UploadedAt}} +
+
+ {{end}} +
+
+{{else}} +
+

No project has uploaded an SBOM yet

+
+{{end}} +{{end}} From 895f22a601b47f31539fa2a9d57e05a2da57b523 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 19:43:43 +0100 Subject: [PATCH 005/198] Skip the project's own packages in uploaded SBOMs A syft SBOM of a repository lists the repository itself: the module in go.mod and the root of package-lock.json, both without a version or with the version from package.json, and the module its Go binaries were built from. The project page showed them as dependencies. Entries without a version are now skipped, as are lock-file roots (lock entries that depend on others while nothing depends on them) and modules syft marks as a binary's main module. --- docs/configuration.md | 4 +- internal/inventory/inventory.go | 67 ++++++++++++++++++++++++++-- internal/inventory/inventory_test.go | 37 +++++++++++++-- internal/server/projects_api_test.go | 6 +-- internal/server/projects_ui_test.go | 2 +- 5 files changed, 103 insertions(+), 13 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c5c6f7a..3a5579d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1545,10 +1545,10 @@ curl -X PUT --data-binary @sbom.json \ The response counts the recorded package versions and the skipped entries: ```json -{"project":"acme/shop/web","components":10,"skipped":5} +{"project":"acme/shop/web","components":9,"skipped":6} ``` -Entries are recorded by their package URL. Entries without one (files, the operating system) and package types the proxy doesn't serve (such as `pkg:github`) are skipped. Duplicates are recorded once. +Entries are recorded by their package URL. Entries without one (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`) and entries without a version are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once. A GitLab CI job on the default branch could look like this: diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 954d46e..9df6ce4 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -91,18 +91,33 @@ func FromPURL(s string) (Component, error) { return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil } +// syftMainModule is the property syft sets on modules found in a Go binary, +// naming the module the binary was built from. +const syftMainModule = "syft:metadata:mainModule" + +// syftMetadataType is the property naming where syft read a package from. +const syftMetadataType = "syft:package:metadataType" + +// unknownVersion is what syft gives a package without a resolved version, +// such as the project's own module in go.mod or the root of a lock file. +const unknownVersion = "UNKNOWN" + // Parse reads a CycloneDX (JSON or XML) or SPDX SBOM and returns the package -// versions it lists, without duplicates. Packages without a package URL, or -// of a type the proxy doesn't serve, are counted in skipped. +// versions it lists, without duplicates. Packages without a package URL, of a +// type the proxy doesn't serve, or without a version are counted in skipped. +// So are the project's own packages: Go modules its binaries were built from, +// and the root of a lock file. func Parse(data []byte) (components []Component, skipped int, err error) { doc, err := sbom.Parse(data) if err != nil { return nil, 0, err } + ownModules := mainModules(doc.Packages) + lockRoots := lockFileRoots(doc) seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok { + if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { skipped++ continue } @@ -128,9 +143,55 @@ func componentOf(pkg sbom.Package) (Component, bool) { if c.Version == "" { c.Version = pkg.Version } + // A package without a version is the project itself (the module in + // go.mod, the root of package-lock.json), not something it depends on. + if c.Version == "" || c.Version == unknownVersion { + return Component{}, false + } return c, true } +// lockFileRoots returns the IDs of packages read from a lock file that depend +// on others while nothing depends on them: the project's own entry, such as +// the root of package-lock.json, which carries the version from package.json. +func lockFileRoots(doc *sbom.SBOM) map[string]bool { + dependents := map[string]bool{} + dependencies := map[string]bool{} + for _, r := range doc.Relationships { + if r.Type != sbom.RelDependsOn { + continue + } + dependencies[r.SourceID] = true + dependents[r.TargetID] = true + } + roots := map[string]bool{} + for _, pkg := range doc.Packages { + if pkg.ID == "" || !dependencies[pkg.ID] || dependents[pkg.ID] { + continue + } + for _, prop := range pkg.Properties { + if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { + roots[pkg.ID] = true + } + } + } + return roots +} + +// mainModules returns the Go modules the SBOM's binaries were built from, +// which are the project's own. +func mainModules(pkgs []sbom.Package) map[string]bool { + own := map[string]bool{} + for _, pkg := range pkgs { + for _, prop := range pkg.Properties { + if prop.Name == syftMainModule && prop.Value != "" { + own[prop.Value] = true + } + } + } + return own +} + // ValidateProjectName checks a project name such as "group/sub/repo": one or // more segments of letters, digits, ".", "_" and "-", none of them "." or // "..". diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 3d55d50..b4c6cc3 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -27,14 +27,14 @@ func TestParseSyftCycloneDX(t *testing.T) { {Ecosystem: "composer", Name: "monolog/monolog", Version: "3.5.0", PackagePURL: "pkg:composer/monolog/monolog"}, {Ecosystem: "alpine", Name: "musl", Version: "1.2.4-r2", PackagePURL: "pkg:apk/alpine/musl"}, {Ecosystem: "composer", Name: "symfony/console", Version: "v6.4.1", PackagePURL: "pkg:composer/symfony/console"}, - {Ecosystem: "npm", Name: "webapp", Version: "1.0.0", PackagePURL: "pkg:npm/webapp"}, } if !slices.Equal(components, want) { t.Errorf("components =\n%v\nwant\n%v", components, want) } - // The operating system and the four lock files syft lists carry no PURL. - if skipped != 5 { - t.Errorf("skipped = %d, want 5", skipped) + // The operating system and the four lock files syft lists carry no PURL, + // and webapp is the root of package-lock.json, the project itself. + if skipped != 6 { + t.Errorf("skipped = %d, want 6", skipped) } } @@ -131,3 +131,32 @@ func TestOutdatedAndAffected(t *testing.T) { t.Error("a vulnerability without a fix should affect every version") } } + +func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"mod","type":"library","name":"github.com/acme/app","version":"UNKNOWN","purl":"pkg:golang/github.com/acme/app"}, + {"bom-ref":"bin","type":"library","name":"github.com/acme/app","version":"v0.0.0-20261027162038-a3e9dd9fd3f4+dirty","purl":"pkg:golang/github.com/acme/app@v0.0.0-20261027162038-a3e9dd9fd3f4%2Bdirty", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/acme/app"}]}, + {"bom-ref":"cobra","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/acme/app"}]}, + {"bom-ref":"root","type":"library","name":"app-frontend","version":"2.1.0","purl":"pkg:npm/app-frontend@2.1.0", + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]}, + {"bom-ref":"nover","type":"library","name":"app-tools","purl":"pkg:npm/app-tools"}, + {"bom-ref":"jsdom","type":"library","name":"jsdom","version":"30.1.3","purl":"pkg:npm/jsdom@30.1.3", + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]} + ],"dependencies":[{"ref":"root","dependsOn":["jsdom"]}]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + want := []Component{ + {Ecosystem: "golang", Name: "github.com/spf13/cobra", Version: "v1.8.0", PackagePURL: "pkg:golang/github.com/spf13/cobra"}, + {Ecosystem: "npm", Name: "jsdom", Version: "30.1.3", PackagePURL: "pkg:npm/jsdom"}, + } + if !slices.Equal(components, want) { + t.Errorf("components =\n%v\nwant\n%v", components, want) + } + if skipped != 4 { + t.Errorf("skipped = %d, want 4", skipped) + } +} diff --git a/internal/server/projects_api_test.go b/internal/server/projects_api_test.go index 880a4cf..9a7d9b9 100644 --- a/internal/server/projects_api_test.go +++ b/internal/server/projects_api_test.go @@ -104,14 +104,14 @@ func TestProjectUploadAndGet(t *testing.T) { ts := newProjectsTestServer(t) resp := ts.upload(t, "/api/projects/acme/shop/web/sbom?ref=0123abcd&branch=main", syftSBOM(t)) - if resp != (ProjectUploadResponse{Project: "acme/shop/web", Components: 10, Skipped: 5}) { + if resp != (ProjectUploadResponse{Project: "acme/shop/web", Components: 9, Skipped: 6}) { t.Errorf("upload response = %+v", resp) } // The same project, URL-encoded as one segment. var project ProjectResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme%2Fshop%2Fweb", "", nil), &project) - if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 10 { + if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 9 { t.Errorf("project = %+v", project.ProjectSummary) } var purls []string @@ -126,7 +126,7 @@ func TestProjectUploadAndGet(t *testing.T) { var list ProjectListResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects", "", nil), &list) - if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 10 { + if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 9 { t.Errorf("list = %+v", list) } } diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index 18fc5c1..46dbaa4 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -83,7 +83,7 @@ func TestProjectPageShowsComponents(t *testing.T) { } list := ts.page(t, "/ui/projects") - assertContains(t, list, `href="/ui/projects/acme/web"`, "10 packages") + assertContains(t, list, `href="/ui/projects/acme/web"`, "9 packages") } func TestPackagePageShowsUsedBy(t *testing.T) { From bd05533bfff6e6876f08dac21786dd84f1900636 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 19:46:51 +0100 Subject: [PATCH 006/198] Delete projects from the project page The page asks for the upload token and sends it to DELETE /api/projects/{project}, so /ui gains no unauthenticated way to delete. The token is kept in session storage for the next delete. --- docs/configuration.md | 2 +- internal/server/projects_ui_test.go | 2 + internal/server/static/project-delete.js | 64 +++++++++++++++++++ .../server/templates/pages/project_show.html | 11 ++++ 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 internal/server/static/project-delete.js diff --git a/docs/configuration.md b/docs/configuration.md index 3a5579d..e44b28a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1578,7 +1578,7 @@ curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ https://proxy.example.com/api/projects/acme/shop/web ``` -Reading needs no token, like the rest of `/api`. +Reading needs no token, like the rest of `/api`. A project's page in the UI also has a delete button; it asks for the token and sends it to `DELETE /api/projects/{project}`, keeping it for the browser session. ### What it shows diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index 46dbaa4..cb1eb5d 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -67,6 +67,8 @@ func TestProjectPageShowsComponents(t *testing.T) { "1 vulnerable", "symfony/console", // not cached: listed without a link "not cached", + `data-project="acme/web"`, // delete control, asks for the token + `/ui/static/project-delete.js`, ) if strings.Contains(page, `href="/ui/package/composer/symfony/console"`) { t.Error("links to the package page of a package the proxy has no record of") diff --git a/internal/server/static/project-delete.js b/internal/server/static/project-delete.js new file mode 100644 index 0000000..b7422d7 --- /dev/null +++ b/internal/server/static/project-delete.js @@ -0,0 +1,64 @@ +// Deleting a project needs the upload token: /ui has no authentication of its +// own, so the page sends the token the user enters to the API, the same as an +// upload. The token is kept for the browser session only. +(() => { + const root = document.getElementById('project-delete'); + if (!root) { + return; + } + const start = document.getElementById('project-delete-start'); + const form = document.getElementById('project-delete-form'); + const input = document.getElementById('project-delete-token'); + const cancel = document.getElementById('project-delete-cancel'); + const errorBox = document.getElementById('project-delete-error'); + const submit = form.querySelector('button[type="submit"]'); + const storageKey = 'projects-token'; + + const show = (open) => { + start.classList.toggle('hidden', open); + form.classList.toggle('hidden', !open); + form.classList.toggle('flex', open); + errorBox.classList.add('hidden'); + }; + + start.addEventListener('click', () => { + try { + input.value = sessionStorage.getItem(storageKey) || ''; + } catch (e) { + input.value = ''; + } + show(true); + input.focus(); + }); + cancel.addEventListener('click', () => show(false)); + + form.addEventListener('submit', async (event) => { + event.preventDefault(); + const url = '/api/projects/' + encodeURIComponent(root.dataset.project); + submit.disabled = true; + errorBox.classList.add('hidden'); + try { + const resp = await fetch(url, { + method: 'DELETE', + headers: { Authorization: 'Bearer ' + input.value }, + }); + if (resp.status === 401) { + throw new Error('the token was rejected'); + } + if (!resp.ok) { + const body = await resp.json().catch(() => ({})); + throw new Error(body.message || body.error || `HTTP ${resp.status}`); + } + try { + sessionStorage.setItem(storageKey, input.value); + } catch (e) { + // Without storage the token is just asked for again. + } + window.location.href = '/ui/projects'; + } catch (err) { + errorBox.textContent = 'Delete failed: ' + err.message; + errorBox.classList.remove('hidden'); + submit.disabled = false; + } + }); +})(); diff --git a/internal/server/templates/pages/project_show.html b/internal/server/templates/pages/project_show.html index 0125f9c..9e0cf9e 100644 --- a/internal/server/templates/pages/project_show.html +++ b/internal/server/templates/pages/project_show.html @@ -13,7 +13,18 @@ {{if .Outdated}}{{.Outdated}} outdated{{end}}
{{end}} +
+ + + +
+ {{if .Components}}
From b5eec9c696ded690a5fa9e95f85c0166aa56fb3f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 20:15:50 +0100 Subject: [PATCH 007/198] Drop vulnerability markers from the project inventory The vulnerabilities table is only written by tests: nothing in the proxy calls UpsertVulnerability, so the dashboard card, the per-project count and the per-component markers were always empty. Remove them along with GetComponentVulnerabilities and inventory.Affected, and keep the outdated marker, which uses the latest version the handlers record. --- README.md | 2 +- docs/architecture.md | 2 +- docs/configuration.md | 5 +- internal/database/projects.go | 39 ------- internal/database/projects_test.go | 21 ---- internal/inventory/inventory.go | 12 -- internal/inventory/inventory_test.go | 23 ++-- internal/server/dashboard.go | 3 - internal/server/projects_ui.go | 108 ------------------ internal/server/projects_ui_test.go | 41 +------ internal/server/server.go | 1 - .../templates/components/severity_badge.html | 3 - .../server/templates/pages/dashboard.html | 27 ----- .../server/templates/pages/project_show.html | 9 +- 14 files changed, 20 insertions(+), 276 deletions(-) delete mode 100644 internal/server/templates/components/severity_badge.html diff --git a/README.md b/README.md index 61edf9d..7060960 100644 --- a/README.md +++ b/README.md @@ -1514,7 +1514,7 @@ The proxy serves a web UI under `/ui`. No separate frontend build is needed -- t - **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts. - **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews. - **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files. -- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated or vulnerable. Package pages then show which projects use each version, and the dashboard lists vulnerable versions in use. +- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated. Package pages then show which projects use each version. ## Monitoring diff --git a/docs/architecture.md b/docs/architecture.md index 0105d3f..f1b5e32 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -365,7 +365,7 @@ The package also provides a `MetadataCache` for storing raw upstream metadata bl ### `internal/inventory` -Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated and vulnerability markers on the project pages. +Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated marker on the project pages. ### `internal/config` diff --git a/docs/configuration.md b/docs/configuration.md index e44b28a..9bc11df 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1508,9 +1508,8 @@ Pushed archives are stored like cached ones but marked as local. The cache size The project inventory answers "which repository uses which version of a package?". Each repository uploads an SBOM of its default branch, and the proxy records the package versions it lists. The UI then shows: -- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version or vulnerabilities for that version. +- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version. - On a package page: which projects use each version, including versions the proxy hasn't cached. -- On the dashboard: vulnerable versions in use and the projects using them. It is disabled by default. Uploads and deletes require a token: @@ -1582,7 +1581,7 @@ Reading needs no token, like the rest of `/api`. A project's page in the UI also ### What it shows -The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. Outdated and vulnerability markers use what the proxy knows about a package: its recorded latest version and the vulnerabilities recorded for it, matched against the version a project uses by the vulnerability's fixed version. A package the proxy has no record of has neither. +The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy. diff --git a/internal/database/projects.go b/internal/database/projects.go index b00596b..729e09f 100644 --- a/internal/database/projects.go +++ b/internal/database/projects.go @@ -42,20 +42,6 @@ type ProjectUsage struct { Version string `db:"version"` } -// ComponentVulnerability is a vulnerability recorded for the package of a -// component a project uses. It is not yet filtered by version: whether it -// affects Version depends on FixedVersion. -type ComponentVulnerability struct { - Project string `db:"project"` - ProjectComponent - CachedEcosystem string `db:"cached_ecosystem"` - CachedName string `db:"cached_name"` - VulnID string `db:"vuln_id"` - Severity sql.NullString `db:"severity"` - FixedVersion sql.NullString `db:"fixed_version"` - CVSSScore sql.NullFloat64 `db:"cvss_score"` -} - const projectColumns = ` SELECT pr.id, pr.name, pr.ref, pr.branch, pr.uploaded_at, (SELECT COUNT(*) FROM project_components pc WHERE pc.project_id = pr.id) AS components` @@ -227,31 +213,6 @@ func (db *DB) GetProjectsUsingPackage(packagePURL, version string) ([]ProjectUsa return usages, nil } -// GetComponentVulnerabilities returns the vulnerabilities the proxy holds for -// packages that projects use, for one project or, with an empty name, for all -// of them. Vulnerabilities are recorded per package, so only packages the -// proxy has a record of are matched. -func (db *DB) GetComponentVulnerabilities(project string) ([]ComponentVulnerability, error) { - query := ` - SELECT pr.name AS project, c.package_purl, c.ecosystem, c.name, c.version, - p.ecosystem AS cached_ecosystem, p.name AS cached_name, - v.vuln_id, v.severity, v.fixed_version, v.cvss_score - FROM project_components c - JOIN projects pr ON pr.id = c.project_id - JOIN packages p ON p.purl = c.package_purl - JOIN vulnerabilities v ON v.ecosystem = p.ecosystem AND v.package_name = p.name` - var args []any - if project != "" { - query += ` WHERE pr.name = ?` - args = append(args, project) - } - vulns := []ComponentVulnerability{} - if err := db.Select(&vulns, db.Rebind(query+` ORDER BY c.package_purl, c.version, pr.name, v.vuln_id`), args...); err != nil { - return nil, err - } - return vulns, nil -} - func nullString(s string) sql.NullString { return sql.NullString{String: s, Valid: s != ""} } diff --git a/internal/database/projects_test.go b/internal/database/projects_test.go index 4eaaf50..2a5dded 100644 --- a/internal/database/projects_test.go +++ b/internal/database/projects_test.go @@ -159,13 +159,6 @@ func TestProjectComponentsJoinPackages(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := db.UpsertVulnerability(&Vulnerability{ - VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", - Severity: sql.NullString{String: "high", Valid: true}, - FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, - }); err != nil { - t.Fatal(err) - } saveProject(t, db, "acme/web", time.Now(), lodash20, console) saveProject(t, db, "acme/api", time.Now(), lodash21) @@ -188,20 +181,6 @@ func TestProjectComponentsJoinPackages(t *testing.T) { t.Errorf("cached component = %+v", components[1]) } - vulns, err := db.GetComponentVulnerabilities("acme/web") - if err != nil { - t.Fatal(err) - } - if len(vulns) != 1 || vulns[0].Version != "4.17.20" || vulns[0].VulnID != "GHSA-35jh-r3h4-6jhm" { - t.Errorf("acme/web vulnerabilities = %+v", vulns) - } - all, err := db.GetComponentVulnerabilities("") - if err != nil { - t.Fatal(err) - } - if len(all) != 2 { - t.Errorf("all vulnerabilities = %+v, want one row per project", all) - } }) } diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 9df6ce4..cfacdb8 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -232,15 +232,3 @@ func Outdated(ecosystem, version, latest string) bool { } return CompareVersions(ecosystem, version, latest) < 0 } - -// Affected reports whether a vulnerability fixed in fixedVersion affects -// version. A vulnerability without a fixed version affects every version. -func Affected(ecosystem, version, fixedVersion string) bool { - if fixedVersion == "" { - return true - } - if version == "" { - return false - } - return CompareVersions(ecosystem, version, fixedVersion) < 0 -} diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index b4c6cc3..998c93b 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -100,36 +100,29 @@ func TestValidateProjectName(t *testing.T) { } } -func TestOutdatedAndAffected(t *testing.T) { +func TestOutdated(t *testing.T) { tests := []struct { name string ecosystem string version string other string outdated bool - affected bool }{ - {"npm older", "npm", "4.17.20", "4.17.21", true, true}, - {"npm same", "npm", "4.17.21", "4.17.21", false, false}, - {"npm newer", "npm", "4.18.0", "4.17.21", false, false}, - {"composer v prefix", "composer", "v6.4.1", "v6.4.2", true, true}, - {"golang", "golang", "v1.8.0", "v1.10.0", true, true}, - {"alpine revisions", "alpine", "1.36.1-r15", "1.36.1-r2", false, false}, - {"empty version", "npm", "", "1.0.0", false, false}, + {"npm older", "npm", "4.17.20", "4.17.21", true}, + {"npm same", "npm", "4.17.21", "4.17.21", false}, + {"npm newer", "npm", "4.18.0", "4.17.21", false}, + {"composer v prefix", "composer", "v6.4.1", "v6.4.2", true}, + {"golang", "golang", "v1.8.0", "v1.10.0", true}, + {"alpine revisions", "alpine", "1.36.1-r15", "1.36.1-r2", false}, + {"empty version", "npm", "", "1.0.0", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := Outdated(tt.ecosystem, tt.version, tt.other); got != tt.outdated { t.Errorf("Outdated = %v, want %v", got, tt.outdated) } - if got := Affected(tt.ecosystem, tt.version, tt.other); got != tt.affected { - t.Errorf("Affected = %v, want %v", got, tt.affected) - } }) } - if !Affected("npm", "1.0.0", "") { - t.Error("a vulnerability without a fix should affect every version") - } } func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index ebcc333..56bfddc 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -15,9 +15,6 @@ type DashboardData struct { EnrichmentStats EnrichmentStatsView RecentPackages []PackageInfo PopularPackages []PackageInfo - // VulnerableInUse lists vulnerable package versions projects use, when - // the project inventory is enabled. - VulnerableInUse []VulnerableVersion } // DashboardStats contains cache statistics for the dashboard. diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go index 76ae8bd..0de4601 100644 --- a/internal/server/projects_ui.go +++ b/internal/server/projects_ui.go @@ -1,7 +1,6 @@ package server import ( - "cmp" "net/http" "net/url" "slices" @@ -31,7 +30,6 @@ type ProjectShowData struct { Layout Project ProjectRow Components []ProjectComponentRow - Vulnerable int Outdated int } @@ -48,8 +46,6 @@ type ProjectComponentRow struct { CachedName string LatestVersion string Outdated bool - VulnCount int - Severity string } // VersionUsage lists the projects using one version of a package. @@ -61,17 +57,6 @@ type VersionUsage struct { Projects []string } -// VulnerableVersion is a package version with known vulnerabilities that -// projects use. -type VulnerableVersion struct { - Ecosystem string - Name string - Version string - Severity string - VulnCount int - Projects []string -} - // mountProjectUI registers the project inventory pages when // projects.enabled is set. func (s *Server) mountProjectUI(ui chi.Router) { @@ -122,11 +107,6 @@ func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { s.logger.Error("failed to get project components", "project", name, "error", err) components = nil } - vulns, err := s.db.GetComponentVulnerabilities(name) - if err != nil { - s.logger.Error("failed to get project vulnerabilities", "project", name, "error", err) - } - affecting := affectingVulnerabilities(vulns) data := ProjectShowData{Layout: s.layoutFor(r), Project: projectRow(*project)} for _, c := range components { @@ -140,10 +120,6 @@ func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { LatestVersion: c.LatestVersion.String, Outdated: inventory.Outdated(c.Ecosystem, c.Version, c.LatestVersion.String), } - if v, ok := affecting[componentKey{c.PackagePURL, c.Version}]; ok { - row.VulnCount, row.Severity = len(v.ids), v.severity - data.Vulnerable++ - } if row.Outdated { data.Outdated++ } @@ -201,87 +177,3 @@ func (s *Server) packageUsedBy(pkg *database.Package, versions []database.Versio } return out } - -// vulnerableVersionsInUse returns the package versions projects use that -// have vulnerabilities, most severe and most used first, at most limit. It -// returns nothing when the project inventory is disabled. -func (s *Server) vulnerableVersionsInUse(limit int) []VulnerableVersion { - if !s.cfg.Projects.Enabled { - return nil - } - vulns, err := s.db.GetComponentVulnerabilities("") - if err != nil { - s.logger.Error("failed to get vulnerable versions in use", "error", err) - return nil - } - affecting := affectingVulnerabilities(vulns) - result := make([]VulnerableVersion, 0, len(affecting)) - for _, v := range affecting { - projects := make([]string, 0, len(v.projects)) - for p := range v.projects { - projects = append(projects, p) - } - slices.Sort(projects) - result = append(result, VulnerableVersion{ - Ecosystem: v.ecosystem, Name: v.name, Version: v.version, - Severity: v.severity, VulnCount: len(v.ids), Projects: projects, - }) - } - slices.SortFunc(result, func(a, b VulnerableVersion) int { - return cmp.Or( - cmp.Compare(severityRank(b.Severity), severityRank(a.Severity)), - cmp.Compare(len(b.Projects), len(a.Projects)), - cmp.Compare(a.Name, b.Name), - cmp.Compare(a.Version, b.Version), - ) - }) - if len(result) > limit { - result = result[:limit] - } - return result -} - -type componentKey struct{ packagePURL, version string } - -// affectingVersion collects the vulnerabilities affecting one package -// version, and the projects using it. -type affectingVersion struct { - ecosystem, name, version string - severity string - ids map[string]bool - projects map[string]bool -} - -// affectingVulnerabilities keeps the vulnerabilities that affect the version -// each component uses, grouped by package version. -func affectingVulnerabilities(vulns []database.ComponentVulnerability) map[componentKey]*affectingVersion { - out := map[componentKey]*affectingVersion{} - for _, v := range vulns { - if !inventory.Affected(v.Ecosystem, v.Version, v.FixedVersion.String) { - continue - } - key := componentKey{v.PackagePURL, v.Version} - a, ok := out[key] - if !ok { - a = &affectingVersion{ - ecosystem: v.CachedEcosystem, name: v.CachedName, version: v.Version, - ids: map[string]bool{}, projects: map[string]bool{}, - } - out[key] = a - } - a.ids[v.VulnID] = true - a.projects[v.Project] = true - if severityRank(v.Severity.String) > severityRank(a.severity) { - a.severity = v.Severity.String - } - } - return out -} - -// severityOrder lists severities from least to most severe. -var severityOrder = []string{"low", "medium", "high", "critical"} - -// severityRank orders severities; unknown ones rank below "low". -func severityRank(severity string) int { - return slices.Index(severityOrder, severity) + 1 -} diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index cb1eb5d..e8031d4 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -10,9 +10,8 @@ import ( ) // seedLodash records lodash with 4.17.21 as its latest and only cached -// version, and a vulnerability fixed in 4.17.21. The syft fixture uses -// 4.17.20. -func seedLodash(t *testing.T, db *database.DB, withVuln bool) { +// version. The syft fixture uses 4.17.20. +func seedLodash(t *testing.T, db *database.DB) { t.Helper() if err := db.UpsertPackage(&database.Package{ PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash", @@ -23,16 +22,6 @@ func seedLodash(t *testing.T, db *database.DB, withVuln bool) { if err := db.UpsertVersion(&database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}); err != nil { t.Fatal(err) } - if !withVuln { - return - } - if err := db.UpsertVulnerability(&database.Vulnerability{ - VulnID: "GHSA-35jh-r3h4-6jhm", Ecosystem: "npm", PackageName: "lodash", - Severity: sql.NullString{String: "high", Valid: true}, - FixedVersion: sql.NullString{String: "4.17.21", Valid: true}, - }); err != nil { - t.Fatal(err) - } } func (ts projectsTestServer) page(t *testing.T, target string) string { @@ -55,7 +44,7 @@ func assertContains(t *testing.T, page string, want ...string) { func TestProjectPageShowsComponents(t *testing.T) { ts := newProjectsTestServer(t) - seedLodash(t, ts.db, true) + seedLodash(t, ts.db) ts.upload(t, "/api/projects/acme/web/sbom?branch=main", syftSBOM(t)) page := ts.page(t, "/ui/projects/acme/web") @@ -63,8 +52,7 @@ func TestProjectPageShowsComponents(t *testing.T) { "acme/web", `href="/ui/package/npm/lodash"`, // cached, so linked "outdated, latest 4.17.21", - "1 vulns", - "1 vulnerable", + "1 outdated", "symfony/console", // not cached: listed without a link "not cached", `data-project="acme/web"`, // delete control, asks for the token @@ -90,7 +78,7 @@ func TestProjectPageShowsComponents(t *testing.T) { func TestPackagePageShowsUsedBy(t *testing.T) { ts := newProjectsTestServer(t) - seedLodash(t, ts.db, false) + seedLodash(t, ts.db) ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) ts.upload(t, "/api/projects/acme/api/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) @@ -116,24 +104,7 @@ func TestPackagePageShowsUsedBy(t *testing.T) { } } -func TestDashboardShowsVulnerableVersionsInUse(t *testing.T) { +func TestNavLinksToProjects(t *testing.T) { ts := newProjectsTestServer(t) - seedLodash(t, ts.db, false) - ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) - - if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { - t.Error("dashboard shows the vulnerable versions card without vulnerabilities") - } assertContains(t, ts.page(t, "/ui/"), `href="/ui/projects"`) - - seedLodash(t, ts.db, true) - page := ts.page(t, "/ui/") - assertContains(t, page, "Vulnerable versions in use", "@4.17.20", `href="/ui/projects/acme/web"`, ">high<") - - // A project on the fixed version doesn't count. - ts.upload(t, "/api/projects/acme/web/sbom", []byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ - {"type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"}]}`)) - if page := ts.page(t, "/ui/"); strings.Contains(page, "Vulnerable versions in use") { - t.Error("dashboard lists a version the vulnerability is fixed in") - } } diff --git a/internal/server/server.go b/internal/server/server.go index d96b9dc..88897f7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -845,7 +845,6 @@ func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) { }, EnrichmentStats: enrichmentStatsView(enrichStats), } - data.VulnerableInUse = s.vulnerableVersionsInUse(dashboardTopN) for _, p := range popular { pkgInfo := PackageInfo{ diff --git a/internal/server/templates/components/severity_badge.html b/internal/server/templates/components/severity_badge.html deleted file mode 100644 index a060ed2..0000000 --- a/internal/server/templates/components/severity_badge.html +++ /dev/null @@ -1,3 +0,0 @@ -{{define "severity_badge"}} -{{if .}}{{.}}{{else}}unknown{{end}} -{{end}} diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index ff9bc1b..e48e205 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -25,33 +25,6 @@ {{template "security_overview" .EnrichmentStats}} {{end}} -{{if .VulnerableInUse}} -
-
-

Vulnerable versions in use

-
-
- {{range .VulnerableInUse}} -
-
- {{template "ecosystem_badge" .Ecosystem}} - {{.Name}} - @{{.Version}} - {{template "severity_badge" .Severity}} - {{.VulnCount}} vulns -
-
- {{range .Projects}}{{.}}{{end}} -
-
- {{end}} -
-
- See all projects → -
-
-{{end}} -
diff --git a/internal/server/templates/pages/project_show.html b/internal/server/templates/pages/project_show.html index 9e0cf9e..2d1b56a 100644 --- a/internal/server/templates/pages/project_show.html +++ b/internal/server/templates/pages/project_show.html @@ -7,10 +7,9 @@

{{.Project.Components}} packages, uploaded {{.Project.UploadedAt}}{{if .Project.Branch}} from {{.Project.Branch}}{{end}}{{if .Project.Ref}} at {{.Project.Ref}}{{end}}

- {{if or .Vulnerable .Outdated}} + {{if .Outdated}}
- {{if .Vulnerable}}{{.Vulnerable}} vulnerable{{end}} - {{if .Outdated}}{{.Outdated}} outdated{{end}} + {{.Outdated}} outdated
{{end}}
@@ -41,10 +40,6 @@ {{.Version}}
- {{if .VulnCount}} - {{template "severity_badge" .Severity}} - {{.VulnCount}} vulns - {{end}} {{if .Outdated}} outdated, latest {{.LatestVersion}} {{end}} From 8855c86fc54f84663fef740fb091cde88ea4e102 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 20:18:50 +0100 Subject: [PATCH 008/198] Match more SBOM names to cached packages FromPURL now names packages the way their handlers store them: - NuGet IDs are lowercased, as the NuGet handler stores them. - PyPI names are normalized as in PEP 503. The PyPI handler takes the name from the file name, which spells "." as "_", so pkg:pypi/zope.interface has to match the cached zope-interface. - Conda package URLs without a channel qualifier are skipped: the conda handler records packages as channel/name, so they can never match. - The Go main-module check ignores case. syft keeps the module path's case in its mainModule property, while golang package URLs are lowercased. A table test checks every mapped PURL type against the name its handler passes when caching. --- docs/configuration.md | 2 +- internal/inventory/inventory.go | 76 +++++++++++++++--- internal/inventory/inventory_test.go | 110 +++++++++++++++++++++++++++ internal/server/projects_api.go | 5 +- 4 files changed, 178 insertions(+), 15 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9bc11df..b8e9ae0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1547,7 +1547,7 @@ The response counts the recorded package versions and the skipped entries: {"project":"acme/shop/web","components":9,"skipped":6} ``` -Entries are recorded by their package URL. Entries without one (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`) and entries without a version are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once. +Entries are recorded by their package URL, with the package named the way the proxy caches it: NuGet IDs lowercased, PyPI names normalized as in PEP 503 (`zope.interface` becomes `zope-interface`), the distribution dropped from `deb`, `rpm` and `apk` package URLs. Entries without a package URL (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`), entries without a version, and conda entries without a `channel` qualifier (the proxy records conda packages as `channel/name`) are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once. A GitLab CI job on the default branch could look like this: diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index cfacdb8..ccd9ecd 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "strings" + "unicode" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" @@ -66,8 +67,13 @@ var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": // doesn't serve. var ErrUnsupportedPURL = errors.New("package type is not served by the proxy") -// FromPURL maps a package URL to a component. Qualifiers and subpath are -// dropped; the version is kept as given. +// ErrNoCondaChannel is returned for a conda package URL without a channel +// qualifier: the proxy records conda packages per channel. +var ErrNoCondaChannel = errors.New("conda package URL has no channel qualifier") + +// FromPURL maps a package URL to a component, with the package named the way +// the handler for its type records it. Qualifiers and subpath are dropped; +// the version is kept as given. func FromPURL(s string) (Component, error) { p, err := purl.Parse(s) if err != nil { @@ -77,12 +83,9 @@ func FromPURL(s string) (Component, error) { if !ok { return Component{}, fmt.Errorf("%w: %s", ErrUnsupportedPURL, p.Type) } - name := p.FullName() - switch { - case distroTypes[p.Type]: - name = p.Name - case p.Type == "conda" && p.Namespace == "" && p.Qualifier("channel") != "": - name = p.Qualifier("channel") + "/" + p.Name + name, err := handlerName(p) + if err != nil { + return Component{}, err } packagePURL, _ := packageurl.MakeCacheStrings(ecosystem, name, "") if packagePURL == "" { @@ -91,6 +94,53 @@ func FromPURL(s string) (Component, error) { return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil } +// handlerName returns the package name the handler for p's type stores. +func handlerName(p *purl.PURL) (string, error) { + switch { + case distroTypes[p.Type]: + // The namespace is the distribution; the handlers store the bare + // package name. + return p.Name, nil + case p.Type == "conda" && p.Namespace == "": + // The conda handler stores "channel/name". + channel := p.Qualifier("channel") + if channel == "" { + return "", ErrNoCondaChannel + } + return channel + "/" + p.Name, nil + case p.Type == "nuget": + // The NuGet handler lowercases package IDs. + return strings.ToLower(p.FullName()), nil + case p.Type == "pypi": + return normalizePyPIName(p.FullName()), nil + } + return p.FullName(), nil +} + +// normalizePyPIName normalizes a PyPI project name as PEP 503 does: runs of +// "-", "_" and "." become one "-", and letters are lowercased. The PyPI +// handler stores the name from the file name, which wheels and current sdists +// spell with "_" for any of these, so "zope.interface" is cached as +// "zope-interface". (An old sdist named "zope.interface-5.0.tar.gz" is +// stored with the dot and doesn't match.) +func normalizePyPIName(name string) string { + var b strings.Builder + b.Grow(len(name)) + separator := false + for _, r := range name { + if r == '-' || r == '_' || r == '.' { + separator = true + continue + } + if separator && b.Len() > 0 { + b.WriteByte('-') + } + separator = false + b.WriteRune(unicode.ToLower(r)) + } + return b.String() +} + // syftMainModule is the property syft sets on modules found in a Go binary, // naming the module the binary was built from. const syftMainModule = "syft:metadata:mainModule" @@ -104,7 +154,8 @@ const unknownVersion = "UNKNOWN" // Parse reads a CycloneDX (JSON or XML) or SPDX SBOM and returns the package // versions it lists, without duplicates. Packages without a package URL, of a -// type the proxy doesn't serve, or without a version are counted in skipped. +// type the proxy doesn't serve, without a version, or conda packages without +// a channel are counted in skipped. // So are the project's own packages: Go modules its binaries were built from, // and the root of a lock file. func Parse(data []byte) (components []Component, skipped int, err error) { @@ -117,7 +168,7 @@ func Parse(data []byte) (components []Component, skipped int, err error) { seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { + if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[strings.ToLower(c.Name)]) { skipped++ continue } @@ -179,13 +230,14 @@ func lockFileRoots(doc *sbom.SBOM) map[string]bool { } // mainModules returns the Go modules the SBOM's binaries were built from, -// which are the project's own. +// which are the project's own, lowercased: syft keeps the module path's case +// while FromPURL lowercases golang names. func mainModules(pkgs []sbom.Package) map[string]bool { own := map[string]bool{} for _, pkg := range pkgs { for _, prop := range pkg.Properties { if prop.Name == syftMainModule && prop.Value != "" { - own[prop.Value] = true + own[strings.ToLower(prop.Value)] = true } } } diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 998c93b..e5b6225 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -6,6 +6,8 @@ import ( "slices" "strings" "testing" + + "github.com/git-pkgs/proxy/internal/packageurl" ) func TestParseSyftCycloneDX(t *testing.T) { @@ -81,6 +83,96 @@ func TestFromPURLUnsupported(t *testing.T) { } } +// TestFromPURLMatchesHandlerNames checks, for every PURL type the inventory +// maps, that an SBOM's package URL ends up as the package URL the handler +// records when it caches that package. handlerName is the name the handler +// passes to GetOrFetchArtifactFromURL, taken from the handler's code. +func TestFromPURLMatchesHandlerNames(t *testing.T) { + tests := []struct { + purlType string + sbomPURL string + handlerName string + }{ + {"alpm", "pkg:alpm/arch/glibc@2.39-1?arch=x86_64", "glibc"}, + {"apk", "pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64", "busybox"}, + {"bsd", "pkg:bsd/freebsd/curl@8.7.1", "curl"}, + {"cargo", "pkg:cargo/serde@1.0.200", "serde"}, + {"chef", "pkg:chef/nginx@12.0.0", "nginx"}, + {"composer", "pkg:composer/symfony/console@v6.4.1", "symfony/console"}, + {"conan", "pkg:conan/zlib@1.3.1", "zlib"}, + {"conda", "pkg:conda/numpy@1.26.4?channel=conda-forge&subdir=linux-64", "conda-forge/numpy"}, + {"cran", "pkg:cran/ggplot2@3.5.1", "ggplot2"}, + {"deb", "pkg:deb/debian/libc6@2.36-9?arch=amd64&distro=debian-12", "libc6"}, + {"gem", "pkg:gem/rails@7.1.3", "rails"}, + // The go handler stores the module path with its case; PURLs of + // golang modules are lowercased on both sides. + {"golang", "pkg:golang/github.com/Azure/azure-sdk-for-go@v68.0.0", "github.com/Azure/azure-sdk-for-go"}, + {"hex", "pkg:hex/phoenix@1.7.12", "phoenix"}, + {"julia", "pkg:julia/Example@0.5.3?uuid=7876af07-990d-54b4-ab0e-23690620f79a", "Example"}, + {"maven", "pkg:maven/org.slf4j/slf4j-api@2.0.13?type=jar", "org.slf4j:slf4j-api"}, + {"npm", "pkg:npm/%40babel/core@7.23.0", "@babel/core"}, + // The NuGet handler lowercases the package ID. + {"nuget", "pkg:nuget/Newtonsoft.Json@13.0.3", "newtonsoft.json"}, + {"pub", "pkg:pub/http@1.2.1", "http"}, + // The PyPI handler takes the name from the wheel or sdist file name, + // which spells "." and "-" as "_". + {"pypi", "pkg:pypi/zope.interface@6.4", "zope_interface"}, + {"rpm", "pkg:rpm/fedora/curl@8.6.0-7.fc40?arch=x86_64", "curl"}, + {"swift", "pkg:swift/github.com/apple/swift-argument-parser@1.3.1", "github.com/apple/swift-argument-parser"}, + } + covered := map[string]bool{} + for _, tt := range tests { + t.Run(tt.purlType, func(t *testing.T) { + covered[tt.purlType] = true + c, err := FromPURL(tt.sbomPURL) + if err != nil { + t.Fatalf("FromPURL(%q): %v", tt.sbomPURL, err) + } + want, _ := packageurl.MakeCacheStrings(ecosystems[tt.purlType], tt.handlerName, "") + if want == "" { + t.Fatalf("handler name %q has no package URL", tt.handlerName) + } + if c.PackagePURL != want { + t.Errorf("PackagePURL = %q, want %q as the handler records it", c.PackagePURL, want) + } + }) + } + for purlType := range ecosystems { + if !covered[purlType] { + t.Errorf("no case for PURL type %q", purlType) + } + } +} + +func TestFromPURLNormalizesPyPINames(t *testing.T) { + for _, s := range []string{"pkg:pypi/zope.interface", "pkg:pypi/Zope_Interface", "pkg:pypi/zope--interface", "pkg:pypi/zope._-interface"} { + c, err := FromPURL(s) + if err != nil { + t.Fatal(err) + } + if c.Name != "zope-interface" || c.PackagePURL != "pkg:pypi/zope-interface" { + t.Errorf("FromPURL(%q) = %+v, want zope-interface", s, c) + } + } +} + +func TestFromPURLCondaNeedsChannel(t *testing.T) { + if _, err := FromPURL("pkg:conda/numpy@1.26.4"); !errors.Is(err, ErrNoCondaChannel) { + t.Errorf("err = %v, want ErrNoCondaChannel", err) + } + doc := `{"bomFormat":"CycloneDX","specVersion":"1.5","components":[ + {"type":"library","name":"numpy","version":"1.26.4","purl":"pkg:conda/numpy@1.26.4"}, + {"type":"library","name":"numpy","version":"1.26.4","purl":"pkg:conda/numpy@1.26.4?channel=conda-forge"} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "conda-forge/numpy" || skipped != 1 { + t.Errorf("components = %+v, skipped = %d; want conda-forge/numpy and 1 skipped", components, skipped) + } +} + func TestValidateProjectName(t *testing.T) { valid := []string{"repo", "group/repo", "group/sub/my_repo-2.0", strings.Repeat("a", MaxProjectNameLength)} for _, name := range valid { @@ -153,3 +245,21 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { t.Errorf("skipped = %d, want 4", skipped) } } + +func TestParseSkipsMainModuleRegardlessOfCase(t *testing.T) { + // syft keeps the module path's case in mainModule; package URLs of Go + // modules are lowercased. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"bin","type":"library","name":"github.com/Azure/foo","version":"v1.2.0","purl":"pkg:golang/github.com/Azure/foo@v1.2.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]}, + {"bom-ref":"cobra","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "github.com/spf13/cobra" || skipped != 1 { + t.Errorf("components = %+v, skipped = %d; want only cobra, 1 skipped", components, skipped) + } +} diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go index a007319..7417830 100644 --- a/internal/server/projects_api.go +++ b/internal/server/projects_api.go @@ -43,8 +43,9 @@ type ProjectUploadResponse struct { Project string `json:"project"` // Components is the number of package versions recorded. Components int `json:"components"` - // Skipped is the number of SBOM entries without a package URL or of a - // package type the proxy doesn't serve. + // Skipped is the number of SBOM entries not recorded: without a package + // URL or a version, of a package type the proxy doesn't serve, conda + // entries without a channel, and the project's own packages. Skipped int `json:"skipped"` } From 1c706fffd0dfe3f0ac589a72988a31c1c8d50da8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 27 Oct 2026 20:23:14 +0100 Subject: [PATCH 009/198] Review fixes for the project inventory - GetProjectsUsingPackage no longer counts each project's components per row; the package page only needs names and versions. The ?purl= listing leaves component_count out accordingly. - The delete button sends to DELETE /ui/api/projects/{project}, the same handler as the API and still behind the token, so it sits under /ui like the cache deletes. - The project page answers 500 when its components can't be read, instead of showing an empty project. - DELETE of a path ending in /sbom that names no project answers 400 with a hint, since it's most likely the upload path. - The upload endpoint lists JSON, XML and plain text (SPDX tag-value) as accepted. - The docs say that versions are compared as the SBOM spells them. --- docs/configuration.md | 6 ++- docs/swagger/docs.go | 14 +++--- docs/swagger/swagger.json | 14 +++--- internal/database/projects.go | 10 ++-- internal/server/projects_api.go | 29 +++++++---- internal/server/projects_api_test.go | 62 +++++++++++++++++++++++- internal/server/projects_ui.go | 10 +++- internal/server/projects_ui_test.go | 12 ++++- internal/server/server.go | 1 + internal/server/static/project-delete.js | 6 +-- 10 files changed, 129 insertions(+), 35 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b8e9ae0..e6019fd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1577,11 +1577,13 @@ curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \ https://proxy.example.com/api/projects/acme/shop/web ``` -Reading needs no token, like the rest of `/api`. A project's page in the UI also has a delete button; it asks for the token and sends it to `DELETE /api/projects/{project}`, keeping it for the browser session. +Reading needs no token, like the rest of `/api`. With `?purl=`, each project carries the `version` it uses and no `component_count`. A delete names the project itself: `DELETE /api/projects/acme/shop/web/sbom` is answered with 400 unless a project `acme/shop/web/sbom` exists. + +A project's page in the UI also has a delete button. It asks for the token and sends it to `DELETE /ui/api/projects/{project}`, which works like the API endpoint and also requires the token; the token is kept for the browser session. Like `/ui/api/cache`, it sits under `/ui`, so rules for the UI at an ingress cover it. ### What it shows -The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. +The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. Versions are compared as the SBOM gives them: a version spelled differently from how the proxy caches it, such as Composer's `v6.4.1` for `6.4.1` or a Debian version with or without its epoch, shows as "not cached" on the package page. Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 7ddf630..1888289 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -540,7 +540,7 @@ const docTemplate = `{ "BearerAuth": [] } ], - "description": "Removes the project and the package versions recorded for it.", + "description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.", "tags": [ "projects" ], @@ -586,9 +586,11 @@ const docTemplate = `{ "BearerAuth": [] } ], - "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.", "consumes": [ - "application/json" + "application/json", + "application/xml", + "text/plain" ], "produces": [ "application/json" @@ -1341,7 +1343,7 @@ const docTemplate = `{ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "components": { @@ -1372,7 +1374,7 @@ const docTemplate = `{ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "name": { @@ -1401,7 +1403,7 @@ const docTemplate = `{ "type": "string" }, "skipped": { - "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.", "type": "integer" } } diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index eb6f3b9..a76f868 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -533,7 +533,7 @@ "BearerAuth": [] } ], - "description": "Removes the project and the package versions recorded for it.", + "description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.", "tags": [ "projects" ], @@ -579,9 +579,11 @@ "BearerAuth": [] } ], - "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment.", + "description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.", "consumes": [ - "application/json" + "application/json", + "application/xml", + "text/plain" ], "produces": [ "application/json" @@ -1334,7 +1336,7 @@ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "components": { @@ -1365,7 +1367,7 @@ "type": "string" }, "component_count": { - "description": "ComponentCount is the number of package versions recorded.", + "description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.", "type": "integer" }, "name": { @@ -1394,7 +1396,7 @@ "type": "string" }, "skipped": { - "description": "Skipped is the number of SBOM entries without a package URL or of a\npackage type the proxy doesn't serve.", + "description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.", "type": "integer" } } diff --git a/internal/database/projects.go b/internal/database/projects.go index 729e09f..c14b0d6 100644 --- a/internal/database/projects.go +++ b/internal/database/projects.go @@ -38,8 +38,11 @@ type ProjectComponentInfo struct { // ProjectUsage is a project using a version of a package. type ProjectUsage struct { - Project - Version string `db:"version"` + Name string `db:"name"` + Ref sql.NullString `db:"ref"` + Branch sql.NullString `db:"branch"` + UploadedAt time.Time `db:"uploaded_at"` + Version string `db:"version"` } const projectColumns = ` @@ -197,7 +200,8 @@ func (db *DB) GetProjectComponents(projectID int64) ([]ProjectComponentInfo, err // GetProjectsUsingPackage returns the projects using a package, or only // those using version when it is not empty, ordered by project and version. func (db *DB) GetProjectsUsingPackage(packagePURL, version string) ([]ProjectUsage, error) { - query := projectColumns + `, c.version + query := ` + SELECT pr.name, pr.ref, pr.branch, pr.uploaded_at, c.version FROM project_components c JOIN projects pr ON pr.id = c.project_id WHERE c.package_purl = ?` diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go index 7417830..f297a9a 100644 --- a/internal/server/projects_api.go +++ b/internal/server/projects_api.go @@ -55,8 +55,9 @@ type ProjectSummary struct { Ref string `json:"ref,omitempty"` Branch string `json:"branch,omitempty"` UploadedAt time.Time `json:"uploaded_at"` - // ComponentCount is the number of package versions recorded. - ComponentCount int64 `json:"component_count"` + // ComponentCount is the number of package versions recorded. It is not + // set when listing the projects using a package. + ComponentCount *int64 `json:"component_count,omitempty"` // Version is set when listing the projects using a package: the version // of it this project uses. Version string `json:"version,omitempty"` @@ -107,9 +108,9 @@ func registerProjectRoutes(r chi.Router, h *ProjectsAPIHandler, token string) { // HandleUpload handles PUT /api/projects/{project}/sbom. // @Summary Upload a project's SBOM -// @Description Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL, or of a package type the proxy doesn't serve, are skipped. The project may be given as path segments or URL-encoded as one segment. +// @Description Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment. // @Tags projects -// @Accept json +// @Accept json,application/xml,plain // @Produce json // @Security BearerAuth // @Param project path string true "Project path, e.g. group/sub/repo" @@ -176,9 +177,10 @@ func (h *ProjectsAPIHandler) HandleUpload(w http.ResponseWriter, r *http.Request writeJSON(w, ProjectUploadResponse{Project: name, Components: len(records), Skipped: skipped}) } -// HandleDelete handles DELETE /api/projects/{project}. +// HandleDelete handles DELETE /api/projects/{project}, and the same under +// /ui/api/projects for the delete button on a project's page. // @Summary Delete a project -// @Description Removes the project and the package versions recorded for it. +// @Description Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400. // @Tags projects // @Security BearerAuth // @Param project path string true "Project path, e.g. group/sub/repo" @@ -197,6 +199,10 @@ func (h *ProjectsAPIHandler) HandleDelete(w http.ResponseWriter, r *http.Request case err != nil: h.logger.Error("failed to delete project", "project", name, "error", err) internalError(w, "failed to delete project") + case !deleted && (name == "sbom" || strings.HasSuffix(name, sbomPathSuffix)): + // Most likely the upload path; a project whose last segment is + // "sbom" is deleted above when it exists. + badRequest(w, "project not found; to delete a project, send DELETE /api/projects/{project} without /sbom") case !deleted: notFound(w, "project not found") default: @@ -246,8 +252,13 @@ func (h *ProjectsAPIHandler) listUsages(w http.ResponseWriter, query string) { } resp := ProjectListResponse{Projects: make([]ProjectSummary, len(usages))} for i, u := range usages { - resp.Projects[i] = projectSummary(u.Project) - resp.Projects[i].Version = u.Version + resp.Projects[i] = ProjectSummary{ + Name: u.Name, + Ref: u.Ref.String, + Branch: u.Branch.String, + UploadedAt: u.UploadedAt, + Version: u.Version, + } } writeJSON(w, resp) } @@ -301,7 +312,7 @@ func projectSummary(p database.Project) ProjectSummary { Ref: p.Ref.String, Branch: p.Branch.String, UploadedAt: p.UploadedAt, - ComponentCount: p.Components, + ComponentCount: &p.Components, } } diff --git a/internal/server/projects_api_test.go b/internal/server/projects_api_test.go index 9a7d9b9..ba68dae 100644 --- a/internal/server/projects_api_test.go +++ b/internal/server/projects_api_test.go @@ -100,6 +100,14 @@ func decodeJSON(t *testing.T, rec *httptest.ResponseRecorder, v any) { } } +// componentCount returns a project's component_count, or -1 if not set. +func componentCount(p ProjectSummary) int64 { + if p.ComponentCount == nil { + return -1 + } + return *p.ComponentCount +} + func TestProjectUploadAndGet(t *testing.T) { ts := newProjectsTestServer(t) @@ -111,7 +119,7 @@ func TestProjectUploadAndGet(t *testing.T) { // The same project, URL-encoded as one segment. var project ProjectResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects/acme%2Fshop%2Fweb", "", nil), &project) - if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || project.ComponentCount != 9 { + if project.Name != "acme/shop/web" || project.Ref != "0123abcd" || project.Branch != "main" || componentCount(project.ProjectSummary) != 9 { t.Errorf("project = %+v", project.ProjectSummary) } var purls []string @@ -126,7 +134,7 @@ func TestProjectUploadAndGet(t *testing.T) { var list ProjectListResponse decodeJSON(t, ts.do(t, http.MethodGet, "/api/projects", "", nil), &list) - if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || list.Projects[0].ComponentCount != 9 { + if len(list.Projects) != 1 || list.Projects[0].Name != "acme/shop/web" || componentCount(list.Projects[0]) != 9 { t.Errorf("list = %+v", list) } } @@ -169,6 +177,9 @@ func TestProjectLookupByPURL(t *testing.T) { var got []string for _, p := range list.Projects { got = append(got, p.Name+"@"+p.Version) + if p.ComponentCount != nil { + t.Errorf("%s has component_count set when listing by purl", p.Name) + } } if strings.Join(got, ",") != strings.Join(tt.want, ",") { t.Errorf("projects = %v, want %v", got, tt.want) @@ -212,6 +223,40 @@ func TestProjectDelete(t *testing.T) { } } +func TestProjectDeleteOfUploadPath(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + rec := ts.do(t, http.MethodDelete, "/api/projects/acme/web/sbom", testProjectsToken, nil) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "without /sbom") { + t.Errorf("DELETE of the upload path = %d: %s, want 400 with a hint", rec.Code, rec.Body) + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusOK { + t.Errorf("project is gone after DELETE of the upload path: %d", rec.Code) + } + + // A project whose last segment is "sbom" is deleted as usual. + ts.upload(t, "/api/projects/acme/sbom/sbom", syftSBOM(t)) + if rec := ts.do(t, http.MethodDelete, "/api/projects/acme/sbom", testProjectsToken, nil); rec.Code != http.StatusNoContent { + t.Errorf("DELETE of project acme/sbom = %d: %s", rec.Code, rec.Body) + } +} + +func TestProjectDeleteFromUI(t *testing.T) { + ts := newProjectsTestServer(t) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + for _, token := range []string{"", "wrong-token"} { + if rec := ts.do(t, http.MethodDelete, "/ui/api/projects/acme%2Fweb", token, nil); rec.Code != http.StatusUnauthorized { + t.Errorf("UI delete with token %q status = %d, want 401", token, rec.Code) + } + } + if rec := ts.do(t, http.MethodDelete, "/ui/api/projects/acme%2Fweb", testProjectsToken, nil); rec.Code != http.StatusNoContent { + t.Fatalf("UI delete status = %d: %s", rec.Code, rec.Body) + } + if rec := ts.do(t, http.MethodGet, "/api/projects/acme/web", "", nil); rec.Code != http.StatusNotFound { + t.Errorf("get after UI delete status = %d, want 404", rec.Code) + } +} + func TestProjectPathsAreValidated(t *testing.T) { ts := newProjectsTestServer(t) uploads := []string{ @@ -331,6 +376,9 @@ func TestProjectRoutesFollowConfig(t *testing.T) { if status, _ := request(t, http.MethodPut, disabled+"/api/projects/acme/web/sbom", testProjectsToken, syftSBOM(t)); status != http.StatusNotFound { t.Errorf("upload with projects disabled = %d, want 404", status) } + if status, _ := request(t, http.MethodDelete, disabled+"/ui/api/projects/acme%2Fweb", testProjectsToken, nil); status != http.StatusNotFound { + t.Errorf("UI delete with projects disabled = %d, want 404", status) + } if _, body := request(t, http.MethodGet, disabled+"/ui/", "", nil); strings.Contains(body, `href="/ui/projects"`) { t.Error("navigation links to projects while they are disabled") } @@ -351,4 +399,14 @@ func TestProjectRoutesFollowConfig(t *testing.T) { if _, body := request(t, http.MethodGet, enabled+"/ui/", "", nil); !strings.Contains(body, `href="/ui/projects"`) { t.Error("navigation does not link to projects") } + if status, _ := request(t, http.MethodDelete, enabled+"/ui/api/projects/acme%2Fweb", "", nil); status != http.StatusUnauthorized { + t.Errorf("UI delete without token = %d, want 401", status) + } + if status, body := request(t, http.MethodDelete, enabled+"/ui/api/projects/acme%2Fweb", testProjectsToken, nil); status != http.StatusNoContent { + t.Errorf("UI delete = %d: %s", status, body) + } + // The delete button sends its request there, too. + if _, body := request(t, http.MethodGet, enabled+"/ui/static/project-delete.js", "", nil); !strings.Contains(body, "'/ui/api/projects/'") { + t.Error("project-delete.js does not send to /ui/api/projects") + } } diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go index 0de4601..5bafb39 100644 --- a/internal/server/projects_ui.go +++ b/internal/server/projects_ui.go @@ -58,13 +58,18 @@ type VersionUsage struct { } // mountProjectUI registers the project inventory pages when -// projects.enabled is set. +// projects.enabled is set, and DELETE /ui/api/projects/{project} for the +// delete button on a project's page. Like DELETE /api/projects/{project} it +// requires the bearer token; it sits under /ui so that an ingress can treat +// the UI's requests together. func (s *Server) mountProjectUI(ui chi.Router) { if !s.cfg.Projects.Enabled { return } ui.Get("/projects", s.handleProjectsList) ui.Get("/projects/*", s.handleProjectShow) + api := NewProjectsAPIHandler(s.db, s.logger) + ui.With(requireBearerToken(s.cfg.Projects.TokenExpanded())).Delete("/api/projects/*", api.HandleDelete) } func (s *Server) handleProjectsList(w http.ResponseWriter, r *http.Request) { @@ -105,7 +110,8 @@ func (s *Server) handleProjectShow(w http.ResponseWriter, r *http.Request) { components, err := s.db.GetProjectComponents(project.ID) if err != nil { s.logger.Error("failed to get project components", "project", name, "error", err) - components = nil + http.Error(w, "failed to get project components", http.StatusInternalServerError) + return } data := ProjectShowData{Layout: s.layoutFor(r), Project: projectRow(*project)} diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index e8031d4..5be2224 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -104,7 +104,15 @@ func TestPackagePageShowsUsedBy(t *testing.T) { } } -func TestNavLinksToProjects(t *testing.T) { +func TestProjectPageFailsWhenComponentsCannotBeRead(t *testing.T) { ts := newProjectsTestServer(t) - assertContains(t, ts.page(t, "/ui/"), `href="/ui/projects"`) + ts.upload(t, "/api/projects/acme/web/sbom", syftSBOM(t)) + // The components query joins packages; without the table it fails while + // the project itself can still be read. + if _, err := ts.db.Exec("ALTER TABLE packages RENAME TO packages_gone"); err != nil { + t.Fatal(err) + } + if rec := ts.do(t, http.MethodGet, "/ui/projects/acme/web", "", nil); rec.Code != http.StatusInternalServerError { + t.Errorf("status = %d, want 500 rather than an empty project", rec.Code) + } } diff --git a/internal/server/server.go b/internal/server/server.go index 88897f7..6946e5c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -42,6 +42,7 @@ // - /ui/projects/... - Project inventory (when projects.enabled) // - /ui/api/browse/... - Archive browsing (used by the UI) // - /ui/api/compare/... - Archive diffing (used by the UI) +// - /ui/api/projects/... - Project deletion (used by the UI, needs the token) // // API endpoints for enrichment data: // - GET /api/package/{ecosystem}/{name} - Package metadata diff --git a/internal/server/static/project-delete.js b/internal/server/static/project-delete.js index b7422d7..637cdc1 100644 --- a/internal/server/static/project-delete.js +++ b/internal/server/static/project-delete.js @@ -1,6 +1,6 @@ // Deleting a project needs the upload token: /ui has no authentication of its -// own, so the page sends the token the user enters to the API, the same as an -// upload. The token is kept for the browser session only. +// own, so the page sends the token the user enters with the request, the same +// as an upload. The token is kept for the browser session only. (() => { const root = document.getElementById('project-delete'); if (!root) { @@ -34,7 +34,7 @@ form.addEventListener('submit', async (event) => { event.preventDefault(); - const url = '/api/projects/' + encodeURIComponent(root.dataset.project); + const url = '/ui/api/projects/' + encodeURIComponent(root.dataset.project); submit.disabled = true; errorBox.classList.add('hidden'); try { From 9240f9edef2152a2873bf578dad97db28b67e056 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 28 Oct 2026 14:10:48 +0100 Subject: [PATCH 010/198] Keep the upstream Composer index in memory Serving packages.json fetched the upstream's on every request with cache_metadata off, and after each TTL with it on, so every Composer command waited on the upstream and on http_timeout when it was down. The parsed index is now kept for the metadata TTL, or an hour when it is 0, and a failed fetch is not retried within that time. --- docs/configuration.md | 2 +- internal/handler/composer.go | 6 +- internal/handler/composer_advisories.go | 72 +++++++++++++++-- internal/handler/composer_advisories_test.go | 85 ++++++++++++++++++++ 4 files changed, 157 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7eb8485..1fe9629 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -347,7 +347,7 @@ upstream: composer_repository: "https://upstream-proxy.example.com/composer" ``` -The proxy reads `packages.json` from `upstream.composer_repository`, cached for the metadata TTL, and passes on its `security-advisories` and `filter` entries (Packagist's malware list) with URLs pointing at the proxy. An upstream that offers neither is still usable; `composer audit` then reports nothing from it. A filter `api-url` is not passed on; Composer reads filter entries from package metadata instead. +The proxy reads `packages.json` from `upstream.composer_repository` and keeps it in memory for the metadata TTL, or an hour when the TTL is 0, whether or not `cache_metadata` is on. A failed fetch is not retried within that time: the last good copy is used, or the entries are left out. It passes on its `security-advisories` and `filter` entries (Packagist's malware list) with URLs pointing at the proxy. An upstream that offers neither is still usable; `composer audit` then reports nothing from it. A filter `api-url` is not passed on; Composer reads filter entries from package metadata instead. `upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint. diff --git a/internal/handler/composer.go b/internal/handler/composer.go index bf04da0..2084ae6 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -55,6 +55,7 @@ type ComposerHandler struct { repoURL string proxyURL string routes packageRoutes + indexCache composerIndexCache } // NewComposerHandler creates a new Composer protocol handler. @@ -110,8 +111,9 @@ func (h *ComposerHandler) Routes() http.Handler { // handleServiceIndex returns the Composer repository service index. It // carries the security advisories and filter list entries of the default -// repository's own index when that has them; if the repository cannot be -// reached and nothing is cached, the index goes out without them. +// repository's own index when that has them, kept in memory for the metadata +// TTL; if the repository has not been reached, the index goes out without +// them. func (h *ComposerHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request) { index := map[string]any{ "packages": map[string]any{}, diff --git a/internal/handler/composer_advisories.go b/internal/handler/composer_advisories.go index e77da37..ed8f4bc 100644 --- a/internal/handler/composer_advisories.go +++ b/internal/handler/composer_advisories.go @@ -8,6 +8,10 @@ import ( "net/http" "net/url" "strings" + "sync" + "time" + + "golang.org/x/sync/singleflight" ) const ( @@ -32,18 +36,76 @@ type composerUpstreamIndex struct { filterSummaryURL string } -// upstreamIndex fetches the default repository's packages.json, cached for -// the metadata TTL. ok is false when the repository could not be reached and -// nothing is cached; a document without the fields yields an empty index. +// composerIndexFallbackTTL is how long the upstream index is kept when the +// metadata TTL is 0. +const composerIndexFallbackTTL = time.Hour + +// composerIndexCache keeps the default repository's parsed packages.json in +// memory, whether or not metadata caching is on, so that serving the proxy's +// own packages.json does not wait on the upstream. A failed fetch is +// remembered for the same interval: the last good copy, or none, is used +// until then instead of retrying on every request. The zero value is ready +// to use. +type composerIndexCache struct { + mu sync.Mutex + index composerUpstreamIndex + ok bool + fetchedAt time.Time + group singleflight.Group + now func() time.Time // for tests; nil means time.Now +} + +func (c *composerIndexCache) clock() time.Time { + if c.now != nil { + return c.now() + } + return time.Now() +} + +// upstreamIndex returns the default repository's packages.json fields. ok is +// false when the repository has not been reached since the proxy started; a +// document without the fields yields an empty index. func (h *ComposerHandler) upstreamIndex(ctx context.Context) (composerUpstreamIndex, bool) { + c := &h.indexCache + ttl := h.proxy.MetadataTTL + if ttl <= 0 { + ttl = composerIndexFallbackTTL + } + c.mu.Lock() + if !c.fetchedAt.IsZero() && c.clock().Sub(c.fetchedAt) < ttl { + index, ok := c.index, c.ok + c.mu.Unlock() + return index, ok + } + c.mu.Unlock() + + // One fetch serves every waiting request. It is detached from the + // request that started it, so a client hanging up neither fails the + // others nor gets remembered as an upstream failure. + _, _, _ = c.group.Do("index", func() (any, error) { + index, err := h.fetchUpstreamIndex(context.WithoutCancel(ctx)) + c.mu.Lock() + defer c.mu.Unlock() + if err == nil { + c.index, c.ok = index, true + } + c.fetchedAt = c.clock() + return nil, nil + }) + c.mu.Lock() + defer c.mu.Unlock() + return c.index, c.ok +} + +func (h *ComposerHandler) fetchUpstreamIndex(ctx context.Context) (composerUpstreamIndex, error) { indexURL := h.repoURL + "/packages.json" cacheKey := hashedCacheKey(h.repoURL, "/packages.json") res := h.proxy.fetchMetadataDocument(ctx, "composer", cacheKey, indexURL, contentTypeJSON, nil) if res.err != nil { h.proxy.Logger.Warn("failed to fetch upstream composer packages.json", "url", indexURL, "error", res.err) - return composerUpstreamIndex{}, false + return composerUpstreamIndex{}, res.err } - return parseComposerUpstreamIndex(indexURL, res.body), true + return parseComposerUpstreamIndex(indexURL, res.body), nil } func parseComposerUpstreamIndex(indexURL string, body []byte) composerUpstreamIndex { diff --git a/internal/handler/composer_advisories_test.go b/internal/handler/composer_advisories_test.go index 6adc7d2..5e1a204 100644 --- a/internal/handler/composer_advisories_test.go +++ b/internal/handler/composer_advisories_test.go @@ -9,7 +9,9 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" + "time" ) // composerAdvisoryUpstream serves a packages.json advertising security @@ -18,6 +20,7 @@ import ( type composerAdvisoryUpstream struct { *httptest.Server index string + down atomic.Bool // answer every request with 503 mu sync.Mutex requests []*recordedRequest @@ -43,6 +46,10 @@ func newComposerAdvisoryUpstream(t *testing.T) *composerAdvisoryUpstream { u.mu.Lock() u.requests = append(u.requests, &recordedRequest{r.Method, r.URL.Path, r.URL.Query(), string(body), r.Header.Clone()}) u.mu.Unlock() + if u.down.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } w.Header().Set("Content-Type", "application/json") switch r.URL.Path { case "/packages.json": @@ -164,6 +171,84 @@ func TestComposerServiceIndexUpstreamDown(t *testing.T) { } } +func advertisesAdvisories(index map[string]any) bool { + _, ok := index["security-advisories"] + return ok +} + +func TestComposerServiceIndexUpstreamDownNotRetried(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + upstream.down.Store(true) + h := newAdvisoryComposerHandler(t, upstream) + + serveComposerIndex(t, h) + fetched := len(upstream.requestsTo("/packages.json")) + if fetched == 0 { + t.Fatal("first packages.json made no upstream request") + } + index := serveComposerIndex(t, h) + if got := len(upstream.requestsTo("/packages.json")); got != fetched { + t.Errorf("second packages.json made %d upstream requests, want none", got-fetched) + } + if advertisesAdvisories(index) { + t.Errorf("security-advisories advertised without an upstream index: %v", index["security-advisories"]) + } +} + +func TestComposerServiceIndexKeptInMemory(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + h.proxy.MetadataTTL = 0 + now := time.Now() + h.indexCache.now = func() time.Time { return now } + + if !advertisesAdvisories(serveComposerIndex(t, h)) { + t.Fatal("security-advisories not advertised") + } + if got := len(upstream.requestsTo("/packages.json")); got != 1 { + t.Fatalf("upstream packages.json requests = %d, want 1", got) + } + + // Within the interval the copy in memory is served without asking. + upstream.down.Store(true) + now = now.Add(composerIndexFallbackTTL - time.Minute) + if !advertisesAdvisories(serveComposerIndex(t, h)) { + t.Error("security-advisories dropped within the interval") + } + if got := len(upstream.requestsTo("/packages.json")); got != 1 { + t.Errorf("upstream packages.json requests = %d within the interval, want 1", got) + } + + // After it the index is fetched again; the failure keeps the last good + // copy and is not retried until the next interval. + now = now.Add(2 * time.Minute) + if !advertisesAdvisories(serveComposerIndex(t, h)) { + t.Error("last good copy dropped after a failed fetch") + } + fetched := len(upstream.requestsTo("/packages.json")) + if fetched < 2 { + t.Fatalf("upstream packages.json requests = %d after the interval, want a new fetch", fetched) + } + serveComposerIndex(t, h) + if got := len(upstream.requestsTo("/packages.json")); got != fetched { + t.Errorf("failed fetch retried %d times within the interval", got-fetched) + } +} + +func TestComposerServiceIndexFetchedOnce(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + + var wg sync.WaitGroup + for range 10 { + wg.Go(func() { serveComposer(h, "/packages.json") }) + } + wg.Wait() + if got := len(upstream.requestsTo("/packages.json")); got != 1 { + t.Errorf("upstream packages.json requests = %d for parallel requests, want 1", got) + } +} + func serveAdvisories(h *ComposerHandler, method, target, body string) *httptest.ResponseRecorder { var reader io.Reader if body != "" { From 39d178f62938a5dba2bf4b5a05c9de14d7bf60e1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 09:33:56 +0100 Subject: [PATCH 011/198] Expire cooldown rewrites when a withheld version becomes available A cached rewrite made with cooldown on stays right until a version it held back leaves cooldown, so it now expires then instead of after a fixed time. Handlers ask cooldown through Proxy.cooldown, which reports each withheld version to the rewrites running at that moment. A rewrite that held nothing back is kept for the shortest cooldown, at most a day. --- docs/configuration.md | 4 +- internal/handler/cargo.go | 2 +- internal/handler/composer.go | 4 +- internal/handler/conda.go | 4 +- internal/handler/cooldown_expiry.go | 164 +++++++++++++++++ internal/handler/cooldown_expiry_test.go | 178 +++++++++++++++++++ internal/handler/cooldown_times.go | 2 +- internal/handler/go_cooldown.go | 2 +- internal/handler/handler.go | 4 + internal/handler/helm.go | 2 +- internal/handler/maven_cooldown.go | 6 +- internal/handler/metadata_access_test.go | 2 +- internal/handler/metadata_gzip_test.go | 2 +- internal/handler/metadata_validators_test.go | 2 +- internal/handler/npm.go | 6 +- internal/handler/npm_rewrite_test.go | 4 +- internal/handler/nuget_cooldown.go | 2 +- internal/handler/pub.go | 2 +- internal/handler/pypi.go | 8 +- internal/handler/rewrite_cache.go | 62 ++----- internal/handler/rewrite_cache_test.go | 43 +++-- 21 files changed, 413 insertions(+), 92 deletions(-) create mode 100644 internal/handler/cooldown_expiry.go create mode 100644 internal/handler/cooldown_expiry_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 941bc0f..7f62207 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1141,7 +1141,7 @@ Maven caveats: RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. -For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, but the filtered result is kept for up to an hour (see [rewritten metadata cache](#rewritten-metadata-cache)), so a version leaving cooldown can show up that much later; a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. +For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, and the filtered result is kept until the first version it held back leaves cooldown (see [rewritten metadata cache](#rewritten-metadata-cache)); a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead, and `npm dist-tag ls` shows the same tags. @@ -1420,7 +1420,7 @@ Or via environment variable: `PROXY_ARTIFACT_MAX_SIZE=2GB`. ### Rewritten metadata cache -The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite expires, because cooldown filtering depends on the current time: it is kept for a 24th of the shortest cooldown configured, at least a minute and at most an hour, so a version shows up at most that long after its cooldown ends. With a cooldown of a day or more that is an hour; with `"2h"` somewhere, five minutes. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. +The npm, Composer, PyPI and Helm handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones; a large Helm `index.yaml` takes over a hundred milliseconds. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. A PyPI simple page is cached per representation (HTML or JSON) and per set of versions filtered out of it. With version cooldown enabled, a cached rewrite expires when the first version it held back leaves cooldown, since versions it let through stay allowed and a new upstream version changes the document. A rewrite that held nothing back is kept for the shortest cooldown configured, at least a minute and at most a day. Cached documents carry an `ETag` of their own, and are dated like any other, so clients that revalidate with `If-None-Match` or `If-Modified-Since`, such as Composer, get `304 Not Modified`. The RubyGems compact index and Hex packages filtered by cooldown are kept the same way; they are never served gzip-compressed. Each cached document is kept gzip-compressed, when that makes it smaller, and served with `Content-Encoding: gzip` to clients that accept it, as npm, pnpm, Yarn and Composer do. A large packument is compressed once rather than on every request, by the proxy or by a reverse proxy in front of it. Only the compressed copy is kept, typically an eighth of the document, and it is decompressed for the clients that don't accept gzip. Documents rewritten outside the cache are served uncompressed. diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 578c9ec..0c26551 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -171,7 +171,7 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr cratePURL := canonicalPackagePURL("cargo", crate.Name) - if !h.proxy.Cooldown.IsAllowed("cargo", cratePURL, publishedAt) { + if !h.proxy.cooldown().IsAllowed("cargo", cratePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering cargo version", "crate", crate.Name, "version", crate.Version, "published", crate.PublishTime) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 116d106..ccb42bb 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -394,7 +394,7 @@ func (h *ComposerHandler) shouldFilterVersion(packagePURL, packageName, version return false } - if !h.proxy.Cooldown.IsAllowed("composer", packagePURL, publishedAt) { + if !h.proxy.cooldown().IsAllowed("composer", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering composer version", "package", packageName, "version", version) return true @@ -731,7 +731,7 @@ func (h *ComposerHandler) serveCachedRelease(w http.ResponseWriter, r *http.Requ if err != nil || ver == nil || !ver.PublishedAt.Valid { return false } - if !h.proxy.Cooldown.IsAllowed("composer", canonicalPackagePURL("composer", packageName), ver.PublishedAt.Time) { + if !h.proxy.cooldown().IsAllowed("composer", canonicalPackagePURL("composer", packageName), ver.PublishedAt.Time) { h.proxy.Logger.Info("cooldown: withholding composer archive", "package", packageName, "version", version) http.Error(w, "version not found", http.StatusNotFound) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 76a43f2..3cbaf93 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -493,7 +493,7 @@ func (f *repodataFilter) inCooldown(filename string, raw []byte) bool { return false } publishedAt := time.Unix(int64(entry.Timestamp)/condaTimestampDivisor, 0) - decision := f.h.proxy.Cooldown.Evaluate("conda", canonicalPackagePURL("conda", entry.Name), publishedAt, f.now) + decision := f.h.proxy.cooldown().Evaluate("conda", canonicalPackagePURL("conda", entry.Name), publishedAt, f.now) if decision.Allowed { return false } @@ -517,7 +517,7 @@ func (h *CondaHandler) recordInCooldown(filename, name, version string, timestam return false } publishedAt := time.Unix(int64(timestamp)/condaTimestampDivisor, 0) - if h.proxy.Cooldown.IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) { + if h.proxy.cooldown().IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) { return false } h.proxy.Logger.Info("cooldown: filtering conda package", diff --git a/internal/handler/cooldown_expiry.go b/internal/handler/cooldown_expiry.go new file mode 100644 index 0000000..1163bde --- /dev/null +++ b/internal/handler/cooldown_expiry.go @@ -0,0 +1,164 @@ +package handler + +import ( + "sync" + "time" + + "github.com/git-pkgs/cooldown" +) + +// A rewrite made with cooldown on stays right until a version it held back +// leaves cooldown: versions it let through stay allowed as time passes, and +// new versions upstream change the document and so the cache key. So the +// cached rewrite expires when the earliest version it withheld becomes +// available, instead of after a fixed time. +// +// Handlers ask cooldown through Proxy.cooldown, which tells every rewrite +// running at that moment when a withheld version becomes available. A +// rewrite may hear of versions another rewrite withheld at the same time; +// that only makes it expire earlier than it has to, never later. + +// maxCooldownRewriteTTL is the longest a rewrite made with cooldown on is +// served, even when it held nothing back. +const maxCooldownRewriteTTL = 24 * time.Hour + +// minCooldownRewriteTTL is the shortest cap on a rewrite made with cooldown +// on, however short the cooldown. +const minCooldownRewriteTTL = time.Minute + +// cooldownRewriteTTL returns the longest a rewrite made with cooldown on is +// served: the shortest cooldown configured, between a minute and a day. It +// only matters when a rewrite held nothing back, or held back a version +// whose availability is unknown. +func (p *Proxy) cooldownRewriteTTL() time.Duration { + ttl := p.ShortestCooldown + switch { + case ttl <= 0 || ttl > maxCooldownRewriteTTL: + return maxCooldownRewriteTTL + case ttl < minCooldownRewriteTTL: + return minCooldownRewriteTTL + } + return ttl +} + +// cooldownWatch holds the rewrites running with cooldown on. +type cooldownWatch struct { + mu sync.Mutex + active map[*cooldownRecorder]struct{} +} + +// cooldownRecorder collects when the versions withheld during one rewrite +// become available. +type cooldownRecorder struct { + // earliest is when the first withheld version becomes available, or + // zero if none was withheld. + earliest time.Time + // unknown is set when a version was withheld without a time it becomes + // available. + unknown bool +} + +// watchCooldown starts recording the versions cooldown holds back for a +// rewrite about to run. unwatchCooldown ends it. +func (p *Proxy) watchCooldown() *cooldownRecorder { + rec := &cooldownRecorder{} + w := &p.cooldownWatch + w.mu.Lock() + defer w.mu.Unlock() + if w.active == nil { + w.active = make(map[*cooldownRecorder]struct{}) + } + w.active[rec] = struct{}{} + return rec +} + +func (p *Proxy) unwatchCooldown(rec *cooldownRecorder) { + w := &p.cooldownWatch + w.mu.Lock() + defer w.mu.Unlock() + delete(w.active, rec) +} + +// cooldownExpiry returns when a rewrite that started at start and recorded +// into rec should expire: when the first version it held back leaves +// cooldown, and no later than cooldownRewriteTTL after start. +func (p *Proxy) cooldownExpiry(rec *cooldownRecorder, start time.Time) time.Time { + w := &p.cooldownWatch + w.mu.Lock() + earliest, unknown := rec.earliest, rec.unknown + w.mu.Unlock() + + limit := start.Add(p.cooldownRewriteTTL()) + if unknown || earliest.IsZero() || earliest.After(limit) { + return limit + } + return earliest +} + +// withheld tells the running rewrites that a version is held back until +// availableAt, zero when that is unknown. +func (p *Proxy) withheld(availableAt time.Time) { + w := &p.cooldownWatch + w.mu.Lock() + defer w.mu.Unlock() + for rec := range w.active { + switch { + case availableAt.IsZero(): + rec.unknown = true + case rec.earliest.IsZero() || availableAt.Before(rec.earliest): + rec.earliest = availableAt + } + } +} + +// cooldown returns the proxy's cooldown policy, telling the running rewrites +// about each version it holds back. Handlers use it rather than p.Cooldown. +func (p *Proxy) cooldown() CooldownPolicy { //nolint:ireturn // nil when cooldown is not configured, as p.Cooldown is + if p.Cooldown == nil { + return nil + } + return recordingCooldown{p: p} +} + +// recordingCooldown is the proxy's cooldown policy, reporting the versions it +// holds back to the running rewrites. +type recordingCooldown struct { + p *Proxy +} + +func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { + if r.p.Cooldown.IsAllowed(ecosystem, packagePURL, publishedAt) { + return true + } + r.p.withheld(r.availableAt(ecosystem, packagePURL, publishedAt)) + return false +} + +func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision { + d := r.p.Cooldown.Evaluate(ecosystem, packagePURL, publishedAt, evaluatedAt) + if !d.Allowed { + r.p.withheld(d.AvailableAt) + } + return d +} + +func (r recordingCooldown) For(ecosystem, packagePURL string) time.Duration { + return r.p.Cooldown.For(ecosystem, packagePURL) +} + +func (r recordingCooldown) Enabled() bool { + return r.p.Cooldown.Enabled() +} + +// availableAt is when a version IsAllowed held back leaves cooldown, or zero +// when that is unknown. +func (r recordingCooldown) availableAt(ecosystem, packagePURL string, publishedAt time.Time) time.Time { + if publishedAt.IsZero() { + return time.Time{} + } + d := r.p.Cooldown.For(ecosystem, packagePURL) + if d <= 0 { + return time.Time{} + } + return publishedAt.Add(d) +} diff --git a/internal/handler/cooldown_expiry_test.go b/internal/handler/cooldown_expiry_test.go new file mode 100644 index 0000000..fd55845 --- /dev/null +++ b/internal/handler/cooldown_expiry_test.go @@ -0,0 +1,178 @@ +package handler + +import ( + "context" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// TestCachedRewrite_ExpiresWhenWithheldVersionIsAvailable checks that a +// rewrite which held a version back is cached until that version leaves +// cooldown, and no longer. +func TestCachedRewrite_ExpiresWhenWithheldVersionIsAvailable(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + clock := &testClock{t: time.Now()} + proxy.now = clock.now + proxy.SetMetadataRewriteCacheSize(1 << 20) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + purl := canonicalPackagePURL("npm", "left-pad") + // One version is 3d-5h old: it leaves cooldown in five hours. + published := clock.now().Add(-72*time.Hour + 5*time.Hour) + + var calls int + var rewrite keepingRewrite = func(b []byte) ([]byte, []string, error) { + calls++ + if proxy.cooldown().IsAllowed("npm", purl, published) { + return b, []string{"1.0.0", "1.1.0"}, nil + } + return b, []string{"1.0.0"}, nil + } + get := func() { + if _, err := proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", rewrite); err != nil { + t.Fatal(err) + } + } + + get() + clock.advance(5*time.Hour - time.Minute) + get() + if calls != 1 { + t.Errorf("rewrites before the withheld version is available = %d, want 1", calls) + } + clock.advance(time.Minute) + get() + if calls != 2 { + t.Errorf("rewrites once the withheld version is available = %d, want 2", calls) + } +} + +// TestCachedRewrite_NothingWithheldCachedForCap checks that a rewrite which +// held nothing back is cached for cooldownRewriteTTL. +func TestCachedRewrite_NothingWithheldCachedForCap(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} + proxy.now = clock.now + proxy.SetMetadataRewriteCacheSize(1 << 20) + proxy.Cooldown = &cooldown.Config{Default: "2h"} + proxy.ShortestCooldown = 2 * time.Hour + var r countingRewrite + + get := func() { + _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), "", r.keeping) + } + get() + clock.advance(2*time.Hour - time.Second) + get() + if got := r.calls.Load(); got != 1 { + t.Errorf("rewrites within the cap = %d, want 1", got) + } + clock.advance(time.Second) + get() + if got := r.calls.Load(); got != 2 { + t.Errorf("rewrites after the cap = %d, want 2", got) + } +} + +// TestCooldownExpiry checks how one rewrite's recorded decisions set its +// expiry, including decisions made for other rewrites running at the same +// time, which may only make it expire earlier. +func TestCooldownExpiry(t *testing.T) { + p := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}, ShortestCooldown: 72 * time.Hour} + // IsAllowed compares with the wall clock, so the times are relative to it. + start := time.Now() + limit := start.Add(24 * time.Hour) + purl := canonicalPackagePURL("npm", "left-pad") + + rec := p.watchCooldown() + if got := p.cooldownExpiry(rec, start); !got.Equal(limit) { + t.Errorf("nothing withheld: expiry = %v, want the cap %v", got, limit) + } + + other := p.watchCooldown() + p.cooldown().IsAllowed("npm", purl, start.Add(-60*time.Hour)) // available in 12h + p.unwatchCooldown(other) + p.cooldown().IsAllowed("npm", purl, start.Add(-66*time.Hour)) // available in 6h, after other ended + if got, want := p.cooldownExpiry(rec, start), start.Add(6*time.Hour); !got.Equal(want) { + t.Errorf("rec expiry = %v, want the earliest availability %v", got, want) + } + if got, want := p.cooldownExpiry(other, start), start.Add(12*time.Hour); !got.Equal(want) { + t.Errorf("other expiry = %v, want %v: decisions after it ended don't count", got, want) + } + + p.cooldown().Evaluate("npm", purl, start.Add(-71*time.Hour), start) // available in 1h + if got, want := p.cooldownExpiry(rec, start), start.Add(time.Hour); !got.Equal(want) { + t.Errorf("after Evaluate: expiry = %v, want %v", got, want) + } + p.unwatchCooldown(rec) + if len(p.cooldownWatch.active) != 0 { + t.Errorf("%d recorders left after unwatch", len(p.cooldownWatch.active)) + } +} + +// TestHandlersAskCooldownThroughRecorder checks that handlers make cooldown +// decisions through Proxy.cooldown. A decision made on Proxy.Cooldown +// directly would not reach the rewrites running, so a cached rewrite could +// keep hiding a version past its cooldown. +func TestHandlersAskCooldownThroughRecorder(t *testing.T) { + direct := regexp.MustCompile(`\.Cooldown\.(IsAllowed|Evaluate)\(|Cooldown:\s+\w+(\.\w+)*\.Cooldown\b`) + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + for _, f := range files { + if strings.HasSuffix(f, "_test.go") || f == "cooldown_expiry.go" { + continue + } + src, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for i, line := range strings.Split(string(src), "\n") { + if direct.MatchString(line) { + t.Errorf("%s:%d asks Proxy.Cooldown directly; use Proxy.cooldown(): %s", f, i+1, strings.TrimSpace(line)) + } + } + } +} + +// TestHandlerRewriteExpiresWithWithheldVersion checks, through the handlers, +// that a cached rewrite expires when the version cooldown held back from it +// becomes available. +func TestHandlerRewriteExpiresWithWithheldVersion(t *testing.T) { + for _, c := range validatorCases { + t.Run(c.ecosystem, func(t *testing.T) { + u := cooldownUpstream(t) // the newest version was published 2h ago + proxy, _, clock := validatorProxyWithDB(t, u, time.Hour) + // Cooldown compares with the wall clock, so the proxy's clock + // follows it here. + clock.t = time.Now() + proxy.Cooldown = &cooldown.Config{Default: "3h"} + proxy.ShortestCooldown = 3 * time.Hour + handler := c.handler(proxy, u) + + if rec := serve(handler, c.path, nil); rec.Code != http.StatusOK { + t.Fatalf("status %d", rec.Code) + } + proxy.rewrites.mu.Lock() + defer proxy.rewrites.mu.Unlock() + if len(proxy.rewrites.entries) != 1 { + t.Fatalf("%d cached rewrites, want 1", len(proxy.rewrites.entries)) + } + for _, el := range proxy.rewrites.entries { + expires := el.Value.(*rewriteEntry).expires + // Published 2h ago with a 3h cooldown: available in an hour, + // to the second the publish time was given in. + if want := time.Now().Add(time.Hour); expires.Before(want.Add(-2*time.Second)) || expires.After(want.Add(time.Second)) { + t.Errorf("rewrite expires %v, want about %v when the withheld version is available", expires, want) + } + } + }) + } +} diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index 88532cd..5d95d16 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -126,7 +126,7 @@ func (d timedDocuments) withheld(p *Proxy, ecosystem, packagePURL, version strin if published.IsZero() { return false } - return !p.Cooldown.IsAllowed(ecosystem, packagePURL, published) + return !p.cooldown().IsAllowed(ecosystem, packagePURL, published) } // decodedMetadataBody returns res's body without its Content-Encoding. The diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index 90189d4..892e77b 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -42,7 +42,7 @@ func (h *GoHandler) allowedAt(module string, published time.Time) bool { if published.IsZero() { return true } - return h.proxy.Cooldown.IsAllowed("golang", canonicalPackagePURL("golang", module), published) + return h.proxy.cooldown().IsAllowed("golang", canonicalPackagePURL("golang", module), published) } // writeGoMetadataError answers a failed upstream metadata fetch. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 5693421..1bc5ef7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -284,6 +284,10 @@ type Proxy struct { // the process started; see metadataDate. now func() time.Time started time.Time + + // cooldownWatch holds the rewrites recording which versions cooldown + // holds back; see cooldown_expiry.go. + cooldownWatch cooldownWatch } // CooldownPolicy supplies version-age filtering and package-specific durations. diff --git a/internal/handler/helm.go b/internal/handler/helm.go index 07e2f74..fdc000e 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -378,7 +378,7 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { return !created.IsZero() && h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() && - !h.proxy.Cooldown.IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created) + !h.proxy.cooldown().IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created) } type helmChartDownload struct { diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index 4705f70..4da839e 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -222,7 +222,7 @@ func (h *MavenHandler) checkCooldown(w http.ResponseWriter, r *http.Request, sou http.Error(w, "failed to check cooldown", http.StatusBadGateway) return true } - if !h.proxy.Cooldown.IsAllowed("maven", mavenPackagePURL(group, artifact), published) { + if !h.proxy.cooldown().IsAllowed("maven", mavenPackagePURL(group, artifact), published) { h.proxy.Logger.Info("cooldown: withholding maven version", "group", group, "artifact", artifact, "version", version) http.Error(w, errMavenCooldown+": "+canonicalVersionPURL("maven", group+":"+artifact, version), http.StatusForbidden) @@ -307,7 +307,7 @@ func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, a packagePURL := mavenPackagePURL(group, artifact) // No version can be newer than the metadata listing it. if lastUpdated, err := time.Parse(mavenLastUpdatedLayout, doc.lastUpdated); err == nil && - h.proxy.Cooldown.IsAllowed("maven", packagePURL, lastUpdated) { + h.proxy.cooldown().IsAllowed("maven", packagePURL, lastUpdated) { return body, nil } @@ -324,7 +324,7 @@ func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, a if published.IsZero() { continue } - if h.proxy.Cooldown.IsAllowed("maven", packagePURL, published) { + if h.proxy.cooldown().IsAllowed("maven", packagePURL, published) { break } h.proxy.Logger.Info("cooldown: filtering maven version", "group", group, "artifact", artifact, "version", v) diff --git a/internal/handler/metadata_access_test.go b/internal/handler/metadata_access_test.go index 0a64b80..7b5e1cf 100644 --- a/internal/handler/metadata_access_test.go +++ b/internal/handler/metadata_access_test.go @@ -143,7 +143,7 @@ func TestStoredRewrite_MarksAccess(t *testing.T) { body := []byte(`{}`) proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } old := ageMetadataAccess(t, db, "npm", "demo") diff --git a/internal/handler/metadata_gzip_test.go b/internal/handler/metadata_gzip_test.go index b5939c7..57d20cc 100644 --- a/internal/handler/metadata_gzip_test.go +++ b/internal/handler/metadata_gzip_test.go @@ -184,7 +184,7 @@ func TestRewriteCache_KeepsGzipCopyOnlyWhenSmaller(t *testing.T) { for _, doc := range [][]byte{large, []byte("abc")} { key := rewriteCacheKey("npm", "http://proxy", string(doc[:3]), doc) - if _, err := c.rewrite(t.Context(), key, doc, 0, identity); err != nil { + if _, err := c.rewrite(t.Context(), key, doc, nil, identity); err != nil { t.Fatal(err) } } diff --git a/internal/handler/metadata_validators_test.go b/internal/handler/metadata_validators_test.go index f7c1475..340e3db 100644 --- a/internal/handler/metadata_validators_test.go +++ b/internal/handler/metadata_validators_test.go @@ -500,7 +500,7 @@ func TestRewriteCacheETag(t *testing.T) { exclaim := func(b []byte) ([]byte, error) { return []byte(string(b) + "!"), nil } key := rewriteCacheKey("npm", "http://proxy", "demo", []byte("doc")) - m, err := c.rewrite(t.Context(), key, []byte("doc"), 0, exclaim) + m, err := c.rewrite(t.Context(), key, []byte("doc"), nil, exclaim) if err != nil || string(m.content()) != "doc!" { t.Fatalf("rewrite = %q, %v", m.content(), err) } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 537c4b3..7263b3b 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -467,7 +467,7 @@ func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, l func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry) npm.Options { return npm.Options{ ProxyURL: h.proxyURL, - Cooldown: h.proxy.Cooldown, + Cooldown: h.proxy.cooldown(), DeniedVersions: h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName)), EvaluatedAt: time.Now(), LocalVersions: local, @@ -685,7 +685,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun versionPURL := canonicalVersionPURL("npm", packageName, version) if !routed { if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time), nil + return !h.proxy.cooldown().IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time), nil } } @@ -717,7 +717,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun } } - return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt), nil + return !h.proxy.cooldown().IsAllowed("npm", canonicalPackagePURL("npm", packageName), publishedAt), nil } // npmMetadataPath returns the package name and the encoded registry path. diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go index b36b229..9b8e35b 100644 --- a/internal/handler/npm_rewrite_test.go +++ b/internal/handler/npm_rewrite_test.go @@ -94,7 +94,7 @@ func TestNPMStoredRewriteSkippedWhenStale(t *testing.T) { body := []byte(`{}`) proxy.cacheMetadataBlob(t.Context(), "npm", "demo", metadataStoragePath("npm", "demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } if _, ok := proxy.storedRewrite("npm", "demo", "http://proxy", "demo"); !ok { @@ -129,7 +129,7 @@ func TestNPMStoredRewriteRecordsMetadataHit(t *testing.T) { body := []byte(`{"name":"demo"}`) proxy.cacheMetadataBlob(t.Context(), "npm", "hit-demo", metadataStoragePath("npm", "hit-demo"), &upstreamMetadata{body: body}) key := rewriteCacheKey("npm", "http://proxy", "hit-demo", body) - if _, err := proxy.rewrites.rewrite(t.Context(), key, body, 0, func(b []byte) ([]byte, error) { return b, nil }); err != nil { + if _, err := proxy.rewrites.rewrite(t.Context(), key, body, nil, func(b []byte) ([]byte, error) { return b, nil }); err != nil { t.Fatal(err) } diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index afb147d..b9345c5 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -204,7 +204,7 @@ func (h *NuGetHandler) nugetLeafAllowed(leaf map[string]any, id string) bool { if err != nil { return true } - return h.proxy.Cooldown.IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) + return h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) } func nugetCatalogEntry(leaf map[string]any) map[string]any { diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 3c81298..24fdd02 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -247,7 +247,7 @@ func (h *PubHandler) shouldFilterVersion(packagePURL, name, version string, vmap return false } - if !h.proxy.Cooldown.IsAllowed("pub", packagePURL, publishedAt) { + if !h.proxy.cooldown().IsAllowed("pub", packagePURL, publishedAt) { h.proxy.Logger.Info("cooldown: filtering pub version", "package", name, "version", version) return true diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index fe018d8..ebb3082 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -318,7 +318,7 @@ func (h *PyPIHandler) fetchFilteredVersions(r *http.Request, name string) (map[s filtered := make(map[string]bool) for version, publishedAt := range times { - if !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) { + if !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) { filtered[version] = true } } @@ -611,7 +611,7 @@ func (h *PyPIHandler) shouldFilterRelease(packagePURL string, files any) bool { } publishedAt := h.newestUploadTime(filesArr) - return !publishedAt.IsZero() && !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, publishedAt) + return !publishedAt.IsZero() && !h.proxy.cooldown().IsAllowed("pypi", packagePURL, publishedAt) } // versionInCooldown reports whether a version is still inside the cooldown @@ -636,7 +636,7 @@ func (h *PyPIHandler) versionInCooldown(r *http.Request, name, version string) ( return false, nil } h.storeUploadTime(name, version, publishedAt) - return !h.proxy.Cooldown.IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil + return !h.proxy.cooldown().IsAllowed("pypi", canonicalPackagePURL("pypi", name), publishedAt), nil } // storeUploadTime records a release's newest upload time under the version @@ -675,7 +675,7 @@ func (h *PyPIHandler) storedCooldown(r *http.Request, name, version, filename st if err != nil || ver == nil || !ver.PublishedAt.Valid { return false, false } - if !h.proxy.Cooldown.IsAllowed("pypi", packagePURL, ver.PublishedAt.Time) { + if !h.proxy.cooldown().IsAllowed("pypi", packagePURL, ver.PublishedAt.Time) { return true, true } cached, err := h.proxy.HasCachedArtifact(r.Context(), "pypi", name, version, filename) diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index b2dc992..fd455a7 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -35,8 +35,8 @@ import ( // depends on is fixed for the life of the process: the proxy URL is part of // the key, and the denylist is loaded at startup. The exception is cooldown, // which depends on the time of the request: with cooldown on, entries expire -// after Proxy.cooldownRewriteTTL, so a version is held back at most that long -// past its cooldown. +// when the first version they held back leaves cooldown (see +// cooldown_expiry.go). // // Each entry keeps the ETag of its output, so clients can revalidate their // copy without the output being hashed again on every request. Dates are not @@ -65,36 +65,6 @@ type rewriteEntry struct { expires time.Time } -// maxCooldownRewriteTTL is the longest a rewrite made with cooldown on is -// served. Versions leave cooldown as time passes, so a cached rewrite would -// otherwise keep hiding them. Cooldowns run to days, and a package manager -// that checks every lockfile entry on each install, as pnpm 12 does, would -// otherwise have the whole lockfile rewritten again on every install more -// than a minute after the last. -const maxCooldownRewriteTTL = time.Hour - -// minCooldownRewriteTTL is the shortest a rewrite made with cooldown on is -// served, however short the cooldown. -const minCooldownRewriteTTL = time.Minute - -// cooldownRewriteShare is the share of the shortest cooldown a rewrite made -// with cooldown on is served for, as a divisor. -const cooldownRewriteShare = 24 - -// cooldownRewriteTTL returns how long a rewrite made with cooldown on is -// served: a 24th of the shortest cooldown configured, so a version is held -// back at most about 4% past its cooldown, kept between a minute and an hour. -func (p *Proxy) cooldownRewriteTTL() time.Duration { - ttl := p.ShortestCooldown / cooldownRewriteShare - switch { - case ttl <= 0 || ttl > maxCooldownRewriteTTL: - return maxCooldownRewriteTTL - case ttl < minCooldownRewriteTTL: - return minCooldownRewriteTTL - } - return ttl -} - // size is how much of the cache's room the entry takes. func (e *rewriteEntry) size() int64 { return int64(len(e.plain) + len(e.gzipped)) @@ -279,7 +249,7 @@ func (c *rewriteCache) remove(el *list.Element) { // shared. A caller waiting on another's rewrite leaves when ctx ends; the // rewrite itself always runs to completion, so the result is cached for the // next request. -func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, ttl time.Duration, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { +func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, expiry func(start time.Time) time.Time, rewrite func([]byte) ([]byte, error)) (rewrittenMetadata, error) { if c == nil { return rewriteWithETag(in, rewrite) } @@ -302,16 +272,17 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, ttl t c.inFlight[key] = f c.mu.Unlock() - // The entry expires counting from before the rewrite, so it never - // outlives the time its cooldown decisions were made at by more than ttl. - var expires time.Time - if ttl > 0 { - expires = c.clock().Add(ttl) - } + // expiry is given the time from before the rewrite, so an entry never + // outlives the time its cooldown decisions were made at. + start := c.clock() defer func() { c.mu.Lock() delete(c.inFlight, key) if f.err == nil { + var expires time.Time + if expiry != nil { + expires = expiry(start) + } c.add(&rewriteEntry{key: key, rewrittenMetadata: f.m.compact(), expires: expires}) } c.mu.Unlock() @@ -370,7 +341,8 @@ func (c *rewriteCache) add(e *rewriteEntry) { // metadata cache stored it, and is empty otherwise; when known it keys the // cache, so the document isn't hashed again. // -// With cooldown on, the cached rewrite expires after p.cooldownRewriteTTL. Its +// With cooldown on, the cached rewrite expires when the first version it +// held back leaves cooldown, and after p.cooldownRewriteTTL at the latest. Its // ETag is the hash of its output, and it is dated like any other, so clients // revalidate with either validator and get the gzip copy. // @@ -390,11 +362,15 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str if digest == "" { key = rewriteCacheKey(ecosystem, proxyURL, name, in) } - var ttl time.Duration + var expiry func(time.Time) time.Time if cooldown { - ttl = p.cooldownRewriteTTL() + // Recording runs from before the rewrite until after it, so it + // hears of every version the rewrite holds back. + rec := p.watchCooldown() + defer p.unwatchCooldown(rec) + expiry = func(start time.Time) time.Time { return p.cooldownExpiry(rec, start) } } - m, err := p.rewrites.rewrite(ctx, key, in, ttl, rewriteOnly) + m, err := p.rewrites.rewrite(ctx, key, in, expiry, rewriteOnly) if err == nil { m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) } diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go index 7188634..3f4c062 100644 --- a/internal/handler/rewrite_cache_test.go +++ b/internal/handler/rewrite_cache_test.go @@ -44,7 +44,7 @@ func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte("doc")) for range 3 { - m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) + m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) if err != nil || string(m.content()) != "DOC" { t.Fatalf("rewrite = %q, %v", m.content(), err) } @@ -60,7 +60,7 @@ func TestRewriteCache_NewUpstreamBytesRewriteAgain(t *testing.T) { for _, doc := range []string{"v1", "v2"} { key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte(doc)) - m, err := c.rewrite(context.Background(), key, []byte(doc), 0, r.fn) + m, err := c.rewrite(context.Background(), key, []byte(doc), nil, r.fn) if err != nil || string(m.content()) != strings.ToUpper(doc) { t.Fatalf("rewrite(%s) = %q, %v", doc, m.content(), err) } @@ -82,7 +82,7 @@ func TestRewriteCache_ConcurrentRequestsShareRewrite(t *testing.T) { outs := make(chan string, n) run := func() { - m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) + m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) if err != nil { t.Error(err) } @@ -115,20 +115,20 @@ func TestRewriteCache_WaiterLeavingKeepsRewrite(t *testing.T) { first := make(chan error, 1) go func() { - _, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn) + _, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn) first <- err }() <-r.entered ctx, cancel := context.WithCancel(context.Background()) cancel() - if _, err := c.rewrite(ctx, key, []byte("doc"), 0, r.fn); !errors.Is(err, context.Canceled) { + if _, err := c.rewrite(ctx, key, []byte("doc"), nil, r.fn); !errors.Is(err, context.Canceled) { t.Errorf("waiter err = %v, want context.Canceled", err) } close(r.release) if err := <-first; err != nil { t.Fatal(err) } - if m, err := c.rewrite(context.Background(), key, []byte("doc"), 0, r.fn); err != nil || string(m.content()) != "DOC" { + if m, err := c.rewrite(context.Background(), key, []byte("doc"), nil, r.fn); err != nil || string(m.content()) != "DOC" { t.Errorf("cached rewrite = %q, %v", m.content(), err) } if got := r.calls.Load(); got != 1 { @@ -142,18 +142,18 @@ func TestRewriteCache_EvictsLeastRecentlyUsed(t *testing.T) { keyFor := func(doc string) string { return rewriteCacheKey("npm", "http://proxy", doc, []byte(doc)) } for _, doc := range []string{"aaa", "bbb"} { - _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), 0, r.fn) + _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), nil, r.fn) } - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), 0, r.fn) // aaa is now the most recent - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), 0, r.fn) // evicts bbb + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), nil, r.fn) // aaa is now the most recent + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), nil, r.fn) // evicts bbb before := r.calls.Load() - _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), 0, r.fn) - _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), 0, r.fn) + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), nil, r.fn) + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), nil, r.fn) if got := r.calls.Load() - before; got != 0 { t.Errorf("aaa and ccc rewritten %d times, want both still cached", got) } - _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), 0, r.fn) + _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), nil, r.fn) if got := r.calls.Load() - before; got != 1 { t.Errorf("bbb should have been evicted and rewritten") } @@ -164,7 +164,7 @@ func TestRewriteCache_OutputLargerThanCacheIsNotStored(t *testing.T) { var r countingRewrite key := rewriteCacheKey("npm", "http://proxy", "big", []byte("big")) - _, _ = c.rewrite(context.Background(), key, []byte("big"), 0, r.fn) + _, _ = c.rewrite(context.Background(), key, []byte("big"), nil, r.fn) if c.len() != 0 { t.Errorf("cached %d entries, want 0", c.len()) } @@ -177,7 +177,7 @@ func TestRewriteCache_ErrorsAreNotCached(t *testing.T) { key := rewriteCacheKey("npm", "http://proxy", "broken", []byte("doc")) for range 2 { - if _, err := c.rewrite(context.Background(), key, []byte("doc"), 0, failing); err == nil { + if _, err := c.rewrite(context.Background(), key, []byte("doc"), nil, failing); err == nil { t.Fatal("expected the rewrite error") } } @@ -232,7 +232,7 @@ func TestRewriteCacheExpiredEntryFreesRoom(t *testing.T) { clock := &testClock{t: time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC)} c.now = clock.now upper := func(b []byte) ([]byte, error) { return bytes.ToUpper(b), nil } - if _, err := c.rewrite(context.Background(), "k", []byte("doc"), time.Minute, upper); err != nil { + if _, err := c.rewrite(context.Background(), "k", []byte("doc"), func(start time.Time) time.Time { return start.Add(time.Minute) }, upper); err != nil { t.Fatal(err) } clock.advance(time.Minute) @@ -302,17 +302,16 @@ func (c *rewriteCache) len() int { return len(c.entries) } -// TestCooldownRewriteTTL checks that a rewrite filtered by cooldown is cached -// for a 24th of the shortest cooldown, between a minute and an hour. +// TestCooldownRewriteTTL checks that the cap on a rewrite filtered by +// cooldown is the shortest cooldown, between a minute and a day. func TestCooldownRewriteTTL(t *testing.T) { for _, tc := range []struct { shortest, want time.Duration }{ - {0, time.Hour}, - {72 * time.Hour, time.Hour}, - {24 * time.Hour, time.Hour}, - {2 * time.Hour, 5 * time.Minute}, - {10 * time.Minute, time.Minute}, + {0, 24 * time.Hour}, + {72 * time.Hour, 24 * time.Hour}, + {2 * time.Hour, 2 * time.Hour}, + {10 * time.Second, time.Minute}, } { p := &Proxy{ShortestCooldown: tc.shortest} if got := p.cooldownRewriteTTL(); got != tc.want { From c4615ff56164f410211ede3732fadf07e4d6c244 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 10:09:21 +0100 Subject: [PATCH 012/198] Read change feeds through git-pkgs/registries The feed readers move to registries' changes package. The proxy keeps the cursor and moves it past the changes only once their metadata is expired, so a failed expiry reads them again. --- internal/changefeed/changefeed.go | 261 ------------------------- internal/changefeed/changefeed_test.go | 198 ------------------- internal/server/change_feeds.go | 54 +++-- internal/server/change_feeds_test.go | 75 +++++-- internal/server/server.go | 48 +++-- 5 files changed, 124 insertions(+), 512 deletions(-) delete mode 100644 internal/changefeed/changefeed.go delete mode 100644 internal/changefeed/changefeed_test.go diff --git a/internal/changefeed/changefeed.go b/internal/changefeed/changefeed.go deleted file mode 100644 index 2e2d48f..0000000 --- a/internal/changefeed/changefeed.go +++ /dev/null @@ -1,261 +0,0 @@ -// Package changefeed reads the feeds in which public registries list the -// packages that changed since a given point, so the proxy can revalidate its -// cached metadata for them before metadata_ttl runs out. -package changefeed - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "slices" - "strconv" - "strings" -) - -// Feed is one registry's change feed. It remembers how far it has read, in -// memory only: after a restart it starts again from the registry's current -// position. -type Feed interface { - // Ecosystem is the metadata cache ecosystem the changed names belong to. - Ecosystem() string - // Changes returns what changed since the previous call. The first call - // only finds the feed's current position and reports no changes. On error - // the position stays where it was, so the next call asks again. - Changes(ctx context.Context) (Changes, error) -} - -// Changes is one read of a feed. -type Changes struct { - // Names are the cache keys of the documents that changed. - Names []string - // All is set when the feed can't say what changed, so every document of - // the ecosystem has to be treated as changed. - All bool -} - -// maxFeedResponse bounds a feed response read into memory. A full page of -// the npm feed is about 1.5 MB. -const maxFeedResponse = 64 << 20 - -// getJSON fetches u and decodes its JSON body into v. A status other than 200 -// is an error unless allowStatus also accepts it. -func getJSON(ctx context.Context, client *http.Client, u string, v any, allowStatus ...int) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) - if err != nil { - return fmt.Errorf("creating request: %w", err) - } - req.Header.Set("Accept", "application/json") - resp, err := client.Do(req) - if err != nil { - return err - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK && !slices.Contains(allowStatus, resp.StatusCode) { - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxFeedResponse)) - return fmt.Errorf("%s: unexpected status %d", redact(u), resp.StatusCode) - } - body, err := io.ReadAll(io.LimitReader(resp.Body, maxFeedResponse+1)) - if err != nil { - return fmt.Errorf("%s: reading response: %w", redact(u), err) - } - if len(body) > maxFeedResponse { - return fmt.Errorf("%s: response larger than %d bytes", redact(u), maxFeedResponse) - } - if err := json.Unmarshal(body, v); err != nil { - return fmt.Errorf("%s: decoding response: %w", redact(u), err) - } - return nil -} - -// redact drops credentials and the query from u for error messages. -func redact(u string) string { - parsed, err := url.Parse(u) - if err != nil { - return "feed" - } - parsed.User = nil - parsed.RawQuery = "" - return parsed.String() -} - -// withQuery returns base with the query parameters in q added. -func withQuery(base string, q url.Values) string { - sep := "?" - if strings.Contains(base, "?") { - sep = "&" - } - return base + sep + q.Encode() -} - -// ComposerFeedURL is where a Packagist instance at apiURL, such as -// https://packagist.org, publishes its metadata change feed. Packagist -// advertises it as "metadata-changes-url" in its packages.json. -func ComposerFeedURL(apiURL string) string { - return strings.TrimSuffix(apiURL, "/") + "/metadata/changes.json" -} - -// ComposerFeed reads Packagist's metadata change feed -// (https://packagist.org/apidoc#track-package-updates). Its position is a -// timestamp in units of 1/10000 s. Asked without one it answers 400 with the -// current timestamp; asked with one older than its 24 hours of history it -// answers with a single "resync" action. Each update names the file that -// changed, "vendor/package" or "vendor/package~dev", which is also the key -// the Composer handler caches it under; a delete names the package, and -// covers both files. -type ComposerFeed struct { - client *http.Client - url string - since int64 -} - -// NewComposerFeed returns the feed at feedURL, see ComposerFeedURL. -func NewComposerFeed(client *http.Client, feedURL string) *ComposerFeed { - return &ComposerFeed{client: client, url: feedURL} -} - -func (f *ComposerFeed) Ecosystem() string { return "composer" } - -type composerChanges struct { - Actions []struct { - Type string `json:"type"` - Package string `json:"package"` - } `json:"actions"` - Timestamp int64 `json:"timestamp"` - Error string `json:"error"` -} - -func (f *ComposerFeed) Changes(ctx context.Context) (Changes, error) { - if f.since == 0 { - var start composerChanges - if err := getJSON(ctx, f.client, f.url, &start, http.StatusBadRequest); err != nil { - return Changes{}, err - } - if start.Timestamp <= 0 { - return Changes{}, errors.New("composer change feed: no timestamp to start from") - } - f.since = start.Timestamp - return Changes{}, nil - } - - var page composerChanges - u := withQuery(f.url, url.Values{"since": {strconv.FormatInt(f.since, 10)}}) - if err := getJSON(ctx, f.client, u, &page); err != nil { - return Changes{}, err - } - if page.Error != "" || page.Timestamp <= 0 { - return Changes{}, fmt.Errorf("composer change feed: %q", page.Error) - } - var changes Changes - for _, a := range page.Actions { - switch a.Type { - case "update": - changes.Names = append(changes.Names, a.Package) - case "delete": - // A deleted package takes its dev versions with it. - changes.Names = append(changes.Names, a.Package, a.Package+"~dev") - case "resync": - // The position is too old for the feed to list what changed. - changes.All = true - } - } - f.since = page.Timestamp - return changes, nil -} - -// NPMFeedURL is the change feed of the public npm registry. -const NPMFeedURL = "https://replicate.npmjs.com/registry/_changes" - -// npmPageLimit is the most changes the npm feed returns in one response. -const npmPageLimit = 10000 - -// npmMaxPages bounds how many pages one Changes call reads. The rest is read -// on the next call. -const npmMaxPages = 10 - -// NPMFeed reads the npm registry's CouchDB-style changes feed at -// replicate.npmjs.com. Its position is a sequence, which npm asks followers -// to treat as opaque; the database document at the feed's parent URL, with -// its trailing slash (without it the URL is 404), gives the current one as -// update_seq. A page holds at most 10000 changes (1000 when no limit is -// given). Each result's id is a package name, which is also the key the npm -// handler caches its packument under. -type NPMFeed struct { - client *http.Client - url string - // since is the sequence exactly as the feed returned it. - since json.RawMessage -} - -// NewNPMFeed returns the feed at feedURL, see NPMFeedURL. -func NewNPMFeed(client *http.Client, feedURL string) *NPMFeed { - return &NPMFeed{client: client, url: feedURL} -} - -func (f *NPMFeed) Ecosystem() string { return "npm" } - -type npmChanges struct { - Results []struct { - ID string `json:"id"` - } `json:"results"` - LastSeq json.RawMessage `json:"last_seq"` -} - -func (f *NPMFeed) Changes(ctx context.Context) (Changes, error) { - if f.since == nil { - var info struct { - UpdateSeq json.RawMessage `json:"update_seq"` - } - if err := getJSON(ctx, f.client, strings.TrimSuffix(f.url, "_changes"), &info); err != nil { - return Changes{}, err - } - if len(info.UpdateSeq) == 0 { - return Changes{}, errors.New("npm change feed: no update_seq to start from") - } - f.since = info.UpdateSeq - return Changes{}, nil - } - - var changes Changes - for range npmMaxPages { - var page npmChanges - u := withQuery(f.url, url.Values{ - "since": {seqParam(f.since)}, - "limit": {strconv.Itoa(npmPageLimit)}, - }) - if err := getJSON(ctx, f.client, u, &page); err != nil { - // Changes already read are reported, and the position kept - // after them, so they aren't lost to a later page's failure. - if len(changes.Names) > 0 { - return changes, nil - } - return Changes{}, err - } - if len(page.LastSeq) == 0 { - return changes, errors.New("npm change feed: response without last_seq") - } - for _, r := range page.Results { - if r.ID != "" && !strings.HasPrefix(r.ID, "_design/") { - changes.Names = append(changes.Names, r.ID) - } - } - f.since = page.LastSeq - if len(page.Results) < npmPageLimit { - break - } - } - return changes, nil -} - -// seqParam returns a sequence as the since parameter takes it: a number as -// is, a string without its quotes. -func seqParam(seq json.RawMessage) string { - var s string - if json.Unmarshal(seq, &s) == nil { - return s - } - return string(seq) -} diff --git a/internal/changefeed/changefeed_test.go b/internal/changefeed/changefeed_test.go deleted file mode 100644 index 21a4765..0000000 --- a/internal/changefeed/changefeed_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package changefeed - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "slices" - "strconv" - "strings" - "sync" - "sync/atomic" - "testing" -) - -// fakePackagist serves a change feed the way packagist.org does: without -// since, 400 with the current timestamp; with an old since, a resync; with a -// known one, the actions recorded after it. -type fakePackagist struct { - mu sync.Mutex - now int64 - actions map[int64]string // since -> JSON actions - asked []string -} - -func (p *fakePackagist) ServeHTTP(w http.ResponseWriter, r *http.Request) { - p.mu.Lock() - defer p.mu.Unlock() - since := r.URL.Query().Get("since") - p.asked = append(p.asked, since) - w.Header().Set("Content-Type", "application/json") - if since == "" { - w.WriteHeader(http.StatusBadRequest) - _, _ = fmt.Fprintf(w, `{"error":"Invalid or missing since query parameter","timestamp":%d}`, p.now) - return - } - n, _ := strconv.ParseInt(since, 10, 64) - actions, ok := p.actions[n] - if !ok { - actions = `[{"type":"resync","package":"*","time":1}]` - } - _, _ = fmt.Fprintf(w, `{"actions":%s,"timestamp":%d}`, actions, p.now) -} - -func TestComposerFeed(t *testing.T) { - p := &fakePackagist{now: 100, actions: map[int64]string{ - 100: `[{"type":"update","package":"acme/lib","time":1},{"type":"update","package":"acme/lib~dev","time":1},{"type":"delete","package":"acme/gone","time":1}]`, - 200: `[]`, - }} - srv := httptest.NewServer(p) - defer srv.Close() - feed := NewComposerFeed(srv.Client(), ComposerFeedURL(srv.URL+"/")) - ctx := context.Background() - - if c, err := feed.Changes(ctx); err != nil || c.Names != nil || c.All { - t.Fatalf("first read = %+v, %v; want the start position only", c, err) - } - p.mu.Lock() - p.now = 200 - p.mu.Unlock() - c, err := feed.Changes(ctx) - if err != nil { - t.Fatal(err) - } - if want := []string{"acme/lib", "acme/lib~dev", "acme/gone", "acme/gone~dev"}; !slices.Equal(c.Names, want) || c.All { - t.Errorf("changes = %+v, want %v", c, want) - } - if c, err := feed.Changes(ctx); err != nil || len(c.Names) != 0 || c.All { - t.Errorf("changes without news = %+v, %v", c, err) - } - - // A position the feed no longer has history for gets a resync. - feed.since = 50 - if c, err := feed.Changes(ctx); err != nil || !c.All { - t.Errorf("changes after a resync = %+v, %v, want All", c, err) - } - if want := []string{"", "100", "200", "50"}; !slices.Equal(p.asked, want) { - t.Errorf("asked since %q, want %q", p.asked, want) - } -} - -func TestComposerFeedErrorKeepsPosition(t *testing.T) { - var fail atomic.Bool - fail.Store(true) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if fail.Load() { - w.WriteHeader(http.StatusServiceUnavailable) - return - } - if r.URL.Query().Get("since") != "7" { - t.Errorf("since = %q, want 7", r.URL.Query().Get("since")) - } - _, _ = w.Write([]byte(`{"actions":[{"type":"update","package":"acme/lib","time":1}],"timestamp":8}`)) - })) - defer srv.Close() - feed := &ComposerFeed{client: srv.Client(), url: srv.URL, since: 7} - if _, err := feed.Changes(context.Background()); err == nil { - t.Fatal("want an error from a failing feed") - } - if feed.since != 7 { - t.Fatalf("since = %d after an error, want 7", feed.since) - } - fail.Store(false) - if c, err := feed.Changes(context.Background()); err != nil || !slices.Equal(c.Names, []string{"acme/lib"}) { - t.Errorf("changes after recovery = %+v, %v", c, err) - } -} - -// fakeReplicate serves a CouchDB-style changes feed over seqs 1..len(ids). -type fakeReplicate struct { - mu sync.Mutex - ids []string - limit int - asked []string -} - -func (f *fakeReplicate) ServeHTTP(w http.ResponseWriter, r *http.Request) { - f.mu.Lock() - defer f.mu.Unlock() - w.Header().Set("Content-Type", "application/json") - if r.URL.Path == "/registry/" { - _, _ = fmt.Fprintf(w, `{"db_name":"registry","update_seq":%d}`, len(f.ids)) - return - } - if r.URL.Path != "/registry/_changes" { - w.WriteHeader(http.StatusNotFound) - return - } - since, _ := strconv.Atoi(r.URL.Query().Get("since")) - limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) - f.asked = append(f.asked, r.URL.Query().Get("since")) - if f.limit > 0 { - limit = min(limit, f.limit) - } - var results []string - last := since - for seq := since + 1; seq <= len(f.ids) && len(results) < limit; seq++ { - results = append(results, fmt.Sprintf(`{"seq":%d,"id":%q,"changes":[{"rev":"1-a"}]}`, seq, f.ids[seq-1])) - last = seq - } - _, _ = fmt.Fprintf(w, `{"results":[%s],"last_seq":%d}`, strings.Join(results, ","), last) -} - -func TestNPMFeed(t *testing.T) { - f := &fakeReplicate{ids: []string{"old"}} - srv := httptest.NewServer(f) - defer srv.Close() - feed := NewNPMFeed(srv.Client(), srv.URL+"/registry/_changes") - ctx := context.Background() - - if c, err := feed.Changes(ctx); err != nil || c.Names != nil { - t.Fatalf("first read = %+v, %v; want the start position only", c, err) - } - f.mu.Lock() - f.ids = append(f.ids, "lodash", "@scope/pkg", "_design/app") - f.mu.Unlock() - c, err := feed.Changes(ctx) - if err != nil { - t.Fatal(err) - } - if want := []string{"lodash", "@scope/pkg"}; !slices.Equal(c.Names, want) { - t.Errorf("changes = %v, want %v", c.Names, want) - } - if c, err := feed.Changes(ctx); err != nil || len(c.Names) != 0 { - t.Errorf("changes without news = %+v, %v", c, err) - } - if want := []string{"1", "4"}; !slices.Equal(f.asked, want) { - t.Errorf("asked since %q, want %q", f.asked, want) - } -} - -// TestNPMFeedPages checks that a backlog longer than a page is read page by -// page, and that one read stops after npmMaxPages. -func TestNPMFeedPages(t *testing.T) { - f := &fakeReplicate{limit: npmPageLimit} - srv := httptest.NewServer(f) - defer srv.Close() - feed := &NPMFeed{client: srv.Client(), url: srv.URL + "/registry/_changes", since: []byte("0")} - f.ids = make([]string, npmPageLimit*(npmMaxPages+1)+5) - for i := range f.ids { - f.ids[i] = "pkg-" + strconv.Itoa(i) - } - - c, err := feed.Changes(context.Background()) - if err != nil { - t.Fatal(err) - } - if len(c.Names) != npmPageLimit*npmMaxPages { - t.Errorf("first read returned %d names, want %d", len(c.Names), npmPageLimit*npmMaxPages) - } - c, err = feed.Changes(context.Background()) - if err != nil { - t.Fatal(err) - } - if len(c.Names) != npmPageLimit+5 { - t.Errorf("second read returned %d names, want %d", len(c.Names), npmPageLimit+5) - } -} diff --git a/internal/server/change_feeds.go b/internal/server/change_feeds.go index 059bd2f..556650f 100644 --- a/internal/server/change_feeds.go +++ b/internal/server/change_feeds.go @@ -6,9 +6,10 @@ import ( "net/http" "time" - "github.com/git-pkgs/proxy/internal/changefeed" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/metrics" + "github.com/git-pkgs/registries/changes" + "github.com/git-pkgs/registries/client" ) // changeFeedInterval is how often an upstream change feed is read. @@ -20,13 +21,14 @@ const changeFeedMaxBackoff = 15 * time.Minute // startChangeFeeds follows the change feeds the configuration turns on. Each // instance sharing a database follows them itself: the reads are small, and // expiring a row twice does no harm. -func (s *Server) startChangeFeeds(ctx context.Context, client *http.Client, metadataTTL time.Duration) { - var feeds []changefeed.Feed +func (s *Server) startChangeFeeds(ctx context.Context, httpClient *http.Client, metadataTTL time.Duration) { + c := changes.WithClient(client.NewClient(client.WithHTTPClient(httpClient))) + var feeds []changes.Feed if s.cfg.Upstream.NPMChangeFeed { - feeds = append(feeds, changefeed.NewNPMFeed(client, changefeed.NPMFeedURL)) + feeds = append(feeds, changes.NewNPM(c)) } if s.cfg.Upstream.ComposerChangeFeed { - feeds = append(feeds, changefeed.NewComposerFeed(client, changefeed.ComposerFeedURL(s.cfg.Upstream.Composer))) + feeds = append(feeds, changes.NewPackagist(c, changes.WithURL(changes.PackagistFeedURL(s.cfg.Upstream.Composer)))) } if len(feeds) == 0 { return @@ -43,11 +45,15 @@ func (s *Server) startChangeFeeds(ctx context.Context, client *http.Client, meta // followChangeFeed reads feed every interval until ctx ends and expires the // cached metadata it lists. After a failed read it waits twice as long as -// before, up to changeFeedMaxBackoff. -func followChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changefeed.Feed, interval time.Duration) { +// before, up to changeFeedMaxBackoff. The position in the feed is kept in +// memory: after a restart the feed is followed from its current position. +func followChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changes.Feed, interval time.Duration) { wait := interval + var cursor changes.Cursor for { - if err := readChangeFeed(ctx, db, logger, feed); err != nil { + next, err := readChangeFeed(ctx, db, logger, feed, cursor) + cursor = next + if err != nil { if ctx.Err() != nil { return } @@ -66,29 +72,37 @@ func followChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, } } -// readChangeFeed reads feed once and expires the cached metadata of what it -// lists, so the next request for it revalidates upstream. Only rows fetched -// before the read finished are expired: a row fetched after it was fetched -// after every change the read returned. If expiring fails, the feed has -// already moved past the changes, and those rows wait out metadata_ttl. -func readChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changefeed.Feed) error { - changes, err := feed.Changes(ctx) +// readChangeFeed reads feed once after cursor, or finds its current position +// when cursor is empty, and expires the cached metadata of what it lists, so +// the next request for it revalidates upstream. Only rows fetched before the +// read finished are expired: a row fetched after it was fetched after every +// change the read returned. It returns the cursor to read on from, which +// moves past the changes only once they are expired, so a failed expiry +// reads them again. +func readChangeFeed(ctx context.Context, db *database.DB, logger *slog.Logger, feed changes.Feed, cursor changes.Cursor) (changes.Cursor, error) { + if cursor == "" { + return feed.Start(ctx) + } + found, next, err := feed.Changes(ctx, cursor) if err != nil { - return err + return cursor, err } cutoff := time.Now() ecosystem := feed.Ecosystem() var expired []string switch { - case changes.All: + case found.All: logger.Info("change feed: upstream asked for a full resync, revalidating all cached metadata", "ecosystem", ecosystem) expired, err = db.ExpireAllMetadataCache(ecosystem, cutoff) - case len(changes.Names) > 0: - expired, err = db.ExpireMetadataCache(ecosystem, changes.Names, cutoff) + case len(found.Names) > 0: + expired, err = db.ExpireMetadataCache(ecosystem, found.Names, cutoff) } for _, name := range expired { logger.Debug("change feed: cached metadata changed upstream", "ecosystem", ecosystem, "key", name) } metrics.RecordMetadataExpiredByChangeFeed(ecosystem, len(expired)) - return err + if err != nil { + return cursor, err + } + return next, nil } diff --git a/internal/server/change_feeds_test.go b/internal/server/change_feeds_test.go index 90441ac..366c1ae 100644 --- a/internal/server/change_feeds_test.go +++ b/internal/server/change_feeds_test.go @@ -9,17 +9,19 @@ import ( "log/slog" "net/http" "net/http/httptest" + "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" - "github.com/git-pkgs/proxy/internal/changefeed" "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/changes" + "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" "github.com/go-chi/chi/v5" ) @@ -186,14 +188,18 @@ func TestChangeFeedRevalidatesChangedMetadata(t *testing.T) { reg.set("/npm/left-pad", npmDoc("1.0.0")) h, db := changeFeedRouter(t, reg) logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - feeds := []changefeed.Feed{ - changefeed.NewComposerFeed(reg.Client(), changefeed.ComposerFeedURL(reg.URL)), - changefeed.NewNPMFeed(reg.Client(), reg.URL+"/replicate/registry/_changes"), + c := changes.WithClient(client.NewClient(client.WithHTTPClient(reg.Client()), client.WithMaxRetries(0))) + feeds := []changes.Feed{ + changes.NewPackagist(c, changes.WithURL(changes.PackagistFeedURL(reg.URL))), + changes.NewNPM(c, changes.WithURL(reg.URL+"/replicate/registry/_changes")), } - for _, feed := range feeds { - if err := readChangeFeed(context.Background(), db, logger, feed); err != nil { - t.Fatalf("starting %s feed: %v", feed.Ecosystem(), err) + cursors := make([]changes.Cursor, len(feeds)) + for i, feed := range feeds { + cursor, err := readChangeFeed(context.Background(), db, logger, feed, "") + if err != nil || cursor == "" { + t.Fatalf("starting %s feed: cursor %q, %v", feed.Ecosystem(), cursor, err) } + cursors[i] = cursor } getBody(t, h, "/composer/p2/acme/lib.json") @@ -209,9 +215,10 @@ func TestChangeFeedRevalidatesChangedMetadata(t *testing.T) { t.Fatal("npm packument revalidated before the feed listed it; the test proves nothing") } - for _, feed := range feeds { - if err := readChangeFeed(context.Background(), db, logger, feed); err != nil { - t.Fatalf("reading %s feed: %v", feed.Ecosystem(), err) + for i, feed := range feeds { + next, err := readChangeFeed(context.Background(), db, logger, feed, cursors[i]) + if err != nil || next == cursors[i] { + t.Fatalf("reading %s feed: cursor %q -> %q, %v", feed.Ecosystem(), cursors[i], next, err) } } @@ -238,11 +245,51 @@ type countingFeed struct { func (f *countingFeed) Ecosystem() string { return "npm" } -func (f *countingFeed) Changes(context.Context) (changefeed.Changes, error) { - if f.reads.Add(1) <= f.failures { - return changefeed.Changes{}, errors.New("feed down") +func (f *countingFeed) Start(context.Context) (changes.Cursor, error) { + return f.read() +} + +func (f *countingFeed) Changes(_ context.Context, cursor changes.Cursor) (changes.Changes, changes.Cursor, error) { + if _, err := f.read(); err != nil { + return changes.Changes{}, cursor, err + } + return changes.Changes{}, cursor, nil +} + +func (f *countingFeed) read() (changes.Cursor, error) { + if f.reads.Add(1) <= f.failures { + return "", errors.New("feed down") + } + return "1", nil +} + +// fixedFeed reports the same names on every read. +type fixedFeed struct{ names []string } + +func (fixedFeed) Ecosystem() string { return "npm" } +func (fixedFeed) Start(context.Context) (changes.Cursor, error) { return "1", nil } +func (f fixedFeed) Changes(context.Context, changes.Cursor) (changes.Changes, changes.Cursor, error) { + return changes.Changes{Names: f.names}, "2", nil +} + +// TestReadChangeFeedKeepsCursorWhenExpiryFails checks that the cursor moves +// past the changes only once they are expired, so changes whose expiry +// failed are read again. +func TestReadChangeFeedKeepsCursorWhenExpiryFails(t *testing.T) { + db, err := database.Create(filepath.Join(t.TempDir(), "proxy.db")) + if err != nil { + t.Fatal(err) + } + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + feed := fixedFeed{names: []string{"left-pad"}} + + if next, err := readChangeFeed(context.Background(), db, logger, feed, "1"); err != nil || next != "2" { + t.Fatalf("read = %q, %v; want cursor 2", next, err) + } + _ = db.Close() + if next, err := readChangeFeed(context.Background(), db, logger, feed, "1"); err == nil || next != "1" { + t.Errorf("read with a failing expiry = %q, %v; want an error and cursor 1", next, err) } - return changefeed.Changes{}, nil } // TestFollowChangeFeedStopsWithContext checks that a failing feed is retried diff --git a/internal/server/server.go b/internal/server/server.go index a5c7b07..574ce3a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -234,6 +234,33 @@ func (s *Server) Start(listeners ...net.Listener) error { return s.serve(listener) } +// configureGitCache sets up the cache of upstream git repositories, when any +// are configured. +func (s *Server) configureGitCache(client *http.Client) error { + if len(s.cfg.Upstream.Git) == 0 { + return nil + } + upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) + for name, upstream := range s.cfg.Upstream.Git { + upstreamDirs[name] = gitcache.UpstreamDir(name, upstream.URL) + } + gitCache, err := gitcache.New(gitcache.Options{ + CacheDir: s.cfg.Git.CacheDir, + MaxSize: s.cfg.Git.MaxSizeBytes(), + MaxRepoSize: s.cfg.Git.MaxRepoSizeBytes(), + RefreshWait: s.cfg.Git.RefreshWaitDuration(), + StaleMaxAge: s.cfg.Git.StaleMaxAgeDuration(), + Client: client, + Logger: s.logger, + Upstreams: upstreamDirs, + }) + if err != nil { + return fmt.Errorf("configuring git cache: %w", err) + } + s.git = gitCache + return nil +} + func (s *Server) serve(listener net.Listener) error { metadataClient, artifactClient := s.newUpstreamClients(s.cfg.ParseArtifactTimeout()) @@ -283,25 +310,8 @@ func (s *Server) serve(listener net.Listener) error { proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts - if len(s.cfg.Upstream.Git) > 0 { - upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) - for name, upstream := range s.cfg.Upstream.Git { - upstreamDirs[name] = gitcache.UpstreamDir(name, upstream.URL) - } - gitCache, err := gitcache.New(gitcache.Options{ - CacheDir: s.cfg.Git.CacheDir, - MaxSize: s.cfg.Git.MaxSizeBytes(), - MaxRepoSize: s.cfg.Git.MaxRepoSizeBytes(), - RefreshWait: s.cfg.Git.RefreshWaitDuration(), - StaleMaxAge: s.cfg.Git.StaleMaxAgeDuration(), - Client: artifactClient, - Logger: s.logger, - Upstreams: upstreamDirs, - }) - if err != nil { - return fmt.Errorf("configuring git cache: %w", err) - } - s.git = gitCache + if err := s.configureGitCache(artifactClient); err != nil { + return err } // Create router with Chi From 77525b0d78cb01ca4bbad82614233ace33a80dc1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 10:26:21 +0100 Subject: [PATCH 013/198] Store each snapshot recording at a path of its own Two requests recording the same index version in the same millisecond stored to one path, and the one whose row was not added deleted the blob the other's row points at. The path now ends in a random suffix. Also trims comments that restated the PR, notes in the docs that turning snapshots off leaves them in place, and moves the git cache setup out of serve, which lint found too long after the merge. --- docs/configuration.md | 4 +++ internal/database/metadata_snapshots.go | 11 +++--- internal/handler/apk.go | 3 +- internal/handler/apk_snapshots.go | 38 ++++++++++---------- internal/handler/apk_snapshots_test.go | 37 +++++++++++++++++++ internal/server/server.go | 48 +++++++++++++++---------- 6 files changed, 95 insertions(+), 46 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 1744431..2d7542e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -662,6 +662,10 @@ Durations take a `d` suffix; empty or `"0"` disables snapshots. Snapshots need `cache_metadata`. Their files live under `_snapshots/` in storage and do not count towards `storage.max_size`. +Turning snapshots off, or `cache_metadata` off, leaves the kept versions in +place. To remove them, first set a short duration such as `"1h"` and let the +hourly prune delete everything but the newest version of each index. + Snapshots keep indexes, not packages. A package listed by an old snapshot can still be evicted by `storage.max_size` or [retention](#retention), and is then only available while upstream still has it. Retention counts from a package's diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go index b174aab..95edacc 100644 --- a/internal/database/metadata_snapshots.go +++ b/internal/database/metadata_snapshots.go @@ -7,13 +7,10 @@ import ( "time" ) -// Metadata snapshots keep earlier versions of a metadata document after -// upstream replaced it, so a request for a past date can be answered with the -// document that was current then. Each row is one version, valid from -// valid_from_ms (Unix milliseconds, the time the proxy first stored it) until -// the next row for the same ecosystem and name. - -// MetadataSnapshot is one kept version of a metadata document. +// MetadataSnapshot is one kept version of a metadata document, so a request +// for a past date can be answered with the document that was current then. +// It is valid from ValidFromMS (Unix milliseconds, the time the proxy first +// stored it) until the next snapshot of the same ecosystem and name. type MetadataSnapshot struct { ID int64 `db:"id"` Ecosystem string `db:"ecosystem"` diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 071ae3a..83fee97 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -53,8 +53,7 @@ type APKHandler struct { proxy *Proxy proxyURL string repositories map[string]string - // snapshots is how long replaced index versions are kept; 0 disables - // snapshots. See WithSnapshots. + // snapshots: see WithSnapshots. snapshots time.Duration } diff --git a/internal/handler/apk_snapshots.go b/internal/handler/apk_snapshots.go index cca04d6..0f29c85 100644 --- a/internal/handler/apk_snapshots.go +++ b/internal/handler/apk_snapshots.go @@ -2,6 +2,7 @@ package handler import ( "context" + "crypto/rand" "io" "net/http" "strconv" @@ -11,23 +12,13 @@ import ( "github.com/git-pkgs/proxy/internal/database" ) -// Dated snapshots of APK indexes. -// -// Alpine mirrors keep only the current build of each package, so once a -// package is upgraded the index no longer lists the old build and -// `apk add name=old-version` fails. With snapshots enabled, every version of an -// index (and of a detached signature) the proxy stores is kept, and -// /apk/{name}@{YYYY-MM-DD}/... serves the version that was current at the end -// of that day (UTC). The kept bytes are the signed upstream bytes, so apk -// verifies them as usual. Packages under a snapshot path are the same -// requests as under the plain path. - // apkSnapshotDateLayout is the date format of a snapshot path segment. const apkSnapshotDateLayout = "2006-01-02" // WithSnapshots keeps earlier versions of the indexes this handler serves and -// makes them available under /apk/{name}@{YYYY-MM-DD}/. keep is how long a -// version is kept after it was replaced; 0 disables snapshots. +// makes them available under /apk/{name}@{YYYY-MM-DD}/, as signed upstream +// bytes that apk verifies as usual. keep is how long a version is kept after +// it was replaced; 0 disables snapshots. func (h *APKHandler) WithSnapshots(keep time.Duration) *APKHandler { h.snapshots = keep return h @@ -125,7 +116,15 @@ func (h *APKHandler) recordSnapshot(ctx context.Context, cacheKey string) { return } - validFrom := time.Now() + h.storeSnapshot(ctx, cacheKey, entry, time.Now()) +} + +// storeSnapshot records the cached document entry as the version current +// from validFrom. Concurrent requests can both get here for the same +// version; each copies to a path of its own, so the one whose row is not +// added only deletes its own copy. +func (h *APKHandler) storeSnapshot(ctx context.Context, cacheKey string, entry *database.MetadataCacheEntry, validFrom time.Time) { + digest := entry.ContentDigest.String storagePath := apkSnapshotStoragePath(cacheKey, digest, validFrom) size, ok := h.copySnapshotBlob(ctx, entry.StoragePath, storagePath, digest) if !ok { @@ -178,10 +177,13 @@ func (h *APKHandler) discardSnapshotBlob(ctx context.Context, path string) { } } -// apkSnapshotStoragePath is where one kept version of the document cached -// under cacheKey is stored. The time it became current keeps a version that -// returns after an upgrade was rolled back apart from its earlier copy. +// apkSnapshotStoragePath returns a new path to store one kept version of the +// document cached under cacheKey at. The time it became current keeps a +// version that returns after an upgrade was rolled back apart from its +// earlier copy, and a random suffix keeps two requests recording the same +// version in the same millisecond apart. func apkSnapshotStoragePath(cacheKey, digest string, validFrom time.Time) string { return "_snapshots/" + apkEcosystem + "/" + cacheKey + "/" + - strconv.FormatInt(validFrom.UnixMilli(), 10) + "-" + strings.TrimPrefix(digest, "sha256:") + strconv.FormatInt(validFrom.UnixMilli(), 10) + "-" + strings.TrimPrefix(digest, "sha256:") + + "-" + rand.Text() } diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go index 50291c1..52b5ff7 100644 --- a/internal/handler/apk_snapshots_test.go +++ b/internal/handler/apk_snapshots_test.go @@ -166,3 +166,40 @@ func TestAPKSnapshots_ConfiguredNameWins(t *testing.T) { expectAPKResponse(t, h, "/alpine@2026-01-10"+apkSnapshotIndexPath, http.StatusOK, "index v1") } + +// TestAPKSnapshots_SameVersionRecordedTwice checks that two requests +// recording the same version in the same millisecond, as coalesced fetches +// after an upstream update can, leave the recorded snapshot readable: the +// request whose row is not added must not delete the other's blob. +func TestAPKSnapshots_SameVersionRecordedTwice(t *testing.T) { + u := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, u.server.URL) + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + + key := h.metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) + entry, err := db.GetMetadataCache(apkEcosystem, key) + if err != nil || entry == nil { + t.Fatalf("cached index row = %v, %v", entry, err) + } + if _, err := db.Exec(`DELETE FROM metadata_snapshots`); err != nil { + t.Fatal(err) + } + validFrom := time.Now() + h.storeSnapshot(t.Context(), key, entry, validFrom) + h.storeSnapshot(t.Context(), key, entry, validFrom) + + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots recorded, want 1", n) + } + snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) + if err != nil || snap == nil { + t.Fatalf("latest snapshot = %v, %v", snap, err) + } + body, err := h.proxy.Storage.Open(t.Context(), snap.StoragePath) + if err != nil { + t.Fatalf("recorded snapshot blob unreadable: %v", err) + } + _ = body.Close() + day := validFrom.UTC().Format(apkSnapshotDateLayout) + expectAPKResponse(t, h, "/alpine@"+day+apkSnapshotIndexPath, http.StatusOK, "index v1") +} diff --git a/internal/server/server.go b/internal/server/server.go index ff06d91..41c4c5a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -234,6 +234,33 @@ func (s *Server) Start(listeners ...net.Listener) error { return s.serve(listener) } +// configureGitCache sets up the cache of upstream git repositories, when any +// are configured. +func (s *Server) configureGitCache(client *http.Client) error { + if len(s.cfg.Upstream.Git) == 0 { + return nil + } + upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) + for name, upstream := range s.cfg.Upstream.Git { + upstreamDirs[name] = gitcache.UpstreamDir(name, upstream.URL) + } + gitCache, err := gitcache.New(gitcache.Options{ + CacheDir: s.cfg.Git.CacheDir, + MaxSize: s.cfg.Git.MaxSizeBytes(), + MaxRepoSize: s.cfg.Git.MaxRepoSizeBytes(), + RefreshWait: s.cfg.Git.RefreshWaitDuration(), + StaleMaxAge: s.cfg.Git.StaleMaxAgeDuration(), + Client: client, + Logger: s.logger, + Upstreams: upstreamDirs, + }) + if err != nil { + return fmt.Errorf("configuring git cache: %w", err) + } + s.git = gitCache + return nil +} + func (s *Server) serve(listener net.Listener) error { metadataClient, artifactClient := s.newUpstreamClients(s.cfg.ParseArtifactTimeout()) @@ -283,25 +310,8 @@ func (s *Server) serve(listener net.Listener) error { proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts - if len(s.cfg.Upstream.Git) > 0 { - upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) - for name, upstream := range s.cfg.Upstream.Git { - upstreamDirs[name] = gitcache.UpstreamDir(name, upstream.URL) - } - gitCache, err := gitcache.New(gitcache.Options{ - CacheDir: s.cfg.Git.CacheDir, - MaxSize: s.cfg.Git.MaxSizeBytes(), - MaxRepoSize: s.cfg.Git.MaxRepoSizeBytes(), - RefreshWait: s.cfg.Git.RefreshWaitDuration(), - StaleMaxAge: s.cfg.Git.StaleMaxAgeDuration(), - Client: artifactClient, - Logger: s.logger, - Upstreams: upstreamDirs, - }) - if err != nil { - return fmt.Errorf("configuring git cache: %w", err) - } - s.git = gitCache + if err := s.configureGitCache(artifactClient); err != nil { + return err } // Create router with Chi From 1205fa759e60f3c484c2646be7e75ac6b036e1f8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 13:54:24 +0100 Subject: [PATCH 014/198] Show versions held back by cooldown on the package page and in the API Part of #617. The package page marks versions in cooldown with when they become available, and lists a version only seen in metadata (no artifact row) only while it is held back. /api/package returns the package's cooldown and available_at on a held-back version. FillVersionPublishedAt stores a publish time without overwriting one, for the cooldown filters to call once they record the versions they hold back. --- docs/configuration.md | 2 +- docs/swagger/docs.go | 4 + docs/swagger/swagger.json | 4 + internal/database/database_test.go | 76 +++++++++++++ internal/database/queries.go | 54 +++++++++ internal/server/api.go | 41 ++++++- internal/server/cooldown_view.go | 45 +++++++- internal/server/cooldown_view_test.go | 104 ++++++++++++++++++ internal/server/dashboard.go | 2 +- internal/server/server.go | 13 ++- internal/server/server_test.go | 5 + .../server/templates/pages/package_show.html | 1 + internal/server/templates_test.go | 11 +- 13 files changed, 352 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0b9c779..9c67e98 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1125,7 +1125,7 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. +The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index e35a86f..57a896f 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -1062,6 +1062,10 @@ const docTemplate = `{ "server.PackageResponse": { "type": "object", "properties": { + "cooldown": { + "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").", + "type": "string" + }, "description": { "type": "string" }, diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 09e4f11..1a95e14 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -1055,6 +1055,10 @@ "server.PackageResponse": { "type": "object", "properties": { + "cooldown": { + "description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").", + "type": "string" + }, "description": { "type": "string" }, diff --git a/internal/database/database_test.go b/internal/database/database_test.go index f8956a6..30c6bff 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1164,3 +1164,79 @@ func TestVersionPublishedAtPreserved(t *testing.T) { } }) } + +func TestFillVersionPublishedAt(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkg, held, known = "pkg:npm/leftpad", "pkg:npm/leftpad@2.0.0", "pkg:npm/leftpad@1.0.0" + stored := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + filtered := time.Date(2021, 6, 7, 8, 9, 10, 0, time.UTC) + + if err := db.SetVersionPublishedAt(known, pkg, stored); err != nil { + t.Fatalf("SetVersionPublishedAt: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: held, PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + + for _, purl := range []string{known, held, "pkg:npm/leftpad@3.0.0"} { + if err := db.FillVersionPublishedAt(purl, pkg, filtered); err != nil { + t.Fatalf("FillVersionPublishedAt(%s): %v", purl, err) + } + } + + want := map[string]time.Time{ + known: stored, + held: filtered, + "pkg:npm/leftpad@3.0.0": filtered, + } + for purl, at := range want { + got, err := db.GetVersionByPURL(purl) + if err != nil || got == nil { + t.Fatalf("GetVersionByPURL(%s): %v", purl, err) + } + if !got.PublishedAt.Valid || !got.PublishedAt.Time.Equal(at) { + t.Errorf("%s PublishedAt = %v, want %v", purl, got.PublishedAt, at) + } + } + }) +} + +func TestGetFetchedVersionPURLs(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const pkg = "pkg:npm/leftpad" + for _, v := range []string{"1.0.0", "2.0.0", "3.0.0"} { + if err := db.UpsertVersion(&Version{PURL: pkg + "@" + v, PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + } + if err := db.UpsertVersion(&Version{PURL: "pkg:npm/other@1.0.0", PackagePURL: "pkg:npm/other"}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + artifacts := []Artifact{ + {VersionPURL: pkg + "@1.0.0", Filename: "leftpad-1.0.0.tgz", UpstreamURL: "https://example.com/1", + StoragePath: sql.NullString{String: "npm/leftpad-1.0.0.tgz", Valid: true}}, + // Evicted: the row stays without a storage path. + {VersionPURL: pkg + "@2.0.0", Filename: "leftpad-2.0.0.tgz", UpstreamURL: "https://example.com/2"}, + {VersionPURL: "pkg:npm/other@1.0.0", Filename: "other-1.0.0.tgz", UpstreamURL: "https://example.com/3"}, + } + for i := range artifacts { + if err := db.UpsertArtifact(&artifacts[i]); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + } + + got, err := db.GetFetchedVersionPURLs(pkg) + if err != nil { + t.Fatalf("GetFetchedVersionPURLs: %v", err) + } + want := map[string]bool{pkg + "@1.0.0": true, pkg + "@2.0.0": true} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for purl := range want { + if !got[purl] { + t.Errorf("missing %s in %v", purl, got) + } + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e8..5ba8edf 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -223,6 +223,60 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } +// FillVersionPublishedAt records a version's publish time like +// SetVersionPublishedAt, but keeps a time already stored. The cooldown +// filters call it for each version they hold back, on every uncached +// metadata request, so it writes without reading first. +func (db *DB) FillVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { + now := time.Now() + var query string + + if db.dialect == DialectPostgres { + query = ` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(purl) DO UPDATE SET + published_at = EXCLUDED.published_at, + updated_at = EXCLUDED.updated_at + WHERE versions.published_at IS NULL + ` + } else { + query = ` + INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(purl) DO UPDATE SET + published_at = excluded.published_at, + updated_at = excluded.updated_at + WHERE versions.published_at IS NULL + ` + } + + _, err := db.Exec(query, versionPURL, packagePURL, publishedAt, now, now) + if err != nil { + return fmt.Errorf("filling version publish time: %w", err) + } + return nil +} + +// GetFetchedVersionPURLs returns the versions of a package that have an +// artifact row, cached now or evicted since. Versions without one were only +// seen in metadata. +func (db *DB) GetFetchedVersionPURLs(packagePURL string) (map[string]bool, error) { + var purls []string + query := db.Rebind(` + SELECT DISTINCT version_purl FROM artifacts + WHERE version_purl IN (SELECT purl FROM versions WHERE package_purl = ?) + `) + if err := db.Select(&purls, query, packagePURL); err != nil { + return nil, fmt.Errorf("listing fetched versions: %w", err) + } + fetched := make(map[string]bool, len(purls)) + for _, p := range purls { + fetched[p] = true + } + return fetched, nil +} + // Artifact queries func (db *DB) GetArtifact(versionPURL, filename string) (*Artifact, error) { diff --git a/internal/server/api.go b/internal/server/api.go index b204367..ca1e626 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "strings" + "time" shared "github.com/git-pkgs/enrichment" "github.com/git-pkgs/proxy/internal/database" @@ -23,6 +24,8 @@ type APIHandler struct { enrichment *enrichment.Service ecosystems *shared.EcosystemsClient db DBSearcher + // cooldown is nil with cooldown off. + cooldown *cooldownView } // DBSearcher defines the interface for database search operations. @@ -46,6 +49,13 @@ func NewAPIHandler(svc *enrichment.Service, db DBSearcher) *APIHandler { return h } +// newAPIHandler returns the server's API handler, with its cooldown policy. +func (s *Server) newAPIHandler(svc *enrichment.Service) *APIHandler { + h := NewAPIHandler(svc, s.db) + h.cooldown = s.cooldown + return h +} + // PackageResponse contains enriched package metadata. type PackageResponse struct { Ecosystem string `json:"ecosystem"` @@ -57,6 +67,9 @@ type PackageResponse struct { Homepage string `json:"homepage,omitempty"` Repository string `json:"repository,omitempty"` RegistryURL string `json:"registry_url,omitempty"` + // Cooldown is the cooldown that applies to the package, in the + // configuration's syntax ("3d", "0"). + Cooldown string `json:"cooldown,omitempty"` } // VersionResponse contains enriched version metadata. @@ -66,11 +79,16 @@ type VersionResponse struct { Version string `json:"version"` License string `json:"license,omitempty"` PublishedAt string `json:"published_at,omitempty"` + // AvailableAt is when a version held back by cooldown leaves it. + AvailableAt string `json:"available_at,omitempty"` Integrity string `json:"integrity,omitempty"` Yanked bool `json:"yanked"` IsOutdated bool `json:"is_outdated"` } +// apiTimeFormat is the format of the API's timestamps. +const apiTimeFormat = "2006-01-02T15:04:05Z" + // VulnResponse contains vulnerability information. type VulnResponse struct { ID string `json:"id"` @@ -179,6 +197,7 @@ func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) { Homepage: info.Homepage, Repository: info.Repository, RegistryURL: info.RegistryURL, + Cooldown: h.packageCooldown(ecosystem, fullName), } writeJSON(w, resp) return @@ -212,6 +231,7 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste Homepage: info.Homepage, Repository: info.Repository, RegistryURL: info.RegistryURL, + Cooldown: h.packageCooldown(ecosystem, name), } writeJSON(w, resp) @@ -240,6 +260,7 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste Homepage: result.Package.Homepage, Repository: result.Package.Repository, RegistryURL: result.Package.RegistryURL, + Cooldown: h.packageCooldown(ecosystem, name), } } @@ -253,8 +274,11 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste Yanked: result.Version.Yanked, IsOutdated: result.IsOutdated, } - if !result.Version.PublishedAt.IsZero() { - resp.Version.PublishedAt = result.Version.PublishedAt.Format("2006-01-02T15:04:05Z") + if published := result.Version.PublishedAt; !published.IsZero() { + resp.Version.PublishedAt = published.UTC().Format(apiTimeFormat) + if at := h.cooldown.availableAt(ecosystem, name, published, time.Now()); !at.IsZero() { + resp.Version.AvailableAt = at.UTC().Format(apiTimeFormat) + } } } @@ -272,6 +296,19 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste writeJSON(w, resp) } +// packageCooldown returns the cooldown for a package in the configuration's +// syntax, or "" with cooldown off or not applied to the ecosystem. +func (h *APIHandler) packageCooldown(ecosystem, name string) string { + c := h.cooldown.forPackage(ecosystem, name) + switch { + case c == nil: + return "" + case c.Duration == cooldownOff: + return "0" + } + return c.Duration +} + // HandleVulnsPath dispatches /api/vulns/{ecosystem}/* to the vulns handler. // Supports both {name} and {name}/{version} paths with namespaced package names. func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { diff --git a/internal/server/cooldown_view.go b/internal/server/cooldown_view.go index f244c85..415d708 100644 --- a/internal/server/cooldown_view.go +++ b/internal/server/cooldown_view.go @@ -7,6 +7,7 @@ import ( "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/packageurl" ) @@ -30,6 +31,14 @@ type CooldownOverride struct { Unsupported bool } +// PackageVersion is a version on the package page. +type PackageVersion struct { + database.Version + // AvailableAt is when the version leaves cooldown, zero when it is not + // held back. + AvailableAt time.Time +} + // PackageCooldown is the cooldown that applies to one package, and the kind // of rule it comes from. type PackageCooldown struct { @@ -104,6 +113,37 @@ func (v *cooldownView) forPackage(ecosystem, name string) *PackageCooldown { return &PackageCooldown{Duration: formatCooldown(v.policy.For(ecosystem, purl)), Rule: rule} } +// availableAt returns when a version published at publishedAt leaves +// cooldown, or zero when it is not held back at now. +func (v *cooldownView) availableAt(ecosystem, name string, publishedAt, now time.Time) time.Time { + if !v.enabled() || !handler.CooldownSupported(ecosystem) || publishedAt.IsZero() { + return time.Time{} + } + d := v.policy.Evaluate(ecosystem, packageurl.MakeString(ecosystem, name, ""), publishedAt, now) + if d.Allowed { + return time.Time{} + } + return d.AvailableAt +} + +// packageVersions returns the versions to list on a package page. A version +// without an artifact row was only seen in metadata, by a cooldown filter or +// a download check, and is listed only while it is held back. +func (v *cooldownView) packageVersions(pkg *database.Package, versions []database.Version, fetched map[string]bool, now time.Time) []PackageVersion { + rows := make([]PackageVersion, 0, len(versions)) + for _, ver := range versions { + var at time.Time + if ver.PublishedAt.Valid { + at = v.availableAt(pkg.Ecosystem, pkg.Name, ver.PublishedAt.Time, now) + } + if at.IsZero() && !fetched[ver.PURL] { + continue + } + rows = append(rows, PackageVersion{Version: ver, AvailableAt: at}) + } + return rows +} + func formatConfigDuration(s string) string { d, err := cooldown.ParseDuration(s) if err != nil { @@ -112,13 +152,16 @@ func formatConfigDuration(s string) string { return formatCooldown(d) } +// cooldownOff is how formatCooldown writes a zero cooldown. +const cooldownOff = "off" + // formatCooldown writes d as the configuration does: "3d", "12h", "30m", // and "off" for zero. func formatCooldown(d time.Duration) string { const day = 24 * time.Hour switch { case d <= 0: - return "off" + return cooldownOff case d%day == 0: return fmt.Sprintf("%dd", d/day) case d%time.Hour == 0: diff --git a/internal/server/cooldown_view_test.go b/internal/server/cooldown_view_test.go index 13e731c..0a347a8 100644 --- a/internal/server/cooldown_view_test.go +++ b/internal/server/cooldown_view_test.go @@ -1,6 +1,7 @@ package server import ( + "database/sql" "net/http" "net/http/httptest" "slices" @@ -152,3 +153,106 @@ func TestCooldownPages(t *testing.T) { t.Error("package page doesn't show the 7d ecosystem override") } } + +func TestPackagePageWithheldVersions(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + ts.server.cooldown = testCooldownView(t, config.CooldownConfig{Default: "3d"}) + + const pkg = "pkg:npm/lodash" + if err := ts.db.UpsertPackage(&database.Package{PURL: pkg, Ecosystem: "npm", Name: "lodash"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + now := time.Now() + held := now.Add(-time.Hour) + versions := map[string]time.Time{ + "1.0.0": now.Add(-30 * 24 * time.Hour), // fetched + "1.1.0": now.Add(-time.Hour), // fetched while held back + "2.0.0": now.Add(-20 * 24 * time.Hour), // only seen, left cooldown + "2.1.0": held, // only seen, held back + } + for v, at := range versions { + if err := ts.db.SetVersionPublishedAt(pkg+"@"+v, pkg, at); err != nil { + t.Fatalf("SetVersionPublishedAt: %v", err) + } + } + // No publish time: the download checks never stored one. + if err := ts.db.UpsertVersion(&database.Version{PURL: pkg + "@0.9.0", PackagePURL: pkg}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + for _, v := range []string{"1.0.0", "1.1.0"} { + if err := ts.db.UpsertArtifact(&database.Artifact{ + VersionPURL: pkg + "@" + v, Filename: "lodash-" + v + ".tgz", UpstreamURL: "https://example.com/" + v, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + } + + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/package/npm/lodash", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status %d", w.Code) + } + page := w.Body.String() + + for _, v := range []string{"1.0.0", "1.1.0", "2.1.0"} { + if !strings.Contains(page, "lodash@"+v) { + t.Errorf("page is missing %s", v) + } + } + for _, v := range []string{"2.0.0", "0.9.0"} { + if strings.Contains(page, "lodash@"+v) { + t.Errorf("page lists %s, which nobody fetched and cooldown no longer holds back", v) + } + } + until := held.Add(3 * 24 * time.Hour).UTC().Format("2006-01-02 15:04") + if got := strings.Count(page, "held back until"); got != 2 { + t.Errorf("page marks %d versions as held back, want 2 (1.1.0 and 2.1.0)", got) + } + if !strings.Contains(page, "held back until "+until+" UTC") { + t.Errorf("page is missing the availability of 2.1.0, %s", until) + } +} + +func TestPackageVersionsCooldownOff(t *testing.T) { + var off *cooldownView + pkg := &database.Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"} + recent := sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true} + versions := []database.Version{ + {PURL: "pkg:npm/lodash@1.0.0", PublishedAt: recent}, + {PURL: "pkg:npm/lodash@2.0.0", PublishedAt: recent}, + } + got := off.packageVersions(pkg, versions, map[string]bool{"pkg:npm/lodash@1.0.0": true}, time.Now()) + if len(got) != 1 || got[0].PURL != "pkg:npm/lodash@1.0.0" || !got[0].AvailableAt.IsZero() { + t.Errorf("packageVersions with cooldown off = %+v, want only the fetched 1.0.0", got) + } +} + +func TestAPIPackageCooldown(t *testing.T) { + h := &APIHandler{cooldown: testCooldownView(t, config.CooldownConfig{ + Default: "3d", + Packages: map[string]string{"pkg:npm/leftpad": "0"}, + })} + tests := []struct{ ecosystem, name, want string }{ + {"npm", "lodash", "3d"}, + {"npm", "leftpad", "0"}, + {"deb", "curl", ""}, + } + for _, tt := range tests { + if got := h.packageCooldown(tt.ecosystem, tt.name); got != tt.want { + t.Errorf("packageCooldown(%s, %s) = %q, want %q", tt.ecosystem, tt.name, got, tt.want) + } + } + if got := (&APIHandler{}).packageCooldown("npm", "lodash"); got != "" { + t.Errorf("packageCooldown with cooldown off = %q, want empty", got) + } + + now := time.Now() + published := now.Add(-time.Hour) + if got, want := h.cooldown.availableAt("npm", "lodash", published, now), published.Add(72*time.Hour); !got.Equal(want) { + t.Errorf("availableAt = %v, want %v", got, want) + } + if got := h.cooldown.availableAt("npm", "lodash", now.Add(-96*time.Hour), now); !got.IsZero() { + t.Errorf("availableAt after cooldown = %v, want zero", got) + } +} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index 7e14af6..08cda58 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -66,7 +66,7 @@ type RegistryConfig struct { type PackageShowData struct { Layout Package *database.Package - Versions []database.Version + Versions []PackageVersion Vulnerabilities []database.Vulnerability LicenseCategory string // Cooldown is nil with cooldown off. diff --git a/internal/server/server.go b/internal/server/server.go index 2f0bd68..5ced4bc 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -372,7 +372,7 @@ func (s *Server) serve(listener net.Listener) error { // API endpoints for enrichment data enrichSvc := enrichment.New(s.logger) - apiHandler := NewAPIHandler(enrichSvc, s.db) + apiHandler := s.newAPIHandler(enrichSvc) r.Get("/api/package/{ecosystem}/*", apiHandler.HandlePackagePath) r.Get("/api/vulns/{ecosystem}/*", apiHandler.HandleVulnsPath) @@ -1163,6 +1163,15 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, s.logger.Error("failed to get versions", "error", err) versions = []database.Version{} } + fetched, err := s.db.GetFetchedVersionPURLs(pkg.PURL) + if err != nil { + // List every version rather than hide fetched ones. + s.logger.Error("failed to get fetched versions", "error", err) + fetched = make(map[string]bool, len(versions)) + for _, v := range versions { + fetched[v.PURL] = true + } + } vulns, err := s.db.GetVulnerabilitiesForPackage(ecosystem, name) if err != nil { @@ -1173,7 +1182,7 @@ func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, data := PackageShowData{ Layout: s.layoutFor(r), Package: pkg, - Versions: versions, + Versions: s.cooldown.packageVersions(pkg, versions, fetched, time.Now()), Vulnerabilities: vulns, LicenseCategory: categorizeLicense(pkg.License), Cooldown: s.cooldown.forPackage(pkg.Ecosystem, pkg.Name), diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 292643c..aad9037 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1098,6 +1098,11 @@ func TestVersionShowPage_PlusInVersion(t *testing.T) { }); err != nil { t.Fatalf("failed to upsert version: %v", err) } + if err := ts.db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, Filename: "nmap.deb", UpstreamURL: "https://example.com/nmap.deb", + }); err != nil { + t.Fatalf("failed to upsert artifact: %v", err) + } // The package page must link to and display the decoded version. req := httptest.NewRequest("GET", "/ui/package/deb/nmap", nil) diff --git a/internal/server/templates/pages/package_show.html b/internal/server/templates/pages/package_show.html index b0dd29a..977ff6e 100644 --- a/internal/server/templates/pages/package_show.html +++ b/internal/server/templates/pages/package_show.html @@ -69,6 +69,7 @@ {{.DisplayPURL}} {{if .Yanked}}yanked{{end}} + {{if not .AvailableAt.IsZero}}held back until {{.AvailableAt.UTC.Format "2006-01-02 15:04"}} UTC{{end}}
{{if .PublishedAt.Valid}}{{.PublishedAt.Time.Format "2006-01-02"}}{{end}}
diff --git a/internal/server/templates_test.go b/internal/server/templates_test.go index 708f02c..c6fa492 100644 --- a/internal/server/templates_test.go +++ b/internal/server/templates_test.go @@ -8,6 +8,7 @@ import ( "regexp" "strings" "testing" + "time" "github.com/git-pkgs/proxy/internal/database" ) @@ -122,8 +123,12 @@ func TestTemplatesRenderAllPages(t *testing.T) { LatestVersion: sql.NullString{String: "4.17.21", Valid: true}, License: sql.NullString{String: "MIT", Valid: true}, }, - Versions: []database.Version{ - {PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}, + Versions: []PackageVersion{ + {Version: database.Version{PURL: "pkg:npm/lodash@4.17.21", PackagePURL: "pkg:npm/lodash"}}, + { + Version: database.Version{PURL: "pkg:npm/lodash@4.17.22", PackagePURL: "pkg:npm/lodash"}, + AvailableAt: time.Date(2026, 1, 2, 3, 4, 0, 0, time.UTC), + }, }, Vulnerabilities: []database.Vulnerability{}, LicenseCategory: "permissive", @@ -134,7 +139,7 @@ func TestTemplatesRenderAllPages(t *testing.T) { Ecosystem: "npm", Name: "minimal", }, - Versions: []database.Version{}, + Versions: []PackageVersion{}, Vulnerabilities: []database.Vulnerability{}, LicenseCategory: "unknown", }}, From 5b57994c09ffccc09d43a26eddf9b4e0e48578ba Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 14:14:10 +0100 Subject: [PATCH 015/198] Stream npm packuments through storage when rewriting them Closes #525. With metadata caching and the rewrite cache on, a packument is streamed from upstream into storage and rewritten from there in two passes: the first collects the time map, dist-tags and version names, the second copies each kept version and rewrites its tarball URL. Neither the packument nor its rewrite is held in memory uncompressed, so metadata_max_size no longer limits packuments. Both passes check the stored bytes against the digest in the cache row, so a document replaced in between is not rewritten from mixed bytes. Documents the stream can't handle (abbreviated, no versions object) and rewrite errors fall back to the in-memory rewrite. --- docs/architecture.md | 2 +- docs/configuration.md | 2 +- internal/handler/conda.go | 3 +- internal/handler/handler.go | 87 ++++++--- internal/handler/metadata_stored.go | 207 ++++++++++++++++++++ internal/handler/npm.go | 80 ++++++++ internal/handler/npm_stream_test.go | 243 +++++++++++++++++++++++ internal/handler/rewrite_cache.go | 94 ++++++++- internal/metadata/npm/imports_test.go | 1 + internal/metadata/npm/stream.go | 270 ++++++++++++++++++++++++++ internal/metadata/npm/stream_test.go | 162 ++++++++++++++++ 11 files changed, 1111 insertions(+), 40 deletions(-) create mode 100644 internal/handler/metadata_stored.go create mode 100644 internal/handler/npm_stream_test.go create mode 100644 internal/metadata/npm/stream.go create mode 100644 internal/metadata/npm/stream_test.go diff --git a/docs/architecture.md b/docs/architecture.md index b6c654f..c859f33 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -316,7 +316,7 @@ HTTP protocol handlers for each registry type. ### `internal/metadata/npm` -The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. It imports no database, storage, metrics, HTTP or logging packages, which a test checks. +The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. `ScanPackument` and `RewriteStream` do the same rewrite over a reader in two passes, the first collecting the time map, dist-tags and version names and the second copying each kept version, so the handler can rewrite a packument from storage without holding it in memory. It imports no database, storage, metrics, HTTP or logging packages, which a test checks. ### `internal/jsonscan` diff --git a/docs/configuration.md b/docs/configuration.md index 7f62207..6adfa38 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1400,7 +1400,7 @@ When upstream is unreachable and the cached entry is past its TTL, the proxy ser ### Metadata size limit -Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. +Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. With `cache_metadata` and the metadata rewrite cache on, an npm packument is streamed from the upstream into storage and rewritten from there in two passes, so it isn't held in memory and the limit doesn't apply to it; the rewritten copy is kept gzip-compressed. Requests for a single version (`//`) still read the packument into memory. ```yaml metadata_max_size: "100MB" # default diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 3cbaf93..93ef0f0 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -409,7 +409,8 @@ func (h *CondaHandler) rememberRepodata(cacheKey, digest string, filter *repodat h.repodataStates[cacheKey] = repodataState{digest: digest, etag: etag, until: filter.until} } -// countingWriter records whether anything has reached the response yet. +// countingWriter counts the bytes written through it, such as whether +// anything has reached the response yet. type countingWriter struct { w io.Writer n int64 diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 1bc5ef7..b2db3c5 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1796,6 +1796,10 @@ type metadataResult struct { // fromCache is set when the body is a stored copy within its TTL, read // without asking upstream. fromCache bool + // entry is the cache row of the stored document, for a lookup that + // leaves the document in storage (see storedMetadataDocument). body is + // nil then. + entry *database.MetadataCacheEntry } // describedBy returns res with the digest that entry, the cache row its body @@ -1935,24 +1939,9 @@ type upstreamMetadata struct { // alongside for the caller to store and replay. An empty acceptEncoding leaves Go to negotiate and // decompress transparently. func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept, acceptEncoding string) (*upstreamMetadata, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) + resp, err := p.requestUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) if err != nil { - return nil, fmt.Errorf("creating request: %w", err) - } - req.Header.Set("Accept", accept) - if acceptEncoding != "" { - req.Header.Set(headerAcceptEncoding, acceptEncoding) - } - - if entry != nil && entry.ETag.Valid { - req.Header.Set("If-None-Match", entry.ETag.String) - } else if entry != nil && entry.LastModified.Valid { - req.Header.Set("If-Modified-Since", entry.LastModified.Time.UTC().Format(http.TimeFormat)) - } - - resp, err := p.HTTPClient.Do(req) - if err != nil { - return nil, fmt.Errorf("fetching metadata: %w", err) + return nil, err } defer func() { _ = resp.Body.Close() }() @@ -1980,12 +1969,8 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e return meta, nil } - if resp.StatusCode == http.StatusNotFound { - body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamNotFoundBodyLimit)) - return nil, &upstreamNotFoundError{body: body} - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("upstream returned %d", resp.StatusCode) + if err := upstreamMetadataStatus(resp); err != nil { + return nil, err } // ContentLength is -1 when the transport decompresses the body, and @@ -2010,6 +1995,44 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e return meta, nil } +// requestUpstreamMetadata sends the request for a metadata document, +// revalidating entry when it is given. +func (p *Proxy) requestUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept, acceptEncoding string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) + if err != nil { + return nil, fmt.Errorf("creating request: %w", err) + } + req.Header.Set("Accept", accept) + if acceptEncoding != "" { + req.Header.Set(headerAcceptEncoding, acceptEncoding) + } + + if entry != nil && entry.ETag.Valid { + req.Header.Set("If-None-Match", entry.ETag.String) + } else if entry != nil && entry.LastModified.Valid { + req.Header.Set("If-Modified-Since", entry.LastModified.Time.UTC().Format(http.TimeFormat)) + } + + resp, err := p.HTTPClient.Do(req) + if err != nil { + return nil, fmt.Errorf("fetching metadata: %w", err) + } + return resp, nil +} + +// upstreamMetadataStatus returns the error for a metadata response other +// than 200. Callers handle a 304 they asked for before. +func upstreamMetadataStatus(resp *http.Response) error { + switch resp.StatusCode { + case http.StatusOK: + return nil + case http.StatusNotFound: + body, _ := io.ReadAll(io.LimitReader(resp.Body, upstreamNotFoundBodyLimit)) + return &upstreamNotFoundError{body: body} + } + return fmt.Errorf("upstream returned %d", resp.StatusCode) +} + // refreshMetadataBlob records that upstream answered 304 for the cached // document meta was read from, by moving the row's fetched_at on. The bytes in // storage are unchanged, so they are neither written nor hashed again. It @@ -2037,14 +2060,22 @@ func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMe // database. It returns the digest it recorded for the bytes, or "" when they // were not cached. func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, meta *upstreamMetadata) string { + digest, _ := p.storeMetadataBlob(ctx, ecosystem, cacheKey, storagePath, bytes.NewReader(meta.body), meta) + return digest +} + +// storeMetadataBlob is cacheMetadataBlob for a document read from body, +// which meta describes. It also returns the error that kept it from caching +// the document. +func (p *Proxy) storeMetadataBlob(ctx context.Context, ecosystem, cacheKey, storagePath string, body io.Reader, meta *upstreamMetadata) (string, error) { if p.DB == nil || p.Storage == nil { - return "" + return "", errMetadataNotStored } - size, hash, err := p.Storage.Store(ctx, storagePath, bytes.NewReader(meta.body)) + size, hash, err := p.Storage.Store(ctx, storagePath, body) if err != nil { p.Logger.Warn("failed to cache metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return "" + return "", err } var digest string if hash != "" { @@ -2074,9 +2105,9 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { p.Logger.Warn("failed to discard metadata blob", "ecosystem", ecosystem, "key", cacheKey, "error", delErr) } - return "" + return "", err } - return digest + return digest, nil } // currentMetadataEntry re-reads the metadata cache row and returns it, or diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go new file mode 100644 index 0000000..e46239f --- /dev/null +++ b/internal/handler/metadata_stored.go @@ -0,0 +1,207 @@ +package handler + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/http" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" +) + +// A metadata document can be rewritten from storage instead of from memory: +// storedMetadataDocument streams it from upstream into storage, and the +// rewrite reads it back as often as it needs. Nothing holds the whole +// document, so its size is not limited by metadata_max_size or by memory. + +// errMetadataNotStored is returned when metadata is not cached in storage, so +// a document has to be read into memory. +var errMetadataNotStored = errors.New("metadata is not cached in storage") + +// errStoredMetadataChanged is returned when a stored document no longer has +// the digest its cache row records: a fetch replaced it while it was read. +var errStoredMetadataChanged = errors.New("stored metadata changed while it was read") + +// storedMetadataDocument returns the cache row of a metadata document stored +// uncompressed with its digest: the stored copy while it is within its TTL, +// or else the document fetched from upstream and stored, or the stored copy +// when upstream fails. The document is streamed from upstream into storage +// without being held in memory, so a fetch takes no metadata_max_size limit. +// Concurrent calls for one document share a fetch, and instances sharing the +// database take turns as for other metadata. +// +// It returns errMetadataNotStored when the proxy doesn't cache metadata. +func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string) (*database.MetadataCacheEntry, error) { + if !p.CacheMetadata || p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 { + return nil, errMetadataNotStored + } + if containsPathTraversal(cacheKey) { + return nil, fmt.Errorf("invalid cache key: %q", cacheKey) + } + if entry, fresh := p.storedMetadataEntryState(ctx, ecosystem, cacheKey); fresh { + metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) + p.recordMetadataRead(entry) + return entry, nil + } + p.recordMetadataCacheMiss(ecosystem) + + key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, "", false) + "\x00stored" + res, _ := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { + entry, fresh := p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey) + if fresh { + return metadataResult{entry: entry} + } + release, outcome, err := p.awaitFetchLease(fetchCtx, metrics.CacheKindMetadata, key, func() bool { + entry, fresh = p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey) + return fresh + }) + if err != nil { + return metadataResult{err: err} + } + switch outcome { + case leaseStored: + return metadataResult{entry: entry} + case leaseFailed: + // Serve the stored copy, as the holder did, rather than trying + // the failing upstream again. + p.reloadUpstreamHolds() + if usableStoredMetadata(entry) { + return p.staleStoredMetadata(ecosystem, cacheKey, entry, errFetchFailedElsewhere) + } + case leaseFetch: + } + res := metadataResult{err: errSharedFetchAbandoned} + defer func() { release(res.err != nil || res.stale) }() + res = p.fetchStoredMetadata(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, entry) + return res + }) + return res.entry, res.err +} + +// storedMetadataEntryState reads the cache row of a document and reports +// whether it is within its TTL and usable by storedMetadataDocument. +func (p *Proxy) storedMetadataEntryState(ctx context.Context, ecosystem, cacheKey string) (*database.MetadataCacheEntry, bool) { + entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) + entry = p.migrateLegacyMetadataEntry(ctx, ecosystem, cacheKey, entry) + fresh := usableStoredMetadata(entry) && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL + return entry, fresh +} + +// usableStoredMetadata reports whether entry describes a document stored +// uncompressed with its digest, which a rewrite can read back and check. +func usableStoredMetadata(entry *database.MetadataCacheEntry) bool { + return entry != nil && entry.ContentDigest.Valid && entry.ContentEncoding.String == "" +} + +// fetchStoredMetadata fetches a document from upstream into storage, +// revalidating entry when there is one, and falls back to entry when +// upstream fails. +func (p *Proxy) fetchStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, entry *database.MetadataCacheEntry) metadataResult { + revalidate := entry + if !usableStoredMetadata(entry) { + revalidate = nil + } + stored, err := p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, revalidate) + if errors.Is(err, errStale304) { + stored, err = p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, nil) + } + if err == nil { + return metadataResult{entry: stored} + } + if !usableStoredMetadata(entry) || errors.Is(err, ErrUpstreamNotFound) { + return metadataResult{err: err} + } + return p.staleStoredMetadata(ecosystem, cacheKey, entry, err) +} + +// staleStoredMetadata falls back to the stored document entry describes after +// the upstream fetch failed with err. +func (p *Proxy) staleStoredMetadata(ecosystem, cacheKey string, entry *database.MetadataCacheEntry, err error) metadataResult { + p.Logger.Warn("upstream metadata fetch failed, serving cached copy", + "ecosystem", ecosystem, "key", cacheKey, "error", err) + entry = p.currentMetadataEntry(ecosystem, cacheKey, entry) + if !usableStoredMetadata(entry) { + return metadataResult{err: fmt.Errorf("upstream failed and no usable cached metadata: %w", err)} + } + return metadataResult{entry: entry, stale: true} +} + +// storeUpstreamMetadata fetches a document and streams it into storage, and +// returns its new cache row. On a 304 for entry it only moves the row's +// fetched_at on; errStale304 means the row was replaced meanwhile and the +// caller should fetch without validators. +func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, entry *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { + resp, err := p.requestUpstreamMetadata(ctx, upstreamURL, entry, accept, "") + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotModified && entry != nil { + refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) + if err != nil || !refreshed { + return nil, errStale304 + } + return p.currentMetadataEntry(ecosystem, cacheKey, entry), nil + } + if err := upstreamMetadataStatus(resp); err != nil { + return nil, err + } + + meta := &upstreamMetadata{ + contentType: resp.Header.Get(headerContentType), + etag: resp.Header.Get(headerETag), + } + if meta.contentType == "" { + meta.contentType = contentTypeJSON + } + if lm := resp.Header.Get(headerLastModified); lm != "" { + meta.lastModified, _ = http.ParseTime(lm) + } + // Go decompresses the body, since no Accept-Encoding was set, so it is + // stored uncompressed. + if _, err := p.storeMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), resp.Body, meta); err != nil { + return nil, err + } + stored, err := p.DB.GetMetadataCache(ecosystem, cacheKey) + if err != nil || !usableStoredMetadata(stored) { + return nil, fmt.Errorf("reading the stored metadata row: %w", errors.Join(err, errMetadataNotStored)) + } + return stored, nil +} + +// openStoredMetadata opens the document entry describes. Reading it to the +// end fails with errStoredMetadataChanged when its bytes don't have the +// digest entry records. +func (p *Proxy) openStoredMetadata(ctx context.Context, entry *database.MetadataCacheEntry) (io.ReadCloser, error) { + r, err := p.Storage.Open(ctx, entry.StoragePath) + if err != nil { + return nil, err + } + return &digestReader{r: r, h: sha256.New(), want: entry.ContentDigest.String}, nil +} + +// digestReader checks at the end of its reader that what it read has the +// digest want, in the form the metadata cache records it. +type digestReader struct { + r io.ReadCloser + h hash.Hash + want string +} + +func (d *digestReader) Read(b []byte) (int, error) { + n, err := d.r.Read(b) + d.h.Write(b[:n]) + if errors.Is(err, io.EOF) && "sha256:"+hex.EncodeToString(d.h.Sum(nil)) != d.want { + return n, errStoredMetadataChanged + } + return n, err +} + +func (d *digestReader) Close() error { return d.r.Close() } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 7263b3b..99c12ae 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -255,6 +255,10 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } + if !versionLookup && h.serveStreamedPackument(w, r, packageName, upstreamURL, accept, metadataCacheKey) { + return + } + var validate func([]byte) error if !versionLookup { validate = h.packumentValidator() @@ -308,6 +312,82 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques serveRewrittenMetadata(w, r, rewritten) } +// serveStreamedPackument serves a packument rewritten from storage: the +// document is streamed from upstream into storage, then read twice, once to +// collect the time map and version names cooldown and the denylist decide +// on, and once to copy each kept version. registry.npmjs.org writes +// dist-tags before versions and time after them, so one pass can't do it. +// Neither the packument nor its rewrite is held in memory uncompressed, and +// packuments larger than metadata_max_size can be served. +// +// It reports false when it leaves the request to the in-memory rewrite: +// without metadata or rewrite caching, for a document it can't stream, such +// as an abbreviated packument where the full one is required, and when the +// rewrite fails. The document is stored by then, so the in-memory path reads +// it from storage. +func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Request, packageName, upstreamURL, accept, metadataCacheKey string) bool { + if h.proxy.rewrites == nil { + return false + } + entry, err := h.proxy.storedMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept) + switch { + case err == nil: + case errors.Is(err, errMetadataNotStored): + return false + case errors.Is(err, ErrUpstreamNotFound): + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) + return true + case writeUpstreamHeld(w, err): + return true + default: + h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) + JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + return true + } + + fullRequired := h.packumentValidator() != nil + options := h.npmRewriteOptions(packageName, nil) + rewritten, err := h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { + var scan *npm.Scan + err := readStored(open, func(in io.Reader) (err error) { + scan, err = npm.ScanPackument(in) + return err + }) + if err != nil { + return err + } + if fullRequired && scan.Abbreviated() { + return npm.ErrNotStreamable + } + return readStored(open, func(in io.Reader) error { + _, err := npm.RewriteStream(packageName, scan, in, out, options) + return err + }) + }) + if err != nil { + if r.Context().Err() != nil { + return true // the client left while waiting on a shared rewrite + } + if !errors.Is(err, npm.ErrNotStreamable) { + h.proxy.Logger.Warn("failed to rewrite stored npm metadata, rewriting in memory", + "package", packageName, "error", err) + } + return false + } + serveRewrittenMetadata(w, r, rewritten) + return true +} + +// readStored opens a stored document and reads it with read. +func readStored(open func() (io.ReadCloser, error), read func(io.Reader) error) error { + in, err := open() + if err != nil { + return err + } + defer func() { _ = in.Close() }() + return read(in) +} + // packumentAccept returns the Accept header packuments are requested with. // It prefers the smaller abbreviated packument format but includes // application/json as a fallback so upstreams that reject the abbreviated type diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go new file mode 100644 index 0000000..565f062 --- /dev/null +++ b/internal/handler/npm_stream_test.go @@ -0,0 +1,243 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// bigPackument returns a packument laid out as registry.npmjs.org writes +// them, dist-tags before versions and time after, with n old versions padded +// so the document is large, and a latest published an hour ago. +func bigPackument(n int) string { + var b strings.Builder + b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"9.0.0","old":"1.0.0"},"versions":{`) + times := []string{`"created":"2015-01-01T00:00:00Z"`} + for i := range n { + v := fmt.Sprintf("1.0.%d", i) + if i == 0 { + v = "1.0.0" + } + fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"readme":%q,"dist":{"tarball":"https://registry.npmjs.org/big/-/big-%s.tgz"}},`, + v, v, strings.Repeat("x", 512), v) + times = append(times, fmt.Sprintf(`%q:"2016-01-01T00:%02d:%02dZ"`, v, i/60%60, i%60)) + } + recent := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) + b.WriteString(`"9.0.0":{"name":"big","version":"9.0.0","dist":{"tarball":"https://registry.npmjs.org/big/-/big-9.0.0.tgz"}}},`) + times = append(times, `"9.0.0":"`+recent+`"`) + b.WriteString(`"time":{` + strings.Join(times, ",") + `},"readme":"big"}`) + return b.String() +} + +type streamUpstream struct { + *httptest.Server + body atomic.Value // string + fail atomic.Bool + requests atomic.Int32 +} + +func newStreamUpstream(t *testing.T, body string) *streamUpstream { + t.Helper() + u := &streamUpstream{} + u.body.Store(body) + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.requests.Add(1) + switch { + case u.fail.Load(): + http.Error(w, "down", http.StatusBadGateway) + case r.URL.Path == "/big" && r.Header.Get("If-None-Match") == `"v1"`: + w.WriteHeader(http.StatusNotModified) + case r.URL.Path == "/big": + w.Header().Set("Content-Type", "application/json") + w.Header().Set("ETag", `"v1"`) + _, _ = io.WriteString(w, u.body.Load().(string)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(u.Close) + return u +} + +func streamProxy(t *testing.T, u *streamUpstream) (*Proxy, *mockStorage, http.Handler) { + t.Helper() + proxy, _, store, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.SetMetadataRewriteCacheSize(64 << 20) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + return proxy, store, NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() +} + +type packumentDoc struct { + DistTags map[string]string `json:"dist-tags"` + Versions map[string]json.RawMessage `json:"versions"` + Time map[string]string `json:"time"` +} + +func decodeStreamed(t *testing.T, rec *httptest.ResponseRecorder) packumentDoc { + t.Helper() + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + body := rec.Body.Bytes() + if rec.Header().Get("Content-Encoding") == "gzip" { + zr, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if body, err = io.ReadAll(zr); err != nil { + t.Fatal(err) + } + } + var doc packumentDoc + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatalf("response is not JSON: %v", err) + } + return doc +} + +// TestNPMStreamedPackument requests a packument larger than +// metadata_max_size with cooldown on, its time map after its versions and +// its latest inside cooldown. The in-memory rewrite can't read it at all. +func TestNPMStreamedPackument(t *testing.T) { + const versions = 400 + body := bigPackument(versions) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + + for _, header := range []http.Header{nil, {"Accept-Encoding": {"gzip"}}} { + doc := decodeStreamed(t, serve(handler, "/big", header)) + if len(doc.Versions) != versions { + t.Errorf("%d versions, want %d without 9.0.0", len(doc.Versions), versions) + } + if _, ok := doc.Versions["9.0.0"]; ok { + t.Error("9.0.0 is listed although it is inside cooldown") + } + if _, ok := doc.Time["9.0.0"]; ok { + t.Error("time still lists 9.0.0") + } + if doc.DistTags["latest"] != fmt.Sprintf("1.0.%d", versions-1) || doc.DistTags["old"] != "1.0.0" { + t.Errorf("dist-tags = %v, want latest moved to the newest version left", doc.DistTags) + } + if !strings.Contains(string(doc.Versions["1.0.0"]), `"tarball":"http://proxy.example/npm/big/-/big-1.0.0.tgz"`) { + t.Errorf("tarball not rewritten: %s", doc.Versions["1.0.0"]) + } + } + if n := u.requests.Load(); n != 1 { + t.Errorf("upstream asked %d times, want once", n) + } + + // Without the rewrite cache the in-memory path runs, which the size + // limit stops. + proxy.SetMetadataRewriteCacheSize(0) + if rec := serve(handler, "/big", nil); rec.Code == http.StatusOK { + t.Error("in-memory rewrite served a packument over metadata_max_size; the test no longer covers the stream") + } +} + +// TestNPMStreamedPackumentMatchesInMemory checks that both paths serve the +// same document and ETag. +func TestNPMStreamedPackumentMatchesInMemory(t *testing.T) { + body := bigPackument(50) + u := newStreamUpstream(t, body) + _, _, streamed := streamProxy(t, u) + inMemory, _, handler := streamProxy(t, u) + inMemory.CacheMetadata = false + + a, b := serve(streamed, "/big", nil), serve(handler, "/big", nil) + if a.Code != http.StatusOK || b.Code != http.StatusOK { + t.Fatalf("status %d and %d", a.Code, b.Code) + } + var ca, cb bytes.Buffer + if err := json.Compact(&ca, a.Body.Bytes()); err != nil { + t.Fatal(err) + } + if err := json.Compact(&cb, b.Body.Bytes()); err != nil { + t.Fatal(err) + } + if !bytes.Equal(ca.Bytes(), cb.Bytes()) { + t.Error("streamed and in-memory rewrites differ") + } +} + +// TestNPMStreamedPackumentStale serves the stored copy when upstream fails +// after the TTL, and 404 for a package upstream doesn't have. +func TestNPMStreamedPackumentStale(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, _, handler := streamProxy(t, u) + decodeStreamed(t, serve(handler, "/big", nil)) + + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) // drop the cached rewrite + u.fail.Store(true) + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { + t.Errorf("stale copy has %d versions, want 10", len(doc.Versions)) + } + + u.fail.Store(false) + if rec := serve(handler, "/missing", nil); rec.Code != http.StatusNotFound { + t.Errorf("missing package: status %d, want 404", rec.Code) + } +} + +// TestNPMStreamedPackumentChangedInStorage checks that a document replaced +// in storage between the two passes isn't rewritten from mixed bytes. +func TestNPMStreamedPackumentChangedInStorage(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, store, _ := streamProxy(t, u) + entry, err := proxy.storedMetadataDocument(context.Background(), "npm", "big", u.URL+"/big", contentTypeJSON) + if err != nil { + t.Fatal(err) + } + store.mu.Lock() + store.files[entry.StoragePath] = []byte(bigPackument(11)) + store.mu.Unlock() + + r, err := proxy.openStoredMetadata(context.Background(), entry) + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + if _, err := io.ReadAll(r); !errors.Is(err, errStoredMetadataChanged) { + t.Errorf("read error %v, want errStoredMetadataChanged", err) + } +} + +// TestNPMStreamedPackumentRevalidates checks that a 304 after the TTL keeps +// the stored document and its cached rewrite, without storing it again. +func TestNPMStreamedPackumentRevalidates(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, store, handler := streamProxy(t, u) + first := serve(handler, "/big", nil) + decodeStreamed(t, first) + + proxy.MetadataTTL = time.Nanosecond + second := serve(handler, "/big", nil) + decodeStreamed(t, second) + if second.Header().Get("ETag") != first.Header().Get("ETag") { + t.Errorf("ETag changed from %s to %s", first.Header().Get("ETag"), second.Header().Get("ETag")) + } + if n := u.requests.Load(); n != 2 { + t.Errorf("upstream asked %d times, want 2", n) + } + store.mu.Lock() + defer store.mu.Unlock() + if store.stores != 1 { + t.Errorf("document stored %d times, want once", store.stores) + } +} diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index fd455a7..a624dba 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -18,6 +18,7 @@ import ( "sync" "time" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/metrics" ) @@ -253,6 +254,18 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, expir if c == nil { return rewriteWithETag(in, rewrite) } + return c.produce(ctx, key, expiry, func() (rewrittenMetadata, error) { + m, err := rewriteWithETag(in, rewrite) + if err == nil { + m.gzipped = gzipIfSmaller(m.plain) + } + return m, err + }) +} + +// produce is rewrite for a document that build makes with its validators +// and gzip copy. +func (c *rewriteCache) produce(ctx context.Context, key string, expiry func(start time.Time) time.Time, build func() (rewrittenMetadata, error)) (rewrittenMetadata, error) { c.mu.Lock() if e, ok := c.lookup(key, c.clock()); ok { @@ -288,10 +301,7 @@ func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, expir c.mu.Unlock() close(f.done) }() - f.m, f.err = rewriteWithETag(in, rewrite) - if f.err == nil { - f.m.gzipped = gzipIfSmaller(f.m.plain) - } + f.m, f.err = build() return f.m, f.err } @@ -391,6 +401,60 @@ func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name str return m, err } +// streamedRewrite rewrites a document stored in storage, through the +// proxy's rewrite cache, without holding the document or its rewrite in +// memory uncompressed. rewrite is given a way to open the stored document, +// as often as it needs to read it, and writes the rewritten document to w. +// The result is cached under the same key cachedRewrite uses for the +// document, and expires the same way with cooldown on. +// +// It returns errMetadataNotStored without the rewrite cache, which the +// result has to be kept in. +func (p *Proxy) streamedRewrite(ctx context.Context, ecosystem, proxyURL, name string, entry *database.MetadataCacheEntry, rewrite func(open func() (io.ReadCloser, error), w io.Writer) error) (rewrittenMetadata, error) { + if p.rewrites == nil { + return rewrittenMetadata{}, errMetadataNotStored + } + var expiry func(time.Time) time.Time + if p.cooldownEnabled() { + rec := p.watchCooldown() + defer p.unwatchCooldown(rec) + expiry = func(start time.Time) time.Time { return p.cooldownExpiry(rec, start) } + } + key := rewriteCacheKeyForDigest(ecosystem, proxyURL, name, entry.ContentDigest.String) + m, err := p.rewrites.produce(ctx, key, expiry, func() (rewrittenMetadata, error) { + // The rewrite runs to completion for the cache even when the + // caller leaves. + readCtx := context.WithoutCancel(ctx) + return gzipRewrite(func(w io.Writer) error { + return rewrite(func() (io.ReadCloser, error) { return p.openStoredMetadata(readCtx, entry) }, w) + }) + }) + if err == nil { + m.lastModified = p.metadataDate(ecosystem, proxyURL, name, m.etag) + } + return m, err +} + +// gzipRewrite runs write and keeps what it writes gzip-compressed only, with +// the ETag and length of the uncompressed output. +func gzipRewrite(write func(w io.Writer) error) (rewrittenMetadata, error) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + sum := sha256.New() + n := &countingWriter{w: io.Discard} + if err := write(io.MultiWriter(zw, sum, n)); err != nil { + return rewrittenMetadata{}, err + } + if err := zw.Close(); err != nil { + return rewrittenMetadata{}, err + } + return rewrittenMetadata{ + metadataValidators: metadataValidators{etag: `"` + hex.EncodeToString(sum.Sum(nil)) + `"`}, + gzipped: buf.Bytes(), + size: int(n.n), + }, nil +} + // uncachedRewrite gives a rewrite made outside the cache its validators: an // ETag of the output itself and, as cachedRewrite does, a date that replicas // agree on. @@ -501,16 +565,28 @@ func serveRewrittenMetadataAs(w http.ResponseWriter, r *http.Request, m rewritte w.WriteHeader(http.StatusNotModified) return } - body := m.gzipped - if !compressed { - body = m.content() - } w.Header().Set(headerContentType, contentType) if compressed { w.Header().Set(headerContentEncoding, "gzip") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(m.gzipped) + return + } + if m.plain == nil && m.gzipped != nil { + // Decompressed as it is sent, rather than into a copy of the + // whole document. + zr, err := gzip.NewReader(bytes.NewReader(m.gzipped)) + if err != nil { + http.Error(w, "failed to read cached metadata", http.StatusInternalServerError) + return + } + w.Header().Set(headerContentLength, strconv.Itoa(m.size)) + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, zr) + return } w.WriteHeader(http.StatusOK) - _, _ = w.Write(body) + _, _ = w.Write(m.plain) } // ensureVary adds field to the Vary header unless it is listed already. diff --git a/internal/metadata/npm/imports_test.go b/internal/metadata/npm/imports_test.go index 0f04bc1..19f828c 100644 --- a/internal/metadata/npm/imports_test.go +++ b/internal/metadata/npm/imports_test.go @@ -23,6 +23,7 @@ func TestPortableImports(t *testing.T) { "github.com/Masterminds/semver/v3", "github.com/git-pkgs/cooldown", "github.com/git-pkgs/proxy/internal/jsonscan", + "github.com/git-pkgs/proxy/internal/jsonstream", "github.com/git-pkgs/proxy/internal/metadata/npm", "github.com/git-pkgs/proxy/internal/packageurl", "github.com/git-pkgs/purl", diff --git a/internal/metadata/npm/stream.go b/internal/metadata/npm/stream.go new file mode 100644 index 0000000..cc70c52 --- /dev/null +++ b/internal/metadata/npm/stream.go @@ -0,0 +1,270 @@ +package npm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "reflect" + "slices" + + "github.com/git-pkgs/proxy/internal/jsonstream" +) + +// ErrNotStreamable is returned by RewriteStream for a document it leaves to +// Rewrite: one without a versions object, such as a single version's +// metadata, or a rewrite that lists pushed versions. +var ErrNotStreamable = errors.New("npm metadata can't be rewritten as a stream") + +// Scan is what a first pass over a packument learns: the version names, and +// the time map and dist-tags the filters work on. It holds no version +// entries, so it stays small whatever the size of the packument. +type Scan struct { + versions []string + versionsObject bool + time any + distTags any + // occurrences counts each filtered key, so the second pass can drop the + // earlier duplicates, as Rewrite does. + occurrences map[string]int + hasModified bool +} + +// Abbreviated reports whether the scanned document is an abbreviated +// packument, as the package function Abbreviated does for a document in +// memory. +func (s *Scan) Abbreviated() bool { + return s.hasModified && s.occurrences["time"] == 0 +} + +// ScanPackument reads a packument once, keeping only what RewriteStream +// needs to decide which versions remain. Version entries are read one at a +// time and dropped. +func ScanPackument(r io.Reader) (*Scan, error) { + s := &Scan{occurrences: map[string]int{}} + dec := json.NewDecoder(r) + if err := expectDelim(dec, '{'); err != nil { + return nil, err + } + for dec.More() { + key, err := objectKey(dec) + if err != nil { + return nil, err + } + switch key { + case "versions": + s.occurrences[key]++ + s.versions, s.versionsObject, err = objectKeys(dec) + case "time": + s.occurrences[key]++ + s.time = nil + err = dec.Decode(&s.time) + case "dist-tags": + s.occurrences[key]++ + s.distTags = nil + err = dec.Decode(&s.distTags) + default: + if key == "modified" { + s.hasModified = true + } + err = skipValue(dec) + } + if err != nil { + return nil, err + } + } + if err := expectDelim(dec, '}'); err != nil { + return nil, err + } + if _, err := dec.Token(); err != io.EOF { + if err == nil { + err = errors.New("unexpected data after the top-level object") + } + return nil, err + } + return s, nil +} + +// RewriteStream is Rewrite for a packument too large to hold in memory. scan +// is the result of ScanPackument over the same bytes r reads again; the +// caller checks they are the same. The rewritten document is written to w +// with the bytes of untouched values copied as they are, and its output is +// the same as Rewrite's for the same document and options. It returns the +// versions it kept. For a document without a versions object, or options +// listing pushed versions, it returns ErrNotStreamable before writing. +func RewriteStream(packageName string, scan *Scan, r io.Reader, w io.Writer, options Options) ([]string, error) { + if !scan.versionsObject || len(options.LocalVersions) > 0 { + return nil, ErrNotStreamable + } + h := &transformer{options} + + versions := make(map[string]any, len(scan.versions)) + for _, v := range scan.versions { + versions[v] = nil + } + metadata := map[string]any{} + timeMap := scan.objectValue(metadata, "time", scan.time) + distTags := scan.objectValue(metadata, "dist-tags", scan.distTags) + timeEntries := len(timeMap) + originalTags := maps.Clone(distTags) + + h.applyCooldownFiltering(metadata, versions, packageName) + h.applyDenylistFiltering(metadata, versions) + + sw := &streamWriter{ + transformer: h, + packageName: packageName, + scan: scan, + versions: versions, + timeMap: timeMap, + distTags: distTags, + timeChanged: timeMap != nil && len(timeMap) != timeEntries, + tagsChanged: distTags != nil && !reflect.DeepEqual(distTags, originalTags), + seen: map[string]int{}, + } + if err := jsonstream.CopyObject(json.NewDecoder(r), w, sw.member); err != nil { + return nil, err + } + h.tarballsRewritten(sw.rewritten) + return slices.Collect(maps.Keys(versions)), nil +} + +// streamWriter writes the second pass of RewriteStream from the filtered +// state the first pass left. +type streamWriter struct { + *transformer + packageName string + scan *Scan + versions map[string]any + timeMap, distTags map[string]any + timeChanged, tagsChanged bool + seen map[string]int + entry bytes.Buffer + rewritten int +} + +// member writes one top-level member. Members the filters don't touch are +// left unconsumed, so CopyObject copies them as they are. +func (s *streamWriter) member(m *jsonstream.Member) error { + total, filtered := s.scan.occurrences[m.Key] + if !filtered { + return nil + } + s.seen[m.Key]++ + if s.seen[m.Key] < total { + _, err := m.Raw() // an earlier duplicate, dropped + return err + } + switch { + case m.Key == "versions": + return m.Object(s.version) + case m.Key == "time" && s.timeChanged: + return m.Object(s.time) + case m.Key == "dist-tags" && s.tagsChanged: + if _, err := m.Raw(); err != nil { + return err + } + encoded, err := json.Marshal(s.distTags) + if err != nil { + return err + } + return m.Write(encoded) + } + return nil +} + +// version writes a kept version with its tarball pointing at the proxy. +func (s *streamWriter) version(v *jsonstream.Member) error { + raw, err := v.Raw() + if err != nil { + return err + } + if _, keep := s.versions[v.Key]; !keep { + return nil + } + s.entry.Reset() + if s.writeNPMVersion(&s.entry, s.packageName, v.Key, raw) { + s.rewritten++ + } + return v.Write(s.entry.Bytes()) +} + +// time writes a time entry the filters kept. +func (s *streamWriter) time(t *jsonstream.Member) error { + if _, keep := s.timeMap[t.Key]; keep { + return t.Copy() + } + _, err := t.Raw() + return err +} + +// objectValue records value under name in metadata, as Rewrite's +// decodeObject does, and returns it when it is an object. +func (s *Scan) objectValue(metadata map[string]any, name string, value any) map[string]any { + if s.occurrences[name] == 0 { + return nil + } + metadata[name] = value + object, _ := value.(map[string]any) + return object +} + +func expectDelim(dec *json.Decoder, want json.Delim) error { + tok, err := dec.Token() + if err != nil { + return err + } + if tok != want { + return fmt.Errorf("npm metadata: expected %v, got %v", want, tok) + } + return nil +} + +func objectKey(dec *json.Decoder) (string, error) { + tok, err := dec.Token() + if err != nil { + return "", err + } + key, ok := tok.(string) + if !ok { + return "", fmt.Errorf("npm metadata: object key is %T, not a string", tok) + } + return key, nil +} + +// objectKeys reads a value and returns its keys when it is an object. +func objectKeys(dec *json.Decoder) ([]string, bool, error) { + tok, err := dec.Token() + if err != nil { + return nil, false, err + } + delim, ok := tok.(json.Delim) + if !ok { + return nil, false, nil // a scalar, read whole by Token + } + var keys []string + for dec.More() { + if delim == '{' { + key, err := objectKey(dec) + if err != nil { + return nil, false, err + } + keys = append(keys, key) + } + if err := skipValue(dec); err != nil { + return nil, false, err + } + } + if _, err := dec.Token(); err != nil { + return nil, false, err + } + return keys, delim == '{', nil +} + +// skipValue reads the next value without keeping it. +func skipValue(dec *json.Decoder) error { + var raw json.RawMessage + return dec.Decode(&raw) +} diff --git a/internal/metadata/npm/stream_test.go b/internal/metadata/npm/stream_test.go new file mode 100644 index 0000000..ef2c954 --- /dev/null +++ b/internal/metadata/npm/stream_test.go @@ -0,0 +1,162 @@ +package npm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// rewriteStream runs both passes of RewriteStream over body. +func rewriteStream(packageName string, body []byte, options Options) ([]byte, []string, error) { + scan, err := ScanPackument(bytes.NewReader(body)) + if err != nil { + return nil, nil, err + } + var out bytes.Buffer + kept, err := RewriteStream(packageName, scan, bytes.NewReader(body), &out, options) + return out.Bytes(), kept, err +} + +// TestRewriteStreamMatchesRewrite checks that the streamed rewrite writes the +// same bytes and keeps the same versions as Rewrite, over documents with the +// filtered members in either order, duplicates, and values Rewrite copies +// verbatim. +func TestRewriteStreamMatchesRewrite(t *testing.T) { + now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) + ts := func(age time.Duration) string { return now.Add(-age).Format(time.RFC3339) } + version := func(v string) string { + return fmt.Sprintf(`%q:{"version":%q,"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-%s.tgz"}}`, v, v, v) + } + versions := `"versions":{` + version("1.0.0") + `,` + version("2.0.0") + `,` + version("3.0.0") + `}` + times := `"time":{"created":"` + ts(1000*time.Hour) + `","1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}` + tags := `"dist-tags":{"latest":"3.0.0","beta":"2.0.0","next":"1.0.0"}` + + docs := map[string]string{ + "testdata": string(packument), + "tags, versions, time": `{"name":"demo",` + tags + `,` + versions + `,` + times + `}`, + "time first": `{` + times + `,"name":"demo",` + versions + `,` + tags + `}`, + "no time": `{"name":"demo",` + tags + `,` + versions + `}`, + "time null": `{"time":null,` + tags + `,` + versions + `}`, + "tags not an object": `{"dist-tags":"x",` + versions + `,` + times + `}`, + "duplicates": `{"versions":{` + version("9.0.0") + `},` + tags + `,"time":{"9.0.0":"` + ts(time.Hour) + `"},` + + versions + `,` + times + `,"dist-tags":{"latest":"3.0.0"}}`, + "verbatim values": `{"_big":12345678901234567890,"readme":"&","nested":{"a":[1, 2 ,{"b":null}]},` + versions + `}`, + "unusual entries": `{"versions":{"1.0.0":"not an object","2.0.0":{"dist":"x"},"3.0.0":{"dist":{"tarball":7}},` + + `"4.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/renamed.tgz"}},"5.0.0":{"dist":{}}}}`, + "whitespace": "{ \"name\" : \"demo\" ,\n\t\"versions\" : {\n \"1.0.0\" : { \"dist\" : { \"tarball\" : \"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz\" } } } }", + "empty versions": `{"name":"demo","versions":{}}`, + } + options := map[string]Options{ + "plain": {ProxyURL: "http://proxy"}, + "cooldown": {ProxyURL: "http://proxy", Cooldown: &cooldown.Config{Default: "3d"}, EvaluatedAt: now}, + "denylist": {ProxyURL: "http://proxy", DeniedVersions: map[string]bool{"2.0.0": true}}, + "both": {ProxyURL: "http://proxy", Cooldown: &cooldown.Config{Default: "3d"}, EvaluatedAt: now, + DeniedVersions: map[string]bool{"1.0.0": true}}, + } + for docName, doc := range docs { + for optName, opts := range options { + t.Run(docName+"/"+optName, func(t *testing.T) { + want, wantKept, err := Rewrite("demo", []byte(doc), opts) + if err != nil { + t.Fatalf("Rewrite: %v", err) + } + got, gotKept, err := rewriteStream("demo", []byte(doc), opts) + if err != nil { + t.Fatalf("RewriteStream: %v", err) + } + // Rewrite keeps the whitespace inside the members of a + // filtered time map, which the stream doesn't see. + if !bytes.Equal(compact(t, got), compact(t, want)) { + t.Errorf("RewriteStream =\n%s\nRewrite =\n%s", got, want) + } + if docName == "verbatim values" && !bytes.Contains(got, []byte(`"_big":12345678901234567890,"readme":"&","nested":{"a":[1, 2 ,{"b":null}]}`)) { + t.Errorf("values not copied verbatim: %s", got) + } + slices.Sort(gotKept) + slices.Sort(wantKept) + if !slices.Equal(gotKept, wantKept) { + t.Errorf("kept %v, Rewrite kept %v", gotKept, wantKept) + } + }) + } + } +} + +func compact(t *testing.T, doc []byte) []byte { + t.Helper() + var out bytes.Buffer + if err := json.Compact(&out, doc); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, doc) + } + return out.Bytes() +} + +func TestRewriteStreamCallbacks(t *testing.T) { + var filtered []string + var rewritten []int + options := Options{ProxyURL: "http://proxy", Cooldown: &cooldown.Config{Default: "7d"}, + EvaluatedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + OnCooldownFilter: func(version, _ string) { filtered = append(filtered, version) }, + OnTarballsRewritten: func(count int) { rewritten = append(rewritten, count) }, + } + if _, _, err := rewriteStream("@example/widgets", packument, options); err != nil { + t.Fatal(err) + } + if !slices.Equal(filtered, []string{"3.0.0"}) || !slices.Equal(rewritten, []int{2}) { + t.Errorf("filtered %v, rewritten %v; want [3.0.0] and [2]", filtered, rewritten) + } +} + +func TestRewriteStreamNotStreamable(t *testing.T) { + for name, tc := range map[string]struct { + doc string + options Options + }{ + "single version": {`{"name":"demo","version":"1.0.0","dist":{"tarball":"t"}}`, Options{}}, + "versions array": {`{"name":"demo","versions":[{"a":1}]}`, Options{}}, + "versions null": {`{"name":"demo","versions":null}`, Options{}}, + "pushed versions": {`{"versions":{}}`, Options{LocalVersions: []LocalVersion{{Version: "1.0.0", Entry: []byte(`{}`)}}}}, + } { + t.Run(name, func(t *testing.T) { + var out bytes.Buffer + scan, err := ScanPackument(strings.NewReader(tc.doc)) + if err != nil { + t.Fatal(err) + } + if _, err := RewriteStream("demo", scan, strings.NewReader(tc.doc), &out, tc.options); !errors.Is(err, ErrNotStreamable) { + t.Errorf("err = %v, want ErrNotStreamable", err) + } + if out.Len() != 0 { + t.Errorf("wrote %q before refusing", out.String()) + } + }) + } +} + +func TestScanPackument(t *testing.T) { + for _, body := range []string{`{"versions": {}`, `[1]`, `{"versions": {}} trailing`, `{"a":}`, ``} { + if _, err := ScanPackument(strings.NewReader(body)); err == nil { + t.Errorf("ScanPackument(%q): expected an error", body) + } + } + for body, want := range map[string]bool{ + `{"name":"demo","modified":"2024-01-01T00:00:00Z","versions":{}}`: true, + `{"name":"demo","modified":"2024-01-01T00:00:00Z","time":{},"versions":{}}`: false, + `{"name":"demo","versions":{}}`: false, + } { + scan, err := ScanPackument(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if scan.Abbreviated() != want || Abbreviated([]byte(body)) != want { + t.Errorf("Abbreviated(%s) = %v, want %v", body, scan.Abbreviated(), want) + } + } +} From 8c9e87f15ff8bc5c5022c5770176baf701e2d14d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 15:24:18 +0100 Subject: [PATCH 016/198] Stage fetched packuments, fall back on storage errors, stream version lookups A fetched packument is now stored under a staging path and checked before it replaces the cached copy, so an abbreviated one no longer overwrites a full copy; the full copy is served as when upstream fails. A failure to store falls back to the in-memory path, as caching is best effort there. Requests for a single version or dist-tag and the tarball download's cooldown and integrity lookup read the stored packument as a stream too, keeping only the one version entry they need. --- internal/handler/metadata_stored.go | 81 ++++++++-- internal/handler/npm.go | 223 +++++++++++++++++++++++---- internal/handler/npm_stream_test.go | 119 +++++++++++++- internal/handler/npm_test.go | 4 +- internal/metadata/npm/stream.go | 204 ++++++++++++++++++++---- internal/metadata/npm/stream_test.go | 73 ++++++++- 6 files changed, 617 insertions(+), 87 deletions(-) diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index e46239f..d831303 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -2,6 +2,7 @@ package handler import ( "context" + "crypto/rand" "crypto/sha256" "encoding/hex" "errors" @@ -36,8 +37,14 @@ var errStoredMetadataChanged = errors.New("stored metadata changed while it was // Concurrent calls for one document share a fetch, and instances sharing the // database take turns as for other metadata. // -// It returns errMetadataNotStored when the proxy doesn't cache metadata. -func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string) (*database.MetadataCacheEntry, error) { +// validate, when not nil, checks a fetched document before it replaces the +// stored one; a document it rejects is not stored, and the stored copy is +// served as when upstream fails. It does not check a stored copy within its +// TTL. +// +// It returns errMetadataNotStored when the proxy doesn't cache metadata or +// storing the document failed, and the caller reads it into memory instead. +func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error) (*database.MetadataCacheEntry, error) { if !p.CacheMetadata || p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 { return nil, errMetadataNotStored } @@ -78,7 +85,7 @@ func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, } res := metadataResult{err: errSharedFetchAbandoned} defer func() { release(res.err != nil || res.stale) }() - res = p.fetchStoredMetadata(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, entry) + res = p.fetchStoredMetadata(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, validate, entry) return res }) return res.entry, res.err @@ -101,20 +108,21 @@ func usableStoredMetadata(entry *database.MetadataCacheEntry) bool { // fetchStoredMetadata fetches a document from upstream into storage, // revalidating entry when there is one, and falls back to entry when -// upstream fails. -func (p *Proxy) fetchStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, entry *database.MetadataCacheEntry) metadataResult { +// upstream fails or validate rejects what it sent. A failure to store is +// returned as it is, for the caller to read the document into memory. +func (p *Proxy) fetchStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, entry *database.MetadataCacheEntry) metadataResult { revalidate := entry if !usableStoredMetadata(entry) { revalidate = nil } - stored, err := p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, revalidate) + stored, err := p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, revalidate) if errors.Is(err, errStale304) { - stored, err = p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, nil) + stored, err = p.storeUpstreamMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, nil) } if err == nil { return metadataResult{entry: stored} } - if !usableStoredMetadata(entry) || errors.Is(err, ErrUpstreamNotFound) { + if !usableStoredMetadata(entry) || errors.Is(err, ErrUpstreamNotFound) || errors.Is(err, errMetadataNotStored) { return metadataResult{err: err} } return p.staleStoredMetadata(ecosystem, cacheKey, entry, err) @@ -136,7 +144,10 @@ func (p *Proxy) staleStoredMetadata(ecosystem, cacheKey string, entry *database. // returns its new cache row. On a 304 for entry it only moves the row's // fetched_at on; errStale304 means the row was replaced meanwhile and the // caller should fetch without validators. -func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, entry *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { +// +// With validate, the document is staged under a path of its own and checked +// there first, so one validate rejects never replaces the stored copy. +func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, entry *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { resp, err := p.requestUpstreamMetadata(ctx, upstreamURL, entry, accept, "") if err != nil { return nil, err @@ -166,8 +177,17 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, } // Go decompresses the body, since no Accept-Encoding was set, so it is // stored uncompressed. - if _, err := p.storeMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), resp.Body, meta); err != nil { - return nil, err + var body io.Reader = resp.Body + if validate != nil { + staged, err := p.stageMetadata(ctx, ecosystem, cacheKey, resp.Body, validate) + if err != nil { + return nil, err + } + defer func() { _ = staged.Close() }() + body = staged + } + if _, err := p.storeMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), body, meta); err != nil { + return nil, fmt.Errorf("%w: %w", errMetadataNotStored, err) } stored, err := p.DB.GetMetadataCache(ecosystem, cacheKey) if err != nil || !usableStoredMetadata(stored) { @@ -176,6 +196,45 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, return stored, nil } +// stageMetadata stores body under a staging path of its own, checks it with +// validate, and returns it opened for reading, to be stored in place of the +// cached copy. The staged copy is deleted when the reader is closed. A +// failure to store it is returned as errMetadataNotStored. +func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, body io.Reader, validate func(io.Reader) error) (io.ReadCloser, error) { + path := metadataStoragePath(ecosystem, cacheKey) + ".staged-" + rand.Text() + discard := func() { + if err := p.Storage.Delete(context.WithoutCancel(ctx), path); err != nil { + p.Logger.Warn("failed to delete staged metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) + } + } + if _, _, err := p.Storage.Store(ctx, path, body); err != nil { + discard() + return nil, fmt.Errorf("%w: staging: %w", errMetadataNotStored, err) + } + if err := readStored(func() (io.ReadCloser, error) { return p.Storage.Open(ctx, path) }, validate); err != nil { + discard() + return nil, err + } + staged, err := p.Storage.Open(ctx, path) + if err != nil { + discard() + return nil, fmt.Errorf("%w: reading staged copy: %w", errMetadataNotStored, err) + } + return &stagedReader{ReadCloser: staged, discard: discard}, nil +} + +// stagedReader deletes the staged copy it reads when closed. +type stagedReader struct { + io.ReadCloser + discard func() +} + +func (s *stagedReader) Close() error { + err := s.ReadCloser.Close() + s.discard() + return err +} + // openStoredMetadata opens the document entry describes. Reading it to the // end fails with errStoredMetadataChanged when its bytes don't have the // digest entry records. diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 99c12ae..baddd97 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -2,6 +2,7 @@ package handler import ( "bytes" + "context" "errors" "fmt" "io" @@ -12,6 +13,7 @@ import ( "time" "github.com/git-pkgs/integrity" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/jsonscan" "github.com/git-pkgs/proxy/internal/metadata/npm" "github.com/git-pkgs/proxy/internal/retention" @@ -329,19 +331,13 @@ func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Reque if h.proxy.rewrites == nil { return false } - entry, err := h.proxy.storedMetadataDocument(r.Context(), "npm", metadataCacheKey, upstreamURL, accept) + entry, err := h.storedPackument(r.Context(), upstreamURL, accept, metadataCacheKey) switch { case err == nil: - case errors.Is(err, errMetadataNotStored): + case errors.Is(err, errInMemory): return false - case errors.Is(err, ErrUpstreamNotFound): - JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) - return true - case writeUpstreamHeld(w, err): - return true default: - h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) - JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + h.writeFetchError(w, err) return true } @@ -378,6 +374,109 @@ func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Reque return true } +// npmFetchError is a failure to fetch a packument, answered as such rather +// than as a failure to filter it. +type npmFetchError struct{ err error } + +func (e *npmFetchError) Error() string { return e.err.Error() } +func (e *npmFetchError) Unwrap() error { return e.err } + +// writeFetchError answers a request whose packument could not be fetched. +func (h *NPMHandler) writeFetchError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, ErrUpstreamNotFound): + JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) + case writeUpstreamHeld(w, err): + default: + h.proxy.Logger.Error("failed to fetch npm metadata", "error", err) + JSONError(w, http.StatusBadGateway, "failed to fetch from upstream") + } +} + +// errInMemory is returned when a packument has to be read into memory to +// answer the request: metadata is not cached in storage, or the stored +// document can't be read as a stream. +var errInMemory = errors.New("npm packument needs the in-memory path") + +// storedPackument returns the cache row of the packument stored for cacheKey, +// fetching it into storage first when needed. It returns errInMemory when +// metadata isn't cached in storage, and ErrUpstreamNotFound and upstream +// errors as they come. +func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string) (*database.MetadataCacheEntry, error) { + entry, err := h.proxy.storedMetadataDocument(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator()) + if errors.Is(err, errMetadataNotStored) { + return nil, errInMemory + } + return entry, err +} + +// packumentStreamValidator is packumentValidator for a packument read from +// storage. +func (h *NPMHandler) packumentStreamValidator() func(io.Reader) error { + if h.packumentValidator() == nil { + return nil + } + return func(r io.Reader) error { + entry, err := npm.LookupVersion(r, "") + if err != nil { + return err + } + if entry.Abbreviated { + return errAbbreviatedPackument + } + return nil + } +} + +// streamedVersionMetadata resolves a version or dist-tag through the +// packument stored in storage, as rewriteNPMVersionFromPackument does in +// memory: one pass collects what cooldown and the denylist decide on, and a +// second finds the version entry. It returns errInMemory when the request +// has to take the in-memory path. +func (h *NPMHandler) streamedVersionMetadata(ctx context.Context, packageName, specifier, packumentURL, accept, cacheKey, metadataCacheKey string) (rewrittenMetadata, error) { + if h.proxy.rewrites == nil { + return rewrittenMetadata{}, errInMemory + } + entry, err := h.storedPackument(ctx, packumentURL, accept, cacheKey) + if errors.Is(err, errInMemory) { + return rewrittenMetadata{}, err + } + if err != nil { + return rewrittenMetadata{}, &npmFetchError{err} + } + fullRequired := h.packumentValidator() != nil + options := h.npmRewriteOptions(packageName, nil) + m, err := h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { + var scan *npm.Scan + err := readStored(open, func(in io.Reader) (err error) { + scan, err = npm.ScanPackument(in) + return err + }) + if err != nil { + return err + } + if fullRequired && scan.Abbreviated() { + return npm.ErrNotStreamable + } + return readStored(open, func(in io.Reader) error { + version, err := npm.VersionStream(packageName, specifier, scan, in, options) + if err != nil { + return npmRewriteError(err) + } + _, err = out.Write(version) + return err + }) + }) + switch { + case err == nil, ctx.Err() != nil, errors.Is(err, ErrVersionDenied), errors.Is(err, ErrVersionNotFound): + return m, err + case !errors.Is(err, npm.ErrNotStreamable): + h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", + "package", packageName, "error", err) + } + return rewrittenMetadata{}, errInMemory +} + // readStored opens a stored document and reads it with read. func readStored(open func() (io.ReadCloser, error), read func(io.Reader) error) error { in, err := open() @@ -442,7 +541,19 @@ func requireFullPackument(body []byte) error { // filtered packument so cooldown and dist-tags match the root metadata response. func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r *http.Request, packageName, specifier, accept, registryURL, cacheKey, metadataCacheKey string) { _, packumentPath := npmMetadataPath(packageName) - packument, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, registryURL+"/"+packumentPath, "", h.packumentValidator(), accept) + packumentURL := registryURL + "/" + packumentPath + rewritten, err := h.streamedVersionMetadata(r.Context(), packageName, specifier, packumentURL, accept, cacheKey, metadataCacheKey) + var fetchErr *npmFetchError + switch { + case errors.As(err, &fetchErr): + h.writeFetchError(w, fetchErr.err) + return + case !errors.Is(err, errInMemory): + h.writeVersionMetadata(w, r, packageName, rewritten, err) + return + } + + packument, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) @@ -456,10 +567,16 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * return } - rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { + rewritten, err = h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, packument, "", func(b []byte) ([]byte, []string, error) { out, err := h.rewriteNPMVersionFromPackument(packageName, specifier, b) return out, nil, err }) + h.writeVersionMetadata(w, r, packageName, rewritten, err) +} + +// writeVersionMetadata answers a version metadata request with its rewrite +// or the error that kept it from being made. +func (h *NPMHandler) writeVersionMetadata(w http.ResponseWriter, r *http.Request, packageName string, rewritten rewrittenMetadata, err error) { if err != nil { if r.Context().Err() != nil { return @@ -620,10 +737,8 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } registryURL, cacheKey := h.sourceFor(packageName) - metadata := sync.OnceValues(func() ([]byte, error) { - upstreamURL := registryURL + "/" + url.PathEscape(packageName) - body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, upstreamURL, "", h.packumentValidator(), h.packumentAccept()) - return body, err + metadata := sync.OnceValues(func() (npmVersionData, error) { + return h.versionData(r.Context(), registryURL+"/"+url.PathEscape(packageName), cacheKey, version) }) inCooldown, err := h.versionInCooldown(packageName, version, metadata) if err != nil { @@ -665,7 +780,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifactRequest(w, r, result) } -func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, version, filename string, metadata func() ([]byte, error)) (*CacheResult, error) { +func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, version, filename string, metadata func() (npmVersionData, error)) (*CacheResult, error) { cacheFilename := h.routes.cacheFilename(packageName, filename) if cached, err := h.proxy.GetCachedArtifact(r.Context(), "npm", packageName, version, cacheFilename); err != nil || cached != nil { return cached, err @@ -673,35 +788,35 @@ func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, versi // The packument says where the tarball is and what it must hash to. // Without it the tarball could not be verified, and an unverified copy // would be cached and served to everyone, so the download fails instead. - body, err := metadata() + data, err := metadata() if err != nil { return nil, fmt.Errorf("fetching npm metadata to verify tarball: %w", err) } downloadURL := fmt.Sprintf("%s/%s/-/%s", registryURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) - if tarball := npmVersionTarball(body, version); tarball != "" { + if tarball := npmEntryTarball(data.entry); tarball != "" { downloadURL, err = validateTarballURL(tarball, registryURL) if err != nil { return nil, err } } - sri := npmVersionIntegrity(body, version) + sri := npmEntryIntegrity(data.entry) return h.proxy.GetOrFetchArtifactFromURLWithIntegrity(r.Context(), "npm", packageName, version, cacheFilename, downloadURL, sri) } -// npmVersionIntegrity returns the integrity a packument declares for a -// version's tarball: dist.integrity when it is a Subresource Integrity string +// npmEntryIntegrity returns the integrity a version's packument entry +// declares for its tarball: dist.integrity when it is a Subresource Integrity string // the proxy can verify, or else the SHA-1 in dist.shasum, which is all // versions published before integrity existed carry, as "sha1-". It // returns "" when neither is usable, and the tarball is fetched unverified as // before. -func npmVersionIntegrity(body []byte, version string) string { - sri, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "integrity") +func npmEntryIntegrity(entry []byte) string { + sri, _, _ := jsonscan.LookupString(entry, "dist", "integrity") if sri != "" { if _, err := integrity.ParseSRI(sri); err == nil { return sri } } - shasum, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "shasum") + shasum, _, _ := jsonscan.LookupString(entry, "dist", "shasum") return sha1Checksum(shasum) } @@ -709,10 +824,57 @@ func npmVersionIntegrity(body []byte, version string) string { // or "" if it has none. It reads that one value in place rather than decoding // every version, since it runs on each tarball the proxy has not cached yet. func npmVersionTarball(body []byte, version string) string { - tarball, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "tarball") + entry, _ := jsonscan.Lookup(body, "versions", version) + return npmEntryTarball(entry) +} + +// npmEntryTarball is npmVersionTarball for the version's entry. +func npmEntryTarball(entry []byte) string { + tarball, _, _ := jsonscan.LookupString(entry, "dist", "tarball") return tarball } +// npmVersionData is what a tarball download reads from the packument: the +// version's entry, nil when it has none, and its publish time from the time +// map, "" when it has none. +type npmVersionData struct { + entry []byte + published string +} + +// versionData reads what a tarball download needs about version from the +// packument: from storage in one pass when metadata is cached there, and +// otherwise from the packument read into memory. +func (h *NPMHandler) versionData(ctx context.Context, packumentURL, cacheKey, version string) (npmVersionData, error) { + accept := h.packumentAccept() + if entry, err := h.storedPackument(ctx, packumentURL, accept, cacheKey); err == nil { + var found npm.VersionEntry + err := readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { + found, err = npm.LookupVersion(in, version) + return err + }) + switch { + case err == nil && (!found.Abbreviated || h.packumentValidator() == nil): + return npmVersionData{entry: found.Entry, published: found.Published}, nil + case err != nil: + h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) + } + } else if !errors.Is(err, errInMemory) { + return npmVersionData{}, err + } + + body, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) + if err != nil { + return npmVersionData{}, err + } + entry, _ := jsonscan.Lookup(body, "versions", version) + published, _, err := jsonscan.LookupString(body, "time", version) + if err != nil { + return npmVersionData{}, fmt.Errorf("parsing npm metadata: %w", err) + } + return npmVersionData{entry: entry, published: published}, nil +} + func validateTarballURL(raw, registryURL string) (string, error) { tarball, err := url.Parse(raw) if err != nil { @@ -756,7 +918,7 @@ func validateTarballURL(raw, registryURL string) (string, error) { // for a private version of the same name, a private time would hold back the // public version, and moving a route to another upstream would keep the old // one's times. -func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() ([]byte, error)) (bool, error) { +func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() (npmVersionData, error)) (bool, error) { if !h.proxy.cooldownApplies("npm", packageName) { return false, nil } @@ -769,7 +931,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun } } - body, err := metadata() + data, err := metadata() if errors.Is(err, ErrUpstreamNotFound) { return false, nil } @@ -777,11 +939,8 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun return false, err } - published, ok, err := jsonscan.LookupString(body, "time", version) - if err != nil { - return false, fmt.Errorf("parsing npm metadata: %w", err) - } - if !ok { + published := data.published + if published == "" { return false, nil } diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go index 565f062..3442ce2 100644 --- a/internal/handler/npm_stream_test.go +++ b/internal/handler/npm_stream_test.go @@ -16,12 +16,18 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" ) // bigPackument returns a packument laid out as registry.npmjs.org writes // them, dist-tags before versions and time after, with n old versions padded // so the document is large, and a latest published an hour ago. func bigPackument(n int) string { + return bigPackumentAt("https://registry.npmjs.org", n) +} + +// bigPackumentAt is bigPackument with tarballs under base. +func bigPackumentAt(base string, n int) string { var b strings.Builder b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"9.0.0","old":"1.0.0"},"versions":{`) times := []string{`"created":"2015-01-01T00:00:00Z"`} @@ -30,12 +36,12 @@ func bigPackument(n int) string { if i == 0 { v = "1.0.0" } - fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"readme":%q,"dist":{"tarball":"https://registry.npmjs.org/big/-/big-%s.tgz"}},`, - v, v, strings.Repeat("x", 512), v) + fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"readme":%q,"dist":{"tarball":"%s/big/-/big-%s.tgz"}},`, + v, v, strings.Repeat("x", 512), base, v) times = append(times, fmt.Sprintf(`%q:"2016-01-01T00:%02d:%02dZ"`, v, i/60%60, i%60)) } recent := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) - b.WriteString(`"9.0.0":{"name":"big","version":"9.0.0","dist":{"tarball":"https://registry.npmjs.org/big/-/big-9.0.0.tgz"}}},`) + b.WriteString(`"9.0.0":{"name":"big","version":"9.0.0","dist":{"tarball":"` + base + `/big/-/big-9.0.0.tgz"}}},`) times = append(times, `"9.0.0":"`+recent+`"`) b.WriteString(`"time":{` + strings.Join(times, ",") + `},"readme":"big"}`) return b.String() @@ -200,7 +206,7 @@ func TestNPMStreamedPackumentStale(t *testing.T) { func TestNPMStreamedPackumentChangedInStorage(t *testing.T) { u := newStreamUpstream(t, bigPackument(10)) proxy, store, _ := streamProxy(t, u) - entry, err := proxy.storedMetadataDocument(context.Background(), "npm", "big", u.URL+"/big", contentTypeJSON) + entry, err := proxy.storedMetadataDocument(context.Background(), "npm", "big", u.URL+"/big", contentTypeJSON, nil) if err != nil { t.Fatal(err) } @@ -225,6 +231,9 @@ func TestNPMStreamedPackumentRevalidates(t *testing.T) { proxy, store, handler := streamProxy(t, u) first := serve(handler, "/big", nil) decodeStreamed(t, first) + store.mu.Lock() + stores := store.stores + store.mu.Unlock() proxy.MetadataTTL = time.Nanosecond second := serve(handler, "/big", nil) @@ -237,7 +246,105 @@ func TestNPMStreamedPackumentRevalidates(t *testing.T) { } store.mu.Lock() defer store.mu.Unlock() - if store.stores != 1 { - t.Errorf("document stored %d times, want once", store.stores) + if store.stores != stores { + t.Errorf("the 304 stored the document again (%d stores, %d before)", store.stores, stores) + } +} + +// TestNPMStreamedPackumentKeepsFullCopy checks that an abbreviated packument +// from upstream, where cooldown needs the full one, doesn't replace the full +// copy in storage, which is served instead as when upstream fails. +func TestNPMStreamedPackumentKeepsFullCopy(t *testing.T) { + full := bigPackument(10) + u := newStreamUpstream(t, full) + proxy, store, handler := streamProxy(t, u) + decodeStreamed(t, serve(handler, "/big", nil)) + + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) + u.body.Store(`{"name":"big","modified":"2026-01-01T00:00:00Z","dist-tags":{"latest":"1.0.0"},"versions":{"1.0.0":{}}}`) + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { + t.Errorf("served %d versions, want the 10 of the full copy", len(doc.Versions)) + } + + store.mu.Lock() + defer store.mu.Unlock() + if got := string(store.files[metadataStoragePath("npm", "big")]); got != full { + t.Errorf("stored packument was replaced: %.80s", got) + } + for path := range store.files { + if strings.Contains(path, ".staged-") { + t.Errorf("staged copy %s left in storage", path) + } + } +} + +// TestNPMStreamedPackumentStorageFails checks that a packument is still +// served, read into memory, when storing it fails. +func TestNPMStreamedPackumentStorageFails(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + _, store, handler := streamProxy(t, u) + store.storeErr = errors.New("no space left on device") + doc := decodeStreamed(t, serve(handler, "/big", nil)) + if len(doc.Versions) != 10 || doc.DistTags["latest"] != "1.0.9" { + t.Errorf("served %d versions, latest %q; want 10 and 1.0.9", len(doc.Versions), doc.DistTags["latest"]) + } +} + +// TestNPMStreamedVersionMetadata resolves versions and dist-tags of a +// packument larger than metadata_max_size. +func TestNPMStreamedVersionMetadata(t *testing.T) { + body := bigPackument(100) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + + for path, want := range map[string]string{ + "/big/1.0.5": "1.0.5", + "/big/latest": "1.0.99", // 9.0.0 is inside cooldown + "/big/old": "1.0.0", + } { + rec := serve(handler, path, nil) + var v struct { + Version string `json:"version"` + Dist struct { + Tarball string `json:"tarball"` + } `json:"dist"` + } + if rec.Code != http.StatusOK || json.Unmarshal(rec.Body.Bytes(), &v) != nil { + t.Errorf("%s: status %d: %.200s", path, rec.Code, rec.Body.String()) + continue + } + if v.Version != want || v.Dist.Tarball != "http://proxy.example/npm/big/-/big-"+want+".tgz" { + t.Errorf("%s = %s with tarball %s, want %s", path, v.Version, v.Dist.Tarball, want) + } + } + if rec := serve(handler, "/big/9.0.0", nil); rec.Code != http.StatusNotFound { + t.Errorf("version inside cooldown: status %d, want 404", rec.Code) + } + if n := u.requests.Load(); n != 1 { + t.Errorf("upstream asked %d times, want once", n) + } +} + +// TestNPMStreamedTarballCooldown checks the tarball download against a +// packument larger than metadata_max_size: the version's publish time and +// tarball URL are read from storage. +func TestNPMStreamedTarballCooldown(t *testing.T) { + u := newStreamUpstream(t, "") + u.body.Store(bigPackumentAt(u.URL, 100)) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = 1024 + fetcher := proxy.Fetcher.(*mockFetcher) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} + + if rec := serve(handler, "/big/-/big-9.0.0.tgz", nil); rec.Code != http.StatusNotFound { + t.Errorf("tarball inside cooldown: status %d, want 404", rec.Code) + } + if rec := serve(handler, "/big/-/big-1.0.5.tgz", nil); rec.Code != http.StatusOK || rec.Body.String() != "package" { + t.Errorf("tarball: status %d, body %q", rec.Code, rec.Body.String()) + } + if want := u.URL + "/big/-/big-1.0.5.tgz"; fetcher.fetchedURL != want { + t.Errorf("fetched %s, want %s from the packument", fetcher.fetchedURL, want) } } diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index 20fda4e..9b9a6c3 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -561,9 +561,9 @@ func TestNPMDownloadCooldownDisabled(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) h := NewNPMHandler(proxy, "http://proxy.test", "") - if inCooldown, _ := h.versionInCooldown("leftpad", testVersion100, func() ([]byte, error) { + if inCooldown, _ := h.versionInCooldown("leftpad", testVersion100, func() (npmVersionData, error) { t.Fatal("cooldown must not request metadata when disabled") - return nil, nil + return npmVersionData{}, nil }); inCooldown { t.Error("versionInCooldown = true, want false when cooldown is not configured") } diff --git a/internal/metadata/npm/stream.go b/internal/metadata/npm/stream.go index cc70c52..3e86faa 100644 --- a/internal/metadata/npm/stream.go +++ b/internal/metadata/npm/stream.go @@ -99,50 +99,63 @@ func RewriteStream(packageName string, scan *Scan, r io.Reader, w io.Writer, opt return nil, ErrNotStreamable } h := &transformer{options} - - versions := make(map[string]any, len(scan.versions)) - for _, v := range scan.versions { - versions[v] = nil - } - metadata := map[string]any{} - timeMap := scan.objectValue(metadata, "time", scan.time) - distTags := scan.objectValue(metadata, "dist-tags", scan.distTags) - timeEntries := len(timeMap) - originalTags := maps.Clone(distTags) - - h.applyCooldownFiltering(metadata, versions, packageName) - h.applyDenylistFiltering(metadata, versions) - + f := scan.filter(h, packageName) sw := &streamWriter{ transformer: h, packageName: packageName, scan: scan, - versions: versions, - timeMap: timeMap, - distTags: distTags, - timeChanged: timeMap != nil && len(timeMap) != timeEntries, - tagsChanged: distTags != nil && !reflect.DeepEqual(distTags, originalTags), + filtered: f, seen: map[string]int{}, } if err := jsonstream.CopyObject(json.NewDecoder(r), w, sw.member); err != nil { return nil, err } h.tarballsRewritten(sw.rewritten) - return slices.Collect(maps.Keys(versions)), nil + return slices.Collect(maps.Keys(f.versions)), nil +} + +// filtered is what cooldown and the denylist leave of a scanned packument. +type filtered struct { + versions map[string]any + timeMap, distTags map[string]any + timeChanged, tagsChanged bool +} + +// filter runs the filters Rewrite runs over what scan collected. The scan is +// left as it is, so it can be filtered again. +func (s *Scan) filter(h *transformer, packageName string) filtered { + versions := make(map[string]any, len(s.versions)) + for _, v := range s.versions { + versions[v] = nil + } + metadata := map[string]any{} + timeMap := s.objectValue(metadata, "time", s.time) + distTags := s.objectValue(metadata, "dist-tags", s.distTags) + timeEntries := len(timeMap) + originalTags := maps.Clone(distTags) + + h.applyCooldownFiltering(metadata, versions, packageName) + h.applyDenylistFiltering(metadata, versions) + + return filtered{ + versions: versions, + timeMap: timeMap, + distTags: distTags, + timeChanged: timeMap != nil && len(timeMap) != timeEntries, + tagsChanged: distTags != nil && !reflect.DeepEqual(distTags, originalTags), + } } // streamWriter writes the second pass of RewriteStream from the filtered // state the first pass left. type streamWriter struct { *transformer - packageName string - scan *Scan - versions map[string]any - timeMap, distTags map[string]any - timeChanged, tagsChanged bool - seen map[string]int - entry bytes.Buffer - rewritten int + filtered + packageName string + scan *Scan + seen map[string]int + entry bytes.Buffer + rewritten int } // member writes one top-level member. Members the filters don't touch are @@ -201,13 +214,19 @@ func (s *streamWriter) time(t *jsonstream.Member) error { } // objectValue records value under name in metadata, as Rewrite's -// decodeObject does, and returns it when it is an object. +// decodeObject does, and returns it when it is an object. An object is +// copied, since the filters edit it. func (s *Scan) objectValue(metadata map[string]any, name string, value any) map[string]any { if s.occurrences[name] == 0 { return nil } - metadata[name] = value - object, _ := value.(map[string]any) + object, ok := value.(map[string]any) + if !ok { + metadata[name] = value + return nil + } + object = maps.Clone(object) + metadata[name] = object return object } @@ -268,3 +287,126 @@ func skipValue(dec *json.Decoder) error { var raw json.RawMessage return dec.Decode(&raw) } + +// VersionStream is Version for a packument too large to hold in memory. scan +// is the result of ScanPackument over the bytes r reads again; the second +// pass only looks for the one version entry it serves. For a document +// without a versions object it returns ErrNotStreamable. +func VersionStream(packageName, specifier string, scan *Scan, r io.Reader, options Options) ([]byte, error) { + if !scan.versionsObject || len(options.LocalVersions) > 0 { + return nil, ErrNotStreamable + } + h := &transformer{options} + f := scan.filter(h, packageName) + + target := specifier + if tag, ok := f.distTags[specifier].(string); ok { + target = tag + } + if options.DeniedVersions[target] { + return nil, ErrVersionDenied + } + if _, kept := f.versions[target]; !kept { + return nil, ErrVersionNotFound + } + entry, err := LookupVersion(r, target) + if err != nil { + return nil, err + } + if entry.Entry == nil { + return nil, ErrVersionNotFound + } + var out bytes.Buffer + h.writeNPMVersion(&out, packageName, target, entry.Entry) + return out.Bytes(), nil +} + +// VersionEntry is what a packument holds for one version. +type VersionEntry struct { + // Entry is versions[version], or nil when there is none. + Entry json.RawMessage + // Published is time[version], or "" when it is missing or not a string. + Published string + // Abbreviated is set for an abbreviated packument (see Abbreviated). + Abbreviated bool +} + +// LookupVersion reads a packument once and returns what it holds for +// version, keeping no other version's entry. Of repeated keys the last one +// counts, as with JSON.parse. +func LookupVersion(r io.Reader, version string) (VersionEntry, error) { + var e VersionEntry + var hasTime, hasModified bool + dec := json.NewDecoder(r) + if err := expectDelim(dec, '{'); err != nil { + return e, err + } + for dec.More() { + key, err := objectKey(dec) + if err != nil { + return e, err + } + switch key { + case "versions": + e.Entry, err = objectMember(dec, version) + case "time": + hasTime = true + var raw json.RawMessage + raw, err = objectMember(dec, version) + e.Published = "" + if err == nil && raw != nil { + _ = json.Unmarshal(raw, &e.Published) + } + default: + hasModified = hasModified || key == "modified" + err = skipValue(dec) + } + if err != nil { + return e, err + } + } + if err := expectDelim(dec, '}'); err != nil { + return e, err + } + if _, err := dec.Token(); err != io.EOF { + if err == nil { + err = errors.New("unexpected data after the top-level object") + } + return e, err + } + e.Abbreviated = hasModified && !hasTime + return e, nil +} + +// objectMember reads a value and returns its member key when it is an object +// that has one, the last if repeated, or nil. +func objectMember(dec *json.Decoder, key string) (json.RawMessage, error) { + tok, err := dec.Token() + if err != nil { + return nil, err + } + delim, ok := tok.(json.Delim) + if !ok { + return nil, nil + } + var found json.RawMessage + for dec.More() { + var k string + if delim == '{' { + if k, err = objectKey(dec); err != nil { + return nil, err + } + } + var raw json.RawMessage + if err := dec.Decode(&raw); err != nil { + return nil, err + } + if delim == '{' && k == key { + found = raw + } + } + if _, err := dec.Token(); err != nil { + return nil, err + } + return found, nil +} diff --git a/internal/metadata/npm/stream_test.go b/internal/metadata/npm/stream_test.go index ef2c954..0c8a4ce 100644 --- a/internal/metadata/npm/stream_test.go +++ b/internal/metadata/npm/stream_test.go @@ -24,11 +24,10 @@ func rewriteStream(packageName string, body []byte, options Options) ([]byte, [] return out.Bytes(), kept, err } -// TestRewriteStreamMatchesRewrite checks that the streamed rewrite writes the -// same bytes and keeps the same versions as Rewrite, over documents with the -// filtered members in either order, duplicates, and values Rewrite copies -// verbatim. -func TestRewriteStreamMatchesRewrite(t *testing.T) { +// streamCases returns packuments with the filtered members in either +// order, duplicates and values Rewrite copies verbatim, and the options to +// rewrite them with. +func streamCases() (map[string]string, map[string]Options) { now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) ts := func(age time.Duration) string { return now.Add(-age).Format(time.RFC3339) } version := func(v string) string { @@ -60,6 +59,15 @@ func TestRewriteStreamMatchesRewrite(t *testing.T) { "both": {ProxyURL: "http://proxy", Cooldown: &cooldown.Config{Default: "3d"}, EvaluatedAt: now, DeniedVersions: map[string]bool{"1.0.0": true}}, } + return docs, options +} + +// TestRewriteStreamMatchesRewrite checks that the streamed rewrite writes the +// same bytes and keeps the same versions as Rewrite, over documents with the +// filtered members in either order, duplicates, and values Rewrite copies +// verbatim. +func TestRewriteStreamMatchesRewrite(t *testing.T) { + docs, options := streamCases() for docName, doc := range docs { for optName, opts := range options { t.Run(docName+"/"+optName, func(t *testing.T) { @@ -160,3 +168,58 @@ func TestScanPackument(t *testing.T) { } } } + +// TestVersionStreamMatchesVersion checks that a single version resolved from +// the stream is the one Version serves, with the same errors. +func TestVersionStreamMatchesVersion(t *testing.T) { + docs, options := streamCases() + specifiers := []string{"latest", "beta", "next", "old", "1.0.0", "2.0.0", "3.0.0", "9.0.0", "missing"} + for docName, doc := range docs { + scan, err := ScanPackument(strings.NewReader(doc)) + if err != nil { + t.Fatalf("%s: %v", docName, err) + } + for optName, opts := range options { + for _, spec := range specifiers { + want, wantErr := Version("demo", spec, []byte(doc), opts) + got, gotErr := VersionStream("demo", spec, scan, strings.NewReader(doc), opts) + name := docName + "/" + optName + "/" + spec + switch { + case wantErr != nil || gotErr != nil: + if !errors.Is(gotErr, ErrVersionDenied) == errors.Is(wantErr, ErrVersionDenied) || + !errors.Is(gotErr, ErrVersionNotFound) == errors.Is(wantErr, ErrVersionNotFound) || + (gotErr == nil) != (wantErr == nil) { + t.Errorf("%s: error %v, Version gave %v", name, gotErr, wantErr) + } + case !bytes.Equal(got, want): + t.Errorf("%s:\n%s\nVersion:\n%s", name, got, want) + } + } + } + } +} + +func TestLookupVersion(t *testing.T) { + doc := `{"modified":"x","versions":{"1.0.0":{"a":1}},"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":7},` + + `"versions":{"1.0.0":{"a":2},"2.0.0":{"b":1}}}` + for version, want := range map[string]VersionEntry{ + "1.0.0": {Entry: json.RawMessage(`{"a":2}`), Published: "2020-01-01T00:00:00Z"}, + "2.0.0": {Entry: json.RawMessage(`{"b":1}`)}, + "3.0.0": {}, + } { + got, err := LookupVersion(strings.NewReader(doc), version) + if err != nil { + t.Fatal(err) + } + if string(got.Entry) != string(want.Entry) || got.Published != want.Published || got.Abbreviated { + t.Errorf("LookupVersion(%s) = %+v, want %+v", version, got, want) + } + } + got, err := LookupVersion(strings.NewReader(`{"modified":"x","versions":{}}`), "1.0.0") + if err != nil || !got.Abbreviated { + t.Errorf("abbreviated packument: %+v, %v", got, err) + } + if _, err := LookupVersion(strings.NewReader(`{"versions":{}`), "1.0.0"); err == nil { + t.Error("expected an error for truncated JSON") + } +} From ddba210a60d082fedf2c98bac8dc1bd60a1d6b40 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 15:31:31 +0100 Subject: [PATCH 017/198] Serialize snapshot recording per document on Postgres AddMetadataSnapshot takes a transaction-scoped advisory lock on the document before its check and insert, so two replicas recording the same version can't both read the previous digest and both insert. SQLite runs one write at a time, so it keeps the single statement. --- internal/database/metadata_snapshots.go | 36 ++++++++++-- internal/database/metadata_snapshots_test.go | 52 +++++++++++++++++ internal/handler/apk_snapshots_test.go | 60 ++++++++++++++++++++ 3 files changed, 143 insertions(+), 5 deletions(-) diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go index 95edacc..fba774a 100644 --- a/internal/database/metadata_snapshots.go +++ b/internal/database/metadata_snapshots.go @@ -62,11 +62,25 @@ func (db *DB) metadataSnapshotBefore(ecosystem, name string, beforeMS int64) (*M // AddMetadataSnapshot records snap as the current version of its document, // unless the newest snapshot already has its digest, and reports whether it -// did. The check and the insert are one statement, so concurrent callers -// recording the same document add it once. The casts let Postgres type the -// parameters of the INSERT ... SELECT. +// did. On Postgres, replicas recording the same document take an advisory +// lock on it first, so their check and insert don't interleave: without it, +// two transactions can both read the previous digest and both insert. SQLite +// runs one write at a time, so the single statement is enough there. A +// document going back to an earlier digest is recorded again, since only the +// newest snapshot is compared. The casts let Postgres type the parameters of +// the INSERT ... SELECT. func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { - res, err := db.Exec(db.Rebind(` + tx, err := db.Beginx() + if err != nil { + return false, fmt.Errorf("recording metadata snapshot: %w", err) + } + defer func() { _ = tx.Rollback() }() + if db.dialect == DialectPostgres { + if _, err := tx.Exec("SELECT pg_advisory_xact_lock($1)", metadataSnapshotLockKey(snap.Ecosystem, snap.Name)); err != nil { + return false, fmt.Errorf("locking metadata snapshots: %w", err) + } + } + res, err := tx.Exec(db.Rebind(` INSERT INTO metadata_snapshots (ecosystem, name, storage_path, content_type, content_encoding, content_digest, size, valid_from_ms) SELECT CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), CAST(? AS TEXT), @@ -84,7 +98,19 @@ func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { return false, fmt.Errorf("recording metadata snapshot: %w", err) } n, err := res.RowsAffected() - return n > 0, err + if err != nil { + return false, err + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("recording metadata snapshot: %w", err) + } + return n > 0, nil +} + +// metadataSnapshotLockKey is the advisory lock AddMetadataSnapshot takes for +// a document. +func metadataSnapshotLockKey(ecosystem, name string) int64 { + return lockKey("metadata-snapshot:" + ecosystem + "\x00" + name) } // PruneMetadataSnapshots deletes the snapshots of ecosystem that stopped diff --git a/internal/database/metadata_snapshots_test.go b/internal/database/metadata_snapshots_test.go index ae859fd..0b4919f 100644 --- a/internal/database/metadata_snapshots_test.go +++ b/internal/database/metadata_snapshots_test.go @@ -106,3 +106,55 @@ func TestMetadataSnapshots_Prune(t *testing.T) { } }) } + +// TestMetadataSnapshots_ConcurrentPostgres records a version while another +// replica's transaction has recorded the same one and not committed yet, as +// two replicas can. The second must wait and then find it, so one row +// remains. Without the lock both inserts succeed. +func TestMetadataSnapshots_ConcurrentPostgres(t *testing.T) { + db := createTestPostgresDB(t) + if db == nil { + t.Skip("PROXY_DATABASE_URL not set, skipping postgres test") + } + defer func() { _ = db.Close() }() + const name = "v3.22/main/x86_64/APKINDEX.tar.gz" + + other, err := db.Beginx() + if err != nil { + t.Fatal(err) + } + defer func() { _ = other.Rollback() }() + if _, err := other.Exec("SELECT pg_advisory_xact_lock($1)", metadataSnapshotLockKey("alpine", name)); err != nil { + t.Fatal(err) + } + if _, err := other.Exec(`INSERT INTO metadata_snapshots (ecosystem, name, storage_path, content_digest, size, valid_from_ms) + VALUES ('alpine', $1, 'other', 'sha256:a', 1, $2)`, name, time.Now().UnixMilli()); err != nil { + t.Fatal(err) + } + + done := make(chan bool, 1) + go func() { done <- addSnapshot(t, db, name, "sha256:a", time.Now()) }() + select { + case added := <-done: + t.Fatalf("recorded without waiting for the other replica (added=%v)", added) + case <-time.After(200 * time.Millisecond): + } + if err := other.Commit(); err != nil { + t.Fatal(err) + } + if added := <-done; added { + t.Error("recorded a version the other replica had just recorded") + } + var n int + if err := db.Get(&n, `SELECT COUNT(*) FROM metadata_snapshots WHERE name = $1`, name); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("%d snapshot rows, want 1", n) + } + + // Going back to an earlier digest is still recorded. + if !addSnapshot(t, db, name, "sha256:b", time.Now()) || !addSnapshot(t, db, name, "sha256:a", time.Now().Add(time.Millisecond)) { + t.Error("a change back to an earlier digest was not recorded") + } +} diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go index 52b5ff7..38d4b08 100644 --- a/internal/handler/apk_snapshots_test.go +++ b/internal/handler/apk_snapshots_test.go @@ -3,6 +3,8 @@ package handler import ( "net/http" "net/http/httptest" + "strings" + "sync" "sync/atomic" "testing" "time" @@ -203,3 +205,61 @@ func TestAPKSnapshots_SameVersionRecordedTwice(t *testing.T) { day := validFrom.UTC().Format(apkSnapshotDateLayout) expectAPKResponse(t, h, "/alpine@"+day+apkSnapshotIndexPath, http.StatusOK, "index v1") } + +// TestAPKSnapshots_ConcurrentReplicas records the same index version from +// two replicas sharing a Postgres database, many times at once. One snapshot +// row must remain, with a readable blob, and the other copies are deleted. +func TestAPKSnapshots_ConcurrentReplicas(t *testing.T) { + a, b, db := sharedInstances(t) + u := newAPKSnapshotUpstream(t) + handlers := make([]*APKHandler, 0, 2) + for _, p := range []*Proxy{a, b} { + p.CacheMetadata = true + p.HTTPClient = http.DefaultClient + handlers = append(handlers, NewAPKHandler(p, "http://proxy.example", map[string]string{"alpine": u.server.URL}). + WithSnapshots(30*24*time.Hour)) + } + expectAPKResponse(t, handlers[0], "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + + key := handlers[0].metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) + entry, err := db.GetMetadataCache(apkEcosystem, key) + if err != nil || entry == nil { + t.Fatalf("cached index row = %v, %v", entry, err) + } + if _, err := db.Exec(`DELETE FROM metadata_snapshots`); err != nil { + t.Fatal(err) + } + // The request above recorded a copy of its own, which the test leaves be. + store := a.Storage.(*mockStorage) + store.mu.Lock() + before := map[string]bool{} + for path := range store.files { + before[path] = true + } + store.mu.Unlock() + + validFrom := time.Now() + var wg sync.WaitGroup + for i := range 16 { + wg.Go(func() { handlers[i%2].storeSnapshot(t.Context(), key, entry, validFrom) }) + } + wg.Wait() + + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots recorded, want 1", n) + } + snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) + if err != nil || snap == nil { + t.Fatalf("latest snapshot = %v, %v", snap, err) + } + store.mu.Lock() + defer store.mu.Unlock() + if got := string(store.files[snap.StoragePath]); got != "index v1" { + t.Errorf("snapshot blob = %q, want the index", got) + } + for path := range store.files { + if strings.HasPrefix(path, "_snapshots/") && path != snap.StoragePath && !before[path] { + t.Errorf("copy %s of a snapshot that wasn't recorded is left in storage", path) + } + } +} From ba95de17f30d9068b2fa61b8dad8eb10a76495ad Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 30 Oct 2026 14:41:45 +0000 Subject: [PATCH 018/198] Handle Swift registry identities in cache and vulnerability APIs --- go.mod | 2 +- go.sum | 4 +- internal/enrichment/enrichment.go | 9 ++-- internal/enrichment/enrichment_test.go | 11 +++-- internal/packageurl/packageurl.go | 24 ++++------- internal/packageurl/packageurl_test.go | 25 +++++------ internal/server/api.go | 9 ++++ internal/server/swift_vulns_test.go | 57 ++++++++++++++++++++++++++ 8 files changed, 102 insertions(+), 39 deletions(-) create mode 100644 internal/server/swift_vulns_test.go diff --git a/go.mod b/go.mod index 4a164c2..79978b1 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.0 - github.com/git-pkgs/enrichment v0.7.4 + github.com/git-pkgs/enrichment v0.7.5 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.6.0 diff --git a/go.sum b/go.sum index 983c87e..7ed7c29 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= -github.com/git-pkgs/enrichment v0.7.4 h1:owiCKjITG8tNxBikcyUCDuDf1i3C/+Yoa5yJseuqIjA= -github.com/git-pkgs/enrichment v0.7.4/go.mod h1:ZN/cH4PnmzsapEVPvmAwREP7+ox+RX0yfR6KvoB8xKE= +github.com/git-pkgs/enrichment v0.7.5 h1:jbfFzf7+HSS7el8AuMri4vHXq2izIIZDVIU0uYI5LFM= +github.com/git-pkgs/enrichment v0.7.5/go.mod h1:TbxC/nGQN96YCbdoiDs8a9z3gk2zXmLVS1989/jvnsU= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.2.0 h1:sxpAk7EUllZnTYttOoPTW/Yfe5J2imglpuJiuBUZOxM= diff --git a/internal/enrichment/enrichment.go b/internal/enrichment/enrichment.go index 4d847af..2246e39 100644 --- a/internal/enrichment/enrichment.go +++ b/internal/enrichment/enrichment.go @@ -100,9 +100,6 @@ func (s *Service) BulkEnrichPackages(ctx context.Context, packages []struct{ Eco // CheckVulnerabilities queries for vulnerabilities affecting a package version. func (s *Service) CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]VulnInfo, error) { - if packageurl.Make(ecosystem, name, version) == nil { - return nil, nil - } return s.vulns.Check(ctx, ecosystem, name, version) } @@ -137,7 +134,8 @@ type EnrichmentResult struct { } // EnrichFull fetches package, version and vulnerability data for a package -// version concurrently. Failed lookups are logged and left empty. +// version concurrently. Unsupported vulnerability identities return an error; +// other failed lookups are logged and left empty. func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version string) (*EnrichmentResult, error) { result := &EnrichmentResult{} @@ -147,6 +145,9 @@ func (s *Service) EnrichFull(ctx context.Context, ecosystem, name, version strin wg.Go(func() { result.Version, verErr = s.EnrichVersion(ctx, ecosystem, name, version) }) wg.Go(func() { result.Vulnerabilities, vulnErr = s.CheckVulnerabilities(ctx, ecosystem, name, version) }) wg.Wait() + if errors.Is(vulnErr, shared.ErrUnsupportedVulnerabilityIdentity) { + return nil, vulnErr + } if pkgErr != nil { s.logger.Debug("failed to enrich package", "ecosystem", ecosystem, "name", name, "error", pkgErr) diff --git a/internal/enrichment/enrichment_test.go b/internal/enrichment/enrichment_test.go index a854e5a..f01bb29 100644 --- a/internal/enrichment/enrichment_test.go +++ b/internal/enrichment/enrichment_test.go @@ -2,10 +2,12 @@ package enrichment import ( "context" + "errors" "log/slog" "os" "testing" + shared "github.com/git-pkgs/enrichment" "github.com/git-pkgs/registries" ) @@ -26,7 +28,7 @@ func TestNew(t *testing.T) { } } -func TestSwiftRegistryIdentitySkipsPURLDependentLookups(t *testing.T) { +func TestSwiftRegistryIdentityLookups(t *testing.T) { svc := New(slog.New(slog.NewTextHandler(os.Stdout, nil))) ctx := context.Background() @@ -41,8 +43,11 @@ func TestSwiftRegistryIdentitySkipsPURLDependentLookups(t *testing.T) { } vulnerabilities, err := svc.CheckVulnerabilities(ctx, "swift", "apple/example", "1.2.3") - if err != nil || vulnerabilities != nil { - t.Errorf("CheckVulnerabilities() = %#v, %v; want nil, nil", vulnerabilities, err) + if !errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) || vulnerabilities != nil { + t.Errorf("CheckVulnerabilities() = %#v, %v; want unsupported identity", vulnerabilities, err) + } + if _, err := svc.EnrichFull(ctx, "swift", "apple/example", "1.2.3"); !errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + t.Errorf("EnrichFull error = %v, want unsupported identity", err) } latest, err := svc.GetLatestVersion(ctx, "swift", "apple/example") diff --git a/internal/packageurl/packageurl.go b/internal/packageurl/packageurl.go index 7110942..be0e9ab 100644 --- a/internal/packageurl/packageurl.go +++ b/internal/packageurl/packageurl.go @@ -29,25 +29,19 @@ func WithVersionString(packagePURL, version string) string { } // MakeCacheStrings returns package and version PURLs suitable for artifact -// cache records. Swift registry identities use an explicit generic PURL until -// their source repository has been resolved. The result is independent of the -// configured upstream so cache entries survive an upstream.swift change, -// matching every other ecosystem. +// cache records. Swift registry identities keep their generic storage PURLs. +// Cache keys are independent of the configured upstream. func MakeCacheStrings(ecosystem, name, version string) (packagePURL, versionPURL string) { + if purl.NormalizeEcosystem(ecosystem) == "swift" { + if identity, ok := swiftRegistryIdentity(name); ok { + pkg := purl.New("generic", "swift-registry", identity, "", nil) + return pkg.String(), pkg.WithVersion(version).String() + } + } if pkg := Make(ecosystem, name, ""); pkg != nil { return pkg.String(), pkg.WithVersion(version).String() } - if purl.NormalizeEcosystem(ecosystem) != "swift" { - return "", "" - } - - identity, ok := swiftRegistryIdentity(name) - if !ok { - return "", "" - } - - pkg := purl.New("generic", "swift-registry", identity, "", nil) - return pkg.String(), pkg.WithVersion(version).String() + return "", "" } func swiftRegistryIdentity(name string) (string, bool) { diff --git a/internal/packageurl/packageurl_test.go b/internal/packageurl/packageurl_test.go index 8982288..5a46776 100644 --- a/internal/packageurl/packageurl_test.go +++ b/internal/packageurl/packageurl_test.go @@ -2,17 +2,12 @@ package packageurl import "testing" -func TestMakeSwiftRegistryIdentityUnsupported(t *testing.T) { - identities := []string{"apple.swift-argument-parser", "apple/swift-argument-parser"} - for _, identity := range identities { - t.Run(identity, func(t *testing.T) { - if got := Make("swift", identity, "1.8.2"); got != nil { - t.Errorf("Make() = %q, want nil", got.String()) - } - if got := MakeString("swift", identity, "1.8.2"); got != "" { - t.Errorf("MakeString() = %q, want empty string", got) - } - }) +func TestMakeSwiftRegistryPathUnsupported(t *testing.T) { + if got := Make("swift", "apple/swift-argument-parser", "1.8.2"); got != nil { + t.Errorf("Make() = %q, want nil", got.String()) + } + if got := MakeString("swift", "apple/swift-argument-parser", "1.8.2"); got != "" { + t.Errorf("MakeString() = %q, want empty string", got) } } @@ -49,9 +44,11 @@ func TestMakeCacheStringsSwiftRegistryIdentity(t *testing.T) { t.Errorf("version PURL = %q, want %q", versionPURL, wantVersion) } - dottedPackage, dottedVersion := MakeCacheStrings("swift", "apple.example", "1.2.3") - if dottedPackage != packagePURL || dottedVersion != versionPURL { - t.Errorf("dotted identity cache PURLs = %q, %q; want %q, %q", dottedPackage, dottedVersion, packagePURL, versionPURL) + for _, identity := range []string{"apple.example", "APPLE.EXAMPLE", "apple/example"} { + dottedPackage, dottedVersion := MakeCacheStrings("Swift", identity, "1.2.3") + if dottedPackage != packagePURL || dottedVersion != versionPURL { + t.Errorf("%q cache PURLs = %q, %q; want %q, %q", identity, dottedPackage, dottedVersion, packagePURL, versionPURL) + } } } diff --git a/internal/server/api.go b/internal/server/api.go index b204367..22bfb8a 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -2,6 +2,7 @@ package server import ( "encoding/json" + "errors" "net/http" "strings" @@ -219,6 +220,10 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) { result, err := h.enrichment.EnrichFull(r.Context(), ecosystem, name, version) + if errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + badRequest(w, "package identity is not supported for vulnerability lookups") + return + } if err != nil { writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to enrich version") return @@ -303,6 +308,10 @@ func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) { } vulns, err := h.enrichment.CheckVulnerabilities(r.Context(), ecosystem, name, version) + if errors.Is(err, shared.ErrUnsupportedVulnerabilityIdentity) { + badRequest(w, "package identity is not supported for vulnerability lookups") + return + } if err != nil { writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to check vulnerabilities") return diff --git a/internal/server/swift_vulns_test.go b/internal/server/swift_vulns_test.go new file mode 100644 index 0000000..f28caaf --- /dev/null +++ b/internal/server/swift_vulns_test.go @@ -0,0 +1,57 @@ +package server + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/git-pkgs/proxy/internal/enrichment" + "github.com/go-chi/chi/v5" +) + +func TestSwiftRegistryVulnerabilityErrors(t *testing.T) { + h := NewAPIHandler(enrichment.New(slog.Default()), nil) + r := chi.NewRouter() + r.Get("/api/vulns/{ecosystem}/*", h.HandleVulnsPath) + r.Get("/api/package/{ecosystem}/*", h.HandlePackagePath) + transport := &swiftVulnsTransport{} + original := http.DefaultTransport + http.DefaultTransport = transport + t.Cleanup(func() { http.DefaultTransport = original }) + + for _, path := range []string{ + "/api/vulns/swift/apple.swift-nio", + "/api/vulns/swift/apple.swift-nio/2.0.0", + "/api/vulns/Swift/Apple.Swift-NIO/2.0.0", + "/api/package/swift/apple.swift-nio/2.0.0", + "/api/package/Swift/Apple.Swift-NIO/2.0.0", + } { + t.Run(path, func(t *testing.T) { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, body = %s", w.Code, w.Body) + } + var response ErrorResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.Code != ErrCodeBadRequest || response.Message != "package identity is not supported for vulnerability lookups" { + t.Fatalf("unexpected response: %+v", response) + } + }) + } + if transport.requests != 0 { + t.Errorf("made %d upstream requests for unsupported identities", transport.requests) + } +} + +type swiftVulnsTransport struct{ requests int } + +func (s *swiftVulnsTransport) RoundTrip(r *http.Request) (*http.Response, error) { + s.requests++ + return nil, fmt.Errorf("unexpected upstream request: %s", r.URL) +} From f32b46b71ae1f1f9cfd92e21b2a9bfb3deaa72d2 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 30 Oct 2026 18:02:30 +0000 Subject: [PATCH 019/198] Preserve Go cache and policy behavior when upgrading PURL --- docs/configuration.md | 3 ++ go.mod | 4 +- go.sum | 8 +-- internal/config/config.go | 3 +- internal/database/rekey_artifact.go | 40 ++++++++++++++ internal/database/rekey_artifact_test.go | 69 ++++++++++++++++++++++++ internal/denylist/denylist.go | 3 +- internal/handler/cooldown_expiry.go | 7 ++- internal/handler/cooldown_fail_closed.go | 2 +- internal/handler/denylist_test.go | 6 +++ internal/handler/go.go | 60 +++++++++++++++------ internal/handler/go_case_test.go | 69 ++++++++++++++++++++++++ internal/handler/go_cooldown.go | 8 +-- internal/handler/go_policy_test.go | 47 ++++++++++++++++ internal/handler/handler.go | 6 +++ internal/packageurl/packageurl.go | 9 ++++ internal/retention/retention.go | 11 ++-- internal/server/retention_golang_test.go | 4 +- 18 files changed, 320 insertions(+), 39 deletions(-) create mode 100644 internal/database/rekey_artifact.go create mode 100644 internal/database/rekey_artifact_test.go create mode 100644 internal/handler/go_policy_test.go diff --git a/docs/configuration.md b/docs/configuration.md index e1a4f9e..b20685f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1119,6 +1119,8 @@ Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to Package PURL keys are normalized to canonical form before matching, so `pkg:npm/@babel/core` and `pkg:npm/%40babel/core` are equivalent, as are `pkg:pypi/Django` and `pkg:pypi/django`. If both forms configure the same package, the canonical entry wins. +Go module names in exact cooldown overrides are matched case-insensitively. If both lowercase and mixed-case keys are configured, the lowercase entry wins. Go package patterns match lowercase module names. + `package_patterns` matches canonical, versionless PURLs using `*` for zero or more characters and `?` for one character. Neither wildcard crosses `/` separators. Character classes (`[...]`) and backslash escapes are rejected at startup. For example, `"pkg:npm/@example/*"` matches packages under the `@example` npm scope. Patterns accept `@` as an alias for `%40`; other characters must use their canonical PURL form. Equivalent patterns with different durations are rejected at startup. Equal durations, such as `1d` and `24h`, are accepted. Exact `packages` entries take precedence over patterns. When several patterns match, longer patterns win after excluding `*` and `?` from the length. Ties use lexical order of the normalized patterns. @@ -1180,6 +1182,7 @@ Names and versions are compared the way each ecosystem treats them: `pkg:composer/symfony/console@6.1.7` also covers `v6.1.7`. - Go versions are matched as written, `pkg:golang/example.com/mod@v1.0.0-RC1`, though download URLs escape uppercase letters (`v1.0.0-!r!c1`). + Go module names in denylist entries are matched case-insensitively. - A Swift entry names the registry scope and package, ignoring case: `pkg:swift/apple/swift-nio@2.62.0`. The stored form, `pkg:generic/swift-registry/apple.swift-nio@2.62.0`, works too. diff --git a/go.mod b/go.mod index 79978b1..2ce65e2 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,8 @@ require ( github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 github.com/git-pkgs/magic v0.6.0 - github.com/git-pkgs/purl v0.1.24 - github.com/git-pkgs/registries v0.12.0 + github.com/git-pkgs/purl v0.2.1 + github.com/git-pkgs/registries v0.13.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 7ed7c29..c322161 100644 --- a/go.sum +++ b/go.sum @@ -133,10 +133,10 @@ github.com/git-pkgs/magic v0.6.0 h1:fINPe67FEIaFRVlZzhOIDClMkm+gftBroBENa8bHSxQ= github.com/git-pkgs/magic v0.6.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= -github.com/git-pkgs/purl v0.1.24 h1:JMEB1uNWI+baEpl6AeVx+Kfq1c+xkou+THxa5KJ3MrU= -github.com/git-pkgs/purl v0.1.24/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= -github.com/git-pkgs/registries v0.12.0 h1:fhxYaR5iR9Zlx6doLkjlXurVqPxkYcXmNFiIrFeV3/k= -github.com/git-pkgs/registries v0.12.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc= +github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= +github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= +github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= +github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= diff --git a/internal/config/config.go b/internal/config/config.go index 5e90150..c09a1ba 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -77,6 +77,7 @@ import ( "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" "gopkg.in/yaml.v3" @@ -261,7 +262,7 @@ func (c *CooldownConfig) NormalizedPackages() map[string]string { for _, key := range keys { canonical := key if parsed, err := purl.Parse(key); err == nil { - canonical = parsed.String() + canonical = packageurl.PolicyPackageKey(parsed.String()) } if _, exists := normalized[canonical]; exists && key != canonical { continue diff --git a/internal/database/rekey_artifact.go b/internal/database/rekey_artifact.go new file mode 100644 index 0000000..68a8dbd --- /dev/null +++ b/internal/database/rekey_artifact.go @@ -0,0 +1,40 @@ +package database + +import ( + "fmt" + "time" +) + +// RekeyCachedArtifact moves one unchanged upstream record without replacing +// an existing destination. Access history and stored bytes are preserved. +func (db *DB) RekeyCachedArtifact(old *Artifact, pkg *Package, version *Version) (bool, error) { + tx, err := db.Beginx() + if err != nil { + return false, err + } + defer func() { _ = tx.Rollback() }() + now := time.Now() + _, err = tx.Exec(db.Rebind(`INSERT INTO packages (purl, ecosystem, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) ON CONFLICT(purl) DO NOTHING`), pkg.PURL, pkg.Ecosystem, pkg.Name, now, now) + if err != nil { + return false, fmt.Errorf("creating artifact package: %w", err) + } + _, err = tx.Exec(db.Rebind(`INSERT INTO versions (purl, package_purl, created_at, updated_at) + VALUES (?, ?, ?, ?) ON CONFLICT(purl) DO NOTHING`), version.PURL, version.PackagePURL, now, now) + if err != nil { + return false, fmt.Errorf("creating artifact version: %w", err) + } + result, err := tx.Exec(db.Rebind(`UPDATE artifacts SET version_purl = ?, updated_at = ? + WHERE id = ? AND version_purl = ? AND storage_path = ? AND upstream_url = ? AND source = ? + AND NOT EXISTS (SELECT 1 FROM artifacts WHERE version_purl = ? AND filename = ?)`), + version.PURL, now, old.ID, old.VersionPURL, old.StoragePath, old.UpstreamURL, SourceUpstream, + version.PURL, old.Filename) + if err != nil { + return false, fmt.Errorf("moving cached artifact: %w", err) + } + count, err := result.RowsAffected() + if err != nil || count == 0 { + return false, err + } + return true, tx.Commit() +} diff --git a/internal/database/rekey_artifact_test.go b/internal/database/rekey_artifact_test.go new file mode 100644 index 0000000..d5fd270 --- /dev/null +++ b/internal/database/rekey_artifact_test.go @@ -0,0 +1,69 @@ +package database + +import ( + "database/sql" + "testing" +) + +func TestRekeyCachedArtifact(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + for _, scenario := range []string{"move", "destination exists", "source changed", "local source"} { + t.Run(scenario, func(t *testing.T) { + oldPackage := &Package{PURL: "pkg:golang/example.com/old", Ecosystem: "golang", Name: "example.com/Old"} + oldVersion := &Version{PURL: oldPackage.PURL + "@v1.0.0", PackagePURL: oldPackage.PURL} + old := &Artifact{VersionPURL: oldVersion.PURL, Filename: scenario + ".zip", + UpstreamURL: "https://proxy.example/example.com/!old/@v/v1.0.0.zip", + StoragePath: sql.NullString{String: scenario + "/old.zip", Valid: true}} + if err := db.RecordCachedArtifact(oldPackage, oldVersion, old); err != nil { + t.Fatal(err) + } + old, err := db.GetArtifact(oldVersion.PURL, old.Filename) + if err != nil || old == nil { + t.Fatalf("source artifact = %+v, %v", old, err) + } + pkg := &Package{PURL: "pkg:golang/example.com/Old", Ecosystem: "golang", Name: "example.com/Old"} + version := &Version{PURL: pkg.PURL + "@v1.0.0", PackagePURL: pkg.PURL} + prepareRekeyConflict(t, db, scenario, old, pkg, version) + moved, err := db.RekeyCachedArtifact(old, pkg, version) + if err != nil || moved != (scenario == "move") { + t.Fatalf("RekeyCachedArtifact = %v, %v", moved, err) + } + checkRekeyResult(t, db, scenario, old, version) + }) + } + }) +} + +func checkRekeyResult(t *testing.T, db *DB, scenario string, old *Artifact, version *Version) { + t.Helper() + current, err := db.GetArtifact(version.PURL, old.Filename) + if err != nil { + t.Fatal(err) + } + if scenario == "move" && (current == nil || current.ID != old.ID || current.StoragePath != old.StoragePath) { + t.Errorf("moved artifact = %+v, old = %+v", current, old) + } + if scenario == "destination exists" && (current == nil || current.StoragePath.String != "destination.zip") { + t.Errorf("destination replaced: %+v", current) + } + if paths := allPendingDeletes(t, db); len(paths) != 0 { + t.Errorf("rekey queued stored bytes for deletion: %v", paths) + } +} + +func prepareRekeyConflict(t *testing.T, db *DB, scenario string, old *Artifact, pkg *Package, version *Version) { + t.Helper() + var err error + switch scenario { + case "destination exists": + err = db.RecordCachedArtifact(pkg, version, &Artifact{VersionPURL: version.PURL, Filename: old.Filename, + StoragePath: sql.NullString{String: "destination.zip", Valid: true}}) + case "source changed": + _, err = db.Exec(db.Rebind(`UPDATE artifacts SET storage_path = ? WHERE id = ?`), "replacement.zip", old.ID) + case "local source": + _, err = db.Exec(db.Rebind(`UPDATE artifacts SET source = ? WHERE id = ?`), SourceLocal, old.ID) + } + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go index 15bb9e5..76f2de5 100644 --- a/internal/denylist/denylist.go +++ b/internal/denylist/denylist.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" ) @@ -138,5 +139,5 @@ func identify(pkg *purl.PURL) identity { channel = c.Qualifier("channel") } key := purl.New(c.Type, c.Namespace, c.Name, "", nil).String() - return identity{key: key, channel: channel, version: c.Version} + return identity{key: packageurl.PolicyPackageKey(key), channel: channel, version: c.Version} } diff --git a/internal/handler/cooldown_expiry.go b/internal/handler/cooldown_expiry.go index 1163bde..c31ace8 100644 --- a/internal/handler/cooldown_expiry.go +++ b/internal/handler/cooldown_expiry.go @@ -5,6 +5,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/packageurl" ) // A rewrite made with cooldown on stays right until a version it held back @@ -127,6 +128,7 @@ type recordingCooldown struct { } func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { + packagePURL = packageurl.PolicyPackageKey(packagePURL) if r.p.Cooldown.IsAllowed(ecosystem, packagePURL, publishedAt) { return true } @@ -135,6 +137,7 @@ func (r recordingCooldown) IsAllowed(ecosystem, packagePURL string, publishedAt } func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision { + packagePURL = packageurl.PolicyPackageKey(packagePURL) d := r.p.Cooldown.Evaluate(ecosystem, packagePURL, publishedAt, evaluatedAt) if !d.Allowed { r.p.withheld(d.AvailableAt) @@ -143,7 +146,7 @@ func (r recordingCooldown) Evaluate(ecosystem, packagePURL string, publishedAt, } func (r recordingCooldown) For(ecosystem, packagePURL string) time.Duration { - return r.p.Cooldown.For(ecosystem, packagePURL) + return r.p.Cooldown.For(ecosystem, packageurl.PolicyPackageKey(packagePURL)) } func (r recordingCooldown) Enabled() bool { @@ -156,7 +159,7 @@ func (r recordingCooldown) availableAt(ecosystem, packagePURL string, publishedA if publishedAt.IsZero() { return time.Time{} } - d := r.p.Cooldown.For(ecosystem, packagePURL) + d := r.For(ecosystem, packagePURL) if d <= 0 { return time.Time{} } diff --git a/internal/handler/cooldown_fail_closed.go b/internal/handler/cooldown_fail_closed.go index da600c1..0dd2c36 100644 --- a/internal/handler/cooldown_fail_closed.go +++ b/internal/handler/cooldown_fail_closed.go @@ -5,7 +5,7 @@ import "net/http" // cooldownApplies reports whether cooldown withholds new versions of the // package, taking package overrides and exemptions into account. func (p *Proxy) cooldownApplies(ecosystem, name string) bool { - return p.cooldownEnabled() && p.Cooldown.For(ecosystem, canonicalPackagePURL(ecosystem, name)) > 0 + return p.cooldownEnabled() && p.cooldown().For(ecosystem, canonicalPackagePURL(ecosystem, name)) > 0 } // mustFilterMetadata reports whether the package's metadata may have diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go index bb41e38..663d0c3 100644 --- a/internal/handler/denylist_test.go +++ b/internal/handler/denylist_test.go @@ -288,6 +288,12 @@ func TestDenylistMatchesEcosystemSpellings(t *testing.T) { {"go case-escaped version", "pkg:golang/github.com/example/mod@v1.0.0-RC1", "/github.com/example/mod/@v/v1.0.0-!r!c1.zip", "golang", "github.com/example/mod", "v1.0.0-!r!c1", "mod@v1.0.0-!r!c1.zip", func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, + {"go lowercase rule", "pkg:golang/github.com/example/mod@v1.0.0", "/github.com/!example/mod/@v/v1.0.0.zip", + "golang", "github.com/Example/mod", "v1.0.0", goZipFilename("github.com/Example/mod", "v1.0.0"), + func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, + {"go mixed-case rule", "pkg:golang/github.com/Example/mod@v1.0.0", "/github.com/example/mod/@v/v1.0.0.zip", + "golang", "github.com/example/mod", "v1.0.0", "mod@v1.0.0.zip", + func(p *Proxy) http.Handler { return NewGoHandler(p, "http://proxy.test").Routes() }}, {"oci repository path", "pkg:oci/library/alpine@" + ociTestDigest, "/library/alpine/blobs/" + ociTestDigest, "oci", "library/alpine", ociTestDigest, ociTestDigest, func(p *Proxy) http.Handler { diff --git a/internal/handler/go.go b/internal/handler/go.go index 1af00ed..08dae57 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -4,11 +4,15 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" "net/http" + "net/url" "strings" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" "github.com/git-pkgs/registries/fetch" gomodule "golang.org/x/mod/module" ) @@ -18,15 +22,13 @@ const ( asciiCaseOffset = 32 // difference between lowercase and uppercase ASCII letters ) -// Module zips are stored under pkg:golang/{module path}, which the PURL -// rules lowercase, so a package rule covers a module whatever case either -// is written in. A major version suffix (/v2) is part of the module path -// and so a module of its own. A key in the escaped form proxy URLs and the -// module cache use (github.com/!azure) does not survive the default -// mapping's round trip and is refused; the handler stores decoded paths, -// so such a key could never match. func init() { - retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: retention.DefaultCanonical("golang")}) + retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: golangRetentionKey}) +} + +func golangRetentionKey(p *purl.PURL) (string, bool) { + key, ok := retention.DefaultCanonical("golang")(p) + return packageurl.PolicyPackageKey(key), ok } // GoHandler handles Go module proxy protocol requests. @@ -200,13 +202,7 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul ServeArtifactRequest(w, r, result) } -// goZipFilename is the name a module zip is cached under. Module paths are -// case-sensitive, but the zip is stored under the module's PURL, which -// lowercases golang names, so github.com/Sirupsen/logrus and -// github.com/sirupsen/logrus share one package. A path with uppercase -// letters therefore gets a hash of its exact spelling in the filename. An -// all-lowercase path keeps the plain name, so existing cache entries stay -// valid. +// Keep the module spelling hash used by caches with lowercased PURL keys. func goZipFilename(module, version string) string { base := lastComponent(module) + "@" + version if module == strings.ToLower(module) { @@ -220,6 +216,40 @@ func goZipFilename(module, version string) string { // filename carries: enough to tell apart paths that differ only in case. const goModuleHashBytes = 8 +// An old cache key may combine case-distinct modules. The upstream path +// identifies which module owns an individual archive. +func (p *Proxy) rekeyLegacyGoArtifact(pkgPURL, versionPURL, filename string) (bool, error) { + if !strings.HasPrefix(versionPURL, "pkg:golang/") { + return false, nil + } + parsed, err := purl.Parse(versionPURL) + if err != nil || parsed.FullName() == strings.ToLower(parsed.FullName()) { + return false, nil + } + name := parsed.FullName() + _, legacyVersion, err := packagePURLStrings("golang", strings.ToLower(name), parsed.Version) + if err != nil { + return false, err + } + record, err := p.DB.GetArtifact(legacyVersion, filename) + if err != nil { + return false, fmt.Errorf("reading legacy Go artifact: %w", err) + } + if record == nil || !record.StoragePath.Valid || record.Source != database.SourceUpstream { + return false, nil + } + escaped, err := gomodule.EscapePath(name) + if err != nil { + return false, nil + } + upstream, err := url.Parse(record.UpstreamURL) + if err != nil || !strings.HasSuffix(upstream.Path, "/"+escaped+"/@v/"+parsed.Version+".zip") { + return false, nil + } + return p.DB.RekeyCachedArtifact(record, &database.Package{PURL: pkgPURL, Ecosystem: "golang", Name: name}, + &database.Version{PURL: versionPURL, PackagePURL: pkgPURL}) +} + // discardOtherModulesZip drops a cached zip that another module stored // under filename. Before module paths kept their case, a module with // uppercase letters was cached under the plain filename of its lowercase diff --git a/internal/handler/go_case_test.go b/internal/handler/go_case_test.go index 9451caa..91bac44 100644 --- a/internal/handler/go_case_test.go +++ b/internal/handler/go_case_test.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "testing" + "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/registries/fetch" ) @@ -115,6 +116,74 @@ func TestGoZipFromAnotherUpstreamHostIsKept(t *testing.T) { } } +func TestGoLegacyCacheKeepsCaseDistinctModules(t *testing.T) { + upstream, counts := goCaseUpstream(t) + h, proxy := newGoCaseHandler(t, upstream) + seedGoZip(t, proxy, upstream.URL+lowerLogrusPath, "zip of "+lowerLogrusPath) + old := seedLegacyUpperGoZip(t, proxy, "https://old-proxy.example"+upperLogrusPath, "zip of "+upperLogrusPath) + upstream.Close() + + for range 2 { + for _, path := range []string{upperLogrusPath, lowerLogrusPath} { + if w := getGo(h, path); w.Code != http.StatusOK || w.Body.String() != "zip of "+path { + t.Fatalf("GET %s: status = %d, body = %q", path, w.Code, w.Body) + } + } + } + current, err := proxy.DB.GetArtifact("pkg:golang/github.com/Sirupsen/logrus@v1.0.6", old.Filename) + if err != nil || current == nil { + t.Fatalf("moved artifact = %+v, %v", current, err) + } + if current.ID != old.ID || current.StoragePath != old.StoragePath || current.FetchedAt != old.FetchedAt || current.HitCount != old.HitCount+2 { + t.Errorf("artifact history changed: before=%+v, after=%+v", old, current) + } + legacy, err := proxy.DB.GetArtifact(old.VersionPURL, old.Filename) + if err != nil || legacy != nil { + t.Fatalf("legacy artifact = %+v, %v", legacy, err) + } + for path, count := range counts { + if count.Load() != 0 { + t.Errorf("unexpected upstream request: %s", path) + } + } +} + +func TestGoLegacyCacheRequiresExactModuleOrigin(t *testing.T) { + for _, origin := range []string{"", "https://proxy.example" + lowerLogrusPath} { + t.Run(origin, func(t *testing.T) { + upstream, counts := goCaseUpstream(t) + h, proxy := newGoCaseHandler(t, upstream) + seedLegacyUpperGoZip(t, proxy, origin, "another module") + if w := getGo(h, upperLogrusPath); w.Code != http.StatusOK || w.Body.String() != "zip of "+upperLogrusPath { + t.Fatalf("status = %d, body = %q", w.Code, w.Body) + } + if counts[upperLogrusPath].Load() != 1 { + t.Fatal("archive with unproven identity was reused") + } + }) + } +} + +func seedLegacyUpperGoZip(t *testing.T, proxy *Proxy, upstreamURL, content string) *database.Artifact { + t.Helper() + const packagePURL = "pkg:golang/github.com/sirupsen/logrus" + const versionPURL = packagePURL + "@v1.0.6" + const storagePath = "golang/logrus/v1.0.6/legacy/upper.zip" + filename := goZipFilename("github.com/Sirupsen/logrus", "v1.0.6") + if _, _, err := proxy.Storage.Store(context.Background(), storagePath, strings.NewReader(content)); err != nil { + t.Fatal(err) + } + artifact := testArtifact(content, versionPURL, filename, "application/zip") + if err := proxy.updateCacheDB("golang", "github.com/Sirupsen/logrus", packagePURL, upstreamURL, storagePath, artifact, ""); err != nil { + t.Fatal(err) + } + old, err := proxy.DB.GetArtifact(versionPURL, filename) + if err != nil || old == nil { + t.Fatalf("seeded artifact = %+v, %v", old, err) + } + return old +} + // seedGoZip caches content as github.com/sirupsen/logrus v1.0.6 under the // plain filename, recorded as fetched from upstreamURL. func seedGoZip(t *testing.T, proxy *Proxy, upstreamURL, content string) { diff --git a/internal/handler/go_cooldown.go b/internal/handler/go_cooldown.go index 892e77b..efcc1e1 100644 --- a/internal/handler/go_cooldown.go +++ b/internal/handler/go_cooldown.go @@ -71,12 +71,8 @@ func (h *GoHandler) fetchGoMetadata(ctx context.Context, escapedModule, module, return body, err } -// goTimeKey returns the versions-table PURLs a module's times are stored -// under. golang PURLs lowercase the name, so the key uses the escaped module -// path (github.com/!azure for github.com/Azure), which keeps modules that -// differ only in case apart and is the plain path for lowercase modules. -// Only canonical versions are stored: a query such as a branch name can -// resolve to another commit later. +// Keep escaped module paths in publish-time keys for cache compatibility. +// Only canonical versions are stored; branch queries can resolve differently. func (h *GoHandler) goTimeKey(module, version string) (versionPURL, packagePURL string, ok bool) { if h.proxy.DB == nil || gomodule.CanonicalVersion(version) != version { return "", "", false diff --git a/internal/handler/go_policy_test.go b/internal/handler/go_policy_test.go new file mode 100644 index 0000000..0ce981f --- /dev/null +++ b/internal/handler/go_policy_test.go @@ -0,0 +1,47 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/config" +) + +func TestGoCooldownOverridesIgnoreModuleCase(t *testing.T) { + for _, tc := range []struct { + name, rule, fallback, override string + status int + }{ + {"lowercase restriction", "example.com/module", "0", "7d", http.StatusForbidden}, + {"mixed-case restriction", "example.com/Module", "0", "7d", http.StatusForbidden}, + {"lowercase exemption", "example.com/module", "7d", "0", http.StatusOK}, + {"mixed-case exemption", "example.com/Module", "7d", "0", http.StatusOK}, + } { + t.Run(tc.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".info") { + _ = json.NewEncoder(w).Encode(goInfo{Version: "v1.0.0", Time: time.Now()}) + return + } + _, _ = w.Write([]byte("module zip")) + })) + defer upstream.Close() + h, proxy := newGoCaseHandler(t, upstream) + cfg := config.CooldownConfig{Packages: map[string]string{"pkg:golang/" + tc.rule: tc.override}} + proxy.Cooldown = &cooldown.Config{Default: tc.fallback, Packages: cfg.NormalizedPackages()} + for _, path := range []string{ + "/example.com/!module/@v/v1.0.0.zip", "/example.com/module/@v/v1.0.0.zip", + "/example.com/!module/@v/v1.0.0.info", "/example.com/module/@v/v1.0.0.info", + } { + if response := getGo(h, path); response.Code != tc.status { + t.Errorf("GET %s = %d %s, want %d", path, response.Code, response.Body, tc.status) + } + } + }) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 3951016..4b7ebe7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -498,6 +498,12 @@ func (p *Proxy) lookupCache(ctx context.Context, pkgPURL, versionPURL, filename if p.StreamArtifacts { return nil, nil } + if cached, err := p.readStored(ctx, pkgPURL, versionPURL, filename); err != nil || cached != nil { + return cached, err + } + if moved, err := p.rekeyLegacyGoArtifact(pkgPURL, versionPURL, filename); err != nil || !moved { + return nil, err + } return p.readStored(ctx, pkgPURL, versionPURL, filename) } diff --git a/internal/packageurl/packageurl.go b/internal/packageurl/packageurl.go index be0e9ab..e01f4b2 100644 --- a/internal/packageurl/packageurl.go +++ b/internal/packageurl/packageurl.go @@ -18,6 +18,15 @@ func MakeString(ecosystem, name, version string) string { return purl.MakePURLString(ecosystem, name, version) } +// PolicyPackageKey maps a canonical versionless PURL to its policy key. +// Go policies match module names case-insensitively for compatibility. +func PolicyPackageKey(packagePURL string) string { + if strings.HasPrefix(packagePURL, "pkg:golang/") { + return strings.ToLower(packagePURL) + } + return packagePURL +} + // WithVersionString returns a package PURL with its version replaced. It // returns an empty string when packagePURL is invalid. func WithVersionString(packagePURL, version string) string { diff --git a/internal/retention/retention.go b/internal/retention/retention.go index a8ff835..df8e538 100644 --- a/internal/retention/retention.go +++ b/internal/retention/retention.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" ) @@ -41,8 +42,8 @@ type Spec struct { // derived from Key that belong to this ecosystem. ExtraDBEcosystems []string - // CanonicalPackage turns a package PURL from the config into the - // package PURL the handler stores in the cache, reporting false when the + // CanonicalPackage turns a package PURL from the config into a + // policy key, reporting false when the // PURL does not belong to this ecosystem. Nil means the ecosystem accepts // no package overrides. CanonicalPackage func(p *purl.PURL) (string, bool) @@ -150,8 +151,7 @@ func (r *Registry) KeyForDBEcosystem(ecosystem string) (string, bool) { return key, ok } -// Canonical returns the stored package PURL a configured package PURL -// stands for, asking the registered specs in key order. +// Canonical returns the policy key for a configured package PURL. func (r *Registry) Canonical(p *purl.PURL) (string, bool) { for _, key := range r.Keys() { s, _ := r.Lookup(key) @@ -173,7 +173,7 @@ type Rules struct { Default time.Duration // Ecosystems holds rules keyed by registered ecosystem key. Ecosystems map[string]time.Duration - // Packages holds rules keyed by the package PURL stored in the cache. + // Packages holds rules keyed by package PURL, with Go keys lowercased. Packages map[string]time.Duration } @@ -181,6 +181,7 @@ type Rules struct { // stored package PURL: the package rule, else the ecosystem rule, else the // default. Generic artifacts require an explicit package or ecosystem rule. func (r Rules) For(key, packagePURL string) time.Duration { + packagePURL = packageurl.PolicyPackageKey(packagePURL) if d, ok := r.Packages[packagePURL]; ok { return d } diff --git a/internal/server/retention_golang_test.go b/internal/server/retention_golang_test.go index f74eaf4..8ff0357 100644 --- a/internal/server/retention_golang_test.go +++ b/internal/server/retention_golang_test.go @@ -188,8 +188,8 @@ func TestGolangRetentionPackageRules(t *testing.T) { if err := db.Get(&stored, db.Rebind(`SELECT package_purl FROM versions WHERE purl = ?`), zip.VersionPURL); err != nil { t.Fatalf("reading stored package PURL: %v", err) } - if stored != "pkg:golang/github.com/burntsushi/toml" { - t.Errorf("stored package PURL = %q, want pkg:golang/github.com/burntsushi/toml", stored) + if stored != "pkg:golang/github.com/BurntSushi/toml" { + t.Errorf("stored package PURL = %q, want pkg:golang/github.com/BurntSushi/toml", stored) } for _, keys := range []map[string]string{ From 9941d79561a750426a751de4660b30e65f3eb7ab Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:19:01 +0100 Subject: [PATCH 020/198] Document the published container image and add a Docker health check --- Dockerfile | 4 ++++ README.md | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a934ed..cc35e9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,5 +31,9 @@ WORKDIR /data EXPOSE 8080 +# /health checks the database and storage and can take up to ten seconds. +HEALTHCHECK --interval=30s --timeout=11s --start-period=10s --retries=3 \ + CMD wget -q -O /dev/null http://127.0.0.1:8080/health || exit 1 + ENTRYPOINT ["proxy"] CMD ["serve"] diff --git a/README.md b/README.md index a324c53..6d84241 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,11 @@ Cooldown requires publish timestamps in metadata. Registries without a "Yes" in brew install git-pkgs/git-pkgs/proxy ``` -Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases). +Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases), or run the container image: + +```bash +docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest +``` ### Helm @@ -1715,13 +1719,21 @@ sudo systemctl start proxy ### Docker -A Dockerfile is included in the repo. Build and run: +Images for linux/amd64 and linux/arm64 are published to `ghcr.io/git-pkgs/proxy` for each release, tagged with the version and `latest`: + +```bash +docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest +``` + +To build the image from the repo instead: ```bash docker build -t proxy . docker run -p 8080:8080 -v proxy-data:/data proxy ``` +The image has a health check that requests `/health` on port 8080. It checks the database and storage, so a container reports unhealthy when either fails. If you change the port with `PROXY_LISTEN`, override it with `--health-cmd`. + With `--read-only`, also pass `--tmpfs /tmp`: the proxy writes some uploads to a temporary file before reading them. With Postgres and S3: From 760623e617d7bed7d3b64b437fb942076eef4504 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:25:09 +0100 Subject: [PATCH 021/198] Serve the stored npm packument on upstream 404 and refetch on a 304 for a missing blob --- internal/handler/metadata_stored.go | 11 +++-- internal/handler/npm_stream_test.go | 72 +++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index d831303..80cbaf4 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -122,7 +122,9 @@ func (p *Proxy) fetchStoredMetadata(ctx context.Context, ecosystem, cacheKey, up if err == nil { return metadataResult{entry: stored} } - if !usableStoredMetadata(entry) || errors.Is(err, ErrUpstreamNotFound) || errors.Is(err, errMetadataNotStored) { + // As in fetchMetadataFromUpstream, any upstream failure, a 404 included, + // falls back to the stored copy. + if !usableStoredMetadata(entry) || errors.Is(err, errMetadataNotStored) { return metadataResult{err: err} } return p.staleStoredMetadata(ecosystem, cacheKey, entry, err) @@ -142,8 +144,8 @@ func (p *Proxy) staleStoredMetadata(ecosystem, cacheKey string, entry *database. // storeUpstreamMetadata fetches a document and streams it into storage, and // returns its new cache row. On a 304 for entry it only moves the row's -// fetched_at on; errStale304 means the row was replaced meanwhile and the -// caller should fetch without validators. +// fetched_at on; errStale304 means the stored copy is gone or the row was +// replaced meanwhile, and the caller should fetch without validators. // // With validate, the document is staged under a path of its own and checked // there first, so one validate rejects never replaces the stored copy. @@ -155,6 +157,9 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusNotModified && entry != nil { + if ok, err := p.Storage.Exists(ctx, entry.StoragePath); err != nil || !ok { + return nil, errStale304 + } refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) if err != nil || !refreshed { return nil, errStale304 diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go index 3442ce2..de5a17d 100644 --- a/internal/handler/npm_stream_test.go +++ b/internal/handler/npm_stream_test.go @@ -50,24 +50,38 @@ func bigPackumentAt(base string, n int) string { type streamUpstream struct { *httptest.Server body atomic.Value // string + etag atomic.Value // string fail atomic.Bool + gone atomic.Bool requests atomic.Int32 + // sent counts the responses that carried the document. + sent atomic.Int32 +} + +// replace makes upstream send body under a new ETag, so a revalidation +// gets it in a 200. +func (u *streamUpstream) replace(body, etag string) { + u.body.Store(body) + u.etag.Store(etag) } func newStreamUpstream(t *testing.T, body string) *streamUpstream { t.Helper() u := &streamUpstream{} - u.body.Store(body) + u.replace(body, `"v1"`) u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { u.requests.Add(1) switch { case u.fail.Load(): http.Error(w, "down", http.StatusBadGateway) - case r.URL.Path == "/big" && r.Header.Get("If-None-Match") == `"v1"`: + case u.gone.Load(): + http.NotFound(w, r) + case r.URL.Path == "/big" && r.Header.Get("If-None-Match") == u.etag.Load().(string): w.WriteHeader(http.StatusNotModified) case r.URL.Path == "/big": + u.sent.Add(1) w.Header().Set("Content-Type", "application/json") - w.Header().Set("ETag", `"v1"`) + w.Header().Set("ETag", u.etag.Load().(string)) _, _ = io.WriteString(w, u.body.Load().(string)) default: http.NotFound(w, r) @@ -201,6 +215,53 @@ func TestNPMStreamedPackumentStale(t *testing.T) { } } +// TestNPMStreamedPackumentStaleOn404 checks that a stored copy past its TTL +// is served when upstream answers 404, as the in-memory path does. +func TestNPMStreamedPackumentStaleOn404(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + proxy, _, handler := streamProxy(t, u) + decodeStreamed(t, serve(handler, "/big", nil)) + + proxy.MetadataTTL = time.Nanosecond + u.gone.Store(true) + for _, size := range []int64{64 << 20, 0} { // streamed, then in memory + proxy.SetMetadataRewriteCacheSize(size) + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { + t.Errorf("rewrite cache %d: stale copy has %d versions, want 10", size, len(doc.Versions)) + } + } +} + +// TestNPMStreamedPackumentBlobMissing checks that a 304 for a row whose +// document is gone from storage fetches the document again, also when it is +// larger than metadata_max_size. +func TestNPMStreamedPackumentBlobMissing(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + decodeStreamed(t, serve(handler, "/big", nil)) + + path := metadataStoragePath("npm", "big") + store.mu.Lock() + delete(store.files, path) + store.mu.Unlock() + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { + t.Errorf("served %d versions, want 400", len(doc.Versions)) + } + if n := u.sent.Load(); n != 2 { + t.Errorf("upstream sent the document %d times, want 2", n) + } + store.mu.Lock() + defer store.mu.Unlock() + if string(store.files[path]) != body { + t.Error("the document was not stored again") + } +} + // TestNPMStreamedPackumentChangedInStorage checks that a document replaced // in storage between the two passes isn't rewritten from mixed bytes. func TestNPMStreamedPackumentChangedInStorage(t *testing.T) { @@ -262,10 +323,13 @@ func TestNPMStreamedPackumentKeepsFullCopy(t *testing.T) { proxy.MetadataTTL = time.Nanosecond proxy.SetMetadataRewriteCacheSize(64 << 20) - u.body.Store(`{"name":"big","modified":"2026-01-01T00:00:00Z","dist-tags":{"latest":"1.0.0"},"versions":{"1.0.0":{}}}`) + u.replace(`{"name":"big","modified":"2026-01-01T00:00:00Z","dist-tags":{"latest":"1.0.0"},"versions":{"1.0.0":{}}}`, `"v2"`) if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 10 { t.Errorf("served %d versions, want the 10 of the full copy", len(doc.Versions)) } + if n := u.sent.Load(); n != 2 { + t.Fatalf("upstream sent a document %d times, want 2: the abbreviated one was not fetched", n) + } store.mu.Lock() defer store.mu.Unlock() From 3ce582c7330335a707df8a286e357e65f10cee89 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:25:41 +0100 Subject: [PATCH 022/198] Keep the last good Composer index when upstream sends one that doesn't parse parseComposerUpstreamIndex now returns an error for a packages.json that is not a JSON object or has the advisories and filter fields in the wrong shape, and fetchUpstreamIndex passes it to the metadata cache as validate. A malformed 200 is no longer stored: with metadata caching the stored copy is served as when upstream fails, and without it the error keeps the copy in memory. --- internal/handler/composer_advisories.go | 31 ++++++-- internal/handler/composer_advisories_test.go | 77 ++++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/internal/handler/composer_advisories.go b/internal/handler/composer_advisories.go index ed8f4bc..aa0fa26 100644 --- a/internal/handler/composer_advisories.go +++ b/internal/handler/composer_advisories.go @@ -1,9 +1,11 @@ package handler import ( + "bytes" "context" "encoding/json" "errors" + "fmt" "io" "net/http" "net/url" @@ -97,18 +99,34 @@ func (h *ComposerHandler) upstreamIndex(ctx context.Context) (composerUpstreamIn return c.index, c.ok } +// fetchUpstreamIndex fetches and parses the default repository's +// packages.json. A document that doesn't parse is rejected before it is +// cached, so a broken response never replaces a good copy: with metadata +// caching the stored copy is used instead, and without it the error keeps +// the copy in memory. func (h *ComposerHandler) fetchUpstreamIndex(ctx context.Context) (composerUpstreamIndex, error) { indexURL := h.repoURL + "/packages.json" cacheKey := hashedCacheKey(h.repoURL, "/packages.json") - res := h.proxy.fetchMetadataDocument(ctx, "composer", cacheKey, indexURL, contentTypeJSON, nil) + validate := func(body []byte) error { + _, err := parseComposerUpstreamIndex(indexURL, body) + return err + } + res := h.proxy.fetchMetadataDocument(ctx, "composer", cacheKey, indexURL, contentTypeJSON, validate) if res.err != nil { h.proxy.Logger.Warn("failed to fetch upstream composer packages.json", "url", indexURL, "error", res.err) return composerUpstreamIndex{}, res.err } - return parseComposerUpstreamIndex(indexURL, res.body), nil + return parseComposerUpstreamIndex(indexURL, res.body) } -func parseComposerUpstreamIndex(indexURL string, body []byte) composerUpstreamIndex { +// errComposerIndexNotObject is returned for a packages.json that is valid +// JSON but not an object, such as null. +var errComposerIndexNotObject = errors.New("composer packages.json is not a JSON object") + +// parseComposerUpstreamIndex reads the advisories and filter fields of a +// packages.json. A document without them yields an empty index; one that is +// not a JSON object, or has them in the wrong shape, is an error. +func parseComposerUpstreamIndex(indexURL string, body []byte) (composerUpstreamIndex, error) { var doc struct { SecurityAdvisories struct { Metadata bool `json:"metadata"` @@ -121,8 +139,11 @@ func parseComposerUpstreamIndex(indexURL string, body []byte) composerUpstreamIn } `json:"filter"` } var index composerUpstreamIndex + if trimmed := bytes.TrimSpace(body); len(trimmed) == 0 || trimmed[0] != '{' { + return index, errComposerIndexNotObject + } if err := json.Unmarshal(body, &doc); err != nil { - return index + return index, fmt.Errorf("parsing composer packages.json: %w", err) } // Composer refuses advisories configured without an api-url unless the // repository lists its packages, which the proxy does not. @@ -137,7 +158,7 @@ func parseComposerUpstreamIndex(indexURL string, body []byte) composerUpstreamIn index.filterLists = doc.Filter.Lists index.filterSummaryURL, _ = resolveComposerURL(indexURL, doc.Filter.SummaryURL) } - return index + return index, nil } // resolveComposerURL resolves a URL from packages.json against the document's diff --git a/internal/handler/composer_advisories_test.go b/internal/handler/composer_advisories_test.go index 5e1a204..097d8cc 100644 --- a/internal/handler/composer_advisories_test.go +++ b/internal/handler/composer_advisories_test.go @@ -2,6 +2,7 @@ package handler import ( "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -12,6 +13,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/git-pkgs/proxy/internal/database" ) // composerAdvisoryUpstream serves a packages.json advertising security @@ -21,6 +24,8 @@ type composerAdvisoryUpstream struct { *httptest.Server index string down atomic.Bool // answer every request with 503 + // malformed answers packages.json with an HTML page and status 200. + malformed atomic.Bool mu sync.Mutex requests []*recordedRequest @@ -53,6 +58,11 @@ func newComposerAdvisoryUpstream(t *testing.T) *composerAdvisoryUpstream { w.Header().Set("Content-Type", "application/json") switch r.URL.Path { case "/packages.json": + if u.malformed.Load() { + w.Header().Set("Content-Type", "text/html") + _, _ = io.WriteString(w, "Service temporarily unavailable") + return + } _, _ = io.WriteString(w, u.index) case "/api/security-advisories/": _, _ = io.WriteString(w, composerAdvisoriesResponse) @@ -235,6 +245,73 @@ func TestComposerServiceIndexKeptInMemory(t *testing.T) { } } +// assertComposerSecurityAdvertised checks that h advertises the upstream's +// advisories API and malware filter, and relays both. +func assertComposerSecurityAdvertised(t *testing.T, h *ComposerHandler, when string) { + t.Helper() + index := serveComposerIndex(t, h) + if !advertisesAdvisories(index) { + t.Errorf("%s: security-advisories not advertised: %v", when, index) + } + if _, ok := index["filter"]; !ok { + t.Errorf("%s: filter not advertised: %v", when, index) + } + w := serveAdvisories(h, http.MethodPost, "/api/security-advisories/", "packages%5B%5D=public%2Flibrary") + if w.Code != http.StatusOK || w.Body.String() != composerAdvisoriesResponse { + t.Errorf("%s: advisories = %d %q, want upstream's advisories", when, w.Code, w.Body) + } + if w := serveComposer(h, "/lists/summary.json"); w.Code != http.StatusOK || w.Body.String() != composerFilterSummary { + t.Errorf("%s: filter summary = %d %q, want upstream's summary", when, w.Code, w.Body) + } +} + +// A packages.json that doesn't parse, sent with status 200, must not replace +// the last good index, with metadata caching on or off: the advisories API +// and the malware filter stay advertised and relayed. +func TestComposerServiceIndexMalformedRefreshKeepsLastGood(t *testing.T) { + for _, cacheMetadata := range []bool{false, true} { + t.Run(fmt.Sprintf("cache_metadata=%v", cacheMetadata), func(t *testing.T) { + upstream := newComposerAdvisoryUpstream(t) + h := newAdvisoryComposerHandler(t, upstream) + h.proxy.CacheMetadata = cacheMetadata + h.proxy.MetadataTTL = time.Hour + now := time.Now() + h.indexCache.now = func() time.Time { return now } + cacheKey := hashedCacheKey(upstream.URL, "/packages.json") + + assertComposerSecurityAdvertised(t, h, "before the refresh") + var stored *database.MetadataCacheEntry + if cacheMetadata { + var err error + if stored, err = h.proxy.DB.GetMetadataCache("composer", cacheKey); err != nil || stored == nil { + t.Fatalf("packages.json not cached: %v, %v", stored, err) + } + expireMetadata(t, h.proxy.DB, "composer", cacheKey) + } + + upstream.malformed.Store(true) + now = now.Add(2 * time.Hour) + fetched := len(upstream.requestsTo("/packages.json")) + assertComposerSecurityAdvertised(t, h, "after a malformed refresh") + if got := len(upstream.requestsTo("/packages.json")); got == fetched { + t.Fatal("packages.json was not fetched again after the TTL") + } + + if !cacheMetadata { + return + } + after, err := h.proxy.DB.GetMetadataCache("composer", cacheKey) + if err != nil || after == nil || after.ContentDigest != stored.ContentDigest { + t.Errorf("cached packages.json replaced by the malformed refresh: %v, %v", after, err) + } + // A fresh handler, as after a restart, has nothing in memory and + // reads the stored copy. + restarted := NewComposerHandlerWithUpstreams(h.proxy, "http://proxy.test", upstream.URL, upstream.URL) + assertComposerSecurityAdvertised(t, restarted, "after a restart") + }) + } +} + func TestComposerServiceIndexFetchedOnce(t *testing.T) { upstream := newComposerAdvisoryUpstream(t) h := newAdvisoryComposerHandler(t, upstream) From 04f732e14744592bea58c1176a64b3f32c887d69 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:26:03 +0100 Subject: [PATCH 023/198] Record APK snapshots from the document a request serves A snapshot was recorded after the response by reading the metadata cache again, so a refresh that replaced the cached copy in between lost the version the request served. Record it from the served bytes before responding instead, and compare a new snapshot with the version current at its own time, so one recorded late keeps its place in the history. --- internal/database/metadata_snapshots.go | 21 ++-- internal/database/metadata_snapshots_test.go | 30 ++++++ internal/handler/apk.go | 6 +- internal/handler/apk_snapshots.go | 104 +++++++++++++++---- internal/handler/apk_snapshots_test.go | 76 ++++++++++++++ 5 files changed, 207 insertions(+), 30 deletions(-) diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go index fba774a..4df1fa1 100644 --- a/internal/database/metadata_snapshots.go +++ b/internal/database/metadata_snapshots.go @@ -60,15 +60,18 @@ func (db *DB) metadataSnapshotBefore(ecosystem, name string, beforeMS int64) (*M return &snap, nil } -// AddMetadataSnapshot records snap as the current version of its document, -// unless the newest snapshot already has its digest, and reports whether it -// did. On Postgres, replicas recording the same document take an advisory -// lock on it first, so their check and insert don't interleave: without it, -// two transactions can both read the previous digest and both insert. SQLite +// AddMetadataSnapshot records snap as the version of its document current +// from snap.ValidFromMS, unless the snapshot current at that time already +// has its digest, and reports whether it did. Comparing with the version +// current at that time rather than the newest one keeps a version a request +// recorded late, after a newer one, in its place in the history. On +// Postgres, replicas recording the same document take an advisory lock on it +// first, so their check and insert don't interleave: without it, two +// transactions can both read the previous digest and both insert. SQLite // runs one write at a time, so the single statement is enough there. A // document going back to an earlier digest is recorded again, since only the -// newest snapshot is compared. The casts let Postgres type the parameters of -// the INSERT ... SELECT. +// snapshot current at that time is compared. The casts let Postgres type the +// parameters of the INSERT ... SELECT. func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { tx, err := db.Beginx() if err != nil { @@ -87,13 +90,13 @@ func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { CAST(? AS TEXT), CAST(? AS BIGINT), CAST(? AS BIGINT) WHERE COALESCE(( SELECT content_digest FROM metadata_snapshots - WHERE ecosystem = ? AND name = ? + WHERE ecosystem = ? AND name = ? AND valid_from_ms <= CAST(? AS BIGINT) ORDER BY valid_from_ms DESC, id DESC LIMIT 1 ), '') <> CAST(? AS TEXT) `), snap.Ecosystem, snap.Name, snap.StoragePath, snap.ContentType, snap.ContentEncoding, snap.ContentDigest, snap.Size, snap.ValidFromMS, - snap.Ecosystem, snap.Name, snap.ContentDigest) + snap.Ecosystem, snap.Name, snap.ValidFromMS, snap.ContentDigest) if err != nil { return false, fmt.Errorf("recording metadata snapshot: %w", err) } diff --git a/internal/database/metadata_snapshots_test.go b/internal/database/metadata_snapshots_test.go index 0b4919f..71865b9 100644 --- a/internal/database/metadata_snapshots_test.go +++ b/internal/database/metadata_snapshots_test.go @@ -70,6 +70,36 @@ func TestMetadataSnapshots_AddAndLookup(t *testing.T) { }) } +// TestMetadataSnapshots_RecordedLate adds a version after a newer one was +// recorded, as a request that fetched it first but recorded it last does. It +// is compared with the version current at its own time, so it keeps its place +// in the history. +func TestMetadataSnapshots_RecordedLate(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + day := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + addSnapshot(t, db, "idx", "sha256:a", day) + addSnapshot(t, db, "idx", "sha256:c", day.Add(2*time.Hour)) + if !addSnapshot(t, db, "idx", "sha256:b", day.Add(time.Hour)) { + t.Fatal("version recorded after a newer one not added") + } + if addSnapshot(t, db, "idx", "sha256:b", day.Add(90*time.Minute)) { + t.Error("version already current at that time added again") + } + for _, tc := range []struct { + at time.Time + want string + }{ + {day.Add(30 * time.Minute), "sha256:a"}, + {day.Add(100 * time.Minute), "sha256:b"}, + {day.Add(3 * time.Hour), "sha256:c"}, + } { + if got := snapshotDigestAt(t, db, "idx", tc.at); got != tc.want { + t.Errorf("snapshot at %s = %q, want %q", tc.at, got, tc.want) + } + } + }) +} + func TestMetadataSnapshots_Prune(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { now := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 83fee97..5de909a 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -1,7 +1,6 @@ package handler import ( - "context" "crypto/sha256" "encoding/hex" "net/http" @@ -169,10 +168,11 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques // and detached signatures valid. func (h *APKHandler) handleMetadata(w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string) { cacheKey := h.metadataCacheKey(repository, upstreamURL, path) - h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, apkEcosystem, cacheKey, "*/*") if h.snapshotsEnabled() { - h.recordSnapshot(context.WithoutCancel(r.Context()), cacheKey) + h.serveMetadataWithSnapshot(w, r, upstreamURL+"/"+path, cacheKey) + return } + h.proxy.ProxyCached(w, r, upstreamURL+"/"+path, apkEcosystem, cacheKey, "*/*") } // metadataCacheKey derives the metadata cache key from the repository name, diff --git a/internal/handler/apk_snapshots.go b/internal/handler/apk_snapshots.go index 0f29c85..d581980 100644 --- a/internal/handler/apk_snapshots.go +++ b/internal/handler/apk_snapshots.go @@ -1,8 +1,12 @@ package handler import ( + "bytes" "context" "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" "io" "net/http" "strconv" @@ -101,22 +105,80 @@ func (h *APKHandler) serveSnapshot(w http.ResponseWriter, r *http.Request, repos } } -// recordSnapshot keeps the document now cached under cacheKey as a snapshot -// when it differs from the newest one kept. It runs after the document was -// served, so a failure only means this version is recorded on a later -// request instead. -func (h *APKHandler) recordSnapshot(ctx context.Context, cacheKey string) { - entry, err := h.proxy.DB.GetMetadataCache(apkEcosystem, cacheKey) - if err != nil || entry == nil || !entry.ContentDigest.Valid { - return - } - digest := entry.ContentDigest.String - latest, err := h.proxy.DB.LatestMetadataSnapshot(apkEcosystem, cacheKey) - if err != nil || (latest != nil && latest.ContentDigest == digest) { - return +// serveMetadataWithSnapshot is ProxyCached for an index or signature that +// keeps each version it serves as a snapshot. A version is recorded from the +// document the request serves, not by reading the cache again afterwards: a +// concurrent refresh can replace the cached copy at any time, and the version +// it replaced would then never be recorded. +func (h *APKHandler) serveMetadataWithSnapshot(w http.ResponseWriter, r *http.Request, upstream, cacheKey string) { + p := h.proxy + ctx := context.WithoutCancel(r.Context()) + + // A cached copy within its TTL was recorded by the request that fetched + // it; this only catches one stored before snapshots were enabled, or + // whose recording failed. + if entry := p.freshMetadataEntry(apkEcosystem, cacheKey); entry != nil { + seen := time.Now() + if p.serveCachedMetadata(w, r, apkEcosystem, cacheKey) { + h.recordStoredSnapshot(ctx, cacheKey, entry, seen) + return + } } - h.storeSnapshot(ctx, cacheKey, entry, time.Now()) + body, contentType, contentEncoding, err := p.fetchOrCacheMetadata(r.Context(), apkEcosystem, cacheKey, upstream, "identity", nil, "*/*") + if err != nil { + p.writeMetadataFetchError(w, err) + return + } + // Record before responding, so the version is kept by the time a + // client has it. + h.recordServedSnapshot(ctx, cacheKey, body, contentType, contentEncoding, time.Now()) + p.writeMetadataCachedResponseWithEncoding(w, r, apkEcosystem, cacheKey, body, contentType, contentEncoding) +} + +// recordServedSnapshot keeps body, the document a request is serving, as the +// version current from observed, unless the version current then already has +// its bytes. +func (h *APKHandler) recordServedSnapshot(ctx context.Context, cacheKey string, body []byte, contentType, contentEncoding string, observed time.Time) { + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + if !h.snapshotNeeded(cacheKey, digest, observed) { + return + } + storagePath := apkSnapshotStoragePath(cacheKey, digest, observed) + if _, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(body)); err != nil { + h.proxy.Logger.Warn("failed to store apk snapshot", "path", storagePath, "error", err) + h.discardSnapshotBlob(ctx, storagePath) + return + } + h.addSnapshot(ctx, &database.MetadataSnapshot{ + Ecosystem: apkEcosystem, + Name: cacheKey, + StoragePath: storagePath, + ContentType: sql.NullString{String: contentType, Valid: contentType != ""}, + ContentEncoding: sql.NullString{String: contentEncoding, Valid: contentEncoding != ""}, + ContentDigest: digest, + Size: int64(len(body)), + ValidFromMS: observed.UnixMilli(), + }) +} + +// recordStoredSnapshot keeps the cached document entry describes as the +// version current from observed, unless the version current then already +// has its digest. +func (h *APKHandler) recordStoredSnapshot(ctx context.Context, cacheKey string, entry *database.MetadataCacheEntry, observed time.Time) { + if !entry.ContentDigest.Valid || !h.snapshotNeeded(cacheKey, entry.ContentDigest.String, observed) { + return + } + h.storeSnapshot(ctx, cacheKey, entry, observed) +} + +// snapshotNeeded reports whether a version with digest seen at observed +// differs from the version current then. It is only a shortcut that saves +// copying the document: AddMetadataSnapshot makes the same check. +func (h *APKHandler) snapshotNeeded(cacheKey, digest string, observed time.Time) bool { + current, err := h.proxy.DB.MetadataSnapshotAt(apkEcosystem, cacheKey, observed.Add(time.Millisecond)) + return err == nil && (current == nil || current.ContentDigest != digest) } // storeSnapshot records the cached document entry as the version current @@ -130,7 +192,7 @@ func (h *APKHandler) storeSnapshot(ctx context.Context, cacheKey string, entry * if !ok { return } - added, err := h.proxy.DB.AddMetadataSnapshot(&database.MetadataSnapshot{ + h.addSnapshot(ctx, &database.MetadataSnapshot{ Ecosystem: apkEcosystem, Name: cacheKey, StoragePath: storagePath, @@ -140,12 +202,18 @@ func (h *APKHandler) storeSnapshot(ctx context.Context, cacheKey string, entry * Size: size, ValidFromMS: validFrom.UnixMilli(), }) +} + +// addSnapshot adds the row for a snapshot whose blob is stored, and deletes +// the blob when the row isn't added, as when another request recorded the +// same version first. +func (h *APKHandler) addSnapshot(ctx context.Context, snap *database.MetadataSnapshot) { + added, err := h.proxy.DB.AddMetadataSnapshot(snap) if err != nil { - h.proxy.Logger.Warn("failed to record apk snapshot", "key", cacheKey, "error", err) + h.proxy.Logger.Warn("failed to record apk snapshot", "key", snap.Name, "error", err) } if err != nil || !added { - // Another request recorded this version first. - h.discardSnapshotBlob(ctx, storagePath) + h.discardSnapshotBlob(ctx, snap.StoragePath) } } diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go index 38d4b08..b0f109d 100644 --- a/internal/handler/apk_snapshots_test.go +++ b/internal/handler/apk_snapshots_test.go @@ -1,6 +1,7 @@ package handler import ( + "io" "net/http" "net/http/httptest" "strings" @@ -206,6 +207,81 @@ func TestAPKSnapshots_SameVersionRecordedTwice(t *testing.T) { expectAPKResponse(t, h, "/alpine@"+day+apkSnapshotIndexPath, http.StatusOK, "index v1") } +// pausingWriter blocks the first Write until release is closed, after +// signalling on writing. +type pausingWriter struct { + *httptest.ResponseRecorder + once sync.Once + writing chan struct{} + release chan struct{} +} + +func (w *pausingWriter) Write(b []byte) (int, error) { + w.once.Do(func() { + close(w.writing) + <-w.release + }) + return w.ResponseRecorder.Write(b) +} + +// TestAPKSnapshots_OverlappingRequests pauses a request while it writes +// index v1, lets a second request fetch and serve v2, and then lets the first +// one finish. Both versions must be kept, in the order they were fetched, +// with readable blobs. +func TestAPKSnapshots_OverlappingRequests(t *testing.T) { + u := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, u.server.URL) + target := "/alpine" + apkSnapshotIndexPath + + first := &pausingWriter{ + ResponseRecorder: httptest.NewRecorder(), + writing: make(chan struct{}), + release: make(chan struct{}), + } + done := make(chan struct{}) + go func() { + defer close(done) + h.Routes().ServeHTTP(first, httptest.NewRequest(http.MethodGet, target, nil)) + }() + select { + case <-first.writing: + case <-done: + t.Fatal("first request finished without writing a body") + } + + u.index.Store("index v2") + expectAPKResponse(t, h, target, http.StatusOK, "index v2") + close(first.release) + <-done + if first.Code != http.StatusOK || first.Body.String() != "index v1" { + t.Fatalf("first request = %d %q, want 200 with index v1", first.Code, first.Body.String()) + } + + var snaps []database.MetadataSnapshot + if err := db.Select(&snaps, `SELECT `+ + `id, ecosystem, name, storage_path, content_type, content_encoding, content_digest, size, valid_from_ms `+ + `FROM metadata_snapshots ORDER BY valid_from_ms, id`); err != nil { + t.Fatal(err) + } + if len(snaps) != 2 { + t.Fatalf("%d snapshots recorded, want 2", len(snaps)) + } + for i, want := range []string{"index v1", "index v2"} { + body, err := h.proxy.Storage.Open(t.Context(), snaps[i].StoragePath) + if err != nil { + t.Fatalf("snapshot %d blob unreadable: %v", i, err) + } + got, _ := io.ReadAll(body) + _ = body.Close() + if string(got) != want { + t.Errorf("snapshot %d = %q, want %q", i, got, want) + } + if snaps[i].Size != int64(len(want)) { + t.Errorf("snapshot %d size = %d, want %d", i, snaps[i].Size, len(want)) + } + } +} + // TestAPKSnapshots_ConcurrentReplicas records the same index version from // two replicas sharing a Postgres database, many times at once. One snapshot // row must remain, with a readable blob, and the other copies are deleted. From 9458dc84e1ccdeace927c990a22510e819bf644a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:29:11 +0100 Subject: [PATCH 024/198] Share one PURL-to-handler mapping between the mirror and the inventory packageurl.HandlerPackage returns the ecosystem and name the handlers store a package URL under. The mirror and the inventory both use it, which settles where they disagreed: distribution namespaces of apk, alpm, bsd, deb and rpm are dropped, PyPI and NuGet names use purl's comparison form, and Maven names are group:artifact as the handler stores them, where the mirror used group/artifact. --- internal/inventory/inventory.go | 93 ++++------------------------ internal/inventory/inventory_test.go | 4 +- internal/mirror/files.go | 14 ----- internal/mirror/source.go | 21 +------ internal/mirror/source_test.go | 27 +++++--- internal/packageurl/handler.go | 59 ++++++++++++++++++ internal/packageurl/handler_test.go | 55 ++++++++++++++++ 7 files changed, 150 insertions(+), 123 deletions(-) create mode 100644 internal/packageurl/handler.go create mode 100644 internal/packageurl/handler_test.go diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index ccd9ecd..9a13a8b 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "strings" - "unicode" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" @@ -33,36 +32,16 @@ type Component struct { PackagePURL string } -// ecosystems maps the PURL types the proxy serves to the ecosystem name its -// handlers store packages under. -var ecosystems = map[string]string{ - "alpm": "arch", - "apk": "alpine", - "bsd": "freebsd", - "cargo": "cargo", - "chef": "chef", - "composer": "composer", - "conan": "conan", - "conda": "conda", - "cran": "cran", - "deb": "deb", - "gem": "gem", - "golang": "golang", - "hex": "hex", - "julia": "julia", - "maven": "maven", - "npm": "npm", - "nuget": "nuget", - "pub": "pub", - "pypi": "pypi", - "rpm": "rpm", - "swift": "swift", +// servedTypes are the PURL types the proxy serves packages of and the +// inventory records. +var servedTypes = map[string]bool{ + "alpm": true, "apk": true, "bsd": true, "cargo": true, "chef": true, + "composer": true, "conan": true, "conda": true, "cran": true, "deb": true, + "gem": true, "golang": true, "hex": true, "julia": true, "maven": true, + "npm": true, "nuget": true, "pub": true, "pypi": true, "rpm": true, + "swift": true, } -// distroTypes are PURL types whose namespace names the distribution -// ("debian", "alpine"), not part of the package name the proxy stores. -var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} - // ErrUnsupportedPURL is returned for a package URL whose type the proxy // doesn't serve. var ErrUnsupportedPURL = errors.New("package type is not served by the proxy") @@ -79,13 +58,12 @@ func FromPURL(s string) (Component, error) { if err != nil { return Component{}, err } - ecosystem, ok := ecosystems[p.Type] - if !ok { + if !servedTypes[p.Type] { return Component{}, fmt.Errorf("%w: %s", ErrUnsupportedPURL, p.Type) } - name, err := handlerName(p) - if err != nil { - return Component{}, err + ecosystem, name := packageurl.HandlerPackage(p) + if ecosystem == "conda" && !strings.Contains(name, "/") { + return Component{}, ErrNoCondaChannel } packagePURL, _ := packageurl.MakeCacheStrings(ecosystem, name, "") if packagePURL == "" { @@ -94,53 +72,6 @@ func FromPURL(s string) (Component, error) { return Component{Ecosystem: ecosystem, Name: name, Version: p.Version, PackagePURL: packagePURL}, nil } -// handlerName returns the package name the handler for p's type stores. -func handlerName(p *purl.PURL) (string, error) { - switch { - case distroTypes[p.Type]: - // The namespace is the distribution; the handlers store the bare - // package name. - return p.Name, nil - case p.Type == "conda" && p.Namespace == "": - // The conda handler stores "channel/name". - channel := p.Qualifier("channel") - if channel == "" { - return "", ErrNoCondaChannel - } - return channel + "/" + p.Name, nil - case p.Type == "nuget": - // The NuGet handler lowercases package IDs. - return strings.ToLower(p.FullName()), nil - case p.Type == "pypi": - return normalizePyPIName(p.FullName()), nil - } - return p.FullName(), nil -} - -// normalizePyPIName normalizes a PyPI project name as PEP 503 does: runs of -// "-", "_" and "." become one "-", and letters are lowercased. The PyPI -// handler stores the name from the file name, which wheels and current sdists -// spell with "_" for any of these, so "zope.interface" is cached as -// "zope-interface". (An old sdist named "zope.interface-5.0.tar.gz" is -// stored with the dot and doesn't match.) -func normalizePyPIName(name string) string { - var b strings.Builder - b.Grow(len(name)) - separator := false - for _, r := range name { - if r == '-' || r == '_' || r == '.' { - separator = true - continue - } - if separator && b.Len() > 0 { - b.WriteByte('-') - } - separator = false - b.WriteRune(unicode.ToLower(r)) - } - return b.String() -} - // syftMainModule is the property syft sets on modules found in a Go binary, // naming the module the binary was built from. const syftMainModule = "syft:metadata:mainModule" diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index e5b6225..818129a 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -128,7 +128,7 @@ func TestFromPURLMatchesHandlerNames(t *testing.T) { if err != nil { t.Fatalf("FromPURL(%q): %v", tt.sbomPURL, err) } - want, _ := packageurl.MakeCacheStrings(ecosystems[tt.purlType], tt.handlerName, "") + want, _ := packageurl.MakeCacheStrings(packageurl.HandlerEcosystem(tt.purlType), tt.handlerName, "") if want == "" { t.Fatalf("handler name %q has no package URL", tt.handlerName) } @@ -137,7 +137,7 @@ func TestFromPURLMatchesHandlerNames(t *testing.T) { } }) } - for purlType := range ecosystems { + for purlType := range servedTypes { if !covered[purlType] { t.Errorf("no case for PURL type %q", purlType) } diff --git a/internal/mirror/files.go b/internal/mirror/files.go index 8cf61fe..7486e8c 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -116,17 +116,3 @@ func fileQualifiers(ecosystem string, qualifiers map[string]string) map[string]s } return kept } - -// condaName returns a conda package name with its channel, taken from the -// namespace or the channel qualifier. Without either it stays bare, and -// mirroring refuses it. -func condaName(namespace, name, channel string) string { - switch { - case namespace != "": - return namespace + "/" + name - case channel != "": - return channel + "/" + name - default: - return name - } -} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index ecc3a0c..22812b6 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" "github.com/git-pkgs/sbom" @@ -28,18 +29,6 @@ func (pv PackageVersion) String() string { return fmt.Sprintf("pkg:%s/%s@%s", pv.Ecosystem, pv.Name, pv.Version) } -// handlerEcosystem returns the ecosystem name the proxy's handlers store -// packages of a PURL type under. purl names gem and composer after their -// registries, rubygems and packagist; the handlers use the type. -func handlerEcosystem(purlType string) string { - switch purlType { - case "gem", "composer": - return purlType - default: - return purl.PURLTypeToEcosystem(purlType) - } -} - // Source produces PackageVersion items for mirroring. type Source interface { Enumerate(ctx context.Context, fn func(PackageVersion) error) error @@ -140,13 +129,7 @@ func (s *PURLSource) listVersions(ctx context.Context, versions VersionLister, e // mirrorTarget returns the ecosystem, name and file qualifiers a PURL is // mirrored under. func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]string) { - ecosystem = handlerEcosystem(p.Type) - name = p.Name - if ecosystem == "conda" { - name = condaName(p.Namespace, p.Name, p.Qualifier("channel")) - } else if p.Namespace != "" { - name = p.Namespace + "/" + p.Name - } + ecosystem, name = packageurl.HandlerPackage(p) qualifiers = fileQualifiers(ecosystem, p.Qualifiers.Map()) if ecosystem == "julia" { qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} diff --git a/internal/mirror/source_test.go b/internal/mirror/source_test.go index 9c69afe..faddf35 100644 --- a/internal/mirror/source_test.go +++ b/internal/mirror/source_test.go @@ -3,6 +3,7 @@ package mirror import ( "context" "encoding/json" + "slices" "strings" "testing" ) @@ -229,26 +230,38 @@ func marshalJSON(t *testing.T, v any) []byte { return data } -// The mirror files packages under the ecosystem names the handlers use, so -// a mirrored package and the same package cached by a client share one name. +// The mirror files packages under the ecosystem and package names the +// handlers use, so a mirrored package and the same package cached by a client +// share one name. func TestPURLSourceUsesHandlerEcosystemNames(t *testing.T) { source := &PURLSource{PURLs: []string{ "pkg:gem/rake@13.2.1", "pkg:composer/psr/log@3.0.2", - "pkg:apk/curl@8.14.1-r2", + "pkg:apk/alpine/curl@8.14.1-r2", "pkg:npm/left-pad@1.3.0", + "pkg:maven/org.slf4j/slf4j-api@2.0.13", + "pkg:pypi/Zope.Interface@6.4", + "pkg:nuget/Newtonsoft.Json@13.0.3", }} var got []string if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { - got = append(got, pv.Ecosystem) + got = append(got, pv.Ecosystem+" "+pv.Name) return nil }); err != nil { t.Fatalf("Enumerate() error = %v", err) } - want := []string{"gem", "composer", "alpine", "npm"} - if strings.Join(got, ",") != strings.Join(want, ",") { - t.Errorf("ecosystems = %v, want %v", got, want) + want := []string{ + "gem rake", + "composer psr/log", + "alpine curl", + "npm left-pad", + "maven org.slf4j:slf4j-api", + "pypi zope-interface", + "nuget newtonsoft.json", + } + if !slices.Equal(got, want) { + t.Errorf("packages = %q, want %q", got, want) } } diff --git a/internal/packageurl/handler.go b/internal/packageurl/handler.go new file mode 100644 index 0000000..6056fcb --- /dev/null +++ b/internal/packageurl/handler.go @@ -0,0 +1,59 @@ +package packageurl + +import ( + "github.com/git-pkgs/purl" +) + +// distroTypes are PURL types whose namespace names the distribution +// ("alpine", "debian"), not part of the package name the handlers store. +var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} + +// HandlerEcosystem returns the ecosystem name the proxy's handlers store +// packages of a PURL type under. purl names gem and composer after their +// registries, rubygems and packagist; the handlers use the type. +func HandlerEcosystem(purlType string) string { + switch purlType { + case "gem", "composer": + return purlType + default: + return purl.PURLTypeToEcosystem(purlType) + } +} + +// HandlerPackage returns the ecosystem and package name the proxy's handlers +// store p under, so a package named in a PURL, such as one from an SBOM, +// matches the package a client fetched through the proxy: +// +// - apk, alpm, bsd, deb and rpm: the namespace names the distribution and +// is dropped +// - conda: the channel, from the namespace or the channel qualifier, is +// part of the name ("conda-forge/numpy"); without either the name stays +// bare +// - pypi and nuget: the name in purl's comparison form, PEP 503 for PyPI +// and lowercased for NuGet, as the handlers store them +// +// Other types use purl's full name, which keeps the namespace: "@babel/core", +// or "org.slf4j:slf4j-api" for Maven, as its handler stores it. +func HandlerPackage(p *purl.PURL) (ecosystem, name string) { + ecosystem = HandlerEcosystem(p.Type) + switch { + case distroTypes[p.Type]: + return ecosystem, p.Name + case p.Type == "conda": + return ecosystem, condaName(p.Namespace, p.Name, p.Qualifier("channel")) + case p.Type == "pypi" || p.Type == "nuget": + return ecosystem, p.ComparisonForm().FullName() + } + return ecosystem, p.FullName() +} + +func condaName(namespace, name, channel string) string { + switch { + case namespace != "": + return namespace + "/" + name + case channel != "": + return channel + "/" + name + default: + return name + } +} diff --git a/internal/packageurl/handler_test.go b/internal/packageurl/handler_test.go new file mode 100644 index 0000000..5ae9443 --- /dev/null +++ b/internal/packageurl/handler_test.go @@ -0,0 +1,55 @@ +package packageurl + +import ( + "testing" + + "github.com/git-pkgs/purl" +) + +// HandlerPackage is used by both the mirror and the project inventory, so a +// mirrored package, a package listed in an SBOM and the same package fetched +// by a client end up under one name. +func TestHandlerPackage(t *testing.T) { + tests := []struct { + purl string + ecosystem string + name string + }{ + // purl names these after their registries; the handlers use the type. + {"pkg:gem/rake@13.2.1", "gem", "rake"}, + {"pkg:composer/psr/log@3.0.2", "composer", "psr/log"}, + // Distribution namespaces are dropped. + {"pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64", "alpine", "busybox"}, + {"pkg:apk/curl@8.14.1-r2", "alpine", "curl"}, + {"pkg:alpm/arch/glibc@2.39-1", "arch", "glibc"}, + {"pkg:bsd/freebsd/curl@8.7.1", "freebsd", "curl"}, + {"pkg:deb/debian/libc6@2.36-9?arch=amd64", "deb", "libc6"}, + {"pkg:rpm/fedora/curl@8.6.0-7.fc40", "rpm", "curl"}, + // The conda channel is part of the name. + {"pkg:conda/numpy@1.26.4?channel=conda-forge", "conda", "conda-forge/numpy"}, + {"pkg:conda/conda-forge/numpy@1.26.4", "conda", "conda-forge/numpy"}, + {"pkg:conda/numpy@1.26.4", "conda", "numpy"}, + // PyPI names are normalized as PEP 503 does, NuGet IDs lowercased. + {"pkg:pypi/Zope.Interface@6.4", "pypi", "zope-interface"}, + {"pkg:pypi/zope_interface", "pypi", "zope-interface"}, + {"pkg:nuget/Newtonsoft.Json@13.0.3", "nuget", "newtonsoft.json"}, + // Other namespaces stay part of the name. + {"pkg:npm/%40babel/core@7.23.0", "npm", "@babel/core"}, + // The Maven handler stores "group:artifact". + {"pkg:maven/org.slf4j/slf4j-api@2.0.13", "maven", "org.slf4j:slf4j-api"}, + {"pkg:docker/library/alpine@3.20", "docker", "library/alpine"}, + {"pkg:julia/Example@0.5.3?uuid=7876af07-990d-54b4-ab0e-23690620f79a", "julia", "Example"}, + } + for _, tt := range tests { + t.Run(tt.purl, func(t *testing.T) { + p, err := purl.Parse(tt.purl) + if err != nil { + t.Fatal(err) + } + ecosystem, name := HandlerPackage(p) + if ecosystem != tt.ecosystem || name != tt.name { + t.Errorf("HandlerPackage(%q) = %q, %q, want %q, %q", tt.purl, ecosystem, name, tt.ecosystem, tt.name) + } + }) + } +} From 479f26e09e6128e71f7c424e335406326d30562f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 30 Oct 2026 19:29:11 +0100 Subject: [PATCH 025/198] Start project pruning with the project API and move enrichment routes out of serve --- internal/server/projects_api.go | 7 +++++-- internal/server/server.go | 25 +++++++++++++------------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/server/projects_api.go b/internal/server/projects_api.go index f297a9a..750be17 100644 --- a/internal/server/projects_api.go +++ b/internal/server/projects_api.go @@ -1,6 +1,7 @@ package server import ( + "context" "errors" "io" "log/slog" @@ -82,11 +83,13 @@ type ProjectResponse struct { Components []ProjectComponent `json:"components"` } -// mountProjectAPI registers /api/projects when projects.enabled is set. -func (s *Server) mountProjectAPI(r chi.Router) { +// mountProjectAPI registers /api/projects when projects.enabled is set, and +// starts pruning projects that sent no SBOM within projects.max_age. +func (s *Server) mountProjectAPI(ctx context.Context, r chi.Router) { if s.cfg.Projects.Enabled { registerProjectRoutes(r, NewProjectsAPIHandler(s.db, s.logger), s.cfg.Projects.TokenExpanded()) } + s.startProjectPruning(ctx) } // registerProjectRoutes mounts /api/projects. Reading is open like the rest diff --git a/internal/server/server.go b/internal/server/server.go index 0ec6470..1812957 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -242,6 +242,17 @@ func (s *Server) Start(listeners ...net.Listener) error { return s.serve(listener) } +// mountEnrichmentAPI registers the API endpoints for enrichment data. +func (s *Server) mountEnrichmentAPI(r chi.Router) { + apiHandler := NewAPIHandler(enrichment.New(s.logger), s.db) + r.Get("/api/package/{ecosystem}/*", apiHandler.HandlePackagePath) + r.Get("/api/vulns/{ecosystem}/*", apiHandler.HandleVulnsPath) + r.Post("/api/outdated", apiHandler.HandleOutdated) + r.Post("/api/bulk", apiHandler.HandleBulkLookup) + r.Get("/api/search", apiHandler.HandleSearch) + r.Get("/api/packages", apiHandler.HandlePackagesList) +} + func (s *Server) serve(listener net.Listener) error { metadataClient, artifactClient := s.newUpstreamClients(s.cfg.ParseArtifactTimeout()) @@ -379,16 +390,7 @@ func (s *Server) serve(listener net.Listener) error { r.Delete("/api/cache/{ecosystem}/*", s.handleCacheDelete) } - // API endpoints for enrichment data - enrichSvc := enrichment.New(s.logger) - apiHandler := NewAPIHandler(enrichSvc, s.db) - - r.Get("/api/package/{ecosystem}/*", apiHandler.HandlePackagePath) - r.Get("/api/vulns/{ecosystem}/*", apiHandler.HandleVulnsPath) - r.Post("/api/outdated", apiHandler.HandleOutdated) - r.Post("/api/bulk", apiHandler.HandleBulkLookup) - r.Get("/api/search", apiHandler.HandleSearch) - r.Get("/api/packages", apiHandler.HandlePackagesList) + s.mountEnrichmentAPI(r) // Start background context (used by mirror jobs and cleanup) bgCtx, bgCancel := context.WithCancel(context.Background()) @@ -419,8 +421,7 @@ func (s *Server) serve(listener net.Listener) error { } // Project inventory endpoints (opt-in via projects.enabled or PROXY_PROJECTS_ENABLED) - s.mountProjectAPI(r) - s.startProjectPruning(bgCtx) + s.mountProjectAPI(bgCtx, r) s.http = &http.Server{ Addr: s.cfg.Listen, From 4ae534eba391826c6c60babf92632063b10e3357 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 07:58:51 +0100 Subject: [PATCH 026/198] Fetch an image's blobs concurrently when mirroring The image mirror fetched the config and layer blobs of a manifest one after another, so a run took as long as all layers together. It now fetches up to three at once, as many as a Docker pull does by default. For a Home Assistant image (550 MB in ten blobs) that would leave mostly the time of its largest layer. --- internal/handler/container_mirror.go | 46 +++++++++++++++++------ internal/handler/container_mirror_test.go | 36 ++++++++++++++++++ 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index ddd3022..82a98cf 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -11,7 +11,10 @@ import ( "regexp" "slices" "strings" + "sync" "time" + + "golang.org/x/sync/errgroup" ) const ( @@ -19,6 +22,9 @@ const ( maxMirrorIndexDepth = 2 // mirrorErrorBodyLimit bounds how much of an error response is quoted. mirrorErrorBodyLimit = 200 + // mirrorBlobConcurrency is how many blobs of one manifest are fetched at + // once, as many as a Docker pull fetches by default. + mirrorBlobConcurrency = 3 dockerReferenceTypeAnnotation = "vnd.docker.reference.type" dockerReferenceDigestAnnotation = "vnd.docker.reference.digest" @@ -439,6 +445,7 @@ func (m *imageMirror) mirrorBlobs(ctx context.Context, manifest *ociManifest) er if manifest.Config != nil { blobs = append([]ociDescriptor{*manifest.Config}, blobs...) } + var digests []string for _, blob := range blobs { if len(blob.URLs) > 0 || strings.Contains(blob.MediaType, "nondistributable") || strings.Contains(blob.MediaType, "foreign") { m.result.Warnings = append(m.result.Warnings, fmt.Sprintf("skipped non-distributable layer %s", blob.Digest)) @@ -451,34 +458,49 @@ func (m *imageMirror) mirrorBlobs(ctx context.Context, manifest *ociManifest) er continue } m.seen[blob.Digest] = true - if err := m.mirrorBlob(ctx, blob.Digest); err != nil { - return err - } + digests = append(digests, blob.Digest) } - return nil + + // Large layers take most of the time, so they are fetched side by side. + var mu sync.Mutex + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(mirrorBlobConcurrency) + for _, digest := range digests { + g.Go(func() error { + size, fetched, err := m.mirrorBlob(gctx, digest) + if err != nil || !fetched { + return err + } + mu.Lock() + m.result.Fetched++ + m.result.Bytes += size + mu.Unlock() + return nil + }) + } + return g.Wait() } -// mirrorBlob stores a blob under the same identity handleBlobDownload uses. -func (m *imageMirror) mirrorBlob(ctx context.Context, digest string) error { +// mirrorBlob stores a blob under the same identity handleBlobDownload uses, +// and returns its size when it had to be fetched. +func (m *imageMirror) mirrorBlob(ctx context.Context, digest string) (int64, bool, error) { proxy := m.h.proxy cached, err := proxy.HasCachedArtifact(ctx, "oci", m.cacheName, digest, digest) if err != nil { - return fmt.Errorf("blob %s: %w", digest, err) + return 0, false, fmt.Errorf("blob %s: %w", digest, err) } if cached { - return nil + return 0, false, nil } result, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "oci", m.cacheName, digest, digest, fmt.Sprintf("%s/v2/%s/blobs/%s", m.registryURL, m.upstreamName, digest), digest) if err != nil { - return fmt.Errorf("blob %s: %w", digest, err) + return 0, false, fmt.Errorf("blob %s: %w", digest, err) } if result.Reader != nil { _ = result.Reader.Close() } - m.result.Fetched++ - m.result.Bytes += result.Artifact.Size - return nil + return result.Artifact.Size, true, nil } // mirrorResponse collects the handler's manifest response in memory. diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index 149fb23..61b0ceb 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -10,8 +10,10 @@ import ( "net/url" "strconv" "strings" + "sync" "sync/atomic" "testing" + "time" "github.com/git-pkgs/registries/fetch" ) @@ -503,3 +505,37 @@ func TestMirrorImageRefetchesBlobMissingFromStorage(t *testing.T) { t.Errorf("second run fetched %d blobs, want the one missing from storage", got) } } + +func TestMirrorImageFetchesBlobsConcurrently(t *testing.T) { + registry, _ := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + + // "single" has a config and a layer blob. Each request waits until the + // other is in flight too, or gives up after a while. + var inFlight atomic.Int32 + var concurrent atomic.Bool + both := make(chan struct{}) + var once sync.Once + registry.onBlob = func(string) { + defer inFlight.Add(-1) + if inFlight.Add(1) >= 2 { + concurrent.Store(true) + once.Do(func() { close(both) }) + } + select { + case <-both: + case <-time.After(2 * time.Second): + } + } + result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"single"}}, nil) + if err != nil { + t.Fatalf("MirrorImage: %v", err) + } + if !concurrent.Load() { + t.Error("blobs were fetched one after another, want them fetched concurrently") + } + if result.Fetched != 3 { + t.Errorf("fetched = %d, want the manifest and its two blobs", result.Fetched) + } +} From a79a58b04ef1a1e84c7d629ce7ec7e810d16e4ba Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 08:27:39 +0100 Subject: [PATCH 027/198] Guard the fake registry's manifests in mirror tests TestMirrorImageKeepsTagMovedByPullDuringWalk moves a tag while a blob is served. With blobs fetched concurrently, another blob request reads the manifest map at the same time, which the race detector caught on Windows. --- internal/handler/container_mirror_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index 61b0ceb..d2cb54b 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -21,8 +21,10 @@ import ( // fakeImageRegistry serves manifests and blobs of a small multi-arch image, // a single-arch image and a moved tag. It fails with 503 once offline. type fakeImageRegistry struct { - server *httptest.Server - offline atomic.Bool + server *httptest.Server + offline atomic.Bool + // mu guards manifests against a test changing it while blobs are served. + mu sync.RWMutex manifests map[string]fakeManifest blobs map[string]string blobGets atomic.Int32 @@ -119,7 +121,10 @@ func newFakeImageRegistry(t *testing.T) (*fakeImageRegistry, fakeImage) { http.Error(w, "offline", http.StatusServiceUnavailable) return } - if doc, ok := r.manifests[req.URL.Path]; ok { + r.mu.RLock() + doc, ok := r.manifests[req.URL.Path] + r.mu.RUnlock() + if ok { w.Header().Set("Content-Type", doc.contentType) w.Header().Set("Docker-Content-Digest", sha256Digest([]byte(doc.body))) if req.Method == http.MethodGet && !strings.Contains(req.URL.Path, "/manifests/sha256:") { @@ -434,7 +439,9 @@ func TestMirrorImageKeepsTagMovedByPullDuringWalk(t *testing.T) { var once atomic.Bool registry.onBlob = func(string) { if once.CompareAndSwap(false, true) { + registry.mu.Lock() registry.manifests["/v2/team/app/manifests/single"] = moved + registry.mu.Unlock() if response := pull(t, h, "/team/app/manifests/single", dockerManifestAccept); response.Code != http.StatusOK { t.Errorf("client pull during the walk: status %d", response.Code) } From 7a77baf01a58f0bfc280fa63499aac18052c53d2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 11:33:31 +0100 Subject: [PATCH 028/198] Keep namespace/name for Swift package URLs in the handler mapping With purl v0.2.1, FullName returns a Swift package as its registry identity, scope.name. The mirror splits namespace/name again to find the scope or the source repository, so HandlerPackage keeps that form, as mirrorTarget did before the two mappings were shared. --- internal/packageurl/handler.go | 5 +++++ internal/packageurl/handler_test.go | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/internal/packageurl/handler.go b/internal/packageurl/handler.go index 6056fcb..ee3005e 100644 --- a/internal/packageurl/handler.go +++ b/internal/packageurl/handler.go @@ -31,6 +31,9 @@ func HandlerEcosystem(purlType string) string { // bare // - pypi and nuget: the name in purl's comparison form, PEP 503 for PyPI // and lowercased for NuGet, as the handlers store them +// - swift: namespace/name, the registry scope or the source repository +// with the package, which the Swift mirror resolves; purl's full name +// is the registry identity scope.name // // Other types use purl's full name, which keeps the namespace: "@babel/core", // or "org.slf4j:slf4j-api" for Maven, as its handler stores it. @@ -43,6 +46,8 @@ func HandlerPackage(p *purl.PURL) (ecosystem, name string) { return ecosystem, condaName(p.Namespace, p.Name, p.Qualifier("channel")) case p.Type == "pypi" || p.Type == "nuget": return ecosystem, p.ComparisonForm().FullName() + case p.Type == "swift" && p.Namespace != "": + return ecosystem, p.Namespace + "/" + p.Name } return ecosystem, p.FullName() } diff --git a/internal/packageurl/handler_test.go b/internal/packageurl/handler_test.go index 5ae9443..7283b9f 100644 --- a/internal/packageurl/handler_test.go +++ b/internal/packageurl/handler_test.go @@ -39,6 +39,13 @@ func TestHandlerPackage(t *testing.T) { {"pkg:maven/org.slf4j/slf4j-api@2.0.13", "maven", "org.slf4j:slf4j-api"}, {"pkg:docker/library/alpine@3.20", "docker", "library/alpine"}, {"pkg:julia/Example@0.5.3?uuid=7876af07-990d-54b4-ab0e-23690620f79a", "julia", "Example"}, + // Swift keeps namespace/name, the scope or the source repository, + // which the Swift mirror splits again; purl's full name would be the + // registry identity "scope.name". + {"pkg:swift/Acme/Widget@1.0.0", "swift", "Acme/Widget"}, + {"pkg:swift/github.com/apple/swift-nio@2.62.0", "swift", "github.com/apple/swift-nio"}, + // Go module paths keep their case. + {"pkg:golang/github.com/Azure/foo@v1.2.0", "golang", "github.com/Azure/foo"}, } for _, tt := range tests { t.Run(tt.purl, func(t *testing.T) { From 1cadc4a7ba68980f8d1c6bd5a78490aa8ad68712 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 11:33:31 +0100 Subject: [PATCH 029/198] Compare Go main modules with their case purl v0.2.1 keeps the case of Go module paths, so the inventory no longer lowercases syft's mainModule. Module paths are case-sensitive: github.com/azure/foo is a dependency of github.com/Azure/foo, not part of it. --- internal/inventory/inventory.go | 8 ++++---- internal/inventory/inventory_test.go | 25 ++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 9a13a8b..d115fb7 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -99,7 +99,7 @@ func Parse(data []byte) (components []Component, skipped int, err error) { seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[strings.ToLower(c.Name)]) { + if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { skipped++ continue } @@ -161,14 +161,14 @@ func lockFileRoots(doc *sbom.SBOM) map[string]bool { } // mainModules returns the Go modules the SBOM's binaries were built from, -// which are the project's own, lowercased: syft keeps the module path's case -// while FromPURL lowercases golang names. +// which are the project's own. Module paths are case-sensitive and package +// URLs keep their case, so they are compared as they are. func mainModules(pkgs []sbom.Package) map[string]bool { own := map[string]bool{} for _, pkg := range pkgs { for _, prop := range pkg.Properties { if prop.Name == syftMainModule && prop.Value != "" { - own[strings.ToLower(prop.Value)] = true + own[prop.Value] = true } } } diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 818129a..3fbe113 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -246,9 +246,9 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { } } -func TestParseSkipsMainModuleRegardlessOfCase(t *testing.T) { - // syft keeps the module path's case in mainModule; package URLs of Go - // modules are lowercased. +func TestParseSkipsMixedCaseMainModule(t *testing.T) { + // syft keeps the module path's case in mainModule, and so does the + // package URL. doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ {"bom-ref":"bin","type":"library","name":"github.com/Azure/foo","version":"v1.2.0","purl":"pkg:golang/github.com/Azure/foo@v1.2.0", "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]}, @@ -263,3 +263,22 @@ func TestParseSkipsMainModuleRegardlessOfCase(t *testing.T) { t.Errorf("components = %+v, skipped = %d; want only cobra, 1 skipped", components, skipped) } } + +func TestParseMainModuleKeepsCase(t *testing.T) { + // Go module paths are case-sensitive, and package URLs keep their case, + // so github.com/azure/foo is another module than the main module + // github.com/Azure/foo, and a dependency. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"bin","type":"library","name":"github.com/Azure/foo","version":"v1.2.0","purl":"pkg:golang/github.com/Azure/foo@v1.2.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]}, + {"bom-ref":"other","type":"library","name":"github.com/azure/foo","version":"v0.1.0","purl":"pkg:golang/github.com/azure/foo@v0.1.0", + "properties":[{"name":"syft:metadata:mainModule","value":"github.com/Azure/foo"}]} + ]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "github.com/azure/foo" || skipped != 1 { + t.Errorf("components = %+v, skipped = %d; want only github.com/azure/foo, 1 skipped", components, skipped) + } +} From 29863e858a62d5c313e97eec627721475023dcd8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 13:53:37 +0100 Subject: [PATCH 030/198] Refresh the most used packages in the background mirror.refresh mirrors, for the top packages per ecosystem by recent downloads, the newest release of each major line a version is cached on. Releases within cooldown or on the denylist are skipped, and PyPI and conda get only the kinds of files clients fetched before. max_versions and max_bytes cap one run. Instances sharing a database take turns as for schedules. --- go.mod | 2 +- internal/config/config.go | 2 + internal/config/mirror.go | 99 ++++++++ internal/config/mirror_test.go | 81 ++++++ internal/database/most_used.go | 78 ++++++ internal/database/most_used_test.go | 94 +++++++ internal/handler/release_allowed.go | 19 ++ internal/handler/release_allowed_test.go | 44 ++++ internal/mirror/files.go | 54 ++++ internal/mirror/refresh.go | 138 ++++++++++ internal/mirror/refresh_test.go | 306 +++++++++++++++++++++++ internal/mirror/refresh_versions.go | 67 +++++ internal/mirror/refresh_versions_test.go | 79 ++++++ internal/mirror/schedule.go | 15 +- internal/mirror/schedule_test.go | 33 +++ internal/mirror/source.go | 3 + internal/server/mirror_schedule.go | 31 ++- internal/server/mirror_schedule_test.go | 22 ++ 18 files changed, 1158 insertions(+), 9 deletions(-) create mode 100644 internal/database/most_used.go create mode 100644 internal/database/most_used_test.go create mode 100644 internal/handler/release_allowed.go create mode 100644 internal/handler/release_allowed_test.go create mode 100644 internal/mirror/refresh.go create mode 100644 internal/mirror/refresh_test.go create mode 100644 internal/mirror/refresh_versions.go create mode 100644 internal/mirror/refresh_versions_test.go diff --git a/go.mod b/go.mod index 2ce65e2..2b1ee11 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/git-pkgs/registries v0.13.0 github.com/git-pkgs/sbom v0.1.7 github.com/git-pkgs/spdx v0.3.3 + github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 github.com/jmoiron/sqlx v1.4.0 github.com/klauspost/compress v1.20.3 @@ -76,7 +77,6 @@ require ( github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect github.com/git-pkgs/pom v0.1.8 // indirect - github.com/git-pkgs/vers v0.7.2 // indirect github.com/git-pkgs/vulns v0.2.4 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index c09a1ba..8f5ca1f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1614,6 +1614,8 @@ func (c *Config) validateCacheArtifacts() error { return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served") case len(c.Mirror.Schedules) > 0: return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror.schedules: mirrored artifacts would never be served") + case c.Mirror.Refresh != nil: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror.refresh: mirrored artifacts would never be served") case c.Hosted.Enabled: return fmt.Errorf("storage.cache_artifacts: false cannot be combined with hosted.enabled: pushed packages would never be served") } diff --git a/internal/config/mirror.go b/internal/config/mirror.go index 7f6c668..473b744 100644 --- a/internal/config/mirror.go +++ b/internal/config/mirror.go @@ -15,6 +15,102 @@ type MirrorConfig struct { // Schedules re-mirror container image tags at a fixed interval, so a // moved tag is picked up even when nobody pulls it. Schedules []MirrorSchedule `json:"schedules" yaml:"schedules"` + + // Refresh, when set, keeps the most used packages up to date in the + // background. + Refresh *MirrorRefresh `json:"refresh" yaml:"refresh"` +} + +const ( + defaultMirrorRefreshTop = 50 + defaultMirrorRefreshSince = 14 * 24 * time.Hour +) + +// MirrorRefresh mirrors new releases of the packages clients use most, so +// they are cached before the first client asks for them. +type MirrorRefresh struct { + // Interval is how often the refresh runs, at least one minute (e.g. + // "24h"). + Interval string `json:"interval" yaml:"interval"` + + // Top is how many packages per ecosystem are refreshed, the ones with + // the most downloads among those used within Since. Default: 50. + Top int `json:"top" yaml:"top"` + + // Since is how recently a package must have been downloaded to be + // refreshed (e.g. "14d"). Default: 14d. + Since string `json:"since" yaml:"since"` + + // MaxVersions caps how many new versions one run mirrors. 0 means no + // cap. + MaxVersions int `json:"max_versions" yaml:"max_versions"` + + // MaxBytes caps how much one run downloads (e.g. "2GB"). Empty means no + // cap. + MaxBytes string `json:"max_bytes" yaml:"max_bytes"` + + interval time.Duration + since time.Duration + maxBytes int64 +} + +// IntervalDuration returns the parsed Interval. Validate must run first. +func (r *MirrorRefresh) IntervalDuration() time.Duration { return r.interval } + +// TopPackages returns Top, or its default. +func (r *MirrorRefresh) TopPackages() int { + if r.Top == 0 { + return defaultMirrorRefreshTop + } + return r.Top +} + +// SinceDuration returns the parsed Since, or its default. Validate must run +// first. +func (r *MirrorRefresh) SinceDuration() time.Duration { return r.since } + +// MaxVersionsPerRun returns MaxVersions; 0 means no cap. +func (r *MirrorRefresh) MaxVersionsPerRun() int { return r.MaxVersions } + +// MaxBytesPerRun returns the parsed MaxBytes; 0 means no cap. Validate must +// run first. +func (r *MirrorRefresh) MaxBytesPerRun() int64 { return r.maxBytes } + +func (r *MirrorRefresh) validate() error { + d, err := time.ParseDuration(r.Interval) + if err != nil { + return fmt.Errorf("invalid mirror.refresh.interval %q: %w", r.Interval, err) + } + if d < minMirrorScheduleInterval { + return fmt.Errorf("invalid mirror.refresh.interval %q: must be at least %s", r.Interval, minMirrorScheduleInterval) + } + r.interval = d + if r.Top < 0 { + return fmt.Errorf("invalid mirror.refresh.top %d: must not be negative", r.Top) + } + r.since = defaultMirrorRefreshSince + if r.Since != "" { + since, err := parseDayDuration(r.Since) + if err != nil { + return fmt.Errorf("invalid mirror.refresh.since %q: %w", r.Since, err) + } + if since <= 0 { + return fmt.Errorf("invalid mirror.refresh.since %q: must be positive", r.Since) + } + r.since = since + } + if r.MaxVersions < 0 { + return fmt.Errorf("invalid mirror.refresh.max_versions %d: must not be negative", r.MaxVersions) + } + r.maxBytes = 0 + if r.MaxBytes != "" { + n, err := ParseSize(r.MaxBytes) + if err != nil { + return fmt.Errorf("invalid mirror.refresh.max_bytes %q: %w", r.MaxBytes, err) + } + r.maxBytes = n + } + return nil } // MirrorSchedule is a set of image PURLs mirrored every Interval. @@ -45,6 +141,9 @@ func (m *MirrorConfig) Validate() error { return err } } + if m.Refresh != nil { + return m.Refresh.validate() + } return nil } diff --git a/internal/config/mirror_test.go b/internal/config/mirror_test.go index ae1220d..2a53585 100644 --- a/internal/config/mirror_test.go +++ b/internal/config/mirror_test.go @@ -87,3 +87,84 @@ mirror: t.Error("Validate() accepted a 30s interval in the second schedule") } } + +func TestMirrorRefreshValidate(t *testing.T) { + tests := []struct { + name string + refresh *MirrorRefresh + wantErr bool + }{ + {name: "defaults", refresh: &MirrorRefresh{Interval: "24h"}}, + {name: "all fields", refresh: &MirrorRefresh{Interval: "12h", Top: 20, Since: "14d", MaxVersions: 100, MaxBytes: "2GB"}}, + {name: "missing interval", refresh: &MirrorRefresh{}, wantErr: true}, + {name: "interval under a minute", refresh: &MirrorRefresh{Interval: "10s"}, wantErr: true}, + {name: "negative top", refresh: &MirrorRefresh{Interval: "1h", Top: -1}, wantErr: true}, + {name: "bad since", refresh: &MirrorRefresh{Interval: "1h", Since: "recently"}, wantErr: true}, + {name: "negative since", refresh: &MirrorRefresh{Interval: "1h", Since: "-1d"}, wantErr: true}, + {name: "negative max_versions", refresh: &MirrorRefresh{Interval: "1h", MaxVersions: -1}, wantErr: true}, + {name: "bad max_bytes", refresh: &MirrorRefresh{Interval: "1h", MaxBytes: "lots"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := MirrorConfig{Refresh: tt.refresh} + err := m.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestMirrorRefreshDefaults(t *testing.T) { + m := MirrorConfig{Refresh: &MirrorRefresh{Interval: "24h"}} + if err := m.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + r := m.Refresh + if r.IntervalDuration() != 24*time.Hour || r.TopPackages() != 50 || r.SinceDuration() != 14*24*time.Hour || + r.MaxVersionsPerRun() != 0 || r.MaxBytesPerRun() != 0 { + t.Errorf("got interval %v top %d since %v max_versions %d max_bytes %d, want 24h 50 336h 0 0", + r.IntervalDuration(), r.TopPackages(), r.SinceDuration(), r.MaxVersionsPerRun(), r.MaxBytesPerRun()) + } +} + +func TestValidateRejectsMirrorRefreshWithoutCachedArtifacts(t *testing.T) { + cfg := Default() + cfg.Mirror.Refresh = &MirrorRefresh{Interval: "24h"} + cfg.Storage.CacheArtifacts = false + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted mirror.refresh with storage.cache_artifacts: false") + } +} + +func TestLoadYAMLMirrorRefresh(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + data := ` +mirror: + refresh: + interval: 12h + top: 10 + since: 7d + max_versions: 200 + max_bytes: 1GB +` + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + r := cfg.Mirror.Refresh + if r == nil { + t.Fatal("mirror.refresh not loaded") + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if r.IntervalDuration() != 12*time.Hour || r.TopPackages() != 10 || r.SinceDuration() != 7*24*time.Hour || + r.MaxVersionsPerRun() != 200 || r.MaxBytesPerRun() != 1<<30 { + t.Errorf("got interval %v top %d since %v max_versions %d max_bytes %d", + r.IntervalDuration(), r.TopPackages(), r.SinceDuration(), r.MaxVersionsPerRun(), r.MaxBytesPerRun()) + } +} diff --git a/internal/database/most_used.go b/internal/database/most_used.go new file mode 100644 index 0000000..c68e833 --- /dev/null +++ b/internal/database/most_used.go @@ -0,0 +1,78 @@ +package database + +import ( + "time" +) + +// UsedPackage is a package clients download from the cache, with the files +// cached for it. +type UsedPackage struct { + Ecosystem string + Name string + PURL string + // Hits counts the downloads of the files downloaded since the cutoff. + Hits int64 + Files []UsedFile +} + +// UsedFile is a cached file of a UsedPackage. +type UsedFile struct { + Version string + Filename string + UpstreamURL string +} + +// MostUsedPackages returns, for each ecosystem, the top packages with the most +// downloads among the cached files downloaded since cutoff, ordered by +// ecosystem and then by hits. Each lists every file cached for it. Files +// from local sources, which have no upstream, are left out. +func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) { + var rows []struct { + Ecosystem string `db:"ecosystem"` + Name string `db:"name"` + PURL string `db:"purl"` + Hits int64 `db:"hits"` + } + err := db.Select(&rows, db.Rebind(` + SELECT ecosystem, name, purl, hits FROM ( + SELECT p.ecosystem, p.name, p.purl, SUM(a.hit_count) AS hits, + ROW_NUMBER() OVER (PARTITION BY p.ecosystem ORDER BY SUM(a.hit_count) DESC, p.name) AS rank + FROM artifacts a + JOIN versions v ON v.purl = a.version_purl + JOIN packages p ON p.purl = v.package_purl + WHERE a.storage_path IS NOT NULL AND a.source = ? + AND COALESCE(a.last_accessed_at, a.fetched_at) >= ? + GROUP BY p.ecosystem, p.name, p.purl + ) ranked + WHERE rank <= ? + ORDER BY ecosystem, hits DESC, name + `), SourceUpstream, cutoff.UTC(), top) + if err != nil { + return nil, err + } + packages := make([]UsedPackage, len(rows)) + for i, row := range rows { + packages[i] = UsedPackage{Ecosystem: row.Ecosystem, Name: row.Name, PURL: row.PURL, Hits: row.Hits} + var files []struct { + VersionPURL string `db:"version_purl"` + Filename string `db:"filename"` + UpstreamURL string `db:"upstream_url"` + } + err := db.Select(&files, db.Rebind(` + SELECT a.version_purl, a.filename, a.upstream_url + FROM artifacts a + JOIN versions v ON v.purl = a.version_purl + WHERE v.package_purl = ? AND a.storage_path IS NOT NULL AND a.source = ? + ORDER BY a.version_purl, a.filename + `), row.PURL, SourceUpstream) + if err != nil { + return nil, err + } + for _, f := range files { + packages[i].Files = append(packages[i].Files, UsedFile{ + Version: VersionFromPURL(f.VersionPURL), Filename: f.Filename, UpstreamURL: f.UpstreamURL, + }) + } + } + return packages, nil +} diff --git a/internal/database/most_used_test.go b/internal/database/most_used_test.go new file mode 100644 index 0000000..8704f90 --- /dev/null +++ b/internal/database/most_used_test.go @@ -0,0 +1,94 @@ +package database + +import ( + "reflect" + "testing" + "time" +) + +// seedUsedArtifact stores a cached artifact with hits, last downloaded at +// accessed. +func seedUsedArtifact(t *testing.T, db *DB, ecosystem, name, version, filename string, hits int64, accessed time.Time) { + t.Helper() + packagePURL := "pkg:" + ecosystem + "/" + name + versionPURL := packagePURL + "@" + version + if err := db.UpsertPackage(&Package{PURL: packagePURL, Ecosystem: ecosystem, Name: name}); err != nil { + t.Fatal(err) + } + if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: packagePURL}); err != nil { + t.Fatal(err) + } + if err := db.UpsertArtifact(&Artifact{VersionPURL: versionPURL, Filename: filename, UpstreamURL: "https://example.test/" + filename}); err != nil { + t.Fatal(err) + } + if err := db.MarkArtifactCached(versionPURL, filename, ecosystem+"/"+filename, "sha256:00", 1, "application/octet-stream"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET hit_count = ?, last_accessed_at = ? WHERE version_purl = ? AND filename = ?"), + hits, accessed.UTC(), versionPURL, filename); err != nil { + t.Fatal(err) + } +} + +func TestMostUsedPackages(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seedUsedArtifact(t, db, "npm", "lodash", "4.17.20", "lodash-4.17.20.tgz", 5, now.Add(-time.Hour)) + seedUsedArtifact(t, db, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", 4, now.Add(-20*24*time.Hour)) + seedUsedArtifact(t, db, "npm", "react", "18.3.1", "react-18.3.1.tgz", 10, now.Add(-time.Hour)) + seedUsedArtifact(t, db, "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", 100, now.Add(-30*24*time.Hour)) + seedUsedArtifact(t, db, "cargo", "serde", "1.0.200", "serde-1.0.200.crate", 1, now.Add(-time.Hour)) + + // Not cached: never counts. + if err := db.UpsertArtifact(&Artifact{VersionPURL: "pkg:npm/react@19.0.0", Filename: "react-19.0.0.tgz", UpstreamURL: "https://example.test/x"}); err != nil { + t.Fatal(err) + } + + got, err := db.MostUsedPackages(now.Add(-14*24*time.Hour), 2) + if err != nil { + t.Fatal(err) + } + want := []UsedPackage{ + {Ecosystem: "cargo", Name: "serde", PURL: "pkg:cargo/serde", Hits: 1, Files: []UsedFile{ + {Version: "1.0.200", Filename: "serde-1.0.200.crate", UpstreamURL: "https://example.test/serde-1.0.200.crate"}, + }}, + // react has more hits than lodash. Every cached file of lodash is + // listed, but only recent downloads count. + {Ecosystem: "npm", Name: "react", PURL: "pkg:npm/react", Hits: 10, Files: []UsedFile{ + {Version: "18.3.1", Filename: "react-18.3.1.tgz", UpstreamURL: "https://example.test/react-18.3.1.tgz"}, + }}, + {Ecosystem: "npm", Name: "lodash", PURL: "pkg:npm/lodash", Hits: 5, Files: []UsedFile{ + {Version: "4.17.20", Filename: "lodash-4.17.20.tgz", UpstreamURL: "https://example.test/lodash-4.17.20.tgz"}, + {Version: "4.17.21", Filename: "lodash-4.17.21.tgz", UpstreamURL: "https://example.test/lodash-4.17.21.tgz"}, + }}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("MostUsedPackages =\n%+v\nwant\n%+v", got, want) + } + + top1, err := db.MostUsedPackages(now.Add(-14*24*time.Hour), 1) + if err != nil { + t.Fatal(err) + } + if len(top1) != 2 || top1[0].Name != "serde" || top1[1].Name != "react" { + t.Errorf("top 1 per ecosystem = %+v, want serde and react", top1) + } + }) +} + +func TestMostUsedPackagesSkipsLocalArtifacts(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seedUsedArtifact(t, db, "npm", "internal-lib", "1.0.0", "internal-lib-1.0.0.tgz", 50, now) + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET source = ? WHERE filename = ?"), SourceLocal, "internal-lib-1.0.0.tgz"); err != nil { + t.Fatal(err) + } + got, err := db.MostUsedPackages(now.Add(-time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Errorf("MostUsedPackages = %+v, want none: local packages have no upstream to refresh from", got) + } + }) +} diff --git a/internal/handler/release_allowed.go b/internal/handler/release_allowed.go new file mode 100644 index 0000000..4c70214 --- /dev/null +++ b/internal/handler/release_allowed.go @@ -0,0 +1,19 @@ +package handler + +import ( + "time" +) + +// ReleaseAllowed reports whether clients would be served a version published +// at publishedAt: the denylist doesn't name it, and cooldown, where the +// ecosystem applies it, doesn't withhold it. Without a publish time a version +// counts as withheld wherever cooldown applies. +func (p *Proxy) ReleaseAllowed(ecosystem, name, version string, publishedAt time.Time) bool { + if p.versionDenied(ecosystem, name, version) { + return false + } + if !CooldownSupported(ecosystem) || !p.cooldownApplies(ecosystem, name) { + return true + } + return !publishedAt.IsZero() && p.cooldown().IsAllowed(ecosystem, canonicalPackagePURL(ecosystem, name), publishedAt) +} diff --git a/internal/handler/release_allowed_test.go b/internal/handler/release_allowed_test.go new file mode 100644 index 0000000..0d8dd88 --- /dev/null +++ b/internal/handler/release_allowed_test.go @@ -0,0 +1,44 @@ +package handler + +import ( + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/denylist" +) + +func TestReleaseAllowed(t *testing.T) { + deny, err := denylist.New([]string{"pkg:npm/lodash@4.17.22"}) + if err != nil { + t.Fatal(err) + } + p := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}, Denylist: deny} + old := time.Now().Add(-10 * 24 * time.Hour) + fresh := time.Now().Add(-time.Hour) + + tests := []struct { + name string + ecosystem, pkg, version string + published time.Time + want bool + }{ + {"old enough", "npm", "lodash", "4.17.21", old, true}, + {"within cooldown", "npm", "lodash", "4.17.23", fresh, false}, + {"denied", "npm", "lodash", "4.17.22", old, false}, + {"no publish time under cooldown", "npm", "lodash", "4.17.24", time.Time{}, false}, + // deb has no publish times, so cooldown never applies to it. + {"ecosystem without cooldown", "deb", "curl", "8.5.0-2", time.Time{}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := p.ReleaseAllowed(tt.ecosystem, tt.pkg, tt.version, tt.published); got != tt.want { + t.Errorf("ReleaseAllowed = %v, want %v", got, tt.want) + } + }) + } + + if !(&Proxy{}).ReleaseAllowed("npm", "lodash", "4.17.23", fresh) { + t.Error("a proxy without cooldown or denylist refused a release") + } +} diff --git a/internal/mirror/files.go b/internal/mirror/files.go index 8cf61fe..87ce895 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -2,6 +2,8 @@ package mirror import ( "fmt" + "path" + "regexp" "slices" "strings" @@ -22,6 +24,12 @@ type fileEcosystem struct { // check refuses a package the ecosystem can't mirror, before its files // are listed. check func(pv PackageVersion) error + // variant names the kind of a file, such as a wheel's tags or a conda + // package's platform, so a refresh mirrors the same kinds of files of a + // new version as were cached of an old one. cachedVariant names it for + // the filename cacheKey returns. + variant func(file registries.Artifact) string + cachedVariant func(cacheFilename string) string } var fileEcosystems = map[string]fileEcosystem{ @@ -37,6 +45,8 @@ var fileEcosystems = map[string]fileEcosystem{ } return name, version, file.Filename, true }, + variant: func(file registries.Artifact) string { return pypiVariant(file.Filename) }, + cachedVariant: pypiVariant, }, "conda": { qualifiers: []string{"subdir", "build", "type"}, @@ -60,9 +70,47 @@ var fileEcosystems = map[string]fileEcosystem{ } return nil }, + variant: func(file registries.Artifact) string { + subdir, _ := file.Metadata["subdir"].(string) + return condaVariant(subdir, file.Filename) + }, + cachedVariant: func(cacheFilename string) string { + subdir, filename := path.Split(cacheFilename) + return condaVariant(strings.TrimSuffix(subdir, "/"), filename) + }, }, } +// pypiVariant names a wheel by its tags, python-abi-platform; every sdist +// is one variant. +func pypiVariant(filename string) string { + stem, wheel := strings.CutSuffix(filename, ".whl") + if !wheel { + return "sdist" + } + parts := strings.Split(stem, "-") + if len(parts) < 5 { //nolint:mnd // name, version and three tags + return stem + } + return strings.Join(parts[len(parts)-3:], "-") +} + +// condaBuildHash matches the hash and build number a conda build string +// ends in, such as h8753938_0 in py312h8753938_0. +var condaBuildHash = regexp.MustCompile(`h[0-9a-f]{7}_\d+$`) + +// condaVariant names a conda package file by its subdir, the part of its +// build string before the hash, such as py312, and its type. +func condaVariant(subdir, filename string) string { + fileType := condaFileType(filename) + stem := strings.TrimSuffix(strings.TrimSuffix(filename, ".conda"), ".tar.bz2") + build := stem[strings.LastIndex(stem, "-")+1:] + if loc := condaBuildHash.FindStringIndex(build); loc != nil { + build = build[:loc[0]] + } + return subdir + "/" + build + "/" + fileType +} + // condaFileType returns the PURL type qualifier of a conda package file. func condaFileType(filename string) string { switch { @@ -78,6 +126,9 @@ func condaFileType(filename string) string { // selection returns the qualifiers of pv that select files, as // "key=value" pairs, for error messages. func (e fileEcosystem) selection(pv PackageVersion) string { + if pv.variants != nil { + return "the kinds of files cached of earlier versions" + } var pairs []string for _, key := range e.qualifiers { if value := pv.Qualifiers[key]; value != "" { @@ -89,6 +140,9 @@ func (e fileEcosystem) selection(pv PackageVersion) string { // selects reports whether file matches every selecting qualifier of pv. func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool { + if pv.variants != nil && !pv.variants[e.variant(file)] { + return false + } attrs := e.attrs(file) for _, key := range e.qualifiers { if want := pv.Qualifiers[key]; want != "" && attrs[key] != want { diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go new file mode 100644 index 0000000..bdc4c29 --- /dev/null +++ b/internal/mirror/refresh.go @@ -0,0 +1,138 @@ +package mirror + +import ( + "context" + "fmt" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries" +) + +// Refresh mirrors new releases of the packages clients download most, so +// they are cached before the first client asks for them. +type Refresh struct { + // Top is how many packages per ecosystem are refreshed. + Top int + // Since is how recently a package must have been downloaded. + Since time.Duration + // MaxVersions stops a run after mirroring this many versions. 0 means + // no cap. + MaxVersions int + // MaxBytes stops a run once it has downloaded this much. It is checked + // before each version, so the last one may go over it. 0 means no cap. + MaxBytes int64 +} + +func (r Refresh) spent(tracker *progressTracker) bool { + return (r.MaxVersions > 0 && tracker.completed.Load() >= int64(r.MaxVersions)) || + (r.MaxBytes > 0 && tracker.bytes.Load() >= r.MaxBytes) +} + +// ReleaseLister lists every release of a package, with its publish time and +// status, for refreshes. +type ReleaseLister interface { + ListReleases(ctx context.Context, ecosystem, name string) ([]registries.Version, error) +} + +// ListReleases implements ReleaseLister. +func (r RegistryVersions) ListReleases(ctx context.Context, ecosystem, name string) ([]registries.Version, error) { + return r.fetchVersions(ctx, ecosystem, name) +} + +// releaseLister lists releases where Versions lists versions, so both come +// from the configured upstream. +func (m *Mirror) releaseLister() ReleaseLister { //nolint:ireturn // any lister will do + if lister, ok := m.Versions.(ReleaseLister); ok { + return lister + } + return RegistryVersions{} +} + +// Refresh mirrors, for each of the most used packages, the newest release +// of each major line it has a version cached on, as clients would be served +// it: releases within cooldown or on the denylist are left out. Packages it +// can't list are counted as failed. +func (m *Mirror) Refresh(ctx context.Context, r Refresh) (*Progress, error) { + packages, err := m.db.MostUsedPackages(time.Now().Add(-r.Since), r.Top) + if err != nil { + return nil, fmt.Errorf("listing the most used packages: %w", err) + } + tracker := newProgressTracker() + tracker.phase.Store("downloading") +refresh: + for _, pkg := range packages { + if ctx.Err() != nil { + break + } + items, err := m.refreshItems(ctx, pkg) + if err != nil { + m.recordFailure(tracker, PackageVersion{Ecosystem: pkg.Ecosystem, Name: pkg.Name}, err) + continue + } + for _, item := range items { + if r.spent(tracker) { + m.logger.Info("mirror refresh reached its cap for one run", + "mirrored", tracker.completed.Load(), "bytes", tracker.bytes.Load()) + break refresh + } + tracker.total.Add(1) + m.mirrorOne(ctx, item, tracker) + } + } + tracker.phase.Store("complete") + p := tracker.snapshot() + return &p, nil +} + +// refreshItems returns the versions to mirror for a used package. +func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([]PackageVersion, error) { + p, err := purl.Parse(pkg.PURL) + if err != nil { + return nil, err + } + ecosystem, name, _ := mirrorTarget(p) + if !refreshable(ecosystem) { + return nil, nil + } + releases, err := m.releaseLister().ListReleases(ctx, ecosystem, name) + if err != nil { + return nil, fmt.Errorf("listing releases: %w", err) + } + + var cached []string + seen := map[string]bool{} + for _, f := range pkg.Files { + if !seen[f.Version] { + seen[f.Version] = true + cached = append(cached, f.Version) + } + } + allowed := func(v registries.Version) bool { + return m.proxy == nil || m.proxy.ReleaseAllowed(pkg.Ecosystem, pkg.Name, v.Number, v.PublishedAt) + } + + var items []PackageVersion + for _, version := range newestPerCachedLine(ecosystem, cached, releases, allowed) { + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: version} + if eco, ok := fileEcosystems[ecosystem]; ok { + pv.variants = map[string]bool{} + for _, f := range pkg.Files { + pv.variants[eco.cachedVariant(f.Filename)] = true + } + } + items = append(items, pv) + } + return items, nil +} + +// refreshable reports whether a refresh can list the releases of an +// ecosystem's packages. +func refreshable(ecosystem string) bool { + if isImageEcosystem(ecosystem) { + return false + } + _, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", nil) + return err == nil +} diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go new file mode 100644 index 0000000..684b44c --- /dev/null +++ b/internal/mirror/refresh_test.go @@ -0,0 +1,306 @@ +package mirror + +import ( + "context" + "fmt" + "io" + "net/http" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries" + "github.com/git-pkgs/registries/fetch" +) + +// servingFetcher answers every fetch with body and records the URLs. +type servingFetcher struct { + body string + mu sync.Mutex + urls []string +} + +func (f *servingFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { + return f.FetchWithHeaders(ctx, url, nil) +} + +func (f *servingFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) { + f.mu.Lock() + f.urls = append(f.urls, url) + f.mu.Unlock() + return &fetch.Artifact{Body: io.NopCloser(strings.NewReader(f.body)), Size: int64(len(f.body)), ContentType: "application/octet-stream"}, nil +} + +func (f *servingFetcher) Head(context.Context, string) (int64, string, error) { + return int64(len(f.body)), "application/octet-stream", nil +} + +func (f *servingFetcher) fetched() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.urls...) +} + +func (f *servingFetcher) reset() { + f.mu.Lock() + defer f.mu.Unlock() + f.urls = nil +} + +// stubReleases lists releases by "ecosystem/name". +type stubReleases map[string][]registries.Version + +func (s stubReleases) ListReleases(_ context.Context, ecosystem, name string) ([]registries.Version, error) { + r, ok := s[ecosystem+"/"+name] + if !ok { + return nil, fmt.Errorf("%w: %s/%s", registries.ErrNotFound, ecosystem, name) + } + return r, nil +} + +func (s stubReleases) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) { + r, err := s.ListReleases(ctx, ecosystem, name) + numbers := make([]string, len(r)) + for i, v := range r { + numbers[i] = v.Number + } + return numbers, err +} + +func (s stubReleases) FindVersion(ctx context.Context, ecosystem, name, version string) (registries.Version, error) { + r, err := s.ListReleases(ctx, ecosystem, name) + if err != nil { + return registries.Version{}, err + } + for _, v := range r { + if v.Number == version { + return v, nil + } + } + return registries.Version{}, registries.ErrNotFound +} + +func oldReleases(numbers ...string) []registries.Version { + r := releases(numbers...) + for i := range r { + r[i].PublishedAt = time.Now().Add(-time.Duration(len(r)-i) * 30 * 24 * time.Hour) + } + return r +} + +func setupRefreshMirror(t *testing.T, body string) (*Mirror, *servingFetcher) { + t.Helper() + m := setupTestMirror(t, 1) + fetcher := &servingFetcher{body: body} + m.proxy.Fetcher = fetcher + return m, fetcher +} + +// cacheNPM downloads a version the way a client would, so it counts as used. +func cacheNPM(t *testing.T, m *Mirror, name, version string) { + t.Helper() + result, err := m.proxy.GetOrFetchArtifact(context.Background(), "npm", name, version, "") + if err != nil { + t.Fatalf("caching %s@%s: %v", name, version, err) + } + _ = result.Reader.Close() +} + +func fetchedMatching(urls []string, part string) []string { + var out []string + for _, u := range urls { + if strings.Contains(u, part) { + out = append(out, u) + } + } + return out +} + +func TestRefreshMirrorsNewerVersionOnCachedLine(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21", "5.0.0")} + fetcher.reset() + + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Completed != 1 || progress.Failed != 0 { + t.Errorf("progress = %+v, want one mirrored", progress) + } + urls := fetcher.fetched() + if len(fetchedMatching(urls, "lodash-4.17.21.tgz")) != 1 || len(fetchedMatching(urls, "5.0.0")) != 0 { + t.Errorf("fetched %v, want lodash 4.17.21 only", urls) + } + + // The new version is cached now, so the next run has nothing to do. + progress, err = m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("second Refresh: %v", err) + } + if progress.Completed != 0 || progress.Failed != 0 { + t.Errorf("second run progress = %+v, want nothing to do", progress) + } +} + +func TestRefreshKeepsCooldown(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + r := oldReleases("4.17.20", "4.17.21") + r[1].PublishedAt = time.Now().Add(-time.Hour) + m.Versions = stubReleases{"npm/lodash": r} + m.proxy.Cooldown = &cooldown.Config{Default: "3d"} + fetcher.reset() + + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want a version within cooldown left alone", progress, fetcher.fetched()) + } +} + +func TestRefreshOnlyTopPackagesRecentlyUsed(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + cacheNPM(t, m, "react", "18.3.0") + cacheNPM(t, m, "react", "18.3.0") // a second download: react is used most + m.Versions = stubReleases{ + "npm/lodash": oldReleases("4.17.20", "4.17.21"), + "npm/react": oldReleases("18.3.0", "18.3.1"), + } + if err := m.db.RecordArtifactHit("pkg:npm/react@18.3.0", "react-18.3.0.tgz"); err != nil { + t.Fatal(err) + } + fetcher.reset() + + progress, err := m.Refresh(context.Background(), Refresh{Top: 1, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + urls := fetcher.fetched() + if progress.Completed != 1 || len(fetchedMatching(urls, "react-18.3.1.tgz")) != 1 || len(fetchedMatching(urls, "lodash")) != 0 { + t.Errorf("progress = %+v, fetched %v; want only react refreshed", progress, urls) + } +} + +func TestRefreshStopsAtMaxVersions(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + cacheNPM(t, m, "react", "18.3.0") + m.Versions = stubReleases{ + "npm/lodash": oldReleases("4.17.20", "4.17.21"), + "npm/react": oldReleases("18.3.0", "18.3.1"), + } + fetcher.reset() + + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour, MaxVersions: 1}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Completed != 1 || len(fetcher.fetched()) != 1 { + t.Errorf("progress = %+v, fetched %v; want one version", progress, fetcher.fetched()) + } +} + +func TestRefreshStopsAtMaxBytes(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "0123456789") + cacheNPM(t, m, "lodash", "4.17.20") + cacheNPM(t, m, "react", "18.3.0") + m.Versions = stubReleases{ + "npm/lodash": oldReleases("4.17.20", "4.17.21"), + "npm/react": oldReleases("18.3.0", "18.3.1"), + } + fetcher.reset() + + // The cap is checked before each download, so the first one goes over it. + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour, MaxBytes: 5}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Completed != 1 || progress.Bytes != 10 { + t.Errorf("progress = %+v, want one version of 10 bytes", progress) + } +} + +func TestRefreshCountsListingFailure(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{} + fetcher.reset() + + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Failed != 1 || len(progress.Errors) != 1 || progress.Errors[0].Name != "lodash" { + t.Errorf("progress = %+v, want lodash failed", progress) + } +} + +func TestRefreshMirrorsTheFileVariantsClientsFetched(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "wheel") + ctx := context.Background() + for _, f := range []string{"demo-1.0.0-py3-none-any.whl", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.whl"} { + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "pypi", "demo", "1.0.0", f, "https://files.example/"+f, "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + } + files := func(version string, names ...string) registries.Version { + v := registries.Version{Number: version, PublishedAt: time.Now().Add(-30 * 24 * time.Hour)} + for _, n := range names { + v.Artifacts = append(v.Artifacts, registries.Artifact{Filename: n, URL: "https://files.example/" + n}) + } + return v + } + m.Versions = stubReleases{"pypi/demo": { + files("1.0.0", "demo-1.0.0-py3-none-any.whl", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.whl"), + files("1.1.0", + "demo-1.1.0.tar.gz", + "demo-1.1.0-py3-none-any.whl", + "demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.whl", + "demo-1.1.0-cp313-cp313-manylinux_2_17_x86_64.whl", + "demo-1.1.0-cp312-cp312-win_amd64.whl"), + }} + fetcher.reset() + + progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + got := fetcher.fetched() + slices.Sort(got) + want := []string{ + "https://files.example/demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.whl", + "https://files.example/demo-1.1.0-py3-none-any.whl", + } + if progress.Completed != 1 || !slices.Equal(got, want) { + t.Errorf("progress = %+v, fetched %v; want %v", progress, got, want) + } +} + +func TestFileVariants(t *testing.T) { + tests := []struct { + ecosystem, cacheFilename, want string + }{ + {"pypi", "demo-1.0.0-py3-none-any.whl", "py3-none-any"}, + {"pypi", "demo-1.0.0-1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64"}, + {"pypi", "demo-1.0.0.tar.gz", "sdist"}, + {"pypi", "demo-1.0.0.zip", "sdist"}, + {"conda", "linux-64/numpy-1.26.4-py312h8753938_0.conda", "linux-64/py312/conda"}, + {"conda", "noarch/requests-2.32.3-pyhd8ed1ab_0.conda", "noarch/py/conda"}, + {"conda", "osx-arm64/zlib-1.3.1-h8359307_2.tar.bz2", "osx-arm64//tar.bz2"}, + } + for _, tt := range tests { + if got := fileEcosystems[tt.ecosystem].cachedVariant(tt.cacheFilename); got != tt.want { + t.Errorf("%s variant of %q = %q, want %q", tt.ecosystem, tt.cacheFilename, got, tt.want) + } + } +} diff --git a/internal/mirror/refresh_versions.go b/internal/mirror/refresh_versions.go new file mode 100644 index 0000000..b489170 --- /dev/null +++ b/internal/mirror/refresh_versions.go @@ -0,0 +1,67 @@ +package mirror + +import ( + "strconv" + + "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries" + "github.com/git-pkgs/vers" +) + +// newestPerCachedLine returns, for each major line a cached version is on, +// the newest release of that line that allowed accepts, when it is newer +// than every cached version of the line. A line is the major version, or +// 0.minor below 1.0, as caret ranges read them. Prereleases and yanked, +// retracted or deprecated releases are skipped. +func newestPerCachedLine(ecosystem string, cached []string, releases []registries.Version, allowed func(registries.Version) bool) []string { + scheme := purl.EcosystemToPURLType(ecosystem) + newestCached := map[string]string{} + var lines []string + for _, v := range cached { + line, ok := versionLine(v) + if !ok { + continue + } + current, seen := newestCached[line] + if !seen { + lines = append(lines, line) + } + if !seen || vers.CompareWithScheme(v, current, scheme) > 0 { + newestCached[line] = v + } + } + + newest := map[string]string{} + for _, r := range releases { + line, ok := versionLine(r.Number) + if !ok || r.Status != registries.StatusNone || vers.IsPrereleaseWithScheme(r.Number, scheme) { + continue + } + if _, cachedLine := newestCached[line]; !cachedLine || !allowed(r) { + continue + } + if current, seen := newest[line]; !seen || vers.CompareWithScheme(r.Number, current, scheme) > 0 { + newest[line] = r.Number + } + } + + var out []string + for _, line := range lines { + if v, ok := newest[line]; ok && vers.CompareWithScheme(v, newestCached[line], scheme) > 0 { + out = append(out, v) + } + } + return out +} + +// versionLine returns the major line of a version. +func versionLine(v string) (string, bool) { + info, err := vers.ParseVersion(v) + if err != nil { + return "", false + } + if info.Major == 0 { + return "0." + strconv.Itoa(info.Minor), true + } + return strconv.Itoa(info.Major), true +} diff --git a/internal/mirror/refresh_versions_test.go b/internal/mirror/refresh_versions_test.go new file mode 100644 index 0000000..6e1e262 --- /dev/null +++ b/internal/mirror/refresh_versions_test.go @@ -0,0 +1,79 @@ +package mirror + +import ( + "reflect" + "testing" + "time" + + "github.com/git-pkgs/registries" +) + +func releases(numbers ...string) []registries.Version { + out := make([]registries.Version, len(numbers)) + for i, n := range numbers { + out[i] = registries.Version{Number: n, PublishedAt: time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC)} + } + return out +} + +func TestNewestPerCachedLine(t *testing.T) { + allowAll := func(registries.Version) bool { return true } + yanked := releases("4.17.20", "4.17.21", "4.17.22") + yanked[2].Status = registries.StatusYanked + + tests := []struct { + name string + cached []string + releases []registries.Version + allowed func(registries.Version) bool + want []string + }{ + {name: "newer patch on the cached line", cached: []string{"4.17.20"}, + releases: releases("4.17.20", "4.17.21", "5.0.0"), want: []string{"4.17.21"}}, + {name: "newest already cached", cached: []string{"4.17.21"}, + releases: releases("4.17.20", "4.17.21", "5.0.0")}, + {name: "one per cached major line", cached: []string{"1.2.0", "2.0.0"}, + releases: releases("1.2.0", "1.3.0", "2.0.0", "2.1.0", "3.0.0"), want: []string{"1.3.0", "2.1.0"}}, + {name: "zero major lines are minor lines", cached: []string{"0.3.1"}, + releases: releases("0.3.1", "0.3.2", "0.4.0"), want: []string{"0.3.2"}}, + {name: "prereleases skipped", cached: []string{"2.0.0"}, + releases: releases("2.0.0", "2.1.0-beta.1")}, + {name: "yanked skipped", cached: []string{"4.17.20"}, + releases: yanked, want: []string{"4.17.21"}}, + {name: "newest withheld, older newer one taken", cached: []string{"4.17.20"}, + releases: releases("4.17.20", "4.17.21", "4.17.22"), + allowed: func(v registries.Version) bool { return v.Number != "4.17.22" }, + want: []string{"4.17.21"}}, + {name: "nothing allowed newer than cached", cached: []string{"4.17.20"}, + releases: releases("4.17.20", "4.17.21"), + allowed: func(v registries.Version) bool { return v.Number == "4.17.20" }}, + {name: "cached newer than every listed one", cached: []string{"4.17.25"}, + releases: releases("4.17.20", "4.17.21")}, + {name: "only the newest cached of a line counts", cached: []string{"4.17.19", "4.17.21"}, + releases: releases("4.17.19", "4.17.20", "4.17.21")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + allowed := tt.allowed + if allowed == nil { + allowed = allowAll + } + got := newestPerCachedLine("npm", tt.cached, tt.releases, allowed) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("newestPerCachedLine = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewestPerCachedLineUsesEcosystemOrdering(t *testing.T) { + // Maven orders 1.0.10 after 1.0.9, and a gem's 1.0.0.rc1 is a prerelease. + got := newestPerCachedLine("maven", []string{"1.0.9"}, releases("1.0.9", "1.0.10"), func(registries.Version) bool { return true }) + if !reflect.DeepEqual(got, []string{"1.0.10"}) { + t.Errorf("maven: got %v, want [1.0.10]", got) + } + got = newestPerCachedLine("gem", []string{"1.0.0"}, releases("1.0.0", "1.0.1.rc1"), func(registries.Version) bool { return true }) + if got != nil { + t.Errorf("gem: got %v, want nothing", got) + } +} diff --git a/internal/mirror/schedule.go b/internal/mirror/schedule.go index 1df9a75..244fd89 100644 --- a/internal/mirror/schedule.go +++ b/internal/mirror/schedule.go @@ -17,10 +17,12 @@ const ( scheduleJitter = 10 ) -// Schedule mirrors a set of image PURLs every Interval. +// Schedule mirrors a set of image PURLs every Interval, or refreshes the +// most used packages when Refresh is set. type Schedule struct { PURLs []string Platforms []string + Refresh *Refresh Interval time.Duration } @@ -102,7 +104,13 @@ func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) { } } started := time.Now() - progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms}) + var progress *Progress + var err error + if schedule.Refresh != nil { + progress, err = s.mirror.Refresh(ctx, *schedule.Refresh) + } else { + progress, err = s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms}) + } if ctx.Err() != nil { if s.Claim != nil && s.Release != nil { if err := s.Release(key); err != nil { @@ -140,6 +148,9 @@ func scheduleDelay(interval time.Duration, first bool, randN func(int64) int64) // does not inherit the old one's last run. func scheduleKey(schedule Schedule) string { h := sha256.New() + if r := schedule.Refresh; r != nil { + _, _ = fmt.Fprintf(h, "refresh top %d since %s max_versions %d max_bytes %d\n", r.Top, r.Since, r.MaxVersions, r.MaxBytes) + } for _, purl := range schedule.PURLs { _, _ = fmt.Fprintf(h, "purl %s\n", purl) } diff --git a/internal/mirror/schedule_test.go b/internal/mirror/schedule_test.go index b5b0143..0a6edc3 100644 --- a/internal/mirror/schedule_test.go +++ b/internal/mirror/schedule_test.go @@ -279,3 +279,36 @@ func TestScheduleKey(t *testing.T) { } } } + +func TestSchedulerRunsRefresh(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21")} + fetcher.reset() + + schedule := Schedule{Refresh: &Refresh{Top: 10, Since: time.Hour}, Interval: time.Hour} + s := NewScheduler(m, []Schedule{schedule}, m.logger) + s.run(context.Background(), 0, schedule) + + if urls := fetcher.fetched(); len(urls) != 1 || len(fetchedMatching(urls, "lodash-4.17.21.tgz")) != 1 { + t.Errorf("fetched %v, want lodash 4.17.21", urls) + } +} + +func TestScheduleKeyRefresh(t *testing.T) { + base := Schedule{Refresh: &Refresh{Top: 50, Since: time.Hour}, Interval: time.Hour} + same := Schedule{Refresh: &Refresh{Top: 50, Since: time.Hour}, Interval: time.Hour} + if scheduleKey(base) != scheduleKey(same) { + t.Error("same refresh gave different keys") + } + for name, other := range map[string]Schedule{ + "no refresh": {Interval: time.Hour}, + "top": {Refresh: &Refresh{Top: 10, Since: time.Hour}, Interval: time.Hour}, + "since": {Refresh: &Refresh{Top: 50, Since: 2 * time.Hour}, Interval: time.Hour}, + "interval": {Refresh: &Refresh{Top: 50, Since: time.Hour}, Interval: 2 * time.Hour}, + } { + if scheduleKey(other) == scheduleKey(base) { + t.Errorf("changing %s kept the key", name) + } + } +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index ecc3a0c..30c4e0b 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -22,6 +22,9 @@ type PackageVersion struct { Qualifiers map[string]string // Platforms limit which platforms of an image index are mirrored. Platforms []string + // variants, when set, limit the files of a version that are mirrored to + // those of these variants (see fileEcosystem.variant). + variants map[string]bool } func (pv PackageVersion) String() string { diff --git a/internal/server/mirror_schedule.go b/internal/server/mirror_schedule.go index 172c2de..78315d4 100644 --- a/internal/server/mirror_schedule.go +++ b/internal/server/mirror_schedule.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/mirror" ) @@ -13,20 +14,38 @@ import ( const mirrorScheduleShutdownWait = 30 * time.Second func (s *Server) startMirrorSchedules(ctx context.Context, proxy *handler.Proxy) { - if len(s.cfg.Mirror.Schedules) == 0 { + schedules := mirrorSchedules(s.cfg.Mirror) + if len(schedules) == 0 { return } // One worker, so a schedule never fans out against the registry. m := s.newMirror(proxy, 1) - schedules := make([]mirror.Schedule, len(s.cfg.Mirror.Schedules)) - for i, schedule := range s.cfg.Mirror.Schedules { - schedules[i] = mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()} - } s.scheduler = mirror.NewScheduler(m, schedules, s.logger) s.scheduler.Claim = s.db.ClaimJobRun s.scheduler.Release = s.db.ReleaseJobRun s.scheduler.Start(ctx) - s.logger.Info("mirror schedules enabled", "schedules", len(schedules)) + s.logger.Info("mirror schedules enabled", "schedules", len(schedules), "refresh", s.cfg.Mirror.Refresh != nil) +} + +// mirrorSchedules returns the schedules of a validated mirror configuration, +// the refresh last. +func mirrorSchedules(cfg config.MirrorConfig) []mirror.Schedule { + var schedules []mirror.Schedule + for _, schedule := range cfg.Schedules { + schedules = append(schedules, mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()}) + } + if r := cfg.Refresh; r != nil { + schedules = append(schedules, mirror.Schedule{ + Refresh: &mirror.Refresh{ + Top: r.TopPackages(), + Since: r.SinceDuration(), + MaxVersions: r.MaxVersionsPerRun(), + MaxBytes: r.MaxBytesPerRun(), + }, + Interval: r.IntervalDuration(), + }) + } + return schedules } func (s *Server) waitForMirrorSchedules(ctx context.Context) { diff --git a/internal/server/mirror_schedule_test.go b/internal/server/mirror_schedule_test.go index d2440e6..1c19559 100644 --- a/internal/server/mirror_schedule_test.go +++ b/internal/server/mirror_schedule_test.go @@ -4,10 +4,12 @@ import ( "context" "io" "log/slog" + "reflect" "sync/atomic" "testing" "time" + "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/mirror" ) @@ -96,3 +98,23 @@ func TestWaitForMirrorSchedulesWithoutSchedules(t *testing.T) { s := &Server{} s.waitForMirrorSchedules(context.Background()) } + +func TestMirrorSchedulesFromConfig(t *testing.T) { + cfg := config.MirrorConfig{ + Schedules: []config.MirrorSchedule{{Images: []string{"pkg:docker/alpine@3"}, Interval: "6h", Platforms: []string{"linux/amd64"}}}, + Refresh: &config.MirrorRefresh{Interval: "24h", Top: 20, Since: "7d", MaxVersions: 100, MaxBytes: "1GB"}, + } + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + want := []mirror.Schedule{ + {PURLs: []string{"pkg:docker/alpine@3"}, Platforms: []string{"linux/amd64"}, Interval: 6 * time.Hour}, + {Refresh: &mirror.Refresh{Top: 20, Since: 7 * 24 * time.Hour, MaxVersions: 100, MaxBytes: 1 << 30}, Interval: 24 * time.Hour}, + } + if got := mirrorSchedules(cfg); !reflect.DeepEqual(got, want) { + t.Errorf("mirrorSchedules =\n%+v\nwant\n%+v", got, want) + } + if got := mirrorSchedules(config.MirrorConfig{}); len(got) != 0 { + t.Errorf("mirrorSchedules of an empty config = %+v, want none", got) + } +} From 3e4491287d3634eb2f6e1e6fa6f6d236246ec09a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 13:57:49 +0100 Subject: [PATCH 031/198] Record the upstream URL of cached metadata The refresh fetches Debian, Alpine and RPM indexes again from the URLs clients fetched them from, which the hashed cache keys don't keep. Rows cached before get their URL on the next revalidation, even on a 304. --- .../database/metadata_upstream_url_test.go | 90 +++++++++++++++++++ internal/database/queries.go | 35 ++++++-- internal/database/schema.go | 19 ++++ internal/database/types.go | 15 ++-- internal/handler/handler.go | 11 ++- internal/handler/handler_test.go | 52 +++++++++++ 6 files changed, 209 insertions(+), 13 deletions(-) create mode 100644 internal/database/metadata_upstream_url_test.go diff --git a/internal/database/metadata_upstream_url_test.go b/internal/database/metadata_upstream_url_test.go new file mode 100644 index 0000000..ef58863 --- /dev/null +++ b/internal/database/metadata_upstream_url_test.go @@ -0,0 +1,90 @@ +package database + +import ( + "database/sql" + "path/filepath" + "reflect" + "testing" + "time" +) + +func TestMigrationAddsMetadataUpstreamURL(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "existing.db") + sqlDB, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + if _, err := sqlDB.Exec(schemaSQLite); err != nil { + t.Fatalf("create schema: %v", err) + } + if _, err := sqlDB.Exec(`ALTER TABLE metadata_cache DROP COLUMN upstream_url`); err != nil { + t.Fatalf("drop upstream_url column: %v", err) + } + if _, err := sqlDB.Exec(`DELETE FROM migrations WHERE name = '020_add_metadata_upstream_url'`); err != nil { + t.Fatalf("forget migration: %v", err) + } + if err := sqlDB.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + db, err := Open(dbPath) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer func() { _ = db.Close() }() + if err := db.MigrateSchema(); err != nil { + t.Fatalf("MigrateSchema: %v", err) + } + has, err := db.HasColumn("metadata_cache", "upstream_url") + if err != nil || !has { + t.Errorf("metadata_cache.upstream_url after migration = %v, %v", has, err) + } +} + +func upsertMetadataURL(t *testing.T, db *DB, ecosystem, name, url string) { + t.Helper() + entry := &MetadataCacheEntry{Ecosystem: ecosystem, Name: name, StoragePath: "_metadata/" + name} + if url != "" { + entry.UpstreamURL = sql.NullString{String: url, Valid: true} + } + if err := db.UpsertMetadataCache(entry); err != nil { + t.Fatal(err) + } +} + +func TestUpsertMetadataCacheKeepsUpstreamURL(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertMetadataURL(t, db, "apk", "k1", "https://dl-cdn.alpinelinux.org/alpine/v3.20/main/aarch64/APKINDEX.tar.gz") + // A writer that doesn't know the URL leaves it as it was. + upsertMetadataURL(t, db, "apk", "k1", "") + got, err := db.GetMetadataCache("apk", "k1") + if err != nil || got == nil { + t.Fatalf("GetMetadataCache: %+v, %v", got, err) + } + if got.UpstreamURL.String != "https://dl-cdn.alpinelinux.org/alpine/v3.20/main/aarch64/APKINDEX.tar.gz" { + t.Errorf("upstream_url = %q", got.UpstreamURL.String) + } + }) +} + +func TestRecentMetadataURLs(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertMetadataURL(t, db, "apk", "k1", "https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz") + upsertMetadataURL(t, db, "apk", "k2", "https://a.test/v3.19/main/aarch64/APKINDEX.tar.gz") + upsertMetadataURL(t, db, "apk", "k3", "") + upsertMetadataURL(t, db, "debian", "k4", "https://d.test/dists/bookworm/InRelease") + old := time.Now().Add(-30 * 24 * time.Hour).UTC() + if _, err := db.Exec(db.Rebind("UPDATE metadata_cache SET last_accessed_at = ?, updated_at = ? WHERE name = ?"), old, old, "k2"); err != nil { + t.Fatal(err) + } + + got, err := db.RecentMetadataURLs("apk", time.Now().Add(-14*24*time.Hour)) + if err != nil { + t.Fatal(err) + } + want := []string{"https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz"} + if !reflect.DeepEqual(got, want) { + t.Errorf("RecentMetadataURLs = %v, want %v", got, want) + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e8..3f74a6d 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -1299,7 +1299,7 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err var entry MetadataCacheEntry query := db.Rebind(` SELECT id, ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at + content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at FROM metadata_cache WHERE ecosystem = ? AND name = ? `) err := db.Get(&entry, query, ecosystem, name) @@ -1312,6 +1312,19 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err return &entry, nil } +// RecentMetadataURLs returns the upstream URLs of the documents of an +// ecosystem cached or served since cutoff, as far as they were recorded. +func (db *DB) RecentMetadataURLs(ecosystem string, cutoff time.Time) ([]string, error) { + var urls []string + err := db.Select(&urls, db.Rebind(` + SELECT DISTINCT upstream_url FROM metadata_cache + WHERE ecosystem = ? AND upstream_url IS NOT NULL + AND COALESCE(last_accessed_at, updated_at) >= ? + ORDER BY upstream_url + `), ecosystem, cutoff.UTC()) + return urls, err +} + // RefreshMetadataCache sets fetched_at on the metadata cache row for ecosystem // and name, provided the row still records digest, and reports whether it did. // It is for an upstream 304, which leaves the stored document as it is. @@ -1328,6 +1341,14 @@ func (db *DB) RefreshMetadataCache(ecosystem, name, digest string, fetchedAt tim return n > 0, err } +// SetMetadataUpstreamURL records the upstream URL of the metadata cache row +// for ecosystem and name, for rows cached before URLs were recorded. +func (db *DB) SetMetadataUpstreamURL(ecosystem, name, upstreamURL string) error { + _, err := db.Exec(db.Rebind(`UPDATE metadata_cache SET upstream_url = ? WHERE ecosystem = ? AND name = ?`), + upstreamURL, ecosystem, name) + return err +} + // MoveMetadataCache points the metadata cache row for ecosystem and name at // to, provided it is still at from and records digest, and reports whether // it did. The caller has copied the document from from to to. @@ -1373,8 +1394,8 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { if db.dialect == DialectPostgres { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = EXCLUDED.storage_path, etag = EXCLUDED.etag, @@ -1382,6 +1403,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { content_type = EXCLUDED.content_type, content_encoding = EXCLUDED.content_encoding, content_digest = EXCLUDED.content_digest, + upstream_url = COALESCE(EXCLUDED.upstream_url, metadata_cache.upstream_url), size = EXCLUDED.size, last_modified = EXCLUDED.last_modified, fetched_at = EXCLUDED.fetched_at, @@ -1391,8 +1413,8 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { } else { query = ` INSERT INTO metadata_cache (ecosystem, name, storage_path, etag, link, content_type, content_encoding, - content_digest, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + content_digest, upstream_url, size, last_modified, fetched_at, last_accessed_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(ecosystem, name) DO UPDATE SET storage_path = excluded.storage_path, etag = excluded.etag, @@ -1400,6 +1422,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { content_type = excluded.content_type, content_encoding = excluded.content_encoding, content_digest = excluded.content_digest, + upstream_url = COALESCE(excluded.upstream_url, metadata_cache.upstream_url), size = excluded.size, last_modified = excluded.last_modified, fetched_at = excluded.fetched_at, @@ -1410,7 +1433,7 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { _, err := db.Exec(query, entry.Ecosystem, entry.Name, entry.StoragePath, entry.ETag, entry.Link, - entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.Size, entry.LastModified, fetchedAt, now.UTC(), now, now, + entry.ContentType, entry.ContentEncoding, entry.ContentDigest, entry.UpstreamURL, entry.Size, entry.LastModified, fetchedAt, now.UTC(), now, now, ) if err != nil { return fmt.Errorf("upserting metadata cache: %w", err) diff --git a/internal/database/schema.go b/internal/database/schema.go index f0ec651..d2ad5f5 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -128,6 +128,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( content_type TEXT, content_encoding TEXT, content_digest TEXT, + upstream_url TEXT, size INTEGER, last_modified DATETIME, fetched_at DATETIME, @@ -271,6 +272,7 @@ CREATE TABLE IF NOT EXISTS metadata_cache ( content_type TEXT, content_encoding TEXT, content_digest TEXT, + upstream_url TEXT, size BIGINT, last_modified TIMESTAMP, fetched_at TIMESTAMP, @@ -468,6 +470,7 @@ var migrations = []migration{ {"017_add_artifact_content_hash_index", migrateAddArtifactContentHashIndex}, {"018_add_upstream_holds", migrateAddUpstreamHolds}, {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, + {"020_add_metadata_upstream_url", migrateAddMetadataUpstreamURL}, } // isTableNotFound returns true if the error indicates a missing table. @@ -728,6 +731,22 @@ func migrateAddMetadataLink(db *DB) error { return nil } +// migrateAddMetadataUpstreamURL adds the URL a metadata document was +// fetched from, which the mirror refresh fetches it from again. +func migrateAddMetadataUpstreamURL(db *DB) error { + hasColumn, err := db.HasColumn("metadata_cache", "upstream_url") + if err != nil { + return fmt.Errorf("checking metadata_cache upstream_url column: %w", err) + } + if hasColumn { + return nil + } + if _, err := db.Exec("ALTER TABLE metadata_cache ADD COLUMN upstream_url TEXT"); err != nil { + return fmt.Errorf("adding metadata_cache upstream_url column: %w", err) + } + return nil +} + func migrateAddMetadataContentEncoding(db *DB) error { hasColumn, err := db.HasColumn("metadata_cache", "content_encoding") if err != nil { diff --git a/internal/database/types.go b/internal/database/types.go index fb49a1c..b759396 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -184,12 +184,15 @@ type MetadataCacheEntry struct { ContentType sql.NullString `db:"content_type" json:"content_type,omitempty"` ContentEncoding sql.NullString `db:"content_encoding" json:"content_encoding,omitempty"` ContentDigest sql.NullString `db:"content_digest" json:"content_digest,omitempty"` - Size sql.NullInt64 `db:"size" json:"size,omitempty"` - LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` - FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` - LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + // UpstreamURL is where the document was fetched from. Rows written + // before it was recorded, and by writers that don't know it, have none. + UpstreamURL sql.NullString `db:"upstream_url" json:"upstream_url,omitempty"` + Size sql.NullInt64 `db:"size" json:"size,omitempty"` + LastModified sql.NullTime `db:"last_modified" json:"last_modified,omitempty"` + FetchedAt sql.NullTime `db:"fetched_at" json:"fetched_at,omitempty"` + LastAccessedAt sql.NullTime `db:"last_accessed_at" json:"last_accessed_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` } // Vulnerability represents a cached vulnerability record. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4b7ebe7..60a7ae4 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1943,6 +1943,8 @@ type upstreamMetadata struct { contentEncoding string etag string lastModified time.Time + // url is the upstream URL the document was fetched from. + url string // notModified is the cache row upstream answered 304 for, whose stored // bytes body holds. It is nil for a document upstream sent. notModified *database.MetadataCacheEntry @@ -1987,7 +1989,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e if readErr != nil { return nil, errStale304 } - meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, notModified: entry} + meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, url: upstreamURL, notModified: entry} if entry.ContentType.Valid { meta.contentType = entry.ContentType.String } @@ -2020,6 +2022,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e contentType: resp.Header.Get(headerContentType), contentEncoding: resp.Header.Get(headerContentEncoding), etag: resp.Header.Get(headerETag), + url: upstreamURL, } if meta.contentType == "" { meta.contentType = contentTypeJSON @@ -2049,6 +2052,11 @@ func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMe if !refreshed { return "" } + if !entry.UpstreamURL.Valid && meta.url != "" { + if err := p.DB.SetMetadataUpstreamURL(ecosystem, cacheKey, meta.url); err != nil { + p.Logger.Warn("failed to record metadata upstream URL", "ecosystem", ecosystem, "key", cacheKey, "error", err) + } + } p.recordMetadataRead(entry) return entry.ContentDigest.String } @@ -2081,6 +2089,7 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor // The digest identifies the stored bytes, so a rewrite cached for them // can be found without reading them back (see storedRewrite). ContentDigest: sql.NullString{String: digest, Valid: digest != ""}, + UpstreamURL: sql.NullString{String: meta.url, Valid: meta.url != ""}, Size: sql.NullInt64{Int64: size, Valid: true}, LastModified: sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 8ce636c..ed56b63 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -13,6 +13,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "testing" "time" @@ -1584,3 +1585,54 @@ func TestCanonicalPackagePURLMatchesConfig(t *testing.T) { }) } } + +func TestProxyCachedRecordsUpstreamURL(t *testing.T) { + proxy, upstream := setupCachedProxy(t, `"abc123"`, "") + req := httptest.NewRequest(http.MethodGet, "/test", nil) + proxy.ProxyCached(httptest.NewRecorder(), req, upstream.URL+"/test", "test-eco", "url-key") + + entry, err := proxy.DB.GetMetadataCache("test-eco", "url-key") + if err != nil || entry == nil { + t.Fatalf("GetMetadataCache: %+v, %v", entry, err) + } + if entry.UpstreamURL.String != upstream.URL+"/test" { + t.Errorf("upstream_url = %q, want %q", entry.UpstreamURL.String, upstream.URL+"/test") + } +} + +// A row cached before upstream URLs were recorded gets its URL on the next +// revalidation, even when upstream answers 304. +func TestProxyCachedRecordsUpstreamURLOnNotModified(t *testing.T) { + var notModified atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"v1"`) + if r.Header.Get("If-None-Match") == `"v1"` { + notModified.Add(1) + w.WriteHeader(http.StatusNotModified) + return + } + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.HTTPClient = upstream.Client() + + proxy.ProxyCached(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/test", nil), upstream.URL+"/test", "test-eco", "url-304-key") + if _, err := proxy.DB.Exec("UPDATE metadata_cache SET upstream_url = NULL"); err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + proxy.ProxyCached(w, httptest.NewRequest(http.MethodGet, "/test", nil), upstream.URL+"/test", "test-eco", "url-304-key") + if w.Code != http.StatusOK || notModified.Load() != 1 { + t.Fatalf("status = %d, upstream 304s = %d; want 200 after one 304", w.Code, notModified.Load()) + } + entry, err := proxy.DB.GetMetadataCache("test-eco", "url-304-key") + if err != nil || entry == nil { + t.Fatalf("GetMetadataCache: %+v, %v", entry, err) + } + if entry.UpstreamURL.String != upstream.URL+"/test" { + t.Errorf("upstream_url after a 304 = %q, want %q", entry.UpstreamURL.String, upstream.URL+"/test") + } +} From 2f2d01a414b3867eab3544ebdd31bc0aee3b790c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 14:05:05 +0100 Subject: [PATCH 032/198] Refresh Debian, Alpine and RPM packages from their indexes These clients find new versions in a repository index. The refresh fetches the InRelease, APKINDEX and repomd.xml files clients fetched again, reads the Packages and primary indexes they point at for the repositories the most used packages came from, and mirrors the newer file per package and architecture, checked against the index's SHA-256 where it has one. --- go.mod | 2 +- .../database/metadata_upstream_url_test.go | 8 +- internal/database/queries.go | 23 +- internal/handler/os_package_identity.go | 17 ++ internal/handler/os_package_identity_test.go | 23 ++ internal/mirror/mirror.go | 9 + internal/mirror/refresh.go | 61 ++++- internal/mirror/refresh_index.go | 254 ++++++++++++++++++ internal/mirror/refresh_index_parse.go | 238 ++++++++++++++++ internal/mirror/refresh_index_parse_test.go | 188 +++++++++++++ internal/mirror/refresh_index_test.go | 231 ++++++++++++++++ internal/mirror/source.go | 4 + 12 files changed, 1032 insertions(+), 26 deletions(-) create mode 100644 internal/handler/os_package_identity.go create mode 100644 internal/handler/os_package_identity_test.go create mode 100644 internal/mirror/refresh_index.go create mode 100644 internal/mirror/refresh_index_parse.go create mode 100644 internal/mirror/refresh_index_parse_test.go create mode 100644 internal/mirror/refresh_index_test.go diff --git a/go.mod b/go.mod index 2b1ee11..69031ac 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/prometheus/client_golang v1.25.0 github.com/prometheus/client_model v0.6.3 github.com/swaggo/swag v1.16.6 + github.com/ulikunitz/xz v0.5.17 github.com/vmihailenco/msgpack/v5 v5.4.1 gocloud.dev v0.47.0 golang.org/x/mod v0.42.0 @@ -108,7 +109,6 @@ require ( github.com/rogpeppe/go-internal v1.16.0 // indirect github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect - github.com/ulikunitz/xz v0.5.17 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.46.0 // indirect diff --git a/internal/database/metadata_upstream_url_test.go b/internal/database/metadata_upstream_url_test.go index ef58863..cb5bafc 100644 --- a/internal/database/metadata_upstream_url_test.go +++ b/internal/database/metadata_upstream_url_test.go @@ -67,7 +67,7 @@ func TestUpsertMetadataCacheKeepsUpstreamURL(t *testing.T) { }) } -func TestRecentMetadataURLs(t *testing.T) { +func TestRecentMetadataDocuments(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { upsertMetadataURL(t, db, "apk", "k1", "https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz") upsertMetadataURL(t, db, "apk", "k2", "https://a.test/v3.19/main/aarch64/APKINDEX.tar.gz") @@ -78,13 +78,13 @@ func TestRecentMetadataURLs(t *testing.T) { t.Fatal(err) } - got, err := db.RecentMetadataURLs("apk", time.Now().Add(-14*24*time.Hour)) + got, err := db.RecentMetadataDocuments("apk", time.Now().Add(-14*24*time.Hour)) if err != nil { t.Fatal(err) } - want := []string{"https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz"} + want := []MetadataDocument{{Name: "k1", UpstreamURL: "https://a.test/v3.20/main/aarch64/APKINDEX.tar.gz"}} if !reflect.DeepEqual(got, want) { - t.Errorf("RecentMetadataURLs = %v, want %v", got, want) + t.Errorf("RecentMetadataDocuments = %+v, want %+v", got, want) } }) } diff --git a/internal/database/queries.go b/internal/database/queries.go index 3f74a6d..20771e1 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -1312,17 +1312,24 @@ func (db *DB) GetMetadataCache(ecosystem, name string) (*MetadataCacheEntry, err return &entry, nil } -// RecentMetadataURLs returns the upstream URLs of the documents of an -// ecosystem cached or served since cutoff, as far as they were recorded. -func (db *DB) RecentMetadataURLs(ecosystem string, cutoff time.Time) ([]string, error) { - var urls []string - err := db.Select(&urls, db.Rebind(` - SELECT DISTINCT upstream_url FROM metadata_cache +// MetadataDocument names a cached metadata document and where it was +// fetched from. +type MetadataDocument struct { + Name string `db:"name"` + UpstreamURL string `db:"upstream_url"` +} + +// RecentMetadataDocuments returns the documents of an ecosystem cached or +// served since cutoff whose upstream URL was recorded. +func (db *DB) RecentMetadataDocuments(ecosystem string, cutoff time.Time) ([]MetadataDocument, error) { + var docs []MetadataDocument + err := db.Select(&docs, db.Rebind(` + SELECT name, upstream_url FROM metadata_cache WHERE ecosystem = ? AND upstream_url IS NOT NULL AND COALESCE(last_accessed_at, updated_at) >= ? - ORDER BY upstream_url + ORDER BY upstream_url, name `), ecosystem, cutoff.UTC()) - return urls, err + return docs, err } // RefreshMetadataCache sets fetched_at on the metadata cache row for ecosystem diff --git a/internal/handler/os_package_identity.go b/internal/handler/os_package_identity.go new file mode 100644 index 0000000..0e0400b --- /dev/null +++ b/internal/handler/os_package_identity.go @@ -0,0 +1,17 @@ +package handler + +// OSPackageIdentity returns the name, version and architecture the deb, +// alpine and rpm download handlers cache a package file under, read from its +// path below the repository the way they read it. It returns empty strings +// for other ecosystems and for paths they would not cache. +func OSPackageIdentity(ecosystem, path string) (name, version, arch string) { + switch ecosystem { + case "deb": + return (*DebianHandler)(nil).parsePoolPath("/" + path) + case apkEcosystem: + return (*APKHandler)(nil).parseAPKPath(path) + case "rpm": + return (*RPMHandler)(nil).parseRPMPath(path) + } + return "", "", "" +} diff --git a/internal/handler/os_package_identity_test.go b/internal/handler/os_package_identity_test.go new file mode 100644 index 0000000..3d487be --- /dev/null +++ b/internal/handler/os_package_identity_test.go @@ -0,0 +1,23 @@ +package handler + +import "testing" + +func TestOSPackageIdentity(t *testing.T) { + tests := []struct { + ecosystem, path string + name, version, arch string + }{ + {"deb", "pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb", "curl", "7.88.1-10+deb12u8", "amd64"}, + {"alpine", "v3.20/main/aarch64/curl-8.9.1-r0.apk", "curl", "8.9.1-r0", "aarch64"}, + {"rpm", "Packages/c/curl-8.6.0-10.fc40.x86_64.rpm", "curl", "8.6.0-10.fc40", "x86_64"}, + {"deb", "pool/main/c/curl/README", "", "", ""}, + {"npm", "lodash-4.17.21.tgz", "", "", ""}, + } + for _, tt := range tests { + name, version, arch := OSPackageIdentity(tt.ecosystem, tt.path) + if name != tt.name || version != tt.version || arch != tt.arch { + t.Errorf("OSPackageIdentity(%s, %s) = %q %q %q, want %q %q %q", + tt.ecosystem, tt.path, name, version, arch, tt.name, tt.version, tt.arch) + } + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index ac20a4e..2068644 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -246,6 +246,15 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "reason", m.RefusePackages) return } + if pv.fileURL != "" { + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, pv.Ecosystem, pv.Name, pv.Version, pv.cacheFilename, pv.fileURL, pv.digest) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + m.recordResult(tracker, pv, result) + return + } if eco, ok := fileEcosystems[pv.Ecosystem]; ok { m.mirrorFiles(ctx, pv, eco, tracker) return diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index bdc4c29..a20ba8b 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -55,37 +55,72 @@ func (m *Mirror) releaseLister() ReleaseLister { //nolint:ireturn // any lister // it: releases within cooldown or on the denylist are left out. Packages it // can't list are counted as failed. func (m *Mirror) Refresh(ctx context.Context, r Refresh) (*Progress, error) { - packages, err := m.db.MostUsedPackages(time.Now().Add(-r.Since), r.Top) + cutoff := time.Now().Add(-r.Since) + packages, err := m.db.MostUsedPackages(cutoff, r.Top) if err != nil { return nil, fmt.Errorf("listing the most used packages: %w", err) } tracker := newProgressTracker() tracker.phase.Store("downloading") -refresh: - for _, pkg := range packages { - if ctx.Err() != nil { - break - } - items, err := m.refreshItems(ctx, pkg) - if err != nil { - m.recordFailure(tracker, PackageVersion{Ecosystem: pkg.Ecosystem, Name: pkg.Name}, err) - continue - } + run := func(items []PackageVersion) bool { for _, item := range items { + if ctx.Err() != nil { + return false + } if r.spent(tracker) { m.logger.Info("mirror refresh reached its cap for one run", "mirrored", tracker.completed.Load(), "bytes", tracker.bytes.Load()) - break refresh + return false } tracker.total.Add(1) m.mirrorOne(ctx, item, tracker) } + return true + } + for _, group := range byEcosystem(packages) { + if ix, ok := osIndexes[group[0].Ecosystem]; ok { + items, errs := m.indexRefreshItems(ctx, ix, group, cutoff) + for _, err := range errs { + m.recordFailure(tracker, PackageVersion{Ecosystem: ix.ecosystem}, err) + } + if !run(items) { + break + } + continue + } + done := true + for _, pkg := range group { + items, err := m.refreshItems(ctx, pkg) + if err != nil { + m.recordFailure(tracker, PackageVersion{Ecosystem: pkg.Ecosystem, Name: pkg.Name}, err) + continue + } + if done = run(items); !done { + break + } + } + if !done { + break + } } tracker.phase.Store("complete") p := tracker.snapshot() return &p, nil } +// byEcosystem splits packages, ordered by ecosystem, into one group per +// ecosystem. +func byEcosystem(packages []database.UsedPackage) [][]database.UsedPackage { + var groups [][]database.UsedPackage + for i, pkg := range packages { + if i == 0 || pkg.Ecosystem != packages[i-1].Ecosystem { + groups = append(groups, nil) + } + groups[len(groups)-1] = append(groups[len(groups)-1], pkg) + } + return groups +} + // refreshItems returns the versions to mirror for a used package. func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([]PackageVersion, error) { p, err := purl.Parse(pkg.PURL) @@ -130,7 +165,7 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] // refreshable reports whether a refresh can list the releases of an // ecosystem's packages. func refreshable(ecosystem string) bool { - if isImageEcosystem(ecosystem) { + if isImageEcosystem(ecosystem) || osIndexes[ecosystem].ecosystem != "" { return false } _, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", nil) diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go new file mode 100644 index 0000000..dc7818a --- /dev/null +++ b/internal/mirror/refresh_index.go @@ -0,0 +1,254 @@ +package mirror + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "path" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/vers" +) + +// A Debian, Alpine or RPM client finds new versions in a repository index +// rather than in metadata per package. A refresh reads the indexes clients +// fetched again, from the URLs the metadata cache recorded, and mirrors the +// newer files they list of the packages downloaded most. + +// osIndex describes the repository index of one ecosystem. +type osIndex struct { + // ecosystem is what package files are cached under, metaEcosystem what + // the index documents are cached under. + ecosystem, metaEcosystem string + // scheme orders versions. + scheme string + // base returns the repository base package paths are relative to, for + // the URL of an index document a refresh starts from, or false for + // other documents. + base func(indexURL string) (string, bool) + // files lists the package files of the index document body fetched + // from indexURL. cached are the files of the most used packages under + // the repository base, for indexes that list their files in parts. + files func(ctx context.Context, m *Mirror, indexURL string, body []byte, cached []cachedOSFile) ([]indexFile, error) +} + +var osIndexes = map[string]osIndex{ + "alpine": { + ecosystem: "alpine", metaEcosystem: "alpine", scheme: "alpine", + base: func(u string) (string, bool) { + return strings.CutSuffix(u, "/APKINDEX.tar.gz") + }, + files: func(_ context.Context, _ *Mirror, _ string, body []byte, _ []cachedOSFile) ([]indexFile, error) { + return parseAPKIndex(bytes.NewReader(body)) + }, + }, + "deb": { + ecosystem: "deb", metaEcosystem: "debian", scheme: "deb", + base: func(u string) (string, bool) { + if !strings.HasSuffix(u, "/InRelease") && !strings.HasSuffix(u, "/Release") { + return "", false + } + base, _, ok := strings.Cut(u, "/dists/") + return base, ok + }, + files: debIndexFiles, + }, + "rpm": { + ecosystem: "rpm", metaEcosystem: "rpm", scheme: "rpm", + base: func(u string) (string, bool) { + return strings.CutSuffix(u, "/repodata/repomd.xml") + }, + files: rpmIndexFiles, + }, +} + +// cachedOSFile is a cached file of a most used package, by its path below a +// repository base. +type cachedOSFile struct { + file database.UsedFile + rel string + name, version, arch string +} + +func (f cachedOSFile) key() string { return f.name + "\x00" + f.arch } + +// indexRefreshItems returns the newer files the recently fetched indexes of +// an ecosystem list for packages, and an error per index it couldn't read. +func (m *Mirror) indexRefreshItems(ctx context.Context, ix osIndex, packages []database.UsedPackage, cutoff time.Time) ([]PackageVersion, []error) { + docs, err := m.db.RecentMetadataDocuments(ix.metaEcosystem, cutoff) + if err != nil { + return nil, []error{fmt.Errorf("listing %s indexes: %w", ix.metaEcosystem, err)} + } + var items []PackageVersion + var errs []error + seenIndex := map[string]bool{} + seenFile := map[string]bool{} + for _, doc := range docs { + base, ok := ix.base(doc.UpstreamURL) + if !ok || seenIndex[urlDir(doc.UpstreamURL)] { + continue + } + cached := cachedUnder(ix.ecosystem, base, packages) + if len(cached) == 0 { + continue + } + seenIndex[urlDir(doc.UpstreamURL)] = true + body, _, err := m.proxy.FetchOrCacheMetadata(ctx, ix.metaEcosystem, doc.Name, doc.UpstreamURL, "*/*") + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", doc.UpstreamURL, err)) + continue + } + files, err := ix.files(ctx, m, doc.UpstreamURL, body, cached) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", doc.UpstreamURL, err)) + continue + } + for _, item := range m.newerIndexFiles(ix, base, cached, files) { + if !seenFile[item.fileURL] { + seenFile[item.fileURL] = true + items = append(items, item) + } + } + } + return items, errs +} + +// cachedUnder returns the cached files of packages below base. +func cachedUnder(ecosystem, base string, packages []database.UsedPackage) []cachedOSFile { + var cached []cachedOSFile + for _, pkg := range packages { + for _, f := range pkg.Files { + rel, ok := strings.CutPrefix(f.UpstreamURL, base+"/") + if !ok { + continue + } + name, version, arch := handler.OSPackageIdentity(ecosystem, rel) + if name != "" { + cached = append(cached, cachedOSFile{file: f, rel: rel, name: name, version: version, arch: arch}) + } + } + } + return cached +} + +// newerIndexFiles returns, per package and architecture cached below base, +// the newest file the index lists when it is newer than every cached one. +func (m *Mirror) newerIndexFiles(ix osIndex, base string, cached []cachedOSFile, files []indexFile) []PackageVersion { + newestCached := map[string]cachedOSFile{} + for _, c := range cached { + if current, ok := newestCached[c.key()]; !ok || vers.CompareWithScheme(c.version, current.version, ix.scheme) > 0 { + newestCached[c.key()] = c + } + } + newest := map[string]PackageVersion{} + var order []string + for _, f := range files { + name, version, arch := handler.OSPackageIdentity(ix.ecosystem, f.Path) + key := name + "\x00" + arch + c, ok := newestCached[key] + if name == "" || !ok || vers.CompareWithScheme(version, c.version, ix.scheme) <= 0 { + continue + } + if m.proxy != nil && !m.proxy.ReleaseAllowed(ix.ecosystem, name, version, time.Time{}) { + continue + } + if current, seen := newest[key]; seen && vers.CompareWithScheme(version, current.Version, ix.scheme) <= 0 { + continue + } else if !seen { + order = append(order, key) + } + cacheFilename := path.Base(f.Path) + if dir := path.Dir(c.file.Filename); dir != "." { + cacheFilename = dir + "/" + cacheFilename + } + newest[key] = PackageVersion{ + Ecosystem: ix.ecosystem, Name: name, Version: version, + fileURL: base + "/" + f.Path, cacheFilename: cacheFilename, digest: f.Digest, + } + } + items := make([]PackageVersion, 0, len(order)) + for _, key := range order { + items = append(items, newest[key]) + } + return items +} + +// debIndexFiles reads the Packages indexes an InRelease or Release file +// lists for the components and architectures of the cached files. +func debIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, cached []cachedOSFile) ([]indexFile, error) { + indexes := debPackagesIndexes(body) + wanted := map[string]bool{} + for _, c := range cached { + rest, ok := strings.CutPrefix(c.rel, "pool/") + if !ok { + continue + } + component, _, _ := strings.Cut(rest, "/") + if dir := component + "/binary-" + c.arch; indexes[dir] != "" { + wanted[dir] = true + continue + } + // Files for every architecture are listed in each binary-* index. + for dir := range indexes { + if strings.HasPrefix(dir, component+"/binary-") { + wanted[dir] = true + break + } + } + } + suite := urlDir(indexURL) + var files []indexFile + for dir := range wanted { + listed, err := m.readIndex(ctx, suite+"/"+indexes[dir], parseDebPackages) + if err != nil { + return nil, err + } + files = append(files, listed...) + } + return files, nil +} + +// rpmIndexFiles reads the primary metadata a repomd.xml points at. +func rpmIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, _ []cachedOSFile) ([]indexFile, error) { + primary, err := repomdPrimary(body) + if err != nil { + return nil, err + } + base := strings.TrimSuffix(indexURL, "/repodata/repomd.xml") + return m.readIndex(ctx, base+"/"+primary, parseRPMPrimary) +} + +// readIndex streams an index from upstream through parse, decompressed as +// its name says. It is read directly rather than through the metadata +// cache, since it can be far larger than a metadata document may be. +func (m *Mirror) readIndex(ctx context.Context, indexURL string, parse func(io.Reader) ([]indexFile, error)) ([]indexFile, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil) + if err != nil { + return nil, err + } + resp, err := m.proxy.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s: upstream returned %d", indexURL, resp.StatusCode) + } + r, err := decompressByName(indexURL, resp.Body) + if err != nil { + return nil, err + } + defer func() { _ = r.Close() }() + return parse(r) +} + +// urlDir returns a URL without its last path segment. path.Dir would fold +// the slashes after the scheme. +func urlDir(u string) string { + return u[:strings.LastIndex(u, "/")] +} diff --git a/internal/mirror/refresh_index_parse.go b/internal/mirror/refresh_index_parse.go new file mode 100644 index 0000000..4ae9857 --- /dev/null +++ b/internal/mirror/refresh_index_parse.go @@ -0,0 +1,238 @@ +package mirror + +import ( + "archive/tar" + "bufio" + "compress/bzip2" + "compress/gzip" + "encoding/xml" + "errors" + "fmt" + "io" + "strings" + + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +// indexFile is a package file a repository index lists: its path below the +// repository base and, where the index gives one, its SHA-256 digest. +type indexFile struct { + Path string + Digest string +} + +// maxIndexLine bounds one line of a Debian or Alpine index. +const maxIndexLine = 1 << 20 + +// parseAPKIndex reads the files an APKINDEX.tar.gz lists. Its checksums +// cover only the control section of a package, so no digest is returned. +func parseAPKIndex(r io.Reader) ([]indexFile, error) { + gz, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + defer func() { _ = gz.Close() }() + // An APKINDEX.tar.gz may be several gzip streams: the signature, then the + // index. Reading on across them finds the index whichever comes first. + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil, errors.New("APKINDEX.tar.gz has no APKINDEX") + } + if err != nil { + return nil, err + } + if hdr.Name == "APKINDEX" { + return parseAPKIndexText(tr) + } + } +} + +func parseAPKIndexText(r io.Reader) ([]indexFile, error) { + var files []indexFile + var name, version string + flush := func() { + if name != "" && version != "" { + files = append(files, indexFile{Path: name + "-" + version + ".apk"}) + } + name, version = "", "" + } + scanner := bufio.NewScanner(r) + scanner.Buffer(nil, maxIndexLine) + for scanner.Scan() { + line := scanner.Text() + switch { + case line == "": + flush() + case strings.HasPrefix(line, "P:"): + name = line[2:] + case strings.HasPrefix(line, "V:"): + version = line[2:] + } + } + flush() + return files, scanner.Err() +} + +// debIndexPreference ranks the compressions of a Packages index, best first. +var debIndexPreference = []string{".xz", ".gz", ".bz2", ""} + +// debPackagesIndexes returns, per component/binary-arch directory, the path +// of the Packages index a Release or InRelease file lists, in the best +// compression it offers. +func debPackagesIndexes(release []byte) map[string]string { + listed := map[string]bool{} + inSHA256 := false + for _, line := range strings.Split(string(release), "\n") { + line = strings.TrimRight(line, "\r") + if !strings.HasPrefix(line, " ") { + inSHA256 = line == "SHA256:" + continue + } + if !inSHA256 { + continue + } + fields := strings.Fields(line) + if len(fields) == 3 { //nolint:mnd // checksum, size, path + listed[fields[2]] = true + } + } + indexes := map[string]string{} + for path := range listed { + dir, file, ok := strings.Cut(path, "/Packages") + if !ok || !strings.Contains(dir, "/binary-") || strings.Contains(file, "/") { + continue + } + for _, ext := range debIndexPreference { + if listed[dir+"/Packages"+ext] { + indexes[dir] = dir + "/Packages" + ext + break + } + } + } + return indexes +} + +// parseDebPackages reads the files a Packages index lists. +func parseDebPackages(r io.Reader) ([]indexFile, error) { + var files []indexFile + var current indexFile + flush := func() { + if current.Path != "" { + files = append(files, current) + } + current = indexFile{} + } + scanner := bufio.NewScanner(r) + scanner.Buffer(nil, maxIndexLine) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + flush() + continue + } + key, value, ok := strings.Cut(line, ": ") + if !ok || strings.HasPrefix(line, " ") { + continue + } + switch key { + case "Filename": + current.Path = value + case "SHA256": + current.Digest = "sha256:" + value + } + } + flush() + return files, scanner.Err() +} + +// repomdPrimary returns the location of the primary metadata a repomd.xml +// points at. +func repomdPrimary(body []byte) (string, error) { + var doc struct { + Data []struct { + Type string `xml:"type,attr"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"location"` + } `xml:"data"` + } + if err := xml.Unmarshal(body, &doc); err != nil { + return "", err + } + for _, d := range doc.Data { + if d.Type == "primary" && d.Location.Href != "" { + return d.Location.Href, nil + } + } + return "", errors.New("repomd.xml lists no primary metadata") +} + +// parseRPMPrimary reads the binary packages a primary.xml lists, one at a +// time. Source packages are left out. +func parseRPMPrimary(r io.Reader) ([]indexFile, error) { + dec := xml.NewDecoder(r) + var files []indexFile + for { + tok, err := dec.Token() + if errors.Is(err, io.EOF) { + return files, nil + } + if err != nil { + return nil, err + } + start, ok := tok.(xml.StartElement) + if !ok || start.Name.Local != "package" { + continue + } + var pkg struct { + Arch string `xml:"arch"` + Checksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` + } `xml:"checksum"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"location"` + } + if err := dec.DecodeElement(&pkg, &start); err != nil { + return nil, err + } + if pkg.Arch == "src" || pkg.Location.Href == "" { + continue + } + file := indexFile{Path: pkg.Location.Href} + if pkg.Checksum.Type == "sha256" { + file.Digest = "sha256:" + strings.TrimSpace(pkg.Checksum.Value) + } + files = append(files, file) + } +} + +// decompressByName returns r decompressed as the extension of name says. +func decompressByName(name string, r io.Reader) (io.ReadCloser, error) { + switch { + case strings.HasSuffix(name, ".gz"): + return gzip.NewReader(r) + case strings.HasSuffix(name, ".xz"): + x, err := xz.NewReader(r) + if err != nil { + return nil, err + } + return io.NopCloser(x), nil + case strings.HasSuffix(name, ".zst"): + z, err := zstd.NewReader(r) + if err != nil { + return nil, err + } + return z.IOReadCloser(), nil + case strings.HasSuffix(name, ".bz2"): + return io.NopCloser(bzip2.NewReader(r)), nil + case strings.HasSuffix(name, ".xml"), !strings.Contains(name[strings.LastIndex(name, "/")+1:], "."): + return io.NopCloser(r), nil + default: + return nil, fmt.Errorf("unsupported compression: %s", name) + } +} diff --git a/internal/mirror/refresh_index_parse_test.go b/internal/mirror/refresh_index_parse_test.go new file mode 100644 index 0000000..3b68805 --- /dev/null +++ b/internal/mirror/refresh_index_parse_test.go @@ -0,0 +1,188 @@ +package mirror + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "reflect" + "strings" + "testing" + + "github.com/klauspost/compress/zstd" +) + +// apkIndexArchive packs an APKINDEX into the tar.gz Alpine serves. +func apkIndexArchive(t *testing.T, index string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for name, body := range map[string]string{"DESCRIPTION": "v3.20.3-1-g0000000", "APKINDEX": index} { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestParseAPKIndex(t *testing.T) { + index := "C:Q1abc=\nP:curl\nV:8.9.1-r0\nA:aarch64\nS:100\n\nC:Q1def=\nP:busybox\nV:1.36.1-r29\nA:aarch64\n\n" + got, err := parseAPKIndex(bytes.NewReader(apkIndexArchive(t, index))) + if err != nil { + t.Fatal(err) + } + want := []indexFile{{Path: "curl-8.9.1-r0.apk"}, {Path: "busybox-1.36.1-r29.apk"}} + if !reflect.DeepEqual(got, want) { + t.Errorf("parseAPKIndex = %+v, want %+v", got, want) + } +} + +const debRelease = `-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA256 + +Origin: Debian +Suite: stable +Codename: bookworm +Architectures: all amd64 arm64 +Components: main contrib +MD5Sum: + 0123 100 main/binary-amd64/Packages.xz +SHA256: + aaaa 1000 main/binary-amd64/Packages + bbbb 200 main/binary-amd64/Packages.xz + cccc 300 main/binary-amd64/Packages.gz + dddd 400 main/binary-arm64/Packages.gz + eeee 500 contrib/binary-amd64/Packages.xz + ffff 600 main/i18n/Translation-en.bz2 +-----BEGIN PGP SIGNATURE----- +xyz +-----END PGP SIGNATURE----- +` + +func TestDebPackagesIndexes(t *testing.T) { + got := debPackagesIndexes([]byte(debRelease)) + want := map[string]string{ + "main/binary-amd64": "main/binary-amd64/Packages.xz", + "main/binary-arm64": "main/binary-arm64/Packages.gz", + "contrib/binary-amd64": "contrib/binary-amd64/Packages.xz", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("debPackagesIndexes = %v, want %v", got, want) + } +} + +func TestParseDebPackages(t *testing.T) { + packages := `Package: curl +Version: 7.88.1-10+deb12u8 +Architecture: amd64 +Description: a tool + with a long description +Filename: pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb +SHA256: 1111 + +Package: zlib1g +Version: 1:1.2.13.dfsg-1 +Architecture: amd64 +Filename: pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb +SHA256: 2222 +` + got, err := parseDebPackages(strings.NewReader(packages)) + if err != nil { + t.Fatal(err) + } + want := []indexFile{ + {Path: "pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb", Digest: "sha256:1111"}, + {Path: "pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb", Digest: "sha256:2222"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("parseDebPackages = %+v, want %+v", got, want) + } +} + +const repomd = ` + + 1700000000 + + + +` + +func TestRepomdPrimary(t *testing.T) { + got, err := repomdPrimary([]byte(repomd)) + if err != nil || got != "repodata/bbbb-primary.xml.zst" { + t.Errorf("repomdPrimary = %q, %v", got, err) + } + if _, err := repomdPrimary([]byte(``)); err == nil { + t.Error("repomdPrimary accepted a repomd without primary") + } +} + +const rpmPrimary = ` + + + curl + x86_64 + + 3333 + + + + curl + src + + 4444 + + + + tzdata + noarch + + 5555 + + + +` + +func TestParseRPMPrimary(t *testing.T) { + got, err := parseRPMPrimary(strings.NewReader(rpmPrimary)) + if err != nil { + t.Fatal(err) + } + want := []indexFile{ + {Path: "Packages/c/curl-8.6.0-10.fc40.x86_64.rpm", Digest: "sha256:3333"}, + {Path: "Packages/t/tzdata-2024a-5.fc40.noarch.rpm"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("parseRPMPrimary = %+v, want %+v", got, want) + } +} + +func TestDecompressByName(t *testing.T) { + var gz bytes.Buffer + w := gzip.NewWriter(&gz) + _, _ = w.Write([]byte("hello")) + _ = w.Close() + enc, _ := zstd.NewWriter(nil) + zst := enc.EncodeAll([]byte("hello"), nil) + + for name, data := range map[string][]byte{"Packages": []byte("hello"), "Packages.gz": gz.Bytes(), "primary.xml.zst": zst} { + r, err := decompressByName(name, bytes.NewReader(data)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + var out bytes.Buffer + if _, err := out.ReadFrom(r); err != nil || out.String() != "hello" { + t.Errorf("%s: got %q, %v", name, out.String(), err) + } + _ = r.Close() + } +} diff --git a/internal/mirror/refresh_index_test.go b/internal/mirror/refresh_index_test.go new file mode 100644 index 0000000..1f66c38 --- /dev/null +++ b/internal/mirror/refresh_index_test.go @@ -0,0 +1,231 @@ +package mirror + +import ( + "bytes" + "compress/gzip" + "context" + "net/http" + "net/http/httptest" + "slices" + "sync" + "testing" + "time" +) + +// indexServer serves repository index documents that a test can change. +type indexServer struct { + *httptest.Server + mu sync.Mutex + docs map[string][]byte +} + +func newIndexServer(t *testing.T) *indexServer { + t.Helper() + s := &indexServer{docs: map[string][]byte{}} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + body, ok := s.docs[r.URL.Path] + s.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(s.Close) + return s +} + +func (s *indexServer) set(path string, body []byte) { + s.mu.Lock() + defer s.mu.Unlock() + s.docs[path] = body +} + +func gzipped(t *testing.T, s string) []byte { + t.Helper() + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + if _, err := w.Write([]byte(s)); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// setupIndexRefresh returns a mirror whose package files come from a +// servingFetcher and whose indexes come from an indexServer. +func setupIndexRefresh(t *testing.T) (*Mirror, *servingFetcher, *indexServer) { + t.Helper() + m, fetcher := setupRefreshMirror(t, "package") + idx := newIndexServer(t) + m.proxy.HTTPClient = idx.Client() + m.proxy.CacheMetadata = true + return m, fetcher, idx +} + +// clientFetchesCurl caches an index document and a curl package file the +// way the handlers do for a client. +func clientFetchesCurl(t *testing.T, m *Mirror, metaEcosystem, indexURL, ecosystem, version, cacheFilename, fileURL string) { + t.Helper() + ctx := context.Background() + if _, _, err := m.proxy.FetchOrCacheMetadata(ctx, metaEcosystem, "key:"+indexURL, indexURL, "*/*"); err != nil { + t.Fatalf("fetching %s: %v", indexURL, err) + } + result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, ecosystem, "curl", version, cacheFilename, fileURL) + if err != nil { + t.Fatalf("fetching %s: %v", fileURL, err) + } + _ = result.Reader.Close() +} + +func refreshOnce(t *testing.T, m *Mirror) *Progress { + t.Helper() + progress, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + return progress +} + +func cachedNames(t *testing.T, m *Mirror, ecosystem, filename string) []string { + t.Helper() + names, err := m.db.CachedArtifactNames(ecosystem, filename) + if err != nil { + t.Fatal(err) + } + return names +} + +func TestRefreshAlpineIndex(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + dir := "/alpine/v3.20/main/aarch64" + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.0-r0\nA:aarch64\n\nP:busybox\nV:1.36.1-r29\n\n")) + clientFetchesCurl(t, m, "alpine", idx.URL+dir+"/APKINDEX.tar.gz", + "alpine", "8.9.0-r0", "edge/v3.20/main/aarch64/curl-8.9.0-r0.apk", idx.URL+dir+"/curl-8.9.0-r0.apk") + + // A new release of curl; busybox was never downloaded and stays out. + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.1-r0\nA:aarch64\n\nP:busybox\nV:1.36.1-r30\n\n")) + fetcher.reset() + progress := refreshOnce(t, m) + + if want := []string{idx.URL + dir + "/curl-8.9.1-r0.apk"}; !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 { + t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) + } + // Cached where the APK handler looks for it. + if names := cachedNames(t, m, "alpine", "edge/v3.20/main/aarch64/curl-8.9.1-r0.apk"); !slices.Equal(names, []string{"curl"}) { + t.Errorf("cached names = %v, want [curl]", names) + } + + fetcher.reset() + if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("second run: progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } +} + +func TestRefreshIndexNeverDowngrades(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + dir := "/alpine/v3.20/main/aarch64" + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.1-r0\n\n")) + clientFetchesCurl(t, m, "alpine", idx.URL+dir+"/APKINDEX.tar.gz", + "alpine", "8.9.1-r0", "edge/v3.20/main/aarch64/curl-8.9.1-r0.apk", idx.URL+dir+"/curl-8.9.1-r0.apk") + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.0-r0\n\n")) + fetcher.reset() + + if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } +} + +func TestRefreshDebianIndex(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + release := []byte("Suite: stable\nSHA256:\n 0000 1 main/binary-amd64/Packages.gz\n 0000 1 main/binary-arm64/Packages.gz\n") + packages := func(curl string) []byte { + return gzipped(t, "Package: curl\nVersion: "+curl+"\nArchitecture: amd64\nFilename: pool/main/c/curl/curl_"+curl+"_amd64.deb\nSHA256: "+ + "bc4a71180870f7945155fbb02f4b0a2e3faa2a62d6d31b7039013055ed19869a\n\n"+ + "Package: zlib1g\nVersion: 1:1.2.13.dfsg-1\nArchitecture: amd64\nFilename: pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb\n\n") + } + idx.set("/debian/dists/bookworm/InRelease", release) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("7.88.1-10")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease", + "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", idx.URL+"/debian/pool/main/c/curl/curl_7.88.1-10_amd64.deb") + + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("7.88.1-10+deb12u8")) + fetcher.reset() + progress := refreshOnce(t, m) + + // Only the amd64 index is read: nothing was downloaded for arm64. The + // file is checked against the index's SHA-256 of "package". + want := []string{idx.URL + "/debian/pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb"} + if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) + } + if names := cachedNames(t, m, "deb", "curl_7.88.1-10+deb12u8_amd64.deb"); !slices.Equal(names, []string{"curl"}) { + t.Errorf("cached names = %v, want [curl]", names) + } +} + +func TestRefreshDebianIndexChecksDigest(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + idx.set("/debian/dists/bookworm/InRelease", []byte("SHA256:\n 0000 1 main/binary-amd64/Packages\n")) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages", []byte("Package: curl\nFilename: pool/main/c/curl/curl_7.88.1-10_amd64.deb\n\n")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease", + "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", idx.URL+"/debian/pool/main/c/curl/curl_7.88.1-10_amd64.deb") + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages", []byte( + "Package: curl\nFilename: pool/main/c/curl/curl_7.88.1-11_amd64.deb\nSHA256: 0000000000000000000000000000000000000000000000000000000000000000\n\n")) + fetcher.reset() + + progress := refreshOnce(t, m) + if progress.Failed != 1 || progress.Completed != 0 { + t.Errorf("progress = %+v, want the file failing its digest", progress) + } + if names := cachedNames(t, m, "deb", "curl_7.88.1-11_amd64.deb"); len(names) != 0 { + t.Errorf("a file failing its digest was cached as %v", names) + } +} + +func TestRefreshRPMIndex(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + base := "/fedora/updates/40/Everything/x86_64" + primary := func(rel string) []byte { + return gzipped(t, `curlx86_64`+ + ``) + } + repomdFor := func(href string) []byte { + return []byte(``) + } + idx.set(base+"/repodata/repomd.xml", repomdFor("repodata/aaaa-primary.xml.gz")) + idx.set(base+"/repodata/aaaa-primary.xml.gz", primary("10")) + clientFetchesCurl(t, m, "rpm", idx.URL+base+"/repodata/repomd.xml", + "rpm", "8.6.0-10.fc40", "curl-8.6.0-10.fc40.x86_64.rpm", idx.URL+base+"/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm") + + idx.set(base+"/repodata/repomd.xml", repomdFor("repodata/bbbb-primary.xml.gz")) + idx.set(base+"/repodata/bbbb-primary.xml.gz", primary("11")) + fetcher.reset() + progress := refreshOnce(t, m) + + want := []string{idx.URL + base + "/Packages/c/curl-8.6.0-11.fc40.x86_64.rpm"} + if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 { + t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) + } + if names := cachedNames(t, m, "rpm", "curl-8.6.0-11.fc40.x86_64.rpm"); !slices.Equal(names, []string{"curl"}) { + t.Errorf("cached names = %v, want [curl]", names) + } +} + +func TestRefreshIndexWithoutRecordedURLDoesNothing(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + result, err := m.proxy.GetOrFetchArtifactFromURL(context.Background(), "alpine", "curl", "8.9.0-r0", + "edge/v3.20/main/aarch64/curl-8.9.0-r0.apk", idx.URL+"/alpine/v3.20/main/aarch64/curl-8.9.0-r0.apk") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + fetcher.reset() + + if progress := refreshOnce(t, m); progress.Completed != 0 || progress.Failed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 30c4e0b..8ad4768 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -25,6 +25,10 @@ type PackageVersion struct { // variants, when set, limit the files of a version that are mirrored to // those of these variants (see fileEcosystem.variant). variants map[string]bool + // fileURL, when set, is the one file to mirror, cached as cacheFilename + // and checked against digest when there is one. A refresh finds it in + // a repository index. + fileURL, cacheFilename, digest string } func (pv PackageVersion) String() string { From f80c826e735791fa38a85d365e7fe469785662e1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 14:09:02 +0100 Subject: [PATCH 033/198] Document mirror.refresh --- config.example.yaml | 14 ++++++++++++-- docs/configuration.md | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index d3b6da4..0ee2601 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -120,8 +120,8 @@ storage: # Store fetched artifacts. Set to false to stream every download from # upstream without storing it; metadata filtering, cooldown and the # denylist still apply. Useful when another cache sits in front of the - # proxy. false is incompatible with direct_serve, scanning, mirror_api and - # mirror.schedules. + # proxy. false is incompatible with direct_serve, scanning, mirror_api, + # mirror.schedules and mirror.refresh. cache_artifacts: true # Redirect cached artifact downloads to presigned storage URLs (HTTP 302) @@ -449,6 +449,16 @@ gradle: # - images: ["pkg:docker/library/alpine@3.20", "pkg:docker/library/nginx@stable"] # interval: "6h" # platforms: ["linux/amd64"] +# +# Mirror new releases of the most used packages in the background, so the +# first install after a release doesn't wait on the upstream. Cooldown and +# the denylist apply. +# refresh: +# interval: "24h" +# top: 50 # packages per ecosystem, by recent downloads +# since: "14d" +# max_versions: 200 # per run; default no cap +# max_bytes: "2GB" # per run; default no cap # Health endpoint configuration. health: diff --git a/docs/configuration.md b/docs/configuration.md index 3e94531..d2cad09 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -91,7 +91,7 @@ storage: Every download is a fresh upstream fetch, and concurrent requests for the same artifact are not combined. Artifacts with a digest known up front (OCI blobs, Swift archives, Helm charts) are verified while streaming: the response is sent chunked, and on a mismatch the connection is aborted before the response completes so the client never receives a tampered artifact as a good one. The same happens when the upstream connection fails mid-download. -`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve`, `mirror_api` or `mirror.schedules`, which all need stored artifacts, and the `mirror` command refuses to run with it. +`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve`, `mirror_api`, `mirror.schedules` or `mirror.refresh`, which all need stored artifacts, and the `mirror` command refuses to run with it. ### Retention @@ -1737,6 +1737,37 @@ Each run checks the tag with a HEAD request. If the tag still points to the cach - Mirrored layers are scanned when `scanning.enabled` is set. A blocked layer fails the run and keeps the previous image. - Schedules are read at startup; changes need a restart. They do not need `mirror_api`. +### Refreshing the most used packages + +For packages, the cache learns about a new release only when a client asks for it, and that first install waits on the upstream or fails while it is down. `mirror.refresh` mirrors new releases of the packages clients download most, so they are cached before anyone asks: + +```yaml +mirror: + refresh: + interval: 24h + top: 50 + since: 14d + max_versions: 200 + max_bytes: 2GB +``` + +| Setting | Env | Description | +|---------|-----|-------------| +| `mirror.refresh.interval` | - | How often the refresh runs, at least `1m` | +| `mirror.refresh.top` | - | Packages per ecosystem to refresh, the ones with the most downloads among those downloaded within `since`. Default: `50` | +| `mirror.refresh.since` | - | How recently a package must have been downloaded, with a `d` suffix for days. Default: `14d` | +| `mirror.refresh.max_versions` | - | Stop a run after mirroring this many new versions. Default: no cap | +| `mirror.refresh.max_bytes` | - | Stop a run once it has downloaded this much, such as `2GB`. It is checked before each version, so the last one can go over it. Default: no cap | + +For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. + +- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python, ABI and platform tags, and the sdist, or conda packages of the same subdir, build variant (such as `py312`) and file type. +- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newer file of each package and architecture. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. +- Container images are not refreshed; list the tags to follow in `mirror.schedules`. +- Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. +- Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. +- What a run mirrors is evicted by `storage.max_size` and retention like any other file. With `max_versions` and `max_bytes` a run stays bounded when many packages release at once. + ## Docker ### SQLite with Local Storage From e50179d0ccb3d80dc2fc9c1faf32775268b32934 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 14:09:59 +0100 Subject: [PATCH 034/198] Refresh Swift and Helm packages, leave Julia out --- docs/configuration.md | 2 +- internal/mirror/refresh.go | 12 +++++++----- internal/mirror/refresh_test.go | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d2cad09..2323ec3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1763,7 +1763,7 @@ For each of those packages a run lists the releases on the configured upstream a - PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python, ABI and platform tags, and the sdist, or conda packages of the same subdir, build variant (such as `py312`) and file type. - Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newer file of each package and architecture. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. -- Container images are not refreshed; list the tags to follow in `mirror.schedules`. +- Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. - Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. - What a run mirrors is evicted by `storage.max_size` and retention like any other file. With `max_versions` and `max_bytes` a run stays bounded when many packages release at once. diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index a20ba8b..fc9317a 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -3,6 +3,7 @@ package mirror import ( "context" "fmt" + "slices" "time" "github.com/git-pkgs/proxy/internal/database" @@ -162,12 +163,13 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] return items, nil } -// refreshable reports whether a refresh can list the releases of an -// ecosystem's packages. +// refreshable reports whether a refresh lists the releases of an +// ecosystem's packages. Images follow mirror.schedules, OS packages their +// repository indexes, and Julia files are cached by tree hash rather than by +// version. func refreshable(ecosystem string) bool { - if isImageEcosystem(ecosystem) || osIndexes[ecosystem].ecosystem != "" { + if isImageEcosystem(ecosystem) || osIndexes[ecosystem].ecosystem != "" || ecosystem == "julia" { return false } - _, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", nil) - return err == nil + return slices.Contains(registries.SupportedEcosystems(), purl.EcosystemToPURLType(ecosystem)) } diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index 684b44c..4a9f083 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -304,3 +304,20 @@ func TestFileVariants(t *testing.T) { } } } + +func TestRefreshable(t *testing.T) { + for ecosystem, want := range map[string]bool{ + "npm": true, "pypi": true, "conda": true, "composer": true, + // Their registries need the configured upstream URL. + "swift": true, "helm": true, + // Julia files are cached by tree hash, not by version. + "julia": false, + // Images follow mirror.schedules; OS packages their indexes. + "docker": false, "oci": false, "deb": false, "alpine": false, "rpm": false, + "generic": false, + } { + if got := refreshable(ecosystem); got != want { + t.Errorf("refreshable(%s) = %v, want %v", ecosystem, got, want) + } + } +} From c4979b9e762096c13ce3a1420f2103c194b4fc7e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 15:19:11 +0100 Subject: [PATCH 035/198] Count stored publish times under their ecosystem in the stats Cooldown stores the publish time of every version it looks up, also of packages nobody downloaded, which have no package row. The ecosystem stats counted those versions as unattributed. They now count under the type of their package PURL; only a PURL that can't be parsed stays unattributed. --- internal/database/analytics_test.go | 39 +++++++++++++++++++++++++++++ internal/database/queries.go | 36 +++++++++++++++++++++----- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/internal/database/analytics_test.go b/internal/database/analytics_test.go index 269b9d2..e537694 100644 --- a/internal/database/analytics_test.go +++ b/internal/database/analytics_test.go @@ -373,3 +373,42 @@ func TestMergeAliasedEcosystemsBreaksSizeTiesByName(t *testing.T) { t.Errorf("input order changed the surviving spelling: %q vs %q", first, second) } } + +// Cooldown stores publish times of versions nobody downloaded, without a +// package row. They count under the ecosystem of their package PURL, not as +// unattributed. +func TestGetEcosystemStatsAttributesPublishTimesByPURL(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) + published := time.Now().Add(-time.Hour) + for _, v := range []struct{ version, pkg string }{ + {"pkg:npm/react@19.0.0", "pkg:npm/react"}, + {"pkg:golang/github.com/swaggo/swag@v1.16.3", "pkg:golang/github.com/swaggo/swag"}, + {"pkg:golang/github.com/swaggo/swag@v1.16.4", "pkg:golang/github.com/swaggo/swag"}, + {"pkg:composer/vendor/pkg@1.0.0", "pkg:composer/vendor/pkg"}, + {"not-a-purl@1", "not-a-purl"}, + } { + if err := db.SetVersionPublishedAt(v.version, v.pkg, published); err != nil { + t.Fatal(err) + } + } + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + versions := map[string]int64{} + for _, e := range stats { + versions[e.Ecosystem] = e.Versions + } + want := map[string]int64{"npm": 2, "golang": 2, "composer": 1, unattributedEcosystem: 1} + for ecosystem, n := range want { + if versions[ecosystem] != n { + t.Errorf("versions of %s = %d, want %d (all: %v)", ecosystem, versions[ecosystem], n, versions) + } + } + if len(versions) != len(want) { + t.Errorf("ecosystems = %v, want %v", versions, want) + } + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index d7150e8..b3ec832 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "sort" + "strings" "time" "github.com/git-pkgs/artifacts" @@ -1458,15 +1459,18 @@ func (db *DB) GetEcosystemStats() ([]EcosystemStats, error) { return nil, err } - // Left joined and bucketed for the same reason as artifacts below: a version - // whose package row is missing would otherwise vanish here while still - // counting in GetCacheStats' COUNT(*), leaving the two unable to reconcile. + // Left joined for the same reason as artifacts below: a version whose + // package row is missing would otherwise vanish here while still counting + // in GetCacheStats' COUNT(*), leaving the two unable to reconcile. Such + // versions are mostly publish times cooldown stored for versions nobody + // downloaded, so they count under the type of their package PURL, and + // only a PURL that can't be parsed is unattributed. if err := db.eachCount(` - SELECT COALESCE(p.ecosystem, '`+unattributedEcosystem+`'), COUNT(*) + SELECT COALESCE(p.ecosystem, '`+orphanVersionPrefix+`' || v.package_purl), COUNT(*) FROM versions v LEFT JOIN packages p ON p.purl = v.package_purl - GROUP BY COALESCE(p.ecosystem, '`+unattributedEcosystem+`') - `, func(ecosystem string, n int64) { get(ecosystem).Versions = n }); err != nil { + GROUP BY COALESCE(p.ecosystem, '`+orphanVersionPrefix+`' || v.package_purl) + `, func(key string, n int64) { get(versionEcosystem(key)).Versions += n }); err != nil { return nil, err } @@ -1549,6 +1553,26 @@ func mergeAliasedEcosystems(stats []EcosystemStats) []EcosystemStats { // Grafana. Bucketing them keeps the two reconcilable. const unattributedEcosystem = "unattributed" +// orphanVersionPrefix marks the package PURL of a version without a package +// row in GetEcosystemStats. No ecosystem name starts with it. +const orphanVersionPrefix = "~" + +// versionEcosystem returns the ecosystem a GetEcosystemStats version count +// is for: the package row's, or the type of the package PURL of a version +// without one. The ecosystems that store publish times are named like their +// PURL types. +func versionEcosystem(key string) string { + packagePURL, orphan := strings.CutPrefix(key, orphanVersionPrefix) + if !orphan { + return key + } + p, err := purl.Parse(packagePURL) + if err != nil { + return unattributedEcosystem + } + return p.Type +} + func (db *DB) eachArtifactStat(get func(string) *EcosystemStats) error { rows, err := db.Query(` SELECT COALESCE(p.ecosystem, '` + unattributedEcosystem + `'), From 11ff4cf314c1d7dbeaf45167d01b2559b6543472 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 16:55:15 +0100 Subject: [PATCH 036/198] Fetch an npm packument again when its stored copy is gone within the TTL A row within its TTL was served without checking its blob, so a missing blob failed the streamed read and the request fell back to the bounded in-memory path. A read that finds the stored copy missing now fetches the packument again into storage, ignoring that row, and reads it once more, for packument, version and tarball requests. --- internal/handler/metadata_stored.go | 28 +++++- internal/handler/npm.go | 149 +++++++++++++++++----------- internal/handler/npm_stream_test.go | 87 +++++++++++----- 3 files changed, 176 insertions(+), 88 deletions(-) diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index 80cbaf4..7d43d01 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -45,13 +45,27 @@ var errStoredMetadataChanged = errors.New("stored metadata changed while it was // It returns errMetadataNotStored when the proxy doesn't cache metadata or // storing the document failed, and the caller reads it into memory instead. func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error) (*database.MetadataCacheEntry, error) { + return p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, "") +} + +// restoreStoredMetadata is storedMetadataDocument for a document whose row +// gone describes but whose stored copy was missing when read: the row is not +// served or revalidated, within its TTL or not, and the document is fetched +// again. +func (p *Proxy) restoreStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { + return p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, gone.ContentDigest.String) +} + +// storedMetadata implements storedMetadataDocument. A row with the digest +// gone is treated as if there were none. +func (p *Proxy) storedMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone string) (*database.MetadataCacheEntry, error) { if !p.CacheMetadata || p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 { return nil, errMetadataNotStored } if containsPathTraversal(cacheKey) { return nil, fmt.Errorf("invalid cache key: %q", cacheKey) } - if entry, fresh := p.storedMetadataEntryState(ctx, ecosystem, cacheKey); fresh { + if entry, fresh := p.storedMetadataEntryState(ctx, ecosystem, cacheKey, gone); fresh { metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) p.recordMetadataRead(entry) return entry, nil @@ -60,12 +74,12 @@ func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, "", false) + "\x00stored" res, _ := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { - entry, fresh := p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey) + entry, fresh := p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey, gone) if fresh { return metadataResult{entry: entry} } release, outcome, err := p.awaitFetchLease(fetchCtx, metrics.CacheKindMetadata, key, func() bool { - entry, fresh = p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey) + entry, fresh = p.storedMetadataEntryState(fetchCtx, ecosystem, cacheKey, gone) return fresh }) if err != nil { @@ -92,10 +106,14 @@ func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, } // storedMetadataEntryState reads the cache row of a document and reports -// whether it is within its TTL and usable by storedMetadataDocument. -func (p *Proxy) storedMetadataEntryState(ctx context.Context, ecosystem, cacheKey string) (*database.MetadataCacheEntry, bool) { +// whether it is within its TTL and usable by storedMetadataDocument. A row +// with the digest gone is returned as nil. +func (p *Proxy) storedMetadataEntryState(ctx context.Context, ecosystem, cacheKey, gone string) (*database.MetadataCacheEntry, bool) { entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) entry = p.migrateLegacyMetadataEntry(ctx, ecosystem, cacheKey, entry) + if gone != "" && entry != nil && entry.ContentDigest.String == gone { + return nil, false + } fresh := usableStoredMetadata(entry) && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL return entry, fresh } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 05e77ea..054400b 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -17,6 +17,7 @@ import ( "github.com/git-pkgs/proxy/internal/jsonscan" "github.com/git-pkgs/proxy/internal/metadata/npm" "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/proxy/internal/storage" ) const ( @@ -340,35 +341,36 @@ func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Reque if h.proxy.rewrites == nil { return false } - entry, err := h.storedPackument(r.Context(), upstreamURL, accept, metadataCacheKey) - switch { - case err == nil: - case errors.Is(err, errInMemory): - return false - default: - h.writeFetchError(w, err) - return true - } - fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) - rewritten, err := h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { - var scan *npm.Scan - err := readStored(open, func(in io.Reader) (err error) { - scan, err = npm.ScanPackument(in) - return err - }) - if err != nil { - return err - } - if fullRequired && scan.Abbreviated() { - return npm.ErrNotStreamable - } - return readStored(open, func(in io.Reader) error { - _, err := npm.RewriteStream(packageName, scan, in, out, options) - return err + var rewritten rewrittenMetadata + err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, func(entry *database.MetadataCacheEntry) (err error) { + rewritten, err = h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { + var scan *npm.Scan + err := readStored(open, func(in io.Reader) (err error) { + scan, err = npm.ScanPackument(in) + return err + }) + if err != nil { + return err + } + if fullRequired && scan.Abbreviated() { + return npm.ErrNotStreamable + } + return readStored(open, func(in io.Reader) error { + _, err := npm.RewriteStream(packageName, scan, in, out, options) + return err + }) }) + return err }) + switch { + case errors.Is(fetchErr, errInMemory): + return false + case fetchErr != nil: + h.writeFetchError(w, fetchErr) + return true + } if err != nil { if r.Context().Err() != nil { return true // the client left while waiting on a shared rewrite @@ -419,6 +421,30 @@ func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, return entry, err } +// readStoredPackument reads the packument stored for cacheKey with read, +// fetching it into storage first when needed. When the stored copy is gone, +// although its row says otherwise, the packument is fetched again and read +// once more. fetchErr is what storedPackument returned when the packument +// couldn't be fetched, and err is what read returned. +func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { + entry, fetchErr := h.storedPackument(ctx, packumentURL, accept, cacheKey) + if fetchErr != nil { + return nil, fetchErr + } + if err = read(entry); !errors.Is(err, storage.ErrNotFound) { + return err, nil + } + h.proxy.Logger.Warn("stored npm metadata is missing, fetching it again", "url", packumentURL) + entry, fetchErr = h.proxy.restoreStoredMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), entry) + if errors.Is(fetchErr, errMetadataNotStored) { + return nil, errInMemory + } + if fetchErr != nil { + return nil, fetchErr + } + return read(entry), nil +} + // packumentStreamValidator is packumentValidator for a packument read from // storage. func (h *NPMHandler) packumentStreamValidator() func(io.Reader) error { @@ -446,37 +472,40 @@ func (h *NPMHandler) streamedVersionMetadata(ctx context.Context, packageName, s if h.proxy.rewrites == nil { return rewrittenMetadata{}, errInMemory } - entry, err := h.storedPackument(ctx, packumentURL, accept, cacheKey) - if errors.Is(err, errInMemory) { - return rewrittenMetadata{}, err - } - if err != nil { - return rewrittenMetadata{}, &npmFetchError{err} - } fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) - m, err := h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { - var scan *npm.Scan - err := readStored(open, func(in io.Reader) (err error) { - scan, err = npm.ScanPackument(in) - return err - }) - if err != nil { - return err - } - if fullRequired && scan.Abbreviated() { - return npm.ErrNotStreamable - } - return readStored(open, func(in io.Reader) error { - version, err := npm.VersionStream(packageName, specifier, scan, in, options) + var m rewrittenMetadata + err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) (err error) { + m, err = h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { + var scan *npm.Scan + err := readStored(open, func(in io.Reader) (err error) { + scan, err = npm.ScanPackument(in) + return err + }) if err != nil { - return npmRewriteError(err) + return err } - _, err = out.Write(version) - return err + if fullRequired && scan.Abbreviated() { + return npm.ErrNotStreamable + } + return readStored(open, func(in io.Reader) error { + version, err := npm.VersionStream(packageName, specifier, scan, in, options) + if err != nil { + return npmRewriteError(err) + } + _, err = out.Write(version) + return err + }) }) + return err }) switch { + case errors.Is(fetchErr, errInMemory): + return rewrittenMetadata{}, fetchErr + case fetchErr != nil: + return rewrittenMetadata{}, &npmFetchError{fetchErr} + } + switch { case err == nil, ctx.Err() != nil, errors.Is(err, ErrVersionDenied), errors.Is(err, ErrVersionNotFound): return m, err case !errors.Is(err, npm.ErrNotStreamable): @@ -856,20 +885,20 @@ type npmVersionData struct { // otherwise from the packument read into memory. func (h *NPMHandler) versionData(ctx context.Context, packumentURL, cacheKey, version string) (npmVersionData, error) { accept := h.packumentAccept() - if entry, err := h.storedPackument(ctx, packumentURL, accept, cacheKey); err == nil { - var found npm.VersionEntry - err := readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { + var found npm.VersionEntry + err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) error { + return readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { found, err = npm.LookupVersion(in, version) return err }) - switch { - case err == nil && (!found.Abbreviated || h.packumentValidator() == nil): - return npmVersionData{entry: found.Entry, published: found.Published}, nil - case err != nil: - h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) - } - } else if !errors.Is(err, errInMemory) { - return npmVersionData{}, err + }) + switch { + case fetchErr == nil && err == nil && (!found.Abbreviated || h.packumentValidator() == nil): + return npmVersionData{entry: found.Entry, published: found.Published}, nil + case fetchErr == nil && err != nil: + h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) + case fetchErr != nil && !errors.Is(fetchErr, errInMemory): + return npmVersionData{}, fetchErr } body, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go index de5a17d..d63ab9b 100644 --- a/internal/handler/npm_stream_test.go +++ b/internal/handler/npm_stream_test.go @@ -232,33 +232,74 @@ func TestNPMStreamedPackumentStaleOn404(t *testing.T) { } } -// TestNPMStreamedPackumentBlobMissing checks that a 304 for a row whose -// document is gone from storage fetches the document again, also when it is -// larger than metadata_max_size. +// TestNPMStreamedPackumentBlobMissing checks that a row whose document is +// gone from storage, within its TTL or revalidated by a 304 after it, fetches +// the document again, also when it is larger than metadata_max_size. func TestNPMStreamedPackumentBlobMissing(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) + for _, tt := range []struct { + name string + ttl time.Duration + }{ + {"fresh row", time.Hour}, + {"expired row", time.Nanosecond}, + } { + t.Run(tt.name, func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + decodeStreamed(t, serve(handler, "/big", nil)) + + path := metadataStoragePath("npm", "big") + store.mu.Lock() + delete(store.files, path) + store.mu.Unlock() + proxy.MetadataTTL = tt.ttl + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { + t.Errorf("served %d versions, want 400", len(doc.Versions)) + } + if n := u.sent.Load(); n != 2 { + t.Errorf("upstream sent the document %d times, want 2", n) + } + store.mu.Lock() + defer store.mu.Unlock() + if string(store.files[path]) != body { + t.Error("the document was not stored again") + } + }) + } +} + +// TestNPMStreamedVersionBlobMissing checks that a version request and a +// tarball download fetch a packument again whose stored copy is gone while +// its row is within the TTL. +func TestNPMStreamedVersionBlobMissing(t *testing.T) { + u := newStreamUpstream(t, "") + u.body.Store(bigPackumentAt(u.URL, 100)) proxy, store, handler := streamProxy(t, u) - proxy.MetadataMaxSize = int64(len(body) / 4) - decodeStreamed(t, serve(handler, "/big", nil)) - + proxy.MetadataMaxSize = 1024 + fetcher := proxy.Fetcher.(*mockFetcher) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} path := metadataStoragePath("npm", "big") - store.mu.Lock() - delete(store.files, path) - store.mu.Unlock() - proxy.MetadataTTL = time.Nanosecond - proxy.SetMetadataRewriteCacheSize(64 << 20) - if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { - t.Errorf("served %d versions, want 400", len(doc.Versions)) - } - if n := u.sent.Load(); n != 2 { - t.Errorf("upstream sent the document %d times, want 2", n) - } - store.mu.Lock() - defer store.mu.Unlock() - if string(store.files[path]) != body { - t.Error("the document was not stored again") + for _, req := range []string{"/big/1.0.5", "/big/-/big-1.0.5.tgz"} { + decodeStreamed(t, serve(handler, "/big", nil)) + store.mu.Lock() + delete(store.files, path) + store.mu.Unlock() + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if rec := serve(handler, req, nil); rec.Code != http.StatusOK { + t.Errorf("%s: status %d: %.200s", req, rec.Code, rec.Body.String()) + } + store.mu.Lock() + _, ok := store.files[path] + store.mu.Unlock() + if !ok { + t.Errorf("%s: the packument was not stored again", req) + } } } From f0c96e87a81ce6f2404ddd45c7527747c6d6dbaf Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 17:50:24 +0100 Subject: [PATCH 037/198] Keep the stored copy when upstream cuts a metadata document off A body that ended before its Content-Length was read as complete in memory: ReadMetadataSized took the transport's io.ErrUnexpectedEOF for the end of a body shorter than its size hint. The half document was served and cached, or failed the filter with a 502, instead of the stored copy being served. Upstream bodies are now read through a reader that marks their read errors, and such a failure is an upstream failure on both paths: streamed, it no longer counts as a failure to store, which had sent the request to the bounded in-memory path. A stored packument whose bytes no longer match its row's digest is now fetched again like a missing one. Tests also cover concurrent requests sharing one fetch, client revalidation and upstream failures with nothing cached. The stream tests build cooldown through cooldown.NewPolicy as the server does, since a bare cooldown.Config parses itself on first use without locking. --- internal/handler/handler.go | 9 +- internal/handler/metadata_stored.go | 36 +++- internal/handler/npm.go | 10 +- internal/handler/npm_stream_harden_test.go | 206 +++++++++++++++++++++ internal/handler/npm_stream_test.go | 27 ++- internal/handler/read_metadata_test.go | 16 ++ 6 files changed, 287 insertions(+), 17 deletions(-) create mode 100644 internal/handler/npm_stream_harden_test.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 96ae655..e0cede7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -157,7 +157,9 @@ func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { // Reading into a buffer of that size avoids io.ReadAll growing its buffer // step by step, which peaks at more than twice the document size. size is // only a hint: a negative one, or one above the limit, reads as ReadMetadata -// does, and a body of a different length is still read correctly. +// does, and a body of a different length is still read correctly. A body +// read through upstreamBodyReader that upstream cuts off fails, even though +// it ends like a body shorter than size. func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { limit := p.MetadataMaxSize if limit <= 0 { @@ -170,7 +172,10 @@ func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { // the buffer and a longer one is noticed. buf := make([]byte, size+1) n, err := io.ReadFull(r, buf) + var cut *upstreamBodyError switch { + case errors.As(err, &cut): + return nil, err case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): if int64(n) < size/2 { // The hint was far off; don't keep the oversized buffer alive. @@ -1989,7 +1994,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e // ContentLength is -1 when the transport decompresses the body, and // otherwise the length of the bytes read here. - body, err := p.ReadMetadataSized(resp.Body, resp.ContentLength) + body, err := p.ReadMetadataSized(upstreamBodyReader{resp.Body}, resp.ContentLength) if err != nil { return nil, fmt.Errorf("reading response: %w", err) } diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index 7d43d01..8bbf3eb 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -200,9 +200,9 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, } // Go decompresses the body, since no Accept-Encoding was set, so it is // stored uncompressed. - var body io.Reader = resp.Body + var body io.Reader = upstreamBodyReader{resp.Body} if validate != nil { - staged, err := p.stageMetadata(ctx, ecosystem, cacheKey, resp.Body, validate) + staged, err := p.stageMetadata(ctx, ecosystem, cacheKey, body, validate) if err != nil { return nil, err } @@ -210,7 +210,7 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, body = staged } if _, err := p.storeMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), body, meta); err != nil { - return nil, fmt.Errorf("%w: %w", errMetadataNotStored, err) + return nil, notStored("", err) } stored, err := p.DB.GetMetadataCache(ecosystem, cacheKey) if err != nil || !usableStoredMetadata(stored) { @@ -232,7 +232,7 @@ func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, b } if _, _, err := p.Storage.Store(ctx, path, body); err != nil { discard() - return nil, fmt.Errorf("%w: staging: %w", errMetadataNotStored, err) + return nil, notStored("staging: ", err) } if err := readStored(func() (io.ReadCloser, error) { return p.Storage.Open(ctx, path) }, validate); err != nil { discard() @@ -246,6 +246,34 @@ func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, b return &stagedReader{ReadCloser: staged, discard: discard}, nil } +// upstreamBodyReader marks a failure to read a document from upstream, so +// it is told apart from a failure to store it. +type upstreamBodyReader struct{ r io.Reader } + +func (u upstreamBodyReader) Read(b []byte) (int, error) { + n, err := u.r.Read(b) + if err != nil && !errors.Is(err, io.EOF) { + err = &upstreamBodyError{err} + } + return n, err +} + +// upstreamBodyError is a failure to read a document from upstream. It is +// handled as an upstream failure: the stored copy, if any, is served. +type upstreamBodyError struct{ err error } + +func (e *upstreamBodyError) Error() string { return "reading upstream metadata: " + e.err.Error() } +func (e *upstreamBodyError) Unwrap() error { return e.err } + +// notStored returns a failure to store a document as errMetadataNotStored, +// unless reading it from upstream failed. +func notStored(what string, err error) error { + if body := (*upstreamBodyError)(nil); errors.As(err, &body) { + return body + } + return fmt.Errorf("%w: %s%w", errMetadataNotStored, what, err) +} + // stagedReader deletes the staged copy it reads when closed. type stagedReader struct { io.ReadCloser diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 054400b..50757a3 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -422,19 +422,19 @@ func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, } // readStoredPackument reads the packument stored for cacheKey with read, -// fetching it into storage first when needed. When the stored copy is gone, -// although its row says otherwise, the packument is fetched again and read -// once more. fetchErr is what storedPackument returned when the packument +// fetching it into storage first when needed. When the stored copy is gone +// or no longer has the digest its row records, the packument is fetched +// again and read once more. fetchErr is what storedPackument returned when the packument // couldn't be fetched, and err is what read returned. func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { entry, fetchErr := h.storedPackument(ctx, packumentURL, accept, cacheKey) if fetchErr != nil { return nil, fetchErr } - if err = read(entry); !errors.Is(err, storage.ErrNotFound) { + if err = read(entry); !errors.Is(err, storage.ErrNotFound) && !errors.Is(err, errStoredMetadataChanged) { return err, nil } - h.proxy.Logger.Warn("stored npm metadata is missing, fetching it again", "url", packumentURL) + h.proxy.Logger.Warn("stored npm metadata is missing or changed, fetching it again", "url", packumentURL, "error", err) entry, fetchErr = h.proxy.restoreStoredMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), entry) if errors.Is(fetchErr, errMetadataNotStored) { return nil, errInMemory diff --git a/internal/handler/npm_stream_harden_test.go b/internal/handler/npm_stream_harden_test.go new file mode 100644 index 0000000..ff8f97e --- /dev/null +++ b/internal/handler/npm_stream_harden_test.go @@ -0,0 +1,206 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// bigMetadataFiles returns a copy of what is in storage under the metadata path +// of big, staged copies included. +func bigMetadataFiles(store *mockStorage) map[string]string { + store.mu.Lock() + defer store.mu.Unlock() + files := map[string]string{} + for path, b := range store.files { + if strings.HasPrefix(path, metadataStoragePath("npm", "big")) { + files[path] = string(b) + } + } + return files +} + +// TestNPMStreamedPackumentBlobCorrupt checks that a stored copy whose bytes +// no longer match the digest of its row, within its TTL, is fetched again +// rather than failing the request, also above metadata_max_size. +func TestNPMStreamedPackumentBlobCorrupt(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + decodeStreamed(t, serve(handler, "/big", nil)) + + path := metadataStoragePath("npm", "big") + store.mu.Lock() + store.files[path] = []byte(strings.Replace(body, "1.0.5", "1.0.X", 1)) + store.mu.Unlock() + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != 400 { + t.Errorf("served %d versions, want 400", len(doc.Versions)) + } + if got := bigMetadataFiles(store)[path]; got != body { + t.Error("the corrupt copy was not replaced") + } +} + +// TestNPMStreamedPackumentTruncated checks that a document upstream cuts +// off halfway neither replaces the stored copy, which is served instead, +// nor leaves anything behind, with and without cooldown, streamed and in +// memory. +func TestNPMStreamedPackumentTruncated(t *testing.T) { + for _, tt := range []struct { + name string + cooldown, inMemory bool + }{ + {"cooldown", true, false}, + {"no cooldown", false, false}, + {"in memory with cooldown", true, true}, + {"in memory without cooldown", false, true}, + } { + t.Run(tt.name, func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + if !tt.cooldown { + proxy.Cooldown = nil + } + if tt.inMemory { + proxy.SetMetadataRewriteCacheSize(0) + } else { + proxy.MetadataMaxSize = int64(len(body) / 4) + } + want := len(decodeStreamed(t, serve(handler, "/big", nil)).Versions) + before := bigMetadataFiles(store) + + u.replace(bigPackument(401), `"v2"`) + u.truncate.Store(true) + proxy.MetadataTTL = time.Nanosecond + if !tt.inMemory { + proxy.SetMetadataRewriteCacheSize(64 << 20) + } + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != want { + t.Errorf("served %d versions, want the stored %d", len(doc.Versions), want) + } + after := bigMetadataFiles(store) + if len(after) != len(before) { + t.Errorf("storage holds %d files, %d before", len(after), len(before)) + } + for path, b := range before { + if after[path] != b { + t.Errorf("%s was changed", path) + } + } + }) + } +} + +// TestNPMStreamedPackumentTruncatedUncached checks that a document upstream +// cuts off halfway fails the request and is not kept, streamed and in memory. +func TestNPMStreamedPackumentTruncatedUncached(t *testing.T) { + for _, inMemory := range []bool{false, true} { + t.Run(map[bool]string{false: "streamed", true: "in memory"}[inMemory], func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.Cooldown = nil + if inMemory { + proxy.SetMetadataRewriteCacheSize(0) + } else { + proxy.MetadataMaxSize = int64(len(body) / 4) + } + u.truncate.Store(true) + + if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { + t.Errorf("status %d, want 502", rec.Code) + } + if files := bigMetadataFiles(store); len(files) != 0 { + t.Errorf("left %d files in storage", len(files)) + } + if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { + t.Error("left a metadata cache row") + } + }) + } +} + +// TestNPMStreamedPackumentUncachedErrors checks the answers to upstream +// failures with nothing cached. +func TestNPMStreamedPackumentUncachedErrors(t *testing.T) { + for _, tt := range []struct { + name string + set func(*streamUpstream) + want int + }{ + {"not found", func(u *streamUpstream) { u.gone.Store(true) }, http.StatusNotFound}, + {"upstream down", func(u *streamUpstream) { u.fail.Store(true) }, http.StatusBadGateway}, + } { + t.Run(tt.name, func(t *testing.T) { + u := newStreamUpstream(t, bigPackument(10)) + _, store, handler := streamProxy(t, u) + tt.set(u) + if rec := serve(handler, "/big", nil); rec.Code != tt.want { + t.Errorf("status %d, want %d", rec.Code, tt.want) + } + if files := bigMetadataFiles(store); len(files) != 0 { + t.Errorf("left %d files in storage", len(files)) + } + }) + } +} + +// TestNPMStreamedPackumentConcurrent checks that concurrent requests for an +// uncached packument share one upstream fetch and all get it. +func TestNPMStreamedPackumentConcurrent(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + const n = 16 + var wg sync.WaitGroup + recs := make([]*httptest.ResponseRecorder, n) + for i := range n { + wg.Go(func() { recs[i] = serve(handler, "/big", nil) }) + } + wg.Wait() + for _, rec := range recs { + if doc := decodeStreamed(t, rec); len(doc.Versions) != 400 { + t.Errorf("served %d versions, want 400", len(doc.Versions)) + } + } + if got := u.sent.Load(); got != 1 { + t.Errorf("upstream sent the document %d times, want once", got) + } +} + +// TestNPMStreamedPackumentClientRevalidates checks that a client sending +// the ETag it got gets a 304, and a new ETag once the document changes. +func TestNPMStreamedPackumentClientRevalidates(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, _, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + + first := serve(handler, "/big", nil) + etag := first.Header().Get("ETag") + if first.Code != http.StatusOK || etag == "" { + t.Fatalf("status %d, ETag %q", first.Code, etag) + } + if rec := serve(handler, "/big", http.Header{"If-None-Match": {etag}}); rec.Code != http.StatusNotModified { + t.Errorf("same document: status %d, want 304", rec.Code) + } + + u.replace(bigPackument(401), `"v2"`) + proxy.MetadataTTL = time.Nanosecond + rec := serve(handler, "/big", http.Header{"If-None-Match": {etag}}) + if rec.Code != http.StatusOK || rec.Header().Get("ETag") == etag { + t.Errorf("changed document: status %d, ETag %q (was %q)", rec.Code, rec.Header().Get("ETag"), etag) + } + if doc := decodeStreamed(t, rec); len(doc.Versions) != 401 { + t.Errorf("served %d versions, want 401", len(doc.Versions)) + } +} diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go index d63ab9b..a8a4ea2 100644 --- a/internal/handler/npm_stream_test.go +++ b/internal/handler/npm_stream_test.go @@ -49,10 +49,13 @@ func bigPackumentAt(base string, n int) string { type streamUpstream struct { *httptest.Server - body atomic.Value // string - etag atomic.Value // string - fail atomic.Bool - gone atomic.Bool + body atomic.Value // string + etag atomic.Value // string + fail atomic.Bool + gone atomic.Bool + // truncate makes upstream close the connection halfway through the + // document it announced. + truncate atomic.Bool requests atomic.Int32 // sent counts the responses that carried the document. sent atomic.Int32 @@ -80,9 +83,15 @@ func newStreamUpstream(t *testing.T, body string) *streamUpstream { w.WriteHeader(http.StatusNotModified) case r.URL.Path == "/big": u.sent.Add(1) + body := u.body.Load().(string) w.Header().Set("Content-Type", "application/json") w.Header().Set("ETag", u.etag.Load().(string)) - _, _ = io.WriteString(w, u.body.Load().(string)) + if u.truncate.Load() { + w.Header().Set("Content-Length", fmt.Sprint(len(body))) + _, _ = io.WriteString(w, body[:len(body)/2]) + return + } + _, _ = io.WriteString(w, body) default: http.NotFound(w, r) } @@ -98,7 +107,13 @@ func streamProxy(t *testing.T, u *streamUpstream) (*Proxy, *mockStorage, http.Ha proxy.CacheMetadata = true proxy.MetadataTTL = time.Hour proxy.SetMetadataRewriteCacheSize(64 << 20) - proxy.Cooldown = &cooldown.Config{Default: "3d"} + // As the server does: a cooldown.Config parses itself on first use, + // which concurrent requests would race on. + policy, err := cooldown.NewPolicy(&cooldown.Config{Default: "3d"}, nil) + if err != nil { + t.Fatal(err) + } + proxy.Cooldown = policy return proxy, store, NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() } diff --git a/internal/handler/read_metadata_test.go b/internal/handler/read_metadata_test.go index 639405c..d87d416 100644 --- a/internal/handler/read_metadata_test.go +++ b/internal/handler/read_metadata_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "runtime" + "strings" "testing" "testing/iotest" ) @@ -131,3 +132,18 @@ func TestReadMetadataSizedAllocatesOnce(t *testing.T) { t.Errorf("allocated %d bytes reading %d", allocated, size) } } + +// TestReadMetadataSizedTruncatedUpstream checks that an upstream body cut +// off before its Content-Length fails, while a body shorter than a size +// hint is still read. +func TestReadMetadataSizedTruncatedUpstream(t *testing.T) { + p := &Proxy{} + cut := io.MultiReader(strings.NewReader(`{"name":`), iotest.ErrReader(io.ErrUnexpectedEOF)) + if _, err := p.ReadMetadataSized(upstreamBodyReader{cut}, 100); !errors.Is(err, io.ErrUnexpectedEOF) { + t.Errorf("cut off body: error %v, want io.ErrUnexpectedEOF", err) + } + got, err := p.ReadMetadataSized(strings.NewReader(`{"name":"x"}`), 100) + if err != nil || string(got) != `{"name":"x"}` { + t.Errorf("short body = %q, %v", got, err) + } +} From b73980e8edaadcd5d0725d80e8b8bff8eaa05098 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 17:58:00 +0100 Subject: [PATCH 038/198] Refresh each suite of a shared Debian pool on its own An index file was mirrored only when it was newer than every cached version of the package and architecture. Debian suites share one pool, so once curl from trixie was cached, bookworm's security updates to curl were older than it and never mirrored. A run now mirrors the newest file an index lists when it isn't cached and an older version is, which still never downgrades and leaves a suite alone that only has a newer version cached. Tests also cover repositories listing several versions, an unreadable index next to working ones, Architecture: all packages, scoped npm packages, denied releases, a listing failure next to other packages, a cancelled run and Go version lines. --- docs/configuration.md | 2 +- internal/mirror/refresh_harden_test.go | 231 +++++++++++++++++++++++++ internal/mirror/refresh_index.go | 59 +++++-- 3 files changed, 272 insertions(+), 20 deletions(-) create mode 100644 internal/mirror/refresh_harden_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 2323ec3..7aea7fc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1762,7 +1762,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. - PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python, ABI and platform tags, and the sdist, or conda packages of the same subdir, build variant (such as `py312`) and file type. -- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newer file of each package and architecture. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. +- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. - Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go new file mode 100644 index 0000000..458f1e1 --- /dev/null +++ b/internal/mirror/refresh_harden_test.go @@ -0,0 +1,231 @@ +package mirror + +import ( + "context" + "reflect" + "slices" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/registries" +) + +// debCurlPackages returns a Packages index listing curl at version. +func debCurlPackages(t *testing.T, version string) []byte { + t.Helper() + return gzipped(t, "Package: curl\nVersion: "+version+"\nArchitecture: amd64\nFilename: pool/main/c/curl/curl_"+version+"_amd64.deb\n\n") +} + +// TestRefreshDebianIndexPerSuite checks that suites sharing a pool are +// refreshed each on its own: a newer curl cached from trixie must not keep +// bookworm's security update from being mirrored. +func TestRefreshDebianIndexPerSuite(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n") + pool := idx.URL + "/debian/pool/main/c/curl/" + for suite, version := range map[string]string{"bookworm": "7.88.1-10", "trixie": "8.14.1-2"} { + idx.set("/debian/dists/"+suite+"/InRelease", release) + idx.set("/debian/dists/"+suite+"/main/binary-amd64/Packages.gz", debCurlPackages(t, version)) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/"+suite+"/InRelease", + "deb", version, "curl_"+version+"_amd64.deb", pool+"curl_"+version+"_amd64.deb") + } + + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10+deb12u8")) + idx.set("/debian/dists/trixie/main/binary-amd64/Packages.gz", debCurlPackages(t, "8.14.1-2+deb13u1")) + fetcher.reset() + progress := refreshOnce(t, m) + + want := []string{pool + "curl_7.88.1-10+deb12u8_amd64.deb", pool + "curl_8.14.1-2+deb13u1_amd64.deb"} + got := fetcher.fetched() + slices.Sort(got) + if !slices.Equal(got, want) || progress.Completed != 2 { + t.Errorf("progress = %+v, fetched %v; want %v", progress, got, want) + } +} + +// TestRefreshDebianIndexSkipsOtherSuite checks that a suite's version of a +// package is not mirrored when only a newer one is cached: clients of that +// suite never fetched the package. +func TestRefreshDebianIndexSkipsOtherSuite(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n") + pool := idx.URL + "/debian/pool/main/c/curl/" + idx.set("/debian/dists/trixie/InRelease", release) + idx.set("/debian/dists/trixie/main/binary-amd64/Packages.gz", debCurlPackages(t, "8.14.1-2")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/trixie/InRelease", + "deb", "8.14.1-2", "curl_8.14.1-2_amd64.deb", pool+"curl_8.14.1-2_amd64.deb") + // A bookworm client fetched its index but never curl. + idx.set("/debian/dists/bookworm/InRelease", release) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10+deb12u8")) + if _, _, err := m.proxy.FetchOrCacheMetadata(t.Context(), "debian", "key:bookworm", idx.URL+"/debian/dists/bookworm/InRelease", "*/*"); err != nil { + t.Fatal(err) + } + fetcher.reset() + + if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } +} + +// TestRefreshRPMIndexListingSeveralVersions checks that a repository +// listing old versions next to the newest mirrors only the newest, and +// nothing once it is cached. +func TestRefreshRPMIndexListingSeveralVersions(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + base := "/fedora/updates/40/Everything/x86_64" + primary := func(rels ...string) []byte { + var pkgs string + for _, rel := range rels { + pkgs += `curlx86_64` + } + return gzipped(t, ``+pkgs+``) + } + idx.set(base+"/repodata/repomd.xml", []byte(``)) + idx.set(base+"/repodata/primary.xml.gz", primary("10")) + clientFetchesCurl(t, m, "rpm", idx.URL+base+"/repodata/repomd.xml", + "rpm", "8.6.0-10.fc40", "curl-8.6.0-10.fc40.x86_64.rpm", idx.URL+base+"/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm") + + idx.set(base+"/repodata/primary.xml.gz", primary("10", "12", "11")) + fetcher.reset() + progress := refreshOnce(t, m) + want := []string{idx.URL + base + "/Packages/c/curl-8.6.0-12.fc40.x86_64.rpm"} + if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 { + t.Fatalf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) + } + + fetcher.reset() + if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("second run: progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } +} + +// TestRefreshIndexFailureDoesNotStopRun checks that an index that can't be +// read is counted as failed and the other indexes are still refreshed. +func TestRefreshIndexFailureDoesNotStopRun(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n") + pool := idx.URL + "/debian/pool/main/c/curl/" + for suite, version := range map[string]string{"bookworm": "7.88.1-10", "trixie": "8.14.1-2"} { + idx.set("/debian/dists/"+suite+"/InRelease", release) + idx.set("/debian/dists/"+suite+"/main/binary-amd64/Packages.gz", debCurlPackages(t, version)) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/"+suite+"/InRelease", + "deb", version, "curl_"+version+"_amd64.deb", pool+"curl_"+version+"_amd64.deb") + } + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", []byte("not gzip")) + idx.set("/debian/dists/trixie/main/binary-amd64/Packages.gz", debCurlPackages(t, "8.14.1-2+deb13u1")) + fetcher.reset() + + progress := refreshOnce(t, m) + want := []string{pool + "curl_8.14.1-2+deb13u1_amd64.deb"} + if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 || progress.Failed != 1 { + t.Errorf("progress = %+v, fetched %v; want %v and one failure", progress, fetcher.fetched(), want) + } +} + +func TestNewestPerCachedLineGo(t *testing.T) { + allowAll := func(registries.Version) bool { return true } + for _, tt := range []struct { + name string + cached []string + releases []registries.Version + want []string + }{ + {"v prefix", []string{"v1.2.3"}, releases("v1.2.3", "v1.3.0"), []string{"v1.3.0"}}, + {"incompatible is its own line", []string{"v1.2.3"}, releases("v1.2.3", "v2.0.0+incompatible"), nil}, + {"pseudo-versions are prereleases", []string{"v1.2.3"}, releases("v1.2.3", "v1.2.4-0.20260101000000-abcdefabcdef"), nil}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := newestPerCachedLine("golang", tt.cached, tt.releases, allowAll); !reflect.DeepEqual(got, tt.want) { + t.Errorf("newestPerCachedLine = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRefreshScopedNPMPackage(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "@babel/core", "7.24.0") + m.Versions = stubReleases{"npm/@babel/core": oldReleases("7.24.0", "7.25.2")} + fetcher.reset() + + progress := refreshOnce(t, m) + if urls := fetcher.fetched(); progress.Completed != 1 || len(fetchedMatching(urls, "core-7.25.2.tgz")) != 1 { + t.Errorf("progress = %+v, fetched %v; want @babel/core 7.25.2", progress, urls) + } +} + +func TestRefreshSkipsDeniedRelease(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21", "4.17.22")} + policy, err := denylist.New([]string{"pkg:npm/lodash@4.17.22"}) + if err != nil { + t.Fatal(err) + } + m.proxy.Denylist = policy + fetcher.reset() + + progress := refreshOnce(t, m) + if urls := fetcher.fetched(); progress.Completed != 1 || len(fetchedMatching(urls, "lodash-4.17.21.tgz")) != 1 || len(fetchedMatching(urls, "4.17.22")) != 0 { + t.Errorf("progress = %+v, fetched %v; want 4.17.21, not the denied 4.17.22", progress, urls) + } +} + +func TestRefreshListingFailureDoesNotStopOthers(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + cacheNPM(t, m, "react", "18.3.0") + m.Versions = stubReleases{"npm/react": oldReleases("18.3.0", "18.3.1")} + fetcher.reset() + + progress := refreshOnce(t, m) + if urls := fetcher.fetched(); progress.Completed != 1 || progress.Failed != 1 || len(fetchedMatching(urls, "react-18.3.1.tgz")) != 1 { + t.Errorf("progress = %+v, fetched %v; want react refreshed and lodash failed", progress, urls) + } +} + +func TestRefreshStopsWhenCancelled(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21")} + fetcher.reset() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want nothing after cancelling", progress, fetcher.fetched()) + } +} + +// TestRefreshDebianArchitectureAll checks that a package for every +// architecture, listed in each binary-* index rather than one of its own, +// is refreshed. +func TestRefreshDebianArchitectureAll(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + packages := func(version string) []byte { + return gzipped(t, "Package: ca-certificates\nVersion: "+version+"\nArchitecture: all\nFilename: pool/main/c/ca-certificates/ca-certificates_"+version+"_all.deb\n\n") + } + idx.set("/debian/dists/bookworm/InRelease", []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n")) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("20230311")) + pool := idx.URL + "/debian/pool/main/c/ca-certificates/" + if _, _, err := m.proxy.FetchOrCacheMetadata(t.Context(), "debian", "key:bookworm", idx.URL+"/debian/dists/bookworm/InRelease", "*/*"); err != nil { + t.Fatal(err) + } + result, err := m.proxy.GetOrFetchArtifactFromURL(t.Context(), "deb", "ca-certificates", "20230311", "ca-certificates_20230311_all.deb", pool+"ca-certificates_20230311_all.deb") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", packages("20230311+deb12u1")) + fetcher.reset() + progress := refreshOnce(t, m) + if want := []string{pool + "ca-certificates_20230311+deb12u1_all.deb"}; !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 { + t.Errorf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) + } +} diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go index dc7818a..01168d2 100644 --- a/internal/mirror/refresh_index.go +++ b/internal/mirror/refresh_index.go @@ -137,43 +137,64 @@ func cachedUnder(ecosystem, base string, packages []database.UsedPackage) []cach } // newerIndexFiles returns, per package and architecture cached below base, -// the newest file the index lists when it is newer than every cached one. +// the newest file the index lists when it isn't cached and an older version +// is. Suites sharing a pool are told apart that way: a newer version cached +// from another suite doesn't hide an update to this one, and a package only +// cached in a newer version than this suite has isn't mirrored for it. func (m *Mirror) newerIndexFiles(ix osIndex, base string, cached []cachedOSFile, files []indexFile) []PackageVersion { - newestCached := map[string]cachedOSFile{} + cachedByKey := map[string][]cachedOSFile{} for _, c := range cached { - if current, ok := newestCached[c.key()]; !ok || vers.CompareWithScheme(c.version, current.version, ix.scheme) > 0 { - newestCached[c.key()] = c - } + cachedByKey[c.key()] = append(cachedByKey[c.key()], c) } - newest := map[string]PackageVersion{} + type candidate struct { + file indexFile + name, version string + } + newest := map[string]candidate{} var order []string for _, f := range files { name, version, arch := handler.OSPackageIdentity(ix.ecosystem, f.Path) key := name + "\x00" + arch - c, ok := newestCached[key] - if name == "" || !ok || vers.CompareWithScheme(version, c.version, ix.scheme) <= 0 { + if name == "" || len(cachedByKey[key]) == 0 { continue } if m.proxy != nil && !m.proxy.ReleaseAllowed(ix.ecosystem, name, version, time.Time{}) { continue } - if current, seen := newest[key]; seen && vers.CompareWithScheme(version, current.Version, ix.scheme) <= 0 { - continue - } else if !seen { + current, seen := newest[key] + if !seen { order = append(order, key) } - cacheFilename := path.Base(f.Path) - if dir := path.Dir(c.file.Filename); dir != "." { - cacheFilename = dir + "/" + cacheFilename - } - newest[key] = PackageVersion{ - Ecosystem: ix.ecosystem, Name: name, Version: version, - fileURL: base + "/" + f.Path, cacheFilename: cacheFilename, digest: f.Digest, + if !seen || vers.CompareWithScheme(version, current.version, ix.scheme) > 0 { + newest[key] = candidate{file: f, name: name, version: version} } } + items := make([]PackageVersion, 0, len(order)) for _, key := range order { - items = append(items, newest[key]) + n := newest[key] + var older *cachedOSFile + for i, c := range cachedByKey[key] { + cmp := vers.CompareWithScheme(n.version, c.version, ix.scheme) + if cmp == 0 { + older = nil + break + } + if cmp > 0 && older == nil { + older = &cachedByKey[key][i] + } + } + if older == nil { + continue + } + cacheFilename := path.Base(n.file.Path) + if dir := path.Dir(older.file.Filename); dir != "." { + cacheFilename = dir + "/" + cacheFilename + } + items = append(items, PackageVersion{ + Ecosystem: ix.ecosystem, Name: n.name, Version: n.version, + fileURL: base + "/" + n.file.Path, cacheFilename: cacheFilename, digest: n.file.Digest, + }) } return items } From 83a018bfd4adb87e3713813b80ee4d0f58ff7619 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 18:09:05 +0100 Subject: [PATCH 039/198] Test that stats versions add up with orphan publish times Covers namespaced npm and Maven PURLs, a publish time under an alias of a package row's ecosystem, and a PURL starting with the orphan prefix, and checks the per-ecosystem versions against GetCacheStats' total. --- internal/database/analytics_test.go | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/internal/database/analytics_test.go b/internal/database/analytics_test.go index e537694..5084acd 100644 --- a/internal/database/analytics_test.go +++ b/internal/database/analytics_test.go @@ -412,3 +412,53 @@ func TestGetEcosystemStatsAttributesPublishTimesByPURL(t *testing.T) { } }) } + +// TestGetEcosystemStatsVersionsReconcile checks that the versions counted +// per ecosystem add up to the cache stats' total, with publish times stored +// under namespaced PURLs, an alias of a package row's ecosystem, and a PURL +// that can't be parsed. +func TestGetEcosystemStatsVersionsReconcile(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) + seedArtifact(t, db, "go", "github.com/pkg/errors", "v0.9.1", 500, 1) + published := time.Now().Add(-time.Hour) + for _, v := range []struct{ version, pkg string }{ + {"pkg:npm/%40babel/core@7.25.2", "pkg:npm/%40babel/core"}, + {"pkg:maven/org.apache.commons/commons-lang3@3.14.0", "pkg:maven/org.apache.commons/commons-lang3"}, + {"pkg:pypi/requests@2.32.3", "pkg:pypi/requests"}, + {"pkg:golang/github.com/swaggo/swag@v1.16.4", "pkg:golang/github.com/swaggo/swag"}, + {"~odd@1", "~odd"}, + } { + if err := db.SetVersionPublishedAt(v.version, v.pkg, published); err != nil { + t.Fatal(err) + } + } + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + versions := map[string]int64{} + var sum int64 + for _, e := range stats { + versions[e.Ecosystem] += e.Versions + sum += e.Versions + } + want := map[string]int64{"npm": 2, "go": 2, "maven": 1, "pypi": 1, unattributedEcosystem: 1} + if len(versions) != len(want) { + t.Errorf("ecosystems = %v, want %v", versions, want) + } + for ecosystem, n := range want { + if versions[ecosystem] != n { + t.Errorf("versions of %s = %d, want %d (all: %v)", ecosystem, versions[ecosystem], n, versions) + } + } + cache, err := db.GetCacheStats() + if err != nil { + t.Fatal(err) + } + if sum != cache.TotalVersions { + t.Errorf("versions per ecosystem add up to %d, cache stats count %d", sum, cache.TotalVersions) + } + }) +} From fd16ba166464632104155d2cc270a26cef7e3dd8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 18:15:11 +0100 Subject: [PATCH 040/198] Let in-flight blobs finish when one fails errgroup.WithContext cancelled the blobs being fetched as soon as one failed. A client pull sharing one of those fetches failed with it, and the next run had to fetch them again. A failure now only stops further blobs from starting. --- internal/handler/container_mirror.go | 18 ++++++++++++--- internal/handler/container_mirror_test.go | 27 +++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index 82a98cf..c8ca65c 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -12,6 +12,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "golang.org/x/sync/errgroup" @@ -462,15 +463,26 @@ func (m *imageMirror) mirrorBlobs(ctx context.Context, manifest *ociManifest) er } // Large layers take most of the time, so they are fetched side by side. + // A failure starts no further blobs but lets those in flight finish + // rather than cancelling them: a client pull may share their fetch, and + // they would have to be fetched again next run. var mu sync.Mutex - g, gctx := errgroup.WithContext(ctx) + var failed atomic.Bool + var g errgroup.Group g.SetLimit(mirrorBlobConcurrency) for _, digest := range digests { g.Go(func() error { - size, fetched, err := m.mirrorBlob(gctx, digest) - if err != nil || !fetched { + if failed.Load() { + return nil + } + size, fetched, err := m.mirrorBlob(ctx, digest) + if err != nil { + failed.Store(true) return err } + if !fetched { + return nil + } mu.Lock() m.result.Fetched++ m.result.Bytes += size diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index d2cb54b..2131928 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -546,3 +546,30 @@ func TestMirrorImageFetchesBlobsConcurrently(t *testing.T) { t.Errorf("fetched = %d, want the manifest and its two blobs", result.Fetched) } } + +// TestMirrorImageBlobFailureKeepsOtherFetches checks that a blob failing +// doesn't cancel the blobs already being fetched: their fetch may be shared +// with a client pull, which would fail with it, and the next run would have +// to fetch them again. +func TestMirrorImageBlobFailureKeepsOtherFetches(t *testing.T) { + registry, image := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + + // The config blob is gone upstream; the layer is still being sent when + // its 404 arrives. + for path := range registry.blobs { + if !strings.HasSuffix(path, image.singleLayer) { + delete(registry.blobs, path) + } + } + registry.onBlob = func(string) { time.Sleep(300 * time.Millisecond) } + + if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"single"}}, nil); err == nil { + t.Fatal("MirrorImage succeeded with a blob missing upstream") + } + registry.offline.Store(true) + if response := pull(t, h, "/team/app/blobs/"+image.singleLayer, ""); response.Code != http.StatusOK { + t.Errorf("layer fetched alongside the failing blob: status %d, want it cached", response.Code) + } +} From 3a0e79d6b4e5e94bc52ccf24bda18eaa8e347159 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 31 Oct 2026 18:26:17 +0100 Subject: [PATCH 041/198] Fuzz SBOM parsing and test concurrent project uploads FuzzParse checks that an uploaded body never makes the inventory panic and that every component it records has a package URL, ecosystem and version. TestSaveProjectConcurrentUploads saves one project from eight uploads at once and checks that each succeeds and the components of exactly one remain. --- internal/database/projects_test.go | 52 ++++++++++++++++++++++++++++++ internal/inventory/fuzz_test.go | 28 ++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 internal/inventory/fuzz_test.go diff --git a/internal/database/projects_test.go b/internal/database/projects_test.go index 2a5dded..bf19d44 100644 --- a/internal/database/projects_test.go +++ b/internal/database/projects_test.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "slices" + "sync" "testing" "time" ) @@ -205,3 +206,54 @@ func TestMigrateAddsProjects(t *testing.T) { } }) } + +// TestSaveProjectConcurrentUploads checks that concurrent uploads of one +// project all succeed and leave the components of exactly one of them. +func TestSaveProjectConcurrentUploads(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + const uploads = 8 + sets := make([][]ProjectComponent, uploads) + for i := range sets { + for j := 0; j <= i; j++ { + sets[i] = append(sets[i], ProjectComponent{ + PackagePURL: "pkg:npm/dep" + string(rune('a'+j)), Ecosystem: "npm", + Name: "dep" + string(rune('a'+j)), Version: "1.0." + string(rune('0'+i)), + }) + } + } + now := time.Now().UTC() + errs := make(chan error, uploads) + var wg sync.WaitGroup + for i := range uploads { + wg.Go(func() { errs <- db.SaveProject("acme/web", "", "", now, sets[i]) }) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Errorf("SaveProject: %v", err) + } + } + + project, err := db.GetProject("acme/web") + if err != nil || project == nil { + t.Fatalf("GetProject = %v, %v", project, err) + } + components, err := db.GetProjectComponents(project.ID) + if err != nil { + t.Fatal(err) + } + if len(components) == 0 || len(components) > uploads { + t.Fatalf("%d components, want those of one upload", len(components)) + } + version := components[0].Version + for _, c := range components { + if c.Version != version { + t.Fatalf("components mix uploads: %+v", components) + } + } + if want := int(version[len(version)-1]-'0') + 1; len(components) != want { + t.Errorf("%d components of the upload with version %s, want %d", len(components), version, want) + } + }) +} diff --git a/internal/inventory/fuzz_test.go b/internal/inventory/fuzz_test.go new file mode 100644 index 0000000..f7fcadd --- /dev/null +++ b/internal/inventory/fuzz_test.go @@ -0,0 +1,28 @@ +package inventory + +import ( + "os" + "testing" +) + +// FuzzParse checks that no uploaded body makes the SBOM parsing panic. +func FuzzParse(f *testing.F) { + if data, err := os.ReadFile("testdata/syft-cyclonedx.json"); err == nil { + f.Add(data) + } + f.Add([]byte(`{"bomFormat":"CycloneDX","specVersion":"1.5","components":[{"type":"library","name":"x","version":"1","purl":"pkg:npm/x@1"}]}`)) + f.Add([]byte(`{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT","packages":[{"SPDXID":"SPDXRef-a","name":"a","versionInfo":"1","externalRefs":[{"referenceCategory":"PACKAGE-MANAGER","referenceType":"purl","referenceLocator":"pkg:conda/a@1?channel=c"}]}]}`)) + f.Add([]byte("SPDXVersion: SPDX-2.3\nPackageName: a\nSPDXID: SPDXRef-a\nExternalRef: PACKAGE-MANAGER purl pkg:deb/debian/a@1\n")) + f.Add([]byte(`xpkg:pypi/X_Y@1`)) + f.Fuzz(func(t *testing.T, data []byte) { + components, _, err := Parse(data) + if err != nil { + return + } + for _, c := range components { + if c.PackagePURL == "" || c.Version == "" || c.Ecosystem == "" { + t.Errorf("component with an empty field: %+v", c) + } + } + }) +} From 358b1355a9704ac61dffd3c9600a0f4f74b03f6a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 1 Nov 2026 12:48:04 +0100 Subject: [PATCH 042/198] Fail metadata reads when upstream cuts a document off ReadMetadataSized took the transport's io.ErrUnexpectedEOF for the end of a body shorter than its size hint. A document upstream closed before its Content-Length was served with a 200 and cached, or failed the cooldown filter with a 502, instead of the cached copy being served. Upstream bodies are now read through a reader that marks their read errors, and such a body fails the read as an upstream failure. --- internal/handler/handler.go | 9 +- internal/handler/metadata_truncated_test.go | 121 ++++++++++++++++++++ internal/handler/read_metadata_test.go | 16 +++ internal/handler/upstream_body.go | 26 +++++ 4 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 internal/handler/metadata_truncated_test.go create mode 100644 internal/handler/upstream_body.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4b7ebe7..f3fdcbb 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -157,7 +157,9 @@ func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { // Reading into a buffer of that size avoids io.ReadAll growing its buffer // step by step, which peaks at more than twice the document size. size is // only a hint: a negative one, or one above the limit, reads as ReadMetadata -// does, and a body of a different length is still read correctly. +// does, and a body of a different length is still read correctly. A body +// read through upstreamBodyReader that upstream cuts off fails, even though +// it ends like a body shorter than size. func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { limit := p.MetadataMaxSize if limit <= 0 { @@ -170,7 +172,10 @@ func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { // the buffer and a longer one is noticed. buf := make([]byte, size+1) n, err := io.ReadFull(r, buf) + var cut *upstreamBodyError switch { + case errors.As(err, &cut): + return nil, err case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): if int64(n) < size/2 { // The hint was far off; don't keep the oversized buffer alive. @@ -2010,7 +2015,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e // ContentLength is -1 when the transport decompresses the body, and // otherwise the length of the bytes read here. - body, err := p.ReadMetadataSized(resp.Body, resp.ContentLength) + body, err := p.ReadMetadataSized(upstreamBodyReader{resp.Body}, resp.ContentLength) if err != nil { return nil, fmt.Errorf("reading response: %w", err) } diff --git a/internal/handler/metadata_truncated_test.go b/internal/handler/metadata_truncated_test.go new file mode 100644 index 0000000..5044374 --- /dev/null +++ b/internal/handler/metadata_truncated_test.go @@ -0,0 +1,121 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/cooldown" +) + +// cutMetadataUpstream serves one npm packument and, when cut is set, closes +// the connection halfway through the document it announced. +type cutMetadataUpstream struct { + *httptest.Server + mu sync.Mutex + body string + cut bool +} + +func newCutMetadataUpstream(t *testing.T, body string) *cutMetadataUpstream { + t.Helper() + u := &cutMetadataUpstream{body: body} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + body, cut := u.body, u.cut + u.mu.Unlock() + if r.URL.Path != "/pkg" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Length", fmt.Sprint(len(body))) + if cut { + _, _ = io.WriteString(w, body[:len(body)/2]) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *cutMetadataUpstream) set(body string, cut bool) { + u.mu.Lock() + defer u.mu.Unlock() + u.body, u.cut = body, cut +} + +// truncPackument is an npm packument with n versions published long ago. +func truncPackument(n int) string { + var versions, times []string + for i := range n { + v := fmt.Sprintf("1.0.%d", i) + versions = append(versions, fmt.Sprintf(`%q:{"name":"pkg","version":%q,"dist":{"tarball":"https://registry.npmjs.org/pkg/-/pkg-%s.tgz"}}`, v, v, v)) + times = append(times, fmt.Sprintf(`%q:"2020-01-01T00:00:00Z"`, v)) + } + return `{"name":"pkg","dist-tags":{"latest":"1.0.0"},"versions":{` + strings.Join(versions, ",") + + `},"time":{` + strings.Join(times, ",") + `}}` +} + +// TestMetadataTruncatedUpstream checks that a document upstream cuts off +// before its Content-Length is neither served nor cached: the cached copy is +// served instead, with and without cooldown. +func TestMetadataTruncatedUpstream(t *testing.T) { + for _, withCooldown := range []bool{false, true} { + t.Run(map[bool]string{false: "no cooldown", true: "cooldown"}[withCooldown], func(t *testing.T) { + u := newCutMetadataUpstream(t, truncPackument(50)) + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + proxy.CacheMetadata = true + if withCooldown { + proxy.Cooldown = &cooldown.Config{Default: "3d"} + } + handler := NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() + + first := serve(handler, "/pkg", nil) + if first.Code != http.StatusOK { + t.Fatalf("first request: status %d", first.Code) + } + before, err := db.GetMetadataCache("npm", "pkg") + if err != nil || before == nil { + t.Fatalf("no metadata row after the first request: %v", err) + } + + u.set(truncPackument(51), true) + expireMetadata(t, db, "npm", "pkg") + proxy.SetMetadataRewriteCacheSize(0) + + rec := serve(handler, "/pkg", nil) + if rec.Code != http.StatusOK || rec.Body.String() != first.Body.String() { + t.Errorf("status %d, want 200 with the cached document", rec.Code) + } + after, err := db.GetMetadataCache("npm", "pkg") + if err != nil || after == nil || after.ContentDigest != before.ContentDigest { + t.Error("the cut-off document replaced the cached copy") + } + }) + } +} + +// TestMetadataTruncatedUpstreamUncached checks that a cut-off document with +// nothing cached fails the request and is not cached. +func TestMetadataTruncatedUpstreamUncached(t *testing.T) { + u := newCutMetadataUpstream(t, truncPackument(50)) + u.set(truncPackument(50), true) + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + proxy.CacheMetadata = true + handler := NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() + + if rec := serve(handler, "/pkg", nil); rec.Code != http.StatusBadGateway { + t.Errorf("status %d, want 502", rec.Code) + } + if entry, _ := db.GetMetadataCache("npm", "pkg"); entry != nil { + t.Error("cached the cut-off document") + } +} diff --git a/internal/handler/read_metadata_test.go b/internal/handler/read_metadata_test.go index 639405c..d87d416 100644 --- a/internal/handler/read_metadata_test.go +++ b/internal/handler/read_metadata_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "runtime" + "strings" "testing" "testing/iotest" ) @@ -131,3 +132,18 @@ func TestReadMetadataSizedAllocatesOnce(t *testing.T) { t.Errorf("allocated %d bytes reading %d", allocated, size) } } + +// TestReadMetadataSizedTruncatedUpstream checks that an upstream body cut +// off before its Content-Length fails, while a body shorter than a size +// hint is still read. +func TestReadMetadataSizedTruncatedUpstream(t *testing.T) { + p := &Proxy{} + cut := io.MultiReader(strings.NewReader(`{"name":`), iotest.ErrReader(io.ErrUnexpectedEOF)) + if _, err := p.ReadMetadataSized(upstreamBodyReader{cut}, 100); !errors.Is(err, io.ErrUnexpectedEOF) { + t.Errorf("cut off body: error %v, want io.ErrUnexpectedEOF", err) + } + got, err := p.ReadMetadataSized(strings.NewReader(`{"name":"x"}`), 100) + if err != nil || string(got) != `{"name":"x"}` { + t.Errorf("short body = %q, %v", got, err) + } +} diff --git a/internal/handler/upstream_body.go b/internal/handler/upstream_body.go new file mode 100644 index 0000000..7db9076 --- /dev/null +++ b/internal/handler/upstream_body.go @@ -0,0 +1,26 @@ +package handler + +import ( + "errors" + "io" +) + +// upstreamBodyReader marks a failure to read a document from upstream, so a +// body upstream cuts off is told apart from one that ends, and from a +// failure to store it. +type upstreamBodyReader struct{ r io.Reader } + +func (u upstreamBodyReader) Read(b []byte) (int, error) { + n, err := u.r.Read(b) + if err != nil && !errors.Is(err, io.EOF) { + err = &upstreamBodyError{err} + } + return n, err +} + +// upstreamBodyError is a failure to read a document from upstream. It is +// handled as an upstream failure: the stored copy, if any, is served. +type upstreamBodyError struct{ err error } + +func (e *upstreamBodyError) Error() string { return "reading upstream metadata: " + e.err.Error() } +func (e *upstreamBodyError) Unwrap() error { return e.err } From 143eef7f05d6b356e604d6a88bd89c2c7554f5e6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 1 Nov 2026 12:48:37 +0100 Subject: [PATCH 043/198] Test snapshots on cache hits and record them through proxyCachedWithEncoding TestAPKSnapshots_CacheHit covers the cache-hit branch with a metadata TTL above 0. proxyCachedWithEncoding takes optional hooks for a cache hit and a fetched body, so the APK handler no longer keeps a copy of it. --- internal/database/metadata_snapshots.go | 16 +++----- internal/handler/apk_snapshots.go | 34 ++++----------- internal/handler/apk_snapshots_test.go | 41 +++++++++++++++++++ internal/handler/conda.go | 2 +- internal/handler/handler.go | 25 +++++++++-- internal/handler/homebrew.go | 2 +- .../handler/proxy_cached_encoding_test.go | 6 +-- 7 files changed, 81 insertions(+), 45 deletions(-) diff --git a/internal/database/metadata_snapshots.go b/internal/database/metadata_snapshots.go index 4df1fa1..268baf7 100644 --- a/internal/database/metadata_snapshots.go +++ b/internal/database/metadata_snapshots.go @@ -61,17 +61,11 @@ func (db *DB) metadataSnapshotBefore(ecosystem, name string, beforeMS int64) (*M } // AddMetadataSnapshot records snap as the version of its document current -// from snap.ValidFromMS, unless the snapshot current at that time already -// has its digest, and reports whether it did. Comparing with the version -// current at that time rather than the newest one keeps a version a request -// recorded late, after a newer one, in its place in the history. On -// Postgres, replicas recording the same document take an advisory lock on it -// first, so their check and insert don't interleave: without it, two -// transactions can both read the previous digest and both insert. SQLite -// runs one write at a time, so the single statement is enough there. A -// document going back to an earlier digest is recorded again, since only the -// snapshot current at that time is compared. The casts let Postgres type the -// parameters of the INSERT ... SELECT. +// from snap.ValidFromMS, unless the snapshot current at that time already has +// its digest, and reports whether it did. On Postgres it first takes an +// advisory lock on the document, so two replicas can't both read the previous +// digest and both insert. The casts let Postgres type the parameters of the +// INSERT ... SELECT. func (db *DB) AddMetadataSnapshot(snap *MetadataSnapshot) (bool, error) { tx, err := db.Beginx() if err != nil { diff --git a/internal/handler/apk_snapshots.go b/internal/handler/apk_snapshots.go index d581980..401dc00 100644 --- a/internal/handler/apk_snapshots.go +++ b/internal/handler/apk_snapshots.go @@ -105,35 +105,17 @@ func (h *APKHandler) serveSnapshot(w http.ResponseWriter, r *http.Request, repos } } -// serveMetadataWithSnapshot is ProxyCached for an index or signature that -// keeps each version it serves as a snapshot. A version is recorded from the -// document the request serves, not by reading the cache again afterwards: a -// concurrent refresh can replace the cached copy at any time, and the version -// it replaced would then never be recorded. +// serveMetadataWithSnapshot is ProxyCached, recording each version it serves. func (h *APKHandler) serveMetadataWithSnapshot(w http.ResponseWriter, r *http.Request, upstream, cacheKey string) { - p := h.proxy ctx := context.WithoutCancel(r.Context()) - - // A cached copy within its TTL was recorded by the request that fetched - // it; this only catches one stored before snapshots were enabled, or - // whose recording failed. - if entry := p.freshMetadataEntry(apkEcosystem, cacheKey); entry != nil { - seen := time.Now() - if p.serveCachedMetadata(w, r, apkEcosystem, cacheKey) { + h.proxy.proxyCachedWithEncoding(w, r, upstream, apkEcosystem, cacheKey, "identity", &metadataHooks{ + hit: func(entry *database.MetadataCacheEntry, seen time.Time) { h.recordStoredSnapshot(ctx, cacheKey, entry, seen) - return - } - } - - body, contentType, contentEncoding, err := p.fetchOrCacheMetadata(r.Context(), apkEcosystem, cacheKey, upstream, "identity", nil, "*/*") - if err != nil { - p.writeMetadataFetchError(w, err) - return - } - // Record before responding, so the version is kept by the time a - // client has it. - h.recordServedSnapshot(ctx, cacheKey, body, contentType, contentEncoding, time.Now()) - p.writeMetadataCachedResponseWithEncoding(w, r, apkEcosystem, cacheKey, body, contentType, contentEncoding) + }, + fetched: func(body []byte, contentType, contentEncoding string) { + h.recordServedSnapshot(ctx, cacheKey, body, contentType, contentEncoding, time.Now()) + }, + }, "*/*") } // recordServedSnapshot keeps body, the document a request is serving, as the diff --git a/internal/handler/apk_snapshots_test.go b/internal/handler/apk_snapshots_test.go index b0f109d..83bb26d 100644 --- a/internal/handler/apk_snapshots_test.go +++ b/internal/handler/apk_snapshots_test.go @@ -339,3 +339,44 @@ func TestAPKSnapshots_ConcurrentReplicas(t *testing.T) { } } } + +// TestAPKSnapshots_CacheHit records an index cached before snapshots were +// enabled when a request is served from the cache within its TTL, and adds +// no row for a later hit on the same version. +func TestAPKSnapshots_CacheHit(t *testing.T) { + u := newAPKSnapshotUpstream(t) + h, db := newSnapshotAPKHandler(t, u.server.URL) + h.proxy.MetadataTTL = 5 * time.Minute + h.WithSnapshots(0) + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if n := snapshotCount(t, db); n != 0 { + t.Fatalf("%d snapshots recorded with snapshots off, want 0", n) + } + + h.WithSnapshots(30 * 24 * time.Hour) + // Upstream moved on; a hit still serves the cached v1. + u.index.Store("index v2") + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots recorded on a cache hit, want 1", n) + } + key := h.metadataCacheKey("alpine", u.server.URL, apkSnapshotIndexPath[1:]) + snap, err := db.LatestMetadataSnapshot(apkEcosystem, key) + if err != nil || snap == nil { + t.Fatalf("latest snapshot = %v, %v", snap, err) + } + body, err := h.proxy.Storage.Open(t.Context(), snap.StoragePath) + if err != nil { + t.Fatalf("recorded snapshot blob unreadable: %v", err) + } + got, _ := io.ReadAll(body) + _ = body.Close() + if string(got) != "index v1" { + t.Fatalf("snapshot blob = %q, want %q", got, "index v1") + } + + expectAPKResponse(t, h, "/alpine"+apkSnapshotIndexPath, http.StatusOK, "index v1") + if n := snapshotCount(t, db); n != 1 { + t.Fatalf("%d snapshots after a second hit on the same version, want 1", n) + } +} diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 3cbaf93..ca8fd2f 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -540,7 +540,7 @@ func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, ".json") { acceptEncoding = "gzip" } - h.proxy.proxyCachedWithEncoding(w, r, h.upstreamURL+r.URL.Path, "conda", cacheKey, acceptEncoding, "*/*") + h.proxy.proxyCachedWithEncoding(w, r, h.upstreamURL+r.URL.Path, "conda", cacheKey, acceptEncoding, nil, "*/*") } // proxyUpstream forwards a request to Anaconda without caching. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 4b7ebe7..7fef62e 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2151,7 +2151,7 @@ func (p *Proxy) cachedMetaFor(entry *database.MetadataCacheEntry) cachedMeta { // When metadata caching is disabled, the response is streamed directly to avoid buffering // large metadata responses (e.g. npm packages with many versions) in memory. func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey string, acceptHeaders ...string) { - p.proxyCachedWithEncoding(w, r, upstreamURL, ecosystem, cacheKey, "identity", acceptHeaders...) + p.proxyCachedWithEncoding(w, r, upstreamURL, ecosystem, cacheKey, "identity", nil, acceptHeaders...) } // proxyCachedWithEncoding is ProxyCached with an explicit upstream Accept-Encoding. @@ -2159,14 +2159,30 @@ func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, // compressed for large, non-hash-pinned metadata whose clients decode gzip // (Homebrew API and Conda repodata). The stored bytes and Content-Encoding are // replayed verbatim either way. -func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, acceptEncoding string, acceptHeaders ...string) { +// metadataHooks are optional callbacks for proxyCachedWithEncoding: hit runs +// after a fresh cached row is served, with that row and when it was read, and +// fetched runs with a fetched document before it is written. +type metadataHooks struct { + hit func(entry *database.MetadataCacheEntry, seen time.Time) + fetched func(body []byte, contentType, contentEncoding string) +} + +func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, acceptEncoding string, hooks *metadataHooks, acceptHeaders ...string) { if !p.CacheMetadata { // Stream directly without buffering when caching is off. p.proxyMetadataStream(w, r, upstreamURL, acceptEncoding, acceptHeaders...) return } - if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { + if hooks != nil && hooks.hit != nil { + if entry := p.freshMetadataEntry(ecosystem, cacheKey); entry != nil { + seen := time.Now() + if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { + hooks.hit(entry, seen) + return + } + } + } else if p.serveCachedMetadata(w, r, ecosystem, cacheKey) { return } @@ -2176,6 +2192,9 @@ func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, return } + if hooks != nil && hooks.fetched != nil { + hooks.fetched(body, contentType, contentEncoding) + } p.writeMetadataCachedResponseWithEncoding(w, r, ecosystem, cacheKey, body, contentType, contentEncoding) } diff --git a/internal/handler/homebrew.go b/internal/handler/homebrew.go index 0af67af..4e40d02 100644 --- a/internal/handler/homebrew.go +++ b/internal/handler/homebrew.go @@ -64,7 +64,7 @@ func (h *HomebrewHandler) Routes() http.Handler { if strings.HasPrefix(requestPath, "analytics/") { acceptEncoding = "identity" } - h.proxy.proxyCachedWithEncoding(w, r, upstreamURL, homebrewMetadataEcosystem, homebrewMetadataCacheKey(requestPath, r.URL.RawQuery), acceptEncoding, "*/*") + h.proxy.proxyCachedWithEncoding(w, r, upstreamURL, homebrewMetadataEcosystem, homebrewMetadataCacheKey(requestPath, r.URL.RawQuery), acceptEncoding, nil, "*/*") }) } diff --git a/internal/handler/proxy_cached_encoding_test.go b/internal/handler/proxy_cached_encoding_test.go index 35c8793..45eaef3 100644 --- a/internal/handler/proxy_cached_encoding_test.go +++ b/internal/handler/proxy_cached_encoding_test.go @@ -56,7 +56,7 @@ func (u *gzipWhenAskedUpstream) sawAcceptEncoding() string { func serveGzip(proxy *Proxy, upstreamURL string) *httptest.ResponseRecorder { w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/index.json", nil) - proxy.proxyCachedWithEncoding(w, r, upstreamURL, "gzip-test", "index", "gzip", "*/*") + proxy.proxyCachedWithEncoding(w, r, upstreamURL, "gzip-test", "index", "gzip", nil, "*/*") return w } @@ -187,7 +187,7 @@ func TestProxyCachedWithEncoding_UpsertFailureDiscardsBlob(t *testing.T) { // Seed an identity row + plain blob, as every key has before the gzip rollout. w := httptest.NewRecorder() proxy.proxyCachedWithEncoding(w, httptest.NewRequest(http.MethodGet, "/index.json", nil), - upstream.URL+"/index.json", "gzip-test", "index", "identity", "*/*") + upstream.URL+"/index.json", "gzip-test", "index", "identity", nil, "*/*") if w.Code != http.StatusOK { t.Fatalf("seed status = %d, want 200", w.Code) } @@ -250,7 +250,7 @@ func TestProxyCachedWithEncoding_StaleFallbackRereadsRow(t *testing.T) { w := httptest.NewRecorder() proxy.proxyCachedWithEncoding(w, httptest.NewRequest(http.MethodGet, "/index.json", nil), - upstream.URL+"/index.json", "gzip-test", "index", "identity", "*/*") + upstream.URL+"/index.json", "gzip-test", "index", "identity", nil, "*/*") if w.Code != http.StatusOK { t.Fatalf("seed status = %d, want 200", w.Code) } From 6dbda21faf5092e663029d5fcb8c59ee5adb4654 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 1 Nov 2026 12:49:09 +0100 Subject: [PATCH 044/198] Move the in-memory truncation fix to its own pull request The change to ReadMetadataSized and fetchUpstreamMetadata, with its tests, is now #649. upstreamBodyReader stays for storeUpstreamMetadata, in a file of its own that #649 adds with the same content. --- internal/handler/handler.go | 9 +-- internal/handler/metadata_stored.go | 19 ------- internal/handler/npm_stream_harden_test.go | 65 +++++++--------------- internal/handler/read_metadata_test.go | 16 ------ internal/handler/upstream_body.go | 26 +++++++++ 5 files changed, 49 insertions(+), 86 deletions(-) create mode 100644 internal/handler/upstream_body.go diff --git a/internal/handler/handler.go b/internal/handler/handler.go index e0cede7..96ae655 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -157,9 +157,7 @@ func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) { // Reading into a buffer of that size avoids io.ReadAll growing its buffer // step by step, which peaks at more than twice the document size. size is // only a hint: a negative one, or one above the limit, reads as ReadMetadata -// does, and a body of a different length is still read correctly. A body -// read through upstreamBodyReader that upstream cuts off fails, even though -// it ends like a body shorter than size. +// does, and a body of a different length is still read correctly. func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { limit := p.MetadataMaxSize if limit <= 0 { @@ -172,10 +170,7 @@ func (p *Proxy) ReadMetadataSized(r io.Reader, size int64) ([]byte, error) { // the buffer and a longer one is noticed. buf := make([]byte, size+1) n, err := io.ReadFull(r, buf) - var cut *upstreamBodyError switch { - case errors.As(err, &cut): - return nil, err case errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF): if int64(n) < size/2 { // The hint was far off; don't keep the oversized buffer alive. @@ -1994,7 +1989,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e // ContentLength is -1 when the transport decompresses the body, and // otherwise the length of the bytes read here. - body, err := p.ReadMetadataSized(upstreamBodyReader{resp.Body}, resp.ContentLength) + body, err := p.ReadMetadataSized(resp.Body, resp.ContentLength) if err != nil { return nil, fmt.Errorf("reading response: %w", err) } diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index 8bbf3eb..b3bc420 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -246,25 +246,6 @@ func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, b return &stagedReader{ReadCloser: staged, discard: discard}, nil } -// upstreamBodyReader marks a failure to read a document from upstream, so -// it is told apart from a failure to store it. -type upstreamBodyReader struct{ r io.Reader } - -func (u upstreamBodyReader) Read(b []byte) (int, error) { - n, err := u.r.Read(b) - if err != nil && !errors.Is(err, io.EOF) { - err = &upstreamBodyError{err} - } - return n, err -} - -// upstreamBodyError is a failure to read a document from upstream. It is -// handled as an upstream failure: the stored copy, if any, is served. -type upstreamBodyError struct{ err error } - -func (e *upstreamBodyError) Error() string { return "reading upstream metadata: " + e.err.Error() } -func (e *upstreamBodyError) Unwrap() error { return e.err } - // notStored returns a failure to store a document as errMetadataNotStored, // unless reading it from upstream failed. func notStored(what string, err error) error { diff --git a/internal/handler/npm_stream_harden_test.go b/internal/handler/npm_stream_harden_test.go index ff8f97e..ca7143f 100644 --- a/internal/handler/npm_stream_harden_test.go +++ b/internal/handler/npm_stream_harden_test.go @@ -49,39 +49,24 @@ func TestNPMStreamedPackumentBlobCorrupt(t *testing.T) { // TestNPMStreamedPackumentTruncated checks that a document upstream cuts // off halfway neither replaces the stored copy, which is served instead, -// nor leaves anything behind, with and without cooldown, streamed and in -// memory. +// nor leaves anything behind, with and without cooldown. func TestNPMStreamedPackumentTruncated(t *testing.T) { - for _, tt := range []struct { - name string - cooldown, inMemory bool - }{ - {"cooldown", true, false}, - {"no cooldown", false, false}, - {"in memory with cooldown", true, true}, - {"in memory without cooldown", false, true}, - } { - t.Run(tt.name, func(t *testing.T) { + for _, withCooldown := range []bool{true, false} { + t.Run(map[bool]string{true: "cooldown", false: "no cooldown"}[withCooldown], func(t *testing.T) { body := bigPackument(400) u := newStreamUpstream(t, body) proxy, store, handler := streamProxy(t, u) - if !tt.cooldown { + if !withCooldown { proxy.Cooldown = nil } - if tt.inMemory { - proxy.SetMetadataRewriteCacheSize(0) - } else { - proxy.MetadataMaxSize = int64(len(body) / 4) - } + proxy.MetadataMaxSize = int64(len(body) / 4) want := len(decodeStreamed(t, serve(handler, "/big", nil)).Versions) before := bigMetadataFiles(store) u.replace(bigPackument(401), `"v2"`) u.truncate.Store(true) proxy.MetadataTTL = time.Nanosecond - if !tt.inMemory { - proxy.SetMetadataRewriteCacheSize(64 << 20) - } + proxy.SetMetadataRewriteCacheSize(64 << 20) if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != want { t.Errorf("served %d versions, want the stored %d", len(doc.Versions), want) @@ -100,31 +85,23 @@ func TestNPMStreamedPackumentTruncated(t *testing.T) { } // TestNPMStreamedPackumentTruncatedUncached checks that a document upstream -// cuts off halfway fails the request and is not kept, streamed and in memory. +// cuts off halfway fails the request and is not kept. func TestNPMStreamedPackumentTruncatedUncached(t *testing.T) { - for _, inMemory := range []bool{false, true} { - t.Run(map[bool]string{false: "streamed", true: "in memory"}[inMemory], func(t *testing.T) { - body := bigPackument(400) - u := newStreamUpstream(t, body) - proxy, store, handler := streamProxy(t, u) - proxy.Cooldown = nil - if inMemory { - proxy.SetMetadataRewriteCacheSize(0) - } else { - proxy.MetadataMaxSize = int64(len(body) / 4) - } - u.truncate.Store(true) + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.Cooldown = nil + proxy.MetadataMaxSize = int64(len(body) / 4) + u.truncate.Store(true) - if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { - t.Errorf("status %d, want 502", rec.Code) - } - if files := bigMetadataFiles(store); len(files) != 0 { - t.Errorf("left %d files in storage", len(files)) - } - if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { - t.Error("left a metadata cache row") - } - }) + if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { + t.Errorf("status %d, want 502", rec.Code) + } + if files := bigMetadataFiles(store); len(files) != 0 { + t.Errorf("left %d files in storage", len(files)) + } + if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { + t.Error("left a metadata cache row") } } diff --git a/internal/handler/read_metadata_test.go b/internal/handler/read_metadata_test.go index d87d416..639405c 100644 --- a/internal/handler/read_metadata_test.go +++ b/internal/handler/read_metadata_test.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "runtime" - "strings" "testing" "testing/iotest" ) @@ -132,18 +131,3 @@ func TestReadMetadataSizedAllocatesOnce(t *testing.T) { t.Errorf("allocated %d bytes reading %d", allocated, size) } } - -// TestReadMetadataSizedTruncatedUpstream checks that an upstream body cut -// off before its Content-Length fails, while a body shorter than a size -// hint is still read. -func TestReadMetadataSizedTruncatedUpstream(t *testing.T) { - p := &Proxy{} - cut := io.MultiReader(strings.NewReader(`{"name":`), iotest.ErrReader(io.ErrUnexpectedEOF)) - if _, err := p.ReadMetadataSized(upstreamBodyReader{cut}, 100); !errors.Is(err, io.ErrUnexpectedEOF) { - t.Errorf("cut off body: error %v, want io.ErrUnexpectedEOF", err) - } - got, err := p.ReadMetadataSized(strings.NewReader(`{"name":"x"}`), 100) - if err != nil || string(got) != `{"name":"x"}` { - t.Errorf("short body = %q, %v", got, err) - } -} diff --git a/internal/handler/upstream_body.go b/internal/handler/upstream_body.go new file mode 100644 index 0000000..7db9076 --- /dev/null +++ b/internal/handler/upstream_body.go @@ -0,0 +1,26 @@ +package handler + +import ( + "errors" + "io" +) + +// upstreamBodyReader marks a failure to read a document from upstream, so a +// body upstream cuts off is told apart from one that ends, and from a +// failure to store it. +type upstreamBodyReader struct{ r io.Reader } + +func (u upstreamBodyReader) Read(b []byte) (int, error) { + n, err := u.r.Read(b) + if err != nil && !errors.Is(err, io.EOF) { + err = &upstreamBodyError{err} + } + return n, err +} + +// upstreamBodyError is a failure to read a document from upstream. It is +// handled as an upstream failure: the stored copy, if any, is served. +type upstreamBodyError struct{ err error } + +func (e *upstreamBodyError) Error() string { return "reading upstream metadata: " + e.err.Error() } +func (e *upstreamBodyError) Unwrap() error { return e.err } From b08cebf9e4d45f723f4e39b9c5d6bf94f596427d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 1 Nov 2026 12:53:00 +0100 Subject: [PATCH 045/198] Cap npm packuments streamed into storage metadata_max_size bounds what is read into memory, but a packument streamed from upstream into storage had no limit, and with cooldown was stored twice. It is now capped by metadata_stream_max_size, 1 GB by default. A document over the cap fails as an upstream failure: nothing is stored, and the stored copy, if any, is served instead. The metadata settings move into configureMetadata to keep serve under the maintidx limit. --- cmd/proxy/main.go | 1 + docs/configuration.md | 7 +-- internal/config/config.go | 31 ++++++++++-- internal/config/config_test.go | 23 +++++++++ internal/handler/handler.go | 23 +++++---- internal/handler/metadata_stored.go | 21 ++++++++- internal/handler/npm_stream_harden_test.go | 55 ++++++++++++++++++++++ internal/server/server.go | 14 ++++-- 8 files changed, 155 insertions(+), 20 deletions(-) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 824f2e2..d39125d 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -561,6 +561,7 @@ func runMirror() { proxy.CacheMetadata = true // mirror always caches metadata proxy.MetadataTTL = cfg.ParseMetadataTTL() proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() + proxy.MetadataStreamMaxSize = cfg.ParseMetadataStreamMaxSize() m := newCommandMirror(cfg, proxy, images, db, store, logger, *concurrency) switch s := source.(type) { diff --git a/docs/configuration.md b/docs/configuration.md index 93a7299..5c33a6e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1403,13 +1403,14 @@ When upstream is unreachable and the cached entry is past its TTL, the proxy ser ### Metadata size limit -Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. With `cache_metadata` and the metadata rewrite cache on, an npm packument is streamed from the upstream into storage and rewritten from there in two passes, so it isn't held in memory and the limit doesn't apply to it; the rewritten copy is kept gzip-compressed. Requests for a single version (`//`) still read the packument into memory. +Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. With `cache_metadata` and the metadata rewrite cache on, an npm packument is streamed from the upstream into storage and rewritten from there in two passes, so it isn't held in memory and the limit doesn't apply to it; the rewritten copy is kept gzip-compressed. Requests for a single version (`//`) and tarball downloads read the version from the stored copy the same way. `metadata_stream_max_size` (1 GB by default) caps a document streamed this way; one over it isn't stored, and the stored copy, if any, is served instead. ```yaml -metadata_max_size: "100MB" # default +metadata_max_size: "100MB" # default +metadata_stream_max_size: "1GB" # default ``` -Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`. +Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`, and `PROXY_METADATA_STREAM_MAX_SIZE=2GB` for `metadata_stream_max_size`. ### Artifact size limit diff --git a/internal/config/config.go b/internal/config/config.go index 5e90150..14790a9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -169,6 +169,11 @@ type Config struct { // size return ErrMetadataTooLarge. Default: "100MB". MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"` + // MetadataStreamMaxSize is the largest metadata document the proxy + // streams from upstream into storage instead of buffering it, which + // metadata_max_size doesn't cover (e.g. "1GB"). Default: "1GB". + MetadataStreamMaxSize string `json:"metadata_stream_max_size" yaml:"metadata_stream_max_size"` + // ArtifactMaxSize is the largest upstream artifact the proxy will fetch // (e.g. "2GB"). Larger artifacts are refused and not cached. Default: "" // (unlimited). "0" is also unlimited. @@ -1480,6 +1485,7 @@ func (c *Config) LoadFromEnv() { setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") + setEnvString(&c.MetadataStreamMaxSize, "PROXY_METADATA_STREAM_MAX_SIZE") setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE") setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE") setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT") @@ -1726,6 +1732,7 @@ const ( defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default defaultHitFlushInterval = time.Second defaultMetadataMaxSize = 100 << 20 + defaultMetadataStreamMaxSize = 1 << 30 defaultMetadataRewriteCacheSize = 256 << 20 defaultGradleBuildCacheMaxUploadSize = 100 << 20 defaultGradleBuildCacheSweepInterval = 10 * time.Minute @@ -1779,22 +1786,25 @@ func validateRequestTimeout(field, s string) error { return nil } -func validateMetadataMaxSize(s string) error { +func validateMetadataMaxSize(name, s string) error { if s == "" { return nil } size, err := ParseSize(s) if err != nil { - return fmt.Errorf("invalid metadata_max_size: %w", err) + return fmt.Errorf("invalid %s: %w", name, err) } if size <= 0 { - return fmt.Errorf("invalid metadata_max_size %q: must be positive", s) + return fmt.Errorf("invalid %s %q: must be positive", name, s) } return nil } func (c *Config) validateDownloadSizeLimits() error { - if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil { + if err := validateMetadataMaxSize("metadata_max_size", c.MetadataMaxSize); err != nil { + return err + } + if err := validateMetadataMaxSize("metadata_stream_max_size", c.MetadataStreamMaxSize); err != nil { return err } return validateArtifactMaxSize(c.ArtifactMaxSize) @@ -1857,6 +1867,19 @@ func (c *Config) ParseMetadataMaxSize() int64 { return size } +// ParseMetadataStreamMaxSize returns the size limit for metadata streamed +// into storage in bytes. Returns 1GB if unset or invalid. +func (c *Config) ParseMetadataStreamMaxSize() int64 { + if c.MetadataStreamMaxSize == "" { + return defaultMetadataStreamMaxSize + } + size, err := ParseSize(c.MetadataStreamMaxSize) + if err != nil || size <= 0 { + return defaultMetadataStreamMaxSize + } + return size +} + // ParseArtifactMaxSize returns the artifact size limit in bytes, or 0 for // no limit. func (c *Config) ParseArtifactMaxSize() int64 { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index a0132cf..00f45d9 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -785,6 +785,29 @@ func TestParseMetadataMaxSize(t *testing.T) { } } +func TestParseMetadataStreamMaxSize(t *testing.T) { + cfg := Default() + if got := cfg.ParseMetadataStreamMaxSize(); got != defaultMetadataStreamMaxSize { + t.Errorf("unset: %d, want %d", got, defaultMetadataStreamMaxSize) + } + cfg.MetadataStreamMaxSize = "2GB" + if got := cfg.ParseMetadataStreamMaxSize(); got != 2<<30 { + t.Errorf("2GB: %d", got) + } + t.Setenv("PROXY_METADATA_STREAM_MAX_SIZE", "300MB") + cfg = Default() + cfg.LoadFromEnv() + if got := cfg.ParseMetadataStreamMaxSize(); got != 300<<20 { + t.Errorf("from env: %d", got) + } + for _, bad := range []string{"0", "lots"} { + cfg.MetadataStreamMaxSize = bad + if err := cfg.Validate(); err == nil { + t.Errorf("metadata_stream_max_size %q validated", bad) + } + } +} + func TestValidateMetadataMaxSize(t *testing.T) { cfg := Default() cfg.MetadataMaxSize = "not-a-size" diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 96ae655..832ac47 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -142,6 +142,10 @@ func ifNoneMatchHits(header, etag string) bool { // defaultMetadataMaxSize is used when Proxy.MetadataMaxSize is unset. const defaultMetadataMaxSize = 100 << 20 +// defaultMetadataStreamMaxSize is used when Proxy.MetadataStreamMaxSize is +// unset. +const defaultMetadataStreamMaxSize = 1 << 30 + // ErrMetadataTooLarge is returned when upstream metadata exceeds the configured limit. var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit") @@ -215,14 +219,17 @@ type Proxy struct { Cooldown CooldownPolicy // ShortestCooldown is the shortest cooldown configured anywhere. It // bounds how long a rewrite filtered by cooldown is cached. - ShortestCooldown time.Duration - Denylist *denylist.Policy - CacheMetadata bool - MetadataTTL time.Duration - MetadataMaxSize int64 - GradleReadOnly bool - HostedEnabled bool - GradleMaxUploadSize int64 + ShortestCooldown time.Duration + Denylist *denylist.Policy + CacheMetadata bool + MetadataTTL time.Duration + MetadataMaxSize int64 + // MetadataStreamMaxSize caps a metadata document streamed from upstream + // into storage, which metadata_max_size doesn't cover. Default 1 GB. + MetadataStreamMaxSize int64 + GradleReadOnly bool + HostedEnabled bool + GradleMaxUploadSize int64 // NPMFullMetadata requests full npm packuments from upstream even when // cooldown is disabled, so served metadata carries publish times. NPMFullMetadata bool diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index b3bc420..ffe524d 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -198,9 +198,13 @@ func (p *Proxy) storeUpstreamMetadata(ctx context.Context, ecosystem, cacheKey, if lm := resp.Header.Get(headerLastModified); lm != "" { meta.lastModified, _ = http.ParseTime(lm) } + limit := p.MetadataStreamMaxSize + if limit <= 0 { + limit = defaultMetadataStreamMaxSize + } // Go decompresses the body, since no Accept-Encoding was set, so it is // stored uncompressed. - var body io.Reader = upstreamBodyReader{resp.Body} + var body io.Reader = &limitedBody{r: upstreamBodyReader{resp.Body}, left: limit} if validate != nil { staged, err := p.stageMetadata(ctx, ecosystem, cacheKey, body, validate) if err != nil { @@ -246,6 +250,21 @@ func (p *Proxy) stageMetadata(ctx context.Context, ecosystem, cacheKey string, b return &stagedReader{ReadCloser: staged, discard: discard}, nil } +// limitedBody fails with ErrMetadataTooLarge, as an upstream failure, once +// r has sent more than left bytes. +type limitedBody struct { + r io.Reader + left int64 +} + +func (l *limitedBody) Read(b []byte) (int, error) { + n, err := l.r.Read(b) + if l.left -= int64(n); l.left < 0 { + return 0, &upstreamBodyError{ErrMetadataTooLarge} + } + return n, err +} + // notStored returns a failure to store a document as errMetadataNotStored, // unless reading it from upstream failed. func notStored(what string, err error) error { diff --git a/internal/handler/npm_stream_harden_test.go b/internal/handler/npm_stream_harden_test.go index ca7143f..0cebf0a 100644 --- a/internal/handler/npm_stream_harden_test.go +++ b/internal/handler/npm_stream_harden_test.go @@ -181,3 +181,58 @@ func TestNPMStreamedPackumentClientRevalidates(t *testing.T) { t.Errorf("served %d versions, want 401", len(doc.Versions)) } } + +// TestNPMStreamedPackumentOverStreamLimit checks that a document over +// MetadataStreamMaxSize is neither stored nor served: the stored copy is +// served instead, with and without cooldown, and with nothing stored the +// request fails. +func TestNPMStreamedPackumentOverStreamLimit(t *testing.T) { + for _, withCooldown := range []bool{true, false} { + t.Run(map[bool]string{true: "cooldown", false: "no cooldown"}[withCooldown], func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + if !withCooldown { + proxy.Cooldown = nil + } + proxy.MetadataMaxSize = int64(len(body) / 4) + proxy.MetadataStreamMaxSize = int64(len(body) + 1000) + want := len(decodeStreamed(t, serve(handler, "/big", nil)).Versions) + before := bigMetadataFiles(store) + + u.replace(bigPackument(500), `"v2"`) + proxy.MetadataTTL = time.Nanosecond + proxy.SetMetadataRewriteCacheSize(64 << 20) + + if doc := decodeStreamed(t, serve(handler, "/big", nil)); len(doc.Versions) != want { + t.Errorf("served %d versions, want the stored %d", len(doc.Versions), want) + } + after := bigMetadataFiles(store) + if len(after) != len(before) { + t.Errorf("storage holds %d files, %d before", len(after), len(before)) + } + for path, b := range before { + if after[path] != b { + t.Errorf("%s was changed", path) + } + } + }) + } + t.Run("uncached", func(t *testing.T) { + body := bigPackument(400) + u := newStreamUpstream(t, body) + proxy, store, handler := streamProxy(t, u) + proxy.MetadataMaxSize = int64(len(body) / 4) + proxy.MetadataStreamMaxSize = int64(len(body) / 2) + + if rec := serve(handler, "/big", nil); rec.Code != http.StatusBadGateway { + t.Errorf("status %d, want 502", rec.Code) + } + if files := bigMetadataFiles(store); len(files) != 0 { + t.Errorf("left %d files in storage", len(files)) + } + if entry, _ := proxy.DB.GetMetadataCache("npm", "big"); entry != nil { + t.Error("left a metadata cache row") + } + }) +} diff --git a/internal/server/server.go b/internal/server/server.go index 2f0bd68..caebb80 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -268,10 +268,7 @@ func (s *Server) serve(listener net.Listener) error { if err != nil { return fmt.Errorf("configuring scanners: %w", err) } - proxy.CacheMetadata = s.cfg.CacheMetadata - proxy.MetadataTTL = s.cfg.ParseMetadataTTL() - proxy.MetadataMaxSize = s.cfg.ParseMetadataMaxSize() - proxy.SetMetadataRewriteCacheSize(s.cfg.ParseMetadataRewriteCacheSize()) + configureMetadata(proxy, s.cfg) proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly proxy.HostedEnabled = s.cfg.Hosted.Enabled proxy.HostedNPMScopes = s.cfg.Hosted.NPMScopes @@ -618,6 +615,15 @@ func (s *Server) newContainerHandler(proxy *handler.Proxy) *handler.ContainerHan return containerHandler } +// configureMetadata applies the metadata cache settings to proxy. +func configureMetadata(proxy *handler.Proxy, cfg *config.Config) { + proxy.CacheMetadata = cfg.CacheMetadata + proxy.MetadataTTL = cfg.ParseMetadataTTL() + proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() + proxy.MetadataStreamMaxSize = cfg.ParseMetadataStreamMaxSize() + proxy.SetMetadataRewriteCacheSize(cfg.ParseMetadataRewriteCacheSize()) +} + func configureScanning(proxy *handler.Proxy, cfg config.ScanningConfig, baseURL string, logger *slog.Logger) (*scanner.Group, error) { scanGroup, err := scanner.NewGroup(cfg, logger) if err != nil { From 26cae44e41feb62f1bcc284fa5623de6edfb8f87 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 2 Nov 2026 11:33:11 +0100 Subject: [PATCH 046/198] Bump cooldown to v0.3.1 and set cooldown directly in stream tests v0.3.1 parses a Config once when it is first used concurrently, so the stream tests no longer need cooldown.NewPolicy. --- go.mod | 2 +- go.sum | 4 ++-- internal/handler/npm_stream_test.go | 8 +------- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index db6695b..0f44186 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 - github.com/git-pkgs/cooldown v0.3.0 + github.com/git-pkgs/cooldown v0.3.1 github.com/git-pkgs/enrichment v0.7.5 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.2.0 diff --git a/go.sum b/go.sum index d1c0563..52f4c62 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/git-pkgs/archives v0.10.0 h1:/qgk0MI6VK2sKOoVTd0B4zs5hwTo+St0X8cYYftz github.com/git-pkgs/archives v0.10.0/go.mod h1:GJ/h7NPTUYiPinOUxpYNsHFCUikYZcJXvugBvP1VGNc= github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx2xk= github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= -github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A= -github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= +github.com/git-pkgs/cooldown v0.3.1 h1:XDt54+ZuYoJclTLysaQ2GUZPhYYuJdjpaqeLSwjMdto= +github.com/git-pkgs/cooldown v0.3.1/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0= github.com/git-pkgs/enrichment v0.7.5 h1:jbfFzf7+HSS7el8AuMri4vHXq2izIIZDVIU0uYI5LFM= github.com/git-pkgs/enrichment v0.7.5/go.mod h1:TbxC/nGQN96YCbdoiDs8a9z3gk2zXmLVS1989/jvnsU= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= diff --git a/internal/handler/npm_stream_test.go b/internal/handler/npm_stream_test.go index a8a4ea2..5d2ab73 100644 --- a/internal/handler/npm_stream_test.go +++ b/internal/handler/npm_stream_test.go @@ -107,13 +107,7 @@ func streamProxy(t *testing.T, u *streamUpstream) (*Proxy, *mockStorage, http.Ha proxy.CacheMetadata = true proxy.MetadataTTL = time.Hour proxy.SetMetadataRewriteCacheSize(64 << 20) - // As the server does: a cooldown.Config parses itself on first use, - // which concurrent requests would race on. - policy, err := cooldown.NewPolicy(&cooldown.Config{Default: "3d"}, nil) - if err != nil { - t.Fatal(err) - } - proxy.Cooldown = policy + proxy.Cooldown = &cooldown.Config{Default: "3d"} return proxy, store, NewNPMHandler(proxy, "http://proxy.example", u.URL).Routes() } From f35d152be0cc34af1ade85399f936fdca164c243 Mon Sep 17 00:00:00 2001 From: Ondrej Kokes Date: Mon, 2 Nov 2026 11:37:28 +0100 Subject: [PATCH 047/198] expose commit sha in UI footer --- .dockerignore | 17 +++++++++++++++++ .github/workflows/publish.yml | 3 +++ Dockerfile | 6 +++++- 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..05fb224 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +# Version and commit come in as build args, so the build doesn't need .git. +.git +.github + +# Local build output and runtime data +/proxy +dist/ +build/ +cache/ +*.db +*.tar.gz + +# Local config and secrets +.env +.env.local +proxy.cfg +config.yaml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0fadcbf..cbb6e15 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -61,6 +61,9 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + COMMIT=${{ github.sha }} provenance: mode=max sbom: true diff --git a/Dockerfile b/Dockerfile index 1a934ed..49be462 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,11 @@ COPY . . # Build the binary for the target platform ARG TARGETARCH -RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /proxy ./cmd/proxy +ARG VERSION=dev +ARG COMMIT=unknown +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ + -ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${COMMIT}" \ + -o /proxy ./cmd/proxy FROM alpine:3.24.2 From dd987f2de78da42343fe7a4cd59eb51d5040b367 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 2 Nov 2026 16:56:33 +0100 Subject: [PATCH 048/198] Document that SQLite needs a local disk or block volume --- README.md | 8 +++++++- deploy/charts/proxy/values.yaml | 6 ++++++ docs/configuration.md | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a324c53..f249c85 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,12 @@ using SQLite and filesystem artifact storage under `/data`. See [`deploy/charts/proxy/values.yaml`](deploy/charts/proxy/values.yaml) for ingress, external database and object-storage configuration options. +The SQLite database needs a local disk or a block-backed volume, even with one +replica; `ReadWriteOnce` alone doesn't guarantee that, since the storage class +can still be NFS. If the cluster only offers network filesystem storage, use +Postgres. Artifact storage can stay on NFS, separately from the database. See +[SQLite](docs/configuration.md#sqlite). + To trust a private CA, for example when upstreams sit behind a TLS-intercepting corporate proxy or are internal registries signed by an in-house CA, mount the certificate with `extraVolumes` and `extraVolumeMounts` and add its directory @@ -1048,7 +1054,7 @@ Run with config file: ### PostgreSQL -SQLite is the default and works well for single-node deployments. For multi-node setups or if you prefer a managed database, switch to Postgres: +SQLite is the default and works well for single-node deployments, with the database file on a local disk or block-backed volume. For multi-node setups, when only network filesystem storage such as NFS is available, or if you prefer a managed database, switch to Postgres: ```yaml database: diff --git a/deploy/charts/proxy/values.yaml b/deploy/charts/proxy/values.yaml index 5936ef3..bbb253f 100644 --- a/deploy/charts/proxy/values.yaml +++ b/deploy/charts/proxy/values.yaml @@ -79,6 +79,12 @@ extraInitContainers: [] # image: busybox:1.37 # command: ["sh", "-c", "until nc -z postgres 5432; do sleep 2; done"] +# With SQLite, the database file needs a local disk or block-backed volume, +# even with one replica. ReadWriteOnce doesn't guarantee that: pick a +# storageClass that isn't NFS or another network filesystem. If only network +# filesystem storage is available, set config.data.database to Postgres. +# Artifact storage can stay on NFS on a separate volume (extraVolumes and +# extraVolumeMounts, with config.data.storage.url pointing at it). persistence: enabled: true # Keep this aligned with config.data.storage.url and config.data.database.path diff --git a/docs/configuration.md b/docs/configuration.md index 3e94531..2089c5f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -201,6 +201,8 @@ database: The database runs in WAL mode. Writes go through a single connection, since SQLite allows one writer at a time. Reads use a separate pool of read-only connections (one per CPU, between 2 and 8), so request lookups don't wait behind writes, stats queries or eviction scans. An in-memory database (`:memory:`) uses one connection for both. +The SQLite database needs a local disk or a block-backed volume, even with one replica. SQLite's WAL mode doesn't work on network filesystems such as NFS ([sqlite.org/wal.html](https://www.sqlite.org/wal.html)), which shows up as stalled requests and `database disk image is malformed`. A Kubernetes `ReadWriteOnce` volume doesn't guarantee suitable storage, since its storage class can still be NFS. If only network filesystem storage is available to the proxy, use `driver: postgres`. Artifact storage can stay on NFS, on a separate volume from the SQLite database. + ### PostgreSQL ```yaml From 7cb905638cbc4a92552051a4cdc61a77179eebce Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 3 Nov 2026 09:28:29 +0000 Subject: [PATCH 049/198] Keep proxyCachedWithEncoding doc on the function it describes --- internal/handler/handler.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 7fef62e..a3fc2b7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2154,11 +2154,6 @@ func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, p.proxyCachedWithEncoding(w, r, upstreamURL, ecosystem, cacheKey, "identity", nil, acceptHeaders...) } -// proxyCachedWithEncoding is ProxyCached with an explicit upstream Accept-Encoding. -// "identity" preserves signed index bytes (the default); "gzip" keeps both hops -// compressed for large, non-hash-pinned metadata whose clients decode gzip -// (Homebrew API and Conda repodata). The stored bytes and Content-Encoding are -// replayed verbatim either way. // metadataHooks are optional callbacks for proxyCachedWithEncoding: hit runs // after a fresh cached row is served, with that row and when it was read, and // fetched runs with a fetched document before it is written. @@ -2167,6 +2162,11 @@ type metadataHooks struct { fetched func(body []byte, contentType, contentEncoding string) } +// proxyCachedWithEncoding is ProxyCached with an explicit upstream Accept-Encoding. +// "identity" preserves signed index bytes (the default); "gzip" keeps both hops +// compressed for large, non-hash-pinned metadata whose clients decode gzip +// (Homebrew API and Conda repodata). The stored bytes and Content-Encoding are +// replayed verbatim either way. func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, acceptEncoding string, hooks *metadataHooks, acceptHeaders ...string) { if !p.CacheMetadata { // Stream directly without buffering when caching is off. From 683455aecb386a8a8cb54616cb7db42268e30dd3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 3 Nov 2026 14:39:36 +0100 Subject: [PATCH 050/198] Test available_at through the API and drop FillVersionPublishedAt The API handler takes the enrichment calls it uses as an interface, so the test can answer them without reaching a registry. Remove FillVersionPublishedAt until the shared cooldown filter calls it. --- internal/database/database_test.go | 36 ------------- internal/database/queries.go | 35 ------------- internal/server/api.go | 14 +++-- internal/server/api_cooldown_test.go | 76 ++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 74 deletions(-) create mode 100644 internal/server/api_cooldown_test.go diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 6b49c32..1374dd9 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1165,42 +1165,6 @@ func TestVersionPublishedAtPreserved(t *testing.T) { }) } -func TestFillVersionPublishedAt(t *testing.T) { - runWithBothDatabases(t, func(t *testing.T, db *DB) { - const pkg, held, known = "pkg:npm/leftpad", "pkg:npm/leftpad@2.0.0", "pkg:npm/leftpad@1.0.0" - stored := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) - filtered := time.Date(2021, 6, 7, 8, 9, 10, 0, time.UTC) - - if err := db.SetVersionPublishedAt(known, pkg, stored); err != nil { - t.Fatalf("SetVersionPublishedAt: %v", err) - } - if err := db.UpsertVersion(&Version{PURL: held, PackagePURL: pkg}); err != nil { - t.Fatalf("UpsertVersion: %v", err) - } - - for _, purl := range []string{known, held, "pkg:npm/leftpad@3.0.0"} { - if err := db.FillVersionPublishedAt(purl, pkg, filtered); err != nil { - t.Fatalf("FillVersionPublishedAt(%s): %v", purl, err) - } - } - - want := map[string]time.Time{ - known: stored, - held: filtered, - "pkg:npm/leftpad@3.0.0": filtered, - } - for purl, at := range want { - got, err := db.GetVersionByPURL(purl) - if err != nil || got == nil { - t.Fatalf("GetVersionByPURL(%s): %v", purl, err) - } - if !got.PublishedAt.Valid || !got.PublishedAt.Time.Equal(at) { - t.Errorf("%s PublishedAt = %v, want %v", purl, got.PublishedAt, at) - } - } - }) -} - func TestGetFetchedVersionPURLs(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { const pkg = "pkg:npm/leftpad" diff --git a/internal/database/queries.go b/internal/database/queries.go index 825841e..5d8f5f3 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -224,41 +224,6 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt return nil } -// FillVersionPublishedAt records a version's publish time like -// SetVersionPublishedAt, but keeps a time already stored. The cooldown -// filters call it for each version they hold back, on every uncached -// metadata request, so it writes without reading first. -func (db *DB) FillVersionPublishedAt(versionPURL, packagePURL string, publishedAt time.Time) error { - now := time.Now() - var query string - - if db.dialect == DialectPostgres { - query = ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5) - ON CONFLICT(purl) DO UPDATE SET - published_at = EXCLUDED.published_at, - updated_at = EXCLUDED.updated_at - WHERE versions.published_at IS NULL - ` - } else { - query = ` - INSERT INTO versions (purl, package_purl, published_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(purl) DO UPDATE SET - published_at = excluded.published_at, - updated_at = excluded.updated_at - WHERE versions.published_at IS NULL - ` - } - - _, err := db.Exec(query, versionPURL, packagePURL, publishedAt, now, now) - if err != nil { - return fmt.Errorf("filling version publish time: %w", err) - } - return nil -} - // GetFetchedVersionPURLs returns the versions of a package that have an // artifact row, cached now or evicted since. Versions without one were only // seen in metadata. diff --git a/internal/server/api.go b/internal/server/api.go index b08055d..07080aa 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -1,6 +1,7 @@ package server import ( + "context" "encoding/json" "errors" "net/http" @@ -22,13 +23,22 @@ const ( // APIHandler provides REST endpoints for package enrichment data. type APIHandler struct { - enrichment *enrichment.Service + enrichment enricher ecosystems *shared.EcosystemsClient db DBSearcher // cooldown is nil with cooldown off. cooldown *cooldownView } +// enricher is the part of enrichment.Service the API uses. +type enricher interface { + EnrichPackage(ctx context.Context, ecosystem, name string) (*enrichment.PackageInfo, error) + EnrichFull(ctx context.Context, ecosystem, name, version string) (*enrichment.EnrichmentResult, error) + CheckVulnerabilities(ctx context.Context, ecosystem, name, version string) ([]enrichment.VulnInfo, error) + GetLatestVersion(ctx context.Context, ecosystem, name string) (string, error) + BulkEnrichPackages(ctx context.Context, packages []struct{ Ecosystem, Name string }) map[string]*enrichment.PackageInfo +} + // DBSearcher defines the interface for database search operations. type DBSearcher interface { SearchPackages(query string, ecosystem string, limit int, offset int) ([]database.SearchResult, error) @@ -50,7 +60,6 @@ func NewAPIHandler(svc *enrichment.Service, db DBSearcher) *APIHandler { return h } -// newAPIHandler returns the server's API handler, with its cooldown policy. func (s *Server) newAPIHandler(svc *enrichment.Service) *APIHandler { h := NewAPIHandler(svc, s.db) h.cooldown = s.cooldown @@ -87,7 +96,6 @@ type VersionResponse struct { IsOutdated bool `json:"is_outdated"` } -// apiTimeFormat is the format of the API's timestamps. const apiTimeFormat = "2006-01-02T15:04:05Z" // VulnResponse contains vulnerability information. diff --git a/internal/server/api_cooldown_test.go b/internal/server/api_cooldown_test.go new file mode 100644 index 0000000..4e3e3c7 --- /dev/null +++ b/internal/server/api_cooldown_test.go @@ -0,0 +1,76 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/enrichment" + "github.com/go-chi/chi/v5" +) + +func TestAPIVersionAvailableAt(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + recent, old := now.Add(-time.Hour), now.Add(-96*time.Hour) + h := &APIHandler{ + enrichment: fakeEnricher{published: map[string]time.Time{"1.0.0": old, "2.0.0": recent}}, + cooldown: testCooldownView(t, config.CooldownConfig{Default: "3d"}), + } + r := chi.NewRouter() + r.Get("/api/package/{ecosystem}/*", h.HandlePackagePath) + + tests := []struct{ version, availableAt string }{ + {"2.0.0", recent.Add(72 * time.Hour).Format(apiTimeFormat)}, + {"1.0.0", ""}, + } + for _, tt := range tests { + t.Run(tt.version, func(t *testing.T) { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/api/package/npm/leftpad/"+tt.version, nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", w.Code, w.Body) + } + var resp EnrichmentResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Package == nil || resp.Package.Cooldown != "3d" { + t.Errorf("package = %+v, want cooldown 3d", resp.Package) + } + if resp.Version == nil || resp.Version.AvailableAt != tt.availableAt { + t.Errorf("version = %+v, want available_at %q", resp.Version, tt.availableAt) + } + }) + } +} + +// fakeEnricher answers version lookups with the given publish times and +// knows no packages by name alone. +type fakeEnricher struct{ published map[string]time.Time } + +func (fakeEnricher) EnrichPackage(context.Context, string, string) (*enrichment.PackageInfo, error) { + return nil, nil +} + +func (f fakeEnricher) EnrichFull(_ context.Context, ecosystem, name, version string) (*enrichment.EnrichmentResult, error) { + return &enrichment.EnrichmentResult{ + Package: &enrichment.PackageInfo{Ecosystem: ecosystem, Name: name}, + Version: &enrichment.VersionInfo{Number: version, PublishedAt: f.published[version]}, + }, nil +} + +func (fakeEnricher) CheckVulnerabilities(context.Context, string, string, string) ([]enrichment.VulnInfo, error) { + return nil, nil +} + +func (fakeEnricher) GetLatestVersion(context.Context, string, string) (string, error) { + return "", nil +} + +func (fakeEnricher) BulkEnrichPackages(context.Context, []struct{ Ecosystem, Name string }) map[string]*enrichment.PackageInfo { + return nil +} From cb330a5cbf87df0c37cd4da480926235b9bf1e73 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 3 Nov 2026 20:10:24 +0100 Subject: [PATCH 051/198] Answer 502 when a Composer download's metadata lookup fails upstream The lookup took any non-200 metadata response for a version that isn't listed, so a Packagist outage or rate limit answered the archive request with 404. Composer treats that as the version being gone. Only 404 and 410 now mean the document is missing. A 429 is passed on as 503 with Retry-After while the upstream is held, and any other status answers 502. --- internal/handler/composer.go | 16 +++-- .../handler/composer_download_lookup_test.go | 67 +++++++++++++++---- 2 files changed, 67 insertions(+), 16 deletions(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index ccb42bb..aa79080 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -905,9 +905,9 @@ type composerDist struct { } // findDownloadURLFromMetadata fetches a metadata document and returns the dist -// of the given version, with an empty URL if the version is not present. An -// error is returned only on transport failure; a missing document (non-200) -// or a missing version both yield an empty URL so the caller can fall back. +// of the given version, with an empty URL if the version is not present. A +// missing document (404 or 410) or a missing version both yield an empty URL +// so the caller can fall back; any other failure is an error. func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaURL, packageName, version string) (composerDist, error) { h.proxy.Logger.Debug("fetching upstream metadata for download lookup", "url", metaURL, "package", packageName, "version", version) @@ -926,8 +926,16 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU h.proxy.Logger.Debug("upstream metadata response", "url", metaURL, "status", resp.StatusCode) - if resp.StatusCode != http.StatusOK { + switch { + case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: return composerDist{}, nil + case resp.StatusCode == http.StatusTooManyRequests && h.proxy.RateLimits != nil: + if err := h.proxy.RateLimits.Held(metaURL); err != nil { + return composerDist{}, err + } + } + if resp.StatusCode != http.StatusOK { + return composerDist{}, fmt.Errorf("upstream metadata returned %d", resp.StatusCode) } body, err := h.proxy.ReadMetadata(resp.Body) diff --git a/internal/handler/composer_download_lookup_test.go b/internal/handler/composer_download_lookup_test.go index 5635000..53a0cfe 100644 --- a/internal/handler/composer_download_lookup_test.go +++ b/internal/handler/composer_download_lookup_test.go @@ -11,23 +11,25 @@ import ( "time" "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" ) // composerLookupUpstream serves v/p's metadata listing versions and the // archives they point at, counts metadata and archive requests, and fails -// every request while down is set. A version listed in published carries that -// publish time. +// every request while down is set, with downStatus or 503. A version listed in +// published carries that publish time. type composerLookupUpstream struct { - h http.Handler - handler *ComposerHandler - proxy *Proxy - versions atomic.Value // []string - published map[string]time.Time - down atomic.Bool - metadata atomic.Int32 - archives atomic.Int32 - routeURL string + h http.Handler + handler *ComposerHandler + proxy *Proxy + versions atomic.Value // []string + published map[string]time.Time + down atomic.Bool + downStatus atomic.Int32 + metadata atomic.Int32 + archives atomic.Int32 + routeURL string } func setupComposerLookup(t *testing.T, cacheMetadata bool, versions ...string) *composerLookupUpstream { @@ -37,7 +39,12 @@ func setupComposerLookup(t *testing.T, cacheMetadata bool, versions ...string) * var srv *httptest.Server srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if u.down.Load() { - http.Error(w, "down", http.StatusServiceUnavailable) + status := int(u.downStatus.Load()) + if status == 0 { + status = http.StatusServiceUnavailable + } + w.Header().Set("Retry-After", "30") + http.Error(w, "down", status) return } switch { @@ -146,6 +153,42 @@ func TestComposerDownloadWithoutMetadataCache(t *testing.T) { } } +// TestComposerDownloadLookupUpstreamError checks that a download whose +// metadata lookup fails upstream answers 502, or 503 while upstream holds +// requests, and not 404: Composer treats 404 as the version being gone. +func TestComposerDownloadLookupUpstreamError(t *testing.T) { + for _, status := range []int{http.StatusInternalServerError, http.StatusServiceUnavailable} { + t.Run(http.StatusText(status), func(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + u.downStatus.Store(int32(status)) + u.down.Store(true) + if w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip"); w.Code != http.StatusBadGateway { + t.Errorf("status = %d, want 502", w.Code) + } + }) + } + + t.Run("rate limited", func(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + gate := registryclient.NewRateLimitGate(http.DefaultTransport) + u.proxy.HTTPClient = &http.Client{Transport: gate} + u.proxy.RateLimits = gate + u.downStatus.Store(http.StatusTooManyRequests) + u.down.Store(true) + w := u.get(t, "/files/v/p/1.0.0/1.0.0.zip") + if w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" { + t.Errorf("status = %d, Retry-After %q; want 503 with Retry-After", w.Code, w.Header().Get("Retry-After")) + } + }) + + t.Run("not listed", func(t *testing.T) { + u := setupComposerLookup(t, false, "1.0.0") + if w := u.get(t, "/files/v/p/2.0.0/2.0.0.zip"); w.Code != http.StatusNotFound { + t.Errorf("status = %d, want 404", w.Code) + } + }) +} + // storedPublishTime returns the publish time recorded for v/p at version. func (u *composerLookupUpstream) storedPublishTime(t *testing.T, version string) (time.Time, bool) { t.Helper() From 88c0ba8ab3be75703296c33d3b1917478f25c8ee Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 13:20:29 +0100 Subject: [PATCH 052/198] Use sbom's root detection for the project's own packages Bump sbom to v0.1.8 and replace lockFileRoots with sbom.Roots. Inferred roots count as the project only when read from a lock file, so an image's packages that nothing depends on are kept. --- go.mod | 2 +- go.sum | 4 +-- internal/inventory/inventory.go | 49 +++++++++++++------------ internal/inventory/inventory_test.go | 53 ++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index e5cb0c8..2fe061f 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/registries v0.13.0 - github.com/git-pkgs/sbom v0.1.7 + github.com/git-pkgs/sbom v0.1.8 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 52f4c62..3e33929 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= -github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA= -github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= +github.com/git-pkgs/sbom v0.1.8 h1:B+ohNEQfk1M+ch1/hMUKAitZp792g6fUrMaO4GTSqhI= +github.com/git-pkgs/sbom v0.1.8/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index d115fb7..a0b6b2b 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -88,18 +88,18 @@ const unknownVersion = "UNKNOWN" // type the proxy doesn't serve, without a version, or conda packages without // a channel are counted in skipped. // So are the project's own packages: Go modules its binaries were built from, -// and the root of a lock file. +// and the roots sbom finds for the dependency graph. func Parse(data []byte) (components []Component, skipped int, err error) { doc, err := sbom.Parse(data) if err != nil { return nil, 0, err } ownModules := mainModules(doc.Packages) - lockRoots := lockFileRoots(doc) + roots := projectRoots(doc) seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) - if !ok || lockRoots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { + if !ok || roots[pkg.ID] || (c.Ecosystem == "golang" && ownModules[c.Name]) { skipped++ continue } @@ -133,33 +133,36 @@ func componentOf(pkg sbom.Package) (Component, bool) { return c, true } -// lockFileRoots returns the IDs of packages read from a lock file that depend -// on others while nothing depends on them: the project's own entry, such as -// the root of package-lock.json, which carries the version from package.json. -func lockFileRoots(doc *sbom.SBOM) map[string]bool { - dependents := map[string]bool{} - dependencies := map[string]bool{} - for _, r := range doc.Relationships { - if r.Type != sbom.RelDependsOn { - continue - } - dependencies[r.SourceID] = true - dependents[r.TargetID] = true +// projectRoots returns the IDs of the roots sbom finds that are the +// project: all of them when the document declares them, otherwise only those +// read from a lock file. An image's packages that nothing depends on, such as +// busybox, are roots of the graph but not the project. +func projectRoots(doc *sbom.SBOM) map[string]bool { + ids, declared := doc.Roots() + roots := make(map[string]bool, len(ids)) + for _, id := range ids { + roots[id] = declared + } + if declared { + return roots } - roots := map[string]bool{} for _, pkg := range doc.Packages { - if pkg.ID == "" || !dependencies[pkg.ID] || dependents[pkg.ID] { - continue - } - for _, prop := range pkg.Properties { - if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { - roots[pkg.ID] = true - } + if _, ok := roots[pkg.ID]; ok && readFromLockFile(pkg) { + roots[pkg.ID] = true } } return roots } +func readFromLockFile(pkg sbom.Package) bool { + for _, prop := range pkg.Properties { + if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { + return true + } + } + return false +} + // mainModules returns the Go modules the SBOM's binaries were built from, // which are the project's own. Module paths are case-sensitive and package // URLs keep their case, so they are compared as they are. diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 3fbe113..36e67ef 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -246,6 +246,59 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { } } +func TestParseKeepsTopLevelDependenciesOfLockFiles(t *testing.T) { + // A poetry.lock doesn't list the project, so requests is a direct + // dependency, not the project's own entry. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6","components":[ + {"bom-ref":"requests","type":"library","name":"requests","version":"2.32.3","purl":"pkg:pypi/requests@2.32.3", + "properties":[{"name":"syft:package:metadataType","value":"python-poetry-lock-entry"},{"name":"syft:location:0:path","value":"/poetry.lock"}]}, + {"bom-ref":"urllib3","type":"library","name":"urllib3","version":"2.2.3","purl":"pkg:pypi/urllib3@2.2.3", + "properties":[{"name":"syft:package:metadataType","value":"python-poetry-lock-entry"},{"name":"syft:location:0:path","value":"/poetry.lock"}]} + ],"dependencies":[{"ref":"requests","dependsOn":["urllib3"]}]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 2 || skipped != 0 { + t.Errorf("components = %+v, skipped = %d; want requests and urllib3, 0 skipped", components, skipped) + } +} + +func TestParseKeepsOSPackagesNothingDependsOn(t *testing.T) { + // syft's CycloneDX of an image: metadata.component is the image, with + // no edges, and busybox is something nothing else depends on. + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6", + "metadata":{"component":{"bom-ref":"image","type":"container","name":"alpine:3.20"}}, + "components":[ + {"bom-ref":"busybox","type":"library","name":"busybox","version":"1.36.1-r29","purl":"pkg:apk/alpine/busybox@1.36.1-r29?distro=alpine-3.20.3", + "properties":[{"name":"syft:package:metadataType","value":"apk-db-entry"}]}, + {"bom-ref":"musl","type":"library","name":"musl","version":"1.2.5-r0","purl":"pkg:apk/alpine/musl@1.2.5-r0?distro=alpine-3.20.3", + "properties":[{"name":"syft:package:metadataType","value":"apk-db-entry"}]} + ],"dependencies":[{"ref":"busybox","dependsOn":["musl"]}]}` + components, skipped, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 2 || skipped != 0 { + t.Errorf("components = %+v, skipped = %d; want busybox and musl, 0 skipped", components, skipped) + } +} + +func TestParseSkipsTheDeclaredRoot(t *testing.T) { + doc := `{"bomFormat":"CycloneDX","specVersion":"1.6", + "metadata":{"component":{"bom-ref":"app","type":"application","name":"app","version":"1.0.0","purl":"pkg:npm/app@1.0.0"}}, + "components":[ + {"bom-ref":"lodash","type":"library","name":"lodash","version":"4.17.21","purl":"pkg:npm/lodash@4.17.21"} + ],"dependencies":[{"ref":"app","dependsOn":["lodash"]}]}` + components, _, err := Parse([]byte(doc)) + if err != nil { + t.Fatal(err) + } + if len(components) != 1 || components[0].Name != "lodash" { + t.Errorf("components = %+v, want only lodash", components) + } +} + func TestParseSkipsMixedCaseMainModule(t *testing.T) { // syft keeps the module path's case in mainModule, and so does the // package URL. From 3e2a84a88ada3196b6dedff0f3f600b27a59ecb1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 15:44:01 +0100 Subject: [PATCH 053/198] Mirror Alpine packages The PURL namespace names the upstream.apk repository, and the distro and arch qualifiers syft writes give the release and architecture. The package is looked up in the main and community indexes of that release and cached under the path the APK handler serves it from. --- docs/configuration.md | 2 + internal/handler/apk.go | 29 +++-- internal/mirror/apk.go | 151 +++++++++++++++++++++++++++ internal/mirror/mirror.go | 9 ++ internal/mirror/source.go | 5 +- internal/server/mirror_apk_test.go | 142 +++++++++++++++++++++++++ internal/server/mirror_fetch_test.go | 1 + internal/server/mirror_versions.go | 5 +- 8 files changed, 333 insertions(+), 11 deletions(-) create mode 100644 internal/mirror/apk.go create mode 100644 internal/server/mirror_apk_test.go diff --git a/docs/configuration.md b/docs/configuration.md index ae2ae79..c71a8ca 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1726,6 +1726,8 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. +An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. + | Flag | Default | Description | |------|---------|-------------| | `--sbom` | | Path to CycloneDX or SPDX SBOM file | diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 5de909a..045dbac 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -60,18 +60,31 @@ type APKHandler struct { // When repositories is empty, a single repository named "alpine" pointing at // the official Alpine mirror is used. func NewAPKHandler(proxy *Proxy, proxyURL string, repositories map[string]string) *APKHandler { - h := &APKHandler{ + return &APKHandler{ proxy: proxy, proxyURL: strings.TrimSuffix(proxyURL, "/"), - repositories: make(map[string]string, len(repositories)), + repositories: APKRepositories(repositories), } - for name, repositoryURL := range repositories { - h.repositories[name] = strings.TrimSuffix(repositoryURL, "/") +} + +// APKRepositories returns the upstream.apk repositories by name without +// trailing slashes, or the official Alpine mirror as "alpine" when none are +// configured. +func APKRepositories(configured map[string]string) map[string]string { + repositories := make(map[string]string, len(configured)) + for name, repositoryURL := range configured { + repositories[name] = strings.TrimSuffix(repositoryURL, "/") } - if len(h.repositories) == 0 { - h.repositories[defaultAPKRepositoryName] = defaultAPKUpstream + if len(repositories) == 0 { + repositories[defaultAPKRepositoryName] = defaultAPKUpstream } - return h + return repositories +} + +// APKCacheFilename returns the filename a package at path +// ({release}/{repo}/{arch}/{file}) of repository is cached under. +func APKCacheFilename(repository, path string) string { + return repository + "/" + path } // Routes returns the HTTP handler for APK requests. @@ -145,7 +158,7 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques } downloadURL := upstreamURL + "/" + path - cacheFilename := repository + "/" + path + cacheFilename := APKCacheFilename(repository, path) h.proxy.Logger.Info("apk package download", "repository", repository, "name", name, "version", version, "arch", arch) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go new file mode 100644 index 0000000..b6b4ea3 --- /dev/null +++ b/internal/mirror/apk.go @@ -0,0 +1,151 @@ +package mirror + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + "sync" + "time" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/git-pkgs/registries" +) + +// apkIndexTTL is how long a repository's APKINDEX is reused, so mirroring +// the packages of an image reads each index once. +const apkIndexTTL = 5 * time.Minute + +var ( + // apkDistroPattern takes the release from syft's distro qualifier, such + // as alpine-3.20.3 for v3.20. + apkDistroPattern = regexp.MustCompile(`^alpine-(\d+)\.(\d+)(?:[.-]|$)`) + apkArchPattern = regexp.MustCompile(`^[a-z0-9_]+$`) + // apkComponents are the repositories of a release a package is looked + // up in, in order. + apkComponents = []string{"main", "community"} +) + +// apkIndexes keeps one registries client per repository directory, which +// holds the parsed APKINDEX. +type apkIndexes struct { + mu sync.Mutex + regs map[string]apkIndex +} + +type apkIndex struct { + reg registries.Registry + fetched time.Time +} + +// mirrorAPK mirrors an Alpine package under the path the handler caches it +// at, {repository}/v{release}/{component}/{arch}/{file}. The PURL namespace +// names the upstream.apk repository; the distro and arch qualifiers give +// the release and architecture, and the package is looked up in the main +// and community indexes of that release. +func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + fail := func(err error) { m.recordFailure(tracker, pv, err) } + repository, name, ok := strings.Cut(pv.Name, "/") + if !ok { + repository, name = "alpine", pv.Name + } + repositoryURL, ok := handler.APKRepositories(m.APKRepositories)[repository] + if !ok { + fail(fmt.Errorf("unknown APK repository %q: it isn't in upstream.apk", repository)) + return + } + match := apkDistroPattern.FindStringSubmatch(pv.Qualifiers["distro"]) + if match == nil { + fail(fmt.Errorf("apk packages need a distro qualifier with the Alpine release, such as distro=alpine-3.20.3, not %q", pv.Qualifiers["distro"])) + return + } + release := "v" + match[1] + "." + match[2] + arch := pv.Qualifiers["arch"] + if !apkArchPattern.MatchString(arch) { + fail(fmt.Errorf("apk packages need an arch qualifier, such as arch=x86_64, not %q", arch)) + return + } + + for _, component := range apkComponents { + dir := release + "/" + component + "/" + arch + artifact, found, err := m.apkArtifact(ctx, repositoryURL+"/"+dir, name, pv.Version) + if err != nil { + fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", dir, err)) + return + } + if !found { + continue + } + cacheFilename := handler.APKCacheFilename(repository, dir+"/"+artifact.Filename) + result, err := m.proxy.GetOrFetchArtifactFromURL(ctx, pv.Ecosystem, name, pv.Version, cacheFilename, artifact.URL) + if err != nil { + fail(err) + return + } + m.recordResult(tracker, pv, result) + return + } + fail(fmt.Errorf("%s %s isn't in %s of %s for %s", name, pv.Version, strings.Join(apkComponents, " or "), release, arch)) +} + +// apkArtifact returns the file of version of name that the index in dirURL +// lists, and whether it lists one. +func (m *Mirror) apkArtifact(ctx context.Context, dirURL, name, version string) (registries.Artifact, bool, error) { + reg, err := m.apkRegistry(dirURL) + if err != nil { + return registries.Artifact{}, false, err + } + versions, err := reg.FetchVersions(ctx, name) + if isNotFound(err) { + return registries.Artifact{}, false, nil + } + if err != nil { + m.apk.forget(dirURL) + return registries.Artifact{}, false, err + } + for _, v := range versions { + if v.Number == version && len(v.Artifacts) > 0 { + return v.Artifacts[0], true, nil + } + } + return registries.Artifact{}, false, nil +} + +// apkRegistry returns the client for the repository directory at dirURL, +// reusing it, and the index it read, for apkIndexTTL. +func (m *Mirror) apkRegistry(dirURL string) (registries.Registry, error) { //nolint:ireturn // registries.New returns the interface + m.apk.mu.Lock() + defer m.apk.mu.Unlock() + if idx, ok := m.apk.regs[dirURL]; ok && time.Since(idx.fetched) < apkIndexTTL { + return idx.reg, nil + } + r, _ := m.Versions.(RegistryVersions) + reg, err := registries.New("apk", dirURL, r.client()) + if err != nil { + return nil, err + } + if m.apk.regs == nil { + m.apk.regs = map[string]apkIndex{} + } + m.apk.regs[dirURL] = apkIndex{reg: reg, fetched: time.Now()} + return reg, nil +} + +// forget drops the client for dirURL, so an index that failed to load is +// read again next time. +func (a *apkIndexes) forget(dirURL string) { + a.mu.Lock() + defer a.mu.Unlock() + delete(a.regs, dirURL) +} + +// isNotFound reports whether err means a package or index doesn't exist, +// such as a release without a community repository for an architecture. +func isNotFound(err error) bool { + if errors.Is(err, registries.ErrNotFound) { + return true + } + var httpErr *registries.HTTPError + return errors.As(err, &httpErr) && httpErr.IsNotFound() +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index ac20a4e..d5ec9a5 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -45,11 +45,16 @@ type Mirror struct { // uses https://pkg.julialang.org. JuliaUpstream string + // APKRepositories are the upstream.apk repositories by name. Nil uses + // the official Alpine mirror as "alpine". + APKRepositories map[string]string + // RefusePackages, when set, fails every item Images does not take with // this reason instead of fetching it, so nothing is cached. RefusePackages string cran cranIndex + apk apkIndexes } // New creates a new Mirror with the given dependencies. @@ -270,6 +275,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorCRAN(ctx, pv, tracker) return } + if pv.Ecosystem == "alpine" { + m.mirrorAPK(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index ecc3a0c..1d3954c 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -148,8 +148,11 @@ func mirrorTarget(p *purl.PURL) (ecosystem, name string, qualifiers map[string]s name = p.Namespace + "/" + p.Name } qualifiers = fileQualifiers(ecosystem, p.Qualifiers.Map()) - if ecosystem == "julia" { + switch ecosystem { + case "julia": qualifiers = map[string]string{"uuid": p.Qualifier("uuid")} + case "alpine": + qualifiers = map[string]string{"arch": p.Qualifier("arch"), "distro": p.Qualifier("distro")} } return ecosystem, name, qualifiers } diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go new file mode 100644 index 0000000..8172744 --- /dev/null +++ b/internal/server/mirror_apk_test.go @@ -0,0 +1,142 @@ +package server + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" +) + +const ( + apkBusyboxPath = "/v3.20/main/x86_64/busybox-1.36.1-r29.apk" + apkHtopPath = "/v3.20/community/x86_64/htop-3.3.0-r0.apk" + apkMainIndex = "/v3.20/main/x86_64/APKINDEX.tar.gz" + apkCommIndex = "/v3.20/community/x86_64/APKINDEX.tar.gz" +) + +// apkUpstream serves an Alpine mirror with busybox in v3.20/main and htop in +// v3.20/community, for x86_64. It counts requests by path. +type apkUpstream struct { + *httptest.Server + mu sync.Mutex + files map[string]string + hits map[string]int +} + +func apkIndexArchive(t *testing.T, index string) string { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + if err := tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(index))}); err != nil { + t.Fatal(err) + } + _, _ = io.WriteString(tw, index) + _ = tw.Close() + _ = zw.Close() + return buf.String() +} + +func newAPKUpstream(t *testing.T) *apkUpstream { + t.Helper() + u := &apkUpstream{ + files: map[string]string{ + apkMainIndex: apkIndexArchive(t, "P:busybox\nV:1.36.1-r29\nA:x86_64\nS:18\n\nP:musl\nV:1.2.5-r0\nA:x86_64\nS:15\n\n"), + apkCommIndex: apkIndexArchive(t, "P:htop\nV:3.3.0-r0\nA:x86_64\nS:15\n\n"), + apkBusyboxPath: "busybox 1.36.1-r29", + apkHtopPath: "htop 3.3.0-r0 x86", + "/v3.20/main/x86_64/musl-1.2.5-r0.apk": "musl 1.2.5-r0", + }, + hits: map[string]int{}, + } + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.mu.Lock() + body, ok := u.files[r.URL.Path] + u.hits[r.URL.Path]++ + u.mu.Unlock() + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(u.Close) + return u +} + +func (u *apkUpstream) hitsFor(path string) int { + u.mu.Lock() + defer u.mu.Unlock() + return u.hits[path] +} + +func apkMirrorConfig(upstream *apkUpstream) *config.Config { + cfg := config.Default() + cfg.Upstream.APK = map[string]string{"alpine": upstream.URL} + return cfg +} + +func getAPK(proxy *handler.Proxy, upstream *apkUpstream, path string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + h := handler.NewAPKHandler(proxy, "http://proxy.test", map[string]string{"alpine": upstream.URL}) + h.Routes().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/alpine"+path, nil)) + return rec +} + +func TestMirrorAPKPackagesAreServedFromCache(t *testing.T) { + upstream := newAPKUpstream(t) + + progress, proxy := mirrorWithConfig(t, apkMirrorConfig(upstream), + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3", + "pkg:apk/alpine/htop@3.3.0-r0?arch=x86_64&distro=alpine-3.20.3", + "pkg:apk/alpine/musl@1.2.5-r0?arch=x86_64&distro=alpine-3.20.3") + + if progress.Completed != 3 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } + for _, path := range []string{apkBusyboxPath, apkHtopPath, apkMainIndex, apkCommIndex} { + if got := upstream.hitsFor(path); got != 1 { + t.Errorf("%s fetched %d times, want 1", path, got) + } + } + + // apk gets the packages from the cache once the upstream is down. + upstream.Close() + for path, want := range map[string]string{ + apkBusyboxPath: "busybox 1.36.1-r29", + apkHtopPath: "htop 3.3.0-r0 x86", + } { + if rec := getAPK(proxy, upstream, path); rec.Code != http.StatusOK || rec.Body.String() != want { + t.Errorf("%s from cache: status = %d, body = %q, want 200 and %q", path, rec.Code, rec.Body, want) + } + } +} + +func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { + upstream := newAPKUpstream(t) + + tests := map[string]string{ + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64": "distro", + "pkg:apk/alpine/busybox@1.36.1-r29?distro=alpine-3.20.3": "arch", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=debian-12": "distro", + "pkg:apk/other/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3": "upstream.apk", + "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "9.9.9-r0", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=../../etc&distro=alpine-3.20.3": "arch", + } + for p, want := range tests { + t.Run(p, func(t *testing.T) { + progress, _ := mirrorWithConfig(t, apkMirrorConfig(upstream), p) + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, want) { + t.Errorf("failed = %d, errors = %v; want one error mentioning %q", progress.Failed, progress.Errors, want) + } + }) + } +} diff --git a/internal/server/mirror_fetch_test.go b/internal/server/mirror_fetch_test.go index dba27d3..3e2cebb 100644 --- a/internal/server/mirror_fetch_test.go +++ b/internal/server/mirror_fetch_test.go @@ -64,6 +64,7 @@ func mirrorWithVersions(t *testing.T, cfg *config.Config, versions mirror.Versio m.Charts = handler.NewHelmHandler(proxy, cfg.BaseURL, cfg.Upstream.Helm) m.Versions = versions m.JuliaUpstream = cfg.Upstream.Julia + m.APKRepositories = cfg.Upstream.APK if versions == nil { m.Versions = NewMirrorVersions(cfg.Upstream, proxy.HTTPClient) } diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 27b3d51..6d4599e 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -37,11 +37,12 @@ func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { } // UseMirrorUpstreams points m at the configured upstreams: it lists -// versions from them, through client, and fetches Julia packages from -// upstream.julia. +// versions from them, through client, fetches Julia packages from +// upstream.julia and Alpine packages from upstream.apk. func UseMirrorUpstreams(m *mirror.Mirror, upstream config.UpstreamConfig, client *http.Client) { m.Versions = NewMirrorVersions(upstream, client) m.JuliaUpstream = upstream.Julia + m.APKRepositories = upstream.APK } // mirrorVersionsURL returns the registry base URL to list a package's From adcda6ca8baa5140eee575fc4ace5655dfd9f98b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 15:54:10 +0100 Subject: [PATCH 054/198] Read APKINDEX.tar.gz also where Packages.adb is served beside it --- internal/mirror/apk.go | 4 +++- internal/server/mirror_apk_test.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index b6b4ea3..7b80261 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -121,7 +121,9 @@ func (m *Mirror) apkRegistry(dirURL string) (registries.Registry, error) { //nol return idx.reg, nil } r, _ := m.Versions.(RegistryVersions) - reg, err := registries.New("apk", dirURL, r.client()) + // Naming the v2 index reads it also where Packages.adb is served beside + // it, which the client would try first and refuse. + reg, err := registries.New("apk", dirURL+"/APKINDEX.tar.gz", r.client()) if err != nil { return nil, err } diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go index 8172744..c82602d 100644 --- a/internal/server/mirror_apk_test.go +++ b/internal/server/mirror_apk_test.go @@ -120,6 +120,20 @@ func TestMirrorAPKPackagesAreServedFromCache(t *testing.T) { } } +func TestMirrorAPKReadsV2IndexBesideV3(t *testing.T) { + // A repository moving to apk v3 serves Packages.adb next to + // APKINDEX.tar.gz; the mirror reads the v2 index. + upstream := newAPKUpstream(t) + upstream.files["/v3.20/main/x86_64/Packages.adb"] = "ADB.v3-index" + + progress, _ := mirrorWithConfig(t, apkMirrorConfig(upstream), + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3") + + if progress.Completed != 1 || progress.Failed != 0 { + t.Fatalf("completed = %d, failed = %d, errors = %v", progress.Completed, progress.Failed, progress.Errors) + } +} + func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { upstream := newAPKUpstream(t) From 6d3410df41f583816513b6bbdcc668b343810786 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:05:16 +0100 Subject: [PATCH 055/198] Use sbom's ProjectRoots for the project's own packages Drop projectRoots and readFromLockFile; sbom now decides which inferred roots are the project. sbom is on the merge commit of git-pkgs/sbom#27 until it is tagged. --- go.mod | 2 +- go.sum | 4 +-- internal/inventory/inventory.go | 38 +++------------------------- internal/inventory/inventory_test.go | 4 +-- 4 files changed, 9 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 2fe061f..b6182d2 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/registries v0.13.0 - github.com/git-pkgs/sbom v0.1.8 + github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 3e33929..6c0e3d1 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= -github.com/git-pkgs/sbom v0.1.8 h1:B+ohNEQfk1M+ch1/hMUKAitZp792g6fUrMaO4GTSqhI= -github.com/git-pkgs/sbom v0.1.8/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= +github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 h1:mghNOVmaxRvz8I0TsY+augx8GcVuvqbtD5fjMrBXGa8= +github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index a0b6b2b..c48f387 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -76,9 +76,6 @@ func FromPURL(s string) (Component, error) { // naming the module the binary was built from. const syftMainModule = "syft:metadata:mainModule" -// syftMetadataType is the property naming where syft read a package from. -const syftMetadataType = "syft:package:metadataType" - // unknownVersion is what syft gives a package without a resolved version, // such as the project's own module in go.mod or the root of a lock file. const unknownVersion = "UNKNOWN" @@ -95,7 +92,10 @@ func Parse(data []byte) (components []Component, skipped int, err error) { return nil, 0, err } ownModules := mainModules(doc.Packages) - roots := projectRoots(doc) + roots := map[string]bool{} + for _, id := range doc.ProjectRoots() { + roots[id] = true + } seen := make(map[[2]string]bool, len(doc.Packages)) for _, pkg := range doc.Packages { c, ok := componentOf(pkg) @@ -133,36 +133,6 @@ func componentOf(pkg sbom.Package) (Component, bool) { return c, true } -// projectRoots returns the IDs of the roots sbom finds that are the -// project: all of them when the document declares them, otherwise only those -// read from a lock file. An image's packages that nothing depends on, such as -// busybox, are roots of the graph but not the project. -func projectRoots(doc *sbom.SBOM) map[string]bool { - ids, declared := doc.Roots() - roots := make(map[string]bool, len(ids)) - for _, id := range ids { - roots[id] = declared - } - if declared { - return roots - } - for _, pkg := range doc.Packages { - if _, ok := roots[pkg.ID]; ok && readFromLockFile(pkg) { - roots[pkg.ID] = true - } - } - return roots -} - -func readFromLockFile(pkg sbom.Package) bool { - for _, prop := range pkg.Properties { - if prop.Name == syftMetadataType && strings.Contains(prop.Value, "lock") { - return true - } - } - return false -} - // mainModules returns the Go modules the SBOM's binaries were built from, // which are the project's own. Module paths are case-sensitive and package // URLs keep their case, so they are compared as they are. diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 36e67ef..89b070b 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -225,10 +225,10 @@ func TestParseSkipsTheProjectsOwnPackages(t *testing.T) { {"bom-ref":"cobra","type":"library","name":"github.com/spf13/cobra","version":"v1.8.0","purl":"pkg:golang/github.com/spf13/cobra@v1.8.0", "properties":[{"name":"syft:metadata:mainModule","value":"github.com/acme/app"}]}, {"bom-ref":"root","type":"library","name":"app-frontend","version":"2.1.0","purl":"pkg:npm/app-frontend@2.1.0", - "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]}, + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]}, {"bom-ref":"nover","type":"library","name":"app-tools","purl":"pkg:npm/app-tools"}, {"bom-ref":"jsdom","type":"library","name":"jsdom","version":"30.1.3","purl":"pkg:npm/jsdom@30.1.3", - "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"}]} + "properties":[{"name":"syft:package:metadataType","value":"javascript-npm-package-lock-entry"},{"name":"syft:location:0:path","value":"/package-lock.json"}]} ],"dependencies":[{"ref":"root","dependsOn":["jsdom"]}]}` components, skipped, err := Parse([]byte(doc)) if err != nil { From 4e2539633180d43765a088c9775f37c9e5aa33b0 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:09:00 +0100 Subject: [PATCH 056/198] Refuse distro qualifiers that aren't an Alpine release --- docs/configuration.md | 2 +- internal/mirror/apk.go | 5 +++-- internal/server/mirror_apk_test.go | 14 ++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c71a8ca..35e2e48 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1726,7 +1726,7 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. -An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. +An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either, and edge packages, are refused. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. | Flag | Default | Description | |------|---------|-------------| diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index 7b80261..3e08572 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -19,8 +19,9 @@ const apkIndexTTL = 5 * time.Minute var ( // apkDistroPattern takes the release from syft's distro qualifier, such - // as alpine-3.20.3 for v3.20. - apkDistroPattern = regexp.MustCompile(`^alpine-(\d+)\.(\d+)(?:[.-]|$)`) + // as alpine-3.20.3 for v3.20. Edge, as alpine-3.21.0_alpha20240807, + // isn't a release and doesn't match. + apkDistroPattern = regexp.MustCompile(`^alpine-(\d+)\.(\d+)(?:\.\d+)?$`) apkArchPattern = regexp.MustCompile(`^[a-z0-9_]+$`) // apkComponents are the repositories of a release a package is looked // up in, in order. diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go index c82602d..d5a8c0d 100644 --- a/internal/server/mirror_apk_test.go +++ b/internal/server/mirror_apk_test.go @@ -138,12 +138,14 @@ func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { upstream := newAPKUpstream(t) tests := map[string]string{ - "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64": "distro", - "pkg:apk/alpine/busybox@1.36.1-r29?distro=alpine-3.20.3": "arch", - "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=debian-12": "distro", - "pkg:apk/other/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3": "upstream.apk", - "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "9.9.9-r0", - "pkg:apk/alpine/busybox@1.36.1-r29?arch=../../etc&distro=alpine-3.20.3": "arch", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64": "distro", + "pkg:apk/alpine/busybox@1.36.1-r29?distro=alpine-3.20.3": "arch", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20-evil": "distro", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.21.0_alpha20240807": "distro", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=debian-12": "distro", + "pkg:apk/other/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3": "upstream.apk", + "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "9.9.9-r0", + "pkg:apk/alpine/busybox@1.36.1-r29?arch=../../etc&distro=alpine-3.20.3": "arch", } for p, want := range tests { t.Run(p, func(t *testing.T) { From 7095ab67ef105c5466f6431a162ba2839eec8a70 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:16:35 +0100 Subject: [PATCH 057/198] Keep the Projects link on an empty projects page ProjectsListData.Projects hid Layout.Projects, so the header left the link out when no project had been uploaded. --- internal/server/projects_ui.go | 8 +++++--- internal/server/projects_ui_test.go | 7 +++++++ internal/server/templates/pages/projects_list.html | 6 +++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/internal/server/projects_ui.go b/internal/server/projects_ui.go index 5bafb39..c7c2c05 100644 --- a/internal/server/projects_ui.go +++ b/internal/server/projects_ui.go @@ -13,7 +13,9 @@ import ( // ProjectsListData contains data for rendering the projects list page. type ProjectsListData struct { Layout - Projects []ProjectRow + // Rows isn't called Projects, which would hide Layout.Projects from + // the header. + Rows []ProjectRow } // ProjectRow is a project in the UI. @@ -79,9 +81,9 @@ func (s *Server) handleProjectsList(w http.ResponseWriter, r *http.Request) { http.Error(w, "failed to list projects", http.StatusInternalServerError) return } - data := ProjectsListData{Layout: s.layoutFor(r), Projects: make([]ProjectRow, len(projects))} + data := ProjectsListData{Layout: s.layoutFor(r), Rows: make([]ProjectRow, len(projects))} for i, p := range projects { - data.Projects[i] = projectRow(p) + data.Rows[i] = projectRow(p) } if err := s.templates.Render(w, "projects_list", data); err != nil { s.logger.Error("failed to render projects list", "error", err) diff --git a/internal/server/projects_ui_test.go b/internal/server/projects_ui_test.go index 5be2224..bd868c9 100644 --- a/internal/server/projects_ui_test.go +++ b/internal/server/projects_ui_test.go @@ -116,3 +116,10 @@ func TestProjectPageFailsWhenComponentsCannotBeRead(t *testing.T) { t.Errorf("status = %d, want 500 rather than an empty project", rec.Code) } } + +func TestProjectsPageLinksProjectsWhenEmpty(t *testing.T) { + ts := newProjectsTestServer(t) + + page := ts.page(t, "/ui/projects") + assertContains(t, page, `href="/ui/projects"`, "0 projects") +} diff --git a/internal/server/templates/pages/projects_list.html b/internal/server/templates/pages/projects_list.html index f5ce62d..d7256da 100644 --- a/internal/server/templates/pages/projects_list.html +++ b/internal/server/templates/pages/projects_list.html @@ -3,13 +3,13 @@ {{define "content"}}

Projects

-

{{len .Projects}} projects, as of their latest SBOM upload

+

{{len .Rows}} projects, as of their latest SBOM upload

-{{if .Projects}} +{{if .Rows}}
- {{range .Projects}} + {{range .Rows}}
{{.Name}} From e4e14696d0f7fdd870c1b8482eac298d2325ff73 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:20:30 +0100 Subject: [PATCH 058/198] Say which edge packages the APK mirror can't recognise --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 35e2e48..2f4e31e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1726,7 +1726,7 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. -An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either, and edge packages, are refused. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. +An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. | Flag | Default | Description | |------|---------|-------------| From c3f0c23dc3624eac050f88f7ee6659df30ea96e5 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Wed, 4 Nov 2026 16:31:50 +0100 Subject: [PATCH 059/198] Name the repository segment of the APK path in the mirror docs --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2f4e31e..73a0b92 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1726,7 +1726,7 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either. -An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{name}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. +An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one. | Flag | Default | Description | |------|---------|-------------| From 60872fe990f32788cb7c79c7eac5fc27b0c8d2d2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 09:34:02 +0100 Subject: [PATCH 060/198] Use sbom v0.1.9 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b6182d2..76a1fad 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/git-pkgs/magic v0.6.0 github.com/git-pkgs/purl v0.2.1 github.com/git-pkgs/registries v0.13.0 - github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 + github.com/git-pkgs/sbom v0.1.9 github.com/git-pkgs/spdx v0.3.3 github.com/git-pkgs/vers v0.7.2 github.com/go-chi/chi/v5 v5.5.0 diff --git a/go.sum b/go.sum index 6c0e3d1..c6b5f02 100644 --- a/go.sum +++ b/go.sum @@ -137,8 +137,8 @@ github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0= github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw= github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU= github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M= -github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9 h1:mghNOVmaxRvz8I0TsY+augx8GcVuvqbtD5fjMrBXGa8= -github.com/git-pkgs/sbom v0.1.9-0.20261104140234-f3632df60df9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= +github.com/git-pkgs/sbom v0.1.9 h1:sfF5cJAwmgrm2N4HTXQWjiBe8tdu8xwhjwfAOyDNMfc= +github.com/git-pkgs/sbom v0.1.9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M= github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= From c20f64ebaa1151e4b65f31452c6d660f054dad4a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 10:48:44 +0100 Subject: [PATCH 061/198] Keep the apk namespace for the mirror's repository lookup The inventory drops the namespace from apk names so they match the handler's, but the apk mirror picks the upstream.apk repository from it. --- internal/mirror/apk.go | 6 +++--- internal/mirror/source.go | 11 +++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index 3e08572..9746f20 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -47,9 +47,9 @@ type apkIndex struct { // and community indexes of that release. func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *progressTracker) { fail := func(err error) { m.recordFailure(tracker, pv, err) } - repository, name, ok := strings.Cut(pv.Name, "/") - if !ok { - repository, name = "alpine", pv.Name + repository, name := pv.Namespace, pv.Name + if repository == "" { + repository = "alpine" } repositoryURL, ok := handler.APKRepositories(m.APKRepositories)[repository] if !ok { diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 1e326ee..2c41705 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -16,6 +16,9 @@ type PackageVersion struct { Ecosystem string Name string Version string + // Namespace is the PURL namespace of apk packages, which Name leaves + // out. It picks the upstream.apk repository. + Namespace string // Qualifiers are the PURL qualifiers. Images use repository_url, tag and // arch; PyPI uses file_name, and conda subdir, build and type, to select // which files of the version are mirrored. Julia uses uuid to identify @@ -61,11 +64,15 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro } ecosystem, name, qualifiers := mirrorTarget(p) + var namespace string + if ecosystem == "alpine" { + namespace = p.Namespace + } // Images are never expanded to all tags, and conda packages without // a channel aren't listed; both fail when they are mirrored. if p.Version != "" || isImageEcosystem(ecosystem) || (ecosystem == "conda" && !strings.Contains(name, "/")) { - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Qualifiers: qualifiers} + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Namespace: namespace, Qualifiers: qualifiers} if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() } @@ -84,7 +91,7 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err) } for _, v := range listed { - pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Qualifiers: qualifiers} + pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: v, Namespace: namespace, Qualifiers: qualifiers} if takesPlatforms(ecosystem) { pv.Platforms = s.Platforms } From 4d707933512e4e4c70adde33e5927e29381efea7 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 11:56:42 +0100 Subject: [PATCH 062/198] Name the apk repository in mirror errors --- internal/mirror/apk.go | 4 ++-- internal/server/mirror_apk_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index 9746f20..c4f6868 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -72,7 +72,7 @@ func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *prog dir := release + "/" + component + "/" + arch artifact, found, err := m.apkArtifact(ctx, repositoryURL+"/"+dir, name, pv.Version) if err != nil { - fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", dir, err)) + fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", repositoryURL+"/"+dir, err)) return } if !found { @@ -87,7 +87,7 @@ func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *prog m.recordResult(tracker, pv, result) return } - fail(fmt.Errorf("%s %s isn't in %s of %s for %s", name, pv.Version, strings.Join(apkComponents, " or "), release, arch)) + fail(fmt.Errorf("%s %s isn't in %s of %s %s for %s", name, pv.Version, strings.Join(apkComponents, " or "), repository, release, arch)) } // apkArtifact returns the file of version of name that the index in dirURL diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go index d5a8c0d..0c46ba1 100644 --- a/internal/server/mirror_apk_test.go +++ b/internal/server/mirror_apk_test.go @@ -144,7 +144,7 @@ func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.21.0_alpha20240807": "distro", "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=debian-12": "distro", "pkg:apk/other/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3": "upstream.apk", - "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "9.9.9-r0", + "pkg:apk/alpine/busybox@9.9.9-r0?arch=x86_64&distro=alpine-3.20.3": "isn't in main or community of alpine v3.20", "pkg:apk/alpine/busybox@1.36.1-r29?arch=../../etc&distro=alpine-3.20.3": "arch", } for p, want := range tests { From eb34a1ac6f0049d6b0f126234afdbe900c48a8c3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 12:33:46 +0100 Subject: [PATCH 063/198] Name the apk repository, not its URL, in index errors --- internal/mirror/apk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mirror/apk.go b/internal/mirror/apk.go index c4f6868..30a9771 100644 --- a/internal/mirror/apk.go +++ b/internal/mirror/apk.go @@ -72,7 +72,7 @@ func (m *Mirror) mirrorAPK(ctx context.Context, pv PackageVersion, tracker *prog dir := release + "/" + component + "/" + arch artifact, found, err := m.apkArtifact(ctx, repositoryURL+"/"+dir, name, pv.Version) if err != nil { - fail(fmt.Errorf("reading %s/APKINDEX.tar.gz: %w", repositoryURL+"/"+dir, err)) + fail(fmt.Errorf("reading %s %s/APKINDEX.tar.gz: %w", repository, dir, err)) return } if !found { From 903fd94df90930fc69c0b768e3143d28e4f02310 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 12:34:32 +0100 Subject: [PATCH 064/198] Test that apk index errors leave out the URL --- internal/server/mirror_apk_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/server/mirror_apk_test.go b/internal/server/mirror_apk_test.go index 0c46ba1..ff74df9 100644 --- a/internal/server/mirror_apk_test.go +++ b/internal/server/mirror_apk_test.go @@ -156,3 +156,19 @@ func TestMirrorAPKRefusesIncompletePURLs(t *testing.T) { }) } } + +func TestMirrorAPKIndexErrorsLeaveOutTheURL(t *testing.T) { + upstream := newAPKUpstream(t) + upstream.files[apkMainIndex] = "not an index" + + progress, _ := mirrorWithConfig(t, apkMirrorConfig(upstream), + "pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3") + + if progress.Failed != 1 || len(progress.Errors) != 1 { + t.Fatalf("failed = %d, errors = %v; want one error", progress.Failed, progress.Errors) + } + msg := progress.Errors[0].Error + if !strings.Contains(msg, "alpine v3.20/main/x86_64") || strings.Contains(msg, upstream.URL) { + t.Errorf("error = %q; want the repository and directory without %s", msg, upstream.URL) + } +} From 17c2d525009fa71d00d9ea29bb7dc24217951890 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 12:34:36 +0100 Subject: [PATCH 065/198] Note that refreshing Debian, Alpine and RPM needs cache_metadata --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index bb18db0..e78fbc0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1842,7 +1842,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. - PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python, ABI and platform tags, and the sdist, or conda packages of the same subdir, build variant (such as `py312`) and file type. -- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. +- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. - Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. From 010ed4761aa17c73f07addb6c1fbc02ade5108cb Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 13:51:07 +0100 Subject: [PATCH 066/198] Skip index entries whose path leaves the repository --- internal/mirror/refresh_harden_test.go | 28 ++++++++++++++++++++++++++ internal/mirror/refresh_index.go | 9 +++++++++ 2 files changed, 37 insertions(+) diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index 458f1e1..599a993 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -229,3 +229,31 @@ func TestRefreshDebianArchitectureAll(t *testing.T) { t.Errorf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) } } + +// TestRefreshSkipsIndexPathsOutsideTheRepository checks that an index entry +// whose path leaves the repository base is not fetched. +func TestRefreshSkipsIndexPathsOutsideTheRepository(t *testing.T) { + for _, filename := range []string{ + "../other/pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb", + "pool/main/../../../other/curl_7.88.1-10+deb12u8_amd64.deb", + "/other/curl_7.88.1-10+deb12u8_amd64.deb", + } { + t.Run(filename, func(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + release := []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n") + pool := idx.URL + "/debian/pool/main/c/curl/" + idx.set("/debian/dists/bookworm/InRelease", release) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease", + "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", pool+"curl_7.88.1-10_amd64.deb") + + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", gzipped(t, + "Package: curl\nVersion: 7.88.1-10+deb12u8\nArchitecture: amd64\nFilename: "+filename+"\n\n")) + fetcher.reset() + + if progress := refreshOnce(t, m); progress.Completed != 0 || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want nothing", progress, fetcher.fetched()) + } + }) + } +} diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go index 01168d2..9385736 100644 --- a/internal/mirror/refresh_index.go +++ b/internal/mirror/refresh_index.go @@ -153,6 +153,9 @@ func (m *Mirror) newerIndexFiles(ix osIndex, base string, cached []cachedOSFile, newest := map[string]candidate{} var order []string for _, f := range files { + if !confinedIndexPath(f.Path) { + continue + } name, version, arch := handler.OSPackageIdentity(ix.ecosystem, f.Path) key := name + "\x00" + arch if name == "" || len(cachedByKey[key]) == 0 { @@ -199,6 +202,12 @@ func (m *Mirror) newerIndexFiles(ix osIndex, base string, cached []cachedOSFile, return items } +// confinedIndexPath reports whether an index entry's path is relative and +// stays below the repository base. +func confinedIndexPath(p string) bool { + return p != "" && !path.IsAbs(p) && path.Clean(p) == p && p != ".." && !strings.HasPrefix(p, "../") +} + // debIndexFiles reads the Packages indexes an InRelease or Release file // lists for the components and architectures of the cached files. func debIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, cached []cachedOSFile) ([]indexFile, error) { From 55f1b1e2da7b894720ba02c1285f4c4c9491177c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 14:53:36 +0100 Subject: [PATCH 067/198] Reject a negative mirror.refresh.max_bytes --- internal/config/mirror.go | 3 +++ internal/config/mirror_test.go | 1 + 2 files changed, 4 insertions(+) diff --git a/internal/config/mirror.go b/internal/config/mirror.go index 473b744..625f58d 100644 --- a/internal/config/mirror.go +++ b/internal/config/mirror.go @@ -108,6 +108,9 @@ func (r *MirrorRefresh) validate() error { if err != nil { return fmt.Errorf("invalid mirror.refresh.max_bytes %q: %w", r.MaxBytes, err) } + if n < 0 { + return fmt.Errorf("invalid mirror.refresh.max_bytes %q: must not be negative", r.MaxBytes) + } r.maxBytes = n } return nil diff --git a/internal/config/mirror_test.go b/internal/config/mirror_test.go index 2a53585..153677b 100644 --- a/internal/config/mirror_test.go +++ b/internal/config/mirror_test.go @@ -103,6 +103,7 @@ func TestMirrorRefreshValidate(t *testing.T) { {name: "negative since", refresh: &MirrorRefresh{Interval: "1h", Since: "-1d"}, wantErr: true}, {name: "negative max_versions", refresh: &MirrorRefresh{Interval: "1h", MaxVersions: -1}, wantErr: true}, {name: "bad max_bytes", refresh: &MirrorRefresh{Interval: "1h", MaxBytes: "lots"}, wantErr: true}, + {name: "negative max_bytes", refresh: &MirrorRefresh{Interval: "1h", MaxBytes: "-1GB"}, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 308cc133abff7fbb750ba7cebc932b0a04860adf Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 5 Nov 2026 15:09:16 +0000 Subject: [PATCH 068/198] Remove connection details from stats response --- README.md | 4 +--- docs/swagger/docs.go | 6 ------ docs/swagger/swagger.json | 6 ------ internal/server/server.go | 4 ---- internal/server/server_test.go | 26 -------------------------- internal/server/stats_test.go | 2 +- 6 files changed, 2 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 2a08eaa..e47216e 100644 --- a/README.md +++ b/README.md @@ -1515,9 +1515,7 @@ Response: { "cached_artifacts": 142, "total_size_bytes": 523456789, - "total_size": "499.2 MB", - "storage_url": "file:///path/to/cache/artifacts", - "database_path": "./cache/proxy.db" + "total_size": "499.2 MB" } ``` diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 3ad61f7..fe8723d 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -1461,9 +1461,6 @@ const docTemplate = `{ "cached_artifacts": { "type": "integer" }, - "database_path": { - "type": "string" - }, "downloaded": { "type": "string" }, @@ -1484,9 +1481,6 @@ const docTemplate = `{ "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", "type": "boolean" }, - "storage_url": { - "type": "string" - }, "total_size": { "type": "string" }, diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index d0f27c3..496aecd 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -1454,9 +1454,6 @@ "cached_artifacts": { "type": "integer" }, - "database_path": { - "type": "string" - }, "downloaded": { "type": "string" }, @@ -1477,9 +1474,6 @@ "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", "type": "boolean" }, - "storage_url": { - "type": "string" - }, "total_size": { "type": "string" }, diff --git a/internal/server/server.go b/internal/server/server.go index 55fe597..8599c48 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1387,8 +1387,6 @@ type StatsResponse struct { CachedArtifacts int64 `json:"cached_artifacts"` TotalSize int64 `json:"total_size_bytes"` TotalSizeHuman string `json:"total_size"` - StorageURL string `json:"storage_url"` - DatabasePath string `json:"database_path"` // DownloadedBytes is the accumulated download volume across every // ecosystem: cache hits multiplied by the artifact size they served. DownloadedBytes int64 `json:"downloaded_bytes"` @@ -1452,8 +1450,6 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) { TotalSizeHuman: formatSize(size), Ecosystems: make([]EcosystemStatsEntry, 0, len(ecosystems)), StatsUnavailable: statsErr != nil && len(ecosystems) == 0, - StorageURL: s.storage.URL(), - DatabasePath: s.cfg.Database.String(), } for _, e := range ecosystems { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index e2d2264..5fa49e9 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -534,10 +534,6 @@ func TestStatsEndpoint(t *testing.T) { if stats.CachedArtifacts != 0 { t.Errorf("expected 0 cached artifacts, got %d", stats.CachedArtifacts) } - - if !strings.HasPrefix(stats.StorageURL, "file://") { - t.Errorf("expected storage_url to start with file://, got %q", stats.StorageURL) - } } func TestDashboard(t *testing.T) { @@ -1676,28 +1672,6 @@ func TestNewServer_InvalidAccessLogFailsBeforeDatabaseInit(t *testing.T) { } } -func TestStatsEndpoint_StorageURL(t *testing.T) { - ts := newTestServer(t) - defer ts.close() - - req := httptest.NewRequest("GET", "/stats", nil) - w := httptest.NewRecorder() - ts.handler.ServeHTTP(w, req) - - if w.Code != http.StatusOK { - t.Fatalf("expected status 200, got %d", w.Code) - } - - // Verify the JSON response uses storage_url (not storage_path) - body := w.Body.String() - if !strings.Contains(body, `"storage_url"`) { - t.Errorf("expected JSON key storage_url in response, got: %s", body) - } - if strings.Contains(body, `"storage_path"`) { - t.Errorf("unexpected JSON key storage_path in response (should be storage_url)") - } -} - // TestUIRefusesFraming checks that other sites cannot frame UI pages, where // a framed Delete button could be clicked by a signed-in user. func TestUIRefusesFraming(t *testing.T) { diff --git a/internal/server/stats_test.go b/internal/server/stats_test.go index 467f3c1..e777758 100644 --- a/internal/server/stats_test.go +++ b/internal/server/stats_test.go @@ -108,7 +108,7 @@ func TestStatsFlagsUnavailableFigures(t *testing.T) { if len(stats.Ecosystems) != 0 { t.Errorf("ecosystems = %v, want empty", stats.Ecosystems) } - if stats.StorageURL == "" { + if stats.TotalSizeHuman == "" { t.Error("the rest of the response must still be served") } } From 17355d2227f0361ed399dd11316b13b4045dd777 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:13:42 +0000 Subject: [PATCH 069/198] Bump alpine from 3.24.2 to 3.24.3 Bumps alpine from 3.24.2 to 3.24.3. --- updated-dependencies: - dependency-name: alpine dependency-version: 3.24.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6a56b03..d434e58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ -ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${COMMIT}" \ -o /proxy ./cmd/proxy -FROM alpine:3.24.2 +FROM alpine:3.24.3 RUN apk add --no-cache ca-certificates git From 12c5cc9e4ca1ebee793c6c26d798882c7279387f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:14:00 +0000 Subject: [PATCH 070/198] Bump docker/login-action from 4.6.0 to 4.7.0 Bumps [docker/login-action](https://github.com/docker/login-action) from 4.6.0 to 4.7.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/dbcb813823bdd20940b903addbd779551569679f...81c6f93d4bc991e510b342a00b419b3bb4efc5c6) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.7.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cbb6e15..e325a8a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Log in to the Container registry - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 with: registry: ghcr.io username: ${{ github.actor }} From d5dbd7a95a23c7a788210e6d8648576457091332 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:14:03 +0000 Subject: [PATCH 071/198] Bump zizmorcore/zizmor-action from 0.7.1 to 0.8.0 Bumps [zizmorcore/zizmor-action](https://github.com/zizmorcore/zizmor-action) from 0.7.1 to 0.8.0. - [Release notes](https://github.com/zizmorcore/zizmor-action/releases) - [Commits](https://github.com/zizmorcore/zizmor-action/compare/85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e...53290badfa369cb0bf7ad16342cb62c2fa47ac7b) --- updated-dependencies: - dependency-name: zizmorcore/zizmor-action dependency-version: 0.8.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/zizmor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 02a0722..00041a6 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -34,4 +34,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e # v0.7.1 + uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0 From 06dd126211658fda388c4b4a64da11912e0921a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:15:18 +0000 Subject: [PATCH 072/198] Bump github.com/aws/aws-sdk-go-v2/config from 1.33.9 to 1.33.12 Bumps [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) from 1.33.9 to 1.33.12. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.33.9...config/v1.33.12) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.33.12 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 28 ++++++++++++++-------------- go.sum | 56 ++++++++++++++++++++++++++++---------------------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index 76a1fad..ad2e259 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.27.3 require ( github.com/BurntSushi/toml v1.6.0 github.com/Masterminds/semver/v3 v3.5.0 - github.com/aws/aws-sdk-go-v2/config v1.33.9 + github.com/aws/aws-sdk-go-v2/config v1.33.12 github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 @@ -52,24 +52,24 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.47.3 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect - github.com/aws/smithy-go v1.28.5 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect + github.com/aws/smithy-go v1.28.7 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index c6b5f02..448e33b 100644 --- a/go.sum +++ b/go.sum @@ -52,46 +52,46 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= -github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= -github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0= -github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg= -github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= -github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= +github.com/aws/aws-sdk-go-v2/config v1.33.12 h1:IV1YARaXahwCqUIyP8IlWsP85Lil+5ozTozDQXPumRk= +github.com/aws/aws-sdk-go-v2/config v1.33.12/go.mod h1:h51R06bzMVwNBRZOX5ylHhNDnkB/azZeq/LvxM6q2pQ= +github.com/aws/aws-sdk-go-v2/credentials v1.20.12 h1:6P7Q7yH/GIwHg6MLKv3ySsBqaCoE1z+C0OTzYMenhEY= +github.com/aws/aws-sdk-go-v2/credentials v1.20.12/go.mod h1:Kw/SaoW/wc7IqJErqXJIzKo8b7Aelxfbuc6XlbrRnsc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 h1:vsFkmFrfzODkxVbyuPLscv5j5lC8KbG48LZwV+p56Y0= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3/go.mod h1:pjQYVtvVBEtlsf2p6nXMsjPuDaWs9GM44FMdHbmnmYU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 h1:163OpmhcdzTzgbXbLV+u2v1LY/QmGT148/eOwkczKCY= github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5/go.mod h1:uDza1Eipo/Olii3CCgLglHmAni24PScSAhpfIKbUl0c= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 h1:CO485zrLOZBeq8mrdsudc8cjyhvC4LY4JFt8ZfHvXZ0= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2/go.mod h1:cBAjVUKBr+5wnx1K3XWS8u5AZsqpA2S/0z7YtI6BGFc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 h1:Enx5JcIy/o90kio1LFZ/HguxIsos3clbGuSpGdyshX8= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2/go.mod h1:Ldmll8/2VqaVerWWDf8IGG+6pAvZY04gs3tevq3nIFs= github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 h1:4PokON/ISKuD6Zl062n4WfQNnEFBMHvfeJy73Jp7K+g= github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0/go.mod h1:6ZFrmv/qIAYFmG+2WhA7mFU8+edldRlISwT+tNnyLuA= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM= -github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= -github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 h1:3vu/eUqh0rjh7Zy4N4YKcpZX4Aud+zinB3VsLhhlQFw= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.0/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 h1:kIEpD5dUry7VET3Ybl7zj7PPHRB4lb52G9iTz+a1Tqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.0/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 h1:WfFev+HgTodggcNon5HieiLxxwWwb8Br8R2+nDkixvk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= From f8c5fa9a1d15b90bbfde5d17b2b7dcf9710ae7de Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:18:56 +0100 Subject: [PATCH 073/198] Match wheels on OS and architecture, and conda builds without the hash anywhere --- internal/mirror/files.go | 42 +++++++++++---- internal/mirror/refresh_test.go | 93 ++++++++++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 12 deletions(-) diff --git a/internal/mirror/files.go b/internal/mirror/files.go index 641dfd9..0c0a216 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -81,8 +81,8 @@ var fileEcosystems = map[string]fileEcosystem{ }, } -// pypiVariant names a wheel by its tags, python-abi-platform; every sdist -// is one variant. +// pypiVariant names a wheel by its python and ABI tags and the OS and +// architecture of its platform tags; every sdist is one variant. func pypiVariant(filename string) string { stem, wheel := strings.CutSuffix(filename, ".whl") if !wheel { @@ -92,22 +92,42 @@ func pypiVariant(filename string) string { if len(parts) < 5 { //nolint:mnd // name, version and three tags return stem } - return strings.Join(parts[len(parts)-3:], "-") + tags := parts[len(parts)-3:] + return tags[0] + "-" + tags[1] + "-" + wheelPlatform(tags[2]) } -// condaBuildHash matches the hash and build number a conda build string -// ends in, such as h8753938_0 in py312h8753938_0. -var condaBuildHash = regexp.MustCompile(`h[0-9a-f]{7}_\d+$`) +// wheelPlatformMinimum matches the glibc, musl or macOS minimum version in a +// platform tag, such as the 2_17 in manylinux_2_17_x86_64 or the 2014 in +// manylinux2014_x86_64. +var wheelPlatformMinimum = regexp.MustCompile(`^(manylinux|musllinux|macosx)_?(?:\d+_\d+|1|2010|2014)_`) -// condaVariant names a conda package file by its subdir, the part of its -// build string before the hash, such as py312, and its type. +// wheelPlatform reduces a platform tag, which may list several joined by +// dots, to OS and architecture, so manylinux_2_17_x86_64 and +// manylinux_2_28_x86_64 name the same platform. +func wheelPlatform(tag string) string { + var platforms []string + for _, t := range strings.Split(tag, ".") { + t = wheelPlatformMinimum.ReplaceAllString(t, "${1}_") + if !slices.Contains(platforms, t) { + platforms = append(platforms, t) + } + } + slices.Sort(platforms) + return strings.Join(platforms, ".") +} + +// condaBuildHash matches the hash and build number in a conda build string, +// such as h8753938_0 in py312h8753938_0 or h5f976f7_0 in +// h5f976f7_0_cpython. +var condaBuildHash = regexp.MustCompile(`h[0-9a-f]{7}_\d+`) + +// condaVariant names a conda package file by its subdir, its build string +// without the hash, such as py312 or _cpython, and its type. func condaVariant(subdir, filename string) string { fileType := condaFileType(filename) stem := strings.TrimSuffix(strings.TrimSuffix(filename, ".conda"), ".tar.bz2") build := stem[strings.LastIndex(stem, "-")+1:] - if loc := condaBuildHash.FindStringIndex(build); loc != nil { - build = build[:loc[0]] - } + build = condaBuildHash.ReplaceAllString(build, "") return subdir + "/" + build + "/" + fileType } diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index 4a9f083..2c23e64 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -291,12 +291,18 @@ func TestFileVariants(t *testing.T) { ecosystem, cacheFilename, want string }{ {"pypi", "demo-1.0.0-py3-none-any.whl", "py3-none-any"}, - {"pypi", "demo-1.0.0-1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64"}, + {"pypi", "demo-1.0.0-1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "cp312-cp312-manylinux_x86_64"}, + {"pypi", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux_2_17_aarch64.whl", "cp312-cp312-manylinux_aarch64.manylinux_x86_64"}, + {"pypi", "demo-1.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", "cp312-cp312-musllinux_x86_64"}, + {"pypi", "demo-1.0.0-cp312-cp312-macosx_10_13_universal2.whl", "cp312-cp312-macosx_universal2"}, + {"pypi", "demo-1.0.0-cp312-cp312-win_amd64.whl", "cp312-cp312-win_amd64"}, {"pypi", "demo-1.0.0.tar.gz", "sdist"}, {"pypi", "demo-1.0.0.zip", "sdist"}, {"conda", "linux-64/numpy-1.26.4-py312h8753938_0.conda", "linux-64/py312/conda"}, {"conda", "noarch/requests-2.32.3-pyhd8ed1ab_0.conda", "noarch/py/conda"}, {"conda", "osx-arm64/zlib-1.3.1-h8359307_2.tar.bz2", "osx-arm64//tar.bz2"}, + {"conda", "linux-64/python-3.12.16-h5f976f7_0_cpython.conda", "linux-64/_cpython/conda"}, + {"conda", "linux-64/python-3.12.16-h5f976f7_0_debug_cpython.conda", "linux-64/_debug_cpython/conda"}, } for _, tt := range tests { if got := fileEcosystems[tt.ecosystem].cachedVariant(tt.cacheFilename); got != tt.want { @@ -321,3 +327,88 @@ func TestRefreshable(t *testing.T) { } } } + +// refreshFetches caches files of an old version of name the way a client +// would, lists newFiles for newVersion and returns what a refresh fetched. +func refreshFetches(t *testing.T, ecosystem, name, oldVersion string, cached []string, newVersion string, newFiles []registries.Artifact) []string { + t.Helper() + m, fetcher := setupRefreshMirror(t, "file") + ctx := context.Background() + old := registries.Version{Number: oldVersion, PublishedAt: time.Now().Add(-60 * 24 * time.Hour)} + for _, f := range cached { + result, err := m.proxy.GetOrFetchArtifactFromURLWithDigest(ctx, ecosystem, name, oldVersion, f, "https://files.example/"+f, "") + if err != nil { + t.Fatal(err) + } + _ = result.Reader.Close() + } + for i := range newFiles { + newFiles[i].URL = "https://files.example/" + newFiles[i].Filename + } + m.Versions = stubReleases{ecosystem + "/" + name: { + old, + {Number: newVersion, PublishedAt: time.Now().Add(-30 * 24 * time.Hour), Artifacts: newFiles}, + }} + fetcher.reset() + + progress, err := m.Refresh(ctx, Refresh{Top: 10, Since: 24 * time.Hour}) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if progress.Failed != 0 { + t.Errorf("progress = %+v, want no failures", progress) + } + got := fetcher.fetched() + slices.Sort(got) + return got +} + +func wheels(names ...string) []registries.Artifact { + files := make([]registries.Artifact, 0, len(names)) + for _, n := range names { + files = append(files, registries.Artifact{Filename: n}) + } + return files +} + +// A new wheel built for a newer glibc or macOS still replaces the cached +// one: platform tags match on OS and architecture. +func TestRefreshMatchesWheelsAcrossPlatformMinimums(t *testing.T) { + got := refreshFetches(t, "pypi", "numpy", "2.2.6", + []string{"numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl"}, + "2.3.0", wheels( + "numpy-2.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", + "numpy-2.3.0-cp312-cp312-manylinux_2_28_aarch64.whl", + "numpy-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", + "numpy-2.3.0-cp313-cp313-manylinux_2_28_x86_64.whl", + "numpy-2.3.0-cp312-cp312-macosx_11_0_x86_64.whl", + "numpy-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", + "numpy-2.3.0-cp312-cp312-win_amd64.whl", + )) + want := []string{ + "https://files.example/numpy-2.3.0-cp312-cp312-macosx_11_0_x86_64.whl", + "https://files.example/numpy-2.3.0-cp312-cp312-manylinux_2_28_x86_64.whl", + } + if !slices.Equal(got, want) { + t.Errorf("fetched %v, want %v", got, want) + } +} + +// conda-forge's python builds carry the hash before a suffix; the suffix +// stays part of the kind. +func TestRefreshMatchesCondaBuildsWithHashInside(t *testing.T) { + conda := func(subdir, filename, build string) registries.Artifact { + return registries.Artifact{Filename: filename, Metadata: map[string]any{"subdir": subdir, "build": build}} + } + got := refreshFetches(t, "conda", "conda-forge/python", "3.12.15", + []string{"linux-64/python-3.12.15-h5f976f7_0_cpython.conda"}, + "3.12.16", []registries.Artifact{ + conda("linux-64", "python-3.12.16-hab00c5b_1_cpython.conda", "hab00c5b_1_cpython"), + conda("linux-64", "python-3.12.16-hd12c33a_1_debug_cpython.conda", "hd12c33a_1_debug_cpython"), + conda("osx-arm64", "python-3.12.16-h99e199e_1_cpython.conda", "h99e199e_1_cpython"), + }) + want := []string{"https://files.example/python-3.12.16-hab00c5b_1_cpython.conda"} + if !slices.Equal(got, want) { + t.Errorf("fetched %v, want %v", got, want) + } +} From 34524a40564e813cf273ed25d0457838561b900d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:20:06 +0100 Subject: [PATCH 074/198] Limit a refreshed index to 512 MiB once decompressed --- internal/mirror/refresh_harden_test.go | 26 +++++++++++++++++++ internal/mirror/refresh_index.go | 28 +++++++++++++++++++-- internal/mirror/refresh_index_limit_test.go | 20 +++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 internal/mirror/refresh_index_limit_test.go diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index 599a993..574a873 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -4,6 +4,7 @@ import ( "context" "reflect" "slices" + "strings" "testing" "time" @@ -257,3 +258,28 @@ func TestRefreshSkipsIndexPathsOutsideTheRepository(t *testing.T) { }) } } + +// TestRefreshRefusesIndexesOverTheSizeLimit checks that an index that +// decompresses to more than maxIndexBytes fails instead of being parsed in +// full. +func TestRefreshRefusesIndexesOverTheSizeLimit(t *testing.T) { + limit := maxIndexBytes + maxIndexBytes = 4 << 10 + t.Cleanup(func() { maxIndexBytes = limit }) + + m, fetcher, idx := setupIndexRefresh(t) + idx.set("/debian/dists/bookworm/InRelease", []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n")) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease", + "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", idx.URL+"/debian/pool/main/c/curl/curl_7.88.1-10_amd64.deb") + + padding := strings.Repeat("Package: filler\nVersion: 1\nArchitecture: amd64\nFilename: pool/main/f/filler/filler_1_amd64.deb\n\n", 100) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", gzipped(t, + "Package: curl\nVersion: 7.88.1-10+deb12u8\nArchitecture: amd64\nFilename: pool/main/c/curl/curl_7.88.1-10+deb12u8_amd64.deb\n\n"+padding)) + fetcher.reset() + + progress := refreshOnce(t, m) + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "larger than") || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want the index failing for its size", progress, fetcher.fetched()) + } +} diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go index 9385736..50bb20d 100644 --- a/internal/mirror/refresh_index.go +++ b/internal/mirror/refresh_index.go @@ -254,7 +254,7 @@ func rpmIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, } // readIndex streams an index from upstream through parse, decompressed as -// its name says. It is read directly rather than through the metadata +// its name says and limited to maxIndexBytes. It is read directly rather than through the metadata // cache, since it can be far larger than a metadata document may be. func (m *Mirror) readIndex(ctx context.Context, indexURL string, parse func(io.Reader) ([]indexFile, error)) ([]indexFile, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil) @@ -274,7 +274,31 @@ func (m *Mirror) readIndex(ctx context.Context, indexURL string, parse func(io.R return nil, err } defer func() { _ = r.Close() }() - return parse(r) + return parse(&indexLimitReader{r: r, url: indexURL, remaining: maxIndexBytes}) +} + +// maxIndexBytes bounds an index once decompressed. Debian's largest +// Packages files are under 100 MB and Fedora's primary.xml under 200 MB. +var maxIndexBytes int64 = 512 << 20 + +// indexLimitReader fails once more than maxIndexBytes have been read, so an +// oversized index fails rather than looking complete. +type indexLimitReader struct { + r io.Reader + url string + remaining int64 +} + +func (l *indexLimitReader) Read(p []byte) (int, error) { + if int64(len(p)) > l.remaining+1 { + p = p[:l.remaining+1] + } + n, err := l.r.Read(p) + if int64(n) > l.remaining { + return int(l.remaining), fmt.Errorf("%s: index larger than %d bytes", l.url, maxIndexBytes) + } + l.remaining -= int64(n) + return n, err } // urlDir returns a URL without its last path segment. path.Dir would fold diff --git a/internal/mirror/refresh_index_limit_test.go b/internal/mirror/refresh_index_limit_test.go new file mode 100644 index 0000000..7512fc5 --- /dev/null +++ b/internal/mirror/refresh_index_limit_test.go @@ -0,0 +1,20 @@ +package mirror + +import ( + "io" + "strings" + "testing" +) + +func TestIndexLimitReaderAllowsExactlyTheLimit(t *testing.T) { + limit := maxIndexBytes + maxIndexBytes = 8 + t.Cleanup(func() { maxIndexBytes = limit }) + + if _, err := io.ReadAll(&indexLimitReader{r: strings.NewReader("12345678"), remaining: maxIndexBytes}); err != nil { + t.Errorf("index of exactly the limit: %v", err) + } + if _, err := io.ReadAll(&indexLimitReader{r: strings.NewReader("123456789"), remaining: maxIndexBytes}); err == nil { + t.Error("index one byte over the limit read without an error") + } +} From 6d971716d9fcf3afa297f98541fa5389bf060686 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:20:16 +0100 Subject: [PATCH 075/198] Describe the looser file kind matching --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index dacc93c..b513158 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1924,7 +1924,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. -- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python, ABI and platform tags, and the sdist, or conda packages of the same subdir, build variant (such as `py312`) and file type. +- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need, and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. - Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. From 0f26841f5661fb39a138022458053665a59bec38 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:39:19 +0100 Subject: [PATCH 076/198] Keep blobs of scheduled images from retention A mirror run that finds a blob cached records it in mirrored_at, which retention counts as use. LRU eviction keeps using last_accessed_at, so #508's decision not to count mirror runs as cache hits stands. Fixes #648. --- docs/configuration.md | 2 +- internal/database/retention.go | 35 +++++++++++++++------ internal/database/retention_test.go | 20 ++++++------ internal/database/schema.go | 33 +++++++++++++++++--- internal/database/types.go | 3 ++ internal/handler/container_mirror.go | 5 +++ internal/handler/container_mirror_test.go | 37 +++++++++++++++++++++++ internal/handler/handler.go | 10 ++++++ 8 files changed, 121 insertions(+), 24 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index bf34b23..a15978c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -111,7 +111,7 @@ storage: sweep_interval: "10m" ``` -Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`. +Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. The blobs of an image that `mirror.schedules` keeps current also count from the last run that found them cached, so the schedule keeps them; for `max_size` eviction such a run doesn't count as use. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`. A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version, written in one of the forms the table below lists for the ecosystem. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems, except `generic`, which requires an explicit rule. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention supports (`covered`) and which ones `default` does not reach (`default_not_applied`), including `generic`. diff --git a/internal/database/retention.go b/internal/database/retention.go index 90fffd2..0036362 100644 --- a/internal/database/retention.go +++ b/internal/database/retention.go @@ -17,28 +17,33 @@ type RetentionCandidate struct { Size sql.NullInt64 `db:"size"` FetchedAt sql.NullTime `db:"fetched_at"` LastAccessedAt sql.NullTime `db:"last_accessed_at"` + MirroredAt sql.NullTime `db:"mirrored_at"` Ecosystem string `db:"ecosystem"` PackagePURL string `db:"package_purl"` } -// ExpiredBefore reports whether the artifact was neither fetched nor -// accessed at or after cutoff. Both are checked: clearing a record keeps its -// last access time, so a record fetched again after an eviction has a fresh -// fetch time next to an old access time. +// ExpiredBefore reports whether the artifact was neither fetched, accessed +// nor found by a mirror run at or after cutoff. Fetch and access are both +// checked: clearing a record keeps its last access time, so a record fetched +// again after an eviction has a fresh fetch time next to an old access time. func (c RetentionCandidate) ExpiredBefore(cutoff time.Time) bool { if !c.FetchedAt.Valid || !c.FetchedAt.Time.Before(cutoff) { return false } + if c.MirroredAt.Valid && !c.MirroredAt.Time.Before(cutoff) { + return false + } return !c.LastAccessedAt.Valid || c.LastAccessedAt.Time.Before(cutoff) } -// expiredCondition selects cached artifacts fetched before the cutoff and not -// accessed since. It takes the cutoff twice. +// expiredCondition selects cached artifacts fetched before the cutoff and +// neither accessed nor mirrored since. It takes the cutoff three times. const expiredCondition = ` storage_path IS NOT NULL AND source <> 'local' AND fetched_at < ? - AND (last_accessed_at IS NULL OR last_accessed_at < ?)` + AND (last_accessed_at IS NULL OR last_accessed_at < ?) + AND (mirrored_at IS NULL OR mirrored_at < ?)` // MaxArtifactID returns the highest artifact id, or zero for an empty table. func (db *DB) MaxArtifactID() (int64, error) { @@ -56,7 +61,7 @@ func (db *DB) GetRetentionCandidates(fromID, toID int64, cutoff time.Time) ([]Re var candidates []RetentionCandidate query := db.Rebind(` SELECT a.id, a.version_purl, a.filename, a.storage_path, a.size, - a.fetched_at, a.last_accessed_at, p.ecosystem, p.purl AS package_purl + a.fetched_at, a.last_accessed_at, a.mirrored_at, p.ecosystem, p.purl AS package_purl FROM ( SELECT * FROM artifacts WHERE id > ? AND id <= ? AND` + expiredCondition + ` @@ -65,7 +70,7 @@ func (db *DB) GetRetentionCandidates(fromID, toID int64, cutoff time.Time) ([]Re JOIN packages p ON p.purl = v.package_purl ORDER BY a.id `) - if err := db.Select(&candidates, query, fromID, toID, cutoff.UTC(), cutoff.UTC()); err != nil { + if err := db.Select(&candidates, query, fromID, toID, cutoff.UTC(), cutoff.UTC(), cutoff.UTC()); err != nil { return nil, err } return candidates, nil @@ -89,7 +94,7 @@ func (db *DB) ClearExpiredArtifact(versionPURL, filename, storagePath string, cu SET storage_path = NULL, content_hash = NULL, size = NULL, content_type = NULL, fetched_at = NULL, updated_at = ? WHERE version_purl = ? AND filename = ? AND storage_path = ? AND` + expiredCondition) - res, err := tx.Exec(query, time.Now(), versionPURL, filename, storagePath, cutoff.UTC(), cutoff.UTC()) + res, err := tx.Exec(query, time.Now(), versionPURL, filename, storagePath, cutoff.UTC(), cutoff.UTC(), cutoff.UTC()) if err != nil { return false, err } @@ -103,6 +108,16 @@ func (db *DB) ClearExpiredArtifact(versionPURL, filename, storagePath string, cu return true, tx.Commit() } +// MarkArtifactMirrored records that a mirror run found the artifact cached, +// which keeps it from retention without counting as a cache hit. +func (db *DB) MarkArtifactMirrored(versionPURL, filename string) error { + _, err := db.Exec(db.Rebind(` + UPDATE artifacts SET mirrored_at = ? + WHERE version_purl = ? AND filename = ? AND storage_path IS NOT NULL + `), time.Now().UTC(), versionPURL, filename) + return err +} + // GetArtifactEcosystem returns the packages.ecosystem value of the package an // artifact version belongs to, or "" when it has no package record. func (db *DB) GetArtifactEcosystem(versionPURL string) (string, error) { diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go index bd9ad28..2382011 100644 --- a/internal/database/retention_test.go +++ b/internal/database/retention_test.go @@ -341,18 +341,20 @@ func TestRetentionCandidateExpiredBefore(t *testing.T) { recent := sql.NullTime{Time: now, Valid: true} tests := []struct { - name string - fetched, accessed sql.NullTime - want bool + name string + fetched, accessed, mirrored sql.NullTime + want bool }{ - {"fetched and accessed before", old, old, true}, - {"never served", old, sql.NullTime{}, true}, - {"accessed since", old, recent, false}, - {"fetched again since", recent, old, false}, - {"not cached", sql.NullTime{}, old, false}, + {"fetched and accessed before", old, old, sql.NullTime{}, true}, + {"never served", old, sql.NullTime{}, sql.NullTime{}, true}, + {"accessed since", old, recent, sql.NullTime{}, false}, + {"fetched again since", recent, old, sql.NullTime{}, false}, + {"not cached", sql.NullTime{}, old, sql.NullTime{}, false}, + {"mirrored since", old, old, recent, false}, + {"mirrored before", old, old, old, true}, } for _, tt := range tests { - c := RetentionCandidate{FetchedAt: tt.fetched, LastAccessedAt: tt.accessed} + c := RetentionCandidate{FetchedAt: tt.fetched, LastAccessedAt: tt.accessed, MirroredAt: tt.mirrored} if got := c.ExpiredBefore(cutoff); got != tt.want { t.Errorf("%s: ExpiredBefore() = %v, want %v", tt.name, got, tt.want) } diff --git a/internal/database/schema.go b/internal/database/schema.go index efc2b2d..6a9adaa 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -75,7 +75,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at DATETIME, updated_at DATETIME, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at DATETIME ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -252,7 +253,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at TIMESTAMP, updated_at TIMESTAMP, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at TIMESTAMP ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -388,7 +390,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at DATETIME, updated_at DATETIME, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at DATETIME ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -413,7 +416,8 @@ CREATE TABLE IF NOT EXISTS artifacts ( created_at TIMESTAMP, updated_at TIMESTAMP, source TEXT NOT NULL DEFAULT 'upstream', - manifest TEXT + manifest TEXT, + mirrored_at TIMESTAMP ); CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(version_purl, filename); CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); @@ -538,6 +542,7 @@ var migrations = []migration{ {"019_add_artifact_filename_index", migrateAddArtifactFilenameIndex}, {"020_add_metadata_snapshots", migrateAddMetadataSnapshots}, {"021_add_projects", migrateAddProjects}, + {"022_add_artifact_mirrored_at", migrateAddArtifactMirroredAt}, } // isTableNotFound returns true if the error indicates a missing table. @@ -1080,6 +1085,26 @@ func migrateAddArtifactSource(db *DB) error { return nil } +// migrateAddArtifactMirroredAt records when a mirror run last found an +// artifact cached, which retention counts as use and LRU eviction doesn't. +func migrateAddArtifactMirroredAt(db *DB) error { + hasCol, err := db.HasColumn("artifacts", "mirrored_at") + if err != nil { + return fmt.Errorf("checking column mirrored_at: %w", err) + } + if hasCol { + return nil + } + ts := sqliteDatetime + if db.dialect == DialectPostgres { + ts = postgresTimestamp + } + if _, err := db.Exec(fmt.Sprintf("ALTER TABLE artifacts ADD COLUMN mirrored_at %s", ts)); err != nil { + return fmt.Errorf("adding column mirrored_at to artifacts: %w", err) + } + return nil +} + // migrateAddArtifactManifest stores the manifest read from a pushed archive, // which the metadata of its ecosystem lists the version with. func migrateAddArtifactManifest(db *DB) error { diff --git a/internal/database/types.go b/internal/database/types.go index fb49a1c..c2675d1 100644 --- a/internal/database/types.go +++ b/internal/database/types.go @@ -151,6 +151,9 @@ type Artifact struct { UpdatedAt time.Time `db:"updated_at" json:"updated_at"` Source string `db:"source" json:"source"` Manifest sql.NullString `db:"manifest" json:"manifest,omitempty"` + // MirroredAt is when a mirror run last found the artifact cached. It + // keeps the artifact from retention but not from LRU eviction. + MirroredAt sql.NullTime `db:"mirrored_at" json:"-"` } func (a *Artifact) sourceOrUpstream() string { diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index c8ca65c..a973afd 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -502,6 +502,11 @@ func (m *imageMirror) mirrorBlob(ctx context.Context, digest string) (int64, boo return 0, false, fmt.Errorf("blob %s: %w", digest, err) } if cached { + // Not a cache hit, so LRU eviction still sees the image as unused, + // but retention keeps it while the schedule does. + if err := proxy.MarkMirrored("oci", m.cacheName, digest, digest); err != nil { + return 0, false, fmt.Errorf("blob %s: %w", digest, err) + } return 0, false, nil } result, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "oci", m.cacheName, digest, digest, diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index 2131928..e6be5a5 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -488,6 +488,43 @@ func TestMirrorImageCountsNoCacheHits(t *testing.T) { } } +// A scheduled image that no client pulls keeps its blobs: a mirror run that +// finds them cached holds them back from retention, though not from LRU +// eviction, which TestMirrorImageCountsNoCacheHits covers. +func TestMirrorImageKeepsCachedBlobsFromRetention(t *testing.T) { + registry, _ := newFakeImageRegistry(t) + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + ref := ImageRef{Name: "team/app", References: []string{"multi"}} + + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("first MirrorImage: %v", err) + } + old := time.Now().Add(-30 * 24 * time.Hour).UTC() + if _, err := proxy.DB.Exec(proxy.DB.Rebind("UPDATE artifacts SET fetched_at = ?, last_accessed_at = ?"), old, old); err != nil { + t.Fatal(err) + } + cutoff := time.Now().Add(-14 * 24 * time.Hour) + maxID, err := proxy.DB.MaxArtifactID() + if err != nil { + t.Fatal(err) + } + if candidates, err := proxy.DB.GetRetentionCandidates(0, maxID, cutoff); err != nil || len(candidates) == 0 { + t.Fatalf("before the second run: %d candidates, err %v; want the aged blobs", len(candidates), err) + } + + if _, err := h.MirrorImage(context.Background(), ref, nil); err != nil { + t.Fatalf("second MirrorImage: %v", err) + } + candidates, err := proxy.DB.GetRetentionCandidates(0, maxID, cutoff) + if err != nil { + t.Fatal(err) + } + if len(candidates) != 0 { + t.Errorf("after the second run %d blobs are retention candidates, want none", len(candidates)) + } +} + func TestMirrorImageRefetchesBlobMissingFromStorage(t *testing.T) { registry, image := newFakeImageRegistry(t) proxy := newMirrorTestProxy(t, registry) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 196f648..dae3bb7 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -451,6 +451,16 @@ func (p *Proxy) HasCachedArtifact(ctx context.Context, ecosystem, name, version, return p.Storage.Exists(ctx, artifact.StoragePath) } +// MarkMirrored records that a mirror run found an artifact cached, which +// keeps it from retention without counting as a cache hit. +func (p *Proxy) MarkMirrored(ecosystem, name, version, filename string) error { + _, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return err + } + return p.DB.MarkArtifactMirrored(versionPURL, filename) +} + // ClearCachedArtifact clears an artifact cache record after an external // integrity check fails, and queues its stored bytes for deletion. func (p *Proxy) ClearCachedArtifact(ecosystem, name, version, filename string) error { From 14d04955b4c62c101a1b9a3d16ae5cf7dc09b8c3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:43:33 +0100 Subject: [PATCH 077/198] Match wheels on any of their alternative tags --- internal/mirror/files.go | 66 +++++++++++++++++---------------- internal/mirror/refresh.go | 4 +- internal/mirror/refresh_test.go | 24 +++++++++++- internal/mirror/source.go | 2 +- 4 files changed, 61 insertions(+), 35 deletions(-) diff --git a/internal/mirror/files.go b/internal/mirror/files.go index 0c0a216..ff0d298 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -24,12 +24,12 @@ type fileEcosystem struct { // check refuses a package the ecosystem can't mirror, before its files // are listed. check func(pv PackageVersion) error - // variant names the kind of a file, such as a wheel's tags or a conda - // package's platform, so a refresh mirrors the same kinds of files of a - // new version as were cached of an old one. cachedVariant names it for - // the filename cacheKey returns. - variant func(file registries.Artifact) string - cachedVariant func(cacheFilename string) string + // variants name the kinds of a file, such as each tag combination of a + // wheel or a conda package's platform, so a refresh mirrors the same + // kinds of files of a new version as were cached of an old one. + // cachedVariants names them for the filename cacheKey returns. + variants func(file registries.Artifact) []string + cachedVariants func(cacheFilename string) []string } var fileEcosystems = map[string]fileEcosystem{ @@ -45,8 +45,8 @@ var fileEcosystems = map[string]fileEcosystem{ } return name, version, file.Filename, true }, - variant: func(file registries.Artifact) string { return pypiVariant(file.Filename) }, - cachedVariant: pypiVariant, + variants: func(file registries.Artifact) []string { return pypiVariants(file.Filename) }, + cachedVariants: pypiVariants, }, "conda": { qualifiers: []string{"subdir", "build", "type"}, @@ -70,30 +70,43 @@ var fileEcosystems = map[string]fileEcosystem{ } return nil }, - variant: func(file registries.Artifact) string { + variants: func(file registries.Artifact) []string { subdir, _ := file.Metadata["subdir"].(string) - return condaVariant(subdir, file.Filename) + return []string{condaVariant(subdir, file.Filename)} }, - cachedVariant: func(cacheFilename string) string { + cachedVariants: func(cacheFilename string) []string { subdir, filename := path.Split(cacheFilename) - return condaVariant(strings.TrimSuffix(subdir, "/"), filename) + return []string{condaVariant(strings.TrimSuffix(subdir, "/"), filename)} }, }, } -// pypiVariant names a wheel by its python and ABI tags and the OS and -// architecture of its platform tags; every sdist is one variant. -func pypiVariant(filename string) string { +// pypiVariants names a wheel by each combination of its python, ABI and +// platform tags, which are alternatives where a tag lists several joined by +// dots, with the platform reduced to OS and architecture. Every sdist is +// one variant. +func pypiVariants(filename string) []string { stem, wheel := strings.CutSuffix(filename, ".whl") if !wheel { - return "sdist" + return []string{"sdist"} } parts := strings.Split(stem, "-") if len(parts) < 5 { //nolint:mnd // name, version and three tags - return stem + return []string{stem} } tags := parts[len(parts)-3:] - return tags[0] + "-" + tags[1] + "-" + wheelPlatform(tags[2]) + var variants []string + for _, python := range strings.Split(tags[0], ".") { + for _, abi := range strings.Split(tags[1], ".") { + for _, platform := range strings.Split(tags[2], ".") { + v := python + "-" + abi + "-" + wheelPlatform(platform) + if !slices.Contains(variants, v) { + variants = append(variants, v) + } + } + } + } + return variants } // wheelPlatformMinimum matches the glibc, musl or macOS minimum version in a @@ -101,19 +114,10 @@ func pypiVariant(filename string) string { // manylinux2014_x86_64. var wheelPlatformMinimum = regexp.MustCompile(`^(manylinux|musllinux|macosx)_?(?:\d+_\d+|1|2010|2014)_`) -// wheelPlatform reduces a platform tag, which may list several joined by -// dots, to OS and architecture, so manylinux_2_17_x86_64 and -// manylinux_2_28_x86_64 name the same platform. +// wheelPlatform reduces a platform tag to OS and architecture, so +// manylinux_2_17_x86_64 and manylinux_2_28_x86_64 name the same platform. func wheelPlatform(tag string) string { - var platforms []string - for _, t := range strings.Split(tag, ".") { - t = wheelPlatformMinimum.ReplaceAllString(t, "${1}_") - if !slices.Contains(platforms, t) { - platforms = append(platforms, t) - } - } - slices.Sort(platforms) - return strings.Join(platforms, ".") + return wheelPlatformMinimum.ReplaceAllString(tag, "${1}_") } // condaBuildHash matches the hash and build number in a conda build string, @@ -160,7 +164,7 @@ func (e fileEcosystem) selection(pv PackageVersion) string { // selects reports whether file matches every selecting qualifier of pv. func (e fileEcosystem) selects(pv PackageVersion, file registries.Artifact) bool { - if pv.variants != nil && !pv.variants[e.variant(file)] { + if pv.variants != nil && !slices.ContainsFunc(e.variants(file), func(v string) bool { return pv.variants[v] }) { return false } attrs := e.attrs(file) diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index fc9317a..6fc036b 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -155,7 +155,9 @@ func (m *Mirror) refreshItems(ctx context.Context, pkg database.UsedPackage) ([] if eco, ok := fileEcosystems[ecosystem]; ok { pv.variants = map[string]bool{} for _, f := range pkg.Files { - pv.variants[eco.cachedVariant(f.Filename)] = true + for _, v := range eco.cachedVariants(f.Filename) { + pv.variants[v] = true + } } } items = append(items, pv) diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index 2c23e64..af0d1be 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -292,7 +292,8 @@ func TestFileVariants(t *testing.T) { }{ {"pypi", "demo-1.0.0-py3-none-any.whl", "py3-none-any"}, {"pypi", "demo-1.0.0-1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "cp312-cp312-manylinux_x86_64"}, - {"pypi", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux_2_17_aarch64.whl", "cp312-cp312-manylinux_aarch64.manylinux_x86_64"}, + {"pypi", "demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux_2_17_aarch64.whl", "cp312-cp312-manylinux_x86_64 cp312-cp312-manylinux_aarch64"}, + {"pypi", "demo-1.0.0-py2.py3-none-any.whl", "py2-none-any py3-none-any"}, {"pypi", "demo-1.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", "cp312-cp312-musllinux_x86_64"}, {"pypi", "demo-1.0.0-cp312-cp312-macosx_10_13_universal2.whl", "cp312-cp312-macosx_universal2"}, {"pypi", "demo-1.0.0-cp312-cp312-win_amd64.whl", "cp312-cp312-win_amd64"}, @@ -305,7 +306,7 @@ func TestFileVariants(t *testing.T) { {"conda", "linux-64/python-3.12.16-h5f976f7_0_debug_cpython.conda", "linux-64/_debug_cpython/conda"}, } for _, tt := range tests { - if got := fileEcosystems[tt.ecosystem].cachedVariant(tt.cacheFilename); got != tt.want { + if got := strings.Join(fileEcosystems[tt.ecosystem].cachedVariants(tt.cacheFilename), " "); got != tt.want { t.Errorf("%s variant of %q = %q, want %q", tt.ecosystem, tt.cacheFilename, got, tt.want) } } @@ -412,3 +413,22 @@ func TestRefreshMatchesCondaBuildsWithHashInside(t *testing.T) { t.Errorf("fetched %v, want %v", got, want) } } + +// The dot-separated tags of a wheel are alternatives: a wheel matches a +// cached one when any of its tags does. +func TestRefreshMatchesWheelsOnAnyAlternativeTag(t *testing.T) { + got := refreshFetches(t, "pypi", "demo", "1.0.0", + []string{"demo-1.0.0-cp312-cp312-manylinux_2_17_x86_64.whl", "demo-1.0.0-py2.py3-none-macosx_11_0_arm64.macosx_11_0_x86_64.whl"}, + "1.1.0", wheels( + "demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.linux_x86_64.whl", + "demo-1.1.0-py3-none-macosx_11_0_x86_64.whl", + "demo-1.1.0-cp312-cp312-win_amd64.whl", + )) + want := []string{ + "https://files.example/demo-1.1.0-cp312-cp312-manylinux_2_17_x86_64.linux_x86_64.whl", + "https://files.example/demo-1.1.0-py3-none-macosx_11_0_x86_64.whl", + } + if !slices.Equal(got, want) { + t.Errorf("fetched %v, want %v", got, want) + } +} diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 5e32ae2..6e01c6d 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -27,7 +27,7 @@ type PackageVersion struct { // Platforms limit which platforms of an image index are mirrored. Platforms []string // variants, when set, limit the files of a version that are mirrored to - // those of these variants (see fileEcosystem.variant). + // those of these variants (see fileEcosystem.variants). variants map[string]bool // fileURL, when set, is the one file to mirror, cached as cacheFilename // and checked against digest when there is one. A refresh finds it in From 0300ad252278b1b22d0d9be9620a47c87033add3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Thu, 5 Nov 2026 17:44:12 +0100 Subject: [PATCH 078/198] Limit APKINDEX to maxIndexBytes once decompressed --- internal/mirror/refresh_harden_test.go | 24 +++++++++++++++++++++ internal/mirror/refresh_index.go | 4 ++-- internal/mirror/refresh_index_parse.go | 9 ++++---- internal/mirror/refresh_index_parse_test.go | 2 +- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index 574a873..9c311dd 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -283,3 +283,27 @@ func TestRefreshRefusesIndexesOverTheSizeLimit(t *testing.T) { t.Errorf("progress = %+v, fetched %v; want the index failing for its size", progress, fetcher.fetched()) } } + +// TestRefreshRefusesAPKIndexesOverTheSizeLimit checks that the limit also +// covers APKINDEX.tar.gz, which comes from the metadata cache rather than +// readIndex. +func TestRefreshRefusesAPKIndexesOverTheSizeLimit(t *testing.T) { + limit := maxIndexBytes + maxIndexBytes = 4 << 10 + t.Cleanup(func() { maxIndexBytes = limit }) + + m, fetcher, idx := setupIndexRefresh(t) + dir := "/alpine/v3.20/main/aarch64" + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.0-r0\nA:aarch64\n\n")) + clientFetchesCurl(t, m, "alpine", idx.URL+dir+"/APKINDEX.tar.gz", + "alpine", "8.9.0-r0", "edge/v3.20/main/aarch64/curl-8.9.0-r0.apk", idx.URL+dir+"/curl-8.9.0-r0.apk") + + padding := strings.Repeat("P:filler\nV:1-r0\nA:aarch64\n\n", 400) + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.1-r0\nA:aarch64\n\n"+padding)) + fetcher.reset() + + progress := refreshOnce(t, m) + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, "larger than") || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want the index failing for its size", progress, fetcher.fetched()) + } +} diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go index 50bb20d..3a7ad7c 100644 --- a/internal/mirror/refresh_index.go +++ b/internal/mirror/refresh_index.go @@ -43,8 +43,8 @@ var osIndexes = map[string]osIndex{ base: func(u string) (string, bool) { return strings.CutSuffix(u, "/APKINDEX.tar.gz") }, - files: func(_ context.Context, _ *Mirror, _ string, body []byte, _ []cachedOSFile) ([]indexFile, error) { - return parseAPKIndex(bytes.NewReader(body)) + files: func(_ context.Context, _ *Mirror, indexURL string, body []byte, _ []cachedOSFile) ([]indexFile, error) { + return parseAPKIndex(bytes.NewReader(body), indexURL) }, }, "deb": { diff --git a/internal/mirror/refresh_index_parse.go b/internal/mirror/refresh_index_parse.go index 4ae9857..9b0cd27 100644 --- a/internal/mirror/refresh_index_parse.go +++ b/internal/mirror/refresh_index_parse.go @@ -25,9 +25,10 @@ type indexFile struct { // maxIndexLine bounds one line of a Debian or Alpine index. const maxIndexLine = 1 << 20 -// parseAPKIndex reads the files an APKINDEX.tar.gz lists. Its checksums -// cover only the control section of a package, so no digest is returned. -func parseAPKIndex(r io.Reader) ([]indexFile, error) { +// parseAPKIndex reads the files an APKINDEX.tar.gz lists, limited to +// maxIndexBytes once decompressed. Its checksums cover only the control +// section of a package, so no digest is returned. +func parseAPKIndex(r io.Reader, indexURL string) ([]indexFile, error) { gz, err := gzip.NewReader(r) if err != nil { return nil, err @@ -35,7 +36,7 @@ func parseAPKIndex(r io.Reader) ([]indexFile, error) { defer func() { _ = gz.Close() }() // An APKINDEX.tar.gz may be several gzip streams: the signature, then the // index. Reading on across them finds the index whichever comes first. - tr := tar.NewReader(gz) + tr := tar.NewReader(&indexLimitReader{r: gz, url: indexURL, remaining: maxIndexBytes}) for { hdr, err := tr.Next() if errors.Is(err, io.EOF) { diff --git a/internal/mirror/refresh_index_parse_test.go b/internal/mirror/refresh_index_parse_test.go index 3b68805..934256e 100644 --- a/internal/mirror/refresh_index_parse_test.go +++ b/internal/mirror/refresh_index_parse_test.go @@ -36,7 +36,7 @@ func apkIndexArchive(t *testing.T, index string) []byte { func TestParseAPKIndex(t *testing.T) { index := "C:Q1abc=\nP:curl\nV:8.9.1-r0\nA:aarch64\nS:100\n\nC:Q1def=\nP:busybox\nV:1.36.1-r29\nA:aarch64\n\n" - got, err := parseAPKIndex(bytes.NewReader(apkIndexArchive(t, index))) + got, err := parseAPKIndex(bytes.NewReader(apkIndexArchive(t, index)), "APKINDEX.tar.gz") if err != nil { t.Fatal(err) } From 550a908d888ce2bb8df714b9f12111d3e2d72517 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:23:50 +0000 Subject: [PATCH 079/198] Bump github.com/aws/aws-sdk-go-v2/service/ecr from 1.66.5 to 1.67.1 Bumps [github.com/aws/aws-sdk-go-v2/service/ecr](https://github.com/aws/aws-sdk-go-v2) from 1.66.5 to 1.67.1. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/ecr/v1.66.5...service/s3/v1.67.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/ecr dependency-version: 1.67.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ad2e259..2f4a0ce 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.6.0 github.com/Masterminds/semver/v3 v3.5.0 github.com/aws/aws-sdk-go-v2/config v1.33.12 - github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 + github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1 github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.1 diff --git a/go.sum b/go.sum index 448e33b..cb9226b 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdY github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 h1:163OpmhcdzTzgbXbLV+u2v1LY/QmGT148/eOwkczKCY= -github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5/go.mod h1:uDza1Eipo/Olii3CCgLglHmAni24PScSAhpfIKbUl0c= +github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1 h1:EJlz0wSlZDKaCN2FHwg9RBR+C+gsuVdYblJnEnrq3M4= +github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1/go.mod h1:6rVJFO9ZO+4j/QL5/FbtAsbb8xHC85j79wXA9ndOCTY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 h1:CO485zrLOZBeq8mrdsudc8cjyhvC4LY4JFt8ZfHvXZ0= From 3456b394beb793db9a5926e26d565c8ccbc7eb52 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 18 Oct 2026 21:30:21 +0200 Subject: [PATCH 080/198] Cargo, gem, hex, pub, conda and helm now check cooldown on artifact downloads too, so a version inside the window is withheld even when it was cached before. Publish times are stored in UTC. Fixes #521. --- internal/database/database_test.go | 41 +++ internal/database/queries.go | 4 +- internal/handler/cargo.go | 28 ++ internal/handler/conda.go | 44 +++ internal/handler/download_cooldown.go | 82 +++++ internal/handler/download_cooldown_test.go | 347 +++++++++++++++++++++ internal/handler/filename_download.go | 11 + internal/handler/gem.go | 30 ++ internal/handler/helm.go | 54 ++++ internal/handler/hex.go | 25 ++ internal/handler/pub.go | 34 ++ 11 files changed, 698 insertions(+), 2 deletions(-) create mode 100644 internal/handler/download_cooldown.go create mode 100644 internal/handler/download_cooldown_test.go diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 9ac4140..91734e1 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1204,3 +1204,44 @@ func TestGetFetchedVersionPURLs(t *testing.T) { } }) } + +// Publish times are written in UTC whatever zone the caller's time is in, so +// a TIMESTAMP column without a zone reads back the same instant. +func TestVersionPublishedAtIsUTC(t *testing.T) { + tokyo, err := time.LoadLocation("Asia/Tokyo") + if err != nil { + t.Skipf("time zone data unavailable: %v", err) + } + prev := time.Local + time.Local = tokyo + t.Cleanup(func() { time.Local = prev }) + + runWithBothDatabases(t, func(t *testing.T, db *DB) { + written := time.Date(2026, 10, 10, 21, 0, 1, 0, tokyo) + want := written.UTC() + check := func(purl string) { + t.Helper() + got, err := db.GetVersionByPURL(purl) + if err != nil || got == nil || !got.PublishedAt.Valid { + t.Fatalf("GetVersionByPURL(%s) = %+v, %v", purl, got, err) + } + if !got.PublishedAt.Time.Equal(want) || got.PublishedAt.Time.Location() != time.UTC { + t.Errorf("%s PublishedAt read back as %v, want %v", purl, got.PublishedAt.Time, want) + } + } + + if err := db.SetVersionPublishedAt("pkg:cargo/demo@1.0.0", "pkg:cargo/demo", written); err != nil { + t.Fatal(err) + } + check("pkg:cargo/demo@1.0.0") + + if err := db.UpsertVersion(&Version{ + PURL: "pkg:cargo/demo@2.0.0", + PackagePURL: "pkg:cargo/demo", + PublishedAt: sql.NullTime{Time: written, Valid: true}, + }); err != nil { + t.Fatal(err) + } + check("pkg:cargo/demo@2.0.0") + }) +} diff --git a/internal/database/queries.go b/internal/database/queries.go index 23d55df..86f31ef 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -185,7 +185,7 @@ func (db *DB) upsertVersion(exec execer, v *Version) error { _, err := exec.Exec(query, v.PURL, v.PackagePURL, v.License, v.Integrity, - v.PublishedAt, v.Yanked, v.EnrichedAt, now, now, + inUTC(v.PublishedAt), v.Yanked, v.EnrichedAt, now, now, ) if err != nil { return fmt.Errorf("upserting version: %w", err) @@ -218,7 +218,7 @@ func (db *DB) SetVersionPublishedAt(versionPURL, packagePURL string, publishedAt ` } - _, err := db.Exec(query, versionPURL, packagePURL, publishedAt, now, now) + _, err := db.Exec(query, versionPURL, packagePURL, publishedAt.UTC(), now, now) if err != nil { return fmt.Errorf("setting version publish time: %w", err) } diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 0c26551..43bcbc4 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -2,6 +2,7 @@ package handler import ( "bufio" + "bytes" "encoding/json" "errors" "fmt" @@ -186,6 +187,26 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr } } +// publishTime reads a crate version's pubtime from the sparse index, sharing +// the index cache with handleIndex. Entries published before crates.io +// recorded pubtime have none. +func (h *CargoHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + upstreamURL := fmt.Sprintf("%s/%s", h.indexURL, h.buildIndexPath(name)) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "cargo", name, upstreamURL, "text/plain") + if err != nil { + return time.Time{}, false, err + } + for line := range bytes.SplitSeq(body, []byte("\n")) { + if vers, _, _ := lookupJSONString(line, "vers"); vers != version { + continue + } + pubtime, _, _ := lookupJSONString(line, "pubtime") + published, err := time.Parse(time.RFC3339, pubtime) + return published, err == nil, nil + } + return time.Time{}, false, nil +} + // buildIndexPath builds the sparse index path for a crate name. func (h *CargoHandler) buildIndexPath(name string) string { name = strings.ToLower(name) @@ -217,6 +238,13 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("cargo download request", "crate", name, "version", version, "filename", filename) + if h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: "cargo", policyName: name, storedName: name, version: version, + lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, + }) { + return + } + downloadURL := fmt.Sprintf( "%s/%s/%s", h.downloadURL, diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 8581c88..22ec603 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -139,6 +139,13 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("conda download request", "channel", channel, "arch", arch, "name", name, "version", version) + if h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: "conda", policyName: name, storedName: packageName, version: version, + lookup: func() (time.Time, bool, error) { return h.publishTime(r, channel, arch, filename) }, + }) { + return + } + upstreamURL := h.upstreamURL + r.URL.Path // The same filename can hold different bytes in each subdir, so the // subdir is part of the cache key. @@ -531,6 +538,43 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { return hashedCacheKey(h.upstreamURL, urlPath) } +// publishTime reads a package file's timestamp from the subdir's +// repodata.json. It shares proxyCached's cache entry and encoding, so the +// repodata a client just fetched is not downloaded again, and reads the one +// entry in place rather than decoding the whole index. +func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { + repodataPath := "/" + channel + "/" + arch + "/repodata.json" + cacheKey := h.metadataCacheKey(repodataPath) + body, _, encoding, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "gzip", nil, "*/*") + if err != nil { + return time.Time{}, false, err + } + if strings.EqualFold(strings.TrimSpace(encoding), "gzip") { + decoded, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return time.Time{}, false, err + } + body, err = h.proxy.ReadMetadata(decoded) + if err != nil { + return time.Time{}, false, err + } + } + + key := "packages" + if strings.HasSuffix(filename, ".conda") { + key = "packages.conda" + } + raw, err := lookupJSON(body, key, filename, "timestamp") + if err != nil || len(raw) == 0 { + return time.Time{}, false, err + } + var ts float64 + if err := json.Unmarshal(raw, &ts); err != nil || ts == 0 { + return time.Time{}, false, nil + } + return time.Unix(int64(ts)/condaTimestampDivisor, 0), true, nil +} + // proxyCached forwards a metadata request with caching. func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) { cacheKey := h.metadataCacheKey(r.URL.Path) diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go new file mode 100644 index 0000000..94554d3 --- /dev/null +++ b/internal/handler/download_cooldown.go @@ -0,0 +1,82 @@ +package handler + +import ( + "net/http" + "time" +) + +// publishTimeLookup reads a version's publish time from upstream metadata. ok +// is false when the metadata has no usable time for the version. +type publishTimeLookup func() (published time.Time, ok bool, err error) + +// downloadVersion names the version an artifact download is for, so its +// cooldown can be checked before the artifact is served. +type downloadVersion struct { + ecosystem string + // policyName is the package name the cooldown policy is keyed by, + // canonicalPackagePURL(ecosystem, policyName). + policyName string + // storedName and version name the versions row the publish time is kept + // in, the same names the artifact is cached under. An empty storedName + // keeps no time, and every check asks lookup. + storedName string + version string + lookup publishTimeLookup +} + +// withholdInCooldown reports whether a download is inside the cooldown +// window, and if so answers it with a 404. Filtering metadata is not enough on +// its own: lockfiles record artifact URLs, so clients reach the download path +// without reading the metadata first, and a cached artifact would be served +// even after the policy was raised. +// +// A publish time never changes, so the stored versions row is read first and +// lookup is only asked for a version the proxy has no time for yet; the time +// it finds is stored. A version with no usable publish time is allowed +// through, matching how the metadata filters treat it. +func (p *Proxy) withholdInCooldown(w http.ResponseWriter, d downloadVersion) bool { + if !p.versionInCooldown(d) { + return false + } + p.Logger.Info("cooldown: withholding "+d.ecosystem+" download", + "name", d.policyName, "version", d.version) + http.Error(w, ErrVersionNotFound.Error(), http.StatusNotFound) + return true +} + +func (p *Proxy) versionInCooldown(d downloadVersion) bool { + if p.Cooldown == nil || !p.Cooldown.Enabled() { + return false + } + policyKey := canonicalPackagePURL(d.ecosystem, d.policyName) + if p.Cooldown.For(d.ecosystem, policyKey) <= 0 { + return false + } + + var versionPURL string + if d.storedName != "" { + versionPURL = canonicalVersionPURL(d.ecosystem, d.storedName, d.version) + if ver, err := p.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, ver.PublishedAt.Time) + } + } + + published, ok, err := d.lookup() + if err != nil { + p.Logger.Warn("cooldown: could not read publish time for download check", + "ecosystem", d.ecosystem, "name", d.policyName, "version", d.version, "error", err) + return false + } + if !ok { + return false + } + + if versionPURL != "" { + if err := p.DB.SetVersionPublishedAt(versionPURL, canonicalPackagePURL(d.ecosystem, d.storedName), published); err != nil { + p.Logger.Warn("cooldown: could not store publish time", + "ecosystem", d.ecosystem, "name", d.policyName, "version", d.version, "error", err) + } + } + + return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, published) +} diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go new file mode 100644 index 0000000..9b791be --- /dev/null +++ b/internal/handler/download_cooldown_test.go @@ -0,0 +1,347 @@ +package handler + +import ( + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// Versions every download cooldown fixture lists: one published before the +// window, one inside it, and one without a publish time. +const ( + cooldownOldVersion = "1.0.0" + cooldownRecentVersion = "2.0.0" + cooldownUntimed = "3.0.0" +) + +// downloadCooldownCase drives one ecosystem's artifact endpoint. The package +// is always named demo, so the policy key is pkg:{ecosystem}/demo. +type downloadCooldownCase struct { + ecosystem string + // storedName is the name the artifact is cached under; empty when the + // ecosystem keeps no publish time in the versions table. + storedName string + // artifactVersion maps a release to the version the artifact is cached + // under, and filename and path to its file and download path. + artifactVersion func(version string) string + filename func(version string) string + path func(version string) string + // metadata serves the upstream document holding the publish times; a + // zero time is listed without one. + metadata func(w http.ResponseWriter, r *http.Request, times map[string]time.Time) + handler func(proxy *Proxy, upstreamURL string) http.Handler +} + +func sameVersion(version string) string { return version } + +func chartContent(version string) string { return "chart " + version } + +func downloadCooldownCases() []downloadCooldownCase { + return []downloadCooldownCase{ + { + ecosystem: "cargo", + storedName: "demo", + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".crate" }, + path: func(v string) string { return "/crates/demo/" + v + "/download" }, + metadata: cargoCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewCargoHandler(p, "http://proxy", u, u).Routes() }, + }, + { + ecosystem: "gem", + storedName: "demo", + // Platform gems carry the platform in the filename and the + // cached version, but not in the versions API number. + artifactVersion: func(v string) string { return v + "-x86_64-linux" }, + filename: func(v string) string { return "demo-" + v + "-x86_64-linux.gem" }, + path: func(v string) string { return "/gems/demo-" + v + "-x86_64-linux.gem" }, + metadata: gemCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewGemHandlerWithUpstream(p, "http://proxy", u).Routes() }, + }, + { + ecosystem: "hex", + storedName: "demo", + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".tar" }, + path: func(v string) string { return "/tarballs/demo-" + v + ".tar" }, + metadata: hexCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewHexHandlerWithUpstreams(p, "http://proxy", u, u).Routes() + }, + }, + { + ecosystem: "pub", + storedName: "demo", + artifactVersion: sameVersion, + filename: func(v string) string { return "demo-" + v + ".tar.gz" }, + path: func(v string) string { return "/packages/demo/versions/" + v + ".tar.gz" }, + metadata: pubCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { return NewPubHandlerWithUpstream(p, "http://proxy", u).Routes() }, + }, + { + // Conda caches under {channel}/{name}, but the policy is keyed by + // the name alone, as the repodata filter keys it. + ecosystem: "conda", + storedName: "conda-forge/demo", + artifactVersion: sameVersion, + filename: func(v string) string { return "noarch/demo-" + v + "-py_0.conda" }, + path: func(v string) string { return "/conda-forge/noarch/demo-" + v + "-py_0.conda" }, + metadata: condaCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewCondaHandlerWithUpstream(p, "http://proxy", u).Routes() + }, + }, + { + // Helm caches charts under the repository and digest, so the + // chart name and time come from the index on every check. + ecosystem: "helm", + artifactVersion: func(v string) string { return sha256Hex(chartContent(v)) }, + filename: func(v string) string { return "demo-" + v + ".tgz" }, + path: func(v string) string { return "/stable/charts/" + sha256Hex(chartContent(v)) + "/demo-" + v + ".tgz" }, + metadata: helmCooldownMetadata, + handler: func(p *Proxy, u string) http.Handler { + return NewHelmHandler(p, "http://proxy", map[string]string{"stable": u + "/charts"}).Routes() + }, + }, + } +} + +func cargoCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/de/mo/demo" { + http.NotFound(w, r) + return + } + for version, published := range times { + line := map[string]string{"name": "demo", "vers": version} + if !published.IsZero() { + line["pubtime"] = published.Format(time.RFC3339) + } + encoded, _ := json.Marshal(line) + _, _ = fmt.Fprintf(w, "%s\n", encoded) + } +} + +func gemCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/api/v1/versions/demo.json" { + http.NotFound(w, r) + return + } + var versions []gemVersion + for version, published := range times { + v := gemVersion{Number: version, Platform: "x86_64-linux"} + if !published.IsZero() { + v.CreatedAt = published.Format(time.RFC3339) + } + versions = append(versions, v) + } + _ = json.NewEncoder(w).Encode(versions) +} + +func hexCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/api/packages/demo" { + http.NotFound(w, r) + return + } + var pkg hexPackageAPI + for version, published := range times { + release := hexRelease{Version: version} + if !published.IsZero() { + release.InsertedAt = published.Format(time.RFC3339Nano) + } + pkg.Releases = append(pkg.Releases, release) + } + _ = json.NewEncoder(w).Encode(pkg) +} + +func pubCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/api/packages/demo" { + http.NotFound(w, r) + return + } + var versions []map[string]string + for version, published := range times { + v := map[string]string{"version": version} + if !published.IsZero() { + v["published"] = published.Format(time.RFC3339) + } + versions = append(versions, v) + } + _ = json.NewEncoder(w).Encode(map[string]any{"name": "demo", "versions": versions}) +} + +func condaCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/conda-forge/noarch/repodata.json" { + http.NotFound(w, r) + return + } + packages := map[string]map[string]any{} + for version, published := range times { + entry := map[string]any{"name": "demo", "version": version} + if !published.IsZero() { + entry["timestamp"] = published.UnixMilli() + } + packages["demo-"+version+"-py_0.conda"] = entry + } + // The download check asks for gzip, as proxyCached does. + w.Header().Set(headerContentEncoding, "gzip") + gz := gzip.NewWriter(w) + _ = json.NewEncoder(gz).Encode(map[string]any{"packages.conda": packages}) + _ = gz.Close() +} + +func helmCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path != "/charts/index.yaml" { + http.NotFound(w, r) + return + } + _, _ = fmt.Fprintln(w, "apiVersion: v1\nentries:\n demo:") + for version, published := range times { + _, _ = fmt.Fprintf(w, " - digest: %s\n urls: [demo-%s.tgz]\n version: %s\n", + sha256Hex(chartContent(version)), version, version) + if !published.IsZero() { + _, _ = fmt.Fprintf(w, " created: %s\n", published.Format(time.RFC3339)) + } + } +} + +// seedCachedArtifact caches the release's artifact the way an earlier, +// cooldown-free download would have. +func (c downloadCooldownCase) seedCachedArtifact(t *testing.T, proxy *Proxy, version string) { + t.Helper() + db, store := proxy.DB, proxy.Storage.(*mockStorage) + name := c.storedName + content := "artifact " + version + if c.ecosystem == "helm" { + name, content = "stable", chartContent(version) + } + seedPackage(t, db, store, c.ecosystem, name, c.artifactVersion(version), c.filename(version), content) +} + +// TestDownloadCooldownWithholdsCachedArtifacts requests each ecosystem's +// artifact endpoint for a version that was cached before the cooldown was +// raised. A version inside the window must be withheld, from the cache too; +// one past the window, or without a known publish time, is served. +func TestDownloadCooldownWithholdsCachedArtifacts(t *testing.T) { + now := time.Now().Truncate(time.Second) + times := map[string]time.Time{ + cooldownOldVersion: now.Add(-30 * 24 * time.Hour), + cooldownRecentVersion: now.Add(-time.Hour), + cooldownUntimed: {}, + } + statuses := map[string]int{ + cooldownOldVersion: http.StatusOK, + cooldownRecentVersion: http.StatusNotFound, + cooldownUntimed: http.StatusOK, + } + + for _, c := range downloadCooldownCases() { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.metadata(w, r, times) + })) + t.Cleanup(upstream.Close) + + for _, version := range []string{cooldownOldVersion, cooldownRecentVersion, cooldownUntimed} { + t.Run(c.ecosystem+"/"+version, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, version) + proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ + canonicalPackagePURL(c.ecosystem, "demo"): "7d", + }} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(version), nil)) + + if w.Code != statuses[version] { + t.Fatalf("status = %d, want %d (body %q)", w.Code, statuses[version], w.Body.String()) + } + if w.Code == http.StatusOK && w.Body.String() == "" { + t.Error("served an empty body") + } + if c.storedName == "" || times[version].IsZero() { + return + } + ver, err := db.GetVersionByPURL(canonicalVersionPURL(c.ecosystem, c.storedName, c.artifactVersion(version))) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("publish time not stored: %+v, %v", ver, err) + } + if got, want := ver.PublishedAt.Time, times[version]; !got.Equal(want) { + t.Errorf("stored publish time = %v, want %v", got, want) + } + }) + } + } +} + +// TestDownloadCooldownUsesStoredPublishTime checks that a stored publish +// time decides the download without asking the upstream, which here fails. +func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(upstream.Close) + + for _, c := range downloadCooldownCases() { + if c.storedName == "" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + versionPURL := canonicalVersionPURL(c.ecosystem, c.storedName, c.artifactVersion(cooldownRecentVersion)) + packagePURL := canonicalPackagePURL(c.ecosystem, c.storedName) + if err := db.SetVersionPublishedAt(versionPURL, packagePURL, time.Now().Add(-time.Hour)); err != nil { + t.Fatalf("seeding publish time: %v", err) + } + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 (body %q)", w.Code, w.Body.String()) + } + }) + } +} + +// TestDownloadCooldownOtherPackagesSkipLookup checks that a download outside +// every cooldown rule is served without reading upstream metadata. Helm +// needs the index to learn the chart name, so it is left out. +func TestDownloadCooldownOtherPackagesSkipLookup(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.storedName == "" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected upstream request %s", r.URL.Path) + http.Error(w, "unexpected", http.StatusInternalServerError) + })) + t.Cleanup(upstream.Close) + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + proxy.Cooldown = &cooldown.Config{Packages: map[string]string{ + canonicalPackagePURL(c.ecosystem, "other"): "7d", + }} + + w := httptest.NewRecorder() + c.handler(proxy, upstream.URL).ServeHTTP(w, httptest.NewRequest(http.MethodGet, c.path(cooldownRecentVersion), nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 (body %q)", w.Code, strings.TrimSpace(w.Body.String())) + } + }) + } +} diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go index 153c0e6..dbc0e64 100644 --- a/internal/handler/filename_download.go +++ b/internal/handler/filename_download.go @@ -3,6 +3,7 @@ package handler import ( "net/http" "strings" + "time" ) type filenameDownload struct { @@ -12,6 +13,9 @@ type filenameDownload struct { parseErr string fetchErr string parse func(string) (name, version string) + // publishTime reads a version's publish time from upstream metadata for + // the download-time cooldown check. + publishTime func(r *http.Request, name, version string) (time.Time, bool, error) } func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d filenameDownload) { @@ -30,6 +34,13 @@ func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d p.Logger.Debug(d.ecosystem+" download request", "name", name, "version", version, "filename", filename) + if p.withholdInCooldown(w, downloadVersion{ + ecosystem: d.ecosystem, policyName: name, storedName: name, version: version, + lookup: func() (time.Time, bool, error) { return d.publishTime(r, name, version) }, + }) { + return + } + downloadURL := d.upstreamURL + r.URL.Path result, err := p.GetOrFetchArtifactFromURL( r.Context(), d.ecosystem, name, version, filename, downloadURL, diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 6e57559..009e0f5 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "regexp" "strings" "time" @@ -82,6 +83,7 @@ func (h *GemHandler) handleDownload(w http.ResponseWriter, r *http.Request) { parseErr: "could not parse gem filename", fetchErr: "failed to fetch gem", parse: h.parseGemFilename, + publishTime: h.publishTime, }) } @@ -304,6 +306,34 @@ func parseGemVersions(body []byte) (publishTimes, error) { return times, nil } +// publishTime reads a gem version's created_at from the versions API. The +// download check caches the response, unlike fetchFilteredVersions, so each +// download does not cost an upstream request. version carries the platform +// the way gem filenames do, as in 1.0.0-x86_64-linux. +func (h *GemHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + versionsURL := fmt.Sprintf("%s/api/v1/versions/%s.json", h.upstreamURL, url.PathEscape(name)) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "gem", "api/v1/versions/"+name, versionsURL) + if err != nil { + return time.Time{}, false, err + } + var versions []gemVersion + if err := json.Unmarshal(body, &versions); err != nil { + return time.Time{}, false, err + } + for _, v := range versions { + number := v.Number + if v.Platform != "" && v.Platform != "ruby" { + number += "-" + v.Platform + } + if number != version { + continue + } + createdAt, err := time.Parse(time.RFC3339, v.CreatedAt) + return createdAt, err == nil, nil + } + return time.Time{}, false, nil +} + // proxyCached forwards a metadata request with caching. func (h *GemHandler) proxyCached(w http.ResponseWriter, r *http.Request) { upstreamURL := h.upstreamURL + r.URL.Path diff --git a/internal/handler/helm.go b/internal/handler/helm.go index fdc000e..9fb947f 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -127,6 +127,12 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } if cached != nil { + if h.withholdCachedChart(w, r, repository, upstreamURL, digest) { + if cached.Reader != nil { + _ = cached.Reader.Close() + } + return + } h.serveChart(w, r, repository, digest, filename, cached) return } @@ -376,6 +382,54 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh return lookup, nil } +// withholdCachedChart checks a cached chart against the cooldown before it is +// served. An uncached chart is checked by findChartDownload, but a cached one +// may have been fetched under an earlier policy. Charts are cached under the +// repository and digest rather than the chart name the policy is keyed by, so +// the name and publish time come from the index on every check. +func (h *HelmHandler) withholdCachedChart(w http.ResponseWriter, r *http.Request, repository, upstreamURL, digest string) bool { + if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + return false + } + index := h.fetchIndex(r, repository, upstreamURL) + if index.err != nil { + h.proxy.Logger.Warn("cooldown: could not fetch Helm index for download check", + "repository", repository, "digest", digest, "error", index.err) + return false + } + chartName, created, ok := h.findChartRelease(upstreamURL, index.body, digest) + if !ok { + return false + } + return h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: helmMetadataEcosystem, policyName: chartName, version: digest, + lookup: func() (time.Time, bool, error) { return created, !created.IsZero(), nil }, + }) +} + +// findChartRelease returns the name and created time of the index release +// with the given digest. +func (h *HelmHandler) findChartRelease(upstreamURL string, body []byte, digest string) (string, time.Time, bool) { + _, entries, err := parseHelmIndex(body) + if err != nil { + return "", time.Time{}, false + } + for i := 0; i < len(entries.Content); i += 2 { + chartName := entries.Content[i].Value + releases := entries.Content[i+1] + if releases.Kind != yaml.SequenceNode { + continue + } + for _, release := range releases.Content { + chart, err := h.parseChartRelease(chartName, upstreamURL, release) + if err == nil && chart.digest == digest { + return chartName, chart.created, true + } + } + } + return "", time.Time{}, false +} + func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { return !created.IsZero() && h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() && !h.proxy.cooldown().IsAllowed(helmMetadataEcosystem, canonicalPackagePURL(helmMetadataEcosystem, chartName), created) diff --git a/internal/handler/hex.go b/internal/handler/hex.go index f2a93ad..6836dad 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "strings" "time" @@ -78,6 +79,7 @@ func (h *HexHandler) handleDownload(w http.ResponseWriter, r *http.Request) { parseErr: "could not parse tarball filename", fetchErr: "failed to fetch package", parse: h.parseTarballFilename, + publishTime: h.publishTime, }) } @@ -241,6 +243,29 @@ func parseHexReleases(body []byte) (publishTimes, error) { return times, nil } +// publishTime reads a release's inserted_at from the Hex HTTP API. The +// download check caches the response, unlike fetchFilteredVersions, so each +// download does not cost an upstream request. +func (h *HexHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + apiURL := fmt.Sprintf("%s/api/packages/%s", h.apiURL, url.PathEscape(name)) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "hex", "api/packages/"+name, apiURL) + if err != nil { + return time.Time{}, false, err + } + var pkg hexPackageAPI + if err := json.Unmarshal(body, &pkg); err != nil { + return time.Time{}, false, err + } + for _, release := range pkg.Releases { + if release.Version != version { + continue + } + insertedAt, err := time.Parse(time.RFC3339Nano, release.InsertedAt) + return insertedAt, err == nil, nil + } + return time.Time{}, false, nil +} + // signedPayload decompresses a gzipped Signed message, with the metadata // limit applied to the decompressed size, and returns its payload. func (h *HexHandler) signedPayload(gzippedData []byte) ([]byte, error) { diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 24fdd02..22540f1 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -94,6 +94,13 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("pub download request", "name", name, "version", version) + if h.proxy.withholdInCooldown(w, downloadVersion{ + ecosystem: "pub", policyName: name, storedName: name, version: version, + lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, + }) { + return + } + downloadURL := fmt.Sprintf("%s/packages/%s/versions/%s%s", h.upstreamURL, url.PathEscape(name), url.PathEscape(version), pubArchiveExt) h.discardVersionPage(name, version, filename) @@ -134,6 +141,33 @@ func (h *PubHandler) discardVersionPage(name, version, filename string) { } } +// publishTime reads a version's published time from the package metadata, +// sharing the metadata cache with handlePackageMetadata. +func (h *PubHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { + upstreamURL := fmt.Sprintf("%s/api/packages/%s", h.upstreamURL, name) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "pub", name, upstreamURL) + if err != nil { + return time.Time{}, false, err + } + var metadata struct { + Versions []struct { + Version string `json:"version"` + Published string `json:"published"` + } `json:"versions"` + } + if err := json.Unmarshal(body, &metadata); err != nil { + return time.Time{}, false, err + } + for _, v := range metadata.Versions { + if v.Version != version { + continue + } + published, err := time.Parse(time.RFC3339, v.Published) + return published, err == nil, nil + } + return time.Time{}, false, nil +} + // handlePackageMetadata proxies package metadata and rewrites archive URLs. func (h *PubHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { name := r.PathValue("name") From 89c60abd1a51a551aceef46d030f55f0f0b94cb6 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Sun, 18 Oct 2026 21:43:57 +0200 Subject: [PATCH 081/198] Conda now checks each package file against its own timestamp, because builds of one version differ. Conda repodata and helm indexes are parsed once per cached document instead of on every download. --- internal/handler/conda.go | 55 +++++++---- internal/handler/download_cooldown.go | 62 ++++++++++++ internal/handler/download_cooldown_test.go | 108 +++++++++++++++++---- internal/handler/handler.go | 4 + internal/handler/helm.go | 62 +++++++----- 5 files changed, 232 insertions(+), 59 deletions(-) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 22ec603..d7cf196 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -139,8 +139,10 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("conda download request", "channel", channel, "arch", arch, "name", name, "version", version) + // Builds and subdirs of one version each have their own timestamp, so + // the check is per file and keeps no time in the shared versions row. if h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: "conda", policyName: name, storedName: packageName, version: version, + ecosystem: "conda", policyName: name, version: version, lookup: func() (time.Time, bool, error) { return h.publishTime(r, channel, arch, filename) }, }) { return @@ -539,40 +541,57 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { } // publishTime reads a package file's timestamp from the subdir's -// repodata.json. It shares proxyCached's cache entry and encoding, so the -// repodata a client just fetched is not downloaded again, and reads the one -// entry in place rather than decoding the whole index. +// repodata.json, fetched with the cache key and encoding proxyCached uses. +// The repodata is indexed once per document (see publishIndex), since a +// subdir's index runs to tens of megabytes and an install downloads many +// files from it. func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { repodataPath := "/" + channel + "/" + arch + "/repodata.json" cacheKey := h.metadataCacheKey(repodataPath) - body, _, encoding, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "gzip", nil, "*/*") + res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) + if res.err != nil { + return time.Time{}, false, res.err + } + index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata) if err != nil { return time.Time{}, false, err } + published, ok := index[filename] + return published, ok, nil +} + +// indexRepodata maps each package file in a repodata.json to its timestamp, +// leaving out files without one. +func (h *CondaHandler) indexRepodata(body []byte, encoding string) (map[string]time.Time, error) { if strings.EqualFold(strings.TrimSpace(encoding), "gzip") { decoded, err := gzip.NewReader(bytes.NewReader(body)) if err != nil { - return time.Time{}, false, err + return nil, err } body, err = h.proxy.ReadMetadata(decoded) if err != nil { - return time.Time{}, false, err + return nil, err } } - - key := "packages" - if strings.HasSuffix(filename, ".conda") { - key = "packages.conda" + type entry struct { + Timestamp float64 `json:"timestamp"` } - raw, err := lookupJSON(body, key, filename, "timestamp") - if err != nil || len(raw) == 0 { - return time.Time{}, false, err + var repodata struct { + Packages map[string]entry `json:"packages"` + PackagesConda map[string]entry `json:"packages.conda"` } - var ts float64 - if err := json.Unmarshal(raw, &ts); err != nil || ts == 0 { - return time.Time{}, false, nil + if err := json.Unmarshal(body, &repodata); err != nil { + return nil, err } - return time.Unix(int64(ts)/condaTimestampDivisor, 0), true, nil + index := make(map[string]time.Time, len(repodata.Packages)+len(repodata.PackagesConda)) + for _, files := range []map[string]entry{repodata.Packages, repodata.PackagesConda} { + for filename, e := range files { + if e.Timestamp != 0 { + index[filename] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0) + } + } + } + return index, nil } // proxyCached forwards a metadata request with caching. diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index 94554d3..5aec959 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -2,7 +2,10 @@ package handler import ( "net/http" + "sync" "time" + + "golang.org/x/sync/singleflight" ) // publishTimeLookup reads a version's publish time from upstream metadata. ok @@ -80,3 +83,62 @@ func (p *Proxy) versionInCooldown(d downloadVersion) bool { return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, published) } + +// maxPublishIndexes bounds how many documents publishIndexCache keeps an +// index for. +const maxPublishIndexes = 64 + +// publishIndexCache keeps, for each large metadata document, the publish +// times a download check reads from it, so a document is decompressed and +// parsed once per version from upstream rather than on every download. +// Concurrent checks for a document not indexed yet share one parse. +type publishIndexCache struct { + mu sync.Mutex + entries map[string]publishIndexEntry + group singleflight.Group +} + +type publishIndexEntry struct { + digest string + index any +} + +// publishIndex returns build's index of res, reusing the one built for the +// same document under key. res.digest identifies the document; without one, +// as when the metadata cache is off, the index is built for this call alone. +func publishIndex[V any](p *Proxy, key string, res metadataResult, build func([]byte, string) (map[string]V, error)) (map[string]V, error) { + c := &p.publishIndexes + if res.digest == "" { + return build(res.body, res.contentEncoding) + } + c.mu.Lock() + if e, ok := c.entries[key]; ok && e.digest == res.digest { + c.mu.Unlock() + return e.index.(map[string]V), nil + } + c.mu.Unlock() + + v, err, _ := c.group.Do(key+"\x00"+res.digest, func() (any, error) { + index, err := build(res.body, res.contentEncoding) + if err != nil { + return nil, err + } + c.mu.Lock() + defer c.mu.Unlock() + if c.entries == nil { + c.entries = make(map[string]publishIndexEntry) + } + if _, ok := c.entries[key]; !ok && len(c.entries) >= maxPublishIndexes { + for evict := range c.entries { + delete(c.entries, evict) + break + } + } + c.entries[key] = publishIndexEntry{digest: res.digest, index: index} + return index, nil + }) + if err != nil { + return nil, err + } + return v.(map[string]V), nil +} diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go index 9b791be..c504edb 100644 --- a/internal/handler/download_cooldown_test.go +++ b/internal/handler/download_cooldown_test.go @@ -26,9 +26,10 @@ const ( // is always named demo, so the policy key is pkg:{ecosystem}/demo. type downloadCooldownCase struct { ecosystem string - // storedName is the name the artifact is cached under; empty when the - // ecosystem keeps no publish time in the versions table. - storedName string + // cachedName is the name the artifact is cached under, and keepsTime + // whether the publish time is kept in its versions row. + cachedName string + keepsTime bool // artifactVersion maps a release to the version the artifact is cached // under, and filename and path to its file and download path. artifactVersion func(version string) string @@ -48,7 +49,8 @@ func downloadCooldownCases() []downloadCooldownCase { return []downloadCooldownCase{ { ecosystem: "cargo", - storedName: "demo", + cachedName: "demo", + keepsTime: true, artifactVersion: sameVersion, filename: func(v string) string { return "demo-" + v + ".crate" }, path: func(v string) string { return "/crates/demo/" + v + "/download" }, @@ -57,7 +59,8 @@ func downloadCooldownCases() []downloadCooldownCase { }, { ecosystem: "gem", - storedName: "demo", + cachedName: "demo", + keepsTime: true, // Platform gems carry the platform in the filename and the // cached version, but not in the versions API number. artifactVersion: func(v string) string { return v + "-x86_64-linux" }, @@ -68,7 +71,8 @@ func downloadCooldownCases() []downloadCooldownCase { }, { ecosystem: "hex", - storedName: "demo", + cachedName: "demo", + keepsTime: true, artifactVersion: sameVersion, filename: func(v string) string { return "demo-" + v + ".tar" }, path: func(v string) string { return "/tarballs/demo-" + v + ".tar" }, @@ -79,7 +83,8 @@ func downloadCooldownCases() []downloadCooldownCase { }, { ecosystem: "pub", - storedName: "demo", + cachedName: "demo", + keepsTime: true, artifactVersion: sameVersion, filename: func(v string) string { return "demo-" + v + ".tar.gz" }, path: func(v string) string { return "/packages/demo/versions/" + v + ".tar.gz" }, @@ -88,9 +93,10 @@ func downloadCooldownCases() []downloadCooldownCase { }, { // Conda caches under {channel}/{name}, but the policy is keyed by - // the name alone, as the repodata filter keys it. + // the name alone, as the repodata filter keys it. Each build has + // its own timestamp, so none is kept in the version's row. ecosystem: "conda", - storedName: "conda-forge/demo", + cachedName: "conda-forge/demo", artifactVersion: sameVersion, filename: func(v string) string { return "noarch/demo-" + v + "-py_0.conda" }, path: func(v string) string { return "/conda-forge/noarch/demo-" + v + "-py_0.conda" }, @@ -102,6 +108,7 @@ func downloadCooldownCases() []downloadCooldownCase { { // Helm caches charts under the repository and digest, so the // chart name and time come from the index on every check. + cachedName: "stable", ecosystem: "helm", artifactVersion: func(v string) string { return sha256Hex(chartContent(v)) }, filename: func(v string) string { return "demo-" + v + ".tgz" }, @@ -217,12 +224,11 @@ func helmCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[stri func (c downloadCooldownCase) seedCachedArtifact(t *testing.T, proxy *Proxy, version string) { t.Helper() db, store := proxy.DB, proxy.Storage.(*mockStorage) - name := c.storedName content := "artifact " + version if c.ecosystem == "helm" { - name, content = "stable", chartContent(version) + content = chartContent(version) } - seedPackage(t, db, store, c.ecosystem, name, c.artifactVersion(version), c.filename(version), content) + seedPackage(t, db, store, c.ecosystem, c.cachedName, c.artifactVersion(version), c.filename(version), content) } // TestDownloadCooldownWithholdsCachedArtifacts requests each ecosystem's @@ -267,10 +273,10 @@ func TestDownloadCooldownWithholdsCachedArtifacts(t *testing.T) { if w.Code == http.StatusOK && w.Body.String() == "" { t.Error("served an empty body") } - if c.storedName == "" || times[version].IsZero() { + if !c.keepsTime || times[version].IsZero() { return } - ver, err := db.GetVersionByPURL(canonicalVersionPURL(c.ecosystem, c.storedName, c.artifactVersion(version))) + ver, err := db.GetVersionByPURL(canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(version))) if err != nil || ver == nil || !ver.PublishedAt.Valid { t.Fatalf("publish time not stored: %+v, %v", ver, err) } @@ -291,7 +297,7 @@ func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) { t.Cleanup(upstream.Close) for _, c := range downloadCooldownCases() { - if c.storedName == "" { + if !c.keepsTime { continue } t.Run(c.ecosystem, func(t *testing.T) { @@ -299,8 +305,8 @@ func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) { proxy.HTTPClient = upstream.Client() fetcher.fetchErr = errors.New("artifact must be served from the cache") c.seedCachedArtifact(t, proxy, cooldownRecentVersion) - versionPURL := canonicalVersionPURL(c.ecosystem, c.storedName, c.artifactVersion(cooldownRecentVersion)) - packagePURL := canonicalPackagePURL(c.ecosystem, c.storedName) + versionPURL := canonicalVersionPURL(c.ecosystem, c.cachedName, c.artifactVersion(cooldownRecentVersion)) + packagePURL := canonicalPackagePURL(c.ecosystem, c.cachedName) if err := db.SetVersionPublishedAt(versionPURL, packagePURL, time.Now().Add(-time.Hour)); err != nil { t.Fatalf("seeding publish time: %v", err) } @@ -320,7 +326,7 @@ func TestDownloadCooldownUsesStoredPublishTime(t *testing.T) { // needs the index to learn the chart name, so it is left out. func TestDownloadCooldownOtherPackagesSkipLookup(t *testing.T) { for _, c := range downloadCooldownCases() { - if c.storedName == "" { + if c.ecosystem == "helm" { continue } t.Run(c.ecosystem, func(t *testing.T) { @@ -345,3 +351,69 @@ func TestDownloadCooldownOtherPackagesSkipLookup(t *testing.T) { }) } } + +// TestCondaDownloadCooldownPerBuild checks that builds of one version are +// judged by their own timestamps, whichever is requested first: a rebuild +// inside the window is withheld and an old build stays available. +func TestCondaDownloadCooldownPerBuild(t *testing.T) { + now := time.Now() + repodata := map[string]any{"packages.conda": map[string]any{ + "demo-1.0.0-py_0.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-400 * 24 * time.Hour).UnixMilli()}, + "demo-1.0.0-py_1.conda": map[string]any{"name": "demo", "version": "1.0.0", "timestamp": now.Add(-time.Hour).UnixMilli()}, + }} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(repodata) + })) + t.Cleanup(upstream.Close) + + builds := map[string]int{"py_0": http.StatusOK, "py_1": http.StatusNotFound} + for _, order := range [][]string{{"py_0", "py_1", "py_0"}, {"py_1", "py_0", "py_1"}} { + t.Run(strings.Join(order, ","), func(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.fetchErr = errors.New("artifact must be served from the cache") + for build := range builds { + filename := "noarch/demo-1.0.0-" + build + ".conda" + seedPackage(t, db, store, "conda", "conda-forge/demo", "1.0.0", filename, "artifact "+build) + } + proxy.Cooldown = &cooldown.Config{Default: "7d"} + h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes() + + for _, build := range order { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/noarch/demo-1.0.0-"+build+".conda", nil)) + if w.Code != builds[build] { + t.Fatalf("%s status = %d, want %d", build, w.Code, builds[build]) + } + } + }) + } +} + +// TestPublishIndexBuildsOncePerDocument checks that an index is reused for +// the same document, rebuilt for a new one, and not kept without a digest. +func TestPublishIndexBuildsOncePerDocument(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + builds := 0 + build := func(body []byte, _ string) (map[string]bool, error) { + builds++ + return map[string]bool{string(body): true}, nil + } + steps := []struct { + res metadataResult + want string + wantBuilds int + }{ + {metadataResult{body: []byte("a"), digest: "d1"}, "a", 1}, + {metadataResult{body: []byte("ignored"), digest: "d1"}, "a", 1}, + {metadataResult{body: []byte("b"), digest: "d2"}, "b", 2}, + {metadataResult{body: []byte("c")}, "c", 3}, + {metadataResult{body: []byte("c")}, "c", 4}, + } + for i, step := range steps { + got, err := publishIndex(proxy, "key", step.res, build) + if err != nil || !got[step.want] || builds != step.wantBuilds { + t.Fatalf("step %d: publishIndex = %v, %v after %d builds, want %q after %d", i, got, err, builds, step.want, step.wantBuilds) + } + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 196f648..125b117 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -292,6 +292,10 @@ type Proxy struct { // Nil leaves every request to rewrite its own copy. rewrites *rewriteCache + // publishIndexes keeps publish times read out of large metadata + // documents for the download cooldown check; see publishIndex. + publishIndexes publishIndexCache + // now is the clock rewritten metadata is dated by, and started the time // the process started; see metadataDate. now func() time.Time diff --git a/internal/handler/helm.go b/internal/handler/helm.go index 9fb947f..cbbf8b0 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -120,6 +120,10 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } + if h.withholdChart(w, r, repository, upstreamURL, digest) { + return + } + cached, err := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename) if err != nil { h.proxy.Logger.Error("failed to check Helm chart cache", "error", err) @@ -127,12 +131,6 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } if cached != nil { - if h.withholdCachedChart(w, r, repository, upstreamURL, digest) { - if cached.Reader != nil { - _ = cached.Reader.Close() - } - return - } h.serveChart(w, r, repository, digest, filename, cached) return } @@ -382,38 +380,55 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh return lookup, nil } -// withholdCachedChart checks a cached chart against the cooldown before it is -// served. An uncached chart is checked by findChartDownload, but a cached one +// withholdChart checks a chart against the cooldown before it is served. +// findChartDownload already leaves out charts in cooldown, but a cached chart // may have been fetched under an earlier policy. Charts are cached under the // repository and digest rather than the chart name the policy is keyed by, so -// the name and publish time come from the index on every check. -func (h *HelmHandler) withholdCachedChart(w http.ResponseWriter, r *http.Request, repository, upstreamURL, digest string) bool { +// the name and publish time come from the index, indexed once per document +// (see publishIndex). +func (h *HelmHandler) withholdChart(w http.ResponseWriter, r *http.Request, repository, upstreamURL, digest string) bool { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { return false } - index := h.fetchIndex(r, repository, upstreamURL) - if index.err != nil { + res := h.fetchIndex(r, repository, upstreamURL) + if res.err != nil { h.proxy.Logger.Warn("cooldown: could not fetch Helm index for download check", - "repository", repository, "digest", digest, "error", index.err) + "repository", repository, "digest", digest, "error", res.err) return false } - chartName, created, ok := h.findChartRelease(upstreamURL, index.body, digest) + index, err := publishIndex(h.proxy, "helm\x00"+h.indexCacheKey(repository, upstreamURL), res, + func(body []byte, _ string) (map[string]helmChartTime, error) { + return h.indexChartTimes(upstreamURL, body) + }) + if err != nil { + h.proxy.Logger.Warn("cooldown: could not read Helm index for download check", + "repository", repository, "digest", digest, "error", err) + return false + } + chart, ok := index[digest] if !ok { return false } return h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: helmMetadataEcosystem, policyName: chartName, version: digest, - lookup: func() (time.Time, bool, error) { return created, !created.IsZero(), nil }, + ecosystem: helmMetadataEcosystem, policyName: chart.name, version: digest, + lookup: func() (time.Time, bool, error) { return chart.created, !chart.created.IsZero(), nil }, }) } -// findChartRelease returns the name and created time of the index release -// with the given digest. -func (h *HelmHandler) findChartRelease(upstreamURL string, body []byte, digest string) (string, time.Time, bool) { +// helmChartTime is the chart a release digest belongs to and when it was +// created. +type helmChartTime struct { + name string + created time.Time +} + +// indexChartTimes maps each release digest in an index to its chart. +func (h *HelmHandler) indexChartTimes(upstreamURL string, body []byte) (map[string]helmChartTime, error) { _, entries, err := parseHelmIndex(body) if err != nil { - return "", time.Time{}, false + return nil, err } + index := make(map[string]helmChartTime) for i := 0; i < len(entries.Content); i += 2 { chartName := entries.Content[i].Value releases := entries.Content[i+1] @@ -422,12 +437,13 @@ func (h *HelmHandler) findChartRelease(upstreamURL string, body []byte, digest s } for _, release := range releases.Content { chart, err := h.parseChartRelease(chartName, upstreamURL, release) - if err == nil && chart.digest == digest { - return chartName, chart.created, true + if err != nil { + continue } + index[chart.digest] = helmChartTime{name: chartName, created: chart.created} } } - return "", time.Time{}, false + return index, nil } func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { From 5db16cbb991c25714673aebd9fd860ca451da9dc Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 26 Oct 2026 07:28:44 +0100 Subject: [PATCH 082/198] Conda reads repodata as a stream for the download check, under the same bound as the repodata filter, so a repodata.json over metadata_max_size no longer lets a package in cooldown through. If even the compressed copy is too large to cache, the check streams it from upstream. --- internal/handler/conda.go | 93 ++++++++++++++++------ internal/handler/download_cooldown_test.go | 48 +++++++++++ 2 files changed, 115 insertions(+), 26 deletions(-) diff --git a/internal/handler/conda.go b/internal/handler/conda.go index d7cf196..5d79b3f 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -543,16 +543,23 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { // publishTime reads a package file's timestamp from the subdir's // repodata.json, fetched with the cache key and encoding proxyCached uses. // The repodata is indexed once per document (see publishIndex), since a -// subdir's index runs to tens of megabytes and an install downloads many -// files from it. +// subdir's index runs to hundreds of megabytes and an install downloads many +// files from it. A compressed copy over metadata_max_size can't be cached, +// so it is streamed from upstream for each check, as handleRepodata does. func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { repodataPath := "/" + channel + "/" + arch + "/repodata.json" cacheKey := h.metadataCacheKey(repodataPath) res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) - if res.err != nil { - return time.Time{}, false, res.err + var index map[string]time.Time + var err error + switch { + case errors.Is(res.err, ErrMetadataTooLarge): + index, err = h.streamRepodataIndex(r, h.upstreamURL+repodataPath) + case res.err != nil: + err = res.err + default: + index, err = publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata) } - index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata) if err != nil { return time.Time{}, false, err } @@ -560,36 +567,70 @@ func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename stri return published, ok, nil } -// indexRepodata maps each package file in a repodata.json to its timestamp, -// leaving out files without one. +// indexRepodata indexes a cached repodata.json body. func (h *CondaHandler) indexRepodata(body []byte, encoding string) (map[string]time.Time, error) { + var reader io.Reader = bytes.NewReader(body) if strings.EqualFold(strings.TrimSpace(encoding), "gzip") { - decoded, err := gzip.NewReader(bytes.NewReader(body)) - if err != nil { - return nil, err - } - body, err = h.proxy.ReadMetadata(decoded) + decoded, err := gzip.NewReader(reader) if err != nil { return nil, err } + reader = decoded } - type entry struct { - Timestamp float64 `json:"timestamp"` - } - var repodata struct { - Packages map[string]entry `json:"packages"` - PackagesConda map[string]entry `json:"packages.conda"` - } - if err := json.Unmarshal(body, &repodata); err != nil { + return readRepodataIndex(reader) +} + +// streamRepodataIndex indexes repodata.json as it streams from upstream. +func (h *CondaHandler) streamRepodataIndex(r *http.Request, upstreamURL string) (map[string]time.Time, error) { + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) + if err != nil { return nil, err } - index := make(map[string]time.Time, len(repodata.Packages)+len(repodata.PackagesConda)) - for _, files := range []map[string]entry{repodata.Packages, repodata.PackagesConda} { - for filename, e := range files { - if e.Timestamp != 0 { - index[filename] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0) - } + req.Header.Set(headerAcceptEncoding, "gzip") + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("repodata returned %d", resp.StatusCode) + } + var reader io.Reader = resp.Body + if strings.EqualFold(strings.TrimSpace(resp.Header.Get(headerContentEncoding)), "gzip") { + decoded, err := gzip.NewReader(resp.Body) + if err != nil { + return nil, err } + defer func() { _ = decoded.Close() }() + reader = decoded + } + return readRepodataIndex(reader) +} + +// readRepodataIndex maps each package file in a repodata.json to its +// timestamp, leaving out files without one. It reads the document as it +// streams, under the same bound as filterRepodata, so the decompressed size +// is not limited by metadata_max_size. +func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { + index := make(map[string]time.Time) + err := jsonstream.CopyObject(json.NewDecoder(io.LimitReader(r, maxRepodataSize)), io.Discard, func(m *jsonstream.Member) error { + if m.Key != "packages" && m.Key != "packages.conda" { + return m.Copy() + } + return m.Object(func(entry *jsonstream.Member) error { + raw, err := entry.Raw() + if err != nil { + return err + } + var e repodataEntry + if json.Unmarshal(raw, &e) == nil && e.Timestamp != 0 { + index[entry.Key] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0) + } + return nil + }) + }) + if err != nil { + return nil, err } return index, nil } diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go index c504edb..1dd44e0 100644 --- a/internal/handler/download_cooldown_test.go +++ b/internal/handler/download_cooldown_test.go @@ -417,3 +417,51 @@ func TestPublishIndexBuildsOncePerDocument(t *testing.T) { } } } + +// TestCondaDownloadCooldownPastMetadataLimit checks that a cached package is +// withheld when its repodata is over metadata_max_size, as the repodata +// endpoint hides it: once only decompressed, and once already compressed. +func TestCondaDownloadCooldownPastMetadataLimit(t *testing.T) { + plain := largeRepodata(500) + compressed := gzipPayload(t, plain) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(headerContentEncoding, "gzip") + _, _ = w.Write(compressed) + })) + t.Cleanup(upstream.Close) + + for _, tt := range []struct { + name string + limit int64 + }{ + {"decompressed over the limit", int64(len(compressed)) + 1}, + {"compressed over the limit", int64(len(compressed)) - 1}, + } { + t.Run(tt.name, func(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.MetadataMaxSize = tt.limit + fetcher.fetchErr = errors.New("artifact must be served from the cache") + for _, file := range []string{"pkg-0-old.conda", "pkg-1-new.conda"} { + seedPackage(t, db, store, "conda", "conda-forge/pkg", strings.Split(file, "-")[1], "linux-64/"+file, "artifact "+file) + } + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := NewCondaHandlerWithUpstream(proxy, "http://proxy", upstream.URL).Routes() + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) + if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "pkg-1-new.conda") { + t.Fatalf("repodata status = %d, want 200 without pkg-1-new.conda", w.Code) + } + for file, want := range map[string]int{"pkg-0-old.conda": http.StatusOK, "pkg-1-new.conda": http.StatusNotFound} { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+file, nil)) + if w.Code != want { + t.Errorf("%s status = %d, want %d (body %q)", file, w.Code, want, w.Body.String()) + } + } + }) + } +} From 3ab7dcc1dc0fdcd821f08cc9eb23d020274a8625 Mon Sep 17 00:00:00 2001 From: Christian Heim Date: Mon, 26 Oct 2026 08:55:25 +0100 Subject: [PATCH 083/198] The UTC test compares the zone offset, since Postgres reads TIMESTAMP back as a fixed +0000 zone rather than time.UTC. --- internal/database/database_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 91734e1..500ad0d 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -1225,7 +1225,9 @@ func TestVersionPublishedAtIsUTC(t *testing.T) { if err != nil || got == nil || !got.PublishedAt.Valid { t.Fatalf("GetVersionByPURL(%s) = %+v, %v", purl, got, err) } - if !got.PublishedAt.Time.Equal(want) || got.PublishedAt.Time.Location() != time.UTC { + // Postgres reads TIMESTAMP back as a fixed +0000 zone rather + // than time.UTC, so compare the offset. + if _, offset := got.PublishedAt.Time.Zone(); !got.PublishedAt.Time.Equal(want) || offset != 0 { t.Errorf("%s PublishedAt read back as %v, want %v", purl, got.PublishedAt.Time, want) } } From b9dcbd27fc065e6e82e8b930b36a243170d9befb Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 5 Nov 2026 18:41:03 +0000 Subject: [PATCH 084/198] Handle cooldown lookup failures and reuse download indexes --- docs/configuration.md | 4 + internal/handler/cargo.go | 16 ++- internal/handler/conda.go | 39 +++--- internal/handler/cooldown_times.go | 15 +++ internal/handler/download_cooldown.go | 98 ++++++++------- .../handler/download_cooldown_cache_test.go | 26 ++++ .../handler/download_cooldown_errors_test.go | 101 +++++++++++++++ internal/handler/download_cooldown_test.go | 11 +- internal/handler/filename_download.go | 4 +- internal/handler/gem.go | 27 +--- internal/handler/helm.go | 117 ++++++++---------- internal/handler/hex.go | 22 +--- internal/handler/pub.go | 6 +- 13 files changed, 294 insertions(+), 192 deletions(-) create mode 100644 internal/handler/download_cooldown_cache_test.go create mode 100644 internal/handler/download_cooldown_errors_test.go diff --git a/docs/configuration.md b/docs/configuration.md index bf34b23..38765e0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1201,6 +1201,10 @@ Maven caveats: Helm uses each release's `created` time from the repository's `index.yaml`. Releases in cooldown are left out of the index the proxy serves, and their archives aren't served; a release without `created` is let through. Charts pulled over OCI have no publish time and aren't held back. +Cargo, RubyGems, Hex, pub.dev, Conda and Helm check artifact downloads against cooldown, including files cached before the policy changed. When a publish-time lookup fails, an already-cached file is still served; an uncached download fails with 502, or 503 with `Retry-After` when the rate-limit gate holds the request. Conda checks timestamps per file because builds of one version can have different publish times. + +Conda repodata and Helm indexes are parsed once per cached document for download checks. With `cache_metadata` off, the parsed index is kept for one minute, including the fetch, so an install can reuse it across files. Conda streams repodata when its compressed copy exceeds `metadata_max_size`, under the same decompressed size bound as the metadata filter. + RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered. For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, and the filtered result is kept until the first version it held back leaves cooldown (see [rewritten metadata cache](#rewritten-metadata-cache)); a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 43bcbc4..fa93a74 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -197,11 +197,17 @@ func (h *CargoHandler) publishTime(r *http.Request, name, version string) (time. return time.Time{}, false, err } for line := range bytes.SplitSeq(body, []byte("\n")) { - if vers, _, _ := lookupJSONString(line, "vers"); vers != version { + if len(bytes.TrimSpace(line)) == 0 { continue } - pubtime, _, _ := lookupJSONString(line, "pubtime") - published, err := time.Parse(time.RFC3339, pubtime) + var entry crateIndexEntry + if err := json.Unmarshal(line, &entry); err != nil { + return time.Time{}, false, err + } + if entry.Version != version { + continue + } + published, err := time.Parse(time.RFC3339, entry.PublishTime) return published, err == nil, nil } return time.Time{}, false, nil @@ -238,8 +244,8 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("cargo download request", "crate", name, "version", version, "filename", filename) - if h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: "cargo", policyName: name, storedName: name, version: version, + if h.proxy.withholdInCooldown(w, r, downloadVersion{ + ecosystem: "cargo", policyName: name, storedName: name, version: version, cacheName: name, filename: filename, lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, }) { return diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 5d79b3f..de6ae8a 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -141,18 +141,16 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // Builds and subdirs of one version each have their own timestamp, so // the check is per file and keeps no time in the shared versions row. - if h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: "conda", policyName: name, version: version, + // Subdirs distinguish cached files with the same filename. + cacheFilename := arch + "/" + filename + if h.proxy.withholdInCooldown(w, r, downloadVersion{ + ecosystem: "conda", policyName: name, version: version, cacheName: packageName, filename: cacheFilename, lookup: func() (time.Time, bool, error) { return h.publishTime(r, channel, arch, filename) }, }) { return } upstreamURL := h.upstreamURL + r.URL.Path - // The same filename can hold different bytes in each subdir, so the - // subdir is part of the cache key. - cacheFilename := arch + "/" + filename - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, cacheFilename, upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") @@ -540,26 +538,21 @@ func (h *CondaHandler) metadataCacheKey(urlPath string) string { return hashedCacheKey(h.upstreamURL, urlPath) } -// publishTime reads a package file's timestamp from the subdir's -// repodata.json, fetched with the cache key and encoding proxyCached uses. -// The repodata is indexed once per document (see publishIndex), since a -// subdir's index runs to hundreds of megabytes and an install downloads many -// files from it. A compressed copy over metadata_max_size can't be cached, -// so it is streamed from upstream for each check, as handleRepodata does. +// publishTime streams compressed repodata when it exceeds metadata_max_size. func (h *CondaHandler) publishTime(r *http.Request, channel, arch, filename string) (time.Time, bool, error) { repodataPath := "/" + channel + "/" + arch + "/repodata.json" cacheKey := h.metadataCacheKey(repodataPath) - res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) - var index map[string]time.Time - var err error - switch { - case errors.Is(res.err, ErrMetadataTooLarge): - index, err = h.streamRepodataIndex(r, h.upstreamURL+repodataPath) - case res.err != nil: - err = res.err - default: - index, err = publishIndex(h.proxy, "conda\x00"+cacheKey, res, h.indexRepodata) - } + index, err := publishIndex(h.proxy, "conda\x00"+cacheKey, func() metadataResult { + return h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, h.upstreamURL+repodataPath, "*/*", "gzip", nil) + }, func(res metadataResult) (map[string]time.Time, error) { + if errors.Is(res.err, ErrMetadataTooLarge) { + return h.streamRepodataIndex(r, h.upstreamURL+repodataPath) + } + if res.err != nil { + return nil, res.err + } + return h.indexRepodata(res.body, res.contentEncoding) + }) if err != nil { return time.Time{}, false, err } diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index 5d95d16..b7a0516 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -16,6 +16,21 @@ import ( // through. type publishTimes map[string]time.Time +func (p *Proxy) lookupPublishTime(ctx context.Context, t timedMetadata, version string) (time.Time, bool, error) { + var times publishTimes + validate := func(body []byte) error { + var err error + times, err = t.parseTimes(body) + return err + } + res := p.lookupMetadata(ctx, t.ecosystem, t.timesKey, t.timesURL, t.timesAccept, "", validate) + if res.err != nil { + return time.Time{}, false, res.err + } + published, ok := times[version] + return published, ok && !published.IsZero(), nil +} + // timedMetadata describes metadata whose publish times live in a second // document, as with the RubyGems compact index and the versions API, or a // Hex package and the Hex HTTP API. Both are read through the metadata cache. diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index 5aec959..47d8b70 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -24,21 +24,22 @@ type downloadVersion struct { // keeps no time, and every check asks lookup. storedName string version string + cacheName string + filename string lookup publishTimeLookup } // withholdInCooldown reports whether a download is inside the cooldown -// window, and if so answers it with a 404. Filtering metadata is not enough on -// its own: lockfiles record artifact URLs, so clients reach the download path -// without reading the metadata first, and a cached artifact would be served -// even after the policy was raised. -// -// A publish time never changes, so the stored versions row is read first and -// lookup is only asked for a version the proxy has no time for yet; the time -// it finds is stored. A version with no usable publish time is allowed -// through, matching how the metadata filters treat it. -func (p *Proxy) withholdInCooldown(w http.ResponseWriter, d downloadVersion) bool { - if !p.versionInCooldown(d) { +// window, or a lookup failure prevents an uncached download. +func (p *Proxy) withholdInCooldown(w http.ResponseWriter, r *http.Request, d downloadVersion) bool { + withheld, err := p.versionInCooldown(d) + if err != nil { + if !p.serveCachedDespiteCooldownError(w, r, d.ecosystem, d.cacheName, d.version, d.filename, err) { + p.writeCooldownLookupError(w, d.ecosystem, d.policyName, err) + } + return true + } + if !withheld { return false } p.Logger.Info("cooldown: withholding "+d.ecosystem+" download", @@ -47,31 +48,26 @@ func (p *Proxy) withholdInCooldown(w http.ResponseWriter, d downloadVersion) boo return true } -func (p *Proxy) versionInCooldown(d downloadVersion) bool { - if p.Cooldown == nil || !p.Cooldown.Enabled() { - return false - } +func (p *Proxy) versionInCooldown(d downloadVersion) (bool, error) { policyKey := canonicalPackagePURL(d.ecosystem, d.policyName) - if p.Cooldown.For(d.ecosystem, policyKey) <= 0 { - return false + if !p.cooldownApplies(d.ecosystem, d.policyName) { + return false, nil } var versionPURL string if d.storedName != "" { versionPURL = canonicalVersionPURL(d.ecosystem, d.storedName, d.version) if ver, err := p.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { - return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, ver.PublishedAt.Time) + return !p.cooldown().IsAllowed(d.ecosystem, policyKey, ver.PublishedAt.Time), nil } } published, ok, err := d.lookup() if err != nil { - p.Logger.Warn("cooldown: could not read publish time for download check", - "ecosystem", d.ecosystem, "name", d.policyName, "version", d.version, "error", err) - return false + return false, err } if !ok { - return false + return false, nil } if versionPURL != "" { @@ -81,7 +77,7 @@ func (p *Proxy) versionInCooldown(d downloadVersion) bool { } } - return !p.Cooldown.IsAllowed(d.ecosystem, policyKey, published) + return !p.cooldown().IsAllowed(d.ecosystem, policyKey, published), nil } // maxPublishIndexes bounds how many documents publishIndexCache keeps an @@ -99,27 +95,40 @@ type publishIndexCache struct { } type publishIndexEntry struct { - digest string - index any + digest string + index any + expires time.Time } -// publishIndex returns build's index of res, reusing the one built for the -// same document under key. res.digest identifies the document; without one, -// as when the metadata cache is off, the index is built for this call alone. -func publishIndex[V any](p *Proxy, key string, res metadataResult, build func([]byte, string) (map[string]V, error)) (map[string]V, error) { +// publishIndex reuses parsed documents by digest, or for a minute without one. +func publishIndex[V any](p *Proxy, key string, load func() metadataResult, build func(metadataResult) (V, error)) (V, error) { //nolint:ireturn // Supports timestamp maps and Helm lookups. c := &p.publishIndexes - if res.digest == "" { - return build(res.body, res.contentEncoding) + fresh := func() (V, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if e, ok := c.entries[key]; ok && !e.expires.IsZero() && p.clock().Before(e.expires) { + return e.index.(V), true + } + var zero V + return zero, false } - c.mu.Lock() - if e, ok := c.entries[key]; ok && e.digest == res.digest { - c.mu.Unlock() - return e.index.(map[string]V), nil + if index, ok := fresh(); ok { + return index, nil } - c.mu.Unlock() - - v, err, _ := c.group.Do(key+"\x00"+res.digest, func() (any, error) { - index, err := build(res.body, res.contentEncoding) + v, err, _ := c.group.Do(key, func() (any, error) { + if index, ok := fresh(); ok { + return index, nil + } + res := load() + if res.err == nil && res.digest != "" { + c.mu.Lock() + entry, ok := c.entries[key] + c.mu.Unlock() + if ok && entry.digest == res.digest { + return entry.index, nil + } + } + index, err := build(res) if err != nil { return nil, err } @@ -134,11 +143,16 @@ func publishIndex[V any](p *Proxy, key string, res metadataResult, build func([] break } } - c.entries[key] = publishIndexEntry{digest: res.digest, index: index} + entry := publishIndexEntry{digest: res.digest, index: index} + if res.digest == "" || !p.CacheMetadata { + entry.expires = p.clock().Add(time.Minute) + } + c.entries[key] = entry return index, nil }) if err != nil { - return nil, err + var zero V + return zero, err } - return v.(map[string]V), nil + return v.(V), nil } diff --git a/internal/handler/download_cooldown_cache_test.go b/internal/handler/download_cooldown_cache_test.go new file mode 100644 index 0000000..be488de --- /dev/null +++ b/internal/handler/download_cooldown_cache_test.go @@ -0,0 +1,26 @@ +package handler + +import ( + "net/http" + "testing" +) + +func TestGemDownloadSharesPublishTimeMetadata(t *testing.T) { + u, proxy, h := gemCacheSetup(t) + wantStatus(t, serveGet(t, h, gemInfoPath), http.StatusOK, "") + seedPackage(t, proxy.DB, proxy.Storage.(*mockStorage), "gem", "testgem", "1.0.0", "testgem-1.0.0.gem", "gem bytes") + wantStatus(t, serveGet(t, h, "/gems/testgem-1.0.0.gem"), http.StatusOK, "") + if n, _ := u.counts(gemVersionsPath); n != 1 { + t.Fatalf("versions API fetched %d times, want 1", n) + } +} + +func TestHexDownloadSharesPublishTimeMetadata(t *testing.T) { + u, proxy, h, _ := hexCacheSetup(t, 0) + wantStatus(t, serveGet(t, h, hexPackagePath), http.StatusOK, "") + seedPackage(t, proxy.DB, proxy.Storage.(*mockStorage), "hex", "testpkg", "1.0.0", "testpkg-1.0.0.tar", "hex bytes") + wantStatus(t, serveGet(t, h, "/tarballs/testpkg-1.0.0.tar"), http.StatusOK, "") + if n, _ := u.counts(hexAPIPath); n != 1 { + t.Fatalf("Hex API fetched %d times, want 1", n) + } +} diff --git a/internal/handler/download_cooldown_errors_test.go b/internal/handler/download_cooldown_errors_test.go new file mode 100644 index 0000000..b214db2 --- /dev/null +++ b/internal/handler/download_cooldown_errors_test.go @@ -0,0 +1,101 @@ +package handler + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" +) + +func TestDownloadCooldownLookupFailures(t *testing.T) { + for _, c := range downloadCooldownCases() { + for _, failure := range []string{"upstream error", "invalid metadata", "rate limited"} { + for _, cached := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/cached=%t", c.ecosystem, failure, cached), func(t *testing.T) { + assertDownloadLookupFailure(t, c, failure, cached) + }) + } + } + } +} + +func assertDownloadLookupFailure(t *testing.T, c downloadCooldownCase, failure string, cached bool) { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if failure == "upstream error" { + w.WriteHeader(http.StatusInternalServerError) + } + _, _ = w.Write([]byte("invalid metadata")) + })) + t.Cleanup(upstream.Close) + proxy, _, _, fetcher := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "7d"} + proxy.HTTPClient = upstream.Client() + if failure == "rate limited" { + proxy.HTTPClient = &http.Client{Transport: roundTripperFunc(func(_ *http.Request) (*http.Response, error) { + return nil, ®istryclient.HeldError{Host: "upstream.example", Until: time.Now().Add(time.Minute), RetryAfter: time.Minute} + })} + } + if cached { + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + } + w := serveGet(t, c.handler(proxy, upstream.URL), c.path(cooldownRecentVersion)) + want := http.StatusBadGateway + if failure == "rate limited" { + want = http.StatusServiceUnavailable + } + if cached { + want = http.StatusOK + if w.Body.Len() == 0 { + t.Fatal("cached artifact body is empty") + } + } + wantStatus(t, w, want, "") + if fetcher.fetchCalled { + t.Fatal("lookup failure fetched an unchecked artifact") + } +} + +func TestDownloadPublishIndexesHaveShortTTLWithoutMetadataCache(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.ecosystem != "conda" && c.ecosystem != "helm" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + var calls atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + c.metadata(w, r, map[string]time.Time{ + cooldownOldVersion: time.Now().Add(-30 * 24 * time.Hour), + cooldownRecentVersion: time.Now().Add(-time.Hour), + }) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + clock := &testClock{t: time.Now()} + proxy.now = clock.now + c.seedCachedArtifact(t, proxy, cooldownOldVersion) + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + h := c.handler(proxy, upstream.URL) + for range 3 { + wantStatus(t, serveGet(t, h, c.path(cooldownOldVersion)), http.StatusOK, "") + wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), http.StatusNotFound, "") + } + if n := calls.Load(); n != 1 { + t.Fatalf("index downloads within TTL = %d, want 1", n) + } + clock.advance(time.Minute + time.Second) + wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), http.StatusNotFound, "") + if n := calls.Load(); n != 2 { + t.Errorf("index downloads after expiry = %d, want 2", n) + } + }) + } +} diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go index 1dd44e0..a3c89ae 100644 --- a/internal/handler/download_cooldown_test.go +++ b/internal/handler/download_cooldown_test.go @@ -391,13 +391,14 @@ func TestCondaDownloadCooldownPerBuild(t *testing.T) { } // TestPublishIndexBuildsOncePerDocument checks that an index is reused for -// the same document, rebuilt for a new one, and not kept without a digest. +// the same document, rebuilt for a new one, and kept briefly without a digest. func TestPublishIndexBuildsOncePerDocument(t *testing.T) { proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true builds := 0 - build := func(body []byte, _ string) (map[string]bool, error) { + build := func(res metadataResult) (map[string]bool, error) { builds++ - return map[string]bool{string(body): true}, nil + return map[string]bool{string(res.body): true}, nil } steps := []struct { res metadataResult @@ -408,10 +409,10 @@ func TestPublishIndexBuildsOncePerDocument(t *testing.T) { {metadataResult{body: []byte("ignored"), digest: "d1"}, "a", 1}, {metadataResult{body: []byte("b"), digest: "d2"}, "b", 2}, {metadataResult{body: []byte("c")}, "c", 3}, - {metadataResult{body: []byte("c")}, "c", 4}, + {metadataResult{body: []byte("c")}, "c", 3}, } for i, step := range steps { - got, err := publishIndex(proxy, "key", step.res, build) + got, err := publishIndex(proxy, "key", func() metadataResult { return step.res }, build) if err != nil || !got[step.want] || builds != step.wantBuilds { t.Fatalf("step %d: publishIndex = %v, %v after %d builds, want %q after %d", i, got, err, builds, step.want, step.wantBuilds) } diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go index dbc0e64..373c8f4 100644 --- a/internal/handler/filename_download.go +++ b/internal/handler/filename_download.go @@ -34,8 +34,8 @@ func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d p.Logger.Debug(d.ecosystem+" download request", "name", name, "version", version, "filename", filename) - if p.withholdInCooldown(w, downloadVersion{ - ecosystem: d.ecosystem, policyName: name, storedName: name, version: version, + if p.withholdInCooldown(w, r, downloadVersion{ + ecosystem: d.ecosystem, policyName: name, storedName: name, version: version, cacheName: name, filename: filename, lookup: func() (time.Time, bool, error) { return d.publishTime(r, name, version) }, }) { return diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 009e0f5..e9ee55a 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "net/http" - "net/url" "regexp" "strings" "time" @@ -306,32 +305,8 @@ func parseGemVersions(body []byte) (publishTimes, error) { return times, nil } -// publishTime reads a gem version's created_at from the versions API. The -// download check caches the response, unlike fetchFilteredVersions, so each -// download does not cost an upstream request. version carries the platform -// the way gem filenames do, as in 1.0.0-x86_64-linux. func (h *GemHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { - versionsURL := fmt.Sprintf("%s/api/v1/versions/%s.json", h.upstreamURL, url.PathEscape(name)) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "gem", "api/v1/versions/"+name, versionsURL) - if err != nil { - return time.Time{}, false, err - } - var versions []gemVersion - if err := json.Unmarshal(body, &versions); err != nil { - return time.Time{}, false, err - } - for _, v := range versions { - number := v.Number - if v.Platform != "" && v.Platform != "ruby" { - number += "-" + v.Platform - } - if number != version { - continue - } - createdAt, err := time.Parse(time.RFC3339, v.CreatedAt) - return createdAt, err == nil, nil - } - return time.Time{}, false, nil + return h.proxy.lookupPublishTime(r.Context(), h.timedIndex(name), version) } // proxyCached forwards a metadata request with caching. diff --git a/internal/handler/helm.go b/internal/handler/helm.go index cbbf8b0..661ca98 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -120,8 +120,22 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } - if h.withholdChart(w, r, repository, upstreamURL, digest) { - return + var charts *helmChartLookup + if h.proxy.cooldownEnabled() { + var err error + charts, err = h.downloadChartLookup(r, repository, upstreamURL) + if err != nil { + cached, cacheErr := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename) + if cacheErr == nil && cached != nil { + h.serveChart(w, r, repository, digest, filename, cached) + } else { + h.proxy.writeCooldownLookupError(w, helmMetadataEcosystem, repository, err) + } + return + } + if h.withholdChart(w, charts, digest, filename) { + return + } } cached, err := h.proxy.GetCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename) @@ -135,13 +149,17 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } - index := h.fetchIndex(r.Context(), repository, upstreamURL) - if index.err != nil { - h.serveIndexError(w, index.err) - return + var downloadURL string + if charts == nil { + index := h.fetchIndex(r.Context(), repository, upstreamURL) + if index.err != nil { + h.serveIndexError(w, index.err) + return + } + downloadURL, err = h.findChartDownload(repository, upstreamURL, index, digest, filename) + } else { + downloadURL, err = h.findChartURL(charts, digest, filename) } - - downloadURL, err := h.findChartDownload(repository, upstreamURL, index, digest, filename) if err != nil { if errors.Is(err, errHelmChartNotFound) { http.NotFound(w, r) @@ -269,6 +287,10 @@ func (h *HelmHandler) findChartDownload(repository, upstreamURL string, index me if err != nil { return "", err } + return h.findChartURL(charts, digest, filename) +} + +func (h *HelmHandler) findChartURL(charts *helmChartLookup, digest, filename string) (string, error) { for _, chart := range charts.byDigest[digest] { if h.chartOnCooldown(chart.name, chart.created) { continue @@ -380,70 +402,35 @@ func (h *HelmHandler) buildChartLookup(upstreamURL string, body []byte) (*helmCh return lookup, nil } -// withholdChart checks a chart against the cooldown before it is served. -// findChartDownload already leaves out charts in cooldown, but a cached chart -// may have been fetched under an earlier policy. Charts are cached under the -// repository and digest rather than the chart name the policy is keyed by, so -// the name and publish time come from the index, indexed once per document -// (see publishIndex). -func (h *HelmHandler) withholdChart(w http.ResponseWriter, r *http.Request, repository, upstreamURL, digest string) bool { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - return false - } - res := h.fetchIndex(r, repository, upstreamURL) - if res.err != nil { - h.proxy.Logger.Warn("cooldown: could not fetch Helm index for download check", - "repository", repository, "digest", digest, "error", res.err) - return false - } - index, err := publishIndex(h.proxy, "helm\x00"+h.indexCacheKey(repository, upstreamURL), res, - func(body []byte, _ string) (map[string]helmChartTime, error) { - return h.indexChartTimes(upstreamURL, body) - }) - if err != nil { - h.proxy.Logger.Warn("cooldown: could not read Helm index for download check", - "repository", repository, "digest", digest, "error", err) - return false - } - chart, ok := index[digest] - if !ok { - return false - } - return h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: helmMetadataEcosystem, policyName: chart.name, version: digest, - lookup: func() (time.Time, bool, error) { return chart.created, !chart.created.IsZero(), nil }, +func (h *HelmHandler) downloadChartLookup(r *http.Request, repository, upstreamURL string) (*helmChartLookup, error) { + key := "helm\x00" + h.indexCacheKey(repository, upstreamURL) + return publishIndex(h.proxy, key, func() metadataResult { + return h.fetchIndex(r.Context(), repository, upstreamURL) + }, func(res metadataResult) (*helmChartLookup, error) { + if res.err != nil { + return nil, res.err + } + return h.chartLookup(repository, upstreamURL, res) }) } -// helmChartTime is the chart a release digest belongs to and when it was -// created. -type helmChartTime struct { - name string - created time.Time -} - -// indexChartTimes maps each release digest in an index to its chart. -func (h *HelmHandler) indexChartTimes(upstreamURL string, body []byte) (map[string]helmChartTime, error) { - _, entries, err := parseHelmIndex(body) - if err != nil { - return nil, err - } - index := make(map[string]helmChartTime) - for i := 0; i < len(entries.Content); i += 2 { - chartName := entries.Content[i].Value - releases := entries.Content[i+1] - if releases.Kind != yaml.SequenceNode { - continue - } - for _, release := range releases.Content { - chart, err := h.parseChartRelease(chartName, upstreamURL, release) - if err != nil { +func (h *HelmHandler) withholdChart(w http.ResponseWriter, charts *helmChartLookup, digest, filename string) bool { + withheld := false + for _, chart := range charts.byDigest[digest] { + for _, download := range chart.downloads { + if download.filename != filename { continue } - index[chart.digest] = helmChartTime{name: chartName, created: chart.created} + if !h.chartOnCooldown(chart.name, chart.created) { + return false + } + withheld = true } } - return index, nil + if withheld { + http.Error(w, ErrVersionNotFound.Error(), http.StatusNotFound) + } + return withheld } func (h *HelmHandler) chartOnCooldown(chartName string, created time.Time) bool { diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 6836dad..ceaf587 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "net/http" - "net/url" "strings" "time" @@ -243,27 +242,8 @@ func parseHexReleases(body []byte) (publishTimes, error) { return times, nil } -// publishTime reads a release's inserted_at from the Hex HTTP API. The -// download check caches the response, unlike fetchFilteredVersions, so each -// download does not cost an upstream request. func (h *HexHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { - apiURL := fmt.Sprintf("%s/api/packages/%s", h.apiURL, url.PathEscape(name)) - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "hex", "api/packages/"+name, apiURL) - if err != nil { - return time.Time{}, false, err - } - var pkg hexPackageAPI - if err := json.Unmarshal(body, &pkg); err != nil { - return time.Time{}, false, err - } - for _, release := range pkg.Releases { - if release.Version != version { - continue - } - insertedAt, err := time.Parse(time.RFC3339Nano, release.InsertedAt) - return insertedAt, err == nil, nil - } - return time.Time{}, false, nil + return h.proxy.lookupPublishTime(r.Context(), h.timedPackage(name), version) } // signedPayload decompresses a gzipped Signed message, with the metadata diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 22540f1..001b0c6 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -94,8 +94,9 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Debug("pub download request", "name", name, "version", version) - if h.proxy.withholdInCooldown(w, downloadVersion{ - ecosystem: "pub", policyName: name, storedName: name, version: version, + h.discardVersionPage(name, version, filename) + if h.proxy.withholdInCooldown(w, r, downloadVersion{ + ecosystem: "pub", policyName: name, storedName: name, version: version, cacheName: name, filename: filename, lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, }) { return @@ -103,7 +104,6 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { downloadURL := fmt.Sprintf("%s/packages/%s/versions/%s%s", h.upstreamURL, url.PathEscape(name), url.PathEscape(version), pubArchiveExt) - h.discardVersionPage(name, version, filename) result, err := h.proxy.GetOrFetchArtifactFromURL( r.Context(), "pub", name, version, filename, downloadURL, ) From 61ac273862069436dac9412c366aa6dff9cec895 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 5 Nov 2026 20:00:19 +0000 Subject: [PATCH 085/198] Check primary metadata before allowing untimed downloads --- internal/handler/cargo.go | 5 +- internal/handler/conda.go | 5 +- internal/handler/cooldown_times.go | 28 +++--- internal/handler/download_cooldown.go | 9 +- .../handler/download_cooldown_cache_test.go | 89 +++++++++++++++++++ internal/handler/download_cooldown_test.go | 15 ++++ internal/handler/filename_download.go | 4 +- internal/handler/gem.go | 4 +- internal/handler/hex.go | 4 +- internal/handler/pub.go | 5 +- 10 files changed, 144 insertions(+), 24 deletions(-) diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index fa93a74..aca7416 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -246,7 +246,10 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { if h.proxy.withholdInCooldown(w, r, downloadVersion{ ecosystem: "cargo", policyName: name, storedName: name, version: version, cacheName: name, filename: filename, - lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, + lookup: func() (time.Time, bool, bool, error) { + published, ok, err := h.publishTime(r, name, version) + return published, ok, false, err + }, }) { return } diff --git a/internal/handler/conda.go b/internal/handler/conda.go index de6ae8a..1c14980 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -145,7 +145,10 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { cacheFilename := arch + "/" + filename if h.proxy.withholdInCooldown(w, r, downloadVersion{ ecosystem: "conda", policyName: name, version: version, cacheName: packageName, filename: cacheFilename, - lookup: func() (time.Time, bool, error) { return h.publishTime(r, channel, arch, filename) }, + lookup: func() (time.Time, bool, bool, error) { + published, ok, err := h.publishTime(r, channel, arch, filename) + return published, ok, false, err + }, }) { return } diff --git a/internal/handler/cooldown_times.go b/internal/handler/cooldown_times.go index b7a0516..b765f9a 100644 --- a/internal/handler/cooldown_times.go +++ b/internal/handler/cooldown_times.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "slices" "strings" "time" ) @@ -16,19 +17,24 @@ import ( // through. type publishTimes map[string]time.Time -func (p *Proxy) lookupPublishTime(ctx context.Context, t timedMetadata, version string) (time.Time, bool, error) { - var times publishTimes - validate := func(body []byte) error { +func (p *Proxy) lookupPublishTime(ctx context.Context, t timedMetadata, version string, listed func(metadataResult) ([]string, error)) (time.Time, bool, bool, error) { + var versions []string + docs, err := p.fetchTimedMetadata(ctx, t, func(doc metadataResult) ([]string, error) { var err error - times, err = t.parseTimes(body) - return err + versions, err = listed(doc) + return versions, err + }) + switch { + case docs.doc.err != nil: + return time.Time{}, false, false, docs.doc.err + case docs.times.err != nil: + return time.Time{}, false, false, docs.times.err + case err != nil: + return time.Time{}, false, false, err } - res := p.lookupMetadata(ctx, t.ecosystem, t.timesKey, t.timesURL, t.timesAccept, "", validate) - if res.err != nil { - return time.Time{}, false, res.err - } - published, ok := times[version] - return published, ok && !published.IsZero(), nil + published, ok := docs.published[version] + withholdUnknown := !ok && docs.withholdUnknown && slices.Contains(versions, version) + return published, ok && !published.IsZero(), withholdUnknown, nil } // timedMetadata describes metadata whose publish times live in a second diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index 47d8b70..c491ef3 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -9,8 +9,9 @@ import ( ) // publishTimeLookup reads a version's publish time from upstream metadata. ok -// is false when the metadata has no usable time for the version. -type publishTimeLookup func() (published time.Time, ok bool, err error) +// is false when the metadata has no usable time for the version. withholdUnknown +// marks a listed version missing from stale publish-time metadata. +type publishTimeLookup func() (published time.Time, ok, withholdUnknown bool, err error) // downloadVersion names the version an artifact download is for, so its // cooldown can be checked before the artifact is served. @@ -62,12 +63,12 @@ func (p *Proxy) versionInCooldown(d downloadVersion) (bool, error) { } } - published, ok, err := d.lookup() + published, ok, withholdUnknown, err := d.lookup() if err != nil { return false, err } if !ok { - return false, nil + return withholdUnknown, nil } if versionPURL != "" { diff --git a/internal/handler/download_cooldown_cache_test.go b/internal/handler/download_cooldown_cache_test.go index be488de..d51b862 100644 --- a/internal/handler/download_cooldown_cache_test.go +++ b/internal/handler/download_cooldown_cache_test.go @@ -1,8 +1,12 @@ package handler import ( + "errors" + "fmt" "net/http" + "net/http/httptest" "testing" + "time" ) func TestGemDownloadSharesPublishTimeMetadata(t *testing.T) { @@ -15,6 +19,91 @@ func TestGemDownloadSharesPublishTimeMetadata(t *testing.T) { } } +type downloadPublishTimeCase struct { + name string + down bool + absent bool + untimed bool + refresh bool + wantStatus int +} + +func TestDownloadCooldownRefreshesMissingPublishTimes(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.ecosystem != "gem" && c.ecosystem != "hex" { + continue + } + for _, scenario := range []downloadPublishTimeCase{ + {name: "refreshed times", refresh: true, wantStatus: http.StatusNotFound}, + {name: "stale times and listed version", down: true, wantStatus: http.StatusNotFound}, + {name: "stale times and absent version", down: true, absent: true, wantStatus: http.StatusOK}, + {name: "fresh times without version", wantStatus: http.StatusOK}, + {name: "stale times with untimed version", down: true, untimed: true, wantStatus: http.StatusOK}, + } { + for _, cached := range []bool{false, true} { + if !cached && scenario.wantStatus == http.StatusOK { + continue + } + t.Run(fmt.Sprintf("%s/%s/cached=%t", c.ecosystem, scenario.name, cached), func(t *testing.T) { + assertDownloadPublishTime(t, c, scenario, cached) + }) + } + } + } +} + +func assertDownloadPublishTime(t *testing.T, c downloadCooldownCase, scenario downloadPublishTimeCase, cached bool) { + t.Helper() + u := newCooldownCountingUpstream(t) + proxy := cooldownCacheProxy(t, u) + fetcher := proxy.Fetcher.(*mockFetcher) + fetcher.fetchErr = errors.New("unchecked artifact download") + h := c.handler(proxy, u.URL) + docPath, timesPath := "/info/demo", "/api/v1/versions/demo.json" + if c.ecosystem == "hex" { + docPath, timesPath = "/packages/demo", "/api/packages/demo" + } + oldTimes := map[string]time.Time{cooldownOldVersion: time.Now().Add(-30 * 24 * time.Hour)} + if scenario.untimed { + oldTimes[cooldownRecentVersion] = time.Time{} + } + setDownloadMetadata(t, u, c, docPath, oldTimes) + setDownloadMetadata(t, u, c, timesPath, oldTimes) + wantStatus(t, serveGet(t, h, docPath), http.StatusOK, "") + + newTimes := map[string]time.Time{ + cooldownOldVersion: oldTimes[cooldownOldVersion], + cooldownUntimed: time.Now().Add(-time.Minute), + } + if !scenario.absent { + newTimes[cooldownRecentVersion] = time.Now().Add(-time.Hour) + } + setDownloadMetadata(t, u, c, docPath, newTimes) + expireCachedMetadata(t, proxy, c.ecosystem, docPath[1:]) + if scenario.refresh { + setDownloadMetadata(t, u, c, timesPath, newTimes) + } + u.setDown(timesPath, scenario.down) + if cached { + c.seedCachedArtifact(t, proxy, cooldownRecentVersion) + } + wantStatus(t, serveGet(t, h, c.path(cooldownRecentVersion)), scenario.wantStatus, "") + if fetcher.fetchCalled { + t.Error("fetched an unchecked artifact") + } + if n, _ := u.counts(timesPath); n != 2 { + t.Errorf("publish times fetched %d times, want 2", n) + } +} + +func setDownloadMetadata(t *testing.T, u *cooldownCountingUpstream, c downloadCooldownCase, path string, times map[string]time.Time) { + t.Helper() + w := httptest.NewRecorder() + c.metadata(w, httptest.NewRequest(http.MethodGet, path, nil), times) + wantStatus(t, w, http.StatusOK, "") + u.set(path, w.Body.Bytes()) +} + func TestHexDownloadSharesPublishTimeMetadata(t *testing.T) { u, proxy, h, _ := hexCacheSetup(t, 0) wantStatus(t, serveGet(t, h, hexPackagePath), http.StatusOK, "") diff --git a/internal/handler/download_cooldown_test.go b/internal/handler/download_cooldown_test.go index a3c89ae..1fd915f 100644 --- a/internal/handler/download_cooldown_test.go +++ b/internal/handler/download_cooldown_test.go @@ -137,6 +137,13 @@ func cargoCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[str } func gemCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path == "/info/demo" { + _, _ = fmt.Fprintln(w, "---") + for version := range times { + _, _ = fmt.Fprintf(w, "%s-x86_64-linux |checksum:a\n", version) + } + return + } if r.URL.Path != "/api/v1/versions/demo.json" { http.NotFound(w, r) return @@ -153,6 +160,14 @@ func gemCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[strin } func hexCooldownMetadata(w http.ResponseWriter, r *http.Request, times map[string]time.Time) { + if r.URL.Path == "/packages/demo" { + var versions []string + for version := range times { + versions = append(versions, version) + } + _, _ = w.Write(buildHexSigned(buildHexPackage("demo", versions))) + return + } if r.URL.Path != "/api/packages/demo" { http.NotFound(w, r) return diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go index 373c8f4..c37d07d 100644 --- a/internal/handler/filename_download.go +++ b/internal/handler/filename_download.go @@ -15,7 +15,7 @@ type filenameDownload struct { parse func(string) (name, version string) // publishTime reads a version's publish time from upstream metadata for // the download-time cooldown check. - publishTime func(r *http.Request, name, version string) (time.Time, bool, error) + publishTime func(r *http.Request, name, version string) (time.Time, bool, bool, error) } func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d filenameDownload) { @@ -36,7 +36,7 @@ func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d if p.withholdInCooldown(w, r, downloadVersion{ ecosystem: d.ecosystem, policyName: name, storedName: name, version: version, cacheName: name, filename: filename, - lookup: func() (time.Time, bool, error) { return d.publishTime(r, name, version) }, + lookup: func() (time.Time, bool, bool, error) { return d.publishTime(r, name, version) }, }) { return } diff --git a/internal/handler/gem.go b/internal/handler/gem.go index e9ee55a..fa7c24b 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -305,8 +305,8 @@ func parseGemVersions(body []byte) (publishTimes, error) { return times, nil } -func (h *GemHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { - return h.proxy.lookupPublishTime(r.Context(), h.timedIndex(name), version) +func (h *GemHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, bool, error) { + return h.proxy.lookupPublishTime(r.Context(), h.timedIndex(name), version, h.indexVersions) } // proxyCached forwards a metadata request with caching. diff --git a/internal/handler/hex.go b/internal/handler/hex.go index ceaf587..d394c4d 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -242,8 +242,8 @@ func parseHexReleases(body []byte) (publishTimes, error) { return times, nil } -func (h *HexHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, error) { - return h.proxy.lookupPublishTime(r.Context(), h.timedPackage(name), version) +func (h *HexHandler) publishTime(r *http.Request, name, version string) (time.Time, bool, bool, error) { + return h.proxy.lookupPublishTime(r.Context(), h.timedPackage(name), version, h.packageVersions) } // signedPayload decompresses a gzipped Signed message, with the metadata diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 001b0c6..31e00d8 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -97,7 +97,10 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.discardVersionPage(name, version, filename) if h.proxy.withholdInCooldown(w, r, downloadVersion{ ecosystem: "pub", policyName: name, storedName: name, version: version, cacheName: name, filename: filename, - lookup: func() (time.Time, bool, error) { return h.publishTime(r, name, version) }, + lookup: func() (time.Time, bool, bool, error) { + published, ok, err := h.publishTime(r, name, version) + return published, ok, false, err + }, }) { return } From 79be33049a05b9f964c203326d9dfdc27a07090e Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Thu, 5 Nov 2026 18:03:21 +0000 Subject: [PATCH 086/198] Add cooldown profiles selected by registry URL --- README.md | 2 + config.example.yaml | 7 + docs/configuration.md | 31 +++ internal/config/config.go | 7 + internal/config/cooldown_profiles.go | 64 ++++++ internal/config/cooldown_profiles_test.go | 81 +++++++ internal/handler/handler.go | 61 +++--- internal/handler/profile.go | 30 +++ internal/handler/profile_test.go | 113 ++++++++++ internal/server/cooldown_profiles.go | 26 +++ internal/server/cooldown_profiles_test.go | 253 ++++++++++++++++++++++ internal/server/middleware.go | 4 + internal/server/server.go | 67 +++--- 13 files changed, 688 insertions(+), 58 deletions(-) create mode 100644 internal/config/cooldown_profiles.go create mode 100644 internal/config/cooldown_profiles_test.go create mode 100644 internal/handler/profile.go create mode 100644 internal/handler/profile_test.go create mode 100644 internal/server/cooldown_profiles.go create mode 100644 internal/server/cooldown_profiles_test.go diff --git a/README.md b/README.md index 2a08eaa..556e676 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ A 3-day cooldown means that when `lodash` publishes version `4.18.0`, your build Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default and carve out exceptions for packages where you need faster updates. See [docs/configuration.md](docs/configuration.md) for the full config reference. +One instance can offer several cooldown policies through [named profiles](docs/configuration.md#cooldown-profiles). Configure `cooldown.profiles` and point clients at paths such as `/cooldown/3d/npm/` or `/cooldown/7d/cargo/`. Profiles share the package cache; existing URLs keep using the global policy. + ## Artifact Scanning Cooldown only looks at a version's publish timestamp — it never inspects the actual bytes. Artifact scanning closes that gap: when enabled, every artifact is staged into storage and scanned by one or more external services (trivy, ClamAV, Wiz, or anything else that speaks a small HTTP/JSON contract) before it's committed to the cache and served to clients. diff --git a/config.example.yaml b/config.example.yaml index e43e434..c8ab11b 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -482,6 +482,13 @@ health: # to spot malicious releases before they're pulled into projects. # Supported durations: "7d" (days), "48h" (hours), "30m" (minutes), "0" (disabled) cooldown: + # Independent policies selected through /cooldown/{name}/{registry}/. + # profiles: + # 3d: + # default: "3d" + # 7d: + # default: "7d" + # Global default cooldown for all ecosystems # default: "3d" diff --git a/docs/configuration.md b/docs/configuration.md index 38765e0..63cc175 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1148,6 +1148,37 @@ Uploads are off by default, so a client that can reach the proxy cannot overwrit `max_age` and `max_size` are independent and can be combined. When both are set, age-based eviction runs first, then size-based eviction trims remaining entries oldest-first. +## Cooldown profiles + +Set `cooldown.profiles` to offer several policies from one instance. Clients select a profile through their package manager's registry URL: + +```yaml +cooldown: + default: "0" + profiles: + 3d: + default: "3d" + 7d: + default: "7d" + packages: + "pkg:npm/@example/tool": "0" +``` + +For the `3d` profile, use these paths after your proxy's base URL: + +| Client | Path | +|--------|------| +| npm registry | `/cooldown/3d/npm/` | +| Go `GOPROXY` | `/cooldown/3d/go` | +| Cargo registry | `sparse+https://your-proxy.example/cooldown/3d/cargo/` | +| Bundler RubyGems mirror | `/cooldown/3d/gem` | + +The prefix works with all package protocol routes. Cooldown has an effect only for [registries with cooldown support](../README.md#supported-registries). Each profile defines its own `default`, `ecosystems`, `packages` and `package_patterns`; it does not inherit the global cooldown overrides. Existing unprefixed URLs, the UI, APIs and background mirror jobs use the global policy. + +Profile names contain lowercase letters, digits, hyphens and underscores, and start with a letter or digit. Profiles must be configured before startup; an unknown profile returns 404. Nested profiles are rejected. Profile paths expose package protocols only, so `/cooldown/3d/ui/` and `/cooldown/3d/api/` return 404. + +Database records, artifacts and upstream metadata are shared. Generated download URLs retain the profile prefix, and filtered metadata and its validators are separated by profile. Changing a registry URL does not rewrite artifact URLs already recorded in a lockfile; regenerate those URLs if the selected policy must cover the whole installation. + ## Cooldown The cooldown feature hides package versions published too recently, giving the community time to spot malicious releases before they reach your projects. When a version is within its cooldown period, it's stripped from metadata responses so package managers won't install it. diff --git a/internal/config/config.go b/internal/config/config.go index 68c578f..3b5c6cb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -237,6 +237,9 @@ type Config struct { // CooldownConfig configures version cooldown periods. // Versions published more recently than the cooldown are hidden from metadata responses. type CooldownConfig struct { + // Profiles are independent policies exposed at /cooldown/{name}/. + Profiles map[string]CooldownConfig `json:"profiles,omitempty" yaml:"profiles,omitempty"` + // Default is the global default cooldown (e.g., "3d", "48h", "0" to disable). Default string `json:"default" yaml:"default"` @@ -1665,6 +1668,10 @@ func (c *Config) validateCacheArtifacts() error { } func (c *Config) validateComponents() error { + if err := c.Cooldown.ValidateProfiles(); err != nil { + return err + } + if err := validateMetadataRewriteCacheSize(c.MetadataRewriteCacheSize); err != nil { return err } diff --git a/internal/config/cooldown_profiles.go b/internal/config/cooldown_profiles.go new file mode 100644 index 0000000..225dd20 --- /dev/null +++ b/internal/config/cooldown_profiles.go @@ -0,0 +1,64 @@ +package config + +import ( + "fmt" + "maps" + "regexp" + "slices" + + "github.com/git-pkgs/cooldown" +) + +var cooldownProfileName = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`) + +// Policy builds the policy for this URL's cooldown settings. +func (c *CooldownConfig) Policy() (*cooldown.Policy, error) { + return cooldown.NewPolicy(&cooldown.Config{ + Default: c.Default, Ecosystems: c.Ecosystems, Packages: c.NormalizedPackages(), + }, c.PackagePatterns) +} + +// ValidateProfiles checks names and policies before they become routes. +func (c *CooldownConfig) ValidateProfiles() error { + for _, name := range slices.Sorted(maps.Keys(c.Profiles)) { + profile := c.Profiles[name] + if !cooldownProfileName.MatchString(name) { + return fmt.Errorf("invalid cooldown profile name %q: use lowercase letters, digits, hyphens or underscores", name) + } + if len(profile.Profiles) > 0 { + return fmt.Errorf("cooldown.profiles.%s cannot contain profiles", name) + } + if err := profile.validateProfileDurations(); err != nil { + return fmt.Errorf("cooldown.profiles.%s: %w", name, err) + } + if _, err := profile.Policy(); err != nil { + return fmt.Errorf("cooldown.profiles.%s: %w", name, err) + } + } + return nil +} + +func (c *CooldownConfig) validateProfileDurations() error { + values := map[string]string{"default": c.Default} + for group, overrides := range map[string]map[string]string{ + "ecosystems": c.Ecosystems, "packages": c.Packages, "package_patterns": c.PackagePatterns, + } { + for key, value := range overrides { + values[group+"."+key] = value + } + } + for _, key := range slices.Sorted(maps.Keys(values)) { + value := values[key] + if key == "default" && value == "" { + continue + } + duration, err := cooldown.ParseDuration(value) + if err != nil { + return fmt.Errorf("invalid %s: %w", key, err) + } + if duration < 0 { + return fmt.Errorf("invalid %s: duration must be non-negative", key) + } + } + return nil +} diff --git a/internal/config/cooldown_profiles_test.go b/internal/config/cooldown_profiles_test.go new file mode 100644 index 0000000..6f10362 --- /dev/null +++ b/internal/config/cooldown_profiles_test.go @@ -0,0 +1,81 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestLoadCooldownProfiles(t *testing.T) { + filename := filepath.Join(t.TempDir(), "config.yaml") + data := `cooldown: + default: 7d + ecosystems: + npm: "0" + profiles: + 3d: + default: 3d + packages: + "pkg:pypi/Django": "0" + strict: + default: 14d + package_patterns: + "pkg:npm/@example/*": 1d +` + if err := os.WriteFile(filename, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(filename) + if err != nil { + t.Fatal(err) + } + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + profile := cfg.Cooldown.Profiles["3d"] + policy, err := profile.Policy() + if err != nil { + t.Fatal(err) + } + if got := policy.For("npm", "pkg:npm/demo"); got != 72*time.Hour { + t.Fatalf("profile inherited global ecosystem override: %v", got) + } + if got := policy.For("pypi", "pkg:pypi/django"); got != 0 { + t.Fatalf("canonical package override = %v", got) + } + strict := cfg.Cooldown.Profiles["strict"] + policy, err = strict.Policy() + if err != nil { + t.Fatal(err) + } + if got := policy.For("npm", "pkg:npm/%40example/tool"); got != 24*time.Hour { + t.Fatalf("profile pattern = %v", got) + } +} + +func TestLoadRejectsInvalidCooldownProfiles(t *testing.T) { + for _, tc := range []struct{ name, profile, want string }{ + {"path", `"a/b": {default: 3d}`, "profile name"}, + {"uppercase", `Strict: {default: 3d}`, "profile name"}, + {"empty", `"": {default: 3d}`, "profile name"}, + {"duration", `strict: {default: nope}`, "cooldown.profiles.strict"}, + {"pattern", `strict: {package_patterns: {"pkg:npm/[": 3d}}`, "cooldown.profiles.strict"}, + {"nested", `strict: {profiles: {nested: {default: 3d}}}`, "cannot contain profiles"}, + } { + t.Run(tc.name, func(t *testing.T) { + filename := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(filename, []byte("cooldown:\n profiles:\n "+tc.profile+"\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(filename) + if err == nil { + err = cfg.Validate() + } + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("Load error = %v, want %q", err, tc.want) + } + }) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 125b117..d188594 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -304,6 +304,7 @@ type Proxy struct { // cooldownWatch holds the rewrites recording which versions cooldown // holds back; see cooldown_expiry.go. cooldownWatch cooldownWatch + shared *Proxy } // cooldownEcosystems are the ecosystems whose handlers apply cooldown. Others @@ -946,18 +947,19 @@ type inflightFetch struct { // proxy_cache_misses_total and upstream fetch observations is what coalescing // saved. func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { - p.fetchMu.Lock() - if p.inFlight == nil { - p.inFlight = make(map[string]*inflightFetch) + shared := p.sharedProxy() + shared.fetchMu.Lock() + if shared.inFlight == nil { + shared.inFlight = make(map[string]*inflightFetch) } - f, joined := p.inFlight[key] + f, joined := shared.inFlight[key] if joined { f.waiters++ } else { f = &inflightFetch{done: make(chan struct{})} - p.inFlight[key] = f + shared.inFlight[key] = f } - p.fetchMu.Unlock() + shared.fetchMu.Unlock() if !joined { if streamsCacheMiss(ctx) { @@ -985,28 +987,30 @@ func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (a // leaveFetch withdraws a waiter that gave up. Once the fetch has finished the // waiter was counted as holding any unrecorded object, so it releases it. func (p *Proxy) leaveFetch(f *inflightFetch) { - p.fetchMu.Lock() + shared := p.sharedProxy() + shared.fetchMu.Lock() select { case <-f.done: - p.fetchMu.Unlock() + shared.fetchMu.Unlock() if f.unrecorded != nil { f.unrecorded.release() } default: f.waiters-- - p.fetchMu.Unlock() + shared.fetchMu.Unlock() } } // runSharedFetch performs the fetch that joined callers are waiting on. It is // never abandoned early, and always releases the key and wakes the waiters. func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { + shared := p.sharedProxy() // Set before running so a panicking commit leaves waiters with an error // rather than a zero-valued artifact. f.err = errSharedFetchAbandoned defer func() { - p.fetchMu.Lock() - delete(p.inFlight, key) + shared.fetchMu.Lock() + delete(shared.inFlight, key) if f.unrecorded != nil { // Hand each waiter a hold, then drop the one kept while the // waiters were still unknown. @@ -1014,7 +1018,7 @@ func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch defer f.unrecorded.release() } close(f.done) - p.fetchMu.Unlock() + shared.fetchMu.Unlock() }() // A caller checks the cache before reaching here, so a fetch that finished @@ -1162,12 +1166,13 @@ func upstreamFetchDetached(ctx context.Context) bool { const maxDetachedFetches = 64 func (p *Proxy) acquireDetachSlot() bool { - p.fetchMu.Lock() - if p.detachSlots == nil { - p.detachSlots = make(chan struct{}, maxDetachedFetches) + shared := p.sharedProxy() + shared.fetchMu.Lock() + if shared.detachSlots == nil { + shared.detachSlots = make(chan struct{}, maxDetachedFetches) } - slots := p.detachSlots - p.fetchMu.Unlock() + slots := shared.detachSlots + shared.fetchMu.Unlock() select { case slots <- struct{}{}: @@ -1178,7 +1183,8 @@ func (p *Proxy) acquireDetachSlot() bool { } func (p *Proxy) releaseDetachSlot() { - <-p.detachSlots + shared := p.sharedProxy() + <-shared.detachSlots } // fetchedArtifact is what a shared fetch hands its callers: metadata and a @@ -1877,18 +1883,19 @@ func metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncodin // fail when the first of them disconnects. It stays bounded by the HTTP // client's timeout. Waiters still leave when their own context ends. func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(context.Context) metadataResult) (metadataResult, bool) { - p.metaMu.Lock() - if p.inFlightMeta == nil { - p.inFlightMeta = make(map[string]*inflightMetadata) + shared := p.sharedProxy() + shared.metaMu.Lock() + if shared.inFlightMeta == nil { + shared.inFlightMeta = make(map[string]*inflightMetadata) } - f, joined := p.inFlightMeta[key] + f, joined := shared.inFlightMeta[key] if joined { f.waiters++ } else { f = &inflightMetadata{done: make(chan struct{})} - p.inFlightMeta[key] = f + shared.inFlightMeta[key] = f } - p.metaMu.Unlock() + shared.metaMu.Unlock() if joined { select { @@ -1903,9 +1910,9 @@ func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(con // rather than an empty body. f.res = metadataResult{err: errSharedFetchAbandoned} defer func() { - p.metaMu.Lock() - delete(p.inFlightMeta, key) - p.metaMu.Unlock() + shared.metaMu.Lock() + delete(shared.inFlightMeta, key) + shared.metaMu.Unlock() close(f.done) }() f.res = fetch(context.WithoutCancel(ctx)) diff --git a/internal/handler/profile.go b/internal/handler/profile.go new file mode 100644 index 0000000..233e7bb --- /dev/null +++ b/internal/handler/profile.go @@ -0,0 +1,30 @@ +package handler + +import "time" + +// WithCooldown creates a policy view sharing storage, fetches and rewrite space. +// Configure the original proxy before constructing its views. +func (p *Proxy) WithCooldown(policy CooldownPolicy, shortest time.Duration) *Proxy { + return &Proxy{ + DB: p.DB, Storage: p.Storage, Fetcher: p.Fetcher, Resolver: p.Resolver, + RouteFor: p.RouteFor, Logger: p.Logger, + Cooldown: policy, ShortestCooldown: shortest, Denylist: p.Denylist, + CacheMetadata: p.CacheMetadata, MetadataTTL: p.MetadataTTL, + MetadataMaxSize: p.MetadataMaxSize, MetadataStreamMaxSize: p.MetadataStreamMaxSize, + GradleReadOnly: p.GradleReadOnly, HostedEnabled: p.HostedEnabled, + GradleMaxUploadSize: p.GradleMaxUploadSize, NPMFullMetadata: p.NPMFullMetadata, + DirectServe: p.DirectServe, DirectServeTTL: p.DirectServeTTL, + DirectServeBaseURL: p.DirectServeBaseURL, HTTPClient: p.HTTPClient, + RateLimits: p.RateLimits, HostedNPMScopes: p.HostedNPMScopes, + HostedComposerVendors: p.HostedComposerVendors, StreamArtifacts: p.StreamArtifacts, + Scanners: p.Scanners, ScanSigningKey: p.ScanSigningKey, ScanFetchBaseURL: p.ScanFetchBaseURL, + rewrites: p.rewrites, now: p.now, started: p.started, shared: p.sharedProxy(), + } +} + +func (p *Proxy) sharedProxy() *Proxy { + if p.shared != nil { + return p.shared + } + return p +} diff --git a/internal/handler/profile_test.go b/internal/handler/profile_test.go new file mode 100644 index 0000000..6377062 --- /dev/null +++ b/internal/handler/profile_test.go @@ -0,0 +1,113 @@ +package handler + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +func profilePolicy(t *testing.T, duration string) *cooldown.Policy { + t.Helper() + policy, err := cooldown.NewPolicy(&cooldown.Config{Default: duration}, nil) + if err != nil { + t.Fatal(err) + } + return policy +} + +func TestProfilesCoalesceUncachedMetadata(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + p := metadataTestProxy(t, u) + views := []*Proxy{p.WithCooldown(profilePolicy(t, "3d"), 72*time.Hour), p.WithCooldown(profilePolicy(t, "7d"), 168*time.Hour)} + first := startMetadataCalls(views[0], context.Background(), 1, u.URL, contentTypeJSON) + <-u.entered + second := startMetadataCalls(views[1], context.Background(), 1, u.URL, contentTypeJSON) + key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) + var release sync.Once + t.Cleanup(func() { release.Do(func() { close(u.release) }) }) + waitForMetadataWaiters(t, p, key, 1) + release.Do(func() { close(u.release) }) + for _, calls := range [][]chan metadataCall{first, second} { + result := <-calls[0] + if result.err != nil || !strings.Contains(string(result.body), "accept") { + t.Fatalf("metadata = %s, error %v", result.body, result.err) + } + } + if n := u.calls.Load(); n != 1 { + t.Fatalf("upstream requests = %d, want 1", n) + } +} + +func TestProfilesCoalesceArtifactRequests(t *testing.T) { + proxy, db, _, _ := setupTestProxy(t) + if err := db.SetVersionPublishedAt("pkg:cargo/demo@1.0.0", "pkg:cargo/demo", time.Now().Add(-10*24*time.Hour)); err != nil { + t.Fatal(err) + } + fetcher := &countingFetcher{content: "crate bytes", delay: fetchHoldTime} + proxy.Fetcher = fetcher + views := []*Proxy{proxy.WithCooldown(profilePolicy(t, "3d"), 72*time.Hour), proxy.WithCooldown(profilePolicy(t, "7d"), 168*time.Hour)} + var wg sync.WaitGroup + start := make(chan struct{}) + for range 4 { + for _, view := range views { + wg.Go(func() { + <-start + h := NewCargoHandler(view, "http://proxy.test", "https://index.crates.io", "https://static.crates.io/crates") + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/crates/demo/1.0.0/download", nil)) + if w.Code != http.StatusOK || w.Body.String() != "crate bytes" { + t.Errorf("artifact response %d: %s", w.Code, w.Body) + } + }) + } + } + close(start) + wg.Wait() + if n := fetcher.calls.Load(); n != 1 { + t.Fatalf("upstream fetches = %d, want 1", n) + } +} + +type profileClockPolicy struct { + *cooldown.Policy + now func() time.Time +} + +func (p profileClockPolicy) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { + return p.Evaluate(ecosystem, packagePURL, publishedAt, p.now()).Allowed +} + +func TestProfileExpiryReleasesVersionsIndependently(t *testing.T) { + u, proxy, _ := gemCacheSetup(t) + clock := &testClock{t: time.Now()} + proxy.now = clock.now + proxy.SetMetadataRewriteCacheSize(1 << 20) + view := proxy.WithCooldown(profileClockPolicy{profilePolicy(t, "3d"), clock.now}, 72*time.Hour) + h := NewGemHandlerWithUpstream(view, "http://proxy.test/cooldown/3d", u.URL).Routes() + strictView := proxy.WithCooldown(profileClockPolicy{profilePolicy(t, "7d"), clock.now}, 168*time.Hour) + strict := NewGemHandlerWithUpstream(strictView, "http://proxy.test/cooldown/7d", u.URL).Routes() + first := getWith(t, h, gemInfoPath, nil) + if first.Code != http.StatusOK || first.Body.String() != gemFiltered { + t.Fatalf("initial response %d: %s", first.Code, first.Body) + } + if first := getWith(t, strict, gemInfoPath, nil); first.Code != http.StatusOK || first.Body.String() != gemFiltered { + t.Fatalf("initial strict response %d: %s", first.Code, first.Body) + } + clock.advance(4 * 24 * time.Hour) + second := getWith(t, h, gemInfoPath, nil) + if second.Code != http.StatusOK || !strings.Contains(second.Body.String(), "2.0.0 |") { + t.Fatalf("version did not leave cooldown: %d %s", second.Code, second.Body) + } + if second := getWith(t, strict, gemInfoPath, nil); second.Code != http.StatusOK || second.Body.String() != gemFiltered { + t.Fatalf("strict profile released version early: %d %s", second.Code, second.Body) + } + if n, _ := u.counts(gemInfoPath); n != 1 { + t.Errorf("expiry refetched raw metadata %d times", n) + } +} diff --git a/internal/server/cooldown_profiles.go b/internal/server/cooldown_profiles.go new file mode 100644 index 0000000..e818a01 --- /dev/null +++ b/internal/server/cooldown_profiles.go @@ -0,0 +1,26 @@ +package server + +import ( + "net/http" + "strings" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/go-chi/chi/v5" +) + +func (s *Server) mountCooldownProfiles(r chi.Router, proxy *handler.Proxy) error { + if err := s.cfg.Cooldown.ValidateProfiles(); err != nil { + return err + } + for name, cfg := range s.cfg.Cooldown.Profiles { + policy, err := cfg.Policy() + if err != nil { + return err + } + prefix := "/cooldown/" + name + profile := chi.NewRouter() + s.mountProtocolHandlersAt(profile, proxy.WithCooldown(policy, cfg.Shortest()), strings.TrimRight(s.cfg.BaseURL, "/")+prefix) + r.Mount(prefix, http.StripPrefix(prefix, profile)) + } + return nil +} diff --git a/internal/server/cooldown_profiles_test.go b/internal/server/cooldown_profiles_test.go new file mode 100644 index 0000000..ad92434 --- /dev/null +++ b/internal/server/cooldown_profiles_test.go @@ -0,0 +1,253 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" +) + +func TestCooldownProfilesThroughServerStart(t *testing.T) { + if os.Getenv("PROXY_TEST_PROFILES") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=^TestCooldownProfilesThroughServerStart$") + cmd.Env = append(os.Environ(), "PROXY_TEST_PROFILES=1") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("profile server: %v\n%s", err, output) + } + return + } + upstream, counts := profileTestUpstream(t) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + cfg := config.Default() + cfg.BaseURL = "http://" + listener.Addr().String() + cfg.Database.Path = filepath.Join(t.TempDir(), "cache.db") + cfg.Storage.URL = "file://" + t.TempDir() + cfg.CacheMetadata = true + cfg.MetadataTTL = "1h" + cfg.Upstream.AllowLoopback = true + cfg.Upstream.NPM = upstream.URL + "/npm" + cfg.Upstream.Cargo = upstream.URL + "/cargo" + cfg.Upstream.CargoDownload = upstream.URL + "/crates" + cfg.Upstream.Gem = upstream.URL + "/gem" + cfg.Upstream.Go = upstream.URL + "/go" + cfg.Cooldown = config.CooldownConfig{Profiles: map[string]config.CooldownConfig{ + "3d": {Default: "3d"}, "7d": {Default: "7d"}, + }} + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + s, err := New(cfg, slog.New(slog.NewTextHandler(io.Discard, nil)), BuildInfo{}) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- s.Start(listener) }() + client := &http.Client{Timeout: 5 * time.Second} + defer func() { + client.CloseIdleConnections() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := s.Shutdown(ctx); err != nil { + t.Error(err) + } + if err := <-done; !errors.Is(err, http.ErrServerClosed) { + t.Error(err) + } + }() + get := func(path string, headers map[string]string) (*http.Response, string) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, cfg.BaseURL+path, nil) + if err != nil { + t.Fatal(err) + } + for name, value := range headers { + req.Header.Set(name, value) + } + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return resp, string(body) + } + assertProfileVersionLists(t, get) + assertProfileRouting(t, cfg.BaseURL, get, counts) + assertConcurrentProfileRequests(t, client, cfg.BaseURL) +} + +type profileGetter func(string, map[string]string) (*http.Response, string) + +func assertProfileVersionLists(t *testing.T, get profileGetter) { + t.Helper() + for _, registry := range []struct{ path, old, middle, newest string }{ + {"npm/demo", `"1.0.0"`, `"2.0.0"`, `"3.0.0"`}, + {"cargo/de/mo/demo", `"vers":"1.0.0"`, `"vers":"2.0.0"`, `"vers":"3.0.0"`}, + {"gem/info/demo", "1.0.0 |", "2.0.0 |", "3.0.0 |"}, + {"go/example.com/demo/@v/list", "v1.0.0", "v2.0.0", "v3.0.0"}, + } { + t.Run(registry.path, func(t *testing.T) { + for _, profile := range []struct { + prefix string + middle, newest bool + }{ + {"/cooldown/3d/", true, false}, {"/cooldown/7d/", false, false}, {"/", true, true}, + } { + resp, body := get(profile.prefix+registry.path, nil) + if resp.StatusCode != http.StatusOK || !strings.Contains(body, registry.old) || + strings.Contains(body, registry.middle) != profile.middle || strings.Contains(body, registry.newest) != profile.newest { + t.Fatalf("%s%s: status %d, body %s", profile.prefix, registry.path, resp.StatusCode, body) + } + } + }) + } +} + +func assertProfileRouting(t *testing.T, baseURL string, get profileGetter, counts func(string) int) { + t.Helper() + for _, profile := range []string{"3d", "7d"} { + _, body := get("/cooldown/"+profile+"/cargo/config.json", nil) + if !strings.Contains(body, baseURL+"/cooldown/"+profile+"/cargo/crates/") { + t.Fatalf("Cargo download URL lost profile: %s", body) + } + _, body = get("/cooldown/"+profile+"/npm/demo", nil) + if !strings.Contains(body, baseURL+"/cooldown/"+profile+"/npm/demo/-/") { + t.Fatalf("npm download URL lost profile: %s", body) + } + } + loose, _ := get("/cooldown/3d/npm/demo", nil) + strict, _ := get("/cooldown/7d/npm/demo", map[string]string{"If-None-Match": loose.Header.Get("ETag")}) + if strict.StatusCode != http.StatusOK || strict.Header.Get("ETag") == loose.Header.Get("ETag") { + t.Fatalf("cross-profile validator returned status %d", strict.StatusCode) + } + resp, _ := get("/cooldown/7d/npm/demo", map[string]string{"If-None-Match": strict.Header.Get("ETag")}) + if resp.StatusCode != http.StatusNotModified { + t.Fatalf("same-profile revalidation returned %d", resp.StatusCode) + } + for _, tc := range []struct { + path string + status int + }{ + {"/cooldown/3d/npm/demo/-/demo-2.0.0.tgz", http.StatusOK}, + {"/cooldown/7d/npm/demo/-/demo-2.0.0.tgz", http.StatusNotFound}, + {"/cooldown/3d/npm/demo/-/demo-2.0.0.tgz", http.StatusOK}, + {"/cooldown/7d/go/example.com/demo/@v/v2.0.0.mod", http.StatusForbidden}, + {"/cooldown/unknown/npm/demo", http.StatusNotFound}, + {"/cooldown/3d/ui/", http.StatusNotFound}, + {"/cooldown/3d/api/packages", http.StatusNotFound}, + {"/cooldown/3d/metrics", http.StatusNotFound}, + } { + resp, _ := get(tc.path, nil) + if resp.StatusCode != tc.status { + t.Errorf("%s: status %d, want %d", tc.path, resp.StatusCode, tc.status) + } + } + if got := counts("/npm/demo/-/demo-2.0.0.tgz"); got != 1 { + t.Errorf("artifact upstream fetches = %d, want 1", got) + } +} + +func assertConcurrentProfileRequests(t *testing.T, client *http.Client, baseURL string) { + t.Helper() + var wg sync.WaitGroup + for range 8 { + for _, profile := range []string{"3d", "7d"} { + wg.Go(func() { + resp, err := client.Get(baseURL + "/cooldown/" + profile + "/npm/demo") + if err != nil { + t.Error(err) + return + } + defer func() { _ = resp.Body.Close() }() + var doc struct{ Versions map[string]json.RawMessage } + if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil { + t.Error(err) + return + } + _, middle := doc.Versions["2.0.0"] + if middle != (profile == "3d") { + t.Errorf("%s served another profile's versions", profile) + } + }) + } + } + wg.Wait() +} + +func profileTestUpstream(t *testing.T) (*httptest.Server, func(string) int) { + t.Helper() + var mu sync.Mutex + counts := make(map[string]int) + ages := []time.Duration{10 * 24 * time.Hour, 5 * 24 * time.Hour, 24 * time.Hour} + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + counts[r.URL.Path]++ + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/npm/demo": + versions := map[string]any{} + times := map[string]string{} + for i, age := range ages { + version := fmt.Sprintf("%d.0.0", i+1) + times[version] = time.Now().Add(-age).UTC().Format(time.RFC3339) + versions[version] = map[string]any{"name": "demo", "version": version, "dist": map[string]string{ + "tarball": upstream.URL + "/npm/demo/-/demo-" + version + ".tgz", + }} + } + _ = json.NewEncoder(w).Encode(map[string]any{"name": "demo", "versions": versions, "time": times, "dist-tags": map[string]string{"latest": "3.0.0"}}) + case "/npm/demo/-/demo-2.0.0.tgz": + _, _ = io.WriteString(w, "cached package bytes") + case "/cargo/de/mo/demo": + for i, age := range ages { + _, _ = fmt.Fprintf(w, "{\"name\":\"demo\",\"vers\":\"%d.0.0\",\"deps\":[],\"features\":{},\"yanked\":false,\"pubtime\":%q}\n", i+1, time.Now().Add(-age).UTC().Format(time.RFC3339)) + } + case "/gem/info/demo": + _, _ = io.WriteString(w, "---\n1.0.0 |checksum:a\n2.0.0 |checksum:b\n3.0.0 |checksum:c\n") + case "/gem/api/v1/versions/demo.json": + versions := make([]map[string]string, 0, len(ages)) + for i, age := range ages { + versions = append(versions, map[string]string{"number": fmt.Sprintf("%d.0.0", i+1), "created_at": time.Now().Add(-age).UTC().Format(time.RFC3339)}) + } + _ = json.NewEncoder(w).Encode(versions) + case "/go/example.com/demo/@v/list": + _, _ = io.WriteString(w, "v1.0.0\nv2.0.0\nv3.0.0\n") + default: + for i, age := range ages { + if r.URL.Path == fmt.Sprintf("/go/example.com/demo/@v/v%d.0.0.info", i+1) { + _ = json.NewEncoder(w).Encode(map[string]string{"Version": fmt.Sprintf("v%d.0.0", i+1), "Time": time.Now().Add(-age).UTC().Format(time.RFC3339)}) + return + } + } + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + return upstream, func(path string) int { + mu.Lock() + defer mu.Unlock() + return counts[path] + } +} diff --git a/internal/server/middleware.go b/internal/server/middleware.go index 7d2bd98..aa4da11 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -121,6 +121,10 @@ func (s *Server) LoggerMiddleware(next http.Handler) http.Handler { // "other" along with the UI, health and metrics paths, pooling a real // ecosystem's traffic with traffic that belongs to no ecosystem at all. func requestEcosystem(path string) string { + if rest, ok := strings.CutPrefix(path, "/cooldown/"); ok { + _, rest, _ = strings.Cut(rest, "/") + path = "/" + rest + } segment, _, _ := strings.Cut(strings.TrimPrefix(path, "/"), "/") switch segment { case "npm", "cargo", "hex", "pub", "pypi", "maven", "gradle", "nuget", diff --git a/internal/server/server.go b/internal/server/server.go index 55fe597..aa12c40 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -77,7 +77,6 @@ import ( "time" "github.com/git-pkgs/archives" - "github.com/git-pkgs/cooldown" shared "github.com/git-pkgs/enrichment" swaggerdoc "github.com/git-pkgs/proxy/docs/swagger" "github.com/git-pkgs/proxy/internal/accesslog" @@ -286,16 +285,11 @@ func (s *Server) serve(listener net.Listener) error { baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(artifactClient), fetch.WithMaxArtifactSize(s.cfg.ParseArtifactMaxSize())) fetcher := fetch.NewCircuitBreakerFetcher(baseFetcher) s.breakers = newBreakerMonitor(fetcher, s.logger) - cd := &cooldown.Config{ - Default: s.cfg.Cooldown.Default, - Ecosystems: s.cfg.Cooldown.Ecosystems, - Packages: s.cfg.Cooldown.NormalizedPackages(), - } proxy := handler.NewProxy(s.db, s.storage, fetcher, nil, s.logger) configureMirrorResolvers(proxy, s.cfg.Upstream) proxy.HTTPClient = metadataClient proxy.RateLimits = s.rateLimits - cooldownPolicy, err := cooldown.NewPolicy(cd, s.cfg.Cooldown.PackagePatterns) + cooldownPolicy, err := s.cfg.Cooldown.Policy() if err != nil { return fmt.Errorf("configuring cooldown policy: %w", err) } @@ -348,6 +342,9 @@ func (s *Server) serve(listener net.Listener) error { // Mount protocol handlers s.mountProtocolHandlers(r, proxy) + if err := s.mountCooldownProfiles(r, proxy); err != nil { + return err + } // Health, stats, and metrics endpoints r.Get("/health", s.handleHealth) @@ -462,71 +459,75 @@ func (s *Server) serve(listener net.Listener) error { // mountProtocolHandlers constructs every ecosystem handler and mounts it on // r under its protocol prefix. func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { - npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.NPM). + s.mountProtocolHandlersAt(r, proxy, s.cfg.BaseURL) +} + +func (s *Server) mountProtocolHandlersAt(r chi.Router, proxy *handler.Proxy, baseURL string) { + npmHandler := handler.NewNPMHandler(proxy, baseURL, s.cfg.Upstream.NPM). WithPackageRoutes(s.cfg.Upstream.NPMRoutes) cargoHandler := handler.NewCargoHandler( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.Cargo, s.cfg.Upstream.CargoDownload, ) - gemHandler := handler.NewGemHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Gem) - goHandler := handler.NewGoHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Go) + gemHandler := handler.NewGemHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.Gem) + goHandler := handler.NewGoHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.Go) goHandler.SetSumDBs(s.cfg.Upstream.GoSumDBs, s.cfg.Upstream.GoSumDBsOffline) goHandler.SetVulnDB(s.cfg.Upstream.GoVulnDB) hexHandler := handler.NewHexHandlerWithUpstreams( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.Hex, s.cfg.Upstream.HexAPI, ) - pubHandler := handler.NewPubHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Pub) + pubHandler := handler.NewPubHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.Pub) pypiHandler := handler.NewPyPIHandlerWithUpstreams( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.PyPI, s.cfg.Upstream.PyPIDownload, ) mavenHandler := handler.NewMavenHandler( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.Maven, s.cfg.Upstream.GradlePluginPortal, ).WithRepositories(s.cfg.Upstream.MavenRepositories) gradleHandler := handler.NewGradleBuildCacheHandler(proxy) nugetHandler := handler.NewNuGetHandlerWithUpstreams( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.NuGet, s.cfg.Upstream.NuGetSearch, ) composerHandler := handler.NewComposerHandlerWithUpstreams( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.Composer, s.cfg.Upstream.ComposerRepository, ).WithPackageRoutes(s.cfg.Upstream.ComposerRoutes) - conanHandler := handler.NewConanHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conan) - chefHandler := handler.NewChefHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Chef) - condaHandler := handler.NewCondaHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conda) - cranHandler := handler.NewCRANHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.CRAN) + conanHandler := handler.NewConanHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.Conan) + chefHandler := handler.NewChefHandler(proxy, baseURL, s.cfg.Upstream.Chef) + condaHandler := handler.NewCondaHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.Conda) + cranHandler := handler.NewCRANHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.CRAN) juliaHandler := handler.NewJuliaHandlerWithUpstream(proxy, s.cfg.Upstream.Julia) - swiftHandler := handler.NewSwiftHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Swift) + swiftHandler := handler.NewSwiftHandler(proxy, baseURL, s.cfg.Upstream.Swift) homebrewHandler := handler.NewHomebrewHandler(proxy, s.cfg.Upstream.HomebrewAPI) - containerHandler := s.newContainerHandler(proxy) + containerHandler := s.newContainerHandlerAt(proxy, baseURL) helmHandler := handler.NewHelmHandlerWithOCIRegistries( - proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI) - apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK). + proxy, baseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI) + apkHandler := handler.NewAPKHandler(proxy, baseURL, s.cfg.Upstream.APK). WithSnapshots(s.cfg.ParseAPKSnapshots()) - archHandler := handler.NewArchHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Arch) - freebsdHandler := handler.NewFreeBSDHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.FreeBSD) + archHandler := handler.NewArchHandler(proxy, baseURL, s.cfg.Upstream.Arch) + freebsdHandler := handler.NewFreeBSDHandler(proxy, baseURL, s.cfg.Upstream.FreeBSD) debianHandler := handler.NewDebianHandler( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.Debian, s.cfg.Upstream.DebianRepositories, ) - rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) + rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, baseURL, s.cfg.Upstream.RPM) genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts). WithPaths(s.cfg.Upstream.GenericPaths) @@ -558,7 +559,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) r.Mount("/generic", http.StripPrefix("/generic", genericHandler.Routes())) if s.git != nil { - gitHandler := handler.NewGitHandler(proxy, s.git, s.cfg.Upstream).WithBaseURL(s.cfg.BaseURL) + gitHandler := handler.NewGitHandler(proxy, s.git, s.cfg.Upstream).WithBaseURL(baseURL) r.Mount("/git", http.StripPrefix("/git", gitHandler.Routes())) } } @@ -629,9 +630,13 @@ func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, // newContainerHandler builds the OCI handler with the configured default // registry, named upstreams, mirrored registries and Homebrew routes. func (s *Server) newContainerHandler(proxy *handler.Proxy) *handler.ContainerHandler { + return s.newContainerHandlerAt(proxy, s.cfg.BaseURL) +} + +func (s *Server) newContainerHandlerAt(proxy *handler.Proxy, baseURL string) *handler.ContainerHandler { containerHandler := handler.NewContainerHandlerWithRegistry( proxy, - s.cfg.BaseURL, + baseURL, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI, ) From 03be03b8fef9f5da0dc7bf886e70d245e593c570 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 6 Nov 2026 07:48:51 +0100 Subject: [PATCH 087/198] Mirror pkg:git commits into the git cache proxy mirror, mirror API jobs and SBOMs take pkg:git/{host}/{path}@{commit} and cache the repository and commit in the cache /git serves, so a later clone is a cache hit, also with the upstream down. Part of #403. --- README.md | 3 + cmd/proxy/main.go | 23 +++++ cmd/proxy/mirror_git_test.go | 132 ++++++++++++++++++++++++++++ docs/architecture.md | 2 +- docs/configuration.md | 14 +++ internal/gitcache/mirror.go | 88 +++++++++++++++++++ internal/gitcache/mirror_test.go | 108 +++++++++++++++++++++++ internal/handler/git_mirror.go | 57 ++++++++++++ internal/handler/git_mirror_test.go | 73 +++++++++++++++ internal/handler/git_test.go | 9 +- internal/mirror/git.go | 49 +++++++++++ internal/mirror/git_test.go | 92 +++++++++++++++++++ internal/mirror/mirror.go | 8 ++ internal/mirror/source.go | 5 +- internal/server/git_test.go | 123 +++++++++++++++++++++----- internal/server/mirror_versions.go | 3 + internal/server/server.go | 15 ++++ 17 files changed, 779 insertions(+), 25 deletions(-) create mode 100644 cmd/proxy/mirror_git_test.go create mode 100644 internal/gitcache/mirror.go create mode 100644 internal/gitcache/mirror_test.go create mode 100644 internal/handler/git_mirror.go create mode 100644 internal/handler/git_mirror_test.go create mode 100644 internal/mirror/git.go create mode 100644 internal/mirror/git_test.go diff --git a/README.md b/README.md index 2a08eaa..2eea25d 100644 --- a/README.md +++ b/README.md @@ -1194,6 +1194,9 @@ proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21 # Mirror a container image for one platform proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20 + +# Mirror a commit of an upstream.git repository into the git cache +proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 ``` The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror. diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 70ae8e6..f9ab849 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -595,7 +595,9 @@ func runMirror() { return } + closeGit := useCommandGit(cfg, proxy, m, logger) progress, err := m.Run(ctx, source) + closeGit() if err != nil { fmt.Fprintf(os.Stderr, "error: %v\n", err) os.Exit(1) @@ -634,6 +636,27 @@ func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler. return m } +// useCommandGit opens the git cache for the pkg:git items of proxy mirror, +// when upstream.git is configured, and returns the function that closes it. +// Where it can't be opened, such as while the running server holds +// git.cache_dir, those items fail with the reason and the rest are mirrored. +func useCommandGit(cfg *config.Config, proxy *handler.Proxy, m *mirror.Mirror, logger *slog.Logger) func() { + git, closeGit, err := server.NewMirrorGit(cfg, proxy, logger) + if err != nil { + m.Git = mirror.UnavailableGit{Reason: fmt.Sprintf("this command can't open the git cache (%v); while the server runs, use its mirror API", err)} + return func() {} + } + if git == nil { + return func() {} + } + m.Git = git + return func() { + if err := closeGit(); err != nil { + logger.Warn("closing the git cache", "error", err) + } + } +} + // mirrorExitCode returns the exit status for a finished mirror run: 1 when any // package version failed to mirror, 0 otherwise. func mirrorExitCode(progress *mirror.Progress) int { diff --git a/cmd/proxy/mirror_git_test.go b/cmd/proxy/mirror_git_test.go new file mode 100644 index 0000000..2789536 --- /dev/null +++ b/cmd/proxy/mirror_git_test.go @@ -0,0 +1,132 @@ +package main + +import ( + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +// gitMirrorUpstream serves team/repo, with one commit on main, from git +// http-backend, and returns the server and the commit. +func gitMirrorUpstream(t *testing.T) (*httptest.Server, string) { + t.Helper() + git := func(dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) + } + backend := "" + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if path := filepath.Join(git("", "--exec-path"), name); fileExistsForTest(path) { + backend = path + } + } + if backend == "" { + t.Skip("git http-backend is not installed") + } + root := t.TempDir() + bare := filepath.Join(root, "team", "repo.git") + git("", "init", "--quiet", "--bare", "--initial-branch=main", bare) + work := t.TempDir() + git("", "clone", "--quiet", bare, work) + git(work, "commit", "--quiet", "--allow-empty", "-m", "first") + git(work, "push", "--quiet", "origin", "HEAD:main") + head := git(work, "rev-parse", "HEAD") + + server := httptest.NewServer(&cgi.Handler{ + Path: backend, + Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}, + }) + t.Cleanup(server.Close) + return server, head +} + +func fileExistsForTest(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// writeGitMirrorConfig writes a config with upstream.git "up" at +// upstreamURL and the git cache, storage and database under dir. +func writeGitMirrorConfig(t *testing.T, dir, upstreamURL string) string { + t.Helper() + cfg := "storage:\n" + + " url: file://" + filepath.Join(dir, "artifacts") + "\n" + + "database:\n" + + " path: " + filepath.Join(dir, "proxy.db") + "\n" + + "upstream:\n" + + " allow_loopback: true\n" + + " git:\n" + + " up: " + upstreamURL + "\n" + + "git:\n" + + " cache_dir: " + filepath.Join(dir, "git") + "\n" + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil { + t.Fatalf("writing config: %v", err) + } + return path +} + +func TestMirrorCommandCachesGitCommit(t *testing.T) { + upstream, head := gitMirrorUpstream(t) + dir := t.TempDir() + configPath := writeGitMirrorConfig(t, dir, upstream.URL) + purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head + + output := runMirrorProcess(t, 0, "-config", configPath, purl) + if !strings.Contains(output, "1 downloaded") { + t.Errorf("output:\n%s", output) + } + cached, _ := filepath.Glob(filepath.Join(dir, "git", "up-*", "team", "repo.git")) + if len(cached) != 1 { + t.Errorf("cached repositories %v, want team/repo.git", cached) + } + + output = runMirrorProcess(t, 0, "-config", configPath, purl) + if !strings.Contains(output, "1 skipped") { + t.Errorf("second run output:\n%s", output) + } +} + +// TestMirrorCommandGitCacheInUse checks pkg:git items fail, pointing at the +// mirror API, while the running server holds git.cache_dir. +func TestMirrorCommandGitCacheInUse(t *testing.T) { + upstream, head := gitMirrorUpstream(t) + dir := t.TempDir() + configPath := writeGitMirrorConfig(t, dir, upstream.URL) + cache, err := gitcache.New(gitcache.Options{ + CacheDir: filepath.Join(dir, "git"), + RefreshWait: time.Second, + Client: &http.Client{}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = cache.Close() }() + purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head + + output := runMirrorProcess(t, 1, "-config", configPath, purl) + for _, want := range []string{"in use by another process", "mirror API", "1 failed"} { + if !strings.Contains(output, want) { + t.Errorf("output missing %q:\n%s", want, output) + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md index d8b294c..655b309 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -363,7 +363,7 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g ### `internal/mirror` -Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. +Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and commit in the git cache that `/git` serves. The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. diff --git a/docs/configuration.md b/docs/configuration.md index 38765e0..b2b830f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1877,6 +1877,20 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403 - Dependencies aren't mirrored with a formula; list them too. brew also needs the JSON API, which the proxy keeps with `cache_metadata`. - Casks, source archives and taps other than homebrew/core aren't mirrored. +### Git repositories + +`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID: + +```bash +proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 +proxy mirror pkg:git/github.com/git-pkgs/purl +``` + +- The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. +- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, for seven days or until a branch or tag includes it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag fails. +- Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. +- The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. + ### Scheduled refresh Tags such as `alpine:3.20` move. The server can re-mirror them on its own, so the cache follows the tag even when nobody pulls it or the proxy is offline later: diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go new file mode 100644 index 0000000..854ea08 --- /dev/null +++ b/internal/gitcache/mirror.go @@ -0,0 +1,88 @@ +package gitcache + +import ( + "context" + "errors" + "fmt" +) + +// MirrorResult is what Mirror did. +type MirrorResult struct { + // Cached reports that the repository, and the commit if one was given, + // were cached already. + Cached bool + // Bytes is how much the cached repository grew. + Bytes int64 +} + +// Mirror brings the repository cached under key up to date with repoURL, +// cloning it if needed, and waits for that to finish. With commit, an object +// ID, it also makes sure the commit is cached. One on no branch or tag is +// fetched by ID when fetchWants allows it (see OpenOptions.FetchWants), and +// is kept as long as a commit a client fetched by ID. +func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWants bool) (MirrorResult, error) { + if commit != "" && !isObjectID(commit) { + return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit) + } + lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants}) + if err != nil { + return MirrorResult{}, err + } + defer lease.Release() + rp := lease.repo + cached := lease.hit + before := int64(0) + var refreshErr error + // Open cloned a repository that wasn't cached; one that was is + // refreshed here, waiting for the fetch a client would not wait for. + if cached { + before = rp.size.Load() + refresh := c.startRefresh(rp, repoURL) + select { + case <-refresh.done: + refreshErr = refresh.err + case <-ctx.Done(): + return MirrorResult{}, ctx.Err() + } + } + + if commit != "" { + fetched, err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants) + if err != nil { + return MirrorResult{}, err + } + cached = cached && !fetched + } + // A refresh that failed leaves the commit cached all the same. + if refreshErr != nil && commit == "" { + return MirrorResult{}, refreshErr + } + return MirrorResult{Cached: cached, Bytes: max(rp.size.Load()-before, 0)}, nil +} + +// ensureCommit makes sure the repository has commit, fetching it by ID when +// it doesn't and fetchWants allows it, and reports whether it was fetched. +func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) (bool, error) { + missing, err := c.missingObjects(ctx, rp.dir, []string{commit}) + if err != nil || len(missing) == 0 { + return false, err + } + if !fetchWants { + return false, fmt.Errorf("commit %s: %w", commit, ErrNotFound) + } + if err := c.fetchMissingWants(ctx, rp, repoURL, missing); err != nil { + if ctx.Err() != nil || errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) { + return false, err + } + // The upstream refuses a commit it doesn't have. + return false, fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err) + } + // fetchMissingWants skips a commit that failed moments ago. + if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil { + return false, err + } + if len(missing) > 0 { + return false, fmt.Errorf("commit %s: %w", commit, ErrNotFound) + } + return true, nil +} diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go new file mode 100644 index 0000000..a1fd07e --- /dev/null +++ b/internal/gitcache/mirror_test.go @@ -0,0 +1,108 @@ +package gitcache + +import ( + "context" + "errors" + "strings" + "testing" +) + +func TestMirrorClonesRepository(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached || got.Bytes <= 0 { + t.Errorf("first mirror = %+v, want fetched bytes", got) + } + if !strings.Contains(tc.cachedRefs(t), "refs/tags/v1") { + t.Errorf("cached refs %q lack the tag", tc.cachedRefs(t)) + } + + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } +} + +func TestMirrorFetchesCommitOnNoBranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("mirror = %+v, want fetched", got) + } + if refs := tc.wantedRefs(t); len(refs) != 1 { + t.Errorf("wanted refs %v, want one", refs) + } + + sessions := upstream.gitSessions.Load() + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + if n := upstream.gitSessions.Load() - sessions; n != 0 { + t.Errorf("second mirror fetched %d times, want none", n) + } +} + +func TestMirrorFindsCommitOnABranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + sha := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, false) + if err != nil || got.Cached { + t.Errorf("mirror = %+v, %v, want fetched", got, err) + } +} + +func TestMirrorDoesNotFetchByIDWhenTurnedOff(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, false) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror of a commit on no branch with fetching by ID off = %v, want ErrNotFound", err) + } + if refs := tc.wantedRefs(t); len(refs) != 0 { + t.Errorf("wanted refs %v, want none", refs) + } +} + +func TestMirrorReportsMissingCommit(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", strings.Repeat("ab", 20), true) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror of a commit the upstream lacks = %v, want ErrNotFound", err) + } +} + +func TestMirrorRefusesInvalidCommit(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "main", true); err == nil { + t.Error("mirrored a branch name as a commit") + } +} diff --git a/internal/handler/git_mirror.go b/internal/handler/git_mirror.go new file mode 100644 index 0000000..eca76c7 --- /dev/null +++ b/internal/handler/git_mirror.go @@ -0,0 +1,57 @@ +package handler + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +// Mirror caches the repository at location, a host and path such as +// github.com/acme/widgets as a pkg:git PURL names it, from the upstream.git +// host serving it, and with commit, a full commit ID, that commit too. It +// is the copy clones through /git are then served. +func (h *GitHandler) Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) { + repo, err := h.mirrorRepository(location) + if err != nil { + return gitcache.MirrorResult{}, err + } + return h.cache.Mirror(ctx, repo.key, repo.url, commit, repo.upstream.fetchWants) +} + +// mirrorRepository resolves location to a repository of the upstream.git +// host whose URL it starts with, the longest such URL if several do. +func (h *GitHandler) mirrorRepository(location string) (gitRepository, error) { + var best, bestPrefix, rest string + for name, upstream := range h.upstreams { + prefix := gitLocationPrefix(upstream.URL) + path, ok := strings.CutPrefix(location, prefix) + if prefix == "" || !ok || path == "" { + continue + } + if len(prefix) > len(bestPrefix) || (len(prefix) == len(bestPrefix) && name < best) { + best, bestPrefix, rest = name, prefix, path + } + } + if best == "" { + return gitRepository{}, fmt.Errorf("no upstream.git host serves %s", location) + } + repo, status, msg := h.repository(best + "/" + rest) + if status != 0 { + return gitRepository{}, fmt.Errorf("%s: %s", location, msg) + } + return repo, nil +} + +// gitLocationPrefix is how a pkg:git PURL names the repositories under an +// upstream URL: its host and path, ending in a slash. Empty for a URL +// without a host. +func gitLocationPrefix(upstreamURL string) string { + u, err := url.Parse(upstreamURL) + if err != nil || u.Host == "" { + return "" + } + return strings.ToLower(u.Host) + strings.TrimSuffix(u.Path, "/") + "/" +} diff --git a/internal/handler/git_mirror_test.go b/internal/handler/git_mirror_test.go new file mode 100644 index 0000000..4ebcf98 --- /dev/null +++ b/internal/handler/git_mirror_test.go @@ -0,0 +1,73 @@ +package handler + +import ( + "context" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/config" +) + +func TestGitMirrorLocation(t *testing.T) { + h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ + "github": {URL: "https://github.com"}, + "wikimedia": {URL: "https://gitlab.wikimedia.org", Repos: []string{"mediawiki/**"}}, + "sub": {URL: "https://git.example/r/"}, + "nested": {URL: "https://git.example/r/deep"}, + }}) + tests := []struct { + location string + ok bool + key, repoURL string + }{ + {"github.com/acme/widgets", true, h.upstreams["github"].dir + "/acme/widgets.git", "https://github.com/acme/widgets"}, + {"github.com/acme/widgets.git", true, h.upstreams["github"].dir + "/acme/widgets.git", "https://github.com/acme/widgets.git"}, + {"gitlab.wikimedia.org/mediawiki/skins/Vector", true, h.upstreams["wikimedia"].dir + "/mediawiki/skins/Vector.git", "https://gitlab.wikimedia.org/mediawiki/skins/Vector"}, + {"gitlab.wikimedia.org/other/repo", false, "", ""}, + {"git.example/r/a", true, h.upstreams["sub"].dir + "/a.git", "https://git.example/r/a"}, + // The longest matching upstream URL wins. + {"git.example/r/deep/a", true, h.upstreams["nested"].dir + "/a.git", "https://git.example/r/deep/a"}, + {"git.example/other", false, "", ""}, + {"git.example/r", false, "", ""}, + {"github.com.evil/acme/widgets", false, "", ""}, + {"github.com/acme/../widgets", false, "", ""}, + {"codeberg.org/forgejo/forgejo", false, "", ""}, + } + for _, tt := range tests { + t.Run(tt.location, func(t *testing.T) { + repo, err := h.mirrorRepository(tt.location) + if (err == nil) != tt.ok { + t.Fatalf("error %v, want ok %v", err, tt.ok) + } + if tt.ok && (repo.key != tt.key || repo.url != tt.repoURL) { + t.Errorf("key %q, URL %q; want %q, %q", repo.key, repo.url, tt.key, tt.repoURL) + } + }) + } +} + +func TestGitMirrorCachesCommitForClones(t *testing.T) { + g := newGitTestSetup(t, 0, 0) + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + location := strings.TrimPrefix(g.upstreamURL, "http://") + "/repo" + + got, err := g.handler.Mirror(context.Background(), location, sha) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("first mirror = %+v, want fetched", got) + } + got, err = g.handler.Mirror(context.Background(), location, sha) + if err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + + // With the upstream gone, a clone through the proxy gets the mirrored copy. + g.upstream.Close() + clone := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", g.url(), clone) + if head := runGitCommand(t, clone, "rev-parse", "HEAD"); head != sha { + t.Errorf("cloned HEAD %s, want %s", head, sha) + } +} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go index 0aeff4b..cf553d6 100644 --- a/internal/handler/git_test.go +++ b/internal/handler/git_test.go @@ -131,7 +131,12 @@ func runGitCommand(t *testing.T, dir string, args ...string) string { type gitTestSetup struct { work string server *httptest.Server - logs *syncLog + // upstream serves the repository, at upstreamURL/repo, and handler is + // the GitHandler under test. + upstream *httptest.Server + upstreamURL string + handler *GitHandler + logs *syncLog // slow delays every upstream request by half a second. slow atomic.Bool // active counts the requests the handler is serving. @@ -172,6 +177,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe backend.ServeHTTP(w, r) })) t.Cleanup(upstream.Close) + g.upstream, g.upstreamURL = upstream, upstream.URL logger := slog.New(slog.NewTextHandler(g.logs, nil)) cache, err := gitcache.New(gitcache.Options{ @@ -190,6 +196,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe if writeStall > 0 { h.writeStall = writeStall } + g.handler = h routes := http.StripPrefix("/git", h.Routes()) g.server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { g.active.Add(1) diff --git a/internal/mirror/git.go b/internal/mirror/git.go new file mode 100644 index 0000000..a4908dc --- /dev/null +++ b/internal/mirror/git.go @@ -0,0 +1,49 @@ +package mirror + +import ( + "context" + "errors" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +// GitMirrorer caches Git repositories for pkg:git items. location is the +// repository's host and path, such as codeberg.org/forgejo/forgejo, and +// commit a full commit ID, or empty to cache the repository's branches and +// tags only. +type GitMirrorer interface { + Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) +} + +// UnavailableGit fails every pkg:git item with Reason. +type UnavailableGit struct { + Reason string +} + +func (u UnavailableGit) Mirror(context.Context, string, string) (gitcache.MirrorResult, error) { + return gitcache.MirrorResult{}, errors.New(u.Reason) +} + +var errNoGitUpstream = errors.New("git repositories are mirrored from upstream.git, and none is configured") + +// mirrorGit caches a pkg:git repository, and the commit its version names, +// in the cache /git serves clones from. +func (m *Mirror) mirrorGit(ctx context.Context, pv PackageVersion, tracker *progressTracker) { + if m.Git == nil { + m.recordFailure(tracker, pv, errNoGitUpstream) + return + } + result, err := m.Git.Mirror(ctx, pv.Name, pv.Version) + if err != nil { + m.recordFailure(tracker, pv, err) + return + } + if result.Cached { + tracker.skipped.Add(1) + m.logger.Debug("already cached", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version) + return + } + tracker.completed.Add(1) + tracker.bytes.Add(result.Bytes) + m.logger.Info("mirrored", "ecosystem", pv.Ecosystem, "name", pv.Name, "version", pv.Version, "size", result.Bytes) +} diff --git a/internal/mirror/git_test.go b/internal/mirror/git_test.go new file mode 100644 index 0000000..aa3aed4 --- /dev/null +++ b/internal/mirror/git_test.go @@ -0,0 +1,92 @@ +package mirror + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/git-pkgs/proxy/internal/gitcache" +) + +type gitCall struct{ location, commit string } + +type fakeGit struct { + mu sync.Mutex + calls []gitCall + result gitcache.MirrorResult + err error +} + +func (f *fakeGit) Mirror(_ context.Context, location, commit string) (gitcache.MirrorResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, gitCall{location, commit}) + return f.result, f.err +} + +const gitTestCommit = "a72d2c07cfca03b55371089de6aa230d8c951fa0" + +func TestMirrorRunCountsGitRepositories(t *testing.T) { + tests := []struct { + name string + result gitcache.MirrorResult + err error + wantCompleted, wantSkipped, wantFailed int64 + }{ + {name: "fetched", result: gitcache.MirrorResult{Bytes: 42}, wantCompleted: 1}, + {name: "cached", result: gitcache.MirrorResult{Cached: true}, wantSkipped: 1}, + {name: "failed", err: errors.New("boom"), wantFailed: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := setupTestMirror(t, 1) + git := &fakeGit{result: tt.result, err: tt.err} + m.Git = git + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo@" + gitTestCommit}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Completed != tt.wantCompleted || progress.Skipped != tt.wantSkipped || progress.Failed != tt.wantFailed { + t.Errorf("progress = %+v", progress) + } + if progress.Bytes != tt.result.Bytes { + t.Errorf("bytes = %d, want %d", progress.Bytes, tt.result.Bytes) + } + want := gitCall{"codeberg.org/forgejo/forgejo", gitTestCommit} + if len(git.calls) != 1 || git.calls[0] != want { + t.Errorf("calls = %+v, want %+v", git.calls, want) + } + }) + } +} + +// TestMirrorRunGitRepositoryWithoutCommit checks a pkg:git PURL without a +// version mirrors the repository instead of listing versions. +func TestMirrorRunGitRepositoryWithoutCommit(t *testing.T) { + m := setupTestMirror(t, 1) + git := &fakeGit{} + m.Git = git + versions := &fakeVersions{err: errors.New("listed versions")} + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo"}, Versions: versions}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(versions.asked) != 0 { + t.Errorf("listed versions of %v", versions.asked) + } + if progress.Failed != 0 || len(git.calls) != 1 || git.calls[0] != (gitCall{"codeberg.org/forgejo/forgejo", ""}) { + t.Errorf("progress = %+v, calls = %+v", progress, git.calls) + } +} + +func TestMirrorRunGitWithoutUpstream(t *testing.T) { + m := setupTestMirror(t, 1) + progress, err := m.Run(context.Background(), &PURLSource{PURLs: []string{"pkg:git/codeberg.org/forgejo/forgejo@" + gitTestCommit}}) + if err != nil { + t.Fatalf("Run: %v", err) + } + if progress.Failed != 1 { + t.Errorf("progress = %+v, want one failure", progress) + } +} diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index d5ec9a5..c08b1fb 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -32,6 +32,10 @@ type Mirror struct { // Charts mirrors helm charts. Without it they fail. Charts ChartMirrorer + // Git mirrors pkg:git repositories into the upstream.git cache. Without + // it they fail. + Git GitMirrorer + // Bottles resolves Homebrew formula versions to the images their // bottles are published under, which Images mirrors. Without it brew // PURLs fail. @@ -279,6 +283,10 @@ func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *prog m.mirrorAPK(ctx, pv, tracker) return } + if pv.Ecosystem == "git" { + m.mirrorGit(ctx, pv, tracker) + return + } result, err := m.proxy.GetOrFetchArtifact(ctx, pv.Ecosystem, pv.Name, pv.Version, "") if err != nil { m.recordFailure(tracker, pv, err) diff --git a/internal/mirror/source.go b/internal/mirror/source.go index 2c41705..c0441c4 100644 --- a/internal/mirror/source.go +++ b/internal/mirror/source.go @@ -70,8 +70,9 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro } // Images are never expanded to all tags, and conda packages without - // a channel aren't listed; both fail when they are mirrored. - if p.Version != "" || isImageEcosystem(ecosystem) || (ecosystem == "conda" && !strings.Contains(name, "/")) { + // a channel aren't listed; both fail when they are mirrored. A Git + // repository without a commit is mirrored as a whole. + if p.Version != "" || isImageEcosystem(ecosystem) || ecosystem == "git" || (ecosystem == "conda" && !strings.Contains(name, "/")) { pv := PackageVersion{Ecosystem: ecosystem, Name: name, Version: p.Version, Namespace: namespace, Qualifiers: qualifiers} if isImageEcosystem(ecosystem) { pv.Qualifiers = p.Qualifiers.Map() diff --git a/internal/server/git_test.go b/internal/server/git_test.go index d52f1a9..e59f30d 100644 --- a/internal/server/git_test.go +++ b/internal/server/git_test.go @@ -3,6 +3,7 @@ package server import ( "bytes" "context" + "encoding/json" "errors" "log/slog" "net" @@ -18,6 +19,7 @@ import ( "time" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/mirror" ) func gitTestEnv(home string) []string { @@ -75,11 +77,11 @@ func (b *syncBuffer) String() string { return b.buf.String() } -// TestGitRouteServesClonesWithUpstreamCredentials clones through the whole -// server, with credentials from upstream.auth that only the proxy holds. -func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { - root := t.TempDir() - bare := filepath.Join(root, "team", "repo.git") +// newGitTestRepo creates the bare repository {root}/{path}.git with one +// commit on main and returns the commit. +func newGitTestRepo(t *testing.T, root, path string) string { + t.Helper() + bare := filepath.Join(root, filepath.FromSlash(path)+".git") gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) work := t.TempDir() gitCommand(t, "", "clone", "--quiet", bare, work) @@ -89,21 +91,22 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { gitCommand(t, work, "add", "file.txt") gitCommand(t, work, "commit", "--quiet", "-m", "first") gitCommand(t, work, "push", "--quiet", "origin", "HEAD:main") - head := gitCommand(t, work, "rev-parse", "HEAD") + return gitCommand(t, work, "rev-parse", "HEAD") +} - backend := &cgi.Handler{ +// gitBackend serves the repositories under root with git http-backend. +func gitBackend(t *testing.T, root string) http.Handler { + t.Helper() + return &cgi.Handler{ Path: gitHTTPBackend(t), Env: append(gitTestEnv(t.TempDir()), "GIT_PROJECT_ROOT="+root, "GIT_HTTP_EXPORT_ALL=1"), } - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Authorization") != "Bearer git-secret" { - http.Error(w, "unauthorized", http.StatusUnauthorized) - return - } - backend.ServeHTTP(w, r) - })) - defer upstream.Close() +} +// startGitTestServer starts the whole server with the git cache, after +// configure has set up cfg, and returns cfg and the server's log. +func startGitTestServer(t *testing.T, configure func(*config.Config)) (*config.Config, *syncBuffer) { + t.Helper() listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -115,21 +118,20 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { cfg.Database.Path = filepath.Join(tempDir, "proxy.db") cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") cfg.Upstream.AllowLoopback = true - cfg.Upstream.Git = map[string]config.GitUpstream{"private": {URL: upstream.URL, Repos: []string{"team/*"}}} - cfg.Upstream.Auth = map[string]config.AuthConfig{upstream.URL: {Type: "bearer", Token: "git-secret"}} cfg.Git.CacheDir = filepath.Join(tempDir, "git") + configure(cfg) if err := cfg.Validate(); err != nil { t.Fatalf("validating config: %v", err) } - var logs syncBuffer - proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(&logs, nil)), BuildInfo{Version: "test", Commit: "test"}) + logs := &syncBuffer{} + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(logs, nil)), BuildInfo{Version: "test", Commit: "test"}) if err != nil { t.Fatalf("creating server: %v", err) } startErr := make(chan error, 1) go func() { startErr <- proxyServer.Start(listener) }() - defer func() { + t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := proxyServer.Shutdown(ctx); err != nil { @@ -138,7 +140,29 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { t.Errorf("Start() error = %v, want %v", err, http.ErrServerClosed) } - }() + }) + return cfg, logs +} + +// TestGitRouteServesClonesWithUpstreamCredentials clones through the whole +// server, with credentials from upstream.auth that only the proxy holds. +func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { + root := t.TempDir() + head := newGitTestRepo(t, root, "team/repo") + backend := gitBackend(t, root) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer git-secret" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + backend.ServeHTTP(w, r) + })) + defer upstream.Close() + + cfg, logs := startGitTestServer(t, func(cfg *config.Config) { + cfg.Upstream.Git = map[string]config.GitUpstream{"private": {URL: upstream.URL, Repos: []string{"team/*"}}} + cfg.Upstream.Auth = map[string]config.AuthConfig{upstream.URL: {Type: "bearer", Token: "git-secret"}} + }) clone := filepath.Join(t.TempDir(), "clone") gitCommand(t, "", "clone", "--quiet", cfg.BaseURL+"/git/private/team/repo.git", clone) @@ -158,3 +182,60 @@ func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { t.Errorf("repository outside repos: status %d, want 404", resp.StatusCode) } } + +// TestMirrorAPICachesGitCommit mirrors a pkg:git commit through the mirror +// API, then clones it through /git while the upstream is down. +func TestMirrorAPICachesGitCommit(t *testing.T) { + root := t.TempDir() + head := newGitTestRepo(t, root, "team/repo") + upstream := httptest.NewServer(gitBackend(t, root)) + defer upstream.Close() + cfg, _ := startGitTestServer(t, func(cfg *config.Config) { + cfg.MirrorAPI = true + cfg.Upstream.Git = map[string]config.GitUpstream{"up": {URL: upstream.URL}} + }) + + location := strings.TrimPrefix(upstream.URL, "http://") + "/team/repo" + body, err := json.Marshal(map[string][]string{"purls": {"pkg:git/" + location + "@" + head}}) + if err != nil { + t.Fatal(err) + } + resp, err := http.Post(cfg.BaseURL+"/api/mirror", "application/json", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + var created struct{ ID string } + err = json.NewDecoder(resp.Body).Decode(&created) + _ = resp.Body.Close() + if err != nil || resp.StatusCode != http.StatusAccepted { + t.Fatalf("creating job: status %d, %v", resp.StatusCode, err) + } + + var job mirror.Job + deadline := time.Now().Add(30 * time.Second) + for job.State != mirror.JobStateComplete && job.State != mirror.JobStateFailed && job.State != mirror.JobStateCanceled { + if time.Now().After(deadline) { + t.Fatalf("job did not finish: %+v", job) + } + time.Sleep(50 * time.Millisecond) + resp, err := http.Get(cfg.BaseURL + "/api/mirror/" + created.ID) + if err != nil { + t.Fatal(err) + } + err = json.NewDecoder(resp.Body).Decode(&job) + _ = resp.Body.Close() + if err != nil { + t.Fatal(err) + } + } + if job.State != mirror.JobStateComplete || job.Progress.Completed != 1 { + t.Fatalf("job = %+v, want one repository mirrored", job) + } + + upstream.Close() + clone := filepath.Join(t.TempDir(), "clone") + gitCommand(t, "", "clone", "--quiet", cfg.BaseURL+"/git/up/team/repo.git", clone) + if got := gitCommand(t, clone, "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want %s", got, head) + } +} diff --git a/internal/server/mirror_versions.go b/internal/server/mirror_versions.go index 6d4599e..81491d9 100644 --- a/internal/server/mirror_versions.go +++ b/internal/server/mirror_versions.go @@ -32,6 +32,9 @@ func (s *Server) newMirror(proxy *handler.Proxy, workers int) *mirror.Mirror { m.Images = s.newContainerHandler(proxy) m.Bottles = handler.NewHomebrewHandler(proxy, s.cfg.Upstream.HomebrewAPI) m.Charts = handler.NewHelmHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm) + if s.git != nil { + m.Git = handler.NewGitHandler(proxy, s.git, s.cfg.Upstream) + } UseMirrorUpstreams(m, s.cfg.Upstream, proxy.HTTPClient) return m } diff --git a/internal/server/server.go b/internal/server/server.go index 55fe597..86b4a4c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -626,6 +626,21 @@ func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, return proxy, s.newContainerHandler(proxy) } +// NewMirrorGit opens the git cache for proxy mirror, with the upstream client +// serve uses, and returns the handler that mirrors pkg:git items into it and +// the function that closes it. Both are nil without upstream.git. +func NewMirrorGit(cfg *config.Config, proxy *handler.Proxy, logger *slog.Logger) (*handler.GitHandler, func() error, error) { + if len(cfg.Upstream.Git) == 0 { + return nil, nil, nil + } + s := &Server{cfg: cfg, logger: logger, ecr: newECRTokens(logger)} + _, artifactClient := s.newUpstreamClients(0) + if err := s.configureGitCache(artifactClient); err != nil { + return nil, nil, err + } + return handler.NewGitHandler(proxy, s.git, cfg.Upstream), s.git.Close, nil +} + // newContainerHandler builds the OCI handler with the configured default // registry, named upstreams, mirrored registries and Homebrew routes. func (s *Server) newContainerHandler(proxy *handler.Proxy) *handler.ContainerHandler { From 0ed91b5872ef9c1cfa6785b0bde4783a02b4de5d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 6 Nov 2026 09:20:43 +0000 Subject: [PATCH 088/198] Redact storage passwords in startup logs --- internal/server/server.go | 6 ++- internal/server/server_test.go | 89 ++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/internal/server/server.go b/internal/server/server.go index 8599c48..b6c67e4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -436,11 +436,15 @@ func (s *Server) serve(listener net.Listener) error { IdleTimeout: serverIdleTimeout, } + storageURL := "[redacted]" + if u, err := url.Parse(s.storage.URL()); err == nil { + storageURL = u.Redacted() + } s.logger.Info("starting server", "listen", s.cfg.Listen, "base_url", s.cfg.BaseURL, "ui_url", s.cfg.UIBaseURL, - "storage", s.storage.URL(), + "storage", storageURL, "database", s.cfg.Database.String()) for _, warning := range s.cfg.EnvWarnings() { s.logger.Warn(warning) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 5fa49e9..02d781d 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -150,6 +150,95 @@ func (ts *testServer) close() { _ = os.RemoveAll(ts.tempDir) } +type storageWithURL struct { + storage.Storage + url string +} + +func (s storageWithURL) URL() string { return s.url } + +func TestStartRedactsStoragePassword(t *testing.T) { + for _, tc := range []struct { + name string + url string + want string + }{ + {"password", "s3://user:secret@bucket?region=us-east-1", "s3://user:xxxxx@bucket?region=us-east-1"}, + {"escaped password", "s3://user:secret%40word@bucket", "s3://user:xxxxx@bucket"}, + {"no password", "s3://bucket?region=us-east-1", "s3://bucket?region=us-east-1"}, + {"invalid URL", "s3://user:secret%zz@bucket", "[redacted]"}, + } { + t.Run(tc.name, func(t *testing.T) { + output := startStorageLog(t, tc.url) + if strings.Contains(output, "secret") { + t.Errorf("startup log contains storage password: %s", output) + } + decoder := json.NewDecoder(strings.NewReader(output)) + for decoder.More() { + var entry struct { + Message string `json:"msg"` + Storage string `json:"storage"` + } + if err := decoder.Decode(&entry); err != nil { + t.Fatal(err) + } + if entry.Message == "starting server" { + if entry.Storage != tc.want { + t.Errorf("logged storage = %q, want %q", entry.Storage, tc.want) + } + t.Logf("logged storage = %q", entry.Storage) + return + } + } + t.Fatal("startup log missing") + }) + } +} + +func startStorageLog(t *testing.T, storageURL string) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(t.TempDir(), "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(t.TempDir(), "artifacts") + var logs syncBuffer + srv, err := New(cfg, slog.New(slog.NewJSONHandler(&logs, nil)), BuildInfo{}) + if err != nil { + t.Fatal(err) + } + srv.storage = storageWithURL{Storage: srv.storage, url: storageURL} + done := make(chan error, 1) + go func() { done <- srv.Start(listener) }() + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + t.Errorf("shutting down server: %v", err) + } + if err := <-done; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("Start() = %v, want %v", err, http.ErrServerClosed) + } + }) + + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Get(cfg.BaseURL + "/health") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("health status = %d, want 200", resp.StatusCode) + } + return logs.String() +} + func TestUpstreamSafeHTTPOptions(t *testing.T) { opts := upstreamSafeHTTPOptions(config.UpstreamConfig{ AllowPrivateHosts: []string{"registry.internal"}, From c2dd16903a9c078dc2f09ea15208916234f30288 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 6 Nov 2026 09:38:07 +0000 Subject: [PATCH 089/198] Share publish indexes across cooldown profiles --- internal/handler/download_cooldown.go | 2 +- internal/handler/profile_test.go | 121 ++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/internal/handler/download_cooldown.go b/internal/handler/download_cooldown.go index c491ef3..4ede9a8 100644 --- a/internal/handler/download_cooldown.go +++ b/internal/handler/download_cooldown.go @@ -103,7 +103,7 @@ type publishIndexEntry struct { // publishIndex reuses parsed documents by digest, or for a minute without one. func publishIndex[V any](p *Proxy, key string, load func() metadataResult, build func(metadataResult) (V, error)) (V, error) { //nolint:ireturn // Supports timestamp maps and Helm lookups. - c := &p.publishIndexes + c := &p.sharedProxy().publishIndexes fresh := func() (V, bool) { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/handler/profile_test.go b/internal/handler/profile_test.go index 6377062..5caf7b8 100644 --- a/internal/handler/profile_test.go +++ b/internal/handler/profile_test.go @@ -2,16 +2,137 @@ package handler import ( "context" + "errors" "net/http" "net/http/httptest" + "reflect" "strings" "sync" + "sync/atomic" "testing" "time" "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/scanner" + registryclient "github.com/git-pkgs/registries/client" ) +func TestWithCooldownCopiesProxyFields(t *testing.T) { + p, _, _, _ := setupTestProxy(t) + p.RouteFor = func(string, string) *ArtifactRoute { return nil } + p.Cooldown = profilePolicy(t, "1d") + p.ShortestCooldown = 24 * time.Hour + p.Denylist = &denylist.Policy{} + p.CacheMetadata = true + p.MetadataTTL = time.Hour + p.MetadataMaxSize = 1024 + p.MetadataStreamMaxSize = 2048 + p.GradleReadOnly = true + p.HostedEnabled = true + p.GradleMaxUploadSize = 4096 + p.NPMFullMetadata = true + p.DirectServe = true + p.DirectServeTTL = time.Minute + p.DirectServeBaseURL = "https://storage.example.com" + p.RateLimits = registryclient.NewRateLimitGate(http.DefaultTransport) + p.HostedNPMScopes = []string{"@example"} + p.HostedComposerVendors = []string{"example"} + p.StreamArtifacts = true + p.Scanners = &scanner.Group{} + p.ScanSigningKey = []byte("signing-key") + p.ScanFetchBaseURL = "http://scanner.example.com" + p.SetMetadataRewriteCacheSize(1024) + p.inFlight = map[string]*inflightFetch{"artifact": {}} + p.detachSlots = make(chan struct{}, 1) + p.inFlightMeta = map[string]*inflightMetadata{"metadata": {}} + p.publishIndexes.entries = map[string]publishIndexEntry{"document": {}} + p.cooldownWatch.active = map[*cooldownRecorder]struct{}{{}: {}} + + policy := profilePolicy(t, "3d") + view := p.WithCooldown(policy, 72*time.Hour) + if view.Cooldown != policy || view.ShortestCooldown != 72*time.Hour || view.shared != p { + t.Fatal("view did not select its policy and shared proxy") + } + original, copied := reflect.ValueOf(p).Elem(), reflect.ValueOf(view).Elem() + for i := range original.NumField() { + name := original.Type().Field(i).Name + before, after := original.Field(i), copied.Field(i) + switch name { + case "Cooldown", "ShortestCooldown", "shared": + continue + case "fetchMu", "inFlight", "detachSlots", "metaMu", "inFlightMeta", "publishIndexes", "cooldownWatch": + if !after.IsZero() { + t.Errorf("view field %s is not empty", name) + } + continue + } + if before.IsZero() { + t.Errorf("populate Proxy.%s to check whether WithCooldown copies it", name) + continue + } + switch before.Kind() { + case reflect.Func, reflect.Slice: + if before.Pointer() != after.Pointer() { + t.Errorf("WithCooldown did not copy %s", name) + } + if before.Kind() == reflect.Slice && (before.Len() != after.Len() || before.Cap() != after.Cap()) { + t.Errorf("WithCooldown changed slice bounds for %s", name) + } + default: + if !before.Equal(after) { + t.Errorf("WithCooldown did not copy %s", name) + } + } + } +} + +func TestProfilesShareDownloadPublishIndexes(t *testing.T) { + for _, c := range downloadCooldownCases() { + if c.ecosystem != "conda" && c.ecosystem != "helm" { + continue + } + t.Run(c.ecosystem, func(t *testing.T) { + var calls atomic.Int64 + times := map[string]time.Time{cooldownRecentVersion: time.Now().Add(-4 * 24 * time.Hour)} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + c.metadata(w, r, times) + })) + t.Cleanup(upstream.Close) + p, _, _, fetcher := setupTestProxy(t) + p.HTTPClient = upstream.Client() + p.Cooldown = profilePolicy(t, "1d") + fetcher.fetchErr = errors.New("artifact must be served from the cache") + c.seedCachedArtifact(t, p, cooldownRecentVersion) + permissive := p.WithCooldown(profilePolicy(t, "3d"), 72*time.Hour) + strict := p.WithCooldown(profilePolicy(t, "7d"), 168*time.Hour) + nested := permissive.WithCooldown(profilePolicy(t, "5d"), 120*time.Hour) + for _, request := range []struct { + proxy *Proxy + want int + }{ + {permissive, http.StatusOK}, + {strict, http.StatusNotFound}, + {p, http.StatusOK}, + {nested, http.StatusNotFound}, + } { + w := serveGet(t, c.handler(request.proxy, upstream.URL), c.path(cooldownRecentVersion)) + wantStatus(t, w, request.want, "") + } + if n := calls.Load(); n != 1 { + t.Errorf("index downloads across profiles = %d, want 1", n) + } + if n := len(p.publishIndexes.entries); n != 1 { + t.Errorf("shared publish indexes = %d, want 1", n) + } + if fetcher.fetchCalled { + t.Error("cached artifact was fetched again") + } + }) + } +} + func profilePolicy(t *testing.T, duration string) *cooldown.Policy { t.Helper() policy, err := cooldown.NewPolicy(&cooldown.Config{Default: duration}, nil) From abe94017a1bfb6e1547e21b1cf43424db023c057 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 6 Nov 2026 11:36:42 +0100 Subject: [PATCH 090/198] Renew wanted refs of mirrored commits and count refreshes that fetched Mirroring a commit on no branch or tag again writes a fresh wanted ref, so its 7 days count from the last mirror run. A cached repository whose refresh fetched new commits is no longer reported as cached. --- docs/configuration.md | 2 +- internal/gitcache/mirror.go | 53 +++++++++++++++------ internal/gitcache/mirror_test.go | 79 ++++++++++++++++++++++++++++++++ internal/gitcache/wants.go | 29 ++++++++---- internal/handler/git_mirror.go | 4 +- 5 files changed, 140 insertions(+), 27 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index b2b830f..540078c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1887,7 +1887,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl ``` - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. -- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, for seven days or until a branch or tag includes it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag fails. +- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag fails. - Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index 854ea08..62ae932 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "strings" + "time" ) // MirrorResult is what Mirror did. @@ -47,42 +49,65 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa } if commit != "" { - fetched, err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants) - if err != nil { + if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { return MirrorResult{}, err } - cached = cached && !fetched } // A refresh that failed leaves the commit cached all the same. if refreshErr != nil && commit == "" { return MirrorResult{}, refreshErr } - return MirrorResult{Cached: cached, Bytes: max(rp.size.Load()-before, 0)}, nil + grown := max(rp.size.Load()-before, 0) + return MirrorResult{Cached: cached && grown == 0, Bytes: grown}, nil } // ensureCommit makes sure the repository has commit, fetching it by ID when -// it doesn't and fetchWants allows it, and reports whether it was fetched. -func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) (bool, error) { +// it doesn't and fetchWants allows it. A commit on no branch or tag that is +// cached already gets a new wanted ref, so it is kept for wantedMaxAge from +// this run on. +func (c *Cache) ensureCommit(ctx context.Context, rp *repo, repoURL, commit string, fetchWants bool) error { missing, err := c.missingObjects(ctx, rp.dir, []string{commit}) - if err != nil || len(missing) == 0 { - return false, err + if err != nil { + return err + } + if len(missing) == 0 { + return c.renewWanted(ctx, rp, commit) } if !fetchWants { - return false, fmt.Errorf("commit %s: %w", commit, ErrNotFound) + return fmt.Errorf("commit %s: %w", commit, ErrNotFound) } if err := c.fetchMissingWants(ctx, rp, repoURL, missing); err != nil { if ctx.Err() != nil || errors.Is(err, ErrTooLarge) || errors.Is(err, ErrCacheFull) || errors.Is(err, errUpstreamUnanswered) { - return false, err + return err } // The upstream refuses a commit it doesn't have. - return false, fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err) + return fmt.Errorf("commit %s: %w: %w", commit, ErrNotFound, err) } // fetchMissingWants skips a commit that failed moments ago. if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil { - return false, err + return err } if len(missing) > 0 { - return false, fmt.Errorf("commit %s: %w", commit, ErrNotFound) + return fmt.Errorf("commit %s: %w", commit, ErrNotFound) } - return true, nil + return nil +} + +// renewWanted writes a new wanted ref for commit when no branch or tag +// includes it. +func (c *Cache) renewWanted(ctx context.Context, rp *repo, commit string) error { + unlock, err := rp.lockWrites(ctx) + if err != nil { + return err + } + defer unlock() + unmerged, err := c.unmergedCommits(ctx, rp.dir, []string{commit}) + if err != nil || !unmerged[commit] { + return err + } + cmd := c.command(ctx, nil, "-C", rp.dir, "update-ref", wantedRef(commit, time.Now()), commit) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("renewing wanted ref: %w: %s", err, strings.TrimSpace(string(out))) + } + return nil } diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index a1fd07e..f87b322 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -3,8 +3,12 @@ package gitcache import ( "context" "errors" + "path/filepath" + "slices" + "strconv" "strings" "testing" + "time" ) func TestMirrorClonesRepository(t *testing.T) { @@ -106,3 +110,78 @@ func TestMirrorRefusesInvalidCommit(t *testing.T) { t.Error("mirrored a branch name as a commit") } } + +func TestMirrorCountsRefreshThatFetched(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true); err != nil { + t.Fatalf("Mirror: %v", err) + } + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached || got.Bytes <= 0 { + t.Errorf("mirror after a push = %+v, want fetched bytes", got) + } +} + +// TestMirrorRenewsWantedCommit checks mirroring a commit on no branch again +// makes its 7 days count from that run, not from the first fetch. +func TestMirrorRenewsWantedCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + cached := filepath.Join(tc.dir, "up", "repo.git") + for _, ref := range tc.wantedRefs(t) { + runGit(t, cached, "update-ref", "-d", ref) + } + old := wantedRef(sha, time.Now().Add(-wantedMaxAge+time.Hour)) + runGit(t, cached, "update-ref", old, sha) + + start := time.Now().Add(-time.Second) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true) + if err != nil || !got.Cached { + t.Fatalf("second mirror = %+v, %v, want cached", got, err) + } + if !slices.ContainsFunc(tc.wantedRefs(t), func(ref string) bool { + return ref != old && wantedRefTime(t, ref, sha).After(start) + }) { + t.Errorf("wanted refs %v, want one written by the second mirror", tc.wantedRefs(t)) + } +} + +func TestMirrorDoesNotPinCommitOnABranch(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + sha := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + for range 2 { + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", sha, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + } + if refs := tc.wantedRefs(t); len(refs) != 0 { + t.Errorf("wanted refs %v, want none for a commit on a branch", refs) + } +} + +func wantedRefTime(t *testing.T, ref, oid string) time.Time { + t.Helper() + created, got, _ := strings.Cut(strings.TrimPrefix(ref, wantedPrefix), "/") + unix, err := strconv.ParseInt(created, 10, 64) + if err != nil || got != oid { + return time.Time{} + } + return time.Unix(unix, 0) +} diff --git a/internal/gitcache/wants.go b/internal/gitcache/wants.go index c715f2f..8755ba1 100644 --- a/internal/gitcache/wants.go +++ b/internal/gitcache/wants.go @@ -161,19 +161,10 @@ func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fe } var mergedErr error if fetched && len(keep) > 0 { - // rev-list prints what the wanted commits, read from stdin, reach - // and no branch or tag does, so a wanted commit it leaves out is - // merged. - cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") - cmd.Stdin = strings.NewReader(strings.Join(slices.Sorted(maps.Keys(keep)), "\n") + "\n") - out, err := cmd.Output() + unmerged, err := c.unmergedCommits(ctx, dir, slices.Sorted(maps.Keys(keep))) if err != nil { mergedErr = fmt.Errorf("listing unmerged wanted commits: %w", err) } else { - unmerged := make(map[string]bool) - for line := range strings.Lines(string(out)) { - unmerged[strings.TrimSpace(line)] = true - } for oid, ref := range keep { if !unmerged[oid] { drop = append(drop, ref) @@ -196,6 +187,24 @@ func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fe return mergedErr } +// unmergedCommits returns which of commits no cached branch or tag +// includes. +func (c *Cache) unmergedCommits(ctx context.Context, dir string, commits []string) (map[string]bool, error) { + // rev-list prints what the commits, read from stdin, reach and no + // branch or tag does, so a commit it leaves out is merged. + cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") + cmd.Stdin = strings.NewReader(strings.Join(commits, "\n") + "\n") + out, err := cmd.Output() + if err != nil { + return nil, err + } + unmerged := make(map[string]bool) + for line := range strings.Lines(string(out)) { + unmerged[strings.TrimSpace(line)] = true + } + return unmerged, nil +} + // withoutFailedWants drops the object IDs a recent fetch for the // repository could not get. func (c *Cache) withoutFailedWants(key string, oids []string) []string { diff --git a/internal/handler/git_mirror.go b/internal/handler/git_mirror.go index eca76c7..709df31 100644 --- a/internal/handler/git_mirror.go +++ b/internal/handler/git_mirror.go @@ -11,8 +11,8 @@ import ( // Mirror caches the repository at location, a host and path such as // github.com/acme/widgets as a pkg:git PURL names it, from the upstream.git -// host serving it, and with commit, a full commit ID, that commit too. It -// is the copy clones through /git are then served. +// host serving it, and with commit, a full commit ID, that commit too. +// Clones through /git are then served from that copy. func (h *GitHandler) Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) { repo, err := h.mirrorRepository(location) if err != nil { From 90ba3f9fd116129348eae34941be9450cb4da85c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 6 Nov 2026 11:44:22 +0100 Subject: [PATCH 091/198] Say which commits fail to mirror without fetching by ID --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 540078c..2b119c2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1887,7 +1887,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl ``` - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. -- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag fails. +- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. - Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. From c037fb9e8b94cb0c5580c6903b72050b2afff39a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 6 Nov 2026 12:04:14 +0100 Subject: [PATCH 092/198] Note that git.max_size can evict a mirrored repository sooner --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2b119c2..d447c0c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1887,7 +1887,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl ``` - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. -- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. +- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. - Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. From b0566aa4568277323a8c84dc7d6b9618398edf49 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 6 Nov 2026 13:00:31 +0100 Subject: [PATCH 093/198] Fail a mirror whose refresh is past max_repo_size The repository is dropped once released, so a cached commit in it would not be served for long. --- internal/gitcache/mirror.go | 5 +++++ internal/gitcache/mirror_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index 62ae932..b5bde40 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -48,6 +48,11 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa } } + // A refresh past max_repo_size gets the repository dropped once it is + // released, commit and all. + if errors.Is(refreshErr, ErrTooLarge) { + return MirrorResult{}, refreshErr + } if commit != "" { if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { return MirrorResult{}, err diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index f87b322..86f652f 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -3,6 +3,7 @@ package gitcache import ( "context" "errors" + "os" "path/filepath" "slices" "strconv" @@ -185,3 +186,29 @@ func wantedRefTime(t *testing.T, ref, oid string) time.Time { } return time.Unix(unix, 0) } + +// TestMirrorReportsRefreshTooLarge checks a commit that is cached still +// fails to mirror when the refresh doesn't fit in max_repo_size, since the +// repository is dropped once released. +func TestMirrorReportsRefreshTooLarge(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + pinned := upstream.pushPullRequest(work, 1, randomBytes(100<<10)) + base := repoSize(t, upstream, "repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 150<<10 }) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", pinned, true); err != nil { + t.Fatalf("Mirror: %v", err) + } + if err := os.WriteFile(filepath.Join(work, "grown.bin"), randomBytes(80<<10), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "grown.bin") + upstream.commit(work, "grown") + runGit(t, work, "push", "--quiet", "origin", "main") + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", pinned, true) + if !errors.Is(err, ErrTooLarge) { + t.Errorf("mirror with a refresh past max_repo_size = %v, want ErrTooLarge", err) + } +} From 292ecc148843699bc0424486404e0956ed51cc64 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Fri, 6 Nov 2026 22:36:47 +0000 Subject: [PATCH 094/198] Correct package manager configuration examples --- README.md | 34 ++++++++++++++++++---------------- docs/configuration.md | 2 +- internal/server/dashboard.go | 27 ++++++++++----------------- 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index be4f0ee..a43ae27 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi Cooldown requires publish timestamps in metadata. Registries without a "Yes" in the cooldown column either don't expose timestamps or haven't been wired up yet. -\* Hex cooldown requires disabling registry signature verification (`HEX_NO_VERIFY_REPO_ORIGIN=1`) since the proxy re-encodes the protobuf payload. +\* Hex cooldown requires disabling registry signature verification (`HEX_UNSAFE_REGISTRY=1`) since the proxy re-encodes the protobuf payload. Origin verification can stay enabled. † Go cooldown uses the time in each version's `.info`, which is the commit time of the tagged revision, not when the tag was pushed. @@ -288,10 +288,10 @@ The equivalent environment variables are `PROXY_UPSTREAM_HOMEBREW_API` and `PROX ### Hex (Elixir) -Configure in `~/.hex/hex.config`: +Configure the mirror: -```erlang -{default_url, <<"http://localhost:8080/hex">>}. +```bash +mix hex.config mirror_url http://localhost:8080/hex ``` Or set the environment variable: @@ -300,6 +300,14 @@ Or set the environment variable: export HEX_MIRROR=http://localhost:8080/hex ``` +When the proxy applies a cooldown, also set: + +```bash +export HEX_UNSAFE_REGISTRY=1 +``` + +Filtering releases removes the registry signature. The proxy preserves the package name and repository fields, so origin verification can stay enabled. Without a cooldown, leave signature verification enabled too. + ### pub.dev (Dart/Flutter) Set the PUB_HOSTED_URL environment variable: @@ -359,17 +367,15 @@ Configure in `settings.gradle.kts`: ```kotlin buildCache { - local { - enabled = false - } remote { url = uri("http://localhost:8080/gradle/") - isPush = System.getenv("CI") != null } } ``` -The proxy rejects uploads unless `gradle.build_cache.read_only` is set to `false`, see [docs/configuration.md](docs/configuration.md#gradle-build-cache). +Enable build caching with `org.gradle.caching=true` in `gradle.properties`, or pass `--build-cache` when running Gradle. The local cache stays enabled and remote uploads are disabled by default. + +To upload from CI, set `gradle.build_cache.read_only` to `false` on the proxy and add `isPush = System.getenv("CI") != null` inside the client's `remote` block. See [docs/configuration.md](docs/configuration.md#gradle-build-cache). ### NuGet @@ -459,8 +465,6 @@ Configure in `~/.condarc`: channels: - http://localhost:8080/conda/main - http://localhost:8080/conda/conda-forge -default_channels: - - http://localhost:8080/conda/main ``` Or set via command: @@ -716,16 +720,14 @@ indexes unchanged. To keep APT from selecting a denied version, configure ### RPM / Yum / DNF -Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`: +In the `[fedora]` section of your existing repository file under `/etc/yum.repos.d/`, replace the `metalink` or `mirrorlist` entry with: ```ini -[proxy-fedora] -name=Fedora via Proxy baseurl=http://localhost:8080/rpm/releases/$releasever/Everything/$basearch/os/ -enabled=1 -gpgcheck=0 ``` +Keep the existing `gpgkey` and signature settings, including `gpgcheck=1`. The proxy serves RPM packages unchanged, so their signatures remain valid. + Then: ```bash diff --git a/docs/configuration.md b/docs/configuration.md index 63cc175..accaefb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1244,7 +1244,7 @@ With cooldown on, the proxy asks npm for full packuments, since the abbreviated Conda cooldown covers `repodata.json`, `current_repodata.json`, `repodata_from_packages.json` and sharded repodata (CEP 16). Other forms of the repodata, such as `repodata.json.zst`, `repodata.json.bz2` and `repodata.jlap` patches, would list versions still in cooldown, so with cooldown on they are answered 404 and conda and mamba fall back to `repodata.json`. With `cache_metadata`, `repodata.json` is read through the metadata cache as it is without cooldown: within `metadata_ttl` the stored copy is filtered as it is read from storage, after it the upstream is asked with its ETag, and while the upstream is down the stale copy is filtered. Once a document has been filtered, the proxy remembers the ETag of the result until the first entry it left out leaves cooldown, so a client revalidating with `If-None-Match` gets `304 Not Modified` without the document being filtered or sent again. The first response for each document has no ETag, as it is sent while it is filtered. Without `cache_metadata`, `repodata.json` is fetched and filtered on every request. Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. -Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. +Hex cooldown requires `HEX_UNSAFE_REGISTRY=1` because filtering releases re-encodes the protobuf payload without the original signature. The proxy preserves the package name and repository fields, so origin verification can stay enabled. `HEX_NO_VERIFY_REPO_ORIGIN` controls origin verification and does not disable signature verification. Without a cooldown, leave signature verification enabled too. ## Version denylist diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index 217f018..f566d55 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -324,11 +324,11 @@ bundle config mirror.https://rubygems.org ` + baseURL + `/gem`), Language: "Elixir", Endpoint: "/hex/", Instructions: template.HTML(`

Configure Hex to use the proxy:

-
# In ~/.hex/hex.config
-{default_url, <<"` + baseURL + `/hex">>}.
+
mix hex.config mirror_url ` + baseURL + `/hex
 
 # Or via environment variable
-export HEX_MIRROR=` + baseURL + `/hex
`),
+export HEX_MIRROR=` + baseURL + `/hex
+

When the proxy applies a cooldown, also set HEX_UNSAFE_REGISTRY=1 to disable registry signature verification. Origin verification can stay enabled.

`), }, { ID: "pub", @@ -378,10 +378,10 @@ index-url = ` + baseURL + `/pypi/simple/`), buildCache { remote<HttpBuildCache> { url = uri("` + baseURL + `/gradle/") - isPush = System.getenv("CI") != null } } -

Uploads need gradle.build_cache.read_only: false on the proxy.

`), +

Enable build caching with org.gradle.caching=true in gradle.properties, or pass --build-cache when running Gradle. The local cache stays enabled and remote uploads are disabled by default.

+

To upload from CI, set gradle.build_cache.read_only: false on the proxy and add isPush = System.getenv("CI") != null inside the client's remote<HttpBuildCache> block.

`), }, { ID: "nuget", @@ -448,8 +448,6 @@ conan remote disable conancenter`), channels: - ` + baseURL + `/conda/main - ` + baseURL + `/conda/conda-forge -default_channels: - - ` + baseURL + `/conda/main # Or via command conda config --add channels ` + baseURL + `/conda/main`), @@ -523,17 +521,12 @@ sudo apt update`), { ID: "rpm", Name: "RPM/Yum", - Language: "Fedora/RHEL", + Language: "Fedora", Endpoint: "/rpm/", - Instructions: template.HTML(`

Configure yum/dnf to use the proxy:

-
# In /etc/yum.repos.d/proxy.repo
-[proxy-fedora]
-name=Fedora via Proxy
-baseurl=` + baseURL + `/rpm/releases/$releasever/Everything/$basearch/os/
-enabled=1
-gpgcheck=0
-
-# Then run:
+			Instructions: template.HTML(`
In the [fedora] section of your existing repository file under /etc/yum.repos.d/, replace the metalink or mirrorlist entry with:

+
baseurl=` + baseURL + `/rpm/releases/$releasever/Everything/$basearch/os/

+
Keep the existing gpgkey and signature settings, including gpgcheck=1. The proxy serves RPM packages unchanged.

+
# Then run:
 sudo dnf clean all
 sudo dnf update
`),
 		},

From 909e2e464abe1a3193ef31bbf2f72c50c6b0fac5 Mon Sep 17 00:00:00 2001
From: Milas Bowman 
Date: Mon, 19 Oct 2026 00:02:05 -0400
Subject: [PATCH 095/198] fix(s3): signatures for alternate hosts

---
 config.example.yaml              |  9 ++--
 internal/handler/handler.go      | 28 +---------
 internal/handler/handler_test.go | 68 -------------------------
 internal/server/server.go        |  7 ++-
 internal/storage/blob.go         | 87 +++++++++++++++++++++++++++++---
 internal/storage/blob_test.go    | 62 +++++++++++++++++++++++
 6 files changed, 151 insertions(+), 110 deletions(-)

diff --git a/config.example.yaml b/config.example.yaml
index e43e434..1967213 100644
--- a/config.example.yaml
+++ b/config.example.yaml
@@ -133,12 +133,9 @@ storage:
   # How long presigned URLs remain valid (e.g. "5m", "1h"). Default: "15m".
   direct_serve_ttl: "15m"
 
-  # Public base URL to substitute into presigned URLs. Set this when the
-  # proxy reaches storage at an internal address (127.0.0.1, a Docker
-  # service name) but clients must use a public hostname. Only scheme and
-  # host are used; the signed path and query are preserved. For S3/MinIO
-  # the reverse proxy at this address must forward requests with the
-  # internal Host header or the SigV4 signature will not validate.
+  # Public base URL for presigned URLs. Set this when the proxy reaches
+  # storage at an internal address (127.0.0.1, a Docker service name, or
+  # VPC DNS) but clients must use a public hostname or CDN.
   # direct_serve_base_url: "https://minio.example.com"
 
 # Database configuration
diff --git a/internal/handler/handler.go b/internal/handler/handler.go
index 125b117..6d38118 100644
--- a/internal/handler/handler.go
+++ b/internal/handler/handler.go
@@ -240,11 +240,7 @@ type Proxy struct {
 	NPMFullMetadata bool
 	DirectServe     bool
 	DirectServeTTL  time.Duration
-	// DirectServeBaseURL, if set, replaces the scheme and host of presigned
-	// URLs so clients receive a public address even when the proxy reaches
-	// storage at an internal one.
-	DirectServeBaseURL string
-	HTTPClient         *http.Client
+	HTTPClient      *http.Client
 
 	// RateLimits is the gate under the upstream clients. Artifact fetches
 	// check it before calling Fetcher, so held hosts never reach its breaker.
@@ -548,7 +544,7 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s
 	if p.DirectServe {
 		signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL)
 		if err == nil {
-			result.RedirectURL = rewriteSignedURLHost(signed, p.DirectServeBaseURL)
+			result.RedirectURL = signed
 			p.recordCacheHit(artifact.Ecosystem, versionPURL, filename)
 			return result, nil
 		}
@@ -595,26 +591,6 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s
 	return result, nil
 }
 
-// rewriteSignedURLHost replaces the scheme and host of a signed URL with those
-// from baseURL, preserving the path and query (which carry the signature).
-// Returns signed unchanged if baseURL is empty or either URL fails to parse.
-func rewriteSignedURLHost(signed, baseURL string) string {
-	if baseURL == "" {
-		return signed
-	}
-	s, err := url.Parse(signed)
-	if err != nil {
-		return signed
-	}
-	b, err := url.Parse(baseURL)
-	if err != nil || b.Scheme == "" || b.Host == "" {
-		return signed
-	}
-	s.Scheme = b.Scheme
-	s.Host = b.Host
-	return s.String()
-}
-
 func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) {
 	_ = p.DB.RecordArtifactHit(versionPURL, filename)
 	metrics.RecordArtifactCacheHit(ecosystem)
diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go
index 8ce636c..3ae885e 100644
--- a/internal/handler/handler_test.go
+++ b/internal/handler/handler_test.go
@@ -638,74 +638,6 @@ func TestGetOrFetchArtifact_DirectServe_Redirect(t *testing.T) {
 	}
 }
 
-func TestGetOrFetchArtifact_DirectServe_BaseURLRewrite(t *testing.T) {
-	proxy, db, store, _ := setupTestProxy(t)
-	seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content")
-
-	proxy.DirectServe = true
-	proxy.DirectServeBaseURL = "https://cdn.example.com"
-	store.signedURL = "http://127.0.0.1:9000/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900"
-
-	result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz")
-	if err != nil {
-		t.Fatalf("unexpected error: %v", err)
-	}
-
-	want := "https://cdn.example.com/bucket/npm/lodash?X-Amz-Signature=abc&X-Amz-Expires=900"
-	if result.RedirectURL != want {
-		t.Errorf("RedirectURL = %q, want %q", result.RedirectURL, want)
-	}
-}
-
-func TestRewriteSignedURLHost(t *testing.T) {
-	tests := []struct {
-		name    string
-		signed  string
-		baseURL string
-		want    string
-	}{
-		{
-			"empty base url is no-op",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-			"",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-		},
-		{
-			"replaces scheme and host",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-			"https://cdn.example.com",
-			"https://cdn.example.com/bucket/key?sig=abc",
-		},
-		{
-			"preserves path and query",
-			"http://minio:9000/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101",
-			"https://files.example.com",
-			"https://files.example.com/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101",
-		},
-		{
-			"ignores base url path",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-			"https://cdn.example.com/ignored",
-			"https://cdn.example.com/bucket/key?sig=abc",
-		},
-		{
-			"invalid base url is no-op",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-			"://bad",
-			"http://127.0.0.1:9000/bucket/key?sig=abc",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := rewriteSignedURLHost(tt.signed, tt.baseURL)
-			if got != tt.want {
-				t.Errorf("rewriteSignedURLHost(%q, %q) = %q, want %q", tt.signed, tt.baseURL, got, tt.want)
-			}
-		})
-	}
-}
-
 func TestGetOrFetchArtifact_DirectServe_FallbackOnUnsupported(t *testing.T) {
 	proxy, db, store, _ := setupTestProxy(t)
 	seedPackage(t, db, store, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", "cached content")
diff --git a/internal/server/server.go b/internal/server/server.go
index 55fe597..4466d06 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -189,7 +189,11 @@ func New(cfg *config.Config, logger *slog.Logger, buildInfo BuildInfo) (*Server,
 		// Fall back to file:// with Path
 		storageURL = "file://" + cfg.Storage.Path //nolint:staticcheck // backwards compat
 	}
-	store, err := storage.OpenBucket(context.Background(), storageURL)
+	var storageOpts []storage.Option
+	if cfg.Storage.DirectServe && cfg.Storage.DirectServeBaseURL != "" {
+		storageOpts = append(storageOpts, storage.WithSignEndpoint(cfg.Storage.DirectServeBaseURL))
+	}
+	store, err := storage.OpenBucket(context.Background(), storageURL, storageOpts...)
 	if err != nil {
 		_ = db.Close()
 		return nil, fmt.Errorf("initializing storage: %w", err)
@@ -317,7 +321,6 @@ func (s *Server) serve(listener net.Listener) error {
 	proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize()
 	proxy.DirectServe = s.cfg.Storage.DirectServe
 	proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL()
-	proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL
 	proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts
 	if err := ConfigureDenylist(s.cfg, proxy); err != nil {
 		return err
diff --git a/internal/storage/blob.go b/internal/storage/blob.go
index 81af78f..cf3ee0f 100644
--- a/internal/storage/blob.go
+++ b/internal/storage/blob.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -30,11 +31,13 @@ const osWindows = "windows"
 const (
 	uploadBufferSize  = 5 << 20
 	uploadConcurrency = 2
-)
 
-// attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an
-// earlier version wrote.
-const attrsExt = ".attrs"
+	s3Scheme = "s3://"
+
+	// attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an
+	// earlier version wrote.
+	attrsExt = ".attrs"
+)
 
 // Blob implements Storage using gocloud.dev/blob.
 // Supports local filesystem (file://) and S3 (s3://) URLs.
@@ -42,11 +45,29 @@ type Blob struct {
 	bucket *blob.Bucket
 	url    string
 
+	// signBucket, when set, is used by SignedURL to sign S3 presigned URLs for
+	// a public endpoint rather than the internal storage endpoint.
+	signBucket *blob.Bucket
+
 	// fileRoot is the directory backing a file:// bucket, empty for cloud
 	// backends. Used only to clear sidecars an earlier version wrote.
 	fileRoot string
 }
 
+// Option configures OpenBucket.
+type Option func(*options)
+
+type options struct {
+	signEndpoint string
+}
+
+// WithSignEndpoint configures S3 presigned URLs to be signed for the scheme and
+// host of signEndpoint (e.g. "https://storage.example.com") instead of the
+// storage endpoint, since SigV4 includes the Host header in the signature.
+func WithSignEndpoint(signEndpoint string) Option {
+	return func(o *options) { o.signEndpoint = signEndpoint }
+}
+
 // OpenBucket opens a blob bucket from a URL.
 //
 // Supported URL schemes:
@@ -60,7 +81,12 @@ type Blob struct {
 // For local filesystem, the directory is created if it doesn't exist.
 //
 //nolint:ireturn // The URL scheme selects the storage implementation.
-func OpenBucket(ctx context.Context, urlStr string) (Storage, error) {
+func OpenBucket(ctx context.Context, urlStr string, opts ...Option) (Storage, error) {
+	var o options
+	for _, opt := range opts {
+		opt(&o)
+	}
+
 	if strings.HasPrefix(urlStr, "gs://") {
 		return OpenGCS(ctx, urlStr)
 	}
@@ -124,7 +150,42 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) {
 		return nil, fmt.Errorf("opening bucket: %w", err)
 	}
 
-	return &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot}, nil
+	b := &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot}
+
+	if o.signEndpoint != "" && strings.HasPrefix(urlStr, s3Scheme) {
+		signBucket, err := openSignBucket(ctx, urlStr, o.signEndpoint)
+		if err != nil {
+			_ = bucket.Close()
+			return nil, err
+		}
+		b.signBucket = signBucket
+	}
+
+	return b, nil
+}
+
+// openSignBucket opens an S3 bucket with the endpoint overridden to signEndpoint
+// while preserving all other storage URL parameters (region, use_path_style, etc.).
+func openSignBucket(ctx context.Context, urlStr, signEndpoint string) (*blob.Bucket, error) {
+	sign, err := url.Parse(signEndpoint)
+	if err != nil || sign.Scheme == "" || sign.Host == "" {
+		return nil, fmt.Errorf("invalid sign endpoint %q: must be an absolute URL", signEndpoint)
+	}
+
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return nil, fmt.Errorf("parsing storage URL: %w", err)
+	}
+	endpoint := sign.Scheme + "://" + sign.Host
+	q := u.Query()
+	q.Set("endpoint", endpoint)
+	u.RawQuery = q.Encode()
+
+	bucket, err := blob.OpenBucket(ctx, u.String())
+	if err != nil {
+		return nil, fmt.Errorf("opening bucket at sign endpoint %s: %w", endpoint, err)
+	}
+	return bucket, nil
 }
 
 // legacySidecarPath gives the ".attrs" path an earlier version wrote for key,
@@ -299,7 +360,11 @@ func (b *Blob) Delete(ctx context.Context, path string) error {
 }
 
 func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
-	url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{
+	bucket := b.bucket
+	if b.signBucket != nil {
+		bucket = b.signBucket
+	}
+	url, err := bucket.SignedURL(ctx, path, &blob.SignedURLOptions{
 		Method: http.MethodGet,
 		Expiry: expiry,
 	})
@@ -371,7 +436,13 @@ func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, err
 }
 
 func (b *Blob) Close() error {
-	return b.bucket.Close()
+	err := b.bucket.Close()
+	if b.signBucket != nil {
+		if signErr := b.signBucket.Close(); err == nil {
+			err = signErr
+		}
+	}
+	return err
 }
 
 func (b *Blob) URL() string {
diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go
index 1a764de..4811c7e 100644
--- a/internal/storage/blob_test.go
+++ b/internal/storage/blob_test.go
@@ -7,6 +7,7 @@ import (
 	"errors"
 	"io"
 	"io/fs"
+	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -260,6 +261,67 @@ func TestBlobSignedURLUnsupported(t *testing.T) {
 	}
 }
 
+func TestBlobSignedURLSignEndpoint(t *testing.T) {
+	t.Setenv("AWS_ACCESS_KEY_ID", "test")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "test")
+
+	const (
+		storageURL   = "s3://bucket?region=us-1&endpoint=http://127.0.0.1:9000&use_path_style=true"
+		signEndpoint = "https://cdn.example.com"
+		artifactPath = "npm/lodash/4.17.21/lodash-4.17.21.tgz"
+	)
+
+	tests := []struct {
+		name string
+		opts []Option
+		host string
+	}{
+		{
+			"storage endpoint without a sign endpoint",
+			nil,
+			"127.0.0.1:9000",
+		},
+		{
+			"configured sign endpoint, reached over another host",
+			[]Option{WithSignEndpoint(signEndpoint)},
+			"cdn.example.com",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+
+			b, err := OpenBucket(ctx, storageURL, tt.opts...)
+			if err != nil {
+				t.Fatalf("OpenBucket failed: %v", err)
+			}
+			defer func() { _ = b.Close() }()
+
+			signed, err := b.SignedURL(ctx, artifactPath, time.Minute)
+			if err != nil {
+				t.Fatalf("SignedURL failed: %v", err)
+			}
+
+			u, err := url.Parse(signed)
+			if err != nil {
+				t.Fatalf("parsing signed URL %q: %v", signed, err)
+			}
+			if u.Host != tt.host {
+				t.Errorf("signed URL %q host = %q, want %q", signed, u.Host, tt.host)
+			}
+			// The bucket and key stay where the storage URL puts them, so the
+			// request the client makes still names the object.
+			if want := "/bucket/" + artifactPath; u.Path != want {
+				t.Errorf("signed URL path = %q, want %q", u.Path, want)
+			}
+			if u.Query().Get("X-Amz-Signature") == "" {
+				t.Errorf("signed URL %q carries no signature", signed)
+			}
+		})
+	}
+}
+
 func TestBlobOverwrite(t *testing.T) {
 	b := createTestBlob(t)
 	ctx := context.Background()

From f82ec690068dce9ec4e5edcad407ea35795b8025 Mon Sep 17 00:00:00 2001
From: Milas Bowman 
Date: Fri, 6 Nov 2026 20:06:36 -0500
Subject: [PATCH 096/198] fix(storage): restore Azure/GCS direct serve signing

---
 internal/storage/blob.go      | 62 +++++++++++++++++++++------
 internal/storage/blob_test.go | 79 +++++++++++++++++++++++++++++++++++
 internal/storage/gcs.go       | 10 +++--
 internal/storage/gcs_test.go  | 56 +++++++++++++++++++++++++
 4 files changed, 192 insertions(+), 15 deletions(-)

diff --git a/internal/storage/blob.go b/internal/storage/blob.go
index cf3ee0f..eeb8b3b 100644
--- a/internal/storage/blob.go
+++ b/internal/storage/blob.go
@@ -49,6 +49,10 @@ type Blob struct {
 	// a public endpoint rather than the internal storage endpoint.
 	signBucket *blob.Bucket
 
+	// signEndpoint, when set on non-S3 backends, replaces the scheme and host
+	// of presigned URLs returned by SignedURL.
+	signEndpoint string
+
 	// fileRoot is the directory backing a file:// bucket, empty for cloud
 	// backends. Used only to clear sidecars an earlier version wrote.
 	fileRoot string
@@ -61,9 +65,11 @@ type options struct {
 	signEndpoint string
 }
 
-// WithSignEndpoint configures S3 presigned URLs to be signed for the scheme and
-// host of signEndpoint (e.g. "https://storage.example.com") instead of the
-// storage endpoint, since SigV4 includes the Host header in the signature.
+// WithSignEndpoint configures presigned URLs to use the scheme and host of
+// signEndpoint (e.g. "https://storage.example.com") instead of the storage
+// endpoint. For S3, the URL is signed directly against signEndpoint so the
+// SigV4 Host signature matches; for other backends, the signed URL's scheme
+// and host are rewritten.
 func WithSignEndpoint(signEndpoint string) Option {
 	return func(o *options) { o.signEndpoint = signEndpoint }
 }
@@ -88,7 +94,12 @@ func OpenBucket(ctx context.Context, urlStr string, opts ...Option) (Storage, er
 	}
 
 	if strings.HasPrefix(urlStr, "gs://") {
-		return OpenGCS(ctx, urlStr)
+		g, err := OpenGCS(ctx, urlStr)
+		if err != nil {
+			return nil, err
+		}
+		g.signEndpoint = o.signEndpoint
+		return g, nil
 	}
 
 	var fileRoot string
@@ -152,13 +163,17 @@ func OpenBucket(ctx context.Context, urlStr string, opts ...Option) (Storage, er
 
 	b := &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot}
 
-	if o.signEndpoint != "" && strings.HasPrefix(urlStr, s3Scheme) {
-		signBucket, err := openSignBucket(ctx, urlStr, o.signEndpoint)
-		if err != nil {
-			_ = bucket.Close()
-			return nil, err
+	if o.signEndpoint != "" {
+		if strings.HasPrefix(urlStr, s3Scheme) {
+			signBucket, err := openSignBucket(ctx, urlStr, o.signEndpoint)
+			if err != nil {
+				_ = bucket.Close()
+				return nil, err
+			}
+			b.signBucket = signBucket
+		} else {
+			b.signEndpoint = o.signEndpoint
 		}
-		b.signBucket = signBucket
 	}
 
 	return b, nil
@@ -364,7 +379,7 @@ func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration)
 	if b.signBucket != nil {
 		bucket = b.signBucket
 	}
-	url, err := bucket.SignedURL(ctx, path, &blob.SignedURLOptions{
+	signed, err := bucket.SignedURL(ctx, path, &blob.SignedURLOptions{
 		Method: http.MethodGet,
 		Expiry: expiry,
 	})
@@ -374,7 +389,30 @@ func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration)
 		}
 		return "", fmt.Errorf("signing URL: %w", err)
 	}
-	return url, nil
+	if b.signBucket == nil && b.signEndpoint != "" {
+		signed = rewriteSignedURLHost(signed, b.signEndpoint)
+	}
+	return signed, nil
+}
+
+// rewriteSignedURLHost replaces the scheme and host of a signed URL with those
+// from baseURL, preserving the path and query (which carry the signature).
+// Returns signed unchanged if baseURL is empty or either URL fails to parse.
+func rewriteSignedURLHost(signed, baseURL string) string {
+	if baseURL == "" {
+		return signed
+	}
+	s, err := url.Parse(signed)
+	if err != nil {
+		return signed
+	}
+	b, err := url.Parse(baseURL)
+	if err != nil || b.Scheme == "" || b.Host == "" {
+		return signed
+	}
+	s.Scheme = b.Scheme
+	s.Host = b.Host
+	return s.String()
 }
 
 func (b *Blob) Size(ctx context.Context, path string) (int64, error) {
diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go
index 4811c7e..bffe09b 100644
--- a/internal/storage/blob_test.go
+++ b/internal/storage/blob_test.go
@@ -322,6 +322,85 @@ func TestBlobSignedURLSignEndpoint(t *testing.T) {
 	}
 }
 
+func TestAzureSignedURLSignEndpoint(t *testing.T) {
+	t.Setenv("AZURE_STORAGE_ACCOUNT", "devstoreaccount1")
+	t.Setenv("AZURE_STORAGE_KEY", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==")
+	t.Setenv("AZURE_STORAGE_PROTOCOL", "http")
+	t.Setenv("AZURE_STORAGE_IS_LOCAL_EMULATOR", "true")
+
+	ctx := context.Background()
+	b, err := OpenBucket(ctx, "azblob://bucket?domain=127.0.0.1:10000", WithSignEndpoint("https://cdn.example.com"))
+	if err != nil {
+		t.Fatalf("OpenBucket failed: %v", err)
+	}
+	defer func() { _ = b.Close() }()
+
+	signed, err := b.SignedURL(ctx, "npm/lodash/4.17.21/lodash-4.17.21.tgz", time.Minute)
+	if err != nil {
+		t.Fatalf("SignedURL failed: %v", err)
+	}
+
+	u, err := url.Parse(signed)
+	if err != nil {
+		t.Fatalf("parsing signed URL %q: %v", signed, err)
+	}
+	if u.Scheme != "https" || u.Host != "cdn.example.com" {
+		t.Errorf("signed URL = %q, want https://cdn.example.com", signed)
+	}
+	if u.Query().Get("sig") == "" {
+		t.Errorf("signed URL %q carries no SAS signature", signed)
+	}
+}
+
+func TestRewriteSignedURLHost(t *testing.T) {
+	tests := []struct {
+		name    string
+		signed  string
+		baseURL string
+		want    string
+	}{
+		{
+			"empty base url is no-op",
+			"http://127.0.0.1:9000/bucket/key?sig=abc",
+			"",
+			"http://127.0.0.1:9000/bucket/key?sig=abc",
+		},
+		{
+			"replaces scheme and host",
+			"http://127.0.0.1:9000/bucket/key?sig=abc",
+			"https://cdn.example.com",
+			"https://cdn.example.com/bucket/key?sig=abc",
+		},
+		{
+			"preserves path and query",
+			"http://minio:9000/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101",
+			"https://files.example.com",
+			"https://files.example.com/bucket/npm/lodash/4.17.21/lodash.tgz?X-Amz-Signature=abc&X-Amz-Date=20260101",
+		},
+		{
+			"ignores base url path",
+			"http://127.0.0.1:9000/bucket/key?sig=abc",
+			"https://cdn.example.com/ignored",
+			"https://cdn.example.com/bucket/key?sig=abc",
+		},
+		{
+			"invalid base url is no-op",
+			"http://127.0.0.1:9000/bucket/key?sig=abc",
+			"://bad",
+			"http://127.0.0.1:9000/bucket/key?sig=abc",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := rewriteSignedURLHost(tt.signed, tt.baseURL)
+			if got != tt.want {
+				t.Errorf("rewriteSignedURLHost(%q, %q) = %q, want %q", tt.signed, tt.baseURL, got, tt.want)
+			}
+		})
+	}
+}
+
 func TestBlobOverwrite(t *testing.T) {
 	b := createTestBlob(t)
 	ctx := context.Background()
diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go
index a096c18..43b14e5 100644
--- a/internal/storage/gcs.go
+++ b/internal/storage/gcs.go
@@ -13,8 +13,9 @@ import (
 
 // GCS adapts a Google Cloud Storage bucket to Storage.
 type GCS struct {
-	bucket *gcstorage.Bucket
-	url    string
+	bucket       *gcstorage.Bucket
+	url          string
+	signEndpoint string
 }
 
 // OpenGCS opens a Google Cloud Storage bucket from a gs:// URL.
@@ -64,7 +65,10 @@ func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration)
 	if errors.Is(err, gcstorage.ErrSignedURLUnsupported) {
 		return "", ErrSignedURLUnsupported
 	}
-	return u, err
+	if err != nil {
+		return "", err
+	}
+	return rewriteSignedURLHost(u, g.signEndpoint), nil
 }
 
 func (g *GCS) UsedSpace(ctx context.Context) (int64, error) {
diff --git a/internal/storage/gcs_test.go b/internal/storage/gcs_test.go
index 1bd8a40..461fb86 100644
--- a/internal/storage/gcs_test.go
+++ b/internal/storage/gcs_test.go
@@ -2,14 +2,20 @@ package storage
 
 import (
 	"context"
+	"crypto/rand"
+	"crypto/rsa"
 	"crypto/sha256"
+	"crypto/x509"
 	"encoding/hex"
 	"encoding/json"
+	"encoding/pem"
 	"errors"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
+	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
@@ -89,6 +95,56 @@ func TestOpenBucketGCSRoundTripWithEmulator(t *testing.T) {
 	}
 }
 
+func TestGCSSignedURLSignEndpoint(t *testing.T) {
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("generating RSA key: %v", err)
+	}
+	pemBytes := pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(key),
+	})
+	credsJSON, err := json.Marshal(map[string]string{
+		"type":         "service_account",
+		"client_email": "test@example.iam.gserviceaccount.com",
+		"private_key":  string(pemBytes),
+	})
+	if err != nil {
+		t.Fatalf("marshaling creds: %v", err)
+	}
+	credsFile := filepath.Join(t.TempDir(), "creds.json")
+	if err := os.WriteFile(credsFile, credsJSON, 0o600); err != nil {
+		t.Fatalf("writing creds: %v", err)
+	}
+	t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", credsFile)
+	t.Setenv("STORAGE_EMULATOR_HOST", "")
+
+	ctx := context.Background()
+	store, err := OpenBucket(ctx, "gs://test-bucket", WithSignEndpoint("https://cdn.example.com"))
+	if err != nil {
+		t.Fatalf("OpenBucket failed: %v", err)
+	}
+	defer func() { _ = store.Close() }()
+
+	signed, err := store.SignedURL(ctx, "npm/pkg/file.tgz", time.Minute)
+	if err != nil {
+		t.Fatalf("SignedURL failed: %v", err)
+	}
+	u, err := url.Parse(signed)
+	if err != nil {
+		t.Fatalf("parsing signed URL %q: %v", signed, err)
+	}
+	if u.Scheme != "https" || u.Host != "cdn.example.com" {
+		t.Errorf("signed URL = %q, want https://cdn.example.com", signed)
+	}
+	if u.Path != "/test-bucket/npm/pkg/file.tgz" {
+		t.Errorf("signed URL path = %q, want /test-bucket/npm/pkg/file.tgz", u.Path)
+	}
+	if u.Query().Get("Signature") == "" {
+		t.Errorf("signed URL %q carries no signature", signed)
+	}
+}
+
 type fakeGCSServer struct {
 	t       *testing.T
 	objects map[string]string

From 611feb003adb611813a1944f73764f28641ba4f5 Mon Sep 17 00:00:00 2001
From: Milas Bowman 
Date: Wed, 4 Nov 2026 18:29:37 -0500
Subject: [PATCH 097/198] fix(debian): route packages by filename rather than
 `/pool` prefix

---
 docs/architecture.md            |   9 +++
 docs/configuration.md           |   4 ++
 internal/handler/debian.go      |  28 +++++++-
 internal/handler/debian_test.go | 112 ++++++++++++++++++++++++++++++++
 4 files changed, 150 insertions(+), 3 deletions(-)

diff --git a/docs/architecture.md b/docs/architecture.md
index d8b294c..cbd6dab 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -304,6 +304,11 @@ HTTP protocol handlers for each registry type.
 - `handleIndex()` - Proxy sparse index
 - `handleDownload()` - Serve cached crate
 
+**DebianHandler:**
+- Serves `/debian` and `/debian/{name}` mirrors of APT archives (`upstream.debian_repositories`)
+- Routes `.deb` packages to the artifact cache, source-package files directly to upstream, `dists/…` to the metadata cache, and proxies all other paths
+- Keys the artifact cache on name, version, and architecture, scoped by repository for named archives
+
 **SwiftHandler:**
 - Proxies the Swift Package Registry v1 read endpoints
 - Rewrites release URLs and caches source archives
@@ -429,6 +434,10 @@ An object a record stops pointing at, because a refetch replaced it or its entry
 - Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table
 - OCI manifests and tag lists are exceptions: they are cached automatically so previously fetched images remain pullable and tag resolution works when the registry or token service is unavailable
 
+**Why are Debian source-package files streamed instead of cached?**
+- Source-package files (`.dsc`, source tarballs, `.buildinfo`, `.changes`) can be re-uploaded or re-signed in place under the same path, whereas the artifact cache keys entries by version and filename without an expected digest from the `Sources` index
+- Streaming them avoids serving stale bytes when an upstream file is replaced or caching them under the `dists/` metadata TTL in flat repositories
+
 **Why stream artifacts?**
 - Memory efficient - don't load large files into RAM
 - Better latency - start sending while still receiving
diff --git a/docs/configuration.md b/docs/configuration.md
index 63cc175..05914eb 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -805,6 +805,10 @@ and `dists` are refused at config load for that reason, and other root paths
 an archive may serve (`indices`, `project`, `doc`, `tools`) are not, so avoid
 those names unless the shadowing is intended.
 
+Binary packages (`.deb`) are cached; source-package files (`.dsc`, source
+tarballs, `.buildinfo`, `.changes`) are streamed directly from upstream because
+their contents can be replaced in place under the same path.
+
 This field has no environment variable, as with the other named upstream maps.
 `PROXY_UPSTREAM_DEBIAN` still sets `upstream.debian`.
 
diff --git a/internal/handler/debian.go b/internal/handler/debian.go
index b2419f6..87dbd65 100644
--- a/internal/handler/debian.go
+++ b/internal/handler/debian.go
@@ -108,11 +108,13 @@ func (h *DebianHandler) Routes() http.Handler {
 			}
 		}
 
-		// Route based on path type
+		// Route by filename rather than path prefix because flat repositories
+		// place pools under dists//pool/....
 		switch {
-		case strings.HasPrefix(rest, "pool/"):
-			// Package downloads - cache these
+		case isDebPackageFile(rest):
 			h.handlePackageDownload(w, r, repository, upstreamURL, rest)
+		case isSourcePackageFile(rest):
+			h.proxyFile(w, r, upstreamURL, rest)
 		case strings.HasPrefix(rest, "dists/"):
 			// Repository metadata - served through the metadata cache
 			h.handleMetadata(w, r, repository, upstreamURL, rest)
@@ -123,6 +125,26 @@ func (h *DebianHandler) Routes() http.Handler {
 	}))
 }
 
+func isDebPackageFile(path string) bool {
+	return strings.HasSuffix(path, ".deb")
+}
+
+// isSourcePackageFile reports whether path is a Debian source-package artifact.
+// Source files can be re-uploaded or re-signed in place under the same filename,
+// so they must be streamed rather than cached without digest verification.
+func isSourcePackageFile(path string) bool {
+	switch {
+	case strings.HasSuffix(path, ".dsc"),
+		strings.HasSuffix(path, ".buildinfo"),
+		strings.HasSuffix(path, ".changes"):
+		return true
+	}
+	// Match Debian source tarballs without matching suite tarballs (e.g. installer netboot.tar.gz).
+	return strings.Contains(path, ".orig.tar.") ||
+		strings.Contains(path, ".orig-") ||
+		strings.Contains(path, ".debian.tar.")
+}
+
 // repositoryNotFound answers a request addressed to a repository that is not
 // configured, naming the ones that are so a misspelling is self-evident.
 func (h *DebianHandler) repositoryNotFound(w http.ResponseWriter, name string) {
diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go
index f31202f..848f9e2 100644
--- a/internal/handler/debian_test.go
+++ b/internal/handler/debian_test.go
@@ -251,6 +251,45 @@ func TestDebianHandler_PackageCacheScopedPerRepository(t *testing.T) {
 	}
 }
 
+// TestDebianHandler_FlatArchivePackageIsCached verifies that .deb packages
+// served under dists//pool/ (as used by flat repositories like Docker's)
+// are cached as artifacts rather than treated as uncached/TTL metadata.
+func TestDebianHandler_FlatArchivePackageIsCached(t *testing.T) {
+	const (
+		flatPath = "/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb"
+		pkgBody  = "docker-ce package bytes"
+	)
+
+	var upstreamHits int
+
+	archive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		upstreamHits++
+		_, _ = fmt.Fprint(w, pkgBody)
+	}))
+	defer archive.Close()
+
+	proxy, _, _, _ := setupTestProxy(t)
+	fetcher := fetch.NewFetcher(fetch.WithHTTPClient(archive.Client()), fetch.WithMaxRetries(0))
+	proxy.Fetcher = fetcher
+	t.Cleanup(func() { _ = fetcher.Close() })
+
+	h := NewDebianHandler(proxy, "http://proxy.example", "https://archive.test", map[string]string{
+		"docker": archive.URL,
+	})
+
+	for attempt := 1; attempt <= 2; attempt++ {
+		w := serveDebianRequest(h, flatPath)
+		if w.Code != http.StatusOK || w.Body.String() != pkgBody {
+			t.Fatalf("request %d: status = %d, body = %q, want 200 %q",
+				attempt, w.Code, w.Body.String(), pkgBody)
+		}
+	}
+
+	if upstreamHits != 1 {
+		t.Errorf("upstream fetched %d times for two requests, want 1: the package is not cached", upstreamHits)
+	}
+}
+
 func serveDebianRequest(h *DebianHandler, target string) *httptest.ResponseRecorder {
 	w := httptest.NewRecorder()
 	h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil))
@@ -285,3 +324,76 @@ func TestDebRetentionKey(t *testing.T) {
 		t.Error("debRetentionKey(nil) accepted")
 	}
 }
+
+func TestDebianHandler_ClassifiesFilesByExtension(t *testing.T) {
+	tests := []struct {
+		path   string
+		pkg    bool
+		source bool
+		why    string
+	}{
+		{"pool/main/h/hello/hello_2.10-3_amd64.deb", true, false, "canonical pool"},
+		{"dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb", true, false, "flat archive"},
+		{"pool/main/h/hello/hello_2.10-3.udeb", false, false, "udeb not matched by debPackagePattern"},
+		{"pool/main/h/hello/hello_2.10-3_amd64.ddeb", false, false, "ddeb not matched by debPackagePattern"},
+		{"pool/main/h/hello/hello_2.10-3.dsc", false, true, "source control file"},
+		{"pool/main/h/hello/hello_2.10.orig.tar.xz", false, true, "source tarball"},
+		{"pool/main/h/hello/hello_2.10.orig-tex.tar.gz", false, true, "component tarball"},
+		{"pool/main/h/hello/hello_2.10-3.debian.tar.xz", false, true, "debian tarball"},
+		{"pool/main/h/hello/hello_2.10-3_amd64.buildinfo", false, true, "buildinfo"},
+		{"pool/main/h/hello/hello_2.10-3_amd64.changes", false, true, "changes"},
+		{"dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1.dsc", false, true, "flat archive source file"},
+		{"dists/trixie/InRelease", false, false, "metadata"},
+		{"dists/trixie/main/binary-amd64/Packages.gz", false, false, "metadata"},
+		{"dists/trixie/main/installer-amd64/current/images/netboot/netboot.tar.gz", false, false, "suite tarball"},
+	}
+	for _, tt := range tests {
+		if got := isDebPackageFile(tt.path); got != tt.pkg {
+			t.Errorf("isDebPackageFile(%q) = %v, want %v (%s)", tt.path, got, tt.pkg, tt.why)
+		}
+		if got := isSourcePackageFile(tt.path); got != tt.source {
+			t.Errorf("isSourcePackageFile(%q) = %v, want %v (%s)", tt.path, got, tt.source, tt.why)
+		}
+	}
+}
+
+func TestDebianHandler_SourcePackageFilesAreNotCached(t *testing.T) {
+	const dscBody = "hello source control bytes"
+
+	var upstreamHits int
+
+	archive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		upstreamHits++
+		_, _ = fmt.Fprint(w, dscBody)
+	}))
+	defer archive.Close()
+
+	proxy, _, store, _ := setupTestProxy(t)
+	proxy.HTTPClient = archive.Client()
+	proxy.CacheMetadata = true
+	proxy.MetadataTTL = time.Hour
+
+	h := NewDebianHandler(proxy, "http://proxy.example", archive.URL, map[string]string{
+		"docker": archive.URL,
+	})
+
+	for _, target := range []string{
+		"/pool/main/h/hello/hello_2.10-3.dsc",
+		"/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1.dsc",
+	} {
+		for attempt := 1; attempt <= 2; attempt++ {
+			w := serveDebianRequest(h, target)
+			if w.Code != http.StatusOK || w.Body.String() != dscBody {
+				t.Fatalf("%s request %d: status = %d, body = %q, want 200 %q",
+					target, attempt, w.Code, w.Body.String(), dscBody)
+			}
+		}
+	}
+
+	if upstreamHits != 4 {
+		t.Errorf("upstream fetched %d times for four requests, want 4: source-package files must not be cached", upstreamHits)
+	}
+	if store.stores != 0 {
+		t.Errorf("storage wrote %d object(s), want 0: a source-package file must not be stored", store.stores)
+	}
+}

From f153a83b3a286b9e3c9f9c04559bd3b674763389 Mon Sep 17 00:00:00 2001
From: Andrew Nesbitt 
Date: Sat, 7 Nov 2026 09:46:54 +0000
Subject: [PATCH 098/198] Cover Fedora updates and Conda defaults in setup docs

---
 README.md                    | 11 +++++++++++
 internal/server/dashboard.go | 36 +++++++++++++++++++++++-------------
 2 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index a43ae27..7739645 100644
--- a/README.md
+++ b/README.md
@@ -465,12 +465,17 @@ Configure in `~/.condarc`:
 channels:
   - http://localhost:8080/conda/main
   - http://localhost:8080/conda/conda-forge
+default_channels:
+  - http://localhost:8080/conda/main
 ```
 
+`default_channels` also routes requests for `defaults` through the proxy, including `-c defaults` and `environment.yml` files that name it.
+
 Or set via command:
 
 ```bash
 conda config --add channels http://localhost:8080/conda/main
+conda config --add default_channels http://localhost:8080/conda/main
 ```
 
 pixi and other rattler-based clients use sharded repodata ([CEP 16](https://github.com/conda/ceps/blob/main/cep-0016.md)) where a channel offers it, as conda-forge does. Point them at the same channel URLs, for example `pixi init -c http://localhost:8080/conda/conda-forge`. The shard index and shards are cached as metadata when `cache_metadata` is on.
@@ -726,6 +731,12 @@ In the `[fedora]` section of your existing repository file under `/etc/yum.repos
 baseurl=http://localhost:8080/rpm/releases/$releasever/Everything/$basearch/os/
 ```
 
+In the `[updates]` section of `fedora-updates.repo`, replace its `metalink` or `mirrorlist` entry with:
+
+```ini
+baseurl=http://localhost:8080/rpm/updates/$releasever/Everything/$basearch/
+```
+
 Keep the existing `gpgkey` and signature settings, including `gpgcheck=1`. The proxy serves RPM packages unchanged, so their signatures remain valid.
 
 Then:
diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go
index f566d55..604a4bf 100644
--- a/internal/server/dashboard.go
+++ b/internal/server/dashboard.go
@@ -448,9 +448,13 @@ conan remote disable conancenter
`), channels: - ` + baseURL + `/conda/main - ` + baseURL + `/conda/conda-forge +default_channels: + - ` + baseURL + `/conda/main # Or via command -conda config --add channels ` + baseURL + `/conda/main`), +conda config --add channels ` + baseURL + `/conda/main +conda config --add default_channels ` + baseURL + `/conda/main +

default_channels also routes requests for defaults through the proxy, including -c defaults and environment.yml files that name it.

`), }, { ID: "cran", @@ -518,18 +522,7 @@ deb ` + baseURL + `/debian stable main contrib # Then run: sudo apt update`), }, - { - ID: "rpm", - Name: "RPM/Yum", - Language: "Fedora", - Endpoint: "/rpm/", - Instructions: template.HTML(`

In the [fedora] section of your existing repository file under /etc/yum.repos.d/, replace the metalink or mirrorlist entry with:

-
baseurl=` + baseURL + `/rpm/releases/$releasever/Everything/$basearch/os/
-

Keep the existing gpgkey and signature settings, including gpgcheck=1. The proxy serves RPM packages unchanged.

-
# Then run:
-sudo dnf clean all
-sudo dnf update
`), - }, + rpmRegistryConfig(baseURL), { ID: "apk", Name: "Alpine APK", @@ -559,6 +552,23 @@ pacman -Syu`), } } +func rpmRegistryConfig(baseURL string) RegistryConfig { + return RegistryConfig{ + ID: "rpm", + Name: "RPM/Yum", + Language: "Fedora", + Endpoint: "/rpm/", + Instructions: template.HTML(`

In the [fedora] section of your existing repository file under /etc/yum.repos.d/, replace the metalink or mirrorlist entry with:

+
baseurl=` + baseURL + `/rpm/releases/$releasever/Everything/$basearch/os/
+

In the [updates] section of fedora-updates.repo, replace its metalink or mirrorlist entry with:

+
baseurl=` + baseURL + `/rpm/updates/$releasever/Everything/$basearch/
+

Keep the existing gpgkey and signature settings, including gpgcheck=1. The proxy serves RPM packages unchanged.

+
# Then run:
+sudo dnf clean all
+sudo dnf update
`), + } +} + // freebsdRegistryConfig describes the FreeBSD pkg endpoint for the dashboard. func freebsdRegistryConfig(baseURL string) RegistryConfig { return RegistryConfig{ From 794ea98a2810cd26eac95e21033c52f878c407aa Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 7 Nov 2026 17:04:29 +0100 Subject: [PATCH 099/198] Match abi3 wheels whatever minimum Python they name The python tag of an abi3 wheel is a minimum, so cp38-abi3 and cp311-abi3 wheels are now the same kind as a cached cp39-abi3 one. --- docs/configuration.md | 2 +- internal/mirror/files.go | 12 +++++++++++- internal/mirror/refresh_test.go | 25 +++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index e9d9195..5459cb6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1973,7 +1973,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. -- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need, and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. +- PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. - Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. diff --git a/internal/mirror/files.go b/internal/mirror/files.go index ff0d298..ab6bafd 100644 --- a/internal/mirror/files.go +++ b/internal/mirror/files.go @@ -99,7 +99,7 @@ func pypiVariants(filename string) []string { for _, python := range strings.Split(tags[0], ".") { for _, abi := range strings.Split(tags[1], ".") { for _, platform := range strings.Split(tags[2], ".") { - v := python + "-" + abi + "-" + wheelPlatform(platform) + v := wheelPython(python, abi) + "-" + abi + "-" + wheelPlatform(platform) if !slices.Contains(variants, v) { variants = append(variants, v) } @@ -109,6 +109,16 @@ func pypiVariants(filename string) []string { return variants } +// wheelPython reduces the python tag of an abi3 wheel, a minimum such as +// cp39, to its interpreter, so cp38-abi3 and cp311-abi3 wheels are the same +// kind. +func wheelPython(python, abi string) string { + if abi != "abi3" { + return python + } + return strings.TrimRight(python, "0123456789") +} + // wheelPlatformMinimum matches the glibc, musl or macOS minimum version in a // platform tag, such as the 2_17 in manylinux_2_17_x86_64 or the 2014 in // manylinux2014_x86_64. diff --git a/internal/mirror/refresh_test.go b/internal/mirror/refresh_test.go index af0d1be..97624d6 100644 --- a/internal/mirror/refresh_test.go +++ b/internal/mirror/refresh_test.go @@ -297,6 +297,7 @@ func TestFileVariants(t *testing.T) { {"pypi", "demo-1.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", "cp312-cp312-musllinux_x86_64"}, {"pypi", "demo-1.0.0-cp312-cp312-macosx_10_13_universal2.whl", "cp312-cp312-macosx_universal2"}, {"pypi", "demo-1.0.0-cp312-cp312-win_amd64.whl", "cp312-cp312-win_amd64"}, + {"pypi", "demo-1.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", "cp-abi3-manylinux_x86_64"}, {"pypi", "demo-1.0.0.tar.gz", "sdist"}, {"pypi", "demo-1.0.0.zip", "sdist"}, {"conda", "linux-64/numpy-1.26.4-py312h8753938_0.conda", "linux-64/py312/conda"}, @@ -432,3 +433,27 @@ func TestRefreshMatchesWheelsOnAnyAlternativeTag(t *testing.T) { t.Errorf("fetched %v, want %v", got, want) } } + +// In an abi3 wheel the python tag is a minimum: cryptography 44.0.0 ships +// cp37-abi3 and cp39-abi3 wheels, 46.0.0 cp38-abi3 and cp311-abi3 ones. A +// refresh stays on the cached major line, so the newer tags are on 44.1.0 +// here. +func TestRefreshMatchesAbi3WheelsAcrossPythonMinimums(t *testing.T) { + got := refreshFetches(t, "pypi", "cryptography", "44.0.0", + []string{"cryptography-44.0.0-cp39-abi3-manylinux_2_28_x86_64.whl"}, + "44.1.0", wheels( + "cryptography-44.1.0-cp38-abi3-manylinux_2_28_x86_64.whl", + "cryptography-44.1.0-cp311-abi3-manylinux_2_28_x86_64.whl", + "cryptography-44.1.0-cp311-abi3-manylinux_2_28_aarch64.whl", + "cryptography-44.1.0-cp311-abi3-macosx_10_9_universal2.whl", + "cryptography-44.1.0-cp311-abi3-win_amd64.whl", + "cryptography-44.1.0-cp314-cp314t-manylinux_2_28_x86_64.whl", + )) + want := []string{ + "https://files.example/cryptography-44.1.0-cp311-abi3-manylinux_2_28_x86_64.whl", + "https://files.example/cryptography-44.1.0-cp38-abi3-manylinux_2_28_x86_64.whl", + } + if !slices.Equal(got, want) { + t.Errorf("fetched %v, want %v", got, want) + } +} From e9aa078a8572160557a4d4cfb09aa9546db952bf Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 7 Nov 2026 17:29:03 +0100 Subject: [PATCH 100/198] Mirror pkg:git tags and the submodules a commit pins A pkg:git version can now be a tag as well as a full commit ID. With a version, the submodules the commit records are mirrored at the commits it pins, from the upstream.git hosts serving them; one no host serves is skipped with a warning. --- docs/architecture.md | 2 +- docs/configuration.md | 6 +- internal/gitcache/mirror.go | 129 +++++++++++++++++++++++++--- internal/gitcache/mirror_test.go | 79 +++++++++++++++++ internal/handler/git_mirror.go | 100 ++++++++++++++++++++- internal/handler/git_mirror_test.go | 81 +++++++++++++++++ internal/handler/git_test.go | 6 +- internal/mirror/git.go | 10 +-- 8 files changed, 387 insertions(+), 26 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 655b309..80120ee 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -363,7 +363,7 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g ### `internal/mirror` -Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and commit in the git cache that `/git` serves. +Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and the commit or tag, with its submodules, in the git cache that `/git` serves. The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints. diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..1c01589 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1910,16 +1910,18 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403 ### Git repositories -`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID: +`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID or a tag: ```bash proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567 +proxy mirror pkg:git/github.com/git-pkgs/purl@v0.2.1 proxy mirror pkg:git/github.com/git-pkgs/purl ``` - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. -- Without a version the repository's branches and tags are mirrored. A tag or branch name as the version fails; give the commit ID. +- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name as the version fails too; give the commit ID. +- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. ### Scheduled refresh diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index b5bde40..047934f 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -4,27 +4,44 @@ import ( "context" "errors" "fmt" + "os/exec" + "slices" "strings" "time" ) // MirrorResult is what Mirror did. type MirrorResult struct { - // Cached reports that the repository, and the commit if one was given, - // were cached already. + // Cached reports that the repository, and the commit if a version was + // given, were cached already. Cached bool // Bytes is how much the cached repository grew. Bytes int64 + // Commit is the commit the version names, empty without a version. + Commit string + // Submodules are the submodules Commit records, sorted by path. + Submodules []Submodule +} + +// Submodule is a submodule a commit records: its path, the URL .gitmodules +// gives for it, which may be relative to the repository's, and the commit +// the gitlink pins. +type Submodule struct { + Path, URL, Commit string } // Mirror brings the repository cached under key up to date with repoURL, -// cloning it if needed, and waits for that to finish. With commit, an object -// ID, it also makes sure the commit is cached. One on no branch or tag is -// fetched by ID when fetchWants allows it (see OpenOptions.FetchWants), and -// is kept as long as a commit a client fetched by ID. -func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWants bool) (MirrorResult, error) { - if commit != "" && !isObjectID(commit) { - return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit) +// cloning it if needed, and waits for that to finish. version, if given, is +// a full commit ID or a tag. A commit is made sure to be cached: one on no +// branch or tag is fetched by ID when fetchWants allows it (see +// OpenOptions.FetchWants), and is kept as long as a commit a client fetched +// by ID. A tag is cached with the repository's tags. +func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) { + tag := version != "" && !isObjectID(version) + if tag { + if err := c.checkTagName(ctx, version); err != nil { + return MirrorResult{}, err + } } lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants}) if err != nil { @@ -53,17 +70,105 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa if errors.Is(refreshErr, ErrTooLarge) { return MirrorResult{}, refreshErr } - if commit != "" { + commit := version + switch { + case tag: + if commit, err = c.resolveTag(ctx, rp.dir, version); err != nil { + if refreshErr != nil { + // The tag may be one the failed refresh would have brought. + return MirrorResult{}, refreshErr + } + return MirrorResult{}, err + } + case version != "": if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { return MirrorResult{}, err } } // A refresh that failed leaves the commit cached all the same. - if refreshErr != nil && commit == "" { + if refreshErr != nil && version == "" { return MirrorResult{}, refreshErr } + var submodules []Submodule + if commit != "" { + if submodules, err = c.submodules(ctx, rp.dir, commit); err != nil { + return MirrorResult{}, err + } + } grown := max(rp.size.Load()-before, 0) - return MirrorResult{Cached: cached && grown == 0, Bytes: grown}, nil + return MirrorResult{Cached: cached && grown == 0, Bytes: grown, Commit: commit, Submodules: submodules}, nil +} + +// checkTagName refuses a version that is no valid tag name, such as one +// with .. or a revision suffix like ^{tree}. +func (c *Cache) checkTagName(ctx context.Context, name string) error { + if err := c.command(ctx, nil, "check-ref-format", "refs/tags/"+name).Run(); err != nil { + return fmt.Errorf("%q is neither a full commit ID nor a tag name", name) + } + return nil +} + +// resolveTag returns the commit tag points at. +func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) { + out, err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", "refs/tags/"+tag+"^{commit}").Output() + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 1 { + return "", fmt.Errorf("tag %s: %w", tag, ErrNotFound) + } + if err != nil { + return "", fmt.Errorf("resolving tag %s: %w", tag, err) + } + return strings.TrimSpace(string(out)), nil +} + +// submodules lists the submodules commit records: its gitlinks that +// .gitmodules names a URL for. +func (c *Cache) submodules(ctx context.Context, dir, commit string) ([]Submodule, error) { + if c.command(ctx, nil, "-C", dir, "cat-file", "-e", commit+":.gitmodules").Run() != nil { + return nil, nil + } + out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--get-regexp", `^submodule\..*\.(path|url)$`) + if err != nil { + return nil, fmt.Errorf("reading .gitmodules: %w", err) + } + paths, urls := map[string]string{}, map[string]string{} + for entry := range strings.SplitSeq(out, "\x00") { + key, value, ok := strings.Cut(entry, "\n") + name, field, dotted := cutLast(strings.TrimPrefix(key, "submodule."), ".") + if !ok || !dotted { + continue + } + if field == "path" { + paths[value] = name + } else { + urls[name] = value + } + } + tree, err := c.run(ctx, dir, "ls-tree", "-r", "-z", "--full-tree", commit) + if err != nil { + return nil, err + } + var submodules []Submodule + for entry := range strings.SplitSeq(tree, "\x00") { + meta, path, ok := strings.Cut(entry, "\t") + fields := strings.Fields(meta) + if !ok || len(fields) != 3 || fields[0] != "160000" { + continue + } + if url := urls[paths[path]]; url != "" && isObjectID(fields[2]) { + submodules = append(submodules, Submodule{Path: path, URL: url, Commit: fields[2]}) + } + } + slices.SortFunc(submodules, func(a, b Submodule) int { return strings.Compare(a.Path, b.Path) }) + return submodules, nil +} + +func cutLast(s, sep string) (before, after string, found bool) { + i := strings.LastIndex(s, sep) + if i < 0 { + return s, "", false + } + return s[:i], s[i+len(sep):], true } // ensureCommit makes sure the repository has commit, fetching it by ID when diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 86f652f..499acfb 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -212,3 +212,82 @@ func TestMirrorReportsRefreshTooLarge(t *testing.T) { t.Errorf("mirror with a refresh past max_repo_size = %v, want ErrTooLarge", err) } } + +func TestMirrorResolvesTag(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tagged := runGit(t, work, "rev-parse", "v1^{commit}") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Commit != tagged { + t.Errorf("commit %q, want %s the tag points at", got.Commit, tagged) + } +} + +func TestMirrorReportsMissingTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, version := range []string{"v9", "main"} { + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) + if !errors.Is(err, ErrNotFound) { + t.Errorf("mirror at %q = %v, want ErrNotFound", version, err) + } + } +} + +func TestMirrorRefusesInvalidTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, version := range []string{"a..b", "v1^{tree}", "v1~1", "v 1", "v1:x"} { + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", version, true) + if err == nil || errors.Is(err, ErrNotFound) { + t.Errorf("mirror at %q = %v, want an invalid version error", version, err) + } + } +} + +func TestMirrorListsSubmodules(t *testing.T) { + upstream := newFakeUpstream(t) + libWork := upstream.newRepo("lib") + libCommit := runGit(t, libWork, "rev-parse", "HEAD") + work := upstream.newRepo("repo") + gitmodules := "[submodule \"vendor/lib\"]\n\tpath = vendor/lib\n\turl = ../lib.git\n" + + "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { + t.Fatal(err) + } + otherCommit := strings.Repeat("ab", 20) + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") + runGit(t, work, "commit", "--quiet", "-m", "submodules") + runGit(t, work, "push", "--quiet", "origin", "main") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + want := []Submodule{ + {Path: "third_party/other", URL: "https://example.com/other.git", Commit: otherCommit}, + {Path: "vendor/lib", URL: "../lib.git", Commit: libCommit}, + } + if !slices.Equal(got.Submodules, want) { + t.Errorf("submodules %+v, want %+v", got.Submodules, want) + } + + // Without a version, nothing names a commit to read submodules from. + got, err = tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "", true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("unversioned mirror = %+v, %v, want no submodules", got, err) + } +} diff --git a/internal/handler/git_mirror.go b/internal/handler/git_mirror.go index 709df31..38be233 100644 --- a/internal/handler/git_mirror.go +++ b/internal/handler/git_mirror.go @@ -3,22 +3,114 @@ package handler import ( "context" "fmt" + "log/slog" "net/url" + "path" "strings" "github.com/git-pkgs/proxy/internal/gitcache" ) +// maxSubmoduleDepth bounds how deeply Mirror follows submodules of +// submodules. +const maxSubmoduleDepth = 8 + // Mirror caches the repository at location, a host and path such as // github.com/acme/widgets as a pkg:git PURL names it, from the upstream.git -// host serving it, and with commit, a full commit ID, that commit too. -// Clones through /git are then served from that copy. -func (h *GitHandler) Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) { +// host serving it. With version, a full commit ID or a tag, it caches that +// commit too, and the commits its submodules pin, from the upstream.git +// hosts serving them. A submodule no upstream.git host serves is skipped. +// Clones through /git are then served from those copies. +func (h *GitHandler) Mirror(ctx context.Context, location, version string) (gitcache.MirrorResult, error) { + return h.mirror(ctx, location, version, map[string]bool{}, 0) +} + +func (h *GitHandler) mirror(ctx context.Context, location, version string, seen map[string]bool, depth int) (gitcache.MirrorResult, error) { repo, err := h.mirrorRepository(location) if err != nil { return gitcache.MirrorResult{}, err } - return h.cache.Mirror(ctx, repo.key, repo.url, commit, repo.upstream.fetchWants) + result, err := h.cache.Mirror(ctx, repo.key, repo.url, version, repo.upstream.fetchWants) + if err != nil { + return result, err + } + for _, sub := range result.Submodules { + subLocation, err := gitSubmoduleLocation(repo.url, sub.URL) + if err == nil { + _, err = h.mirrorRepository(subLocation) + } + if err != nil { + h.logger().Warn("git mirror: skipping submodule", "repository", location, "submodule", sub.Path, "url", sub.URL, "error", err) + continue + } + if seen[subLocation+"@"+sub.Commit] { + continue + } + seen[subLocation+"@"+sub.Commit] = true + if depth >= maxSubmoduleDepth { + return result, fmt.Errorf("submodule %s: nested more than %d deep", sub.Path, maxSubmoduleDepth) + } + subResult, err := h.mirror(ctx, subLocation, sub.Commit, seen, depth+1) + if err != nil { + return result, fmt.Errorf("submodule %s: %w", sub.Path, err) + } + result.Cached = result.Cached && subResult.Cached + result.Bytes += subResult.Bytes + } + return result, nil +} + +func (h *GitHandler) logger() *slog.Logger { + if h.proxy != nil && h.proxy.Logger != nil { + return h.proxy.Logger + } + return slog.Default() +} + +// gitSubmoduleLocation turns the URL .gitmodules gives a submodule into a +// location as Mirror takes it. A relative URL is resolved against the +// superproject's repoURL, as git does; ssh, git and scp-like URLs name the +// host and path the repository has there. +func gitSubmoduleLocation(repoURL, submoduleURL string) (string, error) { + var host, repoPath string + switch { + case strings.HasPrefix(submoduleURL, "./"), strings.HasPrefix(submoduleURL, "../"): + u, err := url.Parse(repoURL) + if err != nil { + return "", err + } + host, repoPath = u.Host, path.Join(u.Path, submoduleURL) + case strings.Contains(submoduleURL, "://"): + u, err := url.Parse(submoduleURL) + if err != nil { + return "", err + } + switch u.Scheme { + case "http", "https": + host = u.Host + case "ssh", "git", "git+ssh", "ssh+git": + host = u.Hostname() + default: + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + repoPath = u.Path + default: + // scp-like: [user@]host:path + hostPart, p, ok := strings.Cut(submoduleURL, ":") + if !ok || strings.Contains(hostPart, "/") { + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + _, host, _ = strings.Cut(hostPart, "@") + if host == "" { + host = hostPart + } + repoPath = p + } + repoPath = strings.Trim(path.Clean("/"+repoPath), "/") + if host == "" || repoPath == "" { + return "", fmt.Errorf("unsupported submodule URL %q", submoduleURL) + } + return strings.ToLower(host) + "/" + repoPath, nil } // mirrorRepository resolves location to a repository of the upstream.git diff --git a/internal/handler/git_mirror_test.go b/internal/handler/git_mirror_test.go index 4ebcf98..227bd71 100644 --- a/internal/handler/git_mirror_test.go +++ b/internal/handler/git_mirror_test.go @@ -2,6 +2,8 @@ package handler import ( "context" + "os" + "path/filepath" "strings" "testing" @@ -71,3 +73,82 @@ func TestGitMirrorCachesCommitForClones(t *testing.T) { t.Errorf("cloned HEAD %s, want %s", head, sha) } } + +func TestGitSubmoduleLocation(t *testing.T) { + tests := []struct { + repoURL, submoduleURL, want string + }{ + {"https://github.com/acme/widgets", "../lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets.git", "../../other/lib", "github.com/other/lib"}, + {"https://github.com/acme/widgets", "./vendored", "github.com/acme/widgets/vendored"}, + {"http://127.0.0.1:8080/team/repo", "../lib.git", "127.0.0.1:8080/team/lib.git"}, + {"https://github.com/acme/widgets", "https://GitLab.com/group/lib.git/", "gitlab.com/group/lib.git"}, + {"https://github.com/acme/widgets", "ssh://git@github.com:22/acme/lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets", "git://git.example/lib", "git.example/lib"}, + {"https://github.com/acme/widgets", "git@github.com:acme/lib.git", "github.com/acme/lib.git"}, + {"https://github.com/acme/widgets", "/srv/git/lib.git", ""}, + {"https://github.com/acme/widgets", "file:///srv/git/lib.git", ""}, + {"https://github.com/acme/widgets", "", ""}, + } + for _, tt := range tests { + got, err := gitSubmoduleLocation(tt.repoURL, tt.submoduleURL) + if tt.want == "" { + if err == nil { + t.Errorf("location of %q = %q, want an error", tt.submoduleURL, got) + } + continue + } + if err != nil || got != tt.want { + t.Errorf("location of %q from %s = %q, %v; want %q", tt.submoduleURL, tt.repoURL, got, err, tt.want) + } + } +} + +// TestGitMirrorCachesSubmodules mirrors a commit with a submodule served by +// the same upstream and one on a host no upstream.git serves, which is +// skipped. +func TestGitMirrorCachesSubmodules(t *testing.T) { + g := newGitTestSetup(t, 0, 0) + libBare := filepath.Join(g.root, "lib.git") + runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", libBare) + libWork := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", libBare, libWork) + runGitCommand(t, libWork, "commit", "--quiet", "--allow-empty", "-m", "lib") + runGitCommand(t, libWork, "push", "--quiet", "origin", "HEAD:main") + libCommit := runGitCommand(t, libWork, "rev-parse", "HEAD") + + gitmodules := "[submodule \"lib\"]\n\tpath = lib\n\turl = ../lib.git\n" + + "[submodule \"elsewhere\"]\n\tpath = elsewhere\n\turl = https://elsewhere.example/x.git\n" + if err := os.WriteFile(filepath.Join(g.work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { + t.Fatal(err) + } + runGitCommand(t, g.work, "add", ".gitmodules") + runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",lib") + runGitCommand(t, g.work, "update-index", "--add", "--cacheinfo", "160000,"+strings.Repeat("ab", 20)+",elsewhere") + runGitCommand(t, g.work, "commit", "--quiet", "-m", "submodules") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + location := strings.TrimPrefix(g.upstreamURL, "http://") + "/repo" + + got, err := g.handler.Mirror(context.Background(), location, sha) + if err != nil { + t.Fatalf("Mirror: %v", err) + } + if got.Cached { + t.Errorf("first mirror = %+v, want fetched", got) + } + if got, err = g.handler.Mirror(context.Background(), location, sha); err != nil || !got.Cached { + t.Errorf("second mirror = %+v, %v, want cached", got, err) + } + if !strings.Contains(g.logs.String(), "elsewhere.example") { + t.Errorf("log doesn't name the skipped submodule:\n%s", g.logs.String()) + } + + // With the upstream gone, the submodule's commit is served from the cache. + g.upstream.Close() + clone := t.TempDir() + runGitCommand(t, "", "clone", "--quiet", g.server.URL+"/git/up/lib", clone) + if head := runGitCommand(t, clone, "rev-parse", "HEAD"); head != libCommit { + t.Errorf("cloned lib HEAD %s, want %s", head, libCommit) + } +} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go index cf553d6..aeec2bc 100644 --- a/internal/handler/git_test.go +++ b/internal/handler/git_test.go @@ -129,7 +129,9 @@ func runGitCommand(t *testing.T, dir string, args ...string) string { // gitTestSetup is a GitHandler in front of a cache of one upstream // repository, "repo", served by git http-backend. type gitTestSetup struct { - work string + work string + // root holds the upstream's bare repositories. + root string server *httptest.Server // upstream serves the repository, at upstreamURL/repo, and handler is // the GitHandler under test. @@ -161,7 +163,7 @@ func newGitTestSetup(t *testing.T, readTimeout, writeStall time.Duration) *gitTe bare := filepath.Join(root, "repo.git") runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) runGitCommand(t, bare, "config", "uploadpack.allowAnySHA1InWant", "true") - g := &gitTestSetup{work: t.TempDir(), logs: &syncLog{}} + g := &gitTestSetup{work: t.TempDir(), root: root, logs: &syncLog{}} runGitCommand(t, "", "clone", "--quiet", bare, g.work) runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "first") runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") diff --git a/internal/mirror/git.go b/internal/mirror/git.go index a4908dc..e9c04ff 100644 --- a/internal/mirror/git.go +++ b/internal/mirror/git.go @@ -9,10 +9,10 @@ import ( // GitMirrorer caches Git repositories for pkg:git items. location is the // repository's host and path, such as codeberg.org/forgejo/forgejo, and -// commit a full commit ID, or empty to cache the repository's branches and -// tags only. +// version a full commit ID or a tag, or empty to cache the repository's +// branches and tags only. type GitMirrorer interface { - Mirror(ctx context.Context, location, commit string) (gitcache.MirrorResult, error) + Mirror(ctx context.Context, location, version string) (gitcache.MirrorResult, error) } // UnavailableGit fails every pkg:git item with Reason. @@ -26,8 +26,8 @@ func (u UnavailableGit) Mirror(context.Context, string, string) (gitcache.Mirror var errNoGitUpstream = errors.New("git repositories are mirrored from upstream.git, and none is configured") -// mirrorGit caches a pkg:git repository, and the commit its version names, -// in the cache /git serves clones from. +// mirrorGit caches a pkg:git repository, and the commit its version names +// with the submodules it pins, in the cache /git serves clones from. func (m *Mirror) mirrorGit(ctx context.Context, pv PackageVersion, tracker *progressTracker) { if m.Git == nil { m.recordFailure(tracker, pv, errNoGitUpstream) From 7f2ac614c421a5b21e6eba71e82fb570e47c9dca Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 7 Nov 2026 17:51:13 +0100 Subject: [PATCH 101/198] Mirror commits whose .gitmodules lists no submodule git config --get-regexp exits 1 when nothing matches, which failed the mirror of a commit with an empty .gitmodules. The submodules are now read from --list. --- internal/gitcache/mirror.go | 13 ++++++++----- internal/gitcache/mirror_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index 047934f..5aaa6fb 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -127,20 +127,23 @@ func (c *Cache) submodules(ctx context.Context, dir, commit string) ([]Submodule if c.command(ctx, nil, "-C", dir, "cat-file", "-e", commit+":.gitmodules").Run() != nil { return nil, nil } - out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--get-regexp", `^submodule\..*\.(path|url)$`) + // --list rather than --get-regexp, which fails when nothing matches. + out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list") if err != nil { return nil, fmt.Errorf("reading .gitmodules: %w", err) } paths, urls := map[string]string{}, map[string]string{} for entry := range strings.SplitSeq(out, "\x00") { key, value, ok := strings.Cut(entry, "\n") - name, field, dotted := cutLast(strings.TrimPrefix(key, "submodule."), ".") - if !ok || !dotted { + section, ok2 := strings.CutPrefix(key, "submodule.") + name, field, dotted := cutLast(section, ".") + if !ok || !ok2 || !dotted { continue } - if field == "path" { + switch field { + case "path": paths[value] = name - } else { + case "url": urls[name] = value } } diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 499acfb..1f85a6c 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -291,3 +291,27 @@ func TestMirrorListsSubmodules(t *testing.T) { t.Errorf("unversioned mirror = %+v, %v, want no submodules", got, err) } } + +// TestMirrorWithEmptyGitmodules checks a .gitmodules with no path or url, +// for which git config --get-regexp finds nothing, lists no submodules. +func TestMirrorWithEmptyGitmodules(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + for i, content := range []string{"", "[submodule \"x\"]\n\tbranch = main\n"} { + if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "--allow-empty", "-m", "gitmodules "+strconv.Itoa(i)) + } + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, rev := range []string{"HEAD~1", "HEAD"} { + commit := runGit(t, work, "rev-parse", rev) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err) + } + } +} From d58b07b2a71ea119d6e2f257a0b053a3aa10446a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 7 Nov 2026 18:02:59 +0100 Subject: [PATCH 102/198] Skip discarded npm packument values with jsontext ScanPackument and LookupVersion read with a jsontext.Decoder, skipping values they don't keep with SkipValue and reading the one LookupVersion returns with ReadValue, instead of decoding each into a json.RawMessage. Repeated names and invalid UTF-8 are still accepted as encoding/json accepts them. Go 1.27 is now required, for encoding/json/jsontext. Benchmarks for both use the 5,000-version packument the handler benchmarks build, now shared in internal/metadata/npm/npmbench. --- go.mod | 4 +- internal/handler/npm_rewrite_bench_test.go | 39 +----- internal/metadata/npm/npmbench/npmbench.go | 42 ++++++ internal/metadata/npm/stream.go | 146 ++++++++++++--------- internal/metadata/npm/stream_bench_test.go | 31 +++++ internal/metadata/npm/stream_test.go | 29 ++++ 6 files changed, 189 insertions(+), 102 deletions(-) create mode 100644 internal/metadata/npm/npmbench/npmbench.go create mode 100644 internal/metadata/npm/stream_bench_test.go diff --git a/go.mod b/go.mod index 2f4a0ce..9b77ac2 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/git-pkgs/proxy -go 1.26.7 - -toolchain go1.27.3 +go 1.27.3 require ( github.com/BurntSushi/toml v1.6.0 diff --git a/internal/handler/npm_rewrite_bench_test.go b/internal/handler/npm_rewrite_bench_test.go index 86552db..cf8b895 100644 --- a/internal/handler/npm_rewrite_bench_test.go +++ b/internal/handler/npm_rewrite_bench_test.go @@ -1,44 +1,13 @@ package handler import ( - "fmt" - "strings" "testing" + + "github.com/git-pkgs/proxy/internal/metadata/npm/npmbench" ) -// benchPackument builds a full packument shaped like a large real one: -// thousands of versions, each with dependencies, scripts and a readme. -func benchPackument(versions int) []byte { - var b strings.Builder - readme := strings.Repeat("Some documentation text. ", 160) - b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"1.0.` + fmt.Sprint(versions-1) + `"},"versions":{`) - for i := range versions { - if i > 0 { - b.WriteByte(',') - } - v := fmt.Sprintf("1.0.%d", i) - fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"description":"A big package","main":"index.js",`+ - `"scripts":{"test":"node test.js","build":"tsc -p ."},"dependencies":{`, v, v) - for d := range 25 { - if d > 0 { - b.WriteByte(',') - } - fmt.Fprintf(&b, `"dep-%d":"^%d.0.0"`, d, d) - } - fmt.Fprintf(&b, `},"readme":%q,"dist":{"shasum":"0123456789abcdef0123456789abcdef01234567",`+ - `"integrity":"sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",`+ - `"tarball":"https://registry.npmjs.org/big/-/big-%s.tgz"}}`, readme, v) - } - b.WriteString(`},"time":{"created":"2015-01-01T00:00:00.000Z"`) - for i := range versions { - fmt.Fprintf(&b, `,"1.0.%d":"2016-01-01T00:00:00.000Z"`, i) - } - b.WriteString(`}}`) - return []byte(b.String()) -} - func BenchmarkNPMRewriteMetadata(b *testing.B) { - body := benchPackument(5000) + body := npmbench.Packument(5000) h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy.example"} b.SetBytes(int64(len(body))) b.ReportAllocs() @@ -50,7 +19,7 @@ func BenchmarkNPMRewriteMetadata(b *testing.B) { } func BenchmarkNPMVersionTarball(b *testing.B) { - body := benchPackument(5000) + body := npmbench.Packument(5000) b.SetBytes(int64(len(body))) b.ReportAllocs() for b.Loop() { diff --git a/internal/metadata/npm/npmbench/npmbench.go b/internal/metadata/npm/npmbench/npmbench.go new file mode 100644 index 0000000..cc4ad57 --- /dev/null +++ b/internal/metadata/npm/npmbench/npmbench.go @@ -0,0 +1,42 @@ +// Package npmbench builds npm packuments for benchmarks. +package npmbench + +import ( + "fmt" + "strings" +) + +// readmeSentences makes each version's readme about 4KB. +const readmeSentences = 160 + +// Packument builds a full packument shaped like a large real one: versions +// 1.0.0 to 1.0.{versions-1}, each with dependencies, scripts and a readme, +// and a time entry for each. +func Packument(versions int) []byte { + var b strings.Builder + readme := strings.Repeat("Some documentation text. ", readmeSentences) + b.WriteString(`{"_id":"big","name":"big","dist-tags":{"latest":"1.0.` + fmt.Sprint(versions-1) + `"},"versions":{`) + for i := range versions { + if i > 0 { + b.WriteByte(',') + } + v := fmt.Sprintf("1.0.%d", i) + fmt.Fprintf(&b, `%q:{"name":"big","version":%q,"description":"A big package","main":"index.js",`+ + `"scripts":{"test":"node test.js","build":"tsc -p ."},"dependencies":{`, v, v) + for d := range 25 { + if d > 0 { + b.WriteByte(',') + } + fmt.Fprintf(&b, `"dep-%d":"^%d.0.0"`, d, d) + } + fmt.Fprintf(&b, `},"readme":%q,"dist":{"shasum":"0123456789abcdef0123456789abcdef01234567",`+ + `"integrity":"sha512-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==",`+ + `"tarball":"https://registry.npmjs.org/big/-/big-%s.tgz"}}`, readme, v) + } + b.WriteString(`},"time":{"created":"2015-01-01T00:00:00.000Z"`) + for i := range versions { + fmt.Fprintf(&b, `,"1.0.%d":"2016-01-01T00:00:00.000Z"`, i) + } + b.WriteString(`}}`) + return []byte(b.String()) +} diff --git a/internal/metadata/npm/stream.go b/internal/metadata/npm/stream.go index 3e86faa..b00f6a2 100644 --- a/internal/metadata/npm/stream.go +++ b/internal/metadata/npm/stream.go @@ -3,6 +3,7 @@ package npm import ( "bytes" "encoding/json" + "encoding/json/jsontext" "errors" "fmt" "io" @@ -44,11 +45,11 @@ func (s *Scan) Abbreviated() bool { // time and dropped. func ScanPackument(r io.Reader) (*Scan, error) { s := &Scan{occurrences: map[string]int{}} - dec := json.NewDecoder(r) + dec := newDecoder(r) if err := expectDelim(dec, '{'); err != nil { return nil, err } - for dec.More() { + for dec.PeekKind() != '}' { key, err := objectKey(dec) if err != nil { return nil, err @@ -60,16 +61,16 @@ func ScanPackument(r io.Reader) (*Scan, error) { case "time": s.occurrences[key]++ s.time = nil - err = dec.Decode(&s.time) + err = decodeValue(dec, &s.time) case "dist-tags": s.occurrences[key]++ s.distTags = nil - err = dec.Decode(&s.distTags) + err = decodeValue(dec, &s.distTags) default: if key == "modified" { s.hasModified = true } - err = skipValue(dec) + err = dec.SkipValue() } if err != nil { return nil, err @@ -78,13 +79,7 @@ func ScanPackument(r io.Reader) (*Scan, error) { if err := expectDelim(dec, '}'); err != nil { return nil, err } - if _, err := dec.Token(); err != io.EOF { - if err == nil { - err = errors.New("unexpected data after the top-level object") - } - return nil, err - } - return s, nil + return s, expectEnd(dec) } // RewriteStream is Rewrite for a packument too large to hold in memory. scan @@ -230,62 +225,82 @@ func (s *Scan) objectValue(metadata map[string]any, name string, value any) map[ return object } -func expectDelim(dec *json.Decoder, want json.Delim) error { - tok, err := dec.Token() +// newDecoder reads JSON as encoding/json does: of repeated names the last +// counts, and invalid UTF-8 in strings is replaced rather than refused. +func newDecoder(r io.Reader) *jsontext.Decoder { + return jsontext.NewDecoder(r, jsontext.AllowDuplicateNames(true), jsontext.AllowInvalidUTF8(true)) +} + +func expectDelim(dec *jsontext.Decoder, want jsontext.Kind) error { + tok, err := dec.ReadToken() if err != nil { return err } - if tok != want { - return fmt.Errorf("npm metadata: expected %v, got %v", want, tok) + if tok.Kind() != want { + return fmt.Errorf("npm metadata: expected %v, got %v", want, tok.Kind()) } return nil } -func objectKey(dec *json.Decoder) (string, error) { - tok, err := dec.Token() +// expectEnd checks nothing follows the top-level object. +func expectEnd(dec *jsontext.Decoder) error { + _, err := dec.ReadToken() + switch err { + case io.EOF: + return nil + case nil: + return errors.New("unexpected data after the top-level object") + } + return err +} + +func objectKey(dec *jsontext.Decoder) (string, error) { + tok, err := dec.ReadToken() if err != nil { return "", err } - key, ok := tok.(string) - if !ok { - return "", fmt.Errorf("npm metadata: object key is %T, not a string", tok) + if tok.Kind() != '"' { + return "", fmt.Errorf("npm metadata: object key is %v, not a string", tok.Kind()) } - return key, nil + return tok.String(), nil } -// objectKeys reads a value and returns its keys when it is an object. -func objectKeys(dec *json.Decoder) ([]string, bool, error) { - tok, err := dec.Token() +// decodeValue reads the next value into v, as encoding/json decodes it. +func decodeValue(dec *jsontext.Decoder, v any) error { + raw, err := dec.ReadValue() if err != nil { + return err + } + return json.Unmarshal(raw, v) +} + +// objectKeys reads a value and returns its keys when it is an object. The +// members' values are skipped without being kept. +func objectKeys(dec *jsontext.Decoder) ([]string, bool, error) { + kind := dec.PeekKind() + if kind != '{' && kind != '[' { + return nil, false, dec.SkipValue() + } + if _, err := dec.ReadToken(); err != nil { return nil, false, err } - delim, ok := tok.(json.Delim) - if !ok { - return nil, false, nil // a scalar, read whole by Token - } var keys []string - for dec.More() { - if delim == '{' { + for next := dec.PeekKind(); next != '}' && next != ']'; next = dec.PeekKind() { + if kind == '{' { key, err := objectKey(dec) if err != nil { return nil, false, err } keys = append(keys, key) } - if err := skipValue(dec); err != nil { + if err := dec.SkipValue(); err != nil { return nil, false, err } } - if _, err := dec.Token(); err != nil { + if _, err := dec.ReadToken(); err != nil { return nil, false, err } - return keys, delim == '{', nil -} - -// skipValue reads the next value without keeping it. -func skipValue(dec *json.Decoder) error { - var raw json.RawMessage - return dec.Decode(&raw) + return keys, kind == '{', nil } // VersionStream is Version for a packument too large to hold in memory. scan @@ -337,11 +352,11 @@ type VersionEntry struct { func LookupVersion(r io.Reader, version string) (VersionEntry, error) { var e VersionEntry var hasTime, hasModified bool - dec := json.NewDecoder(r) + dec := newDecoder(r) if err := expectDelim(dec, '{'); err != nil { return e, err } - for dec.More() { + for dec.PeekKind() != '}' { key, err := objectKey(dec) if err != nil { return e, err @@ -359,7 +374,7 @@ func LookupVersion(r io.Reader, version string) (VersionEntry, error) { } default: hasModified = hasModified || key == "modified" - err = skipValue(dec) + err = dec.SkipValue() } if err != nil { return e, err @@ -368,10 +383,7 @@ func LookupVersion(r io.Reader, version string) (VersionEntry, error) { if err := expectDelim(dec, '}'); err != nil { return e, err } - if _, err := dec.Token(); err != io.EOF { - if err == nil { - err = errors.New("unexpected data after the top-level object") - } + if err := expectEnd(dec); err != nil { return e, err } e.Abbreviated = hasModified && !hasTime @@ -379,33 +391,39 @@ func LookupVersion(r io.Reader, version string) (VersionEntry, error) { } // objectMember reads a value and returns its member key when it is an object -// that has one, the last if repeated, or nil. -func objectMember(dec *json.Decoder, key string) (json.RawMessage, error) { - tok, err := dec.Token() - if err != nil { +// that has one, the last if repeated, or nil. Other members are skipped +// without being kept. +func objectMember(dec *jsontext.Decoder, key string) (json.RawMessage, error) { + kind := dec.PeekKind() + if kind != '{' && kind != '[' { + return nil, dec.SkipValue() + } + if _, err := dec.ReadToken(); err != nil { return nil, err } - delim, ok := tok.(json.Delim) - if !ok { - return nil, nil - } var found json.RawMessage - for dec.More() { - var k string - if delim == '{' { + for next := dec.PeekKind(); next != '}' && next != ']'; next = dec.PeekKind() { + k := "" + if kind == '{' { + var err error if k, err = objectKey(dec); err != nil { return nil, err } } - var raw json.RawMessage - if err := dec.Decode(&raw); err != nil { + if kind == '{' && k == key { + raw, err := dec.ReadValue() + if err != nil { + return nil, err + } + // ReadValue's bytes are only valid until the next read. + found = bytes.Clone(raw) + continue + } + if err := dec.SkipValue(); err != nil { return nil, err } - if delim == '{' && k == key { - found = raw - } } - if _, err := dec.Token(); err != nil { + if _, err := dec.ReadToken(); err != nil { return nil, err } return found, nil diff --git a/internal/metadata/npm/stream_bench_test.go b/internal/metadata/npm/stream_bench_test.go new file mode 100644 index 0000000..827cf94 --- /dev/null +++ b/internal/metadata/npm/stream_bench_test.go @@ -0,0 +1,31 @@ +package npm + +import ( + "bytes" + "testing" + + "github.com/git-pkgs/proxy/internal/metadata/npm/npmbench" +) + +func BenchmarkScanPackument(b *testing.B) { + body := npmbench.Packument(5000) + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if _, err := ScanPackument(bytes.NewReader(body)); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkLookupVersion(b *testing.B) { + body := npmbench.Packument(5000) + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + entry, err := LookupVersion(bytes.NewReader(body), "1.0.2500") + if err != nil || entry.Entry == nil || entry.Published == "" { + b.Fatalf("LookupVersion = %+v, %v", entry, err) + } + } +} diff --git a/internal/metadata/npm/stream_test.go b/internal/metadata/npm/stream_test.go index 0c8a4ce..0bd93ae 100644 --- a/internal/metadata/npm/stream_test.go +++ b/internal/metadata/npm/stream_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "slices" "strings" "testing" @@ -223,3 +224,31 @@ func TestLookupVersion(t *testing.T) { t.Error("expected an error for truncated JSON") } } + +// TestStreamReadsJSONAsEncodingJSON checks what the streaming readers accept +// matches what Rewrite's encoding/json does: names repeated at any depth, +// invalid UTF-8 in strings, and a version entry's bytes kept as they are. +func TestStreamReadsJSONAsEncodingJSON(t *testing.T) { + entry := "{ \"name\" : \"demo\",\n \"x\":1, \"x\":2, \"note\":\"caf\xe9\" }" + doc := "{\"readme\":\"bad \xff byte\",\"meta\":{\"k\":1,\"k\":2},\"dist-tags\":{\"latest\":\"1.0.0\",\"latest\":\"1.0.0\"}," + + "\"versions\":{\"1.0.0\":" + entry + "},\"time\":{\"1.0.0\":\"2020-01-01T00:00:00Z\"}}\n" + if !json.Valid([]byte(strings.ToValidUTF8(doc, "?"))) { + t.Fatal("test document isn't JSON") + } + + scan, err := ScanPackument(strings.NewReader(doc)) + if err != nil { + t.Fatalf("ScanPackument: %v", err) + } + if !slices.Equal(scan.versions, []string{"1.0.0"}) || !reflect.DeepEqual(scan.distTags, map[string]any{"latest": "1.0.0"}) { + t.Errorf("scan versions %v, dist-tags %v", scan.versions, scan.distTags) + } + + got, err := LookupVersion(strings.NewReader(doc), "1.0.0") + if err != nil { + t.Fatalf("LookupVersion: %v", err) + } + if string(got.Entry) != entry || got.Published != "2020-01-01T00:00:00Z" { + t.Errorf("entry %q, published %q; want the entry's bytes %q", got.Entry, got.Published, entry) + } +} From 5c1eaa9fe5a1189476089a414178da0c910690db Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 7 Nov 2026 13:57:40 -0500 Subject: [PATCH 103/198] chore: test reliability / godoc on methods Code review feedback --- internal/storage/blob.go | 11 +++++++++++ internal/storage/blob_test.go | 14 ++++++++++++++ internal/storage/gcs.go | 11 +++++++++++ 3 files changed, 36 insertions(+) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index eeb8b3b..7519da8 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -286,6 +286,7 @@ func (b *Blob) clearLegacySidecar(key string) { } } +// Store writes the contents of r to path and returns the byte count and hex SHA-256 digest. func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { b.clearLegacySidecar(path) @@ -322,6 +323,7 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri return size, hash, nil } +// Open returns a reader for the object stored at path. func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { if localPath := b.localPath(path); localPath != "" { r, err := os.Open(localPath) @@ -349,6 +351,7 @@ func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { // otherwise. func (b *Blob) ReadsSeekable() bool { return true } +// Exists reports whether an object exists at path. func (b *Blob) Exists(ctx context.Context, path string) (bool, error) { exists, err := b.bucket.Exists(ctx, path) if err != nil { @@ -374,6 +377,10 @@ func (b *Blob) Delete(ctx context.Context, path string) error { return nil } +// SignedURL returns a presigned URL granting time-limited GET access to path. +// When signBucket is configured (S3 with WithSignEndpoint), it signs against +// the public endpoint; when signEndpoint is set on other backends (e.g. Azure), +// it rewrites the signed URL's scheme and host. func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { bucket := b.bucket if b.signBucket != nil { @@ -415,6 +422,7 @@ func rewriteSignedURLHost(signed, baseURL string) string { return s.String() } +// Size returns the size in bytes of the object at path. func (b *Blob) Size(ctx context.Context, path string) (int64, error) { attrs, err := b.bucket.Attributes(ctx, path) if err != nil { @@ -426,6 +434,7 @@ func (b *Blob) Size(ctx context.Context, path string) (int64, error) { return attrs.Size, nil } +// UsedSpace returns the total bytes stored across all objects in the bucket. func (b *Blob) UsedSpace(ctx context.Context) (int64, error) { var total int64 @@ -473,6 +482,7 @@ func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, err return objects, nil } +// Close closes the underlying storage bucket and any separate signing bucket. func (b *Blob) Close() error { err := b.bucket.Close() if b.signBucket != nil { @@ -483,6 +493,7 @@ func (b *Blob) Close() error { return err } +// URL returns the storage backend URL. func (b *Blob) URL() string { return b.url } diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index bffe09b..7a46413 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -262,6 +262,19 @@ func TestBlobSignedURLUnsupported(t *testing.T) { } func TestBlobSignedURLSignEndpoint(t *testing.T) { + awsDir := t.TempDir() + awsConfigFile := filepath.Join(awsDir, "config") + awsCredsFile := filepath.Join(awsDir, "credentials") + if err := os.WriteFile(awsConfigFile, nil, 0o600); err != nil { + t.Fatalf("writing empty AWS config file: %v", err) + } + if err := os.WriteFile(awsCredsFile, nil, 0o600); err != nil { + t.Fatalf("writing empty AWS credentials file: %v", err) + } + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_DEFAULT_PROFILE", "") + t.Setenv("AWS_CONFIG_FILE", awsConfigFile) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", awsCredsFile) t.Setenv("AWS_ACCESS_KEY_ID", "test") t.Setenv("AWS_SECRET_ACCESS_KEY", "test") @@ -323,6 +336,7 @@ func TestBlobSignedURLSignEndpoint(t *testing.T) { } func TestAzureSignedURLSignEndpoint(t *testing.T) { + t.Setenv("AZURE_STORAGE_SAS_TOKEN", "") t.Setenv("AZURE_STORAGE_ACCOUNT", "devstoreaccount1") t.Setenv("AZURE_STORAGE_KEY", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==") t.Setenv("AZURE_STORAGE_PROTOCOL", "http") diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go index 43b14e5..c2c98c0 100644 --- a/internal/storage/gcs.go +++ b/internal/storage/gcs.go @@ -27,6 +27,7 @@ func OpenGCS(ctx context.Context, urlStr string) (*GCS, error) { return &GCS{bucket: bucket, url: urlStr}, nil } +// Store writes the contents of r to path and returns the byte count and hex SHA-256 digest. func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { h := sha256.New() size, err := g.bucket.Write(ctx, path, io.TeeReader(r, h)) @@ -36,6 +37,7 @@ func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, strin return size, hex.EncodeToString(h.Sum(nil)), nil } +// Open returns a reader for the object stored at path. func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) { r, err := g.bucket.Open(ctx, path) if errors.Is(err, gcstorage.ErrNotFound) { @@ -44,14 +46,17 @@ func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) { return r, err } +// Exists reports whether an object exists at path. func (g *GCS) Exists(ctx context.Context, path string) (bool, error) { return g.bucket.Exists(ctx, path) } +// Delete removes the object at path. func (g *GCS) Delete(ctx context.Context, path string) error { return g.bucket.Delete(ctx, path) } +// Size returns the size in bytes of the object at path. func (g *GCS) Size(ctx context.Context, path string) (int64, error) { size, err := g.bucket.Size(ctx, path) if errors.Is(err, gcstorage.ErrNotFound) { @@ -60,6 +65,8 @@ func (g *GCS) Size(ctx context.Context, path string) (int64, error) { return size, err } +// SignedURL returns a presigned URL granting time-limited GET access to path, +// rewriting the scheme and host to signEndpoint when configured. func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { u, err := g.bucket.SignedURL(ctx, path, expiry) if errors.Is(err, gcstorage.ErrSignedURLUnsupported) { @@ -71,10 +78,12 @@ func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) return rewriteSignedURLHost(u, g.signEndpoint), nil } +// UsedSpace returns the total bytes stored across all objects in the bucket. func (g *GCS) UsedSpace(ctx context.Context) (int64, error) { return g.bucket.UsedSpace(ctx) } +// ListPrefix returns object metadata for keys under a prefix. func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) { objects, err := g.bucket.ListPrefix(ctx, prefix) if err != nil { @@ -92,10 +101,12 @@ func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, erro return result, nil } +// Close releases any resources held by the GCS client. func (g *GCS) Close() error { return nil } +// URL returns the storage backend URL. func (g *GCS) URL() string { return g.url } From 7d84bf768211382ba40995ee0bbd25ab4c0288df Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 7 Nov 2026 15:28:26 -0500 Subject: [PATCH 104/198] chore: remove deb source pkg logic / fix root-level pkg caching --- docs/architecture.md | 8 +-- docs/configuration.md | 4 -- internal/handler/debian.go | 29 +-------- internal/handler/debian_test.go | 101 +++++--------------------------- 4 files changed, 19 insertions(+), 123 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index cbd6dab..19d9c84 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -306,8 +306,8 @@ HTTP protocol handlers for each registry type. **DebianHandler:** - Serves `/debian` and `/debian/{name}` mirrors of APT archives (`upstream.debian_repositories`) -- Routes `.deb` packages to the artifact cache, source-package files directly to upstream, `dists/…` to the metadata cache, and proxies all other paths -- Keys the artifact cache on name, version, and architecture, scoped by repository for named archives +- Routes `.deb` packages to the artifact cache, `dists/…` to the metadata cache, and proxies all other paths +- Keys the artifact cache on name, version, and filename, scoped by repository for named archives **SwiftHandler:** - Proxies the Swift Package Registry v1 read endpoints @@ -434,10 +434,6 @@ An object a record stops pointing at, because a refetch replaced it or its entry - Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table - OCI manifests and tag lists are exceptions: they are cached automatically so previously fetched images remain pullable and tag resolution works when the registry or token service is unavailable -**Why are Debian source-package files streamed instead of cached?** -- Source-package files (`.dsc`, source tarballs, `.buildinfo`, `.changes`) can be re-uploaded or re-signed in place under the same path, whereas the artifact cache keys entries by version and filename without an expected digest from the `Sources` index -- Streaming them avoids serving stale bytes when an upstream file is replaced or caching them under the `dists/` metadata TTL in flat repositories - **Why stream artifacts?** - Memory efficient - don't load large files into RAM - Better latency - start sending while still receiving diff --git a/docs/configuration.md b/docs/configuration.md index 05914eb..63cc175 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -805,10 +805,6 @@ and `dists` are refused at config load for that reason, and other root paths an archive may serve (`indices`, `project`, `doc`, `tools`) are not, so avoid those names unless the shadowing is intended. -Binary packages (`.deb`) are cached; source-package files (`.dsc`, source -tarballs, `.buildinfo`, `.changes`) are streamed directly from upstream because -their contents can be replaced in place under the same path. - This field has no environment variable, as with the other named upstream maps. `PROXY_UPSTREAM_DEBIAN` still sets `upstream.debian`. diff --git a/internal/handler/debian.go b/internal/handler/debian.go index 87dbd65..565f181 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -108,13 +108,11 @@ func (h *DebianHandler) Routes() http.Handler { } } - // Route by filename rather than path prefix because flat repositories - // place pools under dists//pool/.... + // Route by filename rather than path prefix because repositories such as + // Docker's nest the pool under dists//. switch { case isDebPackageFile(rest): h.handlePackageDownload(w, r, repository, upstreamURL, rest) - case isSourcePackageFile(rest): - h.proxyFile(w, r, upstreamURL, rest) case strings.HasPrefix(rest, "dists/"): // Repository metadata - served through the metadata cache h.handleMetadata(w, r, repository, upstreamURL, rest) @@ -129,22 +127,6 @@ func isDebPackageFile(path string) bool { return strings.HasSuffix(path, ".deb") } -// isSourcePackageFile reports whether path is a Debian source-package artifact. -// Source files can be re-uploaded or re-signed in place under the same filename, -// so they must be streamed rather than cached without digest verification. -func isSourcePackageFile(path string) bool { - switch { - case strings.HasSuffix(path, ".dsc"), - strings.HasSuffix(path, ".buildinfo"), - strings.HasSuffix(path, ".changes"): - return true - } - // Match Debian source tarballs without matching suite tarballs (e.g. installer netboot.tar.gz). - return strings.Contains(path, ".orig.tar.") || - strings.Contains(path, ".orig-") || - strings.Contains(path, ".debian.tar.") -} - // repositoryNotFound answers a request addressed to a repository that is not // configured, naming the ones that are so a misspelling is self-evident. func (h *DebianHandler) repositoryNotFound(w http.ResponseWriter, name string) { @@ -248,12 +230,7 @@ var debPackagePattern = regexp.MustCompile(`^(.+)_([^_]+)_([^_]+)\.deb$`) // parsePoolPath extracts package info from a pool path. // Example: pool/main/n/nginx/nginx_1.18.0-6_amd64.deb func (h *DebianHandler) parsePoolPath(path string) (name, version, arch string) { - // Get the filename - idx := strings.LastIndex(path, "/") - if idx < 0 { - return "", "", "" - } - filename := path[idx+1:] + filename := path[strings.LastIndex(path, "/")+1:] // Parse the filename matches := debPackagePattern.FindStringSubmatch(filename) diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index 848f9e2..54a72b6 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -25,6 +25,7 @@ func TestDebianHandler_parsePoolPath(t *testing.T) { "nmap", "7.91+dfsg1+really7.80+dfsg1-2ubuntu0.1", "amd64", }, {"pool/main/o/openssl/openssl_3.0.2-0ubuntu1.15~build1_amd64.deb", "openssl", "3.0.2-0ubuntu1.15~build1", "amd64"}, + {"nginx_1.18.0-6_amd64.deb", "nginx", "1.18.0-6", "amd64"}, {"invalid/path", "", "", ""}, {"pool/main/n/nginx/nginx.deb", "", "", ""}, }) @@ -251,14 +252,8 @@ func TestDebianHandler_PackageCacheScopedPerRepository(t *testing.T) { } } -// TestDebianHandler_FlatArchivePackageIsCached verifies that .deb packages -// served under dists//pool/ (as used by flat repositories like Docker's) -// are cached as artifacts rather than treated as uncached/TTL metadata. func TestDebianHandler_FlatArchivePackageIsCached(t *testing.T) { - const ( - flatPath = "/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb" - pkgBody = "docker-ce package bytes" - ) + const pkgBody = "docker-ce package bytes" var upstreamHits int @@ -277,16 +272,21 @@ func TestDebianHandler_FlatArchivePackageIsCached(t *testing.T) { "docker": archive.URL, }) - for attempt := 1; attempt <= 2; attempt++ { - w := serveDebianRequest(h, flatPath) - if w.Code != http.StatusOK || w.Body.String() != pkgBody { - t.Fatalf("request %d: status = %d, body = %q, want 200 %q", - attempt, w.Code, w.Body.String(), pkgBody) + for _, path := range []string{ + "/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb", + "/docker/foo_1.0_amd64.deb", + } { + for attempt := 1; attempt <= 2; attempt++ { + w := serveDebianRequest(h, path) + if w.Code != http.StatusOK || w.Body.String() != pkgBody { + t.Fatalf("%s request %d: status = %d, body = %q, want 200 %q", + path, attempt, w.Code, w.Body.String(), pkgBody) + } } } - if upstreamHits != 1 { - t.Errorf("upstream fetched %d times for two requests, want 1: the package is not cached", upstreamHits) + if upstreamHits != 2 { + t.Errorf("upstream fetched %d times for four requests, want 2: the package is not cached", upstreamHits) } } @@ -324,76 +324,3 @@ func TestDebRetentionKey(t *testing.T) { t.Error("debRetentionKey(nil) accepted") } } - -func TestDebianHandler_ClassifiesFilesByExtension(t *testing.T) { - tests := []struct { - path string - pkg bool - source bool - why string - }{ - {"pool/main/h/hello/hello_2.10-3_amd64.deb", true, false, "canonical pool"}, - {"dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1~debian.13~trixie_amd64.deb", true, false, "flat archive"}, - {"pool/main/h/hello/hello_2.10-3.udeb", false, false, "udeb not matched by debPackagePattern"}, - {"pool/main/h/hello/hello_2.10-3_amd64.ddeb", false, false, "ddeb not matched by debPackagePattern"}, - {"pool/main/h/hello/hello_2.10-3.dsc", false, true, "source control file"}, - {"pool/main/h/hello/hello_2.10.orig.tar.xz", false, true, "source tarball"}, - {"pool/main/h/hello/hello_2.10.orig-tex.tar.gz", false, true, "component tarball"}, - {"pool/main/h/hello/hello_2.10-3.debian.tar.xz", false, true, "debian tarball"}, - {"pool/main/h/hello/hello_2.10-3_amd64.buildinfo", false, true, "buildinfo"}, - {"pool/main/h/hello/hello_2.10-3_amd64.changes", false, true, "changes"}, - {"dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1.dsc", false, true, "flat archive source file"}, - {"dists/trixie/InRelease", false, false, "metadata"}, - {"dists/trixie/main/binary-amd64/Packages.gz", false, false, "metadata"}, - {"dists/trixie/main/installer-amd64/current/images/netboot/netboot.tar.gz", false, false, "suite tarball"}, - } - for _, tt := range tests { - if got := isDebPackageFile(tt.path); got != tt.pkg { - t.Errorf("isDebPackageFile(%q) = %v, want %v (%s)", tt.path, got, tt.pkg, tt.why) - } - if got := isSourcePackageFile(tt.path); got != tt.source { - t.Errorf("isSourcePackageFile(%q) = %v, want %v (%s)", tt.path, got, tt.source, tt.why) - } - } -} - -func TestDebianHandler_SourcePackageFilesAreNotCached(t *testing.T) { - const dscBody = "hello source control bytes" - - var upstreamHits int - - archive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - upstreamHits++ - _, _ = fmt.Fprint(w, dscBody) - })) - defer archive.Close() - - proxy, _, store, _ := setupTestProxy(t) - proxy.HTTPClient = archive.Client() - proxy.CacheMetadata = true - proxy.MetadataTTL = time.Hour - - h := NewDebianHandler(proxy, "http://proxy.example", archive.URL, map[string]string{ - "docker": archive.URL, - }) - - for _, target := range []string{ - "/pool/main/h/hello/hello_2.10-3.dsc", - "/docker/dists/trixie/pool/stable/amd64/docker-ce_5.0.1-1.dsc", - } { - for attempt := 1; attempt <= 2; attempt++ { - w := serveDebianRequest(h, target) - if w.Code != http.StatusOK || w.Body.String() != dscBody { - t.Fatalf("%s request %d: status = %d, body = %q, want 200 %q", - target, attempt, w.Code, w.Body.String(), dscBody) - } - } - } - - if upstreamHits != 4 { - t.Errorf("upstream fetched %d times for four requests, want 4: source-package files must not be cached", upstreamHits) - } - if store.stores != 0 { - t.Errorf("storage wrote %d object(s), want 0: a source-package file must not be stored", store.stores) - } -} From e4bc1f5f11b32fad5acf60e62636b3fc28e96bb2 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 7 Nov 2026 16:50:24 -0500 Subject: [PATCH 105/198] fix: do not prepend host on signing --- internal/server/server_test.go | 57 ++++++++++++++++++++++++++++++ internal/storage/blob.go | 39 +++++++++++++++------ internal/storage/blob_test.go | 64 ++++++++++++++++++++++++++++------ 3 files changed, 139 insertions(+), 21 deletions(-) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index e2d2264..93328ac 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1655,6 +1655,63 @@ func TestNewServer_StorageConnectivityCheck(t *testing.T) { _ = srv.db.Close() } +func TestNewServer_DirectServeBaseURL(t *testing.T) { + tempDir := t.TempDir() + awsConfigFile := filepath.Join(tempDir, "aws-config") + awsCredsFile := filepath.Join(tempDir, "aws-credentials") + if err := os.WriteFile(awsConfigFile, nil, 0o600); err != nil { + t.Fatalf("writing empty AWS config file: %v", err) + } + if err := os.WriteFile(awsCredsFile, nil, 0o600); err != nil { + t.Fatalf("writing empty AWS credentials file: %v", err) + } + t.Setenv("AWS_PROFILE", "") + t.Setenv("AWS_DEFAULT_PROFILE", "") + t.Setenv("AWS_CONFIG_FILE", awsConfigFile) + t.Setenv("AWS_SHARED_CREDENTIALS_FILE", awsCredsFile) + t.Setenv("AWS_ACCESS_KEY_ID", "test") + t.Setenv("AWS_SECRET_ACCESS_KEY", "test") + + s3Srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer s3Srv.Close() + + cfg := &config.Config{ + Listen: ":0", + BaseURL: "http://localhost:8080", + Storage: config.StorageConfig{ + URL: "s3://bucket?region=us-1&endpoint=" + s3Srv.URL + "&use_path_style=true", + DirectServe: true, + DirectServeBaseURL: "https://cdn.example.com", + }, + Database: config.DatabaseConfig{Path: filepath.Join(tempDir, "test.db")}, + } + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + srv, err := New(cfg, logger, BuildInfo{}) + if err != nil { + t.Fatalf("New() failed: %v", err) + } + defer func() { + _ = srv.storage.Close() + _ = srv.db.Close() + }() + + signed, err := srv.storage.SignedURL(context.Background(), "npm/lodash/4.17.21/lodash-4.17.21.tgz", time.Minute) + if err != nil { + t.Fatalf("SignedURL() failed: %v", err) + } + + u, err := url.Parse(signed) + if err != nil { + t.Fatalf("parsing signed URL %q: %v", signed, err) + } + if u.Scheme != "https" || u.Host != "cdn.example.com" { + t.Errorf("signed URL = %q, want scheme https and host cdn.example.com", signed) + } +} + func TestNewServer_InvalidAccessLogFailsBeforeDatabaseInit(t *testing.T) { tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test.db") diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 7519da8..0668539 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -14,6 +14,9 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" "gocloud.dev/blob" _ "gocloud.dev/blob/azureblob" _ "gocloud.dev/blob/fileblob" @@ -179,27 +182,41 @@ func OpenBucket(ctx context.Context, urlStr string, opts ...Option) (Storage, er return b, nil } -// openSignBucket opens an S3 bucket with the endpoint overridden to signEndpoint -// while preserving all other storage URL parameters (region, use_path_style, etc.). +// openSignBucket opens an S3 bucket that signs presigned URLs against the +// scheme and host of signEndpoint while preserving the storage URL's +// addressing mode and object path. func openSignBucket(ctx context.Context, urlStr, signEndpoint string) (*blob.Bucket, error) { sign, err := url.Parse(signEndpoint) if err != nil || sign.Scheme == "" || sign.Host == "" { return nil, fmt.Errorf("invalid sign endpoint %q: must be an absolute URL", signEndpoint) } - u, err := url.Parse(urlStr) + bucket, err := blob.OpenBucket(ctx, urlStr) if err != nil { - return nil, fmt.Errorf("parsing storage URL: %w", err) + return nil, fmt.Errorf("opening bucket at sign endpoint %s: %w", signEndpoint, err) } - endpoint := sign.Scheme + "://" + sign.Host - q := u.Query() - q.Set("endpoint", endpoint) - u.RawQuery = q.Encode() - bucket, err := blob.OpenBucket(ctx, u.String()) - if err != nil { - return nil, fmt.Errorf("opening bucket at sign endpoint %s: %w", endpoint, err) + var s3Client *s3.Client + if bucket.As(&s3Client) { + *s3Client = *s3.New(s3Client.Options(), func(o *s3.Options) { + o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { + return stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc( + "signEndpointHost", + func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) ( + middleware.FinalizeOutput, middleware.Metadata, error, + ) { + if req, ok := in.Request.(*smithyhttp.Request); ok { + req.URL.Scheme = sign.Scheme + req.URL.Host = sign.Host + req.Host = sign.Host + } + return next.HandleFinalize(ctx, in) + }, + ), "PresignHTTPRequest", middleware.Before) + }) + }) } + return bucket, nil } diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 7a46413..233b501 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -7,6 +7,7 @@ import ( "errors" "io" "io/fs" + "net/http" "net/url" "os" "path/filepath" @@ -17,6 +18,8 @@ import ( "testing" "time" + "github.com/aws/aws-sdk-go-v2/aws" + awssigv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "gocloud.dev/blob" ) @@ -279,25 +282,39 @@ func TestBlobSignedURLSignEndpoint(t *testing.T) { t.Setenv("AWS_SECRET_ACCESS_KEY", "test") const ( - storageURL = "s3://bucket?region=us-1&endpoint=http://127.0.0.1:9000&use_path_style=true" + pathStyleURL = "s3://bucket?region=us-1&endpoint=http://127.0.0.1:9000&use_path_style=true" + vhostURL = "s3://bucket?region=us-1&endpoint=http://127.0.0.1:9000" signEndpoint = "https://cdn.example.com" artifactPath = "npm/lodash/4.17.21/lodash-4.17.21.tgz" ) tests := []struct { - name string - opts []Option - host string + name string + storageURL string + opts []Option + host string + path string }{ { "storage endpoint without a sign endpoint", + pathStyleURL, nil, "127.0.0.1:9000", + "/bucket/" + artifactPath, }, { "configured sign endpoint, reached over another host", + pathStyleURL, []Option{WithSignEndpoint(signEndpoint)}, "cdn.example.com", + "/bucket/" + artifactPath, + }, + { + "configured sign endpoint with virtual-hosted storage URL", + vhostURL, + []Option{WithSignEndpoint(signEndpoint)}, + "cdn.example.com", + "/" + artifactPath, }, } @@ -305,7 +322,7 @@ func TestBlobSignedURLSignEndpoint(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() - b, err := OpenBucket(ctx, storageURL, tt.opts...) + b, err := OpenBucket(ctx, tt.storageURL, tt.opts...) if err != nil { t.Fatalf("OpenBucket failed: %v", err) } @@ -325,16 +342,43 @@ func TestBlobSignedURLSignEndpoint(t *testing.T) { } // The bucket and key stay where the storage URL puts them, so the // request the client makes still names the object. - if want := "/bucket/" + artifactPath; u.Path != want { - t.Errorf("signed URL path = %q, want %q", u.Path, want) - } - if u.Query().Get("X-Amz-Signature") == "" { - t.Errorf("signed URL %q carries no signature", signed) + if u.Path != tt.path { + t.Errorf("signed URL path = %q, want %q", u.Path, tt.path) } + assertSigV4Signature(t, u, "us-1", aws.Credentials{AccessKeyID: "test", SecretAccessKey: "test"}) }) } } +func assertSigV4Signature(t *testing.T, u *url.URL, region string, creds aws.Credentials) { + t.Helper() + + q := u.Query() + signTime, err := time.Parse("20060102T150405Z", q.Get("X-Amz-Date")) + if err != nil { + t.Fatalf("parsing X-Amz-Date from %q: %v", u, err) + } + q.Del("X-Amz-Signature") + unsigned := &url.URL{Scheme: u.Scheme, Host: u.Host, Path: u.Path, RawQuery: q.Encode()} + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, unsigned.String(), nil) + if err != nil { + t.Fatalf("creating request: %v", err) + } + wantSigned, _, err := awssigv4.NewSigner(func(o *awssigv4.SignerOptions) { + o.DisableURIPathEscaping = true + }).PresignHTTP(context.Background(), creds, req, "UNSIGNED-PAYLOAD", "s3", region, signTime) + if err != nil { + t.Fatalf("PresignHTTP failed: %v", err) + } + wantURL, err := url.Parse(wantSigned) + if err != nil { + t.Fatalf("parsing expected signed URL %q: %v", wantSigned, err) + } + if gotSig, wantSig := u.Query().Get("X-Amz-Signature"), wantURL.Query().Get("X-Amz-Signature"); gotSig != wantSig { + t.Errorf("X-Amz-Signature = %q, want %q (for host %q)", gotSig, wantSig, u.Host) + } +} + func TestAzureSignedURLSignEndpoint(t *testing.T) { t.Setenv("AZURE_STORAGE_SAS_TOKEN", "") t.Setenv("AZURE_STORAGE_ACCOUNT", "devstoreaccount1") From dadc94dd62e47d4a52e6ae49c42b7e2e86ae92d2 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 7 Nov 2026 17:06:44 -0500 Subject: [PATCH 106/198] chore: add some godoc on modified complex funcs --- internal/server/server.go | 1 + internal/server/server_test.go | 2 ++ internal/storage/blob.go | 6 ------ internal/storage/blob_test.go | 7 +++++++ internal/storage/gcs.go | 9 --------- internal/storage/gcs_test.go | 2 ++ 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/internal/server/server.go b/internal/server/server.go index 4466d06..c10d1c5 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -283,6 +283,7 @@ func (s *Server) configureGitCache(client *http.Client) error { return nil } +// serve wires up HTTP routes, background workers, and serves requests on listener. func (s *Server) serve(listener net.Listener) error { metadataClient, artifactClient := s.newUpstreamClients(s.cfg.ParseArtifactTimeout()) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 93328ac..830fb1e 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1655,6 +1655,8 @@ func TestNewServer_StorageConnectivityCheck(t *testing.T) { _ = srv.db.Close() } +// TestNewServer_DirectServeBaseURL verifies that New passes DirectServeBaseURL +// to storage.OpenBucket when DirectServe is enabled. func TestNewServer_DirectServeBaseURL(t *testing.T) { tempDir := t.TempDir() awsConfigFile := filepath.Join(tempDir, "aws-config") diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 0668539..930bd5d 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -303,7 +303,6 @@ func (b *Blob) clearLegacySidecar(key string) { } } -// Store writes the contents of r to path and returns the byte count and hex SHA-256 digest. func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { b.clearLegacySidecar(path) @@ -340,7 +339,6 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri return size, hash, nil } -// Open returns a reader for the object stored at path. func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { if localPath := b.localPath(path); localPath != "" { r, err := os.Open(localPath) @@ -368,7 +366,6 @@ func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { // otherwise. func (b *Blob) ReadsSeekable() bool { return true } -// Exists reports whether an object exists at path. func (b *Blob) Exists(ctx context.Context, path string) (bool, error) { exists, err := b.bucket.Exists(ctx, path) if err != nil { @@ -439,7 +436,6 @@ func rewriteSignedURLHost(signed, baseURL string) string { return s.String() } -// Size returns the size in bytes of the object at path. func (b *Blob) Size(ctx context.Context, path string) (int64, error) { attrs, err := b.bucket.Attributes(ctx, path) if err != nil { @@ -451,7 +447,6 @@ func (b *Blob) Size(ctx context.Context, path string) (int64, error) { return attrs.Size, nil } -// UsedSpace returns the total bytes stored across all objects in the bucket. func (b *Blob) UsedSpace(ctx context.Context) (int64, error) { var total int64 @@ -510,7 +505,6 @@ func (b *Blob) Close() error { return err } -// URL returns the storage backend URL. func (b *Blob) URL() string { return b.url } diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 233b501..cac35f6 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -264,6 +264,8 @@ func TestBlobSignedURLUnsupported(t *testing.T) { } } +// TestBlobSignedURLSignEndpoint verifies that S3 presigned URLs are signed for +// the configured public host while preserving the storage URL's path style. func TestBlobSignedURLSignEndpoint(t *testing.T) { awsDir := t.TempDir() awsConfigFile := filepath.Join(awsDir, "config") @@ -350,6 +352,8 @@ func TestBlobSignedURLSignEndpoint(t *testing.T) { } } +// assertSigV4Signature recomputes the SigV4 presigned signature for u and +// verifies it matches u's X-Amz-Signature query parameter. func assertSigV4Signature(t *testing.T, u *url.URL, region string, creds aws.Credentials) { t.Helper() @@ -379,6 +383,8 @@ func assertSigV4Signature(t *testing.T, u *url.URL, region string, creds aws.Cre } } +// TestAzureSignedURLSignEndpoint verifies that Azure presigned URLs have their +// scheme and host rewritten to the configured sign endpoint. func TestAzureSignedURLSignEndpoint(t *testing.T) { t.Setenv("AZURE_STORAGE_SAS_TOKEN", "") t.Setenv("AZURE_STORAGE_ACCOUNT", "devstoreaccount1") @@ -410,6 +416,7 @@ func TestAzureSignedURLSignEndpoint(t *testing.T) { } } +// TestRewriteSignedURLHost verifies scheme and host substitution on signed URLs. func TestRewriteSignedURLHost(t *testing.T) { tests := []struct { name string diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go index c2c98c0..313cacc 100644 --- a/internal/storage/gcs.go +++ b/internal/storage/gcs.go @@ -27,7 +27,6 @@ func OpenGCS(ctx context.Context, urlStr string) (*GCS, error) { return &GCS{bucket: bucket, url: urlStr}, nil } -// Store writes the contents of r to path and returns the byte count and hex SHA-256 digest. func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { h := sha256.New() size, err := g.bucket.Write(ctx, path, io.TeeReader(r, h)) @@ -37,7 +36,6 @@ func (g *GCS) Store(ctx context.Context, path string, r io.Reader) (int64, strin return size, hex.EncodeToString(h.Sum(nil)), nil } -// Open returns a reader for the object stored at path. func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) { r, err := g.bucket.Open(ctx, path) if errors.Is(err, gcstorage.ErrNotFound) { @@ -46,17 +44,14 @@ func (g *GCS) Open(ctx context.Context, path string) (io.ReadCloser, error) { return r, err } -// Exists reports whether an object exists at path. func (g *GCS) Exists(ctx context.Context, path string) (bool, error) { return g.bucket.Exists(ctx, path) } -// Delete removes the object at path. func (g *GCS) Delete(ctx context.Context, path string) error { return g.bucket.Delete(ctx, path) } -// Size returns the size in bytes of the object at path. func (g *GCS) Size(ctx context.Context, path string) (int64, error) { size, err := g.bucket.Size(ctx, path) if errors.Is(err, gcstorage.ErrNotFound) { @@ -78,12 +73,10 @@ func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) return rewriteSignedURLHost(u, g.signEndpoint), nil } -// UsedSpace returns the total bytes stored across all objects in the bucket. func (g *GCS) UsedSpace(ctx context.Context) (int64, error) { return g.bucket.UsedSpace(ctx) } -// ListPrefix returns object metadata for keys under a prefix. func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) { objects, err := g.bucket.ListPrefix(ctx, prefix) if err != nil { @@ -101,12 +94,10 @@ func (g *GCS) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, erro return result, nil } -// Close releases any resources held by the GCS client. func (g *GCS) Close() error { return nil } -// URL returns the storage backend URL. func (g *GCS) URL() string { return g.url } diff --git a/internal/storage/gcs_test.go b/internal/storage/gcs_test.go index 461fb86..92cf3fd 100644 --- a/internal/storage/gcs_test.go +++ b/internal/storage/gcs_test.go @@ -95,6 +95,8 @@ func TestOpenBucketGCSRoundTripWithEmulator(t *testing.T) { } } +// TestGCSSignedURLSignEndpoint verifies that GCS presigned URLs have their +// scheme and host rewritten to the configured sign endpoint. func TestGCSSignedURLSignEndpoint(t *testing.T) { key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { From 9f400c34af35c5b814486950db5cb22545ee0472 Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 7 Nov 2026 17:13:27 -0500 Subject: [PATCH 107/198] fix: compile error in cooldown after merge (remove direct serve URL) --- internal/handler/profile.go | 2 +- internal/handler/profile_test.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/handler/profile.go b/internal/handler/profile.go index 233e7bb..5236979 100644 --- a/internal/handler/profile.go +++ b/internal/handler/profile.go @@ -14,7 +14,7 @@ func (p *Proxy) WithCooldown(policy CooldownPolicy, shortest time.Duration) *Pro GradleReadOnly: p.GradleReadOnly, HostedEnabled: p.HostedEnabled, GradleMaxUploadSize: p.GradleMaxUploadSize, NPMFullMetadata: p.NPMFullMetadata, DirectServe: p.DirectServe, DirectServeTTL: p.DirectServeTTL, - DirectServeBaseURL: p.DirectServeBaseURL, HTTPClient: p.HTTPClient, + HTTPClient: p.HTTPClient, RateLimits: p.RateLimits, HostedNPMScopes: p.HostedNPMScopes, HostedComposerVendors: p.HostedComposerVendors, StreamArtifacts: p.StreamArtifacts, Scanners: p.Scanners, ScanSigningKey: p.ScanSigningKey, ScanFetchBaseURL: p.ScanFetchBaseURL, diff --git a/internal/handler/profile_test.go b/internal/handler/profile_test.go index 5caf7b8..070b85b 100644 --- a/internal/handler/profile_test.go +++ b/internal/handler/profile_test.go @@ -18,6 +18,8 @@ import ( registryclient "github.com/git-pkgs/registries/client" ) +// TestWithCooldownCopiesProxyFields verifies that WithCooldown copies all +// non-isolated Proxy fields onto the returned view. func TestWithCooldownCopiesProxyFields(t *testing.T) { p, _, _, _ := setupTestProxy(t) p.RouteFor = func(string, string) *ArtifactRoute { return nil } @@ -34,7 +36,6 @@ func TestWithCooldownCopiesProxyFields(t *testing.T) { p.NPMFullMetadata = true p.DirectServe = true p.DirectServeTTL = time.Minute - p.DirectServeBaseURL = "https://storage.example.com" p.RateLimits = registryclient.NewRateLimitGate(http.DefaultTransport) p.HostedNPMScopes = []string{"@example"} p.HostedComposerVendors = []string{"example"} From ae0be48b3688a88fec2aa8a6f74d2b1b5d3d918c Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Sat, 7 Nov 2026 18:29:31 -0500 Subject: [PATCH 108/198] chore: clean-up/clarity around signing across providers --- internal/storage/blob.go | 103 +++++++++++++++------------------------ internal/storage/gcs.go | 14 ++++-- 2 files changed, 48 insertions(+), 69 deletions(-) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 930bd5d..41e1283 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -35,8 +35,6 @@ const ( uploadBufferSize = 5 << 20 uploadConcurrency = 2 - s3Scheme = "s3://" - // attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an // earlier version wrote. attrsExt = ".attrs" @@ -48,13 +46,11 @@ type Blob struct { bucket *blob.Bucket url string - // signBucket, when set, is used by SignedURL to sign S3 presigned URLs for - // a public endpoint rather than the internal storage endpoint. - signBucket *blob.Bucket - - // signEndpoint, when set on non-S3 backends, replaces the scheme and host - // of presigned URLs returned by SignedURL. - signEndpoint string + // postSignRewriteEndpoint, when set, replaces the scheme and host of + // presigned URLs after signing for backends that do not include the host + // in their signature (e.g. Azure). S3 rewrites the host before signing via + // configureS3SignEndpoint so the SigV4 Host signature matches. + postSignRewriteEndpoint string // fileRoot is the directory backing a file:// bucket, empty for cloud // backends. Used only to clear sidecars an earlier version wrote. @@ -101,7 +97,7 @@ func OpenBucket(ctx context.Context, urlStr string, opts ...Option) (Storage, er if err != nil { return nil, err } - g.signEndpoint = o.signEndpoint + g.postSignRewriteEndpoint = o.signEndpoint return g, nil } @@ -167,57 +163,50 @@ func OpenBucket(ctx context.Context, urlStr string, opts ...Option) (Storage, er b := &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot} if o.signEndpoint != "" { - if strings.HasPrefix(urlStr, s3Scheme) { - signBucket, err := openSignBucket(ctx, urlStr, o.signEndpoint) - if err != nil { + var s3Client *s3.Client + if bucket.As(&s3Client) { + if err := configureS3SignEndpoint(s3Client, o.signEndpoint); err != nil { _ = bucket.Close() return nil, err } - b.signBucket = signBucket } else { - b.signEndpoint = o.signEndpoint + b.postSignRewriteEndpoint = o.signEndpoint } } return b, nil } -// openSignBucket opens an S3 bucket that signs presigned URLs against the -// scheme and host of signEndpoint while preserving the storage URL's -// addressing mode and object path. -func openSignBucket(ctx context.Context, urlStr, signEndpoint string) (*blob.Bucket, error) { +// configureS3SignEndpoint registers a middleware on s3Client that rewrites +// presigned request URLs to the scheme and host of signEndpoint before SigV4 +// signing, while leaving regular S3 operations and object paths unchanged. +func configureS3SignEndpoint(s3Client *s3.Client, signEndpoint string) error { sign, err := url.Parse(signEndpoint) if err != nil || sign.Scheme == "" || sign.Host == "" { - return nil, fmt.Errorf("invalid sign endpoint %q: must be an absolute URL", signEndpoint) + return fmt.Errorf("invalid sign endpoint %q: must be an absolute URL", signEndpoint) } - bucket, err := blob.OpenBucket(ctx, urlStr) - if err != nil { - return nil, fmt.Errorf("opening bucket at sign endpoint %s: %w", signEndpoint, err) - } - - var s3Client *s3.Client - if bucket.As(&s3Client) { - *s3Client = *s3.New(s3Client.Options(), func(o *s3.Options) { - o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { - return stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc( - "signEndpointHost", - func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) ( - middleware.FinalizeOutput, middleware.Metadata, error, - ) { - if req, ok := in.Request.(*smithyhttp.Request); ok { - req.URL.Scheme = sign.Scheme - req.URL.Host = sign.Host - req.Host = sign.Host - } - return next.HandleFinalize(ctx, in) - }, - ), "PresignHTTPRequest", middleware.Before) - }) + *s3Client = *s3.New(s3Client.Options(), func(o *s3.Options) { + o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { + if _, ok := stack.Finalize.Get("PresignHTTPRequest"); !ok { + return nil + } + return stack.Finalize.Insert(middleware.FinalizeMiddlewareFunc( + "signEndpointHost", + func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) ( + middleware.FinalizeOutput, middleware.Metadata, error, + ) { + if req, ok := in.Request.(*smithyhttp.Request); ok { + req.URL.Scheme = sign.Scheme + req.URL.Host = sign.Host + req.Host = sign.Host + } + return next.HandleFinalize(ctx, in) + }, + ), "PresignHTTPRequest", middleware.Before) }) - } - - return bucket, nil + }) + return nil } // legacySidecarPath gives the ".attrs" path an earlier version wrote for key, @@ -392,15 +381,8 @@ func (b *Blob) Delete(ctx context.Context, path string) error { } // SignedURL returns a presigned URL granting time-limited GET access to path. -// When signBucket is configured (S3 with WithSignEndpoint), it signs against -// the public endpoint; when signEndpoint is set on other backends (e.g. Azure), -// it rewrites the signed URL's scheme and host. func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { - bucket := b.bucket - if b.signBucket != nil { - bucket = b.signBucket - } - signed, err := bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ + signed, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ Method: http.MethodGet, Expiry: expiry, }) @@ -410,8 +392,8 @@ func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) } return "", fmt.Errorf("signing URL: %w", err) } - if b.signBucket == nil && b.signEndpoint != "" { - signed = rewriteSignedURLHost(signed, b.signEndpoint) + if b.postSignRewriteEndpoint != "" { + signed = rewriteSignedURLHost(signed, b.postSignRewriteEndpoint) } return signed, nil } @@ -494,15 +476,8 @@ func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, err return objects, nil } -// Close closes the underlying storage bucket and any separate signing bucket. func (b *Blob) Close() error { - err := b.bucket.Close() - if b.signBucket != nil { - if signErr := b.signBucket.Close(); err == nil { - err = signErr - } - } - return err + return b.bucket.Close() } func (b *Blob) URL() string { diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go index 313cacc..675a101 100644 --- a/internal/storage/gcs.go +++ b/internal/storage/gcs.go @@ -13,9 +13,13 @@ import ( // GCS adapts a Google Cloud Storage bucket to Storage. type GCS struct { - bucket *gcstorage.Bucket - url string - signEndpoint string + bucket *gcstorage.Bucket + url string + + // postSignRewriteEndpoint replaces the scheme and host of presigned URLs + // after signing (GCS signs against storage.googleapis.com and does not + // include the rewritten host in its V2 signature). + postSignRewriteEndpoint string } // OpenGCS opens a Google Cloud Storage bucket from a gs:// URL. @@ -61,7 +65,7 @@ func (g *GCS) Size(ctx context.Context, path string) (int64, error) { } // SignedURL returns a presigned URL granting time-limited GET access to path, -// rewriting the scheme and host to signEndpoint when configured. +// rewriting the scheme and host to postSignRewriteEndpoint when configured. func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { u, err := g.bucket.SignedURL(ctx, path, expiry) if errors.Is(err, gcstorage.ErrSignedURLUnsupported) { @@ -70,7 +74,7 @@ func (g *GCS) SignedURL(ctx context.Context, path string, expiry time.Duration) if err != nil { return "", err } - return rewriteSignedURLHost(u, g.signEndpoint), nil + return rewriteSignedURLHost(u, g.postSignRewriteEndpoint), nil } func (g *GCS) UsedSpace(ctx context.Context) (int64, error) { From 70d9db29065099cade5ed82f149233d501009342 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:55:24 +0100 Subject: [PATCH 109/198] Add CRAN routes for macOS arch layout and Meta files R 4.3 and later on macOS fetch binaries from a two-segment platform path such as bin/macosx/big-sur-arm64/contrib/4.4/. The router only matched a single platform segment, so both the PACKAGES indexes and the .tgz downloads returned 404. Add routes for that layout and include the arch in the stored version so arm64 and x86_64 builds stay apart. src/contrib/Meta/archive.rds and current.rds, which remotes and pak use to install archived versions, had no route either. Serve .rds files under Meta as cached metadata. --- README.md | 2 + internal/handler/cran.go | 23 ++++++++++++ internal/handler/cran_test.go | 71 +++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/README.md b/README.md index a43c417..dff6964 100644 --- a/README.md +++ b/README.md @@ -498,6 +498,8 @@ local({ }) ``` +Source packages, Windows and macOS binaries (including the `bin/macosx/big-sur-arm64/` layout used by R 4.3 and later) are cached as artifacts. `PACKAGES` indexes and the `src/contrib/Meta/*.rds` files that `remotes::install_version` and pak read are cached as metadata. + ### Julia Set the Pkg server before starting Julia: diff --git a/internal/handler/cran.go b/internal/handler/cran.go index 7cba9f4..10f1750 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -51,21 +51,41 @@ func (h *CRANHandler) Routes() http.Handler { mux.HandleFunc("GET /src/contrib/PACKAGES.gz", h.proxyCached) mux.HandleFunc("GET /src/contrib/PACKAGES.rds", h.proxyCached) + // Repository metadata such as archive.rds and current.rds, used by + // remotes::install_version and pak to find archived versions. + mux.HandleFunc("GET /src/contrib/Meta/{filename}", h.handleMeta) + // Binary package indexes mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES", h.proxyCached) mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES.gz", h.proxyCached) mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/PACKAGES.rds", h.proxyCached) + // macOS binaries for R >= 4.3 use two platform segments, e.g. + // bin/macosx/big-sur-arm64/contrib/4.4/. + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES", h.proxyCached) + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES.gz", h.proxyCached) + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/PACKAGES.rds", h.proxyCached) + // Source package downloads mux.HandleFunc("GET /src/contrib/{filename}", h.handleSourceDownload) mux.HandleFunc("GET /src/contrib/Archive/{name}/{filename}", h.handleSourceDownload) // Binary package downloads mux.HandleFunc("GET /bin/{platform}/contrib/{rversion}/{filename}", h.handleBinaryDownload) + mux.HandleFunc("GET /bin/{platform}/{arch}/contrib/{rversion}/{filename}", h.handleBinaryDownload) return rejectQueryChars(mux) } +// handleMeta serves the .rds files under src/contrib/Meta as cached metadata. +func (h *CRANHandler) handleMeta(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.PathValue("filename"), ".rds") { + http.NotFound(w, r) + return + } + h.proxyCached(w, r) +} + // handleSourceDownload serves a source package, fetching and caching from upstream. func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Request) { filename := r.PathValue("filename") @@ -99,6 +119,9 @@ func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Reques // handleBinaryDownload serves a binary package, fetching and caching from upstream. func (h *CRANHandler) handleBinaryDownload(w http.ResponseWriter, r *http.Request) { platform := r.PathValue("platform") + if arch := r.PathValue("arch"); arch != "" { + platform += "-" + arch + } rversion := r.PathValue("rversion") filename := r.PathValue("filename") diff --git a/internal/handler/cran_test.go b/internal/handler/cran_test.go index 6a9c151..6c156c5 100644 --- a/internal/handler/cran_test.go +++ b/internal/handler/cran_test.go @@ -1,8 +1,14 @@ package handler import ( + "io" "log/slog" + "net/http" + "net/http/httptest" + "strings" "testing" + + "github.com/git-pkgs/registries/fetch" ) func TestCRANParseSourceFilename(t *testing.T) { @@ -50,3 +56,68 @@ func TestCRANParseBinaryFilename(t *testing.T) { } } } + +func TestCRANHandler_MacOSTwoSegmentIndexesAndMeta(t *testing.T) { + for _, path := range []string{ + "/bin/macosx/big-sur-arm64/contrib/4.4/PACKAGES", + "/bin/macosx/big-sur-x86_64/contrib/4.4/PACKAGES.gz", + "/bin/macosx/sonoma-arm64/contrib/4.5/PACKAGES.rds", + "/src/contrib/Meta/archive.rds", + "/src/contrib/Meta/current.rds", + } { + t.Run(path, func(t *testing.T) { + assertUpstreamProxied(t, "index", path, + func(proxy *Proxy, upstreamURL string) http.Handler { + h := &CRANHandler{proxy: proxy, upstreamURL: upstreamURL, proxyURL: "http://localhost"} + return h.Routes() + }, + ) + }) + } +} + +func TestCRANHandler_MacOSTwoSegmentBinaryDownload(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("fetched binary")), + ContentType: "application/x-gzip", + } + + h := NewCRANHandler(proxy, "http://localhost") + h.upstreamURL = "https://cran.r-project.org" + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz") + if err != nil { + t.Fatalf("request failed: %v", err) + } + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + want := "https://cran.r-project.org/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz" + if fetcher.fetchedURL != want { + t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) + } +} + +func TestCRANHandler_MacOSTwoSegmentBinaryCacheIdentity(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackageWithPURL(t, db, store, "cran", "dplyr", "1.1.4_macosx-big-sur-arm64_4.4", "dplyr_1.1.4.tgz", "arm64 binary") + + h := NewCRANHandler(proxy, "http://localhost") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/bin/macosx/big-sur-arm64/contrib/4.4/dplyr_1.1.4.tgz") + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if string(body) != "arm64 binary" { + t.Errorf("body = %q, want %q", body, "arm64 binary") + } +} From cbbeed423b741d7eb64e51cdd79be82d5f5f862c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:56:52 +0100 Subject: [PATCH 110/198] Cache RPM packages under their full path RPM packages were cached under their bare filename. With upstream.rpm pointing at a mirror root that serves several distributions, Rocky and Alma both publish bash-5.1.8-9.el9.x86_64.rpm with different bytes, and whichever was fetched first was served for both, so dnf failed the checksum or signature check. Use the path under the upstream as the cache filename, as the APK and Arch handlers do. Packages cached before this change are no longer found and are fetched again once; retention still matches them by package name. --- docs/configuration.md | 2 +- internal/handler/rpm.go | 9 +++-- internal/handler/rpm_test.go | 50 +++++++++++++++++++++++++++ internal/server/retention_rpm_test.go | 4 +-- 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..9046e28 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1820,7 +1820,7 @@ curl -X DELETE http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1 curl -X DELETE 'http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre?filename=guava-32.1.3-jre.jar' ``` -The name is the package name the UI shows, for example `group:artifact` for Maven. The response lists the deleted files. The next request for a deleted file fetches it from the upstream again. The stored bytes are removed after the same delay as a replaced artifact, so downloads already in progress can finish. +The name is the package name the UI shows, for example `group:artifact` for Maven. An RPM package's `filename` is its path under `upstream.rpm` (`updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm`), since repositories under one upstream can ship different files under one name. The response lists the deleted files. The next request for a deleted file fetches it from the upstream again. The stored bytes are removed after the same delay as a replaced artifact, so downloads already in progress can finish. Pushed archives are not touched: a whole-version delete skips them, and naming one answers 409. Delete those through the [hosted API](#hosted-packages). diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 95163b9..49773a7 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -93,6 +93,10 @@ func (h *RPMHandler) Routes() http.Handler { // Path format varies by repo structure, e.g.: // - releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm // - updates/39/Everything/x86_64/Packages/n/nginx-1.24.0-2.fc39.x86_64.rpm +// +// The cache key is the full path, since repositories under one upstream +// (Rocky and Alma on one mirror, say) can ship different bytes under one +// filename. func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Request, path string) { // Parse the path to extract package info name, version, arch := h.parseRPMPath(path) @@ -102,14 +106,13 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques return } - filename := path[strings.LastIndex(path, "/")+1:] downloadURL := fmt.Sprintf("%s/%s", h.upstreamURL, path) h.proxy.Logger.Info("rpm package download", - "name", name, "version", version, "arch", arch, "filename", filename) + "name", name, "version", version, "arch", arch, "path", path) result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), "rpm", name, version, filename, downloadURL) + r.Context(), "rpm", name, version, path, downloadURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return diff --git a/internal/handler/rpm_test.go b/internal/handler/rpm_test.go index 3526dbd..87275b2 100644 --- a/internal/handler/rpm_test.go +++ b/internal/handler/rpm_test.go @@ -1,9 +1,14 @@ package handler import ( + "io" + "net/http" + "net/http/httptest" + "sync/atomic" "testing" "github.com/git-pkgs/purl" + "github.com/git-pkgs/registries/fetch" ) func TestRPMHandler_parseRPMPath(t *testing.T) { @@ -52,3 +57,48 @@ func TestRPMRetentionKey(t *testing.T) { t.Error("rpmRetentionKey(nil) accepted") } } + +// TestRPMHandler_PackageDownloadCachesPerRepository checks that two +// repositories under one upstream that ship the same filename with different +// bytes are cached apart, as on a mirror root serving several distributions. +func TestRPMHandler_PackageDownloadCachesPerRepository(t *testing.T) { + packages := map[string]string{ + "/rocky/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm": "rocky bash", + "/almalinux/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm": "alma bash", + } + + var available atomic.Bool + available.Store(true) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, ok := packages[r.URL.Path] + if !available.Load() || !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(data)) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + srv := httptest.NewServer(NewRPMHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes()) + defer srv.Close() + + for round := range 2 { + for path, want := range packages { + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || string(body) != want { + t.Errorf("round %d %s: %d %q, want 200 %q", round, path, resp.StatusCode, body, want) + } + } + // The second round must come from the cache. + available.Store(false) + } +} diff --git a/internal/server/retention_rpm_test.go b/internal/server/retention_rpm_test.go index 8c0536b..bc55614 100644 --- a/internal/server/retention_rpm_test.go +++ b/internal/server/retention_rpm_test.go @@ -20,8 +20,8 @@ import ( const ( rpmCurlPath = "/updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm" rpmNMPath = "/updates/40/Everything/x86_64/Packages/n/NetworkManager-1.46.2-1.fc40.x86_64.rpm" - rpmCurlFile = "curl-8.6.0-10.fc40.x86_64.rpm" - rpmNMFile = "NetworkManager-1.46.2-1.fc40.x86_64.rpm" + rpmCurlFile = "updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm" + rpmNMFile = "updates/40/Everything/x86_64/Packages/n/NetworkManager-1.46.2-1.fc40.x86_64.rpm" ) // rpmRetentionUpstream serves package files the way a Fedora mirror does, From b9f047f8dc7f21e1d3ff64ef91508dea4462ba32 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:56:56 +0100 Subject: [PATCH 111/198] Parse boolean environment variables strictly Every boolean PROXY_* variable was true only for exactly "true" or "1". Anything else, including "True", "TRUE" or a typo, silently meant false, so PROXY_SCANNING_ENABLED=True turned scanning off and validation still passed. Parse them with strconv.ParseBool like PROXY_GRADLE_BUILD_CACHE_READ_ONLY already does. An unrecognised value now keeps the current setting and is logged as a warning at startup. --- docs/configuration.md | 2 ++ internal/config/config.go | 55 ++++++++++++++++++---------------- internal/config/config_test.go | 34 +++++++++++++++++++++ 3 files changed, 66 insertions(+), 25 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..40fb55d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,6 +2,8 @@ The proxy can be configured via command line flags, environment variables, or a configuration file. Command line flags take precedence over environment variables, which take precedence over the configuration file. +Boolean environment variables accept `true`, `True`, `TRUE`, `t`, `T` or `1`, and `false`, `False`, `FALSE`, `f`, `F` or `0`. Any other value, such as `yes` or a typo, is ignored: the setting keeps its value from the configuration file or default, and the proxy logs a warning at startup. + ## Configuration File Create a YAML or JSON file and pass it with `-config`: diff --git a/internal/config/config.go b/internal/config/config.go index 3b5c6cb..5c74d3c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1390,11 +1390,20 @@ func setEnvString(dst *string, key string) { } } -// setEnvBool is setEnvString for boolean fields, parsed via envBool. -func setEnvBool(dst *bool, key string) { - if v := os.Getenv(key); v != "" { - *dst = envBool(v) +// setEnvBool is setEnvString for boolean fields. It accepts what +// strconv.ParseBool does; any other value leaves *dst unchanged and is +// reported via EnvWarnings. +func (c *Config) setEnvBool(dst *bool, key string) { + v := os.Getenv(key) + if v == "" { + return } + b, err := strconv.ParseBool(strings.TrimSpace(v)) + if err != nil { + c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping %t", key, v, *dst)) + return + } + *dst = b } // setGradleReadOnlyFromEnv parses strictly: an unrecognised value keeps @@ -1454,16 +1463,16 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Listen, "PROXY_LISTEN") setEnvString(&c.BaseURL, "PROXY_BASE_URL") setEnvString(&c.UIBaseURL, "PROXY_UI_URL") - setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR") - setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES") - setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS") + c.setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR") + c.setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES") + c.setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS") setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL") setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH") setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE") - setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") + c.setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL") setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL") - setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS") + c.setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS") setEnvString(&c.Storage.Retention.Default, "PROXY_STORAGE_RETENTION_DEFAULT") setEnvString(&c.Storage.Retention.SweepInterval, "PROXY_STORAGE_RETENTION_SWEEP_INTERVAL") setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER") @@ -1474,11 +1483,11 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT") setEnvString(&c.AccessLog.Path, "PROXY_ACCESS_LOG_PATH") setEnvStringSlice(&c.Upstream.AllowPrivateHosts, "PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS") - setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK") + c.setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK") setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM") - setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA") - setEnvBool(&c.Upstream.NPMChangeFeed, "PROXY_UPSTREAM_NPM_CHANGE_FEED") - setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE") + c.setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA") + c.setEnvBool(&c.Upstream.NPMChangeFeed, "PROXY_UPSTREAM_NPM_CHANGE_FEED") + c.setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE") setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO") setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD") setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM") @@ -1495,7 +1504,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.NuGetSearch, "PROXY_UPSTREAM_NUGET_SEARCH") setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER") setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY") - setEnvBool(&c.Upstream.ComposerChangeFeed, "PROXY_UPSTREAM_COMPOSER_CHANGE_FEED") + c.setEnvBool(&c.Upstream.ComposerChangeFeed, "PROXY_UPSTREAM_COMPOSER_CHANGE_FEED") setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN") setEnvString(&c.Upstream.Chef, "PROXY_UPSTREAM_CHEF") setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA") @@ -1509,20 +1518,20 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.HomebrewAPI, "PROXY_UPSTREAM_HOMEBREW_API") setEnvString(&c.Upstream.HomebrewArtifact, "PROXY_UPSTREAM_HOMEBREW_ARTIFACT") setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT") - setEnvBool(&c.Scanning.Enabled, "PROXY_SCANNING_ENABLED") - setEnvBool(&c.Scanning.FailOpen, "PROXY_SCANNING_FAIL_OPEN") + c.setEnvBool(&c.Scanning.Enabled, "PROXY_SCANNING_ENABLED") + c.setEnvBool(&c.Scanning.FailOpen, "PROXY_SCANNING_FAIL_OPEN") setEnvString(&c.Scanning.Timeout, "PROXY_SCANNING_TIMEOUT") setEnvString(&c.Scanning.SigningKey, "PROXY_SCANNING_SIGNING_KEY") setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL") - setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") - setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") - setEnvBool(&c.CacheDelete, "PROXY_CACHE_DELETE") - setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") + c.setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") + c.setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") + c.setEnvBool(&c.CacheDelete, "PROXY_CACHE_DELETE") + c.setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN") setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES") setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS") - setEnvBool(&c.Projects.Enabled, "PROXY_PROJECTS_ENABLED") + c.setEnvBool(&c.Projects.Enabled, "PROXY_PROJECTS_ENABLED") setEnvString(&c.Projects.Token, "PROXY_PROJECTS_TOKEN") setEnvString(&c.Projects.MaxAge, "PROXY_PROJECTS_MAX_AGE") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") @@ -2176,7 +2185,3 @@ func (a *AuthConfig) Header() (name, value string) { func expandEnv(s string) string { return os.Expand(s, os.Getenv) } - -func envBool(v string) bool { - return v == "true" || v == "1" -} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 91e9d84..ac6dfff 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2169,3 +2169,37 @@ func TestCooldownShortest(t *testing.T) { } } } + +func TestLoadBoolsFromEnvParseStrictly(t *testing.T) { + tests := []struct { + value string + start bool + want bool + warnings int + }{ + {"true", false, true, 0}, + {"True", false, true, 0}, + {" TRUE ", false, true, 0}, + {"1", false, true, 0}, + {"false", true, false, 0}, + {"FALSE", true, false, 0}, + {"0", true, false, 0}, + {"yes", true, true, 1}, + {"on", false, false, 1}, + {"ture", true, true, 1}, + } + for _, tt := range tests { + t.Run(tt.value, func(t *testing.T) { + t.Setenv("PROXY_SCANNING_ENABLED", tt.value) + cfg := Default() + cfg.Scanning.Enabled = tt.start + cfg.LoadFromEnv() + if cfg.Scanning.Enabled != tt.want { + t.Errorf("Scanning.Enabled = %v, want %v", cfg.Scanning.Enabled, tt.want) + } + if got := len(cfg.EnvWarnings()); got != tt.warnings { + t.Errorf("len(EnvWarnings()) = %d, want %d: %v", got, tt.warnings, cfg.EnvWarnings()) + } + }) + } +} From 0b399c248dfdb257bc9d55fd7e8f6c59c22f8729 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:57:22 +0100 Subject: [PATCH 112/198] Accept sha512 OCI blob digests and check HEAD digests The blob route and blobPathPattern only matched sha256 digests, so a sha512 blob got 404 although the fetch path can verify sha512. The image mirror rejected sha512 digests in index entries, configs and layers for the same reason. Accept sha512 in both, with exact digest lengths. A HEAD for an uncached manifest by digest passed on whatever digest the upstream named. GET checks the body against the requested digest; HEAD now answers DIGEST_INVALID when the upstream names a different digest of the same algorithm. --- internal/handler/container.go | 4 +- internal/handler/container_manifest_fetch.go | 13 ++++ internal/handler/container_mirror.go | 2 +- internal/handler/container_mirror_test.go | 34 ++++++++ internal/handler/container_test.go | 82 +++++++++++++++++--- 5 files changed, 123 insertions(+), 12 deletions(-) diff --git a/internal/handler/container.go b/internal/handler/container.go index d9360fb..40f5d0f 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -353,7 +353,7 @@ func (h *ContainerHandler) Routes() http.Handler { case path == "" || path == "/": // Version check: GET /v2/ h.handleVersionCheck(w, r) - case strings.HasSuffix(path, "/blobs/"+r.URL.Query().Get("digest")) || strings.Contains(path, "/blobs/sha256:"): + case strings.HasSuffix(path, "/blobs/"+r.URL.Query().Get("digest")) || strings.Contains(path, "/blobs/sha256:") || strings.Contains(path, "/blobs/sha512:"): // Blob download: GET /v2/{name}/blobs/{digest} h.handleBlobDownload(w, r, path) case manifestPathPattern.MatchString(path): @@ -692,7 +692,7 @@ func (h *ContainerHandler) containerError(w http.ResponseWriter, status int, cod } // blobPathPattern matches blob paths: {name}/blobs/{digest} -var blobPathPattern = regexp.MustCompile(`^(.+)/blobs/(sha256:[a-f0-9]+)$`) +var blobPathPattern = regexp.MustCompile(`^(.+)/blobs/(sha256:[a-f0-9]{64}|sha512:[a-f0-9]{128})$`) // parseBlobPath extracts repository name and digest from a blob path. func (h *ContainerHandler) parseBlobPath(path string) (name, digest string) { diff --git a/internal/handler/container_manifest_fetch.go b/internal/handler/container_manifest_fetch.go index 4b5ef15..7568671 100644 --- a/internal/handler/container_manifest_fetch.go +++ b/internal/handler/container_manifest_fetch.go @@ -210,6 +210,11 @@ func (h *ContainerHandler) headManifest(ctx context.Context, req manifestRequest digest := resp.Header.Get("Docker-Content-Digest") switch { case resp.StatusCode == http.StatusOK && manifestDigestReferencePattern.MatchString(digest): + if req.immutable() && digestMismatch(req.reference, digest) { + h.proxy.Logger.Error("upstream manifest HEAD names another digest", + "name", req.name, "reference", req.reference, "upstream", digest) + return manifestOutcome{digestInvalid: true} + } if !req.immutable() { if outcome, ok := h.tagFromCachedDigest(ctx, req, cached, fromFallback, resp, digest); ok { return outcome @@ -234,6 +239,14 @@ func (h *ContainerHandler) headManifest(ctx context.Context, req manifestRequest return manifestOutcome{relay: h.captureResponse(resp)} } +// digestMismatch reports whether two digests of the same algorithm differ. +// Digests of different algorithms can't be compared without the body. +func digestMismatch(requested, upstream string) bool { + reqAlg, reqHex, _ := strings.Cut(requested, ":") + upAlg, upHex, _ := strings.Cut(upstream, ":") + return reqAlg == upAlg && !strings.EqualFold(reqHex, upHex) +} + // tagFromCachedDigest answers a tag from the cache once a HEAD named the // digest it points to: the cached tag entry if the tag did not move, or the // manifest stored under the new digest. It reports false when neither is diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index a973afd..e17311c 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -36,7 +36,7 @@ var ( // Grammar from the OCI distribution spec. mirrorNamePattern = regexp.MustCompile(`^[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*$`) mirrorTagPattern = regexp.MustCompile(`^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$`) - mirrorDigestPattern = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`) + mirrorDigestPattern = regexp.MustCompile(`^(sha256:[a-f0-9]{64}|sha512:[a-f0-9]{128})$`) ) // dockerHubMirrorAliases are registry hosts that name Docker Hub in image diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index e6be5a5..f3a15ee 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -2,7 +2,9 @@ package handler import ( "context" + "crypto/sha512" "database/sql" + "encoding/hex" "encoding/json" "io" "net/http" @@ -610,3 +612,35 @@ func TestMirrorImageBlobFailureKeepsOtherFetches(t *testing.T) { t.Errorf("layer fetched alongside the failing blob: status %d, want it cached", response.Code) } } + +func TestMirrorImageAcceptsSHA512Blobs(t *testing.T) { + registry, image := newFakeImageRegistry(t) + const layer = "sha512 layer" + sum := sha512.Sum512([]byte(layer)) + layerDigest := "sha512:" + hex.EncodeToString(sum[:]) + body, err := json.Marshal(map[string]any{ + "schemaVersion": 2, "mediaType": ociManifestMediaType, + "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": image.singleLayer}, + "layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": layerDigest}}, + }) + if err != nil { + t.Fatal(err) + } + registry.mu.Lock() + registry.manifests["/v2/team/app/manifests/sha512"] = fakeManifest{contentType: ociManifestMediaType, body: string(body)} + registry.blobs["/v2/team/app/blobs/"+layerDigest] = layer + registry.mu.Unlock() + + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + if _, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{"sha512"}}, nil); err != nil { + t.Fatalf("MirrorImage: %v", err) + } + cached, err := proxy.GetCachedArtifact(t.Context(), "oci", "team/app", layerDigest, layerDigest) + if err != nil || cached == nil { + t.Fatalf("sha512 layer not cached: %v, %v", cached, err) + } + if cached.Reader != nil { + _ = cached.Reader.Close() + } +} diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index d546725..fd20e38 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -18,6 +18,11 @@ import ( "github.com/git-pkgs/registries/fetch" ) +var ( + testSHA256Digest = "sha256:" + strings.Repeat("ab", 32) + testSHA512Digest = "sha512:" + strings.Repeat("cd", 64) +) + func TestContainerHandler_parseBlobPath(t *testing.T) { h := &ContainerHandler{} @@ -27,19 +32,29 @@ func TestContainerHandler_parseBlobPath(t *testing.T) { wantDigest string }{ { - path: "library/nginx/blobs/sha256:abc123def456", + path: "library/nginx/blobs/" + testSHA256Digest, wantName: "library/nginx", - wantDigest: "sha256:abc123def456", + wantDigest: testSHA256Digest, }, { - path: "myorg/myrepo/blobs/sha256:0123456789abcdef", - wantName: "myorg/myrepo", - wantDigest: "sha256:0123456789abcdef", - }, - { - path: "deep/nested/repo/name/blobs/sha256:fedcba9876543210", + path: "deep/nested/repo/name/blobs/" + testSHA256Digest, wantName: "deep/nested/repo/name", - wantDigest: "sha256:fedcba9876543210", + wantDigest: testSHA256Digest, + }, + { + path: "myorg/myrepo/blobs/" + testSHA512Digest, + wantName: "myorg/myrepo", + wantDigest: testSHA512Digest, + }, + { + path: "library/nginx/blobs/sha256:abc123def456", + wantName: "", + wantDigest: "", + }, + { + path: "library/nginx/blobs/sha512:" + strings.Repeat("a", 64), + wantName: "", + wantDigest: "", }, { path: "invalid/path", @@ -1476,3 +1491,52 @@ func TestContainerHandler_ManifestHonoursSeparateAcceptHeaders(t *testing.T) { t.Errorf("offline body = %q, want the cached manifest", w.Body.String()) } } + +func TestContainerHandler_BlobSHA512(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + const blob = "sha512 layer" + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader(blob)), + ContentType: "application/octet-stream", + } + sum := sha512.Sum512([]byte(blob)) + digest := "sha512:" + hex.EncodeToString(sum[:]) + h := &ContainerHandler{proxy: proxy, registryURL: "https://registry.example.test"} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/library/app/blobs/"+digest, nil)) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if w.Body.String() != blob { + t.Errorf("body = %q, want %q", w.Body.String(), blob) + } + if got := w.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) + } +} + +func TestContainerHandler_HeadManifestByDigestChecksUpstreamDigest(t *testing.T) { + requested := sha256Digest([]byte("requested")) + other := sha256Digest([]byte("other")) + registry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", ociManifestMediaType) + w.Header().Set("Docker-Content-Digest", other) + w.Header().Set("Content-Length", "5") + })) + defer registry.Close() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = registry.Client() + h := &ContainerHandler{proxy: proxy, registryURL: registry.URL} + + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/library/app/manifests/"+requested, nil)) + + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d", w.Code, http.StatusBadGateway) + } + if got := w.Header().Get("Docker-Content-Digest"); got == other { + t.Errorf("Docker-Content-Digest = %q, the upstream's other digest", got) + } +} From 6398ed63f6f1a03a884ad9a0d300a6546d51025c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:57:30 +0100 Subject: [PATCH 113/198] Validate and normalise scanner ecosystem names scanning.scanners[].ecosystems was never checked, and the scanner group compared it exactly against the keys handlers pass. Natural names such as go, docker and rubygems passed validation but never matched the handler keys golang, oci and gem, so a block-mode scanner restricted to them silently never ran. Map common aliases to the handler keys, reject unknown names at startup, and normalise the names again when building the group. --- docs/configuration.md | 2 +- internal/config/config.go | 36 ++++++++++++++++++++++++++++++++++ internal/config/config_test.go | 25 +++++++++++++++++++++++ internal/scanner/group.go | 2 +- internal/scanner/group_test.go | 18 +++++++++++++++++ 5 files changed, 81 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 40fb55d..2ea49c6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1400,7 +1400,7 @@ scanning: | `scanning.scanners[].name` | - | Identifies this scanner in logs and metrics | | `scanning.scanners[].url` | - | Endpoint the proxy POSTs scan notifications to | | `scanning.scanners[].mode` | - | `block` (default) or `monitor` | -| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems | +| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems. Valid names: `alpine`, `arch`, `cargo`, `chef`, `composer`, `conan`, `conda`, `cran`, `deb`, `freebsd`, `gem`, `generic`, `golang`, `helm`, `hex`, `julia`, `maven`, `npm`, `nuget`, `oci`, `pub`, `pypi`, `rpm`, `swift`. The aliases `go`, `docker`, `rubygems`, `packagist`, `debian` and `apk` are accepted too; any other name stops the proxy at startup | | `scanning.scanners[].headers` | - | Extra HTTP headers sent with every scan request (e.g. for authenticating to the scanner service). Values support `${VAR_NAME}` expansion | ### How caching defers to a scan verdict diff --git a/internal/config/config.go b/internal/config/config.go index 5c74d3c..7db98c5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -70,6 +70,7 @@ import ( "path" "path/filepath" "regexp" + "slices" "sort" "strconv" "strings" @@ -436,9 +437,44 @@ func (s *ScannerConfig) Validate() error { default: return fmt.Errorf("invalid mode %q (must be block or monitor)", s.Mode) } + for i, eco := range s.Ecosystems { + key := NormalizeScannerEcosystem(eco) + if !slices.Contains(scannerEcosystems, key) { + return fmt.Errorf("unknown ecosystem %q (valid: %s)", eco, strings.Join(scannerEcosystems, ", ")) + } + s.Ecosystems[i] = key + } return nil } +// scannerEcosystems are the ecosystem keys handlers pass to scanners: the +// retention keys plus the handlers retention does not cover. +var scannerEcosystems = func() []string { + keys := append(slices.Clone(retention.KnownKeys), "arch", "chef") + slices.Sort(keys) + return keys +}() + +// scannerEcosystemAliases maps common ecosystem names to handler keys. +var scannerEcosystemAliases = map[string]string{ + "apk": "alpine", + "debian": "deb", + "docker": "oci", + "go": "golang", + "packagist": "composer", + "rubygems": "gem", +} + +// NormalizeScannerEcosystem returns the handler key for a +// scanning.scanners[].ecosystems entry, resolving case and common aliases. +func NormalizeScannerEcosystem(eco string) string { + eco = strings.ToLower(strings.TrimSpace(eco)) + if key, ok := scannerEcosystemAliases[eco]; ok { + return key + } + return eco +} + // StorageConfig configures artifact storage. type StorageConfig struct { // URL is the storage backend URL. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ac6dfff..e91a4f2 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -2203,3 +2204,27 @@ func TestLoadBoolsFromEnvParseStrictly(t *testing.T) { }) } } + +func TestScannerEcosystems(t *testing.T) { + cfg := ScanningConfig{ + Enabled: true, + SigningKey: "s3cret", + Scanners: []ScannerConfig{{ + Name: "clamav", URL: "http://scanner.invalid/scan", + Ecosystems: []string{"go", "Docker", "rubygems", "npm", "arch"}, + }}, + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() error: %v", err) + } + want := []string{"golang", "oci", "gem", "npm", "arch"} + if got := cfg.Scanners[0].Ecosystems; !slices.Equal(got, want) { + t.Errorf("Ecosystems = %v, want %v", got, want) + } + + cfg.Scanners[0].Ecosystems = []string{"npm", "pypy"} + err := cfg.Validate() + if err == nil || !strings.Contains(err.Error(), "pypy") { + t.Errorf("Validate() error = %v, want unknown ecosystem pypy", err) + } +} diff --git a/internal/scanner/group.go b/internal/scanner/group.go index cf5930f..3066219 100644 --- a/internal/scanner/group.go +++ b/internal/scanner/group.go @@ -67,7 +67,7 @@ func NewGroup(cfg config.ScanningConfig, logger *slog.Logger) (*Group, error) { if len(sc.Ecosystems) > 0 { ecosystems = make(map[string]struct{}, len(sc.Ecosystems)) for _, eco := range sc.Ecosystems { - ecosystems[eco] = struct{}{} + ecosystems[config.NormalizeScannerEcosystem(eco)] = struct{}{} } } diff --git a/internal/scanner/group_test.go b/internal/scanner/group_test.go index cbbaa01..d80166b 100644 --- a/internal/scanner/group_test.go +++ b/internal/scanner/group_test.go @@ -294,3 +294,21 @@ func TestGroupBlocks(t *testing.T) { t.Error("nil group blocks") } } + +func TestGroup_NormalisesEcosystemAliases(t *testing.T) { + g, err := NewGroup(config.ScanningConfig{ + Enabled: true, + SigningKey: "test-signing-key", + Scanners: []config.ScannerConfig{ + {Name: "clamav", URL: "http://clamav.invalid", Ecosystems: []string{"go", "docker", "rubygems"}}, + }, + }, discardLogger()) + if err != nil { + t.Fatalf("NewGroup() error: %v", err) + } + for _, ecosystem := range []string{"golang", "oci", "gem"} { + if !g.Blocks(ecosystem) { + t.Errorf("Blocks(%s) = false, want true", ecosystem) + } + } +} From 2a6c8091032ebb3eb2f30a209884d4a673416990 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:58:49 +0100 Subject: [PATCH 114/198] Reject negative and out-of-range sizes and TTLs ParseSize accepted "-5GB", "NaNGB", "InfGB" and "1e30GB", the last two clamping to the largest int64. A negative or NaN storage.max_size passed validation, and eviction treats a limit of zero or less as unlimited. Reject negative, NaN, infinite and overflowing sizes in ParseSize, and reject negative metadata_ttl and storage.direct_serve_ttl at startup. --- docs/configuration.md | 2 +- internal/config/config.go | 19 ++++++++++++++----- internal/config/config_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 2ea49c6..8baac0f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -67,7 +67,7 @@ storage: |--------|-------------|------|-------------| | `storage.url` | `PROXY_STORAGE_URL` | `-storage-url` | Storage URL (file:// or s3://) | | `storage.path` | `PROXY_STORAGE_PATH` | `-storage-path` | Local path (deprecated, use url) | -| `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB") | +| `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB"). Negative or out-of-range sizes are rejected at startup | | `storage.cache_artifacts` | `PROXY_STORAGE_CACHE_ARTIFACTS` | - | Store fetched artifacts (default: true); `false` streams them from upstream | | `storage.retention.default` | `PROXY_STORAGE_RETENTION_DEFAULT` | - | Evict artifacts not served for this long (e.g. "30d"); see [Retention](#retention) | | `storage.retention.sweep_interval` | `PROXY_STORAGE_RETENTION_SWEEP_INTERVAL` | - | How often expired artifacts are looked for (default: "10m", at least "1m") | diff --git a/internal/config/config.go b/internal/config/config.go index 7db98c5..d9d9b29 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -64,6 +64,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "math" "net" "net/url" "os" @@ -1655,8 +1656,8 @@ func (c *Config) Validate() error { // Validate direct serve TTL if specified if c.Storage.DirectServeTTL != "" { - if _, err := time.ParseDuration(c.Storage.DirectServeTTL); err != nil { - return fmt.Errorf("invalid storage.direct_serve_ttl %q: %w", c.Storage.DirectServeTTL, err) + if _, err := parseNonNegativeDuration("storage.direct_serve_ttl", c.Storage.DirectServeTTL); err != nil { + return err } } @@ -1673,8 +1674,8 @@ func (c *Config) Validate() error { // Validate metadata TTL if specified if c.MetadataTTL != "" && c.MetadataTTL != "0" { - if _, err := time.ParseDuration(c.MetadataTTL); err != nil { - return fmt.Errorf("invalid metadata_ttl %q: %w", c.MetadataTTL, err) + if _, err := parseNonNegativeDuration("metadata_ttl", c.MetadataTTL); err != nil { + return err } } @@ -2172,7 +2173,12 @@ func ParseSize(s string) (int64, error) { if err != nil { return 0, fmt.Errorf("invalid number %q", numStr) } - return int64(num * float64(s2.mult)), nil + // The comparisons also reject NaN. + size := num * float64(s2.mult) + if !(size >= 0 && size < math.MaxInt64) { + return 0, fmt.Errorf("invalid size %q: must be between 0 and %d bytes", s, int64(math.MaxInt64)) + } + return int64(size), nil } } @@ -2181,6 +2187,9 @@ func ParseSize(s string) (int64, error) { if err != nil { return 0, fmt.Errorf("invalid size %q", s) } + if num < 0 { + return 0, fmt.Errorf("invalid size %q: must not be negative", s) + } return num, nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e91a4f2..2139b0e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2228,3 +2228,30 @@ func TestScannerEcosystems(t *testing.T) { t.Errorf("Validate() error = %v, want unknown ecosystem pypy", err) } } + +func TestParseSizeRejectsOutOfRange(t *testing.T) { + for _, s := range []string{"-5GB", "-1", "-0.5K", "NaNGB", "InfGB", "+InfB", "1e30GB", "9999999999TB"} { + if got, err := ParseSize(s); err == nil { + t.Errorf("ParseSize(%q) = %d, want error", s, got) + } + } + cfg := Default() + cfg.Storage.MaxSize = "-5GB" + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted storage.max_size -5GB") + } +} + +func TestValidateRejectsNegativeTTLs(t *testing.T) { + cfg := Default() + cfg.MetadataTTL = "-5m" + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted metadata_ttl -5m") + } + + cfg = Default() + cfg.Storage.DirectServeTTL = "-5m" + if err := cfg.Validate(); err == nil { + t.Error("Validate() accepted storage.direct_serve_ttl -5m") + } +} From d073fdcd52fcf0e988f78941650e37578d8791a4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:00:14 +0100 Subject: [PATCH 115/198] Close npm, PyPI and pub cooldown gaps in metadata and downloads npm: a tarball request for a version the cached packument did not list yet (cached within metadata_ttl before the release) found no publish time and was let through cooldown, and fetched without integrity. When cooldown applies and the cached packument lacks the version, fetch it again and decide on what upstream lists now; if it can't be fetched, fail as when a publish time can't be read. npm: cooldown only moved dist-tags.latest. Other tags kept naming versions in cooldown, while the denylist removes such tags. Cooldown now removes them too, and drops latest when no version is left, in both the in-memory and streamed rewrites. PyPI: the simple JSON rewrite removed filtered files but left their versions in the PEP 700 versions list. Remove them there too. pub: latest was never rewritten, so its archive_url pointed at pub.dev, it kept naming a version in cooldown when no version was left, and the replacement was the last list entry, which may be a prerelease. Point latest's archive_url at the proxy, replace a held back latest with the newest stable version left (or the newest prerelease), and drop it when nothing is left. --- docs/configuration.md | 6 ++- internal/handler/npm.go | 73 +++++++++++++++++++------- internal/handler/npm_attestations.go | 2 +- internal/handler/npm_tarball_test.go | 57 ++++++++++++++++++++ internal/handler/pub.go | 50 ++++++++++++------ internal/handler/pub_test.go | 55 +++++++++++++++++++ internal/handler/pypi.go | 13 +++++ internal/handler/pypi_test.go | 26 +++++++++ internal/metadata/npm/metadata.go | 22 ++++++-- internal/metadata/npm/metadata_test.go | 36 +++++++++++++ 10 files changed, 297 insertions(+), 43 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..93de410 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1240,7 +1240,9 @@ RubyGems, Hex and PyPI keep their publish times outside the metadata clients rea For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, and the filtered result is kept until the first version it held back leaves cooldown (see [rewritten metadata cache](#rewritten-metadata-cache)); a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed. -With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead, and `npm dist-tag ls` shows the same tags. +With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead; other tags that name a version in cooldown are removed, as for the denylist, and `npm dist-tag ls` shows the same tags. A tarball request for a version the cached packument doesn't list yet fetches the packument again, within `metadata_ttl` or not, and is decided on what the upstream lists now; if the packument can't be fetched, the download is not let through, as when a publish time can't be read. + +In pub.dev package metadata, when `latest` is in cooldown it names the newest stable version out of cooldown instead, or the newest prerelease when no stable version is left, and it is left out when every version is in cooldown. Conda cooldown covers `repodata.json`, `current_repodata.json`, `repodata_from_packages.json` and sharded repodata (CEP 16). Other forms of the repodata, such as `repodata.json.zst`, `repodata.json.bz2` and `repodata.jlap` patches, would list versions still in cooldown, so with cooldown on they are answered 404 and conda and mamba fall back to `repodata.json`. With `cache_metadata`, `repodata.json` is read through the metadata cache as it is without cooldown: within `metadata_ttl` the stored copy is filtered as it is read from storage, after it the upstream is asked with its ETag, and while the upstream is down the stale copy is filtered. Once a document has been filtered, the proxy remembers the ETag of the result until the first entry it left out leaves cooldown, so a client revalidating with `If-None-Match` gets `304 Not Modified` without the document being filtered or sent again. The first response for each document has no ETag, as it is sent while it is filtered. Without `cache_metadata`, `repodata.json` is fetched and filtered on every request. Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`. @@ -1296,7 +1298,7 @@ Names and versions are compared the way each ecosystem treats them: channel only. Denied versions are removed from npm packuments, Cargo sparse indexes, and PyPI -simple HTML/JSON and release metadata, including responses served from cached +simple HTML/JSON (files and the JSON `versions` list) and release metadata, including responses served from cached metadata during an upstream outage. No publication timestamp or enabled cooldown is required. npm tags pointing at denied versions are removed; `latest` is moved to the highest remaining stable version when available. Other tags are not diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 50757a3..0e9f32a 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -344,7 +344,7 @@ func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Reque fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) var rewritten rewrittenMetadata - err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, func(entry *database.MetadataCacheEntry) (err error) { + err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, false, func(entry *database.MetadataCacheEntry) (err error) { rewritten, err = h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { var scan *npm.Scan err := readStored(open, func(in io.Reader) (err error) { @@ -413,7 +413,18 @@ var errInMemory = errors.New("npm packument needs the in-memory path") // fetching it into storage first when needed. It returns errInMemory when // metadata isn't cached in storage, and ErrUpstreamNotFound and upstream // errors as they come. -func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string) (*database.MetadataCacheEntry, error) { +// With refresh, a stored copy within its TTL is fetched again rather than +// served. +func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool) (*database.MetadataCacheEntry, error) { + if refresh && h.proxy.DB != nil { + if row, _ := h.proxy.DB.GetMetadataCache("npm", cacheKey); usableStoredMetadata(row) { + entry, err := h.proxy.restoreStoredMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), row) + if errors.Is(err, errMetadataNotStored) { + return nil, errInMemory + } + return entry, err + } + } entry, err := h.proxy.storedMetadataDocument(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator()) if errors.Is(err, errMetadataNotStored) { return nil, errInMemory @@ -426,8 +437,8 @@ func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, // or no longer has the digest its row records, the packument is fetched // again and read once more. fetchErr is what storedPackument returned when the packument // couldn't be fetched, and err is what read returned. -func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { - entry, fetchErr := h.storedPackument(ctx, packumentURL, accept, cacheKey) +func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { + entry, fetchErr := h.storedPackument(ctx, packumentURL, accept, cacheKey, refresh) if fetchErr != nil { return nil, fetchErr } @@ -475,7 +486,7 @@ func (h *NPMHandler) streamedVersionMetadata(ctx context.Context, packageName, s fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) var m rewrittenMetadata - err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) (err error) { + err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, false, func(entry *database.MetadataCacheEntry) (err error) { m, err = h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { var scan *npm.Scan err := readStored(open, func(in io.Reader) (err error) { @@ -776,7 +787,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { } registryURL, cacheKey := h.sourceFor(packageName) metadata := sync.OnceValues(func() (npmVersionData, error) { - return h.versionData(r.Context(), registryURL+"/"+url.PathEscape(packageName), cacheKey, version) + return h.versionData(r.Context(), packageName, registryURL+"/"+url.PathEscape(packageName), cacheKey, version) }) inCooldown, err := h.versionInCooldown(packageName, version, metadata) if err != nil { @@ -883,10 +894,27 @@ type npmVersionData struct { // versionData reads what a tarball download needs about version from the // packument: from storage in one pass when metadata is cached there, and // otherwise from the packument read into memory. -func (h *NPMHandler) versionData(ctx context.Context, packumentURL, cacheKey, version string) (npmVersionData, error) { +// +// A packument cached within its TTL can predate the version. Under cooldown a +// missing entry would let the version through with no publish time, so the +// packument is fetched again and the version decided on what upstream lists +// now, as the Composer handler does. +func (h *NPMHandler) versionData(ctx context.Context, packageName, packumentURL, cacheKey, version string) (npmVersionData, error) { + data, fromCache, err := h.readVersionData(ctx, packumentURL, cacheKey, version, false) + if err == nil && data.entry == nil && fromCache && h.proxy.cooldownApplies("npm", packageName) { + h.proxy.Logger.Info("cooldown: cached npm metadata lacks version, fetching it again", + "package", packageName, "version", version) + data, _, err = h.readVersionData(ctx, packumentURL, cacheKey, version, true) + } + return data, err +} + +// readVersionData implements versionData. fromCache reports whether the +// packument may have been a cached copy; refresh fetches it from upstream. +func (h *NPMHandler) readVersionData(ctx context.Context, packumentURL, cacheKey, version string, refresh bool) (data npmVersionData, fromCache bool, err error) { accept := h.packumentAccept() var found npm.VersionEntry - err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) error { + err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, refresh, func(entry *database.MetadataCacheEntry) error { return readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { found, err = npm.LookupVersion(in, version) return err @@ -894,23 +922,32 @@ func (h *NPMHandler) versionData(ctx context.Context, packumentURL, cacheKey, ve }) switch { case fetchErr == nil && err == nil && (!found.Abbreviated || h.packumentValidator() == nil): - return npmVersionData{entry: found.Entry, published: found.Published}, nil + return npmVersionData{entry: found.Entry, published: found.Published}, !refresh, nil case fetchErr == nil && err != nil: h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) case fetchErr != nil && !errors.Is(fetchErr, errInMemory): - return npmVersionData{}, fetchErr + return npmVersionData{}, false, fetchErr } - body, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) - if err != nil { - return npmVersionData{}, err + var res metadataResult + if refresh { + res = h.proxy.refreshMetadata(ctx, "npm", cacheKey, packumentURL, accept, "", h.packumentValidator()) + } else { + res = h.proxy.fetchMetadataDocument(ctx, "npm", cacheKey, packumentURL, accept, h.packumentValidator()) } - entry, _ := jsonscan.Lookup(body, "versions", version) - published, _, err := jsonscan.LookupString(body, "time", version) - if err != nil { - return npmVersionData{}, fmt.Errorf("parsing npm metadata: %w", err) + if res.err != nil { + return npmVersionData{}, false, res.err } - return npmVersionData{entry: entry, published: published}, nil + // The cached copy is what lacked the version, so it can't decide it. + if refresh && res.stale { + return npmVersionData{}, false, errors.New("npm metadata could not be fetched again") + } + entry, _ := jsonscan.Lookup(res.body, "versions", version) + published, _, err := jsonscan.LookupString(res.body, "time", version) + if err != nil { + return npmVersionData{}, false, fmt.Errorf("parsing npm metadata: %w", err) + } + return npmVersionData{entry: entry, published: published}, res.fromCache, nil } func validateTarballURL(raw, registryURL string) (string, error) { diff --git a/internal/handler/npm_attestations.go b/internal/handler/npm_attestations.go index 02c5751..cc84003 100644 --- a/internal/handler/npm_attestations.go +++ b/internal/handler/npm_attestations.go @@ -41,7 +41,7 @@ func (h *NPMHandler) handleAttestations(w http.ResponseWriter, r *http.Request) registryURL, cacheKey := h.sourceFor(packageName) metadata := sync.OnceValues(func() (npmVersionData, error) { - return h.versionData(r.Context(), registryURL+"/"+url.PathEscape(packageName), cacheKey, version) + return h.versionData(r.Context(), packageName, registryURL+"/"+url.PathEscape(packageName), cacheKey, version) }) inCooldown, err := h.versionInCooldown(packageName, version, metadata) diff --git a/internal/handler/npm_tarball_test.go b/internal/handler/npm_tarball_test.go index fe1508d..d844dd5 100644 --- a/internal/handler/npm_tarball_test.go +++ b/internal/handler/npm_tarball_test.go @@ -201,3 +201,60 @@ func TestNPMTarballRejectsUnsafeMetadataURL(t *testing.T) { }) } } + +// TestNPMTarballCooldownRefetchesStalePackument checks that a tarball for a +// version the cached packument does not list yet is decided on the packument +// upstream serves now, not let through for want of a publish time. +func TestNPMTarballCooldownRefetchesStalePackument(t *testing.T) { + for _, tc := range []struct { + name string + published time.Time + down bool + want int + }{ + {"recent", time.Now().Add(-time.Hour), false, http.StatusNotFound}, + // Without upstream the version can't be checked, so it isn't served. + {"upstream down", time.Now().Add(-time.Hour), true, http.StatusBadGateway}, + // The refetched entry's integrity is checked, so a tarball that + // doesn't match it is refused rather than served unverified. + {"old", time.Now().Add(-30 * 24 * time.Hour), false, http.StatusBadGateway}, + } { + t.Run(tc.name, func(t *testing.T) { + var released atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + old := time.Now().Add(-60 * 24 * time.Hour).Format(time.RFC3339) + if released.Load() && tc.down { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + if !released.Load() { + _, _ = fmt.Fprintf(w, `{"name":"widget","time":{"1.0.0":%q},"versions":{"1.0.0":{"dist":{}}}}`, old) + return + } + _, _ = fmt.Fprintf(w, `{"name":"widget","time":{"1.0.0":%q,"2.0.0":%q},"versions":{"1.0.0":{"dist":{}},"2.0.0":{"dist":{"integrity":%q}}}}`, + old, tc.published.Format(time.RFC3339), sha512SRI("package")) + })) + t.Cleanup(upstream.Close) + p, _, _, fetcher := setupTestProxy(t) + p.CacheMetadata = true + p.MetadataTTL = time.Hour + p.Cooldown = &cooldown.Config{Default: "7d"} + p.HTTPClient = upstream.Client() + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("tampered"))} + h := NewNPMHandler(p, "http://proxy.test", upstream.URL) + + metadata := httptest.NewRecorder() + h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) + if metadata.Code != http.StatusOK { + t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) + } + released.Store(true) + + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) + if response.Code != tc.want { + t.Fatalf("status = %d, want %d: %s", response.Code, tc.want, response.Body.String()) + } + }) + } +} diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 31e00d8..5a6d426 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/Masterminds/semver/v3" "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/retention" ) @@ -293,32 +294,47 @@ func (h *PubHandler) shouldFilterVersion(packagePURL, name, version string, vmap return false } -// updateLatestVersion updates the latest field if the current latest version -// was removed by cooldown filtering. +// updateLatestVersion points latest at the kept entry for its version, so +// its archive_url is the proxy's too. When cooldown removed that version, +// latest becomes the newest stable version left, or the newest version left +// if all are prereleases, and is dropped when none is left. func (h *PubHandler) updateLatestVersion(metadata map[string]any, filtered []any) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - return - } - latest, ok := metadata["latest"].(map[string]any) if !ok { return } + latestVer, _ := latest["version"].(string) - latestVer, ok := latest["version"].(string) - if !ok { - return - } - + var newest, newestStable *semver.Version + var newestEntry, newestStableEntry any for _, vdata := range filtered { - if vmap, ok := vdata.(map[string]any); ok { - if vmap["version"] == latestVer { - return - } + vmap, ok := vdata.(map[string]any) + if !ok { + continue + } + version, _ := vmap["version"].(string) + if version == latestVer { + latest["archive_url"] = vmap["archive_url"] + return + } + parsed, err := semver.NewVersion(version) + if err != nil { + continue + } + if newest == nil || parsed.GreaterThan(newest) { + newest, newestEntry = parsed, vdata + } + if parsed.Prerelease() == "" && (newestStable == nil || parsed.GreaterThan(newestStable)) { + newestStable, newestStableEntry = parsed, vdata } } - if len(filtered) > 0 { - metadata["latest"] = filtered[len(filtered)-1] + switch { + case newestStableEntry != nil: + metadata["latest"] = newestStableEntry + case newestEntry != nil: + metadata["latest"] = newestEntry + default: + delete(metadata, "latest") } } diff --git a/internal/handler/pub_test.go b/internal/handler/pub_test.go index 29526e0..5311e25 100644 --- a/internal/handler/pub_test.go +++ b/internal/handler/pub_test.go @@ -206,3 +206,58 @@ func TestPubDownloadServesCachedArchive(t *testing.T) { t.Errorf("body = %q, want %q", got, "cached archive") } } + +// TestPubRewriteMetadataLatest checks that latest points at the proxy and +// never names a version cooldown holds back, and that a held back latest is +// replaced with the newest stable version left. +func TestPubRewriteMetadataLatest(t *testing.T) { + old := time.Now().Add(-10 * 24 * time.Hour).Format(time.RFC3339) + recent := time.Now().Add(-time.Hour).Format(time.RFC3339) + entry := func(version, published string) string { + return `{"version":"` + version + `","published":"` + published + `","archive_url":"https://pub.dev/packages/demo/versions/` + version + `.tar.gz"}` + } + for _, tc := range []struct { + name, latest, versions string + cooldown bool + want, wantURL string + }{ + {"kept", entry("1.0.0", old), entry("1.0.0", old), false, "1.0.0", + "http://localhost:8080/pub/packages/demo/versions/1.0.0.tar.gz"}, + {"stable over prerelease", entry("2.0.0", recent), + entry("1.0.0", old) + "," + entry("1.1.0", old) + "," + entry("2.0.0-dev.1", old) + "," + entry("2.0.0", recent), true, "1.1.0", + "http://localhost:8080/pub/packages/demo/versions/1.1.0.tar.gz"}, + {"only prereleases", entry("2.0.0", recent), entry("2.0.0-dev.1", old) + "," + entry("2.0.0", recent), true, "2.0.0-dev.1", + "http://localhost:8080/pub/packages/demo/versions/2.0.0-dev.1.tar.gz"}, + {"all held back", entry("2.0.0", recent), entry("2.0.0", recent), true, "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + proxy := &Proxy{Logger: slog.Default()} + if tc.cooldown { + proxy.Cooldown = &cooldown.Config{Default: "3d"} + } + h := &PubHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + output, err := h.rewriteMetadata("demo", []byte(`{"name":"demo","latest":`+tc.latest+`,"versions":[`+tc.versions+`]}`)) + if err != nil { + t.Fatal(err) + } + var result struct { + Latest *struct { + Version string `json:"version"` + ArchiveURL string `json:"archive_url"` + } `json:"latest"` + } + if err := json.Unmarshal(output, &result); err != nil { + t.Fatal(err) + } + if tc.want == "" { + if result.Latest != nil { + t.Errorf("latest = %+v, want none", *result.Latest) + } + return + } + if result.Latest == nil || result.Latest.Version != tc.want || result.Latest.ArchiveURL != tc.wantURL { + t.Errorf("latest = %+v, want %s at %s", result.Latest, tc.want, tc.wantURL) + } + }) + } +} diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index ebb3082..ac4315f 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -467,6 +467,19 @@ func (h *PyPIHandler) rewriteSimpleJSON(body []byte, filteredVersions map[string } metadata["files"] = rewrittenFiles + + // The versions list (PEP 700) loses the filtered versions too, whose + // files were all left out above. + if versions, ok := metadata["versions"].([]any); ok && len(filteredVersions) > 0 { + keptVersions := make([]any, 0, len(versions)) + for _, v := range versions { + if version, ok := v.(string); ok && filteredVersions[version] { + continue + } + keptVersions = append(keptVersions, v) + } + metadata["versions"] = keptVersions + } return json.Marshal(metadata) } diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index dd59974..f38492d 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -276,6 +276,32 @@ func TestPyPISimpleJSONCooldown(t *testing.T) { } } +// TestPyPISimpleJSONFiltersVersions checks that the PEP 700 versions list +// loses a version whose files were all filtered out, and keeps one that +// still has files. +func TestPyPISimpleJSONFiltersVersions(t *testing.T) { + h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}, proxyURL: "http://proxy.test"} + body := `{"meta":{"api-version":"1.1"},"name":"ruff","versions":["1.0.0","2.0.0","3.0.0"],"files":[` + + `{"filename":"ruff-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/ab/ruff-1.0.0.tar.gz"},` + + `{"filename":"ruff-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/cd/ruff-2.0.0.tar.gz"},` + + `{"filename":"ruff-2.0.0-py3-none-any.whl","url":"https://files.pythonhosted.org/packages/ef/ruff-2.0.0-py3-none-any.whl"}` + + `]}` + out, err := h.rewriteSimpleJSON([]byte(body), map[string]bool{"2.0.0": true, "3.0.0": true}) + if err != nil { + t.Fatal(err) + } + var result struct { + Versions []string `json:"versions"` + } + if err := json.Unmarshal(out, &result); err != nil { + t.Fatal(err) + } + // 3.0.0 has no files to filter, but it is still held back. + if strings.Join(result.Versions, ",") != "1.0.0" { + t.Errorf("versions = %v, want [1.0.0]", result.Versions) + } +} + func TestPyPIParseFilename(t *testing.T) { h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}} diff --git a/internal/metadata/npm/metadata.go b/internal/metadata/npm/metadata.go index 3a434c5..7fc4a40 100644 --- a/internal/metadata/npm/metadata.go +++ b/internal/metadata/npm/metadata.go @@ -402,7 +402,7 @@ func (h *transformer) tarballsRewritten(count int) { } // applyCooldownFiltering removes versions that are too recently published, -// and updates dist-tags.latest if the current latest was filtered out. +// and updates the dist-tags that named them. func (h *transformer) applyCooldownFiltering(metadata map[string]any, versions map[string]any, packageName string) { if h.Cooldown == nil || !h.Cooldown.Enabled() { return @@ -415,6 +415,7 @@ func (h *transformer) applyCooldownFiltering(metadata map[string]any, versions m packagePURL := packageurl.MakeString("npm", packageName, "") + withheld := map[string]bool{} for version := range versions { publishedStr, ok := timeMap[version].(string) if !ok { @@ -430,20 +431,29 @@ func (h *transformer) applyCooldownFiltering(metadata map[string]any, versions m } delete(versions, version) delete(timeMap, version) + withheld[version] = true } } - h.updateDistTagsLatest(metadata, versions, timeMap) + h.updateDistTags(metadata, versions, timeMap, withheld) } -// updateDistTagsLatest updates the dist-tags.latest field if the current latest -// version was removed by cooldown filtering. -func (h *transformer) updateDistTagsLatest(metadata, versions, timeMap map[string]any) { +// updateDistTags moves dist-tags.latest to the newest remaining version if +// the current latest was removed by cooldown filtering, and, as the denylist +// does, drops other tags that name a withheld version. +func (h *transformer) updateDistTags(metadata, versions, timeMap map[string]any, withheld map[string]bool) { distTags, ok := metadata["dist-tags"].(map[string]any) if !ok { return } + for tag, value := range distTags { + version, _ := value.(string) + if tag != "latest" && withheld[version] { + delete(distTags, tag) + } + } + latest, ok := distTags["latest"].(string) if !ok { return @@ -455,6 +465,8 @@ func (h *transformer) updateDistTagsLatest(metadata, versions, timeMap map[strin if newLatest := h.findNewestVersion(versions, timeMap); newLatest != "" { distTags["latest"] = newLatest + } else if withheld[latest] { + delete(distTags, "latest") } } diff --git a/internal/metadata/npm/metadata_test.go b/internal/metadata/npm/metadata_test.go index 28e9669..2748ed3 100644 --- a/internal/metadata/npm/metadata_test.go +++ b/internal/metadata/npm/metadata_test.go @@ -194,3 +194,39 @@ func TestNPMRewriteAttestationsURL(t *testing.T) { } } } + +// TestNPMRewriteCooldownDropsTags checks that cooldown, like the denylist, +// drops a tag other than latest that names a version it withholds, in both +// the in-memory and the streamed rewrite. +func TestNPMRewriteCooldownDropsTags(t *testing.T) { + now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) + ts := func(age time.Duration) string { return now.Add(-age).UTC().Format(time.RFC3339) } + options := Options{ProxyURL: "http://proxy", EvaluatedAt: now, Cooldown: &cooldown.Config{Default: "3d"}} + input := `{"name":"demo",` + + `"dist-tags":{"latest":"3.0.0","next":"3.0.0","legacy":"1.0.0"},` + + `"versions":{"1.0.0":{},"2.0.0":{},"3.0.0":{}},` + + `"time":{"1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}}` + + rewrite := map[string]func() ([]byte, []string, error){ + "memory": func() ([]byte, []string, error) { return Rewrite("demo", []byte(input), options) }, + "stream": func() ([]byte, []string, error) { return rewriteStream("demo", []byte(input), options) }, + } + for name, run := range rewrite { + t.Run(name, func(t *testing.T) { + out, _, err := run() + if err != nil { + t.Fatal(err) + } + var doc struct { + DistTags map[string]string `json:"dist-tags"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, out) + } + want := map[string]string{"latest": "2.0.0", "legacy": "1.0.0"} + if fmt.Sprint(doc.DistTags) != fmt.Sprint(want) { + t.Errorf("dist-tags = %v, want %v", doc.DistTags, want) + } + }) + } +} From fcdf8ebc3adb02ad6d3efc0187c79429846f1c2a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:00:26 +0100 Subject: [PATCH 116/198] Serve metadata with the validators of the bytes sent Cached metadata responses took their ETag, Last-Modified and stale flag from the cache row as it was after the fetch, not from the document actually sent. When a refetch got a new document but could not store it, the client got the new body with the old ETag and a Warning header, and a client still holding the old document got a 304 for it. With metadata_ttl 0 the stale fallback had no Warning at all, since staleness was inferred from the row's age. The lookup result now carries the validators of its body (upstream's for a fetched document, the row's for a stored one) and whether it is a stale fallback, and the response is written from those. A 304 also trusted whatever was stored at the row's path. Another instance sharing the store may have replaced it meanwhile, and the new bytes were then served and stored under the old ETag. The stored bytes are now checked against the row's digest, and a 304 whose row can no longer be refreshed is fetched again without validators instead of storing the old document back over the new one. --- docs/configuration.md | 2 +- internal/handler/conda_shards.go | 11 +- internal/handler/go_sumdb.go | 4 +- internal/handler/go_vulndb.go | 2 +- internal/handler/handler.go | 107 ++++++--- internal/handler/metadata_consistency_test.go | 203 ++++++++++++++++++ internal/handler/metadata_immutable.go | 7 +- internal/handler/npm.go | 4 +- internal/handler/nuget_cooldown.go | 2 +- 9 files changed, 299 insertions(+), 43 deletions(-) create mode 100644 internal/handler/metadata_consistency_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..1fdf008 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1499,7 +1499,7 @@ Or via environment variable: `PROXY_METADATA_TTL=10m`. Set to `"0"` to always revalidate with upstream (ETag-based conditional requests still avoid re-downloading unchanged content). -When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. +When upstream is unreachable and the cached entry is past its TTL (with a `metadata_ttl` of `"0"`, on every such request), the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. The `ETag` and `Last-Modified` sent with a document are always those of the bytes served, so a document fetched from upstream but not stored, for example because storage failed, carries the upstream's validators rather than the older cached copy's. ### Change feeds diff --git a/internal/handler/conda_shards.go b/internal/handler/conda_shards.go index 01434e4..f2f9511 100644 --- a/internal/handler/conda_shards.go +++ b/internal/handler/conda_shards.go @@ -58,11 +58,12 @@ func (h *CondaHandler) handleShardIndex(w http.ResponseWriter, r *http.Request) upstreamURL := h.upstreamURL + r.URL.Path cacheKey := h.metadataCacheKey(r.URL.Path) - body, contentType, contentEncoding, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", cacheKey, upstreamURL, "identity", nil, "*/*") - if err != nil { - h.proxy.writeMetadataFetchError(w, err) + res := h.proxy.lookupMetadata(r.Context(), "conda", cacheKey, upstreamURL, "*/*", "identity", nil) + if res.err != nil { + h.proxy.writeMetadataFetchError(w, res.err) return } + body := res.body if !h.cooldownEnabled() { index, err := decodeCondaShardIndex(body) @@ -74,7 +75,7 @@ func (h *CondaHandler) handleShardIndex(w http.ResponseWriter, r *http.Request) http.NotFound(w, r) return } - h.proxy.writeMetadataCachedResponseWithEncoding(w, r, "conda", cacheKey, body, contentType, contentEncoding) + writeMetadataResult(w, r, res) return } @@ -121,7 +122,7 @@ func (h *CondaHandler) handleCooldownShard(w http.ResponseWriter, r *http.Reques } upstreamPath := "/" + channel + "/" + arch + "/" + dir + hex.EncodeToString(masked) + condaShardSuffix - body, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", h.metadataCacheKey(upstreamPath), h.upstreamURL+upstreamPath, "identity", nil, "*/*") + body, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "conda", h.metadataCacheKey(upstreamPath), h.upstreamURL+upstreamPath, "identity", nil, "*/*") if err != nil { h.proxy.writeMetadataFetchError(w, err) return diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index 8b38206..60bd02b 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -383,7 +383,7 @@ func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, name, ups // the upstream timeout before the fetch falls back to it. if h.sumDBOffline && h.sumDBProbes.recentlyFailed(name) { if entry, body, contentType, ok := h.readCachedSumDB(r.Context(), cacheKey, check); ok { - p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, body, contentType, entry.ContentEncoding.String) + writeMetadataBody(w, r, p.cachedMetaFor(entry), body, contentType, entry.ContentEncoding.String) return } } @@ -416,7 +416,7 @@ func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, name, ups return } - p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) + writeMetadataResult(w, r, res) } // fetchSumDB fetches a checksum database response through the metadata diff --git a/internal/handler/go_vulndb.go b/internal/handler/go_vulndb.go index d3c1598..4cd2970 100644 --- a/internal/handler/go_vulndb.go +++ b/internal/handler/go_vulndb.go @@ -57,7 +57,7 @@ func (h *GoHandler) handleVulnDB(w http.ResponseWriter, r *http.Request, path st p.writeMetadataFetchError(w, res.err) return } - p.writeMetadataCachedResponseWithEncoding(w, r, goEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) + writeMetadataResult(w, r, res) } // validGoVulnDBPath reports whether path is a file the vulnerability diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d316aab..c70dea1 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -4,7 +4,9 @@ package handler import ( "bytes" "context" + "crypto/sha256" "database/sql" + "encoding/hex" "errors" "fmt" "io" @@ -1618,8 +1620,7 @@ var errStale304 = fmt.Errorf("upstream returned 304 but cached file is missing") // cacheKey is typically the package name but can include subpath components. // Optional acceptHeaders specify the Accept header(s) to send; defaults to application/json. func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL string, acceptHeaders ...string) ([]byte, string, error) { - body, contentType, _, err := p.fetchOrCacheMetadata(ctx, ecosystem, cacheKey, upstreamURL, "", nil, acceptHeaders...) - return body, contentType, err + return p.fetchOrCacheMetadata(ctx, ecosystem, cacheKey, upstreamURL, "", nil, acceptHeaders...) } // fetchOrCacheMetadata implements FetchOrCacheMetadata. acceptEncoding controls @@ -1635,13 +1636,13 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u // it can also decode the document into request-local state; a caller that // joined a fetch gets its own validation error directly. It must not modify the // body, which joined callers share. -func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, acceptEncoding string, validate func([]byte) error, acceptHeaders ...string) ([]byte, string, string, error) { +func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, acceptEncoding string, validate func([]byte) error, acceptHeaders ...string) ([]byte, string, error) { accept := contentTypeJSON if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { accept = acceptHeaders[0] } res := p.lookupMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) - return res.body, res.contentType, res.contentEncoding, res.err + return res.body, res.contentType, res.err } // fetchMetadataDocument is FetchOrCacheMetadata for callers that rewrite the @@ -1782,22 +1783,23 @@ func (p *Proxy) refreshMetadata(ctx context.Context, ecosystem, cacheKey, upstre // revalidate against, or nil. func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error, entry *database.MetadataCacheEntry) metadataResult { meta, err := p.fetchUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) + var digest string + if err == nil && p.CacheMetadata { + digest, err = p.refreshMetadataBlob(ecosystem, cacheKey, meta) + } if errors.Is(err, errStale304) { - // 304 but cached file is gone; retry without ETag + // 304 but the cached copy is gone or was replaced; retry without + // validators. meta, err = p.fetchUpstreamMetadata(ctx, upstreamURL, nil, accept, acceptEncoding) } if err == nil && validate != nil { err = validate(meta.body) } if err == nil { - var digest string - if p.CacheMetadata { - digest = p.refreshMetadataBlob(ecosystem, cacheKey, meta) - if digest == "" { - digest = p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) - } + if p.CacheMetadata && digest == "" { + digest = p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) } - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} + return fetchedResult(meta, digest) } return p.staleMetadata(ctx, ecosystem, cacheKey, validate, entry, err) @@ -1842,6 +1844,10 @@ type metadataResult struct { // empty otherwise. digest string err error + // etag and lastModified are the validators of body: upstream's for a + // fetched document, the cache row's for a stored one. + etag string + lastModified time.Time // stale is set when the body is the cached copy served because upstream // failed. stale bool @@ -1854,15 +1860,37 @@ type metadataResult struct { entry *database.MetadataCacheEntry } -// describedBy returns res with the digest that entry, the cache row its body -// was read for, records. +// describedBy returns res with the digest and validators that entry, the +// cache row its body was read for, records. func (res metadataResult) describedBy(entry *database.MetadataCacheEntry) metadataResult { if entry.ContentDigest.Valid { res.digest = entry.ContentDigest.String } + res.etag = entry.ETag.String + if entry.LastModified.Valid { + res.lastModified = entry.LastModified.Time + } return res } +// fetchedResult returns the result for meta, a document from upstream (or +// the stored copy it answered 304 for), cached under digest. +func fetchedResult(meta *upstreamMetadata, digest string) metadataResult { + return metadataResult{ + body: meta.body, + contentType: meta.contentType, + contentEncoding: meta.contentEncoding, + digest: digest, + etag: meta.etag, + lastModified: meta.lastModified, + } +} + +// cachedMeta returns the validators and staleness of res's body. +func (res metadataResult) cachedMeta() cachedMeta { + return cachedMeta{etag: res.etag, lastModified: res.lastModified, stale: res.stale} +} + // inflightMetadata is one metadata fetch that concurrent callers share. res is // written before done closes and read only after, so the close is the handoff. type inflightMetadata struct { @@ -2006,7 +2034,8 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e } defer func() { _ = cached.Close() }() data, readErr := p.ReadMetadataSized(cached, metadataEntrySize(entry)) - if readErr != nil { + if readErr != nil || !matchesEntryDigest(entry, data) { + // Gone, or replaced by another instance since entry was read. return nil, errStale304 } meta := &upstreamMetadata{body: data, contentType: contentTypeJSON, etag: entry.ETag.String, notModified: entry} @@ -2073,6 +2102,17 @@ func (p *Proxy) requestUpstreamMetadata(ctx context.Context, upstreamURL string, return resp, nil } +// matchesEntryDigest reports whether data is the document entry records, +// when entry has a sha256 digest to check it by. +func matchesEntryDigest(entry *database.MetadataCacheEntry, data []byte) bool { + want, ok := strings.CutPrefix(entry.ContentDigest.String, "sha256:") + if !entry.ContentDigest.Valid || !ok { + return true + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) == want +} + // upstreamMetadataStatus returns the error for a metadata response other // than 200. Callers handle a 304 they asked for before. func upstreamMetadataStatus(resp *http.Response) error { @@ -2090,23 +2130,25 @@ func upstreamMetadataStatus(resp *http.Response) error { // document meta was read from, by moving the row's fetched_at on. The bytes in // storage are unchanged, so they are neither written nor hashed again. It // returns the row's digest, or "" when the caller has to store meta as for a -// new document: when upstream sent one, the row records no digest, or a -// concurrent fetch has replaced the document since it was read. -func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMetadata) string { +// new document: when upstream sent one or the row records no digest. When +// the row could not be refreshed, for instance because a concurrent fetch has +// replaced the document since it was read, it returns errStale304: storing +// meta would put the old document back under the new one. +func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMetadata) (string, error) { entry := meta.notModified if entry == nil || p.DB == nil || !entry.ContentDigest.Valid { - return "" + return "", nil } refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) if err != nil { p.Logger.Warn("failed to refresh cached metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return "" + return "", errStale304 } if !refreshed { - return "" + return "", errStale304 } p.recordMetadataRead(entry) - return entry.ContentDigest.String + return entry.ContentDigest.String, nil } // cacheMetadataBlob stores metadata bytes in storage and updates the @@ -2250,16 +2292,20 @@ func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, return } - body, contentType, contentEncoding, err := p.fetchOrCacheMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, acceptEncoding, nil, acceptHeaders...) - if err != nil { - p.writeMetadataFetchError(w, err) + accept := contentTypeJSON + if len(acceptHeaders) > 0 && acceptHeaders[0] != "" { + accept = acceptHeaders[0] + } + res := p.lookupMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, nil) + if res.err != nil { + p.writeMetadataFetchError(w, res.err) return } if hooks != nil && hooks.fetched != nil { - hooks.fetched(body, contentType, contentEncoding) + hooks.fetched(res.body, res.contentType, res.contentEncoding) } - p.writeMetadataCachedResponseWithEncoding(w, r, ecosystem, cacheKey, body, contentType, contentEncoding) + writeMetadataResult(w, r, res) } // writeMetadataFetchError answers a failed metadata fetch. @@ -2290,6 +2336,13 @@ func (p *Proxy) writeMetadataCachedResponseWithEncoding(w http.ResponseWriter, r writeMetadataBody(w, r, p.lookupCachedMeta(ecosystem, cacheKey), body, contentType, contentEncoding) } +// writeMetadataResult writes the document res holds with its own validators +// and staleness, rather than the cache row's, which may describe other bytes +// when the cache write failed or another instance replaced it. +func writeMetadataResult(w http.ResponseWriter, r *http.Request, res metadataResult) { + writeMetadataBody(w, r, res.cachedMeta(), res.body, res.contentType, res.contentEncoding) +} + // writeMetadataBody writes a metadata document with the validators and // staleness in cm. func writeMetadataBody(w http.ResponseWriter, r *http.Request, cm cachedMeta, body []byte, contentType, contentEncoding string) { diff --git a/internal/handler/metadata_consistency_test.go b/internal/handler/metadata_consistency_test.go new file mode 100644 index 0000000..7384a10 --- /dev/null +++ b/internal/handler/metadata_consistency_test.go @@ -0,0 +1,203 @@ +package handler + +import ( + "context" + "database/sql" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// proxyCachedRequest runs one ProxyCached request for key "pkg". +func proxyCachedRequest(proxy *Proxy, upstreamURL string, header http.Header) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/pkg", nil) + for k, v := range header { + r.Header[k] = v + } + proxy.ProxyCached(w, r, upstreamURL+"/pkg", "test", "pkg") + return w +} + +// TestFetchedMetadataValidatorsDescribeBody checks that a document fetched +// fresh but not stored is served with upstream's validators, not those of the +// older row still in the cache. +func TestFetchedMetadataValidatorsDescribeBody(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set(headerContentType, contentTypeJSON) + if requests.Add(1) == 1 { + w.Header().Set(headerETag, `"v1"`) + _, _ = w.Write([]byte(`{"v":1}`)) + return + } + w.Header().Set(headerETag, `"v2"`) + _, _ = w.Write([]byte(`{"v":2}`)) + })) + t.Cleanup(upstream.Close) + + proxy, _, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Millisecond + proxy.HTTPClient = upstream.Client() + + if w := proxyCachedRequest(proxy, upstream.URL, nil); w.Code != http.StatusOK { + t.Fatalf("first request: %d", w.Code) + } + time.Sleep(5 * time.Millisecond) + store.mu.Lock() + store.storeErr = errors.New("disk full") + store.mu.Unlock() + + w := proxyCachedRequest(proxy, upstream.URL, nil) + if w.Code != http.StatusOK || w.Body.String() != `{"v":2}` { + t.Fatalf("second request = %d %q, want v2", w.Code, w.Body) + } + if got := w.Header().Get(headerETag); got != `"v2"` { + t.Errorf("ETag = %q, want %q", got, `"v2"`) + } + if got := w.Header().Get("Warning"); got != "" { + t.Errorf("Warning = %q on a fresh document", got) + } + + // A client holding v1 must get v2, not a 304. + w = proxyCachedRequest(proxy, upstream.URL, http.Header{"If-None-Match": {`"v1"`}}) + if w.Code != http.StatusOK || w.Body.String() != `{"v":2}` { + t.Errorf("conditional request = %d %q, want 200 v2", w.Code, w.Body) + } +} + +// TestStaleMetadataMarkedWithoutTTL checks that the stale fallback carries +// the Warning header with metadata_ttl 0, where every request revalidates. +func TestStaleMetadataMarkedWithoutTTL(t *testing.T) { + upstream := upstreamThen(t, `{"v":1}`, http.StatusBadGateway) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 + proxy.HTTPClient = upstream.Client() + + if w := proxyCachedRequest(proxy, upstream.URL, nil); w.Code != http.StatusOK { + t.Fatalf("first request: %d", w.Code) + } + w := proxyCachedRequest(proxy, upstream.URL, nil) + if w.Code != http.StatusOK || w.Body.String() != `{"v":1}` { + t.Fatalf("fallback = %d %q", w.Code, w.Body) + } + if got := w.Header().Get("Warning"); got != staleWarning { + t.Errorf("Warning = %q, want %q", got, staleWarning) + } +} + +// seedMetadataRow stores body for key "pkg" with etag and a row past its TTL. +func seedMetadataRow(t *testing.T, db *database.DB, store *mockStorage, body, etag string) string { + t.Helper() + path := metadataStoragePath("test", "pkg") + store.mu.Lock() + store.files[path] = []byte(body) + store.mu.Unlock() + if err := db.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "test", + Name: "pkg", + StoragePath: path, + ETag: sql.NullString{String: etag, Valid: true}, + ContentType: sql.NullString{String: contentTypeJSON, Valid: true}, + ContentDigest: sql.NullString{String: "sha256:" + sha256Hex(body), Valid: true}, + Size: sql.NullInt64{Int64: int64(len(body)), Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, + }); err != nil { + t.Fatal(err) + } + return path +} + +// v2Upstream answers If-None-Match "v1" with 304 after running onNotModified, +// and any other request with v2. +func v2Upstream(t *testing.T, onNotModified func()) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("If-None-Match") == `"v1"` { + onNotModified() + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set(headerContentType, contentTypeJSON) + w.Header().Set(headerETag, `"v2"`) + _, _ = w.Write([]byte(`{"v":2}`)) + })) + t.Cleanup(srv.Close) + return srv +} + +// TestNotModifiedChecksStoredBytes checks that a 304 for a row whose blob has +// meanwhile been replaced (by another instance sharing the store) does not +// serve the new bytes under the old ETag. +func TestNotModifiedChecksStoredBytes(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Minute + path := seedMetadataRow(t, db, store, `{"v":1}`, `"v1"`) + upstream := v2Upstream(t, func() { + // The other instance has written v2 but not yet its row. + store.mu.Lock() + store.files[path] = []byte(`{"v":2}`) + store.mu.Unlock() + }) + proxy.HTTPClient = upstream.Client() + + w := proxyCachedRequest(proxy, upstream.URL, nil) + if w.Code != http.StatusOK || w.Body.String() != `{"v":2}` { + t.Fatalf("response = %d %q", w.Code, w.Body) + } + if got := w.Header().Get(headerETag); got != `"v2"` { + t.Errorf("ETag = %q, want %q", got, `"v2"`) + } + row, err := db.GetMetadataCache("test", "pkg") + if err != nil || row == nil { + t.Fatal(row, err) + } + if row.ETag.String != `"v2"` || row.ContentDigest.String != "sha256:"+sha256Hex(`{"v":2}`) { + t.Errorf("row = etag %q digest %q, want v2's", row.ETag.String, row.ContentDigest.String) + } +} + +// TestNotModifiedAfterRowReplaced checks that a 304 for a row another +// instance has replaced since it was read does not store the old document +// back over the new one. +func TestNotModifiedAfterRowReplaced(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Minute + path := seedMetadataRow(t, db, store, `{"v":1}`, `"v1"`) + upstream := v2Upstream(t, func() { + // The other instance has recorded v2's row; its blob is not written + // yet. + _ = db.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "test", + Name: "pkg", + StoragePath: path, + ETag: sql.NullString{String: `"v2"`, Valid: true}, + ContentType: sql.NullString{String: contentTypeJSON, Valid: true}, + ContentDigest: sql.NullString{String: "sha256:" + sha256Hex(`{"v":2}`), Valid: true}, + Size: sql.NullInt64{Int64: 7, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + }) + }) + proxy.HTTPClient = upstream.Client() + + res := proxy.fetchMetadataDocument(context.Background(), "test", "pkg", upstream.URL+"/pkg", contentTypeJSON, nil) + if res.err != nil || string(res.body) != `{"v":2}` { + t.Fatalf("fetch = %q, %v; want v2", res.body, res.err) + } + row, err := db.GetMetadataCache("test", "pkg") + if err != nil || row == nil { + t.Fatal(row, err) + } + if row.ETag.String != `"v2"` { + t.Errorf("row etag = %q, want %q", row.ETag.String, `"v2"`) + } +} diff --git a/internal/handler/metadata_immutable.go b/internal/handler/metadata_immutable.go index 691fbc1..214077c 100644 --- a/internal/handler/metadata_immutable.go +++ b/internal/handler/metadata_immutable.go @@ -40,9 +40,8 @@ func (p *Proxy) proxyImmutable(w http.ResponseWriter, r *http.Request, upstreamU p.writeMetadataFetchError(w, res.err) return } - cm := p.lookupCachedMeta(ecosystem, cacheKey) - cm.stale = false - writeMetadataBody(w, r, cm, res.body, res.contentType, res.contentEncoding) + res.stale = false + writeMetadataResult(w, r, res) } // fetchImmutableMetadata returns an immutable metadata document from the @@ -74,7 +73,7 @@ func (p *Proxy) fetchImmutableMetadata(ctx context.Context, ecosystem, cacheKey, return metadataResult{err: err} } digest := p.cacheMetadataBlob(fetchCtx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding, digest: digest} + return fetchedResult(meta, digest) }) return res } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 50757a3..b678f9b 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -591,7 +591,7 @@ func (h *NPMHandler) handleFilteredNPMVersionMetadata(w http.ResponseWriter, r * return } - packument, _, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) + packument, _, err := h.proxy.fetchOrCacheMetadata(r.Context(), "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { JSONError(w, http.StatusNotFound, ErrPackageNotFound.Error()) @@ -901,7 +901,7 @@ func (h *NPMHandler) versionData(ctx context.Context, packumentURL, cacheKey, ve return npmVersionData{}, fetchErr } - body, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) + body, _, err := h.proxy.fetchOrCacheMetadata(ctx, "npm", cacheKey, packumentURL, "", h.packumentValidator(), accept) if err != nil { return npmVersionData{}, err } diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index b9345c5..7a0a17a 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -44,7 +44,7 @@ func (h *NuGetHandler) nugetMetadata(ctx context.Context, path string) (map[stri document, err = h.decodeNuGetMetadata(body) return err } - _, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "nuget", key, target, "", validate) + _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "nuget", key, target, "", validate) if err != nil { return nil, err } From 3860a6cae5a6831641bc02b36ecd5efce70418a4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:00:50 +0100 Subject: [PATCH 117/198] Don't cache Maven paths with dotted group directories parsePath joins the group directories with dots, and the cache filename is the file's base name, so org.foo/bar/1.0/bar-1.0.jar and org/foo/bar/1.0/bar-1.0.jar shared one cache entry. Only upstreams that serve content at such directories, such as raw or hosted repositories, are affected. The Maven layout splits a group at its dots, so a group directory never contains a dot or is empty. Treat such paths as outside the layout and proxy them uncached, which keeps every existing cache entry valid. --- docs/configuration.md | 4 ++++ internal/handler/maven.go | 9 +++++++++ internal/handler/maven_repositories_test.go | 19 +++++++++++++++++++ internal/handler/maven_test.go | 5 +++++ 4 files changed, 37 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 9046e28..59a33b0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1004,6 +1004,10 @@ one doesn't fall back to the Gradle Plugin Portal. Files are cached per repository, so the same coordinates in two repositories never share a cache entry. +In every repository, a path whose group directories contain a dot or are empty +(`org.foo/bar/1.0/bar-1.0.jar`) is not in the Maven layout and is proxied +without caching, since its group would read the same as `org/foo`'s. + `upstream.maven_repositories` is additive. `/maven/…` keeps serving `upstream.maven` with unchanged cache identities. A repository name is matched before the default repository, so it hides every group ID whose first segment diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 6c6a647..ca81b74 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -266,6 +266,7 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so // parsePath extracts Maven coordinates from a URL path. // e.g., "com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar" // -> ("com.google.guava", "guava", "32.1.3-jre", "guava-32.1.3-jre.jar") +// A path that isn't in the Maven layout returns empty strings. func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, filename string) { parts := strings.Split(urlPath, "/") if len(parts) < minMavenParts { @@ -276,6 +277,14 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file version = parts[len(parts)-2] artifact = parts[len(parts)-3] groupParts := parts[:len(parts)-3] + // The Maven layout splits a group at its dots, so a group directory + // never holds a dot or is empty. Such a path would spell the same group + // as another and share its cache entries, so it is not parsed. + for _, part := range groupParts { + if part == "" || strings.Contains(part, ".") { + return "", "", "", "" + } + } group = strings.Join(groupParts, ".") return group, artifact, version, filename diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go index 8272f37..15db066 100644 --- a/internal/handler/maven_repositories_test.go +++ b/internal/handler/maven_repositories_test.go @@ -137,3 +137,22 @@ func TestMavenRepositoryRepointedServesTheNewUpstream(t *testing.T) { } } } + +// TestMavenDottedGroupDirectoryNotCachedAsGroup checks that a path whose +// group directories contain dots is not cached under the group the dots +// spell, where it would share an entry with the standard layout's path. +func TestMavenDottedGroupDirectoryNotCachedAsGroup(t *testing.T) { + const standard = "/org/foo/bar/1.0/bar-1.0.jar" + const dotted = "/org.foo/bar/1.0/bar-1.0.jar" + central, _ := mavenRepoUpstream(t, "central", standard, dotted) + portal, _ := mavenRepoUpstream(t, "portal") + grails, _ := mavenRepoUpstream(t, "grails") + h := newMavenRepositoriesHandler(t, central, portal, grails) + + for _, path := range []string{standard, dotted, standard} { + w := getMaven(h, path) + if want := "central " + path; w.Code != http.StatusOK || w.Body.String() != want { + t.Errorf("GET %s = %d %q, want 200 %q", path, w.Code, w.Body.String(), want) + } + } +} diff --git a/internal/handler/maven_test.go b/internal/handler/maven_test.go index 9d34b09..3d70a32 100644 --- a/internal/handler/maven_test.go +++ b/internal/handler/maven_test.go @@ -31,6 +31,11 @@ func TestMavenParsePath(t *testing.T) { "short/path", "", "", "", "", }, + // Group directories never hold dots or are empty in the Maven + // layout; these would spell the same group as org/foo. + {"org.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + {"org/.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + {"org//foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, } for _, tt := range tests { From 81c6ea26c2350bab1c6c610e419e866e76733022 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:00:50 +0100 Subject: [PATCH 118/198] Treat git keys differing only in case as one repository On a case-insensitive file system, as macOS and Windows use by default, acme/WIDGETS and acme/widgets name the same cache directory but were tracked as two repositories. Users of one spelling didn't keep the other from being evicted, so eviction could move the directory away while it was being read, and each entry had its own write lock, so two refreshes could fetch into the directory at once. Check at startup whether the cache directory ignores case and, if so, track repositories under their lowercased key. The directory and the upstream URL keep the spelling requested. Case-sensitive file systems, and so existing caches on Linux, are unchanged. --- docs/configuration.md | 6 ++++++ internal/gitcache/cache.go | 35 +++++++++++++++++++++++++++++---- internal/gitcache/cache_test.go | 27 +++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 59a33b0..30c9a0c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -946,6 +946,12 @@ those yourself. The proxy locks `cache_dir` while it runs, through a `.gitcache-lock` file in it, so a second process given the same directory, such as another replica on a shared volume, fails to start; give each its own. +Each repository's directory is named by its path as requested. Where +`cache_dir` ignores case in file names, as on macOS and Windows by default, +paths that differ only in case (`acme/widgets` and `acme/WIDGETS`) share one +directory and are treated as one repository. Use a case-sensitive file system +for an upstream whose repositories differ only in case. + Git LFS objects are cached in the artifact cache, so they use the configured storage backend and retention rather than `cache_dir`. git-lfs finds the LFS API under the rewritten remote, at `/git/{name}/{repo}.git/info/lfs`, with no diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index eb1578f..6bc3c20 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -95,6 +95,9 @@ type Cache struct { refCheckTimeout time.Duration lowSpeedTime int maxRequestBody int64 + // foldCase is set where the cache directory ignores case, so keys that + // differ only in case, which name one directory, are one repository. + foldCase bool ctx context.Context cancel context.CancelFunc @@ -214,6 +217,7 @@ func New(opts Options) (*Cache, error) { tooLarge: make(map[string]time.Time), failedWants: make(map[string]time.Time), freed: make(chan struct{}), + foldCase: caseInsensitive(dir), } if opts.refCheckTimeout > 0 { c.refCheckTimeout = opts.refCheckTimeout @@ -284,7 +288,7 @@ func (c *Cache) load() error { return err } rel, _ := filepath.Rel(c.dir, path) - rp := newRepo(filepath.ToSlash(rel), path) + rp := newRepo(c.repoKey(filepath.ToSlash(rel)), path) rp.size.Store(dirSize(path)) rp.lastChecked.Store(stampTime(path, checkedStamp)) rp.lastUsed.Store(stampTime(path, usedStamp)) @@ -352,15 +356,24 @@ func (c *Cache) removeRepointed() { func (c *Cache) acquire(key string) *repo { c.mu.Lock() defer c.mu.Unlock() - rp := c.repos[key] + rp := c.repos[c.repoKey(key)] if rp == nil { - rp = newRepo(key, filepath.Join(c.dir, filepath.FromSlash(key))) - c.repos[key] = rp + rp = newRepo(c.repoKey(key), filepath.Join(c.dir, filepath.FromSlash(key))) + c.repos[rp.key] = rp } rp.users++ return rp } +// repoKey is the key a repository is tracked under: key itself, or key +// lowercased where the cache directory ignores case. +func (c *Cache) repoKey(key string) string { + if c.foldCase { + return strings.ToLower(key) + } + return key +} + func (c *Cache) release(rp *repo) { now := time.Now() rp.lastUsed.Store(now.UnixNano()) @@ -511,3 +524,17 @@ func touch(dir, name string, t time.Time) { } } } + +// caseInsensitive reports whether the file system holding dir ignores case +// in names, as macOS and Windows do by default. +func caseInsensitive(dir string) bool { + f, err := os.CreateTemp(dir, ".gitcache-case-") + if err != nil { + return false + } + name := f.Name() + _ = f.Close() + defer func() { _ = os.Remove(name) }() + _, err = os.Stat(filepath.Join(dir, strings.ToUpper(filepath.Base(name)))) + return err == nil +} diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 4d43c5d..690b11e 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -786,3 +786,30 @@ func TestCloseTwice(t *testing.T) { t.Errorf("closing twice: %v", err) } } + +// TestCaseOnlyDifferentKeysShareOneRepository checks that, where the file +// system ignores case, keys that differ only in case are one repository, so +// a user of one spelling keeps the directory from being evicted under it. +func TestCaseOnlyDifferentKeysShareOneRepository(t *testing.T) { + if !caseInsensitive(t.TempDir()) { + t.Skip("file system is case-sensitive") + } + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + rp := tc.acquire("up/REPO.git") + defer tc.release(rp) + if other := tc.repo("up/repo.git"); other != nil && other != rp { + t.Error("up/REPO.git and up/repo.git are separate repositories") + } + tc.mu.Lock() + tc.opts.MaxSize = 1 + tc.mu.Unlock() + tc.evict() + if !rp.exists() { + t.Error("repository in use as up/REPO.git was evicted") + } +} From 1fbcdc95c8fbd4dc3c929de0765e115c766c0f38 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:58:55 +0100 Subject: [PATCH 119/198] Rewrite project links in the PyPI root simple index pypi.org's HTML root index links each project as /simple//. The proxy relayed it unchanged, but it is mounted at /pypi, so following a link went to /simple// on the proxy and got 404. Rewrite those hrefs to the proxy's /pypi/simple/ while streaming the response, since the index is tens of megabytes. The JSON index has no links and is left alone. --- internal/handler/pypi.go | 70 +++++++++++++++++++++++++++++++++++ internal/handler/pypi_test.go | 50 +++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index ebb3082..7dd3f12 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "maps" "mime" "net/http" @@ -1022,8 +1023,77 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s } defer func() { _ = resp.Body.Close() }() + // The HTML root index links projects as /simple//, which must + // point at the proxy's /pypi mount instead. It is large, so it is + // rewritten as it streams. + if resp.StatusCode == http.StatusOK && resp.Header.Get("Content-Encoding") == "" && + !strings.Contains(resp.Header.Get(headerContentType), "json") { + resp.Body = &replacingReader{ + src: resp.Body, + from: []byte(`href="/simple/`), + to: []byte(`href="` + h.proxyURL + `/pypi/simple/`), + } + resp.Header.Del(headerContentLength) + resp.ContentLength = -1 + } + h.proxy.relayResponse(w, r, resp, func(dst, src http.Header) { copyRelayHeaders(dst, src) ensureVaryAccept(dst) }) } + +const replacingReaderBufferSize = 32 << 10 + +// replacingReader replaces every occurrence of from with to in src as it is +// read. +type replacingReader struct { + src io.ReadCloser + from, to []byte + in []byte // read from src, not yet scanned + out []byte // scanned, not yet returned + buf []byte + err error +} + +func (r *replacingReader) Read(p []byte) (int, error) { + for len(r.out) == 0 { + if r.err != nil { + return 0, r.err + } + if r.buf == nil { + r.buf = make([]byte, replacingReaderBufferSize) + } + n, err := r.src.Read(r.buf) + r.in = append(r.in, r.buf[:n]...) + r.err = err + // Keep a tail that could be the start of a match split across reads. + keep := len(r.from) - 1 + if err != nil { + keep = 0 + } + scan := max(len(r.in)-keep, 0) + var out []byte + done := 0 + for { + i := bytes.Index(r.in[done:], r.from) + if i < 0 || done+i >= scan { + break + } + out = append(out, r.in[done:done+i]...) + out = append(out, r.to...) + done += i + len(r.from) + } + if done < scan { + out = append(out, r.in[done:scan]...) + done = scan + } + r.out = out + r.in = append([]byte(nil), r.in[done:]...) + } + n := copy(p, r.out) + r.out = r.out[n:] + return n, nil +} + +func (r *replacingReader) Close() error { return r.src.Close() } diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index dd59974..73309cb 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -2,13 +2,16 @@ package handler import ( "encoding/json" + "fmt" "io" "log/slog" "net/http" "net/http/httptest" + "strconv" "strings" "sync/atomic" "testing" + "testing/iotest" "time" "github.com/git-pkgs/cooldown" @@ -611,3 +614,50 @@ func TestPyPIDownloadCooldownMetadataCache(t *testing.T) { t.Errorf("upstream metadata JSON requests = %d, want 1 (repeated downloads should reuse the cached metadata)", got) } } + +func TestPyPISimpleIndexRewritesProjectLinks(t *testing.T) { + // Long enough that links cross the rewriter's read boundaries. + var upstream, want strings.Builder + upstream.WriteString(``) + want.WriteString(``) + for i := range 3000 { + fmt.Fprintf(&upstream, `pkg-%d`+"\n", i, i) + fmt.Fprintf(&want, `pkg-%d`+"\n", i, i) + } + upstream.WriteString(``) + want.WriteString(``) + + h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + resp := pypiHTTPResponse(r, "text/html", upstream.String()) + resp.Header.Set("Content-Length", strconv.Itoa(upstream.Len())) + return resp, nil + }) + + req := httptest.NewRequest(http.MethodGet, "/simple/", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + if got := w.Body.String(); got != want.String() { + t.Errorf("body not rewritten; starts %q", got[:min(len(got), 120)]) + } + if got := w.Header().Get("Content-Length"); got != "" && got != strconv.Itoa(want.Len()) { + t.Errorf("Content-Length = %q, want %d or unset", got, want.Len()) + } +} + +func TestReplacingReaderSplitReads(t *testing.T) { + const in = `abxhref="/simplehref="/simple/` + const want = `abxhref="/simplehref="P/` + r := &replacingReader{ + src: io.NopCloser(iotest.OneByteReader(strings.NewReader(in))), + from: []byte(`href="/simple/`), + to: []byte(`href="P/`), + } + got, err := io.ReadAll(r) + if err != nil || string(got) != want { + t.Errorf("got %q, %v; want %q", got, err, want) + } +} From 20cbeeff22f25a0645ab3023e41114ce75ff8e12 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:59:45 +0100 Subject: [PATCH 120/198] Disable install notifications for pushed Composer versions Composer reports installs to the repository's notify-batch URL unless a version has its own notification-url, and the proxy's packages.json sets notify-batch to the upstream. Routed packages get an empty notification-url for this reason, but pushed and hosted-only versions did not, so installs of private packages were reported to packagist.org. Give pushed versions the same empty notification-url. --- docs/configuration.md | 1 + internal/handler/composer_hosted.go | 5 +++++ internal/handler/composer_hosted_test.go | 19 +++++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..3e7abc7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1643,6 +1643,7 @@ Pushed versions are listed in the package metadata clients resolve against, with - Pushed versions are added next to the upstream ones and are never held back by cooldown. npm `dist-tags` stay as the upstream has them, also in `npm dist-tag ls`. - A package the upstream doesn't know is served from its pushed versions alone. Its npm `latest` tag is the highest pushed version. - Pushed Composer dev versions (`dev-*`, `*-dev`) are listed in the `~dev` metadata file. +- Pushed Composer versions carry an empty `notification-url`, as routed packages do, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. - npm entries carry the archive's sha512 as `dist.integrity`. Composer entries carry its sha256 as `dist.reference` and leave `shasum` empty. Metadata of packages with pushed versions is rewritten on every request instead of being cached, so a push or delete shows up on the next request. diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index e94d11c..3c42a6a 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -43,6 +43,11 @@ func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([] manifest["name"] = packageName manifest["version"] = lv.Version manifest["time"] = lv.PushedAt.Format(time.RFC3339) + // As for routed packages, keep Composer from reporting installs of + // a private version to the default repository's notify-batch. + if _, ok := manifest["notification-url"]; !ok { + manifest["notification-url"] = "" + } manifest["dist"] = map[string]string{ "type": "zip", "url": h.localDistURL(packageName, lv.Version, lv.Filename), diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index 539e999..6c439dc 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -243,3 +243,22 @@ func TestComposerDownloadOfPushedArchiveWhileStreamingArtifacts(t *testing.T) { t.Error("a pushed archive was fetched upstream while streaming artifacts") } } + +func TestComposerPushedVersionsDisableDefaultNotification(t *testing.T) { + f := newComposerHostedFixture(t, map[string]string{"/p2/monolog/monolog.json": monologMetadata}) + f.proxy.HostedComposerVendors = []string{"acme"} + f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) + f.push(t, "acme/private", "1.0.0", `{"name":"acme/private"}`) + f.push(t, "other/private", "1.0.0", `{"name":"other/private"}`) + + for _, tc := range []struct{ path, name, version string }{ + {"/p2/monolog/monolog.json", "monolog/monolog", "3.6.0"}, + {"/p2/acme/private.json", "acme/private", "1.0.0"}, + {"/p2/other/private.json", "other/private", "1.0.0"}, + } { + v := f.versions(t, tc.path, tc.name)[tc.version] + if got, ok := v["notification-url"]; !ok || got != "" { + t.Errorf("%s %s notification-url = %v (set %v), want empty", tc.name, tc.version, got, ok) + } + } +} From 12666071ce2d61ed656560781980b52198522cde Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:00:50 +0100 Subject: [PATCH 121/198] Make package search case-insensitive on Postgres too SearchPackages and CountSearchResults matched names with p.name LIKE '%query%'. LIKE ignores ASCII case on SQLite but not on Postgres, so the same search found different packages depending on the database. The % and _ in a query were also taken as wildcards. Compare LOWER(name) with LOWER(pattern) and escape the wildcards. --- internal/database/queries.go | 23 ++++++++--- .../database/queries_packages_list_test.go | 39 +++++++++++++++++++ 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/internal/database/queries.go b/internal/database/queries.go index 86f31ef..7176152 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -849,7 +849,7 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i } var results []SearchResult - searchPattern := "%" + query + "%" + searchPattern := nameSearchPattern(query) var sqlQuery string var args []any @@ -864,7 +864,7 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND ` + condition + ` AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND ` + condition + ` AND a.storage_path IS NOT NULL GROUP BY p.purl, p.ecosystem, p.name, p.latest_version, p.license ORDER BY hits DESC LIMIT ? OFFSET ? @@ -879,7 +879,7 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND a.storage_path IS NOT NULL GROUP BY p.purl, p.ecosystem, p.name, p.latest_version, p.license ORDER BY hits DESC LIMIT ? OFFSET ? @@ -894,6 +894,17 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i return results, nil } +// nameSearchCondition matches package names containing a search pattern +// from nameSearchPattern, ignoring case on SQLite and Postgres alike. +const nameSearchCondition = `LOWER(p.name) LIKE LOWER(?) ESCAPE '!'` + +// nameSearchPattern is the LIKE pattern for names containing query, with +// the LIKE wildcards in query matched literally. +func nameSearchPattern(query string) string { + escaped := strings.NewReplacer("!", "!!", "%", "!%", "_", "!_").Replace(query) + return "%" + escaped + "%" +} + func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) { hasArtifacts, err := db.HasTable("artifacts") if err != nil { @@ -903,7 +914,7 @@ func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) return 0, nil } - searchPattern := "%" + query + "%" + searchPattern := nameSearchPattern(query) var sqlQuery string var args []any @@ -914,7 +925,7 @@ func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND ` + condition + ` AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND ` + condition + ` AND a.storage_path IS NOT NULL `) args = append([]any{searchPattern}, ecosystemArgs...) } else { @@ -923,7 +934,7 @@ func (db *DB) CountSearchResults(query string, ecosystem string) (int64, error) FROM packages p LEFT JOIN versions v ON v.package_purl = p.purl LEFT JOIN artifacts a ON a.version_purl = v.purl - WHERE p.name LIKE ? AND a.storage_path IS NOT NULL + WHERE ` + nameSearchCondition + ` AND a.storage_path IS NOT NULL `) args = []any{searchPattern} } diff --git a/internal/database/queries_packages_list_test.go b/internal/database/queries_packages_list_test.go index 325fee7..704b388 100644 --- a/internal/database/queries_packages_list_test.go +++ b/internal/database/queries_packages_list_test.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "maps" + "slices" "testing" "time" ) @@ -250,3 +251,41 @@ func TestCachedPackageFiltersMatchEcosystemAliases(t *testing.T) { } }) } + +func TestSearchPackagesIgnoresCaseAndEscapesWildcards(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + seedArtifact(t, db, "pypi", "Django", "5.0.0", 100, 1) + seedArtifact(t, db, "npm", "my_pkg", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", "myxpkg", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", "100%-done", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", "1000-done", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", `back\slash`, "1.0.0", 100, 1) + + for _, tc := range []struct { + query string + want []string + }{ + {"django", []string{"Django"}}, + {"DJANGO", []string{"Django"}}, + {"my_pkg", []string{"my_pkg"}}, + {"100%", []string{"100%-done"}}, + {`k\s`, []string{`back\slash`}}, + } { + found, err := db.SearchPackages(tc.query, "", 50, 0) + if err != nil { + t.Fatalf("SearchPackages(%q): %v", tc.query, err) + } + var names []string + for _, p := range found { + names = append(names, p.Name) + } + if !slices.Equal(names, tc.want) { + t.Errorf("SearchPackages(%q) = %v, want %v", tc.query, names, tc.want) + } + count, err := db.CountSearchResults(tc.query, "") + if err != nil || count != int64(len(tc.want)) { + t.Errorf("CountSearchResults(%q) = %d, %v, want %d", tc.query, count, err, len(tc.want)) + } + } + }) +} From b3728b1e1645ddf0d71fc3962b8744f1edd5e0a6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 07:58:23 +0100 Subject: [PATCH 122/198] Check the cached copy before serving upload-pack requests Only GET info/refs checked the upstream. A POST git-upload-pack was served from the cached copy unchecked, so a client posting ls-refs and fetch directly (protocol v2 allows it) kept getting a repository that was deleted or made private upstream, past stale_max_age, indefinitely. Remember when a refresh finds a repository missing or refused upstream, and refuse upload-pack requests for it until a refresh succeeds. Refresh first when the copy was last checked more than a minute past stale_max_age ago; the minute keeps a normal clone (info/refs, then upload-pack) at one upstream check even with stale_max_age 0. Mirror refreshes on its own and skips the check. --- docs/configuration.md | 9 +++- internal/gitcache/cache.go | 7 ++- internal/gitcache/cache_test.go | 89 +++++++++++++++++++++++++++++++++ internal/gitcache/mirror.go | 2 +- internal/gitcache/refresh.go | 6 +++ internal/gitcache/serve.go | 41 +++++++++++++-- 6 files changed, 144 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 584896b..d384bac 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -899,8 +899,13 @@ commits, the request waits for their fetch, so a clone right after a push gets them; if the fetch fails or stalls, the cached copy is served. A `stale_max_age` of `0` never serves a copy the upstream has not just confirmed. A repository the upstream reports missing (404), refuses (401, 403) -or redirects is not served from the cache. Redirects are never followed, so -point the upstream at a repository's current location. +or redirects is not served from the cache, also to requests that skip the ref +listing and go straight to `git-upload-pack`, until a later check succeeds. +Redirects are never followed, so point the upstream at a repository's current +location. A `git-upload-pack` request does not check the upstream itself +unless the copy was last checked more than a minute past `stale_max_age` ago, +so the requests of a clone that follow its ref listing check nothing again; +an older copy is checked first, as for a ref listing. The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it). The `git` processes never reach the network themselves: their fetches go diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index eb1578f..7901380 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -121,14 +121,17 @@ type Cache struct { failedWants map[string]time.Time } -// repo is a cached repository, or one being cloned. users, refreshing and -// the map entry itself are guarded by Cache.mu. +// repo is a cached repository, or one being cloned. users, refreshing, +// refused and the map entry itself are guarded by Cache.mu. type repo struct { key string dir string users int refreshing *refreshCall + // refused is the error of the last refresh when the upstream reported + // the repository missing or refused it, until a refresh succeeds. + refused error // writes serializes git fetches into the repository. writes chan struct{} diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 4d43c5d..5f9d475 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -786,3 +786,92 @@ func TestCloseTwice(t *testing.T) { t.Errorf("closing twice: %v", err) } } + +// lsRefs sends a protocol v2 ls-refs request for "repo" straight to +// git-upload-pack, as a client may without asking for info/refs first, and +// returns the status. +func (tc *testCache) lsRefs(t *testing.T) int { + t.Helper() + body := pktLine("command=ls-refs\n") + pktDelim + pktLine("peel\n") + pktFlush + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, tc.url("repo")+"/git-upload-pack", strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", requestType) + req.Header.Set("Git-Protocol", "version=2") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + return resp.StatusCode +} + +// TestUploadPackChecksCopyPastStaleMaxAge sends upload-pack requests without +// info/refs: a copy not checked within stale_max_age is checked first. +func TestUploadPackChecksCopyPastStaleMaxAge(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + if status := tc.lsRefs(t); status != http.StatusOK { + t.Fatalf("ls-refs on a fresh copy: status %d", status) + } + + waitIdle(t, tc.Cache) + rp := tc.acquire("up/repo.git") + rp.lastChecked.Store(time.Now().Add(-2 * time.Hour).UnixNano()) + tc.release(rp) + upstream.status.Store(http.StatusNotFound) + if status := tc.lsRefs(t); status == http.StatusOK { + t.Errorf("served a copy older than stale_max_age to ls-refs") + } +} + +// TestUploadPackRefusedAfterUpstreamRefusal has the upstream refuse a +// refresh: upload-pack requests are refused too until a check succeeds. +func TestUploadPackRefusedAfterUpstreamRefusal(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusForbidden) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("served the cached copy after 403:\n%s", out) + } + if status := tc.lsRefs(t); status == http.StatusOK { + t.Errorf("ls-refs served after the upstream refused the repository") + } + + upstream.status.Store(0) + tc.clone(t, "repo") + if status := tc.lsRefs(t); status != http.StatusOK { + t.Errorf("ls-refs after a successful check: status %d", status) + } +} + +// TestCloneChecksUpstreamOnce clones with a stale_max_age of 0: the +// upload-pack requests after info/refs do not check the upstream again. +func TestCloneChecksUpstreamOnce(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.StaleMaxAge = 0 }) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + checks := func() int32 { return upstream.requests.Load() - upstream.gitRequests.Load() } + before := checks() + resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + perCheck := checks() - before + + before = checks() + tc.clone(t, "repo") + if got := checks() - before; got != perCheck { + t.Errorf("clone made %d upstream check requests, want %d", got, perCheck) + } +} diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index b5bde40..ad1584f 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -26,7 +26,7 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, commit string, fetchWa if commit != "" && !isObjectID(commit) { return MirrorResult{}, fmt.Errorf("%q is not a full commit ID", commit) } - lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants}) + lease, err := c.Open(ctx, key, repoURL, OpenOptions{FetchWants: fetchWants, NoCheck: true}) if err != nil { return MirrorResult{}, err } diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index 448b3d9..352eb2b 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -157,6 +157,12 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { } c.mu.Lock() rp.refreshing = nil + switch { + case call.err == nil: + rp.refused = nil + case !servesStale(call.err): + rp.refused = call.err + } c.mu.Unlock() c.release(rp) close(call.done) diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index bb5cfb0..68c891a 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -13,6 +13,7 @@ import ( "slices" "strings" "sync/atomic" + "time" ) // gitProtocolPattern is what a Git-Protocol header may hold, such as @@ -42,6 +43,11 @@ type OpenOptions struct { // commits the client may not read through this repository, such as // GitHub forks with credentials configured. FetchWants bool + // NoCheck leaves a cached repository as it is, for a caller that + // refreshes it itself. Otherwise a cached repository is refused when the + // last refresh found it gone or refused upstream, and refreshed first + // when it was not checked within StaleMaxAge. + NoCheck bool } // Open returns a lease on the repository cached under key, a relative path @@ -53,15 +59,40 @@ func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) } rp := c.acquire(key) hit := rp.exists() - if opts.Refresh || !hit { - if err := c.ensureFresh(ctx, rp, repoURL); err != nil { - c.release(rp) - return nil, err - } + var err error + switch { + case opts.Refresh || !hit: + err = c.ensureFresh(ctx, rp, repoURL) + case !opts.NoCheck: + err = c.checkServable(ctx, rp, repoURL) + } + if err != nil { + c.release(rp) + return nil, err } return &Lease{cache: c, repo: rp, repoURL: repoURL, hit: hit, fetchWants: opts.FetchWants}, nil } +// uploadPackGrace is how long past StaleMaxAge a copy is still served +// without a check to the upload-pack requests that follow a client's +// info/refs, so a clone checks the upstream once. +const uploadPackGrace = time.Minute + +// checkServable refuses a cached repository the last refresh found gone or +// refused upstream, and refreshes one not checked recently first. +func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) error { + c.mu.Lock() + err := rp.refused + c.mu.Unlock() + if err != nil { + return err + } + if time.Since(time.Unix(0, rp.lastChecked.Load())) > c.opts.StaleMaxAge+uploadPackGrace { + return c.ensureFresh(ctx, rp, repoURL) + } + return nil +} + // Hit reports whether the repository was already cached when it was opened. func (l *Lease) Hit() bool { return l.hit From 3fdc53d39ac2c47bfbe73963c64f9354d75ed7b8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:02:30 +0100 Subject: [PATCH 123/198] Stop relaying for a fetch once it has ended exec.CommandContext kills only the git parent. Its git-remote-http helper kept its relay connection and went on downloading for up to http.lowSpeedTime after a client disconnect, Close or fetchTimeout. Each read it relayed called cover, which reserved the relayed total again on a reservation already settled or released, so c.reserved grew for good and could evict repositories, and the helpers outlived the fetch cap. Tie each relay route to the fetch's context and cancel it when the fetch returns, so the upstream request ends and the helper's connection is cut. Make cover refuse a reservation that has been given back. --- internal/gitcache/budget.go | 13 +++++ internal/gitcache/budget_test.go | 93 ++++++++++++++++++++++++++++++++ internal/gitcache/refresh.go | 2 +- internal/gitcache/relay.go | 27 +++++++--- internal/gitcache/relay_test.go | 4 +- 5 files changed, 128 insertions(+), 11 deletions(-) diff --git a/internal/gitcache/budget.go b/internal/gitcache/budget.go index 9d4135e..2b0c228 100644 --- a/internal/gitcache/budget.go +++ b/internal/gitcache/budget.go @@ -2,6 +2,7 @@ package gitcache import ( "context" + "errors" "sync/atomic" "time" ) @@ -29,8 +30,15 @@ type reservation struct { // cut, guarded by c.mu, marks a fetch stopped because the cache could // not make room for it. cut bool + // ended, guarded by c.mu, marks a reservation given back. A read + // relayed after that, by a git helper the fetch left behind, reserves + // nothing. + ended bool } +// errReservationEnded is returned by cover once the fetch has ended. +var errReservationEnded = errors.New("fetch has ended") + func (c *Cache) newReservation() *reservation { return &reservation{c: c} } @@ -48,6 +56,10 @@ func (r *reservation) cover(ctx context.Context, relayed int64) error { var timeout <-chan time.Time for { c.mu.Lock() + if r.ended { + c.mu.Unlock() + return errReservationEnded + } if r.cut { c.mu.Unlock() return ErrCacheFull @@ -113,6 +125,7 @@ func (r *reservation) release() { func (r *reservation) releaseLocked() { c := r.c + r.ended = true n := r.reserved.Swap(0) if r.cut { r.cut = false diff --git a/internal/gitcache/budget_test.go b/internal/gitcache/budget_test.go index 506ef2f..8ef58fa 100644 --- a/internal/gitcache/budget_test.go +++ b/internal/gitcache/budget_test.go @@ -1,10 +1,15 @@ package gitcache import ( + "bytes" "context" "crypto/rand" "errors" + "io" "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" "os" "path/filepath" "sync" @@ -216,3 +221,91 @@ func TestConcurrentClonesStayWithinMaxSize(t *testing.T) { t.Errorf("left %d entries in the temporary directory", len(entries)) } } + +// TestCoverAfterReleaseReservesNothing has a relayed read arrive after the +// fetch gave back its reservation, as one from an orphaned git helper would. +func TestCoverAfterReleaseReservesNothing(t *testing.T) { + tc := newTestCache(t, "", func(o *Options) { o.MaxSize = 100 }) + res := tc.newReservation() + if err := res.cover(t.Context(), 10); err != nil { + t.Fatal(err) + } + res.release() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _ = res.cover(ctx, 20) + if reserved, _ := tc.budget(); reserved != 0 { + t.Errorf("reserved %d after release", reserved) + } +} + +// TestCancelledFetchStopsRelaying cancels a fetch while the upstream is +// midway through sending the pack: the upstream connection is closed, and +// nothing stays reserved when the rest arrives. +func TestCancelledFetchStopsRelaying(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + upstreamURL, err := url.Parse(upstream.server.URL) + if err != nil { + t.Fatal(err) + } + arrived := make(chan struct{}) + release := make(chan struct{}) + result := make(chan string, 1) + forward := httputil.NewSingleHostReverseProxy(upstreamURL) + front := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + r.Body = io.NopCloser(bytes.NewReader(body)) + if r.Method != http.MethodPost || !bytes.Contains(body, []byte("command=fetch")) { + forward.ServeHTTP(w, r) + return + } + rec := httptest.NewRecorder() + forward.ServeHTTP(rec, r.Clone(context.Background())) + pack := rec.Body.Bytes() + w.Header().Set("Content-Type", rec.Header().Get("Content-Type")) + _, _ = w.Write(pack[:len(pack)/2]) + w.(http.Flusher).Flush() + close(arrived) + select { + case <-r.Context().Done(): + result <- "closed" + case <-release: + _, _ = w.Write(pack[len(pack)/2:]) + w.(http.Flusher).Flush() + result <- "sent" + } + })) + defer front.Close() + defer close(release) + tc := newTestCache(t, front.URL, func(o *Options) { o.MaxSize = 1 << 30 }) + + dir := t.TempDir() + if _, err := tc.run(t.Context(), dir, "init", "--quiet", "--bare"); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + res := tc.newReservation() + done := make(chan error, 1) + go func() { done <- tc.fetch(ctx, dir, front.URL+"/repo", 0, res, nil, branchesAndTags...) }() + <-arrived + waitFor(t, func() bool { reserved, _ := tc.budget(); return reserved > 0 }) + cancel() + if err := <-done; err == nil { + t.Error("cancelled fetch succeeded") + } + res.release() + + select { + case <-result: + case <-time.After(10 * time.Second): + t.Error("upstream connection still open after the fetch was cancelled") + release <- struct{}{} + <-result + } + // Give a relay still running the time to read what the upstream sent. + time.Sleep(200 * time.Millisecond) + if reserved, _ := tc.budget(); reserved != 0 { + t.Errorf("reserved %d after the fetch ended", reserved) + } +} diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index 352eb2b..d2f8ee4 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -305,7 +305,7 @@ func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, res return ctx.Err() } - relayURL, route, unregister := c.relay.register(repoURL, limit, res) + relayURL, route, unregister := c.relay.register(ctx, repoURL, limit, res) defer unregister() args := append([]string{"fetch", "--quiet", "--no-write-fetch-head"}, flags...) _, err := c.run(ctx, dir, append(append(args, relayURL), refspecs...)...) diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go index 28f87f9..ed42001 100644 --- a/internal/gitcache/relay.go +++ b/internal/gitcache/relay.go @@ -39,6 +39,9 @@ type relay struct { type relayRoute struct { repoURL string + // ctx ends when the fetch does, which stops its relayed requests, so a + // git helper left behind by a killed git does not keep downloading. + ctx context.Context // limit caps the response bytes the route relays, 0 for no cap, and // budget, when set, holds space within MaxSize for them. exceeded and // full record which of them stopped the route. @@ -79,13 +82,15 @@ func (rl *relay) close() error { return rl.server.Close() } -// register adds a route to repoURL and returns the URL git fetches from and -// a function that removes the route. -func (rl *relay) register(repoURL string, limit int64, budget *reservation) (string, *relayRoute, func()) { +// register adds a route to repoURL for a fetch running with ctx and returns +// the URL git fetches from and a function that removes the route and stops +// its requests. +func (rl *relay) register(ctx context.Context, repoURL string, limit int64, budget *reservation) (string, *relayRoute, func()) { var b [relayIDBytes]byte _, _ = rand.Read(b[:]) id := hex.EncodeToString(b[:]) - route := &relayRoute{repoURL: repoURL, limit: limit, budget: budget} + ctx, cancel := context.WithCancel(ctx) + route := &relayRoute{repoURL: repoURL, ctx: ctx, limit: limit, budget: budget} rl.mu.Lock() rl.routes[id] = route @@ -95,6 +100,7 @@ func (rl *relay) register(repoURL string, limit int64, budget *reservation) (str rl.mu.Lock() delete(rl.routes, id) rl.mu.Unlock() + cancel() } } @@ -119,7 +125,10 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - out, err := http.NewRequestWithContext(r.Context(), r.Method, target, r.Body) + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + defer context.AfterFunc(route.ctx, cancel)() + out, err := http.NewRequestWithContext(ctx, r.Method, target, r.Body) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -133,7 +142,9 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { resp, err := rl.client.Do(out) if err != nil { - rl.logger.Warn("git relay: upstream request failed", "upstream", route.repoURL, "error", err) + if ctx.Err() == nil { + rl.logger.Warn("git relay: upstream request failed", "upstream", route.repoURL, "error", err) + } http.Error(w, "upstream request failed", http.StatusBadGateway) return } @@ -151,12 +162,12 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", ct) } w.WriteHeader(resp.StatusCode) - if _, err := io.Copy(w, &routeReader{ctx: r.Context(), r: resp.Body, route: route}); err != nil { + if _, err := io.Copy(w, &routeReader{ctx: ctx, r: resp.Body, route: route}); err != nil { if r.Context().Err() != nil { // git went away; there is no one to tell. return } - if !route.exceeded.Load() && !route.full.Load() { + if ctx.Err() == nil && !route.exceeded.Load() && !route.full.Load() { rl.logger.Warn("git relay: copying upstream response", "upstream", route.repoURL, "error", err) } // Cut the connection so git fails rather than taking a response diff --git a/internal/gitcache/relay_test.go b/internal/gitcache/relay_test.go index ea1d063..309e727 100644 --- a/internal/gitcache/relay_test.go +++ b/internal/gitcache/relay_test.go @@ -28,7 +28,7 @@ func TestRelayOnlyForwardsFetchRequests(t *testing.T) { })) defer upstream.Close() rl := newTestRelay(t) - base, _, unregister := rl.register(upstream.URL+"/repo", 0, nil) + base, _, unregister := rl.register(t.Context(), upstream.URL+"/repo", 0, nil) refused := []struct{ method, url string }{ {http.MethodGet, rl.base + "/unknown/info/refs?service=git-upload-pack"}, @@ -79,7 +79,7 @@ func TestRelayStopsAtLimit(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = rl.close() }) - base, route, unregister := rl.register(upstream.URL+"/repo", limit, nil) + base, route, unregister := rl.register(t.Context(), upstream.URL+"/repo", limit, nil) defer unregister() resp, err := http.Post(base+"/git-upload-pack", requestType, strings.NewReader(pktFlush)) From 4f7a8b1b089fc5abd5ff4b03ea63d2bac8378ce9 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:01:11 +0100 Subject: [PATCH 124/198] Read Conda timestamps in seconds or milliseconds as conda does The Conda handler divided every repodata timestamp by 1000, assuming milliseconds. Some entries (a few in conda-forge noarch) use seconds, so their publish time came out in 1970 and they were never held back by cooldown. Read a timestamp as milliseconds only when it is above 253402300799, as conda does, in one helper used by the repodata filter, the shard filter and the download index. --- docs/configuration.md | 2 +- internal/handler/conda.go | 22 ++++++++++++++----- internal/handler/conda_test.go | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 93de410..e880230 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1232,7 +1232,7 @@ Maven caveats: Helm uses each release's `created` time from the repository's `index.yaml`. Releases in cooldown are left out of the index the proxy serves, and their archives aren't served; a release without `created` is let through. Charts pulled over OCI have no publish time and aren't held back. -Cargo, RubyGems, Hex, pub.dev, Conda and Helm check artifact downloads against cooldown, including files cached before the policy changed. When a publish-time lookup fails, an already-cached file is still served; an uncached download fails with 502, or 503 with `Retry-After` when the rate-limit gate holds the request. Conda checks timestamps per file because builds of one version can have different publish times. +Cargo, RubyGems, Hex, pub.dev, Conda and Helm check artifact downloads against cooldown, including files cached before the policy changed. When a publish-time lookup fails, an already-cached file is still served; an uncached download fails with 502, or 503 with `Retry-After` when the rate-limit gate holds the request. Conda checks timestamps per file because builds of one version can have different publish times. As in conda, a repodata `timestamp` is read as milliseconds when it is above 253402300799 (the end of year 9999 in seconds) and as seconds otherwise, since some older entries use seconds. Conda repodata and Helm indexes are parsed once per cached document for download checks. With `cache_metadata` off, the parsed index is kept for one minute, including the fetch, so an install can reuse it across files. Conda streams repodata when its compressed copy exceeds `metadata_max_size`, under the same decompressed size bound as the metadata filter. diff --git a/internal/handler/conda.go b/internal/handler/conda.go index 1c14980..f593820 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -432,8 +432,18 @@ func (c *countingWriter) Write(p []byte) (int, error) { return n, err } -// condaTimestampDivisor converts Conda's millisecond timestamps to seconds. -const condaTimestampDivisor = 1000 +// condaMaxSecondsTimestamp is the last second of year 9999. Like conda, a +// larger repodata timestamp is read as milliseconds and a smaller one as +// seconds, which some older entries use. +const condaMaxSecondsTimestamp = 253402300799 + +// condaTime returns the time a repodata timestamp names. +func condaTime(timestamp float64) time.Time { + if timestamp > condaMaxSecondsTimestamp { + return time.UnixMilli(int64(timestamp)).Truncate(time.Second) + } + return time.Unix(int64(timestamp), 0) +} const ( repodataWriteBuffer = 64 << 10 @@ -503,7 +513,7 @@ func (f *repodataFilter) inCooldown(filename string, raw []byte) bool { if entry.Name == "" || entry.Timestamp == 0 { return false } - publishedAt := time.Unix(int64(entry.Timestamp)/condaTimestampDivisor, 0) + publishedAt := condaTime(entry.Timestamp) decision := f.h.proxy.cooldown().Evaluate("conda", canonicalPackagePURL("conda", entry.Name), publishedAt, f.now) if decision.Allowed { return false @@ -520,14 +530,14 @@ func (f *repodataFilter) inCooldown(filename string, raw []byte) bool { return true } -// recordInCooldown reports whether a package record with a millisecond +// recordInCooldown reports whether a package record with a repodata // timestamp was published too recently. Records without a name or // timestamp are kept. func (h *CondaHandler) recordInCooldown(filename, name, version string, timestamp float64) bool { if name == "" || timestamp == 0 { return false } - publishedAt := time.Unix(int64(timestamp)/condaTimestampDivisor, 0) + publishedAt := condaTime(timestamp) if h.proxy.cooldown().IsAllowed("conda", canonicalPackagePURL("conda", name), publishedAt) { return false } @@ -620,7 +630,7 @@ func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { } var e repodataEntry if json.Unmarshal(raw, &e) == nil && e.Timestamp != 0 { - index[entry.Key] = time.Unix(int64(e.Timestamp)/condaTimestampDivisor, 0) + index[entry.Key] = condaTime(e.Timestamp) } return nil }) diff --git a/internal/handler/conda_test.go b/internal/handler/conda_test.go index 0c4c52a..63e518e 100644 --- a/internal/handler/conda_test.go +++ b/internal/handler/conda_test.go @@ -3,9 +3,13 @@ package handler import ( "bytes" "encoding/json" + "fmt" "log/slog" + "maps" "net/http" "net/http/httptest" + "slices" + "strings" "testing" "time" @@ -133,6 +137,42 @@ func TestCondaCooldownFiltering(t *testing.T) { } } +// TestCondaCooldownSecondTimestamps checks that a timestamp in seconds, +// which some repodata entries carry, is read as seconds, as conda does, in +// both the filter and the download index. +func TestCondaCooldownSecondTimestamps(t *testing.T) { + recent := time.Now().Add(-time.Hour) + body := fmt.Sprintf(`{"packages":{"seconds-1.0-0.tar.bz2":{"name":"seconds","version":"1.0","timestamp":%d},`+ + `"millis-1.0-0.tar.bz2":{"name":"millis","version":"1.0","timestamp":%d}}}`, recent.Unix(), recent.UnixMilli()) + + proxy := testProxy() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + h := &CondaHandler{proxy: proxy, proxyURL: "http://localhost:8080"} + filtered, err := filterRepodataBytes(h, []byte(body)) + if err != nil { + t.Fatal(err) + } + var result struct { + Packages map[string]json.RawMessage `json:"packages"` + } + if err := json.Unmarshal(filtered, &result); err != nil { + t.Fatal(err) + } + if len(result.Packages) != 0 { + t.Errorf("packages = %v, want both left out", slices.Sorted(maps.Keys(result.Packages))) + } + + index, err := readRepodataIndex(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + for filename, published := range index { + if published.Unix() != recent.Unix() { + t.Errorf("%s published at %v, want %v", filename, published, recent) + } + } +} + func TestCondaCooldownFilteringWithPackageOverride(t *testing.T) { now := time.Now() recentTimestamp := float64(now.Add(-2 * time.Hour).UnixMilli()) From a288afab3283c61277841ae8ce39abe305eef39e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:04:09 +0100 Subject: [PATCH 125/198] Match denylist entries for distro packages as handlers name them The denylist kept the PURL namespace, but the Debian and RPM handlers build pkg:deb/{name} and pkg:rpm/{name} without a vendor, while Alpine and Arch build pkg:apk/alpine/{name} and pkg:alpm/arch/{name} with one. So standard entries such as pkg:deb/debian/curl@... or pkg:rpm/fedora/curl@..., and pkg:apk/busybox@... or pkg:alpm/glibc@..., never matched. Drop the namespace of distro types on both sides, as retention and the inventory already do, using packageurl's list of those types. Debian and RPM handlers read versions from filenames, which leave out the epoch, so an entry such as pkg:deb/foo@1:2.3-1 never matched foo_2.3-1_amd64.deb. An entry with an epoch now also covers the version without it. --- docs/configuration.md | 7 ++++++ internal/denylist/denylist.go | 16 +++++++++++++- internal/denylist/denylist_test.go | 35 ++++++++++++++++++++++++++++++ internal/packageurl/handler.go | 6 +++++ 4 files changed, 63 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index e880230..5acbab9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1270,6 +1270,13 @@ Names and versions are compared the way each ecosystem treats them: - PyPI names ignore case, and runs of `-`, `_` and `.` are equivalent (PEP 503), so `pkg:pypi/zope.interface@6.0` also covers `zope_interface-6.0-*.whl`. - NuGet ids and versions ignore case. +- Debian, RPM, Alpine, Arch and FreeBSD entries ignore the distribution + namespace: `pkg:deb/debian/curl@8.5.0-2` and `pkg:deb/curl@8.5.0-2` are the + same entry. +- Debian and RPM filenames leave out the epoch, so an entry with one also + covers the version without it: `pkg:deb/foo@1:2.3-1` denies + `foo_2.3-1_amd64.deb`. An entry without an epoch only matches a version + without one. - A RubyGems entry also covers that version's platform gems: `pkg:gem/nokogiri@1.16.0` denies `nokogiri-1.16.0-x86_64-linux.gem`. - Composer versions ignore a leading `v`, which Packagist keeps from the tag: diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go index 76f2de5..eea8ee3 100644 --- a/internal/denylist/denylist.go +++ b/internal/denylist/denylist.go @@ -40,6 +40,13 @@ func New(packages []string) (*Policy, error) { p.packages[key] = make(map[string]bool) } p.packages[key][id.version] = true + // Debian and RPM filenames leave the epoch out, so an entry with + // one also matches the version without it. + if pkg.Type == "deb" || pkg.Type == "rpm" { + if _, version, ok := strings.Cut(id.version, ":"); ok && version != "" { + p.packages[key][version] = true + } + } } return p, nil } @@ -128,6 +135,9 @@ func genericSwiftRegistry(pkg *purl.PURL) (scope, name string, ok bool) { // identify compares PURLs in purl's comparison form. The generic // swift-registry spelling is the proxy's own, so it is mapped here first. +// The namespace of a distro type names the distribution, which handlers +// include for some types and not others, so it is dropped as retention and +// the inventory do. func identify(pkg *purl.PURL) identity { if scope, name, ok := genericSwiftRegistry(pkg); ok { pkg = purl.New("swift", scope, name, pkg.Version, nil) @@ -138,6 +148,10 @@ func identify(pkg *purl.PURL) identity { // An entry without a channel matches every channel. channel = c.Qualifier("channel") } - key := purl.New(c.Type, c.Namespace, c.Name, "", nil).String() + namespace := c.Namespace + if packageurl.DistroType(c.Type) { + namespace = "" + } + key := purl.New(c.Type, namespace, c.Name, "", nil).String() return identity{key: packageurl.PolicyPackageKey(key), channel: channel, version: c.Version} } diff --git a/internal/denylist/denylist_test.go b/internal/denylist/denylist_test.go index a110608..03edc99 100644 --- a/internal/denylist/denylist_test.go +++ b/internal/denylist/denylist_test.go @@ -90,3 +90,38 @@ func TestEcosystemSpellings(t *testing.T) { t.Error("pypi versions not found under another spelling") } } + +// TestDistroNamespaces checks that the distribution namespace of a distro +// package is ignored on both sides, as handlers build some of these PURLs +// without one and some with one. +func TestDistroNamespaces(t *testing.T) { + p, err := New([]string{"pkg:deb/debian/curl@8.5.0-2", "pkg:rpm/fedora/curl@8.6.0-1.fc40", "pkg:apk/busybox@1.36.1-r15", "pkg:alpm/glibc@2.39-1"}) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"pkg:deb/curl@8.5.0-2", "pkg:rpm/curl@8.6.0-1.fc40", "pkg:apk/alpine/busybox@1.36.1-r15", "pkg:alpm/arch/glibc@2.39-1"} { + if !p.Denied(value) { + t.Errorf("not denied: %s", value) + } + } + if p.Denied("pkg:deb/curl@8.5.0-3") { + t.Error("other version denied") + } +} + +// TestDistroEpochs checks that a deb or rpm entry with an epoch matches the +// version as it appears in a filename, which leaves the epoch out. +func TestDistroEpochs(t *testing.T) { + p, err := New([]string{"pkg:deb/foo@1:2.3-1", "pkg:rpm/bar@2:1.0-1.fc40"}) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"pkg:deb/foo@2.3-1", "pkg:deb/foo@1:2.3-1", "pkg:rpm/bar@1.0-1.fc40"} { + if !p.Denied(value) { + t.Errorf("not denied: %s", value) + } + } + if p.Denied("pkg:deb/foo@2.3-2") { + t.Error("other version denied") + } +} diff --git a/internal/packageurl/handler.go b/internal/packageurl/handler.go index ee3005e..5edcf80 100644 --- a/internal/packageurl/handler.go +++ b/internal/packageurl/handler.go @@ -8,6 +8,12 @@ import ( // ("alpine", "debian"), not part of the package name the handlers store. var distroTypes = map[string]bool{"alpm": true, "apk": true, "bsd": true, "deb": true, "rpm": true} +// DistroType reports whether a PURL type's namespace names the distribution, +// so that handlers ignore it. +func DistroType(purlType string) bool { + return distroTypes[purlType] +} + // HandlerEcosystem returns the ecosystem name the proxy's handlers store // packages of a PURL type under. purl names gem and composer after their // registries, rubygems and packagist; the handlers use the type. From 92b916d8dbc4340c37f62f13f7b8ef0dd3b5c752 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:05:21 +0100 Subject: [PATCH 126/198] Serve APT and RPM index files that match the cached release file InRelease/Release and repomd.xml had their own metadata cache entries and TTLs, independent of the Packages, Sources or plainly named primary.xml.gz files they list. Within a TTL a client could get the cached release file and a freshly fetched index, or a fresh release file and a cached index, and apt then failed with "Hash Sum mismatch" (repositories without by-hash, or RPM repositories built with --simple-md-filenames). An index file that the cached release file lists is now served by that listing: the cached copy while its digest matches the listed sha256, whatever its age, and otherwise a copy fetched from upstream. If upstream's copy does not match either, the repository has moved on since the release file was cached; that copy is served and the release file is expired so the next update gets a matching pair. --- docs/configuration.md | 4 +- internal/handler/debian.go | 26 +++- internal/handler/metadata_stream.go | 20 ++- internal/handler/release_index.go | 130 ++++++++++++++++++++ internal/handler/release_index_test.go | 164 +++++++++++++++++++++++++ internal/handler/rpm.go | 17 ++- 6 files changed, 353 insertions(+), 8 deletions(-) create mode 100644 internal/handler/release_index.go create mode 100644 internal/handler/release_index_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 1fdf008..457efb7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1479,7 +1479,7 @@ cache_metadata: true Or via environment variable: `PROXY_CACHE_METADATA=true`. -Some metadata never changes once published, and is cached like an artifact, with or without `cache_metadata`, and served without revalidation whatever `metadata_ttl` says: a Go module's `.mod` and `.info` for a canonical version (a release, pre-release, pseudo-version or `+incompatible` version; the checksum database fixes their content), and RPM repodata files named after their own checksum (`repodata/-primary.xml.gz`). A stored copy is checked against the digest recorded for it and fetched again if it doesn't match. Go branch and commit queries, `@v/list` and `@latest`, and RPM `repomd.xml` and plainly named repodata files follow `metadata_ttl` as before. A checksum-named repodata file over `metadata_max_size` is streamed from upstream uncached. +Some metadata never changes once published, and is cached like an artifact, with or without `cache_metadata`, and served without revalidation whatever `metadata_ttl` says: a Go module's `.mod` and `.info` for a canonical version (a release, pre-release, pseudo-version or `+incompatible` version; the checksum database fixes their content), and RPM repodata files named after their own checksum (`repodata/-primary.xml.gz`). A stored copy is checked against the digest recorded for it and fetched again if it doesn't match. Go branch and commit queries, `@v/list` and `@latest`, and RPM `repomd.xml` follow `metadata_ttl` as before; plainly named repodata files follow the cached `repomd.xml` (see [Metadata TTL](#metadata-ttl)). A checksum-named repodata file over `metadata_max_size` is streamed from upstream uncached. Composer archive downloads look up their upstream URL in the same cached metadata, so an archive is resolved without asking Packagist again and stays installable while it is down. Only a version or file the cached metadata does not list yet is looked up upstream. With or without `cache_metadata`, a cached archive of a tagged release from the default repository is served without any metadata lookup; with cooldown enabled it is checked against the publish time recorded when the archive was first resolved. Branch (dev) versions, routed packages and archives picked by a query string such as `?sha=` are always resolved from the metadata. @@ -1501,6 +1501,8 @@ Set to `"0"` to always revalidate with upstream (ETag-based conditional requests When upstream is unreachable and the cached entry is past its TTL (with a `metadata_ttl` of `"0"`, on every such request), the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. The `ETag` and `Last-Modified` sent with a document are always those of the bytes served, so a document fetched from upstream but not stored, for example because storage failed, carries the upstream's validators rather than the older cached copy's. +APT and RPM clients fetch a release file first (`InRelease` or `Release`, `repomd.xml`) and reject an index file whose checksum doesn't match it. So an index file the cached release file lists (`Packages`, `Sources`, a plainly named `primary.xml.gz`, …) does not follow its own `metadata_ttl`: its cached copy is served while it matches the cached release file's sha256, however old, and otherwise it is fetched from upstream. If upstream's copy doesn't match either, the repository has changed since the release file was cached; that copy is served and the release file is expired, so the next `apt update` or `dnf makecache` gets a matching pair. The release file itself follows `metadata_ttl`. Index files it doesn't list, such as `by-hash` paths, follow `metadata_ttl` too. + ### Change feeds Within `metadata_ttl` a new release isn't seen. For npm and Composer the proxy can instead follow the registry's change feed, which lists the packages that changed, and revalidate just those: diff --git a/internal/handler/debian.go b/internal/handler/debian.go index b2419f6..af528e3 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "net/http" + pathpkg "path" "regexp" "sort" "strings" @@ -183,13 +184,36 @@ func (h *DebianHandler) handlePackageDownload( // handleMetadata serves repository metadata files through the metadata cache, // so they keep resolving while the upstream archive is unreachable. +// Index files the cached InRelease or Release lists follow that listing (see +// proxyListedIndex). func (h *DebianHandler) handleMetadata( w http.ResponseWriter, r *http.Request, repository, upstreamURL, path string, ) { cacheKey := h.metadataCacheKeyFor(repository, upstreamURL, path) - h.proxy.ProxyCached(w, r, fmt.Sprintf("%s/%s", upstreamURL, path), "debian", cacheKey, "*/*") + fileURL := fmt.Sprintf("%s/%s", upstreamURL, path) + switch pathpkg.Base(path) { + case "InRelease", "Release", "Release.gpg": + h.proxy.ProxyCached(w, r, fileURL, "debian", cacheKey, "*/*") + return + } + release, want := h.releaseListing(r, repository, upstreamURL, path) + h.proxy.proxyListedIndex(w, r, fileURL, "debian", cacheKey, release, want) +} + +// releaseListing finds the cached release file of the suite path belongs +// to, nearest directory first, and the sha256 it lists for path. +func (h *DebianHandler) releaseListing(r *http.Request, repository, upstreamURL, path string) (*cachedRelease, string) { + for dir := pathpkg.Dir(path); strings.HasPrefix(dir, "dists/"); dir = pathpkg.Dir(dir) { + for _, name := range []string{"InRelease", "Release"} { + release := h.proxy.readCachedRelease(r.Context(), "debian", h.metadataCacheKeyFor(repository, upstreamURL, dir+"/"+name)) + if release != nil { + return release, debianReleaseSHA256(release.body, strings.TrimPrefix(path, dir+"/")) + } + } + } + return nil, "" } // artifactCacheFilename scopes a package by repository, since the same diff --git a/internal/handler/metadata_stream.go b/internal/handler/metadata_stream.go index 56606e2..c165b81 100644 --- a/internal/handler/metadata_stream.go +++ b/internal/handler/metadata_stream.go @@ -35,6 +35,12 @@ func (p *Proxy) serveStoredMetadata(w http.ResponseWriter, r *http.Request, ecos if entry == nil { return false } + return p.serveMetadataEntry(w, r, ecosystem, cacheKey, entry, immutable) +} + +// serveMetadataEntry is serveStoredMetadata for entry, a row +// storedMetadataEntry or streamableMetadata accepted. +func (p *Proxy) serveMetadataEntry(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, entry *database.MetadataCacheEntry, immutable bool) bool { body, err := p.openMetadataEntry(r.Context(), ecosystem, cacheKey, entry) if err != nil { return false @@ -88,14 +94,20 @@ func (p *Proxy) storedMetadataEntry(ecosystem, cacheKey string, immutable bool) if !immutable && time.Since(entry.FetchedAt.Time) >= p.MetadataTTL { return nil } + if !p.streamableMetadata(entry) { + return nil + } + return entry +} + +// streamableMetadata reports whether entry records a sha256 digest and a +// size within the metadata size limit, so its document can be streamed. +func (p *Proxy) streamableMetadata(entry *database.MetadataCacheEntry) bool { limit := p.MetadataMaxSize if limit <= 0 { limit = defaultMetadataMaxSize } - if !entry.ContentDigest.Valid || !strings.HasPrefix(entry.ContentDigest.String, "sha256:") || !entry.Size.Valid || entry.Size.Int64 > limit { - return nil - } - return entry + return entry.ContentDigest.Valid && strings.HasPrefix(entry.ContentDigest.String, "sha256:") && entry.Size.Valid && entry.Size.Int64 <= limit } // openMetadataEntry opens the stored document of a row freshMetadataEntry diff --git a/internal/handler/release_index.go b/internal/handler/release_index.go new file mode 100644 index 0000000..25f3515 --- /dev/null +++ b/internal/handler/release_index.go @@ -0,0 +1,130 @@ +package handler + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "net/http" + "strings" + "time" +) + +// APT and RPM repositories have a release file (InRelease, Release, +// repomd.xml) that lists the sha256 of each index file. Clients fetch the +// release file first and reject an index that doesn't match it. Each file +// has its own metadata cache entry and TTL, so without care a client could +// get a cached release file and a freshly fetched index, or the reverse. +// +// An index the cached release file lists is therefore served by that +// listing rather than by its own TTL: the cached copy when it matches, +// whatever its age, and otherwise a copy fetched from upstream. If upstream +// only has an index the cached release file doesn't list, upstream has moved +// on; that index is served and the release file is expired, so the next +// update gets a matching pair. + +// cachedRelease is a release file in the metadata cache. +type cachedRelease struct { + ecosystem string + cacheKey string + body []byte +} + +// readCachedRelease returns the cached release file stored under cacheKey, +// or nil. +func (p *Proxy) readCachedRelease(ctx context.Context, ecosystem, cacheKey string) *cachedRelease { + if !p.CacheMetadata || p.DB == nil || p.Storage == nil { + return nil + } + entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) + if err != nil || entry == nil || entry.ContentEncoding.String != "" { + return nil + } + body, _, err := p.readCachedMetadata(ctx, entry, nil) + if err != nil { + return nil + } + return &cachedRelease{ecosystem: ecosystem, cacheKey: cacheKey, body: body} +} + +// proxyListedIndex serves an index file that release lists with the sha256 +// hex want. Without a listing it is served as other metadata. +func (p *Proxy) proxyListedIndex(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey string, release *cachedRelease, want string) { + if release == nil || want == "" { + p.ProxyCached(w, r, upstreamURL, ecosystem, cacheKey, "*/*") + return + } + if entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey); entry != nil && p.streamableMetadata(entry) && + entry.ContentDigest.String == "sha256:"+want && entry.ContentEncoding.String == "" { + if p.serveMetadataEntry(w, r, ecosystem, cacheKey, entry, true) { + return + } + } + p.recordMetadataCacheMiss(ecosystem) + res := p.refreshMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, "*/*", "identity", nil) + if res.err != nil { + p.writeMetadataFetchError(w, res.err) + return + } + if sum := sha256.Sum256(res.body); !res.stale && hex.EncodeToString(sum[:]) != want { + p.Logger.Info("index does not match the cached release file, expiring it", + "ecosystem", ecosystem, "url", upstreamURL) + if _, err := p.DB.ExpireMetadataCache(ecosystem, []string{release.cacheKey}, time.Now()); err != nil { + p.Logger.Warn("failed to expire cached release file", "ecosystem", ecosystem, "error", err) + } + } + writeMetadataResult(w, r, res) +} + +// releaseLineMax bounds a line of a Release file. +const releaseLineMax = 1 << 20 + +// debianReleaseSHA256 returns the sha256 an APT Release or InRelease file +// lists for rel, a path relative to the release file's directory, or "". +func debianReleaseSHA256(release []byte, rel string) string { + inSection := false + scanner := bufio.NewScanner(bytes.NewReader(release)) + scanner.Buffer(nil, releaseLineMax) + for scanner.Scan() { + line := strings.TrimRight(scanner.Text(), "\r") + if !strings.HasPrefix(line, " ") { + inSection = line == "SHA256:" + continue + } + if !inSection { + continue + } + if f := strings.Fields(line); len(f) == 3 && f[2] == rel { + return strings.ToLower(f[0]) + } + } + return "" +} + +// repomdSHA256 returns the sha256 an RPM repomd.xml lists for href, a path +// relative to the repository root, or "" when it lists none or another +// checksum type. +func repomdSHA256(repomd []byte, href string) string { + var doc struct { + Data []struct { + Checksum struct { + Type string `xml:"type,attr"` + Value string `xml:",chardata"` + } `xml:"checksum"` + Location struct { + Href string `xml:"href,attr"` + } `xml:"location"` + } `xml:"data"` + } + if xml.Unmarshal(repomd, &doc) != nil { + return "" + } + for _, d := range doc.Data { + if d.Location.Href == href && d.Checksum.Type == "sha256" { + return strings.ToLower(strings.TrimSpace(d.Checksum.Value)) + } + } + return "" +} diff --git a/internal/handler/release_index_test.go b/internal/handler/release_index_test.go new file mode 100644 index 0000000..ccbc62a --- /dev/null +++ b/internal/handler/release_index_test.go @@ -0,0 +1,164 @@ +package handler + +import ( + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +// releaseArchive is an upstream repository whose release file and index +// change together, one generation at a time. +type releaseArchive struct { + gen atomic.Int32 + // releasePath and indexPath are the request paths, release renders the + // release file listing index's sha256 for a generation. + releasePath, indexPath string + release func(indexSHA256 string, size int) string +} + +func indexBody(gen int32) string { return fmt.Sprintf("index generation %d\n", gen) } + +func (a *releaseArchive) serve(w http.ResponseWriter, r *http.Request) { + gen := a.gen.Load() + index := indexBody(gen) + switch r.URL.Path { + case "/" + a.releasePath: + _, _ = w.Write([]byte(a.release(sha256Hex(index), len(index)))) + case "/" + a.indexPath: + _, _ = w.Write([]byte(index)) + default: + http.NotFound(w, r) + } +} + +type releaseIndexCase struct { + name string + ecosystem string + archive *releaseArchive + handler func(p *Proxy, upstreamURL string) (http.Handler, func(path string) string) +} + +func releaseIndexCases() []releaseIndexCase { + return []releaseIndexCase{ + { + name: "debian InRelease", + ecosystem: "debian", + archive: &releaseArchive{ + releasePath: "dists/stable/InRelease", + indexPath: "dists/stable/main/binary-amd64/Packages", + release: func(sum string, size int) string { + return "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\nSuite: stable\nMD5Sum:\n 00000000000000000000000000000000 1 main/binary-amd64/Packages\nSHA256:\n" + + fmt.Sprintf(" %s %d main/binary-amd64/Packages\n", sum, size) + + "-----BEGIN PGP SIGNATURE-----\n\nxx\n-----END PGP SIGNATURE-----\n" + }, + }, + handler: func(p *Proxy, upstreamURL string) (http.Handler, func(string) string) { + h := NewDebianHandler(p, "http://proxy.test", upstreamURL, nil) + return h.Routes(), func(path string) string { return h.metadataCacheKeyFor("", h.upstreamURL, path) } + }, + }, + { + name: "rpm repomd.xml", + ecosystem: "rpm", + archive: &releaseArchive{ + releasePath: "f/39/os/repodata/repomd.xml", + indexPath: "f/39/os/repodata/primary.xml.gz", + release: func(sum string, _ int) string { + return ` +1 +` + sum + ` +` + }, + }, + handler: func(p *Proxy, upstreamURL string) (http.Handler, func(string) string) { + h := NewRPMHandlerWithUpstream(p, "http://proxy.test", upstreamURL) + return h.Routes(), h.metadataCacheKey + }, + }, + } +} + +func getBody(t *testing.T, h http.Handler, path string) string { + t.Helper() + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/"+path, nil)) + if w.Code != http.StatusOK { + t.Fatalf("GET %s = %d %s", path, w.Code, w.Body) + } + return w.Body.String() +} + +// TestIndexFollowsCachedRelease checks that the index files a release file +// lists are served as that release file lists them, whatever the index's own +// cache entry says. +func TestIndexFollowsCachedRelease(t *testing.T) { + for _, tc := range releaseIndexCases() { + t.Run(tc.name, func(t *testing.T) { + a := tc.archive + upstream := httptest.NewServer(http.HandlerFunc(a.serve)) + t.Cleanup(upstream.Close) + proxy, db, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + h, key := tc.handler(proxy, upstream.URL) + expire := func(path string) { + t.Helper() + if _, err := db.ExpireMetadataCache(tc.ecosystem, []string{key(path)}, time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + } + + getBody(t, h, a.releasePath) + if got := getBody(t, h, a.indexPath); got != indexBody(0) { + t.Fatalf("index = %q", got) + } + + // The index entry expires before the release file's while + // upstream moves on: the cached release still lists generation 0. + a.gen.Store(1) + expire(a.indexPath) + if got := getBody(t, h, a.indexPath); got != indexBody(0) { + t.Errorf("index after its entry expired = %q, want the one the cached release lists", got) + } + + // The release file is refetched while the index is still cached: + // the index must follow it. + expire(a.releasePath) + getBody(t, h, a.releasePath) + if got := getBody(t, h, a.indexPath); got != indexBody(1) { + t.Errorf("index after release refetch = %q, want %q", got, indexBody(1)) + } + }) + } +} + +// TestIndexMismatchExpiresRelease checks that when upstream only has an +// index newer than the cached release file, the release file is refetched +// on its next request, so the next update gets a matching pair. +func TestIndexMismatchExpiresRelease(t *testing.T) { + for _, tc := range releaseIndexCases() { + t.Run(tc.name, func(t *testing.T) { + a := tc.archive + upstream := httptest.NewServer(http.HandlerFunc(a.serve)) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + h, _ := tc.handler(proxy, upstream.URL) + + first := getBody(t, h, a.releasePath) + a.gen.Store(1) + if got := getBody(t, h, a.indexPath); got != indexBody(1) { + t.Fatalf("index = %q", got) + } + if got := getBody(t, h, a.releasePath); got == first { + t.Errorf("release file still the cached one, which does not list the index served") + } + }) + } +} diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 95163b9..02cde69 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -122,14 +122,27 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques // handleMetadata proxies repository metadata files (repomd.xml, // primary.xml.gz, etc.). repomd.xml changes with every repository update and // follows the metadata TTL. The files it points at are usually named after -// their own checksum and so never change; those are kept for good. +// their own checksum and so never change; those are kept for good. Plainly +// named ones follow the cached repomd.xml (see proxyListedIndex). func (h *RPMHandler) handleMetadata(w http.ResponseWriter, r *http.Request, path string) { upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, path) if rpmContentAddressed(path) { h.proxy.proxyImmutable(w, r, upstreamURL, "rpm", h.metadataCacheKey(path), "*/*") return } - h.proxy.ProxyCached(w, r, upstreamURL, "rpm", h.metadataCacheKey(path), "*/*") + i := strings.LastIndex(path, "/repodata/") + if strings.HasPrefix(path[i:], "/repodata/repomd.xml") { + h.proxy.ProxyCached(w, r, upstreamURL, "rpm", h.metadataCacheKey(path), "*/*") + return + } + // Plainly named files (createrepo --simple-md-filenames) follow the + // cached repomd.xml that lists them. + repomd := h.proxy.readCachedRelease(r.Context(), "rpm", h.metadataCacheKey(path[:i]+"/repodata/repomd.xml")) + var want string + if repomd != nil { + want = repomdSHA256(repomd.body, path[i+1:]) + } + h.proxy.proxyListedIndex(w, r, upstreamURL, "rpm", h.metadataCacheKey(path), repomd, want) } // rpmChecksumPrefix matches a repodata filename that createrepo prefixed From 682b4e1291ce638525cc91fcb045a94ffbfe55f8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:10:16 +0100 Subject: [PATCH 127/198] Keep Maven metadata checksums and SNAPSHOT files current Without cooldown, maven-metadata.xml and its .md5/.sha1/.sha256/.sha512 files were cached under separate keys with separate TTLs, so a deploy between the two fetches left the checksum describing another document and strict builds failed. The checksum files are now computed from the cached metadata, as the cooldown path already did. The metadata is also written with the validators of the document served. SNAPSHOT files named with SNAPSHOT rather than a build timestamp (lib-1.0-SNAPSHOT.jar, as JitPack and non-unique snapshot repositories publish them) matched the artifact rules and were cached for good, so the first build was served forever. They are now passed through uncached. Timestamped SNAPSHOT files never change and stay cached. --- docs/configuration.md | 11 ++- internal/handler/download_test.go | 17 +++- internal/handler/handler.go | 17 ++-- internal/handler/maven.go | 57 ++++++++++--- internal/handler/maven_consistency_test.go | 97 ++++++++++++++++++++++ internal/handler/maven_cooldown.go | 9 +- 6 files changed, 179 insertions(+), 29 deletions(-) create mode 100644 internal/handler/maven_consistency_test.go diff --git a/docs/configuration.md b/docs/configuration.md index 457efb7..0360d3a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1021,6 +1021,15 @@ A repository's files are cached under its name and a hash of its upstream URL, so pointing a repository at a different URL stops serving what the old one sent. +In every Maven repository, `maven-metadata.xml` follows `metadata_ttl`, and +its `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from the +cached metadata rather than fetched, so they always match it. Other files are +cached for good, except SNAPSHOT files named with `SNAPSHOT` rather than a +build timestamp (`lib-1.0-SNAPSHOT.jar`, as JitPack and non-unique snapshot +repositories publish them), which each build replaces: those are passed +through uncached. Timestamped SNAPSHOT files +(`lib-1.0-20240101.120000-1.jar`) never change and are cached. + This field has no environment variable, as with the other named upstream maps. `PROXY_UPSTREAM_MAVEN` still sets `upstream.maven`. @@ -1221,7 +1230,7 @@ Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyG Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. -Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` or Gradle `.module` itself is downloaded, which clients do before a version's other files, and otherwise with a `HEAD` request. A `.pom` or `.module` refused this way is still cached, so it counts toward `max_size` before the version leaves cooldown. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. +Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` or Gradle `.module` itself is downloaded, which clients do before a version's other files, and otherwise with a `HEAD` request. A `.pom` or `.module` refused this way is still cached, so it counts toward `max_size` before the version leaves cooldown. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served, as they are without cooldown. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. Maven caveats: diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index 216a279..c53585e 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -1,7 +1,9 @@ package handler import ( + "crypto/sha1" "database/sql" + "encoding/hex" "fmt" "io" "net/http" @@ -1074,8 +1076,7 @@ func TestMavenHandler_GradlePluginMarkerFallbackAndCache(t *testing.T) { func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) { paths := map[string]string{ - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "", - "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml.sha1": "sha1", + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "", } primaryHits := map[string]int{} @@ -1131,6 +1132,18 @@ func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) { t.Fatalf("GET %s did not hit plugin portal fallback", reqPath) } } + + // The checksum is computed from the metadata served from the fallback. + resp, err := http.Get(srv.URL + "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml.sha1") + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + sum := sha1.Sum([]byte("")) //nolint:gosec // Maven publishes SHA-1 checksums. + if resp.StatusCode != http.StatusOK || string(body) != hex.EncodeToString(sum[:]) { + t.Fatalf("GET maven-metadata.xml.sha1: %d %q, want %x", resp.StatusCode, body, sum) + } } func TestMavenHandler_GradlePluginChecksumFallbackToPluginPortal(t *testing.T) { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index c70dea1..0b84f0d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2321,17 +2321,12 @@ func (p *Proxy) writeMetadataFetchError(w http.ResponseWriter, err error) { http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) } -// writeMetadataCachedResponse writes a cached metadata response and handles -// conditional request headers using metadata cache validators. -func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType string) { - p.writeMetadataCachedResponseWithEncoding(w, r, ecosystem, cacheKey, body, contentType, "") -} - -// writeMetadataCachedResponseWithEncoding is writeMetadataCachedResponse with -// an explicit Content-Encoding. contentEncoding must describe the body being -// written; it is passed in rather than re-read from the cache row, which is -// missing or stale when the metadata cache write failed and would otherwise -// mislabel the bytes. +// writeMetadataCachedResponseWithEncoding writes a metadata document with the +// validators of its current cache row. Prefer writeMetadataResult, which +// takes them from the lookup the body came from. contentEncoding must +// describe the body being written; it is passed in rather than re-read from +// the cache row, which is missing or stale when the metadata cache write +// failed and would otherwise mislabel the bytes. func (p *Proxy) writeMetadataCachedResponseWithEncoding(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType, contentEncoding string) { writeMetadataBody(w, r, p.lookupCachedMeta(ecosystem, cacheKey), body, contentType, contentEncoding) } diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 6c6a647..30a6976 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "io" "net/http" "path" "strings" @@ -136,6 +137,12 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) { return } + if h.isMutableSnapshotFile(urlPath) { + // Rebuilt in place, so not cached. + h.proxyUpstream(w, r, source, urlPath) + return + } + if h.isArtifactFile(filename) || h.isChecksumFile(filename) || h.isCoordinateFile(urlPath) { h.handleDownload(w, r, source, urlPath) return @@ -175,32 +182,51 @@ func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, so return } - body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, urlPath) - if err != nil { - h.writeMetadataError(w, err) + // The checksums of maven-metadata.xml are computed from the cached + // metadata, as with cooldown, so they never describe another deploy. + if suffix, ok := strings.CutPrefix(path.Base(urlPath), mavenMetadataFile); ok && mavenChecksums[suffix] != nil { + res, _ := h.fetchMetadata(r.Context(), source, path.Join(path.Dir(urlPath), mavenMetadataFile)) + var body []byte + if res.err == nil { + body, res.err = h.proxy.decodedMetadataBody(res) + } + if res.err != nil { + h.writeMetadataError(w, res.err) + return + } + sum := mavenChecksums[suffix]() + _, _ = sum.Write(body) + w.Header().Set(headerContentType, "text/plain") + markStale(w, res.stale) + _, _ = io.WriteString(w, hex.EncodeToString(sum.Sum(nil))) return } - h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType) + res, _ := h.fetchMetadata(r.Context(), source, urlPath) + if res.err != nil { + h.writeMetadataError(w, res.err) + return + } + writeMetadataResult(w, r, res) } // fetchMetadata fetches a metadata file from the source's upstream, or from // its fallback when the upstream doesn't have it, and returns it with the // key it is cached under. -func (h *MavenHandler) fetchMetadata(ctx context.Context, source mavenSource, urlPath string) (body []byte, contentType, cacheKey string, err error) { - cacheKey = source.metadataCacheKey(urlPath) +func (h *MavenHandler) fetchMetadata(ctx context.Context, source mavenSource, urlPath string) (metadataResult, string) { + cacheKey := source.metadataCacheKey(urlPath) upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) - body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, upstreamURL, "*/*") - if err != nil && source.fallbackURL != "" { - if errors.Is(err, ErrUpstreamNotFound) { + res := h.proxy.lookupMetadata(ctx, "maven", cacheKey, upstreamURL, "*/*", "", nil) + if res.err != nil && source.fallbackURL != "" { + if errors.Is(res.err, ErrUpstreamNotFound) { pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal", "path", urlPath) - body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, pluginPortalURL, "*/*") + res = h.proxy.lookupMetadata(ctx, "maven", cacheKey, pluginPortalURL, "*/*", "", nil) } } - return body, contentType, cacheKey, err + return res, cacheKey } func (h *MavenHandler) writeMetadataError(w http.ResponseWriter, err error) { @@ -309,6 +335,15 @@ func (h *MavenHandler) isCoordinateFile(urlPath string) bool { return strings.HasPrefix(filename, artifact+"-"+strings.TrimSuffix(version, "SNAPSHOT")) } +// isMutableSnapshotFile reports whether urlPath is a SNAPSHOT file named +// with "SNAPSHOT" rather than a build timestamp (lib-1.0-SNAPSHOT.jar), which +// each new build replaces. Timestamped SNAPSHOT files never change. +func (h *MavenHandler) isMutableSnapshotFile(urlPath string) bool { + _, artifact, version, filename := h.parsePath(urlPath) + return artifact != "" && strings.HasSuffix(version, mavenSnapshot) && + strings.HasPrefix(filename, artifact+"-"+version) +} + // isVersionListingFile matches maven-metadata.xml and its checksums. The // listing changes with every release, so it is cached for the metadata TTL // rather than kept like an artifact. diff --git a/internal/handler/maven_consistency_test.go b/internal/handler/maven_consistency_test.go new file mode 100644 index 0000000..47ca1f7 --- /dev/null +++ b/internal/handler/maven_consistency_test.go @@ -0,0 +1,97 @@ +package handler + +import ( + "crypto/sha1" //nolint:gosec // Maven publishes SHA-1 checksums. + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/registries/fetch" +) + +// mavenGenerations serves every path with the current generation's bytes, +// and a matching .sha1 for maven-metadata.xml, counting requests per path. +type mavenGenerations struct { + gen atomic.Int32 + requests map[string]*atomic.Int32 +} + +func (m *mavenGenerations) body(path string) string { + return path + " generation " + string(rune('0'+m.gen.Load())) +} + +func newMavenGenerations(t *testing.T, paths ...string) (*mavenGenerations, http.Handler) { + t.Helper() + m := &mavenGenerations{requests: map[string]*atomic.Int32{}} + for _, p := range paths { + m.requests[p] = &atomic.Int32{} + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if n := m.requests[r.URL.Path]; n != nil { + n.Add(1) + } + const metadata = "/com/example/lib/maven-metadata.xml" + if r.URL.Path == metadata+".sha1" { + sum := sha1.Sum([]byte(m.body(metadata))) //nolint:gosec // as above + _, _ = io.WriteString(w, hex.EncodeToString(sum[:])) + return + } + _, _ = io.WriteString(w, m.body(r.URL.Path)) + })) + t.Cleanup(srv.Close) + + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = srv.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(srv.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + return m, NewMavenHandler(proxy, "http://localhost", srv.URL, srv.URL).Routes() +} + +// TestMavenMetadataChecksumMatchesServedMetadata checks that the checksum +// files of maven-metadata.xml describe the metadata the proxy serves, even +// when upstream has deployed since it was cached. +func TestMavenMetadataChecksumMatchesServedMetadata(t *testing.T) { + m, h := newMavenGenerations(t) + const metadata = "/com/example/lib/maven-metadata.xml" + + served := getMaven(h, metadata).Body.String() + m.gen.Store(1) + w := getMaven(h, metadata+".sha1") + sum := sha1.Sum([]byte(served)) //nolint:gosec // as above + if w.Code != http.StatusOK || w.Body.String() != hex.EncodeToString(sum[:]) { + t.Errorf(".sha1 = %d %q, want the sha1 of the served metadata %x", w.Code, w.Body, sum) + } +} + +// TestMavenNonUniqueSnapshotNotKept checks that a SNAPSHOT file without a +// timestamp, which each build replaces, is not served from cache for good, +// while a timestamped one is. +func TestMavenNonUniqueSnapshotNotKept(t *testing.T) { + const ( + snapshot = "/com/example/lib/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar" + timestamped = "/com/example/lib/1.0-SNAPSHOT/lib-1.0-20240101.120000-1.jar" + ) + m, h := newMavenGenerations(t, timestamped) + + getMaven(h, snapshot) + getMaven(h, snapshot+".sha1") + getMaven(h, timestamped) + m.gen.Store(1) + if w := getMaven(h, snapshot); w.Body.String() != m.body(snapshot) { + t.Errorf("SNAPSHOT jar = %q, want the new build %q", w.Body, m.body(snapshot)) + } + if w := getMaven(h, snapshot+".sha1"); w.Body.String() != m.body(snapshot+".sha1") { + t.Errorf("SNAPSHOT jar checksum = %q, want the new build's", w.Body) + } + if w := getMaven(h, timestamped); w.Code != http.StatusOK || m.requests[timestamped].Load() != 1 { + t.Errorf("timestamped SNAPSHOT jar fetched %d times, want once", m.requests[timestamped].Load()) + } +} diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index 4da839e..6155e0d 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -242,11 +242,12 @@ func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Requ return false } metadataPath := path.Join(path.Dir(urlPath), mavenMetadataFile) - body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, metadataPath) - if err != nil { - h.writeMetadataError(w, err) + res, cacheKey := h.fetchMetadata(r.Context(), source, metadataPath) + if res.err != nil { + h.writeMetadataError(w, res.err) return true } + body, contentType := res.body, res.contentType m, err := h.proxy.cachedRewrite(r.Context(), "maven", h.proxyURL, cacheKey, body, "", func(in []byte) ([]byte, []string, error) { // The rewrite is shared by requests waiting on it and cached, so it @@ -273,7 +274,7 @@ func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Requ if contentType == "" || contentType == contentTypeJSON { contentType = "text/xml" } - markStale(w, h.proxy.lookupCachedMeta("maven", cacheKey).stale) + markStale(w, res.stale) serveRewrittenMetadataAs(w, r, m, contentType) case mavenChecksums[suffix] != nil: sum := mavenChecksums[suffix]() From 1d9b9d4883495cd36232cfb599949619fcc19a91 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:50:07 +0100 Subject: [PATCH 128/198] Keep policy checks and fallback for mutable SNAPSHOT files Non-timestamped SNAPSHOT files went straight to upstream, past the denylist, cooldown, scanning and the Plugin Portal fallback. They now take the normal download path, with the cached copy dropped first so each request still gets the current build. A database error while refreshing a verified 304 no longer forces a full download: the bytes were checked against the row and upstream confirmed them. --- docs/configuration.md | 5 ++-- internal/handler/handler.go | 28 ++++++++++++++++++--- internal/handler/maven.go | 13 +++++----- internal/handler/maven_consistency_test.go | 29 ++++++++++++++++++++++ 4 files changed, 63 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0360d3a..ebad0c7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1026,8 +1026,9 @@ its `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from the cached metadata rather than fetched, so they always match it. Other files are cached for good, except SNAPSHOT files named with `SNAPSHOT` rather than a build timestamp (`lib-1.0-SNAPSHOT.jar`, as JitPack and non-unique snapshot -repositories publish them), which each build replaces: those are passed -through uncached. Timestamped SNAPSHOT files +repositories publish them), which each build replaces: those are fetched +again for every request, through the denylist, cooldown, scanning and the +Plugin Portal fallback like any other file. Timestamped SNAPSHOT files (`lib-1.0-20240101.120000-1.jar`) never change and are cached. This field has no environment variable, as with the other named upstream maps. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 0b84f0d..8a86ea8 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2131,9 +2131,10 @@ func upstreamMetadataStatus(resp *http.Response) error { // storage are unchanged, so they are neither written nor hashed again. It // returns the row's digest, or "" when the caller has to store meta as for a // new document: when upstream sent one or the row records no digest. When -// the row could not be refreshed, for instance because a concurrent fetch has -// replaced the document since it was read, it returns errStale304: storing -// meta would put the old document back under the new one. +// a concurrent fetch has replaced the document since it was read, it returns +// errStale304: storing meta would put the old document back under the new +// one. A failed database write only leaves fetched_at behind, so the +// confirmed bytes are still served. func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMetadata) (string, error) { entry := meta.notModified if entry == nil || p.DB == nil || !entry.ContentDigest.Valid { @@ -2141,8 +2142,10 @@ func (p *Proxy) refreshMetadataBlob(ecosystem, cacheKey string, meta *upstreamMe } refreshed, err := p.DB.RefreshMetadataCache(ecosystem, cacheKey, entry.ContentDigest.String, time.Now()) if err != nil { + // The bytes were checked against the row and upstream confirmed + // them; only fetched_at stays behind, so they are still served. p.Logger.Warn("failed to refresh cached metadata", "ecosystem", ecosystem, "key", cacheKey, "error", err) - return "", errStale304 + return entry.ContentDigest.String, nil } if !refreshed { return "", errStale304 @@ -2613,6 +2616,23 @@ func (p *Proxy) getCachedArtifactWithUpstreamHash(ctx context.Context, pkgPURL, return nil, nil } +// forgetCachedArtifact clears the cached entry for a file upstream replaces +// in place, queueing its object for deletion, so the next request fetches +// it again. +func (p *Proxy) forgetCachedArtifact(ecosystem, name, version, filename string) { + if p.DB == nil { + return + } + versionPURL := canonicalVersionPURL(ecosystem, name, version) + record, err := p.DB.GetCachedArtifact(canonicalPackagePURL(ecosystem, name), versionPURL, filename) + if err != nil || record == nil { + return + } + if err := p.DB.DiscardArtifact(versionPURL, filename, record.StoragePath); err != nil { + p.Logger.Warn("failed to clear artifact cache record", "purl", versionPURL, "filename", filename, "error", err) + } +} + // discardStaleArtifact clears the cached entry when its digest disagrees with // upstreamHash, queueing its object for deletion. It runs under the coalescing // key, after the recheck, so an entry a previous fetch refreshed is kept. diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 30a6976..2f8814e 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -137,12 +137,6 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) { return } - if h.isMutableSnapshotFile(urlPath) { - // Rebuilt in place, so not cached. - h.proxyUpstream(w, r, source, urlPath) - return - } - if h.isArtifactFile(filename) || h.isChecksumFile(filename) || h.isCoordinateFile(urlPath) { h.handleDownload(w, r, source, urlPath) return @@ -268,6 +262,13 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) cacheFilename := source.cacheFilename(filename) + // Each build replaces a SNAPSHOT file named without a timestamp, so the + // cached copy is dropped and the file fetched again, through the same + // denylist, cooldown and scanning as any other download. + if h.isMutableSnapshotFile(urlPath) { + h.proxy.forgetCachedArtifact("maven", name, version, cacheFilename) + } + result, err := h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, upstreamURL) if err != nil && source.fallbackURL != "" { if errors.Is(err, ErrUpstreamNotFound) { diff --git a/internal/handler/maven_consistency_test.go b/internal/handler/maven_consistency_test.go index 47ca1f7..a97faf6 100644 --- a/internal/handler/maven_consistency_test.go +++ b/internal/handler/maven_consistency_test.go @@ -95,3 +95,32 @@ func TestMavenNonUniqueSnapshotNotKept(t *testing.T) { t.Errorf("timestamped SNAPSHOT jar fetched %d times, want once", m.requests[timestamped].Load()) } } + +// TestMavenMutableSnapshotKeepsPolicyAndFallback checks that a SNAPSHOT file +// fetched again for each request still goes through the denylist, and falls +// back to the Gradle Plugin Portal when the primary upstream lacks it. +func TestMavenMutableSnapshotKeepsPolicyAndFallback(t *testing.T) { + const snapshot = "/com/example/lib/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar" + primary := httptest.NewServer(http.NotFoundHandler()) + t.Cleanup(primary.Close) + portal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "portal build") + })) + t.Cleanup(portal.Close) + + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.HTTPClient = primary.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(primary.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewMavenHandler(proxy, "http://localhost", primary.URL, portal.URL).Routes() + + if w := getMaven(h, snapshot); w.Code != http.StatusOK || w.Body.String() != "portal build" { + t.Errorf("SNAPSHOT from the fallback = %d %q, want 200 portal build", w.Code, w.Body) + } + setTestDenylist(t, proxy, "pkg:maven/com.example/lib@1.0-SNAPSHOT") + if w := getMaven(h, snapshot); w.Code != http.StatusForbidden { + t.Errorf("denied SNAPSHOT = %d, want 403", w.Code) + } +} From 5c95bd10532dfa178695320694755caf7db4d520 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:50:55 +0100 Subject: [PATCH 129/198] Accept git as a scanner ecosystem Git LFS objects are scanned under the git key, which the new ecosystem check rejected, so a scanner restricted to ecosystems: [git] would have stopped the proxy from starting. --- docs/configuration.md | 2 +- internal/config/config.go | 5 +++-- internal/config/config_test.go | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8baac0f..ba9bbd3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1400,7 +1400,7 @@ scanning: | `scanning.scanners[].name` | - | Identifies this scanner in logs and metrics | | `scanning.scanners[].url` | - | Endpoint the proxy POSTs scan notifications to | | `scanning.scanners[].mode` | - | `block` (default) or `monitor` | -| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems. Valid names: `alpine`, `arch`, `cargo`, `chef`, `composer`, `conan`, `conda`, `cran`, `deb`, `freebsd`, `gem`, `generic`, `golang`, `helm`, `hex`, `julia`, `maven`, `npm`, `nuget`, `oci`, `pub`, `pypi`, `rpm`, `swift`. The aliases `go`, `docker`, `rubygems`, `packagist`, `debian` and `apk` are accepted too; any other name stops the proxy at startup | +| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems. Valid names: `alpine`, `arch`, `cargo`, `chef`, `composer`, `conan`, `conda`, `cran`, `deb`, `freebsd`, `gem`, `generic`, `git` (Git LFS objects), `golang`, `helm`, `hex`, `julia`, `maven`, `npm`, `nuget`, `oci`, `pub`, `pypi`, `rpm`, `swift`. The aliases `go`, `docker`, `rubygems`, `packagist`, `debian` and `apk` are accepted too; any other name stops the proxy at startup | | `scanning.scanners[].headers` | - | Extra HTTP headers sent with every scan request (e.g. for authenticating to the scanner service). Values support `${VAR_NAME}` expansion | ### How caching defers to a scan verdict diff --git a/internal/config/config.go b/internal/config/config.go index d9d9b29..c1a2b08 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -449,9 +449,10 @@ func (s *ScannerConfig) Validate() error { } // scannerEcosystems are the ecosystem keys handlers pass to scanners: the -// retention keys plus the handlers retention does not cover. +// retention keys plus the handlers retention does not cover, and git for +// Git LFS objects. var scannerEcosystems = func() []string { - keys := append(slices.Clone(retention.KnownKeys), "arch", "chef") + keys := append(slices.Clone(retention.KnownKeys), "arch", "chef", "git") slices.Sort(keys) return keys }() diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2139b0e..eb14e6b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2211,13 +2211,13 @@ func TestScannerEcosystems(t *testing.T) { SigningKey: "s3cret", Scanners: []ScannerConfig{{ Name: "clamav", URL: "http://scanner.invalid/scan", - Ecosystems: []string{"go", "Docker", "rubygems", "npm", "arch"}, + Ecosystems: []string{"go", "Docker", "rubygems", "npm", "arch", "git"}, }}, } if err := cfg.Validate(); err != nil { t.Fatalf("Validate() error: %v", err) } - want := []string{"golang", "oci", "gem", "npm", "arch"} + want := []string{"golang", "oci", "gem", "npm", "arch", "git"} if got := cfg.Scanners[0].Ecosystems; !slices.Equal(got, want) { t.Errorf("Ecosystems = %v, want %v", got, want) } From 61947c9c8ccdf33aae67e22cc964cc8e258c5ac7 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:51:39 +0100 Subject: [PATCH 130/198] Treat a null Composer notification-url as absent Composer reads "notification-url": null as missing and reports installs to notify-batch, so a pushed manifest with null now gets an empty one too. The docs say a pushed manifest's own notification-url is kept. --- docs/configuration.md | 2 +- internal/handler/composer_hosted.go | 3 ++- internal/handler/composer_hosted_test.go | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 3e7abc7..8603e1c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1643,7 +1643,7 @@ Pushed versions are listed in the package metadata clients resolve against, with - Pushed versions are added next to the upstream ones and are never held back by cooldown. npm `dist-tags` stay as the upstream has them, also in `npm dist-tag ls`. - A package the upstream doesn't know is served from its pushed versions alone. Its npm `latest` tag is the highest pushed version. - Pushed Composer dev versions (`dev-*`, `*-dev`) are listed in the `~dev` metadata file. -- Pushed Composer versions carry an empty `notification-url`, as routed packages do, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. +- Pushed Composer versions carry an empty `notification-url`, as routed packages do, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. A pushed `composer.json` that sets its own `notification-url` keeps it, and Composer then reports installs there. - npm entries carry the archive's sha512 as `dist.integrity`. Composer entries carry its sha256 as `dist.reference` and leave `shasum` empty. Metadata of packages with pushed versions is rewritten on every request instead of being cached, so a push or delete shows up on the next request. diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index 3c42a6a..3cab559 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -45,7 +45,8 @@ func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([] manifest["time"] = lv.PushedAt.Format(time.RFC3339) // As for routed packages, keep Composer from reporting installs of // a private version to the default repository's notify-batch. - if _, ok := manifest["notification-url"]; !ok { + // Composer reads a null notification-url as absent too. + if url, ok := manifest["notification-url"]; !ok || url == nil { manifest["notification-url"] = "" } manifest["dist"] = map[string]string{ diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index 6c439dc..5b8a687 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -250,11 +250,14 @@ func TestComposerPushedVersionsDisableDefaultNotification(t *testing.T) { f.push(t, "monolog/monolog", "3.6.0", `{"name":"monolog/monolog"}`) f.push(t, "acme/private", "1.0.0", `{"name":"acme/private"}`) f.push(t, "other/private", "1.0.0", `{"name":"other/private"}`) + // Composer reads a null notification-url as absent. + f.push(t, "acme/nulled", "1.0.0", `{"name":"acme/nulled","notification-url":null}`) for _, tc := range []struct{ path, name, version string }{ {"/p2/monolog/monolog.json", "monolog/monolog", "3.6.0"}, {"/p2/acme/private.json", "acme/private", "1.0.0"}, {"/p2/other/private.json", "other/private", "1.0.0"}, + {"/p2/acme/nulled.json", "acme/nulled", "1.0.0"}, } { v := f.versions(t, tc.path, tc.name)[tc.version] if got, ok := v["notification-url"]; !ok || got != "" { From 8f281f1dfa6335f080361b3c00f219cf30875ef3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:55:02 +0100 Subject: [PATCH 131/198] Refuse info/refs while an upstream refusal stands A check that ran past refresh_wait or failed served the cached copy, also right after the upstream had refused the repository. With protocol v0 the refs are in the info/refs response itself, so they were advertised. The stored refusal now refuses those too, until a check succeeds. --- internal/gitcache/cache_test.go | 22 ++++++++++++++++++++++ internal/gitcache/serve.go | 19 +++++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 5f9d475..fc5abef 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -875,3 +875,25 @@ func TestCloneChecksUpstreamOnce(t *testing.T) { t.Errorf("clone made %d upstream check requests, want %d", got, perCheck) } } + +// TestInfoRefsRefusedWhileRefusalStands checks that once the upstream +// refused a repository, a later check that runs past RefreshWait, which +// would otherwise serve the cached copy, doesn't advertise its refs. +func TestInfoRefsRefusedWhileRefusalStands(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = 100 * time.Millisecond }) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusForbidden) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("served the cached copy after 403:\n%s", out) + } + upstream.status.Store(0) + release := upstream.holdRequests() + defer release() + // Protocol v0 advertises the refs in the info/refs response itself. + if out, err := tryGit(t, "", "-c", "protocol.version=0", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("advertised refs while the refusal stands:\n%s", out) + } +} diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index 68c891a..29f54f8 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -63,6 +63,12 @@ func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) switch { case opts.Refresh || !hit: err = c.ensureFresh(ctx, rp, repoURL) + if err == nil && hit && !opts.NoCheck { + // A refresh still running or failed serves the cached copy, + // but not one the upstream refused since and no check has + // cleared. + err = c.refusal(rp) + } case !opts.NoCheck: err = c.checkServable(ctx, rp, repoURL) } @@ -81,10 +87,7 @@ const uploadPackGrace = time.Minute // checkServable refuses a cached repository the last refresh found gone or // refused upstream, and refreshes one not checked recently first. func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) error { - c.mu.Lock() - err := rp.refused - c.mu.Unlock() - if err != nil { + if err := c.refusal(rp); err != nil { return err } if time.Since(time.Unix(0, rp.lastChecked.Load())) > c.opts.StaleMaxAge+uploadPackGrace { @@ -93,6 +96,14 @@ func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) err return nil } +// refusal returns the error of the last refresh that found the repository +// gone or refused upstream, until a refresh succeeds. +func (c *Cache) refusal(rp *repo) error { + c.mu.Lock() + defer c.mu.Unlock() + return rp.refused +} + // Hit reports whether the repository was already cached when it was opened. func (l *Lease) Hit() bool { return l.hit From fc0e94b080fe4b6f2bfec42800f7b0eb9995f159 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 08:56:20 +0100 Subject: [PATCH 132/198] Remove empty directories below a fetch directory on delete APK and Arch packages, and now RPM ones, have filenames with directories in them, which put the object below its fetch directory. Delete only removed the object's own directory when it was the fetch directory, so those directories stayed behind. It now removes them up to and including the fetch directory while they are empty. --- internal/storage/blob.go | 23 +++++++++++++++++++--- internal/storage/blob_test.go | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 81af78f..a999ff5 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -291,13 +291,30 @@ func (b *Blob) Delete(ctx context.Context, path string) error { return fmt.Errorf("deleting object: %w", err) } if p := b.localPath(path); p != "" { - if dir := filepath.Dir(p); dir != b.fileRoot && isFetchDir(filepath.Base(dir)) { - _ = os.Remove(dir) // fails, harmlessly, while the directory holds anything - } + b.removeEmptyFetchDirs(filepath.Dir(p)) } return nil } +// removeEmptyFetchDirs removes dir and its parents while they are empty, up +// to and including the fetch directory they are in, for an object whose +// filename has directories of its own. Outside a fetch directory, as in the +// layout before them, nothing is removed. +func (b *Blob) removeEmptyFetchDirs(dir string) { + var dirs []string + for d := dir; d != b.fileRoot && d != filepath.Dir(d); d = filepath.Dir(d) { + dirs = append(dirs, d) + if isFetchDir(filepath.Base(d)) { + for _, d := range dirs { + if os.Remove(d) != nil { + return // fails, harmlessly, while the directory holds anything + } + } + return + } + } +} + func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { url, err := b.bucket.SignedURL(ctx, path, &blob.SignedURLOptions{ Method: http.MethodGet, diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 1a764de..faaace8 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -180,6 +180,42 @@ func TestBlobDeleteRemovesEmptyFetchDirectory(t *testing.T) { assertReadsBack(t, b, kept, "content") } +// A filename with directories in it, as APK, Arch and RPM packages have, puts +// the object below its fetch directory. Delete removes those directories +// too once empty, up to and including the fetch directory. +func TestBlobDeleteRemovesNestedFetchDirectories(t *testing.T) { + dir := t.TempDir() + b := openFileBlob(t, dir) + ctx := context.Background() + const ( + emptied = "rpm/bash/5.1/0123456789abcdef/rocky/9/BaseOS/bash.rpm" + shared = "rpm/bash/5.1/fedcba9876543210/alma/9/BaseOS/bash.rpm" + kept = "rpm/bash/5.1/fedcba9876543210/alma/9/AppStream/bash.rpm" + ) + for _, key := range []string{emptied, shared, kept} { + if _, _, err := b.Store(ctx, key, strings.NewReader("content")); err != nil { + t.Fatalf("Store(%q): %v", key, err) + } + } + for _, key := range []string{emptied, shared} { + if err := b.Delete(ctx, key); err != nil { + t.Fatalf("Delete(%q): %v", key, err) + } + } + + version := filepath.Join(dir, "rpm", "bash", "5.1") + if _, err := os.Stat(filepath.Join(version, "0123456789abcdef")); !os.IsNotExist(err) { + t.Errorf("emptied fetch directory left behind (stat err %v)", err) + } + if _, err := os.Stat(filepath.Join(version, "fedcba9876543210", "alma", "9", "BaseOS")); !os.IsNotExist(err) { + t.Errorf("emptied directory below the fetch directory left behind (stat err %v)", err) + } + if _, err := os.Stat(version); err != nil { + t.Errorf("version directory removed: %v", err) + } + assertReadsBack(t, b, kept, "content") +} + func TestBlobDeleteNotFound(t *testing.T) { b := createTestBlob(t) ctx := context.Background() From 3b4ddb43d8a0c162d74673495e37041f70e49849 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:04:58 +0100 Subject: [PATCH 133/198] Revert "Remove empty directories below a fetch directory on delete" This reverts commit fc0e94b. The cleanup stopped at the first segment that looked like a fetch id, but named Maven repositories put a hex upstream hash in the filename, so it stopped early. For objects in the old layout, with no fetch directory, the walk could remove version or package directories under a concurrent fetch. Leftover empty directories will be handled separately. --- internal/storage/blob.go | 23 +++------------------- internal/storage/blob_test.go | 36 ----------------------------------- 2 files changed, 3 insertions(+), 56 deletions(-) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index a999ff5..81af78f 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -291,28 +291,11 @@ func (b *Blob) Delete(ctx context.Context, path string) error { return fmt.Errorf("deleting object: %w", err) } if p := b.localPath(path); p != "" { - b.removeEmptyFetchDirs(filepath.Dir(p)) - } - return nil -} - -// removeEmptyFetchDirs removes dir and its parents while they are empty, up -// to and including the fetch directory they are in, for an object whose -// filename has directories of its own. Outside a fetch directory, as in the -// layout before them, nothing is removed. -func (b *Blob) removeEmptyFetchDirs(dir string) { - var dirs []string - for d := dir; d != b.fileRoot && d != filepath.Dir(d); d = filepath.Dir(d) { - dirs = append(dirs, d) - if isFetchDir(filepath.Base(d)) { - for _, d := range dirs { - if os.Remove(d) != nil { - return // fails, harmlessly, while the directory holds anything - } - } - return + if dir := filepath.Dir(p); dir != b.fileRoot && isFetchDir(filepath.Base(dir)) { + _ = os.Remove(dir) // fails, harmlessly, while the directory holds anything } } + return nil } func (b *Blob) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) { diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index faaace8..1a764de 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -180,42 +180,6 @@ func TestBlobDeleteRemovesEmptyFetchDirectory(t *testing.T) { assertReadsBack(t, b, kept, "content") } -// A filename with directories in it, as APK, Arch and RPM packages have, puts -// the object below its fetch directory. Delete removes those directories -// too once empty, up to and including the fetch directory. -func TestBlobDeleteRemovesNestedFetchDirectories(t *testing.T) { - dir := t.TempDir() - b := openFileBlob(t, dir) - ctx := context.Background() - const ( - emptied = "rpm/bash/5.1/0123456789abcdef/rocky/9/BaseOS/bash.rpm" - shared = "rpm/bash/5.1/fedcba9876543210/alma/9/BaseOS/bash.rpm" - kept = "rpm/bash/5.1/fedcba9876543210/alma/9/AppStream/bash.rpm" - ) - for _, key := range []string{emptied, shared, kept} { - if _, _, err := b.Store(ctx, key, strings.NewReader("content")); err != nil { - t.Fatalf("Store(%q): %v", key, err) - } - } - for _, key := range []string{emptied, shared} { - if err := b.Delete(ctx, key); err != nil { - t.Fatalf("Delete(%q): %v", key, err) - } - } - - version := filepath.Join(dir, "rpm", "bash", "5.1") - if _, err := os.Stat(filepath.Join(version, "0123456789abcdef")); !os.IsNotExist(err) { - t.Errorf("emptied fetch directory left behind (stat err %v)", err) - } - if _, err := os.Stat(filepath.Join(version, "fedcba9876543210", "alma", "9", "BaseOS")); !os.IsNotExist(err) { - t.Errorf("emptied directory below the fetch directory left behind (stat err %v)", err) - } - if _, err := os.Stat(version); err != nil { - t.Errorf("version directory removed: %v", err) - } - assertReadsBack(t, b, kept, "content") -} - func TestBlobDeleteNotFound(t *testing.T) { b := createTestBlob(t) ctx := context.Background() From de280a1694f04b7527da5da1f35a9738a7070366 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:05:23 +0100 Subject: [PATCH 134/198] Don't cache Maven paths with a colon in the group or artifact The cache keys Maven artifacts as group:artifact, so org:foo/bar and org/foo:bar both came out as org:foo:bar and shared one entry. Such paths are now proxied without caching, like dotted group directories. --- internal/handler/maven.go | 8 ++++++-- internal/handler/maven_test.go | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/handler/maven.go b/internal/handler/maven.go index ca81b74..0fd81ec 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -279,12 +279,16 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file groupParts := parts[:len(parts)-3] // The Maven layout splits a group at its dots, so a group directory // never holds a dot or is empty. Such a path would spell the same group - // as another and share its cache entries, so it is not parsed. + // as another and share its cache entries, so it is not parsed. Nor is a + // colon in the group or artifact, which are cached as group:artifact. for _, part := range groupParts { - if part == "" || strings.Contains(part, ".") { + if part == "" || strings.ContainsAny(part, ".:") { return "", "", "", "" } } + if strings.Contains(artifact, ":") { + return "", "", "", "" + } group = strings.Join(groupParts, ".") return group, artifact, version, filename diff --git a/internal/handler/maven_test.go b/internal/handler/maven_test.go index 3d70a32..14cb49f 100644 --- a/internal/handler/maven_test.go +++ b/internal/handler/maven_test.go @@ -36,6 +36,10 @@ func TestMavenParsePath(t *testing.T) { {"org.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, {"org/.foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, {"org//foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + // Cache identity is group:artifact, so a colon in either would + // let org:foo/bar and org/foo:bar share one entry. + {"org:foo/bar/1.0/bar-1.0.jar", "", "", "", ""}, + {"org/foo:bar/1.0/bar-1.0.jar", "", "", "", ""}, } for _, tt := range tests { From 3bb7f7c30d3e647d4fc049882ed3809b54c94dea Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:05:43 +0100 Subject: [PATCH 135/198] Keep _ as a wildcard in package search Escaping _ in the search pattern stopped typing_extensions from finding typing-extensions, which worked before because _ matched any single character. Escape only % and the escape character. --- internal/database/queries.go | 7 ++++--- internal/database/queries_packages_list_test.go | 7 +++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/database/queries.go b/internal/database/queries.go index 7176152..d7eaa6e 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -898,10 +898,11 @@ func (db *DB) SearchPackages(query string, ecosystem string, limit int, offset i // from nameSearchPattern, ignoring case on SQLite and Postgres alike. const nameSearchCondition = `LOWER(p.name) LIKE LOWER(?) ESCAPE '!'` -// nameSearchPattern is the LIKE pattern for names containing query, with -// the LIKE wildcards in query matched literally. +// nameSearchPattern is the LIKE pattern for names containing query. A % +// in query is matched literally, but _ stays a single-character wildcard +// so typing_extensions still finds typing-extensions. func nameSearchPattern(query string) string { - escaped := strings.NewReplacer("!", "!!", "%", "!%", "_", "!_").Replace(query) + escaped := strings.NewReplacer("!", "!!", "%", "!%").Replace(query) return "%" + escaped + "%" } diff --git a/internal/database/queries_packages_list_test.go b/internal/database/queries_packages_list_test.go index 704b388..5eaa402 100644 --- a/internal/database/queries_packages_list_test.go +++ b/internal/database/queries_packages_list_test.go @@ -252,11 +252,11 @@ func TestCachedPackageFiltersMatchEcosystemAliases(t *testing.T) { }) } -func TestSearchPackagesIgnoresCaseAndEscapesWildcards(t *testing.T) { +func TestSearchPackagesIgnoresCaseAndEscapesPercent(t *testing.T) { runWithBothDatabases(t, func(t *testing.T, db *DB) { seedArtifact(t, db, "pypi", "Django", "5.0.0", 100, 1) + seedArtifact(t, db, "pypi", "typing-extensions", "4.12.0", 100, 1) seedArtifact(t, db, "npm", "my_pkg", "1.0.0", 100, 1) - seedArtifact(t, db, "npm", "myxpkg", "1.0.0", 100, 1) seedArtifact(t, db, "npm", "100%-done", "1.0.0", 100, 1) seedArtifact(t, db, "npm", "1000-done", "1.0.0", 100, 1) seedArtifact(t, db, "npm", `back\slash`, "1.0.0", 100, 1) @@ -268,6 +268,9 @@ func TestSearchPackagesIgnoresCaseAndEscapesWildcards(t *testing.T) { {"django", []string{"Django"}}, {"DJANGO", []string{"Django"}}, {"my_pkg", []string{"my_pkg"}}, + // _ stays a single-character wildcard, so it matches - too. + {"typing_extensions", []string{"typing-extensions"}}, + {"typing-extensions", []string{"typing-extensions"}}, {"100%", []string{"100%-done"}}, {`k\s`, []string{`back\slash`}}, } { From ae8b2c6d2126145d3930efdf3609414a1960d5ff Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:05:53 +0100 Subject: [PATCH 136/198] Accept yes, no, on and off in boolean environment variables Before the strict parser, PROXY_TRUST_FORWARDED_FOR=no meant false. With only strconv.ParseBool's forms accepted, no was ignored and a setting enabled in the config file stayed on. Accept yes/no/on/off in any case for every boolean variable, the Gradle read-only one included. --- docs/configuration.md | 2 +- internal/config/config.go | 27 ++++++++++++++++++++------- internal/config/config_test.go | 22 ++++++++++++++++++---- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ba9bbd3..8bbf5e1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,7 +2,7 @@ The proxy can be configured via command line flags, environment variables, or a configuration file. Command line flags take precedence over environment variables, which take precedence over the configuration file. -Boolean environment variables accept `true`, `True`, `TRUE`, `t`, `T` or `1`, and `false`, `False`, `FALSE`, `f`, `F` or `0`. Any other value, such as `yes` or a typo, is ignored: the setting keeps its value from the configuration file or default, and the proxy logs a warning at startup. +Boolean environment variables accept `true`, `t`, `1`, `yes` or `on`, and `false`, `f`, `0`, `no` or `off`, in any case. Any other value, such as a typo, is ignored: the setting keeps its value from the configuration file or default, and the proxy logs a warning at startup. ## Configuration File diff --git a/internal/config/config.go b/internal/config/config.go index c1a2b08..a9fc9e2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1428,16 +1428,29 @@ func setEnvString(dst *string, key string) { } } -// setEnvBool is setEnvString for boolean fields. It accepts what -// strconv.ParseBool does; any other value leaves *dst unchanged and is -// reported via EnvWarnings. +// parseEnvBool accepts what strconv.ParseBool does plus yes/no/on/off, +// ignoring case and surrounding space. +func parseEnvBool(v string) (bool, bool) { + v = strings.ToLower(strings.TrimSpace(v)) + switch v { + case "yes", "on": + return true, true + case "no", "off": + return false, true + } + b, err := strconv.ParseBool(v) + return b, err == nil +} + +// setEnvBool is setEnvString for boolean fields, parsed via parseEnvBool. +// Any other value leaves *dst unchanged and is reported via EnvWarnings. func (c *Config) setEnvBool(dst *bool, key string) { v := os.Getenv(key) if v == "" { return } - b, err := strconv.ParseBool(strings.TrimSpace(v)) - if err != nil { + b, ok := parseEnvBool(v) + if !ok { c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping %t", key, v, *dst)) return } @@ -1452,8 +1465,8 @@ func (c *Config) setGradleReadOnlyFromEnv() { if v == "" { return } - readOnly, err := strconv.ParseBool(strings.TrimSpace(v)) - if err != nil { + readOnly, ok := parseEnvBool(v) + if !ok { c.Gradle.BuildCache.ReadOnly = true c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping the Gradle build cache read-only", key, v)) return diff --git a/internal/config/config_test.go b/internal/config/config_test.go index eb14e6b..d89e300 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -467,8 +467,9 @@ func TestLoadGradleBuildCacheReadOnlyFromEnv(t *testing.T) { {" FALSE ", true, false, 0}, {"True", false, true, 0}, {"1", false, true, 0}, - {"yes", false, true, 1}, - {"off", false, true, 1}, + {"yes", false, true, 0}, + {"off", true, false, 0}, + {"maybe", false, true, 1}, {"", false, false, 0}, } for _, tt := range tests { @@ -2172,6 +2173,16 @@ func TestCooldownShortest(t *testing.T) { } func TestLoadBoolsFromEnvParseStrictly(t *testing.T) { + t.Run("no turns off a setting from the config file", func(t *testing.T) { + t.Setenv("PROXY_TRUST_FORWARDED_FOR", "no") + cfg := Default() + cfg.TrustForwardedFor = true + cfg.LoadFromEnv() + if cfg.TrustForwardedFor { + t.Error("TrustForwardedFor = true, want false") + } + }) + tests := []struct { value string start bool @@ -2185,8 +2196,11 @@ func TestLoadBoolsFromEnvParseStrictly(t *testing.T) { {"false", true, false, 0}, {"FALSE", true, false, 0}, {"0", true, false, 0}, - {"yes", true, true, 1}, - {"on", false, false, 1}, + {"yes", false, true, 0}, + {" On ", false, true, 0}, + {"no", true, false, 0}, + {"OFF", true, false, 0}, + {"maybe", false, false, 1}, {"ture", true, true, 1}, } for _, tt := range tests { From 8233ce33399f042bf24d4a87ef1d95c5ce69b3f9 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:05:58 +0100 Subject: [PATCH 137/198] Clean RPM package paths before caching The cache key for an RPM package is its request path, so a path with an empty or "." segment made a second entry for the same upstream file. The path is now cleaned before it is used as the key and the upstream URL. --- internal/handler/rpm.go | 9 +++++++- internal/handler/rpm_test.go | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index 49773a7..62b2798 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -3,6 +3,7 @@ package handler import ( "fmt" "net/http" + pathpkg "path" "regexp" "strings" @@ -78,7 +79,7 @@ func (h *RPMHandler) Routes() http.Handler { switch { case strings.HasSuffix(path, ".rpm"): // Package downloads - cache these - h.handlePackageDownload(w, r, path) + h.handlePackageDownload(w, r, cleanRPMPath(path)) case strings.Contains(path, "/repodata/"): // Repository metadata h.handleMetadata(w, r, path) @@ -89,6 +90,12 @@ func (h *RPMHandler) Routes() http.Handler { })) } +// cleanRPMPath drops empty and "." segments from a relative path, so each +// package has one cache key. ".." segments are rejected before this. +func cleanRPMPath(p string) string { + return strings.TrimPrefix(pathpkg.Clean("/"+p), "/") +} + // handlePackageDownload fetches and caches .rpm packages. // Path format varies by repo structure, e.g.: // - releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm diff --git a/internal/handler/rpm_test.go b/internal/handler/rpm_test.go index 87275b2..d9561e3 100644 --- a/internal/handler/rpm_test.go +++ b/internal/handler/rpm_test.go @@ -102,3 +102,45 @@ func TestRPMHandler_PackageDownloadCachesPerRepository(t *testing.T) { available.Store(false) } } + +// TestRPMHandler_PackageDownloadCleansPath checks that spellings of one +// path with empty or "." segments share one cache entry. +func TestRPMHandler_PackageDownloadCleansPath(t *testing.T) { + const want = "bash" + var hits atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/rocky/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm" { + http.NotFound(w, r) + return + } + hits.Add(1) + _, _ = w.Write([]byte(want)) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + srv := httptest.NewServer(NewRPMHandlerWithUpstream(proxy, testProxyURL, upstream.URL).Routes()) + defer srv.Close() + + for _, path := range []string{ + "/rocky/9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm", + "/rocky//9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm", + "/rocky/./9/BaseOS/x86_64/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm", + } { + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || string(body) != want { + t.Errorf("%s: %d %q, want 200 %q", path, resp.StatusCode, body, want) + } + } + if n := hits.Load(); n != 1 { + t.Errorf("upstream fetched %d times, want 1", n) + } +} From 5c798069a18632652ca7e4d25f36112a67397bcd Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:06:20 +0100 Subject: [PATCH 138/198] Treat a null notification-url as absent for routed packages Composer falls back to the default repository's notify-batch when a version's notification-url is null, which would report installs of a routed private package to Packagist. Replace null with an empty string as is already done for pushed versions. Also drop the pushed-version check's local url variable, which shadowed net/url, and test that a pushed manifest's own URL is kept. --- internal/handler/composer.go | 3 ++- internal/handler/composer_hosted.go | 2 +- internal/handler/composer_hosted_test.go | 17 ++++++++++------- internal/handler/composer_routes_test.go | 7 ++++++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 17a4123..6b82cff 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -431,8 +431,9 @@ func composerPublishTime(published []byte) (time.Time, bool) { // the package name to it. Composer only falls back to notify-batch when a // version has no notification-url of its own, and skips an empty one. Setting // it on the expanded fields keeps it set for the versions that inherit them. +// Composer reads a null notification-url as absent too. func disableDefaultNotification(fields *composerFields) { - if fields.get("notification-url") == nil { + if v := fields.get("notification-url"); v == nil || string(v) == "null" { fields.set("notification-url", []byte(`"notification-url"`), []byte(`""`)) } } diff --git a/internal/handler/composer_hosted.go b/internal/handler/composer_hosted.go index 3cab559..be8a1d6 100644 --- a/internal/handler/composer_hosted.go +++ b/internal/handler/composer_hosted.go @@ -46,7 +46,7 @@ func (h *ComposerHandler) composerLocalEntries(packageName string, dev bool) ([] // As for routed packages, keep Composer from reporting installs of // a private version to the default repository's notify-batch. // Composer reads a null notification-url as absent too. - if url, ok := manifest["notification-url"]; !ok || url == nil { + if manifest["notification-url"] == nil { manifest["notification-url"] = "" } manifest["dist"] = map[string]string{ diff --git a/internal/handler/composer_hosted_test.go b/internal/handler/composer_hosted_test.go index 5b8a687..dd792ca 100644 --- a/internal/handler/composer_hosted_test.go +++ b/internal/handler/composer_hosted_test.go @@ -252,16 +252,19 @@ func TestComposerPushedVersionsDisableDefaultNotification(t *testing.T) { f.push(t, "other/private", "1.0.0", `{"name":"other/private"}`) // Composer reads a null notification-url as absent. f.push(t, "acme/nulled", "1.0.0", `{"name":"acme/nulled","notification-url":null}`) + f.push(t, "acme/own", "1.0.0", `{"name":"acme/own","notification-url":"https://acme.example.com/notify"}`) - for _, tc := range []struct{ path, name, version string }{ - {"/p2/monolog/monolog.json", "monolog/monolog", "3.6.0"}, - {"/p2/acme/private.json", "acme/private", "1.0.0"}, - {"/p2/other/private.json", "other/private", "1.0.0"}, - {"/p2/acme/nulled.json", "acme/nulled", "1.0.0"}, + for _, tc := range []struct{ path, name, version, want string }{ + {"/p2/monolog/monolog.json", "monolog/monolog", "3.6.0", ""}, + {"/p2/acme/private.json", "acme/private", "1.0.0", ""}, + {"/p2/other/private.json", "other/private", "1.0.0", ""}, + {"/p2/acme/nulled.json", "acme/nulled", "1.0.0", ""}, + // A manifest's own URL is kept. + {"/p2/acme/own.json", "acme/own", "1.0.0", "https://acme.example.com/notify"}, } { v := f.versions(t, tc.path, tc.name)[tc.version] - if got, ok := v["notification-url"]; !ok || got != "" { - t.Errorf("%s %s notification-url = %v (set %v), want empty", tc.name, tc.version, got, ok) + if got, ok := v["notification-url"]; !ok || got != tc.want { + t.Errorf("%s %s notification-url = %v (set %v), want %q", tc.name, tc.version, got, ok, tc.want) } } } diff --git a/internal/handler/composer_routes_test.go b/internal/handler/composer_routes_test.go index 5a65440..0ebf5ac 100644 --- a/internal/handler/composer_routes_test.go +++ b/internal/handler/composer_routes_test.go @@ -235,13 +235,16 @@ func TestComposerRoutedDownloadIgnoresDefaultRepositoryArtifact(t *testing.T) { // TestComposerRoutedMinifiedMetadataDisablesNotification checks that every // version of a routed package, as Composer expands minified metadata, has an // empty notification-url: versions that inherit their fields, versions after a -// ~dev reset, and versions after one that set its own URL and then unset it. +// ~dev reset, versions after one that set its own URL and then unset it, and +// versions with a null URL. func TestComposerRoutedMinifiedMetadataDisablesNotification(t *testing.T) { minified := `{"minified":"composer/2.0","packages":{"example/library":[` + `{"name":"example/library","version":"1.0.0","dist":{"url":"https://private.example.com/a/1.0.0.zip","type":"zip"}},` + `{"version":"1.1.0","dist":{"url":"https://private.example.com/a/1.1.0.zip","type":"zip"}},` + `{"version":"1.2.0","notification-url":"https://private.example.com/notify","dist":{"url":"https://private.example.com/a/1.2.0.zip","type":"zip"}},` + `{"version":"1.3.0","notification-url":"__unset","dist":{"url":"https://private.example.com/a/1.3.0.zip","type":"zip"}},` + + `{"version":"1.4.0","notification-url":null,"dist":{"url":"https://private.example.com/a/1.4.0.zip","type":"zip"}},` + + `{"version":"1.5.0","dist":{"url":"https://private.example.com/a/1.5.0.zip","type":"zip"}},` + `"~dev",` + `{"name":"example/library","version":"dev-main","dist":{"url":"https://private.example.com/a/main.zip","type":"zip"}}]}}` public := newComposerUpstream(t, map[string]string{}, http.StatusOK) @@ -258,6 +261,8 @@ func TestComposerRoutedMinifiedMetadataDisablesNotification(t *testing.T) { "1.1.0": "", "1.2.0": "https://private.example.com/notify", // the route's own URL is kept "1.3.0": "", + "1.4.0": "", // Composer reads null as absent + "1.5.0": "", "dev-main": "", } versions := composerVersions(t, w.Body.Bytes(), "example/library") From 767e401a3cf9deae3a43653f0e7ae4d283a10033 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:06:50 +0100 Subject: [PATCH 139/198] Check the upstream again on upload-pack while refused An upload-pack request returned a stored refusal without checking the upstream, so a client that never asks for info/refs, such as a protocol v2 client sending ls-refs and fetch directly, stayed refused after a passing 403 or 404. Refresh first while a refusal stands, as for a copy past stale_max_age, and refuse only if the refusal remains. --- docs/configuration.md | 7 ++++--- internal/gitcache/cache_test.go | 25 +++++++++++++++++++++++++ internal/gitcache/serve.go | 15 ++++++++------- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d384bac..37d39ae 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -903,9 +903,10 @@ or redirects is not served from the cache, also to requests that skip the ref listing and go straight to `git-upload-pack`, until a later check succeeds. Redirects are never followed, so point the upstream at a repository's current location. A `git-upload-pack` request does not check the upstream itself -unless the copy was last checked more than a minute past `stale_max_age` ago, -so the requests of a clone that follow its ref listing check nothing again; -an older copy is checked first, as for a ref listing. +unless the upstream refused the repository at the last check or the copy was +last checked more than a minute past `stale_max_age` ago, so the requests of a +clone that follow its ref listing check nothing again; otherwise the +repository is checked first, as for a ref listing. The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it). The `git` processes never reach the network themselves: their fetches go diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index fc5abef..baf646a 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -897,3 +897,28 @@ func TestInfoRefsRefusedWhileRefusalStands(t *testing.T) { t.Errorf("advertised refs while the refusal stands:\n%s", out) } } + +// refuse has the upstream refuse "repo" with a 403 and checks that the +// cached copy is no longer served. +func (tc *testCache) refuse(t *testing.T, upstream *fakeUpstream) { + t.Helper() + upstream.status.Store(http.StatusForbidden) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("served the cached copy after 403:\n%s", out) + } + upstream.status.Store(0) +} + +// TestUploadPackClearsRefusal has a client that only sends upload-pack +// requests: once the upstream allows the repository again, it is served. +func TestUploadPackClearsRefusal(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + tc.refuse(t, upstream) + + if status := tc.lsRefs(t); status != http.StatusOK { + t.Errorf("ls-refs after the upstream allowed the repository again: status %d", status) + } +} diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index 29f54f8..5a9f1e0 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -84,16 +84,17 @@ func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) // info/refs, so a clone checks the upstream once. const uploadPackGrace = time.Minute -// checkServable refuses a cached repository the last refresh found gone or -// refused upstream, and refreshes one not checked recently first. +// checkServable refreshes a cached repository first when the last refresh +// found it gone or refused upstream, or it was not checked recently, and +// refuses it while that refusal stands. func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) error { - if err := c.refusal(rp); err != nil { + if c.refusal(rp) == nil && time.Since(time.Unix(0, rp.lastChecked.Load())) <= c.opts.StaleMaxAge+uploadPackGrace { + return nil + } + if err := c.ensureFresh(ctx, rp, repoURL); err != nil { return err } - if time.Since(time.Unix(0, rp.lastChecked.Load())) > c.opts.StaleMaxAge+uploadPackGrace { - return c.ensureFresh(ctx, rp, repoURL) - } - return nil + return c.refusal(rp) } // refusal returns the error of the last refresh that found the repository From 39d6b8c7b0320f790ccfd022d5282e23558088b6 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:07:24 +0100 Subject: [PATCH 140/198] Rewrite PyPI root index links under the upstream's path An upstream such as Nexus served from /repository/pypi links projects as /repository/pypi/simple//, which the rewrite missed. Match links under the configured upstream's path, in single-quoted hrefs as well as double-quoted ones, and drop the upstream's ETag, which names the body before the rewrite. --- internal/handler/pypi.go | 64 +++++++++++++++++++--------- internal/handler/pypi_test.go | 79 +++++++++++++++++++++-------------- 2 files changed, 92 insertions(+), 51 deletions(-) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 7dd3f12..50f9964 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -11,6 +11,7 @@ import ( "maps" "mime" "net/http" + "net/url" "regexp" "slices" "strconv" @@ -1023,17 +1024,25 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s } defer func() { _ = resp.Body.Close() }() - // The HTML root index links projects as /simple//, which must - // point at the proxy's /pypi mount instead. It is large, so it is - // rewritten as it streams. + // The HTML root index links projects by absolute path, such as + // /simple//, which must point at the proxy's /pypi mount instead. + // It is large, so it is rewritten as it streams. if resp.StatusCode == http.StatusOK && resp.Header.Get("Content-Encoding") == "" && !strings.Contains(resp.Header.Get(headerContentType), "json") { + prefix := "/simple/" + if u, err := url.Parse(h.upstreamURL); err == nil { + prefix = strings.TrimSuffix(u.Path, "/") + prefix + } + to := h.proxyURL + "/pypi/simple/" resp.Body = &replacingReader{ - src: resp.Body, - from: []byte(`href="/simple/`), - to: []byte(`href="` + h.proxyURL + `/pypi/simple/`), + src: resp.Body, + replacements: []replacement{ + {from: []byte(`href="` + prefix), to: []byte(`href="` + to)}, + {from: []byte(`href='` + prefix), to: []byte(`href='` + to)}, + }, } resp.Header.Del(headerContentLength) + resp.Header.Del("ETag") // it names the upstream body resp.ContentLength = -1 } @@ -1045,15 +1054,28 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s const replacingReaderBufferSize = 32 << 10 -// replacingReader replaces every occurrence of from with to in src as it is -// read. +// replacingReader makes each replacement in src as it is read. type replacingReader struct { - src io.ReadCloser - from, to []byte - in []byte // read from src, not yet scanned - out []byte // scanned, not yet returned - buf []byte - err error + src io.ReadCloser + replacements []replacement + in []byte // read from src, not yet scanned + out []byte // scanned, not yet returned + buf []byte + err error +} + +type replacement struct{ from, to []byte } + +// next finds the earliest replacement in b, returning its index or -1. +func (r *replacingReader) next(b []byte) (int, *replacement) { + at, found := -1, (*replacement)(nil) + for i := range r.replacements { + rep := &r.replacements[i] + if j := bytes.Index(b, rep.from); j >= 0 && (at < 0 || j < at) { + at, found = j, rep + } + } + return at, found } func (r *replacingReader) Read(p []byte) (int, error) { @@ -1068,21 +1090,23 @@ func (r *replacingReader) Read(p []byte) (int, error) { r.in = append(r.in, r.buf[:n]...) r.err = err // Keep a tail that could be the start of a match split across reads. - keep := len(r.from) - 1 - if err != nil { - keep = 0 + keep := 0 + if err == nil { + for _, rep := range r.replacements { + keep = max(keep, len(rep.from)-1) + } } scan := max(len(r.in)-keep, 0) var out []byte done := 0 for { - i := bytes.Index(r.in[done:], r.from) + i, rep := r.next(r.in[done:]) if i < 0 || done+i >= scan { break } out = append(out, r.in[done:done+i]...) - out = append(out, r.to...) - done += i + len(r.from) + out = append(out, rep.to...) + done += i + len(rep.from) } if done < scan { out = append(out, r.in[done:scan]...) diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index 73309cb..c34679d 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -616,45 +616,62 @@ func TestPyPIDownloadCooldownMetadataCache(t *testing.T) { } func TestPyPISimpleIndexRewritesProjectLinks(t *testing.T) { - // Long enough that links cross the rewriter's read boundaries. - var upstream, want strings.Builder - upstream.WriteString(``) - want.WriteString(``) - for i := range 3000 { - fmt.Fprintf(&upstream, `pkg-%d`+"\n", i, i) - fmt.Fprintf(&want, `pkg-%d`+"\n", i, i) - } - upstream.WriteString(``) - want.WriteString(``) + for _, tc := range []struct{ name, upstream, prefix string }{ + {"root", "https://pypi.test", "/simple/"}, + {"subpath", "https://nexus.test/repository/pypi", "/repository/pypi/simple/"}, + } { + t.Run(tc.name, func(t *testing.T) { + // Long enough that links cross the rewriter's read boundaries. + var upstream, want strings.Builder + upstream.WriteString(``) + want.WriteString(``) + for i := range 3000 { + fmt.Fprintf(&upstream, `pkg-%d`+"\n", tc.prefix, i, i) + fmt.Fprintf(&want, `pkg-%d`+"\n", i, i) + } + fmt.Fprintf(&upstream, `q`, tc.prefix) + want.WriteString(`q`) + upstream.WriteString(``) + want.WriteString(``) - h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { - resp := pypiHTTPResponse(r, "text/html", upstream.String()) - resp.Header.Set("Content-Length", strconv.Itoa(upstream.Len())) - return resp, nil - }) + h, _ := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + resp := pypiHTTPResponse(r, "text/html", upstream.String()) + resp.Header.Set("Content-Length", strconv.Itoa(upstream.Len())) + resp.Header.Set("ETag", `"upstream"`) + return resp, nil + }) + h.upstreamURL = tc.upstream - req := httptest.NewRequest(http.MethodGet, "/simple/", nil) - w := httptest.NewRecorder() - h.Routes().ServeHTTP(w, req) + req := httptest.NewRequest(http.MethodGet, "/simple/", nil) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200", w.Code) - } - if got := w.Body.String(); got != want.String() { - t.Errorf("body not rewritten; starts %q", got[:min(len(got), 120)]) - } - if got := w.Header().Get("Content-Length"); got != "" && got != strconv.Itoa(want.Len()) { - t.Errorf("Content-Length = %q, want %d or unset", got, want.Len()) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + if got := w.Body.String(); got != want.String() { + t.Errorf("body not rewritten; starts %q", got[:min(len(got), 120)]) + } + if got := w.Header().Get("Content-Length"); got != "" && got != strconv.Itoa(want.Len()) { + t.Errorf("Content-Length = %q, want %d or unset", got, want.Len()) + } + // The upstream's ETag names the body before the rewrite. + if got := w.Header().Get("ETag"); got != "" { + t.Errorf("ETag = %q, want none on a rewritten body", got) + } + }) } } func TestReplacingReaderSplitReads(t *testing.T) { - const in = `abxhref="/simplehref="/simple/` - const want = `abxhref="/simplehref="P/` + const in = `abxhref="/simplehref="/simple/href='/simple/y/'` + const want = `abxhref="/simplehref="P/href='Q/y/'` r := &replacingReader{ - src: io.NopCloser(iotest.OneByteReader(strings.NewReader(in))), - from: []byte(`href="/simple/`), - to: []byte(`href="P/`), + src: io.NopCloser(iotest.OneByteReader(strings.NewReader(in))), + replacements: []replacement{ + {from: []byte(`href="/simple/`), to: []byte(`href="P/`)}, + {from: []byte(`href='/simple/`), to: []byte(`href='Q/`)}, + }, } got, err := io.ReadAll(r) if err != nil || string(got) != want { From 6381daa4edaf8dc5968be076a76ac0a84a441ae8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:07:53 +0100 Subject: [PATCH 141/198] Test git key case folding on every file system The existing test only runs where the file system ignores case, so it was skipped on Linux, and it passed if the second spelling had no entry at all. Add a test that sets foldCase directly and checks both spellings share one entry, require the entry in the existing test, and work out the repository key once in acquire. --- internal/gitcache/cache.go | 7 ++++--- internal/gitcache/cache_test.go | 23 ++++++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index 6bc3c20..6680057 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -356,10 +356,11 @@ func (c *Cache) removeRepointed() { func (c *Cache) acquire(key string) *repo { c.mu.Lock() defer c.mu.Unlock() - rp := c.repos[c.repoKey(key)] + rk := c.repoKey(key) + rp := c.repos[rk] if rp == nil { - rp = newRepo(c.repoKey(key), filepath.Join(c.dir, filepath.FromSlash(key))) - c.repos[rp.key] = rp + rp = newRepo(rk, filepath.Join(c.dir, filepath.FromSlash(key))) + c.repos[rk] = rp } rp.users++ return rp diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 690b11e..6bef422 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -802,7 +802,7 @@ func TestCaseOnlyDifferentKeysShareOneRepository(t *testing.T) { rp := tc.acquire("up/REPO.git") defer tc.release(rp) - if other := tc.repo("up/repo.git"); other != nil && other != rp { + if other := tc.repo("up/repo.git"); other != rp { t.Error("up/REPO.git and up/repo.git are separate repositories") } tc.mu.Lock() @@ -813,3 +813,24 @@ func TestCaseOnlyDifferentKeysShareOneRepository(t *testing.T) { t.Error("repository in use as up/REPO.git was evicted") } } + +// TestFoldCaseKeysShareOneEntry checks key folding on any file system by +// setting foldCase directly. +func TestFoldCaseKeysShareOneEntry(t *testing.T) { + tc := newTestCache(t, newFakeUpstream(t).server.URL, nil) + tc.foldCase = true + + upper := tc.acquire("up/REPO.git") + defer tc.release(upper) + lower := tc.acquire("up/repo.git") + defer tc.release(lower) + if upper != lower { + t.Error("up/REPO.git and up/repo.git are separate repositories") + } + tc.mu.Lock() + n := len(tc.repos) + tc.mu.Unlock() + if n != 1 { + t.Errorf("%d repository entries, want 1", n) + } +} From 8c04642715f18fc18c7322da3ec5edfa4e698a47 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:07:57 +0100 Subject: [PATCH 142/198] Serve the cached SNAPSHOT file when upstream fails A SNAPSHOT file named without a timestamp was dropped from the cache before it was fetched again, so an unreachable upstream gave a 502 instead of the last build, concurrent requests could drop each other's fresh copy, and HEAD downloaded the file. Fetch it first under the coalescing key and let the new copy replace the cached one once stored. When the fetch fails for any reason but the file being missing or blocked, serve the cached copy. HEAD is answered from cache. --- internal/handler/handler.go | 40 +++++++++++++++------- internal/handler/maven.go | 15 ++++---- internal/handler/maven_consistency_test.go | 31 +++++++++++++++++ 3 files changed, 67 insertions(+), 19 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 8a86ea8..94876fb 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2616,21 +2616,37 @@ func (p *Proxy) getCachedArtifactWithUpstreamHash(ctx context.Context, pkgPURL, return nil, nil } -// forgetCachedArtifact clears the cached entry for a file upstream replaces -// in place, queueing its object for deletion, so the next request fetches -// it again. -func (p *Proxy) forgetCachedArtifact(ecosystem, name, version, filename string) { - if p.DB == nil { - return +// refetchArtifactFromURL fetches a file upstream replaces in place, which +// a cached copy can't stand in for while upstream answers. The new copy +// replaces the cached one only once it is stored, and the cached one is +// served when the fetch fails for any reason but the file being missing or +// blocked. Concurrent callers share one fetch. +func (p *Proxy) refetchArtifactFromURL(ctx context.Context, ecosystem, name, version, filename, downloadURL string) (*CacheResult, error) { + pkgPURL, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return nil, err } - versionPURL := canonicalVersionPURL(ecosystem, name, version) - record, err := p.DB.GetCachedArtifact(canonicalPackagePURL(ecosystem, name), versionPURL, filename) - if err != nil || record == nil { - return + // Streaming never serves a cached copy, and a denied version is refused + // there. + if p.StreamArtifacts || p.versionDenied(ecosystem, name, version) || p.Denylist.Denied(versionPURL) { + return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", "") } - if err := p.DB.DiscardArtifact(versionPURL, filename, record.StoragePath); err != nil { - p.Logger.Warn("failed to clear artifact cache record", "purl", versionPURL, "filename", filename, "error", err) + metrics.RecordArtifactCacheMiss(ecosystem) + key := artifactCoalesceKey(versionPURL, filename, downloadURL, "", "") + noRecheck := func() (artifacts.Artifact, string, bool) { return artifacts.Artifact{}, "", false } + result, err := p.coalesceFetch(ctx, key, noRecheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { + return p.fetchAndCacheFromURL(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, nil, "", "") + }) + if err == nil || errors.Is(err, ErrUpstreamNotFound) || errors.Is(err, ErrArtifactBlocked) || ctx.Err() != nil { + return result, err } + cached, cacheErr := p.checkCache(ctx, pkgPURL, versionPURL, filename) + if cacheErr != nil || cached == nil { + return nil, err + } + p.Logger.Warn("upstream fetch failed, serving cached copy", + "purl", versionPURL, "filename", filename, "error", err) + return cached, nil } // discardStaleArtifact clears the cached entry when its digest disagrees with diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 2f8814e..ecccce7 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -262,20 +262,21 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) cacheFilename := source.cacheFilename(filename) - // Each build replaces a SNAPSHOT file named without a timestamp, so the - // cached copy is dropped and the file fetched again, through the same - // denylist, cooldown and scanning as any other download. - if h.isMutableSnapshotFile(urlPath) { - h.proxy.forgetCachedArtifact("maven", name, version, cacheFilename) + // Each build replaces a SNAPSHOT file named without a timestamp, so a + // GET fetches it again rather than serving the cached copy. HEAD is + // answered from cache. + get := h.proxy.GetOrFetchArtifactFromURL + if h.isMutableSnapshotFile(urlPath) && r.Method != http.MethodHead { + get = h.proxy.refetchArtifactFromURL } - result, err := h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, upstreamURL) + result, err := get(ctx, "maven", name, version, cacheFilename, upstreamURL) if err != nil && source.fallbackURL != "" { if errors.Is(err, ErrUpstreamNotFound) { pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven artifact not found in primary upstream, trying Gradle Plugin Portal", "group", group, "artifact", artifact, "version", version, "filename", filename) - result, err = h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, pluginPortalURL) + result, err = get(ctx, "maven", name, version, cacheFilename, pluginPortalURL) } } if err != nil { diff --git a/internal/handler/maven_consistency_test.go b/internal/handler/maven_consistency_test.go index a97faf6..e548447 100644 --- a/internal/handler/maven_consistency_test.go +++ b/internal/handler/maven_consistency_test.go @@ -18,6 +18,8 @@ import ( type mavenGenerations struct { gen atomic.Int32 requests map[string]*atomic.Int32 + // down makes every request fail with 502. + down atomic.Bool } func (m *mavenGenerations) body(path string) string { @@ -34,6 +36,10 @@ func newMavenGenerations(t *testing.T, paths ...string) (*mavenGenerations, http if n := m.requests[r.URL.Path]; n != nil { n.Add(1) } + if m.down.Load() { + http.Error(w, "unavailable", http.StatusBadGateway) + return + } const metadata = "/com/example/lib/maven-metadata.xml" if r.URL.Path == metadata+".sha1" { sum := sha1.Sum([]byte(m.body(metadata))) //nolint:gosec // as above @@ -124,3 +130,28 @@ func TestMavenMutableSnapshotKeepsPolicyAndFallback(t *testing.T) { t.Errorf("denied SNAPSHOT = %d, want 403", w.Code) } } + +// TestMavenMutableSnapshotServesCacheWhenUpstreamFails checks that a +// SNAPSHOT file fetched again for each request is still served from cache +// while upstream is down, and that HEAD does not download it again. +func TestMavenMutableSnapshotServesCacheWhenUpstreamFails(t *testing.T) { + const snapshot = "/com/example/lib/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar" + m, h := newMavenGenerations(t, snapshot) + + built := getMaven(h, snapshot).Body.String() + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodHead, snapshot, nil)) + if w.Code != http.StatusOK || m.requests[snapshot].Load() != 1 { + t.Errorf("HEAD = %d after %d upstream requests, want 200 from cache", w.Code, m.requests[snapshot].Load()) + } + + m.down.Store(true) + if w := getMaven(h, snapshot); w.Code != http.StatusOK || w.Body.String() != built { + t.Errorf("SNAPSHOT while upstream is down = %d %q, want the cached build %q", w.Code, w.Body, built) + } + m.down.Store(false) + m.gen.Store(1) + if w := getMaven(h, snapshot); w.Body.String() != m.body(snapshot) { + t.Errorf("SNAPSHOT once upstream is back = %q, want the new build", w.Body) + } +} From e2ef2a8083d9dd4e2891154a77c308b6da066c74 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:06:12 +0100 Subject: [PATCH 143/198] Print ignored environment values from proxy mirror server.Start logs the warnings LoadFromEnv collects, but mirror never calls it, so an unparseable boolean there was dropped without a word. Print them to stderr before validating the config. stats and hosted don't load the config file and aren't affected. --- cmd/proxy/main.go | 9 +++++++++ cmd/proxy/main_test.go | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index f9ab849..7c51732 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -495,6 +495,7 @@ func runMirror() { os.Exit(1) } cfg.LoadFromEnv() + printEnvWarnings(cfg) if *storageURL != "" { cfg.Storage.URL = *storageURL @@ -825,6 +826,14 @@ func formatSize(bytes int64) string { } } +// printEnvWarnings reports environment values LoadFromEnv ignored. serve +// logs them from server.Start; mirror never gets there. +func printEnvWarnings(cfg *config.Config) { + for _, warning := range cfg.EnvWarnings() { + fmt.Fprintf(os.Stderr, "warning: %s\n", warning) + } +} + func loadConfig(path string) (*config.Config, error) { if path != "" { return config.Load(path) diff --git a/cmd/proxy/main_test.go b/cmd/proxy/main_test.go index 3f17c27..d5a020d 100644 --- a/cmd/proxy/main_test.go +++ b/cmd/proxy/main_test.go @@ -57,3 +57,24 @@ func TestServeHelpProcess(*testing.T) { os.Args = []string{"proxy", "serve", "-help"} main() } + +func TestMirrorLogsEnvWarnings(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorEnvWarningsProcess$") + // The negative size stops mirror before it opens any storage. + cmd.Env = append(os.Environ(), + "PROXY_TEST_MIRROR_ENV_WARNINGS=1", + "PROXY_CACHE_METADATA=maybe", + "PROXY_STORAGE_MAX_SIZE=-1") + output, _ := cmd.CombinedOutput() + if !strings.Contains(string(output), "PROXY_CACHE_METADATA") { + t.Errorf("mirror did not warn about PROXY_CACHE_METADATA:\n%s", output) + } +} + +func TestMirrorEnvWarningsProcess(*testing.T) { + if os.Getenv("PROXY_TEST_MIRROR_ENV_WARNINGS") != "1" { + return + } + os.Args = []string{"proxy", "mirror", "pkg:npm/lodash@4.17.21"} + main() +} From 7714869af16fa4edbae6a43ff30b2d3fb09b3703 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:06:28 +0100 Subject: [PATCH 144/198] Reject sizes smaller than one byte ParseSize truncated fractional sizes such as 0.5B to 0, and a storage.max_size of 0 means unlimited. Reject a non-zero size that would round down to 0 bytes. --- internal/config/config.go | 4 ++++ internal/config/config_test.go | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/config/config.go b/internal/config/config.go index a9fc9e2..7b20795 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2192,6 +2192,10 @@ func ParseSize(s string) (int64, error) { if !(size >= 0 && size < math.MaxInt64) { return 0, fmt.Errorf("invalid size %q: must be between 0 and %d bytes", s, int64(math.MaxInt64)) } + // max_size 0 means unlimited, so don't let a tiny size become 0. + if num != 0 && int64(size) == 0 { + return 0, fmt.Errorf("invalid size %q: less than one byte", s) + } return int64(size), nil } } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d89e300..ff8fc50 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -297,6 +297,8 @@ func TestParseSize(t *testing.T) { {"1TB", 1024 * 1024 * 1024 * 1024, false}, {"invalid", 0, true}, {"10XB", 0, true}, + {"0GB", 0, false}, + {"0.0MB", 0, false}, } for _, tt := range tests { @@ -2244,7 +2246,7 @@ func TestScannerEcosystems(t *testing.T) { } func TestParseSizeRejectsOutOfRange(t *testing.T) { - for _, s := range []string{"-5GB", "-1", "-0.5K", "NaNGB", "InfGB", "+InfB", "1e30GB", "9999999999TB"} { + for _, s := range []string{"-5GB", "-1", "-0.5K", "NaNGB", "InfGB", "+InfB", "1e30GB", "9999999999TB", "0.5B", "0.000000000001GB"} { if got, err := ParseSize(s); err == nil { t.Errorf("ParseSize(%q) = %d, want error", s, got) } From 7de58c523f21307ab829d6b2cd0606ecb0d9af3d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:08:40 +0100 Subject: [PATCH 145/198] Report errors reading submodules of a mirrored commit Looking for .gitmodules took any error from git cat-file as the file not being there, so a mirror whose context was cancelled reported success with no submodules. Use rev-parse --verify --quiet, whose exit status 1 means the path is missing, and return other errors. A .gitmodules git config can't parse failed the mirror of a commit that is cached all the same. Log a warning and skip its submodules instead, as an unserved submodule host is skipped. --- docs/configuration.md | 2 +- internal/gitcache/helpers_test.go | 3 +- internal/gitcache/mirror.go | 22 +++++++++++--- internal/gitcache/mirror_test.go | 50 +++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 1c01589..a1ced4b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1921,7 +1921,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. - Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name as the version fails too; give the commit ID. -- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log. +- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. ### Scheduled refresh diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go index 55b7333..1f18c5b 100644 --- a/internal/gitcache/helpers_test.go +++ b/internal/gitcache/helpers_test.go @@ -302,6 +302,7 @@ func (u *fakeUpstream) bare() string { type testCache struct { *Cache server *httptest.Server + logs *lockedBuffer } func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *testCache { @@ -333,7 +334,7 @@ func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *t server.Close() _ = c.Close() }) - return &testCache{Cache: c, server: server} + return &testCache{Cache: c, server: server, logs: logs} } // testFront serves /git/up/{repo}/info/refs and git-upload-pack from c, as diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index 5aaa6fb..d6e07f9 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -91,7 +91,7 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchW } var submodules []Submodule if commit != "" { - if submodules, err = c.submodules(ctx, rp.dir, commit); err != nil { + if submodules, err = c.submodules(ctx, key, rp.dir, commit); err != nil { return MirrorResult{}, err } } @@ -123,14 +123,28 @@ func (c *Cache) resolveTag(ctx context.Context, dir, tag string) (string, error) // submodules lists the submodules commit records: its gitlinks that // .gitmodules names a URL for. -func (c *Cache) submodules(ctx context.Context, dir, commit string) ([]Submodule, error) { - if c.command(ctx, nil, "-C", dir, "cat-file", "-e", commit+":.gitmodules").Run() != nil { +func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Submodule, error) { + err := c.command(ctx, nil, "-C", dir, "rev-parse", "--verify", "--quiet", "--end-of-options", commit+":.gitmodules").Run() + var exit *exec.ExitError + if errors.As(err, &exit) && exit.ExitCode() == 1 && ctx.Err() == nil { return nil, nil } + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("looking for .gitmodules: %w", err) + } // --list rather than --get-regexp, which fails when nothing matches. out, err := c.run(ctx, dir, "config", "-z", "--blob", commit+":.gitmodules", "--list") if err != nil { - return nil, fmt.Errorf("reading .gitmodules: %w", err) + if ctx.Err() != nil { + return nil, ctx.Err() + } + // The commit is cached; a .gitmodules git can't read only loses + // its submodules. + c.logger.Warn("git cache: cannot read .gitmodules, skipping submodules", "repository", key, "commit", commit, "error", err) + return nil, nil } paths, urls := map[string]string{}, map[string]string{} for entry := range strings.SplitSeq(out, "\x00") { diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 1f85a6c..e57e64f 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -315,3 +315,53 @@ func TestMirrorWithEmptyGitmodules(t *testing.T) { } } } + +// TestSubmodulesReportsCancel checks a cancelled context is reported, not +// read as a commit without .gitmodules. +func TestSubmodulesReportsCancel(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + tc := newTestCache(t, upstream.server.URL, nil) + + if _, err := tc.submodules(ctx, "up/repo.git", work, commit); !errors.Is(err, context.Canceled) { + t.Errorf("submodules with a cancelled context = %v, want context.Canceled", err) + } +} + +// TestMirrorSkipsBrokenGitmodules checks a .gitmodules git can't read, one +// with a syntax error or committed as a symlink, lists no submodules and +// logs a warning instead of failing the mirror. +func TestMirrorSkipsBrokenGitmodules(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + gitmodules := filepath.Join(work, ".gitmodules") + if err := os.WriteFile(gitmodules, []byte("[submodule \"x\"\n\tpath = x\n"), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "-m", "broken gitmodules") + if err := os.Remove(gitmodules); err != nil { + t.Fatal(err) + } + if err := os.Symlink("../outside/.gitmodules", gitmodules); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", ".gitmodules") + runGit(t, work, "commit", "--quiet", "-m", "symlinked gitmodules") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + + for _, rev := range []string{"HEAD~1", "HEAD"} { + commit := runGit(t, work, "rev-parse", rev) + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit, true) + if err != nil || len(got.Submodules) != 0 { + t.Errorf("mirror of %s = %+v, %v, want no submodules", rev, got, err) + } + if !strings.Contains(tc.logs.String(), "commit="+commit) { + t.Errorf("no warning naming %s in the log", commit) + } + } +} From f4d074d7c471d45131cdbbc05668d5a18c16b5a0 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:08:52 +0100 Subject: [PATCH 146/198] Normalise ecosystems the scanner group is asked about applicable compared the caller's ecosystem raw against the normalised configured keys, so a caller passing an alias such as go or rubygems skipped a scanner restricted to golang or gem. Normalise it first. NewGroup now also rejects unknown names the way Validate does, through a shared ScannerEcosystemKey, instead of building a scanner that can never match. --- internal/config/config.go | 16 +++++++++++++--- internal/scanner/group.go | 8 +++++++- internal/scanner/group_test.go | 15 ++++++++++++++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 7b20795..25536e7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -439,9 +439,9 @@ func (s *ScannerConfig) Validate() error { return fmt.Errorf("invalid mode %q (must be block or monitor)", s.Mode) } for i, eco := range s.Ecosystems { - key := NormalizeScannerEcosystem(eco) - if !slices.Contains(scannerEcosystems, key) { - return fmt.Errorf("unknown ecosystem %q (valid: %s)", eco, strings.Join(scannerEcosystems, ", ")) + key, err := ScannerEcosystemKey(eco) + if err != nil { + return err } s.Ecosystems[i] = key } @@ -477,6 +477,16 @@ func NormalizeScannerEcosystem(eco string) string { return eco } +// ScannerEcosystemKey is NormalizeScannerEcosystem that also rejects +// names no handler uses. +func ScannerEcosystemKey(eco string) (string, error) { + key := NormalizeScannerEcosystem(eco) + if !slices.Contains(scannerEcosystems, key) { + return "", fmt.Errorf("unknown ecosystem %q (valid: %s)", eco, strings.Join(scannerEcosystems, ", ")) + } + return key, nil +} + // StorageConfig configures artifact storage. type StorageConfig struct { // URL is the storage backend URL. diff --git a/internal/scanner/group.go b/internal/scanner/group.go index 3066219..a4856e0 100644 --- a/internal/scanner/group.go +++ b/internal/scanner/group.go @@ -67,7 +67,11 @@ func NewGroup(cfg config.ScanningConfig, logger *slog.Logger) (*Group, error) { if len(sc.Ecosystems) > 0 { ecosystems = make(map[string]struct{}, len(sc.Ecosystems)) for _, eco := range sc.Ecosystems { - ecosystems[config.NormalizeScannerEcosystem(eco)] = struct{}{} + key, err := config.ScannerEcosystemKey(eco) + if err != nil { + return nil, fmt.Errorf("scanner %q: %w", sc.Name, err) + } + ecosystems[key] = struct{}{} } } @@ -112,6 +116,8 @@ func (g *Group) Timeout() time.Duration { } func (g *Group) applicable(ecosystem string) []entry { + // Callers may pass an alias such as go from a PURL type. + ecosystem = config.NormalizeScannerEcosystem(ecosystem) var out []entry for _, e := range g.entries { if len(e.ecosystems) == 0 { diff --git a/internal/scanner/group_test.go b/internal/scanner/group_test.go index d80166b..5dca201 100644 --- a/internal/scanner/group_test.go +++ b/internal/scanner/group_test.go @@ -306,9 +306,22 @@ func TestGroup_NormalisesEcosystemAliases(t *testing.T) { if err != nil { t.Fatalf("NewGroup() error: %v", err) } - for _, ecosystem := range []string{"golang", "oci", "gem"} { + for _, ecosystem := range []string{"golang", "oci", "gem", "go", "Docker", "rubygems"} { if !g.Blocks(ecosystem) { t.Errorf("Blocks(%s) = false, want true", ecosystem) } } } + +func TestNewGroup_RejectsUnknownEcosystem(t *testing.T) { + _, err := NewGroup(config.ScanningConfig{ + Enabled: true, + SigningKey: "test-signing-key", + Scanners: []config.ScannerConfig{ + {Name: "clamav", URL: "http://clamav.invalid", Ecosystems: []string{"npm", "pypy"}}, + }, + }, discardLogger()) + if err == nil || !strings.Contains(err.Error(), "pypy") { + t.Errorf("NewGroup() error = %v, want unknown ecosystem pypy", err) + } +} From c7031e4897e7fef6a826df37693be48597cc33b4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:09:23 +0100 Subject: [PATCH 147/198] Track mirrored manifests by sha256 whatever digest names them Mirroring img:v1@sha512: compared the sha512 digest with the sha256 digest the tag resolved to, so an unmoved tag was reported as moved and its manifest walked twice. Key seen manifests and the returned digest by sha256, as tags are. --- internal/handler/container_mirror.go | 5 ++- internal/handler/container_mirror_test.go | 39 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/internal/handler/container_mirror.go b/internal/handler/container_mirror.go index e17311c..f19ac73 100644 --- a/internal/handler/container_mirror.go +++ b/internal/handler/container_mirror.go @@ -332,7 +332,7 @@ func (m *imageMirror) requestTag(ctx context.Context, method, tag, accept string // mirrorManifest stores the manifest for reference through the handler's own // manifest route, then follows what it references. It returns the manifest -// digest. +// sha256 digest. func (m *imageMirror) mirrorManifest(ctx context.Context, reference string, depth int) (string, error) { cacheAccept := normalizeContainerManifestAccept(containerManifestAccept(&http.Request{Header: http.Header{}})) before, _ := m.h.loadContainerManifestForAccept(ctx, m.registryURL, m.upstreamName, reference, cacheAccept, cacheAccept) @@ -360,6 +360,9 @@ func (m *imageMirror) mirrorManifest(ctx context.Context, reference string, dept if before == nil || before.contentDigest != digest { m.result.Fetched++ } + // Tags resolve to sha256 digests, so a manifest asked for by another + // digest is tracked by its sha256 too. + digest = sha256Digest(response.body.Bytes()) if m.seen[digest] { return digest, nil } diff --git a/internal/handler/container_mirror_test.go b/internal/handler/container_mirror_test.go index f3a15ee..34af48c 100644 --- a/internal/handler/container_mirror_test.go +++ b/internal/handler/container_mirror_test.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "slices" "strconv" "strings" "sync" @@ -36,6 +37,9 @@ type fakeImageRegistry struct { truncateManifests atomic.Bool // onBlob, if set, runs before a blob is served. onBlob func(path string) + // sha512Digests makes manifests requested by a sha512 digest name that + // digest in Docker-Content-Digest, as registries do. + sha512Digests bool } type fakeImage struct { @@ -129,6 +133,9 @@ func newFakeImageRegistry(t *testing.T) (*fakeImageRegistry, fakeImage) { if ok { w.Header().Set("Content-Type", doc.contentType) w.Header().Set("Docker-Content-Digest", sha256Digest([]byte(doc.body))) + if _, ref, _ := strings.Cut(req.URL.Path, "/manifests/"); r.sha512Digests && strings.HasPrefix(ref, "sha512:") { + w.Header().Set("Docker-Content-Digest", ref) + } if req.Method == http.MethodGet && !strings.Contains(req.URL.Path, "/manifests/sha256:") { r.tagGets.Add(1) } @@ -644,3 +651,35 @@ func TestMirrorImageAcceptsSHA512Blobs(t *testing.T) { _ = cached.Reader.Close() } } + +// TestMirrorImageSHA512ReferenceMatchesTag checks that mirroring a tag +// pinned by a sha512 digest compares it with the tag's manifest by sha512, +// so an unmoved tag is neither reported as moved nor counted twice. +func TestMirrorImageSHA512ReferenceMatchesTag(t *testing.T) { + mirror := func(t *testing.T, sha512Ref bool) ImageResult { + registry, image := newFakeImageRegistry(t) + reference := image.singleDigest + if sha512Ref { + body := registry.manifests["/v2/team/app/manifests/single"].body + sum := sha512.Sum512([]byte(body)) + reference = "sha512:" + hex.EncodeToString(sum[:]) + registry.manifests["/v2/team/app/manifests/"+reference] = registry.manifests["/v2/team/app/manifests/single"] + registry.sha512Digests = true + } + proxy := newMirrorTestProxy(t, registry) + h := NewContainerHandlerWithRegistry(proxy, "http://proxy.example", registry.server.URL) + result, err := h.MirrorImage(context.Background(), ImageRef{Name: "team/app", References: []string{reference, "single"}}, nil) + if err != nil { + t.Fatalf("MirrorImage: %v", err) + } + return result + } + want := mirror(t, false) + got := mirror(t, true) + if !slices.Equal(got.Warnings, want.Warnings) { + t.Errorf("warnings = %v, want %v as for a sha256 reference", got.Warnings, want.Warnings) + } + if got.Fetched != want.Fetched { + t.Errorf("fetched = %d, want %d as for a sha256 reference", got.Fetched, want.Fetched) + } +} From 478d59ddd8e9648001b92cb632247a14d35e961a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:09:24 +0100 Subject: [PATCH 148/198] Validate a 304 copy before marking it fresh The refresh of a row confirmed by a 304 had moved ahead of validate, so a stored document validate then rejected was still recorded as fetched now. Validate first, as before. --- internal/handler/handler.go | 15 ++++++--- internal/handler/metadata_consistency_test.go | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 94876fb..7d7dd73 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1782,18 +1782,23 @@ func (p *Proxy) refreshMetadata(ctx context.Context, ecosystem, cacheKey, upstre // and falls back to the cached copy if upstream fails. entry is the row to // revalidate against, or nil. func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error, entry *database.MetadataCacheEntry) metadataResult { - meta, err := p.fetchUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) + fetch := func(entry *database.MetadataCacheEntry) (*upstreamMetadata, error) { + meta, err := p.fetchUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) + if err == nil && validate != nil { + err = validate(meta.body) + } + return meta, err + } + meta, err := fetch(entry) var digest string + // Validated first, so a 304 for a rejected copy doesn't mark it fresh. if err == nil && p.CacheMetadata { digest, err = p.refreshMetadataBlob(ecosystem, cacheKey, meta) } if errors.Is(err, errStale304) { // 304 but the cached copy is gone or was replaced; retry without // validators. - meta, err = p.fetchUpstreamMetadata(ctx, upstreamURL, nil, accept, acceptEncoding) - } - if err == nil && validate != nil { - err = validate(meta.body) + meta, err = fetch(nil) } if err == nil { if p.CacheMetadata && digest == "" { diff --git a/internal/handler/metadata_consistency_test.go b/internal/handler/metadata_consistency_test.go index 7384a10..35a5ea8 100644 --- a/internal/handler/metadata_consistency_test.go +++ b/internal/handler/metadata_consistency_test.go @@ -201,3 +201,34 @@ func TestNotModifiedAfterRowReplaced(t *testing.T) { t.Errorf("row etag = %q, want %q", row.ETag.String, `"v2"`) } } + +// TestNotModifiedRejectedBodyNotRefreshed checks that a 304 for a stored +// document validate rejects does not mark that document fresh. +func TestNotModifiedRejectedBodyNotRefreshed(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Minute + seedMetadataRow(t, db, store, `{"v":1}`, `"v1"`) + before, err := db.GetMetadataCache("test", "pkg") + if err != nil || before == nil { + t.Fatal(before, err) + } + upstream := v2Upstream(t, func() {}) + proxy.HTTPClient = upstream.Client() + + reject := func(body []byte) error { + if string(body) == `{"v":1}` { + return errors.New("damaged") + } + return nil + } + proxy.fetchMetadataDocument(context.Background(), "test", "pkg", upstream.URL+"/pkg", contentTypeJSON, reject) + row, err := db.GetMetadataCache("test", "pkg") + if err != nil || row == nil { + t.Fatal(row, err) + } + if !row.FetchedAt.Time.Equal(before.FetchedAt.Time) { + t.Errorf("rejected v1 row fetched_at moved from %v to %v", before.FetchedAt.Time, row.FetchedAt.Time) + } +} + From b6a1608eca186dca1acfa1ea4d7a37d8ca041e3a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:09:25 +0100 Subject: [PATCH 149/198] Drop validators from a stale copy replaced after its row With metadata_ttl 0 another instance can replace the blob between reading the row and reading the blob, and the fallback then served the new bytes under the old row's ETag, so a client holding that ETag got a wrong 304. Serve the blob without the row's validators when it doesn't match the row's digest. --- internal/handler/handler.go | 8 +++- internal/handler/metadata_consistency_test.go | 39 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 7d7dd73..acbd0bd 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1836,7 +1836,13 @@ func (p *Proxy) staleMetadata(ctx context.Context, ecosystem, cacheKey string, v p.Logger.Info("serving metadata from cache", "ecosystem", ecosystem, "key", cacheKey) - return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true}.describedBy(entry) + res := metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true} + if !matchesEntryDigest(entry, data) { + // Replaced since the row was read; the row's validators are not + // this blob's. + return res + } + return res.describedBy(entry) } // metadataResult is what a metadata lookup hands back. A shared fetch gives diff --git a/internal/handler/metadata_consistency_test.go b/internal/handler/metadata_consistency_test.go index 35a5ea8..115aeba 100644 --- a/internal/handler/metadata_consistency_test.go +++ b/internal/handler/metadata_consistency_test.go @@ -4,8 +4,10 @@ import ( "context" "database/sql" "errors" + "io" "net/http" "net/http/httptest" + "sync" "sync/atomic" "testing" "time" @@ -232,3 +234,40 @@ func TestNotModifiedRejectedBodyNotRefreshed(t *testing.T) { } } +// openHookStorage runs onOpen before the first Open. +type openHookStorage struct { + *mockStorage + once sync.Once + onOpen func() +} + +func (s *openHookStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) { + s.once.Do(s.onOpen) + return s.mockStorage.Open(ctx, path) +} + +// TestStaleMetadataChecksStoredBytes checks that the stale fallback does not +// serve a blob another instance replaced after the row was read under that +// row's ETag. +func TestStaleMetadataChecksStoredBytes(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + t.Cleanup(upstream.Close) + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 + proxy.HTTPClient = upstream.Client() + path := seedMetadataRow(t, db, store, `{"v":1}`, `"v1"`) + proxy.Storage = &openHookStorage{mockStorage: store, onOpen: func() { + // The other instance has written v2 but not yet its row. + store.mu.Lock() + store.files[path] = []byte(`{"v":2}`) + store.mu.Unlock() + }} + + w := proxyCachedRequest(proxy, upstream.URL, http.Header{"If-None-Match": {`"v1"`}}) + if w.Code == http.StatusNotModified || w.Header().Get(headerETag) == `"v1"` { + t.Errorf("response = %d ETag %q for v2's bytes, want no v1 validators", w.Code, w.Header().Get(headerETag)) + } +} From 989697596f9f4fa07e80aa74c198094d25e8bbba Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:09:25 +0100 Subject: [PATCH 150/198] Accept uppercase commit IDs and explain short ones when mirroring A version that wasn't a lowercase full-length hex ID was taken for a tag, so an uppercase commit ID failed as a missing tag. Lowercase a full-length hex version first. A shorter hex version that names no tag now fails saying it is neither a full commit ID nor a tag. --- docs/configuration.md | 2 +- internal/gitcache/mirror.go | 7 +++++++ internal/gitcache/mirror_test.go | 24 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index a1ced4b..0d81646 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1920,7 +1920,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. -- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name as the version fails too; give the commit ID. +- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID. - With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index d6e07f9..f884d37 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -37,6 +37,9 @@ type Submodule struct { // OpenOptions.FetchWants), and is kept as long as a commit a client fetched // by ID. A tag is cached with the repository's tags. func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchWants bool) (MirrorResult, error) { + if lower := strings.ToLower(version); isObjectID(lower) { + version = lower + } tag := version != "" && !isObjectID(version) if tag { if err := c.checkTagName(ctx, version); err != nil { @@ -78,6 +81,10 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchW // The tag may be one the failed refresh would have brought. return MirrorResult{}, refreshErr } + if errors.Is(err, ErrNotFound) && strings.Trim(strings.ToLower(version), "0123456789abcdef") == "" { + // Likely an abbreviated commit ID. + return MirrorResult{}, fmt.Errorf("%s is not a full commit ID or tag: %w", version, ErrNotFound) + } return MirrorResult{}, err } case version != "": diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index e57e64f..b1e27f6 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -365,3 +365,27 @@ func TestMirrorSkipsBrokenGitmodules(t *testing.T) { } } } + +func TestMirrorAcceptsUppercaseCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + got, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", strings.ToUpper(commit), true) + if err != nil || got.Commit != commit { + t.Errorf("mirror of the uppercase commit ID = %+v, %v, want commit %s", got, err, commit) + } +} + +func TestMirrorExplainsShortCommit(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + commit := runGit(t, work, "rev-parse", "HEAD") + tc := newTestCache(t, upstream.server.URL, nil) + + _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", commit[:12], true) + if !errors.Is(err, ErrNotFound) || !strings.Contains(err.Error(), "full commit ID") { + t.Errorf("mirror of a short commit ID = %v, want ErrNotFound saying it is no full commit ID", err) + } +} From ebc1c4646da20050e4c32b1a543124872488bb7e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:09:53 +0100 Subject: [PATCH 151/198] Mark the offline checksum database copy stale Offline mode serves the cached copy without asking the database once it is known to be unreachable, but derived staleness from the row's age. With metadata_ttl 0 that copy had no Warning header. It is always stale, since upstream was not asked. --- internal/handler/go_sumdb.go | 4 +++- internal/handler/go_sumdb_test.go | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/internal/handler/go_sumdb.go b/internal/handler/go_sumdb.go index 60bd02b..e96e5ab 100644 --- a/internal/handler/go_sumdb.go +++ b/internal/handler/go_sumdb.go @@ -383,7 +383,9 @@ func (h *GoHandler) proxySumDB(w http.ResponseWriter, r *http.Request, name, ups // the upstream timeout before the fetch falls back to it. if h.sumDBOffline && h.sumDBProbes.recentlyFailed(name) { if entry, body, contentType, ok := h.readCachedSumDB(r.Context(), cacheKey, check); ok { - writeMetadataBody(w, r, p.cachedMetaFor(entry), body, contentType, entry.ContentEncoding.String) + cm := p.cachedMetaFor(entry) + cm.stale = true // upstream was not asked + writeMetadataBody(w, r, cm, body, contentType, entry.ContentEncoding.String) return } } diff --git a/internal/handler/go_sumdb_test.go b/internal/handler/go_sumdb_test.go index 6a0c47b..d47e432 100644 --- a/internal/handler/go_sumdb_test.go +++ b/internal/handler/go_sumdb_test.go @@ -985,3 +985,26 @@ func TestGoSumDB_ClientRoundTrip(t *testing.T) { t.Fatalf("lookup from the proxy cache with the database down: %v", err) } } + +// TestGoSumDB_OfflineCopyMarkedStaleWithoutTTL: with metadata_ttl 0 the +// cached copy offline mode serves while the database is unreachable still +// carries the Warning header. +func TestGoSumDB_OfflineCopyMarkedStaleWithoutTTL(t *testing.T) { + const lookup = "/lookup/golang.org/x/text@v0.14.0" + db := newFakeSumDB(t, map[string]string{lookup: testSumDBRecord}) + h := newSumDBTestHandler(t, db.server.URL, true, 0) + h.sumDBOffline = true + + if w := serveGoRequest(h, "/sumdb/sum.example"+lookup); w.Code != http.StatusOK { + t.Fatalf("online: status = %d, want 200", w.Code) + } + db.down.Store(true) + serveGoRequest(h, "/sumdb/sum.example"+lookup) + if !h.sumDBProbes.recentlyFailed(testSumDBName) { + t.Fatal("the failure was not remembered") + } + w := serveGoRequest(h, "/sumdb/sum.example"+lookup) + if w.Code != http.StatusOK || w.Header().Get("Warning") != staleWarning { + t.Errorf("offline copy = %d Warning %q, want 200 %q", w.Code, w.Header().Get("Warning"), staleWarning) + } +} From 09a6b21facd2261f93f645f0b6b85f1f110cbb76 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:10:35 +0100 Subject: [PATCH 152/198] Pick the APT release file that lists the index The index lookup always used a cached InRelease over Release, so a stale InRelease decided which index was served to a client that had just fetched a fresh Release, and it stopped at the nearest cached release file even when that file, such as the legacy per-component Release, listed nothing for the path. Skip release files that don't list the path, and where both InRelease and Release do, use the one fetched last: apt reads InRelease and debootstrap Release, and a client asking for an index has most likely just fetched one of them. --- internal/handler/debian.go | 19 +++++- internal/handler/release_index.go | 3 +- internal/handler/release_index_test.go | 82 ++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/internal/handler/debian.go b/internal/handler/debian.go index af528e3..a386f8a 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -203,14 +203,27 @@ func (h *DebianHandler) handleMetadata( } // releaseListing finds the cached release file of the suite path belongs -// to, nearest directory first, and the sha256 it lists for path. +// to and the sha256 it lists for path, nearest directory first, skipping +// release files that list nothing for path (such as the legacy +// per-component Release). Where both InRelease and Release list it, the one +// fetched last wins: apt reads InRelease and debootstrap Release, and a +// client asking for the index has most likely just fetched one of them. func (h *DebianHandler) releaseListing(r *http.Request, repository, upstreamURL, path string) (*cachedRelease, string) { for dir := pathpkg.Dir(path); strings.HasPrefix(dir, "dists/"); dir = pathpkg.Dir(dir) { + var found *cachedRelease + var want string for _, name := range []string{"InRelease", "Release"} { release := h.proxy.readCachedRelease(r.Context(), "debian", h.metadataCacheKeyFor(repository, upstreamURL, dir+"/"+name)) - if release != nil { - return release, debianReleaseSHA256(release.body, strings.TrimPrefix(path, dir+"/")) + if release == nil { + continue } + sum := debianReleaseSHA256(release.body, strings.TrimPrefix(path, dir+"/")) + if sum != "" && (found == nil || release.fetchedAt.After(found.fetchedAt)) { + found, want = release, sum + } + } + if found != nil { + return found, want } } return nil, "" diff --git a/internal/handler/release_index.go b/internal/handler/release_index.go index 25f3515..debf5e5 100644 --- a/internal/handler/release_index.go +++ b/internal/handler/release_index.go @@ -30,6 +30,7 @@ type cachedRelease struct { ecosystem string cacheKey string body []byte + fetchedAt time.Time } // readCachedRelease returns the cached release file stored under cacheKey, @@ -46,7 +47,7 @@ func (p *Proxy) readCachedRelease(ctx context.Context, ecosystem, cacheKey strin if err != nil { return nil } - return &cachedRelease{ecosystem: ecosystem, cacheKey: cacheKey, body: body} + return &cachedRelease{ecosystem: ecosystem, cacheKey: cacheKey, body: body, fetchedAt: entry.FetchedAt.Time} } // proxyListedIndex serves an index file that release lists with the sha256 diff --git a/internal/handler/release_index_test.go b/internal/handler/release_index_test.go index ccbc62a..d1ca689 100644 --- a/internal/handler/release_index_test.go +++ b/internal/handler/release_index_test.go @@ -162,3 +162,85 @@ func TestIndexMismatchExpiresRelease(t *testing.T) { }) } } + +// debianSuite serves an APT suite with both InRelease and Release, and the +// legacy per-component Release that lists no checksums. +func debianSuite(t *testing.T) (*atomic.Int32, http.Handler, func(path string)) { + t.Helper() + const index = "main/binary-amd64/Packages" + var gen atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := indexBody(gen.Load()) + release := fmt.Sprintf("Suite: stable\nSHA256:\n %s %d %s\n", sha256Hex(body), len(body), index) + switch r.URL.Path { + case "/dists/stable/InRelease": + _, _ = w.Write([]byte("-----BEGIN PGP SIGNED MESSAGE-----\n\n" + release + "-----BEGIN PGP SIGNATURE-----\n")) + case "/dists/stable/Release": + _, _ = w.Write([]byte(release)) + case "/dists/stable/main/binary-amd64/Release": + _, _ = w.Write([]byte("Archive: stable\nComponent: main\nArchitecture: amd64\n")) + case "/dists/stable/" + index: + _, _ = w.Write([]byte(body)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + proxy, db, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + h := NewDebianHandler(proxy, "http://proxy.test", upstream.URL, nil) + expire := func(path string) { + t.Helper() + key := h.metadataCacheKeyFor("", h.upstreamURL, path) + if _, err := db.ExpireMetadataCache("debian", []string{key}, time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + } + return &gen, h.Routes(), expire +} + +// TestDebianIndexFollowsLatestReleaseFile checks that with both InRelease +// and Release cached, the index follows the one fetched last, which is what +// the client asking for the index just read. +func TestDebianIndexFollowsLatestReleaseFile(t *testing.T) { + const index = "dists/stable/main/binary-amd64/Packages" + gen, h, expire := debianSuite(t) + + getBody(t, h, "dists/stable/InRelease") + getBody(t, h, index) + gen.Store(1) + time.Sleep(time.Millisecond) + getBody(t, h, "dists/stable/Release") + if got := getBody(t, h, index); got != indexBody(1) { + t.Errorf("index after a fresh Release = %q, want %q", got, indexBody(1)) + } + + // And back: a fresh InRelease wins over the older Release. + gen.Store(2) + expire("dists/stable/InRelease") + time.Sleep(time.Millisecond) + getBody(t, h, "dists/stable/InRelease") + if got := getBody(t, h, index); got != indexBody(2) { + t.Errorf("index after a fresh InRelease = %q, want %q", got, indexBody(2)) + } +} + +// TestDebianIndexSkipsReleaseWithoutListing checks that a cached release +// file that lists nothing for the index, such as the legacy per-component +// Release, does not stop the lookup at the suite's release file. +func TestDebianIndexSkipsReleaseWithoutListing(t *testing.T) { + const index = "dists/stable/main/binary-amd64/Packages" + gen, h, expire := debianSuite(t) + + getBody(t, h, "dists/stable/InRelease") + getBody(t, h, "dists/stable/main/binary-amd64/Release") + getBody(t, h, index) + gen.Store(1) + expire("dists/stable/InRelease") + getBody(t, h, "dists/stable/InRelease") + if got := getBody(t, h, index); got != indexBody(1) { + t.Errorf("index after InRelease refetch = %q, want %q", got, indexBody(1)) + } +} From 6e143432bb1d1d3cb87ec8e2292829708f9d7bf0 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:10:35 +0100 Subject: [PATCH 153/198] Describe SNAPSHOT fallback and release file choice Document that a mutable SNAPSHOT file falls back to the cached copy and that HEAD is answered from cache, and that the most recently fetched of InRelease and Release decides which index is served. --- docs/configuration.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ebad0c7..6322d09 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1027,8 +1027,10 @@ cached metadata rather than fetched, so they always match it. Other files are cached for good, except SNAPSHOT files named with `SNAPSHOT` rather than a build timestamp (`lib-1.0-SNAPSHOT.jar`, as JitPack and non-unique snapshot repositories publish them), which each build replaces: those are fetched -again for every request, through the denylist, cooldown, scanning and the -Plugin Portal fallback like any other file. Timestamped SNAPSHOT files +again for every `GET`, through the denylist, cooldown, scanning and the +Plugin Portal fallback like any other file. The new copy replaces the cached +one once stored; while upstream can't be reached the cached copy is served, +and `HEAD` is answered from cache. Timestamped SNAPSHOT files (`lib-1.0-20240101.120000-1.jar`) never change and are cached. This field has no environment variable, as with the other named upstream maps. @@ -1511,7 +1513,7 @@ Set to `"0"` to always revalidate with upstream (ETag-based conditional requests When upstream is unreachable and the cached entry is past its TTL (with a `metadata_ttl` of `"0"`, on every such request), the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated. The `ETag` and `Last-Modified` sent with a document are always those of the bytes served, so a document fetched from upstream but not stored, for example because storage failed, carries the upstream's validators rather than the older cached copy's. -APT and RPM clients fetch a release file first (`InRelease` or `Release`, `repomd.xml`) and reject an index file whose checksum doesn't match it. So an index file the cached release file lists (`Packages`, `Sources`, a plainly named `primary.xml.gz`, …) does not follow its own `metadata_ttl`: its cached copy is served while it matches the cached release file's sha256, however old, and otherwise it is fetched from upstream. If upstream's copy doesn't match either, the repository has changed since the release file was cached; that copy is served and the release file is expired, so the next `apt update` or `dnf makecache` gets a matching pair. The release file itself follows `metadata_ttl`. Index files it doesn't list, such as `by-hash` paths, follow `metadata_ttl` too. +APT and RPM clients fetch a release file first (`InRelease` or `Release`, `repomd.xml`) and reject an index file whose checksum doesn't match it. So an index file the cached release file lists (`Packages`, `Sources`, a plainly named `primary.xml.gz`, …) does not follow its own `metadata_ttl`: its cached copy is served while it matches the cached release file's sha256, however old, and otherwise it is fetched from upstream. If upstream's copy doesn't match either, the repository has changed since the release file was cached; that copy is served and the release file is expired, so the next `apt update` or `dnf makecache` gets a matching pair. Where a suite has both `InRelease` and `Release` cached, the one fetched last decides. The release file itself follows `metadata_ttl`. Index files it doesn't list, such as `by-hash` paths, follow `metadata_ttl` too. ### Change feeds From fe764ea6408dcc05c7b005534664664cdcde7152 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:07:32 +0100 Subject: [PATCH 154/198] Clear a refusal once the upstream lists the refs A refusal was cleared only by a refresh that succeeded as a whole. When the upstream listed the refs again but the fetch that followed failed, say because the cache was full or the fetch timed out, the old refusal stayed and the repository was still refused, though the upstream had just confirmed access. Clear it as soon as the ref check succeeds. --- internal/gitcache/cache.go | 2 +- internal/gitcache/cache_test.go | 21 +++++++++++++ internal/gitcache/helpers_test.go | 50 ++++++++++++++++++++----------- internal/gitcache/refresh.go | 10 ++++--- internal/gitcache/serve.go | 2 +- 5 files changed, 61 insertions(+), 24 deletions(-) diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index 7901380..9a8628f 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -130,7 +130,7 @@ type repo struct { users int refreshing *refreshCall // refused is the error of the last refresh when the upstream reported - // the repository missing or refused it, until a refresh succeeds. + // the repository missing or refused it, until a ref check succeeds. refused error // writes serializes git fetches into the repository. writes chan struct{} diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index baf646a..931e6d5 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -922,3 +922,24 @@ func TestUploadPackClearsRefusal(t *testing.T) { t.Errorf("ls-refs after the upstream allowed the repository again: status %d", status) } } + +// TestRefCheckClearsRefusal has the upstream list the refs again after a +// refusal but fail the fetch that follows: the check alone clears the +// refusal, and the cached copy is served. +func TestRefCheckClearsRefusal(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + tc.refuse(t, upstream) + + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + upstream.fetchStatus.Store(http.StatusInternalServerError) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err != nil { + t.Errorf("cached copy refused after the upstream listed its refs: %v\n%s", err, out) + } + if status := tc.lsRefs(t); status != http.StatusOK { + t.Errorf("ls-refs after the upstream listed its refs: status %d", status) + } +} diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go index 55b7333..e5f1311 100644 --- a/internal/gitcache/helpers_test.go +++ b/internal/gitcache/helpers_test.go @@ -75,8 +75,10 @@ type fakeUpstream struct { root string server *httptest.Server - // status, when set, is returned instead of serving the request. - status atomic.Int32 + // status, when set, is returned instead of serving the request, and + // fetchStatus instead of serving git's own upload-pack requests. + status atomic.Int32 + fetchStatus atomic.Int32 // requireAuth, when set, is the Authorization every request must carry. requireAuth string // v0Only drops Git-Protocol, so only protocol v0 is spoken. @@ -178,22 +180,7 @@ func newFakeUpstream(t *testing.T) *fakeUpstream { if u.v0Only { r.Header.Del("Git-Protocol") } - if r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { - u.holdMu.Lock() - fetchHold, fetchArrived, fetchOnce := u.fetchHold, u.fetchArrived, u.fetchOnce - u.holdMu.Unlock() - if fetchHold != nil { - fetchOnce.Do(func() { close(fetchArrived) }) - <-fetchHold - } - } - if delay := u.delayFetches.Load(); delay > 0 && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { - time.Sleep(time.Duration(delay)) - } - if u.stallFetches.Load() && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { - // The server notices the client leaving only once the body is read. - _, _ = io.Copy(io.Discard, r.Body) - <-r.Context().Done() + if r.UserAgent() != gitUserAgent && r.Method == http.MethodPost && u.interceptFetch(w, r) { return } if r.Method == http.MethodPost { @@ -205,6 +192,33 @@ func newFakeUpstream(t *testing.T) *fakeUpstream { return u } +// interceptFetch applies the fetch knobs to one of git's own upload-pack +// requests, and reports whether it answered the request. +func (u *fakeUpstream) interceptFetch(w http.ResponseWriter, r *http.Request) bool { + u.holdMu.Lock() + fetchHold, fetchArrived, fetchOnce := u.fetchHold, u.fetchArrived, u.fetchOnce + u.holdMu.Unlock() + if fetchHold != nil { + fetchOnce.Do(func() { close(fetchArrived) }) + <-fetchHold + } + if delay := u.delayFetches.Load(); delay > 0 { + time.Sleep(time.Duration(delay)) + } + if u.stallFetches.Load() { + // The server notices the client leaving only once the body is read. + _, _ = io.Copy(io.Discard, r.Body) + <-r.Context().Done() + return true + } + if status := u.fetchStatus.Load(); status != 0 { + w.Header().Set("Location", u.server.URL+"/elsewhere") + http.Error(w, "unavailable", int(status)) + return true + } + return false +} + // holdFetches holds git's own upload-pack requests until release is called. // arrived is closed when the first one reaches the upstream. func (u *fakeUpstream) holdFetches() (arrived <-chan struct{}, release func()) { diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index d2f8ee4..786603b 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -157,10 +157,7 @@ func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { } c.mu.Lock() rp.refreshing = nil - switch { - case call.err == nil: - rp.refused = nil - case !servesStale(call.err): + if call.err != nil && !servesStale(call.err) { rp.refused = call.err } c.mu.Unlock() @@ -186,6 +183,11 @@ func (c *Cache) refresh(ctx context.Context, rp *repo, repoURL string, checked f if err != nil { return err } + // The upstream lets us read the repository again, even if the fetch + // below fails. + c.mu.Lock() + rp.refused = nil + c.mu.Unlock() if rp.exists() { err = c.update(ctx, rp, repoURL, remote, checked) } else { diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index 5a9f1e0..bda671f 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -98,7 +98,7 @@ func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) err } // refusal returns the error of the last refresh that found the repository -// gone or refused upstream, until a refresh succeeds. +// gone or refused upstream, until a ref check succeeds. func (c *Cache) refusal(rp *repo) error { c.mu.Lock() defer c.mu.Unlock() From 806003e1c112fb7ecd9c41d4a38bf690363e00ec Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:07:51 +0100 Subject: [PATCH 155/198] Saturate the upload-pack check age stale_max_age plus the upload-pack grace overflowed for a stale_max_age near the largest duration, which made every upload-pack request check the upstream. Cap the sum instead. --- internal/gitcache/cache_test.go | 21 +++++++++++++++++++++ internal/gitcache/serve.go | 8 +++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 931e6d5..dc70350 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "maps" + "math" "net/http" "net/http/httptest" "net/http/httputil" @@ -943,3 +944,23 @@ func TestRefCheckClearsRefusal(t *testing.T) { t.Errorf("ls-refs after the upstream listed its refs: status %d", status) } } + +// TestHugeStaleMaxAgeUploadPackChecksNothing sets stale_max_age near the +// largest duration: upload-pack requests on a fresh copy do not check the +// upstream. +func TestHugeStaleMaxAgeUploadPackChecksNothing(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.StaleMaxAge = math.MaxInt64 - time.Second }) + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + + before := upstream.requests.Load() + if status := tc.lsRefs(t); status != http.StatusOK { + t.Fatalf("ls-refs: status %d", status) + } + waitIdle(t, tc.Cache) + if got := upstream.requests.Load() - before; got != 0 { + t.Errorf("ls-refs made %d upstream requests, want 0", got) + } +} diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go index bda671f..5d80938 100644 --- a/internal/gitcache/serve.go +++ b/internal/gitcache/serve.go @@ -6,6 +6,7 @@ import ( "context" "errors" "io" + "math" "net/http" "os" "path/filepath" @@ -88,7 +89,12 @@ const uploadPackGrace = time.Minute // found it gone or refused upstream, or it was not checked recently, and // refuses it while that refusal stands. func (c *Cache) checkServable(ctx context.Context, rp *repo, repoURL string) error { - if c.refusal(rp) == nil && time.Since(time.Unix(0, rp.lastChecked.Load())) <= c.opts.StaleMaxAge+uploadPackGrace { + // The sum saturates rather than overflow for a huge StaleMaxAge. + maxAge := c.opts.StaleMaxAge + uploadPackGrace + if maxAge < c.opts.StaleMaxAge { + maxAge = math.MaxInt64 + } + if c.refusal(rp) == nil && time.Since(time.Unix(0, rp.lastChecked.Load())) <= maxAge { return nil } if err := c.ensureFresh(ctx, rp, repoURL); err != nil { From 06aef28125d80a48607818d58ecca6b694a97756 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:10:34 +0100 Subject: [PATCH 156/198] Refuse a repository whose fetch is redirected The relay turned a redirect into a failed fetch, which counted as the upstream failing rather than refusing, so the cached copy was still served and no refusal was recorded. Report it as a refusal, as for a redirected ref listing. --- internal/gitcache/cache_test.go | 20 ++++++++++++++++++++ internal/gitcache/refresh.go | 3 +++ internal/gitcache/relay.go | 4 ++++ 3 files changed, 27 insertions(+) diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index dc70350..9fd5449 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -964,3 +964,23 @@ func TestHugeStaleMaxAgeUploadPackChecksNothing(t *testing.T) { t.Errorf("ls-refs made %d upstream requests, want 0", got) } } + +// TestFetchRedirectRefusesRepository has the upstream list the refs but +// redirect the fetch that follows: the repository is refused, as when the +// ref listing is redirected. +func TestFetchRedirectRefusesRepository(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + upstream.fetchStatus.Store(http.StatusFound) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("served the cached copy after the fetch was redirected:\n%s", out) + } + if status := tc.lsRefs(t); status == http.StatusOK { + t.Errorf("ls-refs served after the fetch was redirected") + } +} diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index 786603b..f467a3d 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -326,6 +326,9 @@ func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, res if route.full.Load() { return ErrCacheFull } + if status := route.redirected.Load(); status != 0 { + return upstreamStatusError(int(status)) + } if err != nil && !route.answered.Load() { return fmt.Errorf("%w: %w", errUpstreamUnanswered, err) } diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go index ed42001..bd65b25 100644 --- a/internal/gitcache/relay.go +++ b/internal/gitcache/relay.go @@ -53,6 +53,9 @@ type relayRoute struct { // answered records that the upstream answered a git-upload-pack // request, so a failed fetch was not an outage. answered atomic.Bool + // redirected records the status of a redirect the upstream answered + // with, 0 for none. + redirected atomic.Int32 } func newRelay(client *http.Client, logger *slog.Logger) (*relay, error) { @@ -151,6 +154,7 @@ func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer func() { _ = resp.Body.Close() }() // A redirect passed on would send git to the Location itself. if resp.StatusCode >= http.StatusMultipleChoices && resp.StatusCode < http.StatusBadRequest { + route.redirected.Store(int32(resp.StatusCode)) http.Error(w, "upstream redirect not followed", http.StatusBadGateway) return } From 7fdfd9908fbc35af46b3d6f6795aa6dfacb48f63 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:10:43 +0100 Subject: [PATCH 157/198] Refetch npm packuments only when a cached copy lacks the version Under cooldown, a tarball for a version the packument doesn't list is decided on a fresh fetch of the packument. That check had gaps: - A stale copy served because upstream failed counted as fetched, so the version went through with no publish time. It now triggers the refetch like a copy within its TTL. - A packument just stored from upstream counted as cached, so every request for an unlisted version fetched it twice. Only a stored row read within its TTL counts now. - A stored refetch that joined another caller's fetch could get that caller's stale fallback and take it as upstream's answer. The stored lookup now reports stale results, and a stale refetch is refused. - A refused refetch dropped the upstream error, so a rate-limit hold answered 502 instead of 503 with Retry-After, and a 404 was lost. Stale results now carry the upstream error, and the refusal wraps it. --- internal/handler/handler.go | 7 +- internal/handler/metadata_stored.go | 30 +++-- internal/handler/npm.go | 79 +++++++------ internal/handler/npm_tarball_test.go | 166 +++++++++++++++++++++++++++ 4 files changed, 230 insertions(+), 52 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d316aab..be818f2 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1829,7 +1829,7 @@ func (p *Proxy) staleMetadata(ctx context.Context, ecosystem, cacheKey string, v p.Logger.Info("serving metadata from cache", "ecosystem", ecosystem, "key", cacheKey) - return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true}.describedBy(entry) + return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true, upstreamErr: err}.describedBy(entry) } // metadataResult is what a metadata lookup hands back. A shared fetch gives @@ -1843,8 +1843,9 @@ type metadataResult struct { digest string err error // stale is set when the body is the cached copy served because upstream - // failed. - stale bool + // failed, and upstreamErr is that failure. + stale bool + upstreamErr error // fromCache is set when the body is a stored copy within its TTL, read // without asking upstream. fromCache bool diff --git a/internal/handler/metadata_stored.go b/internal/handler/metadata_stored.go index ffe524d..fc16d43 100644 --- a/internal/handler/metadata_stored.go +++ b/internal/handler/metadata_stored.go @@ -45,30 +45,26 @@ var errStoredMetadataChanged = errors.New("stored metadata changed while it was // It returns errMetadataNotStored when the proxy doesn't cache metadata or // storing the document failed, and the caller reads it into memory instead. func (p *Proxy) storedMetadataDocument(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error) (*database.MetadataCacheEntry, error) { - return p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, "") + res := p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, "") + return res.entry, res.err } -// restoreStoredMetadata is storedMetadataDocument for a document whose row -// gone describes but whose stored copy was missing when read: the row is not -// served or revalidated, within its TTL or not, and the document is fetched -// again. -func (p *Proxy) restoreStoredMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone *database.MetadataCacheEntry) (*database.MetadataCacheEntry, error) { - return p.storedMetadata(ctx, ecosystem, cacheKey, upstreamURL, accept, validate, gone.ContentDigest.String) -} - -// storedMetadata implements storedMetadataDocument. A row with the digest -// gone is treated as if there were none. -func (p *Proxy) storedMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone string) (*database.MetadataCacheEntry, error) { +// storedMetadata implements storedMetadataDocument, and reports in the +// result whether the row was read from cache or served stale. A row with +// the digest gone, one whose stored copy was missing or out of date when +// read, is not served or revalidated, within its TTL or not, and the +// document is fetched again. +func (p *Proxy) storedMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept string, validate func(io.Reader) error, gone string) metadataResult { if !p.CacheMetadata || p.DB == nil || p.Storage == nil || p.MetadataTTL <= 0 { - return nil, errMetadataNotStored + return metadataResult{err: errMetadataNotStored} } if containsPathTraversal(cacheKey) { - return nil, fmt.Errorf("invalid cache key: %q", cacheKey) + return metadataResult{err: fmt.Errorf("invalid cache key: %q", cacheKey)} } if entry, fresh := p.storedMetadataEntryState(ctx, ecosystem, cacheKey, gone); fresh { metrics.RecordMetadataCacheHit(ecosystem, int(entry.Size.Int64)) p.recordMetadataRead(entry) - return entry, nil + return metadataResult{entry: entry, fromCache: true} } p.recordMetadataCacheMiss(ecosystem) @@ -102,7 +98,7 @@ func (p *Proxy) storedMetadata(ctx context.Context, ecosystem, cacheKey, upstrea res = p.fetchStoredMetadata(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, validate, entry) return res }) - return res.entry, res.err + return res } // storedMetadataEntryState reads the cache row of a document and reports @@ -157,7 +153,7 @@ func (p *Proxy) staleStoredMetadata(ecosystem, cacheKey string, entry *database. if !usableStoredMetadata(entry) { return metadataResult{err: fmt.Errorf("upstream failed and no usable cached metadata: %w", err)} } - return metadataResult{entry: entry, stale: true} + return metadataResult{entry: entry, stale: true, upstreamErr: err} } // storeUpstreamMetadata fetches a document and streams it into storage, and diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 0e9f32a..7eef424 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -344,7 +344,7 @@ func (h *NPMHandler) serveStreamedPackument(w http.ResponseWriter, r *http.Reque fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) var rewritten rewrittenMetadata - err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, false, func(entry *database.MetadataCacheEntry) (err error) { + err, fetchErr := h.readStoredPackument(r.Context(), upstreamURL, accept, metadataCacheKey, func(entry *database.MetadataCacheEntry) (err error) { rewritten, err = h.proxy.streamedRewrite(r.Context(), "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { var scan *npm.Scan err := readStored(open, func(in io.Reader) (err error) { @@ -409,27 +409,24 @@ func (h *NPMHandler) writeFetchError(w http.ResponseWriter, err error) { // document can't be read as a stream. var errInMemory = errors.New("npm packument needs the in-memory path") -// storedPackument returns the cache row of the packument stored for cacheKey, -// fetching it into storage first when needed. It returns errInMemory when +// storedPackument returns the lookup of the packument stored for cacheKey, +// fetching it into storage first when needed. Its err is errInMemory when // metadata isn't cached in storage, and ErrUpstreamNotFound and upstream // errors as they come. // With refresh, a stored copy within its TTL is fetched again rather than // served. -func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool) (*database.MetadataCacheEntry, error) { +func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool) metadataResult { + var gone string if refresh && h.proxy.DB != nil { if row, _ := h.proxy.DB.GetMetadataCache("npm", cacheKey); usableStoredMetadata(row) { - entry, err := h.proxy.restoreStoredMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), row) - if errors.Is(err, errMetadataNotStored) { - return nil, errInMemory - } - return entry, err + gone = row.ContentDigest.String } } - entry, err := h.proxy.storedMetadataDocument(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator()) - if errors.Is(err, errMetadataNotStored) { - return nil, errInMemory + res := h.proxy.storedMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), gone) + if errors.Is(res.err, errMetadataNotStored) { + res.err = errInMemory } - return entry, err + return res } // readStoredPackument reads the packument stored for cacheKey with read, @@ -437,23 +434,31 @@ func (h *NPMHandler) storedPackument(ctx context.Context, packumentURL, accept, // or no longer has the digest its row records, the packument is fetched // again and read once more. fetchErr is what storedPackument returned when the packument // couldn't be fetched, and err is what read returned. -func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { - entry, fetchErr := h.storedPackument(ctx, packumentURL, accept, cacheKey, refresh) - if fetchErr != nil { - return nil, fetchErr +func (h *NPMHandler) readStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, read func(*database.MetadataCacheEntry) error) (err, fetchErr error) { + fetched, err := h.lookupStoredPackument(ctx, packumentURL, accept, cacheKey, false, read) + return err, fetched.err +} + +// lookupStoredPackument implements readStoredPackument. fetched is the +// lookup of the packument read, whose err is set when it couldn't be +// fetched. +func (h *NPMHandler) lookupStoredPackument(ctx context.Context, packumentURL, accept, cacheKey string, refresh bool, read func(*database.MetadataCacheEntry) error) (fetched metadataResult, err error) { + fetched = h.storedPackument(ctx, packumentURL, accept, cacheKey, refresh) + if fetched.err != nil { + return fetched, nil } - if err = read(entry); !errors.Is(err, storage.ErrNotFound) && !errors.Is(err, errStoredMetadataChanged) { - return err, nil + if err = read(fetched.entry); !errors.Is(err, storage.ErrNotFound) && !errors.Is(err, errStoredMetadataChanged) { + return fetched, err } h.proxy.Logger.Warn("stored npm metadata is missing or changed, fetching it again", "url", packumentURL, "error", err) - entry, fetchErr = h.proxy.restoreStoredMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), entry) - if errors.Is(fetchErr, errMetadataNotStored) { - return nil, errInMemory + fetched = h.proxy.storedMetadata(ctx, "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator(), fetched.entry.ContentDigest.String) + if errors.Is(fetched.err, errMetadataNotStored) { + fetched.err = errInMemory } - if fetchErr != nil { - return nil, fetchErr + if fetched.err != nil { + return fetched, nil } - return read(entry), nil + return fetched, read(fetched.entry) } // packumentStreamValidator is packumentValidator for a packument read from @@ -486,7 +491,7 @@ func (h *NPMHandler) streamedVersionMetadata(ctx context.Context, packageName, s fullRequired := h.packumentValidator() != nil options := h.npmRewriteOptions(packageName, nil) var m rewrittenMetadata - err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, false, func(entry *database.MetadataCacheEntry) (err error) { + err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, func(entry *database.MetadataCacheEntry) (err error) { m, err = h.proxy.streamedRewrite(ctx, "npm", h.proxyURL, metadataCacheKey, entry, func(open func() (io.ReadCloser, error), out io.Writer) error { var scan *npm.Scan err := readStored(open, func(in io.Reader) (err error) { @@ -910,19 +915,24 @@ func (h *NPMHandler) versionData(ctx context.Context, packageName, packumentURL, } // readVersionData implements versionData. fromCache reports whether the -// packument may have been a cached copy; refresh fetches it from upstream. +// packument was a cached copy, within its TTL or served stale; refresh +// fetches it from upstream. func (h *NPMHandler) readVersionData(ctx context.Context, packumentURL, cacheKey, version string, refresh bool) (data npmVersionData, fromCache bool, err error) { accept := h.packumentAccept() var found npm.VersionEntry - err, fetchErr := h.readStoredPackument(ctx, packumentURL, accept, cacheKey, refresh, func(entry *database.MetadataCacheEntry) error { + fetched, err := h.lookupStoredPackument(ctx, packumentURL, accept, cacheKey, refresh, func(entry *database.MetadataCacheEntry) error { return readStored(func() (io.ReadCloser, error) { return h.proxy.openStoredMetadata(ctx, entry) }, func(in io.Reader) (err error) { found, err = npm.LookupVersion(in, version) return err }) }) + fetchErr := fetched.err switch { case fetchErr == nil && err == nil && (!found.Abbreviated || h.packumentValidator() == nil): - return npmVersionData{entry: found.Entry, published: found.Published}, !refresh, nil + if refresh && fetched.stale { + return npmVersionData{}, false, errRefetchFailed(fetched) + } + return npmVersionData{entry: found.Entry, published: found.Published}, fetched.fromCache || fetched.stale, nil case fetchErr == nil && err != nil: h.proxy.Logger.Warn("failed to read stored npm metadata, reading it in memory", "url", packumentURL, "error", err) case fetchErr != nil && !errors.Is(fetchErr, errInMemory): @@ -938,16 +948,21 @@ func (h *NPMHandler) readVersionData(ctx context.Context, packumentURL, cacheKey if res.err != nil { return npmVersionData{}, false, res.err } - // The cached copy is what lacked the version, so it can't decide it. if refresh && res.stale { - return npmVersionData{}, false, errors.New("npm metadata could not be fetched again") + return npmVersionData{}, false, errRefetchFailed(res) } entry, _ := jsonscan.Lookup(res.body, "versions", version) published, _, err := jsonscan.LookupString(res.body, "time", version) if err != nil { return npmVersionData{}, false, fmt.Errorf("parsing npm metadata: %w", err) } - return npmVersionData{entry: entry, published: published}, res.fromCache, nil + return npmVersionData{entry: entry, published: published}, res.fromCache || res.stale, nil +} + +// errRefetchFailed is the error for a refetch that got only the stale copy +// back. That copy is what lacked the version, so it can't decide it. +func errRefetchFailed(res metadataResult) error { + return fmt.Errorf("npm metadata could not be fetched again: %w", res.upstreamErr) } func validateTarballURL(raw, registryURL string) (string, error) { diff --git a/internal/handler/npm_tarball_test.go b/internal/handler/npm_tarball_test.go index d844dd5..6bca0f6 100644 --- a/internal/handler/npm_tarball_test.go +++ b/internal/handler/npm_tarball_test.go @@ -1,6 +1,7 @@ package handler import ( + "context" "encoding/json" "fmt" "io" @@ -13,6 +14,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" ) @@ -258,3 +260,167 @@ func TestNPMTarballCooldownRefetchesStalePackument(t *testing.T) { }) } } + +// widgetPackument is a packument listing only widget 1.0.0, published long +// ago. +func widgetPackument() string { + old := time.Now().Add(-60 * 24 * time.Hour).Format(time.RFC3339) + return fmt.Sprintf(`{"name":"widget","time":{"1.0.0":%q},"versions":{"1.0.0":{"dist":{}}}}`, old) +} + +// cooldownNPMHandler returns an npm handler for upstream with cooldown on +// and a tarball fetcher that would serve anything asked of it. +func cooldownNPMHandler(t *testing.T, upstream *httptest.Server, ttl time.Duration) *NPMHandler { + t.Helper() + p, _, _, fetcher := setupTestProxy(t) + p.CacheMetadata = true + p.MetadataTTL = ttl + p.Cooldown = &cooldown.Config{Default: "7d"} + p.HTTPClient = upstream.Client() + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} + return NewNPMHandler(p, "http://proxy.test", upstream.URL) +} + +// TestNPMTarballCooldownStalePackument checks that a stale packument served +// because upstream failed doesn't let through a version it doesn't list, +// and that the upstream failure decides the response. +func TestNPMTarballCooldownStalePackument(t *testing.T) { + for _, tc := range []struct { + name string + status int + limited bool + want int + }{ + {"upstream down", http.StatusServiceUnavailable, false, http.StatusBadGateway}, + {"not found", http.StatusNotFound, false, http.StatusNotFound}, + {"rate limited", http.StatusTooManyRequests, true, http.StatusServiceUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + var down atomic.Bool + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if down.Load() { + w.Header().Set("Retry-After", "60") + w.WriteHeader(tc.status) + return + } + _, _ = io.WriteString(w, widgetPackument()) + })) + t.Cleanup(upstream.Close) + // With no TTL the packument is read in memory and every read + // asks upstream. + h := cooldownNPMHandler(t, upstream, 0) + if tc.limited { + gate := registryclient.NewRateLimitGate(http.DefaultTransport) + h.proxy.HTTPClient = &http.Client{Transport: gate} + h.proxy.RateLimits = gate + } + + metadata := httptest.NewRecorder() + h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) + if metadata.Code != http.StatusOK { + t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) + } + down.Store(true) + + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) + if response.Code != tc.want { + t.Fatalf("status = %d, want %d: %s", response.Code, tc.want, response.Body.String()) + } + if tc.limited && response.Header().Get("Retry-After") == "" { + t.Error("no Retry-After on a rate-limited response") + } + }) + } +} + +// TestNPMTarballCooldownFetchesMissingVersionOnce checks that a packument +// just fetched from upstream isn't fetched again because it doesn't list the +// requested version. +func TestNPMTarballCooldownFetchesMissingVersionOnce(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + _, _ = io.WriteString(w, widgetPackument()) + })) + t.Cleanup(upstream.Close) + h := cooldownNPMHandler(t, upstream, time.Hour) + + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-99.0.0.tgz", nil)) + if n := requests.Load(); n != 1 { + t.Errorf("upstream packument requests = %d, want 1", n) + } +} + +// TestNPMTarballCooldownRefreshIgnoresSharedStale checks that a refetch of a +// stored packument lacking the version, joining another caller's fetch that +// falls back to the stored copy, doesn't take that copy as upstream's answer. +func TestNPMTarballCooldownRefreshIgnoresSharedStale(t *testing.T) { + var blocked atomic.Bool + arrived := make(chan struct{}, 1) + release := make(chan struct{}) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !blocked.Load() { + _, _ = io.WriteString(w, widgetPackument()) + return + } + arrived <- struct{}{} + select { + case <-release: + case <-r.Context().Done(): + } + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(upstream.Close) + h := cooldownNPMHandler(t, upstream, time.Hour) + p := h.proxy + + metadata := httptest.NewRecorder() + h.Routes().ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/widget", nil)) + if metadata.Code != http.StatusOK { + t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) + } + registryURL, cacheKey := h.sourceFor("widget") + packumentURL := registryURL + "/widget" + accept := h.packumentAccept() + row, err := p.DB.GetMetadataCache("npm", cacheKey) + if err != nil || row == nil { + t.Fatalf("no stored packument row: %v", err) + } + setFetchedAt := func(at time.Time) { + t.Helper() + if ok, err := p.DB.RefreshMetadataCache("npm", cacheKey, row.ContentDigest.String, at); err != nil || !ok { + t.Fatalf("setting fetched_at: %t, %v", ok, err) + } + } + + // Another caller revalidates the expired copy, and upstream holds it. + setFetchedAt(time.Now().Add(-2 * time.Hour)) + blocked.Store(true) + other := make(chan error, 1) + go func() { + _, err := p.storedMetadataDocument(context.Background(), "npm", cacheKey, packumentURL, accept, h.packumentStreamValidator()) + other <- err + }() + <-arrived + + // The tarball request reads the copy as within its TTL and refetches it, + // joining that fetch. + setFetchedAt(time.Now()) + response := httptest.NewRecorder() + done := make(chan struct{}) + go func() { + defer close(done) + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-2.0.0.tgz", nil)) + }() + waitForMetadataWaiters(t, p, metadataCoalesceKey("npm", cacheKey, packumentURL, accept, "", false)+"\x00stored", 1) + close(release) + <-done + if err := <-other; err != nil { + t.Fatalf("other caller: %v", err) + } + if response.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusBadGateway, response.Body.String()) + } +} From fede2463f7a9fa54ae5419a7a34a989575f67c83 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:10:43 +0100 Subject: [PATCH 158/198] Describe conda shard timestamps as condaTime reads them --- internal/handler/conda_shards.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/handler/conda_shards.go b/internal/handler/conda_shards.go index 01434e4..8269f1d 100644 --- a/internal/handler/conda_shards.go +++ b/internal/handler/conda_shards.go @@ -378,7 +378,8 @@ func rewriteCondaShardIndex(body []byte, indexURL string, mask [condaShardHashLe } // condaShardRecord holds the fields cooldown needs from a shard record. -// Timestamps are milliseconds, encoded as an integer or a float. +// Timestamps are an integer or a float, in seconds or milliseconds as +// condaTime reads them. type condaShardRecord struct { Name string `msgpack:"name"` Version string `msgpack:"version"` From 6d80291d22334e14fb928d263c7848b56499e402 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:11:36 +0100 Subject: [PATCH 159/198] Warn when mirroring a cached tag after a failed refresh A tag found in the cache after the refresh failed was reported as mirrored with no sign it might be stale, though tags can move. Keep serving it, as a cached commit is, but log a warning naming the tag, the commit and the refresh error. --- docs/configuration.md | 2 +- internal/gitcache/mirror.go | 4 ++++ internal/gitcache/mirror_test.go | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0d81646..59e5960 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1920,7 +1920,7 @@ proxy mirror pkg:git/github.com/git-pkgs/purl - The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail. - The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails. -- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID. +- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. When the upstream can't be reached, a cached tag is mirrored at the commit it last pointed to, with a warning in the log. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID. - With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`. - The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead. diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index f884d37..ffcd0d9 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -87,6 +87,10 @@ func (c *Cache) Mirror(ctx context.Context, key, repoURL, version string, fetchW } return MirrorResult{}, err } + if refreshErr != nil { + // Tags can move; this is where it pointed when last fetched. + c.logger.Warn("git cache: refresh failed, mirroring the cached tag", "repository", key, "tag", version, "commit", commit, "error", refreshErr) + } case version != "": if err := c.ensureCommit(ctx, rp, repoURL, commit, fetchWants); err != nil { return MirrorResult{}, err diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index b1e27f6..17a19f0 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -3,6 +3,7 @@ package gitcache import ( "context" "errors" + "net/http" "os" "path/filepath" "slices" @@ -389,3 +390,22 @@ func TestMirrorExplainsShortCommit(t *testing.T) { t.Errorf("mirror of a short commit ID = %v, want ErrNotFound saying it is no full commit ID", err) } } + +// TestMirrorWarnsOfStaleTag checks a tag served from the cache when the +// refresh fails is mirrored with a warning, since the tag may have moved. +func TestMirrorWarnsOfStaleTag(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true); err != nil { + t.Fatalf("Mirror: %v", err) + } + upstream.status.Store(http.StatusServiceUnavailable) + + if _, err := tc.Mirror(context.Background(), "up/repo.git", upstream.server.URL+"/repo", "v1", true); err != nil { + t.Fatalf("mirror of a cached tag with the upstream down: %v", err) + } + if !strings.Contains(tc.logs.String(), "tag=v1") { + t.Errorf("no warning naming the tag in the log:\n%s", tc.logs.String()) + } +} From 0ececf3ffa36bfd3eb90e6f30409f814630b9391 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:08:22 +0100 Subject: [PATCH 160/198] Find the Debian component in the security archive's pool security.debian.org keeps files under pool/updates/main while its InRelease lists main/binary-*, so taking the first segment after pool/ as the component found no index and security suites were never refreshed. Take the first or second segment, whichever the release file lists indexes for. --- internal/mirror/refresh_harden_test.go | 23 +++++++++++++++++++++++ internal/mirror/refresh_index.go | 23 +++++++++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index 9c311dd..0ff6d52 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -307,3 +307,26 @@ func TestRefreshRefusesAPKIndexesOverTheSizeLimit(t *testing.T) { t.Errorf("progress = %+v, fetched %v; want the index failing for its size", progress, fetcher.fetched()) } } + +// TestRefreshDebianSecurityPool checks the security archive's layout, whose +// pool is pool/updates/main while its InRelease lists main/binary-*. +func TestRefreshDebianSecurityPool(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + pool := idx.URL + "/debian-security/pool/updates/main/c/curl/" + packages := func(version string) []byte { + return gzipped(t, "Package: curl\nVersion: "+version+"\nArchitecture: amd64\nFilename: pool/updates/main/c/curl/curl_"+version+"_amd64.deb\n\n") + } + idx.set("/debian-security/dists/bookworm-security/InRelease", []byte("SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n")) + idx.set("/debian-security/dists/bookworm-security/main/binary-amd64/Packages.gz", packages("7.88.1-10+deb12u7")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian-security/dists/bookworm-security/InRelease", + "deb", "7.88.1-10+deb12u7", "curl_7.88.1-10+deb12u7_amd64.deb", pool+"curl_7.88.1-10+deb12u7_amd64.deb") + + idx.set("/debian-security/dists/bookworm-security/main/binary-amd64/Packages.gz", packages("7.88.1-10+deb12u8")) + fetcher.reset() + progress := refreshOnce(t, m) + + want := []string{pool + "curl_7.88.1-10+deb12u8_amd64.deb"} + if !slices.Equal(fetcher.fetched(), want) || progress.Completed != 1 || progress.Failed != 0 { + t.Errorf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) + } +} diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go index 3a7ad7c..cda0dbb 100644 --- a/internal/mirror/refresh_index.go +++ b/internal/mirror/refresh_index.go @@ -214,11 +214,10 @@ func debIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, indexes := debPackagesIndexes(body) wanted := map[string]bool{} for _, c := range cached { - rest, ok := strings.CutPrefix(c.rel, "pool/") + component, ok := debComponent(c.rel, indexes) if !ok { continue } - component, _, _ := strings.Cut(rest, "/") if dir := component + "/binary-" + c.arch; indexes[dir] != "" { wanted[dir] = true continue @@ -243,6 +242,26 @@ func debIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, return files, nil } +// debComponent returns the component of a pool file that the release file +// lists indexes for. It is the first segment after pool/, or the second, as +// in the security archive's pool/updates/main. +func debComponent(rel string, indexes map[string]string) (string, bool) { + rest, ok := strings.CutPrefix(rel, "pool/") + if !ok { + return "", false + } + first, rest, _ := strings.Cut(rest, "/") + second, _, _ := strings.Cut(rest, "/") + for _, component := range []string{first, second} { + for dir := range indexes { + if strings.HasPrefix(dir, component+"/binary-") { + return component, true + } + } + } + return "", false +} + // rpmIndexFiles reads the primary metadata a repomd.xml points at. func rpmIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, _ []cachedOSFile) ([]indexFile, error) { primary, err := repomdPrimary(body) From b1875040de329913a6189835c7bd19ed9b11449f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:09:33 +0100 Subject: [PATCH 161/198] Leave the refresh's own downloads out of the most used packages A file the refresh mirrored was fetched just now and never accessed, which counted as recent use, so the refresh kept every package it had refreshed in the most used list forever. Fetches under the new handler.MirrorFetch context set mirrored_at, and a file whose last fetch was a mirror's counts only once a client downloads it. --- docs/configuration.md | 2 +- internal/database/most_used.go | 8 +++++--- internal/database/most_used_test.go | 24 ++++++++++++++++++++++++ internal/handler/handler.go | 18 ++++++++++++++++++ internal/mirror/refresh.go | 2 ++ internal/mirror/refresh_harden_test.go | 23 +++++++++++++++++++++++ 6 files changed, 73 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 5459cb6..dc79b0b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1976,7 +1976,7 @@ For each of those packages a run lists the releases on the configured upstream a - PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. - Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. -- Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. +- Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. A file a run mirrored counts as used only once a client downloads it, so a package nobody fetches drops out after `since`. - Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. - What a run mirrors is evicted by `storage.max_size` and retention like any other file. With `max_versions` and `max_bytes` a run stays bounded when many packages release at once. diff --git a/internal/database/most_used.go b/internal/database/most_used.go index c68e833..a365cb7 100644 --- a/internal/database/most_used.go +++ b/internal/database/most_used.go @@ -25,7 +25,8 @@ type UsedFile struct { // MostUsedPackages returns, for each ecosystem, the top packages with the most // downloads among the cached files downloaded since cutoff, ordered by // ecosystem and then by hits. Each lists every file cached for it. Files -// from local sources, which have no upstream, are left out. +// from local sources, which have no upstream, are left out, and so are +// files a mirror run fetched that no client has downloaded since. func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) { var rows []struct { Ecosystem string `db:"ecosystem"` @@ -41,12 +42,13 @@ func (db *DB) MostUsedPackages(cutoff time.Time, top int) ([]UsedPackage, error) JOIN versions v ON v.purl = a.version_purl JOIN packages p ON p.purl = v.package_purl WHERE a.storage_path IS NOT NULL AND a.source = ? - AND COALESCE(a.last_accessed_at, a.fetched_at) >= ? + AND (a.last_accessed_at >= ? OR (a.last_accessed_at IS NULL AND a.fetched_at >= ? + AND (a.mirrored_at IS NULL OR a.mirrored_at < a.fetched_at))) GROUP BY p.ecosystem, p.name, p.purl ) ranked WHERE rank <= ? ORDER BY ecosystem, hits DESC, name - `), SourceUpstream, cutoff.UTC(), top) + `), SourceUpstream, cutoff.UTC(), cutoff.UTC(), top) if err != nil { return nil, err } diff --git a/internal/database/most_used_test.go b/internal/database/most_used_test.go index 8704f90..4b8693e 100644 --- a/internal/database/most_used_test.go +++ b/internal/database/most_used_test.go @@ -92,3 +92,27 @@ func TestMostUsedPackagesSkipsLocalArtifacts(t *testing.T) { } }) } + +func TestMostUsedPackagesSkipsMirroredArtifactsNotDownloaded(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seedUsedArtifact(t, db, "npm", "lodash", "4.17.21", "lodash-4.17.21.tgz", 0, now) + if _, err := db.Exec(db.Rebind("UPDATE artifacts SET last_accessed_at = NULL, mirrored_at = ?"), time.Now().UTC()); err != nil { + t.Fatal(err) + } + got, err := db.MostUsedPackages(now.Add(-time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Errorf("MostUsedPackages = %+v, want none: a mirror fetched it and no client has since", got) + } + + if err := db.RecordArtifactHit("pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz"); err != nil { + t.Fatal(err) + } + if got, err = db.MostUsedPackages(now.Add(-time.Hour), 10); err != nil || len(got) != 1 { + t.Errorf("MostUsedPackages = %+v, %v; want lodash once a client downloaded it", got, err) + } + }) +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 495a92b..0ad433f 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -797,6 +797,10 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil "path", storagePath, "error", qErr) return sharedArtifact, storagePath, errUnrecorded } + } else if mirrorFetch(ctx) { + if err := p.DB.MarkArtifactMirrored(versionPURL, filename); err != nil { + p.Logger.Warn("failed to record mirrored artifact", "error", err) + } } return sharedArtifact, storagePath, nil @@ -1149,6 +1153,20 @@ func readsSeekable(store storage.Storage) bool { return ok && s.ReadsSeekable() } +type mirrorFetchKey struct{} + +// MirrorFetch marks ctx as a mirror run's rather than a client's. An +// artifact fetched under it is recorded as mirrored, so it doesn't count as +// downloaded by a client. +func MirrorFetch(ctx context.Context) context.Context { + return context.WithValue(ctx, mirrorFetchKey{}, true) +} + +func mirrorFetch(ctx context.Context) bool { + mirror, _ := ctx.Value(mirrorFetchKey{}).(bool) + return mirror +} + type detachUpstreamFetchKey struct{} // DetachUpstreamFetch marks ctx so that an artifact fetch started under it diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index 6fc036b..32852bb 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -7,6 +7,7 @@ import ( "time" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" ) @@ -56,6 +57,7 @@ func (m *Mirror) releaseLister() ReleaseLister { //nolint:ireturn // any lister // it: releases within cooldown or on the denylist are left out. Packages it // can't list are counted as failed. func (m *Mirror) Refresh(ctx context.Context, r Refresh) (*Progress, error) { + ctx = handler.MirrorFetch(ctx) cutoff := time.Now().Add(-r.Since) packages, err := m.db.MostUsedPackages(cutoff, r.Top) if err != nil { diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index 0ff6d52..bcc33d9 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -330,3 +330,26 @@ func TestRefreshDebianSecurityPool(t *testing.T) { t.Errorf("progress = %+v, fetched %v; want %v", progress, fetcher.fetched(), want) } } + +// TestRefreshDownloadsDoNotCountAsUse checks that a package nobody fetched +// within since drops out even though the refresh itself downloaded it. +func TestRefreshDownloadsDoNotCountAsUse(t *testing.T) { + m, _ := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + m.Versions = stubReleases{"npm/lodash": oldReleases("4.17.20", "4.17.21")} + if progress := refreshOnce(t, m); progress.Completed != 1 { + t.Fatalf("progress = %+v, want 4.17.21 mirrored", progress) + } + old := time.Now().Add(-48 * time.Hour).UTC() + if _, err := m.db.Exec(m.db.Rebind("UPDATE artifacts SET fetched_at = ? WHERE filename = ?"), old, "lodash-4.17.20.tgz"); err != nil { + t.Fatal(err) + } + + used, err := m.db.MostUsedPackages(time.Now().Add(-24*time.Hour), 10) + if err != nil { + t.Fatal(err) + } + if len(used) != 0 { + t.Errorf("MostUsedPackages = %+v, want none: only the refresh fetched lodash within since", used) + } +} From e598c8e44efe1bbbbd5e24d877f68f3ca8732f46 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:10:07 +0100 Subject: [PATCH 162/198] Read refreshed index documents without recording an access The refresh read InRelease, APKINDEX and repomd.xml again through the metadata cache, which marked them accessed, so every suite looked fetched within since and never aged out. Read them straight from upstream, as the Packages and primary indexes already are. --- docs/configuration.md | 2 +- internal/mirror/refresh_harden_test.go | 25 ++++++++++++++++++ internal/mirror/refresh_index.go | 36 +++++++++++++++++++++----- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index dc79b0b..8612cc4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1974,7 +1974,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. - PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. -- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`. +- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again from upstream, which doesn't count as a client fetching them, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. A file a run mirrored counts as used only once a client downloads it, so a package nobody fetches drops out after `since`. - Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index bcc33d9..4c398f3 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -353,3 +353,28 @@ func TestRefreshDownloadsDoNotCountAsUse(t *testing.T) { t.Errorf("MostUsedPackages = %+v, want none: only the refresh fetched lodash within since", used) } } + +// TestRefreshIndexReadIsNotAClientAccess checks that a refresh reading an +// index again doesn't keep it among those clients fetched within since. +func TestRefreshIndexReadIsNotAClientAccess(t *testing.T) { + m, _, idx := setupIndexRefresh(t) + dir := "/alpine/v3.20/main/aarch64" + idx.set(dir+"/APKINDEX.tar.gz", apkIndexArchive(t, "P:curl\nV:8.9.0-r0\nA:aarch64\n\n")) + clientFetchesCurl(t, m, "alpine", idx.URL+dir+"/APKINDEX.tar.gz", + "alpine", "8.9.0-r0", "edge/v3.20/main/aarch64/curl-8.9.0-r0.apk", idx.URL+dir+"/curl-8.9.0-r0.apk") + old := time.Now().Add(-2 * time.Hour).UTC() + if _, err := m.db.Exec(m.db.Rebind("UPDATE metadata_cache SET last_accessed_at = ?, updated_at = ?"), old, old); err != nil { + t.Fatal(err) + } + + if _, err := m.Refresh(context.Background(), Refresh{Top: 10, Since: 3 * time.Hour}); err != nil { + t.Fatal(err) + } + docs, err := m.db.RecentMetadataDocuments("alpine", time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(docs) != 0 { + t.Errorf("recent indexes = %+v, want none: only the refresh read the index", docs) + } +} diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go index cda0dbb..890ebd6 100644 --- a/internal/mirror/refresh_index.go +++ b/internal/mirror/refresh_index.go @@ -98,7 +98,7 @@ func (m *Mirror) indexRefreshItems(ctx context.Context, ix osIndex, packages []d continue } seenIndex[urlDir(doc.UpstreamURL)] = true - body, _, err := m.proxy.FetchOrCacheMetadata(ctx, ix.metaEcosystem, doc.Name, doc.UpstreamURL, "*/*") + body, err := m.readIndexDocument(ctx, doc.UpstreamURL) if err != nil { errs = append(errs, fmt.Errorf("%s: %w", doc.UpstreamURL, err)) continue @@ -272,10 +272,20 @@ func rpmIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, return m.readIndex(ctx, base+"/"+primary, parseRPMPrimary) } -// readIndex streams an index from upstream through parse, decompressed as -// its name says and limited to maxIndexBytes. It is read directly rather than through the metadata -// cache, since it can be far larger than a metadata document may be. -func (m *Mirror) readIndex(ctx context.Context, indexURL string, parse func(io.Reader) ([]indexFile, error)) ([]indexFile, error) { +// readIndexDocument reads the index document a refresh starts from directly +// from upstream. Read through the metadata cache it would count as a client +// fetching it, and the suite would never age out of since. +func (m *Mirror) readIndexDocument(ctx context.Context, indexURL string) ([]byte, error) { + body, err := m.openIndex(ctx, indexURL) + if err != nil { + return nil, err + } + defer func() { _ = body.Close() }() + return io.ReadAll(&indexLimitReader{r: body, url: indexURL, remaining: maxIndexBytes}) +} + +// openIndex starts reading an index from upstream. +func (m *Mirror) openIndex(ctx context.Context, indexURL string) (io.ReadCloser, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil) if err != nil { return nil, err @@ -284,11 +294,23 @@ func (m *Mirror) readIndex(ctx context.Context, indexURL string, parse func(io.R if err != nil { return nil, err } - defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { + _ = resp.Body.Close() return nil, fmt.Errorf("%s: upstream returned %d", indexURL, resp.StatusCode) } - r, err := decompressByName(indexURL, resp.Body) + return resp.Body, nil +} + +// readIndex streams an index from upstream through parse, decompressed as +// its name says and limited to maxIndexBytes. It is read directly rather than through the metadata +// cache, since it can be far larger than a metadata document may be. +func (m *Mirror) readIndex(ctx context.Context, indexURL string, parse func(io.Reader) ([]indexFile, error)) ([]indexFile, error) { + body, err := m.openIndex(ctx, indexURL) + if err != nil { + return nil, err + } + defer func() { _ = body.Close() }() + r, err := decompressByName(indexURL, body) if err != nil { return nil, err } From 85dd89187914d2731e634357bb125ec72e04791c Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:10:37 +0100 Subject: [PATCH 163/198] Recover from a panic mirroring one refreshed version Refresh called mirrorOne on the scheduler's goroutine without the recover Run puts around each item, so a panic took the proxy down. Both now go through mirrorRecovered, which counts it as a failure and carries on. --- internal/mirror/mirror.go | 25 ++++++++++------- internal/mirror/refresh.go | 2 +- internal/mirror/refresh_harden_test.go | 37 ++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/internal/mirror/mirror.go b/internal/mirror/mirror.go index b32c7ca..1dded24 100644 --- a/internal/mirror/mirror.go +++ b/internal/mirror/mirror.go @@ -200,16 +200,8 @@ func (m *Mirror) Run(ctx context.Context, source Source, onProgress ...ProgressF g.SetLimit(m.workers) for _, item := range items { - g.Go(func() (err error) { - defer func() { - if r := recover(); r != nil { - m.logger.Error("panic in mirror worker", "recover", r, - "ecosystem", item.Ecosystem, "name", item.Name, "version", item.Version) - tracker.failed.Add(1) - tracker.addError(item.Ecosystem, item.Name, item.Version, fmt.Sprintf("panic: %v", r)) - } - }() - m.mirrorOne(gctx, item, tracker) + g.Go(func() error { + m.mirrorRecovered(gctx, item, tracker) return nil // never fail the group; errors are tracked }) } @@ -239,6 +231,19 @@ func (m *Mirror) RunDryRun(ctx context.Context, source Source) ([]PackageVersion return items, err } +// mirrorRecovered is mirrorOne, counting a panic as a failure. +func (m *Mirror) mirrorRecovered(ctx context.Context, item PackageVersion, tracker *progressTracker) { + defer func() { + if r := recover(); r != nil { + m.logger.Error("panic in mirror worker", "recover", r, + "ecosystem", item.Ecosystem, "name", item.Name, "version", item.Version) + tracker.failed.Add(1) + tracker.addError(item.Ecosystem, item.Name, item.Version, fmt.Sprintf("panic: %v", r)) + } + }() + m.mirrorOne(ctx, item, tracker) +} + func (m *Mirror) mirrorOne(ctx context.Context, pv PackageVersion, tracker *progressTracker) { if isImageEcosystem(pv.Ecosystem) && m.Images != nil { m.mirrorImage(ctx, pv, tracker) diff --git a/internal/mirror/refresh.go b/internal/mirror/refresh.go index 32852bb..3c2e094 100644 --- a/internal/mirror/refresh.go +++ b/internal/mirror/refresh.go @@ -76,7 +76,7 @@ func (m *Mirror) Refresh(ctx context.Context, r Refresh) (*Progress, error) { return false } tracker.total.Add(1) - m.mirrorOne(ctx, item, tracker) + m.mirrorRecovered(ctx, item, tracker) } return true } diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index 4c398f3..f33a833 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -2,6 +2,7 @@ package mirror import ( "context" + "net/http" "reflect" "slices" "strings" @@ -10,6 +11,7 @@ import ( "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/registries" + "github.com/git-pkgs/registries/fetch" ) // debCurlPackages returns a Packages index listing curl at version. @@ -378,3 +380,38 @@ func TestRefreshIndexReadIsNotAClientAccess(t *testing.T) { t.Errorf("recent indexes = %+v, want none: only the refresh read the index", docs) } } + +// panickingFetcher serves files like servingFetcher but panics on URLs +// containing panicOn. +type panickingFetcher struct { + *servingFetcher + panicOn string +} + +func (f *panickingFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { + return f.FetchWithHeaders(ctx, url, nil) +} + +func (f *panickingFetcher) FetchWithHeaders(ctx context.Context, url string, h http.Header) (*fetch.Artifact, error) { + if strings.Contains(url, f.panicOn) { + panic("fetcher bug") + } + return f.servingFetcher.FetchWithHeaders(ctx, url, h) +} + +func TestRefreshRecoversFromPanic(t *testing.T) { + m, fetcher := setupRefreshMirror(t, "tarball") + cacheNPM(t, m, "lodash", "4.17.20") + cacheNPM(t, m, "react", "18.3.0") + m.Versions = stubReleases{ + "npm/lodash": oldReleases("4.17.20", "4.17.21"), + "npm/react": oldReleases("18.3.0", "18.3.1"), + } + m.proxy.Fetcher = &panickingFetcher{servingFetcher: fetcher, panicOn: "lodash-4.17.21"} + fetcher.reset() + + progress := refreshOnce(t, m) + if progress.Failed != 1 || progress.Completed != 1 || len(fetchedMatching(fetcher.fetched(), "react-18.3.1.tgz")) != 1 { + t.Errorf("progress = %+v, fetched %v; want lodash failed and react mirrored", progress, fetcher.fetched()) + } +} From 60c899f1df78af588cdb6132e6816fbf9130737d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:11:11 +0100 Subject: [PATCH 164/198] Treat calendar versions as one line in the refresh A version's first number was taken as its major line, so certifi 2024.8.30 and tzdata 2024.2 never got the next year's release. A first number of 1000 or more is a year, and all such versions are one line. --- docs/configuration.md | 2 +- internal/mirror/refresh_versions.go | 13 +++++++++++-- internal/mirror/refresh_versions_test.go | 13 +++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8612cc4..4e03fb9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1971,7 +1971,7 @@ mirror: | `mirror.refresh.max_versions` | - | Stop a run after mirroring this many new versions. Default: no cap | | `mirror.refresh.max_bytes` | - | Stop a run once it has downloaded this much, such as `2GB`. It is checked before each version, so the last one can go over it. Default: no cap | -For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. +For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Versions numbered by year, such as certifi's `2025.1.31`, are all one line. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. - PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. - Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again from upstream, which doesn't count as a client fetching them, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`. diff --git a/internal/mirror/refresh_versions.go b/internal/mirror/refresh_versions.go index b489170..22fe7c1 100644 --- a/internal/mirror/refresh_versions.go +++ b/internal/mirror/refresh_versions.go @@ -11,7 +11,8 @@ import ( // newestPerCachedLine returns, for each major line a cached version is on, // the newest release of that line that allowed accepts, when it is newer // than every cached version of the line. A line is the major version, or -// 0.minor below 1.0, as caret ranges read them. Prereleases and yanked, +// 0.minor below 1.0, as caret ranges read them, and calendar versions are +// one line. Prereleases and yanked, // retracted or deprecated releases are skipped. func newestPerCachedLine(ecosystem string, cached []string, releases []registries.Version, allowed func(registries.Version) bool) []string { scheme := purl.EcosystemToPURLType(ecosystem) @@ -54,12 +55,20 @@ func newestPerCachedLine(ecosystem string, cached []string, releases []registrie return out } -// versionLine returns the major line of a version. +// firstCalendarYear is the smallest first number read as a year. +const firstCalendarYear = 1000 + +// versionLine returns the major line of a version. Calendar versions, whose +// first number is a year, are all one line, so a new year's release follows +// the last one's. func versionLine(v string) (string, bool) { info, err := vers.ParseVersion(v) if err != nil { return "", false } + if info.Major >= firstCalendarYear { + return "calendar", true + } if info.Major == 0 { return "0." + strconv.Itoa(info.Minor), true } diff --git a/internal/mirror/refresh_versions_test.go b/internal/mirror/refresh_versions_test.go index 6e1e262..bc88947 100644 --- a/internal/mirror/refresh_versions_test.go +++ b/internal/mirror/refresh_versions_test.go @@ -77,3 +77,16 @@ func TestNewestPerCachedLineUsesEcosystemOrdering(t *testing.T) { t.Errorf("gem: got %v, want nothing", got) } } + +func TestNewestPerCachedLineCalendarVersions(t *testing.T) { + allowAll := func(registries.Version) bool { return true } + // certifi and tzdata number releases by year: 2025.1.31 follows 2024.12.14. + got := newestPerCachedLine("pypi", []string{"2024.8.30"}, releases("2024.8.30", "2024.12.14", "2025.1.31"), allowAll) + if !reflect.DeepEqual(got, []string{"2025.1.31"}) { + t.Errorf("certifi: got %v, want [2025.1.31]", got) + } + got = newestPerCachedLine("pypi", []string{"2024.2"}, releases("2024.2", "2025.1"), allowAll) + if !reflect.DeepEqual(got, []string{"2025.1"}) { + t.Errorf("tzdata: got %v, want [2025.1]", got) + } +} From 64eb09e674c4a3c6f9cc54894de3015c02bed669 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:12:37 +0100 Subject: [PATCH 165/198] Warn when mirror.refresh runs without cache_metadata The docs say refreshing Debian, Alpine and RPM packages needs cache_metadata, but nothing said so when it was off. Rejecting the setting would also stop the refresh of every other ecosystem, so the server logs a warning from the new Config.Warnings at startup. --- docs/configuration.md | 2 +- internal/config/mirror.go | 10 ++++++++++ internal/config/mirror_test.go | 17 +++++++++++++++++ internal/server/server.go | 3 ++- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4e03fb9..8bbd52b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1974,7 +1974,7 @@ mirror: For each of those packages a run lists the releases on the configured upstream and mirrors, for each major line a version is cached on, the newest release of that line. A major line is the major version, or `0.minor` below 1.0. Versions numbered by year, such as certifi's `2025.1.31`, are all one line. Prereleases and yanked, retracted or deprecated releases are skipped, and so are releases cooldown still withholds or the denylist names, so a run mirrors what clients would be served rather than the newest published. A release that leaves cooldown later is mirrored by a later run. - PyPI and conda versions publish many files. Only the kinds clients fetched of earlier versions are mirrored: wheels with the same python and ABI tags for the same OS and architecture, whatever glibc or macOS version they need (and, for `abi3` wheels, whatever minimum Python version), and the sdist, or conda packages of the same subdir, build variant (such as `py312` or `_cpython`) and file type. -- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again from upstream, which doesn't count as a client fetching them, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`. +- Debian, Alpine and RPM clients find new versions in repository indexes. A run fetches the `InRelease`, `APKINDEX.tar.gz` and `repomd.xml` files clients fetched within `since` again from upstream, which doesn't count as a client fetching them, reads the `Packages` and primary indexes they point at for the components and architectures of the cached packages, and mirrors the newest file each index lists of a package and architecture when it isn't cached but an older version is. Suites sharing a pool, such as bookworm and trixie, are refreshed each on its own that way. Files are checked against the SHA-256 the index lists, where it lists one. Indexes are found by the URL the metadata cache records with them, which caches from before this release lack until they are next revalidated. This requires `cache_metadata`; without it the proxy logs a warning at startup and refreshes the other ecosystems only. - Container images are not refreshed; list the tags to follow in `mirror.schedules`. Arch Linux and FreeBSD packages, Julia packages, which are cached by tree hash, and other ecosystems the mirror can't list versions of are left out too. - Hit counts are kept per file since it was cached, so `top` ranks by all downloads of the files used within `since`. A file a run mirrored counts as used only once a client downloads it, so a package nobody fetches drops out after `since`. - Runs are spread and shared between instances like schedules: the first run comes within ten minutes of startup, later ones every `interval` ±10 %, and instances sharing a PostgreSQL database take turns. A run fetches one version at a time. diff --git a/internal/config/mirror.go b/internal/config/mirror.go index 625f58d..3b56fd8 100644 --- a/internal/config/mirror.go +++ b/internal/config/mirror.go @@ -137,6 +137,16 @@ func (s *MirrorSchedule) IntervalDuration() time.Duration { return s.interval } +// Warnings returns settings that are valid but do less than they may be +// expected to. The server logs them at startup. +func (c *Config) Warnings() []string { + var warnings []string + if c.Mirror.Refresh != nil && !c.CacheMetadata { + warnings = append(warnings, "mirror.refresh without cache_metadata does not refresh Debian, Alpine or RPM packages: their indexes are found through the metadata cache") + } + return warnings +} + // Validate checks every schedule. func (m *MirrorConfig) Validate() error { for i := range m.Schedules { diff --git a/internal/config/mirror_test.go b/internal/config/mirror_test.go index 153677b..d5dff5f 100644 --- a/internal/config/mirror_test.go +++ b/internal/config/mirror_test.go @@ -3,6 +3,7 @@ package config import ( "os" "path/filepath" + "strings" "testing" "time" ) @@ -169,3 +170,19 @@ mirror: r.IntervalDuration(), r.TopPackages(), r.SinceDuration(), r.MaxVersionsPerRun(), r.MaxBytesPerRun()) } } + +func TestWarningsMirrorRefreshWithoutCacheMetadata(t *testing.T) { + cfg := Default() + cfg.Mirror.Refresh = &MirrorRefresh{Interval: "24h"} + cfg.CacheMetadata = false + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() = %v, want mirror.refresh accepted for the other ecosystems", err) + } + if got := cfg.Warnings(); len(got) != 1 || !strings.Contains(got[0], "cache_metadata") { + t.Errorf("Warnings() = %q, want one about cache_metadata", got) + } + cfg.CacheMetadata = true + if got := cfg.Warnings(); len(got) != 0 { + t.Errorf("Warnings() with cache_metadata = %q, want none", got) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index df0dd25..c836ee3 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -71,6 +71,7 @@ import ( "net" "net/http" "net/url" + "slices" "strconv" "strings" "sync" @@ -443,7 +444,7 @@ func (s *Server) serve(listener net.Listener) error { "ui_url", s.cfg.UIBaseURL, "storage", storageURL, "database", s.cfg.Database.String()) - for _, warning := range s.cfg.EnvWarnings() { + for _, warning := range slices.Concat(s.cfg.EnvWarnings(), s.cfg.Warnings()) { s.logger.Warn(warning) } s.background.Go(func() { s.updateCacheStatsMetrics(bgCtx) }) From 5c42a6dab17c13713110e9aaf9116272d9845e5b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:12:39 +0100 Subject: [PATCH 166/198] Seed the metadata test row without unused parameters Every caller seeds the same v1 document, which the linter flags. --- internal/handler/metadata_consistency_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/handler/metadata_consistency_test.go b/internal/handler/metadata_consistency_test.go index 115aeba..9ac84de 100644 --- a/internal/handler/metadata_consistency_test.go +++ b/internal/handler/metadata_consistency_test.go @@ -95,9 +95,11 @@ func TestStaleMetadataMarkedWithoutTTL(t *testing.T) { } } -// seedMetadataRow stores body for key "pkg" with etag and a row past its TTL. -func seedMetadataRow(t *testing.T, db *database.DB, store *mockStorage, body, etag string) string { +// seedMetadataRow stores {"v":1} for key "pkg" with ETag "v1" and a row past +// its TTL. +func seedMetadataRow(t *testing.T, db *database.DB, store *mockStorage) string { t.Helper() + const body, etag = `{"v":1}`, `"v1"` path := metadataStoragePath("test", "pkg") store.mu.Lock() store.files[path] = []byte(body) @@ -142,7 +144,7 @@ func TestNotModifiedChecksStoredBytes(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) proxy.CacheMetadata = true proxy.MetadataTTL = time.Minute - path := seedMetadataRow(t, db, store, `{"v":1}`, `"v1"`) + path := seedMetadataRow(t, db, store) upstream := v2Upstream(t, func() { // The other instance has written v2 but not yet its row. store.mu.Lock() @@ -174,7 +176,7 @@ func TestNotModifiedAfterRowReplaced(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) proxy.CacheMetadata = true proxy.MetadataTTL = time.Minute - path := seedMetadataRow(t, db, store, `{"v":1}`, `"v1"`) + path := seedMetadataRow(t, db, store) upstream := v2Upstream(t, func() { // The other instance has recorded v2's row; its blob is not written // yet. @@ -210,7 +212,7 @@ func TestNotModifiedRejectedBodyNotRefreshed(t *testing.T) { proxy, db, store, _ := setupTestProxy(t) proxy.CacheMetadata = true proxy.MetadataTTL = time.Minute - seedMetadataRow(t, db, store, `{"v":1}`, `"v1"`) + seedMetadataRow(t, db, store) before, err := db.GetMetadataCache("test", "pkg") if err != nil || before == nil { t.Fatal(before, err) @@ -258,7 +260,7 @@ func TestStaleMetadataChecksStoredBytes(t *testing.T) { proxy.CacheMetadata = true proxy.MetadataTTL = 0 proxy.HTTPClient = upstream.Client() - path := seedMetadataRow(t, db, store, `{"v":1}`, `"v1"`) + path := seedMetadataRow(t, db, store) proxy.Storage = &openHookStorage{mockStorage: store, onOpen: func() { // The other instance has written v2 but not yet its row. store.mu.Lock() From 0ff118b6420ee207c9bc8f8ce18090332635c649 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:57:38 +0100 Subject: [PATCH 167/198] Try the Plugin Portal for Maven paths proxied uncached Paths the proxy doesn't cache, such as ones with a dotted group directory, went to the primary upstream only, so a file available only on the Gradle Plugin Portal returned 404. They now try the Plugin Portal on a 404, as cached files do. --- internal/handler/handler.go | 9 +++++++++ internal/handler/maven.go | 22 ++++++++++++++++++++- internal/handler/maven_repositories_test.go | 15 ++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d316aab..dc54269 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1481,6 +1481,15 @@ func copyArtifactBody(w http.ResponseWriter, reader io.Reader, expectedSize int6 // It copies the request, forwards specified headers, and streams the response back. // forwardHeaders controls the request headers sent upstream. End-to-end response // headers and trailers are relayed independently of that list. +// upstreamRequest sends r's method to upstreamURL with no request headers. +func (p *Proxy) upstreamRequest(r *http.Request, upstreamURL string) (*http.Response, error) { + req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) + if err != nil { + return nil, err + } + return p.HTTPClient.Do(req) +} + func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamURL string, forwardHeaders []string) { p.Logger.Debug("proxying to upstream", "url", upstreamURL) diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 0fd81ec..ed417d7 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -343,5 +343,25 @@ func (h *MavenHandler) isChecksumFile(filename string) bool { // proxyUpstream forwards a request to the source's upstream without caching. func (h *MavenHandler) proxyUpstream(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { - h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, nil) + if source.fallbackURL == "" { + h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, nil) + return + } + // As for cached files, a file the primary upstream lacks is looked for + // on the Gradle Plugin Portal. + resp, err := h.proxy.upstreamRequest(r, source.upstreamURL+"/"+urlPath) + if err == nil && resp.StatusCode == http.StatusNotFound { + _ = resp.Body.Close() + resp, err = h.proxy.upstreamRequest(r, source.fallbackURL+"/"+urlPath) + } + if err != nil { + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("upstream request failed", "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + h.proxy.relayResponse(w, r, resp, nil) } diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go index 15db066..8fb6f27 100644 --- a/internal/handler/maven_repositories_test.go +++ b/internal/handler/maven_repositories_test.go @@ -156,3 +156,18 @@ func TestMavenDottedGroupDirectoryNotCachedAsGroup(t *testing.T) { } } } + +// TestMavenUncachedPathFallsBackToPluginPortal checks that a path proxied +// without caching, such as one with a dotted group directory, still tries +// the Gradle Plugin Portal when the primary upstream lacks it. +func TestMavenUncachedPathFallsBackToPluginPortal(t *testing.T) { + const dotted = "/org.foo/bar/1.0/bar-1.0.jar" + central, _ := mavenRepoUpstream(t, "central") + portal, _ := mavenRepoUpstream(t, "portal", dotted) + grails, _ := mavenRepoUpstream(t, "grails") + h := newMavenRepositoriesHandler(t, central, portal, grails) + + if w := getMaven(h, dotted); w.Code != http.StatusOK || w.Body.String() != "portal "+dotted { + t.Errorf("GET %s = %d %q, want the Plugin Portal's copy", dotted, w.Code, w.Body.String()) + } +} From a5f014d64cda4995c4463502401487fdd64a12ea Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 10:58:43 +0100 Subject: [PATCH 168/198] Check a cached release file against its row before using it The release listing read the cached InRelease, Release or repomd.xml without comparing it with the row's digest, so bytes another instance had replaced could decide which index is served. Such a file is now ignored, and the index falls back to its own cache entry. --- internal/handler/release_index.go | 4 +++- internal/handler/release_index_test.go | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/internal/handler/release_index.go b/internal/handler/release_index.go index debf5e5..626792a 100644 --- a/internal/handler/release_index.go +++ b/internal/handler/release_index.go @@ -44,7 +44,9 @@ func (p *Proxy) readCachedRelease(ctx context.Context, ecosystem, cacheKey strin return nil } body, _, err := p.readCachedMetadata(ctx, entry, nil) - if err != nil { + // Bytes another instance replaced since the row was read don't describe + // the release the row records. + if err != nil || !matchesEntryDigest(entry, body) { return nil } return &cachedRelease{ecosystem: ecosystem, cacheKey: cacheKey, body: body, fetchedAt: entry.FetchedAt.Time} diff --git a/internal/handler/release_index_test.go b/internal/handler/release_index_test.go index d1ca689..b69f4ed 100644 --- a/internal/handler/release_index_test.go +++ b/internal/handler/release_index_test.go @@ -1,6 +1,7 @@ package handler import ( + "context" "fmt" "net/http" "net/http/httptest" @@ -244,3 +245,21 @@ func TestDebianIndexSkipsReleaseWithoutListing(t *testing.T) { t.Errorf("index after InRelease refetch = %q, want %q", got, indexBody(1)) } } + +// TestCachedReleaseChecksStoredBytes checks that a release file whose stored +// bytes no longer match its row, because another instance replaced them, is +// not used to decide which index to serve. +func TestCachedReleaseChecksStoredBytes(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + path := seedMetadataRow(t, db, store) + if proxy.readCachedRelease(context.Background(), "test", "pkg") == nil { + t.Fatal("matching release file not read") + } + store.mu.Lock() + store.files[path] = []byte(`{"v":2}`) + store.mu.Unlock() + if release := proxy.readCachedRelease(context.Background(), "test", "pkg"); release != nil { + t.Errorf("read a release file whose bytes don't match its row: %q", release.body) + } +} From da090443eceb2f99e50f85b33a9a660b1d7eb662 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 12:19:19 +0100 Subject: [PATCH 169/198] Forward Range to upstream for Maven paths proxied uncached Paths proxied without caching sent no request headers, so a resumed download got the whole file back instead of the requested bytes. --- internal/handler/handler.go | 33 ++++++++------------- internal/handler/maven.go | 9 ++++-- internal/handler/maven_repositories_test.go | 25 ++++++++++++++++ 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index dc54269..6c3fef8 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1477,36 +1477,29 @@ func copyArtifactBody(w http.ResponseWriter, reader io.Reader, expectedSize int6 } } -// ProxyUpstream forwards a request to an upstream URL without caching. -// It copies the request, forwards specified headers, and streams the response back. -// forwardHeaders controls the request headers sent upstream. End-to-end response -// headers and trailers are relayed independently of that list. -// upstreamRequest sends r's method to upstreamURL with no request headers. -func (p *Proxy) upstreamRequest(r *http.Request, upstreamURL string) (*http.Response, error) { +// upstreamRequest sends r's method to upstreamURL, copying only the +// forwardHeaders that r sets. +func (p *Proxy) upstreamRequest(r *http.Request, upstreamURL string, forwardHeaders []string) (*http.Response, error) { req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) if err != nil { return nil, err } - return p.HTTPClient.Do(req) -} - -func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamURL string, forwardHeaders []string) { - p.Logger.Debug("proxying to upstream", "url", upstreamURL) - - req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) - if err != nil { - http.Error(w, "failed to create request", http.StatusInternalServerError) - return - } - - // Copy request headers that affect content negotiation / caching for _, header := range forwardHeaders { if v := r.Header.Get(header); v != "" { req.Header.Set(header, v) } } + return p.HTTPClient.Do(req) +} - resp, err := p.HTTPClient.Do(req) +// ProxyUpstream forwards a request to an upstream URL without caching. +// It copies the request, forwards specified headers, and streams the response back. +// forwardHeaders controls the request headers sent upstream. End-to-end response +// headers and trailers are relayed independently of that list. +func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamURL string, forwardHeaders []string) { + p.Logger.Debug("proxying to upstream", "url", upstreamURL) + + resp, err := p.upstreamRequest(r, upstreamURL, forwardHeaders) if err != nil { if writeUpstreamHeld(w, err) { return diff --git a/internal/handler/maven.go b/internal/handler/maven.go index ed417d7..4b170be 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -341,18 +341,21 @@ func (h *MavenHandler) isChecksumFile(filename string) bool { return false } +// mavenForwardHeaders lets clients resume artifacts proxied without caching. +var mavenForwardHeaders = []string{"Range", "If-Range"} + // proxyUpstream forwards a request to the source's upstream without caching. func (h *MavenHandler) proxyUpstream(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { if source.fallbackURL == "" { - h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, nil) + h.proxy.ProxyUpstream(w, r, source.upstreamURL+"/"+urlPath, mavenForwardHeaders) return } // As for cached files, a file the primary upstream lacks is looked for // on the Gradle Plugin Portal. - resp, err := h.proxy.upstreamRequest(r, source.upstreamURL+"/"+urlPath) + resp, err := h.proxy.upstreamRequest(r, source.upstreamURL+"/"+urlPath, mavenForwardHeaders) if err == nil && resp.StatusCode == http.StatusNotFound { _ = resp.Body.Close() - resp, err = h.proxy.upstreamRequest(r, source.fallbackURL+"/"+urlPath) + resp, err = h.proxy.upstreamRequest(r, source.fallbackURL+"/"+urlPath, mavenForwardHeaders) } if err != nil { if writeUpstreamHeld(w, err) { diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go index 8fb6f27..260d3a8 100644 --- a/internal/handler/maven_repositories_test.go +++ b/internal/handler/maven_repositories_test.go @@ -7,6 +7,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "github.com/git-pkgs/registries/fetch" ) @@ -171,3 +172,27 @@ func TestMavenUncachedPathFallsBackToPluginPortal(t *testing.T) { t.Errorf("GET %s = %d %q, want the Plugin Portal's copy", dotted, w.Code, w.Body.String()) } } + +func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { + const dotted = "/org.foo/bar/1.0/bar-1.0.jar" + ranged := func() *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeContent(w, r, "bar.jar", time.Time{}, strings.NewReader("0123456789")) + })) + t.Cleanup(srv.Close) + return srv + } + central, grails := ranged(), ranged() + portal, _ := mavenRepoUpstream(t, "portal") + h := newMavenRepositoriesHandler(t, central, portal, grails) + + for _, p := range []string{dotted, "/grails" + dotted} { + req := httptest.NewRequest(http.MethodGet, p, nil) + req.Header.Set("Range", "bytes=2-4") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusPartialContent || w.Body.String() != "234" { + t.Errorf("GET %s with Range = %d %q, want 206 %q", p, w.Code, w.Body.String(), "234") + } + } +} From 0829eae376be23dedc391d2f73f4eec2965b16d3 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 8 Nov 2026 17:19:03 +0000 Subject: [PATCH 170/198] List only the gitlinks .gitmodules names ls-tree -r read every path in the commit to find a few submodules. --- internal/gitcache/mirror.go | 12 +++++++++++- internal/gitcache/mirror_test.go | 5 ++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index ffcd0d9..0d76a46 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -172,7 +172,17 @@ func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Subm urls[name] = value } } - tree, err := c.run(ctx, dir, "ls-tree", "-r", "-z", "--full-tree", commit) + var pathspecs []string + for path, name := range paths { + if path != "" && urls[name] != "" { + pathspecs = append(pathspecs, path) + } + } + if len(pathspecs) == 0 { + return nil, nil + } + args := append([]string{"--literal-pathspecs", "ls-tree", "-z", "--full-tree", commit, "--"}, pathspecs...) + tree, err := c.run(ctx, dir, args...) if err != nil { return nil, err } diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index 17a19f0..ca734f1 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -261,7 +261,8 @@ func TestMirrorListsSubmodules(t *testing.T) { libCommit := runGit(t, libWork, "rev-parse", "HEAD") work := upstream.newRepo("repo") gitmodules := "[submodule \"vendor/lib\"]\n\tpath = vendor/lib\n\turl = ../lib.git\n" + - "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + "[submodule \"other\"]\n\tpath = third_party/other\n\turl = https://example.com/other.git\n" + + "[submodule \"colon\"]\n\tpath = :colon\n\turl = ../lib.git\n" if err := os.WriteFile(filepath.Join(work, ".gitmodules"), []byte(gitmodules), 0o600); err != nil { t.Fatal(err) } @@ -269,6 +270,7 @@ func TestMirrorListsSubmodules(t *testing.T) { runGit(t, work, "add", ".gitmodules") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") + runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",:colon") runGit(t, work, "commit", "--quiet", "-m", "submodules") runGit(t, work, "push", "--quiet", "origin", "main") commit := runGit(t, work, "rev-parse", "HEAD") @@ -279,6 +281,7 @@ func TestMirrorListsSubmodules(t *testing.T) { t.Fatalf("Mirror: %v", err) } want := []Submodule{ + {Path: ":colon", URL: "../lib.git", Commit: libCommit}, {Path: "third_party/other", URL: "https://example.com/other.git", Commit: otherCommit}, {Path: "vendor/lib", URL: "../lib.git", Commit: libCommit}, } From 1b22f79be2d0021d2bdb41faf0419f7511e57e34 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 8 Nov 2026 17:23:11 +0000 Subject: [PATCH 171/198] Fail the conda timestamp test when the index is empty --- internal/handler/conda_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/handler/conda_test.go b/internal/handler/conda_test.go index 63e518e..f6c37c6 100644 --- a/internal/handler/conda_test.go +++ b/internal/handler/conda_test.go @@ -166,6 +166,9 @@ func TestCondaCooldownSecondTimestamps(t *testing.T) { if err != nil { t.Fatal(err) } + if len(index) != 2 { + t.Fatalf("index has %d entries, want 2", len(index)) + } for filename, published := range index { if published.Unix() != recent.Unix() { t.Errorf("%s published at %v, want %v", filename, published, recent) From 2ecdcced4ca56de5f1e1287bd509d0e001fbb7b4 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 18:51:00 +0100 Subject: [PATCH 172/198] Search again only for the replacement just made in the PyPI root index Each match searched the rest of the buffer for every replacement. The root index has no single-quoted links, so after every double-quoted one the other pattern scanned to the end again, making each chunk quadratic in its links. Each replacement's next match is now kept and only searched for again once a replacement passes it (50,000 links: 18.6 to 818 MB/s). --- internal/handler/pypi.go | 46 ++++++++++++++++--------- internal/handler/pypi_test.go | 63 +++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 16 deletions(-) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 50f9964..a1bee9d 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -1066,18 +1066,6 @@ type replacingReader struct { type replacement struct{ from, to []byte } -// next finds the earliest replacement in b, returning its index or -1. -func (r *replacingReader) next(b []byte) (int, *replacement) { - at, found := -1, (*replacement)(nil) - for i := range r.replacements { - rep := &r.replacements[i] - if j := bytes.Index(b, rep.from); j >= 0 && (at < 0 || j < at) { - at, found = j, rep - } - } - return at, found -} - func (r *replacingReader) Read(p []byte) (int, error) { for len(r.out) == 0 { if r.err != nil { @@ -1099,14 +1087,32 @@ func (r *replacingReader) Read(p []byte) (int, error) { scan := max(len(r.in)-keep, 0) var out []byte done := 0 + // at holds each replacement's next match at or after done, or -1. + // Only a match the last replacement consumed is searched for again, + // so each pass over r.in stays linear. + at := make([]int, len(r.replacements)) + for i := range at { + at[i] = indexFrom(r.in, r.replacements[i].from, 0) + } for { - i, rep := r.next(r.in[done:]) - if i < 0 || done+i >= scan { + k := -1 + for i, j := range at { + if j >= 0 && (k < 0 || j < at[k]) { + k = i + } + } + if k < 0 || at[k] >= scan { break } - out = append(out, r.in[done:done+i]...) + rep := &r.replacements[k] + out = append(out, r.in[done:at[k]]...) out = append(out, rep.to...) - done += i + len(rep.from) + done = at[k] + len(rep.from) + for i, j := range at { + if j >= 0 && j < done { + at[i] = indexFrom(r.in, r.replacements[i].from, done) + } + } } if done < scan { out = append(out, r.in[done:scan]...) @@ -1121,3 +1127,11 @@ func (r *replacingReader) Read(p []byte) (int, error) { } func (r *replacingReader) Close() error { return r.src.Close() } + +// indexFrom returns the index of the first sep in b at or after from, or -1. +func indexFrom(b, sep []byte, from int) int { + if i := bytes.Index(b[from:], sep); i >= 0 { + return from + i + } + return -1 +} diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index c34679d..1b5cbaa 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "log/slog" + "math/rand/v2" "net/http" "net/http/httptest" "strconv" @@ -678,3 +679,65 @@ func TestReplacingReaderSplitReads(t *testing.T) { t.Errorf("got %q, %v; want %q", got, err, want) } } + +// replaceReference makes the replacements the way replacingReader should: +// left to right, the earliest match first, and the first listed on a tie. +func replaceReference(in string, reps []replacement) string { + var out strings.Builder + for len(in) > 0 { + at, found := -1, -1 + for i, rep := range reps { + if j := strings.Index(in, string(rep.from)); j >= 0 && (at < 0 || j < at) { + at, found = j, i + } + } + if at < 0 { + out.WriteString(in) + break + } + out.WriteString(in[:at]) + out.Write(reps[found].to) + in = in[at+len(reps[found].from):] + } + return out.String() +} + +func TestReplacingReaderMatchesReference(t *testing.T) { + reps := []replacement{ + {from: []byte(`href="/s/`), to: []byte(`href="P/`)}, + {from: []byte(`href='/s/`), to: []byte(`href='Q/`)}, + } + pieces := []string{`href="/s/`, `href='/s/`, `href="/s`, `href=`, `x`, `/s/`, `'`, `"`} + rng := rand.New(rand.NewPCG(1, 2)) + for range 500 { + var b strings.Builder + for range rng.IntN(40) { + b.WriteString(pieces[rng.IntN(len(pieces))]) + } + in := b.String() + for _, src := range []io.Reader{strings.NewReader(in), iotest.OneByteReader(strings.NewReader(in))} { + got, err := io.ReadAll(&replacingReader{src: io.NopCloser(src), replacements: reps}) + if want := replaceReference(in, reps); err != nil || string(got) != want { + t.Fatalf("in %q: got %q, %v; want %q", in, got, err, want) + } + } + } +} + +// BenchmarkReplacingReaderRootIndex reads a root index of double-quoted +// links, where the single-quote pattern never matches. +func BenchmarkReplacingReaderRootIndex(b *testing.B) { + var page strings.Builder + for i := range 50000 { + fmt.Fprintf(&page, "project-%d\n", i, i) + } + in := page.String() + reps := []replacement{ + {from: []byte(`href="/simple/`), to: []byte(`href="http://proxy/pypi/simple/`)}, + {from: []byte(`href='/simple/`), to: []byte(`href='http://proxy/pypi/simple/`)}, + } + b.SetBytes(int64(len(in))) + for b.Loop() { + _, _ = io.Copy(io.Discard, &replacingReader{src: io.NopCloser(strings.NewReader(in)), replacements: reps}) + } +} From 140b70233a412fb54b3ddd0c84aa52966ff89fa3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 18:51:46 +0100 Subject: [PATCH 173/198] Move the PyPI root index replacement loop into its own method --- internal/handler/pypi.go | 63 ++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index a1bee9d..b6ffb50 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -1085,35 +1085,7 @@ func (r *replacingReader) Read(p []byte) (int, error) { } } scan := max(len(r.in)-keep, 0) - var out []byte - done := 0 - // at holds each replacement's next match at or after done, or -1. - // Only a match the last replacement consumed is searched for again, - // so each pass over r.in stays linear. - at := make([]int, len(r.replacements)) - for i := range at { - at[i] = indexFrom(r.in, r.replacements[i].from, 0) - } - for { - k := -1 - for i, j := range at { - if j >= 0 && (k < 0 || j < at[k]) { - k = i - } - } - if k < 0 || at[k] >= scan { - break - } - rep := &r.replacements[k] - out = append(out, r.in[done:at[k]]...) - out = append(out, rep.to...) - done = at[k] + len(rep.from) - for i, j := range at { - if j >= 0 && j < done { - at[i] = indexFrom(r.in, r.replacements[i].from, done) - } - } - } + out, done := r.replace(scan) if done < scan { out = append(out, r.in[done:scan]...) done = scan @@ -1126,6 +1098,39 @@ func (r *replacingReader) Read(p []byte) (int, error) { return n, nil } +// replace makes the replacements that start before scan in r.in, returning +// the output and how much of r.in it covers. +func (r *replacingReader) replace(scan int) (out []byte, done int) { + // at holds each replacement's next match at or after done, or -1. + // Only a match the last replacement consumed is searched for again, + // so each pass over r.in stays linear. + at := make([]int, len(r.replacements)) + for i := range at { + at[i] = indexFrom(r.in, r.replacements[i].from, 0) + } + for { + k := -1 + for i, j := range at { + if j >= 0 && (k < 0 || j < at[k]) { + k = i + } + } + if k < 0 || at[k] >= scan { + break + } + rep := &r.replacements[k] + out = append(out, r.in[done:at[k]]...) + out = append(out, rep.to...) + done = at[k] + len(rep.from) + for i, j := range at { + if j >= 0 && j < done { + at[i] = indexFrom(r.in, r.replacements[i].from, done) + } + } + } + return out, done +} + func (r *replacingReader) Close() error { return r.src.Close() } // indexFrom returns the index of the first sep in b at or after from, or -1. From c2f0345e18c20ed5ff2bb183822155a8afafc5c7 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 18:56:24 +0100 Subject: [PATCH 174/198] Pick up the follow-ups from the #676 and #680 reviews The repeated dist-tags name in the jsontext test now has a different first value, so the test checks that the last one counts. Comments that repeat the code go, and the upload-pack paragraph states its check rule once. --- docs/configuration.md | 8 +++----- internal/gitcache/relay.go | 3 +-- internal/metadata/npm/stream.go | 6 ++---- internal/metadata/npm/stream_test.go | 2 +- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6eceb33..868aeb5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -904,11 +904,9 @@ confirmed. A repository the upstream reports missing (404), refuses (401, 403) or redirects is not served from the cache, also to requests that skip the ref listing and go straight to `git-upload-pack`, until a later check succeeds. Redirects are never followed, so point the upstream at a repository's current -location. A `git-upload-pack` request does not check the upstream itself -unless the upstream refused the repository at the last check or the copy was -last checked more than a minute past `stale_max_age` ago, so the requests of a -clone that follow its ref listing check nothing again; otherwise the -repository is checked first, as for a ref listing. +location. A `git-upload-pack` request checks the upstream only when the last +check refused the repository or the copy was last checked more than a minute +past `stale_max_age` ago, so a clone checks once, at its ref listing. The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it). The `git` processes never reach the network themselves: their fetches go diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go index bd65b25..468879c 100644 --- a/internal/gitcache/relay.go +++ b/internal/gitcache/relay.go @@ -39,8 +39,7 @@ type relay struct { type relayRoute struct { repoURL string - // ctx ends when the fetch does, which stops its relayed requests, so a - // git helper left behind by a killed git does not keep downloading. + // ctx ends when the fetch does. ctx context.Context // limit caps the response bytes the route relays, 0 for no cap, and // budget, when set, holds space within MaxSize for them. exceeded and diff --git a/internal/metadata/npm/stream.go b/internal/metadata/npm/stream.go index b00f6a2..aac4071 100644 --- a/internal/metadata/npm/stream.go +++ b/internal/metadata/npm/stream.go @@ -274,8 +274,7 @@ func decodeValue(dec *jsontext.Decoder, v any) error { return json.Unmarshal(raw, v) } -// objectKeys reads a value and returns its keys when it is an object. The -// members' values are skipped without being kept. +// objectKeys reads a value and returns its keys when it is an object. func objectKeys(dec *jsontext.Decoder) ([]string, bool, error) { kind := dec.PeekKind() if kind != '{' && kind != '[' { @@ -391,8 +390,7 @@ func LookupVersion(r io.Reader, version string) (VersionEntry, error) { } // objectMember reads a value and returns its member key when it is an object -// that has one, the last if repeated, or nil. Other members are skipped -// without being kept. +// that has one, the last if repeated, or nil. func objectMember(dec *jsontext.Decoder, key string) (json.RawMessage, error) { kind := dec.PeekKind() if kind != '{' && kind != '[' { diff --git a/internal/metadata/npm/stream_test.go b/internal/metadata/npm/stream_test.go index 0bd93ae..620882d 100644 --- a/internal/metadata/npm/stream_test.go +++ b/internal/metadata/npm/stream_test.go @@ -230,7 +230,7 @@ func TestLookupVersion(t *testing.T) { // invalid UTF-8 in strings, and a version entry's bytes kept as they are. func TestStreamReadsJSONAsEncodingJSON(t *testing.T) { entry := "{ \"name\" : \"demo\",\n \"x\":1, \"x\":2, \"note\":\"caf\xe9\" }" - doc := "{\"readme\":\"bad \xff byte\",\"meta\":{\"k\":1,\"k\":2},\"dist-tags\":{\"latest\":\"1.0.0\",\"latest\":\"1.0.0\"}," + + doc := "{\"readme\":\"bad \xff byte\",\"meta\":{\"k\":1,\"k\":2},\"dist-tags\":{\"latest\":\"0.9.0\",\"latest\":\"1.0.0\"}," + "\"versions\":{\"1.0.0\":" + entry + "},\"time\":{\"1.0.0\":\"2020-01-01T00:00:00Z\"}}\n" if !json.Valid([]byte(strings.ToValidUTF8(doc, "?"))) { t.Fatal("test document isn't JSON") From a330d68564b6034aef3307ea9114e4eff412b7e8 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 18:58:40 +0100 Subject: [PATCH 175/198] Use strings.CutLast for .gitmodules keys Go 1.27 is on main now. --- internal/gitcache/mirror.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/internal/gitcache/mirror.go b/internal/gitcache/mirror.go index c59e905..6e45d43 100644 --- a/internal/gitcache/mirror.go +++ b/internal/gitcache/mirror.go @@ -161,7 +161,7 @@ func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Subm for entry := range strings.SplitSeq(out, "\x00") { key, value, ok := strings.Cut(entry, "\n") section, ok2 := strings.CutPrefix(key, "submodule.") - name, field, dotted := cutLast(section, ".") + name, field, dotted := strings.CutLast(section, ".") if !ok || !ok2 || !dotted { continue } @@ -201,14 +201,6 @@ func (c *Cache) submodules(ctx context.Context, key, dir, commit string) ([]Subm return submodules, nil } -func cutLast(s, sep string) (before, after string, found bool) { - i := strings.LastIndex(s, sep) - if i < 0 { - return s, "", false - } - return s[:i], s[i+len(sep):], true -} - // ensureCommit makes sure the repository has commit, fetching it by ID when // it doesn't and fetchWants allows it. A commit on no branch or tag that is // cached already gets a new wanted ref, so it is kept for wantedMaxAge from From 44afe6997ab69ab70a53f3e85848e7e08bee1cf1 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 19:03:21 +0100 Subject: [PATCH 176/198] Read jsonstream objects with jsontext CopyObject takes the reader and decodes it with jsontext, accepting repeated names and invalid UTF-8 as encoding/json did. Keys are written with the bytes they were read with instead of being re-encoded. Member gets Skip, which reads past a value without keeping it, and npm's RewriteStream skips dropped versions and duplicates with it. conda-forge noarch repodata.json (195 MB): filtering 0.89 to 0.62 s and 379 to 77 MB allocated, the timestamp index 0.84 to 0.61 s. The next packument (32 MB): RewriteStream 69 to 34 ms and 38 to 3.5 MB. --- internal/handler/conda.go | 4 +- internal/handler/conda_repodata_bench_test.go | 59 ++++++++++ internal/jsonstream/jsonstream.go | 103 ++++++++++-------- internal/jsonstream/jsonstream_test.go | 42 ++++++- internal/metadata/npm/stream.go | 16 ++- internal/metadata/npm/stream_bench_test.go | 28 +++++ 6 files changed, 195 insertions(+), 57 deletions(-) create mode 100644 internal/handler/conda_repodata_bench_test.go diff --git a/internal/handler/conda.go b/internal/handler/conda.go index f593820..c9db7a7 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -461,7 +461,7 @@ func (h *CondaHandler) filterRepodata(r io.Reader, w io.Writer) error { // filterRepodataWith is filterRepodata deciding by filter. func (h *CondaHandler) filterRepodataWith(r io.Reader, w io.Writer, filter *repodataFilter) error { - return jsonstream.CopyObject(json.NewDecoder(r), w, func(m *jsonstream.Member) error { + return jsonstream.CopyObject(r, w, func(m *jsonstream.Member) error { if m.Key != "packages" && m.Key != "packages.conda" { return m.Copy() } @@ -619,7 +619,7 @@ func (h *CondaHandler) streamRepodataIndex(r *http.Request, upstreamURL string) // is not limited by metadata_max_size. func readRepodataIndex(r io.Reader) (map[string]time.Time, error) { index := make(map[string]time.Time) - err := jsonstream.CopyObject(json.NewDecoder(io.LimitReader(r, maxRepodataSize)), io.Discard, func(m *jsonstream.Member) error { + err := jsonstream.CopyObject(io.LimitReader(r, maxRepodataSize), io.Discard, func(m *jsonstream.Member) error { if m.Key != "packages" && m.Key != "packages.conda" { return m.Copy() } diff --git a/internal/handler/conda_repodata_bench_test.go b/internal/handler/conda_repodata_bench_test.go new file mode 100644 index 0000000..61e68bd --- /dev/null +++ b/internal/handler/conda_repodata_bench_test.go @@ -0,0 +1,59 @@ +package handler + +import ( + "bytes" + "io" + "log/slog" + "os" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +// repodataBenchBody reads the repodata.json named by CONDA_REPODATA_BENCH, +// such as conda-forge's noarch one, and skips the benchmark without it. +func repodataBenchBody(b *testing.B) []byte { + b.Helper() + path := os.Getenv("CONDA_REPODATA_BENCH") + if path == "" { + b.Skip("CONDA_REPODATA_BENCH names no repodata.json") + } + body, err := os.ReadFile(path) + if err != nil { + b.Fatal(err) + } + return body +} + +func BenchmarkCondaFilterRepodata(b *testing.B) { + body := repodataBenchBody(b) + policy, err := cooldown.NewPolicy(&cooldown.Config{Default: "3d"}, nil) + if err != nil { + b.Fatal(err) + } + proxy := testProxy() + proxy.Logger = slog.New(slog.DiscardHandler) + proxy.Cooldown = policy + h := NewCondaHandler(proxy, "http://proxy.example") + now := time.Now() + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if err := h.filterRepodataWith(bytes.NewReader(body), io.Discard, h.newRepodataFilter(now)); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkCondaReadRepodataIndex(b *testing.B) { + body := repodataBenchBody(b) + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + index, err := readRepodataIndex(bytes.NewReader(body)) + if err != nil || len(index) == 0 { + b.Fatalf("index of %d files, %v", len(index), err) + } + } +} diff --git a/internal/jsonstream/jsonstream.go b/internal/jsonstream/jsonstream.go index 2d3d120..a515f11 100644 --- a/internal/jsonstream/jsonstream.go +++ b/internal/jsonstream/jsonstream.go @@ -5,7 +5,7 @@ package jsonstream import ( "bytes" - "encoding/json" + "encoding/json/jsontext" "errors" "fmt" "io" @@ -20,26 +20,32 @@ var ( ) // Member is one member of an object being copied. Its value must be consumed -// exactly once, with Raw, Copy or Object. +// exactly once, with Raw, Skip, Copy or Object. type Member struct { Key string - dec *json.Decoder + rawKey []byte + dec *jsontext.Decoder out *objectWriter consumed bool } // Raw reads the member's value without writing it. Follow it with Write to -// keep the member. -func (m *Member) Raw() (json.RawMessage, error) { +// keep the member. The bytes are only valid until the next member is read. +func (m *Member) Raw() ([]byte, error) { if m.consumed { return nil, errConsumed } m.consumed = true - var raw json.RawMessage - if err := m.dec.Decode(&raw); err != nil { - return nil, err + return m.dec.ReadValue() +} + +// Skip reads past the member's value, leaving the member out. +func (m *Member) Skip() error { + if m.consumed { + return errConsumed } - return raw, nil + m.consumed = true + return m.dec.SkipValue() } // Write writes the member with value as its raw JSON value. @@ -47,7 +53,7 @@ func (m *Member) Write(value []byte) error { if !m.consumed { return errUnconsumed } - if err := m.out.key(m.Key); err != nil { + if err := m.out.key(m.rawKey); err != nil { return err } _, err := m.out.w.Write(value) @@ -70,54 +76,66 @@ func (m *Member) Object(fn func(*Member) error) error { return errConsumed } m.consumed = true - if err := m.out.key(m.Key); err != nil { + if err := m.out.key(m.rawKey); err != nil { return err } return copyObject(m.dec, m.out.w, fn, true) } -// CopyObject copies the object read from dec to w, calling fn for each -// member. fn decides what is written by how it consumes the member's value; a -// member whose value fn leaves unconsumed is copied unchanged. -func CopyObject(dec *json.Decoder, w io.Writer, fn func(*Member) error) error { +// CopyObject copies the object read from r to w, calling fn for each member. +// fn decides what is written by how it consumes the member's value; a member +// whose value fn leaves unconsumed is copied unchanged. As with +// encoding/json, of repeated names each is passed to fn, and invalid UTF-8 +// is accepted. +func CopyObject(r io.Reader, w io.Writer, fn func(*Member) error) error { + dec := jsontext.NewDecoder(r, jsontext.AllowDuplicateNames(true), jsontext.AllowInvalidUTF8(true)) if err := copyObject(dec, w, fn, false); err != nil { return err } - if _, err := dec.Token(); err != io.EOF { - if err == nil { - err = errors.New("unexpected data after the top-level object") - } - return err + _, err := dec.ReadToken() + switch err { + case io.EOF: + return nil + case nil: + return errors.New("unexpected data after the top-level object") } - return nil + return err } -func copyObject(dec *json.Decoder, w io.Writer, fn func(*Member) error, allowNull bool) error { - tok, err := dec.Token() +func copyObject(dec *jsontext.Decoder, w io.Writer, fn func(*Member) error, allowNull bool) error { + tok, err := dec.ReadToken() if err != nil { return err } - if tok == nil && allowNull { + if tok.Kind() == 'n' && allowNull { _, err := io.WriteString(w, "null") return err } - if tok != json.Delim('{') { + if tok.Kind() != '{' { return ErrNotObject } if _, err := io.WriteString(w, "{"); err != nil { return err } out := &objectWriter{w: w} - for dec.More() { - tok, err := dec.Token() + m := &Member{dec: dec, out: out} + // keyBuf holds the current key; ReadValue's bytes don't outlast fn. + var keyBuf []byte + for dec.PeekKind() != '}' { + // The key is read raw, so it is written with its own bytes. + rawKey, err := dec.ReadValue() if err != nil { return err } - key, ok := tok.(string) - if !ok { - return fmt.Errorf("object key is %T, not a string", tok) + if rawKey.Kind() != '"' { + return fmt.Errorf("object key is %v, not a string", rawKey.Kind()) } - m := &Member{Key: key, dec: dec, out: out} + keyBuf = append(keyBuf[:0], rawKey...) + key, err := jsontext.AppendUnquote(nil, keyBuf) + if err != nil { + return err + } + *m = Member{Key: string(key), rawKey: keyBuf, dec: dec, out: out} if err := fn(m); err != nil { return err } @@ -127,7 +145,7 @@ func copyObject(dec *json.Decoder, w io.Writer, fn func(*Member) error, allowNul } } } - if _, err := dec.Token(); err != nil { + if _, err := dec.ReadToken(); err != nil { return err } _, err = io.WriteString(w, "}") @@ -137,23 +155,18 @@ func copyObject(dec *json.Decoder, w io.Writer, fn func(*Member) error, allowNul type objectWriter struct { w io.Writer wrote bool + buf bytes.Buffer } -// key writes the separator and key for the next member. Keys are re-encoded -// without HTML escaping, so plain keys keep their original bytes. -func (o *objectWriter) key(k string) error { - var buf bytes.Buffer +// key writes the separator and the key's bytes for the next member. +func (o *objectWriter) key(rawKey []byte) error { + o.buf.Reset() if o.wrote { - buf.WriteByte(',') + o.buf.WriteByte(',') } - enc := json.NewEncoder(&buf) - enc.SetEscapeHTML(false) - if err := enc.Encode(k); err != nil { - return err - } - buf.Truncate(buf.Len() - 1) // Encode appends a newline - buf.WriteByte(':') + o.buf.Write(rawKey) + o.buf.WriteByte(':') o.wrote = true - _, err := o.w.Write(buf.Bytes()) + _, err := o.w.Write(o.buf.Bytes()) return err } diff --git a/internal/jsonstream/jsonstream_test.go b/internal/jsonstream/jsonstream_test.go index 44a877e..e7bca69 100644 --- a/internal/jsonstream/jsonstream_test.go +++ b/internal/jsonstream/jsonstream_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "slices" "strings" "testing" ) @@ -11,7 +12,7 @@ import ( func copyString(t *testing.T, in string, fn func(*Member) error) (string, error) { t.Helper() var out bytes.Buffer - err := CopyObject(json.NewDecoder(strings.NewReader(in)), &out, fn) + err := CopyObject(strings.NewReader(in), &out, fn) return out.String(), err } @@ -140,3 +141,42 @@ func TestCopyObjectErrors(t *testing.T) { t.Errorf("err = %v, want ErrNotObject", err) } } + +// TestCopyObjectReadsJSONAsEncodingJSON checks CopyObject accepts what +// encoding/json does: repeated names, each passed to fn, and invalid UTF-8. +// Keys keep their bytes, escapes included, while Key is unquoted. +func TestCopyObjectReadsJSONAsEncodingJSON(t *testing.T) { + var keys []string + in := "{\"a\":1,\"a\":2,\"caf\\u00e9\":\"bad \xff byte\",\"skip\":{\"x\":[1,2]}}" + got, err := copyString(t, in, func(m *Member) error { + keys = append(keys, m.Key) + if m.Key == "skip" { + return m.Skip() + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if want := "{\"a\":1,\"a\":2,\"caf\\u00e9\":\"bad \xff byte\"}"; got != want { + t.Errorf("got %q, want %q", got, want) + } + if want := []string{"a", "a", "café", "skip"}; !slices.Equal(keys, want) { + t.Errorf("keys %q, want %q", keys, want) + } +} + +func TestSkipThenWriteReplaces(t *testing.T) { + got, err := copyString(t, `{"a":{"big":[1,2,3]},"b":2}`, func(m *Member) error { + if m.Key != "a" { + return nil + } + if err := m.Skip(); err != nil { + return err + } + return m.Write([]byte(`"small"`)) + }) + if err != nil || got != `{"a":"small","b":2}` { + t.Errorf("got %s, %v", got, err) + } +} diff --git a/internal/metadata/npm/stream.go b/internal/metadata/npm/stream.go index b00f6a2..8788de1 100644 --- a/internal/metadata/npm/stream.go +++ b/internal/metadata/npm/stream.go @@ -102,7 +102,7 @@ func RewriteStream(packageName string, scan *Scan, r io.Reader, w io.Writer, opt filtered: f, seen: map[string]int{}, } - if err := jsonstream.CopyObject(json.NewDecoder(r), w, sw.member); err != nil { + if err := jsonstream.CopyObject(r, w, sw.member); err != nil { return nil, err } h.tarballsRewritten(sw.rewritten) @@ -162,8 +162,7 @@ func (s *streamWriter) member(m *jsonstream.Member) error { } s.seen[m.Key]++ if s.seen[m.Key] < total { - _, err := m.Raw() // an earlier duplicate, dropped - return err + return m.Skip() // an earlier duplicate, dropped } switch { case m.Key == "versions": @@ -171,7 +170,7 @@ func (s *streamWriter) member(m *jsonstream.Member) error { case m.Key == "time" && s.timeChanged: return m.Object(s.time) case m.Key == "dist-tags" && s.tagsChanged: - if _, err := m.Raw(); err != nil { + if err := m.Skip(); err != nil { return err } encoded, err := json.Marshal(s.distTags) @@ -185,13 +184,13 @@ func (s *streamWriter) member(m *jsonstream.Member) error { // version writes a kept version with its tarball pointing at the proxy. func (s *streamWriter) version(v *jsonstream.Member) error { + if _, keep := s.versions[v.Key]; !keep { + return v.Skip() + } raw, err := v.Raw() if err != nil { return err } - if _, keep := s.versions[v.Key]; !keep { - return nil - } s.entry.Reset() if s.writeNPMVersion(&s.entry, s.packageName, v.Key, raw) { s.rewritten++ @@ -204,8 +203,7 @@ func (s *streamWriter) time(t *jsonstream.Member) error { if _, keep := s.timeMap[t.Key]; keep { return t.Copy() } - _, err := t.Raw() - return err + return t.Skip() } // objectValue records value under name in metadata, as Rewrite's diff --git a/internal/metadata/npm/stream_bench_test.go b/internal/metadata/npm/stream_bench_test.go index 827cf94..12d04d8 100644 --- a/internal/metadata/npm/stream_bench_test.go +++ b/internal/metadata/npm/stream_bench_test.go @@ -2,6 +2,8 @@ package npm import ( "bytes" + "io" + "os" "testing" "github.com/git-pkgs/proxy/internal/metadata/npm/npmbench" @@ -29,3 +31,29 @@ func BenchmarkLookupVersion(b *testing.B) { } } } + +// BenchmarkRewriteStream times RewriteStream's second pass, the copy, over +// the 5,000-version packument, or the one NPM_PACKUMENT_BENCH names. +func BenchmarkRewriteStream(b *testing.B) { + body := npmbench.Packument(5000) + name := "big" + if path := os.Getenv("NPM_PACKUMENT_BENCH"); path != "" { + var err error + if body, err = os.ReadFile(path); err != nil { + b.Fatal(err) + } + name = os.Getenv("NPM_PACKUMENT_BENCH_NAME") + } + scan, err := ScanPackument(bytes.NewReader(body)) + if err != nil { + b.Fatal(err) + } + options := Options{ProxyURL: "http://proxy.example"} + b.SetBytes(int64(len(body))) + b.ReportAllocs() + for b.Loop() { + if _, err := RewriteStream(name, scan, bytes.NewReader(body), io.Discard, options); err != nil { + b.Fatal(err) + } + } +} From 0efe54fd73ff3c2aaf30372db48ee715b25d07f2 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 19:11:36 +0100 Subject: [PATCH 177/198] Let the submodule test add a ':' path on Windows Git for Windows refuses it with core.protectNTFS on. --- internal/gitcache/mirror_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/gitcache/mirror_test.go b/internal/gitcache/mirror_test.go index ca734f1..d016f2a 100644 --- a/internal/gitcache/mirror_test.go +++ b/internal/gitcache/mirror_test.go @@ -270,7 +270,8 @@ func TestMirrorListsSubmodules(t *testing.T) { runGit(t, work, "add", ".gitmodules") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",vendor/lib") runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+otherCommit+",third_party/other") - runGit(t, work, "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",:colon") + // Git for Windows refuses ':' in paths unless NTFS protection is off. + runGit(t, work, "-c", "core.protectNTFS=false", "update-index", "--add", "--cacheinfo", "160000,"+libCommit+",:colon") runGit(t, work, "commit", "--quiet", "-m", "submodules") runGit(t, work, "push", "--quiet", "origin", "main") commit := runGit(t, work, "rev-parse", "HEAD") From 23316c04d297c98e827c8cd6dc1a4121afc13210 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 19:12:30 +0100 Subject: [PATCH 178/198] Accept object keys with invalid UTF-8 in jsonstream AppendUnquote reports invalid UTF-8 after replacing it, which failed the whole copy although the decoder allows it. Key now holds U+FFFD in its place, as with encoding/json and ScanPackument, and the key is still written with its original bytes. --- internal/jsonstream/jsonstream.go | 8 ++++---- internal/jsonstream/jsonstream_test.go | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/internal/jsonstream/jsonstream.go b/internal/jsonstream/jsonstream.go index a515f11..cf213ce 100644 --- a/internal/jsonstream/jsonstream.go +++ b/internal/jsonstream/jsonstream.go @@ -131,10 +131,10 @@ func copyObject(dec *jsontext.Decoder, w io.Writer, fn func(*Member) error, allo return fmt.Errorf("object key is %v, not a string", rawKey.Kind()) } keyBuf = append(keyBuf[:0], rawKey...) - key, err := jsontext.AppendUnquote(nil, keyBuf) - if err != nil { - return err - } + // The decoder has checked the key, so the only error left is invalid + // UTF-8, which AppendUnquote reports after replacing it with U+FFFD, + // as encoding/json does. + key, _ := jsontext.AppendUnquote(nil, keyBuf) *m = Member{Key: string(key), rawKey: keyBuf, dec: dec, out: out} if err := fn(m); err != nil { return err diff --git a/internal/jsonstream/jsonstream_test.go b/internal/jsonstream/jsonstream_test.go index e7bca69..cd201ae 100644 --- a/internal/jsonstream/jsonstream_test.go +++ b/internal/jsonstream/jsonstream_test.go @@ -180,3 +180,23 @@ func TestSkipThenWriteReplaces(t *testing.T) { t.Errorf("got %s, %v", got, err) } } + +// TestCopyObjectInvalidUTF8Key checks a key with invalid UTF-8 is accepted: +// Key has U+FFFD in its place, as with encoding/json, and the key is written +// with its original bytes. +func TestCopyObjectInvalidUTF8Key(t *testing.T) { + var keys []string + got, err := copyString(t, "{\"k\xff\":1}", func(m *Member) error { + keys = append(keys, m.Key) + return nil + }) + if err != nil { + t.Fatal(err) + } + if want := "{\"k\xff\":1}"; got != want { + t.Errorf("got %q, want %q", got, want) + } + if want := []string{"k�"}; !slices.Equal(keys, want) { + t.Errorf("keys %q, want %q", keys, want) + } +} From 79a0ae2dd07a8c3cf8e432eb86eee98117f5a5b9 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 19:14:18 +0100 Subject: [PATCH 179/198] Read the row again when stale metadata bytes don't match it The fallback served replaced bytes with the old row's content type and encoding, so a blob swapped from identity to gzip went out without its Content-Encoding. It now reads the row once more and serves the blob under the row that describes it, and treats the copy as unusable if none does yet. --- internal/handler/handler.go | 34 ++++++------ internal/handler/metadata_consistency_test.go | 53 ++++++++++++++++++- 2 files changed, 69 insertions(+), 18 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index bcbaa42..be3b60e 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1826,23 +1826,25 @@ func (p *Proxy) staleMetadata(ctx context.Context, ecosystem, cacheKey string, v // Re-read the row so the encoding describes the blob as it is now: a // concurrent refetch may have replaced both since entry was read above - // (an identity blob swapped for a gzip one during rollout). - entry = p.currentMetadataEntry(ecosystem, cacheKey, entry) - - data, ct, readErr := p.readCachedMetadata(ctx, entry, validate) - if readErr != nil { - return metadataResult{err: fmt.Errorf("upstream failed and cached metadata unusable (%v): %w", readErr, err)} + // (an identity blob swapped for a gzip one during rollout). Bytes that + // don't match the row were replaced after it was read, so the row is + // read once more; if they still don't match, no row describes them. + for attempt := 0; ; attempt++ { + entry = p.currentMetadataEntry(ecosystem, cacheKey, entry) + data, ct, readErr := p.readCachedMetadata(ctx, entry, validate) + if readErr == nil && !matchesEntryDigest(entry, data) { + if attempt == 0 { + continue + } + readErr = errors.New("replaced while being read") + } + if readErr != nil { + return metadataResult{err: fmt.Errorf("upstream failed and cached metadata unusable (%v): %w", readErr, err)} + } + p.Logger.Info("serving metadata from cache", + "ecosystem", ecosystem, "key", cacheKey) + return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true, upstreamErr: err}.describedBy(entry) } - - p.Logger.Info("serving metadata from cache", - "ecosystem", ecosystem, "key", cacheKey) - res := metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String, stale: true, upstreamErr: err} - if !matchesEntryDigest(entry, data) { - // Replaced since the row was read; the row's validators are not - // this blob's. - return res - } - return res.describedBy(entry) } // metadataResult is what a metadata lookup hands back. A shared fetch gives diff --git a/internal/handler/metadata_consistency_test.go b/internal/handler/metadata_consistency_test.go index 9ac84de..3e0220a 100644 --- a/internal/handler/metadata_consistency_test.go +++ b/internal/handler/metadata_consistency_test.go @@ -1,6 +1,8 @@ package handler import ( + "bytes" + "compress/gzip" "context" "database/sql" "errors" @@ -268,8 +270,55 @@ func TestStaleMetadataChecksStoredBytes(t *testing.T) { store.mu.Unlock() }} + // No row describes v2 yet, so its encoding is unknown: nothing is served. w := proxyCachedRequest(proxy, upstream.URL, http.Header{"If-None-Match": {`"v1"`}}) - if w.Code == http.StatusNotModified || w.Header().Get(headerETag) == `"v1"` { - t.Errorf("response = %d ETag %q for v2's bytes, want no v1 validators", w.Code, w.Header().Get(headerETag)) + if w.Code < http.StatusInternalServerError || w.Header().Get(headerETag) == `"v1"` { + t.Errorf("response = %d ETag %q for v2's bytes, want an error without v1 validators", w.Code, w.Header().Get(headerETag)) + } +} + +// TestStaleMetadataRereadsReplacedRow checks that when the stored bytes no +// longer match the row, the stale fallback reads the row again and serves +// the blob with the encoding and validators of the row that describes it. +func TestStaleMetadataRereadsReplacedRow(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + t.Cleanup(upstream.Close) + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 + proxy.HTTPClient = upstream.Client() + path := seedMetadataRow(t, db, store) + var gz bytes.Buffer + zw := gzip.NewWriter(&gz) + _, _ = zw.Write([]byte(`{"v":2}`)) + _ = zw.Close() + v2 := gz.Bytes() + proxy.Storage = &openHookStorage{mockStorage: store, onOpen: func() { + // The other instance stores v2 gzipped, then its row. + store.mu.Lock() + store.files[path] = v2 + store.mu.Unlock() + if err := db.UpsertMetadataCache(&database.MetadataCacheEntry{ + Ecosystem: "test", + Name: "pkg", + StoragePath: path, + ETag: sql.NullString{String: `"v2"`, Valid: true}, + ContentType: sql.NullString{String: contentTypeJSON, Valid: true}, + ContentEncoding: sql.NullString{String: "gzip", Valid: true}, + ContentDigest: sql.NullString{String: "sha256:" + sha256Hex(string(v2)), Valid: true}, + Size: sql.NullInt64{Int64: int64(len(v2)), Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now().Add(-time.Hour), Valid: true}, + }); err != nil { + t.Error(err) + } + }} + + w := proxyCachedRequest(proxy, upstream.URL, http.Header{"Accept-Encoding": {"gzip"}}) + if w.Code != http.StatusOK || w.Header().Get("Content-Encoding") != "gzip" || + w.Header().Get(headerETag) != `"v2"` || !bytes.Equal(w.Body.Bytes(), v2) { + t.Errorf("response = %d, Content-Encoding %q, ETag %q; want v2 gzipped under its own row", + w.Code, w.Header().Get("Content-Encoding"), w.Header().Get(headerETag)) } } From b61e35205c47238421c0c0b1b0c5a649d621e750 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 21:49:48 +0100 Subject: [PATCH 180/198] Answer 404 for Maven files outside the layout A file whose group directories hold a dot or colon was proxied without caching, which skipped the denylist, scan and cooldown. Such paths aren't in the Maven layout, so they now get 404. Directory listings and catalogs still go through proxyUpstream, which shares its error handling with Proxy.ProxyUpstream in relayUpstream. --- internal/handler/handler.go | 6 +++ internal/handler/maven.go | 18 ++++---- internal/handler/maven_repositories_test.go | 51 +++++++++++++-------- 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index b3ef666..9612e51 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1500,6 +1500,12 @@ func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamUR p.Logger.Debug("proxying to upstream", "url", upstreamURL) resp, err := p.upstreamRequest(r, upstreamURL, forwardHeaders) + p.relayUpstream(w, r, resp, err) +} + +// relayUpstream writes the result of an upstream request: the response, or +// a 502 when the request failed. +func (p *Proxy) relayUpstream(w http.ResponseWriter, r *http.Request, resp *http.Response, err error) { if err != nil { if writeUpstreamHeld(w, err) { return diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 4b170be..5008e00 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -221,6 +221,13 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so // e.g., com/google/guava/guava/32.1.3-jre/guava-32.1.3-jre.jar group, artifact, version, filename := h.parsePath(urlPath) if artifact == "" { + if strings.Count(urlPath, "/") >= minMavenParts-1 { + // A dot or colon in a group directory, or a colon in the + // artifact, which the Maven layout never has. Proxied, the + // file would skip the denylist, scan and cooldown. + http.NotFound(w, r) + return + } h.proxyUpstream(w, r, source, urlPath) return } @@ -357,14 +364,5 @@ func (h *MavenHandler) proxyUpstream(w http.ResponseWriter, r *http.Request, sou _ = resp.Body.Close() resp, err = h.proxy.upstreamRequest(r, source.fallbackURL+"/"+urlPath, mavenForwardHeaders) } - if err != nil { - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("upstream request failed", "error", err) - http.Error(w, "upstream request failed", http.StatusBadGateway) - return - } - defer func() { _ = resp.Body.Close() }() - h.proxy.relayResponse(w, r, resp, nil) + h.proxy.relayUpstream(w, r, resp, err) } diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go index 260d3a8..923af92 100644 --- a/internal/handler/maven_repositories_test.go +++ b/internal/handler/maven_repositories_test.go @@ -139,45 +139,58 @@ func TestMavenRepositoryRepointedServesTheNewUpstream(t *testing.T) { } } -// TestMavenDottedGroupDirectoryNotCachedAsGroup checks that a path whose -// group directories contain dots is not cached under the group the dots -// spell, where it would share an entry with the standard layout's path. +// TestMavenDottedGroupDirectoryNotCachedAsGroup checks that a file whose +// group directories contain dots or colons, which the Maven layout never +// has, is not served: it would either share the cache entry of the group +// the dots spell or be proxied past the denylist, scan and cooldown. func TestMavenDottedGroupDirectoryNotCachedAsGroup(t *testing.T) { const standard = "/org/foo/bar/1.0/bar-1.0.jar" - const dotted = "/org.foo/bar/1.0/bar-1.0.jar" - central, _ := mavenRepoUpstream(t, "central", standard, dotted) + odd := []string{"/org.foo/bar/1.0/bar-1.0.jar", "/org/foo:x/bar/1.0/bar-1.0.jar", "/org/foo/b:ar/1.0/b:ar-1.0.jar"} + central, requests := mavenRepoUpstream(t, "central", append([]string{standard}, odd...)...) portal, _ := mavenRepoUpstream(t, "portal") - grails, _ := mavenRepoUpstream(t, "grails") + var restricted []string + for _, p := range odd { + restricted = append(restricted, "/restricted"+p) + } + grails, grailsRequests := mavenRepoUpstream(t, "grails", restricted...) h := newMavenRepositoriesHandler(t, central, portal, grails) - for _, path := range []string{standard, dotted, standard} { - w := getMaven(h, path) - if want := "central " + path; w.Code != http.StatusOK || w.Body.String() != want { - t.Errorf("GET %s = %d %q, want 200 %q", path, w.Code, w.Body.String(), want) + if w := getMaven(h, standard); w.Code != http.StatusOK || w.Body.String() != "central "+standard { + t.Errorf("GET %s = %d %q", standard, w.Code, w.Body.String()) + } + before := requests.Load() + grailsRequests.Load() + for _, p := range odd { + for _, path := range []string{p, "/grails" + p} { + if w := getMaven(h, path); w.Code != http.StatusNotFound { + t.Errorf("GET %s = %d %q, want 404", path, w.Code, w.Body.String()) + } } } + if n := requests.Load() + grailsRequests.Load() - before; n != 0 { + t.Errorf("%d upstream requests for paths outside the Maven layout", n) + } } // TestMavenUncachedPathFallsBackToPluginPortal checks that a path proxied -// without caching, such as one with a dotted group directory, still tries -// the Gradle Plugin Portal when the primary upstream lacks it. +// without caching, such as a catalog, still tries the Gradle Plugin Portal +// when the primary upstream lacks it. func TestMavenUncachedPathFallsBackToPluginPortal(t *testing.T) { - const dotted = "/org.foo/bar/1.0/bar-1.0.jar" + const uncached = "/org/foo/archetype-catalog.xml" central, _ := mavenRepoUpstream(t, "central") - portal, _ := mavenRepoUpstream(t, "portal", dotted) + portal, _ := mavenRepoUpstream(t, "portal", uncached) grails, _ := mavenRepoUpstream(t, "grails") h := newMavenRepositoriesHandler(t, central, portal, grails) - if w := getMaven(h, dotted); w.Code != http.StatusOK || w.Body.String() != "portal "+dotted { - t.Errorf("GET %s = %d %q, want the Plugin Portal's copy", dotted, w.Code, w.Body.String()) + if w := getMaven(h, uncached); w.Code != http.StatusOK || w.Body.String() != "portal "+uncached { + t.Errorf("GET %s = %d %q, want the Plugin Portal's copy", uncached, w.Code, w.Body.String()) } } func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { - const dotted = "/org.foo/bar/1.0/bar-1.0.jar" + const uncached = "/org/foo/archetype-catalog.xml" ranged := func() *httptest.Server { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.ServeContent(w, r, "bar.jar", time.Time{}, strings.NewReader("0123456789")) + http.ServeContent(w, r, "archetype-catalog.xml", time.Time{}, strings.NewReader("0123456789")) })) t.Cleanup(srv.Close) return srv @@ -186,7 +199,7 @@ func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { portal, _ := mavenRepoUpstream(t, "portal") h := newMavenRepositoriesHandler(t, central, portal, grails) - for _, p := range []string{dotted, "/grails" + dotted} { + for _, p := range []string{uncached, "/grails" + uncached} { req := httptest.NewRequest(http.MethodGet, p, nil) req.Header.Set("Range", "bytes=2-4") w := httptest.NewRecorder() From 3776366ba22582ef2c5940686bf9072bf826491b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 21:58:47 +0100 Subject: [PATCH 181/198] Document the 404 for Maven files outside the layout --- docs/configuration.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index e61c7e9..4272a84 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1018,9 +1018,10 @@ one doesn't fall back to the Gradle Plugin Portal. Files are cached per repository, so the same coordinates in two repositories never share a cache entry. -In every repository, a path whose group directories contain a dot or are empty -(`org.foo/bar/1.0/bar-1.0.jar`) is not in the Maven layout and is proxied -without caching, since its group would read the same as `org/foo`'s. +In every repository, a file whose group directories contain a dot or colon or +are empty (`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, +is not in the Maven layout and gets 404 without an upstream request. Its group +would read the same as `org/foo`'s. `upstream.maven_repositories` is additive. `/maven/…` keeps serving `upstream.maven` with unchanged cache identities. A repository name is matched From 4ae55605b14c52749aa780c673824d74e0140b7f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 22:18:14 +0100 Subject: [PATCH 182/198] Send Maven coordinate files under odd groups to the 404 A file named {artifact}-{version} under a dotted group directory, such as org.foo/bar/1.0/bar-1.0-linux.exe, was not recognised as a coordinate file and so was proxied past the checks. isCoordinateFile no longer checks the group, so handleDownload answers 404 for it. The docs say which files get the 404, and the Range test covers a stale If-Range. --- docs/configuration.md | 10 ++++++---- internal/handler/maven.go | 9 ++++++++- internal/handler/maven_repositories_test.go | 16 +++++++++++++++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 4272a84..9aa7d80 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1018,10 +1018,12 @@ one doesn't fall back to the Gradle Plugin Portal. Files are cached per repository, so the same coordinates in two repositories never share a cache entry. -In every repository, a file whose group directories contain a dot or colon or -are empty (`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, -is not in the Maven layout and gets 404 without an upstream request. Its group -would read the same as `org/foo`'s. +In every repository, an artifact file (one with an artifact extension or named +`{artifact}-{version}…`) whose group directories contain a dot or colon or are +empty (`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, is +not in the Maven layout and gets 404 without an upstream request. Its group +would read the same as `org/foo`'s. Other paths there, such as +`maven-metadata.xml` or a directory listing, are handled as anywhere else. `upstream.maven_repositories` is additive. `/maven/…` keeps serving `upstream.maven` with unchanged cache identities. A repository name is matched diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 5008e00..e841d08 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -321,8 +321,15 @@ func (h *MavenHandler) isArtifactFile(filename string) bool { // artifact and version of its directory, {artifact}-{version}..., as Maven // names every file it publishes, whatever the extension. A SNAPSHOT's // files carry a timestamp in place of "SNAPSHOT". +// +// The group isn't checked, so a file under a group directory outside the +// Maven layout still goes to handleDownload, which answers 404. func (h *MavenHandler) isCoordinateFile(urlPath string) bool { - _, artifact, version, filename := h.parsePath(urlPath) + parts := strings.Split(urlPath, "/") + if len(parts) < minMavenParts { + return false + } + filename, version, artifact := parts[len(parts)-1], parts[len(parts)-2], parts[len(parts)-3] if artifact == "" || version == "" { return false } diff --git a/internal/handler/maven_repositories_test.go b/internal/handler/maven_repositories_test.go index 923af92..e9b2fcf 100644 --- a/internal/handler/maven_repositories_test.go +++ b/internal/handler/maven_repositories_test.go @@ -145,7 +145,10 @@ func TestMavenRepositoryRepointedServesTheNewUpstream(t *testing.T) { // the dots spell or be proxied past the denylist, scan and cooldown. func TestMavenDottedGroupDirectoryNotCachedAsGroup(t *testing.T) { const standard = "/org/foo/bar/1.0/bar-1.0.jar" - odd := []string{"/org.foo/bar/1.0/bar-1.0.jar", "/org/foo:x/bar/1.0/bar-1.0.jar", "/org/foo/b:ar/1.0/b:ar-1.0.jar"} + odd := []string{ + "/org.foo/bar/1.0/bar-1.0.jar", "/org.foo/bar/1.0/bar-1.0-linux.exe", + "/org/foo:x/bar/1.0/bar-1.0.jar", "/org/foo/b:ar/1.0/b:ar-1.0.jar", + } central, requests := mavenRepoUpstream(t, "central", append([]string{standard}, odd...)...) portal, _ := mavenRepoUpstream(t, "portal") var restricted []string @@ -190,6 +193,7 @@ func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { const uncached = "/org/foo/archetype-catalog.xml" ranged := func() *httptest.Server { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("ETag", `"v2"`) http.ServeContent(w, r, "archetype-catalog.xml", time.Time{}, strings.NewReader("0123456789")) })) t.Cleanup(srv.Close) @@ -207,5 +211,15 @@ func TestMavenUncachedPathKeepsByteRanges(t *testing.T) { if w.Code != http.StatusPartialContent || w.Body.String() != "234" { t.Errorf("GET %s with Range = %d %q, want 206 %q", p, w.Code, w.Body.String(), "234") } + + // A Range for an older copy gets the whole current file. + req = httptest.NewRequest(http.MethodGet, p, nil) + req.Header.Set("Range", "bytes=2-4") + req.Header.Set("If-Range", `"v1"`) + w = httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusOK || w.Body.String() != "0123456789" { + t.Errorf("GET %s with stale If-Range = %d %q, want 200 with the whole file", p, w.Code, w.Body.String()) + } } } From d7399b10435925d80815b6f726ae2e5dab8a2a62 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sun, 8 Nov 2026 22:51:25 +0100 Subject: [PATCH 183/198] Say which Maven files get the 404 without listing name rules --- docs/configuration.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9aa7d80..d887509 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1018,12 +1018,12 @@ one doesn't fall back to the Gradle Plugin Portal. Files are cached per repository, so the same coordinates in two repositories never share a cache entry. -In every repository, an artifact file (one with an artifact extension or named -`{artifact}-{version}…`) whose group directories contain a dot or colon or are -empty (`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, is -not in the Maven layout and gets 404 without an upstream request. Its group -would read the same as `org/foo`'s. Other paths there, such as -`maven-metadata.xml` or a directory listing, are handled as anywhere else. +In every repository, a file the proxy would cache (an artifact, checksum or +signature) whose group directories contain a dot or colon +(`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, is not in +the Maven layout and gets 404 without an upstream request. Its group would read +the same as `org/foo`'s. Other paths there, such as `maven-metadata.xml` or a +directory listing, are handled as anywhere else. `upstream.maven_repositories` is additive. `/maven/…` keeps serving `upstream.maven` with unchanged cache identities. A repository name is matched From b7190354249d700cc29826cde81c822bb392176b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 9 Nov 2026 11:57:39 +0100 Subject: [PATCH 184/198] Write Arch databases with their own validators Arch wrote databases and signatures with the validators of the cache row read at write time, which may describe other bytes once another instance rewrote it. Its results now carry validators: fetchedResult for a fetched database, describedBy for a stored one, and stale on the fallback after an upstream failure, which also marks it with metadata_ttl 0. Nothing calls writeMetadataCachedResponseWithEncoding or lookupCachedMeta any more. --- internal/handler/arch.go | 33 +++++++++++++--------- internal/handler/arch_test.go | 53 +++++++++++++++++++++++++++++++++++ internal/handler/handler.go | 22 --------------- 3 files changed, 72 insertions(+), 36 deletions(-) diff --git a/internal/handler/arch.go b/internal/handler/arch.go index b53f25c..da1a6f1 100644 --- a/internal/handler/arch.go +++ b/internal/handler/arch.go @@ -199,12 +199,12 @@ func (h *ArchHandler) handleDatabase(w http.ResponseWriter, r *http.Request, rep d := h.database(repository, upstreamURL, path) if _, hit := p.cachedMetadataState(r.Context(), archEcosystem, d.dbKey, nil); hit != nil { metrics.RecordMetadataCacheHit(archEcosystem, len(hit.body)) - h.writeMetadata(w, r, d.dbKey, *hit) + h.writeMetadata(w, r, *hit) return } p.recordMetadataCacheMiss(archEcosystem) res := h.sharedRefresh(r.Context(), d, true) - h.writeMetadata(w, r, d.dbKey, res) + h.writeMetadata(w, r, res) } // handleDatabaseSignature serves the signature stored for the cached @@ -218,7 +218,7 @@ func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Req d := h.database(repository, upstreamURL, strings.TrimSuffix(path, archSignatureSuffix)) if res, ok := h.cachedSignature(r.Context(), d); ok { metrics.RecordMetadataCacheHit(archEcosystem, len(res.body)) - h.writeMetadata(w, r, d.sigKey, res) + h.writeMetadata(w, r, res) return } p.recordMetadataCacheMiss(archEcosystem) @@ -242,7 +242,7 @@ func (h *ArchHandler) handleDatabaseSignature(w http.ResponseWriter, r *http.Req } return metadataResult{err: errArchSignatureUnpaired} }) - h.writeMetadata(w, r, d.sigKey, res) + h.writeMetadata(w, r, res) } var errArchSignatureUnpaired = errors.New("database signature could not be paired with the database") @@ -324,7 +324,7 @@ func (h *ArchHandler) fetchDatabase(ctx context.Context, d archDatabase, entry * return metadataResult{err: fmt.Errorf("upstream failed and no cached metadata: %w", err)}, nil } p.Logger.Warn("upstream metadata fetch failed, checking cache", "ecosystem", archEcosystem, "key", d.dbKey, "error", err) - return h.readDatabase(ctx, p.currentMetadataEntry(archEcosystem, d.dbKey, entry), err) + return h.readDatabase(ctx, p.currentMetadataEntry(archEcosystem, d.dbKey, entry), err, true) } // storeDatabase caches meta under its digest and returns the database the @@ -345,15 +345,15 @@ func (h *ArchHandler) storeDatabase(ctx context.Context, d archDatabase, meta *u row, _ := p.DB.GetMetadataCache(archEcosystem, d.dbKey) if row == nil { // Nothing cached at all: serve it like an uncached proxy would. - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding}, nil + return fetchedResult(meta, ""), nil } if row.StoragePath != path { - return h.readDatabase(ctx, row, errArchDatabaseNotCached) + return h.readDatabase(ctx, row, errArchDatabaseNotCached, false) } if previous != nil && previous.StoragePath != path { _ = p.Storage.Delete(ctx, previous.StoragePath) } - return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding}, row + return fetchedResult(meta, row.ContentDigest.String), row } // blobExists reports whether path is stored. Size eviction can remove a blob @@ -377,13 +377,18 @@ func (h *ArchHandler) touchDatabase(row *database.MetadataCacheEntry, meta *upst var errArchDatabaseNotCached = errors.New("database could not be cached") // readDatabase returns the cached database row's bytes, or cause if they are -// unreadable. -func (h *ArchHandler) readDatabase(ctx context.Context, row *database.MetadataCacheEntry, cause error) (metadataResult, *database.MetadataCacheEntry) { +// unreadable. stale marks them as served because the upstream failed with +// cause. +func (h *ArchHandler) readDatabase(ctx context.Context, row *database.MetadataCacheEntry, cause error, stale bool) (metadataResult, *database.MetadataCacheEntry) { body, contentType, err := h.proxy.readCachedMetadata(ctx, row, nil) if err != nil { return metadataResult{err: fmt.Errorf("cached database unusable (%v): %w", err, cause)}, nil } - return metadataResult{body: body, contentType: contentType, contentEncoding: row.ContentEncoding.String}, row + res := metadataResult{body: body, contentType: contentType, contentEncoding: row.ContentEncoding.String} + if stale { + res.stale, res.upstreamErr = true, cause + } + return res.describedBy(row), row } // lock serializes cache writes for one database and its signature, so a row @@ -433,7 +438,7 @@ func (h *ArchHandler) storedSignature(ctx context.Context, d archDatabase, dbDig if err != nil { return metadataResult{}, false } - return metadataResult{body: body, contentType: contentType, contentEncoding: entry.ContentEncoding.String}, true + return metadataResult{body: body, contentType: contentType, contentEncoding: entry.ContentEncoding.String}.describedBy(entry), true } return metadataResult{}, false } @@ -478,7 +483,7 @@ func archDigest(body []byte) string { return "sha256:" + hex.EncodeToString(digest[:]) } -func (h *ArchHandler) writeMetadata(w http.ResponseWriter, r *http.Request, cacheKey string, res metadataResult) { +func (h *ArchHandler) writeMetadata(w http.ResponseWriter, r *http.Request, res metadataResult) { if res.err != nil { if errors.Is(res.err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) @@ -491,7 +496,7 @@ func (h *ArchHandler) writeMetadata(w http.ResponseWriter, r *http.Request, cach http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) return } - h.proxy.writeMetadataCachedResponseWithEncoding(w, r, archEcosystem, cacheKey, res.body, res.contentType, res.contentEncoding) + writeMetadataResult(w, r, res) } // metadataCacheKey hashes repository, upstream URL and path, as APKHandler does. diff --git a/internal/handler/arch_test.go b/internal/handler/arch_test.go index 3affad5..02c87fa 100644 --- a/internal/handler/arch_test.go +++ b/internal/handler/arch_test.go @@ -1,6 +1,7 @@ package handler import ( + "database/sql" "fmt" "net/http" "net/http/httptest" @@ -648,3 +649,55 @@ func serveArchRequest(h *ArchHandler, method, target string, header http.Header) h.Routes().ServeHTTP(w, req) return w } + +// TestArchHandler_DatabaseValidatorsDescribeBody checks a fetched database +// goes out with its own Last-Modified, not that of the cache row, which +// another instance may have rewritten while the signature was fetched. +func TestArchHandler_DatabaseValidatorsDescribeBody(t *testing.T) { + const dbPath, sigPath = "/core/os/x86_64/core.db", "/core/os/x86_64/core.db.sig" + fetched := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + rewritten := fetched.Add(time.Hour) + upstream := newArchUpstream(t, map[string][]byte{dbPath: []byte("core-db-v1"), sigPath: []byte("sig")}) + upstream.modified[dbPath] = fetched + proxy := newArchTestProxy(t, upstream) + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + key := h.metadataCacheKey("archlinux", upstream.URL, "core/os/x86_64/core.db") + upstream.served = func(path string) { + if path != sigPath { + return + } + entry, err := proxy.DB.GetMetadataCache(archEcosystem, key) + if err != nil || entry == nil { + t.Errorf("database row while fetching the signature: %v", err) + return + } + entry.LastModified = sql.NullTime{Time: rewritten, Valid: true} + if err := proxy.DB.UpsertMetadataCache(entry); err != nil { + t.Error(err) + } + } + + w := serveArchRequest(h, http.MethodGet, "/archlinux"+dbPath, nil) + assertArchBody(t, w, "database", []byte("core-db-v1")) + if got := w.Header().Get("Last-Modified"); got != fetched.Format(http.TimeFormat) { + t.Errorf("Last-Modified = %q, want the fetched database's %q", got, fetched.Format(http.TimeFormat)) + } +} + +// TestArchHandler_StaleDatabaseMarkedWithoutTTL checks the stale fallback +// carries the Warning header with metadata_ttl 0. +func TestArchHandler_StaleDatabaseMarkedWithoutTTL(t *testing.T) { + db := []byte("core-db-bytes") + upstream := newArchUpstream(t, map[string][]byte{"/core/os/x86_64/core.db": db}) + proxy := newArchTestProxy(t, upstream) + proxy.MetadataTTL = 0 + h := NewArchHandler(proxy, "http://proxy.example", map[string]string{"archlinux": upstream.URL}) + + assertArchBody(t, serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil), "seed", db) + upstream.available.Store(false) + w := serveArchRequest(h, http.MethodGet, "/archlinux/core/os/x86_64/core.db", nil) + assertArchBody(t, w, "stale", db) + if got := w.Header().Get("Warning"); got != staleWarning { + t.Errorf("Warning = %q, want %q", got, staleWarning) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d8c405f..55aa8c9 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2248,18 +2248,6 @@ type cachedMeta struct { stale bool } -// lookupCachedMeta retrieves cache validators for a metadata entry. -func (p *Proxy) lookupCachedMeta(ecosystem, cacheKey string) cachedMeta { - if p.DB == nil { - return cachedMeta{} - } - entry, err := p.DB.GetMetadataCache(ecosystem, cacheKey) - if err != nil || entry == nil { - return cachedMeta{} - } - return p.cachedMetaFor(entry) -} - // cachedMetaFor returns the cache validators and freshness state of entry. func (p *Proxy) cachedMetaFor(entry *database.MetadataCacheEntry) cachedMeta { var cm cachedMeta @@ -2346,16 +2334,6 @@ func (p *Proxy) writeMetadataFetchError(w http.ResponseWriter, err error) { http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) } -// writeMetadataCachedResponseWithEncoding writes a metadata document with the -// validators of its current cache row. Prefer writeMetadataResult, which -// takes them from the lookup the body came from. contentEncoding must -// describe the body being written; it is passed in rather than re-read from -// the cache row, which is missing or stale when the metadata cache write -// failed and would otherwise mislabel the bytes. -func (p *Proxy) writeMetadataCachedResponseWithEncoding(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType, contentEncoding string) { - writeMetadataBody(w, r, p.lookupCachedMeta(ecosystem, cacheKey), body, contentType, contentEncoding) -} - // writeMetadataResult writes the document res holds with its own validators // and staleness, rather than the cache row's, which may describe other bytes // when the cache write failed or another instance replaced it. From 4c3c925cd1f6e32a5b7121ff54382962c7f40795 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 9 Nov 2026 17:44:19 +0100 Subject: [PATCH 185/198] Key Debian and Conan archives by their whole path A Debian package was cached under its repository and file name, but fetched from its full path. Docker's apt repository has a pool under each suite with the same file names, so a bookworm client could get the bullseye build that was fetched first, and fail its hash check. Packages are now cached under their path. Conan joined revision, package reference, package revision and file name with underscores, which revisions can contain, so a recipe file and a package file could share a cache entry. The parts are now joined with slashes. Entries cached under the old names are still served, but only for the URL they were fetched from, so warm caches keep working. --- internal/handler/conan.go | 32 ++++++++-- internal/handler/conan_test.go | 84 +++++++++++++++++++++++-- internal/handler/debian.go | 26 ++++++-- internal/handler/debian_test.go | 84 +++++++++++++++++++++++-- internal/handler/handler.go | 20 ++++++ internal/server/retention_conan_test.go | 8 +-- internal/server/retention_deb_test.go | 4 +- 7 files changed, 230 insertions(+), 28 deletions(-) diff --git a/internal/handler/conan.go b/internal/handler/conan.go index dddd79f..9a07f24 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -89,7 +89,13 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request) "name", name, "version", version, "user", user, "channel", channel, "filename", filename) upstreamURL := h.upstreamURL + r.URL.Path - storageFilename := conanStorageFilename(user, channel, revision+"_"+filename) + storageFilename, err := h.proxy.cacheFilenameKeepingLegacy("conan", name, version, + conanStorageFilename(user, channel, revision, filename), + conanLegacyStorageFilename(user, channel, revision+"_"+filename), upstreamURL) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch file") + return + } result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL) if err != nil { @@ -121,7 +127,13 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) "name", name, "version", version, "pkgref", pkgref, "filename", filename) upstreamURL := h.upstreamURL + r.URL.Path - storageFilename := conanStorageFilename(user, channel, revision+"_"+pkgref+"_"+pkgrev+"_"+filename) + storageFilename, err := h.proxy.cacheFilenameKeepingLegacy("conan", name, version, + conanStorageFilename(user, channel, revision, "packages", pkgref, pkgrev, filename), + conanLegacyStorageFilename(user, channel, revision+"_"+pkgref+"_"+pkgrev+"_"+filename), upstreamURL) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch file") + return + } result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL) if err != nil { @@ -133,9 +145,19 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) } // conanStorageFilename names a cached archive. Archives are stored under -// pkg:conan/{name}@{version}, so the file name carries the revisions and, when -// set, user and channel, to keep every recipe and binary apart. -func conanStorageFilename(user, channel, file string) string { +// pkg:conan/{name}@{version}, so the file name carries user, channel and the +// revisions, to keep every recipe and binary apart. The parts are path +// segments, which can't contain a slash, so joining them with one keeps +// different references apart. +func conanStorageFilename(user, channel string, parts ...string) string { + return strings.Join(append([]string{user, channel}, parts...), "/") +} + +// conanLegacyStorageFilename is the name archives were cached under before +// their parts were joined with slashes. Underscores, which revisions, user +// and channel can contain, joined them, so it is only read for the URL an +// entry was fetched from. +func conanLegacyStorageFilename(user, channel, file string) string { if user == "_" && channel == "_" { return file } diff --git a/internal/handler/conan_test.go b/internal/handler/conan_test.go index cf8038e..5be99fe 100644 --- a/internal/handler/conan_test.go +++ b/internal/handler/conan_test.go @@ -1,6 +1,7 @@ package handler import ( + "context" "io" "log/slog" "net/http" @@ -9,6 +10,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/git-pkgs/registries/fetch" ) const testProxyURL = "http://localhost:8080" @@ -302,13 +305,13 @@ func TestConanV2FileRoutesCacheArchives(t *testing.T) { filename string }{ {"recipe", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", - "rrev1_conan_sources.tgz"}, + "_/_/rrev1/conan_sources.tgz"}, {"package", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tgz", - "rrev1_pkgid1_prev1_conan_package.tgz"}, + "_/_/rrev1/packages/pkgid1/prev1/conan_package.tgz"}, {"recipe xz", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_export.txz", - "rrev1_conan_export.txz"}, + "_/_/rrev1/conan_export.txz"}, {"package zstd", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tzst", - "rrev1_pkgid1_prev1_conan_package.tzst"}, + "_/_/rrev1/packages/pkgid1/prev1/conan_package.tzst"}, } for _, tt := range tests { @@ -362,9 +365,9 @@ func TestConanUserChannelKeptApart(t *testing.T) { for _, ref := range []struct{ path, body, filename string }{ {"/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", - "center sources", "rrev1_conan_sources.tgz"}, + "center sources", "_/_/rrev1/conan_sources.tgz"}, {"/v2/conans/zlib/1.3.1/acme/stable/revisions/rrev1/files/conan_sources.tgz", - "acme sources", "acme@stable_rrev1_conan_sources.tgz"}, + "acme sources", "acme/stable/rrev1/conan_sources.tgz"}, } { fetcher.artifact = artifactBody(ref.body) fetcher.fetchCalled = false @@ -651,3 +654,72 @@ func TestConanMetadataPreservesStatusCodes(t *testing.T) { upstream.Close() } } + +const ( + conanRecipeLookalike = "/v2/conans/zlib/1.3/_/_/revisions/R_P_Q/files/conan_package.tgz" + conanPackageFile = "/v2/conans/zlib/1.3/_/_/revisions/R/packages/P/revisions/Q/files/conan_package.tgz" +) + +// conanPathEchoServer answers every file with its own path, so each URL has +// different bytes. +func conanPathEchoServer(t *testing.T, available *atomic.Bool) *httptest.Server { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte(r.URL.Path)) + })) + t.Cleanup(upstream.Close) + return upstream +} + +func serveConanRequest(h *ConanHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} + +// A recipe revision with underscores must not share a cache entry with a +// package whose revision, reference and package revision spell it. +func TestConanFileKeysKeepPathFieldsApart(t *testing.T) { + var available atomic.Bool + available.Store(true) + upstream := conanPathEchoServer(t, &available) + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) + + for _, target := range []string{conanRecipeLookalike, conanPackageFile} { + if w := serveConanRequest(h, target); w.Code != http.StatusOK || w.Body.String() != target { + t.Errorf("GET %s = %d %q, want its own file", target, w.Code, w.Body.String()) + } + } +} + +// A file cached under the name used before path fields were kept apart is +// still served for the URL it was fetched from, and only for that URL. +func TestConanLegacyFileEntryServedForItsURL(t *testing.T) { + var available atomic.Bool + available.Store(true) + upstream := conanPathEchoServer(t, &available) + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) + + legacy, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "conan", "zlib", "1.3", + "R_P_Q_conan_package.tgz", upstream.URL+conanRecipeLookalike) + if err != nil { + t.Fatalf("seed legacy entry: %v", err) + } + _ = legacy.Reader.Close() + + if w := serveConanRequest(h, conanPackageFile); w.Body.String() != conanPackageFile { + t.Errorf("package file = %d %q, want its own file, not the legacy entry", w.Code, w.Body.String()) + } + available.Store(false) + if w := serveConanRequest(h, conanRecipeLookalike); w.Code != http.StatusOK || w.Body.String() != conanRecipeLookalike { + t.Errorf("recipe file with the upstream down = %d %q, want the legacy entry", w.Code, w.Body.String()) + } +} diff --git a/internal/handler/debian.go b/internal/handler/debian.go index 565f181..dc6d6dc 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -168,7 +168,12 @@ func (h *DebianHandler) handlePackageDownload( filename := path[strings.LastIndex(path, "/")+1:] downloadURL := fmt.Sprintf("%s/%s", upstreamURL, path) - cacheFilename := h.artifactCacheFilename(repository, filename) + cacheFilename, err := h.proxy.cacheFilenameKeepingLegacy("deb", name, version, + h.artifactCacheFilename(repository, path), h.legacyArtifactCacheFilename(repository, filename), downloadURL) + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch package") + return + } h.proxy.Logger.Info("debian package download", "repository", repository, @@ -196,10 +201,21 @@ func (h *DebianHandler) handleMetadata( h.proxy.ProxyCached(w, r, fmt.Sprintf("%s/%s", upstreamURL, path), "debian", cacheKey, "*/*") } -// artifactCacheFilename scopes a package by repository, since the same -// filename can hold different bytes in different archives. The main archive -// keeps the unscoped filename its existing cache entries were stored under. -func (h *DebianHandler) artifactCacheFilename(repository, filename string) string { +// artifactCacheFilename names a package in the cache by its repository and +// its path in it, since the same file name can hold different bytes in +// different archives, and in different directories of one archive: Docker's +// repository has a pool under each suite. +func (h *DebianHandler) artifactCacheFilename(repository, path string) string { + if repository == "" { + return path + } + return repository + "/" + path +} + +// legacyArtifactCacheFilename is the name packages were cached under before +// they were keyed by path: the file name, scoped by repository for a named +// one. It is still read for the URL an entry was fetched from. +func (h *DebianHandler) legacyArtifactCacheFilename(repository, filename string) string { if repository == "" { return filename } diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index 54a72b6..f9c6284 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -1,10 +1,12 @@ package handler import ( + "context" "fmt" "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" @@ -37,10 +39,11 @@ func TestDebianHandler_Routes(t *testing.T) { } // TestDebianHandler_LegacyCacheKeysUnchanged pins the artifact cache identity -// the main archive used before named repositories existed. Deployments carry -// warm caches across upgrades, so a changed key here silently discards them. -// Metadata is cheap to refetch; its keys are covered by -// TestMetadataCacheKeysUnambiguous. +// the main archive used before named repositories existed. Packages are now +// keyed by path, but deployments carry warm caches across upgrades, so +// entries under this name are still read for the URL they were fetched from +// (TestDebianHandler_LegacyPackageEntryServedForItsPath). Metadata is cheap +// to refetch; its keys are covered by TestMetadataCacheKeysUnambiguous. func TestDebianHandler_LegacyCacheKeysUnchanged(t *testing.T) { const ( poolPath = "pool/main/h/hello/hello_2.10-3_amd64.deb" @@ -53,12 +56,12 @@ func TestDebianHandler_LegacyCacheKeysUnchanged(t *testing.T) { // Artifact cache: the bare filename, with no repository prefix. filename := poolPath[strings.LastIndex(poolPath, "/")+1:] - if got := h.artifactCacheFilename("", filename); got != "hello_2.10-3_amd64.deb" { + if got := h.legacyArtifactCacheFilename("", filename); got != "hello_2.10-3_amd64.deb" { t.Errorf("legacy artifact cache filename = %q, want %q", got, "hello_2.10-3_amd64.deb") } // A named repository must not reuse either identity. - if got := h.artifactCacheFilename("security", filename); got == "hello_2.10-3_amd64.deb" { + if got := h.legacyArtifactCacheFilename("security", filename); got == "hello_2.10-3_amd64.deb" { t.Error("named repository reused the legacy artifact cache filename") } if h.metadataCacheKeyFor("security", "https://security.test", distPath) == h.metadataCacheKeyFor("", h.upstreamURL, distPath) { @@ -324,3 +327,72 @@ func TestDebRetentionKey(t *testing.T) { t.Error("debRetentionKey(nil) accepted") } } + +// debianSuiteArchive serves a different build of one package file name per +// suite, as Docker's repository nests a pool under each suite. +func debianSuiteArchive(t *testing.T, available *atomic.Bool) *httptest.Server { + t.Helper() + archive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + suite := strings.Split(strings.TrimPrefix(r.URL.Path, "/dists/"), "/")[0] + _, _ = fmt.Fprint(w, suite+" build") + })) + t.Cleanup(archive.Close) + return archive +} + +const ( + debianBullseyePackage = "/docker/dists/bullseye/pool/stable/amd64/containerd.io_1.7.25-1_amd64.deb" + debianBookwormPackage = "/docker/dists/bookworm/pool/stable/amd64/containerd.io_1.7.25-1_amd64.deb" +) + +// TestDebianHandler_PackageCacheScopedPerPath checks that one file name in +// two directories of a repository does not resolve to one cached artifact. +func TestDebianHandler_PackageCacheScopedPerPath(t *testing.T) { + var available atomic.Bool + available.Store(true) + archive := debianSuiteArchive(t, &available) + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(archive.Client()), fetch.WithMaxRetries(0)) + h := NewDebianHandler(proxy, "http://proxy.example", "https://archive.test", map[string]string{"docker": archive.URL}) + + for _, tc := range []struct{ path, want string }{ + {debianBullseyePackage, "bullseye build"}, + {debianBookwormPackage, "bookworm build"}, + } { + if w := serveDebianRequest(h, tc.path); w.Code != http.StatusOK || w.Body.String() != tc.want { + t.Errorf("GET %s = %d %q, want %q", tc.path, w.Code, w.Body.String(), tc.want) + } + } +} + +// TestDebianHandler_LegacyPackageEntryServedForItsPath checks that a package +// cached under the file name alone, as before packages were keyed by path, is +// still served for the path it was fetched from, and only for that path. +func TestDebianHandler_LegacyPackageEntryServedForItsPath(t *testing.T) { + var available atomic.Bool + available.Store(true) + archive := debianSuiteArchive(t, &available) + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(archive.Client()), fetch.WithMaxRetries(0)) + h := NewDebianHandler(proxy, "http://proxy.example", "https://archive.test", map[string]string{"docker": archive.URL}) + + // The entry an earlier version cached for the bullseye path. + legacy, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "deb", "containerd.io", "1.7.25-1", + "docker/containerd.io_1.7.25-1_amd64.deb", archive.URL+strings.TrimPrefix(debianBullseyePackage, "/docker")) + if err != nil { + t.Fatalf("seed legacy entry: %v", err) + } + _ = legacy.Reader.Close() + + if w := serveDebianRequest(h, debianBookwormPackage); w.Body.String() != "bookworm build" { + t.Errorf("bookworm = %d %q, want its own build, not the legacy bullseye entry", w.Code, w.Body.String()) + } + available.Store(false) + if w := serveDebianRequest(h, debianBullseyePackage); w.Code != http.StatusOK || w.Body.String() != "bullseye build" { + t.Errorf("bullseye with the archive down = %d %q, want the legacy entry", w.Code, w.Body.String()) + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9612e51..62a49e8 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2396,6 +2396,26 @@ func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", "") } +// cacheFilenameKeepingLegacy returns the filename to cache an artifact +// fetched from downloadURL under: legacy, an earlier cache filename that more +// than one URL could share, while an artifact recorded for downloadURL is +// cached under it, and filename otherwise. It keeps entries cached before a +// key change in use, without serving one to a URL it wasn't fetched from. +func (p *Proxy) cacheFilenameKeepingLegacy(ecosystem, name, version, filename, legacy, downloadURL string) (string, error) { + _, versionPURL, err := packagePURLStrings(ecosystem, name, version) + if err != nil { + return "", err + } + record, err := p.DB.GetArtifact(versionPURL, legacy) + if err != nil { + return "", fmt.Errorf("checking artifact cache: %w", err) + } + if record != nil && record.UpstreamURL == downloadURL { + return legacy, nil + } + return filename, nil +} + // GetOrFetchArtifactFromURLWithHeaders retrieves an artifact from cache or fetches from a URL // with additional request-specific HTTP headers. func (p *Proxy) GetOrFetchArtifactFromURLWithHeaders(ctx context.Context, ecosystem, name, version, filename, downloadURL string, headers http.Header) (*CacheResult, error) { diff --git a/internal/server/retention_conan_test.go b/internal/server/retention_conan_test.go index 17607bc..e5023be 100644 --- a/internal/server/retention_conan_test.go +++ b/internal/server/retention_conan_test.go @@ -29,15 +29,15 @@ type conanFile struct { var ( conanZlibSources = conanFile{ "/v2/conans/zlib/1.3.1/_/_/revisions/" + conanRecipeRevision + "/files/conan_sources.tgz", - conanRecipeRevision + "_conan_sources.tgz", + "_/_/" + conanRecipeRevision + "/conan_sources.tgz", } conanZlibPackage = conanFile{ "/v2/conans/zlib/1.3.1/_/_/revisions/" + conanRecipeRevision + "/packages/0123abc/revisions/4567def/files/conan_package.tgz", - conanRecipeRevision + "_0123abc_4567def_conan_package.tgz", + "_/_/" + conanRecipeRevision + "/packages/0123abc/4567def/conan_package.tgz", } conanZlibAcme = conanFile{ "/v2/conans/zlib/1.2.13/acme/stable/revisions/" + conanRecipeRevision + "/files/conan_export.tgz", - "acme@stable_" + conanRecipeRevision + "_conan_export.tgz", + "acme/stable/" + conanRecipeRevision + "/conan_export.tgz", } conanOpenSSLSources = conanFile{ "/v2/conans/openssl/3.3.2/_/_/revisions/" + conanRecipeRevision + "/files/conan_sources.tgz", @@ -209,7 +209,7 @@ func TestConanRetentionPackageRule(t *testing.T) { t.Errorf("%s evicted despite the never-evict rule for zlib", f.filename) } } - if conanArtifact(t, db, "pkg:conan/openssl", conanRecipeRevision+"_conan_sources.tgz").StoragePath.Valid { + if conanArtifact(t, db, "pkg:conan/openssl", "_/_/"+conanRecipeRevision+"/conan_sources.tgz").StoragePath.Valid { t.Error("openssl kept past the conan rule") } } diff --git a/internal/server/retention_deb_test.go b/internal/server/retention_deb_test.go index 8081224..3f6f61d 100644 --- a/internal/server/retention_deb_test.go +++ b/internal/server/retention_deb_test.go @@ -20,8 +20,8 @@ import ( const ( debCurlPath = "/pool/main/c/curl/curl_8.14.1-2_amd64.deb" debBusyboxPath = "/pool/main/b/busybox/busybox_1.37.0-6_amd64.deb" - debCurlFile = "curl_8.14.1-2_amd64.deb" - debBusyboxFile = "busybox_1.37.0-6_amd64.deb" + debCurlFile = "pool/main/c/curl/curl_8.14.1-2_amd64.deb" + debBusyboxFile = "pool/main/b/busybox/busybox_1.37.0-6_amd64.deb" ) // debRetentionUpstream serves package files the way a Debian archive does, From 8cf92d1ec1fb4e78c988e226243d48f8fb1c6268 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 9 Nov 2026 17:49:16 +0100 Subject: [PATCH 186/198] Cap the metadata fetches that outlive their caller A metadata fetch always ran on a context detached from its first caller, so a client could request many uncached documents under different names and hang up at once, leaving one upstream fetch per request running to the HTTP client's timeout, each buffering up to metadata_max_size. Artifact fetches only detach while they hold one of maxDetachedFetches slots. Metadata fetches now do the same with their own 64 slots; beyond that, a fetch ends with its first caller. --- internal/handler/handler.go | 64 +++++++++++++++++----- internal/handler/metadata_coalesce_test.go | 36 ++++++++++++ internal/handler/profile_test.go | 3 +- 3 files changed, 87 insertions(+), 16 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9612e51..d78168c 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -288,6 +288,11 @@ type Proxy struct { metaMu sync.Mutex inFlightMeta map[string]*inflightMetadata + // metaDetachSlots caps the metadata fetches that keep running after + // their caller went away (see coalesceMetadata). Guarded by metaMu + // while it is created. + metaDetachSlots chan struct{} + // rewrites caches metadata documents after their handler rewrites them. // Nil leaves every request to rewrite its own copy. rewrites *rewriteCache @@ -1177,19 +1182,7 @@ const maxDetachedFetches = 64 func (p *Proxy) acquireDetachSlot() bool { shared := p.sharedProxy() - shared.fetchMu.Lock() - if shared.detachSlots == nil { - shared.detachSlots = make(chan struct{}, maxDetachedFetches) - } - slots := shared.detachSlots - shared.fetchMu.Unlock() - - select { - case slots <- struct{}{}: - return true - default: - return false - } + return trySlot(&shared.fetchMu, &shared.detachSlots, maxDetachedFetches) } func (p *Proxy) releaseDetachSlot() { @@ -1197,6 +1190,40 @@ func (p *Proxy) releaseDetachSlot() { <-shared.detachSlots } +// maxDetachedMetadataFetches caps the metadata fetches that may outlive their +// caller, for the same reason as maxDetachedFetches: a client could request +// any number of uncached documents and hang up, leaving each fetch to run to +// the HTTP client's timeout. Fetches beyond the cap end with their caller. +const maxDetachedMetadataFetches = 64 + +func (p *Proxy) acquireMetadataDetachSlot() bool { + shared := p.sharedProxy() + return trySlot(&shared.metaMu, &shared.metaDetachSlots, maxDetachedMetadataFetches) +} + +func (p *Proxy) releaseMetadataDetachSlot() { + shared := p.sharedProxy() + <-shared.metaDetachSlots +} + +// trySlot takes one of size slots without waiting, creating the slots under +// mu on first use. It reports whether a slot was free. +func trySlot(mu *sync.Mutex, slots *chan struct{}, size int) bool { + mu.Lock() + if *slots == nil { + *slots = make(chan struct{}, size) + } + ch := *slots + mu.Unlock() + + select { + case ch <- struct{}{}: + return true + default: + return false + } +} + // fetchedArtifact is what a shared fetch hands its callers: metadata and a // storage path, neither holding reader state. type fetchedArtifact struct { @@ -1900,7 +1927,9 @@ func metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncodin // metadata fetch has no scan or mirror that depends on the caller aborting it, // and CI jobs asking for the same Composer or npm metadata would otherwise all // fail when the first of them disconnects. It stays bounded by the HTTP -// client's timeout. Waiters still leave when their own context ends. +// client's timeout, and at most maxDetachedMetadataFetches run detached at +// once; beyond that, fetch ends with the first caller. Waiters still leave +// when their own context ends. func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(context.Context) metadataResult) (metadataResult, bool) { shared := p.sharedProxy() shared.metaMu.Lock() @@ -1934,7 +1963,12 @@ func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(con shared.metaMu.Unlock() close(f.done) }() - f.res = fetch(context.WithoutCancel(ctx)) + fetchCtx := ctx + if p.acquireMetadataDetachSlot() { + defer p.releaseMetadataDetachSlot() + fetchCtx = context.WithoutCancel(ctx) + } + f.res = fetch(fetchCtx) return f.res, false } diff --git a/internal/handler/metadata_coalesce_test.go b/internal/handler/metadata_coalesce_test.go index af8e2eb..26e0761 100644 --- a/internal/handler/metadata_coalesce_test.go +++ b/internal/handler/metadata_coalesce_test.go @@ -247,3 +247,39 @@ func TestCoalescedMetadataMiss_ServesRowCommittedSinceLookup(t *testing.T) { t.Errorf("upstream requests = %d, want 1 (the priming fetch only)", got) } } + +// TestFetchOrCacheMetadata_DetachCapFallsBackToClientContext checks the cap on +// metadata fetches that outlive their caller: with every slot taken, a caller +// that leaves ends its fetch, so clients that request documents and hang up +// can't leave any number of fetches running. +func TestFetchOrCacheMetadata_DetachCapFallsBackToClientContext(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + defer close(u.release) + p := metadataTestProxy(t, u) + + for range maxDetachedMetadataFetches { + if !p.acquireMetadataDetachSlot() { + t.Fatal("could not take a free metadata detach slot") + } + } + ctx, cancel := context.WithCancel(context.Background()) + call := startMetadataCalls(p, ctx, 1, u.URL, contentTypeJSON) + <-u.entered + cancel() + + select { + case c := <-call[0]: + if !errors.Is(c.err, context.Canceled) { + t.Errorf("err = %v, want context.Canceled once the slots are used up", c.err) + } + case <-time.After(5 * time.Second): + t.Fatal("the fetch kept running after its caller left, with no slot free") + } + + for range maxDetachedMetadataFetches { + p.releaseMetadataDetachSlot() + } + if !p.acquireMetadataDetachSlot() { + t.Error("slots not available again after release") + } +} diff --git a/internal/handler/profile_test.go b/internal/handler/profile_test.go index 5caf7b8..fa18ddf 100644 --- a/internal/handler/profile_test.go +++ b/internal/handler/profile_test.go @@ -46,6 +46,7 @@ func TestWithCooldownCopiesProxyFields(t *testing.T) { p.inFlight = map[string]*inflightFetch{"artifact": {}} p.detachSlots = make(chan struct{}, 1) p.inFlightMeta = map[string]*inflightMetadata{"metadata": {}} + p.metaDetachSlots = make(chan struct{}, 1) p.publishIndexes.entries = map[string]publishIndexEntry{"document": {}} p.cooldownWatch.active = map[*cooldownRecorder]struct{}{{}: {}} @@ -61,7 +62,7 @@ func TestWithCooldownCopiesProxyFields(t *testing.T) { switch name { case "Cooldown", "ShortestCooldown", "shared": continue - case "fetchMu", "inFlight", "detachSlots", "metaMu", "inFlightMeta", "publishIndexes", "cooldownWatch": + case "fetchMu", "inFlight", "detachSlots", "metaMu", "inFlightMeta", "metaDetachSlots", "publishIndexes", "cooldownWatch": if !after.IsZero() { t.Errorf("view field %s is not empty", name) } From 05fd1c8b56f3c92f15d92ae4151250bb37f84146 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 9 Nov 2026 22:42:18 +0100 Subject: [PATCH 187/198] Store blobs without a 1 MiB buffer per write Store copied into the gocloud writer with io.Copy, which uses the writer's ReadFrom. That allocates a 1 MiB buffer on every call, however small the object. A Go version list under cooldown stores one .info file per version, so a module with a few hundred versions allocated several hundred MiB for a response of a few KiB. Store now copies with a pooled 32 KiB buffer. --- internal/storage/blob.go | 18 +++++++++++++++++- internal/storage/blob_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 81af78f..50f8c40 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -11,6 +11,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "time" "gocloud.dev/blob" @@ -32,6 +33,17 @@ const ( uploadConcurrency = 2 ) +// storeCopyBufferSize is the size of the buffers Store copies with, the size +// io.Copy uses itself. +const storeCopyBufferSize = 32 << 10 + +var storeCopyBuffers = sync.Pool{ //nolint:gochecknoglobals // shared across stores + New: func() any { + buffer := make([]byte, storeCopyBufferSize) + return &buffer + }, +} + // attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an // earlier version wrote. const attrsExt = ".attrs" @@ -231,7 +243,11 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri return 0, "", fmt.Errorf("creating writer: %w", err) } - size, err := io.Copy(w, tee) + // Hide the writer's ReadFrom, which allocates a 1 MiB buffer on every + // call, however small the object, so the copy uses a pooled buffer. + buffer := storeCopyBuffers.Get().(*[]byte) + defer storeCopyBuffers.Put(buffer) + size, err := io.CopyBuffer(struct{ io.Writer }{w}, tee, *buffer) if err != nil { abort() _ = w.Close() diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 1a764de..1bb9425 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -1,10 +1,12 @@ package storage import ( + "bytes" "context" "crypto/sha256" "encoding/hex" "errors" + "fmt" "io" "io/fs" "os" @@ -706,3 +708,29 @@ func TestBlobStoreFailedCopyLeavesNoObject(t *testing.T) { t.Error("failed Store left a partial object behind") } } + +// Storing a small object must not allocate a large copy buffer: a Go version +// list under cooldown stores one small .info file per version. +func TestBlobStoreSmallObjectAllocatesLittle(t *testing.T) { + b := createTestBlob(t) + ctx := context.Background() + content := []byte(`{"Version":"v1.0.0","Time":"2024-01-01T00:00:00Z"}`) + + const stores = 32 + const maxBytesPerStore = 128 << 10 + + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + for i := range stores { + path := fmt.Sprintf("_metadata/golang/example.com/mod/v1.0.%d.info", i) + if _, _, err := b.Store(ctx, path, bytes.NewReader(content)); err != nil { + t.Fatalf("Store failed: %v", err) + } + } + runtime.ReadMemStats(&after) + + perStore := (after.TotalAlloc - before.TotalAlloc) / stores + if perStore > maxBytesPerStore { + t.Errorf("Store allocated %d bytes per small object, want at most %d", perStore, maxBytesPerStore) + } +} From da6a5b40273c5e2915a74c1b962d092a748a7bad Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 9 Nov 2026 22:45:10 +0100 Subject: [PATCH 188/198] Read PyPI release times without building the whole document With cooldown on, every simple index request and every download reads the package's JSON API document for its upload times. It was decoded into a generic map, which for a package like botocore (4.4 MB) allocated about 17 MB per request, also when everything was cached. The times are now decoded into a struct that names only that field, so the rest of the document is skipped. --- internal/handler/pypi.go | 40 +++++-- internal/handler/pypi_release_times_test.go | 125 ++++++++++++++++++++ 2 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 internal/handler/pypi_release_times_test.go diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 7db972e..21e4374 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -345,23 +345,41 @@ func (h *PyPIHandler) fetchReleaseTimes(r *http.Request, name string) (map[strin return nil, err } - var metadata map[string]any - if err := json.Unmarshal(body, &metadata); err != nil { + return h.parseReleaseTimes(body) +} + +// pypiReleaseFiles is the part of a JSON API document the release times are +// read from. Decoding into it skips every other field, where a generic +// map would hold the whole document, several times its size, in memory. +type pypiReleaseFiles struct { + Releases map[string][]struct { + UploadTime string `json:"upload_time_iso_8601"` + } `json:"releases"` +} + +// parseReleaseTimes returns each release's newest upload time from a JSON +// API document, zero for a release without one. +func (h *PyPIHandler) parseReleaseTimes(body []byte) (map[string]time.Time, error) { + var metadata pypiReleaseFiles + // A value of another type than expected is skipped and the rest is + // still decoded, so a file entry without a usable time doesn't count. + var mismatch *json.UnmarshalTypeError + if err := json.Unmarshal(body, &metadata); err != nil && !errors.As(err, &mismatch) { return nil, fmt.Errorf("parsing pypi json metadata: %w", err) } - - releases, ok := metadata["releases"].(map[string]any) - if !ok { + if metadata.Releases == nil { return nil, errors.New("pypi json metadata has no releases") } - times := make(map[string]time.Time, len(releases)) - for version, files := range releases { - filesArr, ok := files.([]any) - if !ok { - continue + times := make(map[string]time.Time, len(metadata.Releases)) + for version, files := range metadata.Releases { + var newest time.Time + for _, file := range files { + if t, err := time.Parse(time.RFC3339, file.UploadTime); err == nil && t.After(newest) { + newest = t + } } - times[version] = h.newestUploadTime(filesArr) + times[version] = newest } return times, nil } diff --git a/internal/handler/pypi_release_times_test.go b/internal/handler/pypi_release_times_test.go new file mode 100644 index 0000000..37e8447 --- /dev/null +++ b/internal/handler/pypi_release_times_test.go @@ -0,0 +1,125 @@ +package handler + +import ( + "fmt" + "runtime" + "strings" + "testing" + "time" +) + +func TestPyPIParseReleaseTimes(t *testing.T) { + h := &PyPIHandler{} + body := `{ + "info": {"name": "pkg", "version": "3.0.0"}, + "releases": { + "1.0.0": [ + {"filename": "pkg-1.0.0.tar.gz", "upload_time_iso_8601": "2024-01-01T10:00:00Z"}, + {"filename": "pkg-1.0.0-py3-none-any.whl", "upload_time_iso_8601": "2024-01-02T10:00:00.123456Z"} + ], + "2.0.0": [], + "3.0.0": [ + 42, + {"upload_time_iso_8601": 5}, + {"upload_time_iso_8601": null}, + {"upload_time_iso_8601": "not a time"}, + {"upload_time": "2024-06-01T00:00:00"}, + {"upload_time_iso_8601": "2024-03-01T00:00:00Z"} + ], + "4.0.0": "not a list" + }, + "urls": [{"upload_time_iso_8601": "2030-01-01T00:00:00Z"}] + }` + + times, err := h.parseReleaseTimes([]byte(body)) + if err != nil { + t.Fatalf("parseReleaseTimes failed: %v", err) + } + want := map[string]time.Time{ + "1.0.0": time.Date(2024, 1, 2, 10, 0, 0, 123456000, time.UTC), + "2.0.0": {}, + "3.0.0": time.Date(2024, 3, 1, 0, 0, 0, 0, time.UTC), + "4.0.0": {}, + } + for version, wantTime := range want { + if got := times[version]; !got.Equal(wantTime) { + t.Errorf("times[%q] = %v, want %v", version, got, wantTime) + } + } + for version := range times { + if _, ok := want[version]; !ok { + t.Errorf("unexpected release %q", version) + } + } +} + +func TestPyPIParseReleaseTimesRejectsDocumentWithoutReleases(t *testing.T) { + h := &PyPIHandler{} + for _, body := range []string{ + `{"info": {"name": "pkg"}}`, + `{"releases": null}`, + `{"releases": []}`, + `[]`, + `{"releases": {"1.0.0": [`, + `not json`, + } { + if times, err := h.parseReleaseTimes([]byte(body)); err == nil { + t.Errorf("parseReleaseTimes(%s) = %v, want an error", body, times) + } + } + + times, err := h.parseReleaseTimes([]byte(`{"releases": {}}`)) + if err != nil || len(times) != 0 { + t.Errorf("parseReleaseTimes with no releases = %v, %v; want an empty result", times, err) + } +} + +// pypiJSONDocument builds a JSON API document shaped like PyPI's, with the +// file fields that make up most of a real one. +func pypiJSONDocument(releases, filesPerRelease int) []byte { + var b strings.Builder + b.WriteString(`{"info":{"name":"pkg","summary":"A package"},"releases":{`) + for r := range releases { + if r > 0 { + b.WriteByte(',') + } + fmt.Fprintf(&b, `"1.%d.0":[`, r) + for f := range filesPerRelease { + if f > 0 { + b.WriteByte(',') + } + fmt.Fprintf(&b, `{"comment_text":"","digests":{"blake2b_256":"%064x","md5":"%032x","sha256":"%064x"},`+ + `"downloads":-1,"filename":"pkg-1.%d.0-cp3%d-cp3%d-manylinux_2_17_x86_64.whl","has_sig":false,`+ + `"md5_digest":"%032x","packagetype":"bdist_wheel","python_version":"cp3%d","requires_python":">=3.8",`+ + `"size":123456,"upload_time":"2024-01-02T03:04:05","upload_time_iso_8601":"2024-01-02T03:04:05.123456Z",`+ + `"url":"https://files.pythonhosted.org/packages/ab/cd/%060x/pkg-1.%d.0-cp3%d-cp3%d-manylinux_2_17_x86_64.whl",`+ + `"yanked":false,"yanked_reason":null}`, r, f, r, r, f, f, f, f, r, r, f, f) + } + b.WriteByte(']') + } + b.WriteString(`},"urls":[],"vulnerabilities":[]}`) + return []byte(b.String()) +} + +// Reading the release times must not build the whole document in memory: it +// runs on every simple index request for a package under cooldown. +func TestPyPIParseReleaseTimesAllocatesLessThanDocument(t *testing.T) { + h := &PyPIHandler{} + body := pypiJSONDocument(300, 12) + + const runs = 5 + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + for range runs { + times, err := h.parseReleaseTimes(body) + if err != nil || len(times) != 300 { + t.Fatalf("parseReleaseTimes = %d releases, %v", len(times), err) + } + } + runtime.ReadMemStats(&after) + + perRun := (after.TotalAlloc - before.TotalAlloc) / runs + if perRun > uint64(len(body)) { + t.Errorf("parseReleaseTimes allocated %d bytes for a %d byte document", perRun, len(body)) + } +} From 89fc77ee4b2e4fc8d5887dacbff59ba7d577be07 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 9 Nov 2026 23:01:16 +0100 Subject: [PATCH 189/198] Read NuGet publish times without building the whole registration With cooldown on, the version list reads the package's registration index and all of its pages to find the versions in cooldown. Each document was decoded into a generic map, which for a package like AWSSDK.Core (6.8 MB of pages) allocated 26 MB per request, also when everything was cached. The registration is now decoded into a struct that names only the version and publish time, so the rest is skipped. The download check reads its leaf the same way. --- internal/handler/nuget_cooldown.go | 206 ++++++++++++---- internal/handler/nuget_times_test.go | 337 +++++++++++++++++++++++++++ 2 files changed, 499 insertions(+), 44 deletions(-) create mode 100644 internal/handler/nuget_times_test.go diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index b9345c5..d5a1571 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -36,36 +36,44 @@ func (h *NuGetHandler) nugetCooldownApplies(id string) bool { // Cache upstream documents, not filtered results, so policy changes and elapsed // time take effect even while metadata is fresh. Include the upstream in the key. func (h *NuGetHandler) nugetMetadata(ctx context.Context, path string) (map[string]any, error) { - target := h.upstreamURL + path - key := fmt.Sprintf("_cooldown/%x", sha256.Sum256([]byte(target))) var document map[string]any - validate := func(body []byte) error { - var err error - document, err = h.decodeNuGetMetadata(body) + err := h.readNuGetMetadata(ctx, path, func(body []byte) (err error) { + document, err = decodeNuGetDocument(body) return err - } - _, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "nuget", key, target, "", validate) + }) if err != nil { return nil, err } return document, nil } -func (h *NuGetHandler) decodeNuGetMetadata(body []byte) (map[string]any, error) { - // Normally the HTTP transport decodes gzip. Also support compressed cached - // bytes and clients with transparent decompression disabled, with the same - // metadata limit applied to the decompressed document. - if bytes.HasPrefix(body, []byte{0x1f, 0x8b}) { - reader, err := gzip.NewReader(bytes.NewReader(body)) - if err != nil { - return nil, err - } - defer func() { _ = reader.Close() }() - body, err = h.proxy.ReadMetadata(reader) - if err != nil { - return nil, err +// readNuGetMetadata fetches a document and passes it to decode, which keeps +// what its caller needs. decode can run more than once and must not keep body. +func (h *NuGetHandler) readNuGetMetadata(ctx context.Context, path string, decode func(body []byte) error) error { + target := h.upstreamURL + path + key := fmt.Sprintf("_cooldown/%x", sha256.Sum256([]byte(target))) + validate := func(body []byte) error { + // Normally the HTTP transport decodes gzip. Also support compressed cached + // bytes and clients with transparent decompression disabled, with the same + // metadata limit applied to the decompressed document. + if bytes.HasPrefix(body, []byte{0x1f, 0x8b}) { + reader, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return err + } + defer func() { _ = reader.Close() }() + body, err = h.proxy.ReadMetadata(reader) + if err != nil { + return err + } } + return decode(body) } + _, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "nuget", key, target, "", validate) + return err +} + +func decodeNuGetDocument(body []byte) (map[string]any, error) { var document map[string]any if err := json.Unmarshal(body, &document); err != nil { return nil, fmt.Errorf("parsing NuGet metadata: %w", err) @@ -76,13 +84,85 @@ func (h *NuGetHandler) decodeNuGetMetadata(body []byte) (map[string]any, error) return document, nil } +// nugetTimes holds the fields of a registration index, page or leaf that decide +// which versions are in cooldown. Decoding into it skips everything else, which +// is nearly all of a registration. +type nugetTimes struct { + ID string `json:"@id"` + Version string `json:"version"` + Published string `json:"published"` + CatalogEntry nugetCatalogTimes `json:"catalogEntry"` + Items []nugetTimes `json:"items"` +} + +// nugetCatalogTimes is the catalogEntry of a registration leaf. It stays unset +// unless the entry is an object: standalone leaves carry a link there. +type nugetCatalogTimes struct { + set bool + version string + published string +} + +func (e *nugetCatalogTimes) UnmarshalJSON(data []byte) error { + if !bytes.HasPrefix(data, []byte("{")) { + return nil + } + var fields struct { + Version string `json:"version"` + Published string `json:"published"` + } + var mismatch *json.UnmarshalTypeError + if err := json.Unmarshal(data, &fields); err != nil && !errors.As(err, &mismatch) { + return err + } + *e = nugetCatalogTimes{set: true, version: fields.Version, published: fields.Published} + return nil +} + +// release returns the version and publish time of a leaf. A leaf embedded in a +// page carries them in catalogEntry, a standalone leaf at its root. +func (t *nugetTimes) release() (version, published string) { + if t.CatalogEntry.set { + return t.CatalogEntry.version, t.CatalogEntry.published + } + return t.Version, t.Published +} + +func (h *NuGetHandler) nugetTimesMetadata(ctx context.Context, path string) (*nugetTimes, error) { + var document *nugetTimes + err := h.readNuGetMetadata(ctx, path, func(body []byte) (err error) { + document, err = decodeNuGetTimes(body) + return err + }) + if err != nil { + return nil, err + } + return document, nil +} + +// decodeNuGetTimes accepts the documents decodeNuGetDocument does. A field of +// an unexpected type is left empty, as it is when read from the generic document. +func decodeNuGetTimes(body []byte) (*nugetTimes, error) { + var document *nugetTimes + if err := json.Unmarshal(body, &document); err != nil { + var mismatch *json.UnmarshalTypeError + if !errors.As(err, &mismatch) || mismatch.Field == "" { + return nil, fmt.Errorf("parsing NuGet metadata: %w", err) + } + } + if document == nil { + return nil, fmt.Errorf("empty NuGet metadata") + } + return document, nil +} + // Prefer semver2, but a configured source may advertise only an older hive. // Retry only advertised aliases on 404; transport/validation errors must not // silently switch to a hive with less complete metadata. Keep requests on the // configured upstream, consistent with the service-index route rewriting. -func (h *NuGetHandler) nugetRegistrationMetadata(ctx context.Context, suffix string) (map[string]any, string, error) { +func (h *NuGetHandler) nugetRegistrationMetadata(ctx context.Context, suffix string) (*nugetTimes, string, error) { path := nugetRegistrationPath + suffix - document, err := h.nugetMetadata(ctx, path) + document, err := h.nugetTimesMetadata(ctx, path) if !errors.Is(err, ErrUpstreamNotFound) { return document, path, err } @@ -105,7 +185,7 @@ func (h *NuGetHandler) nugetRegistrationMetadata(ctx context.Context, suffix str } seen[prefix] = true path = prefix + suffix - document, err = h.nugetMetadata(ctx, path) + document, err = h.nugetTimesMetadata(ctx, path) if !errors.Is(err, ErrUpstreamNotFound) { return document, path, err } @@ -142,7 +222,7 @@ func (h *NuGetHandler) handleVersionList(w http.ResponseWriter, r *http.Request) if h.nugetCooldownApplies(id) { registration, registrationPath, err := h.nugetRegistrationMetadata(r.Context(), url.PathEscape(id)+"/index.json") if err == nil { - err = h.expandNuGetPages(r.Context(), registration, registrationPath) + err = h.expandNuGetPageTimes(r.Context(), registration, registrationPath) } if err != nil { h.nugetMetadataError(w, err) @@ -185,7 +265,8 @@ func (h *NuGetHandler) nugetDownloadAllowed(ctx context.Context, id, version str if err != nil { return false, err } - return h.nugetLeafAllowed(leaf, id), nil + _, published := leaf.release() + return h.nugetPublishedAllowed(id, published), nil } // A standalone leaf has published at its root; leaves embedded in pages carry @@ -200,6 +281,10 @@ func (h *NuGetHandler) nugetLeafAllowed(leaf map[string]any, id string) bool { id, _ = entry["id"].(string) } published, _ := entry["published"].(string) + return h.nugetPublishedAllowed(id, published) +} + +func (h *NuGetHandler) nugetPublishedAllowed(id, published string) bool { when, err := time.Parse(time.RFC3339, published) if err != nil { return true @@ -214,16 +299,12 @@ func nugetCatalogEntry(leaf map[string]any) map[string]any { return leaf } -func (h *NuGetHandler) collectNuGetBlockedVersions(document map[string]any, id string, blocked map[string]bool) { - entry := nugetCatalogEntry(document) - if version, ok := entry["version"].(string); ok && !h.nugetLeafAllowed(document, id) { +func (h *NuGetHandler) collectNuGetBlockedVersions(document *nugetTimes, id string, blocked map[string]bool) { + if version, published := document.release(); version != "" && !h.nugetPublishedAllowed(id, published) { blocked[nugetVersionKey(version)] = true } - items, _ := document["items"].([]any) - for _, item := range items { - if child, ok := item.(map[string]any); ok { - h.collectNuGetBlockedVersions(child, id, blocked) - } + for i := range document.Items { + h.collectNuGetBlockedVersions(&document.Items[i], id, blocked) } } @@ -272,11 +353,6 @@ func (h *NuGetHandler) expandNuGetPages(ctx context.Context, document map[string if !ok { return fmt.Errorf("missing registration pages") } - base, err := url.Parse(h.upstreamURL + path) - if err != nil { - return err - } - pagePrefix := strings.TrimSuffix(base.Path, "index.json") + "page/" for _, item := range items { page, ok := item.(map[string]any) if !ok { @@ -286,13 +362,11 @@ func (h *NuGetHandler) expandNuGetPages(ctx context.Context, document map[string continue } link, _ := page["@id"].(string) - target, err := base.Parse(link) - if err != nil || target.Scheme != base.Scheme || target.Host != base.Host || - !strings.HasPrefix(target.Path, pagePrefix) || containsPathTraversal(target.Path) || target.RawQuery != "" || target.Fragment != "" { - return fmt.Errorf("invalid registration page URL: %q", link) + pagePath, err := h.nugetRegistrationPagePath(path, link) + if err != nil { + return err } - upstream, _ := url.Parse(h.upstreamURL) - pageDocument, err := h.nugetMetadata(ctx, strings.TrimPrefix(target.Path, upstream.Path)) + pageDocument, err := h.nugetMetadata(ctx, pagePath) if err != nil { return err } @@ -305,6 +379,50 @@ func (h *NuGetHandler) expandNuGetPages(ctx context.Context, document map[string return nil } +// expandNuGetPageTimes is expandNuGetPages for a registration index decoded +// into nugetTimes. +func (h *NuGetHandler) expandNuGetPageTimes(ctx context.Context, index *nugetTimes, path string) error { + if index.Items == nil { + return fmt.Errorf("missing registration pages") + } + for i := range index.Items { + page := &index.Items[i] + if page.Items != nil { + continue + } + pagePath, err := h.nugetRegistrationPagePath(path, page.ID) + if err != nil { + return err + } + pageDocument, err := h.nugetTimesMetadata(ctx, pagePath) + if err != nil { + return err + } + if pageDocument.Items == nil { + return fmt.Errorf("missing registration leaves") + } + page.Items = pageDocument.Items + } + return nil +} + +// nugetRegistrationPagePath returns the upstream path of a page that the +// registration index at indexPath links to. +func (h *NuGetHandler) nugetRegistrationPagePath(indexPath, link string) (string, error) { + base, err := url.Parse(h.upstreamURL + indexPath) + if err != nil { + return "", err + } + pagePrefix := strings.TrimSuffix(base.Path, "index.json") + "page/" + target, err := base.Parse(link) + if err != nil || target.Scheme != base.Scheme || target.Host != base.Host || + !strings.HasPrefix(target.Path, pagePrefix) || containsPathTraversal(target.Path) || target.RawQuery != "" || target.Fragment != "" { + return "", fmt.Errorf("invalid registration page URL: %q", link) + } + upstream, _ := url.Parse(h.upstreamURL) + return strings.TrimPrefix(target.Path, upstream.Path), nil +} + func (h *NuGetHandler) filterNuGetRegistration(document map[string]any, id string) bool { items, ok := document["items"].([]any) if !ok { diff --git a/internal/handler/nuget_times_test.go b/internal/handler/nuget_times_test.go new file mode 100644 index 0000000..4ebe80d --- /dev/null +++ b/internal/handler/nuget_times_test.go @@ -0,0 +1,337 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "runtime" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +const nugetTestRegistration = nugetRegistrationPath + "testpkg/" + +// nugetDocumentUpstream serves the given documents by path and 404s the rest. +func nugetDocumentUpstream(t *testing.T, documents map[string]string) *httptest.Server { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := documents[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(upstream.Close) + return upstream +} + +func TestNuGetVersionListCooldown(t *testing.T) { + old := time.Now().Add(-30 * 24 * time.Hour).Format(time.RFC3339) + recent := time.Now().Add(-24 * time.Hour).Format(time.RFC3339) + entry := func(version, published string) string { + return `{"catalogEntry":{"id":"TestPkg","version":"` + version + `","published":"` + published + `"}}` + } + const versions = `{"versions":["1.0.0","2.0.0","3.0.0-beta"]}` + const pageOne = nugetTestRegistration + "page/1.0.0/1.0.0.json" + const pageTwo = nugetTestRegistration + "page/2.0.0/3.0.0-beta.json" + + for _, tt := range []struct { + name string + documents map[string]string + want string + }{ + { + name: "inlined pages", + documents: map[string]string{ + nugetTestRegistration + "index.json": `{"items":[{"@id":"` + pageOne + `","items":[` + entry("1.0.0", old) + `]},` + + `{"@id":"` + pageTwo + `","items":[` + entry("2.0.0", recent) + `,` + entry("3.0.0-beta", old) + `]}]}`, + }, + want: `{"versions":["1.0.0","3.0.0-beta"]}`, + }, + { + name: "linked pages", + documents: map[string]string{ + nugetTestRegistration + "index.json": `{"items":[{"@id":"` + pageOne + `"},{"@id":"` + pageTwo + `"}]}`, + pageOne: `{"items":[` + entry("1.0.0", recent) + `]}`, + pageTwo: `{"items":[` + entry("2.0.0", old) + `,` + entry("3.0.0-beta", recent) + `]}`, + }, + want: `{"versions":["2.0.0"]}`, + }, + { + name: "inlined and linked pages", + documents: map[string]string{ + nugetTestRegistration + "index.json": `{"items":[{"@id":"` + pageOne + `","items":[` + entry("1.0.0", recent) + `]},{"@id":"` + pageTwo + `"}]}`, + pageTwo: `{"items":[` + entry("2.0.0", recent) + `,` + entry("3.0.0-beta", old) + `]}`, + }, + want: `{"versions":["3.0.0-beta"]}`, + }, + { + name: "empty inlined page is not fetched", + documents: map[string]string{ + nugetTestRegistration + "index.json": `{"items":[{"@id":"` + pageOne + `","items":[]},{"@id":"` + pageTwo + `","items":[` + entry("2.0.0", recent) + `]}]}`, + }, + want: `{"versions":["1.0.0","3.0.0-beta"]}`, + }, + { + name: "version is compared without case and build metadata", + documents: map[string]string{ + nugetTestRegistration + "index.json": `{"items":[{"items":[` + entry("3.0.0-BETA+build.5", recent) + `]}]}`, + }, + want: `{"versions":["1.0.0","2.0.0"]}`, + }, + { + name: "leaf fields are used when catalogEntry is not an object", + documents: map[string]string{ + nugetTestRegistration + "index.json": `{"items":[{"items":[` + + `{"catalogEntry":"https://example.test/catalog/1.json","version":"1.0.0","published":"` + recent + `"},` + + `{"catalogEntry":null,"version":"2.0.0","published":"` + recent + `"},` + + `{"version":"3.0.0-beta","published":"` + old + `"}]}]}`, + }, + want: `{"versions":["3.0.0-beta"]}`, + }, + { + name: "leaf fields are ignored when catalogEntry is an object", + documents: map[string]string{ + nugetTestRegistration + "index.json": `{"items":[{"items":[` + + `{"catalogEntry":{},"version":"1.0.0","published":"` + recent + `"},` + + `{"catalogEntry":{"version":"2.0.0","published":"` + old + `"},"version":"2.0.0","published":"` + recent + `"}]}]}`, + }, + want: versions, + }, + { + name: "leaves without a usable publish time are allowed", + documents: map[string]string{ + nugetTestRegistration + "index.json": `{"items":[{"items":[` + + entry("1.0.0", "yesterday") + `,` + + `{"catalogEntry":{"version":"2.0.0","published":1700000000}},` + + `{"catalogEntry":{"version":"3.0.0-beta"}}]}]}`, + }, + want: versions, + }, + { + name: "values of the wrong type are skipped", + documents: map[string]string{ + nugetTestRegistration + "index.json": `{"count":"two","items":[{"@id":7,"lower":{},"items":[` + + `{"@id":[],"catalogEntry":{"version":1,"published":"` + recent + `","tags":"a"}},` + + `{"items":"none","catalogEntry":{"version":"2.0.0","published":"` + recent + `"}}]}]}`, + }, + want: `{"versions":["1.0.0","3.0.0-beta"]}`, + }, + } { + t.Run(tt.name, func(t *testing.T) { + tt.documents["/v3-flatcontainer/testpkg/index.json"] = versions + upstream := nugetDocumentUpstream(t, tt.documents) + p := nugetTestProxy() + p.Cooldown = &cooldown.Config{Default: "14d"} + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + w := nugetGet(t, h.Routes(), "/v3-flatcontainer/TestPkg/index.json", http.StatusOK) + if got := strings.TrimSpace(w.Body.String()); got != tt.want { + t.Errorf("version list = %s, want %s", got, tt.want) + } + }) + } +} + +func TestNuGetVersionListRejectsBrokenRegistration(t *testing.T) { + const page = nugetTestRegistration + "page/1.0.0/2.0.0.json" + linked := `{"items":[{"@id":"` + page + `"}]}` + for _, tt := range []struct { + name string + documents map[string]string + }{ + {"null index", map[string]string{nugetTestRegistration + "index.json": `null`}}, + {"index is not an object", map[string]string{nugetTestRegistration + "index.json": `[]`}}, + {"truncated index", map[string]string{nugetTestRegistration + "index.json": `{"items":[{"items":[`}}, + {"index without pages", map[string]string{nugetTestRegistration + "index.json": `{"count":0}`}}, + {"pages are not a list", map[string]string{nugetTestRegistration + "index.json": `{"items":"none"}`}}, + {"page is not an object", map[string]string{nugetTestRegistration + "index.json": `{"items":["` + page + `"]}`}}, + {"page without a link", map[string]string{nugetTestRegistration + "index.json": `{"items":[{"count":1}]}`}}, + {"missing page", map[string]string{nugetTestRegistration + "index.json": linked}}, + {"null page", map[string]string{nugetTestRegistration + "index.json": linked, page: `null`}}, + {"page is not an object", map[string]string{nugetTestRegistration + "index.json": linked, page: `[]`}}, + {"page without leaves", map[string]string{nugetTestRegistration + "index.json": linked, page: `{"count":2}`}}, + {"truncated page", map[string]string{nugetTestRegistration + "index.json": linked, page: `{"items":[{"catalogEntry":{"version":`}}, + } { + t.Run(tt.name, func(t *testing.T) { + tt.documents["/v3-flatcontainer/testpkg/index.json"] = `{"versions":["1.0.0","2.0.0"]}` + upstream := nugetDocumentUpstream(t, tt.documents) + p := nugetTestProxy() + p.Cooldown = &cooldown.Config{Default: "14d"} + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + status := http.StatusBadGateway + if tt.name == "missing page" { + status = http.StatusNotFound + } + nugetGet(t, h.Routes(), "/v3-flatcontainer/testpkg/index.json", status) + }) + } +} + +func TestNuGetVersionListRejectsUnsafePageLinks(t *testing.T) { + for _, link := range []string{ + "https://other.example/page.json", + nugetRegistrationPath + "other/page/1/2.json", + "page/../index.json", + "index.json", + "page/1.json?x=1", + "page/1.json#x", + } { + t.Run(link, func(t *testing.T) { + registrationRequests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v3-flatcontainer/testpkg/index.json" { + _, _ = io.WriteString(w, `{"versions":["1.0.0"]}`) + return + } + registrationRequests++ + _, _ = io.WriteString(w, `{"items":[{"@id":"`+link+`"}]}`) + })) + defer upstream.Close() + p := nugetTestProxy() + p.Cooldown = &cooldown.Config{Default: "14d"} + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + nugetGet(t, h.Routes(), "/v3-flatcontainer/testpkg/index.json", http.StatusBadGateway) + if registrationRequests != 1 { + t.Fatalf("unsafe page link was followed (%d registration requests)", registrationRequests) + } + }) + } +} + +func TestNuGetDownloadCooldownLeafShapes(t *testing.T) { + old := time.Now().Add(-30 * 24 * time.Hour).Format(time.RFC3339) + recent := time.Now().Add(-24 * time.Hour).Format(time.RFC3339) + const catalog = `"catalogEntry":"https://api.nuget.org/v3/catalog0/data/2024.01.02.03.04.05/testpkg.2.0.0.json"` + for _, tt := range []struct { + name, leaf string + want int + }{ + {"recent standalone leaf", `{` + catalog + `,"listed":true,"published":"` + recent + `"}`, http.StatusNotFound}, + {"old standalone leaf", `{` + catalog + `,"listed":true,"published":"` + old + `"}`, http.StatusOK}, + {"catalogEntry object wins", `{"catalogEntry":{"published":"` + recent + `"},"published":"` + old + `"}`, http.StatusNotFound}, + {"catalogEntry object without a time", `{"catalogEntry":{"version":"2.0.0"},"published":"` + recent + `"}`, http.StatusOK}, + {"time is not a string", `{"published":1700000000}`, http.StatusOK}, + {"leaf is not an object", `["` + recent + `"]`, http.StatusBadGateway}, + } { + t.Run(tt.name, func(t *testing.T) { + p, _, _, fetcher := setupTestProxy(t) + p.Cooldown = &cooldown.Config{Default: "14d"} + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package")), ContentType: "application/octet-stream"} + upstream := nugetDocumentUpstream(t, map[string]string{nugetTestRegistration + "2.0.0.json": tt.leaf}) + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + nugetGet(t, h.Routes(), "/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", tt.want) + if fetcher.fetchCalled != (tt.want == http.StatusOK) { + t.Errorf("artifact fetch called = %v", fetcher.fetchCalled) + } + }) + } +} + +// nugetRegistrationPage builds a registration page shaped like nuget.org's, +// with the catalog fields that make up most of a real one. +func nugetRegistrationPage(first, leaves int, published string) string { + var b strings.Builder + b.WriteString(`{"@id":"https://api.nuget.org/v3/registration5-gz-semver2/testpkg/page.json","@type":"catalog:CatalogPage",` + + `"commitId":"0b2f1c8e-5d5c-4b0f-9d0a-0d6f5c1f2a3b","commitTimeStamp":"2024-01-02T03:04:05.1234567+00:00","count":64,"items":[`) + for i := range leaves { + if i > 0 { + b.WriteByte(',') + } + version := fmt.Sprintf("1.%d.0", first+i) + fmt.Fprintf(&b, `{"@id":"https://api.nuget.org/v3/registration5-gz-semver2/testpkg/%[1]s.json","@type":"Package",`+ + `"commitId":"0b2f1c8e-5d5c-4b0f-9d0a-0d6f5c1f2a3b","commitTimeStamp":"2024-01-02T03:04:05.1234567+00:00",`+ + `"catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2024.01.02.03.04.05/testpkg.%[1]s.json","@type":"PackageDetails",`+ + `"authors":"Example Authors","dependencyGroups":[`+ + `{"@id":"https://api.nuget.org/v3/catalog0/data/2024.01.02.03.04.05/testpkg.%[1]s.json#dependencygroup/.netstandard2.0","@type":"PackageDependencyGroup",`+ + `"dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2024.01.02.03.04.05/testpkg.%[1]s.json#dependencygroup/.netstandard2.0/example.dependency",`+ + `"@type":"PackageDependency","id":"Example.Dependency","range":"[8.0.0, )","registration":"https://api.nuget.org/v3/registration5-gz-semver2/example.dependency/index.json"}],`+ + `"targetFramework":".NETStandard2.0"},`+ + `{"@id":"https://api.nuget.org/v3/catalog0/data/2024.01.02.03.04.05/testpkg.%[1]s.json#dependencygroup/net8.0","@type":"PackageDependencyGroup","targetFramework":"net8.0"}],`+ + `"description":"An example package with a description about as long as the ones real packages publish for every version.",`+ + `"iconUrl":"https://api.nuget.org/v3-flatcontainer/testpkg/%[1]s/icon","id":"TestPkg","language":"","licenseExpression":"Apache-2.0",`+ + `"licenseUrl":"https://www.nuget.org/packages/TestPkg/%[1]s/license","listed":true,"minClientVersion":"",`+ + `"packageContent":"https://api.nuget.org/v3-flatcontainer/testpkg/%[1]s/testpkg.%[1]s.nupkg","projectUrl":"https://example.test/testpkg",`+ + `"published":"%[2]s","requireLicenseAcceptance":false,"summary":"","tags":["example","testing","package"],"title":"TestPkg","version":"%[1]s"},`+ + `"packageContent":"https://api.nuget.org/v3-flatcontainer/testpkg/%[1]s/testpkg.%[1]s.nupkg",`+ + `"registration":"https://api.nuget.org/v3/registration5-gz-semver2/testpkg/index.json"}`, version, published) + } + b.WriteString(`],"lower":"1.0.0","parent":"https://api.nuget.org/v3/registration5-gz-semver2/testpkg/index.json","upper":"1.63.0"}`) + return b.String() +} + +// Listing versions must not build every registration page in memory: restore +// asks for the list of each package it resolves, and only the publish times +// of a registration matter for it. +func TestNuGetVersionListAllocatesLessThanRegistration(t *testing.T) { + const pages, leavesPerPage = 20, 64 + old := time.Now().Add(-30 * 24 * time.Hour).Format(time.RFC3339) + recent := time.Now().Add(-24 * time.Hour).Format(time.RFC3339) + + documents := map[string]string{} + var index, versions strings.Builder + registrationBytes := 0 + for page := range pages { + published := old + if page == pages-1 { + published = recent + } + path := fmt.Sprintf("%spage/%d.json", nugetTestRegistration, page) + documents[path] = nugetRegistrationPage(page*leavesPerPage, leavesPerPage, published) + registrationBytes += len(documents[path]) + if page > 0 { + index.WriteByte(',') + } + fmt.Fprintf(&index, `{"@id":%q,"count":%d}`, path, leavesPerPage) + } + for version := range pages * leavesPerPage { + if version > 0 { + versions.WriteByte(',') + } + fmt.Fprintf(&versions, `"1.%d.0"`, version) + } + documents[nugetTestRegistration+"index.json"] = `{"count":` + fmt.Sprint(pages) + `,"items":[` + index.String() + `]}` + documents["/v3-flatcontainer/testpkg/index.json"] = `{"versions":[` + versions.String() + `]}` + upstream := nugetDocumentUpstream(t, documents) + + proxy, _, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "14d"} + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + h := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + routes := h.Routes() + list := func() string { + t.Helper() + return nugetGet(t, routes, "/v3-flatcontainer/testpkg/index.json", http.StatusOK).Body.String() + } + + // The first request fills the metadata cache; the measured ones read from it. + body := list() + allowed := (pages - 1) * leavesPerPage + if got := strings.Count(body, `"1.`); got != allowed { + t.Fatalf("version list has %d versions, want %d", got, allowed) + } + if strings.Contains(body, fmt.Sprintf(`"1.%d.0"`, allowed)) { + t.Fatal("version list contains a version in cooldown") + } + + const runs = 5 + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + for range runs { + list() + } + runtime.ReadMemStats(&after) + + perRun := (after.TotalAlloc - before.TotalAlloc) / runs + t.Logf("allocated %d bytes per request for a %d byte registration", perRun, registrationBytes) + // One copy is read from the cache. Decoding the old way took four more. + if perRun > 3*uint64(registrationBytes) { + t.Errorf("version list allocated %d bytes for a %d byte registration", perRun, registrationBytes) + } +} From 4a6911151980df4977bac41f4d482e03a8bda17f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 9 Nov 2026 23:33:38 +0100 Subject: [PATCH 190/198] Stream NuGet registration documents instead of building them The registration endpoints decoded the index and, with cooldown on, all of its pages into generic maps, filtered and rewrote the tree and encoded it again. For AWSSDK.Core that allocated about 80 MB per request for a 6.8 MB response. The document is now copied token by token. Links are rewritten on the way, leaves in cooldown and pages left empty are cut out of the output buffer, and count, lower and upper are written once a page's leaves are known. The index and its linked pages are still loaded and checked before the response starts, so a registration that can't be filtered stays an error response. The output keeps the upstream's member order and bytes, so it is the same JSON as before but no longer sorted by key. --- internal/handler/nuget_cooldown.go | 180 +++---- internal/handler/nuget_registration.go | 555 ++++++++++++++++++++ internal/handler/nuget_registration_test.go | 370 +++++++++++++ internal/handler/nuget_test.go | 42 +- 4 files changed, 1013 insertions(+), 134 deletions(-) create mode 100644 internal/handler/nuget_registration.go create mode 100644 internal/handler/nuget_registration_test.go diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index d5a1571..6bc2794 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -1,6 +1,7 @@ package handler import ( + "bufio" "bytes" "compress/gzip" "context" @@ -48,7 +49,8 @@ func (h *NuGetHandler) nugetMetadata(ctx context.Context, path string) (map[stri } // readNuGetMetadata fetches a document and passes it to decode, which keeps -// what its caller needs. decode can run more than once and must not keep body. +// what its caller needs. decode can run more than once, and the document is +// the one it last accepted. It must not modify body. func (h *NuGetHandler) readNuGetMetadata(ctx context.Context, path string, decode func(body []byte) error) error { target := h.upstreamURL + path key := fmt.Sprintf("_cooldown/%x", sha256.Sum256([]byte(target))) @@ -129,15 +131,26 @@ func (t *nugetTimes) release() (version, published string) { } func (h *NuGetHandler) nugetTimesMetadata(ctx context.Context, path string) (*nugetTimes, error) { + document, _, err := h.nugetTimesDocument(ctx, path) + return document, err +} + +// nugetTimesDocument is nugetTimesMetadata that also returns the document +// itself, for a caller that goes on to copy it. +func (h *NuGetHandler) nugetTimesDocument(ctx context.Context, path string) (*nugetTimes, []byte, error) { var document *nugetTimes - err := h.readNuGetMetadata(ctx, path, func(body []byte) (err error) { - document, err = decodeNuGetTimes(body) + var body []byte + err := h.readNuGetMetadata(ctx, path, func(candidate []byte) error { + decoded, err := decodeNuGetTimes(candidate) + if err == nil { + document, body = decoded, candidate + } return err }) if err != nil { - return nil, err + return nil, nil, err } - return document, nil + return document, body, nil } // decodeNuGetTimes accepts the documents decodeNuGetDocument does. A field of @@ -222,7 +235,7 @@ func (h *NuGetHandler) handleVersionList(w http.ResponseWriter, r *http.Request) if h.nugetCooldownApplies(id) { registration, registrationPath, err := h.nugetRegistrationMetadata(r.Context(), url.PathEscape(id)+"/index.json") if err == nil { - err = h.expandNuGetPageTimes(r.Context(), registration, registrationPath) + err = h.expandNuGetPageTimes(r.Context(), registration, registrationPath, nil) } if err != nil { h.nugetMetadataError(w, err) @@ -269,21 +282,8 @@ func (h *NuGetHandler) nugetDownloadAllowed(ctx context.Context, id, version str return h.nugetPublishedAllowed(id, published), nil } -// A standalone leaf has published at its root; leaves embedded in pages carry -// it in catalogEntry. Missing/invalid timestamps retain the existing permissive -// behavior, but fetch and JSON errors must not bypass the policy. -func (h *NuGetHandler) nugetLeafAllowed(leaf map[string]any, id string) bool { - if !h.cooldownEnabled() { - return true - } - entry := nugetCatalogEntry(leaf) - if id == "" { - id, _ = entry["id"].(string) - } - published, _ := entry["published"].(string) - return h.nugetPublishedAllowed(id, published) -} - +// Missing/invalid timestamps retain the existing permissive behavior, but +// fetch and JSON errors must not bypass the policy. func (h *NuGetHandler) nugetPublishedAllowed(id, published string) bool { when, err := time.Parse(time.RFC3339, published) if err != nil { @@ -292,13 +292,6 @@ func (h *NuGetHandler) nugetPublishedAllowed(id, published string) bool { return h.proxy.cooldown().IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) } -func nugetCatalogEntry(leaf map[string]any) map[string]any { - if entry, ok := leaf["catalogEntry"].(map[string]any); ok { - return entry - } - return leaf -} - func (h *NuGetHandler) collectNuGetBlockedVersions(document *nugetTimes, id string, blocked map[string]bool) { if version, published := document.release(); version != "" && !h.nugetPublishedAllowed(id, published) { blocked[nugetVersionKey(version)] = true @@ -315,22 +308,41 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request } id := nugetRegistrationID(r.URL.Path) applyCooldown := h.nugetCooldownApplies(id) - document, err := h.nugetMetadata(r.Context(), r.URL.Path) - if err == nil && applyCooldown { - err = h.expandNuGetPages(r.Context(), document, r.URL.Path) + // Everything the response is made of is loaded and checked before any of + // it is written, so a registration that can't be filtered is an error + // response and not a cut-off document. + document, body, err := h.nugetTimesDocument(r.Context(), r.URL.Path) + var pages map[string][]byte + if err == nil && applyCooldown && strings.HasSuffix(r.URL.Path, "/index.json") { + pages = make(map[string][]byte) + err = h.expandNuGetPageTimes(r.Context(), document, r.URL.Path, pages) } if err != nil { h.nugetMetadataError(w, err) return } - _, hasItems := document["items"] - if applyCooldown && !h.filterNuGetRegistration(document, id) && !hasItems { - JSONError(w, http.StatusNotFound, "version not found") - return + if applyCooldown && document.Items == nil { + if _, published := document.release(); !h.nugetPublishedAllowed(id, published) { + JSONError(w, http.StatusNotFound, "version not found") + return + } } - h.rewriteNuGetRegistrationLinks(document) + out := &countingWriter{w: w} + buffered := bufio.NewWriterSize(out, nugetFlushSize) w.Header().Set(headerContentType, contentTypeJSON) - _ = json.NewEncoder(w).Encode(document) + err = h.writeNuGetRegistration(buffered, body, id, applyCooldown, pages) + if err == nil { + err = buffered.Flush() + } + if err != nil { + h.proxy.Logger.Error("failed to write NuGet registration", "path", r.URL.Path, "error", err) + if out.n == 0 { + JSONError(w, http.StatusBadGateway, "failed to process package metadata") + return + } + // Abort rather than end a partial document as if it were complete. + panic(http.ErrAbortHandler) + } } func nugetRegistrationID(path string) string { @@ -343,45 +355,12 @@ func nugetRegistrationID(path string) string { return "" } -// Only expand index pages, never recursively follow arbitrary upstream links. -// Pin requests to this configured upstream and the current package's page path. -func (h *NuGetHandler) expandNuGetPages(ctx context.Context, document map[string]any, path string) error { - if !strings.HasSuffix(path, "/index.json") { - return nil - } - items, ok := document["items"].([]any) - if !ok { - return fmt.Errorf("missing registration pages") - } - for _, item := range items { - page, ok := item.(map[string]any) - if !ok { - return fmt.Errorf("invalid registration page") - } - if _, ok := page["items"].([]any); ok { - continue - } - link, _ := page["@id"].(string) - pagePath, err := h.nugetRegistrationPagePath(path, link) - if err != nil { - return err - } - pageDocument, err := h.nugetMetadata(ctx, pagePath) - if err != nil { - return err - } - leaves, ok := pageDocument["items"].([]any) - if !ok { - return fmt.Errorf("missing registration leaves") - } - page["items"] = leaves - } - return nil -} - -// expandNuGetPageTimes is expandNuGetPages for a registration index decoded -// into nugetTimes. -func (h *NuGetHandler) expandNuGetPageTimes(ctx context.Context, index *nugetTimes, path string) error { +// expandNuGetPageTimes gives each page of a registration index that only +// links to its leaves the leaves of the page it links to. With bodies not nil +// the linked pages themselves are kept in it, by link. Only index pages are +// expanded, never arbitrary upstream links followed: requests are pinned to +// this configured upstream and the current package's page path. +func (h *NuGetHandler) expandNuGetPageTimes(ctx context.Context, index *nugetTimes, path string, bodies map[string][]byte) error { if index.Items == nil { return fmt.Errorf("missing registration pages") } @@ -394,7 +373,7 @@ func (h *NuGetHandler) expandNuGetPageTimes(ctx context.Context, index *nugetTim if err != nil { return err } - pageDocument, err := h.nugetTimesMetadata(ctx, pagePath) + pageDocument, body, err := h.nugetTimesDocument(ctx, pagePath) if err != nil { return err } @@ -402,6 +381,9 @@ func (h *NuGetHandler) expandNuGetPageTimes(ctx context.Context, index *nugetTim return fmt.Errorf("missing registration leaves") } page.Items = pageDocument.Items + if bodies != nil { + bodies[page.ID] = body + } } return nil } @@ -423,50 +405,6 @@ func (h *NuGetHandler) nugetRegistrationPagePath(indexPath, link string) (string return strings.TrimPrefix(target.Path, upstream.Path), nil } -func (h *NuGetHandler) filterNuGetRegistration(document map[string]any, id string) bool { - items, ok := document["items"].([]any) - if !ok { - return h.nugetLeafAllowed(document, id) - } - filtered := make([]any, 0, len(items)) - for _, item := range items { - child, ok := item.(map[string]any) - if ok && h.filterNuGetRegistration(child, id) { - filtered = append(filtered, child) - } - } - document["items"] = filtered - document["count"] = len(filtered) - // Page bounds describe the retained leaves, not versions hidden by cooldown. - if _, isPage := document["lower"]; isPage && len(filtered) > 0 { - first, _ := filtered[0].(map[string]any) - last, _ := filtered[len(filtered)-1].(map[string]any) - document["lower"] = nugetCatalogEntry(first)["version"] - document["upper"] = nugetCatalogEntry(last)["version"] - } - return len(filtered) > 0 -} - -func (h *NuGetHandler) rewriteNuGetRegistrationLinks(value any) { - switch node := value.(type) { - case map[string]any: - for key, child := range node { - if link, ok := child.(string); ok { - switch key { - case "@id", "parent", "registration", "packageContent": - node[key] = h.nugetProxyLink(link) - } - } else { - h.rewriteNuGetRegistrationLinks(child) - } - } - case []any: - for _, child := range node { - h.rewriteNuGetRegistrationLinks(child) - } - } -} - func (h *NuGetHandler) nugetProxyLink(link string) string { u, err := url.Parse(link) if err != nil { diff --git a/internal/handler/nuget_registration.go b/internal/handler/nuget_registration.go new file mode 100644 index 0000000..dafa8d5 --- /dev/null +++ b/internal/handler/nuget_registration.go @@ -0,0 +1,555 @@ +package handler + +import ( + "bytes" + "encoding/json/jsontext" + "fmt" + "io" + "slices" + "strconv" + "strings" +) + +// nugetFlushSize is how much of a document that isn't filtered is collected +// before it is written out. +const nugetFlushSize = 32 << 10 + +// The members of a registration document that are read or rewritten. +const ( + nugetMemberOther = iota + nugetMemberID + nugetMemberLink + nugetMemberItems + nugetMemberCount + nugetMemberLower + nugetMemberUpper + nugetMemberCatalogEntry + nugetMemberVersion + nugetMemberPublished +) + +func nugetMemberKind(name []byte) int { + switch string(name) { + case "@id": + return nugetMemberID + case "parent", "registration", "packageContent": + return nugetMemberLink + case "items": + return nugetMemberItems + case "count": + return nugetMemberCount + case "lower": + return nugetMemberLower + case "upper": + return nugetMemberUpper + case "catalogEntry": + return nugetMemberCatalogEntry + case "version": + return nugetMemberVersion + case "published": + return nugetMemberPublished + } + return nugetMemberOther +} + +// nugetRegistrationWriter copies a registration index, page or leaf with its +// links pointing at the proxy. With filter set it leaves out the leaves in +// cooldown and the pages that end up empty, and corrects the counts and +// bounds of the rest, as it goes: the document is never held as a tree. +// +// A document is made of nodes. A node with an items array is the index or a +// page, and the objects in that array are its nodes in turn; any other node +// is a leaf. Everything else is copied with its upstream bytes. +type nugetRegistrationWriter struct { + h *NuGetHandler + id string + filter bool + // pages holds the pages an index links to, by link. It is nil unless the + // document is an index whose linked pages are to be inlined. + pages map[string][]byte + out io.Writer + // buf holds the output not written yet. A node is only known to be kept + // once it has been read, so it is collected here and cut off again + // otherwise. + buf bytes.Buffer + name []byte // the current member's name, as it is in the document + text []byte // scratch space for a decoded or encoded string + // A link that is the upstream URL followed by plain path characters is + // rewritten by putting proxyPrefix, which opens the JSON string, in place + // of upstreamPrefix. proxyPrefix is nil when links can't be rewritten so. + upstreamPrefix []byte + proxyPrefix []byte +} + +// nugetLinkProbe is a path that nugetProxyLink rewrites. +const nugetLinkProbe = "/v3-flatcontainer/" + +// setLinkPrefixes prepares rewriting plain links without parsing them. Nearly +// every link of a registration is one, and parsing each as a URL allocates +// more than the rest of the copy does. +func (rw *nugetRegistrationWriter) setLinkPrefixes() { + upstream := rw.h.upstreamURL + if strings.ContainsAny(upstream, "?#") { + return + } + proxied, ok := strings.CutSuffix(rw.h.nugetProxyLink(upstream+nugetLinkProbe), nugetLinkProbe) + if !ok { + return + } + quoted, err := jsontext.AppendQuote(nil, proxied) + if err != nil { + return + } + rw.upstreamPrefix = []byte(upstream) + rw.proxyPrefix = quoted[:len(quoted)-1] +} + +// nugetArtifactPrefixBytes is nugetArtifactPrefixes for matching in a document. +var nugetArtifactPrefixBytes = func() [][]byte { + prefixes := make([][]byte, len(nugetArtifactPrefixes)) + for i, prefix := range nugetArtifactPrefixes { + prefixes[i] = []byte(prefix) + } + return prefixes +}() + +// nugetLinkIsForeign reports whether a link can't be to a NuGet resource, so +// that nugetProxyLink leaves it alone: its path names none, however the link +// parses. An escaped link is not judged here. +func nugetLinkIsForeign(link []byte) bool { + if bytes.IndexByte(link, '%') >= 0 { + return false + } + for _, prefix := range nugetArtifactPrefixBytes { + if bytes.Contains(link, prefix) { + return false + } + } + return true +} + +// plainLinkPath returns the path of a link that nugetProxyLink rewrites to +// the same path below the proxy: one on the upstream, to a NuGet resource, +// with nothing in it that a URL or JSON would encode. +func (rw *nugetRegistrationWriter) plainLinkPath(link []byte) ([]byte, bool) { + if rw.proxyPrefix == nil { + return nil, false + } + path, ok := bytes.CutPrefix(link, rw.upstreamPrefix) + if !ok || !slices.ContainsFunc(nugetArtifactPrefixBytes, func(prefix []byte) bool { return bytes.HasPrefix(path, prefix) }) { + return nil, false + } + for _, c := range path { + plain := c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '/' || c == '.' || c == '-' || c == '_' + if !plain { + return nil, false + } + } + return path, true +} + +// writeLink writes a link, given as a JSON string, pointing at the proxy. +func (rw *nugetRegistrationWriter) writeLink(raw []byte) { + link := rw.unquote(raw) + if nugetLinkIsForeign(link) { + rw.buf.Write(raw) + return + } + if path, ok := rw.plainLinkPath(link); ok { + rw.buf.Write(rw.proxyPrefix) + rw.buf.Write(path) + rw.buf.WriteByte('"') + return + } + original := string(link) + if proxied := rw.h.nugetProxyLink(original); proxied != original { + rw.text, _ = jsontext.AppendQuote(rw.text[:0], proxied) + raw = rw.text + } + rw.buf.Write(raw) +} + +// nugetRelease is what an object says about the release it describes. +type nugetRelease struct { + link string + version []byte // as JSON, nil when there is none + published string +} + +// nugetNode is the state of the node being copied. +type nugetNode struct { + depth int + wrote bool // a member has been written + own nugetRelease + entry nugetRelease + hasEntry bool + container bool + kept int + first []byte // version of the first and last child kept, as JSON + last []byte + // Their values are only known at the end of a container, so these members + // are held back and written last. + count, lower, upper []byte +} + +func newNuGetDecoder(document []byte) *jsontext.Decoder { + // A bytes.Buffer is parsed in place. Duplicate names and invalid UTF-8 + // are accepted as encoding/json accepts them. + return jsontext.NewDecoder(bytes.NewBuffer(document), jsontext.AllowDuplicateNames(true), jsontext.AllowInvalidUTF8(true)) +} + +// writeNuGetRegistration writes the registration document to w. The document +// and pages must have been decoded before, so that it only fails once the +// response has started if writing does. +func (h *NuGetHandler) writeNuGetRegistration(w io.Writer, document []byte, id string, filter bool, pages map[string][]byte) error { + rw := &nugetRegistrationWriter{h: h, id: id, filter: filter, pages: pages, out: w} + rw.setLinkPrefixes() + dec := newNuGetDecoder(document) + var err error + if filter { + _, _, err = rw.node(dec, 0) + } else { + err = rw.value(dec) + } + if err != nil { + return err + } + return rw.flush() +} + +func (rw *nugetRegistrationWriter) flush() error { + _, err := rw.out.Write(rw.buf.Bytes()) + rw.buf.Reset() + return err +} + +// unquote returns the text of a JSON string. It is valid until the next call. +func (rw *nugetRegistrationWriter) unquote(raw []byte) []byte { + if bytes.IndexByte(raw, '\\') < 0 { + return raw[1 : len(raw)-1] + } + // The decoder has checked the string, so the only error left is invalid + // UTF-8, which is replaced as encoding/json replaces it. + rw.text, _ = jsontext.AppendUnquote(rw.text[:0], raw) + return rw.text +} + +// readName reads the name of the next member and keeps it for writeName. +func (rw *nugetRegistrationWriter) readName(dec *jsontext.Decoder) (int, error) { + raw, err := dec.ReadValue() + if err != nil { + return nugetMemberOther, err + } + rw.name = append(rw.name[:0], raw...) + return nugetMemberKind(rw.unquote(raw)), nil +} + +// writeName writes the name read last, or name when it is not empty. +func (rw *nugetRegistrationWriter) writeName(wrote *bool, name string) { + if *wrote { + rw.buf.WriteByte(',') + } + *wrote = true + if name == "" { + rw.buf.Write(rw.name) + } else { + rw.buf.WriteString(name) + } + rw.buf.WriteByte(':') +} + +// value copies the value dec is at, rewriting the links in it. +func (rw *nugetRegistrationWriter) value(dec *jsontext.Decoder) error { + switch dec.PeekKind() { + case '{': + return rw.object(dec, nil) + case '[': + return rw.array(dec) + } + raw, err := dec.ReadValue() + if err != nil { + return err + } + rw.buf.Write(raw) + return nil +} + +func (rw *nugetRegistrationWriter) array(dec *jsontext.Decoder) error { + if _, err := dec.ReadToken(); err != nil { + return err + } + rw.buf.WriteByte('[') + for i := 0; dec.PeekKind() != ']'; i++ { + if i > 0 { + rw.buf.WriteByte(',') + } + if err := rw.value(dec); err != nil { + return err + } + } + rw.buf.WriteByte(']') + _, err := dec.ReadToken() + return err +} + +// object copies the object dec is at, rewriting the links in it. release, +// when not nil, receives what the object says about its release. +func (rw *nugetRegistrationWriter) object(dec *jsontext.Decoder, release *nugetRelease) error { + if _, err := dec.ReadToken(); err != nil { + return err + } + rw.buf.WriteByte('{') + wrote := false + for dec.PeekKind() != '}' { + kind, err := rw.readName(dec) + if err != nil { + return err + } + rw.writeName(&wrote, "") + if err := rw.member(dec, kind, release); err != nil { + return err + } + // Nothing written so far can be cut off again unless nodes are + // filtered, so only then does the output wait for the node to end. + if !rw.filter && rw.buf.Len() >= nugetFlushSize { + if err := rw.flush(); err != nil { + return err + } + } + } + rw.buf.WriteByte('}') + _, err := dec.ReadToken() + return err +} + +// member copies the value of a member of the given kind. +func (rw *nugetRegistrationWriter) member(dec *jsontext.Decoder, kind int, release *nugetRelease) error { + if kind == nugetMemberOther || dec.PeekKind() != '"' { + return rw.value(dec) + } + raw, err := dec.ReadValue() + if err != nil { + return err + } + switch kind { + case nugetMemberID, nugetMemberLink: + if kind == nugetMemberID && release != nil && rw.pages != nil { + release.link = string(rw.unquote(raw)) + } + rw.writeLink(raw) + return nil + case nugetMemberVersion: + if release != nil { + release.version = append(release.version[:0], raw...) + } + case nugetMemberPublished: + if release != nil { + release.published = string(rw.unquote(raw)) + } + } + rw.buf.Write(raw) + return nil +} + +// node copies the index, page or leaf dec is at. It reports whether the node +// is to be kept, and the version of its release as JSON. +func (rw *nugetRegistrationWriter) node(dec *jsontext.Decoder, depth int) (bool, []byte, error) { + if dec.PeekKind() != '{' { + return false, nil, rw.value(dec) + } + if _, err := dec.ReadToken(); err != nil { + return false, nil, err + } + rw.buf.WriteByte('{') + n := &nugetNode{depth: depth} + for dec.PeekKind() != '}' { + kind, err := rw.readName(dec) + if err != nil { + return false, nil, err + } + if err := rw.nodeMember(dec, kind, n); err != nil { + return false, nil, err + } + } + if _, err := dec.ReadToken(); err != nil { + return false, nil, err + } + if err := rw.endNode(n); err != nil { + return false, nil, err + } + release := &n.own + if n.hasEntry { + release = &n.entry + } + if n.container { + return n.kept > 0, release.version, nil + } + return rw.h.nugetPublishedAllowed(rw.id, release.published), release.version, nil +} + +// linksPage reports whether a node at depth is a page of an index whose +// linked pages are inlined. +func (rw *nugetRegistrationWriter) linksPage(depth int) bool { + return rw.pages != nil && depth == 1 +} + +func (rw *nugetRegistrationWriter) nodeMember(dec *jsontext.Decoder, kind int, n *nugetNode) error { + var held *[]byte + switch kind { + case nugetMemberItems: + if dec.PeekKind() == '[' { + rw.writeName(&n.wrote, "") + return rw.items(dec, n) + } + if rw.linksPage(n.depth) { + // The leaves of the linked page take its place. + return dec.SkipValue() + } + case nugetMemberCatalogEntry: + // A standalone leaf has a link here and its release in the leaf. + if dec.PeekKind() == '{' { + rw.writeName(&n.wrote, "") + n.hasEntry = true + n.entry = nugetRelease{} + return rw.object(dec, &n.entry) + } + case nugetMemberCount: + held = &n.count + case nugetMemberLower: + held = &n.lower + case nugetMemberUpper: + held = &n.upper + } + if held != nil { + raw, err := dec.ReadValue() + if err != nil { + return err + } + *held = append((*held)[:0], raw...) + return nil + } + rw.writeName(&n.wrote, "") + return rw.member(dec, kind, &n.own) +} + +// items copies the array dec is at, the children of n, without those that +// are not kept. +func (rw *nugetRegistrationWriter) items(dec *jsontext.Decoder, n *nugetNode) error { + if _, err := dec.ReadToken(); err != nil { + return err + } + n.container = true + rw.buf.WriteByte('[') + wrote := false + for dec.PeekKind() != ']' { + // What the document's root has so far is final. Below it a node can + // still be cut off, by the page it is in ending up empty. + if n.depth == 0 { + if err := rw.flush(); err != nil { + return err + } + } + mark := rw.buf.Len() + if wrote { + rw.buf.WriteByte(',') + } + // A child that is not an object is left out. + var keep bool + var version []byte + if dec.PeekKind() == '{' { + var err error + if keep, version, err = rw.node(dec, n.depth+1); err != nil { + return err + } + } else if err := dec.SkipValue(); err != nil { + return err + } + if !keep { + rw.buf.Truncate(mark) + continue + } + wrote = true + if n.kept == 0 { + n.first = append(n.first[:0], version...) + } + n.last = append(n.last[:0], version...) + n.kept++ + } + rw.buf.WriteByte(']') + _, err := dec.ReadToken() + return err +} + +// endNode writes what is left of a node once its members have been read. +func (rw *nugetRegistrationWriter) endNode(n *nugetNode) error { + if rw.linksPage(n.depth) && !n.container { + if err := rw.linkedItems(n); err != nil { + return err + } + } + if n.container { + n.count = append(n.count[:0], strconv.Itoa(n.kept)...) + // Page bounds describe the retained leaves, not versions hidden by cooldown. + if n.lower != nil && n.kept > 0 { + n.lower, n.upper = nugetVersionJSON(n.first), nugetVersionJSON(n.last) + } + } + for _, held := range []struct { + name string + value []byte + }{{`"count"`, n.count}, {`"lower"`, n.lower}, {`"upper"`, n.upper}} { + if held.value == nil { + continue + } + rw.writeName(&n.wrote, held.name) + if held.value[0] != '{' && held.value[0] != '[' { + rw.buf.Write(held.value) + continue + } + // Links are rewritten at any depth, so in here too. + if err := rw.value(newNuGetDecoder(held.value)); err != nil { + return err + } + } + rw.buf.WriteByte('}') + return nil +} + +func nugetVersionJSON(version []byte) []byte { + if version == nil { + return []byte("null") + } + return version +} + +// linkedItems writes the leaves of the page that n, a page of the index +// without leaves of its own, links to. +func (rw *nugetRegistrationWriter) linkedItems(n *nugetNode) error { + page, ok := rw.pages[n.own.link] + if !ok { + return fmt.Errorf("registration page %q was not loaded", n.own.link) + } + dec := newNuGetDecoder(page) + if _, err := dec.ReadToken(); err != nil { + return err + } + for dec.PeekKind() != '}' { + kind, err := rw.readName(dec) + if err != nil { + return err + } + if kind != nugetMemberItems || dec.PeekKind() != '[' { + err = dec.SkipValue() + } else { + rw.writeName(&n.wrote, "") + err = rw.items(dec, n) + } + if err != nil { + return err + } + } + if !n.container { + return fmt.Errorf("registration page %q has no leaves", n.own.link) + } + return nil +} diff --git a/internal/handler/nuget_registration_test.go b/internal/handler/nuget_registration_test.go new file mode 100644 index 0000000..aef46cc --- /dev/null +++ b/internal/handler/nuget_registration_test.go @@ -0,0 +1,370 @@ +package handler + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "runtime" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +const nugetTestProxyBase = "http://proxy.test/nuget" + +// nugetTestLeaf is a registration leaf as a page embeds it. base is empty for +// the upstream document and the proxy's NuGet URL for the expected one. +func nugetTestLeaf(base, version, published string) string { + return `{"@id":"` + base + nugetTestRegistration + version + `.json",` + + `"catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/testpkg.` + version + `.json","id":"TestPkg",` + + `"dependencyGroups":[{"dependencies":[{"id":"Dep","registration":"` + base + nugetRegistrationPath + `dep/index.json"}]}],` + + `"published":"` + published + `","version":"` + version + `"},` + + `"packageContent":"` + base + `/v3-flatcontainer/testpkg/` + version + `/testpkg.` + version + `.nupkg",` + + `"registration":"` + base + nugetTestRegistration + `index.json"}` +} + +func nugetRegistrationRoutes(t *testing.T, policy *cooldown.Config, documents map[string]string) http.Handler { + t.Helper() + upstream := nugetDocumentUpstream(t, documents) + proxy := nugetTestProxy() + proxy.Cooldown = policy + return NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL).Routes() +} + +func assertNuGetJSON(t *testing.T, got, want string) { + t.Helper() + var gotValue, wantValue any + if err := json.Unmarshal([]byte(got), &gotValue); err != nil { + t.Fatalf("response is not JSON: %v: %s", err, got) + } + if err := json.Unmarshal([]byte(want), &wantValue); err != nil { + t.Fatalf("expected document is not JSON: %v: %s", err, want) + } + if !reflect.DeepEqual(gotValue, wantValue) { + t.Errorf("response differs\n got: %s\nwant: %s", got, want) + } +} + +func TestNuGetRegistrationIndexCooldown(t *testing.T) { + old := time.Now().Add(-30 * 24 * time.Hour).Format(time.RFC3339) + recent := time.Now().Add(-24 * time.Hour).Format(time.RFC3339) + const index = nugetTestRegistration + "index.json" + const inlined = nugetTestRegistration + "page/1.0.0/2.0.0.json" + const linked = nugetTestRegistration + "page/3.0.0/4.0.0.json" + const allRecent = nugetTestRegistration + "page/5.0.0/5.0.0.json" + + routes := nugetRegistrationRoutes(t, &cooldown.Config{Default: "14d"}, map[string]string{ + index: `{"@id":"` + index + `","count":3,"items":[` + + `{"@id":"` + inlined + `","count":2,"items":[` + nugetTestLeaf("", "1.0.0", old) + `,` + nugetTestLeaf("", "2.0.0", recent) + `],` + + `"lower":"1.0.0","parent":"` + index + `","upper":"2.0.0"},` + + `{"@id":"` + linked + `","count":2,"lower":"3.0.0","upper":"4.0.0"},` + + `{"@id":"` + allRecent + `","count":1,"lower":"5.0.0","upper":"5.0.0"}]}`, + linked: `{"@id":"` + linked + `","commitId":"only in the page","count":2,"items":[` + + nugetTestLeaf("", "3.0.0", recent) + `,` + nugetTestLeaf("", "4.0.0", old) + `],"lower":"3.0.0","parent":"` + index + `","upper":"4.0.0"}`, + allRecent: `{"items":[` + nugetTestLeaf("", "5.0.0", recent) + `]}`, + }) + + w := nugetGet(t, routes, index, http.StatusOK) + if got := w.Header().Get(headerContentType); got != contentTypeJSON { + t.Errorf("Content-Type = %q, want %q", got, contentTypeJSON) + } + const base = nugetTestProxyBase + assertNuGetJSON(t, w.Body.String(), `{"@id":"`+base+index+`","count":2,"items":[`+ + `{"@id":"`+base+inlined+`","count":1,"items":[`+nugetTestLeaf(base, "1.0.0", old)+`],`+ + `"lower":"1.0.0","parent":"`+base+index+`","upper":"1.0.0"},`+ + `{"@id":"`+base+linked+`","count":1,"items":[`+nugetTestLeaf(base, "4.0.0", old)+`],"lower":"4.0.0","upper":"4.0.0"}]}`) +} + +func TestNuGetRegistrationDocumentCooldown(t *testing.T) { + old := time.Now().Add(-30 * 24 * time.Hour).Format(time.RFC3339) + recent := time.Now().Add(-24 * time.Hour).Format(time.RFC3339) + const base = nugetTestProxyBase + const index = nugetTestRegistration + "index.json" + const page = nugetTestRegistration + "page/1.0.0/3.0.0.json" + const leaf = nugetTestRegistration + "1.0.0.json" + standalone := func(base, published string) string { + return `{"@id":"` + base + leaf + `","catalogEntry":"https://api.nuget.org/v3/catalog0/data/testpkg.1.0.0.json","listed":true,` + + `"packageContent":"` + base + `/v3-flatcontainer/testpkg/1.0.0/testpkg.1.0.0.nupkg","published":"` + published + `",` + + `"registration":"` + base + index + `"}` + } + + for _, tt := range []struct { + name, path, document string + status int + want string + }{ + { + name: "page", + path: page, + document: `{"@id":"` + page + `","count":3,"items":[` + nugetTestLeaf("", "1.0.0", recent) + `,` + nugetTestLeaf("", "2.0.0", old) + `,` + + nugetTestLeaf("", "3.0.0", recent) + `],"lower":"1.0.0","parent":"` + index + `","upper":"3.0.0"}`, + status: http.StatusOK, + want: `{"@id":"` + base + page + `","count":1,"items":[` + nugetTestLeaf(base, "2.0.0", old) + `],` + + `"lower":"2.0.0","parent":"` + base + index + `","upper":"2.0.0"}`, + }, + { + name: "page with every version in cooldown", + path: page, + document: `{"count":1,"items":[` + nugetTestLeaf("", "1.0.0", recent) + `],"lower":"1.0.0","upper":"1.0.0"}`, + status: http.StatusOK, + want: `{"count":0,"items":[],"lower":"1.0.0","upper":"1.0.0"}`, + }, + { + name: "page without a count or bounds", + path: page, + document: `{"items":[` + nugetTestLeaf("", "1.0.0", old) + `]}`, + status: http.StatusOK, + want: `{"count":1,"items":[` + nugetTestLeaf(base, "1.0.0", old) + `]}`, + }, + { + name: "children that are not objects are left out", + path: page, + document: `{"items":[7,` + nugetTestLeaf("", "1.0.0", old) + `,"x",null,[],` + nugetTestLeaf("", "2.0.0", recent) + `],"lower":"0"}`, + status: http.StatusOK, + want: `{"count":1,"items":[` + nugetTestLeaf(base, "1.0.0", old) + `],"lower":"1.0.0","upper":"1.0.0"}`, + }, + { + name: "escaped member names", + path: page, + document: `{"items":[{"catalogEntry":{"published":"` + recent + `"}},{"@id":"` + leaf + `"}]}`, + status: http.StatusOK, + want: `{"count":1,"items":[{"@id":"` + base + leaf + `"}]}`, + }, + { + name: "leaf without a version", + path: page, + document: `{"items":[{"catalogEntry":{"published":"` + old + `"}}],"lower":"1.0.0"}`, + status: http.StatusOK, + want: `{"count":1,"items":[{"catalogEntry":{"published":"` + old + `"}}],"lower":null,"upper":null}`, + }, + { + name: "leaf without a publish time is kept", + path: page, + document: `{"items":[{"catalogEntry":{"version":"1.0.0"}},{"catalogEntry":{"published":7,"version":"2.0.0"}}],"lower":"0"}`, + status: http.StatusOK, + want: `{"count":2,"items":[{"catalogEntry":{"version":"1.0.0"}},{"catalogEntry":{"published":7,"version":"2.0.0"}}],"lower":"1.0.0","upper":"2.0.0"}`, + }, + { + name: "standalone leaf", + path: leaf, + document: standalone("", old), + status: http.StatusOK, + want: standalone(base, old), + }, + { + name: "standalone leaf in cooldown", + path: leaf, + document: standalone("", recent), + status: http.StatusNotFound, + want: `{"error":"version not found"}`, + }, + } { + t.Run(tt.name, func(t *testing.T) { + routes := nugetRegistrationRoutes(t, &cooldown.Config{Default: "14d"}, map[string]string{tt.path: tt.document}) + assertNuGetJSON(t, nugetGet(t, routes, tt.path, tt.status).Body.String(), tt.want) + }) + } +} + +// An exempt package still gets its links rewritten, but nothing is filtered +// and no page is fetched for it. +func TestNuGetRegistrationWithoutEffectiveCooldown(t *testing.T) { + recent := time.Now().Add(-24 * time.Hour).Format(time.RFC3339) + const base = nugetTestProxyBase + const index = nugetTestRegistration + "index.json" + const linked = nugetTestRegistration + "page/2.0.0/2.0.0.json" + document := func(base string) string { + return `{"@id":"` + base + index + `","count":9,"items":[` + + `{"@id":"` + base + nugetTestRegistration + `page/1.0.0/1.0.0.json","count":5,"items":[7,` + nugetTestLeaf(base, "1.0.0", recent) + `],"lower":"0","upper":"9"},` + + `{"@id":"` + base + linked + `","count":1,"lower":"2.0.0","upper":"2.0.0"}]}` + } + policy := &cooldown.Config{Default: "14d", Packages: map[string]string{"pkg:nuget/testpkg": "0"}} + routes := nugetRegistrationRoutes(t, policy, map[string]string{index: document("")}) + assertNuGetJSON(t, nugetGet(t, routes, index, http.StatusOK).Body.String(), document(base)) +} + +// A registration that can't be filtered completely is refused as a whole, so +// a client never reads part of one as the full list. +func TestNuGetRegistrationRejectsBrokenPages(t *testing.T) { + old := time.Now().Add(-30 * 24 * time.Hour).Format(time.RFC3339) + const index = nugetTestRegistration + "index.json" + const first = nugetTestRegistration + "page/1.0.0/1.0.0.json" + const second = nugetTestRegistration + "page/2.0.0/2.0.0.json" + firstPage := `{"items":[` + nugetTestLeaf("", "1.0.0", old) + `]}` + linkedIndex := `{"items":[{"@id":"` + first + `"},{"@id":"` + second + `"}]}` + + for _, tt := range []struct { + name string + documents map[string]string + status int + }{ + {"missing page", map[string]string{index: linkedIndex, first: firstPage}, http.StatusNotFound}, + {"page that is not JSON", map[string]string{index: linkedIndex, first: firstPage, second: `{"items":[`}, http.StatusBadGateway}, + {"page without leaves", map[string]string{index: linkedIndex, first: firstPage, second: `{"count":1}`}, http.StatusBadGateway}, + {"page that is not an object", map[string]string{index: linkedIndex, first: firstPage, second: `[]`}, http.StatusBadGateway}, + {"index without pages", map[string]string{index: `{"count":0}`}, http.StatusBadGateway}, + {"index page that is not an object", map[string]string{index: `{"items":[7]}`}, http.StatusBadGateway}, + {"index that is not an object", map[string]string{index: `[]`}, http.StatusBadGateway}, + {"empty index", map[string]string{index: `null`}, http.StatusBadGateway}, + {"data after the index", map[string]string{index: `{"items":[]}{}`}, http.StatusBadGateway}, + } { + t.Run(tt.name, func(t *testing.T) { + routes := nugetRegistrationRoutes(t, &cooldown.Config{Default: "14d"}, tt.documents) + body := nugetGet(t, routes, index, tt.status).Body.String() + if !strings.HasPrefix(body, `{"error":`) || strings.Contains(body, "1.0.0") { + t.Errorf("response is not only an error: %s", body) + } + }) + } +} + +// Most links are rewritten without parsing them as URLs. That must give what +// parsing them gives, whatever the upstream, the proxy and the link look like. +func TestNuGetRegistrationLinksMatchProxyLink(t *testing.T) { + paths := []string{ + "/v3-flatcontainer/testpkg/1.0.0/testpkg.1.0.0.nupkg", + "/v3/registration5-gz-semver2/testpkg/index.json", + "/v3/registration5-semver1/testpkg/1.0.0.json", + "/v3/catalog0/data/testpkg.1.0.0.json#dependencygroup/net8.0", + "/v3-flatcontainer/test pkg/1.0.0", + "/v3-flatcontainer/test%2Fpkg/1.0.0", + "/v3-flatcontainer/testpkg/1.0.0+build", + "/v3-flatcontainer/testpkg/index.json?semVerLevel=2.0.0#fragment", + "/v3-flatcontainer/../testpkg", + "/v3-flatcontainer//testpkg", + "/v3-flatcontainer/tëstpkg/1.0.0", + `/v3-flatcontainer/<&>"testpkg"`, + "/v3-flatcontainer", + "/V3-flatcontainer/testpkg", + "/%763-flatcontainer/testpkg", + "/other/v3-flatcontainer/testpkg", + "", + } + for _, upstream := range []string{ + "https://api.nuget.org", "https://api.nuget.org/", "https://mirror.test:8443/nuget/base", "https://mirror.test/a%20b", + "https://user:secret@mirror.test", "https://mirror.test/base?key=1", "https://mirror.test/base#fragment", "", + } { + for _, proxyURL := range []string{ + "http://proxy.test", "HTTP://Proxy.Test:8080/pre fix", "http://proxy.test/a?b=1", `http://proxy.test/"quoted"`, "", + } { + h := &NuGetHandler{proxy: nugetTestProxy(), upstreamURL: upstream, proxyURL: proxyURL} + for _, path := range paths { + for _, link := range []string{upstream + path, path, "https://other.test" + path} { + document, err := json.Marshal(map[string]string{"@id": link}) + if err != nil { + t.Fatal(err) + } + var out strings.Builder + if err := h.writeNuGetRegistration(&out, document, "testpkg", false, nil); err != nil { + t.Fatal(err) + } + var got map[string]string + if err := json.Unmarshal([]byte(out.String()), &got); err != nil { + t.Fatalf("upstream %q, proxy %q, link %q: %v: %s", upstream, proxyURL, link, err, out.String()) + } + if want := h.nugetProxyLink(link); got["@id"] != want { + t.Errorf("upstream %q, proxy %q, link %q: rewritten to %q, want %q", upstream, proxyURL, link, got["@id"], want) + } + } + } + } + } +} + +// discardResponse is a response that keeps nothing, so that a test measures +// what the handler allocates and not what a recorder does. +type discardResponse struct { + header http.Header + status int + bytes int +} + +func (d *discardResponse) Header() http.Header { return d.header } + +func (d *discardResponse) WriteHeader(status int) { d.status = status } + +func (d *discardResponse) Write(p []byte) (int, error) { + if d.status == 0 { + d.status = http.StatusOK + } + d.bytes += len(p) + return len(p), nil +} + +// Serving a registration must not build it in memory: a package with many +// versions has megabytes of it, and restore asks for it per package. +func TestNuGetRegistrationAllocatesLessThanRegistration(t *testing.T) { + const pages, leavesPerPage = 20, 64 + old := time.Now().Add(-30 * 24 * time.Hour).Format(time.RFC3339) + recent := time.Now().Add(-24 * time.Hour).Format(time.RFC3339) + const indexPath = nugetTestRegistration + "index.json" + + // As on nuget.org, the links of the registration point at the upstream. + documents := map[string]string{} + upstream := nugetDocumentUpstream(t, documents) + var index strings.Builder + registrationBytes := 0 + for page := range pages { + published := old + if page == pages-1 { + published = recent + } + path := fmt.Sprintf("%spage/%d.json", nugetTestRegistration, page) + documents[path] = strings.ReplaceAll(nugetRegistrationPage(page*leavesPerPage, leavesPerPage, published), "https://api.nuget.org", upstream.URL) + registrationBytes += len(documents[path]) + if page > 0 { + index.WriteByte(',') + } + fmt.Fprintf(&index, `{"@id":%q,"count":%d,"lower":"1.%d.0","upper":"1.%d.0"}`, upstream.URL+path, leavesPerPage, page*leavesPerPage, (page+1)*leavesPerPage-1) + } + documents[indexPath] = `{"count":` + fmt.Sprint(pages) + `,"items":[` + index.String() + `]}` + + proxy, _, _, _ := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "14d"} + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + routes := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL).Routes() + + // The first request fills the metadata cache; the measured ones read from it. + var registration struct { + Count int `json:"count"` + Items []struct { + Count int `json:"count"` + Upper string `json:"upper"` + Items []json.RawMessage `json:"items"` + } `json:"items"` + } + if err := json.Unmarshal(nugetGet(t, routes, indexPath, http.StatusOK).Body.Bytes(), ®istration); err != nil { + t.Fatal(err) + } + if registration.Count != pages-1 || len(registration.Items) != pages-1 { + t.Fatalf("registration has %d pages and count %d, want %d", len(registration.Items), registration.Count, pages-1) + } + last := registration.Items[pages-2] + if last.Count != leavesPerPage || len(last.Items) != leavesPerPage || last.Upper != fmt.Sprintf("1.%d.0", (pages-1)*leavesPerPage-1) { + t.Fatalf("last page has %d leaves, count %d and upper %q", len(last.Items), last.Count, last.Upper) + } + + const runs = 5 + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + for range runs { + response := &discardResponse{header: http.Header{}} + routes.ServeHTTP(response, httptest.NewRequest(http.MethodGet, indexPath, nil)) + if response.status != http.StatusOK || response.bytes < registrationBytes/2 { + t.Fatalf("status %d with %d bytes", response.status, response.bytes) + } + } + runtime.ReadMemStats(&after) + + perRun := (after.TotalAlloc - before.TotalAlloc) / runs + t.Logf("allocated %d bytes per request for a %d byte registration", perRun, registrationBytes) + // One copy is read from the cache. Building the document took eleven more. + if perRun > 3*uint64(registrationBytes) { + t.Errorf("registration allocated %d bytes for a %d byte registration", perRun, registrationBytes) + } +} diff --git a/internal/handler/nuget_test.go b/internal/handler/nuget_test.go index 5ae77dc..579c925 100644 --- a/internal/handler/nuget_test.go +++ b/internal/handler/nuget_test.go @@ -1,6 +1,7 @@ package handler import ( + "bytes" "encoding/json" "io" "log/slog" @@ -784,6 +785,25 @@ func TestNuGetBuildUpstreamURLRegularPath(t *testing.T) { } } +// filterNuGetRegistration returns the registration as it is served for the +// package id. +func filterNuGetRegistration(t *testing.T, h *NuGetHandler, registration map[string]any, id string) map[string]any { + t.Helper() + document, err := json.Marshal(registration) + if err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := h.writeNuGetRegistration(&out, document, id, h.nugetCooldownApplies(id), nil); err != nil { + t.Fatal(err) + } + var filtered map[string]any + if err := json.Unmarshal(out.Bytes(), &filtered); err != nil { + t.Fatal(err) + } + return filtered +} + func TestNuGetCooldownFiltering(t *testing.T) { now := time.Now() oldTime := now.Add(-7 * 24 * time.Hour).Format(time.RFC3339) @@ -823,11 +843,10 @@ func TestNuGetCooldownFiltering(t *testing.T) { proxyURL: "http://localhost:8080", } - if !h.filterNuGetRegistration(registration, "") { + pages := filterNuGetRegistration(t, h, registration, "TestPackage")["items"].([]any) + if len(pages) != 1 { t.Fatal("expected registration items to be retained") } - - pages := registration["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) @@ -841,7 +860,7 @@ func TestNuGetCooldownFiltering(t *testing.T) { } count := page["count"] - if count != 1 { + if count != float64(1) { t.Errorf("expected page count to be 1, got %v", count) } } @@ -878,11 +897,10 @@ func TestNuGetCooldownFilteringWithPackageOverride(t *testing.T) { proxyURL: "http://localhost:8080", } - if !h.filterNuGetRegistration(registration, "") { + pages := filterNuGetRegistration(t, h, registration, "SpecialPackage")["items"].([]any) + if len(pages) != 1 { t.Fatal("expected registration items to be retained") } - - pages := registration["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) @@ -915,11 +933,10 @@ func TestNuGetCooldownNoCooldownConfig(t *testing.T) { proxyURL: "http://localhost:8080", } - if !h.filterNuGetRegistration(registration, "") { + pages := filterNuGetRegistration(t, h, registration, "Test")["items"].([]any) + if len(pages) != 1 { t.Fatal("expected registration items to be retained") } - - pages := registration["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) @@ -961,11 +978,10 @@ func TestNuGetCooldownFilteringNuGetTimestamp(t *testing.T) { proxyURL: "http://localhost:8080", } - if !h.filterNuGetRegistration(registration, "") { + pages := filterNuGetRegistration(t, h, registration, "Test")["items"].([]any) + if len(pages) != 1 { t.Fatal("expected registration items to be retained") } - - pages := registration["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) From 00ae3d3d156a77e1d12f29e8c1b5b258fd5f472b Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Mon, 9 Nov 2026 23:46:00 +0100 Subject: [PATCH 191/198] Give a page whose last leaf has no version a null upper bound A kept leaf without a version after one with a version left an empty, non-nil version behind, and writing the page's upper bound from it panicked. --- internal/handler/nuget_registration.go | 4 +++- internal/handler/nuget_registration_test.go | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/handler/nuget_registration.go b/internal/handler/nuget_registration.go index dafa8d5..a15d279 100644 --- a/internal/handler/nuget_registration.go +++ b/internal/handler/nuget_registration.go @@ -515,8 +515,10 @@ func (rw *nugetRegistrationWriter) endNode(n *nugetNode) error { return nil } +// nugetVersionJSON returns the version of a leaf as JSON. A leaf without one +// leaves an empty version behind, which is not always nil. func nugetVersionJSON(version []byte) []byte { - if version == nil { + if len(version) == 0 { return []byte("null") } return version diff --git a/internal/handler/nuget_registration_test.go b/internal/handler/nuget_registration_test.go index aef46cc..61b0cbc 100644 --- a/internal/handler/nuget_registration_test.go +++ b/internal/handler/nuget_registration_test.go @@ -141,6 +141,14 @@ func TestNuGetRegistrationDocumentCooldown(t *testing.T) { status: http.StatusOK, want: `{"count":1,"items":[{"catalogEntry":{"published":"` + old + `"}}],"lower":null,"upper":null}`, }, + { + name: "leaf without a version after one with a version", + path: page, + document: `{"items":[` + nugetTestLeaf("", "1.0.0", old) + `,{"catalogEntry":{"published":"` + old + `"}}],"lower":"0"}`, + status: http.StatusOK, + want: `{"count":2,"items":[` + nugetTestLeaf(base, "1.0.0", old) + `,{"catalogEntry":{"published":"` + old + `"}}],` + + `"lower":"1.0.0","upper":null}`, + }, { name: "leaf without a publish time is kept", path: page, From 8e6a87a068cc1a047ff6103edeac56539b82bd5d Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Mon, 9 Nov 2026 19:24:44 -0500 Subject: [PATCH 192/198] fix(oci): serve HEAD requests from cache --- internal/handler/container_test.go | 18 +++++++++++++----- internal/handler/handler.go | 12 ++++++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index fd20e38..9ba1535 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -741,6 +741,7 @@ func TestContainerHandler_BlobHead_CacheHitSkipsUpstreamAndAuth(t *testing.T) { } req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil) + req = req.WithContext(ServingHead(req.Context())) w := httptest.NewRecorder() h.Routes().ServeHTTP(w, req) @@ -764,7 +765,7 @@ func TestContainerHandler_BlobHead_CacheHitSkipsUpstreamAndAuth(t *testing.T) { } } -func TestContainerHandler_BlobHead_DirectServeRedirects(t *testing.T) { +func TestContainerHandler_BlobHead_DirectServeAnswersFromCache(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") @@ -778,14 +779,21 @@ func TestContainerHandler_BlobHead_DirectServeRedirects(t *testing.T) { } req := httptest.NewRequest(http.MethodHead, "/library/nginx/blobs/"+digest, nil) + req = req.WithContext(ServingHead(req.Context())) w := httptest.NewRecorder() h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusFound { - t.Fatalf("status = %d, want %d", w.Code, http.StatusFound) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", w.Code, http.StatusOK) } - if got := w.Header().Get("Location"); got != store.signedURL { - t.Errorf("Location = %q, want %q", got, store.signedURL) + if got := w.Header().Get("Location"); got != "" { + t.Errorf("Location = %q, want empty", got) + } + if got := w.Header().Get("Docker-Content-Digest"); got != digest { + t.Errorf("Docker-Content-Digest = %q, want %q", got, digest) + } + if got := w.Header().Get("Content-Length"); got != "11" { + t.Errorf("Content-Length = %q, want %q", got, "11") } wantETag := `"` + sha256Hex("cached blob") + `"` if got := w.Header().Get("ETag"); got != wantETag { diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9612e51..bf9eed4 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -556,6 +556,12 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s storagePath: artifact.StoragePath, } + if servingHead(ctx) { + result.rangeable = readsSeekable(p.Storage) + p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) + return result, nil + } + if p.DirectServe { signed, err := p.Storage.SignedURL(ctx, artifact.StoragePath, p.DirectServeTTL) if err == nil { @@ -569,12 +575,6 @@ func (p *Proxy) readStored(ctx context.Context, pkgPURL, versionPURL, filename s } } - if servingHead(ctx) { - result.rangeable = readsSeekable(p.Storage) - p.recordCacheHit(artifact.Ecosystem, versionPURL, filename) - return result, nil - } - start := time.Now() reader, err := p.Storage.Open(ctx, artifact.StoragePath) metrics.RecordStorageOperation("read", time.Since(start)) From 0fd4b5a83f760d7eee579711f1a16412c6487a32 Mon Sep 17 00:00:00 2001 From: montehurd Date: Mon, 9 Nov 2026 18:13:48 -0800 Subject: [PATCH 193/198] Keep npm packuments in one format across cooldown profiles Cooldown profiles and the unprefixed URLs share a packument's cache row, but each chose its format by its own policy: abbreviated without cooldown, full with it. With cooldown on for only some of them, a request without cooldown that revalidated the row stored the abbreviated packument. The next request under cooldown rejected it and fetched the full one again, even within metadata_ttl, and had no stale copy to serve when the upstream failed. Cooldown in the global policy or in any profile now makes all of them request the full packument, as upstream.npm_full_metadata does. URLs without cooldown are then served full packuments too. --- docs/configuration.md | 2 +- internal/handler/handler.go | 4 ++ internal/handler/npm.go | 5 +- internal/handler/npm_packument_accept_test.go | 50 +++++++++++++++++++ internal/handler/profile.go | 14 +++++- internal/handler/profile_test.go | 2 +- 6 files changed, 72 insertions(+), 5 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d887509..532176e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1198,7 +1198,7 @@ The prefix works with all package protocol routes. Cooldown has an effect only f Profile names contain lowercase letters, digits, hyphens and underscores, and start with a letter or digit. Profiles must be configured before startup; an unknown profile returns 404. Nested profiles are rejected. Profile paths expose package protocols only, so `/cooldown/3d/ui/` and `/cooldown/3d/api/` return 404. -Database records, artifacts and upstream metadata are shared. Generated download URLs retain the profile prefix, and filtered metadata and its validators are separated by profile. Changing a registry URL does not rewrite artifact URLs already recorded in a lockfile; regenerate those URLs if the selected policy must cover the whole installation. +Database records, artifacts and upstream metadata are shared. Generated download URLs retain the profile prefix, and filtered metadata and its validators are separated by profile. Because npm packuments are shared too, cooldown in the global policy or in any profile makes every URL fetch and serve full packuments, as `upstream.npm_full_metadata` does. Changing a registry URL does not rewrite artifact URLs already recorded in a lockfile; regenerate those URLs if the selected policy must cover the whole installation. ## Cooldown diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 9612e51..0c98a7d 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -305,6 +305,10 @@ type Proxy struct { // holds back; see cooldown_expiry.go. cooldownWatch cooldownWatch shared *Proxy + + // viewCooldown is set on the shared proxy once a view of it has cooldown + // on; see anyCooldownEnabled. + viewCooldown atomic.Bool } // cooldownEcosystems are the ecosystems whose handlers apply cooldown. Others diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 7eef424..24942b4 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -553,9 +553,10 @@ func readStored(open func() (io.ReadCloser, error), read func(io.Reader) error) // // Every request for a packument uses it, since they share one cache row: a // request with another Accept would replace the document the others are -// served with one in the other format. +// served with one in the other format. Cooldown profiles share that row too, +// so cooldown in any of them asks for full metadata in all of them. func (h *NPMHandler) packumentAccept() string { - if h.proxy.NPMFullMetadata || h.proxy.cooldownEnabled() { + if h.proxy.NPMFullMetadata || h.proxy.anyCooldownEnabled() { return contentTypeJSON } return npmAcceptDefault diff --git a/internal/handler/npm_packument_accept_test.go b/internal/handler/npm_packument_accept_test.go index 1a87fdd..07f4e63 100644 --- a/internal/handler/npm_packument_accept_test.go +++ b/internal/handler/npm_packument_accept_test.go @@ -208,3 +208,53 @@ func TestNPMCooldownSkipsAbbreviatedStaleFallback(t *testing.T) { t.Errorf("served the abbreviated packument as a stale copy: %s", w.Body.String()) } } + +// Cooldown profiles share a packument's cache row with the unprefixed URLs. +// With cooldown on for only some of them, the others must not replace the +// full packument with the abbreviated one: the next request under cooldown +// would fetch it again, and have no stale copy when upstream fails. +func TestNPMCooldownProfilesShareFullPackument(t *testing.T) { + for _, tc := range []struct{ name, global, profile string }{ + {"cooldown in the profile", "0", "7d"}, + {"cooldown in the global policy", "7d", "0"}, + } { + t.Run(tc.name, func(t *testing.T) { + var fail atomic.Bool + var fullRequests atomic.Int64 + upstream := abbreviatedThenFullRegistry(t, &fail, &fullRequests) + proxy, db, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Nanosecond // every request revalidates + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = profilePolicy(t, tc.global) + global := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + profile := NewNPMHandler(proxy.WithCooldown(profilePolicy(t, tc.profile), 0), "http://proxy.test/cooldown/p", upstream.URL) + filtered, unfiltered := profile, global + if tc.global != "0" { + filtered, unfiltered = global, profile + } + + for _, h := range []*NPMHandler{filtered, unfiltered} { + if w := serveNPM(t, h, http.MethodGet, "/demo", nil, nil); w.Code != http.StatusOK { + t.Fatalf("metadata: status %d: %s", w.Code, w.Body.String()) + } + } + entry, err := db.GetMetadataCache("npm", "demo") + if err != nil || entry == nil { + t.Fatalf("cache row: %v, %v", entry, err) + } + if entry.ContentType.String != contentTypeJSON { + t.Errorf("cached packument Content-Type = %q, want %q", entry.ContentType.String, contentTypeJSON) + } + + fail.Store(true) + w := serveNPM(t, filtered, http.MethodGet, "/demo", nil, nil) + if w.Code != http.StatusOK { + t.Fatalf("metadata while upstream fails: status %d, want the stale full packument", w.Code) + } + if strings.Contains(w.Body.String(), "demo-2.0.0.tgz") { + t.Errorf("stale packument lists a version in cooldown: %s", w.Body.String()) + } + }) + } +} diff --git a/internal/handler/profile.go b/internal/handler/profile.go index 233e7bb..ceff0a3 100644 --- a/internal/handler/profile.go +++ b/internal/handler/profile.go @@ -5,7 +5,7 @@ import "time" // WithCooldown creates a policy view sharing storage, fetches and rewrite space. // Configure the original proxy before constructing its views. func (p *Proxy) WithCooldown(policy CooldownPolicy, shortest time.Duration) *Proxy { - return &Proxy{ + view := &Proxy{ DB: p.DB, Storage: p.Storage, Fetcher: p.Fetcher, Resolver: p.Resolver, RouteFor: p.RouteFor, Logger: p.Logger, Cooldown: policy, ShortestCooldown: shortest, Denylist: p.Denylist, @@ -20,6 +20,10 @@ func (p *Proxy) WithCooldown(policy CooldownPolicy, shortest time.Duration) *Pro Scanners: p.Scanners, ScanSigningKey: p.ScanSigningKey, ScanFetchBaseURL: p.ScanFetchBaseURL, rewrites: p.rewrites, now: p.now, started: p.started, shared: p.sharedProxy(), } + if view.cooldownEnabled() { + view.shared.viewCooldown.Store(true) + } + return view } func (p *Proxy) sharedProxy() *Proxy { @@ -28,3 +32,11 @@ func (p *Proxy) sharedProxy() *Proxy { } return p } + +// anyCooldownEnabled reports whether cooldown is on for this proxy, the one +// it is a view of, or another view of that one. What they keep in one cache +// row has to suit all of them. +func (p *Proxy) anyCooldownEnabled() bool { + shared := p.sharedProxy() + return p.cooldownEnabled() || shared.cooldownEnabled() || shared.viewCooldown.Load() +} diff --git a/internal/handler/profile_test.go b/internal/handler/profile_test.go index 5caf7b8..ca163ee 100644 --- a/internal/handler/profile_test.go +++ b/internal/handler/profile_test.go @@ -61,7 +61,7 @@ func TestWithCooldownCopiesProxyFields(t *testing.T) { switch name { case "Cooldown", "ShortestCooldown", "shared": continue - case "fetchMu", "inFlight", "detachSlots", "metaMu", "inFlightMeta", "publishIndexes", "cooldownWatch": + case "fetchMu", "inFlight", "detachSlots", "metaMu", "inFlightMeta", "publishIndexes", "cooldownWatch", "viewCooldown": if !after.IsZero() { t.Errorf("view field %s is not empty", name) } From 16a8ae23faee3e05b47d0390523acb525d39c772 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 10 Nov 2026 07:34:06 +0100 Subject: [PATCH 194/198] Read a NuGet registration for the cooldown as it is copied The cooldown read a registration with encoding/json into a struct and the copy read it again token by token. The two disagreed on documents with member names in another case and with repeated members: the struct matched Items and Published, and kept the first or merged where the copy took what it saw. Such a document could be served with releases that are in cooldown, by the registration endpoints, the version list and the download check, or fail after part of an index was sent. The cooldown now reads with the copy's decoder and its rules: exact names, as main matched them. A node that repeats @id, items, catalogEntry, version or published is refused. --- internal/handler/nuget_cooldown.go | 184 ++++++++++++++++---- internal/handler/nuget_registration.go | 9 +- internal/handler/nuget_registration_test.go | 90 +++++++++- internal/handler/nuget_times_test.go | 50 ++++++ 4 files changed, 295 insertions(+), 38 deletions(-) diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go index 6bc2794..2585856 100644 --- a/internal/handler/nuget_cooldown.go +++ b/internal/handler/nuget_cooldown.go @@ -7,8 +7,10 @@ import ( "context" "crypto/sha256" "encoding/json" + "encoding/json/jsontext" "errors" "fmt" + "io" "net/http" "net/url" "slices" @@ -86,15 +88,17 @@ func decodeNuGetDocument(body []byte) (map[string]any, error) { return document, nil } -// nugetTimes holds the fields of a registration index, page or leaf that decide -// which versions are in cooldown. Decoding into it skips everything else, which -// is nearly all of a registration. +// nugetTimes holds the members of a registration index, page or leaf that +// decide which versions are in cooldown. Reading only them skips nearly all of +// a registration. type nugetTimes struct { - ID string `json:"@id"` - Version string `json:"version"` - Published string `json:"published"` - CatalogEntry nugetCatalogTimes `json:"catalogEntry"` - Items []nugetTimes `json:"items"` + // ID is the node's link. It is kept for the pages of an index only. + ID string + Version string + Published string + CatalogEntry nugetCatalogTimes + // Items is nil unless the node has an items array. + Items []nugetTimes } // nugetCatalogTimes is the catalogEntry of a registration leaf. It stays unset @@ -105,22 +109,6 @@ type nugetCatalogTimes struct { published string } -func (e *nugetCatalogTimes) UnmarshalJSON(data []byte) error { - if !bytes.HasPrefix(data, []byte("{")) { - return nil - } - var fields struct { - Version string `json:"version"` - Published string `json:"published"` - } - var mismatch *json.UnmarshalTypeError - if err := json.Unmarshal(data, &fields); err != nil && !errors.As(err, &mismatch) { - return err - } - *e = nugetCatalogTimes{set: true, version: fields.Version, published: fields.Published} - return nil -} - // release returns the version and publish time of a leaf. A leaf embedded in a // page carries them in catalogEntry, a standalone leaf at its root. func (t *nugetTimes) release() (version, published string) { @@ -153,20 +141,146 @@ func (h *NuGetHandler) nugetTimesDocument(ctx context.Context, path string) (*nu return document, body, nil } -// decodeNuGetTimes accepts the documents decodeNuGetDocument does. A field of -// an unexpected type is left empty, as it is when read from the generic document. +// decodeNuGetTimes reads a registration document the way writeNuGetRegistration +// copies it, so that what is decided here holds for what is written there: +// with the same decoder, members by their exact names, and a member of an +// unexpected type left empty. A node that has one of these members twice is +// refused, as which of the two counts would be the reader's choice. func decodeNuGetTimes(body []byte) (*nugetTimes, error) { - var document *nugetTimes - if err := json.Unmarshal(body, &document); err != nil { - var mismatch *json.UnmarshalTypeError - if !errors.As(err, &mismatch) || mismatch.Field == "" { - return nil, fmt.Errorf("parsing NuGet metadata: %w", err) - } - } - if document == nil { + dec := newNuGetDecoder(body) + if dec.PeekKind() == 'n' { return nil, fmt.Errorf("empty NuGet metadata") } - return document, nil + document := &nugetTimes{} + var scratch []byte + var err error + if dec.PeekKind() == '{' { + err = document.decode(dec, 0, &scratch) + } else if _, err = dec.ReadValue(); err == nil { + err = errors.New("not an object") + } + if err == nil { + // Nothing may follow the document. + if _, err = dec.ReadToken(); errors.Is(err, io.EOF) { + return document, nil + } else if err == nil { + err = errors.New("data after the document") + } + } + return nil, fmt.Errorf("parsing NuGet metadata: %w", err) +} + +// decode reads the node dec is at, which is depth levels below the document. +func (t *nugetTimes) decode(dec *jsontext.Decoder, depth int, scratch *[]byte) error { + if _, err := dec.ReadToken(); err != nil { + return err + } + seen := 0 + for dec.PeekKind() != '}' { + kind, err := readNuGetMember(dec, &seen, scratch) + if err != nil { + return err + } + switch value := dec.PeekKind(); { + case kind == nugetMemberItems && value == '[': + err = t.decodeItems(dec, depth, scratch) + case kind == nugetMemberCatalogEntry && value == '{': + err = t.CatalogEntry.decode(dec, scratch) + case kind == nugetMemberID && value == '"' && depth == 1: + t.ID, err = readNuGetString(dec, scratch) + case kind == nugetMemberVersion && value == '"': + t.Version, err = readNuGetString(dec, scratch) + case kind == nugetMemberPublished && value == '"': + t.Published, err = readNuGetString(dec, scratch) + default: + err = dec.SkipValue() + } + if err != nil { + return err + } + } + _, err := dec.ReadToken() + return err +} + +// decodeItems reads the items array dec is at. A child that is not an object +// is an empty node. +func (t *nugetTimes) decodeItems(dec *jsontext.Decoder, depth int, scratch *[]byte) error { + if _, err := dec.ReadToken(); err != nil { + return err + } + t.Items = []nugetTimes{} + for dec.PeekKind() != ']' { + t.Items = append(t.Items, nugetTimes{}) + var err error + if dec.PeekKind() == '{' { + err = t.Items[len(t.Items)-1].decode(dec, depth+1, scratch) + } else { + err = dec.SkipValue() + } + if err != nil { + return err + } + } + _, err := dec.ReadToken() + return err +} + +func (e *nugetCatalogTimes) decode(dec *jsontext.Decoder, scratch *[]byte) error { + if _, err := dec.ReadToken(); err != nil { + return err + } + e.set = true + seen := 0 + for dec.PeekKind() != '}' { + kind, err := readNuGetMember(dec, &seen, scratch) + if err != nil { + return err + } + switch value := dec.PeekKind(); { + case kind == nugetMemberVersion && value == '"': + e.version, err = readNuGetString(dec, scratch) + case kind == nugetMemberPublished && value == '"': + e.published, err = readNuGetString(dec, scratch) + default: + err = dec.SkipValue() + } + if err != nil { + return err + } + } + _, err := dec.ReadToken() + return err +} + +// nugetDecidingMembers are the members that decide what a node is and whether +// its release is in cooldown. +const nugetDecidingMembers = 1< Date: Tue, 10 Nov 2026 11:56:20 +0000 Subject: [PATCH 195/198] Limit refreshed index documents to the metadata size limit InRelease, repomd.xml and APKINDEX.tar.gz are read into memory, so cap them at metadata_max_size as the metadata cache did. --- internal/mirror/refresh_harden_test.go | 21 +++++++++++++++++++++ internal/mirror/refresh_index.go | 5 +++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/internal/mirror/refresh_harden_test.go b/internal/mirror/refresh_harden_test.go index f33a833..0af855d 100644 --- a/internal/mirror/refresh_harden_test.go +++ b/internal/mirror/refresh_harden_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/registries" "github.com/git-pkgs/registries/fetch" ) @@ -310,6 +311,26 @@ func TestRefreshRefusesAPKIndexesOverTheSizeLimit(t *testing.T) { } } +// TestRefreshRefusesIndexDocumentsOverTheMetadataLimit checks that an +// InRelease larger than a metadata document may be fails its index. +func TestRefreshRefusesIndexDocumentsOverTheMetadataLimit(t *testing.T) { + m, fetcher, idx := setupIndexRefresh(t) + release := "SHA256:\n 0000 1 main/binary-amd64/Packages.gz\n" + idx.set("/debian/dists/bookworm/InRelease", []byte(release)) + idx.set("/debian/dists/bookworm/main/binary-amd64/Packages.gz", debCurlPackages(t, "7.88.1-10+deb12u8")) + clientFetchesCurl(t, m, "debian", idx.URL+"/debian/dists/bookworm/InRelease", + "deb", "7.88.1-10", "curl_7.88.1-10_amd64.deb", idx.URL+"/debian/pool/main/c/curl/curl_7.88.1-10_amd64.deb") + + m.proxy.MetadataMaxSize = int64(len(release)) + idx.set("/debian/dists/bookworm/InRelease", []byte(release+" 0000 1 contrib/binary-amd64/Packages.gz\n")) + fetcher.reset() + + progress := refreshOnce(t, m) + if progress.Failed != 1 || len(progress.Errors) != 1 || !strings.Contains(progress.Errors[0].Error, handler.ErrMetadataTooLarge.Error()) || len(fetcher.fetched()) != 0 { + t.Errorf("progress = %+v, fetched %v; want the index failing for its size", progress, fetcher.fetched()) + } +} + // TestRefreshDebianSecurityPool checks the security archive's layout, whose // pool is pool/updates/main while its InRelease lists main/binary-*. func TestRefreshDebianSecurityPool(t *testing.T) { diff --git a/internal/mirror/refresh_index.go b/internal/mirror/refresh_index.go index 890ebd6..c54224e 100644 --- a/internal/mirror/refresh_index.go +++ b/internal/mirror/refresh_index.go @@ -274,14 +274,15 @@ func rpmIndexFiles(ctx context.Context, m *Mirror, indexURL string, body []byte, // readIndexDocument reads the index document a refresh starts from directly // from upstream. Read through the metadata cache it would count as a client -// fetching it, and the suite would never age out of since. +// fetching it, and the suite would never age out of since. It is held in +// memory, so it is limited like any metadata document. func (m *Mirror) readIndexDocument(ctx context.Context, indexURL string) ([]byte, error) { body, err := m.openIndex(ctx, indexURL) if err != nil { return nil, err } defer func() { _ = body.Close() }() - return io.ReadAll(&indexLimitReader{r: body, url: indexURL, remaining: maxIndexBytes}) + return m.proxy.ReadMetadata(body) } // openIndex starts reading an index from upstream. From 3e50f45b0d4fb3ead542b1ada48c8639eba7c06d Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 10 Nov 2026 13:03:00 +0100 Subject: [PATCH 196/198] Refuse Conan file requests with an encoded slash A path value is decoded, so a revision of R%2Fpackages%2FP%2FQ gave a recipe file the cache key of a package file. The recipe and package file routes now answer 400 when user, channel, a revision, the package reference or the filename contains a slash. --- internal/handler/conan.go | 23 ++++++++++++++++++++--- internal/handler/conan_test.go | 30 ++++++++++++++++++++++++++++++ internal/handler/handler.go | 7 ++----- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/internal/handler/conan.go b/internal/handler/conan.go index 9a07f24..90edcdb 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -2,6 +2,7 @@ package handler import ( "net/http" + "slices" "strings" "github.com/git-pkgs/proxy/internal/retention" @@ -79,6 +80,11 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request) revision := r.PathValue("revision") filename := r.PathValue("filename") + if conanPartHasSlash(user, channel, revision, filename) { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + // Only the archives are artifacts; the rest goes through the metadata cache. if !h.shouldCacheFile(filename) { h.proxyCached(w, r) @@ -117,6 +123,11 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) pkgrev := r.PathValue("pkgrev") filename := r.PathValue("filename") + if conanPartHasSlash(user, channel, revision, pkgref, pkgrev, filename) { + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + // Only the archives are artifacts; the rest goes through the metadata cache. if !h.shouldCacheFile(filename) { h.proxyCached(w, r) @@ -146,13 +157,19 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) // conanStorageFilename names a cached archive. Archives are stored under // pkg:conan/{name}@{version}, so the file name carries user, channel and the -// revisions, to keep every recipe and binary apart. The parts are path -// segments, which can't contain a slash, so joining them with one keeps -// different references apart. +// revisions, to keep every recipe and binary apart. The handlers refuse a +// part that contains a slash, so joining them with one keeps different +// references apart. func conanStorageFilename(user, channel string, parts ...string) string { return strings.Join(append([]string{user, channel}, parts...), "/") } +// conanPartHasSlash reports whether a path value contains a slash, which it +// can only get from an encoded one (%2F) in the request. +func conanPartHasSlash(parts ...string) bool { + return slices.ContainsFunc(parts, func(part string) bool { return strings.Contains(part, "/") }) +} + // conanLegacyStorageFilename is the name archives were cached under before // their parts were joined with slashes. Underscores, which revisions, user // and channel can contain, joined them, so it is only read for the URL an diff --git a/internal/handler/conan_test.go b/internal/handler/conan_test.go index 5be99fe..200f867 100644 --- a/internal/handler/conan_test.go +++ b/internal/handler/conan_test.go @@ -723,3 +723,33 @@ func TestConanLegacyFileEntryServedForItsURL(t *testing.T) { t.Errorf("recipe file with the upstream down = %d %q, want the legacy entry", w.Code, w.Body.String()) } } + +// A path value is decoded, so one holding an encoded slash could spell the +// cache key of another reference. Those requests are refused. +func TestConanFileRoutesRefuseEncodedSlash(t *testing.T) { + var available atomic.Bool + available.Store(true) + upstream := conanPathEchoServer(t, &available) + proxy, _, _, _ := setupTestProxy(t) + proxy.Fetcher = fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) + + if w := serveConanRequest(h, conanPackageFile); w.Code != http.StatusOK || w.Body.String() != conanPackageFile { + t.Fatalf("GET %s = %d %q, want its own file", conanPackageFile, w.Code, w.Body.String()) + } + + const recipe, pkg = "/v2/conans/zlib/1.3/_/_/revisions/", "/v2/conans/zlib/1.3/_/_/revisions/R/packages/" + for field, target := range map[string]string{ + "revision spelling a package file": recipe + "R%2Fpackages%2FP%2FQ/files/conan_package.tgz", + "user": "/v2/conans/zlib/1.3/a%2Fb/_/revisions/R/files/conan_package.tgz", + "channel": "/v2/conans/zlib/1.3/_/a%2Fb/revisions/R/files/conan_package.tgz", + "recipe filename": recipe + "R/files/a%2Fconan_package.tgz", + "package reference": pkg + "P%2FX/revisions/Q/files/conan_package.tgz", + "package revision": pkg + "P/revisions/Q%2FX/files/conan_package.tgz", + "package filename": pkg + "P/revisions/Q/files/a%2Fconan_package.tgz", + } { + if w := serveConanRequest(h, target); w.Code != http.StatusBadRequest { + t.Errorf("%s: GET %s = %d %q, want 400", field, target, w.Code, w.Body.String()) + } + } +} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 62a49e8..9e8b565 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -2396,11 +2396,8 @@ func (p *Proxy) GetOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, return p.getOrFetchArtifactFromURL(ctx, ecosystem, name, version, filename, downloadURL, nil, "", "") } -// cacheFilenameKeepingLegacy returns the filename to cache an artifact -// fetched from downloadURL under: legacy, an earlier cache filename that more -// than one URL could share, while an artifact recorded for downloadURL is -// cached under it, and filename otherwise. It keeps entries cached before a -// key change in use, without serving one to a URL it wasn't fetched from. +// cacheFilenameKeepingLegacy returns legacy when an artifact recorded under +// it was fetched from downloadURL, and filename otherwise. func (p *Proxy) cacheFilenameKeepingLegacy(ecosystem, name, version, filename, legacy, downloadURL string) (string, error) { _, versionPURL, err := packagePURLStrings(ecosystem, name, version) if err != nil { From b731648123065f64339b1520661d3394308d5025 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Tue, 10 Nov 2026 16:33:21 +0100 Subject: [PATCH 197/198] Add storage.retention.keep_latest to keep the newest versions Retention evicts every file that has not been served for the configured time, also the newest version of a package. keep_latest: N holds back the files of the N highest cached versions of each package from eviction by age. Versions are ordered by the ecosystem's version rules. It applies to cargo, golang, npm and pypi, where the cache stores a release under its version. Other ecosystems ignore it for now. --- config.example.yaml | 3 + docs/configuration.md | 12 + internal/config/config.go | 4 + internal/config/retention.go | 5 + internal/config/retention_test.go | 31 +++ internal/database/retention.go | 16 ++ internal/database/retention_test.go | 43 ++++ internal/handler/cargo.go | 6 +- internal/handler/go.go | 23 +- internal/handler/go_test.go | 24 ++ internal/handler/npm.go | 6 +- internal/handler/pypi.go | 6 +- internal/retention/keep_latest_test.go | 63 +++++ internal/retention/retention.go | 62 +++++ internal/server/retention.go | 91 +++++++- internal/server/retention_keep_latest_test.go | 215 ++++++++++++++++++ internal/server/retention_test.go | 3 +- 17 files changed, 602 insertions(+), 11 deletions(-) create mode 100644 internal/retention/keep_latest_test.go create mode 100644 internal/server/retention_keep_latest_test.go diff --git a/config.example.yaml b/config.example.yaml index 002d506..240e564 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -116,6 +116,9 @@ storage: # "pkg:generic/owner%2Frepo": "90d" # How often to look for expired artifacts; at least "1m". sweep_interval: "10m" + # Keep this many of the newest cached versions of each package from + # eviction by age (cargo, golang, npm and pypi). 0 keeps none. + # keep_latest: 2 # Store fetched artifacts. Set to false to stream every download from # upstream without storing it; metadata filtering, cooldown and the diff --git a/docs/configuration.md b/docs/configuration.md index 9717b1c..1ac89a6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -71,6 +71,7 @@ storage: | `storage.cache_artifacts` | `PROXY_STORAGE_CACHE_ARTIFACTS` | - | Store fetched artifacts (default: true); `false` streams them from upstream | | `storage.retention.default` | `PROXY_STORAGE_RETENTION_DEFAULT` | - | Evict artifacts not served for this long (e.g. "30d"); see [Retention](#retention) | | `storage.retention.sweep_interval` | `PROXY_STORAGE_RETENTION_SWEEP_INTERVAL` | - | How often expired artifacts are looked for (default: "10m", at least "1m") | +| `storage.retention.keep_latest` | - | - | Keep this many of the newest cached versions of each package from eviction by age (default: 0); see [Retention](#retention) | `storage.max_size` counts cached artifacts and cached metadata documents (`cache_metadata`). When the total goes over the limit, the least recently served entries of either kind are evicted first; an evicted metadata document is fetched from upstream again on the next request for it. Pushed artifacts are not counted or evicted. An artifact replaced by a refetch stays in storage for at least an hour, or `storage.direct_serve_ttl` if longer, so requests already reading it can finish, and storage use can exceed the limit by what was replaced in that time. @@ -117,6 +118,17 @@ Each cached file ages on its own, counted from the last time the proxy served it A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version, written in one of the forms the table below lists for the ecosystem. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems, except `generic`, which requires an explicit rule. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention supports (`covered`) and which ones `default` does not reach (`default_not_applied`), including `generic`. +`keep_latest` keeps the newest versions of each package, however long ago they were served: + +```yaml +storage: + retention: + default: "30d" + keep_latest: 2 +``` + +With this, the files of the two highest cached versions of a package are never evicted by age, and older versions expire by their rule as before. Versions are ordered by the ecosystem's version rules, not by when they were fetched, and a prerelease counts like any other version. Only versions in the cache count, not those upstream, so the kept ones need not be the latest releases. `keep_latest` applies to `cargo`, `golang`, `npm` and `pypi`; the other ecosystems ignore it, since many of them cache files under a digest or a build rather than a version. On startup the proxy logs the ecosystems it reaches (`keep_latest_covered`). `storage.max_size` can still evict a kept version. + Generic retention is opt-in because release assets may disappear upstream and cannot then be downloaded again after eviction. An existing `default` never starts evicting generic downloads. Set `ecosystems.generic: "30d"` to cover all generic artifacts, or configure only selected GitHub repositories: ```yaml diff --git a/internal/config/config.go b/internal/config/config.go index 6f8171f..fe92465 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -559,6 +559,10 @@ type RetentionConfig struct { // SweepInterval is how often expired artifacts are looked for, at least // one minute. Default: "10m". SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"` + + // KeepLatest is how many of the highest cached versions of each package + // are never evicted by age, in the ecosystems that support it. Default: 0. + KeepLatest int `json:"keep_latest" yaml:"keep_latest"` } // GradleConfig configures Gradle-specific features. diff --git a/internal/config/retention.go b/internal/config/retention.go index 9493d28..ee78560 100644 --- a/internal/config/retention.go +++ b/internal/config/retention.go @@ -67,6 +67,11 @@ func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error } } + if r.KeepLatest < 0 { + return retention.Rules{}, fmt.Errorf("invalid storage.retention.keep_latest: must not be negative") + } + rules.KeepLatest = r.KeepLatest + for _, key := range sortedKeys(r.Ecosystems) { field := "storage.retention.ecosystems." + key if !retention.IsKnown(key) { diff --git a/internal/config/retention_test.go b/internal/config/retention_test.go index eb614e8..9eb55e7 100644 --- a/internal/config/retention_test.go +++ b/internal/config/retention_test.go @@ -1,6 +1,8 @@ package config import ( + "os" + "path/filepath" "strings" "testing" "time" @@ -69,6 +71,34 @@ func TestRetentionRulesResolves(t *testing.T) { } } +// keep_latest is read from the config file and carried into the rules. +func TestRetentionKeepLatest(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + content := "storage:\n retention:\n default: \"30d\"\n keep_latest: 2\n" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + rules, err := cfg.RetentionRules(retentionTestRegistry()) + if err != nil { + t.Fatalf("RetentionRules() error = %v", err) + } + if rules.KeepLatest != 2 { + t.Errorf("KeepLatest = %d, want 2", rules.KeepLatest) + } + + rules, err = Default().RetentionRules(retentionTestRegistry()) + if err != nil { + t.Fatalf("RetentionRules() error = %v", err) + } + if rules.KeepLatest != 0 { + t.Errorf("KeepLatest = %d by default, want 0", rules.KeepLatest) + } +} + func TestRetentionRulesErrors(t *testing.T) { tests := []struct { name string @@ -76,6 +106,7 @@ func TestRetentionRulesErrors(t *testing.T) { wantErr string }{ {"bad default", func(r *RetentionConfig) { r.Default = "7x" }, "storage.retention.default"}, + {"negative keep_latest", func(r *RetentionConfig) { r.KeepLatest = -1 }, "storage.retention.keep_latest: must not be negative"}, {"negative default", func(r *RetentionConfig) { r.Default = "-1d" }, "must not be negative"}, {"NaN days", func(r *RetentionConfig) { r.Default = "NaNd" }, "out of range"}, {"infinite days", func(r *RetentionConfig) { r.Default = "Infd" }, "out of range"}, diff --git a/internal/database/retention.go b/internal/database/retention.go index 0036362..a2a59e0 100644 --- a/internal/database/retention.go +++ b/internal/database/retention.go @@ -76,6 +76,22 @@ func (db *DB) GetRetentionCandidates(fromID, toID int64, cutoff time.Time) ([]Re return candidates, nil } +// CachedVersionPURLs returns the versions of a package that have at least one +// file cached from upstream. +func (db *DB) CachedVersionPURLs(packagePURL string) ([]string, error) { + var purls []string + query := db.Rebind(` + SELECT DISTINCT a.version_purl + FROM versions v + JOIN artifacts a ON a.version_purl = v.purl + WHERE v.package_purl = ? AND a.storage_path IS NOT NULL AND a.source <> 'local' + `) + if err := db.Select(&purls, query, packagePURL); err != nil { + return nil, err + } + return purls, nil +} + // ClearExpiredArtifact marks an artifact uncached like ClearArtifactCache, // but only while it is still expired before cutoff, so an artifact served // between the scan and the clear stays cached. It queues storagePath for diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go index 2382011..62582ec 100644 --- a/internal/database/retention_test.go +++ b/internal/database/retention_test.go @@ -429,3 +429,46 @@ func TestArtifactFilenameIndex(t *testing.T) { } }) } + +// CachedVersionPURLs lists each version of a package once while it has a +// file cached from upstream, and leaves out other packages, cleared files +// and pushed artifacts. +func TestCachedVersionPURLs(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + now := time.Now() + seed := func(name, version, filename, source string, cached bool) { + t.Helper() + pkgPURL := "pkg:npm/" + name + versionPURL := pkgPURL + "@" + version + if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: "npm", Name: name}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + art := &Artifact{VersionPURL: versionPURL, Filename: filename, UpstreamURL: "https://example.com/" + filename, Source: source} + if cached { + art.StoragePath = sql.NullString{String: "npm/" + name + "/" + version + "/" + filename, Valid: true} + art.FetchedAt = sql.NullTime{Time: now, Valid: true} + } + if err := db.UpsertArtifact(art); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + } + seed("lib", "1.0.0", "lib-1.0.0.tgz", "", true) + seed("lib", "2.0.0", "lib-2.0.0.tgz", "", true) + seed("lib", "2.0.0", "lib-2.0.0-extra.tgz", "", true) + seed("lib", "3.0.0", "lib-3.0.0.tgz", "", false) + seed("lib", "4.0.0", "lib-4.0.0.tgz", "local", true) + seed("other", "9.0.0", "other-9.0.0.tgz", "", true) + + got, err := db.CachedVersionPURLs("pkg:npm/lib") + if err != nil { + t.Fatalf("CachedVersionPURLs: %v", err) + } + slices.Sort(got) + if want := []string{"pkg:npm/lib@1.0.0", "pkg:npm/lib@2.0.0"}; !slices.Equal(got, want) { + t.Errorf("CachedVersionPURLs() = %v, want %v", got, want) + } + }) +} diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index aca7416..0c74214 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -28,7 +28,11 @@ const ( // they are, so package rules use the default key mapping and have to match // that name exactly. func init() { - retention.Default.Register(retention.Spec{Key: "cargo", CanonicalPackage: retention.DefaultCanonical("cargo")}) + retention.Default.Register(retention.Spec{ + Key: "cargo", + CanonicalPackage: retention.DefaultCanonical("cargo"), + CompareVersions: retention.VersionOrder("cargo"), + }) } // CargoHandler handles cargo registry protocol requests. diff --git a/internal/handler/go.go b/internal/handler/go.go index 08dae57..14af7cf 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -23,7 +23,28 @@ const ( ) func init() { - retention.Default.Register(retention.Spec{Key: "golang", CanonicalPackage: golangRetentionKey}) + retention.Default.Register(retention.Spec{ + Key: "golang", + CanonicalPackage: golangRetentionKey, + CompareVersions: golangCompareVersions, + }) +} + +// golangCompareVersions orders two module versions as the zips are cached, +// which is in the escaped form of the download URL (v1.0.0-!r!c1 for +// v1.0.0-RC1). +func golangCompareVersions(a, b string) int { + order := retention.VersionOrder("golang") + return order(unescapedGoVersion(a), unescapedGoVersion(b)) +} + +// unescapedGoVersion returns the version an escaped one stands for, or the +// input when it is not a valid escaped version. +func unescapedGoVersion(escaped string) string { + if version, err := gomodule.UnescapeVersion(escaped); err == nil { + return version + } + return escaped } func golangRetentionKey(p *purl.PURL) (string, bool) { diff --git a/internal/handler/go_test.go b/internal/handler/go_test.go index ae998c9..514cefc 100644 --- a/internal/handler/go_test.go +++ b/internal/handler/go_test.go @@ -80,3 +80,27 @@ func TestLastComponent(t *testing.T) { } } } + +// Module zips are cached under the escaped version of the download URL, so +// the order for keep_latest unescapes before comparing. +func TestGolangCompareVersions(t *testing.T) { + tests := []struct { + a, b string + want int + }{ + {"v1.10.0", "v1.9.0", 1}, + {"v1.0.0", "v1.0.0-!r!c1", 1}, + {"v1.0.0-!r!c2", "v1.0.0-!r!c1", 1}, + {"v1.0.0-!r!c1", "v1.0.0-beta", -1}, + // Compared as stored, "!" would sort the escaped one below "0rc1". + {"v1.0.0-!r!c1", "v1.0.0-0rc1", 1}, + {"v2.0.0+incompatible", "v1.9.0", 1}, + {"v1.2.3", "v1.2.3", 0}, + } + for _, tt := range tests { + got := golangCompareVersions(tt.a, tt.b) + if (got > 0) != (tt.want > 0) || (got < 0) != (tt.want < 0) { + t.Errorf("golangCompareVersions(%q, %q) = %d, want sign of %d", tt.a, tt.b, got, tt.want) + } + } +} diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 95576b5..3cf3244 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -49,7 +49,11 @@ const ( // npm tarballs are stored under pkg:npm/{name} as purl builds it, so package // rules use the default key mapping. func init() { - retention.Default.Register(retention.Spec{Key: "npm", CanonicalPackage: retention.DefaultCanonical("npm")}) + retention.Default.Register(retention.Spec{ + Key: "npm", + CanonicalPackage: retention.DefaultCanonical("npm"), + CompareVersions: retention.VersionOrder("npm"), + }) } // NPMHandler handles npm registry protocol requests. diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 21e4374..c94e151 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -57,7 +57,11 @@ type PyPIHandler struct { // matches typing-extensions. Dots and runs of separators are kept, so files // named zope.interface-* and zope_interface-* end up under two packages. func init() { - retention.Default.Register(retention.Spec{Key: "pypi", CanonicalPackage: retention.DefaultCanonical("pypi")}) + retention.Default.Register(retention.Spec{ + Key: "pypi", + CanonicalPackage: retention.DefaultCanonical("pypi"), + CompareVersions: retention.VersionOrder("pypi"), + }) } // NewPyPIHandlerWithUpstreams creates a PyPI handler with custom API and diff --git a/internal/retention/keep_latest_test.go b/internal/retention/keep_latest_test.go new file mode 100644 index 0000000..5d26eab --- /dev/null +++ b/internal/retention/keep_latest_test.go @@ -0,0 +1,63 @@ +package retention + +import ( + "slices" + "testing" +) + +func TestNewestVersions(t *testing.T) { + tests := []struct { + name string + key string + versions []string + n int + want []string + }{ + {"npm orders by number, not by text", "npm", []string{"1.9.0", "1.10.0", "1.2.0"}, 2, []string{"1.10.0", "1.9.0"}}, + {"a prerelease is below its release", "npm", []string{"2.0.0-beta.1", "2.0.0", "1.0.0"}, 1, []string{"2.0.0"}}, + {"a prerelease counts as a version", "npm", []string{"2.0.0-beta.1", "1.0.0"}, 1, []string{"2.0.0-beta.1"}}, + {"go pseudo-version is below a tag", "golang", []string{"v0.0.0-20200101000000-abcdefabcdef", "v1.10.0", "v1.9.0"}, 1, []string{"v1.10.0"}}, + {"pypi post release", "pypi", []string{"1.0", "1.0.post1", "0.9"}, 1, []string{"1.0.post1"}}, + {"fewer versions than n", "cargo", []string{"0.1.0", "0.2.0"}, 5, []string{"0.2.0", "0.1.0"}}, + {"zero keeps none", "npm", []string{"1.0.0"}, 0, nil}, + {"no versions", "npm", nil, 2, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + given := slices.Clone(tt.versions) + got := NewestVersions(VersionOrder(tt.key), tt.versions, tt.n) + if !slices.Equal(got, tt.want) { + t.Errorf("NewestVersions() = %v, want %v", got, tt.want) + } + if !slices.Equal(tt.versions, given) { + t.Errorf("NewestVersions() reordered its input to %v", tt.versions) + } + }) + } +} + +// Two spellings of one release count as one version, so neither is left out +// for the other. +func TestNewestVersionsCountsEqualVersionsOnce(t *testing.T) { + want := []string{"1.0.0", "1.0"} + for _, versions := range [][]string{{"1.0", "1.0.0", "0.9"}, {"0.9", "1.0.0", "1.0"}} { + if got := NewestVersions(VersionOrder("pypi"), versions, 1); !slices.Equal(got, want) { + t.Errorf("NewestVersions(%v) = %v, want %v", versions, got, want) + } + } + got := NewestVersions(VersionOrder("pypi"), []string{"1.0", "1.0.0", "0.9", "0.8"}, 2) + if want := []string{"1.0.0", "1.0", "0.9"}; !slices.Equal(got, want) { + t.Errorf("NewestVersions() with n=2 = %v, want %v", got, want) + } +} + +func TestRegistryKeepLatestKeys(t *testing.T) { + reg := NewRegistry() + reg.Register(Spec{Key: "npm", CompareVersions: VersionOrder("npm")}) + reg.Register(Spec{Key: "gem"}) + reg.Register(Spec{Key: "cargo", CompareVersions: VersionOrder("cargo")}) + + if got := reg.KeepLatestKeys(); !slices.Equal(got, []string{"cargo", "npm"}) { + t.Errorf("KeepLatestKeys() = %v, want [cargo npm]", got) + } +} diff --git a/internal/retention/retention.go b/internal/retention/retention.go index 156e410..8379dc5 100644 --- a/internal/retention/retention.go +++ b/internal/retention/retention.go @@ -11,11 +11,13 @@ import ( "fmt" "slices" "sort" + "strings" "sync" "time" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" + "github.com/git-pkgs/vers" ) // KnownKeys lists the ecosystem keys storage.retention.ecosystems accepts. @@ -54,6 +56,23 @@ type Spec struct { // config keeps it in Rules.Sources until then. Nil means the ecosystem // has no such form. SourcePackage func(p *purl.PURL) bool + + // CompareVersions orders two versions as the handler stores them, + // returning a negative number, zero or a positive number as a is lower + // than, the same as or higher than b. With it set, Rules.KeepLatest + // applies to the ecosystem. Nil means the stored versions cannot be + // ordered, as with digests and hashes. + CompareVersions func(a, b string) int +} + +// VersionOrder returns a CompareVersions for an ecosystem whose handler +// stores each release under its plain version number: it compares by the +// version rules of the ecosystem key. +func VersionOrder(key string) func(a, b string) int { + scheme := purl.EcosystemToPURLType(key) + return func(a, b string) int { + return vers.CompareWithScheme(a, b, scheme) + } } // dbEcosystems returns every packages.ecosystem value that belongs to the @@ -149,6 +168,45 @@ func (r *Registry) Keys() []string { return keys } +// KeepLatestKeys returns the registered keys Rules.KeepLatest applies to, +// in sorted order. +func (r *Registry) KeepLatestKeys() []string { + var keys []string + for _, key := range r.Keys() { + if s, _ := r.Lookup(key); s.CompareVersions != nil { + keys = append(keys, key) + } + } + return keys +} + +// NewestVersions returns the versions that are among the n highest under +// compare, highest first. Versions that compare equal, such as two spellings +// of one release, count as one version and are returned together. +func NewestVersions(compare func(a, b string) int, versions []string, n int) []string { + if n <= 0 || len(versions) == 0 { + return nil + } + sorted := slices.Clone(versions) + slices.SortFunc(sorted, func(a, b string) int { + if c := compare(b, a); c != 0 { + return c + } + return strings.Compare(b, a) + }) + distinct := 1 + for i := 1; i < len(sorted); i++ { + if compare(sorted[i-1], sorted[i]) == 0 { + continue + } + if distinct == n { + return sorted[:i] + } + distinct++ + } + return sorted +} + // KeyForDBEcosystem returns the registered key a packages.ecosystem value // belongs to. func (r *Registry) KeyForDBEcosystem(ecosystem string) (string, bool) { @@ -198,6 +256,10 @@ type Rules struct { // Spec.SourcePackage). An ecosystem with any is not swept, since one of // them may keep an artifact the sweep would otherwise clear. Sources map[string]map[string]time.Duration + // KeepLatest is how many of the highest cached versions of each package + // are never evicted by age, in ecosystems whose Spec sets + // CompareVersions. Zero keeps none. + KeepLatest int } // Pending reports whether the ecosystem key has package rules still to be diff --git a/internal/server/retention.go b/internal/server/retention.go index ecd88de..d779849 100644 --- a/internal/server/retention.go +++ b/internal/server/retention.go @@ -9,6 +9,7 @@ import ( "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/purl" ) // retentionWindow is how many artifact ids one query scans. @@ -39,10 +40,15 @@ func (s *Server) startRetentionLoop(ctx context.Context, resolvers map[string]so "default_not_applied", notCovered(covered)) return } - s.logger.Info("retention enabled", + attrs := []any{ "covered", covered, "default_not_applied", notCovered(covered), - "sweep_interval", s.cfg.ParseRetentionSweepInterval()) + "sweep_interval", s.cfg.ParseRetentionSweepInterval(), + } + if rules.KeepLatest > 0 { + attrs = append(attrs, "keep_latest", rules.KeepLatest, "keep_latest_covered", reg.KeepLatestKeys()) + } + s.logger.Info("retention enabled", attrs...) ticker := time.NewTicker(s.cfg.ParseRetentionSweepInterval()) defer ticker.Stop() @@ -107,6 +113,7 @@ func sweepRetention(ctx context.Context, db *database.DB, logger *slog.Logger, // under any, so the query uses that cutoff and each row is then checked // against its own rule. scanCutoff := now.Add(-minimum) + keep := newLatestKeeper(db, logger, reg, rules.KeepLatest) cleared := 0 freed := int64(0) capped := false @@ -128,7 +135,7 @@ func sweepRetention(ctx context.Context, db *database.DB, logger *slog.Logger, capped = true break } - if !clearIfExpired(db, logger, reg, rules, now, c) { + if !clearIfExpired(db, logger, reg, rules, keep, now, c) { continue } metrics.RecordArtifactEvicted("retention", c.Ecosystem) @@ -146,9 +153,10 @@ func sweepRetention(ctx context.Context, db *database.DB, logger *slog.Logger, } // clearIfExpired clears one candidate when its own rule says it has expired -// and queues its object for deletion, reporting whether it did. +// and keep does not hold it back, and queues its object for deletion, +// reporting whether it did. func clearIfExpired(db *database.DB, logger *slog.Logger, reg *retention.Registry, - rules retention.Rules, now time.Time, c database.RetentionCandidate) bool { + rules retention.Rules, keep *latestKeeper, now time.Time, c database.RetentionCandidate) bool { key, ok := reg.KeyForDBEcosystem(c.Ecosystem) if !ok || rules.Pending(key) { return false @@ -158,7 +166,7 @@ func clearIfExpired(db *database.DB, logger *slog.Logger, reg *retention.Registr return false } cutoff := now.Add(-d) - if !c.ExpiredBefore(cutoff) { + if !c.ExpiredBefore(cutoff) || keep.keeps(key, c) { return false } @@ -171,6 +179,77 @@ func clearIfExpired(db *database.DB, logger *slog.Logger, reg *retention.Registr return cleared } +// latestKeeper answers, for one sweep, whether an artifact belongs to one of +// the highest cached versions of its package, which keep_latest holds back +// from eviction by age. +type latestKeeper struct { + db *database.DB + logger *slog.Logger + reg *retention.Registry + n int + // kept holds the version PURLs to keep by package PURL, for the + // packages looked at so far. + kept map[string]map[string]struct{} +} + +func newLatestKeeper(db *database.DB, logger *slog.Logger, reg *retention.Registry, n int) *latestKeeper { + return &latestKeeper{db: db, logger: logger, reg: reg, n: n, kept: map[string]map[string]struct{}{}} +} + +// keeps reports whether c, an artifact of the ecosystem key, is a file of +// one of the highest cached versions of its package. +func (k *latestKeeper) keeps(key string, c database.RetentionCandidate) bool { + if k.n <= 0 { + return false + } + spec, _ := k.reg.Lookup(key) + if spec.CompareVersions == nil { + return false + } + kept, ok := k.kept[c.PackagePURL] + if !ok { + var err error + if kept, err = k.newest(spec, c.PackagePURL); err != nil { + // Not knowing which versions are the highest, keep the artifact + // for this sweep rather than evict one of them. + k.logger.Warn("retention: failed to list cached versions, keeping artifact", + "package_purl", c.PackagePURL, "error", err) + return true + } + k.kept[c.PackagePURL] = kept + } + _, keep := kept[c.VersionPURL] + return keep +} + +// newest returns the version PURLs of the highest cached versions of a +// package. +func (k *latestKeeper) newest(spec retention.Spec, packagePURL string) (map[string]struct{}, error) { + purls, err := k.db.CachedVersionPURLs(packagePURL) + if err != nil { + return nil, err + } + byVersion := make(map[string][]string, len(purls)) + versions := make([]string, 0, len(purls)) + for _, versionPURL := range purls { + p, err := purl.Parse(versionPURL) + if err != nil || p.Version == "" { + continue + } + if _, seen := byVersion[p.Version]; !seen { + versions = append(versions, p.Version) + } + byVersion[p.Version] = append(byVersion[p.Version], versionPURL) + } + kept := map[string]struct{}{} + for _, v := range retention.NewestVersions(spec.CompareVersions, versions, k.n) { + for _, versionPURL := range byVersion[v] { + kept[versionPURL] = struct{}{} + } + } + return kept, nil +} + // recordLRUEviction counts an artifact the size limit evicted under the // ecosystem of its package record. func recordLRUEviction(db *database.DB, logger *slog.Logger, versionPURL string) { diff --git a/internal/server/retention_keep_latest_test.go b/internal/server/retention_keep_latest_test.go new file mode 100644 index 0000000..9ee138b --- /dev/null +++ b/internal/server/retention_keep_latest_test.go @@ -0,0 +1,215 @@ +package server + +import ( + "context" + "database/sql" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/retention" + "github.com/git-pkgs/proxy/internal/storage" +) + +// keepLatestRegistry registers npm with keep_latest support and gem without. +func keepLatestRegistry() *retention.Registry { + reg := retention.NewRegistry() + reg.Register(retention.Spec{Key: "npm", CanonicalPackage: retention.DefaultCanonical("npm"), CompareVersions: retention.VersionOrder("npm")}) + reg.Register(retention.Spec{Key: "gem", CanonicalPackage: retention.DefaultCanonical("gem")}) + return reg +} + +// seedAgedVersion stores one cached file of pkg:{ecosystem}/{name}@{version}, +// fetched and last accessed at the given time. +func seedAgedVersion(t *testing.T, db *database.DB, store storage.Storage, ecosystem, name, version, filename string, at time.Time) { + t.Helper() + pkgPURL := "pkg:" + ecosystem + "/" + name + versionPURL := pkgPURL + "@" + version + if err := db.UpsertPackage(&database.Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: name}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&database.Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + path := ecosystem + "/" + name + "/" + version + "/" + filename + size, hash, err := store.Store(context.Background(), path, strings.NewReader("artifact")) + if err != nil { + t.Fatalf("Store: %v", err) + } + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: filename, + UpstreamURL: "https://example.com/" + filename, + StoragePath: sql.NullString{String: path, Valid: true}, + ContentHash: sql.NullString{String: hash, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: at, Valid: true}, + LastAccessedAt: sql.NullTime{Time: at, Valid: true}, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } +} + +// cachedVersions returns the versions of pkg:{ecosystem}/{name} that still +// have a cached file, sorted as text. +func cachedVersions(t *testing.T, db *database.DB, ecosystem, name string) []string { + t.Helper() + pkgPURL := "pkg:" + ecosystem + "/" + name + purls, err := db.CachedVersionPURLs(pkgPURL) + if err != nil { + t.Fatalf("CachedVersionPURLs: %v", err) + } + versions := make([]string, 0, len(purls)) + for _, p := range purls { + versions = append(versions, strings.TrimPrefix(p, pkgPURL+"@")) + } + slices.Sort(versions) + return versions +} + +// With keep_latest, the highest cached versions of a package outlive their +// rule, whichever was fetched first, and the rest are evicted as before. +func TestSweepRetentionKeepsNewestVersions(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + ago := func(days int) time.Time { return now.Add(-time.Duration(days) * retentionDay) } + + // 1.10.0 is the highest version and the one fetched longest ago. + seedAgedVersion(t, db, store, "npm", "lib", "1.10.0", "lib-1.10.0.tgz", ago(90)) + seedAgedVersion(t, db, store, "npm", "lib", "1.9.0", "lib-1.9.0.tgz", ago(60)) + seedAgedVersion(t, db, store, "npm", "lib", "1.2.0", "lib-1.2.0.tgz", ago(30)) + seedAgedVersion(t, db, store, "npm", "lib", "1.1.0", "lib-1.1.0.tgz", ago(20)) + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": 7 * retentionDay}, KeepLatest: 2} + sweepRetention(context.Background(), db, discardLogger(), keepLatestRegistry(), rules, now) + + if got, want := cachedVersions(t, db, "npm", "lib"), []string{"1.10.0", "1.9.0"}; !slices.Equal(got, want) { + t.Errorf("cached versions = %v, want %v", got, want) + } +} + +// A version still within its rule counts as one of the newest, so it takes a +// place an expired version would otherwise keep. +func TestSweepRetentionKeepLatestCountsUnexpiredVersions(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + ago := func(days int) time.Time { return now.Add(-time.Duration(days) * retentionDay) } + + seedAgedVersion(t, db, store, "npm", "lib", "3.0.0", "lib-3.0.0.tgz", ago(1)) + seedAgedVersion(t, db, store, "npm", "lib", "2.0.0", "lib-2.0.0.tgz", ago(30)) + seedAgedVersion(t, db, store, "npm", "lib", "1.0.0", "lib-1.0.0.tgz", ago(30)) + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": 7 * retentionDay}, KeepLatest: 2} + sweepRetention(context.Background(), db, discardLogger(), keepLatestRegistry(), rules, now) + + if got, want := cachedVersions(t, db, "npm", "lib"), []string{"2.0.0", "3.0.0"}; !slices.Equal(got, want) { + t.Errorf("cached versions = %v, want %v", got, want) + } +} + +// Every file of a kept version stays, and an older version unused for less +// than the rule is not evicted for being outside the newest. +func TestSweepRetentionKeepLatestKeepsAllFilesOfAVersion(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + old := now.Add(-30 * retentionDay) + + seedAgedVersion(t, db, store, "npm", "lib", "2.0.0", "lib-2.0.0.tgz", old) + seedAgedVersion(t, db, store, "npm", "lib", "2.0.0", "lib-2.0.0-extra.tgz", old) + seedAgedVersion(t, db, store, "npm", "lib", "1.0.0", "lib-1.0.0.tgz", now.Add(-retentionDay)) + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": 7 * retentionDay}, KeepLatest: 1} + sweepRetention(context.Background(), db, discardLogger(), keepLatestRegistry(), rules, now) + + for _, f := range []struct{ version, filename string }{ + {"2.0.0", "lib-2.0.0.tgz"}, {"2.0.0", "lib-2.0.0-extra.tgz"}, {"1.0.0", "lib-1.0.0.tgz"}, + } { + art, err := db.GetArtifact("pkg:npm/lib@"+f.version, f.filename) + if err != nil { + t.Fatalf("GetArtifact(%s): %v", f.filename, err) + } + if !art.StoragePath.Valid { + t.Errorf("%s was evicted", f.filename) + } + } +} + +// A version evicted earlier no longer counts: the newest versions are the +// newest that are still cached. +func TestSweepRetentionKeepLatestIgnoresUncachedVersions(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + old := now.Add(-30 * retentionDay) + + seedAgedVersion(t, db, store, "npm", "lib", "3.0.0", "lib-3.0.0.tgz", old) + seedAgedVersion(t, db, store, "npm", "lib", "2.0.0", "lib-2.0.0.tgz", old) + seedAgedVersion(t, db, store, "npm", "lib", "1.0.0", "lib-1.0.0.tgz", old) + if _, err := db.ClearArtifactCache("pkg:npm/lib@3.0.0", "lib-3.0.0.tgz", "npm/lib/3.0.0/lib-3.0.0.tgz"); err != nil { + t.Fatalf("ClearArtifactCache: %v", err) + } + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": 7 * retentionDay}, KeepLatest: 1} + sweepRetention(context.Background(), db, discardLogger(), keepLatestRegistry(), rules, now) + + if got, want := cachedVersions(t, db, "npm", "lib"), []string{"2.0.0"}; !slices.Equal(got, want) { + t.Errorf("cached versions = %v, want %v", got, want) + } +} + +// keep_latest counts per package, and an ecosystem that does not support it +// is evicted by age alone. +func TestSweepRetentionKeepLatestIsPerPackageAndEcosystem(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + old := now.Add(-30 * retentionDay) + + seedAgedVersion(t, db, store, "npm", "a", "1.0.0", "a-1.0.0.tgz", old) + seedAgedVersion(t, db, store, "npm", "a", "2.0.0", "a-2.0.0.tgz", old) + seedAgedVersion(t, db, store, "npm", "b", "0.1.0", "b-0.1.0.tgz", old) + seedAgedVersion(t, db, store, "gem", "rails", "7.0.0", "rails-7.0.0.gem", old) + seedAgedVersion(t, db, store, "gem", "rails", "8.0.0", "rails-8.0.0.gem", old) + + rules := retention.Rules{Default: 7 * retentionDay, KeepLatest: 1} + sweepRetention(context.Background(), db, discardLogger(), keepLatestRegistry(), rules, now) + + for _, c := range []struct { + ecosystem, name string + want []string + }{ + {"npm", "a", []string{"2.0.0"}}, + {"npm", "b", []string{"0.1.0"}}, + {"gem", "rails", []string{}}, + } { + if got := cachedVersions(t, db, c.ecosystem, c.name); !slices.Equal(got, c.want) { + t.Errorf("%s/%s cached versions = %v, want %v", c.ecosystem, c.name, got, c.want) + } + } +} + +// Without keep_latest the sweep evicts every expired version, as before. +func TestSweepRetentionWithoutKeepLatestEvictsNewestToo(t *testing.T) { + db, store := setupEvictionTest(t) + now := time.Now() + old := now.Add(-30 * retentionDay) + + seedAgedVersion(t, db, store, "npm", "lib", "2.0.0", "lib-2.0.0.tgz", old) + seedAgedVersion(t, db, store, "npm", "lib", "1.0.0", "lib-1.0.0.tgz", old) + + rules := retention.Rules{Ecosystems: map[string]time.Duration{"npm": 7 * retentionDay}} + sweepRetention(context.Background(), db, discardLogger(), keepLatestRegistry(), rules, now) + + if got := cachedVersions(t, db, "npm", "lib"); len(got) != 0 { + t.Errorf("cached versions = %v, want none", got) + } +} + +// The ecosystems whose handlers store plain release versions support +// keep_latest. +func TestKeepLatestEcosystems(t *testing.T) { + want := []string{"cargo", "golang", "npm", "pypi"} + if got := retention.Default.KeepLatestKeys(); !slices.Equal(got, want) { + t.Errorf("KeepLatestKeys() = %v, want %v", got, want) + } +} diff --git a/internal/server/retention_test.go b/internal/server/retention_test.go index 41acb92..16b2b15 100644 --- a/internal/server/retention_test.go +++ b/internal/server/retention_test.go @@ -313,7 +313,8 @@ func TestClearIfExpiredRechecksAgainstItsRule(t *testing.T) { t.Fatalf("RecordArtifactHit: %v", err) } - if clearIfExpired(db, discardLogger(), retentionRegistry(), rules, now.Add(time.Minute), candidates[0]) { + keep := newLatestKeeper(db, discardLogger(), retentionRegistry(), 0) + if clearIfExpired(db, discardLogger(), retentionRegistry(), rules, keep, now.Add(time.Minute), candidates[0]) { t.Error("cleared an artifact served after the scan") } if !isCached(t, db, "npm", "racing") { From bb6fb2bb372055b39a2cf3f9583a9e19b0edf736 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 10 Nov 2026 16:26:30 +0000 Subject: [PATCH 198/198] Document public-host signing for direct_serve_base_url Also marks the AWS modules as direct dependencies and drops comments that restate function names. --- config.example.yaml | 8 +++++++- go.mod | 6 +++--- internal/config/config.go | 8 ++++---- internal/handler/profile_test.go | 2 -- internal/server/server.go | 1 - internal/server/server_test.go | 2 -- internal/storage/blob_test.go | 5 ----- internal/storage/gcs_test.go | 2 -- 8 files changed, 14 insertions(+), 20 deletions(-) diff --git a/config.example.yaml b/config.example.yaml index 56c02f7..98e9b90 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -135,7 +135,13 @@ storage: # Public base URL for presigned URLs. Set this when the proxy reaches # storage at an internal address (127.0.0.1, a Docker service name, or - # VPC DNS) but clients must use a public hostname or CDN. + # VPC DNS) but clients must use a public hostname. Only scheme and host + # are used. For S3/MinIO the URL is signed for this host, so the server + # there must receive that Host header: a reverse proxy or CDN that sends + # the internal or bucket host to the origin fails signature validation. + # With virtual-hosted addressing (no use_path_style) the URL carries no + # bucket name, so this must be the bucket's own hostname. For GCS and + # Azure the host is replaced after signing. # direct_serve_base_url: "https://minio.example.com" # Database configuration diff --git a/go.mod b/go.mod index dce5841..f1cb4f7 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,11 @@ go 1.27.3 require ( github.com/BurntSushi/toml v1.6.0 github.com/Masterminds/semver/v3 v3.5.0 + github.com/aws/aws-sdk-go-v2 v1.47.5 github.com/aws/aws-sdk-go-v2/config v1.33.12 github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 + github.com/aws/smithy-go v1.28.7 github.com/git-pkgs/archives v0.10.0 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.3.1 @@ -51,7 +54,6 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.47.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.20.12 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect @@ -63,12 +65,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect - github.com/aws/smithy-go v1.28.7 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index 6f8171f..ddfbb93 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -519,10 +519,10 @@ type StorageConfig struct { // Uses Go duration syntax (e.g. "5m", "1h"). Default: "15m". DirectServeTTL string `json:"direct_serve_ttl" yaml:"direct_serve_ttl"` - // DirectServeBaseURL overrides the scheme and host of presigned URLs - // before returning them to clients. Useful when the proxy reaches - // storage at an internal address (e.g. 127.0.0.1 or a Docker hostname) - // but clients must use a public one. + // DirectServeBaseURL sets the scheme and host of presigned URLs. Useful + // when the proxy reaches storage at an internal address (e.g. 127.0.0.1 + // or a Docker hostname) but clients must use a public one. S3 URLs are + // signed for this host; GCS and Azure URLs have it replaced after signing. DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"` // CacheArtifacts stores fetched artifacts so later downloads are served diff --git a/internal/handler/profile_test.go b/internal/handler/profile_test.go index baa179d..a85d6ca 100644 --- a/internal/handler/profile_test.go +++ b/internal/handler/profile_test.go @@ -18,8 +18,6 @@ import ( registryclient "github.com/git-pkgs/registries/client" ) -// TestWithCooldownCopiesProxyFields verifies that WithCooldown copies all -// non-isolated Proxy fields onto the returned view. func TestWithCooldownCopiesProxyFields(t *testing.T) { p, _, _, _ := setupTestProxy(t) p.RouteFor = func(string, string) *ArtifactRoute { return nil } diff --git a/internal/server/server.go b/internal/server/server.go index fedf11b..59717d4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -283,7 +283,6 @@ func (s *Server) configureGitCache(client *http.Client) error { return nil } -// serve wires up HTTP routes, background workers, and serves requests on listener. func (s *Server) serve(listener net.Listener) error { metadataClient, artifactClient := s.newUpstreamClients(s.cfg.ParseArtifactTimeout()) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index c1e5b4a..d631b9a 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1740,8 +1740,6 @@ func TestNewServer_StorageConnectivityCheck(t *testing.T) { _ = srv.db.Close() } -// TestNewServer_DirectServeBaseURL verifies that New passes DirectServeBaseURL -// to storage.OpenBucket when DirectServe is enabled. func TestNewServer_DirectServeBaseURL(t *testing.T) { tempDir := t.TempDir() awsConfigFile := filepath.Join(tempDir, "aws-config") diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index dd357b0..debb16d 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -266,8 +266,6 @@ func TestBlobSignedURLUnsupported(t *testing.T) { } } -// TestBlobSignedURLSignEndpoint verifies that S3 presigned URLs are signed for -// the configured public host while preserving the storage URL's path style. func TestBlobSignedURLSignEndpoint(t *testing.T) { awsDir := t.TempDir() awsConfigFile := filepath.Join(awsDir, "config") @@ -385,8 +383,6 @@ func assertSigV4Signature(t *testing.T, u *url.URL, region string, creds aws.Cre } } -// TestAzureSignedURLSignEndpoint verifies that Azure presigned URLs have their -// scheme and host rewritten to the configured sign endpoint. func TestAzureSignedURLSignEndpoint(t *testing.T) { t.Setenv("AZURE_STORAGE_SAS_TOKEN", "") t.Setenv("AZURE_STORAGE_ACCOUNT", "devstoreaccount1") @@ -418,7 +414,6 @@ func TestAzureSignedURLSignEndpoint(t *testing.T) { } } -// TestRewriteSignedURLHost verifies scheme and host substitution on signed URLs. func TestRewriteSignedURLHost(t *testing.T) { tests := []struct { name string diff --git a/internal/storage/gcs_test.go b/internal/storage/gcs_test.go index 92cf3fd..461fb86 100644 --- a/internal/storage/gcs_test.go +++ b/internal/storage/gcs_test.go @@ -95,8 +95,6 @@ func TestOpenBucketGCSRoundTripWithEmulator(t *testing.T) { } } -// TestGCSSignedURLSignEndpoint verifies that GCS presigned URLs have their -// scheme and host rewritten to the configured sign endpoint. func TestGCSSignedURLSignEndpoint(t *testing.T) { key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil {