package handler import ( "bytes" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "log/slog" "net/http" "net/http/httptest" "strings" "sync/atomic" "testing" "time" "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/registries/fetch" ) const lfsTestContent = "large file content" // lfsSignature is the signature in the upstream's download URLs, which must // not reach logs or the database. const lfsSignature = "s3cr3t-signature" // lfsUpstream is a Git host with an LFS batch API whose download hrefs point // at a separate object store, as GitHub's do. The store only serves objects // to requests carrying the header and signed URL the batch response gave. type lfsUpstream struct { git, store *httptest.Server oid string content atomic.Value // string the store serves for oid batches atomic.Int32 downloads atomic.Int32 // batchStatus, when set, is the status the batch API fails with. batchStatus atomic.Int32 // signature is the signature the batch API hands out and the store // accepts; changing it expires the URLs handed out before. signature atomic.Value // lastBatch is the last batch request the upstream got. lastBatch atomic.Value // logs is what the proxy logged. logs bytes.Buffer } func newLFSUpstream(t *testing.T) *lfsUpstream { t.Helper() sum := sha256.Sum256([]byte(lfsTestContent)) u := &lfsUpstream{oid: hex.EncodeToString(sum[:])} u.content.Store(lfsTestContent) u.signature.Store(lfsSignature) u.store = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/objects/"+u.oid || r.Header.Get("X-Signed") != "token" || r.URL.Query().Get("Signature") != u.signature.Load().(string) { http.Error(w, "forbidden", http.StatusForbidden) return } u.downloads.Add(1) _, _ = io.WriteString(w, u.content.Load().(string)) })) u.git = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // acme/widget has the object, as does its fork; stranger/widget // exists but doesn't have it. repo, _, _ := strings.Cut(r.URL.Path, ".git/info/lfs/objects/batch") hasObject, exists := map[string]bool{"/acme/widget": true, "/fork/widget": true, "/stranger/widget": false}[repo] if r.Method != http.MethodPost || !exists { http.NotFound(w, r) return } u.batches.Add(1) if status := u.batchStatus.Load(); status != 0 { http.Error(w, "failing", int(status)) return } var req lfsBatchRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } u.lastBatch.Store(req) resp := lfsBatchResponse{Transfer: "basic"} for _, object := range req.Objects { if object.OID != u.oid || !hasObject { resp.Objects = append(resp.Objects, lfsObjectResponse{OID: object.OID, Size: object.Size, Error: &lfsObjectError{Code: http.StatusNotFound, Message: "Object does not exist"}}) continue } resp.Objects = append(resp.Objects, lfsObjectResponse{OID: object.OID, Size: object.Size, Actions: map[string]lfsAction{"download": { Href: u.store.URL + "/objects/" + u.oid + "?Expires=1&Signature=" + u.signature.Load().(string), Header: map[string]string{"X-Signed": "token"}, }}}) } // An object nobody asked for is not passed on. resp.Objects = append(resp.Objects, lfsObjectResponse{OID: strings.Repeat("e", 64), Size: 1, Actions: map[string]lfsAction{"download": {Href: u.store.URL + "/other"}}}) w.Header().Set("Content-Type", gitLFSMediaType) _ = json.NewEncoder(w).Encode(resp) })) t.Cleanup(u.git.Close) t.Cleanup(u.store.Close) return u } func newLFSHandler(t *testing.T, u *lfsUpstream, repos ...string) *GitHandler { t.Helper() proxy, _, store, _ := setupTestProxy(t) store.seekable = true proxy.Logger = slog.New(slog.NewTextHandler(&u.logs, nil)) proxy.HTTPClient = u.git.Client() fetcher := fetch.NewFetcher(fetch.WithHTTPClient(u.store.Client()), fetch.WithMaxRetries(0)) proxy.Fetcher = fetcher t.Cleanup(func() { _ = fetcher.Close() }) return NewGitHandler(proxy, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ "up": {URL: u.git.URL, Repos: repos}, }}).WithBaseURL("http://proxy.test/") } func lfsRequest(h *GitHandler, method, target, body string) *httptest.ResponseRecorder { r := httptest.NewRequest(method, target, strings.NewReader(body)) r.Header.Set("Content-Type", gitLFSMediaType) w := httptest.NewRecorder() h.Routes().ServeHTTP(w, r) return w } func lfsBatchBody(operation string, objects ...lfsObject) string { body, _ := json.Marshal(lfsBatchRequest{Operation: operation, Transfers: []string{"basic"}, Ref: json.RawMessage(`{"name":"refs/heads/main"}`), Objects: objects}) return string(body) } func TestGitLFSBatchPointsObjectsAtTheProxy(t *testing.T) { u := newLFSUpstream(t) h := newLFSHandler(t, u) missing := strings.Repeat("0", 64) size := int64(len(lfsTestContent)) w := lfsRequest(h, http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", lfsBatchBody("download", lfsObject{OID: u.oid, Size: size}, lfsObject{OID: missing, Size: 3})) if w.Code != http.StatusOK || w.Header().Get("Content-Type") != gitLFSMediaType { t.Fatalf("batch: status %d, type %q, body %s", w.Code, w.Header().Get("Content-Type"), w.Body) } var resp lfsBatchResponse if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } if len(resp.Objects) != 2 { t.Fatalf("objects = %+v, want the two requested", resp.Objects) } got, notFound := resp.Objects[0], resp.Objects[1] want := fmt.Sprintf("http://proxy.test/git/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size) if download := got.Actions["download"]; download.Href != want || len(download.Header) != 0 || !got.Authenticated { t.Errorf("download = %+v, authenticated %v; want %s without the upstream's headers", download, got.Authenticated, want) } if notFound.Error == nil || notFound.Error.Code != http.StatusNotFound || notFound.Actions != nil { t.Errorf("missing object = %+v, want the upstream's 404", notFound) } if req, _ := u.lastBatch.Load().(lfsBatchRequest); string(req.Ref) != `{"name":"refs/heads/main"}` { t.Errorf("upstream got ref %s, want the client's", req.Ref) } } func TestGitLFSObjectIsCached(t *testing.T) { u := newLFSUpstream(t) h := newLFSHandler(t, u) path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) w := lfsRequest(h, http.MethodGet, path, "") if w.Code != http.StatusOK || w.Body.String() != lfsTestContent { t.Fatalf("first GET: status %d, body %q", w.Code, w.Body) } if u.batches.Load() != 1 || u.downloads.Load() != 1 { t.Errorf("batches %d, downloads %d; want one of each", u.batches.Load(), u.downloads.Load()) } u.git.Close() u.store.Close() w = lfsRequest(h, http.MethodGet, path, "") if w.Code != http.StatusOK || w.Body.String() != lfsTestContent { t.Errorf("GET with the upstream down: status %d, body %q; want it from the cache", w.Code, w.Body) } } func TestGitLFSObjectRefusesWrongContent(t *testing.T) { u := newLFSUpstream(t) u.content.Store("tampered content!!") h := newLFSHandler(t, u) path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) for range 2 { if w := lfsRequest(h, http.MethodGet, path, ""); w.Code == http.StatusOK && w.Body.String() == "tampered content!!" { t.Fatal("served content that does not match its object ID") } } if u.downloads.Load() != 2 { t.Errorf("downloads = %d, want 2: the mismatching object must not be cached", u.downloads.Load()) } } func TestGitLFSRefusals(t *testing.T) { u := newLFSUpstream(t) h := newLFSHandler(t, u, "acme/*") object := lfsObject{OID: u.oid, Size: int64(len(lfsTestContent))} tests := []struct { name, method, target, body string status int }{ {"upload", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", lfsBatchBody("upload", object), http.StatusForbidden}, {"sha512 objects", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", `{"operation":"download","objects":[],"hash_algo":"sha512"}`, http.StatusConflict}, {"other transfers", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", `{"operation":"download","transfers":["ssh"],"objects":[]}`, http.StatusUnprocessableEntity}, {"invalid oid", http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", lfsBatchBody("download", lfsObject{OID: "../x", Size: 1}), http.StatusUnprocessableEntity}, {"batch GET", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/batch", "", http.StatusMethodNotAllowed}, {"repository outside repos", http.MethodPost, "/up/other/widget.git/info/lfs/objects/batch", lfsBatchBody("download", object), http.StatusNotFound}, {"unknown repository", http.MethodPost, "/up/acme/missing.git/info/lfs/objects/batch", lfsBatchBody("download", object), http.StatusNotFound}, {"object without size", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/" + u.oid, "", http.StatusNotFound}, {"uppercase oid", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/" + strings.ToUpper(u.oid) + "/1", "", http.StatusNotFound}, {"object PUT", http.MethodPut, "/up/acme/widget.git/info/lfs/objects/" + u.oid + "/1", "", http.StatusMethodNotAllowed}, {"locks", http.MethodPost, "/up/acme/widget.git/info/lfs/locks/verify", "{}", http.StatusNotFound}, {"missing object", http.MethodGet, "/up/acme/widget.git/info/lfs/objects/" + strings.Repeat("0", 64) + "/3", "", http.StatusNotFound}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { w := lfsRequest(h, tt.method, tt.target, tt.body) if w.Code != tt.status || w.Header().Get("Content-Type") != gitLFSMediaType { t.Errorf("status %d, type %q, body %s; want %d", w.Code, w.Header().Get("Content-Type"), w.Body, tt.status) } }) } if u.downloads.Load() != 0 { t.Errorf("downloads = %d, want none", u.downloads.Load()) } } func TestGitLFSKeepsSignedURLsOutOfLogsAndDatabase(t *testing.T) { u := newLFSUpstream(t) h := newLFSHandler(t, u) path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) if w := lfsRequest(h, http.MethodGet, path, ""); w.Code != http.StatusOK { t.Fatalf("GET: status %d, body %s", w.Code, w.Body) } _, versionPURL, err := packagePURLStrings(gitEcosystem, "up/acme/widget", u.oid) if err != nil { t.Fatal(err) } record, err := h.proxy.DB.GetArtifact(versionPURL, u.oid) if err != nil || record == nil { t.Fatalf("artifact record: %v, %v", record, err) } if want := u.store.URL + "/objects/" + u.oid; record.UpstreamURL != want { t.Errorf("recorded upstream URL %q, want %q", record.UpstreamURL, want) } // A failed download is logged without the signature too, although the // connection error names the URL. other := newLFSUpstream(t) other.store.Close() h = newLFSHandler(t, other) if w := lfsRequest(h, http.MethodGet, path, ""); w.Code == http.StatusOK { t.Fatalf("GET from an unreachable store: status %d", w.Code) } if !strings.Contains(other.logs.String(), other.store.URL+"/objects/"+other.oid) { t.Fatalf("logs don't name the failed download:\n%s", other.logs.String()) } for _, logs := range []string{u.logs.String(), other.logs.String()} { if strings.Contains(logs, lfsSignature) { t.Errorf("logs contain the download URL's signature:\n%s", logs) } } } func TestGitLFSBatchFromCacheWhenUpstreamFails(t *testing.T) { u := newLFSUpstream(t) h := newLFSHandler(t, u) size := int64(len(lfsTestContent)) missing := strings.Repeat("0", 64) batch := lfsBatchBody("download", lfsObject{OID: u.oid, Size: size}, lfsObject{OID: missing, Size: 3}) const batchPath = "/up/acme/widget.git/info/lfs/objects/batch" // Nothing cached yet: a failing upstream fails the batch. u.batchStatus.Store(http.StatusServiceUnavailable) if w := lfsRequest(h, http.MethodPost, batchPath, batch); w.Code != http.StatusBadGateway { t.Fatalf("batch with nothing cached: status %d, body %s; want 502", w.Code, w.Body) } u.batchStatus.Store(0) if w := lfsRequest(h, http.MethodGet, fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size), ""); w.Code != http.StatusOK { t.Fatalf("GET: status %d, body %s", w.Code, w.Body) } // A refusal is the upstream's answer, not an outage. u.batchStatus.Store(http.StatusForbidden) if w := lfsRequest(h, http.MethodPost, batchPath, batch); w.Code != http.StatusBadGateway { t.Errorf("batch the upstream refused: status %d, body %s; want 502", w.Code, w.Body) } u.git.Close() w := lfsRequest(h, http.MethodPost, batchPath, batch) if w.Code != http.StatusOK { t.Fatalf("batch with the upstream down: status %d, body %s", w.Code, w.Body) } var resp lfsBatchResponse if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } if len(resp.Objects) != 2 { t.Fatalf("objects = %+v, want the two requested", resp.Objects) } want := fmt.Sprintf("http://proxy.test/git/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size) if got := resp.Objects[0]; got.Actions["download"].Href != want || !got.Authenticated || got.Error != nil { t.Errorf("cached object = %+v, want a download from %s", got, want) } if got := resp.Objects[1]; got.Actions != nil || got.Error == nil { t.Errorf("uncached object = %+v, want an error", got) } } func TestGitLFSHeadDoesNotDownload(t *testing.T) { u := newLFSUpstream(t) h := newLFSHandler(t, u) path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)) w := lfsRequest(h, http.MethodHead, path, "") if w.Code != http.StatusOK || w.Header().Get("Content-Length") != fmt.Sprint(len(lfsTestContent)) { t.Errorf("HEAD: status %d, Content-Length %q", w.Code, w.Header().Get("Content-Length")) } if u.downloads.Load() != 0 { t.Errorf("downloads = %d, want none for a HEAD", u.downloads.Load()) } missing := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/3", strings.Repeat("0", 64)) if w := lfsRequest(h, http.MethodHead, missing, ""); w.Code != http.StatusNotFound { t.Errorf("HEAD of a missing object: status %d, want 404", w.Code) } } func TestGitLFSObjectAfterBatchNeedsNoSecondBatch(t *testing.T) { u := newLFSUpstream(t) h := newLFSHandler(t, u) size := int64(len(lfsTestContent)) path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, size) batch := lfsBatchBody("download", lfsObject{OID: u.oid, Size: size}) if w := lfsRequest(h, http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", batch); w.Code != http.StatusOK { t.Fatalf("batch: status %d, body %s", w.Code, w.Body) } if w := lfsRequest(h, http.MethodGet, path, ""); w.Code != http.StatusOK || w.Body.String() != lfsTestContent { t.Fatalf("GET: status %d, body %q", w.Code, w.Body) } if u.batches.Load() != 1 || u.downloads.Load() != 1 { t.Errorf("batches %d, downloads %d; want the client's batch and one download", u.batches.Load(), u.downloads.Load()) } // A URL that stopped working before its time is replaced by a fresh one. other := newLFSUpstream(t) h = newLFSHandler(t, other) if w := lfsRequest(h, http.MethodPost, "/up/acme/widget.git/info/lfs/objects/batch", batch); w.Code != http.StatusOK { t.Fatalf("batch: status %d, body %s", w.Code, w.Body) } other.signature.Store("rotated") if w := lfsRequest(h, http.MethodGet, path, ""); w.Code != http.StatusOK || w.Body.String() != lfsTestContent { t.Fatalf("GET with a revoked URL: status %d, body %q", w.Code, w.Body) } if other.batches.Load() != 2 { t.Errorf("batches = %d, want a second one for a fresh URL", other.batches.Load()) } } func TestGitLFSKeptDownloadsExpire(t *testing.T) { now := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC) object := lfsObject{OID: strings.Repeat("a", 64), Size: 3} href := "https://objects.example/a?sig=1" tests := []struct { name string action lfsAction at time.Duration kept bool }{ {"no expiry", lfsAction{Href: href}, 4 * time.Minute, true}, {"no expiry, later", lfsAction{Href: href}, 5 * time.Minute, false}, {"expires_in", lfsAction{Href: href, ExpiresIn: 600}, 9 * time.Minute, true}, {"expires_in, within the margin", lfsAction{Href: href, ExpiresIn: 600}, 10*time.Minute - 10*time.Second, false}, {"expires_in too short to use", lfsAction{Href: href, ExpiresIn: 10}, 0, false}, {"expires_at", lfsAction{Href: href, ExpiresAt: now.Add(20 * time.Minute).Format(time.RFC3339)}, 19 * time.Minute, true}, {"expires_at, past", lfsAction{Href: href, ExpiresAt: now.Add(20 * time.Minute).Format(time.RFC3339)}, 20 * time.Minute, false}, {"long expiry capped", lfsAction{Href: href, ExpiresIn: 86400}, 2 * time.Hour, false}, {"relative href", lfsAction{Href: "/a"}, 0, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var d lfsDownloads d.put("repo", object, tt.action, now) if _, kept := d.get("repo", object, now.Add(tt.at)); kept != tt.kept { t.Errorf("kept = %v, want %v", kept, tt.kept) } }) } var d lfsDownloads d.put("repo", object, lfsAction{Href: href}, now) if _, kept := d.get("repo", lfsObject{OID: object.OID, Size: 4}, now); kept { t.Error("kept for another size") } if _, kept := d.get("other", object, now); kept { t.Error("kept for another repository") } for i := range gitLFSKeptDownloads + 10 { d.put("repo", lfsObject{OID: fmt.Sprintf("%064x", i), Size: 1}, lfsAction{Href: href}, now) } if len(d.entries) > gitLFSKeptDownloads { t.Errorf("%d entries kept, want at most %d", len(d.entries), gitLFSKeptDownloads) } } func TestGitLFSObjectOfAnotherSizeIsRefused(t *testing.T) { u := newLFSUpstream(t) h := newLFSHandler(t, u) path := fmt.Sprintf("/up/acme/widget.git/info/lfs/objects/%s/%d", u.oid, len(lfsTestContent)-1) if w := lfsRequest(h, http.MethodGet, path, ""); w.Code == http.StatusOK { t.Fatalf("GET at the wrong size: status %d, body %q", w.Code, w.Body) } if u.downloads.Load() != 1 || u.batches.Load() != 1 { t.Errorf("downloads %d, batches %d; want one of each, without a retry", u.downloads.Load(), u.batches.Load()) } } func TestExactSizeBody(t *testing.T) { for _, tt := range []struct { body string size int64 ok bool }{ {"abcdef", 6, true}, {"abcdef", 5, false}, {"abcdef", 7, false}, {"", 0, true}, {"a", 0, false}, } { got, err := io.ReadAll(&exactSizeBody{ReadCloser: io.NopCloser(strings.NewReader(tt.body)), left: tt.size}) if (err == nil) != tt.ok || (err != nil && !errors.Is(err, errDownloadSize)) { t.Errorf("%q at size %d: err %v, want ok %v", tt.body, tt.size, err, tt.ok) } if int64(len(got)) > tt.size+1 { t.Errorf("%q at size %d: read %d bytes", tt.body, tt.size, len(got)) } } } func TestGitLFSObjectCopiedFromAnotherRepository(t *testing.T) { u := newLFSUpstream(t) h := newLFSHandler(t, u) object := func(repo string) string { return fmt.Sprintf("/up/%s.git/info/lfs/objects/%s/%d", repo, u.oid, len(lfsTestContent)) } if w := lfsRequest(h, http.MethodGet, object("acme/widget"), ""); w.Code != http.StatusOK { t.Fatalf("GET: status %d, body %s", w.Code, w.Body) } w := lfsRequest(h, http.MethodGet, object("fork/widget"), "") if w.Code != http.StatusOK || w.Body.String() != lfsTestContent { t.Fatalf("GET from the fork: status %d, body %q", w.Code, w.Body) } if u.downloads.Load() != 1 || u.batches.Load() != 2 { t.Errorf("downloads %d, batches %d; want the fork's object copied after its own batch", u.downloads.Load(), u.batches.Load()) } cached, err := h.proxy.GetCachedArtifact(t.Context(), gitEcosystem, "up/fork/widget", u.oid, u.oid) if err != nil || cached == nil { t.Fatalf("fork's object not cached: %v, %v", cached, err) } _ = cached.Reader.Close() // A repository the upstream says lacks the object doesn't get it, though // a copy is stored. if w := lfsRequest(h, http.MethodGet, object("stranger/widget"), ""); w.Code != http.StatusNotFound { t.Errorf("GET from a repository without the object: status %d, body %s; want 404", w.Code, w.Body) } }