2026-01-20 21:52:44 +00:00
|
|
|
package handler
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"encoding/json"
|
Fix metadata caching, 404 propagation, mirror progress, and registry stubs
- ProxyCached now stores upstream Last-Modified in the cache and uses it
(along with ETag) for conditional request handling, returning 304 when
client validators match. Adds Content-Length to cached responses.
- Handlers calling FetchOrCacheMetadata (pypi, composer, pub, nuget) now
check for ErrUpstreamNotFound and return 404 instead of 502, matching
the existing npm and cargo behavior.
- Mirror jobs report live progress via a periodic callback while running,
so API polls return real counts instead of zeroed progress.
- Registry mirroring removed from CLI flags, API acceptance, README, and
docs since every enumerator was a stub returning "not yet implemented".
- Added tests for the conditional metadata path (ETag/If-None-Match,
Last-Modified/If-Modified-Since, 304 responses, header omission).
2026-04-01 20:14:11 +01:00
|
|
|
"errors"
|
2026-01-20 21:52:44 +00:00
|
|
|
"fmt"
|
|
|
|
|
"net/http"
|
|
|
|
|
"strings"
|
2026-10-10 22:55:09 +02:00
|
|
|
|
|
|
|
|
"github.com/git-pkgs/proxy/internal/retention"
|
|
|
|
|
"github.com/git-pkgs/purl"
|
2026-01-20 21:52:44 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
const (
|
2026-08-29 11:19:54 +01:00
|
|
|
nugetUpstream = "https://api.nuget.org"
|
|
|
|
|
nugetSearchUpstream = "https://azuresearch-usnc.nuget.org"
|
2026-01-20 21:52:44 +00:00
|
|
|
)
|
|
|
|
|
|
2026-10-10 22:55:09 +02:00
|
|
|
// Packages are stored under pkg:nuget/{id} with the id lowercased, since
|
|
|
|
|
// NuGet ids are case-insensitive. purl keeps nuget names as written, so
|
|
|
|
|
// package rules are lowercased the same way before the default mapping.
|
|
|
|
|
func init() {
|
|
|
|
|
retention.Default.Register(retention.Spec{Key: "nuget", CanonicalPackage: nugetRetentionKey})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// nugetRetentionKey maps a package rule to the stored package PURL, so
|
|
|
|
|
// pkg:nuget/Newtonsoft.Json matches the stored pkg:nuget/newtonsoft.json.
|
|
|
|
|
func nugetRetentionKey(p *purl.PURL) (string, bool) {
|
|
|
|
|
if p == nil {
|
|
|
|
|
return "", false
|
|
|
|
|
}
|
|
|
|
|
return retention.DefaultCanonical("nuget")(purl.New(p.Type, p.Namespace, strings.ToLower(p.Name), "", nil))
|
|
|
|
|
}
|
|
|
|
|
|
2026-01-20 21:52:44 +00:00
|
|
|
// NuGetHandler handles NuGet V3 API protocol requests.
|
|
|
|
|
type NuGetHandler struct {
|
|
|
|
|
proxy *Proxy
|
|
|
|
|
upstreamURL string
|
2026-08-29 11:19:54 +01:00
|
|
|
searchURL string
|
2026-01-20 21:52:44 +00:00
|
|
|
proxyURL string
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// NewNuGetHandler creates a new NuGet protocol handler.
|
|
|
|
|
func NewNuGetHandler(proxy *Proxy, proxyURL string) *NuGetHandler {
|
|
|
|
|
return &NuGetHandler{
|
|
|
|
|
proxy: proxy,
|
|
|
|
|
upstreamURL: nugetUpstream,
|
2026-08-29 11:19:54 +01:00
|
|
|
searchURL: nugetSearchUpstream,
|
2026-01-20 21:52:44 +00:00
|
|
|
proxyURL: strings.TrimSuffix(proxyURL, "/"),
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-29 11:19:54 +01:00
|
|
|
// NewNuGetHandlerWithUpstreams creates a NuGet handler with custom API and
|
|
|
|
|
// search upstreams.
|
|
|
|
|
func NewNuGetHandlerWithUpstreams(proxy *Proxy, proxyURL, upstreamURL, searchURL string) *NuGetHandler {
|
|
|
|
|
h := NewNuGetHandler(proxy, proxyURL)
|
|
|
|
|
h.upstreamURL = configuredUpstreamURL(upstreamURL, nugetUpstream)
|
|
|
|
|
h.searchURL = configuredUpstreamURL(searchURL, nugetSearchUpstream)
|
|
|
|
|
return h
|
|
|
|
|
}
|
|
|
|
|
|
2026-01-20 21:52:44 +00:00
|
|
|
// Routes returns the HTTP handler for NuGet requests.
|
|
|
|
|
func (h *NuGetHandler) Routes() http.Handler {
|
|
|
|
|
mux := http.NewServeMux()
|
|
|
|
|
|
|
|
|
|
// V3 API service index
|
|
|
|
|
mux.HandleFunc("GET /v3/index.json", h.handleServiceIndex)
|
|
|
|
|
|
|
|
|
|
// Package content (downloads)
|
|
|
|
|
mux.HandleFunc("GET /v3-flatcontainer/{id}/{version}/{filename}", h.handleDownload)
|
2026-09-16 12:57:44 +05:30
|
|
|
mux.HandleFunc("GET /v3-flatcontainer/{id}/index.json", h.handleVersionList)
|
2026-01-20 21:52:44 +00:00
|
|
|
|
|
|
|
|
// Registration (package metadata) - use prefix matching since {version}.json isn't allowed
|
2026-09-16 12:57:44 +05:30
|
|
|
for _, prefix := range nugetRegistrationPrefixes {
|
|
|
|
|
mux.HandleFunc("GET "+prefix, h.handleRegistration)
|
|
|
|
|
}
|
2026-01-20 21:52:44 +00:00
|
|
|
|
|
|
|
|
// Search
|
|
|
|
|
mux.HandleFunc("GET /query", h.proxyUpstream)
|
|
|
|
|
|
|
|
|
|
// Autocomplete
|
|
|
|
|
mux.HandleFunc("GET /autocomplete", h.proxyUpstream)
|
|
|
|
|
|
|
|
|
|
return mux
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// handleServiceIndex serves the NuGet V3 service index with rewritten URLs.
|
|
|
|
|
func (h *NuGetHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
h.proxy.Logger.Info("nuget service index request")
|
|
|
|
|
|
|
|
|
|
upstreamURL := h.upstreamURL + "/v3/index.json"
|
|
|
|
|
|
2026-03-19 21:06:02 +00:00
|
|
|
body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "nuget", "_service_index", upstreamURL)
|
2026-01-20 21:52:44 +00:00
|
|
|
if err != nil {
|
Fix metadata caching, 404 propagation, mirror progress, and registry stubs
- ProxyCached now stores upstream Last-Modified in the cache and uses it
(along with ETag) for conditional request handling, returning 304 when
client validators match. Adds Content-Length to cached responses.
- Handlers calling FetchOrCacheMetadata (pypi, composer, pub, nuget) now
check for ErrUpstreamNotFound and return 404 instead of 502, matching
the existing npm and cargo behavior.
- Mirror jobs report live progress via a periodic callback while running,
so API polls return real counts instead of zeroed progress.
- Registry mirroring removed from CLI flags, API acceptance, README, and
docs since every enumerator was a stub returning "not yet implemented".
- Added tests for the conditional metadata path (ETag/If-None-Match,
Last-Modified/If-Modified-Since, 304 responses, header omission).
2026-04-01 20:14:11 +01:00
|
|
|
if errors.Is(err, ErrUpstreamNotFound) {
|
|
|
|
|
http.Error(w, "not found", http.StatusNotFound)
|
|
|
|
|
return
|
|
|
|
|
}
|
2026-10-15 13:35:12 +01:00
|
|
|
if writeUpstreamHeld(w, err) {
|
|
|
|
|
return
|
|
|
|
|
}
|
2026-01-20 21:52:44 +00:00
|
|
|
h.proxy.Logger.Error("upstream request failed", "error", err)
|
|
|
|
|
http.Error(w, "upstream request failed", http.StatusBadGateway)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
rewritten, err := h.rewriteServiceIndex(body)
|
|
|
|
|
if err != nil {
|
2026-09-16 12:57:44 +05:30
|
|
|
if h.cooldownEnabled() {
|
|
|
|
|
h.nugetMetadataError(w, err)
|
|
|
|
|
return
|
|
|
|
|
}
|
2026-01-20 21:52:44 +00:00
|
|
|
h.proxy.Logger.Warn("failed to rewrite service index, proxying original", "error", err)
|
2026-09-02 13:22:02 +01:00
|
|
|
w.Header().Set(headerContentType, "application/json")
|
2026-01-20 21:52:44 +00:00
|
|
|
_, _ = w.Write(body)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-02 13:22:02 +01:00
|
|
|
w.Header().Set(headerContentType, "application/json")
|
2026-01-20 21:52:44 +00:00
|
|
|
_, _ = w.Write(rewritten)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// rewriteServiceIndex rewrites service URLs in the index to point at this proxy.
|
|
|
|
|
func (h *NuGetHandler) rewriteServiceIndex(body []byte) ([]byte, error) {
|
|
|
|
|
var index map[string]any
|
|
|
|
|
if err := json.Unmarshal(body, &index); err != nil {
|
|
|
|
|
return nil, err
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
resources, ok := index["resources"].([]any)
|
|
|
|
|
if !ok {
|
|
|
|
|
return body, nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, res := range resources {
|
|
|
|
|
rmap, ok := res.(map[string]any)
|
|
|
|
|
if !ok {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
id, _ := rmap["@id"].(string)
|
|
|
|
|
rtype, _ := rmap["@type"].(string)
|
|
|
|
|
|
2026-08-29 11:19:54 +01:00
|
|
|
// Rewrite URLs for services we proxy. The service type determines the
|
|
|
|
|
// local route because an upstream index may advertise a different host.
|
|
|
|
|
if id != "" {
|
|
|
|
|
rmap["@id"] = h.rewriteNuGetURL(id, rtype)
|
2026-01-20 21:52:44 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return json.Marshal(index)
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-29 11:19:54 +01:00
|
|
|
// rewriteNuGetURL rewrites a NuGet service URL based on its advertised type.
|
|
|
|
|
// Service types the proxy does not handle are returned unchanged.
|
|
|
|
|
func (h *NuGetHandler) rewriteNuGetURL(origURL, serviceType string) string {
|
|
|
|
|
switch serviceType {
|
|
|
|
|
case "PackageBaseAddress/3.0.0":
|
|
|
|
|
return h.proxyURL + "/nuget/v3-flatcontainer/"
|
2026-09-16 12:57:44 +05:30
|
|
|
case "RegistrationsBaseUrl", "RegistrationsBaseUrl/3.0.0-beta", "RegistrationsBaseUrl/3.0.0-rc":
|
|
|
|
|
return h.proxyURL + "/nuget/v3/registration5-semver1/"
|
|
|
|
|
case "RegistrationsBaseUrl/3.4.0":
|
|
|
|
|
return h.proxyURL + "/nuget/v3/registration5-gz-semver1/"
|
2026-08-29 11:19:54 +01:00
|
|
|
case "RegistrationsBaseUrl/3.6.0", "RegistrationsBaseUrl/Versioned":
|
|
|
|
|
return h.proxyURL + "/nuget/v3/registration5-gz-semver2/"
|
|
|
|
|
case "SearchQueryService", "SearchQueryService/3.0.0-rc", "SearchQueryService/3.5.0":
|
|
|
|
|
return h.proxyURL + "/nuget/query"
|
|
|
|
|
case "SearchAutocompleteService", "SearchAutocompleteService/3.5.0":
|
|
|
|
|
return h.proxyURL + "/nuget/autocomplete"
|
|
|
|
|
default:
|
|
|
|
|
return origURL
|
2026-01-20 21:52:44 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// handleDownload serves a package file, fetching and caching from upstream if needed.
|
|
|
|
|
func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
id := r.PathValue("id")
|
|
|
|
|
version := r.PathValue("version")
|
|
|
|
|
filename := r.PathValue("filename")
|
|
|
|
|
|
|
|
|
|
if id == "" || version == "" || filename == "" {
|
|
|
|
|
http.Error(w, "invalid request", http.StatusBadRequest)
|
|
|
|
|
return
|
|
|
|
|
}
|
2026-09-16 12:57:44 +05:30
|
|
|
|
|
|
|
|
if h.cooldownEnabled() {
|
|
|
|
|
allowed, err := h.nugetDownloadAllowed(r.Context(), id, version)
|
|
|
|
|
if err != nil {
|
|
|
|
|
h.nugetMetadataError(w, err)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
if !allowed {
|
|
|
|
|
JSONError(w, http.StatusNotFound, "version not found")
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-01-20 21:52:44 +00:00
|
|
|
|
|
|
|
|
// Only cache .nupkg files
|
|
|
|
|
if !strings.HasSuffix(filename, ".nupkg") {
|
|
|
|
|
h.proxyUpstream(w, r)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
h.proxy.Logger.Info("nuget download request",
|
|
|
|
|
"id", id, "version", version, "filename", filename)
|
|
|
|
|
|
|
|
|
|
// NuGet package IDs are case-insensitive, lowercase for storage
|
|
|
|
|
name := strings.ToLower(id)
|
|
|
|
|
upstreamURL := fmt.Sprintf("%s/v3-flatcontainer/%s/%s/%s", h.upstreamURL, name, version, filename)
|
|
|
|
|
|
|
|
|
|
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "nuget", name, version, filename, upstreamURL)
|
|
|
|
|
if err != nil {
|
2026-07-26 20:07:46 +02:00
|
|
|
h.proxy.serveArtifactError(w, err, "failed to fetch package")
|
2026-01-20 21:52:44 +00:00
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
Support HTTP byte-range requests for local file cached artifact downloads (#404)
* feat(storage): expose efficient seek for local files
Open file-backed artifacts with os.Open so local readers support
efficient seeking. Leave cloud storage readers unchanged.
Keep the legacy sidecar mapping test focused on fileblob's reader:
Blob.Open now reads local files directly and no longer consults the
sidecar. Continue checking sidecar cleanup and post-store reads.
* feat(handler): preserve efficient seek through integrity checks
Keep the cache result's single Reader field and preserve io.Seeker
when the underlying storage reader supports efficient seeking.
Sequential reads continue through whole-object integrity verification.
A successful seek switches subsequent reads to the source for range
responses, where full-object verification is not possible.
Test both modes and verify the source is closed once.
* feat(handler): add request-aware byte range responses
Add a request-aware artifact serving helper for bounded, open-ended,
and suffix byte ranges, including If-Range handling, 206 responses,
and 416 responses for valid unsatisfiable ranges.
Ignore malformed and multi-range requests by serving the full artifact.
Advertise byte-range support only for seekable readers, preserving HEAD,
redirect, and non-seekable behavior. Clarify that net/http recovers
ErrAbortHandler per request and keeps the server running.
* feat(handler): enable byte ranges across artifact downloads
Forward each artifact request to the shared range-aware response helper,
including cached generic release assets and OCI blobs. This lets the
handler honor ranges only when its reader supports efficient seeking,
without changing direct-storage redirect behavior.
Add warm-cache endpoint tests that assert partial response headers and
bytes without an upstream refetch. Document supported range behavior,
unsupported reader fallback, and the integrity limitation of partial reads.
2026-10-04 09:48:41 -07:00
|
|
|
ServeArtifactRequest(w, r, result)
|
2026-01-20 21:52:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// proxyUpstream forwards a request to NuGet without caching.
|
|
|
|
|
func (h *NuGetHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
|
|
|
|
// Build upstream URL based on the path
|
|
|
|
|
upstreamURL := h.buildUpstreamURL(r)
|
|
|
|
|
|
|
|
|
|
h.proxy.Logger.Debug("proxying to upstream", "url", upstreamURL)
|
|
|
|
|
|
|
|
|
|
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil)
|
|
|
|
|
if err != nil {
|
|
|
|
|
http.Error(w, "failed to create request", http.StatusInternalServerError)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Copy accept-encoding for compression
|
2026-05-05 10:25:17 +01:00
|
|
|
if ae := r.Header.Get(headerAcceptEncoding); ae != "" {
|
|
|
|
|
req.Header.Set(headerAcceptEncoding, ae)
|
2026-01-20 21:52:44 +00:00
|
|
|
}
|
|
|
|
|
|
2026-03-13 07:44:11 +00:00
|
|
|
resp, err := h.proxy.HTTPClient.Do(req)
|
2026-01-20 21:52:44 +00:00
|
|
|
if err != nil {
|
2026-10-15 13:35:12 +01:00
|
|
|
if writeUpstreamHeld(w, err) {
|
|
|
|
|
return
|
|
|
|
|
}
|
2026-01-20 21:52:44 +00:00
|
|
|
h.proxy.Logger.Error("upstream request failed", "error", err)
|
|
|
|
|
http.Error(w, "upstream request failed", http.StatusBadGateway)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
defer func() { _ = resp.Body.Close() }()
|
|
|
|
|
|
2026-09-26 15:44:28 +05:30
|
|
|
h.proxy.relayResponse(w, r, resp, nil)
|
2026-01-20 21:52:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// buildUpstreamURL constructs the upstream URL for a request.
|
|
|
|
|
func (h *NuGetHandler) buildUpstreamURL(r *http.Request) string {
|
|
|
|
|
path := r.URL.Path
|
|
|
|
|
|
|
|
|
|
// Handle query and autocomplete which go to azuresearch
|
|
|
|
|
if strings.HasPrefix(path, "/query") || strings.HasPrefix(path, "/autocomplete") {
|
2026-08-29 11:19:54 +01:00
|
|
|
return h.searchURL + path + "?" + r.URL.RawQuery
|
2026-01-20 21:52:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return h.upstreamURL + path
|
|
|
|
|
}
|