Watch
1
0
Fork
You've already forked pkg-proxy
1
mirror of https://github.com/git-pkgs/proxy.git synced 2026-09-16 15:52:05 -04:00
pkg-proxy/internal/handler/homebrew.go

74 lines
2.4 KiB
Go
Raw Permalink Normal View History

2026-09-03 16:59:12 +01:00
package handler
import (
"crypto/sha256"
"encoding/hex"
"net/http"
"strings"
)
const (
homebrewArtifactNamespace = "homebrew"
homebrewArtifactRepository = "homebrew/core"
homebrewMetadataEcosystem = "homebrew"
)
// HomebrewHandler proxies Homebrew's JSON API without modifying signed files.
type HomebrewHandler struct {
proxy *Proxy
apiUpstream string
}
// NewHomebrewHandler creates a Homebrew JSON API handler.
func NewHomebrewHandler(proxy *Proxy, apiUpstream string) *HomebrewHandler {
return &HomebrewHandler{
proxy: proxy,
apiUpstream: strings.TrimSuffix(apiUpstream, "/"),
}
}
// RegisterHomebrewArtifacts routes homebrew/core OCI requests to its configured
// registry and blocks other homebrew repositories from reaching the default
// OCI registry.
func RegisterHomebrewArtifacts(container *ContainerHandler, artifactUpstream string) {
container.BlockRegistry(homebrewArtifactNamespace)
container.RegisterRegistry(homebrewArtifactRepository, artifactUpstream)
}
// Routes returns the Homebrew JSON API handler. Mount this at /homebrew.
func (h *HomebrewHandler) Routes() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
w.Header().Set("Allow", "GET, HEAD")
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
requestPath := strings.TrimPrefix(r.URL.EscapedPath(), "/")
if requestPath == "" || containsPathTraversal(requestPath) {
http.NotFound(w, r)
return
}
upstreamURL := h.apiUpstream + "/" + requestPath
if r.URL.RawQuery != "" {
upstreamURL += "?" + r.URL.RawQuery
}
fix(handler): let the ProxyCached path request a per-call Accept-Encoding (#324) * fix(handler): fetch conda repodata gzip-compressed on both hops #304 made the ProxyCached path request Accept-Encoding: identity so the metadata cache stores upstream bytes verbatim. That is required for the signed / hash-pinned index ecosystems, but conda's repodata.json is large plain JSON: linux-64 repodata.json is ~441 MB uncompressed (over the metadata_max_size cap, so it 502s today) versus ~34 MB gzip. Replace the ProxyCached path's verbatim bool with an explicit acceptEncoding string ('' = leave unset / transparent, 'identity', or 'gzip'), reusing #304's existing store-and-replay of Content-Encoding unchanged. ProxyCached keeps its exported signature and continues to send identity, so the nine other ecosystems and helm/maven are untouched; only conda's repodata.json / current_repodata.json now request gzip. Setting Accept-Encoding explicitly disables Go's transparent decompression, so the compressed bytes and the Content-Encoding: gzip header are cached and replayed exactly as identity bytes are. conda, mamba and pixi solicit and decode gzip on .json URLs; repodata.json.bz2 stays identity. Fixes #305 * fix(handler): pass metadata content-encoding with the body it describes The adversarial review of the conda gzip route found a reachable regression: writeMetadataCachedResponse took Content-Encoding from a fresh cache-row read while cacheMetadataBlob skips the row write when Storage.Store fails. Under identity that was benign (the body was plain anyway), but on the new gzip route a disk-full or object-store outage served raw gzip bytes as Content-Type: application/json with no Content-Encoding and HTTP 200 -- conda, mamba and pixi fail to parse them, with no HTTP signal and only a Warn log, on every request until a cache write succeeds. fetchOrCacheMetadata now returns the encoding of the body it hands back (the upstream value on a fetch, the stored row's value on a TTL hit or stale fallback) and proxyCachedWithEncoding passes it to writeMetadataCachedResponse, so the header always describes the bytes actually written. cachedMeta drops its now-unused content_encoding field. helm and maven pass "" -- both fetch transparently, so their stored encoding was always empty and behaviour is unchanged. Also fixes a vacuous assertion in the new conda test: the upstream request counter incremented behind the availability gate, so the cached-replay block could never observe a refetch. * fix(handler): pin the stale-fallback content-encoding and drop a dead guard Follow-ups from the adversarial review of the #305 branch, limited to code this branch introduced: - proxyMetadataStream is only ever reached with an explicit Accept-Encoding (ProxyCached passes identity, conda passes gzip or identity), so the guard around the header set was unreachable; replace it with the plain one-token substitution of the former literal, which is the smallest change from main. - The stale-fallback return of fetchOrCacheMetadata (encoding taken from the cache row) was the one #305 return site no test pinned: replacing it with an empty encoding survived the whole suite. Add a conda test that expires the entry, fails the upstream, and asserts the stored gzip blob is served with Content-Encoding: gzip. Not changed, by scope: cacheMetadataBlob still discards the UpsertMetadataCache error (pre-existing on main). If Storage.Store succeeds and the row write fails, a later stale fallback or TTL hit can serve the gzip blob with the row's stale encoding; that needs a DB write failure plus a second event and is tracked separately. * fix(handler): restore the pre-existing cachedMeta content-encoding field The third adversarial review classified deleting cachedMeta.contentEncoding and its lookupCachedMeta populate as elective: neither line was created by this branch nor forced by the fix (writeMetadataCachedResponse now reads the encoding from its parameter and ignores the row value). Under the rule that pre-existing code this branch did not have to touch stays untouched, restore both as they are on main. No behaviour change. Residuals the review documented, unchanged by scope (both share one root cause: the encoding lives in the cache row and the bytes in the blob, and neither is written or read atomically): - cacheMetadataBlob discards the UpsertMetadataCache error, so after a successful gzip Store and a failed row write a later stale fallback or TTL hit can serve the gzip blob with the row's stale encoding. - During the one-time identity->gzip rollout, a request that read a pre-branch identity row, lost the upstream race to a request that stored the gzip blob, and then failed upstream serves the gzip bytes with no Content-Encoding for that one response; later requests self-heal. - helm and maven now pass an empty encoding; on main a spec-violating upstream that answered a transparent gzip request with an encoding Go does not decode (e.g. br) would have had that header replayed from the row. Degenerate; documented rather than changed. * fix(handler): keep conda's proxyCached and .bz2 route as on main Threading acceptEncoding through CondaHandler.proxyCached changed the form of two pieces of original code the fix did not need to touch: the repodata.json.bz2 route (method value rewritten as a closure) and proxyCached itself (new parameter, new call). Restore both exactly as on main; ProxyCached still sends identity, so the .bz2 route is unchanged in behaviour. handleRepodata's non-cooldown branch now derives the cache key inline and calls proxyCachedWithEncoding with gzip directly, so the only original conda.go line that changes is that one call. * fix(handler): keep writeMetadataCachedResponse and its callers as on main Adding a contentEncoding parameter to writeMetadataCachedResponse changed a signature that predates #304 and dragged its two pre-#304 callers (helm.go, maven.go) into the diff, even though #304 only ever added the cm.contentEncoding block inside the function body. Restore writeMetadataCachedResponse's doc and signature exactly as on main and make it a delegate that passes an empty encoding to a new unexported writeMetadataCachedResponseWithEncoding, which carries the original body with #304's block reading the parameter instead of the cache row. proxyCachedWithEncoding calls the sibling with the encoding returned alongside the body. helm.go and maven.go drop out of the diff; their behaviour is unchanged (both fetch transparently, so their stored encoding was always empty). Same split pattern as ProxyCached -> proxyCachedWithEncoding. * fix(handler): move the conda gzip change to its own branch The conda call site in handleRepodata predates #304 and #304 never touched it, so under the rule that this PR only corrects code and behaviour #304 introduced it does not belong here. Restore conda.go and conda_test.go as on main; the conda change continues on a stacked branch against its own issue. Replace the conda-route tests with tests that exercise proxyCachedWithEncoding directly, so this PR still pins its own plumbing: gzip is requested and the compressed bytes plus Content-Encoding are cached and replayed (cached and streaming paths), the header survives a metadata cache write failure, and the stale fallback keeps the stored encoding. * fix(homebrew): fetch the JSON API gzip-compressed on both hops Homebrew (#254) routes every API path through ProxyCached and so, since #304, fetches formula.jws.json (~33 MB plain, ~5 MB gzip) uncompressed on every refresh -- the case that motivated #305. Request gzip for the JSON API via proxyCachedWithEncoding: brew fetches every API download with curl --compressed and decodes Content-Encoding itself, so the compressed bytes and header are cached and served as-is and both hops stay compressed. The analytics endpoints are the one brew consumer fetched without --compressed; they stay on identity. * fix(handler): leave Accept-Encoding unset in proxyMetadataStream for an empty value fetchUpstreamMetadata treats an empty acceptEncoding as 'do not set the header'; proxyMetadataStream set it unconditionally, which would send an empty Accept-Encoding line if a caller ever passed . Guard it the same way so both paths agree. No caller passes today. * fix(handler): keep the metadata row and blob from describing different bytes Two ways the cache row could stop describing the stored blob once a caller requests gzip, both raised by the review of #324: - cacheMetadataBlob stored the blob and then discarded the UpsertMetadataCache error. After a successful gzip store and a failed row write, a later TTL hit or stale fallback served the gzip blob with the previous row's encoding. On a row-write failure, log it and delete the blob just written, so the next request refetches instead. - fetchOrCacheMetadata read the row once up front and reused it for the stale fallback. A request that read an identity row, lost the upstream race to a request that stored the gzip blob, and then failed upstream labelled the new blob with the old row. Re-read the row before falling back so the encoding matches the blob as it is now. Both only become harmful with an encoding change, which this branch introduces; the pre-existing validator-from-row read is tracked separately. * Drop unused cachedMeta.contentEncoding and fix stale doc reference The field was added by #304 and its only reader is replaced in this branch by the encoding parameter passed alongside the body. The proxyCachedWithEncoding comment named conda repodata, which was moved out of this branch in 5991d95; Homebrew is the caller that ships here. --------- Co-authored-by: Andrew Nesbitt <andrewnez@gmail.com>
2026-09-15 20:50:44 +02:00
// brew fetches every JSON API download with `curl --compressed` and
// decodes Content-Encoding itself, and formula.jws.json is ~33 MB plain
// versus ~5 MB gzip, so keep both hops compressed. The analytics
// endpoints are the one consumer brew fetches without --compressed;
// they stay identity.
acceptEncoding := "gzip"
if strings.HasPrefix(requestPath, "analytics/") {
acceptEncoding = "identity"
}
h.proxy.proxyCachedWithEncoding(w, r, upstreamURL, homebrewMetadataEcosystem, homebrewMetadataCacheKey(requestPath, r.URL.RawQuery), acceptEncoding, "*/*")
2026-09-03 16:59:12 +01:00
})
}
func homebrewMetadataCacheKey(requestPath, rawQuery string) string {
sum := sha256.Sum256([]byte(requestPath + "\x00" + rawQuery))
return hex.EncodeToString(sum[:])
}