mirror of
https://github.com/git-pkgs/proxy.git
synced 2026-11-09 23:59:16 -05:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac137a3289 |
15 changed files with 1083 additions and 754 deletions
|
|
@ -13,6 +13,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/git-pkgs/proxy/internal/jsonscan"
|
||||||
"github.com/git-pkgs/proxy/internal/retention"
|
"github.com/git-pkgs/proxy/internal/retention"
|
||||||
"github.com/git-pkgs/purl"
|
"github.com/git-pkgs/purl"
|
||||||
)
|
)
|
||||||
|
|
@ -226,7 +227,7 @@ func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage str
|
||||||
if !json.Valid(body) {
|
if !json.Valid(body) {
|
||||||
return nil, nil, errors.New("composer metadata is not valid JSON")
|
return nil, nil, errors.New("composer metadata is not valid JSON")
|
||||||
}
|
}
|
||||||
format, _, err := lookupJSONString(body, "minified")
|
format, _, err := jsonscan.LookupString(body, "minified")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -236,17 +237,17 @@ func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage str
|
||||||
var kept []string
|
var kept []string
|
||||||
listed := false
|
listed := false
|
||||||
out.Grow(len(body) + len(body)/8)
|
out.Grow(len(body) + len(body)/8)
|
||||||
err = rewriteJSONMembers(&out, body, func(out *bytes.Buffer, m jsonMember) (bool, error) {
|
err = jsonscan.RewriteMembers(&out, body, func(out *bytes.Buffer, m jsonscan.Member) (bool, error) {
|
||||||
packages := m.value(body)
|
packages := m.Value(body)
|
||||||
if !jsonKeyIs(m.key(body), "packages") || packages[0] != '{' {
|
if !jsonscan.KeyIs(m.Key(body), "packages") || packages[0] != '{' {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
return true, rewriteJSONMembers(out, packages, func(out *bytes.Buffer, p jsonMember) (bool, error) {
|
return true, jsonscan.RewriteMembers(out, packages, func(out *bytes.Buffer, p jsonscan.Member) (bool, error) {
|
||||||
versions := p.value(packages)
|
versions := p.Value(packages)
|
||||||
if versions[0] != '[' {
|
if versions[0] != '[' {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
packageName, err := jsonKey(p.key(packages))
|
packageName, err := jsonscan.Key(p.Key(packages))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
@ -286,8 +287,8 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v
|
||||||
_, routed := h.routes.match(packageName)
|
_, routed := h.routes.match(packageName)
|
||||||
|
|
||||||
out.WriteByte('[')
|
out.WriteByte('[')
|
||||||
err := forEachJSONElement(versions, func(entry []byte) error {
|
err := jsonscan.EachElement(versions, func(entry []byte) error {
|
||||||
if minified && jsonStringIs(entry, composerDevReset) {
|
if minified && jsonscan.StringIs(entry, composerDevReset) {
|
||||||
upstream.reset()
|
upstream.reset()
|
||||||
written.reset()
|
written.reset()
|
||||||
devReset = true
|
devReset = true
|
||||||
|
|
@ -428,15 +429,15 @@ func disableDefaultNotification(fields *composerFields) {
|
||||||
// rewriteDist returns a version's dist object with its url pointing at this
|
// rewriteDist returns a version's dist object with its url pointing at this
|
||||||
// proxy. A dist without a string url is returned unchanged.
|
// proxy. A dist without a string url is returned unchanged.
|
||||||
func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte {
|
func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte {
|
||||||
urlMember, ok, err := findJSONMember(dist, "url")
|
urlMember, ok, err := jsonscan.FindMember(dist, "url")
|
||||||
if err != nil || !ok || dist[urlMember.valStart] != '"' {
|
if err != nil || !ok || dist[urlMember.ValStart] != '"' {
|
||||||
return dist
|
return dist
|
||||||
}
|
}
|
||||||
var upstreamURL string
|
var upstreamURL string
|
||||||
if err := json.Unmarshal(urlMember.value(dist), &upstreamURL); err != nil {
|
if err := json.Unmarshal(urlMember.Value(dist), &upstreamURL); err != nil {
|
||||||
return dist
|
return dist
|
||||||
}
|
}
|
||||||
distType, _, _ := lookupJSONString(dist, "type")
|
distType, _, _ := jsonscan.LookupString(dist, "type")
|
||||||
|
|
||||||
proxyURL, ok := h.proxyDistURL(packageName, version, upstreamURL, distType)
|
proxyURL, ok := h.proxyDistURL(packageName, version, upstreamURL, distType)
|
||||||
if !ok {
|
if !ok {
|
||||||
|
|
@ -446,10 +447,10 @@ func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return dist
|
return dist
|
||||||
}
|
}
|
||||||
rewritten := make([]byte, 0, len(dist)-len(urlMember.value(dist))+len(encoded))
|
rewritten := make([]byte, 0, len(dist)-len(urlMember.Value(dist))+len(encoded))
|
||||||
rewritten = append(rewritten, dist[:urlMember.valStart]...)
|
rewritten = append(rewritten, dist[:urlMember.ValStart]...)
|
||||||
rewritten = append(rewritten, encoded...)
|
rewritten = append(rewritten, encoded...)
|
||||||
return append(rewritten, dist[urlMember.valEnd:]...)
|
return append(rewritten, dist[urlMember.ValEnd:]...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// proxyDistURL returns the URL this proxy serves a version's archive at, or
|
// proxyDistURL returns the URL this proxy serves a version's archive at, or
|
||||||
|
|
@ -593,9 +594,9 @@ func (f *composerFields) set(name string, key, value []byte) {
|
||||||
// apply overlays the members of a version entry. In minified metadata the
|
// apply overlays the members of a version entry. In minified metadata the
|
||||||
// value "__unset" removes the field instead.
|
// value "__unset" removes the field instead.
|
||||||
func (f *composerFields) apply(entry []byte, minified bool) error {
|
func (f *composerFields) apply(entry []byte, minified bool) error {
|
||||||
return forEachJSONMember(entry, func(m jsonMember) error {
|
return jsonscan.EachMember(entry, func(m jsonscan.Member) error {
|
||||||
key, value := m.key(entry), m.value(entry)
|
key, value := m.Key(entry), m.Value(entry)
|
||||||
if minified && jsonStringIs(value, composerUnset) {
|
if minified && jsonscan.StringIs(value, composerUnset) {
|
||||||
value = nil
|
value = nil
|
||||||
}
|
}
|
||||||
if bytes.IndexByte(key, '\\') < 0 {
|
if bytes.IndexByte(key, '\\') < 0 {
|
||||||
|
|
@ -604,7 +605,7 @@ func (f *composerFields) apply(entry []byte, minified bool) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
name, err := jsonKey(key)
|
name, err := jsonscan.Key(key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
@ -837,11 +838,11 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU
|
||||||
// the minified format as it needs, since it runs on each archive the proxy
|
// the minified format as it needs, since it runs on each archive the proxy
|
||||||
// has not cached yet.
|
// has not cached yet.
|
||||||
func composerDistURL(body []byte, packageName, version string) (composerDist, error) {
|
func composerDistURL(body []byte, packageName, version string) (composerDist, error) {
|
||||||
format, _, err := lookupJSONString(body, "minified")
|
format, _, err := jsonscan.LookupString(body, "minified")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return composerDist{}, err
|
return composerDist{}, err
|
||||||
}
|
}
|
||||||
versions, err := lookupJSON(body, "packages", packageName)
|
versions, err := jsonscan.Lookup(body, "packages", packageName)
|
||||||
if err != nil || len(versions) == 0 || versions[0] != '[' {
|
if err != nil || len(versions) == 0 || versions[0] != '[' {
|
||||||
return composerDist{}, err
|
return composerDist{}, err
|
||||||
}
|
}
|
||||||
|
|
@ -849,8 +850,8 @@ func composerDistURL(body []byte, packageName, version string) (composerDist, er
|
||||||
|
|
||||||
fields := newComposerFields()
|
fields := newComposerFields()
|
||||||
var dist composerDist
|
var dist composerDist
|
||||||
err = forEachJSONElement(versions, func(entry []byte) error {
|
err = jsonscan.EachElement(versions, func(entry []byte) error {
|
||||||
if minified && jsonStringIs(entry, composerDevReset) {
|
if minified && jsonscan.StringIs(entry, composerDevReset) {
|
||||||
fields.reset()
|
fields.reset()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
@ -863,20 +864,20 @@ func composerDistURL(body []byte, packageName, version string) (composerDist, er
|
||||||
if err := fields.apply(entry, minified); err != nil {
|
if err := fields.apply(entry, minified); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if !jsonStringIs(fields.get("version"), version) {
|
if !jsonscan.StringIs(fields.get("version"), version) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
distRaw := fields.get("dist")
|
distRaw := fields.get("dist")
|
||||||
dist.url, _, _ = lookupJSONString(distRaw, "url")
|
dist.url, _, _ = jsonscan.LookupString(distRaw, "url")
|
||||||
if dist.url != "" {
|
if dist.url != "" {
|
||||||
dist.distType, _, _ = lookupJSONString(distRaw, "type")
|
dist.distType, _, _ = jsonscan.LookupString(distRaw, "type")
|
||||||
dist.shasum, _, _ = lookupJSONString(distRaw, "shasum")
|
dist.shasum, _, _ = jsonscan.LookupString(distRaw, "shasum")
|
||||||
dist.published = fields.get("time")
|
dist.published = fields.get("time")
|
||||||
return errStopScan
|
return jsonscan.ErrStop
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if errors.Is(err, errStopScan) {
|
if errors.Is(err, jsonscan.ErrStop) {
|
||||||
err = nil
|
err = nil
|
||||||
}
|
}
|
||||||
return dist, err
|
return dist, err
|
||||||
|
|
|
||||||
|
|
@ -79,7 +79,7 @@ func TestNPMDenylistMetadata(t *testing.T) {
|
||||||
h := NewNPMHandler(p, "https://proxy.example", "")
|
h := NewNPMHandler(p, "https://proxy.example", "")
|
||||||
for _, times := range []string{"", `,"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z"}`} {
|
for _, times := range []string{"", `,"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z"}`} {
|
||||||
body := `{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0-beta.1":{}},"dist-tags":{"latest":"2.0.0","beta":"2.0.0","next":"3.0.0-beta.1"}` + times + `}`
|
body := `{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0-beta.1":{}},"dist-tags":{"latest":"2.0.0","beta":"2.0.0","next":"3.0.0-beta.1"}` + times + `}`
|
||||||
out, err := h.rewriteMetadata("@scope/demo", []byte(body))
|
out, _, err := h.rewriteMetadataKeeping("@scope/demo", []byte(body))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
@ -87,7 +87,7 @@ func TestNPMDenylistMetadata(t *testing.T) {
|
||||||
t.Fatalf("incorrect metadata: %s", out)
|
t.Fatalf("incorrect metadata: %s", out)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out, err := h.rewriteMetadata("@scope/demo", []byte(`{"versions":{"2.0.0":{}},"dist-tags":{"latest":"2.0.0"}}`))
|
out, _, err := h.rewriteMetadataKeeping("@scope/demo", []byte(`{"versions":{"2.0.0":{}},"dist-tags":{"latest":"2.0.0"}}`))
|
||||||
if err != nil || strings.Contains(string(out), "2.0.0") {
|
if err != nil || strings.Contains(string(out), "2.0.0") {
|
||||||
t.Fatalf("all denied: %s, %v", out, err)
|
t.Fatalf("all denied: %s, %v", out, err)
|
||||||
}
|
}
|
||||||
|
|
@ -205,7 +205,7 @@ func TestDenylistAndCooldownCompose(t *testing.T) {
|
||||||
setTestDenylist(t, p, "pkg:npm/demo@1.0.0")
|
setTestDenylist(t, p, "pkg:npm/demo@1.0.0")
|
||||||
h := NewNPMHandler(p, "https://proxy.test", "")
|
h := NewNPMHandler(p, "https://proxy.test", "")
|
||||||
body := fmt.Sprintf(`{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0":{}},"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z","3.0.0":%q},"dist-tags":{"latest":"3.0.0"}}`, time.Now().UTC().Format(time.RFC3339))
|
body := fmt.Sprintf(`{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0":{}},"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z","3.0.0":%q},"dist-tags":{"latest":"3.0.0"}}`, time.Now().UTC().Format(time.RFC3339))
|
||||||
out, err := h.rewriteMetadata("demo", []byte(body))
|
out, _, err := h.rewriteMetadataKeeping("demo", []byte(body))
|
||||||
if err != nil || strings.Contains(string(out), "1.0.0") || strings.Contains(string(out), "3.0.0") || !strings.Contains(string(out), `"latest":"2.0.0"`) {
|
if err != nil || strings.Contains(string(out), "1.0.0") || strings.Contains(string(out), "3.0.0") || !strings.Contains(string(out), `"latest":"2.0.0"`) {
|
||||||
t.Fatalf("combined policy: %s, %v", out, err)
|
t.Fatalf("combined policy: %s, %v", out, err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,21 +2,18 @@ package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"maps"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"reflect"
|
|
||||||
"slices"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/git-pkgs/integrity"
|
"github.com/git-pkgs/integrity"
|
||||||
|
"github.com/git-pkgs/proxy/internal/jsonscan"
|
||||||
|
"github.com/git-pkgs/proxy/internal/metadata/npm"
|
||||||
"github.com/git-pkgs/proxy/internal/retention"
|
"github.com/git-pkgs/proxy/internal/retention"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -360,444 +357,51 @@ func (h *NPMHandler) npmVersionMetadataUsesFilteredPackument(packageName string)
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// rewriteNPMVersionFromPackument picks one version entry out of an already
|
|
||||||
// filtered packument and rewrites its tarball URL.
|
|
||||||
func (h *NPMHandler) rewriteNPMVersionFromPackument(packageName, specifier string, packument []byte) ([]byte, error) {
|
func (h *NPMHandler) rewriteNPMVersionFromPackument(packageName, specifier string, packument []byte) ([]byte, error) {
|
||||||
filtered, err := h.rewriteMetadata(packageName, packument)
|
out, err := npm.Version(packageName, specifier, packument, h.npmRewriteOptions(packageName, nil))
|
||||||
if err != nil {
|
return out, npmRewriteError(err)
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
target := specifier
|
|
||||||
if resolved, ok, err := lookupJSONString(filtered, "dist-tags", specifier); err != nil {
|
|
||||||
return nil, err
|
|
||||||
} else if ok {
|
|
||||||
target = resolved
|
|
||||||
}
|
|
||||||
|
|
||||||
if h.proxy.versionDenied("npm", packageName, target) {
|
|
||||||
return nil, ErrVersionDenied
|
|
||||||
}
|
|
||||||
|
|
||||||
versionRaw, err := lookupJSON(filtered, "versions", target)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(versionRaw) == 0 {
|
|
||||||
return nil, ErrVersionNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
h.writeNPMVersion(&out, packageName, target, versionRaw)
|
|
||||||
return out.Bytes(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// rewriteMetadata rewrites tarball URLs in npm package metadata to point at this proxy.
|
|
||||||
// If cooldown is enabled, versions published too recently are filtered out.
|
|
||||||
//
|
|
||||||
// The document is never decoded as a whole. A full packument can run to tens
|
|
||||||
// of megabytes, and decoding it into generic maps took many times that in
|
|
||||||
// memory for every rewrite. Instead the version names, time map and dist-tags
|
|
||||||
// are decoded to decide what to keep, and the response is assembled from
|
|
||||||
// slices of the original bytes: only each version's tarball URL, and the time
|
|
||||||
// map and dist-tags when filtering changed them, are written fresh. Everything
|
|
||||||
// else reaches the client byte for byte as upstream sent it.
|
|
||||||
func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, error) {
|
|
||||||
out, _, err := h.rewriteMetadataKeeping(packageName, body)
|
|
||||||
return out, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// rewriteMetadataKeeping is rewriteMetadata that also reports the versions it
|
|
||||||
// kept.
|
|
||||||
func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([]byte, []string, error) {
|
func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([]byte, []string, error) {
|
||||||
return h.rewriteMetadataWithLocal(packageName, body, nil)
|
return h.rewriteMetadataWithLocal(packageName, body, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
// rewriteMetadataWithLocal is rewriteMetadataKeeping that also lists the
|
|
||||||
// pushed versions in local, in place of any upstream entry with the same
|
|
||||||
// version. Cooldown and the denylist only filter upstream versions, and
|
|
||||||
// dist-tags stay as upstream has them.
|
|
||||||
func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, local []npmLocalEntry) ([]byte, []string, error) {
|
func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, local []npmLocalEntry) ([]byte, []string, error) {
|
||||||
doc, err := indexNPMPackument(body)
|
out, kept, err := npm.Rewrite(packageName, body, h.npmRewriteOptions(packageName, local))
|
||||||
if err != nil {
|
return out, kept, npmRewriteError(err)
|
||||||
return nil, nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
versionsRaw := doc.value(doc.versionsAt)
|
func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry) npm.Options {
|
||||||
if len(versionsRaw) == 0 || versionsRaw[0] != '{' {
|
return npm.Options{
|
||||||
version, ok, err := lookupJSONString(body, "version")
|
ProxyURL: h.proxyURL,
|
||||||
if err != nil {
|
Cooldown: h.proxy.Cooldown,
|
||||||
return nil, nil, err
|
DeniedVersions: h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName)),
|
||||||
|
EvaluatedAt: time.Now(),
|
||||||
|
LocalVersions: local,
|
||||||
|
OnTarballRewrite: func(version, oldURL, newURL string) {
|
||||||
|
h.proxy.Logger.Debug("rewrote tarball URL", "package", packageName, "version", version,
|
||||||
|
"old", oldURL, "new", newURL)
|
||||||
|
},
|
||||||
|
OnCooldownFilter: func(version, publishedAt string) {
|
||||||
|
h.proxy.Logger.Info("cooldown: filtering npm version", "package", packageName, "version", version,
|
||||||
|
"published", publishedAt)
|
||||||
|
},
|
||||||
}
|
}
|
||||||
if ok {
|
|
||||||
if h.proxy.versionDenied("npm", packageName, version) {
|
|
||||||
return nil, nil, ErrVersionDenied
|
|
||||||
}
|
|
||||||
var out bytes.Buffer
|
|
||||||
h.writeNPMVersion(&out, packageName, version, body)
|
|
||||||
return out.Bytes(), []string{version}, nil
|
|
||||||
}
|
|
||||||
if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 {
|
|
||||||
return nil, nil, errors.New("npm metadata has no versions object")
|
|
||||||
}
|
|
||||||
return body, nil, nil // No versions to rewrite
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The filters only look at which versions exist, so they run on a map of
|
func npmRewriteError(err error) error {
|
||||||
// version names alongside the decoded time map and dist-tags.
|
|
||||||
versions := map[string]any{}
|
|
||||||
if err := forEachJSONMember(versionsRaw, func(m jsonMember) error {
|
|
||||||
version, err := jsonKey(m.key(versionsRaw))
|
|
||||||
versions[version] = nil
|
|
||||||
return err
|
|
||||||
}); err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
metadata := map[string]any{}
|
|
||||||
timeMap := doc.decodeObject(doc.timeAt, metadata, "time")
|
|
||||||
distTags := doc.decodeObject(doc.tagsAt, metadata, "dist-tags")
|
|
||||||
timeEntries := len(timeMap)
|
|
||||||
originalTags := maps.Clone(distTags)
|
|
||||||
|
|
||||||
h.applyCooldownFiltering(metadata, versions, packageName)
|
|
||||||
h.applyDenylistFiltering(metadata, versions, packageName)
|
|
||||||
for _, e := range local {
|
|
||||||
delete(versions, e.version)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Untouched members are copied as they are; the three the filters
|
|
||||||
// handled are written from their filtered state, and only when it changed.
|
|
||||||
out, err := doc.write(func(out *bytes.Buffer, i int) error {
|
|
||||||
switch {
|
switch {
|
||||||
case i == doc.versionsAt:
|
case errors.Is(err, npm.ErrVersionDenied):
|
||||||
return h.writeNPMVersions(out, packageName, versionsRaw, versions, local)
|
return ErrVersionDenied
|
||||||
case i == doc.timeAt && timeMap != nil && (len(timeMap) != timeEntries || len(local) > 0):
|
case errors.Is(err, npm.ErrVersionNotFound):
|
||||||
pushedAt := make([][2]string, 0, len(local))
|
return ErrVersionNotFound
|
||||||
for _, e := range local {
|
|
||||||
pushedAt = append(pushedAt, [2]string{e.version, e.time})
|
|
||||||
}
|
|
||||||
return writeFilteredJSONObjectWith(out, doc.value(i), func(key string) bool {
|
|
||||||
_, ok := timeMap[key]
|
|
||||||
_, pushed := findNPMLocalEntry(local, key)
|
|
||||||
return ok && !pushed
|
|
||||||
}, pushedAt)
|
|
||||||
case i == doc.tagsAt && distTags != nil && !reflect.DeepEqual(distTags, originalTags):
|
|
||||||
encoded, err := json.Marshal(distTags)
|
|
||||||
out.Write(encoded)
|
|
||||||
return err
|
|
||||||
default:
|
default:
|
||||||
out.Write(doc.value(i))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
})
|
|
||||||
kept := slices.Collect(maps.Keys(versions))
|
|
||||||
for _, e := range local {
|
|
||||||
kept = append(kept, e.version)
|
|
||||||
}
|
|
||||||
return out, kept, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// npmPackument is a packument's top-level members as offsets into its bytes,
|
|
||||||
// with the positions of the members rewriteMetadata filters, or -1 for those
|
|
||||||
// it lacks. A key that repeats is recorded at its last occurrence, the one
|
|
||||||
// JSON.parse keeps.
|
|
||||||
type npmPackument struct {
|
|
||||||
body []byte
|
|
||||||
members []jsonMember
|
|
||||||
versionsAt, timeAt, tagsAt int
|
|
||||||
}
|
|
||||||
|
|
||||||
func indexNPMPackument(body []byte) (*npmPackument, error) {
|
|
||||||
if !json.Valid(body) {
|
|
||||||
return nil, errors.New("npm metadata is not valid JSON")
|
|
||||||
}
|
|
||||||
doc := &npmPackument{body: body, versionsAt: -1, timeAt: -1, tagsAt: -1}
|
|
||||||
err := forEachJSONMember(body, func(m jsonMember) error {
|
|
||||||
switch key := m.key(body); {
|
|
||||||
case jsonKeyIs(key, "versions"):
|
|
||||||
doc.versionsAt = len(doc.members)
|
|
||||||
case jsonKeyIs(key, "time"):
|
|
||||||
doc.timeAt = len(doc.members)
|
|
||||||
case jsonKeyIs(key, "dist-tags"):
|
|
||||||
doc.tagsAt = len(doc.members)
|
|
||||||
}
|
|
||||||
doc.members = append(doc.members, m)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
return doc, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// value returns the raw value of member i, or nil when i is -1.
|
|
||||||
func (d *npmPackument) value(i int) []byte {
|
|
||||||
if i < 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return d.members[i].value(d.body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// decodeObject decodes member i into metadata[name] and returns it when it is
|
|
||||||
// an object. The filters edit the returned map in place.
|
|
||||||
func (d *npmPackument) decodeObject(i int, metadata map[string]any, name string) map[string]any {
|
|
||||||
raw := d.value(i)
|
|
||||||
if raw == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var value any
|
|
||||||
if err := json.Unmarshal(raw, &value); err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
metadata[name] = value
|
|
||||||
object, _ := value.(map[string]any)
|
|
||||||
return object
|
|
||||||
}
|
|
||||||
|
|
||||||
// write assembles the document in member order, with writeValue writing each
|
|
||||||
// member's value. Earlier duplicates of the filtered keys are dropped: they
|
|
||||||
// would carry unfiltered data, and JSON.parse ignores them anyway.
|
|
||||||
func (d *npmPackument) write(writeValue func(out *bytes.Buffer, i int) error) ([]byte, error) {
|
|
||||||
var out bytes.Buffer
|
|
||||||
out.Grow(len(d.body) + len(d.body)/8)
|
|
||||||
out.WriteByte('{')
|
|
||||||
first := true
|
|
||||||
for i, m := range d.members {
|
|
||||||
if i != d.versionsAt && i != d.timeAt && i != d.tagsAt {
|
|
||||||
key := m.key(d.body)
|
|
||||||
if jsonKeyIs(key, "versions") || jsonKeyIs(key, "time") || jsonKeyIs(key, "dist-tags") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !first {
|
|
||||||
out.WriteByte(',')
|
|
||||||
}
|
|
||||||
first = false
|
|
||||||
out.Write(m.key(d.body))
|
|
||||||
out.WriteByte(':')
|
|
||||||
if err := writeValue(&out, i); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out.WriteByte('}')
|
|
||||||
return out.Bytes(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeNPMVersions writes the versions object, keeping the versions still in
|
|
||||||
// keep and pointing each kept version's tarball at this proxy, followed by the
|
|
||||||
// pushed versions in local.
|
|
||||||
func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any, local []npmLocalEntry) error {
|
|
||||||
out.WriteByte('{')
|
|
||||||
first := true
|
|
||||||
err := forEachJSONMember(versionsRaw, func(m jsonMember) error {
|
|
||||||
version, err := jsonKey(m.key(versionsRaw))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if _, ok := keep[version]; !ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !first {
|
|
||||||
out.WriteByte(',')
|
|
||||||
}
|
|
||||||
first = false
|
|
||||||
out.Write(m.key(versionsRaw))
|
|
||||||
out.WriteByte(':')
|
|
||||||
h.writeNPMVersion(out, packageName, version, m.value(versionsRaw))
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
for _, e := range local {
|
|
||||||
if !first {
|
|
||||||
out.WriteByte(',')
|
|
||||||
}
|
|
||||||
first = false
|
|
||||||
key, _ := json.Marshal(e.version)
|
|
||||||
out.Write(key)
|
|
||||||
out.WriteByte(':')
|
|
||||||
out.Write(e.entry)
|
|
||||||
}
|
|
||||||
out.WriteByte('}')
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeNPMVersion writes one version entry with its dist.tarball pointing at
|
|
||||||
// this proxy. An entry without a string tarball is written unchanged.
|
|
||||||
func (h *NPMHandler) writeNPMVersion(out *bytes.Buffer, packageName, version string, raw []byte) {
|
|
||||||
dist, ok, err := findJSONMember(raw, "dist")
|
|
||||||
if err != nil || !ok {
|
|
||||||
out.Write(raw)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
distRaw := dist.value(raw)
|
|
||||||
tarball, ok, err := findJSONMember(distRaw, "tarball")
|
|
||||||
if err != nil || !ok || distRaw[tarball.valStart] != '"' {
|
|
||||||
out.Write(raw)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var oldTarball string
|
|
||||||
if err := json.Unmarshal(tarball.value(distRaw), &oldTarball); err != nil {
|
|
||||||
out.Write(raw)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
newTarball := h.proxyTarballURL(packageName, version, oldTarball)
|
|
||||||
encoded, err := json.Marshal(newTarball)
|
|
||||||
if err != nil {
|
|
||||||
out.Write(raw)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
out.Write(raw[:dist.valStart+tarball.valStart])
|
|
||||||
out.Write(encoded)
|
|
||||||
out.Write(raw[dist.valStart+tarball.valEnd:])
|
|
||||||
|
|
||||||
h.proxy.Logger.Debug("rewrote tarball URL",
|
|
||||||
"package", packageName, "version", version,
|
|
||||||
"old", oldTarball, "new", newTarball)
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeFilteredJSONObject writes the object obj holds, keeping the members
|
|
||||||
// whose decoded key keep accepts.
|
|
||||||
func writeFilteredJSONObject(out *bytes.Buffer, obj []byte, keep func(string) bool) error {
|
|
||||||
return writeFilteredJSONObjectWith(out, obj, keep, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeFilteredJSONObjectWith is writeFilteredJSONObject followed by extra
|
|
||||||
// string members.
|
|
||||||
func writeFilteredJSONObjectWith(out *bytes.Buffer, obj []byte, keep func(string) bool, extra [][2]string) error {
|
|
||||||
out.WriteByte('{')
|
|
||||||
first := true
|
|
||||||
err := forEachJSONMember(obj, func(m jsonMember) error {
|
|
||||||
key, err := jsonKey(m.key(obj))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !keep(key) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !first {
|
|
||||||
out.WriteByte(',')
|
|
||||||
}
|
|
||||||
first = false
|
|
||||||
out.Write(obj[m.keyStart:m.valEnd])
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
for _, member := range extra {
|
|
||||||
if !first {
|
|
||||||
out.WriteByte(',')
|
|
||||||
}
|
|
||||||
first = false
|
|
||||||
key, _ := json.Marshal(member[0])
|
|
||||||
value, _ := json.Marshal(member[1])
|
|
||||||
out.Write(key)
|
|
||||||
out.WriteByte(':')
|
|
||||||
out.Write(value)
|
|
||||||
}
|
|
||||||
out.WriteByte('}')
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyCooldownFiltering removes versions that are too recently published,
|
|
||||||
// and updates dist-tags.latest if the current latest was filtered out.
|
|
||||||
func (h *NPMHandler) applyCooldownFiltering(metadata map[string]any, versions map[string]any, packageName string) {
|
|
||||||
if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
timeMap, _ := metadata["time"].(map[string]any)
|
|
||||||
if timeMap == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
packagePURL := canonicalPackagePURL("npm", packageName)
|
|
||||||
|
|
||||||
for version := range versions {
|
|
||||||
publishedStr, ok := timeMap[version].(string)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
publishedAt, err := time.Parse(time.RFC3339, publishedStr)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !h.proxy.Cooldown.IsAllowed("npm", packagePURL, publishedAt) {
|
|
||||||
h.proxy.Logger.Info("cooldown: filtering npm version",
|
|
||||||
"package", packageName, "version", version,
|
|
||||||
"published", publishedStr)
|
|
||||||
delete(versions, version)
|
|
||||||
delete(timeMap, version)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
h.updateDistTagsLatest(metadata, versions, timeMap)
|
|
||||||
}
|
|
||||||
|
|
||||||
// updateDistTagsLatest updates the dist-tags.latest field if the current latest
|
|
||||||
// version was removed by cooldown filtering.
|
|
||||||
func (h *NPMHandler) updateDistTagsLatest(metadata, versions, timeMap map[string]any) {
|
|
||||||
distTags, ok := metadata["dist-tags"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
latest, ok := distTags["latest"].(string)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, exists := versions[latest]; exists {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if newLatest := h.findNewestVersion(versions, timeMap); newLatest != "" {
|
|
||||||
distTags["latest"] = newLatest
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// proxyTarballURL returns the proxy URL that serves the tarball upstream
|
|
||||||
// lists at tarball for this version.
|
|
||||||
func (h *NPMHandler) proxyTarballURL(packageName, version, tarball string) string {
|
func (h *NPMHandler) proxyTarballURL(packageName, version, tarball string) string {
|
||||||
filename := tarball
|
return npm.TarballURL(h.proxyURL, packageName, version, tarball)
|
||||||
if idx := strings.LastIndex(tarball, "/"); idx >= 0 {
|
|
||||||
filename = tarball[idx+1:]
|
|
||||||
}
|
|
||||||
if h.extractVersionFromFilename(packageName, filename) != version {
|
|
||||||
_, shortName, scoped := strings.Cut(packageName, "/")
|
|
||||||
if !scoped {
|
|
||||||
shortName = packageName
|
|
||||||
}
|
|
||||||
filename = shortName + "-" + version + ".tgz"
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("%s/npm/%s/-/%s", h.proxyURL, url.PathEscape(packageName), filename)
|
|
||||||
}
|
|
||||||
|
|
||||||
// findNewestVersion returns the version string with the most recent timestamp
|
|
||||||
// from the remaining versions, using the time map.
|
|
||||||
func (h *NPMHandler) findNewestVersion(versions map[string]any, timeMap map[string]any) string {
|
|
||||||
if timeMap == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
type versionTime struct {
|
|
||||||
version string
|
|
||||||
t time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
var vts []versionTime
|
|
||||||
for v := range versions {
|
|
||||||
if ts, ok := timeMap[v].(string); ok {
|
|
||||||
if t, err := time.Parse(time.RFC3339, ts); err == nil {
|
|
||||||
vts = append(vts, versionTime{v, t})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(vts) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Slice(vts, func(i, j int) bool {
|
|
||||||
return vts[i].t.After(vts[j].t)
|
|
||||||
})
|
|
||||||
|
|
||||||
return vts[0].version
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleDownload serves a package tarball, fetching and caching from upstream if needed.
|
// handleDownload serves a package tarball, fetching and caching from upstream if needed.
|
||||||
|
|
@ -898,13 +502,13 @@ func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, versi
|
||||||
// returns "" when neither is usable, and the tarball is fetched unverified as
|
// returns "" when neither is usable, and the tarball is fetched unverified as
|
||||||
// before.
|
// before.
|
||||||
func npmVersionIntegrity(body []byte, version string) string {
|
func npmVersionIntegrity(body []byte, version string) string {
|
||||||
sri, _, _ := lookupJSONString(body, "versions", version, "dist", "integrity")
|
sri, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "integrity")
|
||||||
if sri != "" {
|
if sri != "" {
|
||||||
if _, err := integrity.ParseSRI(sri); err == nil {
|
if _, err := integrity.ParseSRI(sri); err == nil {
|
||||||
return sri
|
return sri
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
shasum, _, _ := lookupJSONString(body, "versions", version, "dist", "shasum")
|
shasum, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "shasum")
|
||||||
return sha1Checksum(shasum)
|
return sha1Checksum(shasum)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -912,7 +516,7 @@ func npmVersionIntegrity(body []byte, version string) string {
|
||||||
// or "" if it has none. It reads that one value in place rather than decoding
|
// or "" if it has none. It reads that one value in place rather than decoding
|
||||||
// every version, since it runs on each tarball the proxy has not cached yet.
|
// every version, since it runs on each tarball the proxy has not cached yet.
|
||||||
func npmVersionTarball(body []byte, version string) string {
|
func npmVersionTarball(body []byte, version string) string {
|
||||||
tarball, _, _ := lookupJSONString(body, "versions", version, "dist", "tarball")
|
tarball, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "tarball")
|
||||||
return tarball
|
return tarball
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -978,7 +582,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
published, ok, err := lookupJSONString(body, "time", version)
|
published, ok, err := jsonscan.LookupString(body, "time", version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.proxy.Logger.Warn("cooldown: could not parse npm metadata for download check",
|
h.proxy.Logger.Warn("cooldown: could not parse npm metadata for download check",
|
||||||
"package", packageName, "version", version, "error", err)
|
"package", packageName, "version", version, "error", err)
|
||||||
|
|
@ -1076,28 +680,6 @@ func (h *NPMHandler) parseDownloadPath(path string) (packageName, filename strin
|
||||||
return packageName, filename
|
return packageName, filename
|
||||||
}
|
}
|
||||||
|
|
||||||
// extractVersionFromFilename extracts version from npm tarball filename.
|
|
||||||
// e.g., "lodash-4.17.21.tgz" -> "4.17.21"
|
|
||||||
// e.g., "core-7.23.0.tgz" for @babel/core -> "7.23.0"
|
|
||||||
func (h *NPMHandler) extractVersionFromFilename(packageName, filename string) string {
|
func (h *NPMHandler) extractVersionFromFilename(packageName, filename string) string {
|
||||||
// Remove .tgz extension
|
return npm.VersionFromFilename(packageName, filename)
|
||||||
if !strings.HasSuffix(filename, ".tgz") {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
base := strings.TrimSuffix(filename, ".tgz")
|
|
||||||
|
|
||||||
// For scoped packages, the filename uses the short name
|
|
||||||
shortName := packageName
|
|
||||||
if strings.Contains(packageName, "/") {
|
|
||||||
parts := strings.SplitN(packageName, "/", scopedParts)
|
|
||||||
shortName = parts[1]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Expected format: {shortName}-{version}
|
|
||||||
prefix := shortName + "-"
|
|
||||||
if !strings.HasPrefix(base, prefix) {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.TrimPrefix(base, prefix)
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,15 +8,10 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/Masterminds/semver/v3"
|
"github.com/Masterminds/semver/v3"
|
||||||
|
"github.com/git-pkgs/proxy/internal/metadata/npm"
|
||||||
)
|
)
|
||||||
|
|
||||||
// npmLocalEntry is a pushed version as a packument lists it: the version
|
type npmLocalEntry = npm.LocalVersion
|
||||||
// entry built from its package.json, and the time it was pushed.
|
|
||||||
type npmLocalEntry struct {
|
|
||||||
version string
|
|
||||||
entry json.RawMessage
|
|
||||||
time string
|
|
||||||
}
|
|
||||||
|
|
||||||
// npmLocalEntries returns the pushed versions of a package as packument
|
// npmLocalEntries returns the pushed versions of a package as packument
|
||||||
// entries, or nil when there are none or hosted packages are disabled.
|
// entries, or nil when there are none or hosted packages are disabled.
|
||||||
|
|
@ -42,14 +37,14 @@ func (h *NPMHandler) npmLocalEntries(packageName string) ([]npmLocalEntry, error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
entries = append(entries, npmLocalEntry{version: lv.Version, entry: entry, time: lv.PushedAt.Format(time.RFC3339)})
|
entries = append(entries, npmLocalEntry{Version: lv.Version, Entry: entry, PublishedAt: lv.PushedAt.Format(time.RFC3339)})
|
||||||
}
|
}
|
||||||
return entries, nil
|
return entries, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func findNPMLocalEntry(local []npmLocalEntry, version string) (npmLocalEntry, bool) {
|
func findNPMLocalEntry(local []npmLocalEntry, version string) (npmLocalEntry, bool) {
|
||||||
for _, e := range local {
|
for _, e := range local {
|
||||||
if e.version == version {
|
if e.Version == version {
|
||||||
return e, true
|
return e, true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -81,7 +76,7 @@ func (h *NPMHandler) serveLocalMetadata(w http.ResponseWriter, r *http.Request,
|
||||||
case req.versionLookup && pushed:
|
case req.versionLookup && pushed:
|
||||||
w.Header().Set(headerContentType, contentTypeJSON)
|
w.Header().Set(headerContentType, contentTypeJSON)
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
_, _ = w.Write(e.entry)
|
_, _ = w.Write(e.Entry)
|
||||||
case h.proxy.hostedOnly("npm", req.packageName):
|
case h.proxy.hostedOnly("npm", req.packageName):
|
||||||
h.serveHostedOnlyMetadata(w, r, req.packageName, req.specifier, req.versionLookup, req.metadataCacheKey, local)
|
h.serveHostedOnlyMetadata(w, r, req.packageName, req.specifier, req.versionLookup, req.metadataCacheKey, local)
|
||||||
case len(local) > 0 && !req.versionLookup:
|
case len(local) > 0 && !req.versionLookup:
|
||||||
|
|
@ -144,7 +139,7 @@ func (h *NPMHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http.Requ
|
||||||
}
|
}
|
||||||
w.Header().Set(headerContentType, contentTypeJSON)
|
w.Header().Set(headerContentType, contentTypeJSON)
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
_, _ = w.Write(e.entry)
|
_, _ = w.Write(e.Entry)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
out, err := localNPMPackument(packageName, local)
|
out, err := localNPMPackument(packageName, local)
|
||||||
|
|
@ -161,8 +156,8 @@ func localNPMPackument(packageName string, local []npmLocalEntry) ([]byte, error
|
||||||
versions := make(map[string]json.RawMessage, len(local))
|
versions := make(map[string]json.RawMessage, len(local))
|
||||||
times := make(map[string]string, len(local))
|
times := make(map[string]string, len(local))
|
||||||
for _, e := range local {
|
for _, e := range local {
|
||||||
versions[e.version] = e.entry
|
versions[e.Version] = e.Entry
|
||||||
times[e.version] = e.time
|
times[e.Version] = e.PublishedAt
|
||||||
}
|
}
|
||||||
return json.Marshal(map[string]any{
|
return json.Marshal(map[string]any{
|
||||||
"name": packageName,
|
"name": packageName,
|
||||||
|
|
@ -179,10 +174,10 @@ func latestNPMVersion(local []npmLocalEntry) string {
|
||||||
var latestSemver *semver.Version
|
var latestSemver *semver.Version
|
||||||
semverOnly := true
|
semverOnly := true
|
||||||
for _, e := range local {
|
for _, e := range local {
|
||||||
if e.time > latestTime {
|
if e.PublishedAt > latestTime {
|
||||||
latest, latestTime = e.version, e.time
|
latest, latestTime = e.Version, e.PublishedAt
|
||||||
}
|
}
|
||||||
v, err := semver.NewVersion(e.version)
|
v, err := semver.NewVersion(e.Version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
semverOnly = false
|
semverOnly = false
|
||||||
continue
|
continue
|
||||||
|
|
|
||||||
107
internal/handler/npm_portable_test.go
Normal file
107
internal/handler/npm_portable_test.go
Normal file
|
|
@ -0,0 +1,107 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/git-pkgs/cooldown"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNPMMetadataLogsPolicyAndURLChanges(t *testing.T) {
|
||||||
|
body, err := os.ReadFile("../metadata/npm/testdata/packument.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f := newNPMHostedFixture(t, map[string]string{"/@example%2Fwidgets": string(body)})
|
||||||
|
f.proxy.Cooldown = &cooldown.Config{Default: "7d"}
|
||||||
|
setTestDenylist(t, f.proxy, "pkg:npm/@example/widgets@2.0.0")
|
||||||
|
var logs bytes.Buffer
|
||||||
|
f.proxy.Logger = slog.New(slog.NewJSONHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
||||||
|
if rec := f.get(t, "/@example/widgets"); rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("HTTP %d: %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
want := map[string]map[string]string{
|
||||||
|
"cooldown: filtering npm version": {"level": "INFO", "package": "@example/widgets", "version": "3.0.0", "published": "2099-01-01T00:00:00Z"},
|
||||||
|
"rewrote tarball URL": {"level": "DEBUG", "package": "@example/widgets", "version": "1.0.0",
|
||||||
|
"old": "https://registry.npmjs.org/@example/widgets/-/widgets-1.0.0.tgz", "new": "http://proxy.local/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz"},
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(&logs)
|
||||||
|
for {
|
||||||
|
var record map[string]any
|
||||||
|
if err := decoder.Decode(&record); err == io.EOF {
|
||||||
|
break
|
||||||
|
} else if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
message, _ := record["msg"].(string)
|
||||||
|
fields, ok := want[message]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for key, value := range fields {
|
||||||
|
if record[key] != value {
|
||||||
|
t.Errorf("%s: %s = %v, want %q", message, key, record[key], value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
delete(want, message)
|
||||||
|
}
|
||||||
|
if len(want) != 0 {
|
||||||
|
t.Errorf("missing log records: %v", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNPMRoutesWithPortableMetadata(t *testing.T) {
|
||||||
|
body, err := os.ReadFile("../metadata/npm/testdata/packument.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, hosted := range []bool{false, true} {
|
||||||
|
name := "upstream"
|
||||||
|
if hosted {
|
||||||
|
name = "hosted override"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
f := newNPMHostedFixture(t, map[string]string{"/@example%2Fwidgets": string(body)})
|
||||||
|
f.proxy.Cooldown = &cooldown.Config{Default: "7d"}
|
||||||
|
setTestDenylist(t, f.proxy, "pkg:npm/@example/widgets@2.0.0")
|
||||||
|
if hosted {
|
||||||
|
f.push(t, "@example/widgets", "3.0.0", `{"name":"@example/widgets","version":"3.0.0","dependencies":{"local":"1.0.0"}}`)
|
||||||
|
}
|
||||||
|
rec := f.get(t, "/@example/widgets")
|
||||||
|
doc := decodePackument(t, rec)
|
||||||
|
count := 1
|
||||||
|
if hosted {
|
||||||
|
count = 2
|
||||||
|
}
|
||||||
|
if len(doc.Versions) != count || doc.DistTags["latest"] != "1.0.0" || len(doc.DistTags) != 1 {
|
||||||
|
t.Fatalf("filtered metadata: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
if doc.Versions["1.0.0"].Dist.Tarball != "http://proxy.local/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz" || !strings.Contains(rec.Body.String(), "12345678901234567890") {
|
||||||
|
t.Fatalf("rewritten metadata: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
if hosted && doc.Versions["3.0.0"].Dependencies["local"] != "1.0.0" {
|
||||||
|
t.Fatalf("hosted override: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
checkNPMVersionResponses(t, f, hosted)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkNPMVersionResponses(t *testing.T, f npmHostedFixture, hosted bool) {
|
||||||
|
t.Helper()
|
||||||
|
for specifier, want := range map[string]int{"latest": http.StatusOK, "1.0.0": http.StatusOK, "2.0.0": http.StatusForbidden, "3.0.0": http.StatusNotFound} {
|
||||||
|
if hosted && specifier == "3.0.0" {
|
||||||
|
want = http.StatusOK
|
||||||
|
}
|
||||||
|
got := f.get(t, "/@example/widgets/"+specifier)
|
||||||
|
if got.Code != want {
|
||||||
|
t.Errorf("%s: HTTP %d, want %d: %s", specifier, got.Code, want, got.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -43,7 +43,7 @@ func BenchmarkNPMRewriteMetadata(b *testing.B) {
|
||||||
b.SetBytes(int64(len(body)))
|
b.SetBytes(int64(len(body)))
|
||||||
b.ReportAllocs()
|
b.ReportAllocs()
|
||||||
for b.Loop() {
|
for b.Loop() {
|
||||||
if _, err := h.rewriteMetadata("big", body); err != nil {
|
if _, _, err := h.rewriteMetadataKeeping("big", body); err != nil {
|
||||||
b.Fatal(err)
|
b.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,7 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
@ -14,159 +12,6 @@ import (
|
||||||
"github.com/git-pkgs/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestNPMRewriteMetadataKeepsUpstreamBytes checks that only tarball URLs
|
|
||||||
// change: everything else, including numbers encoding/json would round
|
|
||||||
// through float64 and characters it would HTML-escape, is copied verbatim.
|
|
||||||
func TestNPMRewriteMetadataKeepsUpstreamBytes(t *testing.T) {
|
|
||||||
h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"}
|
|
||||||
input := `{"name":"demo","_big":12345678901234567890,"readme":"<b>&</b>",` +
|
|
||||||
`"versions":{"1.0.0":{"version":"1.0.0","dist":{"shasum":"s","tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz","integrity":"i"}}}}`
|
|
||||||
|
|
||||||
out, err := h.rewriteMetadata("demo", []byte(input))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := strings.Replace(input, "https://registry.npmjs.org/demo/-/demo-1.0.0.tgz", "http://proxy/npm/demo/-/demo-1.0.0.tgz", 1)
|
|
||||||
if string(out) != want {
|
|
||||||
t.Errorf("rewrite =\n%s\nwant\n%s", out, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNPMRewriteMetadataUnusualEntries(t *testing.T) {
|
|
||||||
h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"}
|
|
||||||
input := `{
|
|
||||||
"versions": {
|
|
||||||
"1.0.0": "not an object",
|
|
||||||
"2.0.0": {"dist": "no object either"},
|
|
||||||
"3.0.0": {"dist": {"tarball": 7}},
|
|
||||||
"4.0.0": {"dist": {"tarball": "https://registry.npmjs.org/demo/-/renamed.tgz"}},
|
|
||||||
"5.0.0": {"dist": {}}
|
|
||||||
}
|
|
||||||
}`
|
|
||||||
out, err := h.rewriteMetadata("demo", []byte(input))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var doc struct {
|
|
||||||
Versions map[string]json.RawMessage `json:"versions"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(out, &doc); err != nil {
|
|
||||||
t.Fatalf("output is not JSON: %v\n%s", err, out)
|
|
||||||
}
|
|
||||||
for v, want := range map[string]string{
|
|
||||||
"1.0.0": `"not an object"`,
|
|
||||||
"2.0.0": `{"dist": "no object either"}`,
|
|
||||||
"3.0.0": `{"dist": {"tarball": 7}}`,
|
|
||||||
"4.0.0": `{"dist": {"tarball": "http://proxy/npm/demo/-/demo-4.0.0.tgz"}}`,
|
|
||||||
"5.0.0": `{"dist": {}}`,
|
|
||||||
} {
|
|
||||||
if got := string(doc.Versions[v]); got != want {
|
|
||||||
t.Errorf("%s = %s, want %s", v, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNPMRewriteMetadataInvalidJSON(t *testing.T) {
|
|
||||||
h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"}
|
|
||||||
for _, input := range []string{`{"versions": {}`, `[1]`, `{"versions": {}} trailing`} {
|
|
||||||
if _, err := h.rewriteMetadata("demo", []byte(input)); err == nil {
|
|
||||||
t.Errorf("%s: expected an error", input)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNPMRewriteMetadataNoVersions(t *testing.T) {
|
|
||||||
h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"}
|
|
||||||
input := `{"name":"demo","versions":[]}`
|
|
||||||
out, err := h.rewriteMetadata("demo", []byte(input))
|
|
||||||
if err != nil || string(out) != input {
|
|
||||||
t.Errorf("rewrite = %s, %v; want input unchanged", out, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
setTestDenylist(t, h.proxy, "pkg:npm/demo@1.0.0")
|
|
||||||
if _, err := h.rewriteMetadata("demo", []byte(input)); err == nil {
|
|
||||||
t.Error("expected an error when a denylisted package has no versions object")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNPMRewriteMetadataCooldownAndDenylist runs both filters together and
|
|
||||||
// checks that versions, their time entries and the tags pointing at them are
|
|
||||||
// all removed, with latest moved to the newest version left.
|
|
||||||
func TestNPMRewriteMetadataCooldownAndDenylist(t *testing.T) {
|
|
||||||
now := time.Now()
|
|
||||||
ts := func(age time.Duration) string { return now.Add(-age).UTC().Format(time.RFC3339) }
|
|
||||||
|
|
||||||
proxy := testProxy()
|
|
||||||
proxy.Cooldown = &cooldown.Config{Default: "3d"}
|
|
||||||
setTestDenylist(t, proxy, "pkg:npm/demo@2.0.0")
|
|
||||||
h := &NPMHandler{proxy: proxy, proxyURL: "http://proxy"}
|
|
||||||
|
|
||||||
version := func(v string) string {
|
|
||||||
return fmt.Sprintf(`%q:{"version":%q,"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-%s.tgz"}}`, v, v, v)
|
|
||||||
}
|
|
||||||
input := `{"name":"demo",` +
|
|
||||||
`"dist-tags":{"latest":"3.0.0","beta":"2.0.0","next":"1.0.0"},` +
|
|
||||||
`"versions":{` + version("1.0.0") + `,` + version("2.0.0") + `,` + version("3.0.0") + `},` +
|
|
||||||
`"time":{"created":"` + ts(1000*time.Hour) + `","1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}}`
|
|
||||||
|
|
||||||
out, err := h.rewriteMetadata("demo", []byte(input))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var doc struct {
|
|
||||||
DistTags map[string]string `json:"dist-tags"`
|
|
||||||
Versions map[string]json.RawMessage `json:"versions"`
|
|
||||||
Time map[string]string `json:"time"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(out, &doc); err != nil {
|
|
||||||
t.Fatalf("output is not JSON: %v\n%s", err, out)
|
|
||||||
}
|
|
||||||
if len(doc.Versions) != 1 || doc.Versions["1.0.0"] == nil {
|
|
||||||
t.Errorf("versions = %v, want only 1.0.0", keysOf(doc.Versions))
|
|
||||||
}
|
|
||||||
if _, ok := doc.Time["2.0.0"]; ok {
|
|
||||||
t.Error("time still lists denylisted 2.0.0")
|
|
||||||
}
|
|
||||||
if _, ok := doc.Time["3.0.0"]; ok {
|
|
||||||
t.Error("time still lists 3.0.0, which is inside cooldown")
|
|
||||||
}
|
|
||||||
if _, ok := doc.Time["created"]; !ok {
|
|
||||||
t.Error("time lost its created entry")
|
|
||||||
}
|
|
||||||
want := map[string]string{"latest": "1.0.0", "next": "1.0.0"}
|
|
||||||
if fmt.Sprint(doc.DistTags) != fmt.Sprint(want) {
|
|
||||||
t.Errorf("dist-tags = %v, want %v", doc.DistTags, want)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(doc.Versions["1.0.0"]), "http://proxy/npm/demo/-/demo-1.0.0.tgz") {
|
|
||||||
t.Errorf("1.0.0 tarball not rewritten: %s", doc.Versions["1.0.0"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNPMRewriteMetadataDuplicateKeys checks that a repeated top-level key
|
|
||||||
// cannot smuggle an unfiltered copy of the versions past the filters.
|
|
||||||
func TestNPMRewriteMetadataDuplicateKeys(t *testing.T) {
|
|
||||||
proxy := testProxy()
|
|
||||||
setTestDenylist(t, proxy, "pkg:npm/demo@2.0.0")
|
|
||||||
h := &NPMHandler{proxy: proxy, proxyURL: "http://proxy"}
|
|
||||||
input := `{"versions":{"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}},` +
|
|
||||||
`"versions":{"1.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"}},"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}}}`
|
|
||||||
out, err := h.rewriteMetadata("demo", []byte(input))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Contains(string(out), "2.0.0") || strings.Count(string(out), `"versions"`) != 1 {
|
|
||||||
t.Errorf("rewrite = %s", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func keysOf(m map[string]json.RawMessage) []string {
|
|
||||||
keys := make([]string, 0, len(m))
|
|
||||||
for k := range m {
|
|
||||||
keys = append(keys, k)
|
|
||||||
}
|
|
||||||
return keys
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestMetadataServesRewriteWithoutReadingStorage checks that a repeat
|
// TestMetadataServesRewriteWithoutReadingStorage checks that a repeat
|
||||||
// request inside the metadata TTL is answered from the rewrite cache by the
|
// request inside the metadata TTL is answered from the rewrite cache by the
|
||||||
// stored digest alone, without reading the cached document back.
|
// stored digest alone, without reading the cached document back.
|
||||||
|
|
|
||||||
|
|
@ -153,7 +153,7 @@ func TestNPMRewriteMetadata(t *testing.T) {
|
||||||
}
|
}
|
||||||
}`
|
}`
|
||||||
|
|
||||||
output, err := h.rewriteMetadata("lodash", []byte(input))
|
output, _, err := h.rewriteMetadataKeeping("lodash", []byte(input))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("rewriteMetadata failed: %v", err)
|
t.Fatalf("rewriteMetadata failed: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -193,7 +193,7 @@ func TestNPMRewriteMetadataScopedPackage(t *testing.T) {
|
||||||
}
|
}
|
||||||
}`
|
}`
|
||||||
|
|
||||||
output, err := h.rewriteMetadata("@babel/core", []byte(input))
|
output, _, err := h.rewriteMetadataKeeping("@babel/core", []byte(input))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("rewriteMetadata failed: %v", err)
|
t.Fatalf("rewriteMetadata failed: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -308,7 +308,7 @@ func TestNPMRewriteMetadataCooldown(t *testing.T) {
|
||||||
}
|
}
|
||||||
}`
|
}`
|
||||||
|
|
||||||
output, err := h.rewriteMetadata("testpkg", []byte(input))
|
output, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("rewriteMetadata failed: %v", err)
|
t.Fatalf("rewriteMetadata failed: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -364,7 +364,7 @@ func TestNPMRewriteMetadataCooldownExemptPackage(t *testing.T) {
|
||||||
}
|
}
|
||||||
}`
|
}`
|
||||||
|
|
||||||
output, err := h.rewriteMetadata("testpkg", []byte(input))
|
output, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("rewriteMetadata failed: %v", err)
|
t.Fatalf("rewriteMetadata failed: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
309
internal/jsonscan/jsonscan.go
Normal file
309
internal/jsonscan/jsonscan.go
Normal file
|
|
@ -0,0 +1,309 @@
|
||||||
|
package jsonscan
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The helpers in this file walk a JSON document in place, reporting where
|
||||||
|
// object members sit as offsets into the original bytes. They let a handler
|
||||||
|
// read or replace one value in a large metadata document without decoding the
|
||||||
|
// rest of it into Go values, which for an npm packument costs many times the
|
||||||
|
// document's size.
|
||||||
|
//
|
||||||
|
// They check structure only as far as they need to find their way through
|
||||||
|
// the document. Callers that copy unvisited bytes into a response validate the
|
||||||
|
// whole document first with json.Valid.
|
||||||
|
|
||||||
|
var ErrMalformed = errors.New("malformed JSON")
|
||||||
|
|
||||||
|
// ErrNotObject is returned when a value expected to be an object is not.
|
||||||
|
var ErrNotObject = errors.New("JSON value is not an object")
|
||||||
|
|
||||||
|
// ErrStop is returned by a callback to end a walk early once it has
|
||||||
|
// found what it was looking for.
|
||||||
|
var ErrStop = errors.New("stop scanning")
|
||||||
|
|
||||||
|
// Member is one member of a JSON object as offsets into the bytes that
|
||||||
|
// were scanned. The key span includes its quotes.
|
||||||
|
type Member struct {
|
||||||
|
KeyStart, KeyEnd int
|
||||||
|
ValStart, ValEnd int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m Member) Key(data []byte) []byte { return data[m.KeyStart:m.KeyEnd] }
|
||||||
|
func (m Member) Value(data []byte) []byte { return data[m.ValStart:m.ValEnd] }
|
||||||
|
|
||||||
|
func skipJSONSpace(data []byte, i int) int {
|
||||||
|
for i < len(data) {
|
||||||
|
switch data[i] {
|
||||||
|
case ' ', '\t', '\n', '\r':
|
||||||
|
i++
|
||||||
|
default:
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
|
||||||
|
// skipJSONString returns the index just past the string starting at data[i].
|
||||||
|
func skipJSONString(data []byte, i int) (int, error) {
|
||||||
|
for j := i + 1; j < len(data); j++ {
|
||||||
|
switch data[j] {
|
||||||
|
case '\\':
|
||||||
|
j++
|
||||||
|
case '"':
|
||||||
|
return j + 1, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, ErrMalformed
|
||||||
|
}
|
||||||
|
|
||||||
|
// skipJSONValue returns the index just past the value starting at data[i].
|
||||||
|
func skipJSONValue(data []byte, i int) (int, error) {
|
||||||
|
if i >= len(data) {
|
||||||
|
return 0, ErrMalformed
|
||||||
|
}
|
||||||
|
switch data[i] {
|
||||||
|
case '"':
|
||||||
|
return skipJSONString(data, i)
|
||||||
|
case '{', '[':
|
||||||
|
depth := 0
|
||||||
|
for j := i; j < len(data); j++ {
|
||||||
|
switch data[j] {
|
||||||
|
case '"':
|
||||||
|
end, err := skipJSONString(data, j)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
j = end - 1
|
||||||
|
case '{', '[':
|
||||||
|
depth++
|
||||||
|
case '}', ']':
|
||||||
|
depth--
|
||||||
|
if depth == 0 {
|
||||||
|
return j + 1, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, ErrMalformed
|
||||||
|
case '}', ']', ',', ':':
|
||||||
|
return 0, ErrMalformed
|
||||||
|
default:
|
||||||
|
// A number, true, false or null runs to the next delimiter.
|
||||||
|
j := i
|
||||||
|
for j < len(data) {
|
||||||
|
switch data[j] {
|
||||||
|
case ',', '}', ']', ' ', '\t', '\n', '\r':
|
||||||
|
return j, nil
|
||||||
|
}
|
||||||
|
j++
|
||||||
|
}
|
||||||
|
return j, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// EachMember calls fn for each member of the object obj holds, in
|
||||||
|
// document order. It returns ErrNotObject if obj is not an object, and
|
||||||
|
// stops early with fn's error if fn returns one.
|
||||||
|
func EachMember(obj []byte, fn func(Member) error) error {
|
||||||
|
i := skipJSONSpace(obj, 0)
|
||||||
|
if i >= len(obj) || obj[i] != '{' {
|
||||||
|
return ErrNotObject
|
||||||
|
}
|
||||||
|
i = skipJSONSpace(obj, i+1)
|
||||||
|
if i < len(obj) && obj[i] == '}' {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
if i >= len(obj) || obj[i] != '"' {
|
||||||
|
return ErrMalformed
|
||||||
|
}
|
||||||
|
KeyEnd, err := skipJSONString(obj, i)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
colon := skipJSONSpace(obj, KeyEnd)
|
||||||
|
if colon >= len(obj) || obj[colon] != ':' {
|
||||||
|
return ErrMalformed
|
||||||
|
}
|
||||||
|
ValStart := skipJSONSpace(obj, colon+1)
|
||||||
|
ValEnd, err := skipJSONValue(obj, ValStart)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := fn(Member{KeyStart: i, KeyEnd: KeyEnd, ValStart: ValStart, ValEnd: ValEnd}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
next := skipJSONSpace(obj, ValEnd)
|
||||||
|
if next >= len(obj) {
|
||||||
|
return ErrMalformed
|
||||||
|
}
|
||||||
|
switch obj[next] {
|
||||||
|
case ',':
|
||||||
|
i = skipJSONSpace(obj, next+1)
|
||||||
|
case '}':
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return ErrMalformed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// EachElement calls fn with each element of the array arr holds, in
|
||||||
|
// order, and stops early with fn's error if fn returns one.
|
||||||
|
func EachElement(arr []byte, fn func(element []byte) error) error {
|
||||||
|
i := skipJSONSpace(arr, 0)
|
||||||
|
if i >= len(arr) || arr[i] != '[' {
|
||||||
|
return ErrMalformed
|
||||||
|
}
|
||||||
|
i = skipJSONSpace(arr, i+1)
|
||||||
|
if i < len(arr) && arr[i] == ']' {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
end, err := skipJSONValue(arr, i)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := fn(arr[i:end]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
next := skipJSONSpace(arr, end)
|
||||||
|
if next >= len(arr) {
|
||||||
|
return ErrMalformed
|
||||||
|
}
|
||||||
|
switch arr[next] {
|
||||||
|
case ',':
|
||||||
|
i = skipJSONSpace(arr, next+1)
|
||||||
|
case ']':
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return ErrMalformed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RewriteMembers copies the object obj holds to out byte for byte,
|
||||||
|
// except that rewrite may write a member's value itself. It returns true
|
||||||
|
// when it did; otherwise the original value is copied.
|
||||||
|
func RewriteMembers(out *bytes.Buffer, obj []byte, rewrite func(out *bytes.Buffer, m Member) (bool, error)) error {
|
||||||
|
copied := 0
|
||||||
|
err := EachMember(obj, func(m Member) error {
|
||||||
|
out.Write(obj[copied:m.ValStart])
|
||||||
|
copied = m.ValStart
|
||||||
|
rewritten, err := rewrite(out, m)
|
||||||
|
if rewritten {
|
||||||
|
copied = m.ValEnd
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
out.Write(obj[copied:])
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Key decodes a quoted key as EachMember reports it.
|
||||||
|
func Key(raw []byte) (string, error) {
|
||||||
|
if bytes.IndexByte(raw, '\\') < 0 {
|
||||||
|
return string(raw[1 : len(raw)-1]), nil
|
||||||
|
}
|
||||||
|
var key string
|
||||||
|
err := json.Unmarshal(raw, &key)
|
||||||
|
return key, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeyIs reports whether a quoted key decodes to name.
|
||||||
|
func KeyIs(raw []byte, name string) bool {
|
||||||
|
if bytes.IndexByte(raw, '\\') < 0 {
|
||||||
|
return string(raw[1:len(raw)-1]) == name
|
||||||
|
}
|
||||||
|
key, err := Key(raw)
|
||||||
|
return err == nil && key == name
|
||||||
|
}
|
||||||
|
|
||||||
|
// StringIs reports whether raw is a JSON string that decodes to s.
|
||||||
|
func StringIs(raw []byte, s string) bool {
|
||||||
|
return len(raw) >= 2 && raw[0] == '"' && KeyIs(raw, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindMember returns the member of obj named name. When the key repeats,
|
||||||
|
// the last one wins, as it does for JSON.parse and encoding/json.
|
||||||
|
func FindMember(obj []byte, name string) (Member, bool, error) {
|
||||||
|
var found Member
|
||||||
|
ok := false
|
||||||
|
err := EachMember(obj, func(m Member) error {
|
||||||
|
if KeyIs(m.Key(obj), name) {
|
||||||
|
found, ok = m, true
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return found, ok, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Lookup follows path through nested objects in doc and returns the
|
||||||
|
// value at its end. It returns nil and no error when a key along the path is
|
||||||
|
// missing or names something other than an object.
|
||||||
|
func Lookup(doc []byte, path ...string) ([]byte, error) {
|
||||||
|
value := doc
|
||||||
|
for _, name := range path {
|
||||||
|
m, ok, err := FindMember(value, name)
|
||||||
|
if errors.Is(err, ErrNotObject) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil || !ok {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
value = m.Value(value)
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// LookupString is Lookup for a string value. ok is false when the
|
||||||
|
// value is missing or is not a string.
|
||||||
|
func LookupString(doc []byte, path ...string) (string, bool, error) {
|
||||||
|
raw, err := Lookup(doc, path...)
|
||||||
|
if err != nil || len(raw) == 0 || raw[0] != '"' {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
var s string
|
||||||
|
if err := json.Unmarshal(raw, &s); err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
return s, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteFilteredObject writes the object obj holds, keeping the members
|
||||||
|
// whose decoded key keep accepts, followed by extra string members.
|
||||||
|
func WriteFilteredObject(out *bytes.Buffer, obj []byte, keep func(string) bool, extra [][2]string) error {
|
||||||
|
out.WriteByte('{')
|
||||||
|
first := true
|
||||||
|
err := EachMember(obj, func(m Member) error {
|
||||||
|
key, err := Key(m.Key(obj))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !keep(key) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !first {
|
||||||
|
out.WriteByte(',')
|
||||||
|
}
|
||||||
|
first = false
|
||||||
|
out.Write(obj[m.KeyStart:m.ValEnd])
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
for _, member := range extra {
|
||||||
|
if !first {
|
||||||
|
out.WriteByte(',')
|
||||||
|
}
|
||||||
|
first = false
|
||||||
|
key, _ := json.Marshal(member[0])
|
||||||
|
value, _ := json.Marshal(member[1])
|
||||||
|
out.Write(key)
|
||||||
|
out.WriteByte(':')
|
||||||
|
out.Write(value)
|
||||||
|
}
|
||||||
|
out.WriteByte('}')
|
||||||
|
return err
|
||||||
|
}
|
||||||
155
internal/jsonscan/jsonscan_test.go
Normal file
155
internal/jsonscan/jsonscan_test.go
Normal file
|
|
@ -0,0 +1,155 @@
|
||||||
|
package jsonscan
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLookupJSONString(t *testing.T) {
|
||||||
|
doc := []byte(`{
|
||||||
|
"name": "demo",
|
||||||
|
"versions": {
|
||||||
|
"1.0.0": {"dist": {"tarball": "https://example.com/a.tgz", "shasum": "x"}},
|
||||||
|
"2.0.0": {"readme": "a } tricky \" string ] with {brackets}", "dist": {"tarball": "https://example.com/b.tgz"}},
|
||||||
|
"3.0.0": "not an object",
|
||||||
|
"4.0.0": {"dist": {"tarball": 42}}
|
||||||
|
},
|
||||||
|
"time": {"1.0.0": "2020-01-01T00:00:00Z", "1.0.0": "2021-01-01T00:00:00Z"},
|
||||||
|
"n": -1.5e3, "t": true, "z": null, "list": [1, {"a": []}, "]"]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
path []string
|
||||||
|
want string
|
||||||
|
wantOK bool
|
||||||
|
}{
|
||||||
|
{[]string{"versions", "1.0.0", "dist", "tarball"}, "https://example.com/a.tgz", true},
|
||||||
|
{[]string{"versions", "2.0.0", "dist", "tarball"}, "https://example.com/b.tgz", true},
|
||||||
|
{[]string{"versions", "3.0.0", "dist", "tarball"}, "", false},
|
||||||
|
{[]string{"versions", "4.0.0", "dist", "tarball"}, "", false},
|
||||||
|
{[]string{"versions", "9.9.9", "dist", "tarball"}, "", false},
|
||||||
|
{[]string{"time", "1.0.0"}, "2021-01-01T00:00:00Z", true}, // last duplicate wins
|
||||||
|
{[]string{"name"}, "demo", true},
|
||||||
|
{[]string{"name", "x"}, "", false},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
got, ok, err := LookupString(doc, c.path...)
|
||||||
|
if err != nil || ok != c.wantOK || got != c.want {
|
||||||
|
t.Errorf("lookup %v = %q, %v, %v; want %q, %v", c.path, got, ok, err, c.want, c.wantOK)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLookupJSONEscapedKey(t *testing.T) {
|
||||||
|
doc := []byte(`{"versions": {"1.0.0": {"dist": {"tarball": "u"}}}}`)
|
||||||
|
got, ok, err := LookupString(doc, "versions", "1.0.0", "dist", "tarball")
|
||||||
|
if err != nil || !ok || got != "u" {
|
||||||
|
t.Errorf("lookup = %q, %v, %v", got, ok, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLookupJSONMalformed(t *testing.T) {
|
||||||
|
for _, doc := range []string{
|
||||||
|
`{"versions": {"1.0.0": {"dist": `,
|
||||||
|
`{"versions" {}}`,
|
||||||
|
`{"versions": {"a": 1 "b": 2}}`,
|
||||||
|
`{"a": "unterminated}`,
|
||||||
|
`{"a": [1, 2}`,
|
||||||
|
} {
|
||||||
|
_, _, err := LookupString([]byte(doc), "versions", "1.0.0", "dist", "tarball")
|
||||||
|
if !errors.Is(err, ErrMalformed) {
|
||||||
|
t.Errorf("%s: err = %v, want ErrMalformed", doc, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForEachJSONMemberOffsets(t *testing.T) {
|
||||||
|
doc := []byte(` { "a" : 1 , "b":{"c":[true,null]} ,"d":"x\"y" } `)
|
||||||
|
var got [][2]string
|
||||||
|
err := EachMember(doc, func(m Member) error {
|
||||||
|
got = append(got, [2]string{string(m.Key(doc)), string(m.Value(doc))})
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
want := [][2]string{{`"a"`, `1`}, {`"b"`, `{"c":[true,null]}`}, {`"d"`, `"x\"y"`}}
|
||||||
|
if err != nil || len(got) != len(want) {
|
||||||
|
t.Fatalf("members = %v, %v", got, err)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] {
|
||||||
|
t.Errorf("member %d = %v, want %v", i, got[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := EachMember([]byte(`[1]`), func(Member) error { return nil }); !errors.Is(err, ErrNotObject) {
|
||||||
|
t.Errorf("array: err = %v, want ErrNotObject", err)
|
||||||
|
}
|
||||||
|
if err := EachMember([]byte(`{}`), func(Member) error { t.Error("called for empty object"); return nil }); err != nil {
|
||||||
|
t.Errorf("empty object: err = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteFilteredJSONObject(t *testing.T) {
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := WriteFilteredObject(&out, []byte(`{"a": 1, "b": {"x": 2}, "c": 3}`), func(k string) bool { return k != "b" }, nil)
|
||||||
|
if err != nil || out.String() != `{"a": 1,"c": 3}` {
|
||||||
|
t.Errorf("filtered = %s, %v", out.String(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForEachJSONElement(t *testing.T) {
|
||||||
|
var got []string
|
||||||
|
err := EachElement([]byte(` [ 1, "a,]", {"b": [2]} ,[] ] `), func(e []byte) error {
|
||||||
|
got = append(got, string(e))
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
want := []string{`1`, `"a,]"`, `{"b": [2]}`, `[]`}
|
||||||
|
if err != nil || len(got) != len(want) {
|
||||||
|
t.Fatalf("elements = %q, %v; want %q", got, err, want)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] {
|
||||||
|
t.Errorf("element %d = %s, want %s", i, got[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := EachElement([]byte(`[]`), func([]byte) error { t.Error("called for empty array"); return nil }); err != nil {
|
||||||
|
t.Errorf("empty array: err = %v", err)
|
||||||
|
}
|
||||||
|
for _, bad := range []string{`{}`, `[1,`, `[1 2]`, `[1`} {
|
||||||
|
if err := EachElement([]byte(bad), func([]byte) error { return nil }); err == nil {
|
||||||
|
t.Errorf("%s: expected an error", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRewriteJSONMembers(t *testing.T) {
|
||||||
|
obj := []byte(`{ "a": 1, "b": [2], "c" : 3 }`)
|
||||||
|
var out bytes.Buffer
|
||||||
|
err := RewriteMembers(&out, obj, func(out *bytes.Buffer, m Member) (bool, error) {
|
||||||
|
if !KeyIs(m.Key(obj), "b") {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
out.WriteString(`"B"`)
|
||||||
|
return true, nil
|
||||||
|
})
|
||||||
|
if want := `{ "a": 1, "b": "B", "c" : 3 }`; err != nil || out.String() != want {
|
||||||
|
t.Errorf("rewritten = %s, %v; want %s", out.String(), err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJSONStringIs(t *testing.T) {
|
||||||
|
for raw, want := range map[string]bool{
|
||||||
|
`"~dev"`: true,
|
||||||
|
`"~\u0064ev"`: true,
|
||||||
|
`"~dev2"`: false,
|
||||||
|
`{"~dev":1}`: false,
|
||||||
|
`"`: false,
|
||||||
|
``: false,
|
||||||
|
`["~dev"]`: false,
|
||||||
|
} {
|
||||||
|
if got := StringIs([]byte(raw), "~dev"); got != want {
|
||||||
|
t.Errorf("StringIs(%s) = %v, want %v", raw, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
46
internal/metadata/npm/denylist.go
Normal file
46
internal/metadata/npm/denylist.go
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
package npm
|
||||||
|
|
||||||
|
import "github.com/Masterminds/semver/v3"
|
||||||
|
|
||||||
|
func (h *transformer) applyDenylistFiltering(metadata, versions map[string]any) {
|
||||||
|
denied := h.DeniedVersions
|
||||||
|
if len(denied) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
times, _ := metadata["time"].(map[string]any)
|
||||||
|
for version := range denied {
|
||||||
|
delete(versions, version)
|
||||||
|
delete(times, version)
|
||||||
|
}
|
||||||
|
tags, _ := metadata["dist-tags"].(map[string]any)
|
||||||
|
for tag, value := range tags {
|
||||||
|
version, _ := value.(string)
|
||||||
|
if !denied[version] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
delete(tags, tag)
|
||||||
|
// Custom tags describe publisher intent; do not retarget them. For
|
||||||
|
// latest, use the highest remaining stable version, even in abbreviated
|
||||||
|
// packuments without publication timestamps.
|
||||||
|
if tag == "latest" {
|
||||||
|
if latest := newestStableNPMVersion(versions); latest != "" {
|
||||||
|
tags[tag] = latest
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newestStableNPMVersion(versions map[string]any) string {
|
||||||
|
var best *semver.Version
|
||||||
|
result := ""
|
||||||
|
for version := range versions {
|
||||||
|
parsed, err := semver.StrictNewVersion(version)
|
||||||
|
if err != nil || parsed.Prerelease() != "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if best == nil || parsed.GreaterThan(best) || (parsed.Equal(best) && version > result) {
|
||||||
|
best, result = parsed, version
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
454
internal/metadata/npm/metadata.go
Normal file
454
internal/metadata/npm/metadata.go
Normal file
|
|
@ -0,0 +1,454 @@
|
||||||
|
// Package npm transforms registry metadata without storage or network access.
|
||||||
|
package npm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
"net/url"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/git-pkgs/cooldown"
|
||||||
|
"github.com/git-pkgs/proxy/internal/jsonscan"
|
||||||
|
"github.com/git-pkgs/proxy/internal/packageurl"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrVersionDenied = errors.New("package version blocked by denylist")
|
||||||
|
var ErrVersionNotFound = errors.New("version not found")
|
||||||
|
|
||||||
|
// CooldownPolicy evaluates version age at the caller's supplied time.
|
||||||
|
type CooldownPolicy interface {
|
||||||
|
Enabled() bool
|
||||||
|
Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision
|
||||||
|
}
|
||||||
|
|
||||||
|
// LocalVersion is an already approved hosted version and its publication time.
|
||||||
|
type LocalVersion struct {
|
||||||
|
Version string
|
||||||
|
Entry json.RawMessage
|
||||||
|
PublishedAt string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Options supplies the public URL, policy and hosted versions for a rewrite.
|
||||||
|
// EvaluatedAt is used as given; the transformation never reads the clock.
|
||||||
|
type Options struct {
|
||||||
|
ProxyURL string
|
||||||
|
Cooldown CooldownPolicy
|
||||||
|
DeniedVersions map[string]bool
|
||||||
|
EvaluatedAt time.Time
|
||||||
|
LocalVersions []LocalVersion
|
||||||
|
// Optional callbacks report changes synchronously without retaining events.
|
||||||
|
OnTarballRewrite func(version, oldURL, newURL string)
|
||||||
|
OnCooldownFilter func(version, publishedAt string)
|
||||||
|
}
|
||||||
|
|
||||||
|
type transformer struct{ Options }
|
||||||
|
|
||||||
|
// Rewrite filters upstream versions and rewrites their tarball URLs, preserving
|
||||||
|
// untouched JSON values verbatim. Hosted entries override upstream versions.
|
||||||
|
// The returned version set is unordered. The document, denylist and hosted
|
||||||
|
// entries are not modified.
|
||||||
|
func Rewrite(packageName string, body []byte, options Options) ([]byte, []string, error) {
|
||||||
|
return (&transformer{options}).rewriteMetadataWithLocal(packageName, body, options.LocalVersions)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Version resolves a version or dist-tag through the filtered packument.
|
||||||
|
func Version(packageName, specifier string, packument []byte, options Options) ([]byte, error) {
|
||||||
|
filtered, _, err := Rewrite(packageName, packument, options)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
target := specifier
|
||||||
|
if resolved, ok, err := jsonscan.LookupString(filtered, "dist-tags", specifier); err != nil {
|
||||||
|
return nil, err
|
||||||
|
} else if ok {
|
||||||
|
target = resolved
|
||||||
|
}
|
||||||
|
|
||||||
|
if options.DeniedVersions[target] {
|
||||||
|
return nil, ErrVersionDenied
|
||||||
|
}
|
||||||
|
|
||||||
|
versionRaw, err := jsonscan.Lookup(filtered, "versions", target)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(versionRaw) == 0 {
|
||||||
|
return nil, ErrVersionNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
(&transformer{options}).writeNPMVersion(&out, packageName, target, versionRaw)
|
||||||
|
return out.Bytes(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *transformer) rewriteMetadataWithLocal(packageName string, body []byte, local []LocalVersion) ([]byte, []string, error) {
|
||||||
|
doc, err := indexNPMPackument(body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
versionsRaw := doc.value(doc.versionsAt)
|
||||||
|
if len(versionsRaw) == 0 || versionsRaw[0] != '{' {
|
||||||
|
version, ok, err := jsonscan.LookupString(body, "version")
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
if h.DeniedVersions[version] {
|
||||||
|
return nil, nil, ErrVersionDenied
|
||||||
|
}
|
||||||
|
var out bytes.Buffer
|
||||||
|
h.writeNPMVersion(&out, packageName, version, body)
|
||||||
|
return out.Bytes(), []string{version}, nil
|
||||||
|
}
|
||||||
|
if len(h.DeniedVersions) != 0 {
|
||||||
|
return nil, nil, errors.New("npm metadata has no versions object")
|
||||||
|
}
|
||||||
|
return body, nil, nil // No versions to rewrite
|
||||||
|
}
|
||||||
|
|
||||||
|
// The filters only look at which versions exist, so they run on a map of
|
||||||
|
// version names alongside the decoded time map and dist-tags.
|
||||||
|
versions := map[string]any{}
|
||||||
|
if err := jsonscan.EachMember(versionsRaw, func(m jsonscan.Member) error {
|
||||||
|
version, err := jsonscan.Key(m.Key(versionsRaw))
|
||||||
|
versions[version] = nil
|
||||||
|
return err
|
||||||
|
}); err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
metadata := map[string]any{}
|
||||||
|
timeMap := doc.decodeObject(doc.timeAt, metadata, "time")
|
||||||
|
distTags := doc.decodeObject(doc.tagsAt, metadata, "dist-tags")
|
||||||
|
timeEntries := len(timeMap)
|
||||||
|
originalTags := maps.Clone(distTags)
|
||||||
|
|
||||||
|
h.applyCooldownFiltering(metadata, versions, packageName)
|
||||||
|
h.applyDenylistFiltering(metadata, versions)
|
||||||
|
for _, e := range local {
|
||||||
|
delete(versions, e.Version)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Untouched members are copied as they are; the three the filters
|
||||||
|
// handled are written from their filtered state, and only when it changed.
|
||||||
|
out, err := doc.write(func(out *bytes.Buffer, i int) error {
|
||||||
|
switch {
|
||||||
|
case i == doc.versionsAt:
|
||||||
|
return h.writeNPMVersions(out, packageName, versionsRaw, versions, local)
|
||||||
|
case i == doc.timeAt && timeMap != nil && (len(timeMap) != timeEntries || len(local) > 0):
|
||||||
|
pushedAt := make([][2]string, 0, len(local))
|
||||||
|
for _, e := range local {
|
||||||
|
pushedAt = append(pushedAt, [2]string{e.Version, e.PublishedAt})
|
||||||
|
}
|
||||||
|
return jsonscan.WriteFilteredObject(out, doc.value(i), func(key string) bool {
|
||||||
|
_, ok := timeMap[key]
|
||||||
|
pushed := slices.ContainsFunc(local, func(e LocalVersion) bool { return e.Version == key })
|
||||||
|
return ok && !pushed
|
||||||
|
}, pushedAt)
|
||||||
|
case i == doc.tagsAt && distTags != nil && !reflect.DeepEqual(distTags, originalTags):
|
||||||
|
encoded, err := json.Marshal(distTags)
|
||||||
|
out.Write(encoded)
|
||||||
|
return err
|
||||||
|
default:
|
||||||
|
out.Write(doc.value(i))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
})
|
||||||
|
kept := slices.Collect(maps.Keys(versions))
|
||||||
|
for _, e := range local {
|
||||||
|
kept = append(kept, e.Version)
|
||||||
|
}
|
||||||
|
return out, kept, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// npmPackument is a packument's top-level members as offsets into its bytes,
|
||||||
|
// with the positions of the members rewriteMetadata filters, or -1 for those
|
||||||
|
// it lacks. A key that repeats is recorded at its last occurrence, the one
|
||||||
|
// JSON.parse keeps.
|
||||||
|
type npmPackument struct {
|
||||||
|
body []byte
|
||||||
|
members []jsonscan.Member
|
||||||
|
versionsAt, timeAt, tagsAt int
|
||||||
|
}
|
||||||
|
|
||||||
|
func indexNPMPackument(body []byte) (*npmPackument, error) {
|
||||||
|
if !json.Valid(body) {
|
||||||
|
return nil, errors.New("npm metadata is not valid JSON")
|
||||||
|
}
|
||||||
|
doc := &npmPackument{body: body, versionsAt: -1, timeAt: -1, tagsAt: -1}
|
||||||
|
err := jsonscan.EachMember(body, func(m jsonscan.Member) error {
|
||||||
|
switch key := m.Key(body); {
|
||||||
|
case jsonscan.KeyIs(key, "versions"):
|
||||||
|
doc.versionsAt = len(doc.members)
|
||||||
|
case jsonscan.KeyIs(key, "time"):
|
||||||
|
doc.timeAt = len(doc.members)
|
||||||
|
case jsonscan.KeyIs(key, "dist-tags"):
|
||||||
|
doc.tagsAt = len(doc.members)
|
||||||
|
}
|
||||||
|
doc.members = append(doc.members, m)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return doc, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// value returns the raw value of member i, or nil when i is -1.
|
||||||
|
func (d *npmPackument) value(i int) []byte {
|
||||||
|
if i < 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return d.members[i].Value(d.body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeObject decodes member i into metadata[name] and returns it when it is
|
||||||
|
// an object. The filters edit the returned map in place.
|
||||||
|
func (d *npmPackument) decodeObject(i int, metadata map[string]any, name string) map[string]any {
|
||||||
|
raw := d.value(i)
|
||||||
|
if raw == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var value any
|
||||||
|
if err := json.Unmarshal(raw, &value); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
metadata[name] = value
|
||||||
|
object, _ := value.(map[string]any)
|
||||||
|
return object
|
||||||
|
}
|
||||||
|
|
||||||
|
// write assembles the document in member order, with writeValue writing each
|
||||||
|
// member's value. Earlier duplicates of the filtered keys are dropped: they
|
||||||
|
// would carry unfiltered data, and JSON.parse ignores them anyway.
|
||||||
|
func (d *npmPackument) write(writeValue func(out *bytes.Buffer, i int) error) ([]byte, error) {
|
||||||
|
var out bytes.Buffer
|
||||||
|
out.Grow(len(d.body) + len(d.body)/8)
|
||||||
|
out.WriteByte('{')
|
||||||
|
first := true
|
||||||
|
for i, m := range d.members {
|
||||||
|
if i != d.versionsAt && i != d.timeAt && i != d.tagsAt {
|
||||||
|
key := m.Key(d.body)
|
||||||
|
if jsonscan.KeyIs(key, "versions") || jsonscan.KeyIs(key, "time") || jsonscan.KeyIs(key, "dist-tags") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !first {
|
||||||
|
out.WriteByte(',')
|
||||||
|
}
|
||||||
|
first = false
|
||||||
|
out.Write(m.Key(d.body))
|
||||||
|
out.WriteByte(':')
|
||||||
|
if err := writeValue(&out, i); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.WriteByte('}')
|
||||||
|
return out.Bytes(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeNPMVersions writes the versions object, keeping the versions still in
|
||||||
|
// keep and pointing each kept version's tarball at this proxy, followed by the
|
||||||
|
// pushed versions in local.
|
||||||
|
func (h *transformer) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any, local []LocalVersion) error {
|
||||||
|
out.WriteByte('{')
|
||||||
|
first := true
|
||||||
|
err := jsonscan.EachMember(versionsRaw, func(m jsonscan.Member) error {
|
||||||
|
version, err := jsonscan.Key(m.Key(versionsRaw))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, ok := keep[version]; !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !first {
|
||||||
|
out.WriteByte(',')
|
||||||
|
}
|
||||||
|
first = false
|
||||||
|
out.Write(m.Key(versionsRaw))
|
||||||
|
out.WriteByte(':')
|
||||||
|
h.writeNPMVersion(out, packageName, version, m.Value(versionsRaw))
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
for _, e := range local {
|
||||||
|
if !first {
|
||||||
|
out.WriteByte(',')
|
||||||
|
}
|
||||||
|
first = false
|
||||||
|
key, _ := json.Marshal(e.Version)
|
||||||
|
out.Write(key)
|
||||||
|
out.WriteByte(':')
|
||||||
|
out.Write(e.Entry)
|
||||||
|
}
|
||||||
|
out.WriteByte('}')
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeNPMVersion writes one version entry with its dist.tarball pointing at
|
||||||
|
// this proxy. An entry without a string tarball is written unchanged.
|
||||||
|
func (h *transformer) writeNPMVersion(out *bytes.Buffer, packageName, version string, raw []byte) {
|
||||||
|
dist, ok, err := jsonscan.FindMember(raw, "dist")
|
||||||
|
if err != nil || !ok {
|
||||||
|
out.Write(raw)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
distRaw := dist.Value(raw)
|
||||||
|
tarball, ok, err := jsonscan.FindMember(distRaw, "tarball")
|
||||||
|
if err != nil || !ok || distRaw[tarball.ValStart] != '"' {
|
||||||
|
out.Write(raw)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var oldTarball string
|
||||||
|
if err := json.Unmarshal(tarball.Value(distRaw), &oldTarball); err != nil {
|
||||||
|
out.Write(raw)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
newTarball := TarballURL(h.ProxyURL, packageName, version, oldTarball)
|
||||||
|
encoded, err := json.Marshal(newTarball)
|
||||||
|
if err != nil {
|
||||||
|
out.Write(raw)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
out.Write(raw[:dist.ValStart+tarball.ValStart])
|
||||||
|
out.Write(encoded)
|
||||||
|
out.Write(raw[dist.ValStart+tarball.ValEnd:])
|
||||||
|
if h.OnTarballRewrite != nil {
|
||||||
|
h.OnTarballRewrite(version, oldTarball, newTarball)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyCooldownFiltering removes versions that are too recently published,
|
||||||
|
// and updates dist-tags.latest if the current latest was filtered out.
|
||||||
|
func (h *transformer) applyCooldownFiltering(metadata map[string]any, versions map[string]any, packageName string) {
|
||||||
|
if h.Cooldown == nil || !h.Cooldown.Enabled() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
timeMap, _ := metadata["time"].(map[string]any)
|
||||||
|
if timeMap == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
packagePURL := packageurl.MakeString("npm", packageName, "")
|
||||||
|
|
||||||
|
for version := range versions {
|
||||||
|
publishedStr, ok := timeMap[version].(string)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
publishedAt, err := time.Parse(time.RFC3339, publishedStr)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !h.Cooldown.Evaluate("npm", packagePURL, publishedAt, h.EvaluatedAt).Allowed {
|
||||||
|
if h.OnCooldownFilter != nil {
|
||||||
|
h.OnCooldownFilter(version, publishedStr)
|
||||||
|
}
|
||||||
|
delete(versions, version)
|
||||||
|
delete(timeMap, version)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
h.updateDistTagsLatest(metadata, versions, timeMap)
|
||||||
|
}
|
||||||
|
|
||||||
|
// updateDistTagsLatest updates the dist-tags.latest field if the current latest
|
||||||
|
// version was removed by cooldown filtering.
|
||||||
|
func (h *transformer) updateDistTagsLatest(metadata, versions, timeMap map[string]any) {
|
||||||
|
distTags, ok := metadata["dist-tags"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
latest, ok := distTags["latest"].(string)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, exists := versions[latest]; exists {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if newLatest := h.findNewestVersion(versions, timeMap); newLatest != "" {
|
||||||
|
distTags["latest"] = newLatest
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TarballURL returns the proxy URL that serves the tarball upstream
|
||||||
|
// lists at tarball for this version.
|
||||||
|
func TarballURL(proxyURL, packageName, version, tarball string) string {
|
||||||
|
filename := tarball
|
||||||
|
if idx := strings.LastIndex(tarball, "/"); idx >= 0 {
|
||||||
|
filename = tarball[idx+1:]
|
||||||
|
}
|
||||||
|
if VersionFromFilename(packageName, filename) != version {
|
||||||
|
_, shortName, scoped := strings.Cut(packageName, "/")
|
||||||
|
if !scoped {
|
||||||
|
shortName = packageName
|
||||||
|
}
|
||||||
|
filename = shortName + "-" + version + ".tgz"
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Sprintf("%s/npm/%s/-/%s", proxyURL, url.PathEscape(packageName), filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
// findNewestVersion returns the version string with the most recent timestamp
|
||||||
|
// from the remaining versions, using the time map.
|
||||||
|
func (h *transformer) findNewestVersion(versions map[string]any, timeMap map[string]any) string {
|
||||||
|
if timeMap == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type versionTime struct {
|
||||||
|
version string
|
||||||
|
t time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
var vts []versionTime
|
||||||
|
for v := range versions {
|
||||||
|
if ts, ok := timeMap[v].(string); ok {
|
||||||
|
if t, err := time.Parse(time.RFC3339, ts); err == nil {
|
||||||
|
vts = append(vts, versionTime{v, t})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(vts) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(vts, func(i, j int) bool {
|
||||||
|
return vts[i].t.After(vts[j].t)
|
||||||
|
})
|
||||||
|
|
||||||
|
return vts[0].version
|
||||||
|
}
|
||||||
|
|
||||||
|
// VersionFromFilename extracts a version from an npm tarball filename.
|
||||||
|
func VersionFromFilename(packageName, filename string) string {
|
||||||
|
// Remove .tgz extension
|
||||||
|
if !strings.HasSuffix(filename, ".tgz") {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
base := strings.TrimSuffix(filename, ".tgz")
|
||||||
|
|
||||||
|
// For scoped packages, the filename uses the short name
|
||||||
|
shortName := packageName
|
||||||
|
if _, name, scoped := strings.Cut(packageName, "/"); scoped {
|
||||||
|
shortName = name
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expected format: {shortName}-{version}
|
||||||
|
prefix := shortName + "-"
|
||||||
|
if !strings.HasPrefix(base, prefix) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.TrimPrefix(base, prefix)
|
||||||
|
}
|
||||||
153
internal/metadata/npm/metadata_test.go
Normal file
153
internal/metadata/npm/metadata_test.go
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
package npm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"github.com/git-pkgs/cooldown"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestNPMRewriteMetadataKeepsUpstreamBytes checks that only tarball URLs
|
||||||
|
// change: everything else, including numbers encoding/json would round
|
||||||
|
// through float64 and characters it would HTML-escape, is copied verbatim.
|
||||||
|
func TestNPMRewriteMetadataKeepsUpstreamBytes(t *testing.T) {
|
||||||
|
options := Options{ProxyURL: "http://proxy"}
|
||||||
|
input := `{"name":"demo","_big":12345678901234567890,"readme":"<b>&</b>",` +
|
||||||
|
`"versions":{"1.0.0":{"version":"1.0.0","dist":{"shasum":"s","tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz","integrity":"i"}}}}`
|
||||||
|
|
||||||
|
out, _, err := Rewrite("demo", []byte(input), options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := strings.Replace(input, "https://registry.npmjs.org/demo/-/demo-1.0.0.tgz", "http://proxy/npm/demo/-/demo-1.0.0.tgz", 1)
|
||||||
|
if string(out) != want {
|
||||||
|
t.Errorf("rewrite =\n%s\nwant\n%s", out, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNPMRewriteMetadataUnusualEntries(t *testing.T) {
|
||||||
|
options := Options{ProxyURL: "http://proxy"}
|
||||||
|
input := `{
|
||||||
|
"versions": {
|
||||||
|
"1.0.0": "not an object",
|
||||||
|
"2.0.0": {"dist": "no object either"},
|
||||||
|
"3.0.0": {"dist": {"tarball": 7}},
|
||||||
|
"4.0.0": {"dist": {"tarball": "https://registry.npmjs.org/demo/-/renamed.tgz"}},
|
||||||
|
"5.0.0": {"dist": {}}
|
||||||
|
}
|
||||||
|
}`
|
||||||
|
out, _, err := Rewrite("demo", []byte(input), options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
Versions map[string]json.RawMessage `json:"versions"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(out, &doc); err != nil {
|
||||||
|
t.Fatalf("output is not JSON: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
for v, want := range map[string]string{
|
||||||
|
"1.0.0": `"not an object"`,
|
||||||
|
"2.0.0": `{"dist": "no object either"}`,
|
||||||
|
"3.0.0": `{"dist": {"tarball": 7}}`,
|
||||||
|
"4.0.0": `{"dist": {"tarball": "http://proxy/npm/demo/-/demo-4.0.0.tgz"}}`,
|
||||||
|
"5.0.0": `{"dist": {}}`,
|
||||||
|
} {
|
||||||
|
if got := string(doc.Versions[v]); got != want {
|
||||||
|
t.Errorf("%s = %s, want %s", v, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNPMRewriteMetadataInvalidJSON(t *testing.T) {
|
||||||
|
options := Options{ProxyURL: "http://proxy"}
|
||||||
|
for _, input := range []string{`{"versions": {}`, `[1]`, `{"versions": {}} trailing`} {
|
||||||
|
if _, _, err := Rewrite("demo", []byte(input), options); err == nil {
|
||||||
|
t.Errorf("%s: expected an error", input)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNPMRewriteMetadataNoVersions(t *testing.T) {
|
||||||
|
options := Options{ProxyURL: "http://proxy"}
|
||||||
|
input := `{"name":"demo","versions":[]}`
|
||||||
|
out, _, err := Rewrite("demo", []byte(input), options)
|
||||||
|
if err != nil || string(out) != input {
|
||||||
|
t.Errorf("rewrite = %s, %v; want input unchanged", out, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
options.DeniedVersions = map[string]bool{"1.0.0": true}
|
||||||
|
if _, _, err := Rewrite("demo", []byte(input), options); err == nil {
|
||||||
|
t.Error("expected an error when a denylisted package has no versions object")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNPMRewriteMetadataCooldownAndDenylist runs both filters together and
|
||||||
|
// checks that versions, their time entries and the tags pointing at them are
|
||||||
|
// all removed, with latest moved to the newest version left.
|
||||||
|
func TestNPMRewriteMetadataCooldownAndDenylist(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)
|
||||||
|
ts := func(age time.Duration) string { return now.Add(-age).UTC().Format(time.RFC3339) }
|
||||||
|
|
||||||
|
options := Options{ProxyURL: "http://proxy", EvaluatedAt: time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)}
|
||||||
|
options.Cooldown = &cooldown.Config{Default: "3d"}
|
||||||
|
options.DeniedVersions = map[string]bool{"2.0.0": true}
|
||||||
|
|
||||||
|
version := func(v string) string {
|
||||||
|
return fmt.Sprintf(`%q:{"version":%q,"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-%s.tgz"}}`, v, v, v)
|
||||||
|
}
|
||||||
|
input := `{"name":"demo",` +
|
||||||
|
`"dist-tags":{"latest":"3.0.0","beta":"2.0.0","next":"1.0.0"},` +
|
||||||
|
`"versions":{` + version("1.0.0") + `,` + version("2.0.0") + `,` + version("3.0.0") + `},` +
|
||||||
|
`"time":{"created":"` + ts(1000*time.Hour) + `","1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}}`
|
||||||
|
|
||||||
|
out, _, err := Rewrite("demo", []byte(input), options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
DistTags map[string]string `json:"dist-tags"`
|
||||||
|
Versions map[string]json.RawMessage `json:"versions"`
|
||||||
|
Time map[string]string `json:"time"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(out, &doc); err != nil {
|
||||||
|
t.Fatalf("output is not JSON: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
if len(doc.Versions) != 1 || doc.Versions["1.0.0"] == nil {
|
||||||
|
t.Errorf("versions = %v, want only 1.0.0", doc.Versions)
|
||||||
|
}
|
||||||
|
if _, ok := doc.Time["2.0.0"]; ok {
|
||||||
|
t.Error("time still lists denylisted 2.0.0")
|
||||||
|
}
|
||||||
|
if _, ok := doc.Time["3.0.0"]; ok {
|
||||||
|
t.Error("time still lists 3.0.0, which is inside cooldown")
|
||||||
|
}
|
||||||
|
if _, ok := doc.Time["created"]; !ok {
|
||||||
|
t.Error("time lost its created entry")
|
||||||
|
}
|
||||||
|
want := map[string]string{"latest": "1.0.0", "next": "1.0.0"}
|
||||||
|
if fmt.Sprint(doc.DistTags) != fmt.Sprint(want) {
|
||||||
|
t.Errorf("dist-tags = %v, want %v", doc.DistTags, want)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(doc.Versions["1.0.0"]), "http://proxy/npm/demo/-/demo-1.0.0.tgz") {
|
||||||
|
t.Errorf("1.0.0 tarball not rewritten: %s", doc.Versions["1.0.0"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestNPMRewriteMetadataDuplicateKeys checks that a repeated top-level key
|
||||||
|
// cannot smuggle an unfiltered copy of the versions past the filters.
|
||||||
|
func TestNPMRewriteMetadataDuplicateKeys(t *testing.T) {
|
||||||
|
options := Options{ProxyURL: "http://proxy", EvaluatedAt: time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)}
|
||||||
|
options.DeniedVersions = map[string]bool{"2.0.0": true}
|
||||||
|
input := `{"versions":{"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}},` +
|
||||||
|
`"versions":{"1.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"}},"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}}}`
|
||||||
|
out, _, err := Rewrite("demo", []byte(input), options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(out), "2.0.0") || strings.Count(string(out), `"versions"`) != 1 {
|
||||||
|
t.Errorf("rewrite = %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
148
internal/metadata/npm/policy_test.go
Normal file
148
internal/metadata/npm/policy_test.go
Normal file
|
|
@ -0,0 +1,148 @@
|
||||||
|
package npm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
_ "embed"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/git-pkgs/cooldown"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed testdata/packument.json
|
||||||
|
var packument []byte
|
||||||
|
|
||||||
|
func TestRewriteCallbacks(t *testing.T) {
|
||||||
|
options := Options{ProxyURL: "https://proxy.example", Cooldown: &cooldown.Config{Default: "7d"},
|
||||||
|
DeniedVersions: map[string]bool{"2.0.0": true}, EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)}
|
||||||
|
want, kept, err := Rewrite("@example/widgets", packument, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var rewrites, filtered int
|
||||||
|
options.OnTarballRewrite = func(version, oldURL, newURL string) {
|
||||||
|
rewrites++
|
||||||
|
if version != "1.0.0" || oldURL != "https://registry.npmjs.org/@example/widgets/-/widgets-1.0.0.tgz" || newURL != "https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz" {
|
||||||
|
t.Errorf("rewrite event: %s, %s, %s", version, oldURL, newURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
options.OnCooldownFilter = func(version, publishedAt string) {
|
||||||
|
filtered++
|
||||||
|
if version != "3.0.0" || publishedAt != "2099-01-01T00:00:00Z" {
|
||||||
|
t.Errorf("filter event: %s, %s", version, publishedAt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out, gotKept, err := Rewrite("@example/widgets", packument, options)
|
||||||
|
if err != nil || !bytes.Equal(out, want) || !slices.Equal(gotKept, kept) || rewrites != 1 || filtered != 1 {
|
||||||
|
t.Fatalf("callbacks: rewrites=%d, filtered=%d, kept=%v, error=%v", rewrites, filtered, gotKept, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRewriteHostedOverride(t *testing.T) {
|
||||||
|
options := Options{
|
||||||
|
ProxyURL: "https://proxy.example",
|
||||||
|
Cooldown: &cooldown.Config{Default: "7d"},
|
||||||
|
DeniedVersions: map[string]bool{"2.0.0": true},
|
||||||
|
EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||||
|
LocalVersions: []LocalVersion{{Version: "3.0.0", PublishedAt: "2030-01-01T00:00:00Z",
|
||||||
|
Entry: json.RawMessage(`{"version":"3.0.0","dist":{"tarball":"https://proxy.example/npm/@example%2Fwidgets/-/widgets-3.0.0.tgz","integrity":"sha512-local"}}`)}},
|
||||||
|
}
|
||||||
|
before := bytes.Clone(packument)
|
||||||
|
localBefore := bytes.Clone(options.LocalVersions[0].Entry)
|
||||||
|
out, kept, err := Rewrite("@example/widgets", packument, options)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
slices.Sort(kept)
|
||||||
|
if !slices.Equal(kept, []string{"1.0.0", "3.0.0"}) {
|
||||||
|
t.Fatalf("kept = %v", kept)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
Versions map[string]json.RawMessage `json:"versions"`
|
||||||
|
Tags map[string]string `json:"dist-tags"`
|
||||||
|
Time map[string]string `json:"time"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(out, &doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(doc.Versions) != 2 || !bytes.Equal(doc.Versions["3.0.0"], localBefore) || doc.Time["3.0.0"] != options.LocalVersions[0].PublishedAt {
|
||||||
|
t.Fatalf("hosted override lost: %s", out)
|
||||||
|
}
|
||||||
|
if len(doc.Tags) != 1 || doc.Tags["latest"] != "1.0.0" || doc.Time["2.0.0"] != "" || doc.Time["created"] == "" {
|
||||||
|
t.Fatalf("policy output: %s", out)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"12345678901234567890", "<b>Widgets & tools</b>", "https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz"} {
|
||||||
|
if !strings.Contains(string(out), want) {
|
||||||
|
t.Errorf("output lost %q", want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !bytes.Equal(packument, before) || !bytes.Equal(options.LocalVersions[0].Entry, localBefore) || len(options.DeniedVersions) != 1 || !options.DeniedVersions["2.0.0"] {
|
||||||
|
t.Fatal("rewrite mutated its input")
|
||||||
|
}
|
||||||
|
again, _, err := Rewrite("@example/widgets", packument, options)
|
||||||
|
if err != nil || !bytes.Equal(out, again) {
|
||||||
|
t.Fatalf("repeat changed output: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVersionPolicyErrors(t *testing.T) {
|
||||||
|
options := Options{ProxyURL: "https://proxy.example", Cooldown: &cooldown.Config{Default: "7d"},
|
||||||
|
DeniedVersions: map[string]bool{"2.0.0": true}, EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
specifier string
|
||||||
|
want error
|
||||||
|
}{
|
||||||
|
{"1.0.0", nil}, {"latest", nil}, {"2.0.0", ErrVersionDenied}, {"3.0.0", ErrVersionNotFound}, {"beta", ErrVersionNotFound}, {"missing", ErrVersionNotFound},
|
||||||
|
} {
|
||||||
|
t.Run(tc.specifier, func(t *testing.T) {
|
||||||
|
out, err := Version("@example/widgets", tc.specifier, packument, options)
|
||||||
|
if !errors.Is(err, tc.want) {
|
||||||
|
t.Fatalf("error = %v, want %v", err, tc.want)
|
||||||
|
}
|
||||||
|
if tc.want == nil && (!bytes.Contains(out, []byte(`"version": "1.0.0"`)) || !bytes.Contains(out, []byte("https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz"))) {
|
||||||
|
t.Fatalf("resolved version: %s", out)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRewriteCooldownEvaluationTime(t *testing.T) {
|
||||||
|
body := []byte(`{"versions":{"1.0.0":{}},"time":{"1.0.0":"2030-01-01T00:00:00Z"}}`)
|
||||||
|
boundary := time.Date(2030, 1, 8, 0, 0, 0, 0, time.UTC)
|
||||||
|
options := Options{Cooldown: &cooldown.Config{Default: "7d"}}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
at time.Time
|
||||||
|
count int
|
||||||
|
}{
|
||||||
|
{boundary.Add(-time.Nanosecond), 0}, {boundary, 1}, {boundary.Add(time.Second), 1},
|
||||||
|
} {
|
||||||
|
options.EvaluatedAt = tc.at
|
||||||
|
_, kept, err := Rewrite("@example/widgets", body, options)
|
||||||
|
if err != nil || len(kept) != tc.count {
|
||||||
|
t.Errorf("at %s: kept %v, error %v", tc.at, kept, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
options.Cooldown = &cooldown.Config{Default: "7d", Packages: map[string]string{"pkg:npm/%40example/widgets": "0"}}
|
||||||
|
options.EvaluatedAt = boundary.Add(-time.Hour)
|
||||||
|
_, kept, err := Rewrite("@example/widgets", body, options)
|
||||||
|
if err != nil || len(kept) != 1 {
|
||||||
|
t.Fatalf("scoped exemption: kept %v, error %v", kept, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRewriteVersionDocument(t *testing.T) {
|
||||||
|
body := []byte(`{"name":"@example/widgets","version":"1.0.0","dist":{"tarball":"https://cdn.example/abcdef.tgz"}}`)
|
||||||
|
options := Options{ProxyURL: "https://proxy.example"}
|
||||||
|
out, kept, err := Rewrite("@example/widgets", body, options)
|
||||||
|
if err != nil || !slices.Equal(kept, []string{"1.0.0"}) || !bytes.Contains(out, []byte("https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz")) {
|
||||||
|
t.Fatalf("rewrite: %s, %v, %v", out, kept, err)
|
||||||
|
}
|
||||||
|
options.DeniedVersions = map[string]bool{"1.0.0": true}
|
||||||
|
if _, _, err := Rewrite("@example/widgets", body, options); !errors.Is(err, ErrVersionDenied) {
|
||||||
|
t.Fatalf("denied version: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
12
internal/metadata/npm/testdata/packument.json
vendored
Normal file
12
internal/metadata/npm/testdata/packument.json
vendored
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
{
|
||||||
|
"name": "@example/widgets",
|
||||||
|
"dist-tags": {"latest": "3.0.0", "beta": "2.0.0"},
|
||||||
|
"versions": {
|
||||||
|
"1.0.0": {"name": "@example/widgets", "version": "1.0.0", "dependencies": {"left-pad": "^1.3.0"}, "dist": {"integrity": "sha512-example", "tarball": "https://registry.npmjs.org/@example/widgets/-/widgets-1.0.0.tgz"}},
|
||||||
|
"2.0.0": {"name": "@example/widgets", "version": "2.0.0", "dist": {"tarball": "https://registry.npmjs.org/@example/widgets/-/widgets-2.0.0.tgz"}},
|
||||||
|
"3.0.0": {"name": "@example/widgets", "version": "3.0.0", "dist": {"tarball": "https://cdn.example/abcdef.tgz"}}
|
||||||
|
},
|
||||||
|
"time": {"created": "2020-01-01T00:00:00Z", "1.0.0": "2020-01-01T00:00:00Z", "2.0.0": "2021-01-01T00:00:00Z", "3.0.0": "2099-01-01T00:00:00Z"},
|
||||||
|
"readme": "<b>Widgets & tools</b>",
|
||||||
|
"downloads": 12345678901234567890
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue