pkg-proxy/internal/retention/retention.go

// Package retention holds the per-ecosystem rules for evicting cached
// artifacts that have not been accessed for a configured time.
//
// An ecosystem takes part only once its handler registers a Spec with
// Default, usually from an init function in the handler's own file. Until
// then configuring retention for it is a validation error, and the global
// default does not apply to it.
package retention

import (
	"fmt"
	"slices"
	"sort"
	"sync"
	"time"

	"github.com/git-pkgs/proxy/internal/packageurl"
	"github.com/git-pkgs/purl"
)

// KnownKeys lists the ecosystem keys storage.retention.ecosystems accepts.
// They are the literals the handlers pass when they fetch an artifact. A key
// outside this list is a typo; a key in it whose handler has not registered
// yet is not supported by this version.
var KnownKeys = []string{
	"alpine", "cargo", "composer", "conan", "conda", "cran", "deb", "freebsd", "gem",
	"generic", "golang", "helm", "hex", "julia", "maven", "npm", "nuget", "oci", "pub",
	"pypi", "rpm", "swift",
}

// IsKnown reports whether key is one of KnownKeys.
func IsKnown(key string) bool {
	return slices.Contains(KnownKeys, key)
}

// Spec describes how one ecosystem takes part in retention.
type Spec struct {
	// Key is the ecosystem's config key, one of KnownKeys.
	Key string

	// ExtraDBEcosystems lists packages.ecosystem values besides the ones
	// derived from Key that belong to this ecosystem.
	ExtraDBEcosystems []string

	// CanonicalPackage turns a package PURL from the config into a
	// policy key, reporting false when the
	// PURL does not belong to this ecosystem. Nil means the ecosystem accepts
	// no package overrides.
	CanonicalPackage func(p *purl.PURL) (string, bool)

	// SourcePackage reports a package PURL that names the package by where
	// its source lives, such as a Swift package by its repository. It has to
	// be resolved through the upstream before it can be compared, so the
	// config keeps it in Rules.Sources until then. Nil means the ecosystem
	// has no such form.
	SourcePackage func(p *purl.PURL) bool
}

// dbEcosystems returns every packages.ecosystem value that belongs to the
// spec: the key, its PURL type and the ecosystem name that type maps back
// to, the same spellings git-pkgs matches. The proxy's mirror command and
// git-pkgs store "rubygems" where the gem handler stores "gem", for example.
func (s Spec) dbEcosystems() []string {
	purlType := purl.EcosystemToPURLType(s.Key)
	values := []string{s.Key, purlType, purl.PURLTypeToEcosystem(purlType)}
	values = append(values, s.ExtraDBEcosystems...)
	slices.Sort(values)
	return slices.Compact(values)
}

// DefaultCanonical returns a CanonicalPackage for ecosystems that store the
// package PURL exactly as purl.MakePURLString builds it from the PURL's full
// name. It accepts a PURL only when building it back that way gives the same
// package, so for an ecosystem that stores a different form, such as one
// with a default namespace, it refuses the key instead of producing one
// that never matches. Such ecosystems need their own CanonicalPackage.
func DefaultCanonical(key string) func(p *purl.PURL) (string, bool) {
	purlType := purl.EcosystemToPURLType(key)
	return func(p *purl.PURL) (string, bool) {
		if p == nil || p.Type != purlType {
			return "", false
		}
		canonical := purl.MakePURLString(key, p.FullName(), "")
		if canonical == "" {
			return "", false
		}
		given := purl.New(p.Type, p.Namespace, p.Name, "", nil).String()
		return canonical, canonical == given
	}
}

// Registry maps ecosystem keys to their specs.
type Registry struct {
	mu    sync.RWMutex
	specs map[string]Spec
	byDB  map[string]string
}

// NewRegistry returns an empty registry.
func NewRegistry() *Registry {
	return &Registry{specs: map[string]Spec{}, byDB: map[string]string{}}
}

// Default is the registry the handlers register with.
var Default = NewRegistry()

// Register adds s. It panics when the key is not one of KnownKeys, is
// already registered, or claims a packages.ecosystem value another spec
// claims, since each of those is a programming error.
func (r *Registry) Register(s Spec) {
	r.mu.Lock()
	defer r.mu.Unlock()

	if !IsKnown(s.Key) {
		panic(fmt.Sprintf("retention: unknown ecosystem key %q", s.Key))
	}
	if _, ok := r.specs[s.Key]; ok {
		panic(fmt.Sprintf("retention: ecosystem %q registered twice", s.Key))
	}
	values := s.dbEcosystems()
	for _, v := range values {
		if owner, ok := r.byDB[v]; ok {
			panic(fmt.Sprintf("retention: ecosystem value %q claimed by both %q and %q", v, owner, s.Key))
		}
	}
	r.specs[s.Key] = s
	for _, v := range values {
		r.byDB[v] = s.Key
	}
}

// Lookup returns the spec registered for key.
func (r *Registry) Lookup(key string) (Spec, bool) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	s, ok := r.specs[key]
	return s, ok
}

// Keys returns the registered keys in sorted order.
func (r *Registry) Keys() []string {
	r.mu.RLock()
	defer r.mu.RUnlock()
	keys := make([]string, 0, len(r.specs))
	for k := range r.specs {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// KeyForDBEcosystem returns the registered key a packages.ecosystem value
// belongs to.
func (r *Registry) KeyForDBEcosystem(ecosystem string) (string, bool) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	key, ok := r.byDB[ecosystem]
	return key, ok
}

// Canonical returns the policy key for a configured package PURL.
func (r *Registry) Canonical(p *purl.PURL) (string, bool) {
	for _, key := range r.Keys() {
		s, _ := r.Lookup(key)
		if s.CanonicalPackage == nil {
			continue
		}
		if canonical, ok := s.CanonicalPackage(p); ok {
			return canonical, true
		}
	}
	return "", false
}

// Source returns the ecosystem key whose SourcePackage claims p.
func (r *Registry) Source(p *purl.PURL) (string, bool) {
	for _, key := range r.Keys() {
		s, _ := r.Lookup(key)
		if s.SourcePackage != nil && s.SourcePackage(p) {
			return key, true
		}
	}
	return "", false
}

// Rules are resolved retention durations. A duration of zero means an
// artifact is never evicted by age.
type Rules struct {
	// Default applies to registered ecosystems without their own rule,
	// except generic, which requires an explicit rule.
	Default time.Duration
	// Ecosystems holds rules keyed by registered ecosystem key.
	Ecosystems map[string]time.Duration
	// Packages holds rules keyed by package PURL, with Go keys lowercased.
	Packages map[string]time.Duration
	// Sources holds package rules not resolved to a stored package PURL
	// yet, by ecosystem key and then by the configured PURL (see
	// Spec.SourcePackage). An ecosystem with any is not swept, since one of
	// them may keep an artifact the sweep would otherwise clear.
	Sources map[string]map[string]time.Duration
}

// Pending reports whether the ecosystem key has package rules still to be
// resolved, and must not be swept.
func (r Rules) Pending(key string) bool {
	return len(r.Sources[key]) > 0
}

// Resolve moves the rule for source in ecosystem key to every stored package
// PURL it resolved to. A different rule already set for any of them is a
// conflict, and the source stays pending with none of them set.
func (r Rules) Resolve(key, source string, canonical []string) error {
	d, ok := r.Sources[key][source]
	if !ok {
		return nil
	}
	if len(canonical) == 0 {
		return fmt.Errorf("%s resolves to no stored package", source)
	}
	for _, c := range canonical {
		if prev, set := r.Packages[c]; set && prev != d {
			return fmt.Errorf("%s resolves to %s, which another rule sets differently", source, c)
		}
	}
	for _, c := range canonical {
		r.Packages[c] = d
	}
	delete(r.Sources[key], source)
	return nil
}

// For returns the retention for an artifact of the given ecosystem key and
// stored package PURL: the package rule, else the ecosystem rule, else the
// default. Generic artifacts require an explicit package or ecosystem rule.
func (r Rules) For(key, packagePURL string) time.Duration {
	packagePURL = packageurl.PolicyPackageKey(packagePURL)
	if d, ok := r.Packages[packagePURL]; ok {
		return d
	}
	if d, ok := r.Ecosystems[key]; ok {
		return d
	}
	// Release assets may disappear upstream, so keep generic opt-in.
	if key == "generic" {
		return 0
	}
	return r.Default
}

// MinPositive returns the shortest non-zero duration among all rules, or
// zero when every rule is zero.
func (r Rules) MinPositive() time.Duration {
	minimum := time.Duration(0)
	consider := func(d time.Duration) {
		if d > 0 && (minimum == 0 || d < minimum) {
			minimum = d
		}
	}
	consider(r.Default)
	for _, d := range r.Ecosystems {
		consider(d)
	}
	for _, d := range r.Packages {
		consider(d)
	}
	for _, sources := range r.Sources {
		for _, d := range sources {
			consider(d)
		}
	}
	return minimum
}