// Package retention holds the per-ecosystem rules for evicting cached // artifacts that have not been accessed for a configured time. // // An ecosystem takes part only once its handler registers a Spec with // Default, usually from an init function in the handler's own file. Until // then configuring retention for it is a validation error, and the global // default does not apply to it. package retention import ( "fmt" "slices" "sort" "sync" "time" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/purl" ) // KnownKeys lists the ecosystem keys storage.retention.ecosystems accepts. // They are the literals the handlers pass when they fetch an artifact. A key // outside this list is a typo; a key in it whose handler has not registered // yet is not supported by this version. var KnownKeys = []string{ "alpine", "cargo", "composer", "conan", "conda", "cran", "deb", "freebsd", "gem", "generic", "golang", "helm", "hex", "julia", "maven", "npm", "nuget", "oci", "pub", "pypi", "rpm", "swift", } // IsKnown reports whether key is one of KnownKeys. func IsKnown(key string) bool { return slices.Contains(KnownKeys, key) } // Spec describes how one ecosystem takes part in retention. type Spec struct { // Key is the ecosystem's config key, one of KnownKeys. Key string // ExtraDBEcosystems lists packages.ecosystem values besides the ones // derived from Key that belong to this ecosystem. ExtraDBEcosystems []string // CanonicalPackage turns a package PURL from the config into a // policy key, reporting false when the // PURL does not belong to this ecosystem. Nil means the ecosystem accepts // no package overrides. CanonicalPackage func(p *purl.PURL) (string, bool) // SourcePackage reports a package PURL that names the package by where // its source lives, such as a Swift package by its repository. It has to // be resolved through the upstream before it can be compared, so the // config keeps it in Rules.Sources until then. Nil means the ecosystem // has no such form. SourcePackage func(p *purl.PURL) bool } // dbEcosystems returns every packages.ecosystem value that belongs to the // spec: the key, its PURL type and the ecosystem name that type maps back // to, the same spellings git-pkgs matches. The proxy's mirror command and // git-pkgs store "rubygems" where the gem handler stores "gem", for example. func (s Spec) dbEcosystems() []string { purlType := purl.EcosystemToPURLType(s.Key) values := []string{s.Key, purlType, purl.PURLTypeToEcosystem(purlType)} values = append(values, s.ExtraDBEcosystems...) slices.Sort(values) return slices.Compact(values) } // DefaultCanonical returns a CanonicalPackage for ecosystems that store the // package PURL exactly as purl.MakePURLString builds it from the PURL's full // name. It accepts a PURL only when building it back that way gives the same // package, so for an ecosystem that stores a different form, such as one // with a default namespace, it refuses the key instead of producing one // that never matches. Such ecosystems need their own CanonicalPackage. func DefaultCanonical(key string) func(p *purl.PURL) (string, bool) { purlType := purl.EcosystemToPURLType(key) return func(p *purl.PURL) (string, bool) { if p == nil || p.Type != purlType { return "", false } canonical := purl.MakePURLString(key, p.FullName(), "") if canonical == "" { return "", false } given := purl.New(p.Type, p.Namespace, p.Name, "", nil).String() return canonical, canonical == given } } // Registry maps ecosystem keys to their specs. type Registry struct { mu sync.RWMutex specs map[string]Spec byDB map[string]string } // NewRegistry returns an empty registry. func NewRegistry() *Registry { return &Registry{specs: map[string]Spec{}, byDB: map[string]string{}} } // Default is the registry the handlers register with. var Default = NewRegistry() // Register adds s. It panics when the key is not one of KnownKeys, is // already registered, or claims a packages.ecosystem value another spec // claims, since each of those is a programming error. func (r *Registry) Register(s Spec) { r.mu.Lock() defer r.mu.Unlock() if !IsKnown(s.Key) { panic(fmt.Sprintf("retention: unknown ecosystem key %q", s.Key)) } if _, ok := r.specs[s.Key]; ok { panic(fmt.Sprintf("retention: ecosystem %q registered twice", s.Key)) } values := s.dbEcosystems() for _, v := range values { if owner, ok := r.byDB[v]; ok { panic(fmt.Sprintf("retention: ecosystem value %q claimed by both %q and %q", v, owner, s.Key)) } } r.specs[s.Key] = s for _, v := range values { r.byDB[v] = s.Key } } // Lookup returns the spec registered for key. func (r *Registry) Lookup(key string) (Spec, bool) { r.mu.RLock() defer r.mu.RUnlock() s, ok := r.specs[key] return s, ok } // Keys returns the registered keys in sorted order. func (r *Registry) Keys() []string { r.mu.RLock() defer r.mu.RUnlock() keys := make([]string, 0, len(r.specs)) for k := range r.specs { keys = append(keys, k) } sort.Strings(keys) return keys } // KeyForDBEcosystem returns the registered key a packages.ecosystem value // belongs to. func (r *Registry) KeyForDBEcosystem(ecosystem string) (string, bool) { r.mu.RLock() defer r.mu.RUnlock() key, ok := r.byDB[ecosystem] return key, ok } // Canonical returns the policy key for a configured package PURL. func (r *Registry) Canonical(p *purl.PURL) (string, bool) { for _, key := range r.Keys() { s, _ := r.Lookup(key) if s.CanonicalPackage == nil { continue } if canonical, ok := s.CanonicalPackage(p); ok { return canonical, true } } return "", false } // Source returns the ecosystem key whose SourcePackage claims p. func (r *Registry) Source(p *purl.PURL) (string, bool) { for _, key := range r.Keys() { s, _ := r.Lookup(key) if s.SourcePackage != nil && s.SourcePackage(p) { return key, true } } return "", false } // Rules are resolved retention durations. A duration of zero means an // artifact is never evicted by age. type Rules struct { // Default applies to registered ecosystems without their own rule, // except generic, which requires an explicit rule. Default time.Duration // Ecosystems holds rules keyed by registered ecosystem key. Ecosystems map[string]time.Duration // Packages holds rules keyed by package PURL, with Go keys lowercased. Packages map[string]time.Duration // Sources holds package rules not resolved to a stored package PURL // yet, by ecosystem key and then by the configured PURL (see // Spec.SourcePackage). An ecosystem with any is not swept, since one of // them may keep an artifact the sweep would otherwise clear. Sources map[string]map[string]time.Duration } // Pending reports whether the ecosystem key has package rules still to be // resolved, and must not be swept. func (r Rules) Pending(key string) bool { return len(r.Sources[key]) > 0 } // Resolve moves the rule for source in ecosystem key to every stored package // PURL it resolved to. A different rule already set for any of them is a // conflict, and the source stays pending with none of them set. func (r Rules) Resolve(key, source string, canonical []string) error { d, ok := r.Sources[key][source] if !ok { return nil } if len(canonical) == 0 { return fmt.Errorf("%s resolves to no stored package", source) } for _, c := range canonical { if prev, set := r.Packages[c]; set && prev != d { return fmt.Errorf("%s resolves to %s, which another rule sets differently", source, c) } } for _, c := range canonical { r.Packages[c] = d } delete(r.Sources[key], source) return nil } // For returns the retention for an artifact of the given ecosystem key and // stored package PURL: the package rule, else the ecosystem rule, else the // default. Generic artifacts require an explicit package or ecosystem rule. func (r Rules) For(key, packagePURL string) time.Duration { packagePURL = packageurl.PolicyPackageKey(packagePURL) if d, ok := r.Packages[packagePURL]; ok { return d } if d, ok := r.Ecosystems[key]; ok { return d } // Release assets may disappear upstream, so keep generic opt-in. if key == "generic" { return 0 } return r.Default } // MinPositive returns the shortest non-zero duration among all rules, or // zero when every rule is zero. func (r Rules) MinPositive() time.Duration { minimum := time.Duration(0) consider := func(d time.Duration) { if d > 0 && (minimum == 0 || d < minimum) { minimum = d } } consider(r.Default) for _, d := range r.Ecosystems { consider(d) } for _, d := range r.Packages { consider(d) } for _, sources := range r.Sources { for _, d := range sources { consider(d) } } return minimum }