mirror of
https://github.com/git-pkgs/proxy.git
synced 2026-11-09 23:59:16 -05:00
1389 lines
45 KiB
Go
1389 lines
45 KiB
Go
// Package server provides the HTTP server and router for the proxy.
|
|
//
|
|
// The server mounts protocol handlers at their respective paths:
|
|
// - /npm/* - npm registry protocol
|
|
// - /cargo/* - Cargo registry protocol (sparse index)
|
|
// - /gem/* - RubyGems registry protocol
|
|
// - /go/* - Go module proxy protocol
|
|
// - /hex/* - Hex.pm registry protocol
|
|
// - /pub/* - pub.dev registry protocol
|
|
// - /pypi/* - PyPI registry protocol
|
|
// - /maven/* - Maven repository protocol
|
|
// - /gradle/* - Gradle HttpBuildCache protocol
|
|
// - /nuget/* - NuGet V3 API protocol
|
|
// - /composer/* - Composer/Packagist protocol
|
|
// - /conan/* - Conan C/C++ protocol
|
|
// - /conda/* - Conda/Anaconda protocol
|
|
// - /cran/* - CRAN (R) protocol
|
|
// - /julia/* - Julia Pkg server protocol
|
|
// - /swift/* - Swift Package Registry protocol
|
|
// - /v2/* - OCI/Docker container registry protocol
|
|
// - /apk/* - Alpine APK repository protocol
|
|
// - /arch/* - Arch Linux pacman repository protocol
|
|
// - /debian/* - Debian/APT repository protocol
|
|
// - /rpm/* - RPM/Yum repository protocol
|
|
//
|
|
// Additional endpoints:
|
|
// - /health - Health check endpoint
|
|
// - /stats - Cache statistics (JSON)
|
|
// - /openapi.json - OpenAPI spec (JSON)
|
|
// - /metrics - Prometheus metrics
|
|
//
|
|
// Web UI (HTML), mounted under /ui so reverse proxies can gate it
|
|
// separately from the package endpoints:
|
|
// - /ui/ - Dashboard
|
|
// - /ui/analytics - Download and cache analytics
|
|
// - /ui/install - Client configuration guide
|
|
// - /ui/packages - List all cached packages
|
|
// - /ui/search - Search packages
|
|
// - /ui/package/... - Package and version detail pages
|
|
// - /ui/api/browse/... - Archive browsing (used by the UI)
|
|
// - /ui/api/compare/... - Archive diffing (used by the UI)
|
|
//
|
|
// API endpoints for enrichment data:
|
|
// - GET /api/package/{ecosystem}/{name} - Package metadata
|
|
// - GET /api/package/{ecosystem}/{name}/{version} - Version metadata with vulns
|
|
// - GET /api/vulns/{ecosystem}/{name} - Package vulnerabilities
|
|
// - GET /api/vulns/{ecosystem}/{name}/{version} - Version vulnerabilities
|
|
// - POST /api/outdated - Check outdated packages
|
|
// - POST /api/bulk - Bulk package lookup
|
|
// - GET /api/packages - List cached packages (JSON)
|
|
package server
|
|
|
|
import (
|
|
"cmp"
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/git-pkgs/archives"
|
|
"github.com/git-pkgs/cooldown"
|
|
swaggerdoc "github.com/git-pkgs/proxy/docs/swagger"
|
|
"github.com/git-pkgs/proxy/internal/accesslog"
|
|
"github.com/git-pkgs/proxy/internal/config"
|
|
"github.com/git-pkgs/proxy/internal/cooldownpolicy"
|
|
"github.com/git-pkgs/proxy/internal/database"
|
|
"github.com/git-pkgs/proxy/internal/denylist"
|
|
"github.com/git-pkgs/proxy/internal/enrichment"
|
|
"github.com/git-pkgs/proxy/internal/handler"
|
|
upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient"
|
|
"github.com/git-pkgs/proxy/internal/metrics"
|
|
"github.com/git-pkgs/proxy/internal/mirror"
|
|
"github.com/git-pkgs/proxy/internal/packageurl"
|
|
"github.com/git-pkgs/proxy/internal/scanner"
|
|
"github.com/git-pkgs/proxy/internal/storage"
|
|
"github.com/git-pkgs/registries/fetch"
|
|
"github.com/git-pkgs/registries/safehttp"
|
|
"github.com/git-pkgs/spdx"
|
|
"github.com/go-chi/chi/v5"
|
|
"github.com/go-chi/chi/v5/middleware"
|
|
)
|
|
|
|
const (
|
|
serverReadTimeout = 30 * time.Second
|
|
serverWriteTimeout = 5 * time.Minute
|
|
serverIdleTimeout = 60 * time.Second
|
|
dashboardTopN = 10
|
|
hoursPerDay = 24
|
|
|
|
// Upstream transport defaults, matching what fetch.NewFetcher would use
|
|
// if we did not hand it our own client. Go's default transport keeps only
|
|
// two idle connections per host and never times out waiting for response
|
|
// headers.
|
|
upstreamMaxIdleConnsPerHost = 10
|
|
upstreamResponseHeaderTimeout = 60 * time.Second
|
|
)
|
|
|
|
// Server is the main proxy server.
|
|
type Server struct {
|
|
cfg *config.Config
|
|
db *database.DB
|
|
storage storage.Storage
|
|
logger *slog.Logger
|
|
buildInfo BuildInfo
|
|
http *http.Server
|
|
templates *Templates
|
|
cancel context.CancelFunc
|
|
healthCache *healthCache
|
|
accessLog *accesslog.Logger
|
|
ecr *ecrTokens
|
|
breakers *breakerMonitor
|
|
rateLimits *upstreamhttp.RateLimitGate
|
|
ecoStats ecosystemStatsCache
|
|
sources sourceTracker
|
|
|
|
// browseLimits overrides defaultBrowseLimits when set; tests use it.
|
|
browseLimits archives.StreamOptions
|
|
|
|
compares compareLimiter
|
|
}
|
|
|
|
// New creates a new Server with the given configuration.
|
|
func New(cfg *config.Config, logger *slog.Logger, buildInfo BuildInfo) (*Server, error) {
|
|
var activityLog *accesslog.Logger
|
|
if cfg.AccessLog.Path != "" {
|
|
var err error
|
|
activityLog, err = accesslog.Open(cfg.AccessLog.Path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("initializing access log: %w", err)
|
|
}
|
|
}
|
|
closeAccessLog := true
|
|
defer func() {
|
|
if closeAccessLog && activityLog != nil {
|
|
_ = activityLog.Close()
|
|
}
|
|
}()
|
|
|
|
// Initialize database
|
|
var db *database.DB
|
|
var err error
|
|
|
|
switch cfg.Database.Driver {
|
|
case "postgres":
|
|
db, err = database.OpenPostgresOrCreate(cfg.Database.URL)
|
|
default:
|
|
db, err = database.OpenOrCreate(cfg.Database.Path)
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("opening database: %w", err)
|
|
}
|
|
|
|
// Run schema migration to add missing columns
|
|
if err := db.MigrateSchema(); err != nil {
|
|
_ = db.Close()
|
|
return nil, fmt.Errorf("migrating database schema: %w", err)
|
|
}
|
|
db.BatchHits(cfg.ParseHitFlushInterval(), logger)
|
|
|
|
// Initialize storage
|
|
storageURL := cfg.Storage.URL
|
|
if storageURL == "" {
|
|
// Fall back to file:// with Path
|
|
storageURL = "file://" + cfg.Storage.Path //nolint:staticcheck // backwards compat
|
|
}
|
|
store, err := storage.OpenBucket(context.Background(), storageURL)
|
|
if err != nil {
|
|
_ = db.Close()
|
|
return nil, fmt.Errorf("initializing storage: %w", err)
|
|
}
|
|
|
|
// Verify storage is accessible (catches bad S3 credentials/endpoints early).
|
|
// Exists returns (false, nil) for a missing key, so only real connectivity
|
|
// or permission errors surface here.
|
|
if _, err := store.Exists(context.Background(), ".health-check"); err != nil {
|
|
_ = store.Close()
|
|
_ = db.Close()
|
|
return nil, fmt.Errorf("verifying storage connectivity: %w", err)
|
|
}
|
|
|
|
hc, err := newHealthCache(store, cfg.Health.StorageProbeInterval, logger)
|
|
if err != nil {
|
|
_ = store.Close()
|
|
_ = db.Close()
|
|
return nil, fmt.Errorf("initializing health cache: %w", err)
|
|
}
|
|
|
|
server := &Server{
|
|
cfg: cfg,
|
|
db: db,
|
|
storage: store,
|
|
logger: logger,
|
|
buildInfo: buildInfo,
|
|
templates: &Templates{},
|
|
healthCache: hc,
|
|
accessLog: activityLog,
|
|
ecr: newECRTokens(logger),
|
|
}
|
|
closeAccessLog = false
|
|
return server, nil
|
|
}
|
|
|
|
// Start starts the HTTP server.
|
|
func (s *Server) Start(listeners ...net.Listener) error {
|
|
if len(listeners) > 1 {
|
|
return errors.New("only one listener is supported")
|
|
}
|
|
var listener net.Listener
|
|
if len(listeners) == 1 {
|
|
listener = listeners[0]
|
|
if listener == nil {
|
|
return errors.New("listener is required")
|
|
}
|
|
}
|
|
return s.serve(listener)
|
|
}
|
|
|
|
func (s *Server) serve(listener net.Listener) error {
|
|
metadataClient, artifactClient := s.newUpstreamClients(serverWriteTimeout)
|
|
|
|
// Create shared components with circuit breaker.
|
|
baseFetcher := fetch.NewFetcher(fetch.WithHTTPClient(artifactClient))
|
|
fetcher := fetch.NewCircuitBreakerFetcher(baseFetcher)
|
|
s.breakers = newBreakerMonitor(fetcher, s.logger)
|
|
resolver := fetch.NewResolver()
|
|
cd := &cooldown.Config{
|
|
Default: s.cfg.Cooldown.Default,
|
|
Ecosystems: s.cfg.Cooldown.Ecosystems,
|
|
Packages: s.cfg.Cooldown.NormalizedPackages(),
|
|
}
|
|
proxy := handler.NewProxy(s.db, s.storage, handler.LimitArtifactSize(fetcher, s.cfg.ParseArtifactMaxSize()), resolver, s.logger)
|
|
proxy.HTTPClient = metadataClient
|
|
proxy.RateLimits = s.rateLimits
|
|
cooldownPolicy, err := cooldownpolicy.New(cd, s.cfg.Cooldown.PackagePatterns)
|
|
if err != nil {
|
|
return fmt.Errorf("configuring cooldown policy: %w", err)
|
|
}
|
|
proxy.Cooldown = cooldownPolicy
|
|
policy, err := denylist.New(s.cfg.Denylist.Packages)
|
|
if err != nil {
|
|
return fmt.Errorf("configuring denylist: %w", err)
|
|
}
|
|
proxy.Denylist = policy
|
|
scanGroup, err := configureScanning(proxy, s.cfg.Scanning, s.cfg.BaseURL, s.logger)
|
|
if err != nil {
|
|
return fmt.Errorf("configuring scanners: %w", err)
|
|
}
|
|
proxy.CacheMetadata = s.cfg.CacheMetadata
|
|
proxy.MetadataTTL = s.cfg.ParseMetadataTTL()
|
|
proxy.MetadataMaxSize = s.cfg.ParseMetadataMaxSize()
|
|
proxy.SetMetadataRewriteCacheSize(s.cfg.ParseMetadataRewriteCacheSize())
|
|
proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly
|
|
proxy.HostedEnabled = s.cfg.Hosted.Enabled
|
|
proxy.HostedNPMScopes = s.cfg.Hosted.NPMScopes
|
|
proxy.HostedComposerVendors = s.cfg.Hosted.ComposerVendors
|
|
proxy.NPMFullMetadata = s.cfg.Upstream.NPMFullMetadata
|
|
proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize()
|
|
proxy.DirectServe = s.cfg.Storage.DirectServe
|
|
proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL()
|
|
proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL
|
|
proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts
|
|
|
|
// Create router with Chi
|
|
r := chi.NewRouter()
|
|
|
|
// Add middleware
|
|
r.Use(middleware.RequestID)
|
|
r.Use(RequestIDMiddleware)
|
|
r.Use(s.LoggerMiddleware)
|
|
r.Use(middleware.Recoverer)
|
|
r.Use(func(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/metrics" {
|
|
metrics.IncrementActiveRequests()
|
|
defer metrics.DecrementActiveRequests()
|
|
}
|
|
next.ServeHTTP(w, r)
|
|
})
|
|
})
|
|
r.Use(func(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
next.ServeHTTP(w, r.WithContext(handler.DetachUpstreamFetch(r.Context())))
|
|
})
|
|
})
|
|
|
|
// Mount protocol handlers
|
|
s.mountProtocolHandlers(r, proxy)
|
|
|
|
// Health, stats, and metrics endpoints
|
|
r.Get("/health", s.handleHealth)
|
|
r.Get("/stats", s.handleStats)
|
|
r.Get("/openapi.json", s.handleOpenAPIJSON)
|
|
r.Get("/metrics", func(w http.ResponseWriter, r *http.Request) {
|
|
// Breaker state is only held in the fetcher, so publish it on scrape.
|
|
s.breakers.snapshot()
|
|
metrics.Handler().ServeHTTP(w, r)
|
|
})
|
|
|
|
// Internal route used by external scanners to pull staged artifact
|
|
// bytes before they're committed to the cache. Only wired up when
|
|
// scanning is actually configured, so there's no unauthenticated path
|
|
// to storage objects sitting in the router when the feature is unused.
|
|
// Restrict this to internal-network access only at the
|
|
// ingress/network-policy layer.
|
|
if scanGroup.Enabled() && len(proxy.ScanSigningKey) > 0 {
|
|
r.Get("/_internal/scan-fetch", proxy.ServeScanFetch)
|
|
}
|
|
|
|
// Web UI. Mounted under /ui so a reverse proxy can apply different
|
|
// access rules to it than to the package endpoints above (#123).
|
|
r.Route("/ui", func(ui chi.Router) {
|
|
ui.Mount("/static", http.StripPrefix("/ui/static/", staticHandler()))
|
|
ui.Get("/", s.handleRoot)
|
|
ui.Get("/analytics", s.handleAnalytics)
|
|
ui.Get("/install", s.handleInstall)
|
|
ui.Get("/search", s.handleSearch)
|
|
ui.Get("/packages", s.handlePackagesList)
|
|
ui.Get("/package/{ecosystem}/*", s.handlePackagePath)
|
|
ui.Get("/api/browse/{ecosystem}/*", s.handleBrowsePath)
|
|
ui.Get("/api/compare/{ecosystem}/*", s.handleComparePath)
|
|
})
|
|
r.Get("/", func(w http.ResponseWriter, r *http.Request) {
|
|
http.Redirect(w, r, "/ui/", http.StatusFound)
|
|
})
|
|
|
|
// API endpoints for enrichment data
|
|
enrichSvc := enrichment.New(s.logger)
|
|
apiHandler := NewAPIHandler(enrichSvc, s.db)
|
|
|
|
r.Get("/api/package/{ecosystem}/*", apiHandler.HandlePackagePath)
|
|
r.Get("/api/vulns/{ecosystem}/*", apiHandler.HandleVulnsPath)
|
|
r.Post("/api/outdated", apiHandler.HandleOutdated)
|
|
r.Post("/api/bulk", apiHandler.HandleBulkLookup)
|
|
r.Get("/api/search", apiHandler.HandleSearch)
|
|
r.Get("/api/packages", apiHandler.HandlePackagesList)
|
|
|
|
// Start background context (used by mirror jobs and cleanup)
|
|
bgCtx, bgCancel := context.WithCancel(context.Background())
|
|
s.cancel = bgCancel
|
|
s.startGradleBuildCacheEviction(bgCtx)
|
|
|
|
// Mirror API endpoints (opt-in via mirror_api config or PROXY_MIRROR_API env)
|
|
if s.cfg.MirrorAPI {
|
|
mirrorSvc := mirror.New(proxy, s.db, s.storage, s.logger, 4) //nolint:mnd // default concurrency
|
|
mirrorSvc.Images = s.newContainerHandler(proxy)
|
|
jobStore := mirror.NewJobStore(bgCtx, mirrorSvc)
|
|
mirrorAPI := NewMirrorAPIHandler(jobStore)
|
|
r.Post("/api/mirror", mirrorAPI.HandleCreate)
|
|
r.Get("/api/mirror/{id}", mirrorAPI.HandleGet)
|
|
r.Delete("/api/mirror/{id}", mirrorAPI.HandleCancel)
|
|
go jobStore.StartCleanup(bgCtx)
|
|
}
|
|
|
|
// Hosted package endpoints (opt-in via hosted.enabled or PROXY_HOSTED_ENABLED)
|
|
if s.cfg.Hosted.Enabled {
|
|
hostedAPI := NewHostedAPIHandler(proxy, s.cfg.Hosted.MaxUploadBytes(), s.logger)
|
|
registerHostedRoutes(r, hostedAPI, s.cfg.Hosted.TokenExpanded())
|
|
}
|
|
|
|
s.http = &http.Server{
|
|
Addr: s.cfg.Listen,
|
|
Handler: r,
|
|
ReadTimeout: serverReadTimeout,
|
|
WriteTimeout: serverWriteTimeout, // Large artifacts need time
|
|
IdleTimeout: serverIdleTimeout,
|
|
}
|
|
|
|
s.logger.Info("starting server",
|
|
"listen", s.cfg.Listen,
|
|
"base_url", s.cfg.BaseURL,
|
|
"ui_url", s.cfg.UIBaseURL,
|
|
"storage", s.storage.URL(),
|
|
"database", s.cfg.Database.String())
|
|
go s.updateCacheStatsMetrics()
|
|
go s.startEvictionLoop(bgCtx)
|
|
go s.startRetentionLoop(bgCtx)
|
|
go s.startReclaimLoop(bgCtx)
|
|
|
|
if listener != nil {
|
|
return s.http.Serve(listener)
|
|
}
|
|
return s.http.ListenAndServe()
|
|
}
|
|
|
|
// mountProtocolHandlers constructs every ecosystem handler and mounts it on
|
|
// r under its protocol prefix.
|
|
func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) {
|
|
npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.NPM).
|
|
WithPackageRoutes(s.cfg.Upstream.NPMRoutes)
|
|
cargoHandler := handler.NewCargoHandler(
|
|
proxy,
|
|
s.cfg.BaseURL,
|
|
s.cfg.Upstream.Cargo,
|
|
s.cfg.Upstream.CargoDownload,
|
|
)
|
|
gemHandler := handler.NewGemHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Gem)
|
|
goHandler := handler.NewGoHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Go)
|
|
goHandler.SetSumDBs(s.cfg.Upstream.GoSumDBs, s.cfg.Upstream.GoSumDBsOffline)
|
|
hexHandler := handler.NewHexHandlerWithUpstreams(
|
|
proxy,
|
|
s.cfg.BaseURL,
|
|
s.cfg.Upstream.Hex,
|
|
s.cfg.Upstream.HexAPI,
|
|
)
|
|
pubHandler := handler.NewPubHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Pub)
|
|
pypiHandler := handler.NewPyPIHandlerWithUpstreams(
|
|
proxy,
|
|
s.cfg.BaseURL,
|
|
s.cfg.Upstream.PyPI,
|
|
s.cfg.Upstream.PyPIDownload,
|
|
)
|
|
mavenHandler := handler.NewMavenHandler(
|
|
proxy,
|
|
s.cfg.BaseURL,
|
|
s.cfg.Upstream.Maven,
|
|
s.cfg.Upstream.GradlePluginPortal,
|
|
)
|
|
gradleHandler := handler.NewGradleBuildCacheHandler(proxy)
|
|
nugetHandler := handler.NewNuGetHandlerWithUpstreams(
|
|
proxy,
|
|
s.cfg.BaseURL,
|
|
s.cfg.Upstream.NuGet,
|
|
s.cfg.Upstream.NuGetSearch,
|
|
)
|
|
composerHandler := handler.NewComposerHandlerWithUpstreams(
|
|
proxy,
|
|
s.cfg.BaseURL,
|
|
s.cfg.Upstream.Composer,
|
|
s.cfg.Upstream.ComposerRepository,
|
|
).WithPackageRoutes(s.cfg.Upstream.ComposerRoutes)
|
|
conanHandler := handler.NewConanHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conan)
|
|
condaHandler := handler.NewCondaHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conda)
|
|
cranHandler := handler.NewCRANHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.CRAN)
|
|
juliaHandler := handler.NewJuliaHandlerWithUpstream(proxy, s.cfg.Upstream.Julia)
|
|
swiftHandler := handler.NewSwiftHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Swift)
|
|
homebrewHandler := handler.NewHomebrewHandler(proxy, s.cfg.Upstream.HomebrewAPI)
|
|
containerHandler := s.newContainerHandler(proxy)
|
|
helmHandler := handler.NewHelmHandlerWithOCIRegistries(
|
|
proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI)
|
|
apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK)
|
|
archHandler := handler.NewArchHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Arch)
|
|
debianHandler := handler.NewDebianHandler(
|
|
proxy,
|
|
s.cfg.BaseURL,
|
|
s.cfg.Upstream.Debian,
|
|
s.cfg.Upstream.DebianRepositories,
|
|
)
|
|
rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM)
|
|
genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts)
|
|
|
|
r = r.With(sandboxUpstreamContent, rejectQueryInPath)
|
|
r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes()))
|
|
r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes()))
|
|
r.Mount("/gem", http.StripPrefix("/gem", gemHandler.Routes()))
|
|
r.Mount("/go", http.StripPrefix("/go", goHandler.Routes()))
|
|
r.Mount("/hex", http.StripPrefix("/hex", hexHandler.Routes()))
|
|
r.Mount("/pub", http.StripPrefix("/pub", pubHandler.Routes()))
|
|
r.Mount("/pypi", http.StripPrefix("/pypi", pypiHandler.Routes()))
|
|
r.Mount("/maven", http.StripPrefix("/maven", mavenHandler.Routes()))
|
|
r.Mount("/gradle", http.StripPrefix("/gradle", gradleHandler.Routes()))
|
|
r.Mount("/nuget", http.StripPrefix("/nuget", nugetHandler.Routes()))
|
|
r.Mount("/composer", http.StripPrefix("/composer", composerHandler.Routes()))
|
|
r.Mount("/conan", http.StripPrefix("/conan", conanHandler.Routes()))
|
|
r.Mount("/conda", http.StripPrefix("/conda", condaHandler.Routes()))
|
|
r.Mount("/cran", http.StripPrefix("/cran", cranHandler.Routes()))
|
|
r.Mount("/julia", http.StripPrefix("/julia", juliaHandler.Routes()))
|
|
r.Mount("/swift", http.StripPrefix("/swift", swiftHandler.Routes()))
|
|
r.Mount("/homebrew", http.StripPrefix("/homebrew", homebrewHandler.Routes()))
|
|
r.Mount("/v2", http.StripPrefix("/v2", containerHandler.Routes()))
|
|
r.Mount("/helm", http.StripPrefix("/helm", helmHandler.Routes()))
|
|
r.Mount("/apk", http.StripPrefix("/apk", apkHandler.Routes()))
|
|
r.Mount("/arch", http.StripPrefix("/arch", archHandler.Routes()))
|
|
r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes()))
|
|
r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes()))
|
|
r.Mount("/generic", http.StripPrefix("/generic", genericHandler.Routes()))
|
|
}
|
|
|
|
// configureScanning builds the scanner group from cfg and wires it into
|
|
// proxy, returning the group so the caller can decide whether to mount the
|
|
// internal scan-fetch route.
|
|
// newUpstreamClients builds the metadata and artifact clients. Both share one
|
|
// authentication-aware transport so configured credentials and cached OCI
|
|
// challenges apply consistently. An artifactTimeout of zero means no limit.
|
|
func (s *Server) newUpstreamClients(artifactTimeout time.Duration) (metadataClient, artifactClient *http.Client) {
|
|
safeClient := newUpstreamClient(s.cfg.Upstream)
|
|
baseTransport := safeClient.Transport
|
|
if s.accessLog != nil {
|
|
baseTransport = upstreamhttp.NewAccessLogTransport(baseTransport, s.accessLog, s.logger)
|
|
}
|
|
// The gate sits under the auth transport so it sees every final upstream
|
|
// answer, including the retry after a bearer challenge.
|
|
gate := upstreamhttp.NewRateLimitGate(baseTransport)
|
|
gate.Logger = s.logger
|
|
gate.OnHold = metrics.RecordUpstreamRateLimitHold
|
|
s.rateLimits = gate
|
|
authTransport := upstreamhttp.NewTransport(gate, upstreamhttp.AuthFunc(s.authForURL))
|
|
metadata := *safeClient
|
|
metadata.Timeout = s.cfg.ParseHTTPTimeout()
|
|
metadata.Transport = authTransport
|
|
artifact := metadata
|
|
artifact.Timeout = artifactTimeout
|
|
return &metadata, &artifact
|
|
}
|
|
|
|
// NewMirrorProxy builds the proxy for the mirror command with the same
|
|
// upstream clients, SSRF guard and upstream authentication as serve, and a
|
|
// container handler with the same OCI routes for mirroring images. Artifact
|
|
// downloads have no overall time limit; the caller cancels through ctx.
|
|
func NewMirrorProxy(cfg *config.Config, db *database.DB, store storage.Storage, logger *slog.Logger) (*handler.Proxy, *handler.ContainerHandler) {
|
|
s := &Server{cfg: cfg, db: db, storage: store, logger: logger, ecr: newECRTokens(logger)}
|
|
metadataClient, artifactClient := s.newUpstreamClients(0)
|
|
fetcher := handler.LimitArtifactSize(fetch.NewFetcher(fetch.WithHTTPClient(artifactClient)), cfg.ParseArtifactMaxSize())
|
|
proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), logger)
|
|
proxy.HTTPClient = metadataClient
|
|
proxy.RateLimits = s.rateLimits
|
|
return proxy, s.newContainerHandler(proxy)
|
|
}
|
|
|
|
// newContainerHandler builds the OCI handler with the configured default
|
|
// registry, named upstreams, mirrored registries and Homebrew routes.
|
|
func (s *Server) newContainerHandler(proxy *handler.Proxy) *handler.ContainerHandler {
|
|
containerHandler := handler.NewContainerHandlerWithRegistry(
|
|
proxy,
|
|
s.cfg.BaseURL,
|
|
s.cfg.Upstream.OCIDefault,
|
|
s.cfg.Upstream.OCI,
|
|
)
|
|
containerHandler.SetMirroredRegistries(s.cfg.Upstream.OCIMirrors)
|
|
handler.RegisterHomebrewArtifacts(containerHandler, s.cfg.Upstream.HomebrewArtifact)
|
|
return containerHandler
|
|
}
|
|
|
|
func configureScanning(proxy *handler.Proxy, cfg config.ScanningConfig, baseURL string, logger *slog.Logger) (*scanner.Group, error) {
|
|
scanGroup, err := scanner.NewGroup(cfg, logger)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
proxy.Scanners = scanGroup
|
|
proxy.ScanSigningKey = []byte(cfg.SigningKeyExpanded())
|
|
proxy.ScanFetchBaseURL = cmp.Or(cfg.FetchBaseURL, baseURL)
|
|
return scanGroup, nil
|
|
}
|
|
|
|
// newUpstreamClient builds the shared upstream client: a safehttp client whose
|
|
// transport keeps upstreamMaxIdleConnsPerHost idle connections per host and
|
|
// gives up after upstreamResponseHeaderTimeout when an upstream accepts a
|
|
// request but stalls before sending headers.
|
|
func newUpstreamClient(upstream config.UpstreamConfig) *http.Client {
|
|
client := safehttp.New(nil, upstreamSafeHTTPOptions(upstream))
|
|
if transport, ok := client.Transport.(*http.Transport); ok {
|
|
transport.MaxIdleConnsPerHost = upstreamMaxIdleConnsPerHost
|
|
transport.ResponseHeaderTimeout = upstreamResponseHeaderTimeout
|
|
}
|
|
return client
|
|
}
|
|
|
|
func upstreamSafeHTTPOptions(upstream config.UpstreamConfig) safehttp.Options {
|
|
return safehttp.Options{
|
|
AllowLoopback: upstream.AllowLoopback,
|
|
AllowPrivateHosts: upstream.AllowPrivateHosts,
|
|
}
|
|
}
|
|
|
|
// updateCacheStatsMetrics periodically updates cache statistics in Prometheus metrics.
|
|
func (s *Server) updateCacheStatsMetrics() {
|
|
ticker := time.NewTicker(1 * time.Minute)
|
|
defer ticker.Stop()
|
|
|
|
// Update once immediately
|
|
s.updateCacheStats()
|
|
|
|
for range ticker.C {
|
|
s.updateCacheStats()
|
|
}
|
|
}
|
|
|
|
func (s *Server) updateCacheStats() {
|
|
stats, err := s.db.GetCacheStats()
|
|
if err != nil {
|
|
s.logger.Warn("failed to get cache stats for metrics", "error", err)
|
|
return
|
|
}
|
|
metrics.UpdateCacheStats(stats.TotalSize, stats.TotalArtifacts)
|
|
|
|
ecosystems, err := s.ecoStats.Refresh(s.db)
|
|
if err != nil {
|
|
s.logger.Warn("failed to get ecosystem stats for metrics", "error", err)
|
|
return
|
|
}
|
|
metrics.UpdateEcosystemStats(ecosystemMetrics(ecosystems))
|
|
}
|
|
|
|
// ecosystemMetrics converts database rows into the metrics package's own
|
|
// snapshot type, so that package keeps no dependency on the database schema.
|
|
func ecosystemMetrics(stats []database.EcosystemStats) []metrics.EcosystemStats {
|
|
out := make([]metrics.EcosystemStats, 0, len(stats))
|
|
for _, e := range stats {
|
|
out = append(out, metrics.EcosystemStats{
|
|
Ecosystem: e.Ecosystem,
|
|
Packages: e.Packages,
|
|
Versions: e.Versions,
|
|
Artifacts: e.Artifacts,
|
|
CacheSize: e.CacheSize,
|
|
Downloads: e.Downloads,
|
|
DownloadedBytes: e.DownloadedBytes,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Shutdown gracefully shuts down the server.
|
|
func (s *Server) Shutdown(ctx context.Context) error {
|
|
s.logger.Info("shutting down server")
|
|
|
|
if s.cancel != nil {
|
|
s.cancel()
|
|
}
|
|
|
|
var errs []error
|
|
|
|
if s.http != nil {
|
|
if err := s.http.Shutdown(ctx); err != nil {
|
|
errs = append(errs, fmt.Errorf("http shutdown: %w", err))
|
|
}
|
|
}
|
|
|
|
if s.storage != nil {
|
|
if err := s.storage.Close(); err != nil {
|
|
errs = append(errs, fmt.Errorf("storage close: %w", err))
|
|
}
|
|
}
|
|
|
|
if s.accessLog != nil {
|
|
if err := s.accessLog.Close(); err != nil {
|
|
errs = append(errs, fmt.Errorf("access log close: %w", err))
|
|
}
|
|
}
|
|
|
|
if s.db != nil {
|
|
if err := s.db.Close(); err != nil {
|
|
errs = append(errs, fmt.Errorf("database close: %w", err))
|
|
}
|
|
}
|
|
|
|
if len(errs) > 0 {
|
|
return errs[0]
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// authForURL returns the authentication header for a given URL based on config.
|
|
func (s *Server) authForURL(url string) (headerName, headerValue string) {
|
|
auth := s.cfg.Upstream.AuthForURL(url)
|
|
if auth == nil {
|
|
return "", ""
|
|
}
|
|
if strings.EqualFold(auth.Type, "ecr") {
|
|
region := auth.Region
|
|
if region == "" {
|
|
region = ecrRegion(url)
|
|
}
|
|
return s.ecr.header(region)
|
|
}
|
|
return auth.Header()
|
|
}
|
|
|
|
func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) {
|
|
// Get cache statistics
|
|
stats, err := s.db.GetCacheStats()
|
|
if err != nil {
|
|
s.logger.Error("failed to get cache stats", "error", err)
|
|
stats = &database.CacheStats{}
|
|
}
|
|
|
|
// Get enrichment statistics
|
|
enrichStats, err := s.db.GetEnrichmentStats()
|
|
if err != nil {
|
|
s.logger.Error("failed to get enrichment stats", "error", err)
|
|
enrichStats = &database.EnrichmentStats{}
|
|
}
|
|
|
|
// Get popular packages
|
|
popular, err := s.db.GetMostPopularPackages(dashboardTopN)
|
|
if err != nil {
|
|
s.logger.Error("failed to get popular packages", "error", err)
|
|
}
|
|
|
|
// Get recent packages
|
|
recent, err := s.db.GetRecentlyCachedPackages(dashboardTopN)
|
|
if err != nil {
|
|
s.logger.Error("failed to get recent packages", "error", err)
|
|
}
|
|
|
|
// Build dashboard data
|
|
data := DashboardData{
|
|
Layout: s.layoutFor(r),
|
|
Stats: DashboardStats{
|
|
CachedArtifacts: stats.TotalArtifacts,
|
|
TotalSize: formatSize(stats.TotalSize),
|
|
TotalPackages: stats.TotalPackages,
|
|
TotalVersions: stats.TotalVersions,
|
|
},
|
|
EnrichmentStats: enrichmentStatsView(enrichStats),
|
|
}
|
|
|
|
for _, p := range popular {
|
|
pkgInfo := PackageInfo{
|
|
Ecosystem: p.Ecosystem,
|
|
Name: p.Name,
|
|
Hits: p.Hits,
|
|
Size: formatSize(p.Size),
|
|
}
|
|
|
|
// Fetch enrichment data for this package
|
|
if pkg, err := s.db.GetPackageByEcosystemName(p.Ecosystem, p.Name); err == nil && pkg != nil {
|
|
if pkg.License.Valid {
|
|
pkgInfo.License = pkg.License.String
|
|
pkgInfo.LicenseCategory = categorizeLicenseCSS(pkg.License.String)
|
|
}
|
|
if pkg.LatestVersion.Valid {
|
|
pkgInfo.LatestVersion = pkg.LatestVersion.String
|
|
}
|
|
}
|
|
|
|
// Get vulnerability count
|
|
if vulnCount, err := s.db.GetVulnCountForPackage(p.Ecosystem, p.Name); err == nil {
|
|
pkgInfo.VulnCount = vulnCount
|
|
}
|
|
|
|
data.PopularPackages = append(data.PopularPackages, pkgInfo)
|
|
}
|
|
|
|
for _, p := range recent {
|
|
pkgInfo := PackageInfo{
|
|
Ecosystem: p.Ecosystem,
|
|
Name: p.Name,
|
|
Version: p.Version,
|
|
Size: formatSize(p.Size),
|
|
CachedAt: formatTimeAgo(p.CachedAt),
|
|
}
|
|
|
|
// Fetch enrichment data for this package
|
|
if pkg, err := s.db.GetPackageByEcosystemName(p.Ecosystem, p.Name); err == nil && pkg != nil {
|
|
if pkg.License.Valid {
|
|
pkgInfo.License = pkg.License.String
|
|
pkgInfo.LicenseCategory = categorizeLicenseCSS(pkg.License.String)
|
|
}
|
|
if pkg.LatestVersion.Valid {
|
|
pkgInfo.LatestVersion = pkg.LatestVersion.String
|
|
pkgInfo.IsOutdated = p.Version != "" && pkg.LatestVersion.String != "" && p.Version != pkg.LatestVersion.String
|
|
}
|
|
}
|
|
|
|
// Get vulnerability count
|
|
if vulnCount, err := s.db.GetVulnCountForPackage(p.Ecosystem, p.Name); err == nil {
|
|
pkgInfo.VulnCount = vulnCount
|
|
}
|
|
|
|
data.RecentPackages = append(data.RecentPackages, pkgInfo)
|
|
}
|
|
|
|
if err := s.templates.Render(w, "dashboard", data); err != nil {
|
|
s.logger.Error("failed to render dashboard", "error", err)
|
|
}
|
|
}
|
|
|
|
func (s *Server) handleOpenAPIJSON(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
_, _ = w.Write([]byte(swaggerdoc.SwaggerInfo.ReadDoc()))
|
|
}
|
|
|
|
func (s *Server) handleInstall(w http.ResponseWriter, r *http.Request) {
|
|
data := struct {
|
|
Layout
|
|
BaseURL string
|
|
Registries []RegistryConfig
|
|
}{
|
|
Layout: s.layoutFor(r),
|
|
BaseURL: s.cfg.BaseURL,
|
|
Registries: getRegistryConfigs(s.cfg.BaseURL),
|
|
}
|
|
|
|
if err := s.templates.Render(w, "install", data); err != nil {
|
|
s.logger.Error("failed to render install page", "error", err)
|
|
}
|
|
}
|
|
|
|
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
|
query := r.URL.Query().Get("q")
|
|
ecosystem := r.URL.Query().Get("ecosystem")
|
|
|
|
if query == "" {
|
|
http.Redirect(w, r, "/ui/", http.StatusSeeOther)
|
|
return
|
|
}
|
|
|
|
page := 1
|
|
if pageStr := r.URL.Query().Get("page"); pageStr != "" {
|
|
if p, err := strconv.Atoi(pageStr); err == nil && p > 0 {
|
|
page = p
|
|
}
|
|
}
|
|
limit := 50
|
|
|
|
results, err := s.db.SearchPackages(query, ecosystem, limit, (page-1)*limit)
|
|
if err != nil {
|
|
s.logger.Error("search failed", "error", err)
|
|
http.Error(w, "search failed", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
total, err := s.db.CountSearchResults(query, ecosystem)
|
|
if err != nil {
|
|
s.logger.Error("failed to count search results", "error", err)
|
|
total = 0
|
|
}
|
|
|
|
items := make([]SearchResultItem, len(results))
|
|
for i, result := range results {
|
|
latestVersion := ""
|
|
if result.LatestVersion.Valid {
|
|
latestVersion = result.LatestVersion.String
|
|
}
|
|
license := ""
|
|
if result.License.Valid {
|
|
license = result.License.String
|
|
}
|
|
items[i] = SearchResultItem{
|
|
Ecosystem: result.Ecosystem,
|
|
Name: result.Name,
|
|
LatestVersion: latestVersion,
|
|
License: license,
|
|
Hits: result.Hits,
|
|
Size: result.Size,
|
|
SizeFormatted: formatSize(result.Size),
|
|
}
|
|
}
|
|
|
|
totalPages := int((total + int64(limit) - 1) / int64(limit))
|
|
|
|
data := SearchPageData{
|
|
Layout: s.layoutFor(r),
|
|
Query: query,
|
|
Ecosystem: ecosystem,
|
|
Results: items,
|
|
Count: int(total),
|
|
Page: page,
|
|
PerPage: limit,
|
|
TotalPages: totalPages,
|
|
}
|
|
|
|
if err := s.templates.Render(w, "search", data); err != nil {
|
|
s.logger.Error("failed to render search page", "error", err)
|
|
}
|
|
}
|
|
|
|
func (s *Server) handlePackagesList(w http.ResponseWriter, r *http.Request) {
|
|
ecosystem := r.URL.Query().Get("ecosystem")
|
|
sortBy := r.URL.Query().Get("sort")
|
|
if sortBy == "" {
|
|
sortBy = defaultSortBy
|
|
}
|
|
|
|
page := 1
|
|
if pageStr := r.URL.Query().Get("page"); pageStr != "" {
|
|
if p, err := strconv.Atoi(pageStr); err == nil && p > 0 {
|
|
page = p
|
|
}
|
|
}
|
|
limit := 50
|
|
|
|
packages, err := s.db.ListCachedPackages(ecosystem, sortBy, limit, (page-1)*limit)
|
|
if err != nil {
|
|
s.logger.Error("failed to list packages", "error", err)
|
|
http.Error(w, "failed to list packages", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
|
|
counts, err := s.db.CountCachedPackagesByEcosystem()
|
|
if err != nil {
|
|
s.logger.Error("failed to count packages", "error", err)
|
|
http.Error(w, "failed to count packages", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
var totalPackages int64
|
|
for _, count := range counts {
|
|
totalPackages += count
|
|
}
|
|
total := totalPackages
|
|
if ecosystem != "" {
|
|
total = counts[ecosystem]
|
|
}
|
|
|
|
items := make([]SearchResultItem, len(packages))
|
|
for i, pkg := range packages {
|
|
latestVersion := ""
|
|
if pkg.LatestVersion.Valid {
|
|
latestVersion = pkg.LatestVersion.String
|
|
}
|
|
license := ""
|
|
if pkg.License.Valid {
|
|
license = pkg.License.String
|
|
}
|
|
cachedAt := ""
|
|
if pkg.CachedAt.Valid && pkg.CachedAt.String != "" {
|
|
if t, err := time.Parse("2006-01-02 15:04:05.999999999-07:00", pkg.CachedAt.String); err == nil {
|
|
cachedAt = formatTimeAgo(t)
|
|
}
|
|
}
|
|
items[i] = SearchResultItem{
|
|
Ecosystem: pkg.Ecosystem,
|
|
Name: pkg.Name,
|
|
LatestVersion: latestVersion,
|
|
License: license,
|
|
LicenseCategory: categorizeLicenseCSS(license),
|
|
Hits: pkg.Hits,
|
|
Size: pkg.Size,
|
|
SizeFormatted: formatSize(pkg.Size),
|
|
CachedAt: cachedAt,
|
|
VulnCount: pkg.VulnCount,
|
|
}
|
|
}
|
|
|
|
totalPages := int((total + int64(limit) - 1) / int64(limit))
|
|
|
|
data := PackagesListPageData{
|
|
Layout: s.layoutFor(r),
|
|
Ecosystem: ecosystem,
|
|
SortBy: sortBy,
|
|
Results: items,
|
|
Count: int(total),
|
|
TotalPackages: totalPackages,
|
|
EcosystemFilters: buildEcosystemFilters(counts),
|
|
Page: page,
|
|
PerPage: limit,
|
|
TotalPages: totalPages,
|
|
}
|
|
|
|
if err := s.templates.Render(w, "packages_list", data); err != nil {
|
|
s.logger.Error("failed to render packages list page", "error", err)
|
|
}
|
|
}
|
|
|
|
// handlePackagePath dispatches wildcard package routes to the appropriate handler.
|
|
// It resolves namespaced package names (e.g., Composer vendor/name) by consulting
|
|
// the database to determine which path segments are part of the package name.
|
|
//
|
|
// Supported paths:
|
|
//
|
|
// {name} -> package show
|
|
// {name}/{version} -> version show
|
|
// {name}/{version}/browse -> browse source
|
|
// {name}/compare/{v1}...{v2} -> compare versions
|
|
func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) {
|
|
ecosystem := chi.URLParam(r, "ecosystem")
|
|
segments, err := packagePathSegments(r)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
if ecosystem == "" || len(segments) == 0 {
|
|
http.Error(w, "ecosystem and package name required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
// Check for compare route: {name}/compare/{versions}
|
|
for i, seg := range segments {
|
|
if seg == "compare" && i > 0 && i < len(segments)-1 {
|
|
name := strings.Join(segments[:i], "/")
|
|
versions := strings.Join(segments[i+1:], "/")
|
|
s.showComparePage(w, r, ecosystem, name, versions)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Check for browse suffix
|
|
browse := false
|
|
if len(segments) > 1 && segments[len(segments)-1] == "browse" {
|
|
browse = true
|
|
segments = segments[:len(segments)-1]
|
|
}
|
|
|
|
// Resolve package name from the remaining segments using DB lookup.
|
|
name, rest := resolvePackageName(s.db, ecosystem, segments)
|
|
|
|
if name == "" {
|
|
// No package found in DB. Fall back to heuristic: assume the last
|
|
// segment is a version (if present) and everything else is the name.
|
|
if len(segments) == 1 {
|
|
// Single segment, no DB match: try package show (will 404).
|
|
s.showPackage(w, r, ecosystem, segments[0])
|
|
return
|
|
}
|
|
name = strings.Join(segments[:len(segments)-1], "/")
|
|
rest = segments[len(segments)-1:]
|
|
}
|
|
|
|
switch {
|
|
case len(rest) == 0 && !browse:
|
|
s.showPackage(w, r, ecosystem, name)
|
|
case len(rest) == 1 && browse:
|
|
s.showBrowseSource(w, r, ecosystem, name, rest[0])
|
|
case len(rest) == 1:
|
|
s.showVersion(w, r, ecosystem, name, rest[0])
|
|
default:
|
|
http.Error(w, "not found", http.StatusNotFound)
|
|
}
|
|
}
|
|
|
|
func (s *Server) showPackage(w http.ResponseWriter, r *http.Request, ecosystem, name string) {
|
|
pkg, err := s.db.GetPackageByEcosystemName(ecosystem, name)
|
|
if err != nil {
|
|
s.logger.Error("failed to get package", "error", err, "ecosystem", ecosystem, "name", name)
|
|
http.Error(w, "package not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
if pkg == nil {
|
|
http.Error(w, "package not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
versions, err := s.db.GetVersionsByPackagePURL(pkg.PURL)
|
|
if err != nil {
|
|
s.logger.Error("failed to get versions", "error", err)
|
|
versions = []database.Version{}
|
|
}
|
|
|
|
vulns, err := s.db.GetVulnerabilitiesForPackage(ecosystem, name)
|
|
if err != nil {
|
|
s.logger.Error("failed to get vulnerabilities", "error", err)
|
|
vulns = []database.Vulnerability{}
|
|
}
|
|
|
|
data := PackageShowData{
|
|
Layout: s.layoutFor(r),
|
|
Package: pkg,
|
|
Versions: versions,
|
|
Vulnerabilities: vulns,
|
|
LicenseCategory: categorizeLicense(pkg.License),
|
|
}
|
|
|
|
if err := s.templates.Render(w, "package_show", data); err != nil {
|
|
s.logger.Error("failed to render package show", "error", err)
|
|
}
|
|
}
|
|
|
|
func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) {
|
|
pkg, err := s.db.GetPackageByEcosystemName(ecosystem, name)
|
|
if err != nil || pkg == nil {
|
|
s.logger.Error("failed to get package", "error", err)
|
|
http.Error(w, "package not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
versionPURL := packageurl.WithVersionString(pkg.PURL, version)
|
|
ver, err := s.db.GetVersionByPURL(versionPURL)
|
|
if err != nil || ver == nil {
|
|
s.logger.Error("failed to get version", "error", err)
|
|
http.Error(w, "version not found", http.StatusNotFound)
|
|
return
|
|
}
|
|
|
|
artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL)
|
|
if err != nil {
|
|
s.logger.Error("failed to get artifacts", "error", err)
|
|
artifacts = []database.Artifact{}
|
|
}
|
|
|
|
vulns, err := s.db.GetVulnerabilitiesForPackage(ecosystem, name)
|
|
if err != nil {
|
|
s.logger.Error("failed to get vulnerabilities", "error", err)
|
|
vulns = []database.Vulnerability{}
|
|
}
|
|
|
|
isOutdated := pkg.LatestVersion.Valid && pkg.LatestVersion.String != version
|
|
|
|
// A version whose only cached artifact is a metadata sidecar cannot be
|
|
// browsed, so it must not be advertised as cached.
|
|
hasCached := firstBrowsableArtifact(artifacts) != nil
|
|
|
|
data := VersionShowData{
|
|
Layout: s.layoutFor(r),
|
|
Package: pkg,
|
|
Version: ver,
|
|
Artifacts: artifacts,
|
|
Vulnerabilities: vulns,
|
|
IsOutdated: isOutdated,
|
|
LicenseCategory: categorizeLicense(ver.License),
|
|
HasCachedArtifact: hasCached,
|
|
}
|
|
|
|
if err := s.templates.Render(w, "version_show", data); err != nil {
|
|
s.logger.Error("failed to render version show", "error", err)
|
|
}
|
|
}
|
|
|
|
func (s *Server) cachedVersionPURL(ecosystem, name, version string) string {
|
|
pkg, err := s.db.GetPackageByEcosystemName(ecosystem, name)
|
|
if err != nil || pkg == nil {
|
|
return ""
|
|
}
|
|
return packageurl.WithVersionString(pkg.PURL, version)
|
|
}
|
|
|
|
func (s *Server) showBrowseSource(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) {
|
|
data := BrowseSourceData{
|
|
Layout: s.layoutFor(r),
|
|
Ecosystem: ecosystem,
|
|
PackageName: name,
|
|
Version: version,
|
|
EscapedVersion: url.PathEscape(version),
|
|
}
|
|
|
|
if err := s.templates.Render(w, "browse_source", data); err != nil {
|
|
s.logger.Error("failed to render browse source page", "error", err)
|
|
http.Error(w, "internal server error", http.StatusInternalServerError)
|
|
}
|
|
}
|
|
|
|
func (s *Server) showComparePage(w http.ResponseWriter, r *http.Request, ecosystem, name, versions string) {
|
|
const compareVersionParts = 2
|
|
parts := strings.Split(versions, "...")
|
|
if len(parts) != compareVersionParts {
|
|
http.Error(w, "invalid version format, use: version1...version2", http.StatusBadRequest)
|
|
return
|
|
}
|
|
|
|
data := ComparePageData{
|
|
Layout: s.layoutFor(r),
|
|
Ecosystem: ecosystem,
|
|
PackageName: name,
|
|
FromVersion: parts[0],
|
|
ToVersion: parts[1],
|
|
EscapedFromVersion: url.PathEscape(parts[0]),
|
|
EscapedToVersion: url.PathEscape(parts[1]),
|
|
}
|
|
|
|
if err := s.templates.Render(w, "compare_versions", data); err != nil {
|
|
s.logger.Error("failed to render compare page", "error", err)
|
|
http.Error(w, "internal server error", http.StatusInternalServerError)
|
|
}
|
|
}
|
|
|
|
// handleHealth responds with a structured JSON health report.
|
|
//
|
|
// @Summary Health check
|
|
// @Tags meta
|
|
// @Produce json
|
|
// @Success 200 {object} HealthResponse
|
|
// @Failure 503 {object} HealthResponse
|
|
// @Router /health [get]
|
|
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
|
|
resp := HealthResponse{
|
|
Status: "ok",
|
|
Checks: map[string]HealthCheck{},
|
|
CircuitBreakers: s.breakers.snapshot(),
|
|
UpstreamHolds: formatHolds(s.rateLimits.Holds()),
|
|
}
|
|
|
|
// Database check (short-circuit; do not waste a storage probe call when DB is down).
|
|
// On DB failure the storage entry reports "skipped" rather than being omitted so
|
|
// the response always carries the same key set for monitors that expect it.
|
|
if _, err := s.db.SchemaVersion(); err != nil {
|
|
resp.Status = "error"
|
|
resp.Checks["database"] = HealthCheck{Status: "error", Error: err.Error()}
|
|
resp.Checks["storage"] = HealthCheck{Status: "skipped"}
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
_ = json.NewEncoder(w).Encode(resp)
|
|
return
|
|
}
|
|
resp.Checks["database"] = HealthCheck{Status: "ok"}
|
|
|
|
// Storage probe (via cache).
|
|
if err := s.healthCache.Check(); err != nil {
|
|
resp.Status = "error"
|
|
sc := HealthCheck{Status: "error", Error: err.Error()}
|
|
var pe *probeError
|
|
if errors.As(err, &pe) {
|
|
sc.Step = pe.step
|
|
}
|
|
resp.Checks["storage"] = sc
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
_ = json.NewEncoder(w).Encode(resp)
|
|
return
|
|
}
|
|
resp.Checks["storage"] = HealthCheck{Status: "ok"}
|
|
|
|
w.WriteHeader(http.StatusOK)
|
|
_ = json.NewEncoder(w).Encode(resp)
|
|
}
|
|
|
|
// StatsResponse contains cache statistics.
|
|
type StatsResponse struct {
|
|
CachedArtifacts int64 `json:"cached_artifacts"`
|
|
TotalSize int64 `json:"total_size_bytes"`
|
|
TotalSizeHuman string `json:"total_size"`
|
|
StorageURL string `json:"storage_url"`
|
|
DatabasePath string `json:"database_path"`
|
|
// DownloadedBytes is the accumulated download volume across every
|
|
// ecosystem: cache hits multiplied by the artifact size they served.
|
|
DownloadedBytes int64 `json:"downloaded_bytes"`
|
|
DownloadedBytesHuman string `json:"downloaded"`
|
|
Downloads int64 `json:"downloads"`
|
|
Ecosystems []EcosystemStatsEntry `json:"ecosystems"`
|
|
// StatsUnavailable distinguishes a proxy that has served nothing from one
|
|
// whose aggregation failed with no snapshot to fall back on. Without it
|
|
// both report zeros and an empty array.
|
|
StatsUnavailable bool `json:"stats_unavailable,omitempty"`
|
|
}
|
|
|
|
// EcosystemStatsEntry is one ecosystem's slice of the cache statistics.
|
|
type EcosystemStatsEntry struct {
|
|
Ecosystem string `json:"ecosystem"`
|
|
DownloadedBytes int64 `json:"downloaded_bytes"`
|
|
Downloaded string `json:"downloaded"`
|
|
Downloads int64 `json:"downloads"`
|
|
CacheSize int64 `json:"cache_size_bytes"`
|
|
Artifacts int64 `json:"cached_artifacts"`
|
|
Packages int64 `json:"packages"`
|
|
Versions int64 `json:"versions"`
|
|
}
|
|
|
|
// handleStats returns cache statistics.
|
|
// @Summary Cache statistics
|
|
// @Tags meta
|
|
// @Produce json
|
|
// @Success 200 {object} StatsResponse
|
|
// @Failure 500 {object} ErrorResponse
|
|
// @Router /stats [get]
|
|
func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
|
|
ctx := r.Context()
|
|
|
|
count, err := s.db.GetCachedArtifactCount()
|
|
if err != nil {
|
|
internalError(w, "failed to get artifact count")
|
|
return
|
|
}
|
|
|
|
size, err := s.db.GetTotalCacheSize()
|
|
if err != nil {
|
|
internalError(w, "failed to get cache size")
|
|
return
|
|
}
|
|
|
|
// A failing per-ecosystem aggregation must not take down an endpoint that
|
|
// answered from two cheap counters before it existed. Get returns the last
|
|
// good snapshot alongside the error, so the breakdown is served stale when
|
|
// there is one, and flagged unavailable when there is not.
|
|
ecosystems, statsErr := s.ecoStats.Get(s.db)
|
|
if statsErr != nil {
|
|
s.logger.Error("failed to get ecosystem stats for /stats", "error", statsErr)
|
|
}
|
|
|
|
_ = ctx // Could use for storage.UsedSpace if needed
|
|
|
|
stats := StatsResponse{
|
|
CachedArtifacts: count,
|
|
TotalSize: size,
|
|
TotalSizeHuman: formatSize(size),
|
|
Ecosystems: make([]EcosystemStatsEntry, 0, len(ecosystems)),
|
|
StatsUnavailable: statsErr != nil && len(ecosystems) == 0,
|
|
StorageURL: s.storage.URL(),
|
|
DatabasePath: s.cfg.Database.String(),
|
|
}
|
|
|
|
for _, e := range ecosystems {
|
|
stats.DownloadedBytes += e.DownloadedBytes
|
|
stats.Downloads += e.Downloads
|
|
stats.Ecosystems = append(stats.Ecosystems, EcosystemStatsEntry{
|
|
Ecosystem: e.Ecosystem,
|
|
DownloadedBytes: e.DownloadedBytes,
|
|
Downloaded: formatSize(e.DownloadedBytes),
|
|
Downloads: e.Downloads,
|
|
CacheSize: e.CacheSize,
|
|
Artifacts: e.Artifacts,
|
|
Packages: e.Packages,
|
|
Versions: e.Versions,
|
|
})
|
|
}
|
|
stats.DownloadedBytesHuman = formatSize(stats.DownloadedBytes)
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(stats)
|
|
}
|
|
|
|
func formatSize(bytes int64) string {
|
|
const unit = 1024
|
|
if bytes < unit {
|
|
return fmt.Sprintf("%d B", bytes)
|
|
}
|
|
div, exp := int64(unit), 0
|
|
for n := bytes / unit; n >= unit; n /= unit {
|
|
div *= unit
|
|
exp++
|
|
}
|
|
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
|
}
|
|
|
|
func formatTimeAgo(t time.Time) string {
|
|
if t.IsZero() {
|
|
return ""
|
|
}
|
|
d := time.Since(t)
|
|
switch {
|
|
case d < time.Minute:
|
|
return "just now"
|
|
case d < time.Hour:
|
|
m := int(d.Minutes())
|
|
if m == 1 {
|
|
return "1 min ago"
|
|
}
|
|
return fmt.Sprintf("%d mins ago", m)
|
|
case d < hoursPerDay*time.Hour:
|
|
h := int(d.Hours())
|
|
if h == 1 {
|
|
return "1 hour ago"
|
|
}
|
|
return fmt.Sprintf("%d hours ago", h)
|
|
case d < 7*hoursPerDay*time.Hour:
|
|
days := int(d.Hours() / hoursPerDay)
|
|
if days == 1 {
|
|
return "1 day ago"
|
|
}
|
|
return fmt.Sprintf("%d days ago", days)
|
|
default:
|
|
return t.Format("Jan 2")
|
|
}
|
|
}
|
|
|
|
// categorizeLicenseCSS returns the CSS class suffix for a license category using the spdx module.
|
|
func categorizeLicenseCSS(license string) string {
|
|
if license == "" {
|
|
return licenseCategoryUnknown
|
|
}
|
|
|
|
if spdx.HasCopyleft(license) {
|
|
return "copyleft"
|
|
}
|
|
|
|
if spdx.IsFullyPermissive(license) {
|
|
return "permissive"
|
|
}
|
|
|
|
return licenseCategoryUnknown
|
|
}
|
|
|
|
// categorizeLicense is a helper that handles sql.NullString.
|
|
func categorizeLicense(license sql.NullString) string {
|
|
if !license.Valid {
|
|
return licenseCategoryUnknown
|
|
}
|
|
return categorizeLicenseCSS(license.String)
|
|
}
|
|
|
|
// responseWriter wraps http.ResponseWriter to capture the status code and the
|
|
// number of body bytes written, which is what a client actually downloaded.
|
|
type responseWriter struct {
|
|
http.ResponseWriter
|
|
status int
|
|
bytes int64
|
|
}
|
|
|
|
func (rw *responseWriter) Write(b []byte) (int, error) {
|
|
n, err := rw.ResponseWriter.Write(b)
|
|
rw.bytes += int64(n)
|
|
return n, err
|
|
}
|
|
|
|
// Unwrap lets ResponseController reach capabilities such as flushing when a
|
|
// relayed response discovers trailers only after its body has been copied.
|
|
func (rw *responseWriter) Unwrap() http.ResponseWriter {
|
|
return rw.ResponseWriter
|
|
}
|
|
|
|
func (rw *responseWriter) WriteHeader(code int) {
|
|
rw.status = code
|
|
rw.ResponseWriter.WriteHeader(code)
|
|
}
|