WIP

config.example.yaml

@@ -49,6 +49,145 @@ log:
   # Log format: "text" or "json"
   format: "text"
 
+# Ecosystem support - routes and upstream repositories
+#
+# This section is optional, since 'include_default' in each section
+# defaults to 'true' and the route map will be populated with all of
+# the default routes if no configuration is provided.
+ecosystem:
+  cargo:
+    include_default: true
+    # the default route for crates.io
+    # route:
+    #   - path: /cargo
+    #     upstream:
+    #       - name: crates.io
+    #         index: https://index.crates.io
+    #         crates: https://static.crates.io/crates
+  composer:
+    include_default: true
+    # the default route for packagist.org
+    # route:
+    #   - path: /composer
+    #     upstream:
+    #       - name: packagist.org
+    #         upstream: https://packagist.org
+    #         repository: https://repo.packagist.org
+  conan:
+    include_default: true
+    # the default route for conan.io
+    # route:
+    #   - path: /conan
+    #     upstream:
+    #       - name: conan.io
+    #         upstream: https://center.conan.io
+  conda:
+    include_default: true
+    # the default route for anaconda.org
+    # route:
+    #   - path: /conda
+    #     upstream:
+    #       - name: anaconda.org
+    #         upstream: https://conda.anaconda.org
+  cran:
+    include_default: true
+    # the default route for r-project.org
+    # route:
+    #   - path: /cran
+    #     upstream:
+    #       - name: r-project.org
+    #         upstream: https://cloud.r-project.org
+  debian:
+    include_default: true
+    # the default route for debian.org
+    # route:
+    #   - path: /debian
+    #     upstream:
+    #       - name: debian.org
+    #         upstream: http://deb.debian.org/debian
+  gem:
+    include_default: true
+    # the default route for rubygems.org
+    # route:
+    #   - path: /gem
+    #     upstream:
+    #       - name: rubygems.org
+    #         upstream: https://rubygems.org
+  go:
+    include_default: true
+    # the default route for golang.org
+    # route:
+    #   - path: /go
+    #     upstream:
+    #       - name: golang.org
+    #         upstream: https://proxy.golang.org
+  hex:
+    include_default: true
+    # the default route for hex.pm
+    # route:
+    #   - path: /hex
+    #     upstream:
+    #       - name: hex.pm
+    #         upstream: https://repo.hex.pm
+  maven:
+    include_default: true
+    # the default route for maven.org
+    # route:
+    #   - path: /maven
+    #     upstream:
+    #       - name: maven.org
+    #         upstream: https://repo1.maven.org/maven2
+  npm:
+    include_default: true
+    # the default route for npmjs.org
+    # route:
+    #   - path: /npm
+    #     upstream:
+    #       - name: npmjs.org
+    #         upstream: https://registry.npmjs.org
+  nuget:
+    include_default: true
+    # the default route for nuget.org
+    # route:
+    #   - path: /nuget
+    #     upstream:
+    #       - name: nuget.org
+    #         upstream: https://api.nuget.org
+  oci:
+    include_default: true
+    # the default route for docker.io
+    # route:
+    #   - path: /v2
+    #     upstream:
+    #       - name: docker.io
+    #         registry: https://registry-1.docker.io
+    #         auth: https://auth.docker.io
+  pub:
+    include_default: true
+    # the default route for pub.dev
+    # route:
+    #   - path: /pub
+    #     upstream:
+    #       - name: pub.dev
+    #         upstream: https://pub.dev
+  pypi:
+    include_default: true
+    # the default route for pypi.org
+    # route:
+    #   - path: /pypi
+    #     upstream:
+    #       - name: pypi.org
+    #         index: https://pypi.org
+    #         files_host: files.pythonhosted.org
+  rpm:
+    include_default: true
+    # the default route for fedoraproject.org
+    # route:
+    #   - path: /rpm
+    #     upstream:
+    #       - name: fedoraproject.org
+    #         upstream: https://dl.fedoraproject.org/pub/fedora/linux
+
 # Upstream registry URLs and authentication
 upstream:
   # npm registry URL

internal/config/cargo/cargo.go

@@ -0,0 +1,78 @@
+package cargo
+
+import (
+	"fmt"
+	"net/url"
+)
+
+// Config configures routes
+type Config struct {
+	IncludeDefault bool          `json:"include_default" yaml:"include_default"`
+	Route          []RouteConfig `json:"route" yaml:"route"`
+}
+
+// RouteConfig configures a route
+type RouteConfig struct {
+	Path     string           `json:"path" yaml:"path"`
+	Upstream []UpstreamConfig `json:"upstream" yaml:"upstream"`
+}
+
+// UpstreamConfig configures an upstream (source)
+type UpstreamConfig struct {
+	Name   string `json:"name" yaml:"name"`
+	Index  string `json:"index" yaml:"index"`
+	Crates string `json:"crates" yaml:"crates"`
+}
+
+// RouteDefault is the default route
+var RouteDefault = RouteConfig{
+	Path: "/cargo",
+	Upstream: []UpstreamConfig{
+		{
+			Name:   "crates.io",
+			Index:  "https://index.crates.io",
+			Crates: "https://static.crates.io/crates",
+		},
+	},
+}
+
+func (c *Config) Validate() error {
+	for _, route := range c.Route {
+		if err := route.Validate(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *RouteConfig) Validate() error {
+	// TODO: validate Path
+
+	if len(r.Upstream) == 0 {
+		return fmt.Errorf("cargo route %q does not have any upstreams", r.Path)
+	}
+	if len(r.Upstream) > 1 {
+		return fmt.Errorf("cargo route %q has multiple upstreams; this is not yet supported", r.Path)
+	}
+
+	for _, upstream := range r.Upstream {
+		if err := upstream.Validate(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (u *UpstreamConfig) Validate() error {
+	if _, err := url.Parse(u.Index); err != nil {
+		return fmt.Errorf("cargo upstream index %q is not a valid URL", u.Index)
+	}
+
+	if _, err := url.Parse(u.Crates); err != nil {
+		return fmt.Errorf("cargo upstream crates %q is not a valid URL", u.Crates)
+	}
+
+	return nil
+}

internal/config/config.go

@@ -57,6 +57,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/git-pkgs/proxy/internal/config/cargo"
+	"github.com/git-pkgs/proxy/internal/config/debian"
 	"gopkg.in/yaml.v3"
 )
 
@@ -82,6 +84,9 @@ type Config struct {
 	// Upstream configures upstream registry URLs (optional overrides).
 	Upstream UpstreamConfig `json:"upstream" yaml:"upstream"`
 
+	// Ecosystem configures ecosystem routes and upstreams
+	Ecosystem EcosystemConfig `json:"ecosystem" yaml:"ecosystem"`
+
 	// Cooldown configures version age filtering to mitigate supply chain attacks.
 	Cooldown CooldownConfig `json:"cooldown" yaml:"cooldown"`
 
@@ -239,6 +244,14 @@ func Default() *Config {
 			Level:  "info",
 			Format: "text",
 		},
+		Ecosystem: EcosystemConfig{
+			Cargo: cargo.Config{
+				IncludeDefault: true,
+			},
+			Debian: debian.Config{
+				IncludeDefault: true,
+			},
+		},
 		Upstream: UpstreamConfig{
 			NPM:           "https://registry.npmjs.org",
 			Cargo:         "https://index.crates.io",
@@ -334,6 +347,14 @@ func (c *Config) LoadFromEnv() {
 
 // Validate checks the configuration for errors.
 func (c *Config) Validate() error {
+	// finalize the configuration by injecting default routes if requested
+	if c.Ecosystem.Cargo.IncludeDefault {
+		c.Ecosystem.Cargo.Route = append(c.Ecosystem.Cargo.Route, cargo.RouteDefault)
+	}
+	if c.Ecosystem.Debian.IncludeDefault {
+		c.Ecosystem.Debian.Route = append(c.Ecosystem.Debian.Route, debian.RouteDefault)
+	}
+
 	if c.Listen == "" {
 		return fmt.Errorf("listen address is required")
 	}
@@ -386,6 +407,13 @@ func (c *Config) Validate() error {
 		}
 	}
 
+	if err := c.Ecosystem.Cargo.Validate(); err != nil {
+		return err
+	}
+	if err := c.Ecosystem.Debian.Validate(); err != nil {
+		return err
+	}
+
 	return nil
 }
 

internal/config/debian/debian.go

@@ -0,0 +1,72 @@
+package debian
+
+import (
+	"fmt"
+	"net/url"
+)
+
+// Config configures routes
+type Config struct {
+	IncludeDefault bool          `json:"include_default" yaml:"include_default"`
+	Route          []RouteConfig `json:"route" yaml:"route"`
+}
+
+// RouteConfig configures a route
+type RouteConfig struct {
+	Path     string           `json:"path" yaml:"path"`
+	Upstream []UpstreamConfig `json:"upstream" yaml:"upstream"`
+}
+
+// UpstreamConfig configures an upstream (source)
+type UpstreamConfig struct {
+	Name     string `json:"name" yaml:"name"`
+	Upstream string `json:"upstream" yaml:"upstream"`
+}
+
+// RouteDefault is the default route
+var RouteDefault = RouteConfig{
+	Path: "/debian",
+	Upstream: []UpstreamConfig{
+		{
+			Name:     "debian.org",
+			Upstream: "http://deb.debian.org/debian",
+		},
+	},
+}
+
+func (c *Config) Validate() error {
+	for _, route := range c.Route {
+		if err := route.Validate(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *RouteConfig) Validate() error {
+	// TODO: validate Path
+
+	if len(r.Upstream) == 0 {
+		return fmt.Errorf("debian route %q does not have any upstreams", r.Path)
+	}
+	if len(r.Upstream) > 1 {
+		return fmt.Errorf("debian route %q has multiple upstreams; this is not yet supported", r.Path)
+	}
+
+	for _, upstream := range r.Upstream {
+		if err := upstream.Validate(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (u *UpstreamConfig) Validate() error {
+	if _, err := url.Parse(u.Upstream); err != nil {
+		return fmt.Errorf("debian upstream upstream %q is not a valid URL", u.Upstream)
+	}
+
+	return nil
+}

internal/config/ecosystem.go

@@ -0,0 +1,12 @@
+package config
+
+import (
+	"github.com/git-pkgs/proxy/internal/config/cargo"
+	"github.com/git-pkgs/proxy/internal/config/debian"
+)
+
+// Ecosystem configuration (routes and upstreams)
+type EcosystemConfig struct {
+	Cargo  cargo.Config  `json:"cargo" yaml:"cargo"`
+	Debian debian.Config `json:"debian" yaml:"debian"`
+}

internal/handler/cargo.go

@@ -9,13 +9,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/git-pkgs/proxy/internal/config/cargo"
 	"github.com/git-pkgs/purl"
 )
 
 const (
-	cargoUpstream     = "https://index.crates.io"
-	cargoDownloadBase = "https://static.crates.io/crates"
-
 	cargoIndexLen1 = 1
 	cargoIndexLen2 = 2
 	cargoIndexLen3 = 3
@@ -24,21 +22,27 @@ const (
 // CargoHandler handles cargo registry protocol requests.
 type CargoHandler struct {
 	proxy       *Proxy
+	path        string
 	indexURL    string
 	downloadURL string
 	proxyURL    string
 }
 
 // NewCargoHandler creates a new cargo protocol handler.
-func NewCargoHandler(proxy *Proxy, proxyURL string) *CargoHandler {
+func NewCargoHandler(proxy *Proxy, proxyURL string, cfg cargo.RouteConfig) *CargoHandler {
 	return &CargoHandler{
 		proxy:       proxy,
-		indexURL:    cargoUpstream,
-		downloadURL: cargoDownloadBase,
+		path:        cfg.Path,
+		indexURL:    cfg.Upstream[0].Index,
+		downloadURL: cfg.Upstream[0].Crates,
 		proxyURL:    strings.TrimSuffix(proxyURL, "/"),
 	}
 }
 
+func (h *CargoHandler) Path() string {
+	return h.path
+}
+
 // Routes returns the HTTP handler for cargo requests.
 // Mount this at /cargo on your router.
 func (h *CargoHandler) Routes() http.Handler {
@@ -71,7 +75,7 @@ type CargoConfig struct {
 // handleConfig returns the registry configuration.
 func (h *CargoHandler) handleConfig(w http.ResponseWriter, r *http.Request) {
 	config := CargoConfig{
-		DL: h.proxyURL + "/cargo/crates/{crate}/{version}/download",
+		DL: h.proxyURL + h.path + "/crates/{crate}/{version}/download",
 	}
 
 	w.Header().Set("Content-Type", "application/json")

internal/handler/cargo_test.go

@@ -48,6 +48,7 @@ func TestCargoBuildIndexPath(t *testing.T) {
 func TestCargoConfigEndpoint(t *testing.T) {
 	h := &CargoHandler{
 		proxyURL: "http://localhost:8080",
+		path:     "/xyzzy",
 	}
 
 	req := httptest.NewRequest(http.MethodGet, "/config.json", nil)
@@ -64,7 +65,7 @@ func TestCargoConfigEndpoint(t *testing.T) {
 		t.Fatalf("failed to parse config: %v", err)
 	}
 
-	expectedDL := "http://localhost:8080/cargo/crates/{crate}/{version}/download"
+	expectedDL := "http://localhost:8080/xyzzy/crates/{crate}/{version}/download"
 	if config.DL != expectedDL {
 		t.Errorf("DL = %q, want %q", config.DL, expectedDL)
 	}

internal/handler/debian.go

@@ -2,13 +2,13 @@ package handler
 
 import (
 	"fmt"
+	"github.com/git-pkgs/proxy/internal/config/debian"
 	"net/http"
 	"regexp"
 	"strings"
 )
 
 const (
-	debianUpstream = "http://deb.debian.org/debian"
 	debMatchCount = 4 // full match + name + version + arch
 )
 
@@ -16,19 +16,25 @@ const (
 // It proxies requests to upstream Debian/Ubuntu repositories and caches .deb packages.
 type DebianHandler struct {
 	proxy       *Proxy
+	path        string
 	upstreamURL string
 	proxyURL    string
 }
 
 // NewDebianHandler creates a new Debian/APT protocol handler.
-func NewDebianHandler(proxy *Proxy, proxyURL string) *DebianHandler {
+func NewDebianHandler(proxy *Proxy, proxyURL string, cfg debian.RouteConfig) *DebianHandler {
 	return &DebianHandler{
 		proxy:       proxy,
-		upstreamURL: debianUpstream,
+		path:        cfg.Path,
+		upstreamURL: cfg.Upstream[0].Upstream,
 		proxyURL:    strings.TrimSuffix(proxyURL, "/"),
 	}
 }
 
+func (h *DebianHandler) Path() string {
+	return h.path
+}
+
 // Routes returns the HTTP handler for Debian requests.
 // Mount this at /debian on your router.
 func (h *DebianHandler) Routes() http.Handler {

internal/handler/debian_test.go

@@ -2,6 +2,7 @@ package handler
 
 import (
 	"testing"
+	"github.com/git-pkgs/proxy/internal/config/debian"
 )
 
 func TestDebianHandler_parsePoolPath(t *testing.T) {
@@ -18,6 +19,6 @@ func TestDebianHandler_parsePoolPath(t *testing.T) {
 }
 
 func TestDebianHandler_Routes(t *testing.T) {
-	h := NewDebianHandler(nil, "http://localhost:8080")
+	h := NewDebianHandler(nil, "http://localhost:8080", debian.RouteDefault)
 	assertRoutesBasics(t, h.Routes(), "/dists/stable/Release", "/pool/../../../etc/passwd")
 }

internal/handler/download_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/git-pkgs/proxy/internal/config/debian"
 	"github.com/git-pkgs/proxy/internal/database"
 	"github.com/git-pkgs/proxy/internal/storage"
 	"github.com/git-pkgs/purl"
@@ -897,7 +898,7 @@ func TestDebianHandler_DownloadCacheMiss(t *testing.T) {
 		ContentType: "application/vnd.debian.binary-package",
 	}
 
-	h := NewDebianHandler(proxy, "http://localhost")
+	h := NewDebianHandler(proxy, "http://localhost", debian.RouteDefault)
 	srv := httptest.NewServer(h.Routes())
 	defer srv.Close()
 

internal/server/server.go

@@ -170,7 +170,6 @@ func (s *Server) Start() error {
 
 	// Mount protocol handlers
 	npmHandler := handler.NewNPMHandler(proxy, s.cfg.BaseURL)
-	cargoHandler := handler.NewCargoHandler(proxy, s.cfg.BaseURL)
 	gemHandler := handler.NewGemHandler(proxy, s.cfg.BaseURL)
 	goHandler := handler.NewGoHandler(proxy, s.cfg.BaseURL)
 	hexHandler := handler.NewHexHandler(proxy, s.cfg.BaseURL)
@@ -183,11 +182,21 @@ func (s *Server) Start() error {
 	condaHandler := handler.NewCondaHandler(proxy, s.cfg.BaseURL)
 	cranHandler := handler.NewCRANHandler(proxy, s.cfg.BaseURL)
 	containerHandler := handler.NewContainerHandler(proxy, s.cfg.BaseURL)
-	debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL)
 	rpmHandler := handler.NewRPMHandler(proxy, s.cfg.BaseURL)
 
+	for _, route := range s.cfg.Ecosystem.Cargo.Route {
+		routeHandler := handler.NewCargoHandler(proxy, s.cfg.BaseURL, route)
+		r.Mount(routeHandler.Path(), http.StripPrefix(routeHandler.Path(), routeHandler.Routes()))
+		s.logger.Info("mounted handler", "ecosystem", "cargo", "path", routeHandler.Path())
+	}
+
+	for _, route := range s.cfg.Ecosystem.Debian.Route {
+		routeHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL, route)
+		r.Mount(routeHandler.Path(), http.StripPrefix(routeHandler.Path(), routeHandler.Routes()))
+		s.logger.Info("mounted handler", "ecosystem", "debian", "path", routeHandler.Path())
+	}
+
 	r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes()))
-	r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes()))
 	r.Mount("/gem", http.StripPrefix("/gem", gemHandler.Routes()))
 	r.Mount("/go", http.StripPrefix("/go", goHandler.Routes()))
 	r.Mount("/hex", http.StripPrefix("/hex", hexHandler.Routes()))
@@ -200,7 +209,6 @@ func (s *Server) Start() error {
 	r.Mount("/conda", http.StripPrefix("/conda", condaHandler.Routes()))
 	r.Mount("/cran", http.StripPrefix("/cran", cranHandler.Routes()))
 	r.Mount("/v2", http.StripPrefix("/v2", containerHandler.Routes()))
-	r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes()))
 	r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes()))
 
 	// Health, stats, and static endpoints

internal/server/server_test.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/git-pkgs/proxy/internal/config"
+	"github.com/git-pkgs/proxy/internal/config/cargo"
 	"github.com/git-pkgs/proxy/internal/database"
 	"github.com/git-pkgs/proxy/internal/handler"
 	"github.com/git-pkgs/proxy/internal/storage"
@@ -68,13 +69,13 @@ func newTestServer(t *testing.T) *testServer {
 
 	// Mount handlers
 	npmHandler := handler.NewNPMHandler(proxy, cfg.BaseURL)
-	cargoHandler := handler.NewCargoHandler(proxy, cfg.BaseURL)
+	cargoHandler := handler.NewCargoHandler(proxy, cfg.BaseURL, cargo.RouteDefault)
 	gemHandler := handler.NewGemHandler(proxy, cfg.BaseURL)
 	goHandler := handler.NewGoHandler(proxy, cfg.BaseURL)
 	pypiHandler := handler.NewPyPIHandler(proxy, cfg.BaseURL)
 
 	r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes()))
-	r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes()))
+	r.Mount(cargoHandler.Path(), http.StripPrefix(cargoHandler.Path(), cargoHandler.Routes()))
 	r.Mount("/gem", http.StripPrefix("/gem", gemHandler.Routes()))
 	r.Mount("/go", http.StripPrefix("/go", goHandler.Routes()))
 	r.Mount("/pypi", http.StripPrefix("/pypi", pypiHandler.Routes()))