mirror of
https://github.com/git-pkgs/proxy.git
synced 2026-06-02 16:48:16 -04:00
Compare commits
26 commits
validate-p
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0e7af4aed6 |
||
|
|
65474c77e8 |
||
|
|
946d39f193 |
||
|
|
ee57878386 |
||
|
|
00b032cb5b |
||
|
|
552c8ac4e5 |
||
|
|
76f41cf271 |
||
|
|
a8b6b0a417 |
||
|
|
1e6a0d1ab8 |
||
|
|
d39e312710 |
||
|
|
7586beb37b |
||
|
|
7832db2adb |
||
|
|
497e16f90b |
||
|
|
e8363e2fec |
||
|
|
ffd28ad856 |
||
|
|
17a10bda55 |
||
|
|
f2a5b704f0 |
||
|
|
5315883c3b |
||
|
|
ebc2ea9cf9 |
||
|
|
992f5c68a7 |
||
|
|
31a9ca75b2 |
||
|
|
61741123bf |
||
|
|
8d2740624f |
||
|
|
e912227e3b |
||
|
|
522c6f233e |
||
|
|
a4fd333d48 |
68 changed files with 4243 additions and 552 deletions
2
.github/workflows/publish.yml
vendored
2
.github/workflows/publish.yml
vendored
|
|
@ -38,7 +38,7 @@ jobs:
|
||||||
images: ghcr.io/${{ github.repository }}
|
images: ghcr.io/${{ github.repository }}
|
||||||
|
|
||||||
- name: Build and push Docker image
|
- name: Build and push Docker image
|
||||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
|
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
push: true
|
push: true
|
||||||
|
|
|
||||||
4
.github/workflows/release.yml
vendored
4
.github/workflows/release.yml
vendored
|
|
@ -19,7 +19,7 @@ jobs:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
- uses: sigstore/cosign-installer@cad07c2e89fa2edd6e2d7bab4c1aa38e53f76003 # v4.1.1
|
- uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||||
|
|
||||||
- name: Set up Go
|
- name: Set up Go
|
||||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||||
|
|
@ -27,7 +27,7 @@ jobs:
|
||||||
go-version-file: go.mod
|
go-version-file: go.mod
|
||||||
cache: false
|
cache: false
|
||||||
|
|
||||||
- uses: goreleaser/goreleaser-action@e24998b8b67b290c2fa8b7c14fcfa7de2c5c9b8c # v7.1.0
|
- uses: goreleaser/goreleaser-action@5daf1e915a5f0af01ddbcd89a43b8061ff4f1a89 # v7.2.2
|
||||||
with:
|
with:
|
||||||
version: "~> v2"
|
version: "~> v2"
|
||||||
args: release --clean
|
args: release --clean
|
||||||
|
|
|
||||||
2
.github/workflows/zizmor.yml
vendored
2
.github/workflows/zizmor.yml
vendored
|
|
@ -26,4 +26,4 @@ jobs:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
|
|
||||||
- name: Run zizmor
|
- name: Run zizmor
|
||||||
uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3
|
uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
|
||||||
|
|
|
||||||
28
.golangci.yml
Normal file
28
.golangci.yml
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
version: "2"
|
||||||
|
|
||||||
|
linters:
|
||||||
|
enable:
|
||||||
|
- gocritic
|
||||||
|
- gocognit
|
||||||
|
- gocyclo
|
||||||
|
- maintidx
|
||||||
|
- dupl
|
||||||
|
- mnd
|
||||||
|
- unparam
|
||||||
|
- ireturn
|
||||||
|
- goconst
|
||||||
|
- errcheck
|
||||||
|
settings:
|
||||||
|
goconst:
|
||||||
|
min-len: 4
|
||||||
|
min-occurrences: 5
|
||||||
|
ignore-tests: true
|
||||||
|
ignore-string-values:
|
||||||
|
- "^[a-z]+$"
|
||||||
|
exclusions:
|
||||||
|
rules:
|
||||||
|
- path: _test\.go
|
||||||
|
linters:
|
||||||
|
- goconst
|
||||||
|
- dupl
|
||||||
|
- mnd
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
FROM golang:1.25-alpine AS builder
|
FROM golang:1.26.3-alpine AS builder
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
|
|
@ -15,7 +15,7 @@ COPY . .
|
||||||
# Build the binary
|
# Build the binary
|
||||||
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /proxy ./cmd/proxy
|
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /proxy ./cmd/proxy
|
||||||
|
|
||||||
FROM alpine:3.21
|
FROM alpine:3.23.4
|
||||||
|
|
||||||
RUN apk add --no-cache ca-certificates
|
RUN apk add --no-cache ca-certificates
|
||||||
|
|
||||||
|
|
|
||||||
69
README.md
69
README.md
|
|
@ -32,11 +32,13 @@ Resolution order: package override, then ecosystem override, then global default
|
||||||
| pub.dev | Dart | Yes | ✓ |
|
| pub.dev | Dart | Yes | ✓ |
|
||||||
| PyPI | Python | Yes | ✓ |
|
| PyPI | Python | Yes | ✓ |
|
||||||
| Maven | Java | | ✓ |
|
| Maven | Java | | ✓ |
|
||||||
|
| Gradle Build Cache | Java/Kotlin | | ✓ |
|
||||||
| NuGet | .NET | Yes | ✓ |
|
| NuGet | .NET | Yes | ✓ |
|
||||||
| Composer | PHP | Yes | ✓ |
|
| Composer | PHP | Yes | ✓ |
|
||||||
| Conan | C/C++ | | ✓ |
|
| Conan | C/C++ | | ✓ |
|
||||||
| Conda | Python/R | Yes | ✓ |
|
| Conda | Python/R | Yes | ✓ |
|
||||||
| CRAN | R | | ✓ |
|
| CRAN | R | | ✓ |
|
||||||
|
| Julia | Julia | | ✓ |
|
||||||
| Container | Docker/OCI | | ✓ |
|
| Container | Docker/OCI | | ✓ |
|
||||||
| Debian | Debian/Ubuntu | | ✓ |
|
| Debian | Debian/Ubuntu | | ✓ |
|
||||||
| RPM | RHEL/Fedora | | ✓ |
|
| RPM | RHEL/Fedora | | ✓ |
|
||||||
|
|
@ -208,6 +210,34 @@ Add to your `~/.m2/settings.xml`:
|
||||||
</settings>
|
</settings>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The `/maven/` endpoint uses Maven Central as primary upstream and falls back to the Gradle Plugin Portal for Gradle plugin marker metadata and related artifacts when the primary upstream returns not found.
|
||||||
|
|
||||||
|
For Gradle plugin resolution via the same proxy endpoint:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
pluginManagement {
|
||||||
|
repositories {
|
||||||
|
maven(url = "http://localhost:8080/maven/")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Gradle HTTP Build Cache
|
||||||
|
|
||||||
|
Configure in `settings.gradle(.kts)`:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
buildCache {
|
||||||
|
local {
|
||||||
|
enabled = false
|
||||||
|
}
|
||||||
|
remote<HttpBuildCache> {
|
||||||
|
url = uri("http://localhost:8080/gradle/")
|
||||||
|
push = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
### NuGet
|
### NuGet
|
||||||
|
|
||||||
Configure in `nuget.config`:
|
Configure in `nuget.config`:
|
||||||
|
|
@ -295,6 +325,21 @@ local({
|
||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Julia
|
||||||
|
|
||||||
|
Set the Pkg server before starting Julia:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export JULIA_PKG_SERVER=http://localhost:8080/julia
|
||||||
|
```
|
||||||
|
|
||||||
|
Or inside a running session:
|
||||||
|
|
||||||
|
```julia
|
||||||
|
ENV["JULIA_PKG_SERVER"] = "http://localhost:8080/julia"
|
||||||
|
using Pkg; Pkg.update()
|
||||||
|
```
|
||||||
|
|
||||||
### Docker / Container Registry
|
### Docker / Container Registry
|
||||||
|
|
||||||
Configure Docker to use the proxy as a registry mirror in `/etc/docker/daemon.json`:
|
Configure Docker to use the proxy as a registry mirror in `/etc/docker/daemon.json`:
|
||||||
|
|
@ -353,6 +398,7 @@ sudo dnf update
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
The proxy can be configured via:
|
The proxy can be configured via:
|
||||||
|
|
||||||
1. Command line flags (highest priority)
|
1. Command line flags (highest priority)
|
||||||
2. Environment variables
|
2. Environment variables
|
||||||
3. Configuration file (YAML or JSON)
|
3. Configuration file (YAML or JSON)
|
||||||
|
|
@ -560,7 +606,7 @@ Recently cached:
|
||||||
| Endpoint | Description |
|
| Endpoint | Description |
|
||||||
|----------|-------------|
|
|----------|-------------|
|
||||||
| `GET /` | Dashboard (web UI) |
|
| `GET /` | Dashboard (web UI) |
|
||||||
| `GET /health` | Health check (returns "ok" if healthy) |
|
| `GET /health` | Health check (JSON; HTTP 200 healthy, 503 unhealthy) |
|
||||||
| `GET /stats` | Cache statistics (JSON) |
|
| `GET /stats` | Cache statistics (JSON) |
|
||||||
| `GET /metrics` | Prometheus metrics |
|
| `GET /metrics` | Prometheus metrics |
|
||||||
| `GET /npm/*` | npm registry protocol |
|
| `GET /npm/*` | npm registry protocol |
|
||||||
|
|
@ -576,6 +622,7 @@ Recently cached:
|
||||||
| `GET /conan/*` | Conan C/C++ protocol |
|
| `GET /conan/*` | Conan C/C++ protocol |
|
||||||
| `GET /conda/*` | Conda/Anaconda protocol |
|
| `GET /conda/*` | Conda/Anaconda protocol |
|
||||||
| `GET /cran/*` | CRAN (R) protocol |
|
| `GET /cran/*` | CRAN (R) protocol |
|
||||||
|
| `GET /julia/*` | Julia Pkg server protocol |
|
||||||
| `GET /v2/*` | OCI/Docker registry protocol |
|
| `GET /v2/*` | OCI/Docker registry protocol |
|
||||||
| `GET /debian/*` | Debian/APT repository protocol |
|
| `GET /debian/*` | Debian/APT repository protocol |
|
||||||
| `GET /rpm/*` | RPM/Yum repository protocol |
|
| `GET /rpm/*` | RPM/Yum repository protocol |
|
||||||
|
|
@ -798,9 +845,28 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre
|
||||||
| `proxy_storage_operation_duration_seconds` | histogram | `operation` | Storage read/write latency |
|
| `proxy_storage_operation_duration_seconds` | histogram | `operation` | Storage read/write latency |
|
||||||
| `proxy_storage_errors_total` | counter | `operation` | Storage read/write failures |
|
| `proxy_storage_errors_total` | counter | `operation` | Storage read/write failures |
|
||||||
| `proxy_active_requests` | gauge | | In-flight requests |
|
| `proxy_active_requests` | gauge | | In-flight requests |
|
||||||
|
| `proxy_health_probe_failures_total` | counter | `step` | Storage health probe failures by failing step (`write`, `size`, `read`, `verify`, `delete`). |
|
||||||
|
|
||||||
Cache size and artifact count are refreshed every 60 seconds. The remaining metrics update on each request.
|
Cache size and artifact count are refreshed every 60 seconds. The remaining metrics update on each request.
|
||||||
|
|
||||||
|
### Health Check
|
||||||
|
|
||||||
|
`/health` returns a structured JSON report of subsystem health. HTTP 200 if all checks pass; 503 if any fail.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"checks": {
|
||||||
|
"database": {"status": "ok"},
|
||||||
|
"storage": {"status": "ok"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Failing checks include an `"error"` field. Storage failures also include a `"step"` field identifying which probe step failed (`write`, `size`, `read`, `verify`, `delete`). When the database check fails, the storage entry reports `{"status": "skipped"}` so the response always carries the same key set.
|
||||||
|
|
||||||
|
Storage probe results are cached for `health.storage_probe_interval` (default 30s) to bound the cost of probing remote backends. A probe holds an internal mutex for up to 10 seconds (the hardcoded per-probe timeout), so `/health` is intended as a Kubernetes **readiness** probe rather than a liveness probe — a slow S3 round-trip should pull the pod from rotation, not restart it.
|
||||||
|
|
||||||
Scrape config for Prometheus:
|
Scrape config for Prometheus:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
|
|
@ -924,6 +990,7 @@ The proxy will recreate the database on next start.
|
||||||
## Building from Source
|
## Building from Source
|
||||||
|
|
||||||
Requirements:
|
Requirements:
|
||||||
|
|
||||||
- Go 1.25 or later
|
- Go 1.25 or later
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,14 @@
|
||||||
// PROXY_DATABASE_URL - PostgreSQL connection URL
|
// PROXY_DATABASE_URL - PostgreSQL connection URL
|
||||||
// PROXY_LOG_LEVEL - Log level
|
// PROXY_LOG_LEVEL - Log level
|
||||||
// PROXY_LOG_FORMAT - Log format
|
// PROXY_LOG_FORMAT - Log format
|
||||||
|
// PROXY_UPSTREAM_MAVEN - Maven repository upstream URL
|
||||||
|
// PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL - Gradle Plugin Portal upstream URL
|
||||||
|
// PROXY_GRADLE_BUILD_CACHE_READ_ONLY - Disable Gradle PUT uploads
|
||||||
|
// PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE - Max Gradle PUT request body size
|
||||||
|
// PROXY_GRADLE_BUILD_CACHE_MAX_AGE - Gradle cache max age eviction
|
||||||
|
// PROXY_GRADLE_BUILD_CACHE_MAX_SIZE - Gradle cache max total size
|
||||||
|
// PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL - Gradle cache eviction sweep interval
|
||||||
|
// PROXY_HEALTH_STORAGE_PROBE_INTERVAL - Storage health probe cache interval (default "30s")
|
||||||
//
|
//
|
||||||
// Example:
|
// Example:
|
||||||
//
|
//
|
||||||
|
|
@ -193,6 +201,14 @@ func runServe() {
|
||||||
fmt.Fprintf(os.Stderr, " PROXY_DATABASE_URL PostgreSQL connection URL\n")
|
fmt.Fprintf(os.Stderr, " PROXY_DATABASE_URL PostgreSQL connection URL\n")
|
||||||
fmt.Fprintf(os.Stderr, " PROXY_LOG_LEVEL Log level\n")
|
fmt.Fprintf(os.Stderr, " PROXY_LOG_LEVEL Log level\n")
|
||||||
fmt.Fprintf(os.Stderr, " PROXY_LOG_FORMAT Log format\n")
|
fmt.Fprintf(os.Stderr, " PROXY_LOG_FORMAT Log format\n")
|
||||||
|
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_MAVEN Maven repository upstream URL\n")
|
||||||
|
fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL Gradle Plugin Portal upstream URL\n")
|
||||||
|
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_READ_ONLY Disable Gradle PUT uploads\n")
|
||||||
|
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE Max Gradle PUT request body size\n")
|
||||||
|
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_AGE Gradle cache max age eviction\n")
|
||||||
|
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_MAX_SIZE Gradle cache max total size\n")
|
||||||
|
fmt.Fprintf(os.Stderr, " PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL Gradle cache eviction sweep interval\n")
|
||||||
|
fmt.Fprintf(os.Stderr, " PROXY_HEALTH_STORAGE_PROBE_INTERVAL Storage health probe cache interval\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
_ = fs.Parse(os.Args[1:])
|
_ = fs.Parse(os.Args[1:])
|
||||||
|
|
@ -454,6 +470,7 @@ func runMirror() {
|
||||||
proxy := handler.NewProxy(db, store, fetcher, resolver, logger)
|
proxy := handler.NewProxy(db, store, fetcher, resolver, logger)
|
||||||
proxy.CacheMetadata = true // mirror always caches metadata
|
proxy.CacheMetadata = true // mirror always caches metadata
|
||||||
proxy.MetadataTTL = cfg.ParseMetadataTTL()
|
proxy.MetadataTTL = cfg.ParseMetadataTTL()
|
||||||
|
proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
|
||||||
|
|
||||||
m := mirror.New(proxy, db, store, logger, *concurrency)
|
m := mirror.New(proxy, db, store, logger, *concurrency)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -71,6 +71,12 @@ upstream:
|
||||||
# npm registry URL
|
# npm registry URL
|
||||||
npm: "https://registry.npmjs.org"
|
npm: "https://registry.npmjs.org"
|
||||||
|
|
||||||
|
# Maven repository URL (used by /maven endpoint)
|
||||||
|
maven: "https://repo1.maven.org/maven2"
|
||||||
|
|
||||||
|
# Gradle Plugin Portal Maven URL (fallback for plugin marker artifacts)
|
||||||
|
gradle_plugin_portal: "https://plugins.gradle.org/m2"
|
||||||
|
|
||||||
# Cargo sparse index URL
|
# Cargo sparse index URL
|
||||||
cargo: "https://index.crates.io"
|
cargo: "https://index.crates.io"
|
||||||
|
|
||||||
|
|
@ -108,6 +114,35 @@ upstream:
|
||||||
# header_name: "X-Auth-Token"
|
# header_name: "X-Auth-Token"
|
||||||
# header_value: "${MAVEN_TOKEN}"
|
# header_value: "${MAVEN_TOKEN}"
|
||||||
|
|
||||||
|
# Gradle HttpBuildCache configuration
|
||||||
|
gradle:
|
||||||
|
build_cache:
|
||||||
|
# Set to true to disable PUT uploads (read-only cache mode)
|
||||||
|
read_only: false
|
||||||
|
|
||||||
|
# Maximum accepted Gradle cache upload body size
|
||||||
|
# Required and must be > 0
|
||||||
|
max_upload_size: "100MB"
|
||||||
|
|
||||||
|
# Evict entries older than this age (set to "0" to disable age-based eviction)
|
||||||
|
max_age: "168h"
|
||||||
|
|
||||||
|
# Cap total Gradle cache size; oldest entries are deleted first
|
||||||
|
# ("0" disables size-based eviction)
|
||||||
|
# max_size: "20GB"
|
||||||
|
|
||||||
|
# How often eviction runs when max_age or max_size is set
|
||||||
|
sweep_interval: "10m"
|
||||||
|
|
||||||
|
# Health endpoint configuration.
|
||||||
|
health:
|
||||||
|
# Minimum time between storage backend probes.
|
||||||
|
# The /health endpoint runs a write/read/verify/delete round-trip
|
||||||
|
# against the configured storage backend and caches the result for
|
||||||
|
# this interval. Set to "0" to probe on every request.
|
||||||
|
# Default: "30s".
|
||||||
|
storage_probe_interval: "30s"
|
||||||
|
|
||||||
# Version cooldown configuration
|
# Version cooldown configuration
|
||||||
# Hides package versions published too recently, giving the community time
|
# Hides package versions published too recently, giving the community time
|
||||||
# to spot malicious releases before they're pulled into projects.
|
# to spot malicious releases before they're pulled into projects.
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ The proxy is a caching HTTP server that sits between package manager clients and
|
||||||
│ │ /npm/* -> NPMHandler /health -> healthHandler │ │
|
│ │ /npm/* -> NPMHandler /health -> healthHandler │ │
|
||||||
│ │ /cargo/* -> CargoHandler /stats -> statsHandler │ │
|
│ │ /cargo/* -> CargoHandler /stats -> statsHandler │ │
|
||||||
│ │ /gem/* -> GemHandler /metrics -> prometheus │ │
|
│ │ /gem/* -> GemHandler /metrics -> prometheus │ │
|
||||||
│ │ ...16 ecosystems /api/* -> APIHandler │ │
|
│ │ ...17 ecosystems /api/* -> APIHandler │ │
|
||||||
│ │ / -> Web UI │ │
|
│ │ / -> Web UI │ │
|
||||||
│ └──────────────────────────────────────────────────────────┘ │
|
│ └──────────────────────────────────────────────────────────┘ │
|
||||||
│ │ │ │ │
|
│ │ │ │ │
|
||||||
|
|
@ -277,15 +277,15 @@ HTTP server setup, web UI, and API handlers.
|
||||||
- Web UI: dashboard, package browser, source browser, version comparison
|
- Web UI: dashboard, package browser, source browser, version comparison
|
||||||
- Templates are embedded in the binary via `//go:embed`
|
- Templates are embedded in the binary via `//go:embed`
|
||||||
- Enrichment API for package metadata, vulnerability scanning, and outdated detection
|
- Enrichment API for package metadata, vulnerability scanning, and outdated detection
|
||||||
- Health, stats, and Prometheus metrics endpoints
|
- Health, stats, and Prometheus metrics endpoints. `/health` runs an active write → size-check → read → verify → delete probe against the storage backend and returns a structured JSON response (`HealthResponse`) with `"ok"` / `"error"` status per subsystem. Probe results are cached (default 30 s, configurable via `health.storage_probe_interval`) to avoid overwhelming remote backends.
|
||||||
|
|
||||||
### `internal/metrics`
|
### `internal/metrics`
|
||||||
|
|
||||||
Prometheus metrics for cache performance, upstream latency, storage operations, and active requests. See the Monitoring section of the README for the full metric list.
|
Prometheus metrics for cache performance, upstream latency, storage operations, and active requests. See the Monitoring section of the README for the full metric list.
|
||||||
|
|
||||||
### `internal/cooldown`
|
### Cooldown
|
||||||
|
|
||||||
Version age filtering for supply chain attack mitigation. Configurable at global, ecosystem, and per-package levels. Supported by npm, PyPI, pub.dev, and Composer handlers.
|
Version age filtering for supply chain attack mitigation, provided by [github.com/git-pkgs/cooldown](https://github.com/git-pkgs/cooldown). Configurable at global, ecosystem, and per-package levels. Supported by npm, PyPI, pub.dev, and Composer handlers.
|
||||||
|
|
||||||
### `internal/enrichment`
|
### `internal/enrichment`
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -114,6 +114,8 @@ Override default upstream registry URLs:
|
||||||
```yaml
|
```yaml
|
||||||
upstream:
|
upstream:
|
||||||
npm: "https://registry.npmjs.org"
|
npm: "https://registry.npmjs.org"
|
||||||
|
maven: "https://repo1.maven.org/maven2"
|
||||||
|
gradle_plugin_portal: "https://plugins.gradle.org/m2"
|
||||||
cargo: "https://index.crates.io"
|
cargo: "https://index.crates.io"
|
||||||
cargo_download: "https://static.crates.io/crates"
|
cargo_download: "https://static.crates.io/crates"
|
||||||
```
|
```
|
||||||
|
|
@ -184,6 +186,30 @@ upstream:
|
||||||
token: "${PRIVATE_TOKEN}"
|
token: "${PRIVATE_TOKEN}"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Gradle Build Cache
|
||||||
|
|
||||||
|
The `/gradle` endpoint supports optional safeguards for upload control and cache retention.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
gradle:
|
||||||
|
build_cache:
|
||||||
|
read_only: false
|
||||||
|
max_upload_size: "100MB"
|
||||||
|
max_age: "168h"
|
||||||
|
max_size: "20GB"
|
||||||
|
sweep_interval: "10m"
|
||||||
|
```
|
||||||
|
|
||||||
|
| Config | Environment | Description |
|
||||||
|
|--------|-------------|-------------|
|
||||||
|
| `gradle.build_cache.read_only` | `PROXY_GRADLE_BUILD_CACHE_READ_ONLY` | Disable PUT uploads and keep GET/HEAD read-only |
|
||||||
|
| `gradle.build_cache.max_upload_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE` | Maximum accepted PUT body size (must be > 0) |
|
||||||
|
| `gradle.build_cache.max_age` | `PROXY_GRADLE_BUILD_CACHE_MAX_AGE` | Delete entries older than this duration (default `168h`, set `0` to disable) |
|
||||||
|
| `gradle.build_cache.max_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_SIZE` | Total size cap for `_gradle/http-build-cache`, deleting oldest first (`0` disables) |
|
||||||
|
| `gradle.build_cache.sweep_interval` | `PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL` | Frequency for background eviction sweeps |
|
||||||
|
|
||||||
|
`max_age` and `max_size` are independent and can be combined. When both are set, age-based eviction runs first, then size-based eviction trims remaining entries oldest-first.
|
||||||
|
|
||||||
## Cooldown
|
## Cooldown
|
||||||
|
|
||||||
The cooldown feature hides package versions published too recently, giving the community time to spot malicious releases before they reach your projects. When a version is within its cooldown period, it's stripped from metadata responses so package managers won't install it.
|
The cooldown feature hides package versions published too recently, giving the community time to spot malicious releases before they reach your projects. When a version is within its cooldown period, it's stripped from metadata responses so package managers won't install it.
|
||||||
|
|
@ -239,6 +265,16 @@ Set to `"0"` to always revalidate with upstream (ETag-based conditional requests
|
||||||
|
|
||||||
When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated.
|
When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated.
|
||||||
|
|
||||||
|
### Metadata size limit
|
||||||
|
|
||||||
|
Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
metadata_max_size: "100MB" # default
|
||||||
|
```
|
||||||
|
|
||||||
|
Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`.
|
||||||
|
|
||||||
## Mirror API
|
## Mirror API
|
||||||
|
|
||||||
The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP:
|
The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP:
|
||||||
|
|
|
||||||
|
|
@ -64,13 +64,13 @@ const docTemplate = `{
|
||||||
"404": {
|
"404": {
|
||||||
"description": "Not Found",
|
"description": "Not Found",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -126,19 +126,19 @@ const docTemplate = `{
|
||||||
"400": {
|
"400": {
|
||||||
"description": "Bad Request",
|
"description": "Bad Request",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"404": {
|
"404": {
|
||||||
"description": "Not Found",
|
"description": "Not Found",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -177,13 +177,13 @@ const docTemplate = `{
|
||||||
"400": {
|
"400": {
|
||||||
"description": "Bad Request",
|
"description": "Bad Request",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -240,13 +240,13 @@ const docTemplate = `{
|
||||||
"404": {
|
"404": {
|
||||||
"description": "Not Found",
|
"description": "Not Found",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -285,13 +285,13 @@ const docTemplate = `{
|
||||||
"400": {
|
"400": {
|
||||||
"description": "Bad Request",
|
"description": "Bad Request",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -338,13 +338,13 @@ const docTemplate = `{
|
||||||
"400": {
|
"400": {
|
||||||
"description": "Bad Request",
|
"description": "Bad Request",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -384,13 +384,13 @@ const docTemplate = `{
|
||||||
"400": {
|
"400": {
|
||||||
"description": "Bad Request",
|
"description": "Bad Request",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -399,7 +399,7 @@ const docTemplate = `{
|
||||||
"/health": {
|
"/health": {
|
||||||
"get": {
|
"get": {
|
||||||
"produces": [
|
"produces": [
|
||||||
"text/plain"
|
"application/json"
|
||||||
],
|
],
|
||||||
"tags": [
|
"tags": [
|
||||||
"meta"
|
"meta"
|
||||||
|
|
@ -409,13 +409,13 @@ const docTemplate = `{
|
||||||
"200": {
|
"200": {
|
||||||
"description": "OK",
|
"description": "OK",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.HealthResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"503": {
|
"503": {
|
||||||
"description": "Service Unavailable",
|
"description": "Service Unavailable",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.HealthResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -440,7 +440,7 @@ const docTemplate = `{
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -504,6 +504,45 @@ const docTemplate = `{
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"server.ErrorResponse": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"code": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"server.HealthCheck": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"error": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"step": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"server.HealthResponse": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"checks": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": {
|
||||||
|
"$ref": "#/definitions/server.HealthCheck"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"server.OutdatedPackage": {
|
"server.OutdatedPackage": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
|
|
|
||||||
|
|
@ -57,13 +57,13 @@
|
||||||
"404": {
|
"404": {
|
||||||
"description": "Not Found",
|
"description": "Not Found",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -119,19 +119,19 @@
|
||||||
"400": {
|
"400": {
|
||||||
"description": "Bad Request",
|
"description": "Bad Request",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"404": {
|
"404": {
|
||||||
"description": "Not Found",
|
"description": "Not Found",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -170,13 +170,13 @@
|
||||||
"400": {
|
"400": {
|
||||||
"description": "Bad Request",
|
"description": "Bad Request",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -233,13 +233,13 @@
|
||||||
"404": {
|
"404": {
|
||||||
"description": "Not Found",
|
"description": "Not Found",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -278,13 +278,13 @@
|
||||||
"400": {
|
"400": {
|
||||||
"description": "Bad Request",
|
"description": "Bad Request",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -331,13 +331,13 @@
|
||||||
"400": {
|
"400": {
|
||||||
"description": "Bad Request",
|
"description": "Bad Request",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -377,13 +377,13 @@
|
||||||
"400": {
|
"400": {
|
||||||
"description": "Bad Request",
|
"description": "Bad Request",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -392,7 +392,7 @@
|
||||||
"/health": {
|
"/health": {
|
||||||
"get": {
|
"get": {
|
||||||
"produces": [
|
"produces": [
|
||||||
"text/plain"
|
"application/json"
|
||||||
],
|
],
|
||||||
"tags": [
|
"tags": [
|
||||||
"meta"
|
"meta"
|
||||||
|
|
@ -402,13 +402,13 @@
|
||||||
"200": {
|
"200": {
|
||||||
"description": "OK",
|
"description": "OK",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.HealthResponse"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"503": {
|
"503": {
|
||||||
"description": "Service Unavailable",
|
"description": "Service Unavailable",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.HealthResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -433,7 +433,7 @@
|
||||||
"500": {
|
"500": {
|
||||||
"description": "Internal Server Error",
|
"description": "Internal Server Error",
|
||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"$ref": "#/definitions/server.ErrorResponse"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -497,6 +497,45 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"server.ErrorResponse": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"code": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"message": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"server.HealthCheck": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"error": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"step": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"server.HealthResponse": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"checks": {
|
||||||
|
"type": "object",
|
||||||
|
"additionalProperties": {
|
||||||
|
"$ref": "#/definitions/server.HealthCheck"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"server.OutdatedPackage": {
|
"server.OutdatedPackage": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": {
|
"properties": {
|
||||||
|
|
|
||||||
21
go.mod
21
go.mod
|
|
@ -3,13 +3,15 @@ module github.com/git-pkgs/proxy
|
||||||
go 1.25.6
|
go 1.25.6
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/CycloneDX/cyclonedx-go v0.10.0
|
github.com/BurntSushi/toml v1.6.0
|
||||||
github.com/git-pkgs/archives v0.2.3
|
github.com/CycloneDX/cyclonedx-go v0.11.0
|
||||||
github.com/git-pkgs/enrichment v0.2.2
|
github.com/git-pkgs/archives v0.3.0
|
||||||
|
github.com/git-pkgs/cooldown v0.1.1
|
||||||
|
github.com/git-pkgs/enrichment v0.2.3
|
||||||
github.com/git-pkgs/purl v0.1.12
|
github.com/git-pkgs/purl v0.1.12
|
||||||
github.com/git-pkgs/registries v0.5.1
|
github.com/git-pkgs/registries v0.6.1
|
||||||
github.com/git-pkgs/spdx v0.1.3
|
github.com/git-pkgs/spdx v0.1.4
|
||||||
github.com/git-pkgs/vers v0.2.5
|
github.com/git-pkgs/vers v0.2.6
|
||||||
github.com/git-pkgs/vulns v0.1.5
|
github.com/git-pkgs/vulns v0.1.5
|
||||||
github.com/go-chi/chi/v5 v5.2.5
|
github.com/go-chi/chi/v5 v5.2.5
|
||||||
github.com/jmoiron/sqlx v1.4.0
|
github.com/jmoiron/sqlx v1.4.0
|
||||||
|
|
@ -22,7 +24,7 @@ require (
|
||||||
golang.org/x/sync v0.20.0
|
golang.org/x/sync v0.20.0
|
||||||
google.golang.org/protobuf v1.36.11
|
google.golang.org/protobuf v1.36.11
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
modernc.org/sqlite v1.50.0
|
modernc.org/sqlite v1.50.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
|
@ -49,7 +51,6 @@ require (
|
||||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||||
github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect
|
github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect
|
||||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||||
github.com/BurntSushi/toml v1.6.0 // indirect
|
|
||||||
github.com/Djarvur/go-err113 v0.1.1 // indirect
|
github.com/Djarvur/go-err113 v0.1.1 // indirect
|
||||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||||
github.com/Masterminds/semver/v3 v3.4.0 // indirect
|
github.com/Masterminds/semver/v3 v3.4.0 // indirect
|
||||||
|
|
@ -128,7 +129,7 @@ require (
|
||||||
github.com/ghostiam/protogetter v0.3.20 // indirect
|
github.com/ghostiam/protogetter v0.3.20 // indirect
|
||||||
github.com/git-pkgs/packageurl-go v0.3.1 // indirect
|
github.com/git-pkgs/packageurl-go v0.3.1 // indirect
|
||||||
github.com/git-pkgs/pom v0.1.4 // indirect
|
github.com/git-pkgs/pom v0.1.4 // indirect
|
||||||
github.com/github/go-spdx/v2 v2.6.0 // indirect
|
github.com/github/go-spdx/v2 v2.7.0 // indirect
|
||||||
github.com/go-critic/go-critic v0.14.3 // indirect
|
github.com/go-critic/go-critic v0.14.3 // indirect
|
||||||
github.com/go-logr/logr v1.4.3 // indirect
|
github.com/go-logr/logr v1.4.3 // indirect
|
||||||
github.com/go-logr/stdr v1.2.2 // indirect
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
|
|
@ -309,7 +310,7 @@ require (
|
||||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||||
honnef.co/go/tools v0.7.0 // indirect
|
honnef.co/go/tools v0.7.0 // indirect
|
||||||
modernc.org/libc v1.72.0 // indirect
|
modernc.org/libc v1.72.3 // indirect
|
||||||
modernc.org/mathutil v1.7.1 // indirect
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
modernc.org/memory v1.11.0 // indirect
|
modernc.org/memory v1.11.0 // indirect
|
||||||
mvdan.cc/gofumpt v0.9.2 // indirect
|
mvdan.cc/gofumpt v0.9.2 // indirect
|
||||||
|
|
|
||||||
50
go.sum
50
go.sum
|
|
@ -66,8 +66,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQ
|
||||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
|
github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
|
||||||
github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
||||||
github.com/CycloneDX/cyclonedx-go v0.10.0 h1:7xyklU7YD+CUyGzSFIARG18NYLsKVn4QFg04qSsu+7Y=
|
github.com/CycloneDX/cyclonedx-go v0.11.0 h1:GokP8FiRC+foiuwWhSSLpSD5H4hSWtGnR3wo7apkBFI=
|
||||||
github.com/CycloneDX/cyclonedx-go v0.10.0/go.mod h1:vUvbCXQsEm48OI6oOlanxstwNByXjCZ2wuleUlwGEO8=
|
github.com/CycloneDX/cyclonedx-go v0.11.0/go.mod h1:vUvbCXQsEm48OI6oOlanxstwNByXjCZ2wuleUlwGEO8=
|
||||||
github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g=
|
github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g=
|
||||||
github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k=
|
github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k=
|
||||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
|
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
|
||||||
|
|
@ -250,26 +250,28 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo=
|
||||||
github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
|
github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
|
||||||
github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0=
|
github.com/ghostiam/protogetter v0.3.20 h1:oW7OPFit2FxZOpmMRPP9FffU4uUpfeE/rEdE1f+MzD0=
|
||||||
github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI=
|
github.com/ghostiam/protogetter v0.3.20/go.mod h1:FjIu5Yfs6FT391m+Fjp3fbAYJ6rkL/J6ySpZBfnODuI=
|
||||||
github.com/git-pkgs/archives v0.2.3 h1:iyKo4CY/KA3HJyshAj2iTVloq4gXSp8vv2+0H+IE4gc=
|
github.com/git-pkgs/archives v0.3.0 h1:iXKyO83jEFub1PGEDlHmk2tQ7XeV5LySTc0sEkH3x78=
|
||||||
github.com/git-pkgs/archives v0.2.3/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU=
|
github.com/git-pkgs/archives v0.3.0/go.mod h1:LTJ1iQVFA7otizWMOyiI82NYVmyBWAPRzwu/e30rcXU=
|
||||||
github.com/git-pkgs/enrichment v0.2.2 h1:vaQu5vs3tjQB5JI0gzBrUCynUc9z3l5byPhgKFaNZrc=
|
github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg7w=
|
||||||
github.com/git-pkgs/enrichment v0.2.2/go.mod h1:5JWGmlHWcv5HQHUrctcpnRUNpEF5VAixD2z4zvqKejs=
|
github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E=
|
||||||
|
github.com/git-pkgs/enrichment v0.2.3 h1:42mqoUhQZNGhlEO671pboI/Cu6F+DoffJoFbVhb2jlw=
|
||||||
|
github.com/git-pkgs/enrichment v0.2.3/go.mod h1:MBv5nhHzjwLxeSgx2+7waCcpReUjhCD+9B0bvufpMO0=
|
||||||
github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M=
|
github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M=
|
||||||
github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4=
|
github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4=
|
||||||
github.com/git-pkgs/pom v0.1.4 h1:C6st+XSbF75eKuwfdkDZZtYHoTcaWRIEQYar5VtszUo=
|
github.com/git-pkgs/pom v0.1.4 h1:C6st+XSbF75eKuwfdkDZZtYHoTcaWRIEQYar5VtszUo=
|
||||||
github.com/git-pkgs/pom v0.1.4/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk=
|
github.com/git-pkgs/pom v0.1.4/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk=
|
||||||
github.com/git-pkgs/purl v0.1.12 h1:qCskrEU1LWQhCkIVZd992W5++Bsxazvx2Cx1/65qCvU=
|
github.com/git-pkgs/purl v0.1.12 h1:qCskrEU1LWQhCkIVZd992W5++Bsxazvx2Cx1/65qCvU=
|
||||||
github.com/git-pkgs/purl v0.1.12/go.mod h1:ofp4mHsR0cUeVONQaf33n6Wxg2QTEvtUdRfCedI8ouA=
|
github.com/git-pkgs/purl v0.1.12/go.mod h1:ofp4mHsR0cUeVONQaf33n6Wxg2QTEvtUdRfCedI8ouA=
|
||||||
github.com/git-pkgs/registries v0.5.1 h1:UPE42CyZAsOfqO3N5bDelu28wS4Ifx/aOj0XZS4qYeI=
|
github.com/git-pkgs/registries v0.6.1 h1:xZfVZQmffIfdeJthn5o2EozbVJ6gBeImYwKQnfdKUfU=
|
||||||
github.com/git-pkgs/registries v0.5.1/go.mod h1:BY0YW+V0WDGBMuDR2aSMR3NzOPFK4K+F3j6+ch+cq3M=
|
github.com/git-pkgs/registries v0.6.1/go.mod h1:a3BP/56VW3O/CFRqiJCtSy+OqRrSH25wF1PWHP76ka0=
|
||||||
github.com/git-pkgs/spdx v0.1.3 h1:YQou23mLfzbW//6JlHUuc5x1P5VNIIDSku5gvauf86I=
|
github.com/git-pkgs/spdx v0.1.4 h1:eQ0waEV3uUeItpWAOvdN1K1rL9hTgsU7fF74r1mDXMs=
|
||||||
github.com/git-pkgs/spdx v0.1.3/go.mod h1:4HGGWyC8tg4DjOhrtBTYl4Lu+5i2BFuauGX8zcVcYPg=
|
github.com/git-pkgs/spdx v0.1.4/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto=
|
||||||
github.com/git-pkgs/vers v0.2.5 h1:tDtUMik9Iw1lyPHdT5V6LXjLo9LsJc0xOawURz7ibQU=
|
github.com/git-pkgs/vers v0.2.6 h1:IelZd7BP/JhzTloUTDY67nehUgoYva3g9viqAMCHJg8=
|
||||||
github.com/git-pkgs/vers v0.2.5/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo=
|
github.com/git-pkgs/vers v0.2.6/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo=
|
||||||
github.com/git-pkgs/vulns v0.1.5 h1:mtX88/27toFl+B95kaH5QbAdOCQ3YIDGjJrlrrnqQTE=
|
github.com/git-pkgs/vulns v0.1.5 h1:mtX88/27toFl+B95kaH5QbAdOCQ3YIDGjJrlrrnqQTE=
|
||||||
github.com/git-pkgs/vulns v0.1.5/go.mod h1:bZFikfrR/5gC0ZMwXh7qcEu2gpKfXMBhVsy4kF12Ae0=
|
github.com/git-pkgs/vulns v0.1.5/go.mod h1:bZFikfrR/5gC0ZMwXh7qcEu2gpKfXMBhVsy4kF12Ae0=
|
||||||
github.com/github/go-spdx/v2 v2.6.0 h1:Y/Chr7L8oG85Ilbzl11xkUSQFUfG1kGkLP18LyInvhg=
|
github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ=
|
||||||
github.com/github/go-spdx/v2 v2.6.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0=
|
github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0=
|
||||||
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
||||||
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
||||||
github.com/go-critic/go-critic v0.14.3 h1:5R1qH2iFeo4I/RJU8vTezdqs08Egi4u5p6vOESA0pog=
|
github.com/go-critic/go-critic v0.14.3 h1:5R1qH2iFeo4I/RJU8vTezdqs08Egi4u5p6vOESA0pog=
|
||||||
|
|
@ -882,10 +884,10 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU=
|
honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU=
|
||||||
honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc=
|
honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc=
|
||||||
modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U=
|
modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY=
|
||||||
modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8=
|
modernc.org/cc/v4 v4.28.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||||
modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU=
|
modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ=
|
||||||
modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0=
|
modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A=
|
||||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||||
|
|
@ -894,18 +896,18 @@ modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo=
|
||||||
modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||||
modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c=
|
modernc.org/libc v1.72.3 h1:ZnDF4tXn4NBXFutMMQC4vtbTFSXhhKzR73fv0beZEAU=
|
||||||
modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ=
|
modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs=
|
||||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||||
modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
|
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||||
modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||||
modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM=
|
modernc.org/sqlite v1.50.1 h1:l+cQvn0sd0zJJtfygGHuQJ5AjlrwXmWPw4KP3ZMwr9w=
|
||||||
modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
|
modernc.org/sqlite v1.50.1/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM=
|
||||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||||
|
|
|
||||||
|
|
@ -96,9 +96,20 @@ type Config struct {
|
||||||
// Default: "5m". Set to "0" to always revalidate.
|
// Default: "5m". Set to "0" to always revalidate.
|
||||||
MetadataTTL string `json:"metadata_ttl" yaml:"metadata_ttl"`
|
MetadataTTL string `json:"metadata_ttl" yaml:"metadata_ttl"`
|
||||||
|
|
||||||
|
// MetadataMaxSize is the maximum size of an upstream metadata response
|
||||||
|
// the proxy will buffer (e.g. "100MB", "250MB"). Responses over this
|
||||||
|
// size return ErrMetadataTooLarge. Default: "100MB".
|
||||||
|
MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"`
|
||||||
|
|
||||||
// MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP.
|
// MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP.
|
||||||
// Disabled by default to prevent unauthenticated users from triggering downloads.
|
// Disabled by default to prevent unauthenticated users from triggering downloads.
|
||||||
MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"`
|
MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"`
|
||||||
|
|
||||||
|
// Gradle configures Gradle HttpBuildCache behavior.
|
||||||
|
Gradle GradleConfig `json:"gradle" yaml:"gradle"`
|
||||||
|
|
||||||
|
// Health configures the /health endpoint behavior.
|
||||||
|
Health HealthConfig `json:"health" yaml:"health"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CooldownConfig configures version cooldown periods.
|
// CooldownConfig configures version cooldown periods.
|
||||||
|
|
@ -151,6 +162,42 @@ type StorageConfig struct {
|
||||||
DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"`
|
DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GradleConfig configures Gradle-specific features.
|
||||||
|
type GradleConfig struct {
|
||||||
|
// BuildCache configures the /gradle HttpBuildCache endpoint.
|
||||||
|
BuildCache GradleBuildCacheConfig `json:"build_cache" yaml:"build_cache"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GradleBuildCacheConfig configures Gradle HttpBuildCache safeguards.
|
||||||
|
type GradleBuildCacheConfig struct {
|
||||||
|
// ReadOnly disables PUT uploads and keeps cache reads (GET/HEAD) enabled.
|
||||||
|
ReadOnly bool `json:"read_only" yaml:"read_only"`
|
||||||
|
|
||||||
|
// MaxUploadSize caps a single PUT body size (e.g., "100MB"). Must be > 0.
|
||||||
|
// Default: "100MB".
|
||||||
|
MaxUploadSize string `json:"max_upload_size" yaml:"max_upload_size"`
|
||||||
|
|
||||||
|
// MaxAge evicts entries older than this duration (e.g., "24h", "7d").
|
||||||
|
// Empty or "0" disables age-based eviction.
|
||||||
|
MaxAge string `json:"max_age" yaml:"max_age"`
|
||||||
|
|
||||||
|
// MaxSize evicts oldest entries until total Gradle cache size is <= MaxSize.
|
||||||
|
// Empty or "0" disables size-based eviction.
|
||||||
|
MaxSize string `json:"max_size" yaml:"max_size"`
|
||||||
|
|
||||||
|
// SweepInterval controls periodic eviction frequency.
|
||||||
|
// Default: "10m".
|
||||||
|
SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HealthConfig configures the /health endpoint.
|
||||||
|
type HealthConfig struct {
|
||||||
|
// StorageProbeInterval is the minimum time between storage backend probes.
|
||||||
|
// Uses Go duration syntax (e.g. "30s", "1m"). Default: "30s".
|
||||||
|
// Set to "0" to probe on every /health request (useful for low-traffic deployments).
|
||||||
|
StorageProbeInterval string `json:"storage_probe_interval" yaml:"storage_probe_interval"`
|
||||||
|
}
|
||||||
|
|
||||||
// DatabaseConfig configures the cache database.
|
// DatabaseConfig configures the cache database.
|
||||||
type DatabaseConfig struct {
|
type DatabaseConfig struct {
|
||||||
// Driver is the database driver: "sqlite" or "postgres".
|
// Driver is the database driver: "sqlite" or "postgres".
|
||||||
|
|
@ -179,6 +226,15 @@ type UpstreamConfig struct {
|
||||||
// Default: https://registry.npmjs.org
|
// Default: https://registry.npmjs.org
|
||||||
NPM string `json:"npm" yaml:"npm"`
|
NPM string `json:"npm" yaml:"npm"`
|
||||||
|
|
||||||
|
// Maven is the upstream Maven repository URL.
|
||||||
|
// Default: https://repo1.maven.org/maven2
|
||||||
|
Maven string `json:"maven" yaml:"maven"`
|
||||||
|
|
||||||
|
// GradlePluginPortal is the upstream Gradle Plugin Portal Maven URL.
|
||||||
|
// Used to resolve Gradle plugin marker artifacts.
|
||||||
|
// Default: https://plugins.gradle.org/m2
|
||||||
|
GradlePluginPortal string `json:"gradle_plugin_portal" yaml:"gradle_plugin_portal"`
|
||||||
|
|
||||||
// Cargo is the upstream cargo index URL.
|
// Cargo is the upstream cargo index URL.
|
||||||
// Default: https://index.crates.io
|
// Default: https://index.crates.io
|
||||||
Cargo string `json:"cargo" yaml:"cargo"`
|
Cargo string `json:"cargo" yaml:"cargo"`
|
||||||
|
|
@ -257,9 +313,20 @@ func Default() *Config {
|
||||||
},
|
},
|
||||||
Upstream: UpstreamConfig{
|
Upstream: UpstreamConfig{
|
||||||
NPM: "https://registry.npmjs.org",
|
NPM: "https://registry.npmjs.org",
|
||||||
|
Maven: "https://repo1.maven.org/maven2",
|
||||||
|
GradlePluginPortal: "https://plugins.gradle.org/m2",
|
||||||
Cargo: "https://index.crates.io",
|
Cargo: "https://index.crates.io",
|
||||||
CargoDownload: "https://static.crates.io/crates",
|
CargoDownload: "https://static.crates.io/crates",
|
||||||
},
|
},
|
||||||
|
Gradle: GradleConfig{
|
||||||
|
BuildCache: GradleBuildCacheConfig{
|
||||||
|
ReadOnly: false,
|
||||||
|
MaxUploadSize: defaultGradleMaxUploadSizeStr,
|
||||||
|
MaxAge: "168h",
|
||||||
|
MaxSize: "",
|
||||||
|
SweepInterval: defaultGradleSweepIntervalStr,
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -303,6 +370,7 @@ func Load(path string) (*Config, error) {
|
||||||
// - PROXY_DATABASE_PATH
|
// - PROXY_DATABASE_PATH
|
||||||
// - PROXY_LOG_LEVEL
|
// - PROXY_LOG_LEVEL
|
||||||
// - PROXY_LOG_FORMAT
|
// - PROXY_LOG_FORMAT
|
||||||
|
// - PROXY_HEALTH_STORAGE_PROBE_INTERVAL
|
||||||
func (c *Config) LoadFromEnv() {
|
func (c *Config) LoadFromEnv() {
|
||||||
if v := os.Getenv("PROXY_LISTEN"); v != "" {
|
if v := os.Getenv("PROXY_LISTEN"); v != "" {
|
||||||
c.Listen = v
|
c.Listen = v
|
||||||
|
|
@ -343,6 +411,12 @@ func (c *Config) LoadFromEnv() {
|
||||||
if v := os.Getenv("PROXY_LOG_FORMAT"); v != "" {
|
if v := os.Getenv("PROXY_LOG_FORMAT"); v != "" {
|
||||||
c.Log.Format = v
|
c.Log.Format = v
|
||||||
}
|
}
|
||||||
|
if v := os.Getenv("PROXY_UPSTREAM_MAVEN"); v != "" {
|
||||||
|
c.Upstream.Maven = v
|
||||||
|
}
|
||||||
|
if v := os.Getenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL"); v != "" {
|
||||||
|
c.Upstream.GradlePluginPortal = v
|
||||||
|
}
|
||||||
if v := os.Getenv("PROXY_COOLDOWN_DEFAULT"); v != "" {
|
if v := os.Getenv("PROXY_COOLDOWN_DEFAULT"); v != "" {
|
||||||
c.Cooldown.Default = v
|
c.Cooldown.Default = v
|
||||||
}
|
}
|
||||||
|
|
@ -355,6 +429,27 @@ func (c *Config) LoadFromEnv() {
|
||||||
if v := os.Getenv("PROXY_METADATA_TTL"); v != "" {
|
if v := os.Getenv("PROXY_METADATA_TTL"); v != "" {
|
||||||
c.MetadataTTL = v
|
c.MetadataTTL = v
|
||||||
}
|
}
|
||||||
|
if v := os.Getenv("PROXY_METADATA_MAX_SIZE"); v != "" {
|
||||||
|
c.MetadataMaxSize = v
|
||||||
|
}
|
||||||
|
if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY"); v != "" {
|
||||||
|
c.Gradle.BuildCache.ReadOnly = v == "true" || v == "1"
|
||||||
|
}
|
||||||
|
if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE"); v != "" {
|
||||||
|
c.Gradle.BuildCache.MaxUploadSize = v
|
||||||
|
}
|
||||||
|
if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE"); v != "" {
|
||||||
|
c.Gradle.BuildCache.MaxAge = v
|
||||||
|
}
|
||||||
|
if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_MAX_SIZE"); v != "" {
|
||||||
|
c.Gradle.BuildCache.MaxSize = v
|
||||||
|
}
|
||||||
|
if v := os.Getenv("PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL"); v != "" {
|
||||||
|
c.Gradle.BuildCache.SweepInterval = v
|
||||||
|
}
|
||||||
|
if v := os.Getenv("PROXY_HEALTH_STORAGE_PROBE_INTERVAL"); v != "" {
|
||||||
|
c.Health.StorageProbeInterval = v
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate checks the configuration for errors.
|
// Validate checks the configuration for errors.
|
||||||
|
|
@ -426,12 +521,84 @@ func (c *Config) Validate() error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.Health.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.Gradle.BuildCache.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate checks the /health configuration. An unset interval is allowed
|
||||||
|
// (the cache uses its default); explicit values must parse and be non-negative.
|
||||||
|
func (h *HealthConfig) Validate() error {
|
||||||
|
if h.StorageProbeInterval == "" || h.StorageProbeInterval == "0" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(h.StorageProbeInterval)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid health.storage_probe_interval %q: %w", h.StorageProbeInterval, err)
|
||||||
|
}
|
||||||
|
if d < 0 {
|
||||||
|
return fmt.Errorf("invalid health.storage_probe_interval %q: must be non-negative", h.StorageProbeInterval)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate checks Gradle build cache settings, applying the default upload
|
||||||
|
// size if unset.
|
||||||
|
func (g *GradleBuildCacheConfig) Validate() error {
|
||||||
|
if g.MaxUploadSize == "" {
|
||||||
|
g.MaxUploadSize = defaultGradleMaxUploadSizeStr
|
||||||
|
}
|
||||||
|
uploadSize, err := ParseSize(g.MaxUploadSize)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid gradle.build_cache.max_upload_size: %w", err)
|
||||||
|
}
|
||||||
|
if uploadSize <= 0 {
|
||||||
|
return fmt.Errorf("invalid gradle.build_cache.max_upload_size %q: must be > 0", g.MaxUploadSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
if g.MaxAge != "" && g.MaxAge != "0" {
|
||||||
|
if _, err := time.ParseDuration(g.MaxAge); err != nil {
|
||||||
|
return fmt.Errorf("invalid gradle.build_cache.max_age %q: %w", g.MaxAge, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if g.MaxSize != "" {
|
||||||
|
if _, err := ParseSize(g.MaxSize); err != nil {
|
||||||
|
return fmt.Errorf("invalid gradle.build_cache.max_size: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if g.SweepInterval != "" {
|
||||||
|
d, err := time.ParseDuration(g.SweepInterval)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: %w", g.SweepInterval, err)
|
||||||
|
}
|
||||||
|
if d <= 0 {
|
||||||
|
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: must be > 0", g.SweepInterval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default
|
defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default
|
||||||
defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default
|
defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default
|
||||||
|
defaultMetadataMaxSize = 100 << 20
|
||||||
|
defaultGradleBuildCacheMaxUploadSize = 100 << 20
|
||||||
|
defaultGradleBuildCacheSweepInterval = 10 * time.Minute
|
||||||
|
defaultGradleMaxUploadSizeStr = "100MB"
|
||||||
|
defaultGradleSweepIntervalStr = "10m"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ParseMaxSize returns the maximum cache size in bytes.
|
// ParseMaxSize returns the maximum cache size in bytes.
|
||||||
|
|
@ -447,6 +614,33 @@ func (c *Config) ParseMaxSize() int64 {
|
||||||
return size
|
return size
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateMetadataMaxSize(s string) error {
|
||||||
|
if s == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
size, err := ParseSize(s)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid metadata_max_size: %w", err)
|
||||||
|
}
|
||||||
|
if size <= 0 {
|
||||||
|
return fmt.Errorf("invalid metadata_max_size %q: must be positive", s)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseMetadataMaxSize returns the maximum metadata response size in bytes.
|
||||||
|
// Returns 100MB if unset or invalid.
|
||||||
|
func (c *Config) ParseMetadataMaxSize() int64 {
|
||||||
|
if c.MetadataMaxSize == "" {
|
||||||
|
return defaultMetadataMaxSize
|
||||||
|
}
|
||||||
|
size, err := ParseSize(c.MetadataMaxSize)
|
||||||
|
if err != nil || size <= 0 {
|
||||||
|
return defaultMetadataMaxSize
|
||||||
|
}
|
||||||
|
return size
|
||||||
|
}
|
||||||
|
|
||||||
// ParseMetadataTTL returns the metadata TTL duration.
|
// ParseMetadataTTL returns the metadata TTL duration.
|
||||||
// Returns 5 minutes if unset, 0 if explicitly disabled.
|
// Returns 5 minutes if unset, 0 if explicitly disabled.
|
||||||
func (c *Config) ParseMetadataTTL() time.Duration {
|
func (c *Config) ParseMetadataTTL() time.Duration {
|
||||||
|
|
@ -463,6 +657,58 @@ func (c *Config) ParseMetadataTTL() time.Duration {
|
||||||
return d
|
return d
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParseGradleBuildCacheMaxUploadSize returns the max accepted PUT body size.
|
||||||
|
// Defaults to 100MB if unset or invalid.
|
||||||
|
func (c *Config) ParseGradleBuildCacheMaxUploadSize() int64 {
|
||||||
|
if c.Gradle.BuildCache.MaxUploadSize == "" {
|
||||||
|
return defaultGradleBuildCacheMaxUploadSize
|
||||||
|
}
|
||||||
|
size, err := ParseSize(c.Gradle.BuildCache.MaxUploadSize)
|
||||||
|
if err != nil || size <= 0 {
|
||||||
|
return defaultGradleBuildCacheMaxUploadSize
|
||||||
|
}
|
||||||
|
return size
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseGradleBuildCacheMaxAge returns age-based eviction threshold.
|
||||||
|
// Returns 0 when disabled or invalid.
|
||||||
|
func (c *Config) ParseGradleBuildCacheMaxAge() time.Duration {
|
||||||
|
if c.Gradle.BuildCache.MaxAge == "" || c.Gradle.BuildCache.MaxAge == "0" {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(c.Gradle.BuildCache.MaxAge)
|
||||||
|
if err != nil || d <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseGradleBuildCacheMaxSize returns total-size cap in bytes.
|
||||||
|
// Returns 0 when disabled or invalid.
|
||||||
|
func (c *Config) ParseGradleBuildCacheMaxSize() int64 {
|
||||||
|
if c.Gradle.BuildCache.MaxSize == "" || c.Gradle.BuildCache.MaxSize == "0" {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
size, err := ParseSize(c.Gradle.BuildCache.MaxSize)
|
||||||
|
if err != nil || size <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return size
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseGradleBuildCacheSweepInterval returns eviction sweep cadence.
|
||||||
|
// Defaults to 10m if unset or invalid.
|
||||||
|
func (c *Config) ParseGradleBuildCacheSweepInterval() time.Duration {
|
||||||
|
if c.Gradle.BuildCache.SweepInterval == "" {
|
||||||
|
return defaultGradleBuildCacheSweepInterval
|
||||||
|
}
|
||||||
|
d, err := time.ParseDuration(c.Gradle.BuildCache.SweepInterval)
|
||||||
|
if err != nil || d <= 0 {
|
||||||
|
return defaultGradleBuildCacheSweepInterval
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
// ParseDirectServeTTL returns the presigned URL expiry duration.
|
// ParseDirectServeTTL returns the presigned URL expiry duration.
|
||||||
// Returns 15 minutes if unset.
|
// Returns 15 minutes if unset.
|
||||||
func (c *Config) ParseDirectServeTTL() time.Duration {
|
func (c *Config) ParseDirectServeTTL() time.Duration {
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,18 @@ func TestDefault(t *testing.T) {
|
||||||
if cfg.Database.Path == "" {
|
if cfg.Database.Path == "" {
|
||||||
t.Error("Database.Path should not be empty")
|
t.Error("Database.Path should not be empty")
|
||||||
}
|
}
|
||||||
|
if cfg.Gradle.BuildCache.MaxUploadSize != "100MB" {
|
||||||
|
t.Errorf("Gradle.BuildCache.MaxUploadSize = %q, want %q", cfg.Gradle.BuildCache.MaxUploadSize, "100MB")
|
||||||
|
}
|
||||||
|
if cfg.Gradle.BuildCache.MaxAge != "168h" {
|
||||||
|
t.Errorf("Gradle.BuildCache.MaxAge = %q, want %q", cfg.Gradle.BuildCache.MaxAge, "168h")
|
||||||
|
}
|
||||||
|
if cfg.Upstream.Maven != "https://repo1.maven.org/maven2" {
|
||||||
|
t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://repo1.maven.org/maven2")
|
||||||
|
}
|
||||||
|
if cfg.Upstream.GradlePluginPortal != "https://plugins.gradle.org/m2" {
|
||||||
|
t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.gradle.org/m2")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidate(t *testing.T) {
|
func TestValidate(t *testing.T) {
|
||||||
|
|
@ -98,6 +110,41 @@ func TestValidate(t *testing.T) {
|
||||||
modify: func(c *Config) { c.Storage.MaxSize = "10GB" },
|
modify: func(c *Config) { c.Storage.MaxSize = "10GB" },
|
||||||
wantErr: false,
|
wantErr: false,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "invalid gradle upload size",
|
||||||
|
modify: func(c *Config) { c.Gradle.BuildCache.MaxUploadSize = testInvalid },
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "zero gradle upload size",
|
||||||
|
modify: func(c *Config) { c.Gradle.BuildCache.MaxUploadSize = "0" },
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid gradle max age",
|
||||||
|
modify: func(c *Config) { c.Gradle.BuildCache.MaxAge = testInvalid },
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid gradle max age",
|
||||||
|
modify: func(c *Config) { c.Gradle.BuildCache.MaxAge = "24h" },
|
||||||
|
wantErr: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid gradle max size",
|
||||||
|
modify: func(c *Config) { c.Gradle.BuildCache.MaxSize = testInvalid },
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid gradle sweep interval",
|
||||||
|
modify: func(c *Config) { c.Gradle.BuildCache.SweepInterval = "0" },
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "valid gradle sweep interval",
|
||||||
|
modify: func(c *Config) { c.Gradle.BuildCache.SweepInterval = "30m" },
|
||||||
|
wantErr: false,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|
@ -223,6 +270,13 @@ func TestLoadFromEnv(t *testing.T) {
|
||||||
t.Setenv("PROXY_BASE_URL", "https://env.example.com")
|
t.Setenv("PROXY_BASE_URL", "https://env.example.com")
|
||||||
t.Setenv("PROXY_STORAGE_PATH", "/env/cache")
|
t.Setenv("PROXY_STORAGE_PATH", "/env/cache")
|
||||||
t.Setenv("PROXY_LOG_LEVEL", testLevelDebug)
|
t.Setenv("PROXY_LOG_LEVEL", testLevelDebug)
|
||||||
|
t.Setenv("PROXY_UPSTREAM_MAVEN", "https://maven.example.com/repository/maven-public")
|
||||||
|
t.Setenv("PROXY_UPSTREAM_GRADLE_PLUGIN_PORTAL", "https://plugins.example.com/m2")
|
||||||
|
t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true")
|
||||||
|
t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE", "32MB")
|
||||||
|
t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_AGE", "12h")
|
||||||
|
t.Setenv("PROXY_GRADLE_BUILD_CACHE_MAX_SIZE", "10GB")
|
||||||
|
t.Setenv("PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL", "15m")
|
||||||
|
|
||||||
cfg.LoadFromEnv()
|
cfg.LoadFromEnv()
|
||||||
|
|
||||||
|
|
@ -238,6 +292,27 @@ func TestLoadFromEnv(t *testing.T) {
|
||||||
if cfg.Log.Level != testLevelDebug {
|
if cfg.Log.Level != testLevelDebug {
|
||||||
t.Errorf("Log.Level = %q, want %q", cfg.Log.Level, testLevelDebug)
|
t.Errorf("Log.Level = %q, want %q", cfg.Log.Level, testLevelDebug)
|
||||||
}
|
}
|
||||||
|
if cfg.Upstream.Maven != "https://maven.example.com/repository/maven-public" {
|
||||||
|
t.Errorf("Upstream.Maven = %q, want %q", cfg.Upstream.Maven, "https://maven.example.com/repository/maven-public")
|
||||||
|
}
|
||||||
|
if cfg.Upstream.GradlePluginPortal != "https://plugins.example.com/m2" {
|
||||||
|
t.Errorf("Upstream.GradlePluginPortal = %q, want %q", cfg.Upstream.GradlePluginPortal, "https://plugins.example.com/m2")
|
||||||
|
}
|
||||||
|
if !cfg.Gradle.BuildCache.ReadOnly {
|
||||||
|
t.Error("Gradle.BuildCache.ReadOnly = false, want true")
|
||||||
|
}
|
||||||
|
if cfg.Gradle.BuildCache.MaxUploadSize != "32MB" {
|
||||||
|
t.Errorf("Gradle.BuildCache.MaxUploadSize = %q, want %q", cfg.Gradle.BuildCache.MaxUploadSize, "32MB")
|
||||||
|
}
|
||||||
|
if cfg.Gradle.BuildCache.MaxAge != "12h" {
|
||||||
|
t.Errorf("Gradle.BuildCache.MaxAge = %q, want %q", cfg.Gradle.BuildCache.MaxAge, "12h")
|
||||||
|
}
|
||||||
|
if cfg.Gradle.BuildCache.MaxSize != "10GB" {
|
||||||
|
t.Errorf("Gradle.BuildCache.MaxSize = %q, want %q", cfg.Gradle.BuildCache.MaxSize, "10GB")
|
||||||
|
}
|
||||||
|
if cfg.Gradle.BuildCache.SweepInterval != "15m" {
|
||||||
|
t.Errorf("Gradle.BuildCache.SweepInterval = %q, want %q", cfg.Gradle.BuildCache.SweepInterval, "15m")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoadCooldownConfig(t *testing.T) {
|
func TestLoadCooldownConfig(t *testing.T) {
|
||||||
|
|
@ -353,6 +428,52 @@ func TestParseMetadataTTL(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseMetadataMaxSize(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
size string
|
||||||
|
want int64
|
||||||
|
}{
|
||||||
|
{"unset uses default", "", defaultMetadataMaxSize},
|
||||||
|
{"explicit value", "250MB", 250 << 20},
|
||||||
|
{"bytes", "1024", 1024},
|
||||||
|
{"invalid uses default", "lots", defaultMetadataMaxSize},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
cfg.MetadataMaxSize = tt.size
|
||||||
|
got := cfg.ParseMetadataMaxSize()
|
||||||
|
if got != tt.want {
|
||||||
|
t.Errorf("ParseMetadataMaxSize() = %d, want %d", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateMetadataMaxSize(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
cfg.MetadataMaxSize = "not-a-size"
|
||||||
|
if err := cfg.Validate(); err == nil {
|
||||||
|
t.Error("expected validation error for invalid metadata_max_size")
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.MetadataMaxSize = "0"
|
||||||
|
if err := cfg.Validate(); err == nil {
|
||||||
|
t.Error("expected validation error for zero metadata_max_size")
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.MetadataMaxSize = "250MB"
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Errorf("unexpected error for valid metadata_max_size: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.MetadataMaxSize = ""
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Errorf("unexpected error for unset metadata_max_size: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateMetadataTTL(t *testing.T) {
|
func TestValidateMetadataTTL(t *testing.T) {
|
||||||
cfg := Default()
|
cfg := Default()
|
||||||
cfg.MetadataTTL = "invalid"
|
cfg.MetadataTTL = "invalid"
|
||||||
|
|
@ -371,6 +492,34 @@ func TestValidateMetadataTTL(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateHealthStorageProbeInterval(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
cfg.Health.StorageProbeInterval = "not-a-duration"
|
||||||
|
if err := cfg.Validate(); err == nil {
|
||||||
|
t.Error("expected validation error for invalid health.storage_probe_interval")
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Health.StorageProbeInterval = "30s"
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Errorf("unexpected error for valid health.storage_probe_interval: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Health.StorageProbeInterval = "0"
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Errorf("unexpected error for zero health.storage_probe_interval: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Health.StorageProbeInterval = ""
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Errorf("unexpected error for empty health.storage_probe_interval: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Health.StorageProbeInterval = "-5s"
|
||||||
|
if err := cfg.Validate(); err == nil {
|
||||||
|
t.Error("expected validation error for negative health.storage_probe_interval")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadMetadataTTLFromEnv(t *testing.T) {
|
func TestLoadMetadataTTLFromEnv(t *testing.T) {
|
||||||
cfg := Default()
|
cfg := Default()
|
||||||
t.Setenv("PROXY_METADATA_TTL", "10m")
|
t.Setenv("PROXY_METADATA_TTL", "10m")
|
||||||
|
|
@ -381,6 +530,41 @@ func TestLoadMetadataTTLFromEnv(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseGradleBuildCacheConfig(t *testing.T) {
|
||||||
|
cfg := Default()
|
||||||
|
|
||||||
|
if got := cfg.ParseGradleBuildCacheMaxUploadSize(); got != 100*1024*1024 {
|
||||||
|
t.Errorf("ParseGradleBuildCacheMaxUploadSize() = %d, want %d", got, 100*1024*1024)
|
||||||
|
}
|
||||||
|
if got := cfg.ParseGradleBuildCacheMaxAge(); got != 168*time.Hour {
|
||||||
|
t.Errorf("ParseGradleBuildCacheMaxAge() = %v, want %v", got, 168*time.Hour)
|
||||||
|
}
|
||||||
|
if got := cfg.ParseGradleBuildCacheMaxSize(); got != 0 {
|
||||||
|
t.Errorf("ParseGradleBuildCacheMaxSize() = %d, want 0", got)
|
||||||
|
}
|
||||||
|
if got := cfg.ParseGradleBuildCacheSweepInterval(); got != 10*time.Minute {
|
||||||
|
t.Errorf("ParseGradleBuildCacheSweepInterval() = %v, want %v", got, 10*time.Minute)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Gradle.BuildCache.MaxUploadSize = "64MB"
|
||||||
|
cfg.Gradle.BuildCache.MaxAge = "48h"
|
||||||
|
cfg.Gradle.BuildCache.MaxSize = "2GB"
|
||||||
|
cfg.Gradle.BuildCache.SweepInterval = "20m"
|
||||||
|
|
||||||
|
if got := cfg.ParseGradleBuildCacheMaxUploadSize(); got != 64*1024*1024 {
|
||||||
|
t.Errorf("ParseGradleBuildCacheMaxUploadSize() = %d, want %d", got, 64*1024*1024)
|
||||||
|
}
|
||||||
|
if got := cfg.ParseGradleBuildCacheMaxAge(); got != 48*time.Hour {
|
||||||
|
t.Errorf("ParseGradleBuildCacheMaxAge() = %v, want %v", got, 48*time.Hour)
|
||||||
|
}
|
||||||
|
if got := cfg.ParseGradleBuildCacheMaxSize(); got != 2*1024*1024*1024 {
|
||||||
|
t.Errorf("ParseGradleBuildCacheMaxSize() = %d, want %d", got, 2*1024*1024*1024)
|
||||||
|
}
|
||||||
|
if got := cfg.ParseGradleBuildCacheSweepInterval(); got != 20*time.Minute {
|
||||||
|
t.Errorf("ParseGradleBuildCacheSweepInterval() = %v, want %v", got, 20*time.Minute)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseDirectServeTTL(t *testing.T) {
|
func TestParseDirectServeTTL(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
|
|
|
||||||
|
|
@ -1,125 +0,0 @@
|
||||||
package cooldown
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const hoursPerDay = 24
|
|
||||||
|
|
||||||
// Config holds cooldown settings for version filtering.
|
|
||||||
// Cooldown hides package versions published too recently, giving the community
|
|
||||||
// time to spot malicious releases before they're pulled into projects.
|
|
||||||
type Config struct {
|
|
||||||
// Default is the global default cooldown duration (e.g., "3d", "48h").
|
|
||||||
Default string `json:"default" yaml:"default"`
|
|
||||||
|
|
||||||
// Ecosystems overrides the default for specific ecosystems.
|
|
||||||
// Keys are ecosystem names (e.g., "npm", "pypi").
|
|
||||||
Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"`
|
|
||||||
|
|
||||||
// Packages overrides the cooldown for specific packages.
|
|
||||||
// Keys are PURLs (e.g., "pkg:npm/lodash", "pkg:npm/@babel/core").
|
|
||||||
Packages map[string]string `json:"packages" yaml:"packages"`
|
|
||||||
|
|
||||||
defaultDuration time.Duration
|
|
||||||
ecosystemDurations map[string]time.Duration
|
|
||||||
packageDurations map[string]time.Duration
|
|
||||||
parsed bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// parse resolves all string durations into time.Duration values.
|
|
||||||
// Called lazily on first use.
|
|
||||||
func (c *Config) parse() {
|
|
||||||
if c.parsed {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
c.parsed = true
|
|
||||||
|
|
||||||
c.defaultDuration, _ = ParseDuration(c.Default)
|
|
||||||
|
|
||||||
c.ecosystemDurations = make(map[string]time.Duration, len(c.Ecosystems))
|
|
||||||
for k, v := range c.Ecosystems {
|
|
||||||
d, _ := ParseDuration(v)
|
|
||||||
c.ecosystemDurations[k] = d
|
|
||||||
}
|
|
||||||
|
|
||||||
c.packageDurations = make(map[string]time.Duration, len(c.Packages))
|
|
||||||
for k, v := range c.Packages {
|
|
||||||
d, _ := ParseDuration(v)
|
|
||||||
c.packageDurations[k] = d
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// For returns the effective cooldown duration for a given ecosystem and package PURL.
|
|
||||||
// Resolution order: package override > ecosystem override > global default.
|
|
||||||
func (c *Config) For(ecosystem, packagePURL string) time.Duration {
|
|
||||||
c.parse()
|
|
||||||
|
|
||||||
if d, ok := c.packageDurations[packagePURL]; ok {
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
if d, ok := c.ecosystemDurations[ecosystem]; ok {
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
return c.defaultDuration
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsAllowed returns true if a version with the given publish time has passed
|
|
||||||
// the cooldown period for this ecosystem/package.
|
|
||||||
func (c *Config) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool {
|
|
||||||
d := c.For(ecosystem, packagePURL)
|
|
||||||
if d == 0 {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if publishedAt.IsZero() {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return time.Since(publishedAt) >= d
|
|
||||||
}
|
|
||||||
|
|
||||||
// Enabled returns true if any cooldown is configured.
|
|
||||||
func (c *Config) Enabled() bool {
|
|
||||||
c.parse()
|
|
||||||
if c.defaultDuration > 0 {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
for _, d := range c.ecosystemDurations {
|
|
||||||
if d > 0 {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, d := range c.packageDurations {
|
|
||||||
if d > 0 {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// ParseDuration parses a duration string supporting days (e.g., "3d"),
|
|
||||||
// in addition to Go's standard time.ParseDuration formats ("48h", "30m").
|
|
||||||
// "0" means disabled (returns 0).
|
|
||||||
func ParseDuration(s string) (time.Duration, error) {
|
|
||||||
s = strings.TrimSpace(s)
|
|
||||||
if s == "" || s == "0" {
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle day suffix
|
|
||||||
if numStr, ok := strings.CutSuffix(s, "d"); ok {
|
|
||||||
days, err := strconv.ParseFloat(numStr, 64)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("invalid duration %q: %w", s, err)
|
|
||||||
}
|
|
||||||
return time.Duration(days * float64(hoursPerDay*time.Hour)), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
d, err := time.ParseDuration(s)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("invalid duration %q: %w", s, err)
|
|
||||||
}
|
|
||||||
return d, nil
|
|
||||||
}
|
|
||||||
|
|
@ -1,133 +0,0 @@
|
||||||
package cooldown
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseDuration(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
input string
|
|
||||||
want time.Duration
|
|
||||||
wantErr bool
|
|
||||||
}{
|
|
||||||
{"", 0, false},
|
|
||||||
{"0", 0, false},
|
|
||||||
{"3d", 3 * 24 * time.Hour, false},
|
|
||||||
{"7d", 7 * 24 * time.Hour, false},
|
|
||||||
{"14d", 14 * 24 * time.Hour, false},
|
|
||||||
{"1.5d", 36 * time.Hour, false},
|
|
||||||
{"48h", 48 * time.Hour, false},
|
|
||||||
{"30m", 30 * time.Minute, false},
|
|
||||||
{"1h30m", 90 * time.Minute, false},
|
|
||||||
{"invalid", 0, true},
|
|
||||||
{"d", 0, true},
|
|
||||||
{"xd", 0, true},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
got, err := ParseDuration(tt.input)
|
|
||||||
if (err != nil) != tt.wantErr {
|
|
||||||
t.Errorf("ParseDuration(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if got != tt.want {
|
|
||||||
t.Errorf("ParseDuration(%q) = %v, want %v", tt.input, got, tt.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigFor(t *testing.T) {
|
|
||||||
c := &Config{
|
|
||||||
Default: "3d",
|
|
||||||
Ecosystems: map[string]string{
|
|
||||||
"npm": "7d",
|
|
||||||
"cargo": "0",
|
|
||||||
},
|
|
||||||
Packages: map[string]string{
|
|
||||||
"pkg:npm/lodash": "0",
|
|
||||||
"pkg:npm/@babel/core": "14d",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
ecosystem string
|
|
||||||
packagePURL string
|
|
||||||
want time.Duration
|
|
||||||
}{
|
|
||||||
// Package override takes priority
|
|
||||||
{"npm", "pkg:npm/lodash", 0},
|
|
||||||
{"npm", "pkg:npm/@babel/core", 14 * 24 * time.Hour},
|
|
||||||
// Ecosystem override
|
|
||||||
{"npm", "pkg:npm/express", 7 * 24 * time.Hour},
|
|
||||||
{"cargo", "pkg:cargo/serde", 0},
|
|
||||||
// Global default
|
|
||||||
{"pypi", "pkg:pypi/requests", 3 * 24 * time.Hour},
|
|
||||||
{"pub", "pkg:pub/flutter", 3 * 24 * time.Hour},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
got := c.For(tt.ecosystem, tt.packagePURL)
|
|
||||||
if got != tt.want {
|
|
||||||
t.Errorf("For(%q, %q) = %v, want %v", tt.ecosystem, tt.packagePURL, got, tt.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigIsAllowed(t *testing.T) {
|
|
||||||
c := &Config{
|
|
||||||
Default: "3d",
|
|
||||||
Packages: map[string]string{
|
|
||||||
"pkg:npm/lodash": "0",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
ecosystem string
|
|
||||||
packagePURL string
|
|
||||||
publishedAt time.Time
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{"old enough", "npm", "pkg:npm/express", now.Add(-4 * 24 * time.Hour), true},
|
|
||||||
{"too recent", "npm", "pkg:npm/express", now.Add(-1 * 24 * time.Hour), false},
|
|
||||||
{"exactly at boundary", "npm", "pkg:npm/express", now.Add(-3 * 24 * time.Hour), true},
|
|
||||||
{"exempt package", "npm", "pkg:npm/lodash", now.Add(-1 * time.Minute), true},
|
|
||||||
{"zero time", "npm", "pkg:npm/express", time.Time{}, true},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
got := c.IsAllowed(tt.ecosystem, tt.packagePURL, tt.publishedAt)
|
|
||||||
if got != tt.want {
|
|
||||||
t.Errorf("IsAllowed(%q, %q, %v) = %v, want %v",
|
|
||||||
tt.ecosystem, tt.packagePURL, tt.publishedAt, got, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigEnabled(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
cfg Config
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{"empty config", Config{}, false},
|
|
||||||
{"default only", Config{Default: "3d"}, true},
|
|
||||||
{"ecosystem only", Config{Ecosystems: map[string]string{"npm": "7d"}}, true},
|
|
||||||
{"package only", Config{Packages: map[string]string{"pkg:npm/x": "1d"}}, true},
|
|
||||||
{"all zero", Config{Default: "0", Ecosystems: map[string]string{"npm": "0"}}, false},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
got := tt.cfg.Enabled()
|
|
||||||
if got != tt.want {
|
|
||||||
t.Errorf("Enabled() = %v, want %v", got, tt.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -6,7 +6,11 @@ import (
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
const postgresTimestamp = "TIMESTAMP"
|
const (
|
||||||
|
postgresTimestamp = "TIMESTAMP"
|
||||||
|
sqliteDatetime = "DATETIME"
|
||||||
|
colTypeText = "TEXT"
|
||||||
|
)
|
||||||
|
|
||||||
// Schema for proxy-specific tables. The packages and versions tables
|
// Schema for proxy-specific tables. The packages and versions tables
|
||||||
// are compatible with git-pkgs, allowing the proxy to use an existing
|
// are compatible with git-pkgs, allowing the proxy to use an existing
|
||||||
|
|
@ -369,9 +373,9 @@ func isTableNotFound(err error) bool {
|
||||||
func (db *DB) createMigrationsTable() error {
|
func (db *DB) createMigrationsTable() error {
|
||||||
var ts string
|
var ts string
|
||||||
if db.dialect == DialectPostgres {
|
if db.dialect == DialectPostgres {
|
||||||
ts = "TIMESTAMP"
|
ts = postgresTimestamp
|
||||||
} else {
|
} else {
|
||||||
ts = "DATETIME"
|
ts = sqliteDatetime
|
||||||
}
|
}
|
||||||
|
|
||||||
query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS migrations (
|
query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS migrations (
|
||||||
|
|
@ -457,12 +461,12 @@ func (db *DB) MigrateSchema() error {
|
||||||
|
|
||||||
func migrateAddPackagesEnrichmentColumns(db *DB) error {
|
func migrateAddPackagesEnrichmentColumns(db *DB) error {
|
||||||
columns := map[string]string{
|
columns := map[string]string{
|
||||||
"registry_url": "TEXT",
|
"registry_url": colTypeText,
|
||||||
"supplier_name": "TEXT",
|
"supplier_name": colTypeText,
|
||||||
"supplier_type": "TEXT",
|
"supplier_type": colTypeText,
|
||||||
"source": "TEXT",
|
"source": colTypeText,
|
||||||
"enriched_at": "DATETIME",
|
"enriched_at": sqliteDatetime,
|
||||||
"vulns_synced_at": "DATETIME",
|
"vulns_synced_at": sqliteDatetime,
|
||||||
}
|
}
|
||||||
|
|
||||||
if db.dialect == DialectPostgres {
|
if db.dialect == DialectPostgres {
|
||||||
|
|
@ -487,10 +491,10 @@ func migrateAddPackagesEnrichmentColumns(db *DB) error {
|
||||||
|
|
||||||
func migrateAddVersionsEnrichmentColumns(db *DB) error {
|
func migrateAddVersionsEnrichmentColumns(db *DB) error {
|
||||||
columns := map[string]string{
|
columns := map[string]string{
|
||||||
"integrity": "TEXT",
|
"integrity": colTypeText,
|
||||||
"yanked": "INTEGER DEFAULT 0",
|
"yanked": "INTEGER DEFAULT 0",
|
||||||
"source": "TEXT",
|
"source": colTypeText,
|
||||||
"enriched_at": "DATETIME",
|
"enriched_at": sqliteDatetime,
|
||||||
}
|
}
|
||||||
|
|
||||||
if db.dialect == DialectPostgres {
|
if db.dialect == DialectPostgres {
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
)
|
)
|
||||||
|
|
||||||
func cargoTestProxy() *Proxy {
|
func cargoTestProxy() *Proxy {
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,7 @@ import (
|
||||||
const (
|
const (
|
||||||
composerUpstream = "https://packagist.org"
|
composerUpstream = "https://packagist.org"
|
||||||
composerRepo = "https://repo.packagist.org"
|
composerRepo = "https://repo.packagist.org"
|
||||||
|
composerUnset = "__unset"
|
||||||
vendorPackageParts = 2
|
vendorPackageParts = 2
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -150,7 +151,8 @@ func (h *ComposerHandler) rewriteMetadata(body []byte) ([]byte, error) {
|
||||||
|
|
||||||
// expandMinifiedVersions expands the Composer v2 minified format where each
|
// expandMinifiedVersions expands the Composer v2 minified format where each
|
||||||
// version entry only contains fields that differ from the previous entry.
|
// version entry only contains fields that differ from the previous entry.
|
||||||
// The "~dev" sentinel string resets the inheritance chain.
|
// The "~dev" sentinel string resets the inheritance chain, and the "__unset"
|
||||||
|
// value removes a field from the inherited state.
|
||||||
func expandMinifiedVersions(versionList []any) []any {
|
func expandMinifiedVersions(versionList []any) []any {
|
||||||
expanded := make([]any, 0, len(versionList))
|
expanded := make([]any, 0, len(versionList))
|
||||||
inherited := map[string]any{}
|
inherited := map[string]any{}
|
||||||
|
|
@ -174,6 +176,10 @@ func expandMinifiedVersions(versionList []any) []any {
|
||||||
merged[k] = deepCopyValue(val)
|
merged[k] = deepCopyValue(val)
|
||||||
}
|
}
|
||||||
for k, val := range vmap {
|
for k, val := range vmap {
|
||||||
|
if val == composerUnset {
|
||||||
|
delete(merged, k)
|
||||||
|
continue
|
||||||
|
}
|
||||||
merged[k] = val
|
merged[k] = val
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestComposerRewriteMetadata(t *testing.T) {
|
func TestComposerRewriteMetadata(t *testing.T) {
|
||||||
|
|
@ -177,6 +177,80 @@ func TestComposerRewriteMetadataMinifiedDevReset(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestComposerRewriteMetadataUnset(t *testing.T) {
|
||||||
|
h := &ComposerHandler{
|
||||||
|
proxy: &Proxy{Logger: slog.Default()},
|
||||||
|
proxyURL: "http://localhost:8080",
|
||||||
|
}
|
||||||
|
|
||||||
|
// In the minified format, "__unset" removes a field from the inherited
|
||||||
|
// state. v1.29.0 has require-dev, v1.28.0 unsets it, v1.27.0 inherits the
|
||||||
|
// unset state. Composer rejects metadata where require-dev (or any link
|
||||||
|
// field) is the literal string "__unset" rather than an object.
|
||||||
|
input := `{
|
||||||
|
"minified": "composer/2.0",
|
||||||
|
"packages": {
|
||||||
|
"venturecraft/revisionable": [
|
||||||
|
{
|
||||||
|
"name": "venturecraft/revisionable",
|
||||||
|
"version": "1.29.0",
|
||||||
|
"require": {"php": ">=5.4"},
|
||||||
|
"require-dev": {"orchestra/testbench": "~3.0"},
|
||||||
|
"dist": {"url": "https://example.com/a.zip", "type": "zip"}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "1.28.0",
|
||||||
|
"require-dev": "__unset"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "1.27.0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"version": "1.26.0",
|
||||||
|
"require-dev": {"foo/bar": "1.0"}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}`
|
||||||
|
|
||||||
|
output, err := h.rewriteMetadata([]byte(input))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("rewriteMetadata failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var result map[string]any
|
||||||
|
if err := json.Unmarshal(output, &result); err != nil {
|
||||||
|
t.Fatalf("failed to parse output: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
versions := result["packages"].(map[string]any)["venturecraft/revisionable"].([]any)
|
||||||
|
if len(versions) != 4 {
|
||||||
|
t.Fatalf("expected 4 versions, got %d", len(versions))
|
||||||
|
}
|
||||||
|
|
||||||
|
byVersion := map[string]map[string]any{}
|
||||||
|
for _, v := range versions {
|
||||||
|
vmap := v.(map[string]any)
|
||||||
|
byVersion[vmap["version"].(string)] = vmap
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := byVersion["1.29.0"]["require-dev"].(map[string]any); !ok {
|
||||||
|
t.Errorf("1.29.0 require-dev should be an object, got %T", byVersion["1.29.0"]["require-dev"])
|
||||||
|
}
|
||||||
|
if rd, ok := byVersion["1.28.0"]["require-dev"]; ok {
|
||||||
|
t.Errorf("1.28.0 require-dev should be absent, got %v", rd)
|
||||||
|
}
|
||||||
|
if rd, ok := byVersion["1.27.0"]["require-dev"]; ok {
|
||||||
|
t.Errorf("1.27.0 require-dev should be absent (inherited unset), got %v", rd)
|
||||||
|
}
|
||||||
|
if _, ok := byVersion["1.26.0"]["require-dev"].(map[string]any); !ok {
|
||||||
|
t.Errorf("1.26.0 require-dev should be an object, got %T", byVersion["1.26.0"]["require-dev"])
|
||||||
|
}
|
||||||
|
if _, ok := byVersion["1.27.0"]["require"].(map[string]any); !ok {
|
||||||
|
t.Error("1.27.0 should still inherit require from 1.29.0")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) {
|
func TestComposerRewriteMetadataCooldownPreservesNames(t *testing.T) {
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339)
|
old := now.Add(-10 * 24 * time.Hour).Format(time.RFC3339)
|
||||||
|
|
|
||||||
|
|
@ -140,7 +140,7 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) {
|
||||||
http.Error(w, "failed to create request", http.StatusInternalServerError)
|
http.Error(w, "failed to create request", http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
req.Header.Set("Accept-Encoding", "gzip")
|
req.Header.Set(headerAcceptEncoding, "gzip")
|
||||||
|
|
||||||
resp, err := h.proxy.HTTPClient.Do(req)
|
resp, err := h.proxy.HTTPClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -161,7 +161,7 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := ReadMetadata(resp.Body)
|
body, err := h.proxy.ReadMetadata(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "failed to read response", http.StatusInternalServerError)
|
http.Error(w, "failed to read response", http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
|
|
@ -241,5 +241,5 @@ func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
// proxyUpstream forwards a request to Anaconda without caching.
|
// proxyUpstream forwards a request to Anaconda without caching.
|
||||||
func (h *CondaHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
func (h *CondaHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
||||||
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{"Accept-Encoding"})
|
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{headerAcceptEncoding})
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCondaParseFilename(t *testing.T) {
|
func TestCondaParseFilename(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -159,5 +159,5 @@ func (h *CRANHandler) proxyCached(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
// proxyUpstream forwards a request to CRAN without caching.
|
// proxyUpstream forwards a request to CRAN without caching.
|
||||||
func (h *CRANHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
func (h *CRANHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
||||||
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{"Accept-Encoding"})
|
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{headerAcceptEncoding})
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -673,7 +673,7 @@ func TestMavenHandler_DownloadCacheHit(t *testing.T) {
|
||||||
proxy, db, store, _ := setupTestProxy(t)
|
proxy, db, store, _ := setupTestProxy(t)
|
||||||
seedPackageWithPURL(t, db, store, "maven", "com.google.guava:guava", "32.1.3-jre", "guava-32.1.3-jre.jar", "jar content")
|
seedPackageWithPURL(t, db, store, "maven", "com.google.guava:guava", "32.1.3-jre", "guava-32.1.3-jre.jar", "jar content")
|
||||||
|
|
||||||
h := NewMavenHandler(proxy, "http://localhost")
|
h := NewMavenHandler(proxy, "http://localhost", "", "")
|
||||||
srv := httptest.NewServer(h.Routes())
|
srv := httptest.NewServer(h.Routes())
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
|
|
@ -730,7 +730,7 @@ func TestMavenHandler_MetadataProxied(t *testing.T) {
|
||||||
|
|
||||||
func TestMavenHandler_EmptyPathNotFound(t *testing.T) {
|
func TestMavenHandler_EmptyPathNotFound(t *testing.T) {
|
||||||
proxy, _, _, _ := setupTestProxy(t)
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
h := NewMavenHandler(proxy, "http://localhost")
|
h := NewMavenHandler(proxy, "http://localhost", "", "")
|
||||||
srv := httptest.NewServer(h.Routes())
|
srv := httptest.NewServer(h.Routes())
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
|
|
@ -748,7 +748,7 @@ func TestMavenHandler_EmptyPathNotFound(t *testing.T) {
|
||||||
func TestMavenHandler_ArtifactExtensions(t *testing.T) {
|
func TestMavenHandler_ArtifactExtensions(t *testing.T) {
|
||||||
proxy, _, _, fetcher := setupTestProxy(t)
|
proxy, _, _, fetcher := setupTestProxy(t)
|
||||||
|
|
||||||
extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib"}
|
extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module"}
|
||||||
for _, ext := range extensions {
|
for _, ext := range extensions {
|
||||||
fetcher.artifact = &fetch.Artifact{
|
fetcher.artifact = &fetch.Artifact{
|
||||||
Body: io.NopCloser(strings.NewReader("artifact")),
|
Body: io.NopCloser(strings.NewReader("artifact")),
|
||||||
|
|
@ -756,7 +756,7 @@ func TestMavenHandler_ArtifactExtensions(t *testing.T) {
|
||||||
}
|
}
|
||||||
fetcher.fetchCalled = false
|
fetcher.fetchCalled = false
|
||||||
|
|
||||||
h := NewMavenHandler(proxy, "http://localhost")
|
h := NewMavenHandler(proxy, "http://localhost", "", "")
|
||||||
|
|
||||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
t.Errorf("should not proxy artifact file %s to upstream", ext)
|
t.Errorf("should not proxy artifact file %s to upstream", ext)
|
||||||
|
|
@ -789,7 +789,7 @@ func TestMavenHandler_CacheMiss(t *testing.T) {
|
||||||
ContentType: "application/java-archive",
|
ContentType: "application/java-archive",
|
||||||
}
|
}
|
||||||
|
|
||||||
h := NewMavenHandler(proxy, "http://localhost")
|
h := NewMavenHandler(proxy, "http://localhost", "", "")
|
||||||
srv := httptest.NewServer(h.Routes())
|
srv := httptest.NewServer(h.Routes())
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
|
|
@ -809,6 +809,274 @@ func TestMavenHandler_CacheMiss(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMavenHandler_GradlePluginMarkerFallbackAndCache(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
markerPath string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Spotless",
|
||||||
|
markerPath: "/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "BenManes",
|
||||||
|
markerPath: "/com/github/ben-manes/versions/com.github.ben-manes.versions.gradle.plugin/0.54.0/com.github.ben-manes.versions.gradle.plugin-0.54.0.pom",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
proxy, _, _, fetcher := setupTestProxy(t)
|
||||||
|
|
||||||
|
primaryUpstream := "https://repo1.maven.org/maven2"
|
||||||
|
pluginPortalUpstream := "https://plugins.gradle.org/m2"
|
||||||
|
primaryURL := primaryUpstream + tt.markerPath
|
||||||
|
|
||||||
|
fetcher.fetchErrByURL = map[string]error{
|
||||||
|
primaryURL: ErrUpstreamNotFound,
|
||||||
|
}
|
||||||
|
fetcher.artifact = &fetch.Artifact{
|
||||||
|
Body: io.NopCloser(strings.NewReader("<project/>")),
|
||||||
|
ContentType: "application/xml",
|
||||||
|
}
|
||||||
|
|
||||||
|
h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
resp, err := http.Get(srv.URL + tt.markerPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("request failed: %v", err)
|
||||||
|
}
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
|
||||||
|
}
|
||||||
|
if string(body) != "<project/>" {
|
||||||
|
t.Fatalf("body = %q, want %q", body, "<project/>")
|
||||||
|
}
|
||||||
|
|
||||||
|
wantFallbackURL := pluginPortalUpstream + tt.markerPath
|
||||||
|
if fetcher.fetchedURL != wantFallbackURL {
|
||||||
|
t.Fatalf("fallback URL = %q, want %q", fetcher.fetchedURL, wantFallbackURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
fetcher.fetchCalled = false
|
||||||
|
resp, err = http.Get(srv.URL + tt.markerPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second request failed: %v", err)
|
||||||
|
}
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("second status = %d, want %d", resp.StatusCode, http.StatusOK)
|
||||||
|
}
|
||||||
|
if fetcher.fetchCalled {
|
||||||
|
t.Fatal("expected plugin marker POM to be served from cache on second request")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMavenHandler_GradlePluginMarkerMetadataFallback(t *testing.T) {
|
||||||
|
paths := map[string]string{
|
||||||
|
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1": "sha1",
|
||||||
|
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha256": "sha256",
|
||||||
|
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/8.4.0/com.diffplug.spotless.gradle.plugin-8.4.0.pom.md5": "md5",
|
||||||
|
"/com/diffplug/spotless/com.diffplug.spotless.gradle.plugin/maven-metadata.xml": "<metadata/>",
|
||||||
|
}
|
||||||
|
|
||||||
|
primaryHits := map[string]int{}
|
||||||
|
pluginHits := map[string]int{}
|
||||||
|
|
||||||
|
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
primaryHits[r.URL.Path]++
|
||||||
|
if _, ok := paths[r.URL.Path]; ok {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.Fatalf("unexpected path to primary upstream: %s", r.URL.Path)
|
||||||
|
}))
|
||||||
|
defer primary.Close()
|
||||||
|
|
||||||
|
pluginPortal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
pluginHits[r.URL.Path]++
|
||||||
|
body, ok := paths[r.URL.Path]
|
||||||
|
if !ok {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = io.WriteString(w, body)
|
||||||
|
}))
|
||||||
|
defer pluginPortal.Close()
|
||||||
|
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
proxy.HTTPClient = primary.Client()
|
||||||
|
|
||||||
|
h := NewMavenHandler(proxy, "http://localhost", primary.URL, pluginPortal.URL)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
for reqPath, wantBody := range paths {
|
||||||
|
resp, err := http.Get(srv.URL + reqPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET %s failed: %v", reqPath, err)
|
||||||
|
}
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("GET %s: status = %d, want %d", reqPath, resp.StatusCode, http.StatusOK)
|
||||||
|
}
|
||||||
|
if string(body) != wantBody {
|
||||||
|
t.Fatalf("GET %s: body = %q, want %q", reqPath, body, wantBody)
|
||||||
|
}
|
||||||
|
|
||||||
|
if primaryHits[reqPath] == 0 {
|
||||||
|
t.Fatalf("GET %s did not hit primary upstream", reqPath)
|
||||||
|
}
|
||||||
|
if pluginHits[reqPath] == 0 {
|
||||||
|
t.Fatalf("GET %s did not hit plugin portal fallback", reqPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMavenHandler_GradlePluginImplementationMetadataFallback(t *testing.T) {
|
||||||
|
paths := map[string]string{
|
||||||
|
"/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha1": "impl-sha1",
|
||||||
|
"/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar.sha256": "impl-sha256",
|
||||||
|
}
|
||||||
|
|
||||||
|
primaryHits := map[string]int{}
|
||||||
|
pluginHits := map[string]int{}
|
||||||
|
|
||||||
|
primary := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
primaryHits[r.URL.Path]++
|
||||||
|
if _, ok := paths[r.URL.Path]; ok {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.Fatalf("unexpected path to primary upstream: %s", r.URL.Path)
|
||||||
|
}))
|
||||||
|
defer primary.Close()
|
||||||
|
|
||||||
|
pluginPortal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
pluginHits[r.URL.Path]++
|
||||||
|
body, ok := paths[r.URL.Path]
|
||||||
|
if !ok {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = io.WriteString(w, body)
|
||||||
|
}))
|
||||||
|
defer pluginPortal.Close()
|
||||||
|
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
proxy.HTTPClient = primary.Client()
|
||||||
|
|
||||||
|
h := NewMavenHandler(proxy, "http://localhost", primary.URL, pluginPortal.URL)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
for reqPath, wantBody := range paths {
|
||||||
|
resp, err := http.Get(srv.URL + reqPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET %s failed: %v", reqPath, err)
|
||||||
|
}
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("GET %s: status = %d, want %d", reqPath, resp.StatusCode, http.StatusOK)
|
||||||
|
}
|
||||||
|
if string(body) != wantBody {
|
||||||
|
t.Fatalf("GET %s: body = %q, want %q", reqPath, body, wantBody)
|
||||||
|
}
|
||||||
|
|
||||||
|
if primaryHits[reqPath] == 0 {
|
||||||
|
t.Fatalf("GET %s did not hit primary upstream", reqPath)
|
||||||
|
}
|
||||||
|
if pluginHits[reqPath] == 0 {
|
||||||
|
t.Fatalf("GET %s did not hit plugin portal fallback", reqPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMavenHandler_GradlePluginImplementation_FallbackToPluginPortal(t *testing.T) {
|
||||||
|
proxy, _, _, fetcher := setupTestProxy(t)
|
||||||
|
|
||||||
|
primaryUpstream := "https://repo1.maven.org/maven2"
|
||||||
|
pluginPortalUpstream := "https://plugins.gradle.org/m2"
|
||||||
|
implPath := "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar"
|
||||||
|
primaryURL := primaryUpstream + implPath
|
||||||
|
pluginPortalURL := pluginPortalUpstream + implPath
|
||||||
|
|
||||||
|
fetcher.fetchErrByURL = map[string]error{
|
||||||
|
primaryURL: ErrUpstreamNotFound,
|
||||||
|
}
|
||||||
|
fetcher.artifact = &fetch.Artifact{
|
||||||
|
Body: io.NopCloser(strings.NewReader("plugin impl jar")),
|
||||||
|
ContentType: "application/java-archive",
|
||||||
|
}
|
||||||
|
|
||||||
|
h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
resp, err := http.Get(srv.URL + implPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("request failed: %v", err)
|
||||||
|
}
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK)
|
||||||
|
}
|
||||||
|
if string(body) != "plugin impl jar" {
|
||||||
|
t.Fatalf("body = %q, want %q", body, "plugin impl jar")
|
||||||
|
}
|
||||||
|
|
||||||
|
if fetcher.fetchedURL != pluginPortalURL {
|
||||||
|
t.Fatalf("implementation artifact should fallback to plugin portal; fetched URL = %q, want %q", fetcher.fetchedURL, pluginPortalURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMavenHandler_GradlePluginImplementation_NotFoundInBothUpstreams(t *testing.T) {
|
||||||
|
proxy, _, _, fetcher := setupTestProxy(t)
|
||||||
|
|
||||||
|
primaryUpstream := "https://repo1.maven.org/maven2"
|
||||||
|
pluginPortalUpstream := "https://plugins.gradle.org/m2"
|
||||||
|
implPath := "/com/diffplug/spotless/spotless-plugin-gradle/8.4.0/spotless-plugin-gradle-8.4.0.jar"
|
||||||
|
primaryURL := primaryUpstream + implPath
|
||||||
|
pluginPortalURL := pluginPortalUpstream + implPath
|
||||||
|
|
||||||
|
fetcher.fetchErrByURL = map[string]error{
|
||||||
|
primaryURL: ErrUpstreamNotFound,
|
||||||
|
pluginPortalURL: ErrUpstreamNotFound,
|
||||||
|
}
|
||||||
|
|
||||||
|
h := NewMavenHandler(proxy, "http://localhost", primaryUpstream, pluginPortalUpstream)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
resp, err := http.Get(srv.URL + implPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("request failed: %v", err)
|
||||||
|
}
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusNotFound {
|
||||||
|
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fetcher.fetchedURL != pluginPortalURL {
|
||||||
|
t.Fatalf("expected fallback attempt to plugin portal; fetched URL = %q, want %q", fetcher.fetchedURL, pluginPortalURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestNuGetHandler_DownloadCacheMiss(t *testing.T) {
|
func TestNuGetHandler_DownloadCacheMiss(t *testing.T) {
|
||||||
proxy, _, _, fetcher := setupTestProxy(t)
|
proxy, _, _, fetcher := setupTestProxy(t)
|
||||||
fetcher.artifact = &fetch.Artifact{
|
fetcher.artifact = &fetch.Artifact{
|
||||||
|
|
|
||||||
|
|
@ -182,7 +182,7 @@ func (h *GemHandler) fetchCompactIndex(r *http.Request, name string) (*http.Resp
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
for _, hdr := range []string{"Accept", "Accept-Encoding", "If-None-Match", "If-Modified-Since"} {
|
for _, hdr := range []string{"Accept", headerAcceptEncoding, "If-None-Match", "If-Modified-Since"} {
|
||||||
if v := r.Header.Get(hdr); v != "" {
|
if v := r.Header.Get(hdr); v != "" {
|
||||||
req.Header.Set(hdr, v)
|
req.Header.Set(hdr, v)
|
||||||
}
|
}
|
||||||
|
|
@ -311,7 +311,7 @@ func (h *GemHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy relevant headers
|
// Copy relevant headers
|
||||||
for _, h := range []string{"Accept", "Accept-Encoding", "If-None-Match", "If-Modified-Since"} {
|
for _, h := range []string{"Accept", headerAcceptEncoding, "If-None-Match", "If-Modified-Since"} {
|
||||||
if v := r.Header.Get(h); v != "" {
|
if v := r.Header.Get(h); v != "" {
|
||||||
req.Header.Set(h, v)
|
req.Header.Set(h, v)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGemParseFilename(t *testing.T) {
|
func TestGemParseFilename(t *testing.T) {
|
||||||
|
|
|
||||||
178
internal/handler/gradle.go
Normal file
178
internal/handler/gradle.go
Normal file
|
|
@ -0,0 +1,178 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/git-pkgs/proxy/internal/metrics"
|
||||||
|
"github.com/git-pkgs/proxy/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
gradleBuildCacheContentType = "application/vnd.gradle.build-cache-artifact.v2"
|
||||||
|
gradleBuildCacheStorageRoot = "_gradle/http-build-cache"
|
||||||
|
defaultGradleMaxUploadSize = 100 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
var gradleBuildCacheKeyPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||||
|
|
||||||
|
// GradleBuildCacheHandler handles Gradle HttpBuildCache GET/HEAD/PUT requests.
|
||||||
|
//
|
||||||
|
// This handler accepts /{key} when mounted under a base URL.
|
||||||
|
type GradleBuildCacheHandler struct {
|
||||||
|
proxy *Proxy
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGradleBuildCacheHandler creates a Gradle HttpBuildCache handler.
|
||||||
|
func NewGradleBuildCacheHandler(proxy *Proxy) *GradleBuildCacheHandler {
|
||||||
|
return &GradleBuildCacheHandler{proxy: proxy}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Routes returns the HTTP handler for Gradle HttpBuildCache requests.
|
||||||
|
func (h *GradleBuildCacheHandler) Routes() http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.Method {
|
||||||
|
case http.MethodGet, http.MethodHead, http.MethodPut:
|
||||||
|
default:
|
||||||
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
key, statusCode := h.parseCacheKey(r.URL.Path)
|
||||||
|
if statusCode != http.StatusOK {
|
||||||
|
if statusCode == http.StatusNotFound {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
http.Error(w, "invalid cache key", statusCode)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.Method == http.MethodPut {
|
||||||
|
if h.proxy.GradleReadOnly {
|
||||||
|
http.Error(w, "gradle build cache is read-only", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.handlePut(w, r, key)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
h.handleGetOrHead(w, r, key)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *GradleBuildCacheHandler) parseCacheKey(urlPath string) (string, int) {
|
||||||
|
keyPath := strings.TrimPrefix(urlPath, "/")
|
||||||
|
if keyPath == "" {
|
||||||
|
return "", http.StatusNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
if containsPathTraversal(keyPath) {
|
||||||
|
return "", http.StatusBadRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(keyPath, "/") {
|
||||||
|
return "", http.StatusNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
if !gradleBuildCacheKeyPattern.MatchString(keyPath) {
|
||||||
|
return "", http.StatusBadRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
return keyPath, http.StatusOK
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *GradleBuildCacheHandler) cacheStoragePath(key string) string {
|
||||||
|
return gradleBuildCacheStorageRoot + "/" + key
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *GradleBuildCacheHandler) handleGetOrHead(w http.ResponseWriter, r *http.Request, key string) {
|
||||||
|
storagePath := h.cacheStoragePath(key)
|
||||||
|
w.Header().Set("Content-Type", gradleBuildCacheContentType)
|
||||||
|
|
||||||
|
if r.Method == http.MethodHead {
|
||||||
|
existsStart := time.Now()
|
||||||
|
exists, err := h.proxy.Storage.Exists(r.Context(), storagePath)
|
||||||
|
metrics.RecordStorageOperation("read", time.Since(existsStart))
|
||||||
|
if err != nil {
|
||||||
|
metrics.RecordStorageError("read")
|
||||||
|
h.proxy.Logger.Error("failed to check gradle build cache entry", "key", key, "error", err)
|
||||||
|
http.Error(w, "failed to read cache entry", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
metrics.RecordCacheMiss("gradle")
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
metrics.RecordCacheHit("gradle")
|
||||||
|
|
||||||
|
sizeStart := time.Now()
|
||||||
|
size, err := h.proxy.Storage.Size(r.Context(), storagePath)
|
||||||
|
metrics.RecordStorageOperation("read", time.Since(sizeStart))
|
||||||
|
if err != nil {
|
||||||
|
metrics.RecordStorageError("read")
|
||||||
|
} else if size >= 0 {
|
||||||
|
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
|
||||||
|
}
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
readStart := time.Now()
|
||||||
|
reader, err := h.proxy.Storage.Open(r.Context(), storagePath)
|
||||||
|
metrics.RecordStorageOperation("read", time.Since(readStart))
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, storage.ErrNotFound) {
|
||||||
|
metrics.RecordCacheMiss("gradle")
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
metrics.RecordStorageError("read")
|
||||||
|
h.proxy.Logger.Error("failed to open gradle build cache entry", "key", key, "error", err)
|
||||||
|
http.Error(w, "failed to read cache entry", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer func() { _ = reader.Close() }()
|
||||||
|
metrics.RecordCacheHit("gradle")
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
_, _ = io.Copy(w, reader)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *GradleBuildCacheHandler) handlePut(w http.ResponseWriter, r *http.Request, key string) {
|
||||||
|
storagePath := h.cacheStoragePath(key)
|
||||||
|
maxUploadSize := h.proxy.GradleMaxUploadSize
|
||||||
|
if maxUploadSize <= 0 {
|
||||||
|
maxUploadSize = defaultGradleMaxUploadSize
|
||||||
|
}
|
||||||
|
|
||||||
|
r.Body = http.MaxBytesReader(w, r.Body, maxUploadSize)
|
||||||
|
|
||||||
|
storeStart := time.Now()
|
||||||
|
_, hash, err := h.proxy.Storage.Store(r.Context(), storagePath, r.Body)
|
||||||
|
metrics.RecordStorageOperation("write", time.Since(storeStart))
|
||||||
|
if err != nil {
|
||||||
|
var maxBytesErr *http.MaxBytesError
|
||||||
|
if errors.As(err, &maxBytesErr) {
|
||||||
|
http.Error(w, "cache entry too large", http.StatusRequestEntityTooLarge)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
metrics.RecordStorageError("write")
|
||||||
|
h.proxy.Logger.Error("failed to store gradle build cache entry", "key", key, "error", err)
|
||||||
|
http.Error(w, "failed to write cache entry", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Length", "0")
|
||||||
|
w.Header().Set("ETag", `"`+hash+`"`)
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
}
|
||||||
285
internal/handler/gradle_test.go
Normal file
285
internal/handler/gradle_test.go
Normal file
|
|
@ -0,0 +1,285 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/git-pkgs/proxy/internal/metrics"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGradleBuildCacheHandler_PutGetHead(t *testing.T) {
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
h := NewGradleBuildCacheHandler(proxy)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
key := "a1b2c3d4e5f6"
|
||||||
|
payload := "cache entry content"
|
||||||
|
|
||||||
|
putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader(payload))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create PUT request: %v", err)
|
||||||
|
}
|
||||||
|
putResp, err := http.DefaultClient.Do(putReq)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PUT request failed: %v", err)
|
||||||
|
}
|
||||||
|
_ = putResp.Body.Close()
|
||||||
|
|
||||||
|
if putResp.StatusCode != http.StatusCreated {
|
||||||
|
t.Fatalf("PUT status = %d, want %d", putResp.StatusCode, http.StatusCreated)
|
||||||
|
}
|
||||||
|
|
||||||
|
getResp, err := http.Get(srv.URL + "/" + key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET request failed: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = getResp.Body.Close() }()
|
||||||
|
|
||||||
|
if getResp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("GET status = %d, want %d", getResp.StatusCode, http.StatusOK)
|
||||||
|
}
|
||||||
|
if getResp.Header.Get("Content-Type") != gradleBuildCacheContentType {
|
||||||
|
t.Fatalf("GET Content-Type = %q, want %q", getResp.Header.Get("Content-Type"), gradleBuildCacheContentType)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, _ := io.ReadAll(getResp.Body)
|
||||||
|
if string(body) != payload {
|
||||||
|
t.Fatalf("GET body = %q, want %q", body, payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
headReq, err := http.NewRequest(http.MethodHead, srv.URL+"/"+key, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create HEAD request: %v", err)
|
||||||
|
}
|
||||||
|
headResp, err := http.DefaultClient.Do(headReq)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("HEAD request failed: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = headResp.Body.Close() }()
|
||||||
|
|
||||||
|
if headResp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("HEAD status = %d, want %d", headResp.StatusCode, http.StatusOK)
|
||||||
|
}
|
||||||
|
body, _ = io.ReadAll(headResp.Body)
|
||||||
|
if len(body) != 0 {
|
||||||
|
t.Fatalf("HEAD body length = %d, want 0", len(body))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGradleBuildCacheHandler_RootKeyPath(t *testing.T) {
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
h := NewGradleBuildCacheHandler(proxy)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
key := "rootpathkey"
|
||||||
|
putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader("root"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create PUT request: %v", err)
|
||||||
|
}
|
||||||
|
putResp, err := http.DefaultClient.Do(putReq)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PUT request failed: %v", err)
|
||||||
|
}
|
||||||
|
_ = putResp.Body.Close()
|
||||||
|
|
||||||
|
if putResp.StatusCode != http.StatusCreated {
|
||||||
|
t.Fatalf("PUT status = %d, want %d", putResp.StatusCode, http.StatusCreated)
|
||||||
|
}
|
||||||
|
|
||||||
|
getResp, err := http.Get(srv.URL + "/" + key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET request failed: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = getResp.Body.Close() }()
|
||||||
|
|
||||||
|
if getResp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("GET status = %d, want %d", getResp.StatusCode, http.StatusOK)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGradleBuildCacheHandler_GetMiss(t *testing.T) {
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
h := NewGradleBuildCacheHandler(proxy)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
resp, err := http.Get(srv.URL + "/missing-key")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET request failed: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusNotFound {
|
||||||
|
t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGradleBuildCacheHandler_MethodNotAllowed(t *testing.T) {
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
h := NewGradleBuildCacheHandler(proxy)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/key", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.Routes().ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusMethodNotAllowed {
|
||||||
|
t.Fatalf("status = %d, want %d", w.Code, http.StatusMethodNotAllowed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGradleBuildCacheHandler_PathTraversalRejected(t *testing.T) {
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
h := NewGradleBuildCacheHandler(proxy)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/../secret", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.Routes().ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want %d", w.Code, http.StatusBadRequest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGradleBuildCacheHandler_CachePrefixRejected(t *testing.T) {
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
h := NewGradleBuildCacheHandler(proxy)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/cache/key", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.Routes().ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusNotFound {
|
||||||
|
t.Fatalf("status = %d, want %d", w.Code, http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGradleBuildCacheHandler_PutOverwriteReturnsCreated(t *testing.T) {
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
h := NewGradleBuildCacheHandler(proxy)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
key := "overwrite-key"
|
||||||
|
|
||||||
|
for i, payload := range []string{"first", "second"} {
|
||||||
|
req, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader(payload))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create PUT request: %v", err)
|
||||||
|
}
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PUT request failed: %v", err)
|
||||||
|
}
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
|
want := http.StatusCreated
|
||||||
|
if resp.StatusCode != want {
|
||||||
|
t.Fatalf("PUT #%d status = %d, want %d", i+1, resp.StatusCode, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGradleBuildCacheHandler_PutReadOnly(t *testing.T) {
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
proxy.GradleReadOnly = true
|
||||||
|
|
||||||
|
h := NewGradleBuildCacheHandler(proxy)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
req, err := http.NewRequest(http.MethodPut, srv.URL+"/readonly-key", strings.NewReader("payload"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create PUT request: %v", err)
|
||||||
|
}
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PUT request failed: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusMethodNotAllowed {
|
||||||
|
t.Fatalf("PUT status = %d, want %d", resp.StatusCode, http.StatusMethodNotAllowed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGradleBuildCacheHandler_PutTooLarge(t *testing.T) {
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
proxy.GradleMaxUploadSize = 4
|
||||||
|
|
||||||
|
h := NewGradleBuildCacheHandler(proxy)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
req, err := http.NewRequest(http.MethodPut, srv.URL+"/oversized-key", strings.NewReader("12345"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create PUT request: %v", err)
|
||||||
|
}
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PUT request failed: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusRequestEntityTooLarge {
|
||||||
|
t.Fatalf("PUT status = %d, want %d", resp.StatusCode, http.StatusRequestEntityTooLarge)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGradleBuildCacheHandler_RecordsMetrics(t *testing.T) {
|
||||||
|
proxy, _, _, _ := setupTestProxy(t)
|
||||||
|
h := NewGradleBuildCacheHandler(proxy)
|
||||||
|
srv := httptest.NewServer(h.Routes())
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
hitsBefore := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle"))
|
||||||
|
missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle"))
|
||||||
|
|
||||||
|
key := "metrics-key"
|
||||||
|
putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/"+key, strings.NewReader("payload"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create PUT request: %v", err)
|
||||||
|
}
|
||||||
|
putResp, err := http.DefaultClient.Do(putReq)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("PUT request failed: %v", err)
|
||||||
|
}
|
||||||
|
_ = putResp.Body.Close()
|
||||||
|
|
||||||
|
getResp, err := http.Get(srv.URL + "/" + key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET request failed: %v", err)
|
||||||
|
}
|
||||||
|
_ = getResp.Body.Close()
|
||||||
|
|
||||||
|
headReq, err := http.NewRequest(http.MethodHead, srv.URL+"/"+key, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create HEAD request: %v", err)
|
||||||
|
}
|
||||||
|
headResp, err := http.DefaultClient.Do(headReq)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("HEAD request failed: %v", err)
|
||||||
|
}
|
||||||
|
_ = headResp.Body.Close()
|
||||||
|
|
||||||
|
missResp, err := http.Get(srv.URL + "/missing-key")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GET miss request failed: %v", err)
|
||||||
|
}
|
||||||
|
_ = missResp.Body.Close()
|
||||||
|
|
||||||
|
hitsAfter := testutil.ToFloat64(metrics.CacheHits.WithLabelValues("gradle"))
|
||||||
|
missesAfter := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("gradle"))
|
||||||
|
|
||||||
|
if diff := hitsAfter - hitsBefore; diff != 2 {
|
||||||
|
t.Fatalf("cache hits delta = %.0f, want 2", diff)
|
||||||
|
}
|
||||||
|
if diff := missesAfter - missesBefore; diff != 1 {
|
||||||
|
t.Fatalf("cache misses delta = %.0f, want 1", diff)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -15,7 +15,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
"github.com/git-pkgs/proxy/internal/database"
|
"github.com/git-pkgs/proxy/internal/database"
|
||||||
"github.com/git-pkgs/proxy/internal/metrics"
|
"github.com/git-pkgs/proxy/internal/metrics"
|
||||||
"github.com/git-pkgs/proxy/internal/storage"
|
"github.com/git-pkgs/proxy/internal/storage"
|
||||||
|
|
@ -24,9 +24,21 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
// containsPathTraversal returns true if the path contains ".." segments
|
// containsPathTraversal returns true if the path contains ".." segments
|
||||||
// that could be used to escape the intended directory.
|
// that could be used to escape the intended directory. It checks the path
|
||||||
|
// as given and after URL-decoding, and treats backslashes as separators.
|
||||||
func containsPathTraversal(path string) bool {
|
func containsPathTraversal(path string) bool {
|
||||||
for _, segment := range strings.Split(path, "/") {
|
if hasDotDotSegment(path) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if decoded, err := url.PathUnescape(path); err == nil && decoded != path {
|
||||||
|
return hasDotDotSegment(decoded)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasDotDotSegment(path string) bool {
|
||||||
|
path = strings.ReplaceAll(path, "\\", "/")
|
||||||
|
for segment := range strings.SplitSeq(path, "/") {
|
||||||
if segment == ".." {
|
if segment == ".." {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
@ -38,23 +50,27 @@ const defaultHTTPTimeout = 30 * time.Second
|
||||||
|
|
||||||
const contentTypeJSON = "application/json"
|
const contentTypeJSON = "application/json"
|
||||||
|
|
||||||
// maxMetadataSize is the maximum size of upstream metadata responses (100 MB).
|
const headerAcceptEncoding = "Accept-Encoding"
|
||||||
// Package metadata (e.g. npm with many versions) can be large, but unbounded
|
|
||||||
// reads risk OOM if an upstream misbehaves.
|
|
||||||
const maxMetadataSize = 100 << 20
|
|
||||||
|
|
||||||
// ErrMetadataTooLarge is returned when upstream metadata exceeds maxMetadataSize.
|
// defaultMetadataMaxSize is used when Proxy.MetadataMaxSize is unset.
|
||||||
|
const defaultMetadataMaxSize = 100 << 20
|
||||||
|
|
||||||
|
// ErrMetadataTooLarge is returned when upstream metadata exceeds the configured limit.
|
||||||
var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit")
|
var ErrMetadataTooLarge = errors.New("metadata response exceeds size limit")
|
||||||
|
|
||||||
// ReadMetadata reads an upstream response body with a size limit to prevent OOM
|
// ReadMetadata reads an upstream response body with a size limit to prevent OOM
|
||||||
// from unexpectedly large responses. Returns ErrMetadataTooLarge if the response
|
// from unexpectedly large responses. Returns ErrMetadataTooLarge if the response
|
||||||
// is truncated by the limit.
|
// is truncated by the limit.
|
||||||
func ReadMetadata(r io.Reader) ([]byte, error) {
|
func (p *Proxy) ReadMetadata(r io.Reader) ([]byte, error) {
|
||||||
data, err := io.ReadAll(io.LimitReader(r, maxMetadataSize+1))
|
limit := p.MetadataMaxSize
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = defaultMetadataMaxSize
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(io.LimitReader(r, limit+1))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if int64(len(data)) > maxMetadataSize {
|
if int64(len(data)) > limit {
|
||||||
return nil, ErrMetadataTooLarge
|
return nil, ErrMetadataTooLarge
|
||||||
}
|
}
|
||||||
return data, nil
|
return data, nil
|
||||||
|
|
@ -70,6 +86,9 @@ type Proxy struct {
|
||||||
Cooldown *cooldown.Config
|
Cooldown *cooldown.Config
|
||||||
CacheMetadata bool
|
CacheMetadata bool
|
||||||
MetadataTTL time.Duration
|
MetadataTTL time.Duration
|
||||||
|
MetadataMaxSize int64
|
||||||
|
GradleReadOnly bool
|
||||||
|
GradleMaxUploadSize int64
|
||||||
DirectServe bool
|
DirectServe bool
|
||||||
DirectServeTTL time.Duration
|
DirectServeTTL time.Duration
|
||||||
// DirectServeBaseURL, if set, replaces the scheme and host of presigned
|
// DirectServeBaseURL, if set, replaces the scheme and host of presigned
|
||||||
|
|
@ -176,7 +195,16 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
result.Reader = reader
|
result.Reader = newVerifyingReader(reader, artifact.ContentHash.String, ver.Integrity.String,
|
||||||
|
func(reason string) {
|
||||||
|
p.Logger.Error("cached artifact failed integrity check",
|
||||||
|
"purl", versionPURL, "filename", filename,
|
||||||
|
"path", artifact.StoragePath.String, "reason", reason)
|
||||||
|
metrics.RecordIntegrityFailure(pkg.Ecosystem)
|
||||||
|
if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil {
|
||||||
|
p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
p.recordCacheHit(pkgPURL, versionPURL, filename)
|
p.recordCacheHit(pkgPURL, versionPURL, filename)
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
@ -449,7 +477,7 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u
|
||||||
cached, readErr := p.Storage.Open(ctx, entry.StoragePath)
|
cached, readErr := p.Storage.Open(ctx, entry.StoragePath)
|
||||||
if readErr == nil {
|
if readErr == nil {
|
||||||
defer func() { _ = cached.Close() }()
|
defer func() { _ = cached.Close() }()
|
||||||
data, readErr := ReadMetadata(cached)
|
data, readErr := p.ReadMetadata(cached)
|
||||||
if readErr == nil {
|
if readErr == nil {
|
||||||
ct := contentTypeJSON
|
ct := contentTypeJSON
|
||||||
if entry.ContentType.Valid {
|
if entry.ContentType.Valid {
|
||||||
|
|
@ -494,7 +522,7 @@ func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u
|
||||||
}
|
}
|
||||||
defer func() { _ = cached.Close() }()
|
defer func() { _ = cached.Close() }()
|
||||||
|
|
||||||
data, readErr := ReadMetadata(cached)
|
data, readErr := p.ReadMetadata(cached)
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
return nil, "", fmt.Errorf("upstream failed and cached read error: %w", err)
|
return nil, "", fmt.Errorf("upstream failed and cached read error: %w", err)
|
||||||
}
|
}
|
||||||
|
|
@ -536,7 +564,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
|
||||||
return nil, "", "", zeroTime, errStale304
|
return nil, "", "", zeroTime, errStale304
|
||||||
}
|
}
|
||||||
defer func() { _ = cached.Close() }()
|
defer func() { _ = cached.Close() }()
|
||||||
data, readErr := ReadMetadata(cached)
|
data, readErr := p.ReadMetadata(cached)
|
||||||
if readErr != nil {
|
if readErr != nil {
|
||||||
return nil, "", "", zeroTime, errStale304
|
return nil, "", "", zeroTime, errStale304
|
||||||
}
|
}
|
||||||
|
|
@ -558,7 +586,7 @@ func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, e
|
||||||
return nil, "", "", zeroTime, fmt.Errorf("upstream returned %d", resp.StatusCode)
|
return nil, "", "", zeroTime, fmt.Errorf("upstream returned %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := ReadMetadata(resp.Body)
|
body, err := p.ReadMetadata(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", "", zeroTime, fmt.Errorf("reading response: %w", err)
|
return nil, "", "", zeroTime, fmt.Errorf("reading response: %w", err)
|
||||||
}
|
}
|
||||||
|
|
@ -655,9 +683,14 @@ func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL,
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
p.writeMetadataCachedResponse(w, r, ecosystem, cacheKey, body, contentType)
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeMetadataCachedResponse writes a cached metadata response and handles
|
||||||
|
// conditional request headers using metadata cache validators.
|
||||||
|
func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType string) {
|
||||||
cm := p.lookupCachedMeta(ecosystem, cacheKey)
|
cm := p.lookupCachedMeta(ecosystem, cacheKey)
|
||||||
|
|
||||||
// Honor client conditional request headers
|
|
||||||
if cm.etag != "" {
|
if cm.etag != "" {
|
||||||
if match := r.Header.Get("If-None-Match"); match != "" && match == cm.etag {
|
if match := r.Header.Get("If-None-Match"); match != "" && match == cm.etag {
|
||||||
w.WriteHeader(http.StatusNotModified)
|
w.WriteHeader(http.StatusNotModified)
|
||||||
|
|
@ -703,7 +736,7 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst
|
||||||
}
|
}
|
||||||
req.Header.Set("Accept", accept)
|
req.Header.Set("Accept", accept)
|
||||||
|
|
||||||
for _, header := range []string{"Accept-Encoding", "If-Modified-Since", "If-None-Match"} {
|
for _, header := range []string{headerAcceptEncoding, "If-Modified-Since", "If-None-Match"} {
|
||||||
if v := r.Header.Get(header); v != "" {
|
if v := r.Header.Get(header); v != "" {
|
||||||
req.Header.Set(header, v)
|
req.Header.Set(header, v)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -99,6 +99,7 @@ func (s *mockStorage) Close() error { return nil }
|
||||||
type mockFetcher struct {
|
type mockFetcher struct {
|
||||||
artifact *fetch.Artifact
|
artifact *fetch.Artifact
|
||||||
fetchErr error
|
fetchErr error
|
||||||
|
fetchErrByURL map[string]error
|
||||||
fetchCalled bool
|
fetchCalled bool
|
||||||
fetchedURL string
|
fetchedURL string
|
||||||
}
|
}
|
||||||
|
|
@ -110,6 +111,11 @@ func (f *mockFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, e
|
||||||
func (f *mockFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) {
|
func (f *mockFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) {
|
||||||
f.fetchCalled = true
|
f.fetchCalled = true
|
||||||
f.fetchedURL = url
|
f.fetchedURL = url
|
||||||
|
if f.fetchErrByURL != nil {
|
||||||
|
if err, ok := f.fetchErrByURL[url]; ok {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
if f.fetchErr != nil {
|
if f.fetchErr != nil {
|
||||||
return nil, f.fetchErr
|
return nil, f.fetchErr
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
"google.golang.org/protobuf/encoding/protowire"
|
"google.golang.org/protobuf/encoding/protowire"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
140
internal/handler/integrity.go
Normal file
140
internal/handler/integrity.go
Normal file
|
|
@ -0,0 +1,140 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"crypto/sha512"
|
||||||
|
"crypto/subtle"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"hash"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// parseSRI parses a Subresource Integrity string (e.g. "sha512-abc==") into
|
||||||
|
// an algorithm name and raw digest bytes. Returns ok=false for empty,
|
||||||
|
// malformed, or unsupported entries. Only the first hash in a multi-hash
|
||||||
|
// SRI string is considered.
|
||||||
|
func parseSRI(s string) (algo string, digest []byte, ok bool) {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" {
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
if i := strings.IndexByte(s, ' '); i >= 0 {
|
||||||
|
s = s[:i]
|
||||||
|
}
|
||||||
|
algo, b64, found := strings.Cut(s, "-")
|
||||||
|
if !found {
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
d, err := base64.StdEncoding.DecodeString(b64)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
switch algo {
|
||||||
|
case "sha256", "sha384", "sha512":
|
||||||
|
return algo, d, true
|
||||||
|
default:
|
||||||
|
return "", nil, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSRIHash(algo string) hash.Hash {
|
||||||
|
switch algo {
|
||||||
|
case "sha256":
|
||||||
|
return sha256.New()
|
||||||
|
case "sha384":
|
||||||
|
return sha512.New384()
|
||||||
|
case "sha512":
|
||||||
|
return sha512.New()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// verifyingReader wraps an io.ReadCloser and computes SHA256 (and optionally
|
||||||
|
// a second SRI hash) as bytes are read. When the underlying reader reaches
|
||||||
|
// EOF it compares the digests against the expected values and calls
|
||||||
|
// onMismatch for each failure. Verification is skipped if the stream was
|
||||||
|
// not fully consumed (e.g. client disconnect) to avoid false positives.
|
||||||
|
type verifyingReader struct {
|
||||||
|
r io.ReadCloser
|
||||||
|
sha256 hash.Hash
|
||||||
|
wantSHA256 string
|
||||||
|
sri hash.Hash
|
||||||
|
sriAlgo string
|
||||||
|
wantSRI []byte
|
||||||
|
onMismatch func(reason string)
|
||||||
|
eof bool
|
||||||
|
verified bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newVerifyingReader(r io.ReadCloser, contentHash, sri string, onMismatch func(string)) io.ReadCloser {
|
||||||
|
if contentHash == "" && sri == "" {
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
v := &verifyingReader{
|
||||||
|
r: r,
|
||||||
|
onMismatch: onMismatch,
|
||||||
|
}
|
||||||
|
if contentHash != "" {
|
||||||
|
v.sha256 = sha256.New()
|
||||||
|
v.wantSHA256 = contentHash
|
||||||
|
}
|
||||||
|
if algo, digest, ok := parseSRI(sri); ok {
|
||||||
|
v.sri = newSRIHash(algo)
|
||||||
|
v.sriAlgo = algo
|
||||||
|
v.wantSRI = digest
|
||||||
|
}
|
||||||
|
if v.sha256 == nil && v.sri == nil {
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *verifyingReader) Read(p []byte) (int, error) {
|
||||||
|
n, err := v.r.Read(p)
|
||||||
|
if n > 0 {
|
||||||
|
if v.sha256 != nil {
|
||||||
|
v.sha256.Write(p[:n])
|
||||||
|
}
|
||||||
|
if v.sri != nil {
|
||||||
|
v.sri.Write(p[:n])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err == io.EOF {
|
||||||
|
v.eof = true
|
||||||
|
v.verify()
|
||||||
|
}
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *verifyingReader) Close() error {
|
||||||
|
if v.eof {
|
||||||
|
v.verify()
|
||||||
|
}
|
||||||
|
return v.r.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *verifyingReader) verify() {
|
||||||
|
if v.verified {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
v.verified = true
|
||||||
|
|
||||||
|
if v.sha256 != nil {
|
||||||
|
got := hex.EncodeToString(v.sha256.Sum(nil))
|
||||||
|
if subtle.ConstantTimeCompare([]byte(got), []byte(v.wantSHA256)) != 1 {
|
||||||
|
v.onMismatch(fmt.Sprintf("content_hash mismatch: stored=%s computed=%s", v.wantSHA256, got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v.sri != nil {
|
||||||
|
got := v.sri.Sum(nil)
|
||||||
|
if subtle.ConstantTimeCompare(got, v.wantSRI) != 1 {
|
||||||
|
v.onMismatch(fmt.Sprintf("integrity mismatch: %s expected=%s computed=%s",
|
||||||
|
v.sriAlgo,
|
||||||
|
base64.StdEncoding.EncodeToString(v.wantSRI),
|
||||||
|
base64.StdEncoding.EncodeToString(got)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
136
internal/handler/integrity_test.go
Normal file
136
internal/handler/integrity_test.go
Normal file
|
|
@ -0,0 +1,136 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"crypto/sha512"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func sha256Hex(data string) string {
|
||||||
|
sum := sha256.Sum256([]byte(data))
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func sha512SRI(data string) string {
|
||||||
|
sum := sha512.Sum512([]byte(data))
|
||||||
|
return "sha512-" + base64.StdEncoding.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseSRI(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
algo string
|
||||||
|
ok bool
|
||||||
|
}{
|
||||||
|
{"sha512", sha512SRI("hello"), "sha512", true},
|
||||||
|
{"sha256", "sha256-" + base64.StdEncoding.EncodeToString([]byte("0123456789012345678901234567890123456789")), "sha256", true},
|
||||||
|
{"empty", "", "", false},
|
||||||
|
{"no dash", "sha512abc", "", false},
|
||||||
|
{"bad base64", "sha512-not!base64", "", false},
|
||||||
|
{"unsupported algo", "md5-" + base64.StdEncoding.EncodeToString([]byte("x")), "", false},
|
||||||
|
{"multi hash takes first", sha512SRI("a") + " " + sha512SRI("b"), "sha512", true},
|
||||||
|
{"whitespace", " " + sha512SRI("x") + " ", "sha512", true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
algo, digest, ok := parseSRI(tt.input)
|
||||||
|
if ok != tt.ok {
|
||||||
|
t.Fatalf("ok = %v, want %v", ok, tt.ok)
|
||||||
|
}
|
||||||
|
if !tt.ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if algo != tt.algo {
|
||||||
|
t.Errorf("algo = %q, want %q", algo, tt.algo)
|
||||||
|
}
|
||||||
|
if len(digest) == 0 {
|
||||||
|
t.Error("digest is empty")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVerifyingReader(t *testing.T) {
|
||||||
|
const data = "hello world"
|
||||||
|
goodSHA := sha256Hex(data)
|
||||||
|
goodSRI := sha512SRI(data)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
hash string
|
||||||
|
sri string
|
||||||
|
wantCalls int
|
||||||
|
}{
|
||||||
|
{"both match", goodSHA, goodSRI, 0},
|
||||||
|
{"sha256 only match", goodSHA, "", 0},
|
||||||
|
{"sri only match", "", goodSRI, 0},
|
||||||
|
{"sha256 mismatch", sha256Hex("other"), "", 1},
|
||||||
|
{"sri mismatch", "", sha512SRI("other"), 1},
|
||||||
|
{"both mismatch", sha256Hex("other"), sha512SRI("other"), 2},
|
||||||
|
{"no checks", "", "", 0},
|
||||||
|
{"unparseable sri ignored", goodSHA, "garbage", 0},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
var calls []string
|
||||||
|
r := newVerifyingReader(io.NopCloser(strings.NewReader(data)), tt.hash, tt.sri,
|
||||||
|
func(reason string) { calls = append(calls, reason) })
|
||||||
|
|
||||||
|
got, err := io.ReadAll(r)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ReadAll: %v", err)
|
||||||
|
}
|
||||||
|
if string(got) != data {
|
||||||
|
t.Errorf("data corrupted: got %q", got)
|
||||||
|
}
|
||||||
|
if err := r.Close(); err != nil {
|
||||||
|
t.Fatalf("Close: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(calls) != tt.wantCalls {
|
||||||
|
t.Errorf("onMismatch called %d times, want %d: %v", len(calls), tt.wantCalls, calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVerifyingReaderPassthrough(t *testing.T) {
|
||||||
|
src := io.NopCloser(strings.NewReader("x"))
|
||||||
|
r := newVerifyingReader(src, "", "", func(string) { t.Fatal("should not be called") })
|
||||||
|
if r != src {
|
||||||
|
t.Error("expected passthrough when no hashes provided")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVerifyingReaderPartialRead(t *testing.T) {
|
||||||
|
var calls int
|
||||||
|
r := newVerifyingReader(io.NopCloser(strings.NewReader("hello world")),
|
||||||
|
sha256Hex("hello world"), "", func(string) { calls++ })
|
||||||
|
|
||||||
|
buf := make([]byte, 5)
|
||||||
|
_, _ = r.Read(buf)
|
||||||
|
_ = r.Close()
|
||||||
|
|
||||||
|
if calls != 0 {
|
||||||
|
t.Errorf("onMismatch called %d times for partial read, want 0", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVerifyingReaderVerifyOnce(t *testing.T) {
|
||||||
|
var calls int
|
||||||
|
r := newVerifyingReader(io.NopCloser(strings.NewReader("x")), sha256Hex("y"), "",
|
||||||
|
func(string) { calls++ })
|
||||||
|
_, _ = io.ReadAll(r)
|
||||||
|
_ = r.Close()
|
||||||
|
_ = r.Close()
|
||||||
|
if calls != 1 {
|
||||||
|
t.Errorf("onMismatch called %d times, want 1", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
347
internal/handler/julia.go
Normal file
347
internal/handler/julia.go
Normal file
|
|
@ -0,0 +1,347 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/BurntSushi/toml"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
juliaUpstream = "https://pkg.julialang.org"
|
||||||
|
juliaGeneralRegistryUUID = "23338594-aafe-5451-b93e-139f81909106"
|
||||||
|
juliaArtifactName = "_artifact"
|
||||||
|
juliaRegistryName = "_registry"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
juliaHexPattern = regexp.MustCompile(`^[0-9a-f]{40,64}$`)
|
||||||
|
juliaUUIDPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// JuliaHandler handles Julia Pkg server protocol requests.
|
||||||
|
//
|
||||||
|
// See https://pkgdocs.julialang.org/v1/registries/ and the PkgServer.jl
|
||||||
|
// reference implementation. The protocol is content-addressed: registry,
|
||||||
|
// package and artifact resources are all identified by git tree hashes
|
||||||
|
// and are immutable once published.
|
||||||
|
type JuliaHandler struct {
|
||||||
|
proxy *Proxy
|
||||||
|
upstreamURL string
|
||||||
|
|
||||||
|
mu sync.RWMutex
|
||||||
|
names map[string]string
|
||||||
|
namesHash string
|
||||||
|
loadMu sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewJuliaHandler creates a new Julia Pkg server handler.
|
||||||
|
func NewJuliaHandler(proxy *Proxy, _ string) *JuliaHandler {
|
||||||
|
return &JuliaHandler{
|
||||||
|
proxy: proxy,
|
||||||
|
upstreamURL: juliaUpstream,
|
||||||
|
names: make(map[string]string),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Routes returns the HTTP handler for Julia requests.
|
||||||
|
func (h *JuliaHandler) Routes() http.Handler {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
|
mux.HandleFunc("GET /registries", h.handleRegistries)
|
||||||
|
mux.HandleFunc("GET /registries.eager", h.handleRegistries)
|
||||||
|
mux.HandleFunc("GET /registries.conservative", h.handleRegistries)
|
||||||
|
mux.HandleFunc("GET /registry/{uuid}/{hash}", h.handleRegistry)
|
||||||
|
mux.HandleFunc("GET /package/{uuid}/{hash}", h.handlePackage)
|
||||||
|
mux.HandleFunc("GET /artifact/{hash}", h.handleArtifact)
|
||||||
|
mux.HandleFunc("GET /meta", h.proxyUpstream)
|
||||||
|
|
||||||
|
return mux
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleRegistries serves the list of available registries. This is the only
|
||||||
|
// mutable endpoint in the protocol so it goes through the metadata cache.
|
||||||
|
func (h *JuliaHandler) handleRegistries(w http.ResponseWriter, r *http.Request) {
|
||||||
|
cacheKey := strings.TrimPrefix(r.URL.Path, "/")
|
||||||
|
h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "julia", cacheKey, "*/*")
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleRegistry serves an immutable registry tarball and refreshes the
|
||||||
|
// UUID→name map from its Registry.toml.
|
||||||
|
func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) {
|
||||||
|
uuid := r.PathValue("uuid")
|
||||||
|
hash := r.PathValue("hash")
|
||||||
|
if !validJuliaUUID(uuid) || !juliaHexPattern.MatchString(hash) {
|
||||||
|
http.Error(w, "invalid registry reference", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
h.proxy.Logger.Info("julia registry request", "uuid", uuid, "hash", hash)
|
||||||
|
|
||||||
|
upstreamURL := h.upstreamURL + r.URL.Path
|
||||||
|
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaRegistryName, hash, hash+".tar.gz", upstreamURL)
|
||||||
|
if err != nil {
|
||||||
|
h.proxy.Logger.Error("failed to get registry", "error", err)
|
||||||
|
http.Error(w, "failed to fetch registry", http.StatusBadGateway)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
go h.refreshNamesFromRegistry(uuid, hash)
|
||||||
|
|
||||||
|
ServeArtifact(w, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handlePackage serves an immutable package source tarball.
|
||||||
|
func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
uuid := r.PathValue("uuid")
|
||||||
|
hash := r.PathValue("hash")
|
||||||
|
if !validJuliaUUID(uuid) || !juliaHexPattern.MatchString(hash) {
|
||||||
|
http.Error(w, "invalid package reference", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := h.ensureNames(r.Context()); err != nil {
|
||||||
|
h.proxy.Logger.Warn("julia name map unavailable, using uuid", "error", err)
|
||||||
|
}
|
||||||
|
name := h.resolveName(uuid)
|
||||||
|
|
||||||
|
h.proxy.Logger.Info("julia package request", "name", name, "uuid", uuid, "hash", hash)
|
||||||
|
|
||||||
|
upstreamURL := h.upstreamURL + r.URL.Path
|
||||||
|
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", name, hash, hash+".tar.gz", upstreamURL)
|
||||||
|
if err != nil {
|
||||||
|
h.proxy.Logger.Error("failed to get package", "error", err)
|
||||||
|
http.Error(w, "failed to fetch package", http.StatusBadGateway)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ServeArtifact(w, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleArtifact serves an immutable binary artifact tarball. Artifacts are
|
||||||
|
// anonymous content-addressed blobs with no associated package name.
|
||||||
|
func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) {
|
||||||
|
hash := r.PathValue("hash")
|
||||||
|
if !juliaHexPattern.MatchString(hash) {
|
||||||
|
http.Error(w, "invalid artifact hash", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
h.proxy.Logger.Info("julia artifact request", "hash", hash)
|
||||||
|
|
||||||
|
upstreamURL := h.upstreamURL + r.URL.Path
|
||||||
|
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "julia", juliaArtifactName, hash, hash+".tar.gz", upstreamURL)
|
||||||
|
if err != nil {
|
||||||
|
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||||
|
http.Error(w, "failed to fetch artifact", http.StatusBadGateway)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ServeArtifact(w, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// proxyUpstream forwards a request to the upstream Pkg server without caching.
|
||||||
|
func (h *JuliaHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
||||||
|
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveName returns the human-readable package name for a UUID, falling
|
||||||
|
// back to the UUID itself if it is not present in the loaded registry.
|
||||||
|
func (h *JuliaHandler) resolveName(uuid string) string {
|
||||||
|
h.mu.RLock()
|
||||||
|
defer h.mu.RUnlock()
|
||||||
|
if name, ok := h.names[uuid]; ok {
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
return uuid
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureNames lazily populates the UUID→name map from the General registry.
|
||||||
|
// Returns immediately if the map is already populated; otherwise blocks until
|
||||||
|
// a single in-flight load completes. Failed loads are retried on the next call.
|
||||||
|
func (h *JuliaHandler) ensureNames(ctx context.Context) error {
|
||||||
|
if h.namesLoaded() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
h.loadMu.Lock()
|
||||||
|
defer h.loadMu.Unlock()
|
||||||
|
|
||||||
|
if h.namesLoaded() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return h.loadNamesFromUpstream(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *JuliaHandler) namesLoaded() bool {
|
||||||
|
h.mu.RLock()
|
||||||
|
defer h.mu.RUnlock()
|
||||||
|
return len(h.names) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadNamesFromUpstream fetches the current /registries listing, downloads the
|
||||||
|
// General registry tarball at its current hash, and parses Registry.toml.
|
||||||
|
func (h *JuliaHandler) loadNamesFromUpstream(ctx context.Context) error {
|
||||||
|
hash, err := h.fetchGeneralRegistryHash(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return h.loadRegistryTarball(ctx, juliaGeneralRegistryUUID, hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchGeneralRegistryHash reads /registries and returns the current tree hash
|
||||||
|
// for the General registry.
|
||||||
|
func (h *JuliaHandler) fetchGeneralRegistryHash(ctx context.Context) (string, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.upstreamURL+"/registries", nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
resp, err := h.proxy.HTTPClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return "", fmt.Errorf("upstream /registries returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(resp.Body)
|
||||||
|
for scanner.Scan() {
|
||||||
|
uuid, hash, ok := parseRegistryLine(scanner.Text())
|
||||||
|
if ok && uuid == juliaGeneralRegistryUUID {
|
||||||
|
return hash, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("general registry not listed in /registries")
|
||||||
|
}
|
||||||
|
|
||||||
|
// refreshNamesFromRegistry reloads the UUID→name map from a registry tarball
|
||||||
|
// that has just been cached. Errors are logged but do not affect the response.
|
||||||
|
func (h *JuliaHandler) refreshNamesFromRegistry(uuid, hash string) {
|
||||||
|
if uuid != juliaGeneralRegistryUUID {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.mu.RLock()
|
||||||
|
current := h.namesHash
|
||||||
|
h.mu.RUnlock()
|
||||||
|
if current == hash {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := h.loadRegistryTarball(context.Background(), uuid, hash); err != nil {
|
||||||
|
h.proxy.Logger.Warn("failed to refresh julia name map", "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadRegistryTarball downloads a registry tarball and replaces the name map
|
||||||
|
// with the contents of its Registry.toml.
|
||||||
|
func (h *JuliaHandler) loadRegistryTarball(ctx context.Context, uuid, hash string) error {
|
||||||
|
url := fmt.Sprintf("%s/registry/%s/%s", h.upstreamURL, uuid, hash)
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
resp, err := h.proxy.HTTPClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return fmt.Errorf("upstream registry returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
names, err := extractRegistryNames(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
h.mu.Lock()
|
||||||
|
h.names = names
|
||||||
|
h.namesHash = hash
|
||||||
|
h.mu.Unlock()
|
||||||
|
|
||||||
|
h.proxy.Logger.Info("loaded julia registry name map", "packages", len(names), "hash", hash)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractRegistryNames reads a gzipped registry tarball, finds Registry.toml
|
||||||
|
// at the root, and returns its [packages] table as a UUID→name map.
|
||||||
|
func extractRegistryNames(r io.Reader) (map[string]string, error) {
|
||||||
|
gz, err := gzip.NewReader(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("opening gzip stream: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = gz.Close() }()
|
||||||
|
|
||||||
|
tr := tar.NewReader(gz)
|
||||||
|
for {
|
||||||
|
hdr, err := tr.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
return nil, fmt.Errorf("no Registry.toml in tarball")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if strings.TrimPrefix(hdr.Name, "./") != "Registry.toml" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := io.ReadAll(tr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return parseRegistryToml(data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type juliaRegistryFile struct {
|
||||||
|
Packages map[string]struct {
|
||||||
|
Name string `toml:"name"`
|
||||||
|
} `toml:"packages"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseRegistryToml decodes the [packages] table of a Registry.toml file.
|
||||||
|
func parseRegistryToml(data []byte) (map[string]string, error) {
|
||||||
|
var reg juliaRegistryFile
|
||||||
|
if _, err := toml.NewDecoder(bytes.NewReader(data)).Decode(®); err != nil {
|
||||||
|
return nil, fmt.Errorf("parsing Registry.toml: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
names := make(map[string]string, len(reg.Packages))
|
||||||
|
for uuid, pkg := range reg.Packages {
|
||||||
|
if pkg.Name != "" {
|
||||||
|
names[uuid] = pkg.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseRegistryLine parses a single line from /registries of the form
|
||||||
|
// "/registry/{uuid}/{hash}" and returns the uuid and hash.
|
||||||
|
func parseRegistryLine(line string) (uuid, hash string, ok bool) {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
line = strings.TrimPrefix(line, "/registry/")
|
||||||
|
uuid, hash, found := strings.Cut(line, "/")
|
||||||
|
if !found || !validJuliaUUID(uuid) || !juliaHexPattern.MatchString(hash) {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
return uuid, hash, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// validJuliaUUID reports whether s looks like a lowercase RFC 4122 UUID.
|
||||||
|
func validJuliaUUID(s string) bool {
|
||||||
|
return juliaUUIDPattern.MatchString(s)
|
||||||
|
}
|
||||||
167
internal/handler/julia_test.go
Normal file
167
internal/handler/julia_test.go
Normal file
|
|
@ -0,0 +1,167 @@
|
||||||
|
package handler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestJuliaParseRegistryLine(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
line string
|
||||||
|
wantUUID string
|
||||||
|
wantHash string
|
||||||
|
wantOK bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
"/registry/23338594-aafe-5451-b93e-139f81909106/342327538ed6c1ec54c69fa145e7b6bf5934201e",
|
||||||
|
"23338594-aafe-5451-b93e-139f81909106",
|
||||||
|
"342327538ed6c1ec54c69fa145e7b6bf5934201e",
|
||||||
|
true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
" /registry/23338594-aafe-5451-b93e-139f81909106/342327538ed6c1ec54c69fa145e7b6bf5934201e\n",
|
||||||
|
"23338594-aafe-5451-b93e-139f81909106",
|
||||||
|
"342327538ed6c1ec54c69fa145e7b6bf5934201e",
|
||||||
|
true,
|
||||||
|
},
|
||||||
|
{"/registry/not-a-uuid/0000", "", "", false},
|
||||||
|
{"junk", "", "", false},
|
||||||
|
{"", "", "", false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
uuid, hash, ok := parseRegistryLine(tt.line)
|
||||||
|
if uuid != tt.wantUUID || hash != tt.wantHash || ok != tt.wantOK {
|
||||||
|
t.Errorf("parseRegistryLine(%q) = (%q, %q, %v), want (%q, %q, %v)",
|
||||||
|
tt.line, uuid, hash, ok, tt.wantUUID, tt.wantHash, tt.wantOK)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJuliaValidUUID(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
s string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"23338594-aafe-5451-b93e-139f81909106", true},
|
||||||
|
{"295af30f-e4ad-537b-8983-00126c2a3abe", true},
|
||||||
|
{"23338594-AAFE-5451-b93e-139f81909106", false},
|
||||||
|
{"23338594aafe5451b93e139f81909106", false},
|
||||||
|
{"23338594-aafe-5451-b93e-139f8190910", false},
|
||||||
|
{"23338594-aafe-5451-b93e-139f81909106-", false},
|
||||||
|
{"23338594-gafe-5451-b93e-139f81909106", false},
|
||||||
|
{"", false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
if got := validJuliaUUID(tt.s); got != tt.want {
|
||||||
|
t.Errorf("validJuliaUUID(%q) = %v, want %v", tt.s, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJuliaParseRegistryToml(t *testing.T) {
|
||||||
|
data := []byte(`name = "General"
|
||||||
|
uuid = "23338594-aafe-5451-b93e-139f81909106"
|
||||||
|
|
||||||
|
[packages]
|
||||||
|
295af30f-e4ad-537b-8983-00126c2a3abe = { name = "Revise", path = "R/Revise" }
|
||||||
|
91a5bcdd-55d7-5caf-9e0b-520d859cae80 = { name = "Plots", path = "P/Plots" }
|
||||||
|
`)
|
||||||
|
|
||||||
|
names, err := parseRegistryToml(data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parseRegistryToml: %v", err)
|
||||||
|
}
|
||||||
|
if got := names["295af30f-e4ad-537b-8983-00126c2a3abe"]; got != "Revise" {
|
||||||
|
t.Errorf("names[Revise uuid] = %q, want Revise", got)
|
||||||
|
}
|
||||||
|
if got := names["91a5bcdd-55d7-5caf-9e0b-520d859cae80"]; got != "Plots" {
|
||||||
|
t.Errorf("names[Plots uuid] = %q, want Plots", got)
|
||||||
|
}
|
||||||
|
if len(names) != 2 {
|
||||||
|
t.Errorf("len(names) = %d, want 2", len(names))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJuliaExtractRegistryNames(t *testing.T) {
|
||||||
|
registryToml := `name = "General"
|
||||||
|
[packages]
|
||||||
|
295af30f-e4ad-537b-8983-00126c2a3abe = { name = "Revise", path = "R/Revise" }
|
||||||
|
`
|
||||||
|
var buf bytes.Buffer
|
||||||
|
gw := gzip.NewWriter(&buf)
|
||||||
|
tw := tar.NewWriter(gw)
|
||||||
|
|
||||||
|
for _, f := range []struct{ name, body string }{
|
||||||
|
{"R/Revise/Package.toml", "name = \"Revise\"\n"},
|
||||||
|
{"Registry.toml", registryToml},
|
||||||
|
} {
|
||||||
|
if err := tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0o644, Size: int64(len(f.body))}); err != nil {
|
||||||
|
t.Fatalf("WriteHeader: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := tw.Write([]byte(f.body)); err != nil {
|
||||||
|
t.Fatalf("Write: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := tw.Close(); err != nil {
|
||||||
|
t.Fatalf("tar Close: %v", err)
|
||||||
|
}
|
||||||
|
if err := gw.Close(); err != nil {
|
||||||
|
t.Fatalf("gzip Close: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
names, err := extractRegistryNames(bytes.NewReader(buf.Bytes()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("extractRegistryNames: %v", err)
|
||||||
|
}
|
||||||
|
if got := names["295af30f-e4ad-537b-8983-00126c2a3abe"]; got != "Revise" {
|
||||||
|
t.Errorf("names[Revise uuid] = %q, want Revise", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJuliaResolveName(t *testing.T) {
|
||||||
|
h := &JuliaHandler{
|
||||||
|
proxy: &Proxy{Logger: slog.Default()},
|
||||||
|
names: map[string]string{
|
||||||
|
"295af30f-e4ad-537b-8983-00126c2a3abe": "Revise",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := h.resolveName("295af30f-e4ad-537b-8983-00126c2a3abe"); got != "Revise" {
|
||||||
|
t.Errorf("resolveName(known) = %q, want Revise", got)
|
||||||
|
}
|
||||||
|
if got := h.resolveName("00000000-0000-0000-0000-000000000000"); got != "00000000-0000-0000-0000-000000000000" {
|
||||||
|
t.Errorf("resolveName(unknown) = %q, want uuid fallback", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJuliaRoutesValidation(t *testing.T) {
|
||||||
|
h := NewJuliaHandler(&Proxy{Logger: slog.Default()}, "")
|
||||||
|
routes := h.Routes()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
path string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"/package/not-a-uuid/342327538ed6c1ec54c69fa145e7b6bf5934201e", http.StatusBadRequest},
|
||||||
|
{"/package/295af30f-e4ad-537b-8983-00126c2a3abe/short", http.StatusBadRequest},
|
||||||
|
{"/registry/295af30f-e4ad-537b-8983-00126c2a3abe/zzzz", http.StatusBadRequest},
|
||||||
|
{"/artifact/nothex", http.StatusBadRequest},
|
||||||
|
{"/nope", http.StatusNotFound},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
req := httptest.NewRequest(http.MethodGet, tt.path, nil)
|
||||||
|
rr := httptest.NewRecorder()
|
||||||
|
routes.ServeHTTP(rr, req)
|
||||||
|
if rr.Code != tt.want {
|
||||||
|
t.Errorf("GET %s = %d, want %d", tt.path, rr.Code, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"path"
|
"path"
|
||||||
|
|
@ -8,7 +9,8 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
mavenUpstream = "https://repo1.maven.org/maven2"
|
mavenCentralUpstream = "https://repo1.maven.org/maven2"
|
||||||
|
gradlePluginPortalUpstream = "https://plugins.gradle.org/m2"
|
||||||
minMavenParts = 4 // group path segments + artifact + version + filename
|
minMavenParts = 4 // group path segments + artifact + version + filename
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -16,14 +18,23 @@ const (
|
||||||
type MavenHandler struct {
|
type MavenHandler struct {
|
||||||
proxy *Proxy
|
proxy *Proxy
|
||||||
upstreamURL string
|
upstreamURL string
|
||||||
|
pluginPortalUpstreamURL string
|
||||||
proxyURL string
|
proxyURL string
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewMavenHandler creates a new Maven repository handler.
|
// NewMavenHandler creates a new Maven repository handler.
|
||||||
func NewMavenHandler(proxy *Proxy, proxyURL string) *MavenHandler {
|
func NewMavenHandler(proxy *Proxy, proxyURL, upstreamURL, pluginPortalUpstreamURL string) *MavenHandler {
|
||||||
|
if strings.TrimSpace(upstreamURL) == "" {
|
||||||
|
upstreamURL = mavenCentralUpstream
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(pluginPortalUpstreamURL) == "" {
|
||||||
|
pluginPortalUpstreamURL = gradlePluginPortalUpstream
|
||||||
|
}
|
||||||
|
|
||||||
return &MavenHandler{
|
return &MavenHandler{
|
||||||
proxy: proxy,
|
proxy: proxy,
|
||||||
upstreamURL: mavenUpstream,
|
upstreamURL: strings.TrimSuffix(upstreamURL, "/"),
|
||||||
|
pluginPortalUpstreamURL: strings.TrimSuffix(pluginPortalUpstreamURL, "/"),
|
||||||
proxyURL: strings.TrimSuffix(proxyURL, "/"),
|
proxyURL: strings.TrimSuffix(proxyURL, "/"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -51,8 +62,7 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) {
|
||||||
filename := path.Base(urlPath)
|
filename := path.Base(urlPath)
|
||||||
|
|
||||||
if h.isMetadataFile(filename) {
|
if h.isMetadataFile(filename) {
|
||||||
cacheKey := strings.ReplaceAll(urlPath, "/", "_")
|
h.handleMetadata(w, r, urlPath)
|
||||||
h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "maven", cacheKey, "*/*")
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -66,6 +76,32 @@ func (h *MavenHandler) handleRequest(w http.ResponseWriter, r *http.Request) {
|
||||||
h.proxyUpstream(w, r)
|
h.proxyUpstream(w, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, urlPath string) {
|
||||||
|
cacheKey := strings.ReplaceAll(urlPath, "/", "_")
|
||||||
|
upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, urlPath)
|
||||||
|
|
||||||
|
body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, upstreamURL, "*/*")
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrUpstreamNotFound) {
|
||||||
|
pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath)
|
||||||
|
h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal",
|
||||||
|
"path", urlPath)
|
||||||
|
body, contentType, err = h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, pluginPortalURL, "*/*")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrUpstreamNotFound) {
|
||||||
|
http.Error(w, "not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.proxy.Logger.Error("metadata fetch failed", "error", err)
|
||||||
|
http.Error(w, "failed to fetch from upstream", http.StatusBadGateway)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType)
|
||||||
|
}
|
||||||
|
|
||||||
// handleDownload serves an artifact file, fetching and caching from upstream if needed.
|
// handleDownload serves an artifact file, fetching and caching from upstream if needed.
|
||||||
func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, urlPath string) {
|
func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, urlPath string) {
|
||||||
// Parse Maven path: group/artifact/version/filename
|
// Parse Maven path: group/artifact/version/filename
|
||||||
|
|
@ -86,6 +122,18 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, ur
|
||||||
|
|
||||||
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, upstreamURL)
|
result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, upstreamURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrUpstreamNotFound) {
|
||||||
|
pluginPortalURL := fmt.Sprintf("%s/%s", h.pluginPortalUpstreamURL, urlPath)
|
||||||
|
h.proxy.Logger.Info("maven artifact not found in primary upstream, trying Gradle Plugin Portal",
|
||||||
|
"group", group, "artifact", artifact, "version", version, "filename", filename)
|
||||||
|
result, err = h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, filename, pluginPortalURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrUpstreamNotFound) {
|
||||||
|
http.Error(w, "not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
h.proxy.Logger.Error("failed to get artifact", "error", err)
|
||||||
http.Error(w, "failed to fetch artifact", http.StatusBadGateway)
|
http.Error(w, "failed to fetch artifact", http.StatusBadGateway)
|
||||||
return
|
return
|
||||||
|
|
@ -115,7 +163,7 @@ func (h *MavenHandler) parsePath(urlPath string) (group, artifact, version, file
|
||||||
// isArtifactFile returns true if the filename looks like a Maven artifact.
|
// isArtifactFile returns true if the filename looks like a Maven artifact.
|
||||||
func (h *MavenHandler) isArtifactFile(filename string) bool {
|
func (h *MavenHandler) isArtifactFile(filename string) bool {
|
||||||
// Common artifact extensions
|
// Common artifact extensions
|
||||||
extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib"}
|
extensions := []string{".jar", ".war", ".ear", ".pom", ".aar", ".klib", ".module"}
|
||||||
for _, ext := range extensions {
|
for _, ext := range extensions {
|
||||||
if strings.HasSuffix(filename, ext) {
|
if strings.HasSuffix(filename, ext) {
|
||||||
return true
|
return true
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,7 @@ func TestMavenIsArtifactFile(t *testing.T) {
|
||||||
}{
|
}{
|
||||||
{"guava-32.1.3-jre.jar", true},
|
{"guava-32.1.3-jre.jar", true},
|
||||||
{"guava-32.1.3-jre.pom", true},
|
{"guava-32.1.3-jre.pom", true},
|
||||||
|
{"guava-32.1.3-jre.module", true},
|
||||||
{"app-1.0.war", true},
|
{"app-1.0.war", true},
|
||||||
{"lib-1.0.aar", true},
|
{"lib-1.0.aar", true},
|
||||||
{"maven-metadata.xml", false},
|
{"maven-metadata.xml", false},
|
||||||
|
|
@ -65,3 +66,63 @@ func TestMavenIsArtifactFile(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMavenIsMetadataFile(t *testing.T) {
|
||||||
|
h := &MavenHandler{}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
filename string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "pom is artifact, not metadata",
|
||||||
|
filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "pom checksum is metadata",
|
||||||
|
filename: "com.diffplug.spotless.gradle.plugin-8.4.0.pom.sha1",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "metadata file",
|
||||||
|
filename: "maven-metadata.xml",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "metadata checksum",
|
||||||
|
filename: "maven-metadata.xml.sha256",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "jar checksum is metadata",
|
||||||
|
filename: "guava-32.1.3-jre.jar.sha1",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "asc signature is metadata",
|
||||||
|
filename: "guava-32.1.3-jre.jar.asc",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "regular jar is not metadata",
|
||||||
|
filename: "guava-32.1.3-jre.jar",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "pom checksum is metadata",
|
||||||
|
filename: "guava-32.1.3-jre.pom.sha1",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := h.isMetadataFile(tt.filename)
|
||||||
|
if got != tt.want {
|
||||||
|
t.Errorf("isMetadataFile(%q) = %v, want %v", tt.filename, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
)
|
)
|
||||||
|
|
||||||
const testVersion100 = "1.0.0"
|
const testVersion100 = "1.0.0"
|
||||||
|
|
|
||||||
|
|
@ -172,7 +172,7 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request
|
||||||
http.Error(w, "failed to create request", http.StatusInternalServerError)
|
http.Error(w, "failed to create request", http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
req.Header.Set("Accept-Encoding", "gzip")
|
req.Header.Set(headerAcceptEncoding, "gzip")
|
||||||
|
|
||||||
resp, err := h.proxy.HTTPClient.Do(req)
|
resp, err := h.proxy.HTTPClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -193,7 +193,7 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := ReadMetadata(resp.Body)
|
body, err := h.proxy.ReadMetadata(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "failed to read response", http.StatusInternalServerError)
|
http.Error(w, "failed to read response", http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
|
|
@ -338,8 +338,8 @@ func (h *NuGetHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy accept-encoding for compression
|
// Copy accept-encoding for compression
|
||||||
if ae := r.Header.Get("Accept-Encoding"); ae != "" {
|
if ae := r.Header.Get(headerAcceptEncoding); ae != "" {
|
||||||
req.Header.Set("Accept-Encoding", ae)
|
req.Header.Set(headerAcceptEncoding, ae)
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err := h.proxy.HTTPClient.Do(req)
|
resp, err := h.proxy.HTTPClient.Do(req)
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
)
|
)
|
||||||
|
|
||||||
func nugetTestProxy() *Proxy {
|
func nugetTestProxy() *Proxy {
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,15 @@ func TestContainsPathTraversal(t *testing.T) {
|
||||||
{"pool/main/../../../etc/shadow", true},
|
{"pool/main/../../../etc/shadow", true},
|
||||||
{"pool/..hidden/file", false}, // ".." as a segment, not "..hidden"
|
{"pool/..hidden/file", false}, // ".." as a segment, not "..hidden"
|
||||||
{"", false},
|
{"", false},
|
||||||
|
{"%2e%2e/etc/passwd", true},
|
||||||
|
{"%2e%2e%2fetc%2fpasswd", true},
|
||||||
|
{"pool/%2e%2e/%2e%2e/etc/shadow", true},
|
||||||
|
{"%2E%2E%2Fetc", true},
|
||||||
|
{`..\\etc\\passwd`, true},
|
||||||
|
{`pool\\..\\..\\etc`, true},
|
||||||
|
{"%2e%2e%5cetc%5cpasswd", true},
|
||||||
|
{"pool/%2e%2ehidden/file", false},
|
||||||
|
{"pool/%zz/bad-encoding", false},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestPubRewriteMetadata(t *testing.T) {
|
func TestPubRewriteMetadata(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
"github.com/git-pkgs/registries/fetch"
|
"github.com/git-pkgs/registries/fetch"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,9 +7,12 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestReadMetadata(t *testing.T) {
|
func TestReadMetadata(t *testing.T) {
|
||||||
|
const limit = 1024
|
||||||
|
p := &Proxy{MetadataMaxSize: limit}
|
||||||
|
|
||||||
t.Run("small body", func(t *testing.T) {
|
t.Run("small body", func(t *testing.T) {
|
||||||
data := []byte("hello world")
|
data := []byte("hello world")
|
||||||
got, err := ReadMetadata(bytes.NewReader(data))
|
got, err := p.ReadMetadata(bytes.NewReader(data))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -19,27 +22,39 @@ func TestReadMetadata(t *testing.T) {
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("exactly at limit", func(t *testing.T) {
|
t.Run("exactly at limit", func(t *testing.T) {
|
||||||
data := make([]byte, maxMetadataSize)
|
data := make([]byte, limit)
|
||||||
for i := range data {
|
for i := range data {
|
||||||
data[i] = 'x'
|
data[i] = 'x'
|
||||||
}
|
}
|
||||||
got, err := ReadMetadata(bytes.NewReader(data))
|
got, err := p.ReadMetadata(bytes.NewReader(data))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
if len(got) != int(maxMetadataSize) {
|
if len(got) != limit {
|
||||||
t.Errorf("got length %d, want %d", len(got), maxMetadataSize)
|
t.Errorf("got length %d, want %d", len(got), limit)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("over limit returns error", func(t *testing.T) {
|
t.Run("over limit returns error", func(t *testing.T) {
|
||||||
data := make([]byte, maxMetadataSize+100)
|
data := make([]byte, limit+100)
|
||||||
for i := range data {
|
for i := range data {
|
||||||
data[i] = 'x'
|
data[i] = 'x'
|
||||||
}
|
}
|
||||||
_, err := ReadMetadata(bytes.NewReader(data))
|
_, err := p.ReadMetadata(bytes.NewReader(data))
|
||||||
if !errors.Is(err, ErrMetadataTooLarge) {
|
if !errors.Is(err, ErrMetadataTooLarge) {
|
||||||
t.Errorf("got error %v, want ErrMetadataTooLarge", err)
|
t.Errorf("got error %v, want ErrMetadataTooLarge", err)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("zero limit uses default", func(t *testing.T) {
|
||||||
|
p := &Proxy{}
|
||||||
|
data := make([]byte, 1<<20)
|
||||||
|
got, err := p.ReadMetadata(bytes.NewReader(data))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(got) != len(data) {
|
||||||
|
t.Errorf("got length %d, want %d", len(got), len(data))
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -120,6 +120,22 @@ var (
|
||||||
Help: "Number of currently active requests",
|
Help: "Number of currently active requests",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
IntegrityFailures = prometheus.NewCounterVec(
|
||||||
|
prometheus.CounterOpts{
|
||||||
|
Name: "proxy_integrity_failures_total",
|
||||||
|
Help: "Cached artifacts that failed hash verification on read",
|
||||||
|
},
|
||||||
|
[]string{"ecosystem"},
|
||||||
|
)
|
||||||
|
|
||||||
|
HealthProbeFailures = prometheus.NewCounterVec(
|
||||||
|
prometheus.CounterOpts{
|
||||||
|
Name: "proxy_health_probe_failures_total",
|
||||||
|
Help: "Total number of storage health probe failures, by step (write|size|read|verify|delete).",
|
||||||
|
},
|
||||||
|
[]string{"step"},
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
|
@ -138,6 +154,8 @@ func init() {
|
||||||
StorageOperationDuration,
|
StorageOperationDuration,
|
||||||
StorageErrors,
|
StorageErrors,
|
||||||
ActiveRequests,
|
ActiveRequests,
|
||||||
|
IntegrityFailures,
|
||||||
|
HealthProbeFailures,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -178,6 +196,17 @@ func RecordStorageOperation(operation string, duration time.Duration) {
|
||||||
StorageOperationDuration.WithLabelValues(operation).Observe(duration.Seconds())
|
StorageOperationDuration.WithLabelValues(operation).Observe(duration.Seconds())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RecordIntegrityFailure increments the integrity failure counter.
|
||||||
|
func RecordIntegrityFailure(ecosystem string) {
|
||||||
|
IntegrityFailures.WithLabelValues(ecosystem).Inc()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordHealthProbeFailure increments the health probe failure counter.
|
||||||
|
// step is one of: "write", "size", "read", "verify", "delete".
|
||||||
|
func RecordHealthProbeFailure(step string) {
|
||||||
|
HealthProbeFailures.WithLabelValues(step).Inc()
|
||||||
|
}
|
||||||
|
|
||||||
// RecordStorageError increments storage error counter.
|
// RecordStorageError increments storage error counter.
|
||||||
func RecordStorageError(operation string) {
|
func RecordStorageError(operation string) {
|
||||||
StorageErrors.WithLabelValues(operation).Inc()
|
StorageErrors.WithLabelValues(operation).Inc()
|
||||||
|
|
|
||||||
|
|
@ -140,10 +140,14 @@ type BulkResponse struct {
|
||||||
func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) {
|
func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) {
|
||||||
ecosystem := chi.URLParam(r, "ecosystem")
|
ecosystem := chi.URLParam(r, "ecosystem")
|
||||||
wildcard := chi.URLParam(r, "*")
|
wildcard := chi.URLParam(r, "*")
|
||||||
|
if err := validatePackagePath(wildcard); err != nil {
|
||||||
|
badRequest(w, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
segments := splitWildcardPath(wildcard)
|
segments := splitWildcardPath(wildcard)
|
||||||
|
|
||||||
if ecosystem == "" || len(segments) == 0 {
|
if ecosystem == "" || len(segments) == 0 {
|
||||||
http.Error(w, "ecosystem and name are required", http.StatusBadRequest)
|
badRequest(w, "ecosystem and name are required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -190,12 +194,12 @@ func (h *APIHandler) HandlePackagePath(w http.ResponseWriter, r *http.Request) {
|
||||||
func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosystem, name string) {
|
func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosystem, name string) {
|
||||||
info, err := h.enrichment.EnrichPackage(r.Context(), ecosystem, name)
|
info, err := h.enrichment.EnrichPackage(r.Context(), ecosystem, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "failed to enrich package", http.StatusInternalServerError)
|
writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to enrich package")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if info == nil {
|
if info == nil {
|
||||||
http.Error(w, "package not found", http.StatusNotFound)
|
notFound(w, "package not found")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -217,7 +221,7 @@ func (h *APIHandler) getPackage(w http.ResponseWriter, r *http.Request, ecosyste
|
||||||
func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) {
|
func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosystem, name, version string) {
|
||||||
result, err := h.enrichment.EnrichFull(r.Context(), ecosystem, name, version)
|
result, err := h.enrichment.EnrichFull(r.Context(), ecosystem, name, version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "failed to enrich version", http.StatusInternalServerError)
|
writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to enrich version")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -274,10 +278,14 @@ func (h *APIHandler) getVersion(w http.ResponseWriter, r *http.Request, ecosyste
|
||||||
func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
|
func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
|
||||||
ecosystem := chi.URLParam(r, "ecosystem")
|
ecosystem := chi.URLParam(r, "ecosystem")
|
||||||
wildcard := chi.URLParam(r, "*")
|
wildcard := chi.URLParam(r, "*")
|
||||||
|
if err := validatePackagePath(wildcard); err != nil {
|
||||||
|
badRequest(w, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
segments := splitWildcardPath(wildcard)
|
segments := splitWildcardPath(wildcard)
|
||||||
|
|
||||||
if ecosystem == "" || len(segments) == 0 {
|
if ecosystem == "" || len(segments) == 0 {
|
||||||
http.Error(w, "ecosystem and name are required", http.StatusBadRequest)
|
badRequest(w, "ecosystem and name are required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -298,7 +306,7 @@ func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|
||||||
vulns, err := h.enrichment.CheckVulnerabilities(r.Context(), ecosystem, name, version)
|
vulns, err := h.enrichment.CheckVulnerabilities(r.Context(), ecosystem, name, version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "failed to check vulnerabilities", http.StatusInternalServerError)
|
writeError(w, http.StatusBadGateway, ErrCodeUpstream, "failed to check vulnerabilities")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -330,19 +338,19 @@ func (h *APIHandler) HandleVulnsPath(w http.ResponseWriter, r *http.Request) {
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Param request body OutdatedRequest true "Packages to check"
|
// @Param request body OutdatedRequest true "Packages to check"
|
||||||
// @Success 200 {object} OutdatedResponse
|
// @Success 200 {object} OutdatedResponse
|
||||||
// @Failure 400 {string} string
|
// @Failure 400 {object} ErrorResponse
|
||||||
// @Failure 500 {string} string
|
// @Failure 500 {object} ErrorResponse
|
||||||
// @Router /api/outdated [post]
|
// @Router /api/outdated [post]
|
||||||
func (h *APIHandler) HandleOutdated(w http.ResponseWriter, r *http.Request) {
|
func (h *APIHandler) HandleOutdated(w http.ResponseWriter, r *http.Request) {
|
||||||
r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
|
r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
|
||||||
var req OutdatedRequest
|
var req OutdatedRequest
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
http.Error(w, "invalid request body", http.StatusBadRequest)
|
badRequest(w, "invalid request body")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(req.Packages) == 0 {
|
if len(req.Packages) == 0 {
|
||||||
http.Error(w, "packages list is required", http.StatusBadRequest)
|
badRequest(w, "packages list is required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -376,19 +384,19 @@ func (h *APIHandler) HandleOutdated(w http.ResponseWriter, r *http.Request) {
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Param request body BulkRequest true "PURLs"
|
// @Param request body BulkRequest true "PURLs"
|
||||||
// @Success 200 {object} BulkResponse
|
// @Success 200 {object} BulkResponse
|
||||||
// @Failure 400 {string} string
|
// @Failure 400 {object} ErrorResponse
|
||||||
// @Failure 500 {string} string
|
// @Failure 500 {object} ErrorResponse
|
||||||
// @Router /api/bulk [post]
|
// @Router /api/bulk [post]
|
||||||
func (h *APIHandler) HandleBulkLookup(w http.ResponseWriter, r *http.Request) {
|
func (h *APIHandler) HandleBulkLookup(w http.ResponseWriter, r *http.Request) {
|
||||||
r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
|
r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
|
||||||
var req BulkRequest
|
var req BulkRequest
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
http.Error(w, "invalid request body", http.StatusBadRequest)
|
badRequest(w, "invalid request body")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(req.PURLs) == 0 {
|
if len(req.PURLs) == 0 {
|
||||||
http.Error(w, "purls list is required", http.StatusBadRequest)
|
badRequest(w, "purls list is required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -476,15 +484,15 @@ type SearchPackageResult struct {
|
||||||
// @Param q query string true "Query"
|
// @Param q query string true "Query"
|
||||||
// @Param ecosystem query string false "Ecosystem"
|
// @Param ecosystem query string false "Ecosystem"
|
||||||
// @Success 200 {object} SearchResponse
|
// @Success 200 {object} SearchResponse
|
||||||
// @Failure 400 {string} string
|
// @Failure 400 {object} ErrorResponse
|
||||||
// @Failure 500 {string} string
|
// @Failure 500 {object} ErrorResponse
|
||||||
// @Router /api/search [get]
|
// @Router /api/search [get]
|
||||||
func (h *APIHandler) HandleSearch(w http.ResponseWriter, r *http.Request) {
|
func (h *APIHandler) HandleSearch(w http.ResponseWriter, r *http.Request) {
|
||||||
query := r.URL.Query().Get("q")
|
query := r.URL.Query().Get("q")
|
||||||
ecosystem := r.URL.Query().Get("ecosystem")
|
ecosystem := r.URL.Query().Get("ecosystem")
|
||||||
|
|
||||||
if query == "" {
|
if query == "" {
|
||||||
http.Error(w, "query parameter 'q' is required", http.StatusBadRequest)
|
badRequest(w, "query parameter 'q' is required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -494,7 +502,7 @@ func (h *APIHandler) HandleSearch(w http.ResponseWriter, r *http.Request) {
|
||||||
// Search in database
|
// Search in database
|
||||||
results, err := h.db.SearchPackages(query, ecosystem, limit, (page-1)*limit)
|
results, err := h.db.SearchPackages(query, ecosystem, limit, (page-1)*limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "search failed", http.StatusInternalServerError)
|
internalError(w, "search failed")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -538,7 +546,7 @@ func (h *APIHandler) HandleSearch(w http.ResponseWriter, r *http.Request) {
|
||||||
func writeJSON(w http.ResponseWriter, v any) {
|
func writeJSON(w http.ResponseWriter, v any) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
if err := json.NewEncoder(w).Encode(v); err != nil {
|
if err := json.NewEncoder(w).Encode(v); err != nil {
|
||||||
http.Error(w, "failed to encode response", http.StatusInternalServerError)
|
internalError(w, "failed to encode response")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -573,8 +581,8 @@ type PackageListResult struct {
|
||||||
// @Param ecosystem query string false "Ecosystem"
|
// @Param ecosystem query string false "Ecosystem"
|
||||||
// @Param sort query string false "Sort" Enums(hits,name,size,cached_at,ecosystem,vulns)
|
// @Param sort query string false "Sort" Enums(hits,name,size,cached_at,ecosystem,vulns)
|
||||||
// @Success 200 {object} PackagesListResponse
|
// @Success 200 {object} PackagesListResponse
|
||||||
// @Failure 400 {string} string
|
// @Failure 400 {object} ErrorResponse
|
||||||
// @Failure 500 {string} string
|
// @Failure 500 {object} ErrorResponse
|
||||||
// @Router /api/packages [get]
|
// @Router /api/packages [get]
|
||||||
func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request) {
|
func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request) {
|
||||||
ecosystem := r.URL.Query().Get("ecosystem")
|
ecosystem := r.URL.Query().Get("ecosystem")
|
||||||
|
|
@ -592,7 +600,7 @@ func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request)
|
||||||
"vulns": true,
|
"vulns": true,
|
||||||
}
|
}
|
||||||
if !validSorts[sortBy] {
|
if !validSorts[sortBy] {
|
||||||
http.Error(w, "invalid sort parameter", http.StatusBadRequest)
|
badRequest(w, "invalid sort parameter")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -601,7 +609,7 @@ func (h *APIHandler) HandlePackagesList(w http.ResponseWriter, r *http.Request)
|
||||||
|
|
||||||
packages, err := h.db.ListCachedPackages(ecosystem, sortBy, limit, (page-1)*limit)
|
packages, err := h.db.ListCachedPackages(ecosystem, sortBy, limit, (page-1)*limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "failed to list packages", http.StatusInternalServerError)
|
internalError(w, "failed to list packages")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,7 @@ import (
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/database"
|
"github.com/git-pkgs/proxy/internal/database"
|
||||||
|
|
@ -48,6 +49,35 @@ func TestHandlePackagePath_MissingParams(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlePackagePath_InvalidName(t *testing.T) {
|
||||||
|
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
||||||
|
svc := enrichment.New(logger)
|
||||||
|
h := NewAPIHandler(svc, nil)
|
||||||
|
|
||||||
|
r := chi.NewRouter()
|
||||||
|
r.Get("/api/package/{ecosystem}/*", h.HandlePackagePath)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
}{
|
||||||
|
{"null byte", "/api/package/npm/lodash%00"},
|
||||||
|
{"too long", "/api/package/npm/" + strings.Repeat("a", maxPackagePathLen+1)},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
req := httptest.NewRequest("GET", tt.path, nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
r.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusBadRequest {
|
||||||
|
t.Errorf("expected status 400, got %d", w.Code)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandleVulnsPath_MissingParams(t *testing.T) {
|
func TestHandleVulnsPath_MissingParams(t *testing.T) {
|
||||||
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
|
||||||
svc := enrichment.New(logger)
|
svc := enrichment.New(logger)
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
@ -69,11 +68,6 @@ func detectSingleRootDir(reader archives.Reader) string {
|
||||||
func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations
|
func openArchive(filename string, content io.Reader, ecosystem string) (archives.Reader, error) { //nolint:ireturn // wraps multiple archive implementations
|
||||||
fname := archiveFilename(filename)
|
fname := archiveFilename(filename)
|
||||||
|
|
||||||
// npm always uses package/ prefix
|
|
||||||
if ecosystem == "npm" {
|
|
||||||
return archives.OpenWithPrefix(fname, content, "package/")
|
|
||||||
}
|
|
||||||
|
|
||||||
limited := io.LimitReader(content, maxBrowseArchiveSize+1)
|
limited := io.LimitReader(content, maxBrowseArchiveSize+1)
|
||||||
data, err := io.ReadAll(limited)
|
data, err := io.ReadAll(limited)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -83,15 +77,18 @@ func openArchive(filename string, content io.Reader, ecosystem string) (archives
|
||||||
return nil, fmt.Errorf("artifact too large for browsing (%d bytes)", len(data))
|
return nil, fmt.Errorf("artifact too large for browsing (%d bytes)", len(data))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open once to detect root prefix
|
if ecosystem == "npm" {
|
||||||
probe, err := archives.Open(fname, bytes.NewReader(data))
|
return archives.OpenBytesWithPrefix(fname, data, "package/")
|
||||||
|
}
|
||||||
|
|
||||||
|
probe, err := archives.OpenBytes(fname, data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
prefix := detectSingleRootDir(probe)
|
prefix := detectSingleRootDir(probe)
|
||||||
_ = probe.Close()
|
_ = probe.Close()
|
||||||
|
|
||||||
return archives.OpenWithPrefix(fname, bytes.NewReader(data), prefix)
|
return archives.OpenBytesWithPrefix(fname, data, prefix)
|
||||||
}
|
}
|
||||||
|
|
||||||
// BrowseListResponse contains the file listing for a directory in an archives.
|
// BrowseListResponse contains the file listing for a directory in an archives.
|
||||||
|
|
@ -120,8 +117,8 @@ type BrowseFileInfo struct {
|
||||||
// @Param version path string true "Version"
|
// @Param version path string true "Version"
|
||||||
// @Param path query string false "Directory path inside the archive"
|
// @Param path query string false "Directory path inside the archive"
|
||||||
// @Success 200 {object} BrowseListResponse
|
// @Success 200 {object} BrowseListResponse
|
||||||
// @Failure 404 {string} string
|
// @Failure 404 {object} ErrorResponse
|
||||||
// @Failure 500 {string} string
|
// @Failure 500 {object} ErrorResponse
|
||||||
// @Router /api/browse/{ecosystem}/{name}/{version} [get]
|
// @Router /api/browse/{ecosystem}/{name}/{version} [get]
|
||||||
// handleBrowsePath dispatches /api/browse/{ecosystem}/* to the appropriate browse handler.
|
// handleBrowsePath dispatches /api/browse/{ecosystem}/* to the appropriate browse handler.
|
||||||
// It resolves namespaced package names by consulting the database.
|
// It resolves namespaced package names by consulting the database.
|
||||||
|
|
@ -133,10 +130,14 @@ type BrowseFileInfo struct {
|
||||||
func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
|
||||||
ecosystem := chi.URLParam(r, "ecosystem")
|
ecosystem := chi.URLParam(r, "ecosystem")
|
||||||
wildcard := chi.URLParam(r, "*")
|
wildcard := chi.URLParam(r, "*")
|
||||||
|
if err := validatePackagePath(wildcard); err != nil {
|
||||||
|
badRequest(w, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
segments := splitWildcardPath(wildcard)
|
segments := splitWildcardPath(wildcard)
|
||||||
|
|
||||||
if ecosystem == "" || len(segments) < 2 {
|
if ecosystem == "" || len(segments) < 2 {
|
||||||
http.Error(w, "ecosystem, name, and version required", http.StatusBadRequest)
|
badRequest(w, "ecosystem, name, and version required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -160,7 +161,7 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
|
||||||
rest = nameVersionSegments[len(nameVersionSegments)-1:]
|
rest = nameVersionSegments[len(nameVersionSegments)-1:]
|
||||||
}
|
}
|
||||||
if len(rest) != 1 {
|
if len(rest) != 1 {
|
||||||
http.Error(w, "not found", http.StatusNotFound)
|
notFound(w, "not found")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.browseFile(w, r, ecosystem, name, rest[0], filePath)
|
s.browseFile(w, r, ecosystem, name, rest[0], filePath)
|
||||||
|
|
@ -174,7 +175,7 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
|
||||||
rest = segments[len(segments)-1:]
|
rest = segments[len(segments)-1:]
|
||||||
}
|
}
|
||||||
if len(rest) != 1 {
|
if len(rest) != 1 {
|
||||||
http.Error(w, "not found", http.StatusNotFound)
|
notFound(w, "not found")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.browseList(w, r, ecosystem, name, rest[0])
|
s.browseList(w, r, ecosystem, name, rest[0])
|
||||||
|
|
@ -185,10 +186,14 @@ func (s *Server) handleBrowsePath(w http.ResponseWriter, r *http.Request) {
|
||||||
func (s *Server) handleComparePath(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleComparePath(w http.ResponseWriter, r *http.Request) {
|
||||||
ecosystem := chi.URLParam(r, "ecosystem")
|
ecosystem := chi.URLParam(r, "ecosystem")
|
||||||
wildcard := chi.URLParam(r, "*")
|
wildcard := chi.URLParam(r, "*")
|
||||||
|
if err := validatePackagePath(wildcard); err != nil {
|
||||||
|
badRequest(w, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
segments := splitWildcardPath(wildcard)
|
segments := splitWildcardPath(wildcard)
|
||||||
|
|
||||||
if ecosystem == "" || len(segments) < 3 {
|
if ecosystem == "" || len(segments) < 3 {
|
||||||
http.Error(w, "ecosystem, name, fromVersion, and toVersion required", http.StatusBadRequest)
|
badRequest(w, "ecosystem, name, fromVersion, and toVersion required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -208,12 +213,12 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
||||||
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
||||||
artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL)
|
artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "version not found", http.StatusNotFound)
|
notFound(w, "version not found")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(artifacts) == 0 {
|
if len(artifacts) == 0 {
|
||||||
http.Error(w, "no artifacts cached", http.StatusNotFound)
|
notFound(w, "no artifacts cached")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -227,7 +232,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
||||||
}
|
}
|
||||||
|
|
||||||
if cachedArtifact == nil {
|
if cachedArtifact == nil {
|
||||||
http.Error(w, "artifact not cached", http.StatusNotFound)
|
notFound(w, "artifact not cached")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -235,7 +240,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
||||||
artifactReader, err := s.storage.Open(r.Context(), cachedArtifact.StoragePath.String)
|
artifactReader, err := s.storage.Open(r.Context(), cachedArtifact.StoragePath.String)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("failed to read artifact from storage", "error", err)
|
s.logger.Error("failed to read artifact from storage", "error", err)
|
||||||
http.Error(w, "failed to read artifact", http.StatusInternalServerError)
|
internalError(w, "failed to read artifact")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer func() { _ = artifactReader.Close() }()
|
defer func() { _ = artifactReader.Close() }()
|
||||||
|
|
@ -244,7 +249,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
||||||
archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem)
|
archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename)
|
s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename)
|
||||||
http.Error(w, "failed to open archive", http.StatusInternalServerError)
|
internalError(w, "failed to open archive")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer func() { _ = archiveReader.Close() }()
|
defer func() { _ = archiveReader.Close() }()
|
||||||
|
|
@ -253,7 +258,7 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
||||||
files, err := archiveReader.ListDir(dirPath)
|
files, err := archiveReader.ListDir(dirPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("failed to list directory", "error", err, "path", dirPath)
|
s.logger.Error("failed to list directory", "error", err, "path", dirPath)
|
||||||
http.Error(w, "failed to list directory", http.StatusInternalServerError)
|
internalError(w, "failed to list directory")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -288,13 +293,13 @@ func (s *Server) browseList(w http.ResponseWriter, r *http.Request, ecosystem, n
|
||||||
// @Param version path string true "Version"
|
// @Param version path string true "Version"
|
||||||
// @Param filepath path string true "File path inside the archive"
|
// @Param filepath path string true "File path inside the archive"
|
||||||
// @Success 200 {file} file
|
// @Success 200 {file} file
|
||||||
// @Failure 400 {string} string
|
// @Failure 400 {object} ErrorResponse
|
||||||
// @Failure 404 {string} string
|
// @Failure 404 {object} ErrorResponse
|
||||||
// @Failure 500 {string} string
|
// @Failure 500 {object} ErrorResponse
|
||||||
// @Router /api/browse/{ecosystem}/{name}/{version}/file/{filepath} [get]
|
// @Router /api/browse/{ecosystem}/{name}/{version}/file/{filepath} [get]
|
||||||
func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, name, version, filePath string) {
|
func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, name, version, filePath string) {
|
||||||
if filePath == "" {
|
if filePath == "" {
|
||||||
http.Error(w, "file path required", http.StatusBadRequest)
|
badRequest(w, "file path required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -302,12 +307,12 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
|
||||||
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
versionPURL := purl.MakePURLString(ecosystem, name, version)
|
||||||
artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL)
|
artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "version not found", http.StatusNotFound)
|
notFound(w, "version not found")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(artifacts) == 0 {
|
if len(artifacts) == 0 {
|
||||||
http.Error(w, "no artifacts cached", http.StatusNotFound)
|
notFound(w, "no artifacts cached")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -321,7 +326,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
|
||||||
}
|
}
|
||||||
|
|
||||||
if cachedArtifact == nil {
|
if cachedArtifact == nil {
|
||||||
http.Error(w, "artifact not cached", http.StatusNotFound)
|
notFound(w, "artifact not cached")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -329,7 +334,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
|
||||||
artifactReader, err := s.storage.Open(r.Context(), cachedArtifact.StoragePath.String)
|
artifactReader, err := s.storage.Open(r.Context(), cachedArtifact.StoragePath.String)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("failed to read artifact from storage", "error", err)
|
s.logger.Error("failed to read artifact from storage", "error", err)
|
||||||
http.Error(w, "failed to read artifact", http.StatusInternalServerError)
|
internalError(w, "failed to read artifact")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer func() { _ = artifactReader.Close() }()
|
defer func() { _ = artifactReader.Close() }()
|
||||||
|
|
@ -338,7 +343,7 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
|
||||||
archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem)
|
archiveReader, err := openArchive(cachedArtifact.Filename, artifactReader, ecosystem)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename)
|
s.logger.Error("failed to open archive", "error", err, "filename", cachedArtifact.Filename)
|
||||||
http.Error(w, "failed to open archive", http.StatusInternalServerError)
|
internalError(w, "failed to open archive")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer func() { _ = archiveReader.Close() }()
|
defer func() { _ = archiveReader.Close() }()
|
||||||
|
|
@ -347,11 +352,11 @@ func (s *Server) browseFile(w http.ResponseWriter, r *http.Request, ecosystem, n
|
||||||
fileReader, err := archiveReader.Extract(filePath)
|
fileReader, err := archiveReader.Extract(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if strings.Contains(err.Error(), "not found") {
|
if strings.Contains(err.Error(), "not found") {
|
||||||
http.Error(w, "file not found", http.StatusNotFound)
|
notFound(w, "file not found")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.logger.Error("failed to extract file", "error", err, "path", filePath)
|
s.logger.Error("failed to extract file", "error", err, "path", filePath)
|
||||||
http.Error(w, "failed to extract file", http.StatusInternalServerError)
|
internalError(w, "failed to extract file")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer func() { _ = fileReader.Close() }()
|
defer func() { _ = fileReader.Close() }()
|
||||||
|
|
@ -491,8 +496,8 @@ type BrowseSourceData struct {
|
||||||
// @Param fromVersion path string true "From version"
|
// @Param fromVersion path string true "From version"
|
||||||
// @Param toVersion path string true "To version"
|
// @Param toVersion path string true "To version"
|
||||||
// @Success 200 {object} map[string]any
|
// @Success 200 {object} map[string]any
|
||||||
// @Failure 404 {string} string
|
// @Failure 404 {object} ErrorResponse
|
||||||
// @Failure 500 {string} string
|
// @Failure 500 {object} ErrorResponse
|
||||||
// @Router /api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion} [get]
|
// @Router /api/compare/{ecosystem}/{name}/{fromVersion}/{toVersion} [get]
|
||||||
func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, name, fromVersion, toVersion string) {
|
func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem, name, fromVersion, toVersion string) {
|
||||||
// Get artifacts for both versions
|
// Get artifacts for both versions
|
||||||
|
|
@ -501,13 +506,13 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
||||||
|
|
||||||
fromArtifacts, err := s.db.GetArtifactsByVersionPURL(fromPURL)
|
fromArtifacts, err := s.db.GetArtifactsByVersionPURL(fromPURL)
|
||||||
if err != nil || len(fromArtifacts) == 0 {
|
if err != nil || len(fromArtifacts) == 0 {
|
||||||
http.Error(w, "from version not found or not cached", http.StatusNotFound)
|
notFound(w, "from version not found or not cached")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
toArtifacts, err := s.db.GetArtifactsByVersionPURL(toPURL)
|
toArtifacts, err := s.db.GetArtifactsByVersionPURL(toPURL)
|
||||||
if err != nil || len(toArtifacts) == 0 {
|
if err != nil || len(toArtifacts) == 0 {
|
||||||
http.Error(w, "to version not found or not cached", http.StatusNotFound)
|
notFound(w, "to version not found or not cached")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -527,7 +532,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
||||||
}
|
}
|
||||||
|
|
||||||
if fromArtifact == nil || toArtifact == nil {
|
if fromArtifact == nil || toArtifact == nil {
|
||||||
http.Error(w, "one or both versions not cached", http.StatusNotFound)
|
notFound(w, "one or both versions not cached")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -535,7 +540,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
||||||
fromReader, err := s.storage.Open(r.Context(), fromArtifact.StoragePath.String)
|
fromReader, err := s.storage.Open(r.Context(), fromArtifact.StoragePath.String)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("failed to open from artifact", "error", err)
|
s.logger.Error("failed to open from artifact", "error", err)
|
||||||
http.Error(w, "failed to read from version", http.StatusInternalServerError)
|
internalError(w, "failed to read from version")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer func() { _ = fromReader.Close() }()
|
defer func() { _ = fromReader.Close() }()
|
||||||
|
|
@ -543,7 +548,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
||||||
toReader, err := s.storage.Open(r.Context(), toArtifact.StoragePath.String)
|
toReader, err := s.storage.Open(r.Context(), toArtifact.StoragePath.String)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("failed to open to artifact", "error", err)
|
s.logger.Error("failed to open to artifact", "error", err)
|
||||||
http.Error(w, "failed to read to version", http.StatusInternalServerError)
|
internalError(w, "failed to read to version")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer func() { _ = toReader.Close() }()
|
defer func() { _ = toReader.Close() }()
|
||||||
|
|
@ -551,7 +556,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
||||||
fromArchive, err := openArchive(fromArtifact.Filename, fromReader, ecosystem)
|
fromArchive, err := openArchive(fromArtifact.Filename, fromReader, ecosystem)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("failed to open from archive", "error", err)
|
s.logger.Error("failed to open from archive", "error", err)
|
||||||
http.Error(w, "failed to open from archive", http.StatusInternalServerError)
|
internalError(w, "failed to open from archive")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer func() { _ = fromArchive.Close() }()
|
defer func() { _ = fromArchive.Close() }()
|
||||||
|
|
@ -559,7 +564,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
||||||
toArchive, err := openArchive(toArtifact.Filename, toReader, ecosystem)
|
toArchive, err := openArchive(toArtifact.Filename, toReader, ecosystem)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("failed to open to archive", "error", err)
|
s.logger.Error("failed to open to archive", "error", err)
|
||||||
http.Error(w, "failed to open to archive", http.StatusInternalServerError)
|
internalError(w, "failed to open to archive")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer func() { _ = toArchive.Close() }()
|
defer func() { _ = toArchive.Close() }()
|
||||||
|
|
@ -568,7 +573,7 @@ func (s *Server) compareDiff(w http.ResponseWriter, r *http.Request, ecosystem,
|
||||||
result, err := diff.Compare(fromArchive, toArchive)
|
result, err := diff.Compare(fromArchive, toArchive)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("failed to generate diff", "error", err)
|
s.logger.Error("failed to generate diff", "error", err)
|
||||||
http.Error(w, "failed to generate diff", http.StatusInternalServerError)
|
internalError(w, "failed to generate diff")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
57
internal/server/browse_bench_test.go
Normal file
57
internal/server/browse_bench_test.go
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"fmt"
|
||||||
|
"math/rand"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func createBenchTarGz(prefix string, fileCount, fileSize int) []byte {
|
||||||
|
rnd := rand.New(rand.NewSource(1)) //nolint:gosec
|
||||||
|
buf := new(bytes.Buffer)
|
||||||
|
gw := gzip.NewWriter(buf)
|
||||||
|
tw := tar.NewWriter(gw)
|
||||||
|
|
||||||
|
payload := make([]byte, fileSize)
|
||||||
|
for i := range fileCount {
|
||||||
|
rnd.Read(payload)
|
||||||
|
_ = tw.WriteHeader(&tar.Header{
|
||||||
|
Name: fmt.Sprintf("%sfile%04d.dat", prefix, i),
|
||||||
|
Size: int64(fileSize),
|
||||||
|
Mode: 0644,
|
||||||
|
})
|
||||||
|
_, _ = tw.Write(payload)
|
||||||
|
}
|
||||||
|
_ = tw.Close()
|
||||||
|
_ = gw.Close()
|
||||||
|
return buf.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkOpenArchive(b *testing.B) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
ecosystem string
|
||||||
|
filename string
|
||||||
|
data []byte
|
||||||
|
}{
|
||||||
|
{"npm", "npm", "pkg.tgz", createBenchTarGz("package/", 64, 16*1024)},
|
||||||
|
{"go", "go", "v1.2.3.tar.gz", createBenchTarGz("repo-abc123/", 64, 16*1024)},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
b.Run(tc.name, func(b *testing.B) {
|
||||||
|
b.SetBytes(int64(len(tc.data)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
for b.Loop() {
|
||||||
|
r, err := openArchive(tc.filename, bytes.NewReader(tc.data), tc.ecosystem)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = r.Close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -204,17 +204,14 @@ func TestDetectContentType(t *testing.T) {
|
||||||
|
|
||||||
func TestOpenArchiveSizeLimit(t *testing.T) {
|
func TestOpenArchiveSizeLimit(t *testing.T) {
|
||||||
huge := bytes.Repeat([]byte("x"), int(maxBrowseArchiveSize)+1)
|
huge := bytes.Repeat([]byte("x"), int(maxBrowseArchiveSize)+1)
|
||||||
_, err := openArchive("test.tar.gz", bytes.NewReader(huge), "npm")
|
for _, eco := range []string{"npm", "go"} {
|
||||||
if err != nil {
|
_, err := openArchive("test.tar.gz", bytes.NewReader(huge), eco)
|
||||||
t.Log("npm path streams directly, error is acceptable:", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = openArchive("test.tar.gz", bytes.NewReader(huge), "go")
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("expected error for oversized archive, got nil")
|
t.Fatalf("%s: expected error for oversized archive, got nil", eco)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "too large") {
|
if !strings.Contains(err.Error(), "too large") {
|
||||||
t.Fatalf("expected 'too large' error, got: %v", err)
|
t.Fatalf("%s: expected 'too large' error, got: %v", eco, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -127,6 +127,7 @@ func supportedEcosystems() []string {
|
||||||
"gem",
|
"gem",
|
||||||
"golang",
|
"golang",
|
||||||
"hex",
|
"hex",
|
||||||
|
"julia",
|
||||||
"maven",
|
"maven",
|
||||||
"npm",
|
"npm",
|
||||||
"nuget",
|
"nuget",
|
||||||
|
|
@ -176,6 +177,8 @@ func ecosystemBadgeClasses(ecosystem string) string {
|
||||||
return base + " bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300"
|
return base + " bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300"
|
||||||
case "cran":
|
case "cran":
|
||||||
return base + " bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300"
|
return base + " bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300"
|
||||||
|
case "julia":
|
||||||
|
return base + " bg-emerald-100 text-emerald-700 dark:bg-emerald-900/50 dark:text-emerald-300"
|
||||||
case "oci":
|
case "oci":
|
||||||
return base + " bg-sky-100 text-sky-700 dark:bg-sky-900/50 dark:text-sky-300"
|
return base + " bg-sky-100 text-sky-700 dark:bg-sky-900/50 dark:text-sky-300"
|
||||||
case "deb":
|
case "deb":
|
||||||
|
|
@ -286,6 +289,20 @@ index-url = ` + baseURL + `/pypi/simple/</code></pre>`),
|
||||||
</mirror>
|
</mirror>
|
||||||
</mirrors>
|
</mirrors>
|
||||||
</settings></code></pre>`),
|
</settings></code></pre>`),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "gradle",
|
||||||
|
Name: "Gradle Build Cache",
|
||||||
|
Language: "Java/Kotlin",
|
||||||
|
Endpoint: "/gradle/",
|
||||||
|
Instructions: template.HTML(`<p class="config-note">Configure Gradle to use the proxy for HttpBuildCache:</p>
|
||||||
|
<pre><code>// In settings.gradle(.kts)
|
||||||
|
buildCache {
|
||||||
|
remote<HttpBuildCache> {
|
||||||
|
url = uri("` + baseURL + `/gradle/")
|
||||||
|
push = true
|
||||||
|
}
|
||||||
|
}</code></pre>`),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
ID: "nuget",
|
ID: "nuget",
|
||||||
|
|
@ -363,6 +380,17 @@ local({
|
||||||
r["CRAN"] <- "` + baseURL + `/cran"
|
r["CRAN"] <- "` + baseURL + `/cran"
|
||||||
options(repos = r)
|
options(repos = r)
|
||||||
})</code></pre>`),
|
})</code></pre>`),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "julia",
|
||||||
|
Name: "Julia",
|
||||||
|
Language: "Julia",
|
||||||
|
Endpoint: "/julia/",
|
||||||
|
Instructions: template.HTML(`<p class="config-note">Set the Pkg server before starting Julia:</p>
|
||||||
|
<pre><code>export JULIA_PKG_SERVER=` + baseURL + `/julia</code></pre>
|
||||||
|
<p class="config-note">Or inside a running session:</p>
|
||||||
|
<pre><code>ENV["JULIA_PKG_SERVER"] = "` + baseURL + `/julia"
|
||||||
|
using Pkg; Pkg.update()</code></pre>`),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
ID: "oci",
|
ID: "oci",
|
||||||
|
|
|
||||||
42
internal/server/errors.go
Normal file
42
internal/server/errors.go
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Error codes returned in API error responses. These are stable identifiers
|
||||||
|
// that clients can match on; the message text is for humans and may change.
|
||||||
|
const (
|
||||||
|
ErrCodeBadRequest = "BAD_REQUEST"
|
||||||
|
ErrCodeNotFound = "NOT_FOUND"
|
||||||
|
ErrCodeUpstream = "UPSTREAM_ERROR"
|
||||||
|
ErrCodeInternal = "INTERNAL_ERROR"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrorResponse is the JSON body returned for API errors.
|
||||||
|
type ErrorResponse struct {
|
||||||
|
Code string `json:"code"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeError sends a JSON error response with the given status, code and
|
||||||
|
// user-facing message. Internal error details should be logged separately
|
||||||
|
// by the caller, never passed as the message.
|
||||||
|
func writeError(w http.ResponseWriter, status int, code, message string) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_ = json.NewEncoder(w).Encode(ErrorResponse{Code: code, Message: message})
|
||||||
|
}
|
||||||
|
|
||||||
|
func badRequest(w http.ResponseWriter, message string) {
|
||||||
|
writeError(w, http.StatusBadRequest, ErrCodeBadRequest, message)
|
||||||
|
}
|
||||||
|
|
||||||
|
func notFound(w http.ResponseWriter, message string) {
|
||||||
|
writeError(w, http.StatusNotFound, ErrCodeNotFound, message)
|
||||||
|
}
|
||||||
|
|
||||||
|
func internalError(w http.ResponseWriter, message string) {
|
||||||
|
writeError(w, http.StatusInternalServerError, ErrCodeInternal, message)
|
||||||
|
}
|
||||||
93
internal/server/errors_test.go
Normal file
93
internal/server/errors_test.go
Normal file
|
|
@ -0,0 +1,93 @@
|
||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWriteError(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
fn func(w http.ResponseWriter)
|
||||||
|
status int
|
||||||
|
code string
|
||||||
|
message string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "badRequest",
|
||||||
|
fn: func(w http.ResponseWriter) { badRequest(w, "missing field") },
|
||||||
|
status: http.StatusBadRequest,
|
||||||
|
code: ErrCodeBadRequest,
|
||||||
|
message: "missing field",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "notFound",
|
||||||
|
fn: func(w http.ResponseWriter) { notFound(w, "package not found") },
|
||||||
|
status: http.StatusNotFound,
|
||||||
|
code: ErrCodeNotFound,
|
||||||
|
message: "package not found",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "internalError",
|
||||||
|
fn: func(w http.ResponseWriter) { internalError(w, "boom") },
|
||||||
|
status: http.StatusInternalServerError,
|
||||||
|
code: ErrCodeInternal,
|
||||||
|
message: "boom",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "upstream",
|
||||||
|
fn: func(w http.ResponseWriter) {
|
||||||
|
writeError(w, http.StatusBadGateway, ErrCodeUpstream, "registry unreachable")
|
||||||
|
},
|
||||||
|
status: http.StatusBadGateway,
|
||||||
|
code: ErrCodeUpstream,
|
||||||
|
message: "registry unreachable",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
tt.fn(w)
|
||||||
|
|
||||||
|
if w.Code != tt.status {
|
||||||
|
t.Errorf("status = %d, want %d", w.Code, tt.status)
|
||||||
|
}
|
||||||
|
if ct := w.Header().Get("Content-Type"); ct != "application/json" {
|
||||||
|
t.Errorf("Content-Type = %q, want application/json", ct)
|
||||||
|
}
|
||||||
|
|
||||||
|
var resp ErrorResponse
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("response body is not valid JSON: %v (body: %q)", err, w.Body.String())
|
||||||
|
}
|
||||||
|
if resp.Code != tt.code {
|
||||||
|
t.Errorf("code = %q, want %q", resp.Code, tt.code)
|
||||||
|
}
|
||||||
|
if resp.Message != tt.message {
|
||||||
|
t.Errorf("message = %q, want %q", resp.Message, tt.message)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIErrorResponseShape(t *testing.T) {
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
badRequest(w, "x")
|
||||||
|
|
||||||
|
var raw map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &raw); err != nil {
|
||||||
|
t.Fatalf("invalid JSON: %v", err)
|
||||||
|
}
|
||||||
|
if _, ok := raw["code"]; !ok {
|
||||||
|
t.Error("response missing 'code' field")
|
||||||
|
}
|
||||||
|
if _, ok := raw["message"]; !ok {
|
||||||
|
t.Error("response missing 'message' field")
|
||||||
|
}
|
||||||
|
if len(raw) != 2 {
|
||||||
|
t.Errorf("response has unexpected fields: %v", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
149
internal/server/gradle_cache_eviction.go
Normal file
149
internal/server/gradle_cache_eviction.go
Normal file
|
|
@ -0,0 +1,149 @@
|
||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/git-pkgs/proxy/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
const gradleBuildCacheStoragePrefix = "_gradle/http-build-cache/"
|
||||||
|
|
||||||
|
type gradleBuildCacheLister interface {
|
||||||
|
ListPrefix(ctx context.Context, prefix string) ([]storage.ObjectInfo, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) startGradleBuildCacheEviction(ctx context.Context) {
|
||||||
|
maxAge := s.cfg.ParseGradleBuildCacheMaxAge()
|
||||||
|
maxSize := s.cfg.ParseGradleBuildCacheMaxSize()
|
||||||
|
if maxAge <= 0 && maxSize <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
lister, ok := s.storage.(gradleBuildCacheLister)
|
||||||
|
if !ok {
|
||||||
|
s.logger.Warn("gradle cache eviction is enabled, but storage backend cannot list objects")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
interval := s.cfg.ParseGradleBuildCacheSweepInterval()
|
||||||
|
s.logger.Info("gradle cache eviction enabled",
|
||||||
|
"max_age", maxAge,
|
||||||
|
"max_size_bytes", maxSize,
|
||||||
|
"interval", interval)
|
||||||
|
|
||||||
|
sweep := func() {
|
||||||
|
deletedCount, freedBytes, err := sweepGradleBuildCache(ctx, s.storage, lister, maxAge, maxSize, time.Now())
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Warn("gradle cache eviction sweep failed", "error", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if deletedCount > 0 {
|
||||||
|
s.logger.Info("gradle cache eviction sweep completed",
|
||||||
|
"deleted_entries", deletedCount,
|
||||||
|
"freed_bytes", freedBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sweep()
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
ticker := time.NewTicker(interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
sweep()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
func sweepGradleBuildCache(
|
||||||
|
ctx context.Context,
|
||||||
|
store storage.Storage,
|
||||||
|
lister gradleBuildCacheLister,
|
||||||
|
maxAge time.Duration,
|
||||||
|
maxSize int64,
|
||||||
|
now time.Time,
|
||||||
|
) (int, int64, error) {
|
||||||
|
entries, err := lister.ListPrefix(ctx, gradleBuildCacheStoragePrefix)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("listing gradle cache entries: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(entries) == 0 {
|
||||||
|
return 0, 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sortOldestFirst(entries)
|
||||||
|
|
||||||
|
deletedCount := 0
|
||||||
|
freedBytes := int64(0)
|
||||||
|
var firstDeleteErr error
|
||||||
|
|
||||||
|
deleteEntry := func(entry storage.ObjectInfo) bool {
|
||||||
|
if err := store.Delete(ctx, entry.Path); err != nil {
|
||||||
|
if firstDeleteErr == nil {
|
||||||
|
firstDeleteErr = err
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
deletedCount++
|
||||||
|
freedBytes += entry.Size
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
remaining := entries
|
||||||
|
if maxAge > 0 {
|
||||||
|
cutoff := now.Add(-maxAge)
|
||||||
|
kept := make([]storage.ObjectInfo, 0, len(entries))
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
if !entry.ModTime.IsZero() && entry.ModTime.Before(cutoff) {
|
||||||
|
if deleteEntry(entry) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
kept = append(kept, entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
remaining = kept
|
||||||
|
}
|
||||||
|
|
||||||
|
if maxSize > 0 {
|
||||||
|
totalSize := int64(0)
|
||||||
|
for _, entry := range remaining {
|
||||||
|
totalSize += entry.Size
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range remaining {
|
||||||
|
if totalSize <= maxSize {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if deleteEntry(entry) {
|
||||||
|
totalSize -= entry.Size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if firstDeleteErr != nil {
|
||||||
|
return deletedCount, freedBytes, fmt.Errorf("deleting gradle cache entries: %w", firstDeleteErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
return deletedCount, freedBytes, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortOldestFirst(entries []storage.ObjectInfo) {
|
||||||
|
sort.Slice(entries, func(i, j int) bool {
|
||||||
|
if entries[i].ModTime.Equal(entries[j].ModTime) {
|
||||||
|
return entries[i].Path < entries[j].Path
|
||||||
|
}
|
||||||
|
return entries[i].ModTime.Before(entries[j].ModTime)
|
||||||
|
})
|
||||||
|
}
|
||||||
138
internal/server/gradle_cache_eviction_test.go
Normal file
138
internal/server/gradle_cache_eviction_test.go
Normal file
|
|
@ -0,0 +1,138 @@
|
||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/git-pkgs/proxy/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakeGradleCacheStore struct {
|
||||||
|
objects map[string]storage.ObjectInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFakeGradleCacheStore(objects []storage.ObjectInfo) *fakeGradleCacheStore {
|
||||||
|
m := make(map[string]storage.ObjectInfo, len(objects))
|
||||||
|
for _, obj := range objects {
|
||||||
|
m[obj.Path] = obj
|
||||||
|
}
|
||||||
|
return &fakeGradleCacheStore{objects: m}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeGradleCacheStore) Store(_ context.Context, path string, r io.Reader) (int64, string, error) {
|
||||||
|
data, _ := io.ReadAll(r)
|
||||||
|
s.objects[path] = storage.ObjectInfo{Path: path, Size: int64(len(data)), ModTime: time.Now()}
|
||||||
|
return int64(len(data)), "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeGradleCacheStore) Open(_ context.Context, path string) (io.ReadCloser, error) {
|
||||||
|
obj, ok := s.objects[path]
|
||||||
|
if !ok {
|
||||||
|
return nil, storage.ErrNotFound
|
||||||
|
}
|
||||||
|
return io.NopCloser(bytes.NewReader(make([]byte, obj.Size))), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeGradleCacheStore) Exists(_ context.Context, path string) (bool, error) {
|
||||||
|
_, ok := s.objects[path]
|
||||||
|
return ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeGradleCacheStore) Delete(_ context.Context, path string) error {
|
||||||
|
delete(s.objects, path)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeGradleCacheStore) Size(_ context.Context, path string) (int64, error) {
|
||||||
|
obj, ok := s.objects[path]
|
||||||
|
if !ok {
|
||||||
|
return 0, storage.ErrNotFound
|
||||||
|
}
|
||||||
|
return obj.Size, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeGradleCacheStore) SignedURL(_ context.Context, _ string, _ time.Duration) (string, error) {
|
||||||
|
return "", storage.ErrSignedURLUnsupported
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeGradleCacheStore) UsedSpace(_ context.Context) (int64, error) {
|
||||||
|
var total int64
|
||||||
|
for _, obj := range s.objects {
|
||||||
|
total += obj.Size
|
||||||
|
}
|
||||||
|
return total, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeGradleCacheStore) URL() string { return "mem://" }
|
||||||
|
|
||||||
|
func (s *fakeGradleCacheStore) Close() error { return nil }
|
||||||
|
|
||||||
|
func (s *fakeGradleCacheStore) ListPrefix(_ context.Context, prefix string) ([]storage.ObjectInfo, error) {
|
||||||
|
objects := make([]storage.ObjectInfo, 0)
|
||||||
|
for _, obj := range s.objects {
|
||||||
|
if strings.HasPrefix(obj.Path, prefix) {
|
||||||
|
objects = append(objects, obj)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return objects, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSweepGradleBuildCache_MaxAge(t *testing.T) {
|
||||||
|
now := time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC)
|
||||||
|
store := newFakeGradleCacheStore([]storage.ObjectInfo{
|
||||||
|
{Path: "_gradle/http-build-cache/old", Size: 10, ModTime: now.Add(-48 * time.Hour)},
|
||||||
|
{Path: "_gradle/http-build-cache/new", Size: 10, ModTime: now.Add(-2 * time.Hour)},
|
||||||
|
})
|
||||||
|
|
||||||
|
deleted, freed, err := sweepGradleBuildCache(context.Background(), store, store, 24*time.Hour, 0, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("sweepGradleBuildCache() error = %v", err)
|
||||||
|
}
|
||||||
|
if deleted != 1 {
|
||||||
|
t.Fatalf("deleted entries = %d, want 1", deleted)
|
||||||
|
}
|
||||||
|
if freed != 10 {
|
||||||
|
t.Fatalf("freed bytes = %d, want 10", freed)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := store.objects["_gradle/http-build-cache/old"]; ok {
|
||||||
|
t.Fatal("old entry was not deleted")
|
||||||
|
}
|
||||||
|
if _, ok := store.objects["_gradle/http-build-cache/new"]; !ok {
|
||||||
|
t.Fatal("new entry should remain")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSweepGradleBuildCache_MaxSizeOldestFirst(t *testing.T) {
|
||||||
|
now := time.Date(2026, 4, 27, 12, 0, 0, 0, time.UTC)
|
||||||
|
store := newFakeGradleCacheStore([]storage.ObjectInfo{
|
||||||
|
{Path: "_gradle/http-build-cache/a", Size: 5, ModTime: now.Add(-3 * time.Hour)},
|
||||||
|
{Path: "_gradle/http-build-cache/b", Size: 5, ModTime: now.Add(-2 * time.Hour)},
|
||||||
|
{Path: "_gradle/http-build-cache/c", Size: 5, ModTime: now.Add(-1 * time.Hour)},
|
||||||
|
})
|
||||||
|
|
||||||
|
deleted, freed, err := sweepGradleBuildCache(context.Background(), store, store, 0, 10, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("sweepGradleBuildCache() error = %v", err)
|
||||||
|
}
|
||||||
|
if deleted != 1 {
|
||||||
|
t.Fatalf("deleted entries = %d, want 1", deleted)
|
||||||
|
}
|
||||||
|
if freed != 5 {
|
||||||
|
t.Fatalf("freed bytes = %d, want 5", freed)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := store.objects["_gradle/http-build-cache/a"]; ok {
|
||||||
|
t.Fatal("oldest entry was not deleted")
|
||||||
|
}
|
||||||
|
if _, ok := store.objects["_gradle/http-build-cache/b"]; !ok {
|
||||||
|
t.Fatal("middle entry should remain")
|
||||||
|
}
|
||||||
|
if _, ok := store.objects["_gradle/http-build-cache/c"]; !ok {
|
||||||
|
t.Fatal("newest entry should remain")
|
||||||
|
}
|
||||||
|
}
|
||||||
182
internal/server/health.go
Normal file
182
internal/server/health.go
Normal file
|
|
@ -0,0 +1,182 @@
|
||||||
|
// Package server implements the proxy HTTP server.
|
||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/git-pkgs/proxy/internal/metrics"
|
||||||
|
"github.com/git-pkgs/proxy/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
probePathPrefix = ".healthcheck/"
|
||||||
|
probeMarker = "proxy-healthcheck:"
|
||||||
|
probeSuffixBytes = 8
|
||||||
|
defaultProbeTTL = 30 * time.Second
|
||||||
|
defaultProbeTimeout = 10 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// HealthResponse is the JSON payload returned by /health.
|
||||||
|
type HealthResponse struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Checks map[string]HealthCheck `json:"checks"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HealthCheck reports the status of a single subsystem check.
|
||||||
|
type HealthCheck struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
Step string `json:"step,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeError tags a storage probe failure with the step that failed.
|
||||||
|
type probeError struct {
|
||||||
|
step string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *probeError) Error() string { return e.step + ": " + e.err.Error() }
|
||||||
|
func (e *probeError) Unwrap() error { return e.err }
|
||||||
|
|
||||||
|
// storageProbe runs a write → size-check → read → verify → delete round-trip
|
||||||
|
// against the storage backend. Returns nil on success or a *probeError on failure.
|
||||||
|
func storageProbe(ctx context.Context, s storage.Storage) (err error) {
|
||||||
|
suffix, suffixErr := randomSuffix()
|
||||||
|
if suffixErr != nil {
|
||||||
|
return &probeError{step: "write", err: fmt.Errorf("generating random suffix: %w", suffixErr)}
|
||||||
|
}
|
||||||
|
path := probePathPrefix + strconv.FormatInt(time.Now().UnixNano(), 10) + "-" + suffix
|
||||||
|
payload := []byte(probeMarker + suffix)
|
||||||
|
|
||||||
|
// 1. Store
|
||||||
|
size, _, storeErr := s.Store(ctx, path, bytes.NewReader(payload))
|
||||||
|
if storeErr != nil {
|
||||||
|
return &probeError{step: "write", err: storeErr}
|
||||||
|
}
|
||||||
|
// After Store succeeds, always attempt to delete on the way out so probe
|
||||||
|
// objects don't accumulate when a later step (size/open/read/verify) fails.
|
||||||
|
// Delete is reported as the primary error only if no earlier failure
|
||||||
|
// already set one.
|
||||||
|
defer func() {
|
||||||
|
if delErr := s.Delete(ctx, path); delErr != nil && err == nil {
|
||||||
|
err = &probeError{step: "delete", err: delErr}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
// 2. Size check
|
||||||
|
if size != int64(len(payload)) {
|
||||||
|
return &probeError{step: "size", err: fmt.Errorf("wrote %d bytes, expected %d", size, len(payload))}
|
||||||
|
}
|
||||||
|
// 3. Open
|
||||||
|
rc, openErr := s.Open(ctx, path)
|
||||||
|
if openErr != nil {
|
||||||
|
return &probeError{step: "read", err: openErr}
|
||||||
|
}
|
||||||
|
// 4. Read all (classify mid-stream errors as read, not verify).
|
||||||
|
// Close explicitly (not deferred) so the file handle is released before
|
||||||
|
// Delete — on Windows, an open handle prevents deletion.
|
||||||
|
data, readErr := io.ReadAll(rc)
|
||||||
|
_ = rc.Close()
|
||||||
|
if readErr != nil {
|
||||||
|
return &probeError{step: "read", err: readErr}
|
||||||
|
}
|
||||||
|
// 5. Verify
|
||||||
|
if !bytes.Equal(data, payload) {
|
||||||
|
return &probeError{step: "verify", err: fmt.Errorf("content mismatch")}
|
||||||
|
}
|
||||||
|
// 6. Delete is handled via the deferred cleanup above.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// randomSuffix returns 8 cryptographically random bytes hex-encoded.
|
||||||
|
func randomSuffix() (string, error) {
|
||||||
|
b := make([]byte, probeSuffixBytes)
|
||||||
|
if _, err := rand.Read(b); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(b), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthCache memoizes the result of storageProbe for a configurable TTL.
|
||||||
|
// It is safe for concurrent use.
|
||||||
|
type healthCache struct {
|
||||||
|
storage storage.Storage
|
||||||
|
interval time.Duration
|
||||||
|
probeTimeout time.Duration
|
||||||
|
logger *slog.Logger
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
lastAt time.Time
|
||||||
|
lastErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
// newHealthCache builds a cache, parsing the interval from a duration string.
|
||||||
|
// Empty interval string defaults to 30s. "0" or "0s" disables caching.
|
||||||
|
func newHealthCache(s storage.Storage, intervalStr string, logger *slog.Logger) (*healthCache, error) {
|
||||||
|
interval := defaultProbeTTL
|
||||||
|
if intervalStr != "" {
|
||||||
|
d, err := time.ParseDuration(intervalStr)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parsing storage_probe_interval %q: %w", intervalStr, err)
|
||||||
|
}
|
||||||
|
interval = d
|
||||||
|
}
|
||||||
|
return &healthCache{
|
||||||
|
storage: s,
|
||||||
|
interval: interval,
|
||||||
|
probeTimeout: defaultProbeTimeout,
|
||||||
|
logger: logger,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check returns the cached probe result if still fresh, otherwise runs a fresh probe.
|
||||||
|
// The probe runs under a context derived from context.Background() with a fixed
|
||||||
|
// timeout so that caller cancellation (e.g. client disconnect) cannot poison the
|
||||||
|
// cache with context.Canceled.
|
||||||
|
func (c *healthCache) Check() error {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
// Cache hit
|
||||||
|
if c.interval > 0 && !c.lastAt.IsZero() && time.Since(c.lastAt) < c.interval {
|
||||||
|
return c.lastErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fresh probe under a detached context
|
||||||
|
probeCtx, cancel := context.WithTimeout(context.Background(), c.probeTimeout)
|
||||||
|
defer cancel()
|
||||||
|
err := storageProbe(probeCtx, c.storage)
|
||||||
|
|
||||||
|
// Transition logging and metric increment happen only on the fresh-probe path.
|
||||||
|
c.logTransition(c.lastErr, err)
|
||||||
|
if err != nil {
|
||||||
|
var pe *probeError
|
||||||
|
if errors.As(err, &pe) {
|
||||||
|
metrics.RecordHealthProbeFailure(pe.step)
|
||||||
|
} else {
|
||||||
|
metrics.RecordHealthProbeFailure("unknown")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
c.lastErr = err
|
||||||
|
c.lastAt = time.Now()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *healthCache) logTransition(prev, curr error) {
|
||||||
|
switch {
|
||||||
|
case prev != nil && curr == nil:
|
||||||
|
c.logger.Info("storage probe recovered")
|
||||||
|
case prev == nil && curr != nil:
|
||||||
|
c.logger.Error("storage probe failed", "error", curr.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
448
internal/server/health_test.go
Normal file
448
internal/server/health_test.go
Normal file
|
|
@ -0,0 +1,448 @@
|
||||||
|
package server
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/git-pkgs/proxy/internal/metrics"
|
||||||
|
"github.com/git-pkgs/proxy/internal/storage"
|
||||||
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeStorage is a minimal storage.Storage for probe tests with per-step failure injection.
|
||||||
|
type fakeStorage struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
|
||||||
|
storeCalls atomic.Int64
|
||||||
|
openCalls atomic.Int64
|
||||||
|
closeCalls atomic.Int64
|
||||||
|
deleteCalls atomic.Int64
|
||||||
|
|
||||||
|
paths []string
|
||||||
|
payloads [][]byte
|
||||||
|
|
||||||
|
// Failure injection.
|
||||||
|
storeErr error
|
||||||
|
openErr error
|
||||||
|
readErr error // returned by the io.ReadCloser.Read after partial bytes
|
||||||
|
deleteErr error
|
||||||
|
|
||||||
|
// Misbehavior knobs.
|
||||||
|
sizeDelta int64 // added to the reported size from Store
|
||||||
|
readOverride []byte // if non-nil, Open returns a reader yielding these bytes instead of stored content
|
||||||
|
|
||||||
|
// storeBlock, if non-nil, causes Store to block until the channel is closed or ctx is done.
|
||||||
|
storeBlock chan struct{}
|
||||||
|
|
||||||
|
stored map[string][]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFakeStorage() *fakeStorage { return &fakeStorage{stored: map[string][]byte{}} }
|
||||||
|
|
||||||
|
func (f *fakeStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) {
|
||||||
|
f.storeCalls.Add(1)
|
||||||
|
if f.storeErr != nil {
|
||||||
|
return 0, "", f.storeErr
|
||||||
|
}
|
||||||
|
if f.storeBlock != nil {
|
||||||
|
select {
|
||||||
|
case <-f.storeBlock:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return 0, "", ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(r)
|
||||||
|
if err != nil {
|
||||||
|
return 0, "", err
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
f.stored[path] = data
|
||||||
|
f.paths = append(f.paths, path)
|
||||||
|
f.payloads = append(f.payloads, data)
|
||||||
|
f.mu.Unlock()
|
||||||
|
return int64(len(data)) + f.sizeDelta, "fakehash", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeReadCloser struct {
|
||||||
|
data []byte
|
||||||
|
pos int
|
||||||
|
readErr error
|
||||||
|
closed *atomic.Int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (rc *fakeReadCloser) Read(p []byte) (int, error) {
|
||||||
|
if rc.pos >= len(rc.data) {
|
||||||
|
if rc.readErr != nil {
|
||||||
|
return 0, rc.readErr
|
||||||
|
}
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
n := copy(p, rc.data[rc.pos:])
|
||||||
|
rc.pos += n
|
||||||
|
if rc.pos >= len(rc.data) && rc.readErr != nil {
|
||||||
|
return n, rc.readErr
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (rc *fakeReadCloser) Close() error { rc.closed.Add(1); return nil }
|
||||||
|
|
||||||
|
func (f *fakeStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) {
|
||||||
|
f.openCalls.Add(1)
|
||||||
|
if f.openErr != nil {
|
||||||
|
return nil, f.openErr
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
data := f.stored[path]
|
||||||
|
f.mu.Unlock()
|
||||||
|
if f.readOverride != nil {
|
||||||
|
data = f.readOverride
|
||||||
|
}
|
||||||
|
return &fakeReadCloser{data: data, readErr: f.readErr, closed: &f.closeCalls}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeStorage) Exists(ctx context.Context, path string) (bool, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
_, ok := f.stored[path]
|
||||||
|
return ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeStorage) Delete(ctx context.Context, path string) error {
|
||||||
|
f.deleteCalls.Add(1)
|
||||||
|
if f.deleteErr != nil {
|
||||||
|
return f.deleteErr
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
delete(f.stored, path)
|
||||||
|
f.mu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeStorage) Size(ctx context.Context, path string) (int64, error) { return 0, nil }
|
||||||
|
func (f *fakeStorage) SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error) {
|
||||||
|
return "", storage.ErrSignedURLUnsupported
|
||||||
|
}
|
||||||
|
func (f *fakeStorage) UsedSpace(ctx context.Context) (int64, error) { return 0, nil }
|
||||||
|
func (f *fakeStorage) URL() string { return "fake://" }
|
||||||
|
func (f *fakeStorage) Close() error { return nil }
|
||||||
|
|
||||||
|
// --- Tests follow. First test: happy path ---
|
||||||
|
|
||||||
|
func TestStorageProbe_HappyPath(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
if err := storageProbe(context.Background(), fs); err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if got := fs.storeCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("Store calls = %d, want 1", got)
|
||||||
|
}
|
||||||
|
if got := fs.openCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("Open calls = %d, want 1", got)
|
||||||
|
}
|
||||||
|
if got := fs.closeCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("Close calls = %d, want 1", got)
|
||||||
|
}
|
||||||
|
if got := fs.deleteCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("Delete calls = %d, want 1", got)
|
||||||
|
}
|
||||||
|
if len(fs.paths) != 1 || !strings.HasPrefix(fs.paths[0], ".healthcheck/") {
|
||||||
|
t.Errorf("unexpected probe path: %v", fs.paths)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageProbe_WriteFails(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
fs.storeErr = errors.New("disk full")
|
||||||
|
err := storageProbe(context.Background(), fs)
|
||||||
|
var pe *probeError
|
||||||
|
if !errors.As(err, &pe) {
|
||||||
|
t.Fatalf("expected *probeError, got %T: %v", err, err)
|
||||||
|
}
|
||||||
|
if pe.step != "write" {
|
||||||
|
t.Errorf("step = %q, want write", pe.step)
|
||||||
|
}
|
||||||
|
if fs.openCalls.Load() != 0 {
|
||||||
|
t.Errorf("Open should not be called after write failure")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageProbe_SizeMismatch(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
fs.sizeDelta = -1 // Report 1 byte fewer than actually written
|
||||||
|
err := storageProbe(context.Background(), fs)
|
||||||
|
var pe *probeError
|
||||||
|
if !errors.As(err, &pe) || pe.step != "size" {
|
||||||
|
t.Fatalf("step = %v, want size; err = %v", pe, err)
|
||||||
|
}
|
||||||
|
if fs.openCalls.Load() != 0 {
|
||||||
|
t.Errorf("Open should not be called after size mismatch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageProbe_OpenFails(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
fs.openErr = errors.New("access denied")
|
||||||
|
err := storageProbe(context.Background(), fs)
|
||||||
|
var pe *probeError
|
||||||
|
if !errors.As(err, &pe) || pe.step != "read" {
|
||||||
|
t.Fatalf("step = %v, want read; err = %v", pe, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageProbe_ReadMidStreamFails(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
fs.readErr = errors.New("connection reset")
|
||||||
|
err := storageProbe(context.Background(), fs)
|
||||||
|
var pe *probeError
|
||||||
|
if !errors.As(err, &pe) || pe.step != "read" {
|
||||||
|
t.Fatalf("step = %v, want read (NOT verify); err = %v", pe, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageProbe_ContentMismatch(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
fs.readOverride = []byte("wrong content")
|
||||||
|
err := storageProbe(context.Background(), fs)
|
||||||
|
var pe *probeError
|
||||||
|
if !errors.As(err, &pe) || pe.step != "verify" {
|
||||||
|
t.Fatalf("step = %v, want verify; err = %v", pe, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageProbe_DeleteFails(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
fs.deleteErr = errors.New("permission denied")
|
||||||
|
err := storageProbe(context.Background(), fs)
|
||||||
|
var pe *probeError
|
||||||
|
if !errors.As(err, &pe) || pe.step != "delete" {
|
||||||
|
t.Fatalf("step = %v, want delete; err = %v", pe, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStorageProbe_CleanupOnNonDeleteFailure asserts that the probe object is
|
||||||
|
// deleted even when a step after Store (size/open/read/verify) fails, so
|
||||||
|
// probe artifacts don't accumulate in the storage backend.
|
||||||
|
func TestStorageProbe_CleanupOnNonDeleteFailure(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
inject func(*fakeStorage)
|
||||||
|
wantErr string
|
||||||
|
}{
|
||||||
|
{"size mismatch", func(fs *fakeStorage) { fs.sizeDelta = -1 }, "size"},
|
||||||
|
{"open fails", func(fs *fakeStorage) { fs.openErr = errors.New("open boom") }, "read"},
|
||||||
|
{"read mid-stream", func(fs *fakeStorage) { fs.readErr = errors.New("mid-stream boom") }, "read"},
|
||||||
|
{"content mismatch", func(fs *fakeStorage) { fs.readOverride = []byte("wrong") }, "verify"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
tc.inject(fs)
|
||||||
|
err := storageProbe(context.Background(), fs)
|
||||||
|
var pe *probeError
|
||||||
|
if !errors.As(err, &pe) || pe.step != tc.wantErr {
|
||||||
|
t.Fatalf("step = %v, want %q; err = %v", pe, tc.wantErr, err)
|
||||||
|
}
|
||||||
|
if got := fs.deleteCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("deleteCalls = %d, want 1 (cleanup should run on non-delete failures)", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageProbe_ReaderClosedOnReadFailure(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
fs.readErr = errors.New("read error")
|
||||||
|
_ = storageProbe(context.Background(), fs)
|
||||||
|
if got := fs.closeCalls.Load(); got != fs.openCalls.Load() {
|
||||||
|
t.Errorf("closeCalls = %d, openCalls = %d (should match)", got, fs.openCalls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStorageProbe_PathUniqueness(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
if err := storageProbe(context.Background(), fs); err != nil {
|
||||||
|
t.Fatalf("probe %d: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
for _, p := range fs.paths {
|
||||||
|
if !strings.HasPrefix(p, ".healthcheck/") {
|
||||||
|
t.Errorf("path missing prefix: %q", p)
|
||||||
|
}
|
||||||
|
if seen[p] {
|
||||||
|
t.Errorf("duplicate path: %q", p)
|
||||||
|
}
|
||||||
|
seen[p] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// helper: a healthCache wired to a fakeStorage and a discard logger.
|
||||||
|
func newTestCache(fs *fakeStorage, interval time.Duration) *healthCache {
|
||||||
|
return &healthCache{
|
||||||
|
storage: fs,
|
||||||
|
interval: interval,
|
||||||
|
probeTimeout: 5 * time.Second,
|
||||||
|
logger: discardLogger(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func discardLogger() *slog.Logger {
|
||||||
|
return slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthCache_CacheHit(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
c := newTestCache(fs, 30*time.Second)
|
||||||
|
if err := c.Check(); err != nil {
|
||||||
|
t.Fatalf("first check: %v", err)
|
||||||
|
}
|
||||||
|
if err := c.Check(); err != nil {
|
||||||
|
t.Fatalf("second check: %v", err)
|
||||||
|
}
|
||||||
|
if got := fs.storeCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("storeCalls = %d, want 1 (second call should be cached)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthCache_MissAfterTTL(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
c := newTestCache(fs, 10*time.Millisecond)
|
||||||
|
_ = c.Check()
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
_ = c.Check()
|
||||||
|
if got := fs.storeCalls.Load(); got != 2 {
|
||||||
|
t.Errorf("storeCalls = %d, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthCache_Disabled(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
c := newTestCache(fs, 0) // interval = 0 means probe every call
|
||||||
|
_ = c.Check()
|
||||||
|
_ = c.Check()
|
||||||
|
if got := fs.storeCalls.Load(); got != 2 {
|
||||||
|
t.Errorf("storeCalls = %d, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthCache_LastAtNotAdvancedOnHit(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
c := newTestCache(fs, 30*time.Second)
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
_ = c.Check()
|
||||||
|
}
|
||||||
|
if got := fs.storeCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("storeCalls = %d, want 1 across 100 hits", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthCache_ConcurrentSingleFlight(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
c := newTestCache(fs, 30*time.Second)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < 20; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() { defer wg.Done(); _ = c.Check() }()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
if got := fs.storeCalls.Load(); got != 1 {
|
||||||
|
t.Errorf("storeCalls = %d, want 1 with 20 concurrent callers", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthCache_FailureCounterIncrement(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
fs.storeErr = errors.New("boom")
|
||||||
|
c := newTestCache(fs, 30*time.Second)
|
||||||
|
|
||||||
|
before := testutil.ToFloat64(metrics.HealthProbeFailures.WithLabelValues("write"))
|
||||||
|
|
||||||
|
// First call: fresh probe → counter +1
|
||||||
|
_ = c.Check()
|
||||||
|
afterFirst := testutil.ToFloat64(metrics.HealthProbeFailures.WithLabelValues("write"))
|
||||||
|
if afterFirst-before != 1 {
|
||||||
|
t.Errorf("counter delta after first call = %v, want 1", afterFirst-before)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Second call: cache hit → counter NOT re-incremented
|
||||||
|
_ = c.Check()
|
||||||
|
afterSecond := testutil.ToFloat64(metrics.HealthProbeFailures.WithLabelValues("write"))
|
||||||
|
if afterSecond != afterFirst {
|
||||||
|
t.Errorf("counter changed on cache hit: %v → %v", afterFirst, afterSecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthCache_ProbeTimeout(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
fs.storeBlock = make(chan struct{}) // Store will block until channel is closed (or never)
|
||||||
|
t.Cleanup(func() { close(fs.storeBlock) })
|
||||||
|
|
||||||
|
c := &healthCache{
|
||||||
|
storage: fs,
|
||||||
|
interval: 30 * time.Second,
|
||||||
|
probeTimeout: 50 * time.Millisecond,
|
||||||
|
logger: discardLogger(),
|
||||||
|
}
|
||||||
|
start := time.Now()
|
||||||
|
err := c.Check()
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected timeout error, got nil")
|
||||||
|
}
|
||||||
|
if elapsed > 500*time.Millisecond {
|
||||||
|
t.Errorf("probe took %v, expected ~50ms (timeout not respected)", elapsed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHealthCache_TransitionLogging(t *testing.T) {
|
||||||
|
fs := newFakeStorage()
|
||||||
|
var buf bytes.Buffer
|
||||||
|
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||||
|
c := &healthCache{
|
||||||
|
storage: fs,
|
||||||
|
interval: 0, // probe every call
|
||||||
|
probeTimeout: 5 * time.Second,
|
||||||
|
logger: logger,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Steady ok state — should not log
|
||||||
|
_ = c.Check()
|
||||||
|
_ = c.Check()
|
||||||
|
if got := strings.Count(buf.String(), "storage probe"); got != 0 {
|
||||||
|
t.Errorf("steady-state logs = %d, want 0; output: %s", got, buf.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ok → err transition: exactly one Error log
|
||||||
|
buf.Reset()
|
||||||
|
fs.storeErr = errors.New("boom")
|
||||||
|
_ = c.Check()
|
||||||
|
if !strings.Contains(buf.String(), "storage probe failed") {
|
||||||
|
t.Errorf("missing failure log on transition; output: %s", buf.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// err steady state — should not log again
|
||||||
|
buf.Reset()
|
||||||
|
_ = c.Check()
|
||||||
|
if buf.Len() != 0 {
|
||||||
|
t.Errorf("steady-err logs = %q, want empty", buf.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// err → ok transition: exactly one Info log
|
||||||
|
buf.Reset()
|
||||||
|
fs.storeErr = nil
|
||||||
|
_ = c.Check()
|
||||||
|
if !strings.Contains(buf.String(), "storage probe recovered") {
|
||||||
|
t.Errorf("missing recovery log on transition; output: %s", buf.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -23,17 +23,13 @@ func (h *MirrorAPIHandler) HandleCreate(w http.ResponseWriter, r *http.Request)
|
||||||
r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
|
r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
|
||||||
var req mirror.JobRequest
|
var req mirror.JobRequest
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
badRequest(w, "invalid request body")
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
writeJSON(w, map[string]string{"error": "invalid request body"})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
id, err := h.jobs.Create(req)
|
id, err := h.jobs.Create(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
badRequest(w, "invalid mirror job request")
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
writeJSON(w, map[string]string{"error": err.Error()})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -47,13 +43,10 @@ func (h *MirrorAPIHandler) HandleGet(w http.ResponseWriter, r *http.Request) {
|
||||||
id := chi.URLParam(r, "id")
|
id := chi.URLParam(r, "id")
|
||||||
job := h.jobs.Get(id)
|
job := h.jobs.Get(id)
|
||||||
if job == nil {
|
if job == nil {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
notFound(w, "job not found")
|
||||||
w.WriteHeader(http.StatusNotFound)
|
|
||||||
writeJSON(w, map[string]string{"error": "job not found"})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
writeJSON(w, job)
|
writeJSON(w, job)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -61,11 +54,8 @@ func (h *MirrorAPIHandler) HandleGet(w http.ResponseWriter, r *http.Request) {
|
||||||
func (h *MirrorAPIHandler) HandleCancel(w http.ResponseWriter, r *http.Request) {
|
func (h *MirrorAPIHandler) HandleCancel(w http.ResponseWriter, r *http.Request) {
|
||||||
id := chi.URLParam(r, "id")
|
id := chi.URLParam(r, "id")
|
||||||
if h.jobs.Cancel(id) {
|
if h.jobs.Cancel(id) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
writeJSON(w, map[string]string{"status": "canceled"})
|
writeJSON(w, map[string]string{"status": "canceled"})
|
||||||
} else {
|
} else {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
notFound(w, "job not found or not running")
|
||||||
w.WriteHeader(http.StatusNotFound)
|
|
||||||
writeJSON(w, map[string]string{"error": "job not found or not running"})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,39 @@
|
||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/database"
|
"github.com/git-pkgs/proxy/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// maxPackagePathLen bounds the wildcard portion of package routes (name plus
|
||||||
|
// version and any suffix). npm caps names at 214 and Maven coordinates can be
|
||||||
|
// longer, so 512 leaves room without admitting pathological inputs.
|
||||||
|
const maxPackagePathLen = 512
|
||||||
|
|
||||||
|
// validatePackagePath rejects wildcard package paths that cannot be valid in
|
||||||
|
// any supported ecosystem. It is a coarse filter applied before database or
|
||||||
|
// enrichment lookups; ecosystem-specific name rules are layered on top.
|
||||||
|
func validatePackagePath(path string) error {
|
||||||
|
if path == "" {
|
||||||
|
return fmt.Errorf("package name required")
|
||||||
|
}
|
||||||
|
if len(path) > maxPackagePathLen {
|
||||||
|
return fmt.Errorf("package path exceeds %d bytes", maxPackagePathLen)
|
||||||
|
}
|
||||||
|
for _, r := range path {
|
||||||
|
if r == 0 {
|
||||||
|
return fmt.Errorf("package path contains null byte")
|
||||||
|
}
|
||||||
|
if unicode.IsControl(r) {
|
||||||
|
return fmt.Errorf("package path contains control character %#U", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// resolvePackageName determines the package name from a wildcard path by
|
// resolvePackageName determines the package name from a wildcard path by
|
||||||
// checking the database. This handles namespaced packages like Composer's
|
// checking the database. This handles namespaced packages like Composer's
|
||||||
// vendor/name format where the package name contains a slash.
|
// vendor/name format where the package name contains a slash.
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ package server
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/git-pkgs/proxy/internal/database"
|
"github.com/git-pkgs/proxy/internal/database"
|
||||||
|
|
@ -118,3 +119,36 @@ func TestSplitWildcardPath(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidatePackagePath(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{"simple", "lodash", false},
|
||||||
|
{"with version", "lodash/4.17.21", false},
|
||||||
|
{"npm scoped", "@babel/core/7.0.0", false},
|
||||||
|
{"composer namespaced", "symfony/console/6.0.0", false},
|
||||||
|
{"maven coordinates", "org.apache.commons/commons-lang3/3.12.0", false},
|
||||||
|
{"unicode", "café/1.0.0", false},
|
||||||
|
{"empty", "", true},
|
||||||
|
{"null byte", "lodash\x00/4.17.21", true},
|
||||||
|
{"null byte suffix", "lodash\x00", true},
|
||||||
|
{"newline", "lodash\n4.17.21", true},
|
||||||
|
{"carriage return", "lodash\r", true},
|
||||||
|
{"escape", "lodash\x1b[31m", true},
|
||||||
|
{"delete", "lodash\x7f", true},
|
||||||
|
{"too long", strings.Repeat("a", maxPackagePathLen+1), true},
|
||||||
|
{"at limit", strings.Repeat("a", maxPackagePathLen), false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := validatePackagePath(tt.path)
|
||||||
|
if (err != nil) != tt.wantErr {
|
||||||
|
t.Errorf("validatePackagePath(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,11 +9,13 @@
|
||||||
// - /pub/* - pub.dev registry protocol
|
// - /pub/* - pub.dev registry protocol
|
||||||
// - /pypi/* - PyPI registry protocol
|
// - /pypi/* - PyPI registry protocol
|
||||||
// - /maven/* - Maven repository protocol
|
// - /maven/* - Maven repository protocol
|
||||||
|
// - /gradle/* - Gradle HttpBuildCache protocol
|
||||||
// - /nuget/* - NuGet V3 API protocol
|
// - /nuget/* - NuGet V3 API protocol
|
||||||
// - /composer/* - Composer/Packagist protocol
|
// - /composer/* - Composer/Packagist protocol
|
||||||
// - /conan/* - Conan C/C++ protocol
|
// - /conan/* - Conan C/C++ protocol
|
||||||
// - /conda/* - Conda/Anaconda protocol
|
// - /conda/* - Conda/Anaconda protocol
|
||||||
// - /cran/* - CRAN (R) protocol
|
// - /cran/* - CRAN (R) protocol
|
||||||
|
// - /julia/* - Julia Pkg server protocol
|
||||||
// - /v2/* - OCI/Docker container registry protocol
|
// - /v2/* - OCI/Docker container registry protocol
|
||||||
// - /debian/* - Debian/APT repository protocol
|
// - /debian/* - Debian/APT repository protocol
|
||||||
// - /rpm/* - RPM/Yum repository protocol
|
// - /rpm/* - RPM/Yum repository protocol
|
||||||
|
|
@ -39,6 +41,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
@ -48,7 +51,7 @@ import (
|
||||||
|
|
||||||
swaggerdoc "github.com/git-pkgs/proxy/docs/swagger"
|
swaggerdoc "github.com/git-pkgs/proxy/docs/swagger"
|
||||||
"github.com/git-pkgs/proxy/internal/config"
|
"github.com/git-pkgs/proxy/internal/config"
|
||||||
"github.com/git-pkgs/proxy/internal/cooldown"
|
"github.com/git-pkgs/cooldown"
|
||||||
"github.com/git-pkgs/proxy/internal/database"
|
"github.com/git-pkgs/proxy/internal/database"
|
||||||
"github.com/git-pkgs/proxy/internal/enrichment"
|
"github.com/git-pkgs/proxy/internal/enrichment"
|
||||||
"github.com/git-pkgs/proxy/internal/handler"
|
"github.com/git-pkgs/proxy/internal/handler"
|
||||||
|
|
@ -79,6 +82,7 @@ type Server struct {
|
||||||
http *http.Server
|
http *http.Server
|
||||||
templates *Templates
|
templates *Templates
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
|
healthCache *healthCache
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new Server with the given configuration.
|
// New creates a new Server with the given configuration.
|
||||||
|
|
@ -124,12 +128,20 @@ func New(cfg *config.Config, logger *slog.Logger) (*Server, error) {
|
||||||
return nil, fmt.Errorf("verifying storage connectivity: %w", err)
|
return nil, fmt.Errorf("verifying storage connectivity: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
hc, err := newHealthCache(store, cfg.Health.StorageProbeInterval, logger)
|
||||||
|
if err != nil {
|
||||||
|
_ = store.Close()
|
||||||
|
_ = db.Close()
|
||||||
|
return nil, fmt.Errorf("initializing health cache: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
return &Server{
|
return &Server{
|
||||||
cfg: cfg,
|
cfg: cfg,
|
||||||
db: db,
|
db: db,
|
||||||
storage: store,
|
storage: store,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
templates: &Templates{},
|
templates: &Templates{},
|
||||||
|
healthCache: hc,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -148,6 +160,9 @@ func (s *Server) Start() error {
|
||||||
proxy.Cooldown = cd
|
proxy.Cooldown = cd
|
||||||
proxy.CacheMetadata = s.cfg.CacheMetadata
|
proxy.CacheMetadata = s.cfg.CacheMetadata
|
||||||
proxy.MetadataTTL = s.cfg.ParseMetadataTTL()
|
proxy.MetadataTTL = s.cfg.ParseMetadataTTL()
|
||||||
|
proxy.MetadataMaxSize = s.cfg.ParseMetadataMaxSize()
|
||||||
|
proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly
|
||||||
|
proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize()
|
||||||
proxy.DirectServe = s.cfg.Storage.DirectServe
|
proxy.DirectServe = s.cfg.Storage.DirectServe
|
||||||
proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL()
|
proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL()
|
||||||
proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL
|
proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL
|
||||||
|
|
@ -179,12 +194,19 @@ func (s *Server) Start() error {
|
||||||
hexHandler := handler.NewHexHandler(proxy, s.cfg.BaseURL)
|
hexHandler := handler.NewHexHandler(proxy, s.cfg.BaseURL)
|
||||||
pubHandler := handler.NewPubHandler(proxy, s.cfg.BaseURL)
|
pubHandler := handler.NewPubHandler(proxy, s.cfg.BaseURL)
|
||||||
pypiHandler := handler.NewPyPIHandler(proxy, s.cfg.BaseURL)
|
pypiHandler := handler.NewPyPIHandler(proxy, s.cfg.BaseURL)
|
||||||
mavenHandler := handler.NewMavenHandler(proxy, s.cfg.BaseURL)
|
mavenHandler := handler.NewMavenHandler(
|
||||||
|
proxy,
|
||||||
|
s.cfg.BaseURL,
|
||||||
|
s.cfg.Upstream.Maven,
|
||||||
|
s.cfg.Upstream.GradlePluginPortal,
|
||||||
|
)
|
||||||
|
gradleHandler := handler.NewGradleBuildCacheHandler(proxy)
|
||||||
nugetHandler := handler.NewNuGetHandler(proxy, s.cfg.BaseURL)
|
nugetHandler := handler.NewNuGetHandler(proxy, s.cfg.BaseURL)
|
||||||
composerHandler := handler.NewComposerHandler(proxy, s.cfg.BaseURL)
|
composerHandler := handler.NewComposerHandler(proxy, s.cfg.BaseURL)
|
||||||
conanHandler := handler.NewConanHandler(proxy, s.cfg.BaseURL)
|
conanHandler := handler.NewConanHandler(proxy, s.cfg.BaseURL)
|
||||||
condaHandler := handler.NewCondaHandler(proxy, s.cfg.BaseURL)
|
condaHandler := handler.NewCondaHandler(proxy, s.cfg.BaseURL)
|
||||||
cranHandler := handler.NewCRANHandler(proxy, s.cfg.BaseURL)
|
cranHandler := handler.NewCRANHandler(proxy, s.cfg.BaseURL)
|
||||||
|
juliaHandler := handler.NewJuliaHandler(proxy, s.cfg.BaseURL)
|
||||||
containerHandler := handler.NewContainerHandler(proxy, s.cfg.BaseURL)
|
containerHandler := handler.NewContainerHandler(proxy, s.cfg.BaseURL)
|
||||||
debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL)
|
debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL)
|
||||||
rpmHandler := handler.NewRPMHandler(proxy, s.cfg.BaseURL)
|
rpmHandler := handler.NewRPMHandler(proxy, s.cfg.BaseURL)
|
||||||
|
|
@ -197,11 +219,13 @@ func (s *Server) Start() error {
|
||||||
r.Mount("/pub", http.StripPrefix("/pub", pubHandler.Routes()))
|
r.Mount("/pub", http.StripPrefix("/pub", pubHandler.Routes()))
|
||||||
r.Mount("/pypi", http.StripPrefix("/pypi", pypiHandler.Routes()))
|
r.Mount("/pypi", http.StripPrefix("/pypi", pypiHandler.Routes()))
|
||||||
r.Mount("/maven", http.StripPrefix("/maven", mavenHandler.Routes()))
|
r.Mount("/maven", http.StripPrefix("/maven", mavenHandler.Routes()))
|
||||||
|
r.Mount("/gradle", http.StripPrefix("/gradle", gradleHandler.Routes()))
|
||||||
r.Mount("/nuget", http.StripPrefix("/nuget", nugetHandler.Routes()))
|
r.Mount("/nuget", http.StripPrefix("/nuget", nugetHandler.Routes()))
|
||||||
r.Mount("/composer", http.StripPrefix("/composer", composerHandler.Routes()))
|
r.Mount("/composer", http.StripPrefix("/composer", composerHandler.Routes()))
|
||||||
r.Mount("/conan", http.StripPrefix("/conan", conanHandler.Routes()))
|
r.Mount("/conan", http.StripPrefix("/conan", conanHandler.Routes()))
|
||||||
r.Mount("/conda", http.StripPrefix("/conda", condaHandler.Routes()))
|
r.Mount("/conda", http.StripPrefix("/conda", condaHandler.Routes()))
|
||||||
r.Mount("/cran", http.StripPrefix("/cran", cranHandler.Routes()))
|
r.Mount("/cran", http.StripPrefix("/cran", cranHandler.Routes()))
|
||||||
|
r.Mount("/julia", http.StripPrefix("/julia", juliaHandler.Routes()))
|
||||||
r.Mount("/v2", http.StripPrefix("/v2", containerHandler.Routes()))
|
r.Mount("/v2", http.StripPrefix("/v2", containerHandler.Routes()))
|
||||||
r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes()))
|
r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes()))
|
||||||
r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes()))
|
r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes()))
|
||||||
|
|
@ -238,6 +262,7 @@ func (s *Server) Start() error {
|
||||||
// Start background context (used by mirror jobs and cleanup)
|
// Start background context (used by mirror jobs and cleanup)
|
||||||
bgCtx, bgCancel := context.WithCancel(context.Background())
|
bgCtx, bgCancel := context.WithCancel(context.Background())
|
||||||
s.cancel = bgCancel
|
s.cancel = bgCancel
|
||||||
|
s.startGradleBuildCacheEviction(bgCtx)
|
||||||
|
|
||||||
// Mirror API endpoints (opt-in via mirror_api config or PROXY_MIRROR_API env)
|
// Mirror API endpoints (opt-in via mirror_api config or PROXY_MIRROR_API env)
|
||||||
if s.cfg.MirrorAPI {
|
if s.cfg.MirrorAPI {
|
||||||
|
|
@ -615,6 +640,10 @@ func (s *Server) handlePackagesList(w http.ResponseWriter, r *http.Request) {
|
||||||
func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handlePackagePath(w http.ResponseWriter, r *http.Request) {
|
||||||
ecosystem := chi.URLParam(r, "ecosystem")
|
ecosystem := chi.URLParam(r, "ecosystem")
|
||||||
wildcard := chi.URLParam(r, "*")
|
wildcard := chi.URLParam(r, "*")
|
||||||
|
if err := validatePackagePath(wildcard); err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
segments := splitWildcardPath(wildcard)
|
segments := splitWildcardPath(wildcard)
|
||||||
|
|
||||||
if ecosystem == "" || len(segments) == 0 {
|
if ecosystem == "" || len(segments) == 0 {
|
||||||
|
|
@ -789,23 +818,49 @@ func (s *Server) showComparePage(w http.ResponseWriter, ecosystem, name, version
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleHealth responds with a simple health check.
|
// handleHealth responds with a structured JSON health report.
|
||||||
|
//
|
||||||
// @Summary Health check
|
// @Summary Health check
|
||||||
// @Tags meta
|
// @Tags meta
|
||||||
// @Produce plain
|
// @Produce json
|
||||||
// @Success 200 {string} string
|
// @Success 200 {object} HealthResponse
|
||||||
// @Failure 503 {string} string
|
// @Failure 503 {object} HealthResponse
|
||||||
// @Router /health [get]
|
// @Router /health [get]
|
||||||
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
|
||||||
// Check database connectivity
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp := HealthResponse{Status: "ok", Checks: map[string]HealthCheck{}}
|
||||||
|
|
||||||
|
// Database check (short-circuit; do not waste a storage probe call when DB is down).
|
||||||
|
// On DB failure the storage entry reports "skipped" rather than being omitted so
|
||||||
|
// the response always carries the same key set for monitors that expect it.
|
||||||
if _, err := s.db.SchemaVersion(); err != nil {
|
if _, err := s.db.SchemaVersion(); err != nil {
|
||||||
|
resp.Status = "error"
|
||||||
|
resp.Checks["database"] = HealthCheck{Status: "error", Error: err.Error()}
|
||||||
|
resp.Checks["storage"] = HealthCheck{Status: "skipped"}
|
||||||
w.WriteHeader(http.StatusServiceUnavailable)
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
_, _ = fmt.Fprint(w, "database error")
|
_ = json.NewEncoder(w).Encode(resp)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
resp.Checks["database"] = HealthCheck{Status: "ok"}
|
||||||
|
|
||||||
|
// Storage probe (via cache).
|
||||||
|
if err := s.healthCache.Check(); err != nil {
|
||||||
|
resp.Status = "error"
|
||||||
|
sc := HealthCheck{Status: "error", Error: err.Error()}
|
||||||
|
var pe *probeError
|
||||||
|
if errors.As(err, &pe) {
|
||||||
|
sc.Step = pe.step
|
||||||
|
}
|
||||||
|
resp.Checks["storage"] = sc
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
_ = json.NewEncoder(w).Encode(resp)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
resp.Checks["storage"] = HealthCheck{Status: "ok"}
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
_, _ = fmt.Fprint(w, "ok")
|
_ = json.NewEncoder(w).Encode(resp)
|
||||||
}
|
}
|
||||||
|
|
||||||
// StatsResponse contains cache statistics.
|
// StatsResponse contains cache statistics.
|
||||||
|
|
@ -822,20 +877,20 @@ type StatsResponse struct {
|
||||||
// @Tags meta
|
// @Tags meta
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Success 200 {object} StatsResponse
|
// @Success 200 {object} StatsResponse
|
||||||
// @Failure 500 {string} string
|
// @Failure 500 {object} ErrorResponse
|
||||||
// @Router /stats [get]
|
// @Router /stats [get]
|
||||||
func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := r.Context()
|
ctx := r.Context()
|
||||||
|
|
||||||
count, err := s.db.GetCachedArtifactCount()
|
count, err := s.db.GetCachedArtifactCount()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "failed to get artifact count", http.StatusInternalServerError)
|
internalError(w, "failed to get artifact count")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
size, err := s.db.GetTotalCacheSize()
|
size, err := s.db.GetTotalCacheSize()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, "failed to get cache size", http.StatusInternalServerError)
|
internalError(w, "failed to get cache size")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -72,12 +72,21 @@ func newTestServer(t *testing.T) *testServer {
|
||||||
gemHandler := handler.NewGemHandler(proxy, cfg.BaseURL)
|
gemHandler := handler.NewGemHandler(proxy, cfg.BaseURL)
|
||||||
goHandler := handler.NewGoHandler(proxy, cfg.BaseURL)
|
goHandler := handler.NewGoHandler(proxy, cfg.BaseURL)
|
||||||
pypiHandler := handler.NewPyPIHandler(proxy, cfg.BaseURL)
|
pypiHandler := handler.NewPyPIHandler(proxy, cfg.BaseURL)
|
||||||
|
gradleHandler := handler.NewGradleBuildCacheHandler(proxy)
|
||||||
|
|
||||||
r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes()))
|
r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes()))
|
||||||
r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes()))
|
r.Mount("/cargo", http.StripPrefix("/cargo", cargoHandler.Routes()))
|
||||||
r.Mount("/gem", http.StripPrefix("/gem", gemHandler.Routes()))
|
r.Mount("/gem", http.StripPrefix("/gem", gemHandler.Routes()))
|
||||||
r.Mount("/go", http.StripPrefix("/go", goHandler.Routes()))
|
r.Mount("/go", http.StripPrefix("/go", goHandler.Routes()))
|
||||||
r.Mount("/pypi", http.StripPrefix("/pypi", pypiHandler.Routes()))
|
r.Mount("/pypi", http.StripPrefix("/pypi", pypiHandler.Routes()))
|
||||||
|
r.Mount("/gradle", http.StripPrefix("/gradle", gradleHandler.Routes()))
|
||||||
|
|
||||||
|
hc, err := newHealthCache(store, "30s", logger)
|
||||||
|
if err != nil {
|
||||||
|
_ = db.Close()
|
||||||
|
_ = os.RemoveAll(tempDir)
|
||||||
|
t.Fatalf("failed to create health cache: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Create a minimal server struct for the handlers
|
// Create a minimal server struct for the handlers
|
||||||
s := &Server{
|
s := &Server{
|
||||||
|
|
@ -86,6 +95,7 @@ func newTestServer(t *testing.T) *testServer {
|
||||||
storage: store,
|
storage: store,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
templates: &Templates{},
|
templates: &Templates{},
|
||||||
|
healthCache: hc,
|
||||||
}
|
}
|
||||||
|
|
||||||
r.Get("/health", s.handleHealth)
|
r.Get("/health", s.handleHealth)
|
||||||
|
|
@ -177,12 +187,55 @@ func TestHealthEndpoint(t *testing.T) {
|
||||||
ts.handler.ServeHTTP(w, req)
|
ts.handler.ServeHTTP(w, req)
|
||||||
|
|
||||||
if w.Code != http.StatusOK {
|
if w.Code != http.StatusOK {
|
||||||
t.Errorf("expected status 200, got %d", w.Code)
|
t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if got := w.Header().Get("Content-Type"); got != "application/json" {
|
||||||
|
t.Errorf("Content-Type = %q, want application/json", got)
|
||||||
|
}
|
||||||
|
var resp HealthResponse
|
||||||
|
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
|
||||||
|
t.Fatalf("decoding response: %v", err)
|
||||||
|
}
|
||||||
|
if resp.Status != "ok" {
|
||||||
|
t.Errorf("status = %q, want ok", resp.Status)
|
||||||
|
}
|
||||||
|
if resp.Checks["database"].Status != "ok" {
|
||||||
|
t.Errorf("database check = %+v, want ok", resp.Checks["database"])
|
||||||
|
}
|
||||||
|
if resp.Checks["storage"].Status != "ok" {
|
||||||
|
t.Errorf("storage check = %+v, want ok", resp.Checks["storage"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
body := w.Body.String()
|
func TestHealthEndpoint_DBFailureShortCircuits(t *testing.T) {
|
||||||
if body != "ok" {
|
ts := newTestServer(t)
|
||||||
t.Errorf("expected body 'ok', got %q", body)
|
defer ts.close()
|
||||||
|
|
||||||
|
// Force DB failure by closing the connection.
|
||||||
|
_ = ts.db.Close()
|
||||||
|
|
||||||
|
req := httptest.NewRequest("GET", "/health", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
ts.handler.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("status = %d, want 503; body: %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var resp HealthResponse
|
||||||
|
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
|
||||||
|
t.Fatalf("decoding: %v", err)
|
||||||
|
}
|
||||||
|
if resp.Status != "error" {
|
||||||
|
t.Errorf("status = %q, want error", resp.Status)
|
||||||
|
}
|
||||||
|
if resp.Checks["database"].Status != "error" {
|
||||||
|
t.Errorf("database check = %+v, want error", resp.Checks["database"])
|
||||||
|
}
|
||||||
|
storage, present := resp.Checks["storage"]
|
||||||
|
if !present {
|
||||||
|
t.Error("storage key should be present (with status=skipped) on DB short-circuit")
|
||||||
|
} else if storage.Status != "skipped" {
|
||||||
|
t.Errorf("storage check = %+v, want status=skipped", storage)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -344,6 +397,33 @@ func TestPyPISimple(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestGradleBuildCachePutGet(t *testing.T) {
|
||||||
|
ts := newTestServer(t)
|
||||||
|
defer ts.close()
|
||||||
|
|
||||||
|
key := "abc123def456"
|
||||||
|
body := "build-cache-bytes"
|
||||||
|
|
||||||
|
putReq := httptest.NewRequest(http.MethodPut, "/gradle/"+key, strings.NewReader(body))
|
||||||
|
putW := httptest.NewRecorder()
|
||||||
|
ts.handler.ServeHTTP(putW, putReq)
|
||||||
|
|
||||||
|
if putW.Code != http.StatusCreated {
|
||||||
|
t.Fatalf("expected status 201, got %d: %s", putW.Code, putW.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
getReq := httptest.NewRequest(http.MethodGet, "/gradle/"+key, nil)
|
||||||
|
getW := httptest.NewRecorder()
|
||||||
|
ts.handler.ServeHTTP(getW, getReq)
|
||||||
|
|
||||||
|
if getW.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected status 200, got %d: %s", getW.Code, getW.Body.String())
|
||||||
|
}
|
||||||
|
if got := getW.Body.String(); got != body {
|
||||||
|
t.Fatalf("expected body %q, got %q", body, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestGemSpecs(t *testing.T) {
|
func TestGemSpecs(t *testing.T) {
|
||||||
ts := newTestServer(t)
|
ts := newTestServer(t)
|
||||||
defer ts.close()
|
defer ts.close()
|
||||||
|
|
|
||||||
|
|
@ -193,7 +193,7 @@ func TestInstallPage(t *testing.T) {
|
||||||
body := w.Body.String()
|
body := w.Body.String()
|
||||||
|
|
||||||
// Should contain instructions for all registries
|
// Should contain instructions for all registries
|
||||||
registries := []string{"npm", "Cargo", "RubyGems", "Go Modules", "PyPI", "Maven", "NuGet", "Composer", "Conan", "Conda", "CRAN"}
|
registries := []string{"npm", "Cargo", "RubyGems", "Go Modules", "PyPI", "Maven", "Gradle Build Cache", "NuGet", "Composer", "Conan", "Conda", "CRAN"}
|
||||||
for _, reg := range registries {
|
for _, reg := range registries {
|
||||||
if !strings.Contains(body, reg) {
|
if !strings.Contains(body, reg) {
|
||||||
t.Errorf("install page should contain %s instructions", reg)
|
t.Errorf("install page should contain %s instructions", reg)
|
||||||
|
|
|
||||||
|
|
@ -184,6 +184,35 @@ func (b *Blob) UsedSpace(ctx context.Context) (int64, error) {
|
||||||
return total, nil
|
return total, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListPrefix returns object metadata for keys under a prefix.
|
||||||
|
func (b *Blob) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||||
|
iter := b.bucket.List(&blob.ListOptions{Prefix: prefix})
|
||||||
|
objects := make([]ObjectInfo, 0)
|
||||||
|
|
||||||
|
for {
|
||||||
|
obj, err := iter.Next(ctx)
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("listing objects: %w", err)
|
||||||
|
}
|
||||||
|
if obj.IsDir {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
info := ObjectInfo{
|
||||||
|
Path: obj.Key,
|
||||||
|
Size: obj.Size,
|
||||||
|
ModTime: obj.ModTime,
|
||||||
|
}
|
||||||
|
|
||||||
|
objects = append(objects, info)
|
||||||
|
}
|
||||||
|
|
||||||
|
return objects, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (b *Blob) Close() error {
|
func (b *Blob) Close() error {
|
||||||
return b.bucket.Close()
|
return b.bucket.Close()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
fsys "io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
@ -187,6 +188,54 @@ func (fs *Filesystem) UsedSpace(ctx context.Context) (int64, error) {
|
||||||
return total, nil
|
return total, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListPrefix returns object metadata for paths under a prefix.
|
||||||
|
func (fs *Filesystem) ListPrefix(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||||
|
searchRoot, err := fs.fullPath(prefix)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Stat(searchRoot); err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return []ObjectInfo{}, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("stat prefix: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
objects := make([]ObjectInfo, 0)
|
||||||
|
err = filepath.WalkDir(searchRoot, func(path string, entry fsys.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if entry.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := entry.Info()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
relPath, err := filepath.Rel(fs.root, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
objects = append(objects, ObjectInfo{
|
||||||
|
Path: filepath.ToSlash(relPath),
|
||||||
|
Size: info.Size(),
|
||||||
|
ModTime: info.ModTime(),
|
||||||
|
})
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("walking prefix: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return objects, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Root returns the root directory of the storage.
|
// Root returns the root directory of the storage.
|
||||||
func (fs *Filesystem) Root() string {
|
func (fs *Filesystem) Root() string {
|
||||||
return fs.root
|
return fs.root
|
||||||
|
|
|
||||||
|
|
@ -28,6 +28,13 @@ var (
|
||||||
ErrSignedURLUnsupported = errors.New("signed URLs not supported by storage backend")
|
ErrSignedURLUnsupported = errors.New("signed URLs not supported by storage backend")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ObjectInfo contains metadata for a stored object.
|
||||||
|
type ObjectInfo struct {
|
||||||
|
Path string
|
||||||
|
Size int64
|
||||||
|
ModTime time.Time
|
||||||
|
}
|
||||||
|
|
||||||
// Storage defines the interface for artifact storage backends.
|
// Storage defines the interface for artifact storage backends.
|
||||||
type Storage interface {
|
type Storage interface {
|
||||||
// Store writes content from r to the given path.
|
// Store writes content from r to the given path.
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue