mirror of
https://github.com/git-pkgs/proxy.git
synced 2026-11-09 23:59:16 -05:00
Compare commits
173 changed files with 787 additions and 16238 deletions
|
|
@ -1,17 +0,0 @@
|
|||
# Version and commit come in as build args, so the build doesn't need .git.
|
||||
.git
|
||||
.github
|
||||
|
||||
# Local build output and runtime data
|
||||
/proxy
|
||||
dist/
|
||||
build/
|
||||
cache/
|
||||
*.db
|
||||
*.tar.gz
|
||||
|
||||
# Local config and secrets
|
||||
.env
|
||||
.env.local
|
||||
proxy.cfg
|
||||
config.yaml
|
||||
5
.github/workflows/publish.yml
vendored
5
.github/workflows/publish.yml
vendored
|
|
@ -38,7 +38,7 @@ jobs:
|
|||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
|
@ -61,9 +61,6 @@ jobs:
|
|||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.meta.outputs.version }}
|
||||
COMMIT=${{ github.sha }}
|
||||
provenance: mode=max
|
||||
sbom: true
|
||||
|
||||
|
|
|
|||
2
.github/workflows/zizmor.yml
vendored
2
.github/workflows/zizmor.yml
vendored
|
|
@ -34,4 +34,4 @@ jobs:
|
|||
persist-credentials: false
|
||||
|
||||
- name: Run zizmor
|
||||
uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0
|
||||
uses: zizmorcore/zizmor-action@85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e # v0.7.1
|
||||
|
|
|
|||
12
Dockerfile
12
Dockerfile
|
|
@ -14,13 +14,9 @@ COPY . .
|
|||
|
||||
# Build the binary for the target platform
|
||||
ARG TARGETARCH
|
||||
ARG VERSION=dev
|
||||
ARG COMMIT=unknown
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \
|
||||
-ldflags="-s -w -X main.Version=${VERSION} -X main.Commit=${COMMIT}" \
|
||||
-o /proxy ./cmd/proxy
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /proxy ./cmd/proxy
|
||||
|
||||
FROM alpine:3.24.3
|
||||
FROM alpine:3.24.2
|
||||
|
||||
RUN apk add --no-cache ca-certificates git
|
||||
|
||||
|
|
@ -35,9 +31,5 @@ WORKDIR /data
|
|||
|
||||
EXPOSE 8080
|
||||
|
||||
# /health checks the database and storage and can take up to ten seconds.
|
||||
HEALTHCHECK --interval=30s --timeout=11s --start-period=10s --retries=3 \
|
||||
CMD wget -q -O /dev/null http://127.0.0.1:8080/health || exit 1
|
||||
|
||||
ENTRYPOINT ["proxy"]
|
||||
CMD ["serve"]
|
||||
|
|
|
|||
111
README.md
111
README.md
|
|
@ -20,8 +20,6 @@ A 3-day cooldown means that when `lodash` publishes version `4.18.0`, your build
|
|||
|
||||
Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default and carve out exceptions for packages where you need faster updates. See [docs/configuration.md](docs/configuration.md) for the full config reference.
|
||||
|
||||
One instance can offer several cooldown policies through [named profiles](docs/configuration.md#cooldown-profiles). Configure `cooldown.profiles` and point clients at paths such as `/cooldown/3d/npm/` or `/cooldown/7d/cargo/`. Profiles share the package cache; existing URLs keep using the global policy.
|
||||
|
||||
## Artifact Scanning
|
||||
|
||||
Cooldown only looks at a version's publish timestamp — it never inspects the actual bytes. Artifact scanning closes that gap: when enabled, every artifact is staged into storage and scanned by one or more external services (trivy, ClamAV, Wiz, or anything else that speaks a small HTTP/JSON contract) before it's committed to the cache and served to clients.
|
||||
|
|
@ -77,7 +75,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi
|
|||
|
||||
Cooldown requires publish timestamps in metadata. Registries without a "Yes" in the cooldown column either don't expose timestamps or haven't been wired up yet.
|
||||
|
||||
\* Hex cooldown requires disabling registry signature verification (`HEX_UNSAFE_REGISTRY=1`) since the proxy re-encodes the protobuf payload. Origin verification can stay enabled.
|
||||
\* Hex cooldown requires disabling registry signature verification (`HEX_NO_VERIFY_REPO_ORIGIN=1`) since the proxy re-encodes the protobuf payload.
|
||||
|
||||
† Go cooldown uses the time in each version's `.info`, which is the commit time of the tagged revision, not when the tag was pushed.
|
||||
|
||||
|
|
@ -89,11 +87,7 @@ Cooldown requires publish timestamps in metadata. Registries without a "Yes" in
|
|||
brew install git-pkgs/git-pkgs/proxy
|
||||
```
|
||||
|
||||
Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases), or run the container image:
|
||||
|
||||
```bash
|
||||
docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest
|
||||
```
|
||||
Or download a binary from the [releases page](https://github.com/git-pkgs/proxy/releases).
|
||||
|
||||
### Helm
|
||||
|
||||
|
|
@ -110,12 +104,6 @@ using SQLite and filesystem artifact storage under `/data`. See
|
|||
[`deploy/charts/proxy/values.yaml`](deploy/charts/proxy/values.yaml) for ingress,
|
||||
external database and object-storage configuration options.
|
||||
|
||||
The SQLite database needs a local disk or a block-backed volume, even with one
|
||||
replica; `ReadWriteOnce` alone doesn't guarantee that, since the storage class
|
||||
can still be NFS. If the cluster only offers network filesystem storage, use
|
||||
Postgres. Artifact storage can stay on NFS, separately from the database. See
|
||||
[SQLite](docs/configuration.md#sqlite).
|
||||
|
||||
To trust a private CA, for example when upstreams sit behind a TLS-intercepting
|
||||
corporate proxy or are internal registries signed by an in-house CA, mount the
|
||||
certificate with `extraVolumes` and `extraVolumeMounts` and add its directory
|
||||
|
|
@ -204,7 +192,7 @@ the proxy, which fetches them from the package's registry and caches them;
|
|||
attestations of versions held back by cooldown or blocked by the denylist are
|
||||
not served.
|
||||
|
||||
To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds).
|
||||
To serve private packages through the same registry URL, see [Private package routes](docs/configuration.md#private-package-routes).
|
||||
|
||||
### Cargo
|
||||
|
||||
|
|
@ -288,10 +276,10 @@ The equivalent environment variables are `PROXY_UPSTREAM_HOMEBREW_API` and `PROX
|
|||
|
||||
### Hex (Elixir)
|
||||
|
||||
Configure the mirror:
|
||||
Configure in `~/.hex/hex.config`:
|
||||
|
||||
```bash
|
||||
mix hex.config mirror_url http://localhost:8080/hex
|
||||
```erlang
|
||||
{default_url, <<"http://localhost:8080/hex">>}.
|
||||
```
|
||||
|
||||
Or set the environment variable:
|
||||
|
|
@ -300,14 +288,6 @@ Or set the environment variable:
|
|||
export HEX_MIRROR=http://localhost:8080/hex
|
||||
```
|
||||
|
||||
When the proxy applies a cooldown, also set:
|
||||
|
||||
```bash
|
||||
export HEX_UNSAFE_REGISTRY=1
|
||||
```
|
||||
|
||||
Filtering releases removes the registry signature. The proxy preserves the package name and repository fields, so origin verification can stay enabled. Without a cooldown, leave signature verification enabled too.
|
||||
|
||||
### pub.dev (Dart/Flutter)
|
||||
|
||||
Set the PUB_HOSTED_URL environment variable:
|
||||
|
|
@ -367,15 +347,17 @@ Configure in `settings.gradle.kts`:
|
|||
|
||||
```kotlin
|
||||
buildCache {
|
||||
local {
|
||||
enabled = false
|
||||
}
|
||||
remote<HttpBuildCache> {
|
||||
url = uri("http://localhost:8080/gradle/")
|
||||
isPush = System.getenv("CI") != null
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Enable build caching with `org.gradle.caching=true` in `gradle.properties`, or pass `--build-cache` when running Gradle. The local cache stays enabled and remote uploads are disabled by default.
|
||||
|
||||
To upload from CI, set `gradle.build_cache.read_only` to `false` on the proxy and add `isPush = System.getenv("CI") != null` inside the client's `remote<HttpBuildCache>` block. See [docs/configuration.md](docs/configuration.md#gradle-build-cache).
|
||||
The proxy rejects uploads unless `gradle.build_cache.read_only` is set to `false`, see [docs/configuration.md](docs/configuration.md#gradle-build-cache).
|
||||
|
||||
### NuGet
|
||||
|
||||
|
|
@ -417,16 +399,7 @@ Or set globally:
|
|||
composer config -g repositories.proxy composer http://localhost:8080/composer
|
||||
```
|
||||
|
||||
`composer audit`, `audit.block-insecure` and Packagist's malware filter list
|
||||
work through the proxy when `upstream.composer_repository` offers them in its
|
||||
`packages.json`, as repo.packagist.org does. The proxy advertises them in its
|
||||
own `packages.json` and passes the advisories and filter list requests through
|
||||
to the upstream uncached. Names of packages in `upstream.composer_routes` or
|
||||
`hosted.composer_vendors` are removed from advisories requests before they are
|
||||
sent. Versions withheld by [cooldown](#version-cooldown) are not excluded from
|
||||
the report.
|
||||
|
||||
To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes). To see new releases before `metadata_ttl` runs out, see [Change feeds](docs/configuration.md#change-feeds).
|
||||
To serve private packages through the same URL, see [Private package routes](docs/configuration.md#private-package-routes).
|
||||
|
||||
### Conan (C/C++)
|
||||
|
||||
|
|
@ -469,13 +442,10 @@ default_channels:
|
|||
- http://localhost:8080/conda/main
|
||||
```
|
||||
|
||||
`default_channels` also routes requests for `defaults` through the proxy, including `-c defaults` and `environment.yml` files that name it.
|
||||
|
||||
Or set via command:
|
||||
|
||||
```bash
|
||||
conda config --add channels http://localhost:8080/conda/main
|
||||
conda config --add default_channels http://localhost:8080/conda/main
|
||||
```
|
||||
|
||||
pixi and other rattler-based clients use sharded repodata ([CEP 16](https://github.com/conda/ceps/blob/main/cep-0016.md)) where a channel offers it, as conda-forge does. Point them at the same channel URLs, for example `pixi init -c http://localhost:8080/conda/conda-forge`. The shard index and shards are cached as metadata when `cache_metadata` is on.
|
||||
|
|
@ -498,8 +468,6 @@ local({
|
|||
})
|
||||
```
|
||||
|
||||
Source packages, Windows and macOS binaries (including the `bin/macosx/big-sur-arm64/` layout used by R 4.3 and later) are cached as artifacts. `PACKAGES` indexes and the `src/contrib/Meta/*.rds` files that `remotes::install_version` and pak read are cached as metadata.
|
||||
|
||||
### Julia
|
||||
|
||||
Set the Pkg server before starting Julia:
|
||||
|
|
@ -727,20 +695,16 @@ indexes unchanged. To keep APT from selecting a denied version, configure
|
|||
|
||||
### RPM / Yum / DNF
|
||||
|
||||
In the `[fedora]` section of your existing repository file under `/etc/yum.repos.d/`, replace the `metalink` or `mirrorlist` entry with:
|
||||
Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`:
|
||||
|
||||
```ini
|
||||
[proxy-fedora]
|
||||
name=Fedora via Proxy
|
||||
baseurl=http://localhost:8080/rpm/releases/$releasever/Everything/$basearch/os/
|
||||
enabled=1
|
||||
gpgcheck=0
|
||||
```
|
||||
|
||||
In the `[updates]` section of `fedora-updates.repo`, replace its `metalink` or `mirrorlist` entry with:
|
||||
|
||||
```ini
|
||||
baseurl=http://localhost:8080/rpm/updates/$releasever/Everything/$basearch/
|
||||
```
|
||||
|
||||
Keep the existing `gpgkey` and signature settings, including `gpgcheck=1`. The proxy serves RPM packages unchanged, so their signatures remain valid.
|
||||
|
||||
Then:
|
||||
|
||||
```bash
|
||||
|
|
@ -770,11 +734,6 @@ signature verification keeps working. Indexes use the metadata cache
|
|||
(`metadata_ttl`, stale fallback); `.apk` packages are stored in the shared
|
||||
artifact cache and remain available when the upstream is unreachable.
|
||||
|
||||
Set `upstream.apk_snapshots` (e.g. `"90d"`) to keep replaced indexes and serve
|
||||
them by date at `/apk/alpine@2026-10-01/v3.22/main`, so pinned package versions
|
||||
stay installable after the mirror moves on. See
|
||||
[Alpine APK snapshots](docs/configuration.md#alpine-apk-snapshots).
|
||||
|
||||
To proxy other mirrors or private repositories, configure named upstreams
|
||||
under `upstream.apk` (this replaces the built-in default; re-add `alpine` if
|
||||
you still want it):
|
||||
|
|
@ -1089,7 +1048,7 @@ Run with config file:
|
|||
|
||||
### PostgreSQL
|
||||
|
||||
SQLite is the default and works well for single-node deployments, with the database file on a local disk or block-backed volume. For multi-node setups, when only network filesystem storage such as NFS is available, or if you prefer a managed database, switch to Postgres:
|
||||
SQLite is the default and works well for single-node deployments. For multi-node setups or if you prefer a managed database, switch to Postgres:
|
||||
|
||||
```yaml
|
||||
database:
|
||||
|
|
@ -1211,9 +1170,6 @@ proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21
|
|||
|
||||
# Mirror a container image for one platform
|
||||
proxy mirror --platform linux/amd64 pkg:docker/library/alpine@3.20
|
||||
|
||||
# Mirror a commit of an upstream.git repository into the git cache
|
||||
proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567
|
||||
```
|
||||
|
||||
The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped. If any package version fails to mirror, the command lists the errors and exits with status 1, so a CI step that warms the cache fails on a partial mirror.
|
||||
|
|
@ -1324,8 +1280,6 @@ Recently cached:
|
|||
| `GET /maven/*` | Maven repository protocol |
|
||||
| `GET /nuget/*` | NuGet V3 API |
|
||||
| `GET /composer/*` | Composer/Packagist protocol |
|
||||
| `GET`, `POST /composer/api/security-advisories/` | Composer security advisories, passed through to upstream without private package names |
|
||||
| `GET /composer/lists/summary.json` | Composer filter list summary, passed through to upstream uncached |
|
||||
| `GET /conan/*` | Conan C/C++ protocol |
|
||||
| `GET /chef/*` | Chef Supermarket protocol (universe, cookbook API and downloads) |
|
||||
| `GET /conda/*` | Conda/Anaconda protocol |
|
||||
|
|
@ -1363,17 +1317,6 @@ Registered when `hosted.enabled` is set. Pushing and deleting require `Authoriza
|
|||
| `GET /api/hosted/{ecosystem}/{name}` | List the archives pushed for a package |
|
||||
| `DELETE /api/hosted/{ecosystem}/{name}/{version}` | Delete a pushed version |
|
||||
|
||||
### Projects API
|
||||
|
||||
Registered when `projects.enabled` is set. Uploading and deleting require `Authorization: Bearer <projects.token>`. See [Project inventory](docs/configuration.md#project-inventory).
|
||||
|
||||
| Endpoint | Description |
|
||||
|----------|-------------|
|
||||
| `PUT /api/projects/{project}/sbom` | Replace a project's recorded package versions with the ones the SBOM in the body lists |
|
||||
| `GET /api/projects` | List projects, or with `?purl=pkg:npm/lodash@4.17.20` the projects using a package or version |
|
||||
| `GET /api/projects/{project}` | A project's recorded package versions |
|
||||
| `DELETE /api/projects/{project}` | Delete a project |
|
||||
|
||||
### Cache API
|
||||
|
||||
Registered when `cache_delete` is set. It has no authentication of its own.
|
||||
|
|
@ -1535,7 +1478,9 @@ Response:
|
|||
{
|
||||
"cached_artifacts": 142,
|
||||
"total_size_bytes": 523456789,
|
||||
"total_size": "499.2 MB"
|
||||
"total_size": "499.2 MB",
|
||||
"storage_url": "file:///path/to/cache/artifacts",
|
||||
"database_path": "./cache/proxy.db"
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -1575,7 +1520,6 @@ The proxy serves a web UI under `/ui`. No separate frontend build is needed -- t
|
|||
- **Version detail** (`/ui/package/{ecosystem}/{name}/{version}`) -- per-version metadata, integrity hash, artifact cache status, and hit counts.
|
||||
- **Source browser** (`/ui/package/{ecosystem}/{name}/{version}/browse`) -- browse files inside cached archives with syntax highlighting for text files and image previews.
|
||||
- **Version diff** (`/ui/package/{ecosystem}/{name}/compare/{v1}...{v2}`) -- side-by-side diff of two cached versions showing added, removed, and changed files.
|
||||
- **Projects** (`/ui/projects`) -- with the [project inventory](docs/configuration.md#project-inventory) enabled, the package versions each repository's uploaded SBOM lists, marked when outdated. Package pages then show which projects use each version.
|
||||
|
||||
## Monitoring
|
||||
|
||||
|
|
@ -1594,7 +1538,6 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre
|
|||
| `proxy_metadata_cache_entries` | gauge | `ecosystem` | Number of cached metadata documents per ecosystem. |
|
||||
| `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) |
|
||||
| `proxy_metadata_evicted_total` | counter | `ecosystem` | Cached metadata documents evicted by the size limit |
|
||||
| `proxy_metadata_expired_by_change_feed_total` | counter | `ecosystem` | Cached metadata documents set to revalidate because an upstream [change feed](docs/configuration.md#change-feeds) listed them |
|
||||
| `proxy_fetch_lease_waits_total` | counter | `kind` | Cache misses (`artifact` or `metadata`) that waited for another instance sharing the database to fetch them |
|
||||
| `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream |
|
||||
| `proxy_upstream_errors_total` | counter | `ecosystem`, `error_type` | Upstream fetch failures |
|
||||
|
|
@ -1665,7 +1608,7 @@ The `kind` label on `proxy_cache_hits_total` and `proxy_cache_misses_total` keep
|
|||
|
||||
`ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which.
|
||||
|
||||
**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_metadata_expired_by_change_feed_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`.
|
||||
**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, the three `proxy_metadata_cache_*` metrics, `proxy_artifacts_evicted_total`, `proxy_metadata_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`.
|
||||
|
||||
**From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all.
|
||||
|
||||
|
|
@ -1772,21 +1715,13 @@ sudo systemctl start proxy
|
|||
|
||||
### Docker
|
||||
|
||||
Images for linux/amd64 and linux/arm64 are published to `ghcr.io/git-pkgs/proxy` for each release, tagged with the version and `latest`:
|
||||
|
||||
```bash
|
||||
docker run -p 8080:8080 -v proxy-data:/data ghcr.io/git-pkgs/proxy:latest
|
||||
```
|
||||
|
||||
To build the image from the repo instead:
|
||||
A Dockerfile is included in the repo. Build and run:
|
||||
|
||||
```bash
|
||||
docker build -t proxy .
|
||||
docker run -p 8080:8080 -v proxy-data:/data proxy
|
||||
```
|
||||
|
||||
The image has a health check that requests `/health` on port 8080. It checks the database and storage, so a container reports unhealthy when either fails. If you change the port with `PROXY_LISTEN`, override it with `--health-cmd`.
|
||||
|
||||
With `--read-only`, also pass `--tmpfs /tmp`: the proxy writes some uploads to a temporary file before reading them.
|
||||
|
||||
With Postgres and S3:
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
// @securityDefinitions.apikey BearerAuth
|
||||
// @in header
|
||||
// @name Authorization
|
||||
// @description "Bearer " followed by hosted.token, or projects.token for /api/projects
|
||||
// @description "Bearer " followed by hosted.token
|
||||
//
|
||||
// The proxy caches package artifacts from upstream registries (npm, cargo, etc.)
|
||||
// providing faster, more reliable access for development teams.
|
||||
|
|
@ -93,9 +93,6 @@
|
|||
// PROXY_HOSTED_MAX_UPLOAD_SIZE - Max size of one pushed archive (default "100MB")
|
||||
// PROXY_HOSTED_NPM_SCOPES - npm scopes served from pushed packages only
|
||||
// PROXY_HOSTED_COMPOSER_VENDORS - Composer vendors served from pushed packages only
|
||||
// PROXY_PROJECTS_ENABLED - Enable the project inventory (/api/projects, /ui/projects)
|
||||
// PROXY_PROJECTS_TOKEN - Bearer token for uploading and deleting project SBOMs
|
||||
// PROXY_PROJECTS_MAX_AGE - Remove projects not uploaded for this long (e.g. "90d")
|
||||
// PROXY_URL - Proxy URL for proxy hosted
|
||||
//
|
||||
// Example:
|
||||
|
|
@ -495,7 +492,6 @@ func runMirror() {
|
|||
os.Exit(1)
|
||||
}
|
||||
cfg.LoadFromEnv()
|
||||
printEnvWarnings(cfg)
|
||||
|
||||
if *storageURL != "" {
|
||||
cfg.Storage.URL = *storageURL
|
||||
|
|
@ -558,7 +554,6 @@ func runMirror() {
|
|||
proxy.CacheMetadata = true // mirror always caches metadata
|
||||
proxy.MetadataTTL = cfg.ParseMetadataTTL()
|
||||
proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize()
|
||||
proxy.MetadataStreamMaxSize = cfg.ParseMetadataStreamMaxSize()
|
||||
// As serve does, resolve Swift source repositories in the denylist through
|
||||
// the upstream, and refuse to run when one can't be.
|
||||
if err := server.ConfigureDenylist(cfg, proxy); err != nil {
|
||||
|
|
@ -596,9 +591,7 @@ func runMirror() {
|
|||
return
|
||||
}
|
||||
|
||||
closeGit := useCommandGit(cfg, proxy, m, logger)
|
||||
progress, err := m.Run(ctx, source)
|
||||
closeGit()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
|
|
@ -637,27 +630,6 @@ func newCommandMirror(cfg *config.Config, proxy *handler.Proxy, images *handler.
|
|||
return m
|
||||
}
|
||||
|
||||
// useCommandGit opens the git cache for the pkg:git items of proxy mirror,
|
||||
// when upstream.git is configured, and returns the function that closes it.
|
||||
// Where it can't be opened, such as while the running server holds
|
||||
// git.cache_dir, those items fail with the reason and the rest are mirrored.
|
||||
func useCommandGit(cfg *config.Config, proxy *handler.Proxy, m *mirror.Mirror, logger *slog.Logger) func() {
|
||||
git, closeGit, err := server.NewMirrorGit(cfg, proxy, logger)
|
||||
if err != nil {
|
||||
m.Git = mirror.UnavailableGit{Reason: fmt.Sprintf("this command can't open the git cache (%v); while the server runs, use its mirror API", err)}
|
||||
return func() {}
|
||||
}
|
||||
if git == nil {
|
||||
return func() {}
|
||||
}
|
||||
m.Git = git
|
||||
return func() {
|
||||
if err := closeGit(); err != nil {
|
||||
logger.Warn("closing the git cache", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mirrorExitCode returns the exit status for a finished mirror run: 1 when any
|
||||
// package version failed to mirror, 0 otherwise.
|
||||
func mirrorExitCode(progress *mirror.Progress) int {
|
||||
|
|
@ -826,14 +798,6 @@ func formatSize(bytes int64) string {
|
|||
}
|
||||
}
|
||||
|
||||
// printEnvWarnings reports environment values LoadFromEnv ignored. serve
|
||||
// logs them from server.Start; mirror never gets there.
|
||||
func printEnvWarnings(cfg *config.Config) {
|
||||
for _, warning := range cfg.EnvWarnings() {
|
||||
fmt.Fprintf(os.Stderr, "warning: %s\n", warning)
|
||||
}
|
||||
}
|
||||
|
||||
func loadConfig(path string) (*config.Config, error) {
|
||||
if path != "" {
|
||||
return config.Load(path)
|
||||
|
|
|
|||
|
|
@ -57,24 +57,3 @@ func TestServeHelpProcess(*testing.T) {
|
|||
os.Args = []string{"proxy", "serve", "-help"}
|
||||
main()
|
||||
}
|
||||
|
||||
func TestMirrorLogsEnvWarnings(t *testing.T) {
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestMirrorEnvWarningsProcess$")
|
||||
// The negative size stops mirror before it opens any storage.
|
||||
cmd.Env = append(os.Environ(),
|
||||
"PROXY_TEST_MIRROR_ENV_WARNINGS=1",
|
||||
"PROXY_CACHE_METADATA=maybe",
|
||||
"PROXY_STORAGE_MAX_SIZE=-1")
|
||||
output, _ := cmd.CombinedOutput()
|
||||
if !strings.Contains(string(output), "PROXY_CACHE_METADATA") {
|
||||
t.Errorf("mirror did not warn about PROXY_CACHE_METADATA:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMirrorEnvWarningsProcess(*testing.T) {
|
||||
if os.Getenv("PROXY_TEST_MIRROR_ENV_WARNINGS") != "1" {
|
||||
return
|
||||
}
|
||||
os.Args = []string{"proxy", "mirror", "pkg:npm/lodash@4.17.21"}
|
||||
main()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,132 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/cgi"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/git-pkgs/proxy/internal/gitcache"
|
||||
)
|
||||
|
||||
// gitMirrorUpstream serves team/repo, with one commit on main, from git
|
||||
// http-backend, and returns the server and the commit.
|
||||
func gitMirrorUpstream(t *testing.T) (*httptest.Server, string) {
|
||||
t.Helper()
|
||||
git := func(dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = []string{
|
||||
"PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1",
|
||||
"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com",
|
||||
"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com",
|
||||
}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
backend := ""
|
||||
for _, name := range []string{"git-http-backend", "git-http-backend.exe"} {
|
||||
if path := filepath.Join(git("", "--exec-path"), name); fileExistsForTest(path) {
|
||||
backend = path
|
||||
}
|
||||
}
|
||||
if backend == "" {
|
||||
t.Skip("git http-backend is not installed")
|
||||
}
|
||||
root := t.TempDir()
|
||||
bare := filepath.Join(root, "team", "repo.git")
|
||||
git("", "init", "--quiet", "--bare", "--initial-branch=main", bare)
|
||||
work := t.TempDir()
|
||||
git("", "clone", "--quiet", bare, work)
|
||||
git(work, "commit", "--quiet", "--allow-empty", "-m", "first")
|
||||
git(work, "push", "--quiet", "origin", "HEAD:main")
|
||||
head := git(work, "rev-parse", "HEAD")
|
||||
|
||||
server := httptest.NewServer(&cgi.Handler{
|
||||
Path: backend,
|
||||
Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"},
|
||||
})
|
||||
t.Cleanup(server.Close)
|
||||
return server, head
|
||||
}
|
||||
|
||||
func fileExistsForTest(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// writeGitMirrorConfig writes a config with upstream.git "up" at
|
||||
// upstreamURL and the git cache, storage and database under dir.
|
||||
func writeGitMirrorConfig(t *testing.T, dir, upstreamURL string) string {
|
||||
t.Helper()
|
||||
cfg := "storage:\n" +
|
||||
" url: file://" + filepath.Join(dir, "artifacts") + "\n" +
|
||||
"database:\n" +
|
||||
" path: " + filepath.Join(dir, "proxy.db") + "\n" +
|
||||
"upstream:\n" +
|
||||
" allow_loopback: true\n" +
|
||||
" git:\n" +
|
||||
" up: " + upstreamURL + "\n" +
|
||||
"git:\n" +
|
||||
" cache_dir: " + filepath.Join(dir, "git") + "\n"
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
if err := os.WriteFile(path, []byte(cfg), 0o600); err != nil {
|
||||
t.Fatalf("writing config: %v", err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestMirrorCommandCachesGitCommit(t *testing.T) {
|
||||
upstream, head := gitMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeGitMirrorConfig(t, dir, upstream.URL)
|
||||
purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
|
||||
|
||||
output := runMirrorProcess(t, 0, "-config", configPath, purl)
|
||||
if !strings.Contains(output, "1 downloaded") {
|
||||
t.Errorf("output:\n%s", output)
|
||||
}
|
||||
cached, _ := filepath.Glob(filepath.Join(dir, "git", "up-*", "team", "repo.git"))
|
||||
if len(cached) != 1 {
|
||||
t.Errorf("cached repositories %v, want team/repo.git", cached)
|
||||
}
|
||||
|
||||
output = runMirrorProcess(t, 0, "-config", configPath, purl)
|
||||
if !strings.Contains(output, "1 skipped") {
|
||||
t.Errorf("second run output:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMirrorCommandGitCacheInUse checks pkg:git items fail, pointing at the
|
||||
// mirror API, while the running server holds git.cache_dir.
|
||||
func TestMirrorCommandGitCacheInUse(t *testing.T) {
|
||||
upstream, head := gitMirrorUpstream(t)
|
||||
dir := t.TempDir()
|
||||
configPath := writeGitMirrorConfig(t, dir, upstream.URL)
|
||||
cache, err := gitcache.New(gitcache.Options{
|
||||
CacheDir: filepath.Join(dir, "git"),
|
||||
RefreshWait: time.Second,
|
||||
Client: &http.Client{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = cache.Close() }()
|
||||
purl := "pkg:git/" + strings.TrimPrefix(upstream.URL, "http://") + "/team/repo@" + head
|
||||
|
||||
output := runMirrorProcess(t, 1, "-config", configPath, purl)
|
||||
for _, want := range []string{"in use by another process", "mirror API", "1 failed"} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Errorf("output missing %q:\n%s", want, output)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -189,10 +189,6 @@ upstream:
|
|||
# npm_routes:
|
||||
# "@example/*": "https://npm.example.com/registry"
|
||||
|
||||
# Follow the npm registry's change feed, so a new release is fetched on the
|
||||
# next request instead of after metadata_ttl. Requires cache_metadata.
|
||||
# npm_change_feed: true
|
||||
|
||||
# Cargo sparse index URL
|
||||
cargo: "https://index.crates.io"
|
||||
|
||||
|
|
@ -249,11 +245,6 @@ upstream:
|
|||
# composer_routes:
|
||||
# "example/*": "https://composer.example.com/packages"
|
||||
|
||||
# Follow the change feed of the Packagist at composer, so a new release is
|
||||
# fetched on the next request instead of after metadata_ttl. Requires
|
||||
# cache_metadata.
|
||||
# composer_change_feed: true
|
||||
|
||||
# Conan registry URL
|
||||
conan: "https://center.conan.io"
|
||||
|
||||
|
|
@ -451,15 +442,6 @@ gradle:
|
|||
# npm_scopes: ["@acme"]
|
||||
# composer_vendors: ["acme"]
|
||||
|
||||
# Project inventory (disabled by default). Repositories upload an SBOM to
|
||||
# PUT /api/projects/{project}/sbom, and the UI shows which project uses
|
||||
# which package versions. Project names are visible on the unauthenticated UI.
|
||||
# projects:
|
||||
# enabled: true
|
||||
# token: "${PROXY_PROJECTS_TOKEN}"
|
||||
# # Drop projects that have not uploaded for this long ("" keeps them)
|
||||
# max_age: "90d"
|
||||
|
||||
# Re-mirror container image tags on a schedule, so a moved tag is picked up
|
||||
# even when nobody pulls it. An unchanged tag costs one HEAD request.
|
||||
# mirror:
|
||||
|
|
@ -482,13 +464,6 @@ health:
|
|||
# to spot malicious releases before they're pulled into projects.
|
||||
# Supported durations: "7d" (days), "48h" (hours), "30m" (minutes), "0" (disabled)
|
||||
cooldown:
|
||||
# Independent policies selected through /cooldown/{name}/{registry}/.
|
||||
# profiles:
|
||||
# 3d:
|
||||
# default: "3d"
|
||||
# 7d:
|
||||
# default: "7d"
|
||||
|
||||
# Global default cooldown for all ecosystems
|
||||
# default: "3d"
|
||||
|
||||
|
|
|
|||
|
|
@ -79,12 +79,6 @@ extraInitContainers: []
|
|||
# image: busybox:1.37
|
||||
# command: ["sh", "-c", "until nc -z postgres 5432; do sleep 2; done"]
|
||||
|
||||
# With SQLite, the database file needs a local disk or block-backed volume,
|
||||
# even with one replica. ReadWriteOnce doesn't guarantee that: pick a
|
||||
# storageClass that isn't NFS or another network filesystem. If only network
|
||||
# filesystem storage is available, set config.data.database to Postgres.
|
||||
# Artifact storage can stay on NFS on a separate volume (extraVolumes and
|
||||
# extraVolumeMounts, with config.data.storage.url pointing at it).
|
||||
persistence:
|
||||
enabled: true
|
||||
# Keep this aligned with config.data.storage.url and config.data.database.path
|
||||
|
|
|
|||
|
|
@ -304,11 +304,6 @@ HTTP protocol handlers for each registry type.
|
|||
- `handleIndex()` - Proxy sparse index
|
||||
- `handleDownload()` - Serve cached crate
|
||||
|
||||
**DebianHandler:**
|
||||
- Serves `/debian` and `/debian/{name}` mirrors of APT archives (`upstream.debian_repositories`)
|
||||
- Routes `.deb` packages to the artifact cache, `dists/…` to the metadata cache, and proxies all other paths
|
||||
- Keys the artifact cache on name, version, and filename, scoped by repository for named archives
|
||||
|
||||
**SwiftHandler:**
|
||||
- Proxies the Swift Package Registry v1 read endpoints
|
||||
- Rewrites release URLs and caches source archives
|
||||
|
|
@ -321,7 +316,7 @@ HTTP protocol handlers for each registry type.
|
|||
|
||||
### `internal/metadata/npm`
|
||||
|
||||
The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. `ScanPackument` and `RewriteStream` do the same rewrite over a reader in two passes, the first collecting the time map, dist-tags and version names and the second copying each kept version, so the handler can rewrite a packument from storage without holding it in memory. It imports no database, storage, metrics, HTTP or logging packages, which a test checks.
|
||||
The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. It imports no database, storage, metrics, HTTP or logging packages, which a test checks.
|
||||
|
||||
### `internal/jsonscan`
|
||||
|
||||
|
|
@ -368,14 +363,10 @@ Package metadata enrichment, built on [github.com/git-pkgs/enrichment](https://g
|
|||
|
||||
### `internal/mirror`
|
||||
|
||||
Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads. Git repositories (`pkg:git`) go through `handler.GitHandler.Mirror()`, which caches the repository and the commit or tag, with its submodules, in the git cache that `/git` serves.
|
||||
Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work. Container images (`pkg:docker`, `pkg:oci`) go through `handler.ContainerHandler.MirrorImage()` instead, which walks index, platform manifests and blobs and stores them under the keys a pull reads.
|
||||
|
||||
The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints.
|
||||
|
||||
### `internal/inventory`
|
||||
|
||||
Turns an SBOM uploaded to `/api/projects` into the package versions recorded for the project. Package URLs are mapped to the ecosystem names and unversioned PURLs the handlers record packages under, so a project's components join with `packages` by `purl`. Also compares versions for the outdated marker on the project pages.
|
||||
|
||||
### `internal/config`
|
||||
|
||||
Configuration loading.
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@
|
|||
|
||||
The proxy can be configured via command line flags, environment variables, or a configuration file. Command line flags take precedence over environment variables, which take precedence over the configuration file.
|
||||
|
||||
Boolean environment variables accept `true`, `t`, `1`, `yes` or `on`, and `false`, `f`, `0`, `no` or `off`, in any case. Any other value, such as a typo, is ignored: the setting keeps its value from the configuration file or default, and the proxy logs a warning at startup.
|
||||
|
||||
## Configuration File
|
||||
|
||||
Create a YAML or JSON file and pass it with `-config`:
|
||||
|
|
@ -67,7 +65,7 @@ storage:
|
|||
|--------|-------------|------|-------------|
|
||||
| `storage.url` | `PROXY_STORAGE_URL` | `-storage-url` | Storage URL (file:// or s3://) |
|
||||
| `storage.path` | `PROXY_STORAGE_PATH` | `-storage-path` | Local path (deprecated, use url) |
|
||||
| `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB"). Negative or out-of-range sizes are rejected at startup |
|
||||
| `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB") |
|
||||
| `storage.cache_artifacts` | `PROXY_STORAGE_CACHE_ARTIFACTS` | - | Store fetched artifacts (default: true); `false` streams them from upstream |
|
||||
| `storage.retention.default` | `PROXY_STORAGE_RETENTION_DEFAULT` | - | Evict artifacts not served for this long (e.g. "30d"); see [Retention](#retention) |
|
||||
| `storage.retention.sweep_interval` | `PROXY_STORAGE_RETENTION_SWEEP_INTERVAL` | - | How often expired artifacts are looked for (default: "10m", at least "1m") |
|
||||
|
|
@ -113,7 +111,7 @@ storage:
|
|||
sweep_interval: "10m"
|
||||
```
|
||||
|
||||
Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. The blobs of an image that `mirror.schedules` keeps current also count from the last run that found them cached, so the schedule keeps them; for `max_size` eviction such a run doesn't count as use. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`.
|
||||
Each cached file ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`.
|
||||
|
||||
A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version, written in one of the forms the table below lists for the ecosystem. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems, except `generic`, which requires an explicit rule. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention supports (`covered`) and which ones `default` does not reach (`default_not_applied`), including `generic`.
|
||||
|
||||
|
|
@ -203,8 +201,6 @@ database:
|
|||
|
||||
The database runs in WAL mode. Writes go through a single connection, since SQLite allows one writer at a time. Reads use a separate pool of read-only connections (one per CPU, between 2 and 8), so request lookups don't wait behind writes, stats queries or eviction scans. An in-memory database (`:memory:`) uses one connection for both.
|
||||
|
||||
The SQLite database needs a local disk or a block-backed volume, even with one replica. SQLite's WAL mode doesn't work on network filesystems such as NFS ([sqlite.org/wal.html](https://www.sqlite.org/wal.html)), which shows up as stalled requests and `database disk image is malformed`. A Kubernetes `ReadWriteOnce` volume doesn't guarantee suitable storage, since its storage class can still be NFS. If only network filesystem storage is available to the proxy, use `driver: postgres`. Artifact storage can stay on NFS, on a separate volume from the SQLite database.
|
||||
|
||||
### PostgreSQL
|
||||
|
||||
```yaml
|
||||
|
|
@ -299,7 +295,6 @@ Each upstream used by a built-in package route can be set in YAML or JSON under
|
|||
| `upstream.allow_loopback` | `PROXY_UPSTREAM_ALLOW_LOOPBACK` | `false` |
|
||||
| `upstream.npm` | `PROXY_UPSTREAM_NPM` | `https://registry.npmjs.org` |
|
||||
| `upstream.npm_full_metadata` | `PROXY_UPSTREAM_NPM_FULL_METADATA` | `false` |
|
||||
| `upstream.npm_change_feed` | `PROXY_UPSTREAM_NPM_CHANGE_FEED` | `false` (see [change feeds](#change-feeds)) |
|
||||
| `upstream.cargo` | `PROXY_UPSTREAM_CARGO` | `https://index.crates.io` |
|
||||
| `upstream.cargo_download` | `PROXY_UPSTREAM_CARGO_DOWNLOAD` | `https://static.crates.io/crates` |
|
||||
| `upstream.gem` | `PROXY_UPSTREAM_GEM` | `https://rubygems.org` |
|
||||
|
|
@ -317,7 +312,6 @@ Each upstream used by a built-in package route can be set in YAML or JSON under
|
|||
| `upstream.nuget_search` | `PROXY_UPSTREAM_NUGET_SEARCH` | `https://azuresearch-usnc.nuget.org` |
|
||||
| `upstream.composer` | `PROXY_UPSTREAM_COMPOSER` | `https://packagist.org` |
|
||||
| `upstream.composer_repository` | `PROXY_UPSTREAM_COMPOSER_REPOSITORY` | `https://repo.packagist.org` |
|
||||
| `upstream.composer_change_feed` | `PROXY_UPSTREAM_COMPOSER_CHANGE_FEED` | `false` (see [change feeds](#change-feeds)) |
|
||||
| `upstream.conan` | `PROXY_UPSTREAM_CONAN` | `https://center.conan.io` |
|
||||
| `upstream.chef` | `PROXY_UPSTREAM_CHEF` | `https://supermarket.chef.io` |
|
||||
| `upstream.conda` | `PROXY_UPSTREAM_CONDA` | `https://conda.anaconda.org` |
|
||||
|
|
@ -329,7 +323,6 @@ Each upstream used by a built-in package route can be set in YAML or JSON under
|
|||
| `upstream.rpm` | `PROXY_UPSTREAM_RPM` | `https://dl.fedoraproject.org/pub/fedora/linux` |
|
||||
| `upstream.homebrew_api` | `PROXY_UPSTREAM_HOMEBREW_API` | `https://formulae.brew.sh/api` |
|
||||
| `upstream.homebrew_artifact` | `PROXY_UPSTREAM_HOMEBREW_ARTIFACT` | `https://ghcr.io` |
|
||||
| `upstream.apk_snapshots` | `PROXY_UPSTREAM_APK_SNAPSHOTS` | disabled; see [Alpine APK snapshots](#alpine-apk-snapshots) |
|
||||
|
||||
Private, ULA, CGNAT, and loopback addresses are rejected by default. Add each private upstream hostname or IP address to `upstream.allow_private_hosts`. The matching environment variable accepts a comma-separated list. Loopback upstreams also require `upstream.allow_loopback: true`. That setting permits upstream requests and redirects to reach any loopback address.
|
||||
|
||||
|
|
@ -355,8 +348,6 @@ upstream:
|
|||
composer_repository: "https://upstream-proxy.example.com/composer"
|
||||
```
|
||||
|
||||
The proxy reads `packages.json` from `upstream.composer_repository` and keeps it in memory for the metadata TTL, or an hour when the TTL is 0, whether or not `cache_metadata` is on. A failed fetch is not retried within that time: the last good copy is used, or the entries are left out. It passes on its `security-advisories` and `filter` entries (Packagist's malware list) with URLs pointing at the proxy. An upstream that offers neither is still usable; `composer audit` then reports nothing from it. A filter `api-url` is not passed on; Composer reads filter entries from package metadata instead.
|
||||
|
||||
`upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint.
|
||||
|
||||
`upstream.chef` is the Supermarket served at `/chef/`, for Berkshelf (`source "https://proxy.example.com/chef"`) and Policyfiles (`default_source :supermarket, "https://proxy.example.com/chef"`). The upstream has to serve the Supermarket API: `/universe`, `/api/v1/cookbooks/{name}`, its versions and their downloads. A private Supermarket that needs credentials takes them from `upstream.auth`, scoped to its URL. A download redirect to another host, such as the storage bucket behind Supermarket, does not receive them:
|
||||
|
|
@ -397,7 +388,6 @@ upstream:
|
|||
- Composer metadata of routed packages carries an empty `notification-url`, so Composer does not report their installs to the default upstream's `notify-batch` endpoint.
|
||||
- Use `upstream.auth` for the registry's credentials. The proxy has no authentication of its own, so every client that can reach it can download routed packages.
|
||||
- Search (`/composer/search.json`) and the package list (`/composer/packages/list.json`) still come from the default upstream only.
|
||||
- Composer security advisories and filter lists come from the default upstream. Routed package names, and those in `hosted.composer_vendors`, are removed from advisories requests before they are sent, so they only get advisories that their route's own metadata carries.
|
||||
- Attestations of a routed npm version (`/npm/-/npm/v1/attestations/...`) come from the route, like its tarballs.
|
||||
- npm audit, signing-key, search and ping requests still go to `upstream.npm`. An audit request lists every package in the dependency tree, routed ones included.
|
||||
- Routes can only be set in the configuration file, not with environment variables.
|
||||
|
|
@ -586,8 +576,6 @@ with the metadata TTL and served byte-for-byte unchanged so apk signature
|
|||
verification keeps working; `.apk` packages use the shared artifact cache.
|
||||
When `upstream.apk` is empty, a single repository named `alpine` pointing at
|
||||
the official mirror is available; configuring any entry replaces that default.
|
||||
Earlier index versions can be kept and served by date; see
|
||||
[Alpine APK snapshots](#alpine-apk-snapshots).
|
||||
|
||||
```yaml
|
||||
upstream:
|
||||
|
|
@ -635,53 +623,6 @@ as signatures and keys, is passed through uncached. When `upstream.freebsd` is e
|
|||
repository named `freebsd` pointing at `https://pkg.freebsd.org` is available;
|
||||
configuring any entry replaces that default.
|
||||
|
||||
### Alpine APK snapshots
|
||||
|
||||
```yaml
|
||||
upstream:
|
||||
# Keep replaced APK indexes for 90 days.
|
||||
apk_snapshots: "90d"
|
||||
```
|
||||
|
||||
Alpine mirrors only keep the current build of each package, so after an
|
||||
upgrade `apk add curl=8.5.0-r0` fails because the index no longer lists that
|
||||
build. With `upstream.apk_snapshots` set, the proxy keeps every version of an
|
||||
index (and of a detached signature) it stores, and serves them under a dated
|
||||
repository name:
|
||||
|
||||
```
|
||||
http://localhost:8080/apk/alpine@2026-10-01/v3.22/main
|
||||
```
|
||||
|
||||
`/apk/{name}@{YYYY-MM-DD}/…` returns the index that was current at the end of
|
||||
that day (UTC). The kept files are the signed upstream bytes, so apk verifies
|
||||
them as usual. A day before the first stored version of an index, or a day
|
||||
that has not started yet, is `404`. Today's snapshot can still change until the
|
||||
day ends. Packages under a snapshot path are the same as under the plain path:
|
||||
they come from the artifact cache, or from upstream if not cached; a build the
|
||||
upstream has dropped and the proxy never cached is `404`. A configured
|
||||
repository whose name contains `@` takes precedence over a snapshot path.
|
||||
|
||||
Snapshots are recorded when an index is requested through the plain path, so
|
||||
they only exist for repositories clients use. A version counts from the first
|
||||
time the proxy served it, which can be up to `metadata_ttl` after upstream
|
||||
published it. A version is deleted once it was replaced more than the
|
||||
configured duration ago; the newest version of each index is always kept.
|
||||
Durations take a `d` suffix; empty or `"0"` disables snapshots. Snapshots need
|
||||
`cache_metadata`. Their files live under `_snapshots/` in storage and do not
|
||||
count towards `storage.max_size`.
|
||||
|
||||
Turning snapshots off, or `cache_metadata` off, leaves the kept versions in
|
||||
place. To remove them, first set a short duration such as `"1h"` and let the
|
||||
hourly prune delete everything but the newest version of each index.
|
||||
|
||||
Snapshots keep indexes, not packages. A package listed by an old snapshot can
|
||||
still be evicted by `storage.max_size` or [retention](#retention), and is then
|
||||
only available while upstream still has it. Retention counts from a package's
|
||||
last download, so a pinned build nobody installed for longer than the `alpine`
|
||||
retention is evicted even while a snapshot lists it. Use package rules with
|
||||
`"0"` for builds that must stay installable.
|
||||
|
||||
### Go checksum database
|
||||
|
||||
```yaml
|
||||
|
|
@ -901,14 +842,8 @@ commits, the request waits for their fetch, so a clone right after a push gets
|
|||
them; if the fetch fails or stalls, the cached copy is served. A
|
||||
`stale_max_age` of `0` never serves a copy the upstream has not just
|
||||
confirmed. A repository the upstream reports missing (404), refuses (401, 403)
|
||||
or redirects is not served from the cache, also to requests that skip the ref
|
||||
listing and go straight to `git-upload-pack`, until a later check succeeds.
|
||||
Redirects are never followed, so point the upstream at a repository's current
|
||||
location. A `git-upload-pack` request does not check the upstream itself
|
||||
unless the upstream refused the repository at the last check or the copy was
|
||||
last checked more than a minute past `stale_max_age` ago, so the requests of a
|
||||
clone that follow its ref listing check nothing again; otherwise the
|
||||
repository is checked first, as for a ref listing.
|
||||
or redirects is not served from the cache. Redirects are never followed, so
|
||||
point the upstream at a repository's current location.
|
||||
|
||||
The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it).
|
||||
The `git` processes never reach the network themselves: their fetches go
|
||||
|
|
@ -954,12 +889,6 @@ those yourself. The proxy locks `cache_dir` while it runs, through a
|
|||
`.gitcache-lock` file in it, so a second process given the same directory,
|
||||
such as another replica on a shared volume, fails to start; give each its own.
|
||||
|
||||
Each repository's directory is named by its path as requested. Where
|
||||
`cache_dir` ignores case in file names, as on macOS and Windows by default,
|
||||
paths that differ only in case (`acme/widgets` and `acme/WIDGETS`) share one
|
||||
directory and are treated as one repository. Use a case-sensitive file system
|
||||
for an upstream whose repositories differ only in case.
|
||||
|
||||
Git LFS objects are cached in the artifact cache, so they use the configured
|
||||
storage backend and retention rather than `cache_dir`. git-lfs finds the LFS
|
||||
API under the rewritten remote, at `/git/{name}/{repo}.git/info/lfs`, with no
|
||||
|
|
@ -1018,13 +947,6 @@ one doesn't fall back to the Gradle Plugin Portal. Files are cached per
|
|||
repository, so the same coordinates in two repositories never share a cache
|
||||
entry.
|
||||
|
||||
In every repository, a file the proxy would cache (an artifact, checksum or
|
||||
signature) whose group directories contain a dot or colon
|
||||
(`org.foo/bar/1.0/bar-1.0.jar`), or whose artifact contains a colon, is not in
|
||||
the Maven layout and gets 404 without an upstream request. Its group would read
|
||||
the same as `org/foo`'s. Other paths there, such as `maven-metadata.xml` or a
|
||||
directory listing, are handled as anywhere else.
|
||||
|
||||
`upstream.maven_repositories` is additive. `/maven/…` keeps serving
|
||||
`upstream.maven` with unchanged cache identities. A repository name is matched
|
||||
before the default repository, so it hides every group ID whose first segment
|
||||
|
|
@ -1169,37 +1091,6 @@ Uploads are off by default, so a client that can reach the proxy cannot overwrit
|
|||
|
||||
`max_age` and `max_size` are independent and can be combined. When both are set, age-based eviction runs first, then size-based eviction trims remaining entries oldest-first.
|
||||
|
||||
## Cooldown profiles
|
||||
|
||||
Set `cooldown.profiles` to offer several policies from one instance. Clients select a profile through their package manager's registry URL:
|
||||
|
||||
```yaml
|
||||
cooldown:
|
||||
default: "0"
|
||||
profiles:
|
||||
3d:
|
||||
default: "3d"
|
||||
7d:
|
||||
default: "7d"
|
||||
packages:
|
||||
"pkg:npm/@example/tool": "0"
|
||||
```
|
||||
|
||||
For the `3d` profile, use these paths after your proxy's base URL:
|
||||
|
||||
| Client | Path |
|
||||
|--------|------|
|
||||
| npm registry | `/cooldown/3d/npm/` |
|
||||
| Go `GOPROXY` | `/cooldown/3d/go` |
|
||||
| Cargo registry | `sparse+https://your-proxy.example/cooldown/3d/cargo/` |
|
||||
| Bundler RubyGems mirror | `/cooldown/3d/gem` |
|
||||
|
||||
The prefix works with all package protocol routes. Cooldown has an effect only for [registries with cooldown support](../README.md#supported-registries). Each profile defines its own `default`, `ecosystems`, `packages` and `package_patterns`; it does not inherit the global cooldown overrides. Existing unprefixed URLs, the UI, APIs and background mirror jobs use the global policy.
|
||||
|
||||
Profile names contain lowercase letters, digits, hyphens and underscores, and start with a letter or digit. Profiles must be configured before startup; an unknown profile returns 404. Nested profiles are rejected. Profile paths expose package protocols only, so `/cooldown/3d/ui/` and `/cooldown/3d/api/` return 404.
|
||||
|
||||
Database records, artifacts and upstream metadata are shared. Generated download URLs retain the profile prefix, and filtered metadata and its validators are separated by profile. Changing a registry URL does not rewrite artifact URLs already recorded in a lockfile; regenerate those URLs if the selected policy must cover the whole installation.
|
||||
|
||||
## Cooldown
|
||||
|
||||
The cooldown feature hides package versions published too recently, giving the community time to spot malicious releases before they reach your projects. When a version is within its cooldown period, it's stripped from metadata responses so package managers won't install it.
|
||||
|
|
@ -1236,7 +1127,7 @@ Exact `packages` entries take precedence over patterns. When several patterns ma
|
|||
|
||||
Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families.
|
||||
|
||||
The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported. The version list marks versions held back with when they become available, and leaves out versions the proxy only saw in metadata once they're out of cooldown. `/api/package/{ecosystem}/{name}` returns the package's `cooldown` in the configuration's syntax, and a held-back version has `available_at`.
|
||||
The web UI's dashboard shows the default and the ecosystem overrides, and how many package overrides and patterns there are; their names aren't shown, since the UI has no authentication. A package's page shows the cooldown that applies to it and which kind of rule it comes from; it shows none for ecosystems without cooldown support, and the dashboard marks an ecosystem override for one as not supported.
|
||||
|
||||
Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go, Maven and Helm. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below.
|
||||
|
||||
|
|
@ -1253,21 +1144,15 @@ Maven caveats:
|
|||
|
||||
Helm uses each release's `created` time from the repository's `index.yaml`. Releases in cooldown are left out of the index the proxy serves, and their archives aren't served; a release without `created` is let through. Charts pulled over OCI have no publish time and aren't held back.
|
||||
|
||||
Cargo, RubyGems, Hex, pub.dev, Conda and Helm check artifact downloads against cooldown, including files cached before the policy changed. When a publish-time lookup fails, an already-cached file is still served; an uncached download fails with 502, or 503 with `Retry-After` when the rate-limit gate holds the request. Conda checks timestamps per file because builds of one version can have different publish times. As in conda, a repodata `timestamp` is read as milliseconds when it is above 253402300799 (the end of year 9999 in seconds) and as seconds otherwise, since some older entries use seconds.
|
||||
|
||||
Conda repodata and Helm indexes are parsed once per cached document for download checks. With `cache_metadata` off, the parsed index is kept for one minute, including the fetch, so an install can reuse it across files. Conda streams repodata when its compressed copy exceeds `metadata_max_size`, under the same decompressed size bound as the metadata filter.
|
||||
|
||||
RubyGems, Hex and PyPI keep their publish times outside the metadata clients read: the RubyGems versions API, the Hex HTTP API (`upstream.hex_api`) and the PyPI JSON API. The proxy fetches them alongside each compact index, package or simple page it filters, and for PyPI downloads. Cooldown fails closed: when it applies to a package and those times can't be fetched or parsed, or the metadata can't be rewritten, the request fails with 502, or with 503 and `Retry-After` while the upstream is rate limiting the proxy, instead of serving the metadata unfiltered. The same goes for npm, Composer and pub.dev metadata that can't be rewritten, and for npm tarball downloads whose publish time can't be read. A PyPI file or npm tarball that is already cached is still served when its publish time can't be read; only a download that would have to be fetched fails. A PyPI release's time is the upload time of its newest file. The proxy stores it when a download looks it up, and serves files of that release that are already cached against the stored time without reading the JSON API again. A file that isn't cached yet is always looked up, since it may have been added to the release since, and the newer time is stored. Packages exempted with a duration of `0` are served as before and don't need the times. A PyPI package the JSON API returns 404 for has no times to go by and is served unfiltered.
|
||||
|
||||
For RubyGems and Hex, both documents go through the metadata cache. With [`cache_metadata`](#metadata-caching) they are served from it within `metadata_ttl` without asking the upstream, revalidated with their ETags after it, and filtered from the last cached copies, marked stale, while the upstream is down; without it they are still fetched once for concurrent requests. Each document has its own TTL, so the compact index or Hex package can be newer than the cached times and list a version they don't have. The times are then fetched again before filtering, and if that fails, such versions are left out until the times can be read. The filtered index or package carries an ETag of its own, so Bundler and Mix revalidate it with `304 Not Modified`. Publish times are compared with the current time on each request, and the filtered result is kept until the first version it held back leaves cooldown (see [rewritten metadata cache](#rewritten-metadata-cache)); a newly published version shows up within `metadata_ttl`, as with other cached metadata. Upstream errors are answered as for other cached metadata (404, or 502) rather than relayed.
|
||||
|
||||
With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead; other tags that name a version in cooldown are removed, as for the denylist, and `npm dist-tag ls` shows the same tags. A tarball request for a version the cached packument doesn't list yet fetches the packument again, within `metadata_ttl` or not, and is decided on what the upstream lists now; if the packument can't be fetched, the download is not let through, as when a publish time can't be read.
|
||||
|
||||
In pub.dev package metadata, when `latest` is in cooldown it names the newest stable version out of cooldown instead, or the newest prerelease when no stable version is left, and it is left out when every version is in cooldown.
|
||||
With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. When `latest` is in cooldown, the packument tags the newest version out of cooldown instead, and `npm dist-tag ls` shows the same tags.
|
||||
|
||||
Conda cooldown covers `repodata.json`, `current_repodata.json`, `repodata_from_packages.json` and sharded repodata (CEP 16). Other forms of the repodata, such as `repodata.json.zst`, `repodata.json.bz2` and `repodata.jlap` patches, would list versions still in cooldown, so with cooldown on they are answered 404 and conda and mamba fall back to `repodata.json`. With `cache_metadata`, `repodata.json` is read through the metadata cache as it is without cooldown: within `metadata_ttl` the stored copy is filtered as it is read from storage, after it the upstream is asked with its ETag, and while the upstream is down the stale copy is filtered. Once a document has been filtered, the proxy remembers the ETag of the result until the first entry it left out leaves cooldown, so a client revalidating with `If-None-Match` gets `304 Not Modified` without the document being filtered or sent again. The first response for each document has no ETag, as it is sent while it is filtered. Without `cache_metadata`, `repodata.json` is fetched and filtered on every request. Clients cache shards by the hash in the shard index, so with cooldown on the proxy serves a rewritten index whose shard hashes change every hour and filters each shard as it is fetched. Versions leaving cooldown show up within the hour, and clients fetch the shards they need again once an hour. If a channel's shard index points packages or shards somewhere the proxy doesn't route, the proxy answers 404 for it and clients fall back to `repodata.json`.
|
||||
|
||||
Hex cooldown requires `HEX_UNSAFE_REGISTRY=1` because filtering releases re-encodes the protobuf payload without the original signature. The proxy preserves the package name and repository fields, so origin verification can stay enabled. `HEX_NO_VERIFY_REPO_ORIGIN` controls origin verification and does not disable signature verification. Without a cooldown, leave signature verification enabled too.
|
||||
Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`.
|
||||
|
||||
## Version denylist
|
||||
|
||||
|
|
@ -1291,13 +1176,6 @@ Names and versions are compared the way each ecosystem treats them:
|
|||
- PyPI names ignore case, and runs of `-`, `_` and `.` are equivalent (PEP 503),
|
||||
so `pkg:pypi/zope.interface@6.0` also covers `zope_interface-6.0-*.whl`.
|
||||
- NuGet ids and versions ignore case.
|
||||
- Debian, RPM, Alpine, Arch and FreeBSD entries ignore the distribution
|
||||
namespace: `pkg:deb/debian/curl@8.5.0-2` and `pkg:deb/curl@8.5.0-2` are the
|
||||
same entry.
|
||||
- Debian and RPM filenames leave out the epoch, so an entry with one also
|
||||
covers the version without it: `pkg:deb/foo@1:2.3-1` denies
|
||||
`foo_2.3-1_amd64.deb`. An entry without an epoch only matches a version
|
||||
without one.
|
||||
- A RubyGems entry also covers that version's platform gems:
|
||||
`pkg:gem/nokogiri@1.16.0` denies `nokogiri-1.16.0-x86_64-linux.gem`.
|
||||
- Composer versions ignore a leading `v`, which Packagist keeps from the tag:
|
||||
|
|
@ -1326,7 +1204,7 @@ Names and versions are compared the way each ecosystem treats them:
|
|||
channel only.
|
||||
|
||||
Denied versions are removed from npm packuments, Cargo sparse indexes, and PyPI
|
||||
simple HTML/JSON (files and the JSON `versions` list) and release metadata, including responses served from cached
|
||||
simple HTML/JSON and release metadata, including responses served from cached
|
||||
metadata during an upstream outage. No publication timestamp or enabled cooldown
|
||||
is required. npm tags pointing at denied versions are removed; `latest` is moved
|
||||
to the highest remaining stable version when available. Other tags are not
|
||||
|
|
@ -1428,7 +1306,7 @@ scanning:
|
|||
| `scanning.scanners[].name` | - | Identifies this scanner in logs and metrics |
|
||||
| `scanning.scanners[].url` | - | Endpoint the proxy POSTs scan notifications to |
|
||||
| `scanning.scanners[].mode` | - | `block` (default) or `monitor` |
|
||||
| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems. Valid names: `alpine`, `arch`, `cargo`, `chef`, `composer`, `conan`, `conda`, `cran`, `deb`, `freebsd`, `gem`, `generic`, `git` (Git LFS objects), `golang`, `helm`, `hex`, `julia`, `maven`, `npm`, `nuget`, `oci`, `pub`, `pypi`, `rpm`, `swift`. The aliases `go`, `docker`, `rubygems`, `packagist`, `debian` and `apk` are accepted too; any other name stops the proxy at startup |
|
||||
| `scanning.scanners[].ecosystems` | - | Restricts this scanner to specific ecosystems (e.g. `npm`, `pypi`). Empty means all ecosystems |
|
||||
| `scanning.scanners[].headers` | - | Extra HTTP headers sent with every scan request (e.g. for authenticating to the scanner service). Values support `${VAR_NAME}` expansion |
|
||||
|
||||
### How caching defers to a scan verdict
|
||||
|
|
@ -1531,36 +1409,15 @@ Set to `"0"` to always revalidate with upstream (ETag-based conditional requests
|
|||
|
||||
When upstream is unreachable and the cached entry is past its TTL, the proxy serves the stale cached copy with a `Warning: 110 - "Response is Stale"` header so clients can tell the data may be outdated.
|
||||
|
||||
### Change feeds
|
||||
|
||||
Within `metadata_ttl` a new release isn't seen. For npm and Composer the proxy can instead follow the registry's change feed, which lists the packages that changed, and revalidate just those:
|
||||
|
||||
```yaml
|
||||
cache_metadata: true
|
||||
upstream:
|
||||
npm_change_feed: true
|
||||
composer_change_feed: true
|
||||
```
|
||||
|
||||
Or via environment variables: `PROXY_UPSTREAM_NPM_CHANGE_FEED=true`, `PROXY_UPSTREAM_COMPOSER_CHANGE_FEED=true`. Both require `cache_metadata`, and do nothing with a `metadata_ttl` of `"0"`.
|
||||
|
||||
Every minute the proxy reads what changed since its last read. Cached metadata of a listed package that was fetched before the read is marked as needing revalidation, and the next request for it asks the upstream with its `ETag` (a `304` when nothing changed). Artifacts are not touched. Listed packages the proxy hasn't cached cost nothing beyond the read. The feed is read in full, so the upstream doesn't learn which packages the proxy has cached. Packages in `upstream.npm_routes` or `upstream.composer_routes` are cached under other keys and never matched.
|
||||
|
||||
- Composer reads `{upstream.composer}/metadata/changes.json`, which packagist.org advertises as `metadata-changes-url`. Both `vendor/package` and its `vendor/package~dev` file are covered. Packagist keeps 24 hours of changes; if the proxy's position is older than that, Packagist asks for a resync and all cached Composer metadata is revalidated. repo.packagist.org sends `Cache-Control: max-age=900`, but Packagist purges its CDN when a package changes, so revalidation gets the new file within seconds; the proxy doesn't apply that header either way.
|
||||
- npm reads `https://replicate.npmjs.com/registry/_changes`, the public registry's feed, whatever `upstream.npm` is. It is only useful when `upstream.npm` is registry.npmjs.org or serves the same packages. Packuments (`/npm/{name}`) are covered; version and dist-tag documents (`/npm/{name}/latest`) still wait out `metadata_ttl`. A busy minute on npm lists a few thousand packages; the proxy reads up to 100,000 per pass and continues where it stopped on the next.
|
||||
|
||||
A new release therefore shows up about a minute after the registry lists it rather than up to `metadata_ttl` later. The feed position is kept in memory: after a restart the proxy starts from the feed's current position, and changes made while it was down are covered by `metadata_ttl` as before. While a feed can't be read the proxy logs a warning and retries with a growing delay, up to 15 minutes, and `metadata_ttl` applies meanwhile. A request whose upstream fetch overlaps the change and the feed read can still store the old document for one TTL. Each instance sharing a database reads the feeds itself. `proxy_metadata_expired_by_change_feed_total` counts the cached documents marked, by ecosystem, and debug logging names each one.
|
||||
|
||||
### Metadata size limit
|
||||
|
||||
Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached. With `cache_metadata` and the metadata rewrite cache on, an npm packument is streamed from the upstream into storage and rewritten from there in two passes, so it isn't held in memory and the limit doesn't apply to it; the rewritten copy is kept gzip-compressed. Requests for a single version (`/<package>/<version>`) and tarball downloads read the version from the stored copy the same way. `metadata_stream_max_size` (1 GB by default) caps a document streamed this way; one over it isn't stored, and the stored copy, if any, is served instead.
|
||||
Upstream metadata responses are buffered in memory before being rewritten and served. `metadata_max_size` caps that buffer to protect against OOM from a misbehaving upstream. Some npm packages with thousands of versions (for example `renovate`) exceed the 100 MB default, so raise this if you see `metadata response exceeds size limit` in the logs. Conda `repodata.json` with cooldown enabled is filtered as it streams and isn't held in memory decompressed. With `cache_metadata` the limit applies to the compressed document as it does without cooldown; one over it is filtered as it streams from the upstream and not cached.
|
||||
|
||||
```yaml
|
||||
metadata_max_size: "100MB" # default
|
||||
metadata_stream_max_size: "1GB" # default
|
||||
metadata_max_size: "100MB" # default
|
||||
```
|
||||
|
||||
Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`, and `PROXY_METADATA_STREAM_MAX_SIZE=2GB` for `metadata_stream_max_size`.
|
||||
Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`.
|
||||
|
||||
### Artifact size limit
|
||||
|
||||
|
|
@ -1673,7 +1530,6 @@ Pushed versions are listed in the package metadata clients resolve against, with
|
|||
- Pushed versions are added next to the upstream ones and are never held back by cooldown. npm `dist-tags` stay as the upstream has them, also in `npm dist-tag ls`.
|
||||
- A package the upstream doesn't know is served from its pushed versions alone. Its npm `latest` tag is the highest pushed version.
|
||||
- Pushed Composer dev versions (`dev-*`, `*-dev`) are listed in the `~dev` metadata file.
|
||||
- Pushed Composer versions carry an empty `notification-url`, as routed packages do, so Composer does not report their installs to the default upstream's `notify-batch` endpoint. A pushed `composer.json` that sets its own `notification-url` keeps it, and Composer then reports installs there.
|
||||
- npm entries carry the archive's sha512 as `dist.integrity`. Composer entries carry its sha256 as `dist.reference` and leave `shasum` empty.
|
||||
|
||||
Metadata of packages with pushed versions is rewritten on every request instead of being cached, so a push or delete shows up on the next request.
|
||||
|
|
@ -1713,89 +1569,6 @@ Generic files never come from an upstream: anything not pushed answers 404. Whil
|
|||
|
||||
Pushed archives are stored like cached ones but marked as local. The cache size limit and storage retention never remove them; only a delete does. Scanning, when enabled, checks pushed archives like fetched ones.
|
||||
|
||||
## Project inventory
|
||||
|
||||
The project inventory answers "which repository uses which version of a package?". Each repository uploads an SBOM of its default branch, and the proxy records the package versions it lists. The UI then shows:
|
||||
|
||||
- `/ui/projects`: every project, and per project its package versions, marked when the proxy knows a newer version.
|
||||
- On a package page: which projects use each version, including versions the proxy hasn't cached.
|
||||
|
||||
It is disabled by default. Uploads and deletes require a token:
|
||||
|
||||
```yaml
|
||||
projects:
|
||||
enabled: true
|
||||
token: "${PROXY_PROJECTS_TOKEN}"
|
||||
max_age: "90d"
|
||||
```
|
||||
|
||||
| Config | Environment | Description |
|
||||
|--------|-------------|-------------|
|
||||
| `projects.enabled` | `PROXY_PROJECTS_ENABLED` | Register the `/api/projects` endpoints and the UI pages (default `false`) |
|
||||
| `projects.token` | `PROXY_PROJECTS_TOKEN` | Bearer token required to upload and delete; required when enabled, supports `${VAR}` |
|
||||
| `projects.max_age` | `PROXY_PROJECTS_MAX_AGE` | Remove projects that haven't uploaded for this long, e.g. `90d` or `720h`; empty or `0` keeps them (default) |
|
||||
|
||||
Removing old projects runs hourly. With several instances on one database, one of them does it.
|
||||
|
||||
### Uploading an SBOM
|
||||
|
||||
`PUT /api/projects/{project}/sbom` takes a CycloneDX (JSON or XML) or SPDX (JSON or tag-value) document of up to 32 MiB and replaces everything recorded for the project with what it lists. The optional `ref` and `branch` query parameters are stored and shown with the project.
|
||||
|
||||
The project name is a path such as `group/sub/repo`: segments of letters, digits, `.`, `_` and `-`, at most 255 characters. It can be sent as path segments (`/api/projects/group/sub/repo/sbom`) or URL-encoded as one segment (`/api/projects/group%2Fsub%2Frepo/sbom`).
|
||||
|
||||
```bash
|
||||
syft dir:. -o cyclonedx-json > sbom.json
|
||||
curl -X PUT --data-binary @sbom.json \
|
||||
-H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \
|
||||
"https://proxy.example.com/api/projects/acme/shop/web/sbom?ref=$(git rev-parse HEAD)&branch=main"
|
||||
```
|
||||
|
||||
The response counts the recorded package versions and the skipped entries:
|
||||
|
||||
```json
|
||||
{"project":"acme/shop/web","components":9,"skipped":6}
|
||||
```
|
||||
|
||||
Entries are recorded by their package URL, with the package named the way the proxy caches it: NuGet IDs lowercased, PyPI names normalized as in PEP 503 (`zope.interface` becomes `zope-interface`), the distribution dropped from `deb`, `rpm` and `apk` package URLs. Entries without a package URL (files, the operating system), package types the proxy doesn't serve (such as `pkg:github`), entries without a version, and conda entries without a `channel` qualifier (the proxy records conda packages as `channel/name`) are skipped. So is the project's own package: the root of a lock file such as `package-lock.json`, and with syft, the Go module its binaries were built from. Duplicates are recorded once.
|
||||
|
||||
A GitLab CI job on the default branch could look like this:
|
||||
|
||||
```yaml
|
||||
sbom:
|
||||
image: alpine:3.20
|
||||
rules:
|
||||
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
||||
script:
|
||||
- apk add --no-cache curl
|
||||
- curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
|
||||
- syft dir:. -o cyclonedx-json > sbom.json
|
||||
- >
|
||||
curl -sSf -X PUT --data-binary @sbom.json
|
||||
-H "Authorization: Bearer $PROXY_PROJECTS_TOKEN"
|
||||
"https://proxy.example.com/api/projects/$CI_PROJECT_PATH/sbom?ref=$CI_COMMIT_SHA&branch=$CI_COMMIT_BRANCH"
|
||||
```
|
||||
|
||||
### Querying
|
||||
|
||||
```bash
|
||||
curl https://proxy.example.com/api/projects # all projects
|
||||
curl https://proxy.example.com/api/projects/acme/shop/web # one project's package versions
|
||||
curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash' # projects using lodash
|
||||
curl 'https://proxy.example.com/api/projects?purl=pkg:npm/lodash@4.17.20' # projects using lodash 4.17.20
|
||||
curl -X DELETE -H "Authorization: Bearer $PROXY_PROJECTS_TOKEN" \
|
||||
https://proxy.example.com/api/projects/acme/shop/web
|
||||
```
|
||||
|
||||
Reading needs no token, like the rest of `/api`. With `?purl=`, each project carries the `version` it uses and no `component_count`. A delete names the project itself: `DELETE /api/projects/acme/shop/web/sbom` is answered with 400 unless a project `acme/shop/web/sbom` exists.
|
||||
|
||||
A project's page in the UI also has a delete button. It asks for the token and sends it to `DELETE /ui/api/projects/{project}`, which works like the API endpoint and also requires the token; the token is kept for the browser session. Like `/ui/api/cache`, it sits under `/ui`, so rules for the UI at an ingress cover it.
|
||||
|
||||
### What it shows
|
||||
|
||||
The inventory reflects what the projects' SBOMs list, not what was downloaded through the proxy: a project that installs from another registry still shows up, and the proxy may not have cached any of its packages. The outdated marker compares the version a project uses with the latest version the proxy has recorded for the package; a package the proxy has no record of has none. Versions are compared as the SBOM gives them: a version spelled differently from how the proxy caches it, such as Composer's `v6.4.1` for `6.4.1` or a Debian version with or without its epoch, shows as "not cached" on the package page.
|
||||
|
||||
Project names and the packages they use are shown on `/ui` and `/api/projects` without authentication. If that is sensitive, restrict both at your ingress or reverse proxy.
|
||||
|
||||
## Mirror API
|
||||
|
||||
The `/api/mirror` endpoints are disabled by default. Enable them to allow starting mirror jobs via HTTP:
|
||||
|
|
@ -1851,7 +1624,7 @@ curl -X DELETE http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1
|
|||
curl -X DELETE 'http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre?filename=guava-32.1.3-jre.jar'
|
||||
```
|
||||
|
||||
The name is the package name the UI shows, for example `group:artifact` for Maven. An RPM package's `filename` is its path under `upstream.rpm` (`updates/40/Everything/x86_64/Packages/c/curl-8.6.0-10.fc40.x86_64.rpm`), since repositories under one upstream can ship different files under one name. The response lists the deleted files. The next request for a deleted file fetches it from the upstream again. The stored bytes are removed after the same delay as a replaced artifact, so downloads already in progress can finish.
|
||||
The name is the package name the UI shows, for example `group:artifact` for Maven. The response lists the deleted files. The next request for a deleted file fetches it from the upstream again. The stored bytes are removed after the same delay as a replaced artifact, so downloads already in progress can finish.
|
||||
|
||||
Pushed archives are not touched: a whole-version delete skips them, and naming one answers 409. Delete those through the [hosted API](#hosted-packages).
|
||||
|
||||
|
|
@ -1875,8 +1648,6 @@ A Swift release is mirrored from `upstream.swift` with its source archive, check
|
|||
|
||||
A CRAN version is mirrored as its source package from `upstream.cran`, for example `pkg:cran/glue@1.8.1`, and cached under the file name, so it is served for both `/cran/src/contrib/{file}` and `/cran/src/contrib/Archive/{name}/{file}`, where CRAN moves it once a newer version is published. The current version is fetched from `src/contrib` and checked against the MD5 sum in `src/contrib/PACKAGES.gz`; older versions are fetched unchecked from `src/contrib/Archive/{name}/`, as the CRAN handler fetches them. Binary packages aren't mirrored, and neither are the `PACKAGES` indexes, so `install.packages` needs the upstream or an index the proxy already cached; installing the tarball by URL works without either.
|
||||
|
||||
An Alpine package is mirrored from the repository in `upstream.apk` named as the namespace, `alpine` by default, as syft writes it in image SBOMs: `pkg:apk/alpine/busybox@1.36.1-r29?arch=x86_64&distro=alpine-3.20.3`. The `distro` qualifier gives the release (`v3.20`) and `arch` the architecture; packages without either are refused. Edge isn't mirrored: a `distro` with a pre-release suffix, such as `alpine-3.21.0_alpha20240807`, is refused, but an edge system that reports a plain release number can't be told apart and is mirrored from that release. The package is looked up in the `main` and then the `community` `APKINDEX.tar.gz` of that release and cached under the path `/apk/{repository}/{release}/{repo}/{arch}/{file}` serves it from. The file isn't checked by the proxy, as when serving, since the index checksum covers only the package's control section; apk checks it on install. The indexes aren't mirrored, so `apk add` needs the upstream or an index the proxy already cached. Give a version: Alpine repositories keep only the current one.
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--sbom` | | Path to CycloneDX or SPDX SBOM file |
|
||||
|
|
@ -1939,22 +1710,6 @@ proxy mirror --platform linux/arm64 pkg:brew/hello@2.12.3 pkg:brew/openssl%403
|
|||
- Dependencies aren't mirrored with a formula; list them too. brew also needs the JSON API, which the proxy keeps with `cache_metadata`.
|
||||
- Casks, source archives and taps other than homebrew/core aren't mirrored.
|
||||
|
||||
### Git repositories
|
||||
|
||||
`pkg:git` PURLs mirror a repository of an [`upstream.git`](#git-repositories) host into the git cache, so a clone through `/git` is served from it, also when the upstream is unreachable. The PURL names the repository by host and path, and the version is a full commit ID or a tag:
|
||||
|
||||
```bash
|
||||
proxy mirror pkg:git/github.com/git-pkgs/purl@0123456789abcdef0123456789abcdef01234567
|
||||
proxy mirror pkg:git/github.com/git-pkgs/purl@v0.2.1
|
||||
proxy mirror pkg:git/github.com/git-pkgs/purl
|
||||
```
|
||||
|
||||
- The repository is served by the `upstream.git` host whose URL its host and path start with, the longest one if several do, and has to match that host's `repos`. Others fail.
|
||||
- The repository's branches and tags are cached, as a clone through the proxy caches them. A commit on none of them is fetched by ID and kept, like one a client fetches by ID, until a branch or tag includes it, or for seven days from the last mirror run that named it, as long as `git.max_size` doesn't evict the repository. Where `upstream.auth` has credentials for the host, commits aren't fetched by ID, and one on no branch or tag that isn't cached yet fails.
|
||||
- Without a version the repository's branches and tags are mirrored. A tag as the version has to be one of them, so a tag the upstream doesn't have fails. When the upstream can't be reached, a cached tag is mirrored at the commit it last pointed to, with a warning in the log. A branch name or an abbreviated commit ID as the version fails too; give the full commit ID.
|
||||
- With a version, the submodules that commit records are mirrored at the commits it pins, with their submodules in turn. A relative submodule URL is resolved against the repository's, and an `ssh://`, `git://` or `git@host:path` URL names the repository by its host and path. A submodule no `upstream.git` host serves is skipped with a warning in the log, as are all of a commit's submodules when git can't read its `.gitmodules`.
|
||||
- The git cache is used by one process. While the server runs, the command can't open `git.cache_dir`, so `pkg:git` items fail and the rest are mirrored; use the mirror API instead.
|
||||
|
||||
### Scheduled refresh
|
||||
|
||||
Tags such as `alpine:3.20` move. The server can re-mirror them on its own, so the cache follows the tag even when nobody pulls it or the proxy is offline later:
|
||||
|
|
|
|||
|
|
@ -460,194 +460,6 @@ const docTemplate = `{
|
|||
}
|
||||
}
|
||||
},
|
||||
"/api/projects": {
|
||||
"get": {
|
||||
"description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"projects"
|
||||
],
|
||||
"summary": "List projects",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20",
|
||||
"name": "purl",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ProjectListResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/projects/{project}": {
|
||||
"get": {
|
||||
"description": "Returns the project with the package versions recorded for it.",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"projects"
|
||||
],
|
||||
"summary": "Get a project",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Project path, e.g. group/sub/repo",
|
||||
"name": "project",
|
||||
"in": "path",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ProjectResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"delete": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.",
|
||||
"tags": [
|
||||
"projects"
|
||||
],
|
||||
"summary": "Delete a project",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Project path, e.g. group/sub/repo",
|
||||
"name": "project",
|
||||
"in": "path",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"204": {
|
||||
"description": "No Content"
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "Unauthorized",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/projects/{project}/sbom": {
|
||||
"put": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of ` + "`" + `syft dir:. -o cyclonedx-json` + "`" + `. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.",
|
||||
"consumes": [
|
||||
"application/json",
|
||||
"application/xml",
|
||||
"text/plain"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"projects"
|
||||
],
|
||||
"summary": "Upload a project's SBOM",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Project path, e.g. group/sub/repo",
|
||||
"name": "project",
|
||||
"in": "path",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Commit the SBOM was generated from",
|
||||
"name": "ref",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Branch the SBOM was generated from",
|
||||
"name": "branch",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ProjectUploadResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "Unauthorized",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"413": {
|
||||
"description": "Request Entity Too Large",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/search": {
|
||||
"get": {
|
||||
"produces": [
|
||||
|
|
@ -1250,10 +1062,6 @@ const docTemplate = `{
|
|||
"server.PackageResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"cooldown": {
|
||||
"description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").",
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
|
|
@ -1312,106 +1120,6 @@ const docTemplate = `{
|
|||
}
|
||||
}
|
||||
},
|
||||
"server.ProjectComponent": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ecosystem": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"purl": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.ProjectListResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"projects": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/server.ProjectSummary"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.ProjectResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"branch": {
|
||||
"type": "string"
|
||||
},
|
||||
"component_count": {
|
||||
"description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.",
|
||||
"type": "integer"
|
||||
},
|
||||
"components": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/server.ProjectComponent"
|
||||
}
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"ref": {
|
||||
"type": "string"
|
||||
},
|
||||
"uploaded_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"description": "Version is set when listing the projects using a package: the version\nof it this project uses.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.ProjectSummary": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"branch": {
|
||||
"type": "string"
|
||||
},
|
||||
"component_count": {
|
||||
"description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.",
|
||||
"type": "integer"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"ref": {
|
||||
"type": "string"
|
||||
},
|
||||
"uploaded_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"description": "Version is set when listing the projects using a package: the version\nof it this project uses.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.ProjectUploadResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"components": {
|
||||
"description": "Components is the number of package versions recorded.",
|
||||
"type": "integer"
|
||||
},
|
||||
"project": {
|
||||
"type": "string"
|
||||
},
|
||||
"skipped": {
|
||||
"description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.",
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.SearchPackageResult": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
@ -1461,6 +1169,9 @@ const docTemplate = `{
|
|||
"cached_artifacts": {
|
||||
"type": "integer"
|
||||
},
|
||||
"database_path": {
|
||||
"type": "string"
|
||||
},
|
||||
"downloaded": {
|
||||
"type": "string"
|
||||
},
|
||||
|
|
@ -1481,6 +1192,9 @@ const docTemplate = `{
|
|||
"description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"storage_url": {
|
||||
"type": "string"
|
||||
},
|
||||
"total_size": {
|
||||
"type": "string"
|
||||
},
|
||||
|
|
@ -1492,7 +1206,7 @@ const docTemplate = `{
|
|||
},
|
||||
"securityDefinitions": {
|
||||
"BearerAuth": {
|
||||
"description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects",
|
||||
"description": "\"Bearer \" followed by hosted.token",
|
||||
"type": "apiKey",
|
||||
"name": "Authorization",
|
||||
"in": "header"
|
||||
|
|
|
|||
|
|
@ -453,194 +453,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"/api/projects": {
|
||||
"get": {
|
||||
"description": "Lists the projects that uploaded an SBOM. With purl, lists the projects using that package, or only that version of it when the purl has one; each entry then carries the version the project uses.",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"projects"
|
||||
],
|
||||
"summary": "List projects",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Package URL, e.g. pkg:npm/lodash or pkg:npm/lodash@4.17.20",
|
||||
"name": "purl",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ProjectListResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/projects/{project}": {
|
||||
"get": {
|
||||
"description": "Returns the project with the package versions recorded for it.",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"projects"
|
||||
],
|
||||
"summary": "Get a project",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Project path, e.g. group/sub/repo",
|
||||
"name": "project",
|
||||
"in": "path",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ProjectResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"delete": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Removes the project and the package versions recorded for it. The path names the project itself, without the /sbom of the upload path: a path ending in /sbom that names no project is answered with 400.",
|
||||
"tags": [
|
||||
"projects"
|
||||
],
|
||||
"summary": "Delete a project",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Project path, e.g. group/sub/repo",
|
||||
"name": "project",
|
||||
"in": "path",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"204": {
|
||||
"description": "No Content"
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "Unauthorized",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/projects/{project}/sbom": {
|
||||
"put": {
|
||||
"security": [
|
||||
{
|
||||
"BearerAuth": []
|
||||
}
|
||||
],
|
||||
"description": "Replaces the package versions recorded for the project with the ones the SBOM lists. CycloneDX (JSON or XML) and SPDX (JSON or tag-value) are accepted, e.g. the output of `syft dir:. -o cyclonedx-json`. Entries without a package URL or version, of a package type the proxy doesn't serve, and the project's own packages are skipped and counted. The project may be given as path segments or URL-encoded as one segment.",
|
||||
"consumes": [
|
||||
"application/json",
|
||||
"application/xml",
|
||||
"text/plain"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"projects"
|
||||
],
|
||||
"summary": "Upload a project's SBOM",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Project path, e.g. group/sub/repo",
|
||||
"name": "project",
|
||||
"in": "path",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Commit the SBOM was generated from",
|
||||
"name": "ref",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Branch the SBOM was generated from",
|
||||
"name": "branch",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ProjectUploadResponse"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"description": "Unauthorized",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"413": {
|
||||
"description": "Request Entity Too Large",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/server.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/search": {
|
||||
"get": {
|
||||
"produces": [
|
||||
|
|
@ -1243,10 +1055,6 @@
|
|||
"server.PackageResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"cooldown": {
|
||||
"description": "Cooldown is the cooldown that applies to the package, in the\nconfiguration's syntax (\"3d\", \"0\").",
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
|
|
@ -1305,106 +1113,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"server.ProjectComponent": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ecosystem": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"purl": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.ProjectListResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"projects": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/server.ProjectSummary"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.ProjectResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"branch": {
|
||||
"type": "string"
|
||||
},
|
||||
"component_count": {
|
||||
"description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.",
|
||||
"type": "integer"
|
||||
},
|
||||
"components": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/server.ProjectComponent"
|
||||
}
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"ref": {
|
||||
"type": "string"
|
||||
},
|
||||
"uploaded_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"description": "Version is set when listing the projects using a package: the version\nof it this project uses.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.ProjectSummary": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"branch": {
|
||||
"type": "string"
|
||||
},
|
||||
"component_count": {
|
||||
"description": "ComponentCount is the number of package versions recorded. It is not\nset when listing the projects using a package.",
|
||||
"type": "integer"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"ref": {
|
||||
"type": "string"
|
||||
},
|
||||
"uploaded_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"version": {
|
||||
"description": "Version is set when listing the projects using a package: the version\nof it this project uses.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.ProjectUploadResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"components": {
|
||||
"description": "Components is the number of package versions recorded.",
|
||||
"type": "integer"
|
||||
},
|
||||
"project": {
|
||||
"type": "string"
|
||||
},
|
||||
"skipped": {
|
||||
"description": "Skipped is the number of SBOM entries not recorded: without a package\nURL or a version, of a package type the proxy doesn't serve, conda\nentries without a channel, and the project's own packages.",
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"server.SearchPackageResult": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
@ -1454,6 +1162,9 @@
|
|||
"cached_artifacts": {
|
||||
"type": "integer"
|
||||
},
|
||||
"database_path": {
|
||||
"type": "string"
|
||||
},
|
||||
"downloaded": {
|
||||
"type": "string"
|
||||
},
|
||||
|
|
@ -1474,6 +1185,9 @@
|
|||
"description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"storage_url": {
|
||||
"type": "string"
|
||||
},
|
||||
"total_size": {
|
||||
"type": "string"
|
||||
},
|
||||
|
|
@ -1485,7 +1199,7 @@
|
|||
},
|
||||
"securityDefinitions": {
|
||||
"BearerAuth": {
|
||||
"description": "\"Bearer \" followed by hosted.token, or projects.token for /api/projects",
|
||||
"description": "\"Bearer \" followed by hosted.token",
|
||||
"type": "apiKey",
|
||||
"name": "Authorization",
|
||||
"in": "header"
|
||||
|
|
|
|||
40
go.mod
40
go.mod
|
|
@ -1,24 +1,25 @@
|
|||
module github.com/git-pkgs/proxy
|
||||
|
||||
go 1.27.3
|
||||
go 1.26.7
|
||||
|
||||
toolchain go1.27.3
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.6.0
|
||||
github.com/Masterminds/semver/v3 v3.5.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.12
|
||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9
|
||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5
|
||||
github.com/git-pkgs/archives v0.10.0
|
||||
github.com/git-pkgs/artifacts v0.2.1
|
||||
github.com/git-pkgs/cooldown v0.3.1
|
||||
github.com/git-pkgs/cooldown v0.3.0
|
||||
github.com/git-pkgs/enrichment v0.7.5
|
||||
github.com/git-pkgs/gcs v0.1.0
|
||||
github.com/git-pkgs/integrity v0.2.0
|
||||
github.com/git-pkgs/magic v0.6.0
|
||||
github.com/git-pkgs/purl v0.2.1
|
||||
github.com/git-pkgs/registries v0.13.0
|
||||
github.com/git-pkgs/sbom v0.1.9
|
||||
github.com/git-pkgs/sbom v0.1.7
|
||||
github.com/git-pkgs/spdx v0.3.3
|
||||
github.com/git-pkgs/vers v0.7.2
|
||||
github.com/go-chi/chi/v5 v5.5.0
|
||||
github.com/jmoiron/sqlx v1.4.0
|
||||
github.com/klauspost/compress v1.20.3
|
||||
|
|
@ -50,24 +51,24 @@ require (
|
|||
github.com/PuerkitoBio/purell v1.1.1 // indirect
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect
|
||||
github.com/aws/smithy-go v1.28.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect
|
||||
github.com/aws/smithy-go v1.28.5 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenk/backoff v2.2.1+incompatible // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
|
|
@ -75,6 +76,7 @@ require (
|
|||
github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect
|
||||
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
|
||||
github.com/git-pkgs/pom v0.1.8 // indirect
|
||||
github.com/git-pkgs/vers v0.7.2 // indirect
|
||||
github.com/git-pkgs/vulns v0.2.4 // indirect
|
||||
github.com/github/go-spdx/v2 v2.7.0 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
|
|
|
|||
68
go.sum
68
go.sum
|
|
@ -52,46 +52,46 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
|
|||
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.12 h1:IV1YARaXahwCqUIyP8IlWsP85Lil+5ozTozDQXPumRk=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.12/go.mod h1:h51R06bzMVwNBRZOX5ylHhNDnkB/azZeq/LvxM6q2pQ=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.12 h1:6P7Q7yH/GIwHg6MLKv3ySsBqaCoE1z+C0OTzYMenhEY=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.12/go.mod h1:Kw/SaoW/wc7IqJErqXJIzKo8b7Aelxfbuc6XlbrRnsc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3 h1:vsFkmFrfzODkxVbyuPLscv5j5lC8KbG48LZwV+p56Y0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.3/go.mod h1:pjQYVtvVBEtlsf2p6nXMsjPuDaWs9GM44FMdHbmnmYU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
|
||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1 h1:EJlz0wSlZDKaCN2FHwg9RBR+C+gsuVdYblJnEnrq3M4=
|
||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.67.1/go.mod h1:6rVJFO9ZO+4j/QL5/FbtAsbb8xHC85j79wXA9ndOCTY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
|
||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5 h1:163OpmhcdzTzgbXbLV+u2v1LY/QmGT148/eOwkczKCY=
|
||||
github.com/aws/aws-sdk-go-v2/service/ecr v1.66.5/go.mod h1:uDza1Eipo/Olii3CCgLglHmAni24PScSAhpfIKbUl0c=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 h1:CO485zrLOZBeq8mrdsudc8cjyhvC4LY4JFt8ZfHvXZ0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2/go.mod h1:cBAjVUKBr+5wnx1K3XWS8u5AZsqpA2S/0z7YtI6BGFc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 h1:Enx5JcIy/o90kio1LFZ/HguxIsos3clbGuSpGdyshX8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2/go.mod h1:Ldmll8/2VqaVerWWDf8IGG+6pAvZY04gs3tevq3nIFs=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0 h1:4PokON/ISKuD6Zl062n4WfQNnEFBMHvfeJy73Jp7K+g=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.111.0/go.mod h1:6ZFrmv/qIAYFmG+2WhA7mFU8+edldRlISwT+tNnyLuA=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 h1:3vu/eUqh0rjh7Zy4N4YKcpZX4Aud+zinB3VsLhhlQFw=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.12.0/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 h1:kIEpD5dUry7VET3Ybl7zj7PPHRB4lb52G9iTz+a1Tqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.39.0/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 h1:WfFev+HgTodggcNon5HieiLxxwWwb8Br8R2+nDkixvk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
|
||||
github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE=
|
||||
github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM=
|
||||
github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k=
|
||||
github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
|
||||
|
|
@ -121,8 +121,8 @@ github.com/git-pkgs/archives v0.10.0 h1:/qgk0MI6VK2sKOoVTd0B4zs5hwTo+St0X8cYYftz
|
|||
github.com/git-pkgs/archives v0.10.0/go.mod h1:GJ/h7NPTUYiPinOUxpYNsHFCUikYZcJXvugBvP1VGNc=
|
||||
github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx2xk=
|
||||
github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA=
|
||||
github.com/git-pkgs/cooldown v0.3.1 h1:XDt54+ZuYoJclTLysaQ2GUZPhYYuJdjpaqeLSwjMdto=
|
||||
github.com/git-pkgs/cooldown v0.3.1/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0=
|
||||
github.com/git-pkgs/cooldown v0.3.0 h1:eAl2CQFwqXxluoVP9hHmropAf1VoV3gSxLMkN+NEu8A=
|
||||
github.com/git-pkgs/cooldown v0.3.0/go.mod h1:FnTg8wufEALATC9Sm/9AHjLdA+u9eVn96e9BqXXjGB0=
|
||||
github.com/git-pkgs/enrichment v0.7.5 h1:jbfFzf7+HSS7el8AuMri4vHXq2izIIZDVIU0uYI5LFM=
|
||||
github.com/git-pkgs/enrichment v0.7.5/go.mod h1:TbxC/nGQN96YCbdoiDs8a9z3gk2zXmLVS1989/jvnsU=
|
||||
github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo=
|
||||
|
|
@ -137,8 +137,8 @@ github.com/git-pkgs/purl v0.2.1 h1:b46hRlOMbrG6aXjjFJdYixGi5WTh0TLlAnH9iNeuOS0=
|
|||
github.com/git-pkgs/purl v0.2.1/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw=
|
||||
github.com/git-pkgs/registries v0.13.0 h1:5i9dHnbiXseIbXLIZ61YnuFySJ9dQX+KEu1MrmwB2BU=
|
||||
github.com/git-pkgs/registries v0.13.0/go.mod h1:1g/jXVi1M/3jWvW1Wom1NMnPrhglJ7UVEZ215l2Zh0M=
|
||||
github.com/git-pkgs/sbom v0.1.9 h1:sfF5cJAwmgrm2N4HTXQWjiBe8tdu8xwhjwfAOyDNMfc=
|
||||
github.com/git-pkgs/sbom v0.1.9/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M=
|
||||
github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA=
|
||||
github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M=
|
||||
github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI=
|
||||
github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU=
|
||||
github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs=
|
||||
|
|
|
|||
|
|
@ -64,14 +64,12 @@ import (
|
|||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
|
@ -172,11 +170,6 @@ type Config struct {
|
|||
// size return ErrMetadataTooLarge. Default: "100MB".
|
||||
MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"`
|
||||
|
||||
// MetadataStreamMaxSize is the largest metadata document the proxy
|
||||
// streams from upstream into storage instead of buffering it, which
|
||||
// metadata_max_size doesn't cover (e.g. "1GB"). Default: "1GB".
|
||||
MetadataStreamMaxSize string `json:"metadata_stream_max_size" yaml:"metadata_stream_max_size"`
|
||||
|
||||
// ArtifactMaxSize is the largest upstream artifact the proxy will fetch
|
||||
// (e.g. "2GB"). Larger artifacts are refused and not cached. Default: ""
|
||||
// (unlimited). "0" is also unlimited.
|
||||
|
|
@ -220,9 +213,6 @@ type Config struct {
|
|||
// Hosted configures pushing packages to the proxy (/api/hosted).
|
||||
Hosted HostedConfig `json:"hosted" yaml:"hosted"`
|
||||
|
||||
// Projects configures the project inventory (/api/projects).
|
||||
Projects ProjectsConfig `json:"projects" yaml:"projects"`
|
||||
|
||||
// Gradle configures Gradle HttpBuildCache behavior.
|
||||
Gradle GradleConfig `json:"gradle" yaml:"gradle"`
|
||||
|
||||
|
|
@ -239,9 +229,6 @@ type Config struct {
|
|||
// CooldownConfig configures version cooldown periods.
|
||||
// Versions published more recently than the cooldown are hidden from metadata responses.
|
||||
type CooldownConfig struct {
|
||||
// Profiles are independent policies exposed at /cooldown/{name}/.
|
||||
Profiles map[string]CooldownConfig `json:"profiles,omitempty" yaml:"profiles,omitempty"`
|
||||
|
||||
// Default is the global default cooldown (e.g., "3d", "48h", "0" to disable).
|
||||
Default string `json:"default" yaml:"default"`
|
||||
|
||||
|
|
@ -438,55 +425,9 @@ func (s *ScannerConfig) Validate() error {
|
|||
default:
|
||||
return fmt.Errorf("invalid mode %q (must be block or monitor)", s.Mode)
|
||||
}
|
||||
for i, eco := range s.Ecosystems {
|
||||
key, err := ScannerEcosystemKey(eco)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.Ecosystems[i] = key
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// scannerEcosystems are the ecosystem keys handlers pass to scanners: the
|
||||
// retention keys plus the handlers retention does not cover, and git for
|
||||
// Git LFS objects.
|
||||
var scannerEcosystems = func() []string {
|
||||
keys := append(slices.Clone(retention.KnownKeys), "arch", "chef", "git")
|
||||
slices.Sort(keys)
|
||||
return keys
|
||||
}()
|
||||
|
||||
// scannerEcosystemAliases maps common ecosystem names to handler keys.
|
||||
var scannerEcosystemAliases = map[string]string{
|
||||
"apk": "alpine",
|
||||
"debian": "deb",
|
||||
"docker": "oci",
|
||||
"go": "golang",
|
||||
"packagist": "composer",
|
||||
"rubygems": "gem",
|
||||
}
|
||||
|
||||
// NormalizeScannerEcosystem returns the handler key for a
|
||||
// scanning.scanners[].ecosystems entry, resolving case and common aliases.
|
||||
func NormalizeScannerEcosystem(eco string) string {
|
||||
eco = strings.ToLower(strings.TrimSpace(eco))
|
||||
if key, ok := scannerEcosystemAliases[eco]; ok {
|
||||
return key
|
||||
}
|
||||
return eco
|
||||
}
|
||||
|
||||
// ScannerEcosystemKey is NormalizeScannerEcosystem that also rejects
|
||||
// names no handler uses.
|
||||
func ScannerEcosystemKey(eco string) (string, error) {
|
||||
key := NormalizeScannerEcosystem(eco)
|
||||
if !slices.Contains(scannerEcosystems, key) {
|
||||
return "", fmt.Errorf("unknown ecosystem %q (valid: %s)", eco, strings.Join(scannerEcosystems, ", "))
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// StorageConfig configures artifact storage.
|
||||
type StorageConfig struct {
|
||||
// URL is the storage backend URL.
|
||||
|
|
@ -673,12 +614,6 @@ type UpstreamConfig struct {
|
|||
// wins.
|
||||
NPMRoutes map[string]string `json:"npm_routes" yaml:"npm_routes"`
|
||||
|
||||
// NPMChangeFeed follows the public npm registry's change feed and
|
||||
// revalidates cached packuments of packages listed in it on their next
|
||||
// request, rather than serving them until metadata_ttl runs out.
|
||||
// Requires cache_metadata. Default: false.
|
||||
NPMChangeFeed bool `json:"npm_change_feed" yaml:"npm_change_feed"`
|
||||
|
||||
// Cargo is the upstream cargo index URL.
|
||||
// Default: https://index.crates.io
|
||||
Cargo string `json:"cargo" yaml:"cargo"`
|
||||
|
|
@ -776,12 +711,6 @@ type UpstreamConfig struct {
|
|||
// matching pattern wins.
|
||||
ComposerRoutes map[string]string `json:"composer_routes" yaml:"composer_routes"`
|
||||
|
||||
// ComposerChangeFeed follows the metadata change feed of
|
||||
// upstream.composer and revalidates cached metadata of packages listed
|
||||
// in it on their next request, rather than serving it until
|
||||
// metadata_ttl runs out. Requires cache_metadata. Default: false.
|
||||
ComposerChangeFeed bool `json:"composer_change_feed" yaml:"composer_change_feed"`
|
||||
|
||||
// Conan is the upstream Conan registry URL.
|
||||
// Default: https://center.conan.io
|
||||
Conan string `json:"conan" yaml:"conan"`
|
||||
|
|
@ -844,12 +773,6 @@ type UpstreamConfig struct {
|
|||
// Default when empty: {"alpine": "https://dl-cdn.alpinelinux.org/alpine"}.
|
||||
APK map[string]string `json:"apk" yaml:"apk"`
|
||||
|
||||
// APKSnapshots keeps earlier versions of Alpine APK indexes for this
|
||||
// long after upstream replaced them, and serves them at
|
||||
// /apk/{name}@{YYYY-MM-DD}/. Accepts a "d" suffix, e.g. "90d".
|
||||
// Empty or "0" disables snapshots.
|
||||
APKSnapshots string `json:"apk_snapshots" yaml:"apk_snapshots"`
|
||||
|
||||
// Arch maps repository names to Arch Linux pacman mirror base URLs,
|
||||
// served at /arch/{name}/. The remaining request path mirrors the
|
||||
// upstream layout, e.g. /arch/archlinux/core/os/x86_64/core.db.
|
||||
|
|
@ -958,11 +881,6 @@ func (u *UpstreamConfig) Validate() error {
|
|||
if err := validateNamedUpstreams("upstream.apk", u.APK); err != nil {
|
||||
return err
|
||||
}
|
||||
if u.APKSnapshots != "" {
|
||||
if _, err := parseRetentionDuration("upstream.apk_snapshots", u.APKSnapshots); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -1080,12 +998,6 @@ func (c *Config) validateUpstream() error {
|
|||
if c.Upstream.GoSumDBsOffline && !c.CacheMetadata {
|
||||
return fmt.Errorf("upstream.go_sumdbs_offline requires cache_metadata")
|
||||
}
|
||||
if c.Upstream.NPMChangeFeed && !c.CacheMetadata {
|
||||
return fmt.Errorf("upstream.npm_change_feed requires cache_metadata")
|
||||
}
|
||||
if c.Upstream.ComposerChangeFeed && !c.CacheMetadata {
|
||||
return fmt.Errorf("upstream.composer_change_feed requires cache_metadata")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -1438,33 +1350,11 @@ func setEnvString(dst *string, key string) {
|
|||
}
|
||||
}
|
||||
|
||||
// parseEnvBool accepts what strconv.ParseBool does plus yes/no/on/off,
|
||||
// ignoring case and surrounding space.
|
||||
func parseEnvBool(v string) (bool, bool) {
|
||||
v = strings.ToLower(strings.TrimSpace(v))
|
||||
switch v {
|
||||
case "yes", "on":
|
||||
return true, true
|
||||
case "no", "off":
|
||||
return false, true
|
||||
// setEnvBool is setEnvString for boolean fields, parsed via envBool.
|
||||
func setEnvBool(dst *bool, key string) {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
*dst = envBool(v)
|
||||
}
|
||||
b, err := strconv.ParseBool(v)
|
||||
return b, err == nil
|
||||
}
|
||||
|
||||
// setEnvBool is setEnvString for boolean fields, parsed via parseEnvBool.
|
||||
// Any other value leaves *dst unchanged and is reported via EnvWarnings.
|
||||
func (c *Config) setEnvBool(dst *bool, key string) {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return
|
||||
}
|
||||
b, ok := parseEnvBool(v)
|
||||
if !ok {
|
||||
c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping %t", key, v, *dst))
|
||||
return
|
||||
}
|
||||
*dst = b
|
||||
}
|
||||
|
||||
// setGradleReadOnlyFromEnv parses strictly: an unrecognised value keeps
|
||||
|
|
@ -1475,8 +1365,8 @@ func (c *Config) setGradleReadOnlyFromEnv() {
|
|||
if v == "" {
|
||||
return
|
||||
}
|
||||
readOnly, ok := parseEnvBool(v)
|
||||
if !ok {
|
||||
readOnly, err := strconv.ParseBool(strings.TrimSpace(v))
|
||||
if err != nil {
|
||||
c.Gradle.BuildCache.ReadOnly = true
|
||||
c.envWarnings = append(c.envWarnings, fmt.Sprintf("ignoring %s=%q: not a boolean, keeping the Gradle build cache read-only", key, v))
|
||||
return
|
||||
|
|
@ -1524,16 +1414,16 @@ func (c *Config) LoadFromEnv() {
|
|||
setEnvString(&c.Listen, "PROXY_LISTEN")
|
||||
setEnvString(&c.BaseURL, "PROXY_BASE_URL")
|
||||
setEnvString(&c.UIBaseURL, "PROXY_UI_URL")
|
||||
c.setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR")
|
||||
c.setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES")
|
||||
c.setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS")
|
||||
setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR")
|
||||
setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES")
|
||||
setEnvBool(&c.UIFailureDetails, "PROXY_UI_FAILURE_DETAILS")
|
||||
setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL")
|
||||
setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH")
|
||||
setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE")
|
||||
c.setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE")
|
||||
setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE")
|
||||
setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL")
|
||||
setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL")
|
||||
c.setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS")
|
||||
setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS")
|
||||
setEnvString(&c.Storage.Retention.Default, "PROXY_STORAGE_RETENTION_DEFAULT")
|
||||
setEnvString(&c.Storage.Retention.SweepInterval, "PROXY_STORAGE_RETENTION_SWEEP_INTERVAL")
|
||||
setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER")
|
||||
|
|
@ -1544,11 +1434,10 @@ func (c *Config) LoadFromEnv() {
|
|||
setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT")
|
||||
setEnvString(&c.AccessLog.Path, "PROXY_ACCESS_LOG_PATH")
|
||||
setEnvStringSlice(&c.Upstream.AllowPrivateHosts, "PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS")
|
||||
c.setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK")
|
||||
setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK")
|
||||
setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM")
|
||||
c.setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA")
|
||||
c.setEnvBool(&c.Upstream.NPMChangeFeed, "PROXY_UPSTREAM_NPM_CHANGE_FEED")
|
||||
c.setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE")
|
||||
setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA")
|
||||
setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE")
|
||||
setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO")
|
||||
setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD")
|
||||
setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM")
|
||||
|
|
@ -1565,7 +1454,6 @@ func (c *Config) LoadFromEnv() {
|
|||
setEnvString(&c.Upstream.NuGetSearch, "PROXY_UPSTREAM_NUGET_SEARCH")
|
||||
setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER")
|
||||
setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY")
|
||||
c.setEnvBool(&c.Upstream.ComposerChangeFeed, "PROXY_UPSTREAM_COMPOSER_CHANGE_FEED")
|
||||
setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN")
|
||||
setEnvString(&c.Upstream.Chef, "PROXY_UPSTREAM_CHEF")
|
||||
setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA")
|
||||
|
|
@ -1574,30 +1462,25 @@ func (c *Config) LoadFromEnv() {
|
|||
setEnvString(&c.Upstream.Swift, "PROXY_UPSTREAM_SWIFT")
|
||||
setEnvString(&c.Upstream.OCIDefault, "PROXY_UPSTREAM_OCI_DEFAULT")
|
||||
setEnvString(&c.Upstream.Debian, "PROXY_UPSTREAM_DEBIAN")
|
||||
setEnvString(&c.Upstream.APKSnapshots, "PROXY_UPSTREAM_APK_SNAPSHOTS")
|
||||
setEnvString(&c.Upstream.RPM, "PROXY_UPSTREAM_RPM")
|
||||
setEnvString(&c.Upstream.HomebrewAPI, "PROXY_UPSTREAM_HOMEBREW_API")
|
||||
setEnvString(&c.Upstream.HomebrewArtifact, "PROXY_UPSTREAM_HOMEBREW_ARTIFACT")
|
||||
setEnvString(&c.Cooldown.Default, "PROXY_COOLDOWN_DEFAULT")
|
||||
c.setEnvBool(&c.Scanning.Enabled, "PROXY_SCANNING_ENABLED")
|
||||
c.setEnvBool(&c.Scanning.FailOpen, "PROXY_SCANNING_FAIL_OPEN")
|
||||
setEnvBool(&c.Scanning.Enabled, "PROXY_SCANNING_ENABLED")
|
||||
setEnvBool(&c.Scanning.FailOpen, "PROXY_SCANNING_FAIL_OPEN")
|
||||
setEnvString(&c.Scanning.Timeout, "PROXY_SCANNING_TIMEOUT")
|
||||
setEnvString(&c.Scanning.SigningKey, "PROXY_SCANNING_SIGNING_KEY")
|
||||
setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL")
|
||||
c.setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA")
|
||||
c.setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API")
|
||||
c.setEnvBool(&c.CacheDelete, "PROXY_CACHE_DELETE")
|
||||
c.setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED")
|
||||
setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA")
|
||||
setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API")
|
||||
setEnvBool(&c.CacheDelete, "PROXY_CACHE_DELETE")
|
||||
setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED")
|
||||
setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN")
|
||||
setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE")
|
||||
setEnvStringSlice(&c.Hosted.NPMScopes, "PROXY_HOSTED_NPM_SCOPES")
|
||||
setEnvStringSlice(&c.Hosted.ComposerVendors, "PROXY_HOSTED_COMPOSER_VENDORS")
|
||||
c.setEnvBool(&c.Projects.Enabled, "PROXY_PROJECTS_ENABLED")
|
||||
setEnvString(&c.Projects.Token, "PROXY_PROJECTS_TOKEN")
|
||||
setEnvString(&c.Projects.MaxAge, "PROXY_PROJECTS_MAX_AGE")
|
||||
setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL")
|
||||
setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE")
|
||||
setEnvString(&c.MetadataStreamMaxSize, "PROXY_METADATA_STREAM_MAX_SIZE")
|
||||
setEnvString(&c.ArtifactMaxSize, "PROXY_ARTIFACT_MAX_SIZE")
|
||||
setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE")
|
||||
setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT")
|
||||
|
|
@ -1680,8 +1563,8 @@ func (c *Config) Validate() error {
|
|||
|
||||
// Validate direct serve TTL if specified
|
||||
if c.Storage.DirectServeTTL != "" {
|
||||
if _, err := parseNonNegativeDuration("storage.direct_serve_ttl", c.Storage.DirectServeTTL); err != nil {
|
||||
return err
|
||||
if _, err := time.ParseDuration(c.Storage.DirectServeTTL); err != nil {
|
||||
return fmt.Errorf("invalid storage.direct_serve_ttl %q: %w", c.Storage.DirectServeTTL, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1698,8 +1581,8 @@ func (c *Config) Validate() error {
|
|||
|
||||
// Validate metadata TTL if specified
|
||||
if c.MetadataTTL != "" && c.MetadataTTL != "0" {
|
||||
if _, err := parseNonNegativeDuration("metadata_ttl", c.MetadataTTL); err != nil {
|
||||
return err
|
||||
if _, err := time.ParseDuration(c.MetadataTTL); err != nil {
|
||||
return fmt.Errorf("invalid metadata_ttl %q: %w", c.MetadataTTL, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1738,10 +1621,6 @@ func (c *Config) validateCacheArtifacts() error {
|
|||
}
|
||||
|
||||
func (c *Config) validateComponents() error {
|
||||
if err := c.Cooldown.ValidateProfiles(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := validateMetadataRewriteCacheSize(c.MetadataRewriteCacheSize); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -1774,10 +1653,6 @@ func (c *Config) validateComponents() error {
|
|||
HostedGenericRepository, HostedGenericRepository)
|
||||
}
|
||||
|
||||
if err := c.Projects.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := c.Mirror.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -1852,7 +1727,6 @@ const (
|
|||
defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default
|
||||
defaultHitFlushInterval = time.Second
|
||||
defaultMetadataMaxSize = 100 << 20
|
||||
defaultMetadataStreamMaxSize = 1 << 30
|
||||
defaultMetadataRewriteCacheSize = 256 << 20
|
||||
defaultGradleBuildCacheMaxUploadSize = 100 << 20
|
||||
defaultGradleBuildCacheSweepInterval = 10 * time.Minute
|
||||
|
|
@ -1906,25 +1780,22 @@ func validateRequestTimeout(field, s string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func validateMetadataMaxSize(name, s string) error {
|
||||
func validateMetadataMaxSize(s string) error {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
size, err := ParseSize(s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid %s: %w", name, err)
|
||||
return fmt.Errorf("invalid metadata_max_size: %w", err)
|
||||
}
|
||||
if size <= 0 {
|
||||
return fmt.Errorf("invalid %s %q: must be positive", name, s)
|
||||
return fmt.Errorf("invalid metadata_max_size %q: must be positive", s)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Config) validateDownloadSizeLimits() error {
|
||||
if err := validateMetadataMaxSize("metadata_max_size", c.MetadataMaxSize); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateMetadataMaxSize("metadata_stream_max_size", c.MetadataStreamMaxSize); err != nil {
|
||||
if err := validateMetadataMaxSize(c.MetadataMaxSize); err != nil {
|
||||
return err
|
||||
}
|
||||
return validateArtifactMaxSize(c.ArtifactMaxSize)
|
||||
|
|
@ -1987,19 +1858,6 @@ func (c *Config) ParseMetadataMaxSize() int64 {
|
|||
return size
|
||||
}
|
||||
|
||||
// ParseMetadataStreamMaxSize returns the size limit for metadata streamed
|
||||
// into storage in bytes. Returns 1GB if unset or invalid.
|
||||
func (c *Config) ParseMetadataStreamMaxSize() int64 {
|
||||
if c.MetadataStreamMaxSize == "" {
|
||||
return defaultMetadataStreamMaxSize
|
||||
}
|
||||
size, err := ParseSize(c.MetadataStreamMaxSize)
|
||||
if err != nil || size <= 0 {
|
||||
return defaultMetadataStreamMaxSize
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// ParseArtifactMaxSize returns the artifact size limit in bytes, or 0 for
|
||||
// no limit.
|
||||
func (c *Config) ParseArtifactMaxSize() int64 {
|
||||
|
|
@ -2101,20 +1959,6 @@ func (c *Config) ParseGradleBuildCacheMaxUploadSize() int64 {
|
|||
return size
|
||||
}
|
||||
|
||||
// ParseAPKSnapshots returns how long earlier Alpine APK indexes are kept
|
||||
// after upstream replaced them. Returns 0 when snapshots are disabled or the
|
||||
// value is invalid.
|
||||
func (c *Config) ParseAPKSnapshots() time.Duration {
|
||||
if c.Upstream.APKSnapshots == "" {
|
||||
return 0
|
||||
}
|
||||
d, err := parseDayDuration(c.Upstream.APKSnapshots)
|
||||
if err != nil || d < 0 {
|
||||
return 0
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// ParseGradleBuildCacheMaxAge returns age-based eviction threshold.
|
||||
// Returns 0 when disabled or invalid.
|
||||
func (c *Config) ParseGradleBuildCacheMaxAge() time.Duration {
|
||||
|
|
@ -2197,16 +2041,7 @@ func ParseSize(s string) (int64, error) {
|
|||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid number %q", numStr)
|
||||
}
|
||||
// The comparisons also reject NaN.
|
||||
size := num * float64(s2.mult)
|
||||
if !(size >= 0 && size < math.MaxInt64) {
|
||||
return 0, fmt.Errorf("invalid size %q: must be between 0 and %d bytes", s, int64(math.MaxInt64))
|
||||
}
|
||||
// max_size 0 means unlimited, so don't let a tiny size become 0.
|
||||
if num != 0 && int64(size) == 0 {
|
||||
return 0, fmt.Errorf("invalid size %q: less than one byte", s)
|
||||
}
|
||||
return int64(size), nil
|
||||
return int64(num * float64(s2.mult)), nil
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2215,9 +2050,6 @@ func ParseSize(s string) (int64, error) {
|
|||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid size %q", s)
|
||||
}
|
||||
if num < 0 {
|
||||
return 0, fmt.Errorf("invalid size %q: must not be negative", s)
|
||||
}
|
||||
return num, nil
|
||||
}
|
||||
|
||||
|
|
@ -2258,3 +2090,7 @@ func (a *AuthConfig) Header() (name, value string) {
|
|||
func expandEnv(s string) string {
|
||||
return os.Expand(s, os.Getenv)
|
||||
}
|
||||
|
||||
func envBool(v string) bool {
|
||||
return v == "true" || v == "1"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package config
|
|||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
|
@ -297,8 +296,6 @@ func TestParseSize(t *testing.T) {
|
|||
{"1TB", 1024 * 1024 * 1024 * 1024, false},
|
||||
{"invalid", 0, true},
|
||||
{"10XB", 0, true},
|
||||
{"0GB", 0, false},
|
||||
{"0.0MB", 0, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
|
@ -469,9 +466,8 @@ func TestLoadGradleBuildCacheReadOnlyFromEnv(t *testing.T) {
|
|||
{" FALSE ", true, false, 0},
|
||||
{"True", false, true, 0},
|
||||
{"1", false, true, 0},
|
||||
{"yes", false, true, 0},
|
||||
{"off", true, false, 0},
|
||||
{"maybe", false, true, 1},
|
||||
{"yes", false, true, 1},
|
||||
{"off", false, true, 1},
|
||||
{"", false, false, 0},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
|
|
@ -789,29 +785,6 @@ func TestParseMetadataMaxSize(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestParseMetadataStreamMaxSize(t *testing.T) {
|
||||
cfg := Default()
|
||||
if got := cfg.ParseMetadataStreamMaxSize(); got != defaultMetadataStreamMaxSize {
|
||||
t.Errorf("unset: %d, want %d", got, defaultMetadataStreamMaxSize)
|
||||
}
|
||||
cfg.MetadataStreamMaxSize = "2GB"
|
||||
if got := cfg.ParseMetadataStreamMaxSize(); got != 2<<30 {
|
||||
t.Errorf("2GB: %d", got)
|
||||
}
|
||||
t.Setenv("PROXY_METADATA_STREAM_MAX_SIZE", "300MB")
|
||||
cfg = Default()
|
||||
cfg.LoadFromEnv()
|
||||
if got := cfg.ParseMetadataStreamMaxSize(); got != 300<<20 {
|
||||
t.Errorf("from env: %d", got)
|
||||
}
|
||||
for _, bad := range []string{"0", "lots"} {
|
||||
cfg.MetadataStreamMaxSize = bad
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Errorf("metadata_stream_max_size %q validated", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMetadataMaxSize(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.MetadataMaxSize = "not-a-size"
|
||||
|
|
@ -1168,17 +1141,6 @@ func TestLoadMetadataTTLFromEnv(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestLoadChangeFeedsFromEnv(t *testing.T) {
|
||||
cfg := Default()
|
||||
t.Setenv("PROXY_UPSTREAM_NPM_CHANGE_FEED", "true")
|
||||
t.Setenv("PROXY_UPSTREAM_COMPOSER_CHANGE_FEED", "true")
|
||||
cfg.LoadFromEnv()
|
||||
|
||||
if !cfg.Upstream.NPMChangeFeed || !cfg.Upstream.ComposerChangeFeed {
|
||||
t.Errorf("change feeds = npm %t, composer %t, want both on", cfg.Upstream.NPMChangeFeed, cfg.Upstream.ComposerChangeFeed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGradleBuildCacheConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
|
|
@ -1767,30 +1729,6 @@ func TestValidateGoSumDBs(t *testing.T) {
|
|||
cfg.Upstream.GoSumDBsOffline = true
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "npm change feed requires metadata caching",
|
||||
modify: func(cfg *Config) {
|
||||
cfg.CacheMetadata = false
|
||||
cfg.Upstream.NPMChangeFeed = true
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Composer change feed requires metadata caching",
|
||||
modify: func(cfg *Config) {
|
||||
cfg.CacheMetadata = false
|
||||
cfg.Upstream.ComposerChangeFeed = true
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "change feeds with metadata caching",
|
||||
modify: func(cfg *Config) {
|
||||
cfg.CacheMetadata = true
|
||||
cfg.Upstream.NPMChangeFeed = true
|
||||
cfg.Upstream.ComposerChangeFeed = true
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
|
@ -2119,41 +2057,6 @@ upstream:
|
|||
}
|
||||
}
|
||||
|
||||
func TestAPKSnapshots(t *testing.T) {
|
||||
cfg := Default()
|
||||
if got := cfg.ParseAPKSnapshots(); got != 0 {
|
||||
t.Errorf("default ParseAPKSnapshots = %v, want 0", got)
|
||||
}
|
||||
|
||||
for value, want := range map[string]time.Duration{
|
||||
"90d": 90 * 24 * time.Hour,
|
||||
"12h": 12 * time.Hour,
|
||||
"0": 0,
|
||||
} {
|
||||
cfg.Upstream.APKSnapshots = value
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Errorf("Validate(%q): %v", value, err)
|
||||
}
|
||||
if got := cfg.ParseAPKSnapshots(); got != want {
|
||||
t.Errorf("ParseAPKSnapshots(%q) = %v, want %v", value, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
for _, value := range []string{"forever", "-1d"} {
|
||||
cfg.Upstream.APKSnapshots = value
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Errorf("Validate(%q) succeeded, want an error", value)
|
||||
}
|
||||
}
|
||||
|
||||
cfg = Default()
|
||||
t.Setenv("PROXY_UPSTREAM_APK_SNAPSHOTS", "30d")
|
||||
cfg.LoadFromEnv()
|
||||
if cfg.Upstream.APKSnapshots != "30d" {
|
||||
t.Errorf("APKSnapshots from env = %q, want 30d", cfg.Upstream.APKSnapshots)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCooldownShortest(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
|
|
@ -2173,101 +2076,3 @@ func TestCooldownShortest(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadBoolsFromEnvParseStrictly(t *testing.T) {
|
||||
t.Run("no turns off a setting from the config file", func(t *testing.T) {
|
||||
t.Setenv("PROXY_TRUST_FORWARDED_FOR", "no")
|
||||
cfg := Default()
|
||||
cfg.TrustForwardedFor = true
|
||||
cfg.LoadFromEnv()
|
||||
if cfg.TrustForwardedFor {
|
||||
t.Error("TrustForwardedFor = true, want false")
|
||||
}
|
||||
})
|
||||
|
||||
tests := []struct {
|
||||
value string
|
||||
start bool
|
||||
want bool
|
||||
warnings int
|
||||
}{
|
||||
{"true", false, true, 0},
|
||||
{"True", false, true, 0},
|
||||
{" TRUE ", false, true, 0},
|
||||
{"1", false, true, 0},
|
||||
{"false", true, false, 0},
|
||||
{"FALSE", true, false, 0},
|
||||
{"0", true, false, 0},
|
||||
{"yes", false, true, 0},
|
||||
{" On ", false, true, 0},
|
||||
{"no", true, false, 0},
|
||||
{"OFF", true, false, 0},
|
||||
{"maybe", false, false, 1},
|
||||
{"ture", true, true, 1},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.value, func(t *testing.T) {
|
||||
t.Setenv("PROXY_SCANNING_ENABLED", tt.value)
|
||||
cfg := Default()
|
||||
cfg.Scanning.Enabled = tt.start
|
||||
cfg.LoadFromEnv()
|
||||
if cfg.Scanning.Enabled != tt.want {
|
||||
t.Errorf("Scanning.Enabled = %v, want %v", cfg.Scanning.Enabled, tt.want)
|
||||
}
|
||||
if got := len(cfg.EnvWarnings()); got != tt.warnings {
|
||||
t.Errorf("len(EnvWarnings()) = %d, want %d: %v", got, tt.warnings, cfg.EnvWarnings())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerEcosystems(t *testing.T) {
|
||||
cfg := ScanningConfig{
|
||||
Enabled: true,
|
||||
SigningKey: "s3cret",
|
||||
Scanners: []ScannerConfig{{
|
||||
Name: "clamav", URL: "http://scanner.invalid/scan",
|
||||
Ecosystems: []string{"go", "Docker", "rubygems", "npm", "arch", "git"},
|
||||
}},
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error: %v", err)
|
||||
}
|
||||
want := []string{"golang", "oci", "gem", "npm", "arch", "git"}
|
||||
if got := cfg.Scanners[0].Ecosystems; !slices.Equal(got, want) {
|
||||
t.Errorf("Ecosystems = %v, want %v", got, want)
|
||||
}
|
||||
|
||||
cfg.Scanners[0].Ecosystems = []string{"npm", "pypy"}
|
||||
err := cfg.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), "pypy") {
|
||||
t.Errorf("Validate() error = %v, want unknown ecosystem pypy", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSizeRejectsOutOfRange(t *testing.T) {
|
||||
for _, s := range []string{"-5GB", "-1", "-0.5K", "NaNGB", "InfGB", "+InfB", "1e30GB", "9999999999TB", "0.5B", "0.000000000001GB"} {
|
||||
if got, err := ParseSize(s); err == nil {
|
||||
t.Errorf("ParseSize(%q) = %d, want error", s, got)
|
||||
}
|
||||
}
|
||||
cfg := Default()
|
||||
cfg.Storage.MaxSize = "-5GB"
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("Validate() accepted storage.max_size -5GB")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsNegativeTTLs(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.MetadataTTL = "-5m"
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("Validate() accepted metadata_ttl -5m")
|
||||
}
|
||||
|
||||
cfg = Default()
|
||||
cfg.Storage.DirectServeTTL = "-5m"
|
||||
if err := cfg.Validate(); err == nil {
|
||||
t.Error("Validate() accepted storage.direct_serve_ttl -5m")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,64 +0,0 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"regexp"
|
||||
"slices"
|
||||
|
||||
"github.com/git-pkgs/cooldown"
|
||||
)
|
||||
|
||||
var cooldownProfileName = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`)
|
||||
|
||||
// Policy builds the policy for this URL's cooldown settings.
|
||||
func (c *CooldownConfig) Policy() (*cooldown.Policy, error) {
|
||||
return cooldown.NewPolicy(&cooldown.Config{
|
||||
Default: c.Default, Ecosystems: c.Ecosystems, Packages: c.NormalizedPackages(),
|
||||
}, c.PackagePatterns)
|
||||
}
|
||||
|
||||
// ValidateProfiles checks names and policies before they become routes.
|
||||
func (c *CooldownConfig) ValidateProfiles() error {
|
||||
for _, name := range slices.Sorted(maps.Keys(c.Profiles)) {
|
||||
profile := c.Profiles[name]
|
||||
if !cooldownProfileName.MatchString(name) {
|
||||
return fmt.Errorf("invalid cooldown profile name %q: use lowercase letters, digits, hyphens or underscores", name)
|
||||
}
|
||||
if len(profile.Profiles) > 0 {
|
||||
return fmt.Errorf("cooldown.profiles.%s cannot contain profiles", name)
|
||||
}
|
||||
if err := profile.validateProfileDurations(); err != nil {
|
||||
return fmt.Errorf("cooldown.profiles.%s: %w", name, err)
|
||||
}
|
||||
if _, err := profile.Policy(); err != nil {
|
||||
return fmt.Errorf("cooldown.profiles.%s: %w", name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CooldownConfig) validateProfileDurations() error {
|
||||
values := map[string]string{"default": c.Default}
|
||||
for group, overrides := range map[string]map[string]string{
|
||||
"ecosystems": c.Ecosystems, "packages": c.Packages, "package_patterns": c.PackagePatterns,
|
||||
} {
|
||||
for key, value := range overrides {
|
||||
values[group+"."+key] = value
|
||||
}
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(values)) {
|
||||
value := values[key]
|
||||
if key == "default" && value == "" {
|
||||
continue
|
||||
}
|
||||
duration, err := cooldown.ParseDuration(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid %s: %w", key, err)
|
||||
}
|
||||
if duration < 0 {
|
||||
return fmt.Errorf("invalid %s: duration must be non-negative", key)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoadCooldownProfiles(t *testing.T) {
|
||||
filename := filepath.Join(t.TempDir(), "config.yaml")
|
||||
data := `cooldown:
|
||||
default: 7d
|
||||
ecosystems:
|
||||
npm: "0"
|
||||
profiles:
|
||||
3d:
|
||||
default: 3d
|
||||
packages:
|
||||
"pkg:pypi/Django": "0"
|
||||
strict:
|
||||
default: 14d
|
||||
package_patterns:
|
||||
"pkg:npm/@example/*": 1d
|
||||
`
|
||||
if err := os.WriteFile(filename, []byte(data), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := Load(filename)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := cfg.Cooldown.Profiles["3d"]
|
||||
policy, err := profile.Policy()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := policy.For("npm", "pkg:npm/demo"); got != 72*time.Hour {
|
||||
t.Fatalf("profile inherited global ecosystem override: %v", got)
|
||||
}
|
||||
if got := policy.For("pypi", "pkg:pypi/django"); got != 0 {
|
||||
t.Fatalf("canonical package override = %v", got)
|
||||
}
|
||||
strict := cfg.Cooldown.Profiles["strict"]
|
||||
policy, err = strict.Policy()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := policy.For("npm", "pkg:npm/%40example/tool"); got != 24*time.Hour {
|
||||
t.Fatalf("profile pattern = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidCooldownProfiles(t *testing.T) {
|
||||
for _, tc := range []struct{ name, profile, want string }{
|
||||
{"path", `"a/b": {default: 3d}`, "profile name"},
|
||||
{"uppercase", `Strict: {default: 3d}`, "profile name"},
|
||||
{"empty", `"": {default: 3d}`, "profile name"},
|
||||
{"duration", `strict: {default: nope}`, "cooldown.profiles.strict"},
|
||||
{"pattern", `strict: {package_patterns: {"pkg:npm/[": 3d}}`, "cooldown.profiles.strict"},
|
||||
{"nested", `strict: {profiles: {nested: {default: 3d}}}`, "cannot contain profiles"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
filename := filepath.Join(t.TempDir(), "config.yaml")
|
||||
if err := os.WriteFile(filename, []byte("cooldown:\n profiles:\n "+tc.profile+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := Load(filename)
|
||||
if err == nil {
|
||||
err = cfg.Validate()
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("Load error = %v, want %q", err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue