forked from mirrors/pkg-proxy
beddf8357a* Fix startup message and add connectivity check for S3 storage When S3 storage is configured, the startup log incorrectly showed the default local path (./cache/artifacts) instead of the actual S3 URL. This also adds a lightweight connectivity check at startup so bad credentials or endpoints fail immediately rather than on first request. Add URL() and Close() to the Storage interface so all backends report their URL and can be cleaned up properly. Rename the stats JSON field from storage_path to storage_url. Close storage in error paths and during graceful shutdown. Fixes #49 * Fix Windows test assertion for file:// URL format OpenBucket normalizes Windows paths to file:///C:/path (three slashes) but the test expected file://C:/path (two slashes).
105 lines
2.8 KiB
Go
105 lines
2.8 KiB
Go
// Package storage provides artifact storage backends for the proxy cache.
|
|
//
|
|
// Storage backends are accessed via gocloud.dev/blob URLs:
|
|
//
|
|
// - file:///path/to/dir - Local filesystem storage
|
|
// - s3://bucket-name - Amazon S3
|
|
// - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
|
|
//
|
|
// Use OpenBucket to create a storage backend from a URL.
|
|
package storage
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"io"
|
|
)
|
|
|
|
const dirPermissions = 0755
|
|
|
|
var (
|
|
ErrNotFound = errors.New("artifact not found")
|
|
)
|
|
|
|
// Storage defines the interface for artifact storage backends.
|
|
type Storage interface {
|
|
// Store writes content from r to the given path.
|
|
// Returns the number of bytes written and the SHA256 hash of the content.
|
|
Store(ctx context.Context, path string, r io.Reader) (size int64, hash string, err error)
|
|
|
|
// Open returns a reader for the content at path.
|
|
// The caller must close the reader when done.
|
|
// Returns ErrNotFound if the path does not exist.
|
|
Open(ctx context.Context, path string) (io.ReadCloser, error)
|
|
|
|
// Exists returns true if content exists at path.
|
|
Exists(ctx context.Context, path string) (bool, error)
|
|
|
|
// Delete removes the content at path.
|
|
// Returns nil if the path does not exist.
|
|
Delete(ctx context.Context, path string) error
|
|
|
|
// Size returns the size in bytes of content at path.
|
|
// Returns ErrNotFound if the path does not exist.
|
|
Size(ctx context.Context, path string) (int64, error)
|
|
|
|
// UsedSpace returns the total bytes used by all stored content.
|
|
UsedSpace(ctx context.Context) (int64, error)
|
|
|
|
// URL returns the storage backend URL (e.g. "file:///path" or "s3://bucket").
|
|
URL() string
|
|
|
|
// Close releases any resources held by the storage backend.
|
|
Close() error
|
|
}
|
|
|
|
// ArtifactPath builds a storage path for an artifact.
|
|
// Format: {ecosystem}/{namespace}/{name}/{version}/{filename}
|
|
// For packages without namespace: {ecosystem}/{name}/{version}/{filename}
|
|
func ArtifactPath(ecosystem, namespace, name, version, filename string) string {
|
|
if namespace != "" {
|
|
return ecosystem + "/" + namespace + "/" + name + "/" + version + "/" + filename
|
|
}
|
|
return ecosystem + "/" + name + "/" + version + "/" + filename
|
|
}
|
|
|
|
// HashingReader wraps a reader and computes SHA256 hash as content is read.
|
|
type HashingReader struct {
|
|
r io.Reader
|
|
hash []byte
|
|
h interface{ Sum([]byte) []byte }
|
|
size int64
|
|
done bool
|
|
}
|
|
|
|
func NewHashingReader(r io.Reader) *HashingReader {
|
|
h := sha256.New()
|
|
return &HashingReader{
|
|
r: io.TeeReader(r, h),
|
|
h: h,
|
|
}
|
|
}
|
|
|
|
func (hr *HashingReader) Read(p []byte) (n int, err error) {
|
|
n, err = hr.r.Read(p)
|
|
hr.size += int64(n)
|
|
if err == io.EOF {
|
|
hr.done = true
|
|
hr.hash = hr.h.Sum(nil)
|
|
}
|
|
return
|
|
}
|
|
|
|
func (hr *HashingReader) Sum() string {
|
|
if !hr.done {
|
|
hr.hash = hr.h.Sum(nil)
|
|
hr.done = true
|
|
}
|
|
return hex.EncodeToString(hr.hash)
|
|
}
|
|
|
|
func (hr *HashingReader) Size() int64 {
|
|
return hr.size
|
|
}
|