pkg-proxy/internal/storage/storage.go

// Package storage provides artifact storage backends for the proxy cache.
//
// Storage backends are accessed via gocloud.dev/blob URLs:
//
//   - file:///path/to/dir - Local filesystem storage
//   - s3://bucket-name - Amazon S3
//   - s3://bucket?endpoint=http://localhost:9000 - S3-compatible (MinIO)
//   - gs://bucket-name - Google Cloud Storage (supports GKE Workload Identity
//     via Application Default Credentials)
//   - azblob://container-name - Azure Blob Storage
//
// Use OpenBucket to create a storage backend from a URL.
package storage

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"errors"
	"io"
	"time"
)

const dirPermissions = 0755

var (
	ErrNotFound = errors.New("artifact not found")

	// ErrSignedURLUnsupported is returned by SignedURL when the backend
	// cannot generate presigned URLs (e.g. local filesystem).
	ErrSignedURLUnsupported = errors.New("signed URLs not supported by storage backend")
)

// ObjectInfo contains metadata for a stored object.
type ObjectInfo struct {
	Path    string
	Size    int64
	ModTime time.Time
}

// Storage defines the interface for artifact storage backends.
type Storage interface {
	// Store writes content from r to the given path.
	// Returns the number of bytes written and the SHA256 hash of the content.
	Store(ctx context.Context, path string, r io.Reader) (size int64, hash string, err error)

	// Open returns a reader for the content at path.
	// The caller must close the reader when done.
	// The reader may also implement io.Seeker when seeking is efficient for the
	// storage backend. Backends must not expose io.Seeker when seeking requires
	// reading and discarding the bytes before the requested position.
	// Returns ErrNotFound if the path does not exist.
	Open(ctx context.Context, path string) (io.ReadCloser, error)

	// Exists returns true if content exists at path.
	Exists(ctx context.Context, path string) (bool, error)

	// Delete removes the content at path.
	// Returns nil if the path does not exist.
	Delete(ctx context.Context, path string) error

	// Size returns the size in bytes of content at path.
	// Returns ErrNotFound if the path does not exist.
	Size(ctx context.Context, path string) (int64, error)

	// SignedURL returns a presigned URL granting time-limited GET access to path.
	// Returns ErrSignedURLUnsupported if the backend cannot generate presigned URLs.
	SignedURL(ctx context.Context, path string, expiry time.Duration) (string, error)

	// UsedSpace returns the total bytes used by all stored content.
	UsedSpace(ctx context.Context) (int64, error)

	// URL returns the storage backend URL (e.g. "file:///path" or "s3://bucket").
	URL() string

	// Close releases any resources held by the storage backend.
	Close() error
}

// ArtifactPath builds the storage path artifacts were cached under before each
// fetch got its own; records from then still point at such paths. Only tests
// call it, to build such records.
// Format: {ecosystem}/{namespace}/{name}/{version}/{filename}
// For packages without namespace: {ecosystem}/{name}/{version}/{filename}
func ArtifactPath(ecosystem, namespace, name, version, filename string) string {
	if namespace != "" {
		return ecosystem + "/" + namespace + "/" + name + "/" + version + "/" + filename
	}
	return ecosystem + "/" + name + "/" + version + "/" + filename
}

// FetchPath builds the storage path for one fetch of an artifact:
// {ecosystem}/{name}/{version}/{fetchID}/{filename}. Each fetch writes its own
// object, so no fetch overwrites or deletes another's.
func FetchPath(ecosystem, name, version, fetchID, filename string) string {
	return ecosystem + "/" + name + "/" + version + "/" + fetchID + "/" + filename
}

// fetchIDBytes sizes a fetch id: 16 hex characters keeps paths short, which
// matters on Windows, and a collision between fetches of one artifact version
// is out of reach.
const fetchIDBytes = 8

// NewFetchID returns a random id for FetchPath.
func NewFetchID() string {
	b := make([]byte, fetchIDBytes)
	_, _ = rand.Read(b)
	return hex.EncodeToString(b)
}

// isFetchDir reports whether a directory name has the shape of a fetch id.
// Only one fetch ever writes into such a directory.
func isFetchDir(name string) bool {
	if len(name) != 2*fetchIDBytes {
		return false
	}
	for _, c := range name {
		if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
			return false
		}
	}
	return true
}