commit ddbd19f15359e0fdc877b483fcb7211b7c72b62e Author: Maxfield Luke Date: Wed Jul 29 04:56:34 2026 -0400 Initial commit on Forgejo Fresh repository history for elektrine/magpie hosted at https://git.elektrine.com/elektrine/magpie. diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b08715e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +.git/ +.env +.env.* +!.env.production.example +data/ +.multipart/ +dist/ +tmp/ +*.db +*.db-* +*.sqlite +*.sqlite-* +*.meta.json +*.tar +*.tar.gz +*.tgz +*.log +coverage.out +magpie diff --git a/.env.production.example b/.env.production.example new file mode 100644 index 0000000..5c7ace2 --- /dev/null +++ b/.env.production.example @@ -0,0 +1,62 @@ +MAGPIE_ADDR=127.0.0.1:8090 +MAGPIE_DATA_DIR=/var/lib/magpie + +# Optional bearer-token API for /objects/* routes. +MAGPIE_AUTH_TOKEN=change-this-long-random-token + +# S3-compatible API credentials for /:bucket/:key routes. +MAGPIE_S3_ACCESS_KEY_ID=magpie +MAGPIE_S3_SECRET_ACCESS_KEY=change-this-long-random-s3-secret +# Optional multi-key form: id:secret:read,write,admin;id2:secret2:read +MAGPIE_S3_KEYS= +MAGPIE_S3_REGION=auto + +# Temporarily set true only for trusted private migrations with clients that +# upload using SigV4 UNSIGNED-PAYLOAD, such as some rclone versions. +MAGPIE_ALLOW_UNSIGNED_PAYLOADS=false + +# Comma-separated peer base URLs for async write/delete replication. +# MAGPIE_REPLICATION_SECRET is required when peers are set. +MAGPIE_REPLICATION_PEERS= +MAGPIE_REPLICATION_SECRET= +# Replication peers must use https unless this is explicitly enabled for a +# trusted local/private network. +MAGPIE_ALLOW_INSECURE_REPLICATION=false +MAGPIE_REPLICATION_MAX_JOBS=10000 +MAGPIE_REPLICATION_MAX_ATTEMPTS=100 + +# Comma-separated bucket allowlist. Empty allows any bucket name. +MAGPIE_ALLOWED_BUCKETS=app-uploads + +# Comma-separated bucket/prefix values that allow unsigned public GET/HEAD requests. +# Empty/private by default. Set to `app-uploads/avatars` only for public media. +# MAGPIE_PUBLIC_PREFIXES=app-uploads/avatars + +# Per-IP request limit per minute. Set 0 to disable. +MAGPIE_RATE_LIMIT_PER_MINUTE=600 +# Trust X-Forwarded-For/X-Real-IP for rate limiting only behind a trusted proxy. +MAGPIE_TRUST_PROXY_HEADERS=false + +# Upload safety limits. +# Default: 1073741824 (1 GiB) +MAGPIE_MAX_OBJECT_SIZE=1073741824 + +# Reject writes if disk free space would fall below this value. +# Default: 1073741824 (1 GiB) +MAGPIE_MIN_FREE_BYTES=1073741824 + +# Maximum accepted SigV4 presigned URL lifetime. +# Default: 24h +MAGPIE_PRESIGNED_MAX_EXPIRY=24h + +# Remove incomplete multipart uploads older than this age. +# Default: 24h +MAGPIE_MULTIPART_MAX_AGE=24h + +# Maximum total expanded bytes accepted by restore. +# Default: 10737418240 (10 GiB) +MAGPIE_MAX_RESTORE_BYTES=10737418240 + +# Rebuild SQLite metadata from existing files on startup. +# Useful after manual restores/migrations; usually false for normal startup. +MAGPIE_REINDEX_ON_START=false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..8647729 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,51 @@ +name: release + +on: + push: + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff + with: + go-version-file: go.mod + cache: true + + - name: Test + run: go test ./... + + - name: Build release artifacts + run: make release VERSION="$GITHUB_REF_NAME" COMMIT="${GITHUB_SHA::7}" + + - name: Upload artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: magpie-${{ github.ref_name }} + path: dist/* + + publish: + if: startsWith(github.ref, 'refs/tags/') + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: magpie-${{ github.ref_name }} + path: dist + + - name: Publish GitHub release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 + with: + files: dist/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f5e1f37 --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Local binaries and build output +magpie +dist/ +coverage.out + +# Runtime data +data/ +*.db +*.db-* +*.sqlite +*.sqlite-* +*.meta.json +.multipart/ + +# Local environment files +.env +.env.* +!.env.production.example + +# Logs and temporary files +*.log +*.tmp +tmp/ + +# Editor and OS files +.DS_Store +.idea/ +.vscode/ +*.swp +*.swo diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dcea72a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM golang:1.26-alpine@sha256:f44b851aa23dfa219d18db6eab743203245429d355cb619cf96a2ffe2a84ba7a AS build + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY *.go ./ +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/magpie . + +FROM alpine:3.22@sha256:2039be0c5ec6ce8566809626a252c930216a92109c043f282504accb5ee3c0c6 + +RUN addgroup -S magpie && adduser -S -G magpie magpie +RUN mkdir -p /var/lib/magpie && chown -R magpie:magpie /var/lib/magpie + +USER magpie +EXPOSE 8090 +ENV MAGPIE_ADDR=0.0.0.0:8090 +ENV MAGPIE_DATA_DIR=/var/lib/magpie + +COPY --from=build /out/magpie /usr/local/bin/magpie +ENTRYPOINT ["/usr/local/bin/magpie"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bc715cf --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +BINARY := magpie +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || printf dev) +COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || printf unknown) +DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) +LDFLAGS := -s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.date=$(DATE) +DIST := dist + +.PHONY: all fmt test build clean checksums release snapshot + +all: fmt test build + +fmt: + gofmt -w . + +test: + go test ./... + +build: + mkdir -p $(DIST) + CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY) . + +snapshot: clean + mkdir -p $(DIST) + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_linux_amd64 . + GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_linux_arm64 . + GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_darwin_amd64 . + GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_darwin_arm64 . + GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_windows_amd64.exe . + +checksums: + cd $(DIST) && sha256sum * > SHA256SUMS + +release: snapshot checksums + +clean: + rm -rf $(DIST) diff --git a/README.md b/README.md new file mode 100644 index 0000000..a62d79d --- /dev/null +++ b/README.md @@ -0,0 +1,271 @@ +# Magpie + +Magpie is a small object store for app-owned files. It stores objects on local disk and exposes a minimal S3-compatible path-style API. + +It is intended to run behind your application or edge proxy, on localhost, a private Docker network, or a VPN-only address. Do not expose write-capable endpoints directly to the public internet. + +## Status + +Magpie supports common application object-storage operations: + +- `PUT`, `GET`, `HEAD`, and `DELETE` objects +- path-style S3 routes: `/:bucket/:key` +- SigV4 header authentication for writes and private reads +- presigned `GET` and `HEAD` URLs +- prefix listing with continuation tokens +- batch delete +- copy object +- multipart upload +- bucket allowlist and opt-in public-read buckets +- local metadata index in SQLite +- async static-peer replication +- backup, restore, scrub, reindex, stats, and repair commands + +It does not implement bucket creation, ACLs, bucket policies, object versioning, lifecycle rules, quorum writes, or consensus replication. + +## Quick Start + +```sh +export MAGPIE_ADDR='127.0.0.1:8090' +export MAGPIE_DATA_DIR='/var/lib/magpie' +export MAGPIE_S3_ACCESS_KEY_ID='magpie' +export MAGPIE_S3_SECRET_ACCESS_KEY='replace-with-a-long-random-secret' +export MAGPIE_ALLOWED_BUCKETS='app-uploads' + +magpie serve +``` + +For local development from source: + +```sh +go run . serve +``` + +## Configuration + +Required authentication config: + +```env +MAGPIE_S3_ACCESS_KEY_ID=magpie +MAGPIE_S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret +``` + +Or define multiple scoped keys: + +```env +MAGPIE_S3_KEYS=app:secret1:read,write;auditor:secret2:read;admin:secret3:admin +``` + +Supported scopes are `read`, `write`, and `admin`. `admin` implies all scopes. + +Common runtime config: + +```env +MAGPIE_ADDR=127.0.0.1:8090 +MAGPIE_DATA_DIR=/var/lib/magpie +MAGPIE_ALLOWED_BUCKETS=app-uploads +# MAGPIE_PUBLIC_PREFIXES=app-uploads/avatars +MAGPIE_RATE_LIMIT_PER_MINUTE=600 +MAGPIE_TRUST_PROXY_HEADERS=false +MAGPIE_MAX_OBJECT_SIZE=1073741824 +MAGPIE_MIN_FREE_BYTES=1073741824 +MAGPIE_PRESIGNED_MAX_EXPIRY=24h +MAGPIE_MULTIPART_MAX_AGE=24h +MAGPIE_REINDEX_ON_START=false +MAGPIE_MAX_RESTORE_BYTES=10737418240 +``` + +`MAGPIE_ALLOWED_BUCKETS` should be set in production. Leave it empty only if you intentionally want any bucket name accepted. + +Buckets are private by default. Set `MAGPIE_PUBLIC_PREFIXES` to comma-separated `bucket/prefix` values, such as `app-uploads/avatars`, only when an app intentionally serves unsigned public media through an edge proxy. Set `MAGPIE_PUBLIC_PREFIXES=none` or leave it unset to require signed reads. + +Public reads are hardened against stored-XSS from uploader-controlled content types: responses carry `Content-Security-Policy: default-src 'none'; sandbox`, and any object that is not inert inline media (images, audio, video — SVG excluded) is served with `Content-Disposition: attachment` so browsers download it instead of rendering it. For untrusted user uploads, still prefer serving public objects from an origin separate from your application. + +## Application Integration + +Use any S3-compatible client that supports path-style endpoints and custom hosts. + +If Magpie runs on the same host as your application outside Docker: + +```env +S3_ACCESS_KEY_ID=magpie +S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret +S3_ENDPOINT=127.0.0.1:8090 +S3_BUCKET_NAME=app-uploads +S3_PUBLIC_URL=http://127.0.0.1:8090/app-uploads +S3_SCHEME=http:// +S3_PORT=8090 +``` + +If your app and Magpie are in the same Docker Compose network, use the Magpie service name: + +```env +S3_ENDPOINT=magpie:8090 +S3_BUCKET_NAME=app-uploads +S3_PUBLIC_URL=http://magpie:8090/app-uploads +S3_SCHEME=http:// +S3_PORT=8090 +``` + +Keep Magpie write access private. Your application should enforce user permissions before it writes to Magpie. Public buckets are intended for already-public objects such as avatars and timeline media. + +## API Notes + +S3 object routes use path-style URLs: + +```text +PUT /:bucket/:key +GET /:bucket/:key +HEAD /:bucket/:key +DELETE /:bucket/:key +POST /:bucket?delete +``` + +Objects are stored under: + +```text +MAGPIE_DATA_DIR// +``` + +Internal names are reserved and cannot be used as object keys: + +- `.multipart` +- `.magpie.db` +- `.magpie.db-*` +- `*.meta.json` + +Presigned URLs are read-only. Magpie accepts presigned `GET` and `HEAD` requests, but rejects presigned writes and deletes. + +Normal SigV4 write requests must include a real `X-Amz-Content-Sha256` payload hash. `UNSIGNED-PAYLOAD` is reserved for authenticated replication traffic. + +## Health And Metrics + +Health and metrics require authentication. + +```sh +curl -H "Authorization: Bearer $MAGPIE_AUTH_TOKEN" \ + http://127.0.0.1:8090/health +``` + +```sh +curl -H "Authorization: Bearer $MAGPIE_AUTH_TOKEN" \ + http://127.0.0.1:8090/metrics +``` + +The health endpoint returns `503` when free disk space is below `MAGPIE_MIN_FREE_BYTES`. Writes are rejected in that state. + +If you only use S3 credentials and do not set `MAGPIE_AUTH_TOKEN`, sign health and metrics requests with an admin S3 key instead. + +## CLI + +```sh +magpie serve +magpie stats +magpie scrub +magpie reindex +magpie repair +magpie backup /backup/magpie.tar.gz +magpie restore /backup/magpie.tar.gz +magpie smoke +magpie version +``` + +Command summary: + +- `stats`: object count, logical bytes, disk status, replication queue status +- `scrub`: verifies metadata against files and reports missing, corrupt, and orphaned objects +- `reindex`: rebuilds SQLite metadata from files on disk +- `repair`: queues known objects for replication to configured peers +- `backup`: creates a tar/gzip archive of the data directory +- `restore`: restores a tar/gzip archive into the data directory +- `smoke`: runs a local write/read/list/scrub/delete check + +Restore rejects unsafe archive paths, symlinks, unsupported archive entries, and archives whose expanded file content exceeds `MAGPIE_MAX_RESTORE_BYTES`. Backup archives are created with owner-only permissions. For consistent backups, stop Magpie or use a filesystem snapshot before running `magpie backup`. + +## Replication + +Replication is asynchronous and best-effort. Local writes complete before peers acknowledge them. + +Configure every node with matching S3 credentials and a shared replication secret: + +```env +MAGPIE_REPLICATION_PEERS=https://magpie-b.internal,https://magpie-c.internal +MAGPIE_REPLICATION_SECRET=replace-with-a-long-random-peer-secret +MAGPIE_REPLICATION_MAX_JOBS=10000 +MAGPIE_REPLICATION_MAX_ATTEMPTS=100 +``` + +`MAGPIE_REPLICATION_SECRET` is required when peers are configured. Replication-marked requests without the shared secret are rejected. + +Replication peer URLs must use `https://` by default because Magpie sends SigV4 authorization and the replication secret to peers. Set `MAGPIE_ALLOW_INSECURE_REPLICATION=true` only for trusted local/private networks where plaintext HTTP is acceptable. + +Use `magpie repair` after outages to queue a full object repair pass to all configured peers. + +## Deployment + +Docker Compose example: + +```sh +cp .env.production.example .env.production +``` + +```yaml +services: + magpie: + image: magpie:latest + container_name: magpie + build: . + restart: unless-stopped + env_file: + - .env.production + ports: + - "127.0.0.1:8090:8090" + volumes: + - /var/lib/magpie:/var/lib/magpie +``` + +The default Compose file is standalone and binds Magpie to localhost only. + +If another Compose project needs to reach Magpie by service name, use the optional network override. It attaches Magpie to an external shared network with the `magpie` network alias: + +```sh +docker network create elektrine-magpie-shared +docker compose -f docker-compose.yml -f docker-compose.network.yml up -d --build +``` + +Then attach your application container to the same external `elektrine-magpie-shared` network and use: + +```env +S3_ENDPOINT=magpie:8090 +S3_PUBLIC_URL=http://magpie:8090/app-uploads +S3_SCHEME=http:// +S3_PORT=8090 +``` + +You can rename the shared network without editing the override file: + +```sh +MAGPIE_DOCKER_NETWORK=my-existing-shared-network \ + docker compose -f docker-compose.yml -f docker-compose.network.yml up -d --build +``` + +Bare-metal installs can use `deploy/systemd/magpie.service`. The systemd unit is optional; use it only if you want Magpie managed directly by systemd. + +Back up `/var/lib/magpie` or whatever directory you set as `MAGPIE_DATA_DIR`. + +## Build And Test + +```sh +go test ./... +go build ./... +``` + +Release artifacts: + +```sh +make test +make build +make release +``` + +Artifacts are written to `dist/`, with checksums in `dist/SHA256SUMS`. diff --git a/backup.go b/backup.go new file mode 100644 index 0000000..952dbc3 --- /dev/null +++ b/backup.go @@ -0,0 +1,181 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +func BackupDir(src string, dst string) error { + out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + defer out.Close() + gz := gzip.NewWriter(out) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + root, err := filepath.Abs(src) + if err != nil { + return err + } + return filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing to back up symlink %q", path) + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + info, err := entry.Info() + if err != nil { + return err + } + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = filepath.ToSlash(rel) + if err := tw.WriteHeader(header); err != nil { + return err + } + file, err := os.Open(path) + if err != nil { + return err + } + fileInfo, statErr := file.Stat() + currentInfo, lstatErr := os.Lstat(path) + if statErr != nil || lstatErr != nil { + _ = file.Close() + if statErr != nil { + return statErr + } + return lstatErr + } + if currentInfo.Mode()&os.ModeSymlink != 0 || !os.SameFile(currentInfo, fileInfo) { + _ = file.Close() + return fmt.Errorf("refusing to back up replaced or symlinked file %q", path) + } + _, copyErr := io.Copy(tw, file) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + return closeErr + }) +} + +func RestoreDir(src string, dst string) error { + return RestoreDirWithLimit(src, dst, defaultMaxRestoreBytes) +} + +func RestoreDirWithLimit(src string, dst string, maxBytes int64) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + gz, err := gzip.NewReader(in) + if err != nil { + return err + } + defer gz.Close() + tr := tar.NewReader(gz) + root, err := filepath.Abs(dst) + if err != nil { + return err + } + if err := os.MkdirAll(root, 0o700); err != nil { + return err + } + var restoredBytes int64 + for { + header, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if strings.Contains(header.Name, "..") || strings.HasPrefix(header.Name, "/") { + return fmt.Errorf("unsafe archive path %q", header.Name) + } + if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA && header.Typeflag != tar.TypeDir { + return fmt.Errorf("unsupported archive entry %q", header.Name) + } + if header.Size < 0 { + return fmt.Errorf("invalid archive size for %q", header.Name) + } + if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeRegA { + restoredBytes += header.Size + if maxBytes > 0 && restoredBytes > maxBytes { + return fmt.Errorf("restore exceeds maximum size") + } + } + path := filepath.Join(root, filepath.FromSlash(header.Name)) + if !strings.HasPrefix(path, root+string(os.PathSeparator)) && path != root { + return fmt.Errorf("unsafe archive path %q", header.Name) + } + if err := rejectSymlinkParents(root, path); err != nil { + return err + } + if header.FileInfo().IsDir() { + if err := os.MkdirAll(path, 0o700); err != nil { + return err + } + continue + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + if info, err := os.Lstat(path); err == nil && info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing to restore through symlink %q", header.Name) + } + out, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + _, copyErr := io.CopyN(out, tr, header.Size) + closeErr := out.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + } +} + +func rejectSymlinkParents(root string, path string) error { + current := root + rel, err := filepath.Rel(root, filepath.Dir(path)) + if err != nil { + return err + } + if rel == "." { + return nil + } + for _, part := range strings.Split(rel, string(os.PathSeparator)) { + current = filepath.Join(current, part) + info, err := os.Lstat(current) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing to restore through symlink parent %q", current) + } + } + return nil +} diff --git a/deploy/systemd/magpie.service b/deploy/systemd/magpie.service new file mode 100644 index 0000000..8d2f34c --- /dev/null +++ b/deploy/systemd/magpie.service @@ -0,0 +1,21 @@ +[Unit] +Description=Magpie object storage service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=magpie +Group=magpie +EnvironmentFile=/etc/magpie/.env.production +ExecStart=/usr/local/bin/magpie +Restart=on-failure +RestartSec=5s +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/magpie + +[Install] +WantedBy=multi-user.target diff --git a/disk_unix.go b/disk_unix.go new file mode 100644 index 0000000..12d3846 --- /dev/null +++ b/disk_unix.go @@ -0,0 +1,21 @@ +//go:build !windows + +package main + +import ( + "fmt" + "syscall" +) + +func (s *FileStore) DiskStatus() (DiskStatus, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(s.root, &stat); err != nil { + return DiskStatus{}, err + } + if stat.Bsize <= 0 { + return DiskStatus{}, fmt.Errorf("invalid filesystem block size") + } + total := stat.Blocks * uint64(stat.Bsize) + free := stat.Bavail * uint64(stat.Bsize) + return DiskStatus{TotalBytes: total, FreeBytes: free, UsedBytes: total - free}, nil +} diff --git a/disk_windows.go b/disk_windows.go new file mode 100644 index 0000000..ecd0b7e --- /dev/null +++ b/disk_windows.go @@ -0,0 +1,7 @@ +//go:build windows + +package main + +func (s *FileStore) DiskStatus() (DiskStatus, error) { + return DiskStatus{}, nil +} diff --git a/docker-compose.network.yml b/docker-compose.network.yml new file mode 100644 index 0000000..e821a0e --- /dev/null +++ b/docker-compose.network.yml @@ -0,0 +1,11 @@ +services: + magpie: + networks: + magpie_shared: + aliases: + - magpie + +networks: + magpie_shared: + external: true + name: ${MAGPIE_DOCKER_NETWORK:-elektrine-magpie-shared} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..eaff37a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +services: + magpie: + image: magpie:latest + container_name: magpie + build: . + restart: unless-stopped + env_file: + - .env.production + ports: + - "127.0.0.1:8090:8090" + volumes: + - magpie-data:/var/lib/magpie + +volumes: + magpie-data: diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..499852d --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module magpie + +go 1.25.0 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.42.0 // indirect + modernc.org/libc v1.72.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.50.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..08ff8cd --- /dev/null +++ b/go.sum @@ -0,0 +1,21 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= +modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM= +modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= diff --git a/main.go b/main.go new file mode 100644 index 0000000..f423de9 --- /dev/null +++ b/main.go @@ -0,0 +1,415 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" +) + +const ( + defaultMaxObjectSize = 1024 * 1024 * 1024 + defaultMinFreeBytes = 1024 * 1024 * 1024 + defaultPresignedMaxExpiry = 24 * time.Hour + defaultMultipartMaxAge = 24 * time.Hour + defaultMaxRestoreBytes = 10 * 1024 * 1024 * 1024 + defaultReplicationMaxJobs = 10_000 + defaultReplicationMaxAttempts = 100 +) + +func main() { + cmd := "serve" + if len(os.Args) > 1 { + cmd = os.Args[1] + } + if cmd == "version" { + printJSON(versionInfo()) + return + } + if cmd == "help" || cmd == "-h" || cmd == "--help" { + printUsage() + return + } + + cfg := configFromEnv() + var store *FileStore + openStore := func() *FileStore { + if store != nil { + return store + } + opened, err := NewFileStore(cfg.DataDir) + if err != nil { + log.Fatalf("init store: %v", err) + } + store = opened + return store + } + + switch cmd { + case "serve": + serve(cfg, openStore()) + case "reindex": + mustJSON(openStore().Reindex()) + case "stats": + stats, err := openStore().Stats() + if err != nil { + log.Fatal(err) + } + printJSON(stats) + case "scrub": + report, err := openStore().Scrub() + if err != nil { + log.Fatal(err) + } + printJSON(report) + case "repair": + count, err := openStore().EnqueueRepair(cfg.ReplicationPeers) + if err != nil { + log.Fatal(err) + } + printJSON(map[string]any{"status": "queued", "jobs": count}) + case "backup": + if len(os.Args) < 3 { + log.Fatal("usage: magpie backup ") + } + mustJSON(BackupDir(cfg.DataDir, os.Args[2])) + case "restore": + if len(os.Args) < 3 { + log.Fatal("usage: magpie restore ") + } + mustJSON(RestoreDirWithLimit(os.Args[2], cfg.DataDir, cfg.MaxRestoreBytes)) + case "smoke": + mustJSON(runSmoke(openStore())) + default: + log.Fatalf("unknown command %q", sanitizeLogValue(cmd)) + } +} + +func serve(cfg Config, store *FileStore) { + if cfg.ReindexOnStart { + log.Printf("reindexing metadata from %s", cfg.DataDir) + if err := store.Reindex(); err != nil { + log.Fatalf("reindex store: %v", err) + } + } + + server := NewServerInstance(store, cfg) + srv := &http.Server{ + Addr: cfg.Addr, + Handler: server.Handler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 2 * time.Minute, + WriteTimeout: 2 * time.Minute, + IdleTimeout: 60 * time.Second, + MaxHeaderBytes: 16 * 1024, + } + + go func() { + log.Printf("object store listening on %s, data=%s", cfg.Addr, cfg.DataDir) + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("listen: %v", err) + } + }() + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + server.startMultipartCleanup(ctx) + server.startReplicationWorker(ctx) + <-ctx.Done() + + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + log.Printf("shutdown: %v", err) + } +} + +type Config struct { + Addr string + DataDir string + AuthToken string + S3AccessKeyID string + S3SecretAccessKey string + S3Keys []AccessKey + S3Region string + AllowUnsignedLocal bool + MaxObjectSize int64 + MinFreeBytes uint64 + PresignedMaxExpiry time.Duration + MultipartMaxAge time.Duration + ReindexOnStart bool + ReplicationPeers []string + ReplicationSecret string + AllowInsecureReplication bool + ReplicationMaxJobs int + ReplicationMaxAttempts int + MaxRestoreBytes int64 + AllowedBuckets map[string]bool + PublicPrefixes map[string]bool + RateLimitPerMinute int + TrustProxyHeaders bool +} + +type AccessKey struct { + ID string + Secret string + Permissions map[string]bool +} + +func configFromEnv() Config { + addr := getenv("MAGPIE_ADDR", "127.0.0.1:8090") + dataDir := getenv("MAGPIE_DATA_DIR", "./data") + authToken := os.Getenv("MAGPIE_AUTH_TOKEN") + s3AccessKeyID := os.Getenv("MAGPIE_S3_ACCESS_KEY_ID") + s3SecretAccessKey := os.Getenv("MAGPIE_S3_SECRET_ACCESS_KEY") + s3Keys := parseAccessKeys(os.Getenv("MAGPIE_S3_KEYS"), s3AccessKeyID, s3SecretAccessKey) + s3Region := getenv("MAGPIE_S3_REGION", "auto") + maxObjectSize := mustParseBytes("MAGPIE_MAX_OBJECT_SIZE", defaultMaxObjectSize) + minFreeBytes := mustParseUint("MAGPIE_MIN_FREE_BYTES", defaultMinFreeBytes) + presignedMaxExpiry := mustParseDuration("MAGPIE_PRESIGNED_MAX_EXPIRY", defaultPresignedMaxExpiry) + multipartMaxAge := mustParseDuration("MAGPIE_MULTIPART_MAX_AGE", defaultMultipartMaxAge) + reindexOnStart := mustParseBool("MAGPIE_REINDEX_ON_START", false) + allowInsecureReplication := mustParseBool("MAGPIE_ALLOW_INSECURE_REPLICATION", false) + replicationPeers := parseReplicationPeers(os.Getenv("MAGPIE_REPLICATION_PEERS"), allowInsecureReplication) + replicationSecret := os.Getenv("MAGPIE_REPLICATION_SECRET") + replicationMaxJobs := mustParseInt("MAGPIE_REPLICATION_MAX_JOBS", defaultReplicationMaxJobs) + replicationMaxAttempts := mustParseInt("MAGPIE_REPLICATION_MAX_ATTEMPTS", defaultReplicationMaxAttempts) + maxRestoreBytes := mustParseBytes("MAGPIE_MAX_RESTORE_BYTES", defaultMaxRestoreBytes) + allowedBuckets := parseSet(os.Getenv("MAGPIE_ALLOWED_BUCKETS")) + publicPrefixes := parsePublicPrefixes(os.Getenv("MAGPIE_PUBLIC_PREFIXES")) + rateLimitPerMinute := mustParseInt("MAGPIE_RATE_LIMIT_PER_MINUTE", 600) + trustProxyHeaders := mustParseBool("MAGPIE_TRUST_PROXY_HEADERS", false) + allowUnsignedLocal := mustParseBool("MAGPIE_ALLOW_UNSIGNED_PAYLOADS", false) + + if authToken == "" && len(s3Keys) == 0 { + log.Fatal("MAGPIE_AUTH_TOKEN or MAGPIE_S3_ACCESS_KEY_ID/MAGPIE_S3_SECRET_ACCESS_KEY or MAGPIE_S3_KEYS is required") + } + if len(replicationPeers) > 0 && replicationSecret == "" { + log.Fatal("MAGPIE_REPLICATION_SECRET is required when MAGPIE_REPLICATION_PEERS is set") + } + + if raw := os.Getenv("MAGPIE_MAX_PROCS"); raw != "" { + if _, err := strconv.Atoi(raw); err != nil { + log.Fatalf("MAGPIE_MAX_PROCS must be an integer: %v", err) + } + } + + return Config{ + Addr: addr, + DataDir: dataDir, + AuthToken: authToken, + S3AccessKeyID: s3AccessKeyID, + S3SecretAccessKey: s3SecretAccessKey, + S3Keys: s3Keys, + S3Region: s3Region, + AllowUnsignedLocal: allowUnsignedLocal, + MaxObjectSize: maxObjectSize, + MinFreeBytes: minFreeBytes, + PresignedMaxExpiry: presignedMaxExpiry, + MultipartMaxAge: multipartMaxAge, + ReindexOnStart: reindexOnStart, + ReplicationPeers: replicationPeers, + ReplicationSecret: replicationSecret, + AllowInsecureReplication: allowInsecureReplication, + ReplicationMaxJobs: replicationMaxJobs, + ReplicationMaxAttempts: replicationMaxAttempts, + MaxRestoreBytes: maxRestoreBytes, + AllowedBuckets: allowedBuckets, + PublicPrefixes: publicPrefixes, + RateLimitPerMinute: rateLimitPerMinute, + TrustProxyHeaders: trustProxyHeaders, + } +} + +func parseAccessKeys(raw string, legacyID string, legacySecret string) []AccessKey { + keys := make([]AccessKey, 0) + if legacyID != "" && legacySecret != "" { + keys = append(keys, AccessKey{ID: legacyID, Secret: legacySecret, Permissions: permissions("read,write,admin")}) + } + for _, entry := range strings.Split(raw, ";") { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + parts := strings.Split(entry, ":") + if len(parts) < 2 || parts[0] == "" || parts[1] == "" { + log.Fatal("MAGPIE_S3_KEYS entries must be id:secret[:read,write,admin]") + } + perms := "read,write" + if len(parts) >= 3 { + perms = parts[2] + } + keys = append(keys, AccessKey{ID: parts[0], Secret: parts[1], Permissions: permissions(perms)}) + } + return keys +} + +func permissions(raw string) map[string]bool { + result := map[string]bool{} + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(strings.ToLower(part)) + if part != "" { + result[part] = true + } + } + return result +} + +func parseCSV(raw string) []string { + parts := make([]string, 0) + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part != "" { + parts = append(parts, strings.TrimRight(part, "/")) + } + } + return parts +} + +func parseReplicationPeers(raw string, allowInsecure bool) []string { + peers := parseCSV(raw) + for _, peer := range peers { + if err := validateReplicationPeer(peer, allowInsecure); err != nil { + log.Fatal(err) + } + } + return peers +} + +func parseSet(raw string) map[string]bool { + values := map[string]bool{} + for _, value := range parseCSV(raw) { + values[value] = true + } + return values +} + +func parsePublicPrefixes(raw string) map[string]bool { + value := strings.TrimSpace(strings.ToLower(raw)) + if value == "none" || value == "false" || value == "0" { + return map[string]bool{} + } + if strings.TrimSpace(raw) != "" { + prefixes := parseSet(raw) + for prefix := range prefixes { + if err := ValidateKey(strings.TrimSuffix(prefix, "/")); err != nil || !strings.Contains(strings.Trim(prefix, "/"), "/") { + log.Fatal("MAGPIE_PUBLIC_PREFIXES entries must be bucket/prefix values") + } + } + return prefixes + } + return map[string]bool{} +} + +func printUsage() { + fmt.Println("usage: magpie [serve|reindex|stats|scrub|repair|version|backup|restore|smoke]") +} + +func mustJSON(err error) { + if err != nil { + log.Fatal(err) + } + printJSON(map[string]string{"status": "ok"}) +} + +func printJSON(value any) { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(value); err != nil { + log.Fatal(err) + } +} + +func getenv(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} + +func mustParseBytes(key string, fallback int64) int64 { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed < 0 { + log.Fatal(fmt.Sprintf("%s must be a non-negative integer byte count", key)) + } + return parsed +} + +func mustParseUint(key string, fallback uint64) uint64 { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := strconv.ParseUint(value, 10, 64) + if err != nil { + log.Fatal(fmt.Sprintf("%s must be a non-negative integer byte count", key)) + } + return parsed +} + +func mustParseDuration(key string, fallback time.Duration) time.Duration { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := time.ParseDuration(value) + if err != nil || parsed <= 0 { + log.Fatal(fmt.Sprintf("%s must be a positive duration like 1h or 24h", key)) + } + return parsed +} + +func mustParseBool(key string, fallback bool) bool { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := strconv.ParseBool(value) + if err != nil { + log.Fatal(fmt.Sprintf("%s must be true or false", key)) + } + return parsed +} + +func mustParseInt(key string, fallback int) int { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := strconv.Atoi(value) + if err != nil || parsed < 0 { + log.Fatal(fmt.Sprintf("%s must be a non-negative integer", key)) + } + return parsed +} + +func runSmoke(store *FileStore) error { + key := "smoke/test.txt" + if _, err := store.Put(key, strings.NewReader("ok"), "text/plain"); err != nil { + return err + } + obj, err := store.Open(key) + if err != nil { + return err + } + _ = obj.Close() + if _, err := store.List("smoke/", 10); err != nil { + return err + } + if _, err := store.Scrub(); err != nil { + return err + } + return store.Delete(key) +} diff --git a/replication.go b/replication.go new file mode 100644 index 0000000..bd6b4ba --- /dev/null +++ b/replication.go @@ -0,0 +1,330 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log" + "net/http" + "net/url" + "strings" + "time" +) + +const ( + replicationOpPut = "put" + replicationOpDelete = "delete" +) + +type ReplicationJob struct { + ID int64 + Peer string + Op string + Key string + Attempts int +} + +func (s *Server) enqueueReplicationPut(key string) { + s.enqueueReplication(replicationOpPut, key) +} + +func (s *Server) enqueueReplicationDelete(key string) { + s.enqueueReplication(replicationOpDelete, key) +} + +func (s *Server) enqueueReplication(op string, key string) { + if len(s.cfg.ReplicationPeers) == 0 || len(s.cfg.S3Keys) == 0 || s.cfg.ReplicationSecret == "" { + return + } + for _, peer := range s.cfg.ReplicationPeers { + if err := s.store.(*FileStore).EnqueueReplication(peer, op, key, s.cfg.ReplicationMaxJobs); err != nil { + log.Printf("replication enqueue op=%s key=%s peer=%s error=%v", op, sanitizeLogValue(key), sanitizeLogValue(peer), err) + } + } + go s.processReplicationQueue(context.Background()) +} + +func (s *Server) startReplicationWorker(ctx context.Context) { + if len(s.cfg.ReplicationPeers) == 0 || len(s.cfg.S3Keys) == 0 || s.cfg.ReplicationSecret == "" { + return + } + go func() { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.processReplicationQueue(ctx) + } + } + }() +} + +func (s *Server) processReplicationQueue(ctx context.Context) { + jobs, err := s.store.(*FileStore).DueReplicationJobs(25) + if err != nil { + log.Printf("replication due jobs error=%v", err) + return + } + for _, job := range jobs { + if ctx.Err() != nil { + return + } + if err := s.performReplicationJob(job); err != nil { + log.Printf("replication job id=%d op=%s key=%s peer=%s error=%v", job.ID, job.Op, sanitizeLogValue(job.Key), sanitizeLogValue(job.Peer), err) + if s.cfg.ReplicationMaxAttempts > 0 && job.Attempts+1 >= s.cfg.ReplicationMaxAttempts { + _ = s.store.(*FileStore).CompleteReplicationJob(job.ID) + continue + } + _ = s.store.(*FileStore).FailReplicationJob(job.ID, job.Attempts+1, err.Error()) + continue + } + _ = s.store.(*FileStore).CompleteReplicationJob(job.ID) + } +} + +func (s *Server) performReplicationJob(job ReplicationJob) error { + switch job.Op { + case replicationOpPut: + return s.performReplicationPut(job.Peer, job.Key) + case replicationOpDelete: + return s.performReplicationDelete(job.Peer, job.Key) + default: + return fmt.Errorf("unknown replication op %q", job.Op) + } +} + +func (s *Server) performReplicationPut(peer string, key string) error { + obj, err := s.store.Open(key) + if err != nil { + return err + } + defer obj.Close() + + peerURL, err := replicationObjectURL(peer, key, s.cfg.AllowInsecureReplication) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPut, peerURL, obj) + if err != nil { + return err + } + req.Header.Set("Content-Type", obj.Info.ContentType) + req.Header.Set("X-Magpie-Replication", "1") + if s.cfg.ReplicationSecret != "" { + req.Header.Set("X-Magpie-Replication-Secret", s.cfg.ReplicationSecret) + } + s.signReplication(req) + resp, err := (&http.Client{Timeout: 5 * time.Minute}).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("peer status %d", resp.StatusCode) + } + return nil +} + +func (s *Server) performReplicationDelete(peer string, key string) error { + peerURL, err := replicationObjectURL(peer, key, s.cfg.AllowInsecureReplication) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodDelete, peerURL, nil) + if err != nil { + return err + } + req.Header.Set("X-Magpie-Replication", "1") + if s.cfg.ReplicationSecret != "" { + req.Header.Set("X-Magpie-Replication-Secret", s.cfg.ReplicationSecret) + } + s.signReplication(req) + resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("peer status %d", resp.StatusCode) + } + return nil +} + +func (s *Server) signReplication(req *http.Request) { + key := s.cfg.S3Keys[0] + amzDate := time.Now().UTC().Format("20060102T150405Z") + date := amzDate[:8] + scope := date + "/" + s.cfg.S3Region + "/s3/aws4_request" + signedHeaders := "host;x-amz-content-sha256;x-amz-date;x-magpie-replication" + if req.Header.Get("X-Magpie-Replication-Secret") != "" { + signedHeaders += ";x-magpie-replication-secret" + } + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD") + canonical := canonicalRequest(req, signedHeaders, "UNSIGNED-PAYLOAD", false) + sig := signature(key.Secret, amzDate, scope, canonical) + req.Header.Set("Authorization", awsAlgorithm+" Credential="+key.ID+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig) +} + +func replicationObjectURL(peer string, key string, allowInsecure bool) (string, error) { + if err := ValidateKey(key); err != nil { + return "", err + } + if err := validateReplicationPeer(peer, allowInsecure); err != nil { + return "", err + } + parsed, err := url.Parse(peer) + if err != nil { + return "", err + } + parsed.RawQuery = "" + parsed.Fragment = "" + parsed.Path = strings.TrimRight(parsed.Path, "/") + "/" + key + parsed.RawPath = "" + return parsed.String(), nil +} + +func validateReplicationPeer(peer string, allowInsecure bool) error { + parsed, err := url.Parse(peer) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return fmt.Errorf("invalid replication peer %q", peer) + } + if parsed.RawQuery != "" || parsed.Fragment != "" { + return fmt.Errorf("replication peer %q must not include query or fragment", peer) + } + if parsed.Scheme == "https" { + return nil + } + if allowInsecure && parsed.Scheme == "http" { + return nil + } + return fmt.Errorf("replication peer %q must use https or set MAGPIE_ALLOW_INSECURE_REPLICATION=true", peer) +} + +func (s *FileStore) EnqueueReplication(peer string, op string, key string, maxJobs int) error { + now := time.Now().UTC().Format(time.RFC3339Nano) + result, err := s.db.Exec(` +UPDATE replication_jobs +SET next_attempt_at = ?, updated_at = ? +WHERE peer = ? AND op = ? AND key = ? +`, now, now, peer, op, key) + if err != nil { + return err + } + if rows, err := result.RowsAffected(); err == nil && rows > 0 { + return nil + } + if maxJobs <= 0 { + maxJobs = defaultReplicationMaxJobs + } + var pending int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM replication_jobs`).Scan(&pending); err != nil { + return err + } + if pending >= maxJobs { + return fmt.Errorf("replication queue is full") + } + _, err = s.db.Exec(` +INSERT INTO replication_jobs (peer, op, key, next_attempt_at, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?) +`, peer, op, key, now, now, now) + return err +} + +func (s *FileStore) DueReplicationJobs(limit int) ([]ReplicationJob, error) { + if limit <= 0 || limit > 100 { + limit = 25 + } + now := time.Now().UTC().Format(time.RFC3339Nano) + rows, err := s.db.Query(` +SELECT id, peer, op, key, attempts +FROM replication_jobs +WHERE next_attempt_at <= ? +ORDER BY id ASC +LIMIT ? +`, now, limit) + if err != nil { + return nil, err + } + defer rows.Close() + jobs := make([]ReplicationJob, 0) + for rows.Next() { + var job ReplicationJob + if err := rows.Scan(&job.ID, &job.Peer, &job.Op, &job.Key, &job.Attempts); err != nil { + return nil, err + } + jobs = append(jobs, job) + } + return jobs, rows.Err() +} + +func (s *FileStore) CompleteReplicationJob(id int64) error { + _, err := s.db.Exec(`DELETE FROM replication_jobs WHERE id = ?`, id) + return err +} + +func (s *FileStore) FailReplicationJob(id int64, attempts int, message string) error { + delay := time.Duration(attempts*attempts) * time.Second + if delay > 5*time.Minute { + delay = 5 * time.Minute + } + next := time.Now().UTC().Add(delay).Format(time.RFC3339Nano) + now := time.Now().UTC().Format(time.RFC3339Nano) + _, err := s.db.Exec(` +UPDATE replication_jobs +SET attempts = ?, next_attempt_at = ?, last_error = ?, updated_at = ? +WHERE id = ? +`, attempts, next, truncateError(message), now, id) + return err +} + +func (s *FileStore) EnqueueRepair(peers []string) (int, error) { + if len(peers) == 0 { + return 0, nil + } + count := 0 + after := "" + for { + page, err := s.ListPage("", after, 1000) + if err != nil { + return count, err + } + for _, object := range page.Objects { + for _, peer := range peers { + if err := s.EnqueueReplication(peer, replicationOpPut, object.Key, defaultReplicationMaxJobs); err != nil { + return count, err + } + count++ + } + } + if page.NextAfter == "" { + return count, nil + } + after = page.NextAfter + } +} + +func (s *FileStore) ReplicationQueueStats() (ReplicationQueueStats, error) { + var stats ReplicationQueueStats + err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(attempts), 0) FROM replication_jobs`).Scan(&stats.PendingJobs, &stats.MaxAttempts) + if errors.Is(err, sql.ErrNoRows) { + return stats, nil + } + return stats, err +} + +func truncateError(message string) string { + if len(message) <= 500 { + return message + } + return message[:500] +} + +type ReplicationQueueStats struct { + PendingJobs int64 `json:"pending_jobs"` + MaxAttempts int `json:"max_attempts"` +} diff --git a/s3.go b/s3.go new file mode 100644 index 0000000..2bd19eb --- /dev/null +++ b/s3.go @@ -0,0 +1,938 @@ +package main + +import ( + "context" + "crypto/md5" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +const ( + maxXMLRequestBytes = 1 << 20 + maxBatchDeleteItems = 1000 +) + +var errPayloadHashMismatch = errors.New("payload hash mismatch") + +func (s *Server) s3Object(w http.ResponseWriter, r *http.Request) { + if s.s3BucketList(w, r) { + return + } + if s.s3BucketPost(w, r) { + return + } + + bucket, key, ok := s.s3BucketKey(r.URL.Path) + if !ok { + s3Error(w, http.StatusNotFound, "NoSuchBucket", "bucket not found") + return + } + if !s.bucketAllowed(bucket) { + s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed") + return + } + + storeKey := bucket + "/" + key + if err := ValidateKey(storeKey); err != nil { + s3Error(w, http.StatusBadRequest, "InvalidObjectName", err.Error()) + return + } + publicRead := (r.Method == http.MethodGet || r.Method == http.MethodHead) && s.publicReadAllowed(storeKey) + if !publicRead && !s.authorizedS3(r) { + s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid") + return + } + if !s.replicationAllowed(r) { + s3Error(w, http.StatusForbidden, "AccessDenied", "replication secret is invalid") + return + } + + switch r.Method { + case http.MethodPost: + if _, ok := r.URL.Query()["uploads"]; ok { + s.s3CreateMultipartUpload(w, r, storeKey) + } else if r.URL.Query().Get("uploadId") != "" { + s.s3CompleteMultipartUpload(w, r, storeKey) + } else if _, ok := r.URL.Query()["delete"]; ok { + s.s3DeleteObjects(w, r, bucket) + } else { + s3Error(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed") + } + case http.MethodPut: + if r.URL.Query().Get("uploadId") != "" && r.URL.Query().Get("partNumber") != "" { + s.s3UploadPart(w, r, storeKey) + } else if r.Header.Get("X-Amz-Copy-Source") != "" { + s.s3CopyObject(w, r, storeKey) + } else { + s.s3Put(w, r, storeKey) + } + case http.MethodGet: + s.s3Get(w, r, storeKey, publicRead) + case http.MethodHead: + s.s3Head(w, r, storeKey, publicRead) + case http.MethodDelete: + if r.URL.Query().Get("uploadId") != "" { + s.s3AbortMultipartUpload(w, r, storeKey) + } else { + s.s3Delete(w, r, storeKey) + } + default: + w.Header().Set("Allow", "PUT, GET, HEAD, DELETE") + s3Error(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed") + } +} + +func (s *Server) s3BucketPost(w http.ResponseWriter, r *http.Request) bool { + path := strings.Trim(r.URL.Path, "/") + if r.Method != http.MethodPost || path == "" || strings.Contains(path, "/") { + return false + } + if _, ok := r.URL.Query()["delete"]; !ok { + return false + } + if err := ValidateKey(path); err != nil { + s3Error(w, http.StatusBadRequest, "InvalidBucketName", err.Error()) + return true + } + if !s.bucketAllowed(path) { + s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed") + return true + } + if !s.authorizedS3(r) { + s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid") + return true + } + if !s.replicationAllowed(r) { + s3Error(w, http.StatusForbidden, "AccessDenied", "replication secret is invalid") + return true + } + s.s3DeleteObjects(w, r, path) + return true +} + +func (s *Server) s3CreateMultipartUpload(w http.ResponseWriter, r *http.Request, key string) { + uploadID, err := randomID() + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "upload id failed") + return + } + dir := s.multipartDir(uploadID) + if err := os.MkdirAll(dir, 0o700); err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "upload init failed") + return + } + meta := multipartMeta{Key: key, ContentType: r.Header.Get("Content-Type")} + data, _ := xml.Marshal(meta) + if err := os.WriteFile(filepath.Join(dir, "meta.xml"), data, 0o600); err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "upload init failed") + return + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _ = xml.NewEncoder(w).Encode(createMultipartUploadResult{UploadID: uploadID}) +} + +func (s *Server) s3UploadPart(w http.ResponseWriter, r *http.Request, key string) { + release, ok := s.acceptWrite(w, r, true) + if !ok { + return + } + defer release() + uploadID := r.URL.Query().Get("uploadId") + partNumber, err := strconv.Atoi(r.URL.Query().Get("partNumber")) + if err != nil || partNumber <= 0 || partNumber > 10_000 { + s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid part number") + return + } + meta, err := s.multipartMeta(uploadID) + if err != nil || meta.Key != key { + s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found") + return + } + dir := s.multipartDir(uploadID) + if !s.acceptMultipartPartSize(w, dir, r.ContentLength) { + return + } + partPath := filepath.Join(dir, fmt.Sprintf("part-%05d", partNumber)) + body, verifyPayload, ok := s.s3StreamingBody(w, r) + if !ok { + return + } + tmp, err := os.CreateTemp(dir, ".part-*") + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed") + return + } + hash := md5Hash() + _, copyErr := io.Copy(io.MultiWriter(tmp, hash), http.MaxBytesReader(w, body, s.cfg.MaxObjectSize)) + closeErr := tmp.Close() + if copyErr != nil || closeErr != nil { + _ = os.Remove(tmp.Name()) + if errors.Is(copyErr, errPayloadHashMismatch) { + s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch") + return + } + s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed") + return + } + if !verifyPayload() { + _ = os.Remove(tmp.Name()) + s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch") + return + } + if err := os.Rename(tmp.Name(), partPath); err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed") + return + } + etag := fmt.Sprintf("%x", hash.Sum(nil)) + w.Header().Set("ETag", quoteETag(etag)) + w.WriteHeader(http.StatusOK) +} + +func (s *Server) acceptMultipartPartSize(w http.ResponseWriter, dir string, incomingSize int64) bool { + if incomingSize < 0 { + s3Error(w, http.StatusLengthRequired, "MissingContentLength", "content length is required") + return false + } + if s.cfg.MaxObjectSize <= 0 { + return true + } + + existingSize := int64(0) + parts, _ := filepath.Glob(filepath.Join(dir, "part-*")) + for _, part := range parts { + info, err := os.Stat(part) + if err == nil && !info.IsDir() { + existingSize += info.Size() + } + } + + if existingSize+incomingSize > s.cfg.MaxObjectSize { + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "multipart upload exceeds object size limit") + return false + } + + return true +} + +func (s *Server) s3CompleteMultipartUpload(w http.ResponseWriter, r *http.Request, key string) { + uploadID := r.URL.Query().Get("uploadId") + meta, err := s.multipartMeta(uploadID) + if err != nil || meta.Key != key { + s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found") + return + } + var complete completeMultipartUpload + completeBody, ok := s.s3BoundedBody(w, r, maxXMLRequestBytes) + if !ok { + return + } + if err := xml.Unmarshal(completeBody, &complete); err != nil && len(completeBody) > 0 { + s3Error(w, http.StatusBadRequest, "MalformedXML", "invalid complete request") + return + } + parts := complete.Parts + if len(parts) == 0 { + entries, _ := filepath.Glob(filepath.Join(s.multipartDir(uploadID), "part-*")) + for _, entry := range entries { + base := filepath.Base(entry) + n, _ := strconv.Atoi(strings.TrimPrefix(base, "part-")) + parts = append(parts, completedPart{PartNumber: n}) + } + } + if len(parts) > 10_000 { + s3Error(w, http.StatusBadRequest, "InvalidRequest", "too many parts") + return + } + sortParts(parts) + readers := make([]io.Reader, 0, len(parts)) + files := make([]*os.File, 0, len(parts)) + defer func() { + for _, file := range files { + _ = file.Close() + } + }() + var totalSize int64 + for _, part := range parts { + if part.PartNumber <= 0 || part.PartNumber > 10_000 { + s3Error(w, http.StatusBadRequest, "InvalidPart", "invalid part number") + return + } + file, err := os.Open(filepath.Join(s.multipartDir(uploadID), fmt.Sprintf("part-%05d", part.PartNumber))) + if err != nil { + s3Error(w, http.StatusBadRequest, "InvalidPart", "part missing") + return + } + stat, err := file.Stat() + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "part stat failed") + return + } + totalSize += stat.Size() + if s.cfg.MaxObjectSize > 0 && totalSize > s.cfg.MaxObjectSize { + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large") + return + } + files = append(files, file) + readers = append(readers, file) + } + release, ok := s.acceptWriteSize(w, totalSize, true) + if !ok { + return + } + defer release() + info, err := s.store.Put(key, io.MultiReader(readers...), meta.ContentType) + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "complete failed") + return + } + _ = os.RemoveAll(s.multipartDir(uploadID)) + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _ = xml.NewEncoder(w).Encode(completeMultipartUploadResult{Key: strings.TrimPrefix(key, strings.SplitN(key, "/", 2)[0]+"/"), ETag: quoteETag(info.ETag)}) + if r.Header.Get("X-Magpie-Replication") != "1" { + s.enqueueReplicationPut(key) + } +} + +func (s *Server) s3AbortMultipartUpload(w http.ResponseWriter, r *http.Request, _ string) { + uploadID := r.URL.Query().Get("uploadId") + if !validUploadID(uploadID) { + s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found") + return + } + _ = os.RemoveAll(s.multipartDir(uploadID)) + w.WriteHeader(http.StatusNoContent) +} + +func (s *Server) s3CopyObject(w http.ResponseWriter, r *http.Request, destinationKey string) { + if !signedHeaderIncludes(r, "x-amz-copy-source") { + s3Error(w, http.StatusForbidden, "AccessDenied", "copy source header must be signed") + return + } + sourceKey, err := copySourceKey(r.Header.Get("X-Amz-Copy-Source")) + if err != nil { + s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid copy source") + return + } + sourceBucket, _, _ := strings.Cut(sourceKey, "/") + if !s.bucketAllowed(sourceBucket) { + s3Error(w, http.StatusForbidden, "AccessDenied", "copy source bucket is not allowed") + return + } + if len(s.cfg.S3Keys) > 0 { + key, ok := s.requestAccessKey(r) + if !ok || !key.can("read") || !key.can("write") { + s3Error(w, http.StatusForbidden, "AccessDenied", "copy requires read and write permission") + return + } + } + obj, err := s.store.Open(sourceKey) + if err != nil { + if errors.Is(err, ErrNotFound) { + s3Error(w, http.StatusNotFound, "NoSuchKey", "copy source not found") + return + } + s3Error(w, http.StatusInternalServerError, "InternalError", "copy source failed") + return + } + defer obj.Close() + if s.cfg.MaxObjectSize > 0 && obj.Info.Size > s.cfg.MaxObjectSize { + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large") + return + } + release, ok := s.acceptWriteSize(w, obj.Info.Size, true) + if !ok { + return + } + defer release() + contentType := obj.Info.ContentType + if override := r.Header.Get("Content-Type"); override != "" && strings.EqualFold(r.Header.Get("X-Amz-Metadata-Directive"), "REPLACE") { + contentType = override + } + info, err := s.store.Put(destinationKey, obj, contentType) + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "copy failed") + return + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _ = xml.NewEncoder(w).Encode(copyObjectResult{ETag: quoteETag(info.ETag), LastModified: info.ModTime.UTC().Format(time.RFC3339)}) + if r.Header.Get("X-Magpie-Replication") != "1" { + s.enqueueReplicationPut(destinationKey) + } +} + +func (s *Server) s3DeleteObjects(w http.ResponseWriter, r *http.Request, bucket string) { + var request deleteObjectsRequest + deleteBody, ok := s.s3BoundedBody(w, r, maxXMLRequestBytes) + if !ok { + return + } + if err := xml.Unmarshal(deleteBody, &request); err != nil { + s3Error(w, http.StatusBadRequest, "MalformedXML", "invalid delete request") + return + } + if len(request.Objects) > maxBatchDeleteItems { + s3Error(w, http.StatusBadRequest, "InvalidRequest", "too many objects in delete request") + return + } + result := deleteObjectsResult{Deleted: make([]deletedObject, 0, len(request.Objects))} + for _, object := range request.Objects { + key := strings.TrimPrefix(object.Key, "/") + storeKey := bucket + "/" + key + if err := ValidateKey(storeKey); err != nil { + result.Errors = append(result.Errors, deleteObjectError{Key: object.Key, Code: "InvalidObjectName", Message: err.Error()}) + continue + } + if err := s.store.Delete(storeKey); err != nil && !errors.Is(err, ErrNotFound) { + result.Errors = append(result.Errors, deleteObjectError{Key: object.Key, Code: "InternalError", Message: "delete failed"}) + continue + } + result.Deleted = append(result.Deleted, deletedObject{Key: object.Key}) + if r.Header.Get("X-Magpie-Replication") != "1" { + s.enqueueReplicationDelete(storeKey) + } + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _ = xml.NewEncoder(w).Encode(result) +} + +func (s *Server) s3BucketList(w http.ResponseWriter, r *http.Request) bool { + if r.Method != http.MethodGet || strings.Trim(r.URL.Path, "/") == "" || strings.Contains(strings.Trim(r.URL.Path, "/"), "/") { + return false + } + if r.URL.RawQuery == "" && r.URL.Query().Get("prefix") == "" && r.URL.Query().Get("list-type") == "" { + return false + } + + bucket := strings.Trim(r.URL.Path, "/") + if err := ValidateKey(bucket); err != nil { + s3Error(w, http.StatusBadRequest, "InvalidBucketName", err.Error()) + return true + } + if !s.bucketAllowed(bucket) { + s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed") + return true + } + if !s.authorizedS3(r) { + s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid") + return true + } + + prefix := r.URL.Query().Get("prefix") + storePrefix := bucket + "/" + if prefix != "" { + storePrefix = bucket + "/" + strings.TrimPrefix(prefix, "/") + } + maxKeys := parseMaxKeys(r.URL.Query().Get("max-keys")) + continuationToken := r.URL.Query().Get("continuation-token") + after, err := decodeContinuationToken(continuationToken) + if err != nil { + s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid continuation token") + return true + } + page, err := s.store.ListPage(storePrefix, after, maxKeys) + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "list failed") + return true + } + + contents := make([]s3ListContent, 0, len(page.Objects)) + for _, object := range page.Objects { + key := strings.TrimPrefix(object.Key, bucket+"/") + contents = append(contents, s3ListContent{ + Key: key, + LastModified: object.ModTime.UTC().Format(time.RFC3339), + ETag: quoteETag(object.ETag), + Size: object.Size, + StorageClass: "STANDARD", + }) + } + + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _ = xml.NewEncoder(w).Encode(s3ListBucketResult{ + Name: bucket, + Prefix: prefix, + ContinuationToken: continuationToken, + NextContinuationToken: encodeContinuationToken(page.NextAfter), + KeyCount: len(contents), + MaxKeys: maxKeys, + IsTruncated: page.NextAfter != "", + Contents: contents, + }) + return true +} + +func encodeContinuationToken(key string) string { + if key == "" { + return "" + } + return base64.RawURLEncoding.EncodeToString([]byte(key)) +} + +func decodeContinuationToken(token string) (string, error) { + if token == "" { + return "", nil + } + decoded, err := base64.RawURLEncoding.DecodeString(token) + if err != nil { + return "", err + } + key := string(decoded) + if err := ValidateKey(key); err != nil { + return "", err + } + return key, nil +} + +func copySourceKey(value string) (string, error) { + value = strings.TrimPrefix(value, "/") + if value == "" { + return "", fmt.Errorf("missing copy source") + } + if decoded, err := url.PathUnescape(value); err == nil { + value = decoded + } + if err := ValidateKey(value); err != nil { + return "", err + } + return value, nil +} + +func (s *Server) s3BucketKey(path string) (string, string, bool) { + path = strings.TrimPrefix(path, "/") + parts := strings.SplitN(path, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", false + } + if err := ValidateKey(parts[0]); err != nil { + return "", "", false + } + if err := ValidateKey(parts[1]); err != nil { + return "", "", false + } + return parts[0], parts[1], true +} + +func (s *Server) s3Put(w http.ResponseWriter, r *http.Request, key string) { + release, ok := s.acceptWrite(w, r, true) + if !ok { + return + } + defer release() + contentType := r.Header.Get("Content-Type") + body, verifyPayload, ok := s.s3StreamingBody(w, r) + if !ok { + return + } + store, ok := s.store.(*FileStore) + if !ok { + s3Error(w, http.StatusInternalServerError, "InternalError", "store failed") + return + } + info, err := store.PutVerified(key, http.MaxBytesReader(w, body, s.cfg.MaxObjectSize), contentType, func() error { + if !verifyPayload() { + return errPayloadHashMismatch + } + return nil + }) + if err != nil { + if maxBytesError(err) { + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large") + return + } + if errors.Is(err, errPayloadHashMismatch) { + s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch") + return + } + s3Error(w, http.StatusInternalServerError, "InternalError", "store failed") + return + } + w.Header().Set("ETag", quoteETag(info.ETag)) + w.WriteHeader(http.StatusOK) + if r.Header.Get("X-Magpie-Replication") != "1" { + s.enqueueReplicationPut(key) + } +} + +func (s *Server) s3Get(w http.ResponseWriter, r *http.Request, key string, publicRead bool) { + obj, err := s.store.Open(key) + if err != nil { + if errors.Is(err, ErrNotFound) { + s3Error(w, http.StatusNotFound, "NoSuchKey", "object not found") + return + } + s3Error(w, http.StatusInternalServerError, "InternalError", "read failed") + return + } + defer obj.Close() + + s3ObjectHeaders(w, obj.Info) + if publicRead { + hardenPublicRead(w, obj.Info.ContentType) + } else if disposition := r.URL.Query().Get("response-content-disposition"); disposition != "" { + w.Header().Set("Content-Disposition", disposition) + } + http.ServeContent(w, r, obj.Info.Key, obj.Info.ModTime, obj) +} + +func (s *Server) s3Head(w http.ResponseWriter, _ *http.Request, key string, publicRead bool) { + info, err := s.store.Stat(key) + if err != nil { + if errors.Is(err, ErrNotFound) { + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(http.StatusInternalServerError) + return + } + s3ObjectHeaders(w, info) + if publicRead { + hardenPublicRead(w, info.ContentType) + } + w.WriteHeader(http.StatusOK) +} + +// hardenPublicRead applies defense-in-depth headers to unauthenticated public +// reads. Objects carry an uploader-supplied Content-Type, so a public prefix +// could otherwise serve active content (HTML, SVG) that executes script in +// Magpie's own origin. The CSP sandbox disables scripting even when a browser +// navigates directly to such an object, and any content type that is not inert +// inline media is forced to download rather than render. The attacker-influenced +// response-content-disposition override is intentionally not honored here. +func hardenPublicRead(w http.ResponseWriter, contentType string) { + w.Header().Set("Content-Security-Policy", "default-src 'none'; sandbox") + if !inlineSafeContentType(contentType) { + w.Header().Set("Content-Disposition", "attachment") + } +} + +// inlineSafeContentType reports whether a content type is inert raster/media +// that browsers render without executing script. SVG is excluded because it can +// carry script when loaded as a top-level document. +func inlineSafeContentType(contentType string) bool { + mediaType := contentType + if i := strings.IndexByte(mediaType, ';'); i >= 0 { + mediaType = mediaType[:i] + } + mediaType = strings.ToLower(strings.TrimSpace(mediaType)) + if mediaType == "image/svg+xml" || mediaType == "image/svg" { + return false + } + return strings.HasPrefix(mediaType, "image/") || + strings.HasPrefix(mediaType, "audio/") || + strings.HasPrefix(mediaType, "video/") +} + +func (s *Server) s3Delete(w http.ResponseWriter, r *http.Request, key string) { + if err := s.store.Delete(key); err != nil && !errors.Is(err, ErrNotFound) { + s3Error(w, http.StatusInternalServerError, "InternalError", "delete failed") + return + } + w.WriteHeader(http.StatusNoContent) + if r.Header.Get("X-Magpie-Replication") != "1" { + s.enqueueReplicationDelete(key) + } +} + +func s3ObjectHeaders(w http.ResponseWriter, info ObjectInfo) { + contentType := info.ContentType + if contentType == "" { + contentType = "application/octet-stream" + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size)) + w.Header().Set("ETag", quoteETag(info.ETag)) + w.Header().Set("Last-Modified", info.ModTime.UTC().Format(http.TimeFormat)) + w.Header().Set("Accept-Ranges", "bytes") + w.Header().Set("X-Content-Type-Options", "nosniff") +} + +func quoteETag(etag string) string { + if etag == "" { + return "\"\"" + } + if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") { + return etag + } + return "\"" + etag + "\"" +} + +type s3ErrorBody struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` +} + +func s3Error(w http.ResponseWriter, status int, code string, message string) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(status) + _ = xml.NewEncoder(w).Encode(s3ErrorBody{Code: code, Message: message}) +} + +func parseMaxKeys(value string) int { + if value == "" { + return 1000 + } + parsed, err := strconv.Atoi(value) + if err != nil || parsed <= 0 { + return 1000 + } + if parsed > 1000 { + return 1000 + } + return parsed +} + +func randomID() (string, error) { + buf := make([]byte, 18) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(buf), nil +} + +func md5Hash() hashHash { + return md5.New() +} + +type hashHash interface { + io.Writer + Sum([]byte) []byte +} + +func (s *Server) multipartDir(uploadID string) string { + if !validUploadID(uploadID) { + return filepath.Join(s.store.(*FileStore).root, ".multipart", "invalid") + } + return filepath.Join(s.store.(*FileStore).root, ".multipart", uploadID) +} + +func validUploadID(uploadID string) bool { + if uploadID == "" || len(uploadID) > 128 { + return false + } + for _, ch := range uploadID { + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '-' || ch == '_' { + continue + } + return false + } + return true +} + +func (s *Server) startMultipartCleanup(ctx context.Context) { + store, ok := s.store.(*FileStore) + if !ok || s.cfg.MultipartMaxAge <= 0 { + return + } + if err := store.CleanupMultipart(s.cfg.MultipartMaxAge); err != nil { + log.Printf("multipart cleanup error=%v", err) + } + go func() { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := store.CleanupMultipart(s.cfg.MultipartMaxAge); err != nil { + log.Printf("multipart cleanup error=%v", err) + } + } + } + }() +} + +func (s *Server) s3BoundedBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, bool) { + body, verifyPayload, ok := s.s3StreamingBody(w, r) + if !ok { + return nil, false + } + data, err := io.ReadAll(http.MaxBytesReader(w, body, limit)) + if err != nil { + if errors.Is(err, errPayloadHashMismatch) { + s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch") + return nil, false + } + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "request body too large") + return nil, false + } + if !verifyPayload() { + s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch") + return nil, false + } + return data, true +} + +func (s *Server) s3StreamingBody(w http.ResponseWriter, r *http.Request) (io.ReadCloser, func() bool, bool) { + payloadHash := r.Header.Get("X-Amz-Content-Sha256") + if payloadHash == "" || payloadHash == "UNSIGNED-PAYLOAD" { + return r.Body, func() bool { return true }, true + } + if len(payloadHash) != sha256.Size*2 { + s3Error(w, http.StatusBadRequest, "InvalidDigest", "unsupported payload hash") + return nil, nil, false + } + if _, err := hex.DecodeString(payloadHash); err != nil { + s3Error(w, http.StatusBadRequest, "InvalidDigest", "invalid payload hash") + return nil, nil, false + } + hash := sha256.New() + verified := false + return hashVerifyingReadCloser{reader: r.Body, hash: hash, expected: strings.ToLower(payloadHash)}, func() bool { + if verified { + return true + } + verified = hex.EncodeToString(hash.Sum(nil)) == strings.ToLower(payloadHash) + return verified + }, true +} + +func signedHeaderIncludes(r *http.Request, header string) bool { + signedHeaders := r.URL.Query().Get("X-Amz-SignedHeaders") + if signedHeaders == "" { + auth := r.Header.Get("Authorization") + if strings.HasPrefix(auth, awsAlgorithm+" ") { + signedHeaders = parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" "))["SignedHeaders"] + } + } + for _, signed := range strings.Split(signedHeaders, ";") { + if strings.EqualFold(strings.TrimSpace(signed), header) { + return true + } + } + return false +} + +type hashVerifyingReadCloser struct { + reader io.ReadCloser + hash hashHash + expected string +} + +func (r hashVerifyingReadCloser) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if n > 0 { + _, _ = r.hash.Write(p[:n]) + } + if errors.Is(err, io.EOF) && hex.EncodeToString(r.hash.Sum(nil)) != r.expected { + return n, errPayloadHashMismatch + } + return n, err +} + +func (r hashVerifyingReadCloser) Close() error { + return r.reader.Close() +} + +func (s *Server) multipartMeta(uploadID string) (multipartMeta, error) { + data, err := os.ReadFile(filepath.Join(s.multipartDir(uploadID), "meta.xml")) + if err != nil { + return multipartMeta{}, err + } + var meta multipartMeta + if err := xml.Unmarshal(data, &meta); err != nil { + return multipartMeta{}, err + } + return meta, nil +} + +func sortParts(parts []completedPart) { + sort.Slice(parts, func(i, j int) bool { return parts[i].PartNumber < parts[j].PartNumber }) +} + +type s3ListBucketResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Xmlns string `xml:"xmlns,attr,omitempty"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + ContinuationToken string `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + KeyCount int `xml:"KeyCount"` + MaxKeys int `xml:"MaxKeys"` + IsTruncated bool `xml:"IsTruncated"` + Contents []s3ListContent `xml:"Contents"` +} + +type s3ListContent struct { + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` + StorageClass string `xml:"StorageClass"` +} + +type multipartMeta struct { + Key string `xml:"Key"` + ContentType string `xml:"ContentType"` +} + +type createMultipartUploadResult struct { + XMLName xml.Name `xml:"InitiateMultipartUploadResult"` + UploadID string `xml:"UploadId"` +} + +type completeMultipartUpload struct { + Parts []completedPart `xml:"Part"` +} + +type completedPart struct { + PartNumber int `xml:"PartNumber"` + ETag string `xml:"ETag"` +} + +type completeMultipartUploadResult struct { + XMLName xml.Name `xml:"CompleteMultipartUploadResult"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` +} + +type copyObjectResult struct { + XMLName xml.Name `xml:"CopyObjectResult"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` +} + +type deleteObjectsRequest struct { + Objects []deleteObjectRequest `xml:"Object"` +} + +type deleteObjectRequest struct { + Key string `xml:"Key"` +} + +type deleteObjectsResult struct { + XMLName xml.Name `xml:"DeleteResult"` + Deleted []deletedObject `xml:"Deleted"` + Errors []deleteObjectError `xml:"Error,omitempty"` +} + +type deletedObject struct { + Key string `xml:"Key"` +} + +type deleteObjectError struct { + Key string `xml:"Key"` + Code string `xml:"Code"` + Message string `xml:"Message"` +} diff --git a/s3auth.go b/s3auth.go new file mode 100644 index 0000000..c8fd6ee --- /dev/null +++ b/s3auth.go @@ -0,0 +1,312 @@ +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +const awsAlgorithm = "AWS4-HMAC-SHA256" + +func (s *Server) authorizedS3(r *http.Request) bool { + if len(s.cfg.S3Keys) == 0 { + return s.authorized(r) + } + + if r.URL.Query().Get("X-Amz-Algorithm") != "" { + return s.authorizedS3Presigned(r) + } + + return s.authorizedS3Header(r) +} + +func (s *Server) authorizedS3Presigned(r *http.Request) bool { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + return false + } + query := r.URL.Query() + if query.Get("X-Amz-Algorithm") != awsAlgorithm { + return false + } + if query.Get("X-Amz-Signature") == "" { + return false + } + + credential := query.Get("X-Amz-Credential") + accessKey, scope, ok := splitCredential(credential) + key, ok := s.accessKey(accessKey) + if !ok || !key.can(permissionForMethod(r.Method)) { + return false + } + + amzDate := query.Get("X-Amz-Date") + if !s.presignedTimeValid(amzDate, query.Get("X-Amz-Expires")) { + return false + } + if !s.scopeAllowed(scope, amzDate) { + return false + } + + signedHeaders := query.Get("X-Amz-SignedHeaders") + canonicalRequest := canonicalRequest(r, signedHeaders, "UNSIGNED-PAYLOAD", true) + expected := signature(key.Secret, amzDate, scope, canonicalRequest) + return hmac.Equal([]byte(expected), []byte(query.Get("X-Amz-Signature"))) +} + +func (s *Server) authorizedS3Header(r *http.Request) bool { + return s.authorizedS3HeaderPermission(r, permissionForMethod(r.Method)) +} + +func (s *Server) authorizedS3HeaderPermission(r *http.Request, permission string) bool { + auth := r.Header.Get("Authorization") + if !strings.HasPrefix(auth, awsAlgorithm+" ") { + return false + } + parts := parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" ")) + credential := parts["Credential"] + signedHeaders := parts["SignedHeaders"] + providedSignature := parts["Signature"] + accessKey, scope, ok := splitCredential(credential) + key, keyOK := s.accessKey(accessKey) + if !ok || !keyOK || !key.can(permission) || signedHeaders == "" || providedSignature == "" { + return false + } + + amzDate := r.Header.Get("X-Amz-Date") + if !headerTimeValid(amzDate) { + return false + } + if !s.scopeAllowed(scope, amzDate) { + return false + } + payloadHash := r.Header.Get("X-Amz-Content-Sha256") + if payloadHash == "" { + payloadHash = "UNSIGNED-PAYLOAD" + } + if permission == "write" && r.Header.Get("X-Magpie-Replication") != "1" && payloadHash == "UNSIGNED-PAYLOAD" && !s.cfg.AllowUnsignedLocal { + return false + } + + canonicalRequest := canonicalRequest(r, signedHeaders, payloadHash, false) + expected := signature(key.Secret, amzDate, scope, canonicalRequest) + return hmac.Equal([]byte(expected), []byte(providedSignature)) +} + +func (s *Server) accessKey(id string) (AccessKey, bool) { + for _, key := range s.cfg.S3Keys { + if subtle.ConstantTimeCompare([]byte(key.ID), []byte(id)) == 1 { + return key, true + } + } + return AccessKey{}, false +} + +func (s *Server) requestAccessKey(r *http.Request) (AccessKey, bool) { + credential := r.URL.Query().Get("X-Amz-Credential") + if credential == "" { + auth := r.Header.Get("Authorization") + if !strings.HasPrefix(auth, awsAlgorithm+" ") { + return AccessKey{}, false + } + credential = parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" "))["Credential"] + } + accessKey, _, ok := splitCredential(credential) + if !ok { + return AccessKey{}, false + } + return s.accessKey(accessKey) +} + +func (key AccessKey) can(permission string) bool { + return key.Permissions["admin"] || key.Permissions[permission] +} + +func permissionForMethod(method string) string { + switch method { + case http.MethodGet, http.MethodHead: + return "read" + case http.MethodPut, http.MethodPost, http.MethodDelete: + return "write" + default: + return "admin" + } +} + +func signature(secret, amzDate, scope, canonicalRequest string) string { + date := strings.Split(scope, "/")[0] + stringToSign := strings.Join([]string{ + awsAlgorithm, + amzDate, + scope, + hexSHA256(canonicalRequest), + }, "\n") + key := signingKey(secret, date, scope) + sig := hmacSHA256(key, []byte(stringToSign)) + return hex.EncodeToString(sig) +} + +func signingKey(secret, date, scope string) []byte { + parts := strings.Split(scope, "/") + region := "auto" + service := "s3" + if len(parts) >= 3 { + region = parts[1] + service = parts[2] + } + kDate := hmacSHA256([]byte("AWS4"+secret), []byte(date)) + kRegion := hmacSHA256(kDate, []byte(region)) + kService := hmacSHA256(kRegion, []byte(service)) + return hmacSHA256(kService, []byte("aws4_request")) +} + +func (s *Server) scopeAllowed(scope string, amzDate string) bool { + parts := strings.Split(scope, "/") + if len(parts) != 4 { + return false + } + if len(amzDate) < 8 || parts[0] != amzDate[:8] { + return false + } + if parts[1] != s.cfg.S3Region || parts[2] != "s3" || parts[3] != "aws4_request" { + return false + } + return true +} + +func canonicalRequest(r *http.Request, signedHeaders, payloadHash string, presigned bool) string { + return strings.Join([]string{ + r.Method, + canonicalURI(r.URL.Path), + canonicalQuery(r.URL.Query(), presigned), + canonicalHeaders(r, signedHeaders), + signedHeaders, + payloadHash, + }, "\n") +} + +func canonicalURI(path string) string { + if path == "" { + return "/" + } + segments := strings.Split(path, "/") + for i, segment := range segments { + segments[i] = awsEscape(segment) + } + return strings.Join(segments, "/") +} + +func canonicalQuery(values url.Values, presigned bool) string { + type pair struct{ key, value string } + pairs := make([]pair, 0) + for key, vals := range values { + if presigned && key == "X-Amz-Signature" { + continue + } + for _, value := range vals { + pairs = append(pairs, pair{awsEscape(key), awsEscape(value)}) + } + } + sort.Slice(pairs, func(i, j int) bool { + if pairs[i].key == pairs[j].key { + return pairs[i].value < pairs[j].value + } + return pairs[i].key < pairs[j].key + }) + parts := make([]string, len(pairs)) + for i, pair := range pairs { + parts[i] = pair.key + "=" + pair.value + } + return strings.Join(parts, "&") +} + +func canonicalHeaders(r *http.Request, signedHeaders string) string { + headers := strings.Split(signedHeaders, ";") + var b strings.Builder + for _, header := range headers { + name := strings.ToLower(strings.TrimSpace(header)) + value := "" + if name == "host" { + value = r.Host + } else { + value = strings.Join(r.Header.Values(name), ",") + } + b.WriteString(name) + b.WriteString(":") + b.WriteString(compressSpaces(value)) + b.WriteString("\n") + } + return b.String() +} + +func splitCredential(credential string) (string, string, bool) { + parts := strings.SplitN(credential, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", false + } + scopeParts := strings.Split(parts[1], "/") + if len(scopeParts) != 4 || scopeParts[3] != "aws4_request" { + return "", "", false + } + return parts[0], parts[1], true +} + +func parseAuthParams(value string) map[string]string { + params := map[string]string{} + for _, part := range strings.Split(value, ",") { + keyValue := strings.SplitN(strings.TrimSpace(part), "=", 2) + if len(keyValue) == 2 { + params[keyValue[0]] = keyValue[1] + } + } + return params +} + +func (s *Server) presignedTimeValid(amzDate, expiresRaw string) bool { + created, err := time.Parse("20060102T150405Z", amzDate) + if err != nil { + return false + } + expires, err := time.ParseDuration(expiresRaw + "s") + if err != nil || expires <= 0 || expires > s.cfg.PresignedMaxExpiry { + return false + } + now := time.Now().UTC() + return now.After(created.Add(-15*time.Minute)) && now.Before(created.Add(expires)) +} + +func headerTimeValid(amzDate string) bool { + created, err := time.Parse("20060102T150405Z", amzDate) + if err != nil { + return false + } + now := time.Now().UTC() + return now.After(created.Add(-15*time.Minute)) && now.Before(created.Add(15*time.Minute)) +} + +func awsEscape(value string) string { + escaped := url.QueryEscape(value) + escaped = strings.ReplaceAll(escaped, "+", "%20") + escaped = strings.ReplaceAll(escaped, "%7E", "~") + return escaped +} + +func compressSpaces(value string) string { + return strings.Join(strings.Fields(strings.TrimSpace(value)), " ") +} + +func hexSHA256(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +func hmacSHA256(key []byte, value []byte) []byte { + mac := hmac.New(sha256.New, key) + _, _ = mac.Write(value) + return mac.Sum(nil) +} diff --git a/server.go b/server.go new file mode 100644 index 0000000..896b808 --- /dev/null +++ b/server.go @@ -0,0 +1,530 @@ +package main + +import ( + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "runtime/debug" + "strings" + "sync" + "sync/atomic" + "time" +) + +const objectPrefix = "/objects/" + +type ObjectStore interface { + Put(key string, body io.Reader, contentType string) (ObjectInfo, error) + Open(key string) (*ObjectRead, error) + Stat(key string) (ObjectInfo, error) + Delete(key string) error + DiskStatus() (DiskStatus, error) + List(prefix string, maxKeys int) ([]ObjectInfo, error) + ListPage(prefix string, after string, maxKeys int) (ListPage, error) +} + +type Server struct { + store ObjectStore + cfg Config + rateLimiter *RateLimiter + metrics *Metrics + writeMu sync.Mutex + reservedBytes uint64 +} + +func NewServer(store ObjectStore, cfg Config) http.Handler { + return NewServerInstance(store, cfg).Handler() +} + +func NewServerInstance(store ObjectStore, cfg Config) *Server { + cfg = cfg.withDefaults() + return &Server{store: store, cfg: cfg, rateLimiter: NewRateLimiter(cfg.RateLimitPerMinute), metrics: &Metrics{}} +} + +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/health", s.health) + mux.HandleFunc("/metrics", s.metricsHandler) + mux.HandleFunc(objectPrefix, s.object) + mux.HandleFunc("/", s.s3Object) + return s.recoverPanics(s.logRequests(s.rateLimit(mux))) +} + +func (s *Server) recoverPanics(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if recovered := recover(); recovered != nil { + log.Printf("panic method=%s path=%s remote=%s error=%v stack=%s", r.Method, safeLogPath(r.URL.Path), sanitizeLogValue(r.RemoteAddr), recovered, debug.Stack()) + s3Error(w, http.StatusInternalServerError, "InternalError", "internal server error") + } + }() + next.ServeHTTP(w, r) + }) +} + +func (s *Server) metricsHandler(w http.ResponseWriter, r *http.Request) { + if !s.opsAuthorized(r) { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"}) + return + } + w.Header().Set("Content-Type", "text/plain; version=0.0.4") + fmt.Fprintf(w, "magpie_requests_total %d\n", atomic.LoadUint64(&s.metrics.Requests)) + fmt.Fprintf(w, "magpie_errors_total %d\n", atomic.LoadUint64(&s.metrics.Errors)) + fmt.Fprintf(w, "magpie_response_bytes_total %d\n", atomic.LoadUint64(&s.metrics.ResponseBytes)) +} + +func (s *Server) opsAuthorized(r *http.Request) bool { + return s.authorized(r) || s.authorizedS3HeaderPermission(r, "admin") +} + +func (cfg Config) withDefaults() Config { + if len(cfg.S3Keys) == 0 && cfg.S3AccessKeyID != "" && cfg.S3SecretAccessKey != "" { + cfg.S3Keys = []AccessKey{{ID: cfg.S3AccessKeyID, Secret: cfg.S3SecretAccessKey, Permissions: permissions("read,write,admin")}} + } + if cfg.S3Region == "" { + cfg.S3Region = "auto" + } + if cfg.MaxObjectSize <= 0 { + cfg.MaxObjectSize = defaultMaxObjectSize + } + if cfg.PresignedMaxExpiry <= 0 { + cfg.PresignedMaxExpiry = defaultPresignedMaxExpiry + } + if cfg.MultipartMaxAge <= 0 { + cfg.MultipartMaxAge = defaultMultipartMaxAge + } + if cfg.PublicPrefixes == nil { + cfg.PublicPrefixes = map[string]bool{} + } + if cfg.ReplicationMaxJobs <= 0 { + cfg.ReplicationMaxJobs = defaultReplicationMaxJobs + } + if cfg.ReplicationMaxAttempts <= 0 { + cfg.ReplicationMaxAttempts = defaultReplicationMaxAttempts + } + if cfg.MaxRestoreBytes <= 0 { + cfg.MaxRestoreBytes = defaultMaxRestoreBytes + } + return cfg +} + +func (s *Server) health(w http.ResponseWriter, r *http.Request) { + if !s.opsAuthorized(r) { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"}) + return + } + disk, err := s.store.DiskStatus() + if err != nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"status": "error", "error": err.Error()}) + return + } + status := "ok" + code := http.StatusOK + if disk.FreeBytes < s.cfg.MinFreeBytes { + status = "low_disk" + code = http.StatusServiceUnavailable + } + writeJSON(w, code, map[string]any{"status": status, "disk": disk}) +} + +func (s *Server) object(w http.ResponseWriter, r *http.Request) { + if !s.authorized(r) { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"}) + return + } + + key := strings.TrimPrefix(r.URL.Path, objectPrefix) + if err := ValidateKey(key); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) + return + } + bucket, _, _ := strings.Cut(key, "/") + if !s.bucketAllowed(bucket) { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "bucket is not allowed"}) + return + } + + switch r.Method { + case http.MethodPut: + s.put(w, r, key) + case http.MethodGet: + s.get(w, r, key) + case http.MethodHead: + s.head(w, r, key) + case http.MethodDelete: + s.delete(w, key) + default: + w.Header().Set("Allow", "PUT, GET, HEAD, DELETE") + writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"}) + } +} + +func (s *Server) put(w http.ResponseWriter, r *http.Request, key string) { + release, ok := s.acceptWrite(w, r, false) + if !ok { + return + } + defer release() + contentType := r.Header.Get("Content-Type") + info, err := s.store.Put(key, http.MaxBytesReader(w, r.Body, s.cfg.MaxObjectSize), contentType) + if err != nil { + if maxBytesError(err) { + writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "object too large"}) + return + } + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "store failed"}) + return + } + writeJSON(w, http.StatusCreated, info) +} + +func (s *Server) acceptWrite(w http.ResponseWriter, r *http.Request, s3 bool) (func(), bool) { + size := r.ContentLength + if r.ContentLength < 0 { + if s.cfg.MaxObjectSize <= 0 { + disk, err := s.store.DiskStatus() + if err != nil || disk.FreeBytes < s.cfg.MinFreeBytes { + if s3 { + s3Error(w, http.StatusInsufficientStorage, "InsufficientStorage", "insufficient free disk space for unknown length write") + } else { + writeJSON(w, http.StatusInsufficientStorage, map[string]string{"error": "insufficient free disk space for unknown length write"}) + } + return nil, false + } + return func() {}, true + } + size = s.cfg.MaxObjectSize + } + if s.cfg.MaxObjectSize > 0 && r.ContentLength > s.cfg.MaxObjectSize { + if s3 { + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large") + } else { + writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "object too large"}) + } + return nil, false + } + return s.acceptWriteSize(w, size, s3) +} + +func (s *Server) acceptWriteSize(w http.ResponseWriter, size int64, s3 bool) (func(), bool) { + if size < 0 { + return func() {}, true + } + disk, err := s.store.DiskStatus() + if err != nil { + if s3 { + s3Error(w, http.StatusInternalServerError, "InternalError", "disk status unavailable") + } else { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "disk status unavailable"}) + } + return nil, false + } + needed := uint64(size) + s.writeMu.Lock() + defer s.writeMu.Unlock() + reserved := s.reservedBytes + available := uint64(0) + if disk.FreeBytes > reserved { + available = disk.FreeBytes - reserved + } + if available < s.cfg.MinFreeBytes || needed > available || available-needed < s.cfg.MinFreeBytes { + if s3 { + s3Error(w, http.StatusInsufficientStorage, "InsufficientStorage", "insufficient free disk space") + } else { + writeJSON(w, http.StatusInsufficientStorage, map[string]string{"error": "insufficient free disk space"}) + } + return nil, false + } + s.reservedBytes += needed + released := false + return func() { + s.writeMu.Lock() + defer s.writeMu.Unlock() + if released { + return + } + released = true + if s.reservedBytes >= needed { + s.reservedBytes -= needed + } else { + s.reservedBytes = 0 + } + }, true +} + +func maxBytesError(err error) bool { + var maxErr *http.MaxBytesError + return errors.As(err, &maxErr) +} + +func (s *Server) get(w http.ResponseWriter, r *http.Request, key string) { + obj, err := s.store.Open(key) + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, ErrNotFound) { + status = http.StatusNotFound + } + writeJSON(w, status, map[string]string{"error": "not found"}) + return + } + defer obj.Close() + + contentType := obj.Info.ContentType + if contentType == "" { + contentType = "application/octet-stream" + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("X-Object-Key", obj.Info.Key) + w.Header().Set("X-Object-Size", fmt.Sprintf("%d", obj.Info.Size)) + w.Header().Set("X-Content-Type-Options", "nosniff") + http.ServeContent(w, r, obj.Info.Key, obj.Info.ModTime, obj) +} + +func (s *Server) head(w http.ResponseWriter, r *http.Request, key string) { + info, err := s.store.Stat(key) + if err != nil { + if errors.Is(err, ErrNotFound) { + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(http.StatusInternalServerError) + return + } + + contentType := info.ContentType + if contentType == "" { + contentType = "application/octet-stream" + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size)) + w.Header().Set("X-Object-Key", info.Key) + w.Header().Set("X-Object-Size", fmt.Sprintf("%d", info.Size)) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + _ = r.Body.Close() +} + +func (s *Server) delete(w http.ResponseWriter, key string) { + err := s.store.Delete(key) + if err != nil && !errors.Is(err, ErrNotFound) { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "delete failed"}) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *Server) authorized(r *http.Request) bool { + const prefix = "Bearer " + value := r.Header.Get("Authorization") + if !strings.HasPrefix(value, prefix) { + return false + } + provided := strings.TrimPrefix(value, prefix) + return s.cfg.AuthToken != "" && subtle.ConstantTimeCompare([]byte(provided), []byte(s.cfg.AuthToken)) == 1 +} + +func (s *Server) bucketAllowed(bucket string) bool { + return len(s.cfg.AllowedBuckets) == 0 || s.cfg.AllowedBuckets[bucket] +} + +func (s *Server) publicReadAllowed(key string) bool { + for prefix := range s.cfg.PublicPrefixes { + prefix = strings.TrimSuffix(prefix, "/") + if !strings.Contains(prefix, "/") { + continue + } + if key == prefix || strings.HasPrefix(key, prefix+"/") { + return true + } + } + return false +} + +func (s *Server) replicationAllowed(r *http.Request) bool { + if r.Header.Get("X-Magpie-Replication") != "1" { + return true + } + if s.cfg.ReplicationSecret == "" { + return false + } + return subtle.ConstantTimeCompare([]byte(r.Header.Get("X-Magpie-Replication-Secret")), []byte(s.cfg.ReplicationSecret)) == 1 +} + +func writeJSON(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(body) +} + +type statusWriter struct { + http.ResponseWriter + status int + bytes int +} + +func (w *statusWriter) WriteHeader(status int) { + w.status = status + w.ResponseWriter.WriteHeader(status) +} + +func (w *statusWriter) Write(b []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + n, err := w.ResponseWriter.Write(b) + w.bytes += n + return n, err +} + +func (s *Server) logRequests(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + recorder := &statusWriter{ResponseWriter: w} + next.ServeHTTP(recorder, r) + status := recorder.status + if status == 0 { + status = http.StatusOK + } + atomic.AddUint64(&s.metrics.Requests, 1) + if recorder.bytes > 0 { + atomic.AddUint64(&s.metrics.ResponseBytes, uint64(recorder.bytes)) + } + if status >= 400 { + atomic.AddUint64(&s.metrics.Errors, 1) + } + log.Printf( + "method=%s path=%s status=%d bytes=%d duration_ms=%d remote=%s", + r.Method, + safeLogPath(r.URL.Path), + status, + recorder.bytes, + time.Since(start).Milliseconds(), + sanitizeLogValue(r.RemoteAddr), + ) + }) +} + +func (s *Server) rateLimit(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !s.rateLimiter.Allow(s.rateLimitKey(r)) { + writeJSON(w, http.StatusTooManyRequests, map[string]string{"error": "rate limited"}) + return + } + next.ServeHTTP(w, r) + }) +} + +func (s *Server) rateLimitKey(r *http.Request) string { + if s.cfg.TrustProxyHeaders { + if forwarded := r.Header.Get("X-Forwarded-For"); forwarded != "" { + candidate := strings.TrimSpace(strings.Split(forwarded, ",")[0]) + if net.ParseIP(candidate) != nil { + return candidate + } + } + if realIP := strings.TrimSpace(r.Header.Get("X-Real-IP")); net.ParseIP(realIP) != nil { + return realIP + } + } + return r.RemoteAddr +} + +type Metrics struct { + Requests uint64 + Errors uint64 + ResponseBytes uint64 +} + +type RateLimiter struct { + limit int + mu sync.Mutex + buckets map[string]rateBucket +} + +type rateBucket struct { + minute int64 + count int +} + +func NewRateLimiter(limit int) *RateLimiter { + return &RateLimiter{limit: limit, buckets: map[string]rateBucket{}} +} + +func (r *RateLimiter) Allow(remote string) bool { + if r.limit <= 0 { + return true + } + ip, _, err := net.SplitHostPort(remote) + if err != nil { + ip = remote + } + minute := time.Now().Unix() / 60 + r.mu.Lock() + defer r.mu.Unlock() + for candidate, bucket := range r.buckets { + if bucket.minute < minute-1 { + delete(r.buckets, candidate) + } + } + bucket := r.buckets[ip] + if bucket.minute != minute { + bucket = rateBucket{minute: minute} + } + bucket.count++ + r.buckets[ip] = bucket + return bucket.count <= r.limit +} + +func safeLogPath(path string) string { + if len(path) <= 256 { + return sanitizeLogValue(path) + } + return sanitizeLogValue(path[:256]) + "..." +} + +func sanitizeLogValue(value string) string { + return strings.Map(func(r rune) rune { + if r < 0x20 || r == 0x7f { + return ' ' + } + return r + }, value) +} + +type ObjectInfo struct { + Key string `json:"key"` + Size int64 `json:"size"` + ContentType string `json:"content_type"` + ModTime time.Time `json:"mod_time"` + ETag string `json:"etag"` +} + +type DiskStatus struct { + TotalBytes uint64 `json:"total_bytes"` + FreeBytes uint64 `json:"free_bytes"` + UsedBytes uint64 `json:"used_bytes"` +} + +type StoreStats struct { + ObjectCount int64 `json:"object_count"` + TotalBytes int64 `json:"total_bytes"` + Disk DiskStatus `json:"disk"` + ReplicationQueue ReplicationQueueStats `json:"replication_queue"` +} + +type ScrubReport struct { + CheckedObjects int `json:"checked_objects"` + MissingKeys []string `json:"missing_keys"` + CorruptKeys []string `json:"corrupt_keys"` + OrphanKeys []string `json:"orphan_keys"` +} + +type ListPage struct { + Objects []ObjectInfo + NextAfter string +} diff --git a/server_test.go b/server_test.go new file mode 100644 index 0000000..ee685cb --- /dev/null +++ b/server_test.go @@ -0,0 +1,1737 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +const testAccessKey = "test-access" +const testSecretKey = "test-secret" + +func TestObjectLifecycle(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret"})) + defer srv.Close() + + putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/objects/attachments/test.txt", strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + putReq.Header.Set("Authorization", "Bearer secret") + putReq.Header.Set("Content-Type", "text/plain") + putResp, err := http.DefaultClient.Do(putReq) + if err != nil { + t.Fatal(err) + } + if putResp.StatusCode != http.StatusCreated { + t.Fatalf("PUT status = %d", putResp.StatusCode) + } + _ = putResp.Body.Close() + + getReq, err := http.NewRequest(http.MethodGet, srv.URL+"/objects/attachments/test.txt", nil) + if err != nil { + t.Fatal(err) + } + getReq.Header.Set("Authorization", "Bearer secret") + getResp, err := http.DefaultClient.Do(getReq) + if err != nil { + t.Fatal(err) + } + defer getResp.Body.Close() + if getResp.StatusCode != http.StatusOK { + t.Fatalf("GET status = %d", getResp.StatusCode) + } + body, err := io.ReadAll(getResp.Body) + if err != nil { + t.Fatal(err) + } + if string(body) != "hello" { + t.Fatalf("GET body = %q", string(body)) + } + if got := getResp.Header.Get("Content-Type"); !strings.HasPrefix(got, "text/plain") { + t.Fatalf("Content-Type = %q", got) + } + + delReq, err := http.NewRequest(http.MethodDelete, srv.URL+"/objects/attachments/test.txt", nil) + if err != nil { + t.Fatal(err) + } + delReq.Header.Set("Authorization", "Bearer secret") + delResp, err := http.DefaultClient.Do(delReq) + if err != nil { + t.Fatal(err) + } + if delResp.StatusCode != http.StatusNoContent { + t.Fatalf("DELETE status = %d", delResp.StatusCode) + } + _ = delResp.Body.Close() +} + +func TestUnauthorized(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret"})) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/objects/test.txt") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("status = %d", resp.StatusCode) + } +} + +func TestRejectsBadKeys(t *testing.T) { + bad := []string{"", "/absolute", "../escape", "a/../b", "a//b", "a\\b", "bucket/a.meta.json", ".multipart/a", ".magpie.db"} + for _, key := range bad { + if err := ValidateKey(key); err == nil { + t.Fatalf("ValidateKey(%q) returned nil", key) + } + } +} + +func TestRangeRequest(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("video/sample.txt", strings.NewReader("abcdef"), "text/plain"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret"})) + defer srv.Close() + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/objects/video/sample.txt", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer secret") + req.Header.Set("Range", "bytes=1-3") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusPartialContent { + t.Fatalf("status = %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if string(body) != "bcd" { + t.Fatalf("range body = %q", string(body)) + } +} + +func TestHealthIncludesDiskStatus(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret"})) + defer srv.Close() + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/health", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer secret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("health status = %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(body), "free_bytes") { + t.Fatalf("health body missing disk status: %s", string(body)) + } +} + +func TestMetricsEndpoint(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret"})) + defer srv.Close() + req, err := http.NewRequest(http.MethodGet, srv.URL+"/metrics", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer secret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body := readResponseBody(t, resp, http.StatusOK) + if !strings.Contains(string(body), "magpie_requests_total") { + t.Fatalf("metrics body = %s", string(body)) + } +} + +func TestRateLimit(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret", RateLimitPerMinute: 1})) + defer srv.Close() + firstReq, err := http.NewRequest(http.MethodGet, srv.URL+"/health", nil) + if err != nil { + t.Fatal(err) + } + firstReq.Header.Set("Authorization", "Bearer secret") + first, err := http.DefaultClient.Do(firstReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, first, http.StatusOK) + secondReq, err := http.NewRequest(http.MethodGet, srv.URL+"/health", nil) + if err != nil { + t.Fatal(err) + } + secondReq.Header.Set("Authorization", "Bearer secret") + second, err := http.DefaultClient.Do(secondReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, second, http.StatusTooManyRequests) +} + +func TestRejectsOversizedBearerUpload(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret", MaxObjectSize: 4})) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/objects/attachments/big.txt", strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer secret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusRequestEntityTooLarge { + t.Fatalf("oversized status = %d", resp.StatusCode) + } +} + +func TestS3LifecycleWithHeaderAuth(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/attachments/test.txt", strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + putReq.Header.Set("Content-Type", "text/plain") + signS3Header(t, putReq) + putResp, err := http.DefaultClient.Do(putReq) + if err != nil { + t.Fatal(err) + } + if putResp.StatusCode != http.StatusOK { + t.Fatalf("S3 PUT status = %d", putResp.StatusCode) + } + _ = putResp.Body.Close() + + headReq, err := http.NewRequest(http.MethodHead, srv.URL+"/bucket/attachments/test.txt", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, headReq) + headResp, err := http.DefaultClient.Do(headReq) + if err != nil { + t.Fatal(err) + } + if headResp.StatusCode != http.StatusOK { + t.Fatalf("S3 HEAD status = %d", headResp.StatusCode) + } + if got := headResp.Header.Get("ETag"); got == "" || got == "\"\"" { + t.Fatalf("S3 HEAD ETag = %q", got) + } + _ = headResp.Body.Close() + + getReq, err := http.NewRequest(http.MethodGet, srv.URL+"/bucket/attachments/test.txt", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, getReq) + getResp, err := http.DefaultClient.Do(getReq) + if err != nil { + t.Fatal(err) + } + defer getResp.Body.Close() + if getResp.StatusCode != http.StatusOK { + t.Fatalf("S3 GET status = %d", getResp.StatusCode) + } + body, err := io.ReadAll(getResp.Body) + if err != nil { + t.Fatal(err) + } + if string(body) != "hello" { + t.Fatalf("S3 GET body = %q", string(body)) + } +} + +func TestBucketAllowlist(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.AllowedBuckets = map[string]bool{"allowed": true} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + req, err := http.NewRequest(http.MethodPut, srv.URL+"/blocked/test.txt", strings.NewReader("x")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3ReadOnlyKeyCannotWrite(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := Config{S3Keys: []AccessKey{{ID: testAccessKey, Secret: testSecretKey, Permissions: permissions("read")}}} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/attachments/test.txt", strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("read-only PUT status = %d", resp.StatusCode) + } +} + +func TestS3MultipartUpload(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + initReq, err := http.NewRequest(http.MethodPost, srv.URL+"/bucket/large/object.txt?uploads", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, initReq) + initResp, err := http.DefaultClient.Do(initReq) + if err != nil { + t.Fatal(err) + } + initBody := readResponseBody(t, initResp, http.StatusOK) + var initResult createMultipartUploadResult + if err := xml.Unmarshal(initBody, &initResult); err != nil { + t.Fatal(err) + } + if initResult.UploadID == "" { + t.Fatalf("empty upload id: %s", string(initBody)) + } + + for partNumber, body := range map[int]string{1: "hello ", 2: "world"} { + partReq, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/large/object.txt?uploadId="+url.QueryEscape(initResult.UploadID)+"&partNumber="+strconv.Itoa(partNumber), strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + signS3Header(t, partReq) + partResp, err := http.DefaultClient.Do(partReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, partResp, http.StatusOK) + } + + completeXML := `12` + completeReq, err := http.NewRequest(http.MethodPost, srv.URL+"/bucket/large/object.txt?uploadId="+url.QueryEscape(initResult.UploadID), strings.NewReader(completeXML)) + if err != nil { + t.Fatal(err) + } + signS3Header(t, completeReq) + completeResp, err := http.DefaultClient.Do(completeReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, completeResp, http.StatusOK) + + getReq, err := http.NewRequest(http.MethodGet, srv.URL+"/bucket/large/object.txt", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, getReq) + getResp, err := http.DefaultClient.Do(getReq) + if err != nil { + t.Fatal(err) + } + body := readResponseBody(t, getResp, http.StatusOK) + if string(body) != "hello world" { + t.Fatalf("multipart body = %q", string(body)) + } +} + +func TestS3MultipartUploadIDCannotEscapeRoot(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/kept.txt", strings.NewReader("keep"), "text/plain"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodDelete, srv.URL+"/bucket/kept.txt?uploadId=..", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusNotFound) + + obj, err := store.Open("bucket/kept.txt") + if err != nil { + t.Fatalf("stored object was removed by invalid upload id: %v", err) + } + _ = obj.Close() +} + +func TestS3CopyObject(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/source.txt", strings.NewReader("copy body"), "text/plain"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + copyReq, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/dest.txt", nil) + if err != nil { + t.Fatal(err) + } + copyReq.Header.Set("X-Amz-Copy-Source", "/bucket/source.txt") + signS3Header(t, copyReq) + copyResp, err := http.DefaultClient.Do(copyReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, copyResp, http.StatusOK) + + obj, err := store.Open("bucket/dest.txt") + if err != nil { + t.Fatal(err) + } + defer obj.Close() + body, err := io.ReadAll(obj) + if err != nil { + t.Fatal(err) + } + if string(body) != "copy body" { + t.Fatalf("copied body = %q", string(body)) + } +} + +func TestS3DeleteObjects(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"bucket/a.txt", "bucket/b.txt", "bucket/c.txt"} { + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + deleteXML := `a.txtb.txt` + deleteReq, err := http.NewRequest(http.MethodPost, srv.URL+"/bucket?delete", strings.NewReader(deleteXML)) + if err != nil { + t.Fatal(err) + } + signS3Header(t, deleteReq) + deleteResp, err := http.DefaultClient.Do(deleteReq) + if err != nil { + t.Fatal(err) + } + body := readResponseBody(t, deleteResp, http.StatusOK) + if !strings.Contains(string(body), "a.txt") || !strings.Contains(string(body), "b.txt") { + t.Fatalf("delete response = %s", string(body)) + } + for _, key := range []string{"bucket/a.txt", "bucket/b.txt"} { + if _, err := store.Stat(key); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected %s deleted, err=%v", key, err) + } + } + if _, err := store.Stat("bucket/c.txt"); err != nil { + t.Fatalf("expected c.txt preserved: %v", err) + } +} + +func TestStaticPeerReplicationPutAndDelete(t *testing.T) { + peerStore, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + peerCfg := s3TestConfig() + peerCfg.ReplicationSecret = "replication-secret" + peer := httptest.NewServer(NewServer(peerStore, peerCfg)) + defer peer.Close() + + primaryStore, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.ReplicationPeers = []string{peer.URL} + cfg.ReplicationSecret = "replication-secret" + cfg.AllowInsecureReplication = true + primary := httptest.NewServer(NewServer(primaryStore, cfg)) + defer primary.Close() + + putReq, err := http.NewRequest(http.MethodPut, primary.URL+"/bucket/replicated.txt", strings.NewReader("copy me")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, putReq) + putResp, err := http.DefaultClient.Do(putReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, putResp, http.StatusOK) + + assertEventually(t, func() bool { + info, err := peerStore.Stat("bucket/replicated.txt") + return err == nil && info.Size == int64(len("copy me")) + }) + + deleteReq, err := http.NewRequest(http.MethodDelete, primary.URL+"/bucket/replicated.txt", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, deleteReq) + deleteResp, err := http.DefaultClient.Do(deleteReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, deleteResp, http.StatusNoContent) + + assertEventually(t, func() bool { + _, err := peerStore.Stat("bucket/replicated.txt") + return errors.Is(err, ErrNotFound) + }) + + specialName := "a?b#c%.txt" + specialReq, err := http.NewRequest(http.MethodPut, primary.URL+"/bucket/"+url.PathEscape(specialName), strings.NewReader("special")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, specialReq) + specialResp, err := http.DefaultClient.Do(specialReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, specialResp, http.StatusOK) + + assertEventually(t, func() bool { + info, err := peerStore.Stat("bucket/" + specialName) + return err == nil && info.Size == int64(len("special")) + }) +} + +func TestReplicationQueueRetriesFailedPeer(t *testing.T) { + primaryStore, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.ReplicationPeers = []string{"http://127.0.0.1:1"} + cfg.ReplicationSecret = "replication-secret" + cfg.AllowInsecureReplication = true + primary := httptest.NewServer(NewServer(primaryStore, cfg)) + defer primary.Close() + + putReq, err := http.NewRequest(http.MethodPut, primary.URL+"/bucket/retry.txt", strings.NewReader("retry")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, putReq) + putResp, err := http.DefaultClient.Do(putReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, putResp, http.StatusOK) + + assertEventually(t, func() bool { + stats, err := primaryStore.ReplicationQueueStats() + return err == nil && stats.PendingJobs == 1 && stats.MaxAttempts > 0 + }) +} + +func TestRepairQueuesAllObjectsForPeers(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"bucket/a.txt", "bucket/b.txt"} { + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 1001; i++ { + key := fmt.Sprintf("many/%04d.txt", i) + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + count, err := store.EnqueueRepair([]string{"http://peer-a", "http://peer-b"}) + if err != nil { + t.Fatal(err) + } + if count != 2006 { + t.Fatalf("repair jobs = %d", count) + } + stats, err := store.ReplicationQueueStats() + if err != nil { + t.Fatal(err) + } + if stats.PendingJobs != 2006 { + t.Fatalf("pending jobs = %d", stats.PendingJobs) + } +} + +func TestBackupAndRestore(t *testing.T) { + source := t.TempDir() + store, err := NewFileStore(source) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/object.txt", strings.NewReader("backup"), "text/plain"); err != nil { + t.Fatal(err) + } + archive := filepath.Join(t.TempDir(), "backup.tar.gz") + if err := BackupDir(source, archive); err != nil { + t.Fatal(err) + } + info, err := os.Stat(archive) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("backup mode = %o", info.Mode().Perm()) + } + destination := t.TempDir() + if err := RestoreDir(archive, destination); err != nil { + t.Fatal(err) + } + restored, err := NewFileStore(destination) + if err != nil { + t.Fatal(err) + } + obj, err := restored.Open("bucket/object.txt") + if err != nil { + t.Fatal(err) + } + defer obj.Close() + body, err := io.ReadAll(obj) + if err != nil { + t.Fatal(err) + } + if string(body) != "backup" { + t.Fatalf("restored body = %q", string(body)) + } +} + +func TestBackupRefusesSymlink(t *testing.T) { + source := t.TempDir() + if err := os.MkdirAll(filepath.Join(source, "bucket"), 0o700); err != nil { + t.Fatal(err) + } + outside := filepath.Join(t.TempDir(), "outside.txt") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(source, "bucket", "link.txt")); err != nil { + t.Fatal(err) + } + archive := filepath.Join(t.TempDir(), "backup.tar.gz") + if err := BackupDir(source, archive); err == nil { + t.Fatal("expected backup with symlink to fail") + } +} + +func TestRestoreEnforcesMaxBytes(t *testing.T) { + archive := filepath.Join(t.TempDir(), "archive.tar.gz") + if err := writeTestTarGz(archive, "bucket/object.txt", "body"); err != nil { + t.Fatal(err) + } + if err := RestoreDirWithLimit(archive, t.TempDir(), 3); err == nil { + t.Fatal("expected restore above max bytes to fail") + } +} + +func TestPutVerifiedRejectsBeforeObjectVisible(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + _, err = store.PutVerified("bucket/object.txt", strings.NewReader("body"), "text/plain", func() error { + return errPayloadHashMismatch + }) + if !errors.Is(err, errPayloadHashMismatch) { + t.Fatalf("PutVerified error = %v", err) + } + if _, err := store.Stat("bucket/object.txt"); !errors.Is(err, ErrNotFound) { + t.Fatalf("object became visible, err=%v", err) + } +} + +func TestS3PresignedGet(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/attachments/test.txt", strings.NewReader("hello"), "text/plain"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + presignedURL := presignS3URL(t, http.MethodGet, srv.URL+"/bucket/attachments/test.txt", map[string]string{ + "response-content-disposition": "attachment", + }) + resp, err := http.Get(presignedURL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("presigned GET status = %d", resp.StatusCode) + } + if got := resp.Header.Get("Content-Disposition"); got != "attachment" { + t.Fatalf("Content-Disposition = %q", got) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if string(body) != "hello" { + t.Fatalf("presigned GET body = %q", string(body)) + } +} + +func TestS3AllowedBucketsArePrivateByDefault(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/avatars/test.txt", strings.NewReader("hello"), "text/plain"); err != nil { + t.Fatal(err) + } + + cfg := s3TestConfig() + cfg.AllowedBuckets = map[string]bool{"app-uploads": true} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/app-uploads/avatars/test.txt") + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3PublicPrefixReadCanBeEnabled(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/avatars/test.txt", strings.NewReader("hello"), "text/plain"); err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/private/test.txt", strings.NewReader("secret"), "text/plain"); err != nil { + t.Fatal(err) + } + + cfg := s3TestConfig() + cfg.AllowedBuckets = map[string]bool{"app-uploads": true} + cfg.PublicPrefixes = map[string]bool{"app-uploads/avatars": true, "app-uploads": true} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/app-uploads/avatars/test.txt") + if err != nil { + t.Fatal(err) + } + body := readResponseBody(t, resp, http.StatusOK) + if string(body) != "hello" { + t.Fatalf("public GET body = %q", string(body)) + } + + privateResp, err := http.Get(srv.URL + "/app-uploads/private/test.txt") + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, privateResp, http.StatusForbidden) +} + +func TestS3PublicReadHardensActiveContent(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/avatars/evil.html", strings.NewReader(""), "text/html"); err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/avatars/evil.svg", strings.NewReader(""), "image/svg+xml"); err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/avatars/pic.png", strings.NewReader("\x89PNG"), "image/png"); err != nil { + t.Fatal(err) + } + + cfg := s3TestConfig() + cfg.AllowedBuckets = map[string]bool{"app-uploads": true} + cfg.PublicPrefixes = map[string]bool{"app-uploads/avatars": true} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + // HTML must be served with a restrictive CSP and forced to download. + htmlResp, err := http.Get(srv.URL + "/app-uploads/avatars/evil.html") + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, htmlResp, http.StatusOK) + if got := htmlResp.Header.Get("Content-Security-Policy"); got != "default-src 'none'; sandbox" { + t.Fatalf("html Content-Security-Policy = %q", got) + } + if got := htmlResp.Header.Get("Content-Disposition"); got != "attachment" { + t.Fatalf("html Content-Disposition = %q", got) + } + + // SVG is an image type but scriptable, so it must also be forced to download. + svgResp, err := http.Get(srv.URL + "/app-uploads/avatars/evil.svg") + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, svgResp, http.StatusOK) + if got := svgResp.Header.Get("Content-Disposition"); got != "attachment" { + t.Fatalf("svg Content-Disposition = %q", got) + } + + // Inert raster media stays inline so avatars/images still render. + pngResp, err := http.Get(srv.URL + "/app-uploads/avatars/pic.png") + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, pngResp, http.StatusOK) + if got := pngResp.Header.Get("Content-Disposition"); got != "" { + t.Fatalf("png Content-Disposition = %q, want inline (empty)", got) + } + if got := pngResp.Header.Get("Content-Security-Policy"); got != "default-src 'none'; sandbox" { + t.Fatalf("png Content-Security-Policy = %q", got) + } +} + +func TestS3AuthenticatedReadKeepsDispositionOverride(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/attachments/page.html", strings.NewReader("

ok

"), "text/html"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + presignedURL := presignS3URL(t, http.MethodGet, srv.URL+"/bucket/attachments/page.html", map[string]string{ + "response-content-disposition": "inline", + }) + resp, err := http.Get(presignedURL) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusOK) + // Authenticated reads are not public, so the caller's override is honored and + // no public-read hardening is applied. + if got := resp.Header.Get("Content-Disposition"); got != "inline" { + t.Fatalf("authenticated Content-Disposition = %q", got) + } + if got := resp.Header.Get("Content-Security-Policy"); got != "" { + t.Fatalf("authenticated Content-Security-Policy = %q, want none", got) + } +} + +func TestS3RejectsTamperedPresignedPath(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + presignedURL := presignS3URL(t, http.MethodGet, srv.URL+"/bucket/attachments/test.txt", nil) + tampered := strings.Replace(presignedURL, "test.txt", "other.txt", 1) + resp, err := http.Get(tampered) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("tampered status = %d", resp.StatusCode) + } +} + +func TestS3RejectsOversizedUpload(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.MaxObjectSize = 4 + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/attachments/big.txt", strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusRequestEntityTooLarge { + t.Fatalf("S3 oversized status = %d", resp.StatusCode) + } +} + +func TestS3RejectsTooLongPresignedExpiry(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.PresignedMaxExpiry = time.Hour + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + presignedURL := presignS3URLWithExpiry(t, http.MethodGet, srv.URL+"/bucket/attachments/test.txt", nil, "7200") + resp, err := http.Get(presignedURL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("long-expiry status = %d", resp.StatusCode) + } +} + +func TestS3RejectsWrongCredentialScope(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/scope.txt", strings.NewReader("body")) + if err != nil { + t.Fatal(err) + } + signS3HeaderWithScope(t, req, "wrong", "s3") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3ReadOnlyPresignedURLCannotWrite(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := Config{S3Keys: []AccessKey{{ID: testAccessKey, Secret: testSecretKey, Permissions: permissions("read")}}} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + presignedURL := presignS3URL(t, http.MethodPut, srv.URL+"/bucket/attachments/test.txt", nil) + req, err := http.NewRequest(http.MethodPut, presignedURL, strings.NewReader("nope")) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3PresignedURLCannotWrite(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + presignedURL := presignS3URL(t, http.MethodPut, srv.URL+"/bucket/attachments/test.txt", nil) + req, err := http.NewRequest(http.MethodPut, presignedURL, strings.NewReader("nope")) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3RejectsStaleHeaderSignature(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/replay.txt", strings.NewReader("replay")) + if err != nil { + t.Fatal(err) + } + signS3HeaderAt(t, req, time.Now().UTC().Add(-time.Hour)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3CopyObjectRequiresAllowedSourceBucket(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("blocked/secret.txt", strings.NewReader("secret"), "text/plain"); err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.AllowedBuckets = map[string]bool{"allowed": true} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/allowed/leak.txt", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Amz-Copy-Source", "/blocked/secret.txt") + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3RejectsPayloadHashMismatch(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/tampered.txt", strings.NewReader("tampered")) + if err != nil { + t.Fatal(err) + } + signS3HeaderWithPayloadHash(t, req, sha256Hex("original")) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) + if _, err := store.Stat("bucket/tampered.txt"); !errors.Is(err, ErrNotFound) { + t.Fatalf("tampered object persisted, err=%v", err) + } +} + +func TestS3CopyObjectRequiresReadAndWritePermission(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/source.txt", strings.NewReader("secret"), "text/plain"); err != nil { + t.Fatal(err) + } + cfg := Config{S3Keys: []AccessKey{{ID: testAccessKey, Secret: testSecretKey, Permissions: permissions("write")}}} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/copy.txt", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Amz-Copy-Source", "/bucket/source.txt") + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestBearerAPIHonorsBucketAllowlist(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := Config{AuthToken: "secret", AllowedBuckets: map[string]bool{"allowed": true}} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/objects/blocked/file.txt", strings.NewReader("x")) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer secret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestCleanupMultipartRemovesStaleUploads(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + dir := filepath.Join(store.root, ".multipart", "old-upload") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-2 * time.Hour) + if err := os.Chtimes(dir, old, old); err != nil { + t.Fatal(err) + } + if err := store.CleanupMultipart(time.Hour); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("multipart dir still exists, err=%v", err) + } +} + +func TestRestoreRefusesExistingSymlinkTarget(t *testing.T) { + archive := filepath.Join(t.TempDir(), "archive.tar.gz") + if err := writeTestTarGz(archive, "bucket/object.txt", "body"); err != nil { + t.Fatal(err) + } + destination := t.TempDir() + if err := os.MkdirAll(filepath.Join(destination, "bucket"), 0o700); err != nil { + t.Fatal(err) + } + outside := filepath.Join(t.TempDir(), "outside.txt") + if err := os.Symlink(outside, filepath.Join(destination, "bucket", "object.txt")); err != nil { + t.Fatal(err) + } + if err := RestoreDir(archive, destination); err == nil { + t.Fatal("expected restore through symlink to fail") + } +} + +func TestRestoreRefusesSymlinkParent(t *testing.T) { + archive := filepath.Join(t.TempDir(), "archive.tar.gz") + if err := writeTestTarGz(archive, "bucket/object.txt", "body"); err != nil { + t.Fatal(err) + } + destination := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(destination, "bucket")); err != nil { + t.Fatal(err) + } + if err := RestoreDir(archive, destination); err == nil { + t.Fatal("expected restore through symlink parent to fail") + } +} + +func TestS3MultipartCompleteEnforcesAggregateMaxSize(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.MaxObjectSize = 6 + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + initReq, err := http.NewRequest(http.MethodPost, srv.URL+"/bucket/large.txt?uploads", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, initReq) + initResp, err := http.DefaultClient.Do(initReq) + if err != nil { + t.Fatal(err) + } + initBody := readResponseBody(t, initResp, http.StatusOK) + var initResult createMultipartUploadResult + if err := xml.Unmarshal(initBody, &initResult); err != nil { + t.Fatal(err) + } + + parts := []struct { + number int + body string + }{ + {number: 1, body: "abcd"}, + {number: 2, body: "efgh"}, + } + + for _, part := range parts { + partNumber := part.number + body := part.body + partReq, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/large.txt?uploadId="+url.QueryEscape(initResult.UploadID)+"&partNumber="+strconv.Itoa(partNumber), strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + signS3Header(t, partReq) + partResp, err := http.DefaultClient.Do(partReq) + if err != nil { + t.Fatal(err) + } + wantStatus := http.StatusOK + if partNumber == 2 { + wantStatus = http.StatusRequestEntityTooLarge + } + _ = readResponseBody(t, partResp, wantStatus) + } +} + +func TestS3RejectsReplicationHeaderWithoutSecret(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/suppressed.txt", strings.NewReader("x")) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Magpie-Replication", "1") + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3RejectsOversizedBatchDelete(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + var body strings.Builder + body.WriteString("") + for i := 0; i <= maxBatchDeleteItems; i++ { + body.WriteString("key-") + body.WriteString(strconv.Itoa(i)) + body.WriteString("") + } + body.WriteString("") + req, err := http.NewRequest(http.MethodPost, srv.URL+"/bucket?delete", strings.NewReader(body.String())) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusBadRequest) +} + +func TestS3ListBucketWithPrefix(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + fixtures := []string{ + "bucket/attachments/a.txt", + "bucket/attachments/b.txt", + "bucket/avatars/c.txt", + "otherbucket/attachments/d.txt", + } + for _, key := range fixtures { + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/bucket?list-type=2&prefix=attachments/", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("list status = %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + text := string(body) + if !strings.Contains(text, "attachments/a.txt") { + t.Fatalf("list missing a.txt: %s", text) + } + if !strings.Contains(text, "attachments/b.txt") { + t.Fatalf("list missing b.txt: %s", text) + } + if strings.Contains(text, "avatars/c.txt") || strings.Contains(text, "otherbucket") { + t.Fatalf("list included wrong keys: %s", text) + } + if !strings.Contains(text, "2") { + t.Fatalf("list key count wrong: %s", text) + } +} + +func TestS3ListBucketMaxKeys(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"bucket/a.txt", "bucket/b.txt", "bucket/c.txt"} { + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/bucket?list-type=2&max-keys=2", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("max-keys list status = %d body=%s", resp.StatusCode, string(body)) + } + if !strings.Contains(string(body), "2") { + t.Fatalf("max-keys not honored: %s", string(body)) + } +} + +func TestS3ListBucketContinuationToken(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"bucket/a.txt", "bucket/b.txt", "bucket/c.txt"} { + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + first := signedGET(t, srv.URL+"/bucket?list-type=2&max-keys=2") + firstBody := readResponseBody(t, first, http.StatusOK) + var firstPage s3ListBucketResult + if err := xml.Unmarshal(firstBody, &firstPage); err != nil { + t.Fatal(err) + } + if !firstPage.IsTruncated || firstPage.NextContinuationToken == "" || firstPage.KeyCount != 2 { + t.Fatalf("first page = %#v body=%s", firstPage, string(firstBody)) + } + + second := signedGET(t, srv.URL+"/bucket?list-type=2&max-keys=2&continuation-token="+url.QueryEscape(firstPage.NextContinuationToken)) + secondBody := readResponseBody(t, second, http.StatusOK) + var secondPage s3ListBucketResult + if err := xml.Unmarshal(secondBody, &secondPage); err != nil { + t.Fatal(err) + } + if secondPage.IsTruncated || secondPage.NextContinuationToken != "" || secondPage.KeyCount != 1 { + t.Fatalf("second page = %#v body=%s", secondPage, string(secondBody)) + } + if len(secondPage.Contents) != 1 || secondPage.Contents[0].Key != "c.txt" { + t.Fatalf("second page contents = %#v", secondPage.Contents) + } +} + +func TestS3ListBucketRejectsInvalidContinuationToken(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + resp := signedGET(t, srv.URL+"/bucket?list-type=2&continuation-token=not-base64!!!") + _ = readResponseBody(t, resp, http.StatusBadRequest) +} + +func TestMetadataPersistsAcrossStoreRestart(t *testing.T) { + dir := t.TempDir() + store, err := NewFileStore(dir) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/attachments/a.txt", strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + + reopened, err := NewFileStore(dir) + if err != nil { + t.Fatal(err) + } + objects, err := reopened.List("bucket/attachments/", 1000) + if err != nil { + t.Fatal(err) + } + if len(objects) != 1 || objects[0].Key != "bucket/attachments/a.txt" { + t.Fatalf("persisted objects = %#v", objects) + } +} + +func TestReindexBackfillsMetadataFromFilesystem(t *testing.T) { + dir := t.TempDir() + objectPath := filepath.Join(dir, "bucket", "attachments", "manual.txt") + if err := os.MkdirAll(filepath.Dir(objectPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(objectPath, []byte("manual"), 0o600); err != nil { + t.Fatal(err) + } + + store, err := NewFileStore(dir) + if err != nil { + t.Fatal(err) + } + objects, err := store.List("bucket/attachments/", 1000) + if err != nil { + t.Fatal(err) + } + if len(objects) != 0 { + t.Fatalf("objects before reindex = %#v", objects) + } + if err := store.Reindex(); err != nil { + t.Fatal(err) + } + objects, err = store.List("bucket/attachments/", 1000) + if err != nil { + t.Fatal(err) + } + if len(objects) != 1 || objects[0].Key != "bucket/attachments/manual.txt" || objects[0].ETag == "" { + t.Fatalf("objects after reindex = %#v", objects) + } +} + +func TestScrubReportsMissingCorruptAndOrphanedObjects(t *testing.T) { + dir := t.TempDir() + store, err := NewFileStore(dir) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/good.txt", strings.NewReader("good"), "text/plain"); err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/missing.txt", strings.NewReader("missing"), "text/plain"); err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/corrupt.txt", strings.NewReader("before"), "text/plain"); err != nil { + t.Fatal(err) + } + missingPath, _ := store.pathFor("bucket/missing.txt") + _ = os.Remove(missingPath) + corruptPath, _ := store.pathFor("bucket/corrupt.txt") + if err := os.WriteFile(corruptPath, []byte("after"), 0o600); err != nil { + t.Fatal(err) + } + orphanPath := filepath.Join(dir, "bucket", "orphan.txt") + if err := os.WriteFile(orphanPath, []byte("orphan"), 0o600); err != nil { + t.Fatal(err) + } + + report, err := store.Scrub() + if err != nil { + t.Fatal(err) + } + if !contains(report.MissingKeys, "bucket/missing.txt") { + t.Fatalf("missing keys = %#v", report.MissingKeys) + } + if !contains(report.CorruptKeys, "bucket/corrupt.txt") { + t.Fatalf("corrupt keys = %#v", report.CorruptKeys) + } + if !contains(report.OrphanKeys, "bucket/orphan.txt") { + t.Fatalf("orphan keys = %#v", report.OrphanKeys) + } +} + +func contains(values []string, value string) bool { + for _, candidate := range values { + if candidate == value { + return true + } + } + return false +} + +func assertEventually(t *testing.T, condition func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if condition() { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("condition not met before timeout") +} + +func s3TestConfig() Config { + return Config{S3AccessKeyID: testAccessKey, S3SecretAccessKey: testSecretKey, S3Region: "auto"} +} + +func signedGET(t *testing.T, rawURL string) *http.Response { + t.Helper() + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + return resp +} + +func readResponseBody(t *testing.T, resp *http.Response, wantStatus int) []byte { + t.Helper() + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != wantStatus { + t.Fatalf("status = %d want=%d body=%s", resp.StatusCode, wantStatus, string(body)) + } + return body +} + +func signS3Header(t *testing.T, req *http.Request) { + t.Helper() + signS3HeaderAt(t, req, time.Now().UTC()) +} + +func signS3HeaderAt(t *testing.T, req *http.Request, at time.Time) { + t.Helper() + amzDate := at.UTC().Format("20060102T150405Z") + date := amzDate[:8] + scope := date + "/auto/s3/aws4_request" + signedHeaders := "host;x-amz-content-sha256;x-amz-date" + if req.Header.Get("X-Amz-Copy-Source") != "" { + signedHeaders += ";x-amz-copy-source" + } + payloadHash := requestPayloadHash(t, req) + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + canonical := canonicalRequest(req, signedHeaders, payloadHash, false) + sig := signature(testSecretKey, amzDate, scope, canonical) + req.Header.Set( + "Authorization", + awsAlgorithm+" Credential="+testAccessKey+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig, + ) +} + +func requestPayloadHash(t *testing.T, req *http.Request) string { + t.Helper() + if req.Body == nil { + return sha256Hex("") + } + if req.GetBody == nil { + return "UNSIGNED-PAYLOAD" + } + body, err := req.GetBody() + if err != nil { + t.Fatal(err) + } + defer body.Close() + hash := sha256.New() + if _, err := io.Copy(hash, body); err != nil { + t.Fatal(err) + } + return hex.EncodeToString(hash.Sum(nil)) +} + +func signS3HeaderWithPayloadHash(t *testing.T, req *http.Request, payloadHash string) { + t.Helper() + amzDate := time.Now().UTC().Format("20060102T150405Z") + date := amzDate[:8] + scope := date + "/auto/s3/aws4_request" + signedHeaders := "host;x-amz-content-sha256;x-amz-date" + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + canonical := canonicalRequest(req, signedHeaders, payloadHash, false) + sig := signature(testSecretKey, amzDate, scope, canonical) + req.Header.Set( + "Authorization", + awsAlgorithm+" Credential="+testAccessKey+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig, + ) +} + +func signS3HeaderWithScope(t *testing.T, req *http.Request, region string, service string) { + t.Helper() + amzDate := time.Now().UTC().Format("20060102T150405Z") + date := amzDate[:8] + scope := date + "/" + region + "/" + service + "/aws4_request" + signedHeaders := "host;x-amz-content-sha256;x-amz-date" + payloadHash := requestPayloadHash(t, req) + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + canonical := canonicalRequest(req, signedHeaders, payloadHash, false) + sig := signature(testSecretKey, amzDate, scope, canonical) + req.Header.Set( + "Authorization", + awsAlgorithm+" Credential="+testAccessKey+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig, + ) +} + +func presignS3URL(t *testing.T, method string, rawURL string, extra map[string]string) string { + return presignS3URLWithExpiry(t, method, rawURL, extra, "3600") +} + +func presignS3URLWithExpiry(t *testing.T, method string, rawURL string, extra map[string]string, expires string) string { + t.Helper() + parsed, err := url.Parse(rawURL) + if err != nil { + t.Fatal(err) + } + amzDate := time.Now().UTC().Format("20060102T150405Z") + date := amzDate[:8] + scope := date + "/auto/s3/aws4_request" + query := parsed.Query() + for key, value := range extra { + query.Set(key, value) + } + query.Set("X-Amz-Algorithm", awsAlgorithm) + query.Set("X-Amz-Credential", testAccessKey+"/"+scope) + query.Set("X-Amz-Date", amzDate) + query.Set("X-Amz-Expires", expires) + query.Set("X-Amz-SignedHeaders", "host") + parsed.RawQuery = query.Encode() + + req, err := http.NewRequest(method, parsed.String(), nil) + if err != nil { + t.Fatal(err) + } + canonical := canonicalRequest(req, "host", "UNSIGNED-PAYLOAD", true) + query.Set("X-Amz-Signature", signature(testSecretKey, amzDate, scope, canonical)) + parsed.RawQuery = query.Encode() + return parsed.String() +} + +func sha256Hex(value string) string { + sum := sha256.Sum256([]byte(value)) + return fmt.Sprintf("%x", sum[:]) +} + +func writeTestTarGz(path string, name string, body string) error { + out, err := os.Create(path) + if err != nil { + return err + } + defer out.Close() + gz := gzip.NewWriter(out) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(body))}); err != nil { + return err + } + _, err = tw.Write([]byte(body)) + return err +} diff --git a/store.go b/store.go new file mode 100644 index 0000000..fd14fd6 --- /dev/null +++ b/store.go @@ -0,0 +1,563 @@ +package main + +import ( + "crypto/md5" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +var ErrNotFound = errors.New("object not found") + +type FileStore struct { + root string + db *sql.DB +} + +type ObjectRead struct { + *os.File + Info ObjectInfo +} + +func NewFileStore(root string) (*FileStore, error) { + abs, err := filepath.Abs(root) + if err != nil { + return nil, err + } + if err := os.MkdirAll(abs, 0o700); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", filepath.Join(abs, ".magpie.db")) + if err != nil { + return nil, err + } + store := &FileStore{root: abs, db: db} + if err := store.initMetadata(); err != nil { + _ = db.Close() + return nil, err + } + return store, nil +} + +func (s *FileStore) Put(key string, body io.Reader, contentType string) (ObjectInfo, error) { + return s.PutVerified(key, body, contentType, nil) +} + +func (s *FileStore) PutVerified(key string, body io.Reader, contentType string, verify func() error) (ObjectInfo, error) { + if err := ValidateKey(key); err != nil { + return ObjectInfo{}, err + } + + path, err := s.pathFor(key) + if err != nil { + return ObjectInfo{}, err + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return ObjectInfo{}, err + } + + tmp, err := os.CreateTemp(filepath.Dir(path), ".upload-*") + if err != nil { + return ObjectInfo{}, err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + + hash := md5.New() + size, copyErr := io.Copy(io.MultiWriter(tmp, hash), body) + closeErr := tmp.Close() + if copyErr != nil { + return ObjectInfo{}, copyErr + } + if closeErr != nil { + return ObjectInfo{}, closeErr + } + if verify != nil { + if err := verify(); err != nil { + return ObjectInfo{}, err + } + } + + info := ObjectInfo{ + Key: key, + Size: size, + ContentType: contentType, + ModTime: time.Now().UTC(), + ETag: fmt.Sprintf("%x", hash.Sum(nil)), + } + if err := s.writeMeta(key, info); err != nil { + return ObjectInfo{}, err + } + if err := s.upsertMetadata(info); err != nil { + _ = os.Remove(path + ".meta.json") + return ObjectInfo{}, err + } + if err := os.Rename(tmpName, path); err != nil { + _ = os.Remove(path + ".meta.json") + return ObjectInfo{}, err + } + return info, nil +} + +func (s *FileStore) Open(key string) (*ObjectRead, error) { + if err := ValidateKey(key); err != nil { + return nil, err + } + path, err := s.pathFor(key) + if err != nil { + return nil, err + } + file, err := os.Open(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, ErrNotFound + } + return nil, err + } + info, err := s.Stat(key) + if err != nil { + _ = file.Close() + return nil, err + } + return &ObjectRead{File: file, Info: info}, nil +} + +func (s *FileStore) Stat(key string) (ObjectInfo, error) { + if err := ValidateKey(key); err != nil { + return ObjectInfo{}, err + } + path, err := s.pathFor(key) + if err != nil { + return ObjectInfo{}, err + } + stat, err := os.Stat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return ObjectInfo{}, ErrNotFound + } + return ObjectInfo{}, err + } + + if info, err := s.getMetadata(key); err == nil { + info.Size = stat.Size() + return info, nil + } + + info := ObjectInfo{Key: key, Size: stat.Size(), ModTime: stat.ModTime().UTC()} + if meta, err := s.readMeta(key); err == nil { + info.ContentType = meta.ContentType + info.ETag = meta.ETag + if !meta.ModTime.IsZero() { + info.ModTime = meta.ModTime + } + } + _ = s.upsertMetadata(info) + return info, nil +} + +func (s *FileStore) Delete(key string) error { + if err := ValidateKey(key); err != nil { + return err + } + path, err := s.pathFor(key) + if err != nil { + return err + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + _ = os.Remove(path + ".meta.json") + _, _ = s.db.Exec(`DELETE FROM objects WHERE key = ?`, key) + return nil +} + +func (s *FileStore) List(prefix string, maxKeys int) ([]ObjectInfo, error) { + page, err := s.ListPage(prefix, "", maxKeys) + if err != nil { + return nil, err + } + return page.Objects, nil +} + +func (s *FileStore) ListPage(prefix string, after string, maxKeys int) (ListPage, error) { + validatePrefix := strings.TrimSuffix(prefix, "/") + if validatePrefix != "" { + if err := ValidateKey(validatePrefix); err != nil { + return ListPage{}, err + } + } + if after != "" { + if err := ValidateKey(after); err != nil { + return ListPage{}, err + } + } + if maxKeys <= 0 || maxKeys > 1000 { + maxKeys = 1000 + } + + return s.listMetadata(prefix, after, maxKeys) +} + +func (s *FileStore) Reindex() error { + return filepath.WalkDir(s.root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + if entry.Name() == ".magpie.db" || strings.HasPrefix(entry.Name(), ".magpie.db-") || strings.HasSuffix(entry.Name(), ".meta.json") { + return nil + } + + rel, err := filepath.Rel(s.root, path) + if err != nil { + return err + } + key := filepath.ToSlash(rel) + if err := ValidateKey(key); err != nil { + return nil + } + info, err := s.statFromFilesystem(key) + if err != nil { + return err + } + return s.upsertMetadata(info) + }) +} + +func (s *FileStore) Stats() (StoreStats, error) { + var stats StoreStats + if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(size), 0) FROM objects`).Scan(&stats.ObjectCount, &stats.TotalBytes); err != nil { + return StoreStats{}, err + } + queue, err := s.ReplicationQueueStats() + if err != nil { + return StoreStats{}, err + } + stats.ReplicationQueue = queue + disk, err := s.DiskStatus() + if err != nil { + return StoreStats{}, err + } + stats.Disk = disk + return stats, nil +} + +func (s *FileStore) Scrub() (ScrubReport, error) { + report := ScrubReport{} + rows, err := s.db.Query(`SELECT key, size, etag FROM objects ORDER BY key ASC`) + if err != nil { + return report, err + } + defer rows.Close() + + metadataKeys := map[string]bool{} + for rows.Next() { + var key string + var size int64 + var etag string + if err := rows.Scan(&key, &size, &etag); err != nil { + return report, err + } + metadataKeys[key] = true + report.CheckedObjects++ + path, err := s.pathFor(key) + if err != nil { + report.CorruptKeys = append(report.CorruptKeys, key) + continue + } + stat, err := os.Stat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + report.MissingKeys = append(report.MissingKeys, key) + continue + } + return report, err + } + if stat.Size() != size { + report.CorruptKeys = append(report.CorruptKeys, key) + continue + } + computed, err := fileETag(path) + if err != nil { + return report, err + } + if etag != "" && computed != etag { + report.CorruptKeys = append(report.CorruptKeys, key) + } + } + if err := rows.Err(); err != nil { + return report, err + } + + if err := filepath.WalkDir(s.root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || entry.Name() == ".magpie.db" || strings.HasPrefix(entry.Name(), ".magpie.db-") || strings.HasSuffix(entry.Name(), ".meta.json") || strings.Contains(path, string(os.PathSeparator)+".multipart"+string(os.PathSeparator)) { + return nil + } + rel, err := filepath.Rel(s.root, path) + if err != nil { + return err + } + key := filepath.ToSlash(rel) + if !metadataKeys[key] { + report.OrphanKeys = append(report.OrphanKeys, key) + } + return nil + }); err != nil { + return report, err + } + sort.Strings(report.MissingKeys) + sort.Strings(report.CorruptKeys) + sort.Strings(report.OrphanKeys) + return report, nil +} + +func (s *FileStore) CleanupMultipart(maxAge time.Duration) error { + if maxAge <= 0 { + return nil + } + root := filepath.Join(s.root, ".multipart") + entries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + cutoff := time.Now().Add(-maxAge) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + info, err := entry.Info() + if err != nil { + return err + } + if info.ModTime().Before(cutoff) { + if err := os.RemoveAll(filepath.Join(root, entry.Name())); err != nil { + return err + } + } + } + return nil +} + +func (s *FileStore) pathFor(key string) (string, error) { + clean := filepath.Clean(filepath.FromSlash(key)) + path := filepath.Join(s.root, clean) + if path != s.root && !strings.HasPrefix(path, s.root+string(os.PathSeparator)) { + return "", fmt.Errorf("invalid object key") + } + return path, nil +} + +func (s *FileStore) writeMeta(key string, info ObjectInfo) error { + path, err := s.pathFor(key) + if err != nil { + return err + } + data, err := json.Marshal(info) + if err != nil { + return err + } + return os.WriteFile(path+".meta.json", data, 0o600) +} + +func (s *FileStore) readMeta(key string) (ObjectInfo, error) { + path, err := s.pathFor(key) + if err != nil { + return ObjectInfo{}, err + } + data, err := os.ReadFile(path + ".meta.json") + if err != nil { + return ObjectInfo{}, err + } + var info ObjectInfo + if err := json.Unmarshal(data, &info); err != nil { + return ObjectInfo{}, err + } + return info, nil +} + +func (s *FileStore) statFromFilesystem(key string) (ObjectInfo, error) { + path, err := s.pathFor(key) + if err != nil { + return ObjectInfo{}, err + } + stat, err := os.Stat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return ObjectInfo{}, ErrNotFound + } + return ObjectInfo{}, err + } + info := ObjectInfo{Key: key, Size: stat.Size(), ModTime: stat.ModTime().UTC()} + if meta, err := s.readMeta(key); err == nil { + info.ContentType = meta.ContentType + info.ETag = meta.ETag + if !meta.ModTime.IsZero() { + info.ModTime = meta.ModTime + } + } + if info.ETag == "" { + etag, err := fileETag(path) + if err != nil { + return ObjectInfo{}, err + } + info.ETag = etag + } + return info, nil +} + +func fileETag(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := md5.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func (s *FileStore) initMetadata() error { + _, err := s.db.Exec(` +PRAGMA journal_mode = WAL; +PRAGMA synchronous = NORMAL; +CREATE TABLE IF NOT EXISTS objects ( + key TEXT PRIMARY KEY, + size INTEGER NOT NULL, + content_type TEXT NOT NULL DEFAULT '', + etag TEXT NOT NULL DEFAULT '', + mod_time TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS objects_key_idx ON objects(key); +CREATE TABLE IF NOT EXISTS replication_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + peer TEXT NOT NULL, + op TEXT NOT NULL, + key TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + next_attempt_at TEXT NOT NULL, + last_error TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS replication_jobs_due_idx ON replication_jobs(next_attempt_at, id); +`) + return err +} + +func (s *FileStore) upsertMetadata(info ObjectInfo) error { + _, err := s.db.Exec(` +INSERT INTO objects (key, size, content_type, etag, mod_time) +VALUES (?, ?, ?, ?, ?) +ON CONFLICT(key) DO UPDATE SET + size = excluded.size, + content_type = excluded.content_type, + etag = excluded.etag, + mod_time = excluded.mod_time +`, info.Key, info.Size, info.ContentType, info.ETag, info.ModTime.UTC().Format(time.RFC3339Nano)) + return err +} + +func (s *FileStore) getMetadata(key string) (ObjectInfo, error) { + var info ObjectInfo + var modTime string + err := s.db.QueryRow(`SELECT key, size, content_type, etag, mod_time FROM objects WHERE key = ?`, key). + Scan(&info.Key, &info.Size, &info.ContentType, &info.ETag, &modTime) + if err != nil { + return ObjectInfo{}, err + } + parsed, err := time.Parse(time.RFC3339Nano, modTime) + if err != nil { + return ObjectInfo{}, err + } + info.ModTime = parsed + return info, nil +} + +func (s *FileStore) listMetadata(prefix string, after string, maxKeys int) (ListPage, error) { + rows, err := s.db.Query(` +SELECT key, size, content_type, etag, mod_time +FROM objects +WHERE key LIKE ? ESCAPE '\' AND key > ? +ORDER BY key ASC +LIMIT ? +`, escapeLike(prefix)+"%", after, maxKeys+1) + if err != nil { + return ListPage{}, err + } + defer rows.Close() + + objects := make([]ObjectInfo, 0) + for rows.Next() { + var info ObjectInfo + var modTime string + if err := rows.Scan(&info.Key, &info.Size, &info.ContentType, &info.ETag, &modTime); err != nil { + return ListPage{}, err + } + parsed, err := time.Parse(time.RFC3339Nano, modTime) + if err != nil { + return ListPage{}, err + } + info.ModTime = parsed + objects = append(objects, info) + } + if err := rows.Err(); err != nil { + return ListPage{}, err + } + + page := ListPage{Objects: objects} + if len(objects) > maxKeys { + page.NextAfter = objects[maxKeys-1].Key + page.Objects = objects[:maxKeys] + } + return page, nil +} + +func escapeLike(value string) string { + value = strings.ReplaceAll(value, `\`, `\\`) + value = strings.ReplaceAll(value, `%`, `\%`) + value = strings.ReplaceAll(value, `_`, `\_`) + return value +} + +func ValidateKey(key string) error { + if key == "" { + return fmt.Errorf("object key is required") + } + if len(key) > 1024 { + return fmt.Errorf("object key is too long") + } + if strings.HasPrefix(key, "/") || strings.HasPrefix(key, "\\") { + return fmt.Errorf("object key cannot be absolute") + } + if strings.Contains(key, "\x00") || strings.Contains(key, "\\") || strings.Contains(key, "//") { + return fmt.Errorf("object key contains invalid characters") + } + for _, part := range strings.Split(key, "/") { + if part == "" || part == "." || part == ".." { + return fmt.Errorf("object key contains invalid path segment") + } + if part == ".multipart" || part == ".magpie.db" || strings.HasPrefix(part, ".magpie.db-") || strings.HasSuffix(part, ".meta.json") { + return fmt.Errorf("object key uses reserved internal name") + } + } + return nil +} diff --git a/version.go b/version.go new file mode 100644 index 0000000..4b4e477 --- /dev/null +++ b/version.go @@ -0,0 +1,29 @@ +package main + +import "runtime" + +var ( + version = "dev" + commit = "unknown" + date = "unknown" +) + +type VersionInfo struct { + Version string `json:"version"` + Commit string `json:"commit"` + Date string `json:"date"` + Go string `json:"go"` + OS string `json:"os"` + Arch string `json:"arch"` +} + +func versionInfo() VersionInfo { + return VersionInfo{ + Version: version, + Commit: commit, + Date: date, + Go: runtime.Version(), + OS: runtime.GOOS, + Arch: runtime.GOARCH, + } +}