From ddbd19f15359e0fdc877b483fcb7211b7c72b62e Mon Sep 17 00:00:00 2001 From: Maxfield Luke Date: Wed, 29 Jul 2026 04:56:34 -0400 Subject: [PATCH] Initial commit on Forgejo Fresh repository history for elektrine/magpie hosted at https://git.elektrine.com/elektrine/magpie. --- .dockerignore | 19 + .env.production.example | 62 ++ .github/workflows/release.yml | 51 + .gitignore | 30 + Dockerfile | 20 + Makefile | 36 + README.md | 271 +++++ backup.go | 181 ++++ deploy/systemd/magpie.service | 21 + disk_unix.go | 21 + disk_windows.go | 7 + docker-compose.network.yml | 11 + docker-compose.yml | 15 + go.mod | 16 + go.sum | 21 + main.go | 415 ++++++++ replication.go | 330 +++++++ s3.go | 938 ++++++++++++++++++ s3auth.go | 312 ++++++ server.go | 530 ++++++++++ server_test.go | 1737 +++++++++++++++++++++++++++++++++ store.go | 563 +++++++++++ version.go | 29 + 23 files changed, 5636 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.production.example create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 README.md create mode 100644 backup.go create mode 100644 deploy/systemd/magpie.service create mode 100644 disk_unix.go create mode 100644 disk_windows.go create mode 100644 docker-compose.network.yml create mode 100644 docker-compose.yml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 replication.go create mode 100644 s3.go create mode 100644 s3auth.go create mode 100644 server.go create mode 100644 server_test.go create mode 100644 store.go create mode 100644 version.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b08715e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +.git/ +.env +.env.* +!.env.production.example +data/ +.multipart/ +dist/ +tmp/ +*.db +*.db-* +*.sqlite +*.sqlite-* +*.meta.json +*.tar +*.tar.gz +*.tgz +*.log +coverage.out +magpie diff --git a/.env.production.example b/.env.production.example new file mode 100644 index 0000000..5c7ace2 --- /dev/null +++ b/.env.production.example @@ -0,0 +1,62 @@ +MAGPIE_ADDR=127.0.0.1:8090 +MAGPIE_DATA_DIR=/var/lib/magpie + +# Optional bearer-token API for /objects/* routes. +MAGPIE_AUTH_TOKEN=change-this-long-random-token + +# S3-compatible API credentials for /:bucket/:key routes. +MAGPIE_S3_ACCESS_KEY_ID=magpie +MAGPIE_S3_SECRET_ACCESS_KEY=change-this-long-random-s3-secret +# Optional multi-key form: id:secret:read,write,admin;id2:secret2:read +MAGPIE_S3_KEYS= +MAGPIE_S3_REGION=auto + +# Temporarily set true only for trusted private migrations with clients that +# upload using SigV4 UNSIGNED-PAYLOAD, such as some rclone versions. +MAGPIE_ALLOW_UNSIGNED_PAYLOADS=false + +# Comma-separated peer base URLs for async write/delete replication. +# MAGPIE_REPLICATION_SECRET is required when peers are set. +MAGPIE_REPLICATION_PEERS= +MAGPIE_REPLICATION_SECRET= +# Replication peers must use https unless this is explicitly enabled for a +# trusted local/private network. +MAGPIE_ALLOW_INSECURE_REPLICATION=false +MAGPIE_REPLICATION_MAX_JOBS=10000 +MAGPIE_REPLICATION_MAX_ATTEMPTS=100 + +# Comma-separated bucket allowlist. Empty allows any bucket name. +MAGPIE_ALLOWED_BUCKETS=app-uploads + +# Comma-separated bucket/prefix values that allow unsigned public GET/HEAD requests. +# Empty/private by default. Set to `app-uploads/avatars` only for public media. +# MAGPIE_PUBLIC_PREFIXES=app-uploads/avatars + +# Per-IP request limit per minute. Set 0 to disable. +MAGPIE_RATE_LIMIT_PER_MINUTE=600 +# Trust X-Forwarded-For/X-Real-IP for rate limiting only behind a trusted proxy. +MAGPIE_TRUST_PROXY_HEADERS=false + +# Upload safety limits. +# Default: 1073741824 (1 GiB) +MAGPIE_MAX_OBJECT_SIZE=1073741824 + +# Reject writes if disk free space would fall below this value. +# Default: 1073741824 (1 GiB) +MAGPIE_MIN_FREE_BYTES=1073741824 + +# Maximum accepted SigV4 presigned URL lifetime. +# Default: 24h +MAGPIE_PRESIGNED_MAX_EXPIRY=24h + +# Remove incomplete multipart uploads older than this age. +# Default: 24h +MAGPIE_MULTIPART_MAX_AGE=24h + +# Maximum total expanded bytes accepted by restore. +# Default: 10737418240 (10 GiB) +MAGPIE_MAX_RESTORE_BYTES=10737418240 + +# Rebuild SQLite metadata from existing files on startup. +# Useful after manual restores/migrations; usually false for normal startup. +MAGPIE_REINDEX_ON_START=false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..8647729 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,51 @@ +name: release + +on: + push: + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff + with: + go-version-file: go.mod + cache: true + + - name: Test + run: go test ./... + + - name: Build release artifacts + run: make release VERSION="$GITHUB_REF_NAME" COMMIT="${GITHUB_SHA::7}" + + - name: Upload artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: magpie-${{ github.ref_name }} + path: dist/* + + publish: + if: startsWith(github.ref, 'refs/tags/') + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: magpie-${{ github.ref_name }} + path: dist + + - name: Publish GitHub release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 + with: + files: dist/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f5e1f37 --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Local binaries and build output +magpie +dist/ +coverage.out + +# Runtime data +data/ +*.db +*.db-* +*.sqlite +*.sqlite-* +*.meta.json +.multipart/ + +# Local environment files +.env +.env.* +!.env.production.example + +# Logs and temporary files +*.log +*.tmp +tmp/ + +# Editor and OS files +.DS_Store +.idea/ +.vscode/ +*.swp +*.swo diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dcea72a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM golang:1.26-alpine@sha256:f44b851aa23dfa219d18db6eab743203245429d355cb619cf96a2ffe2a84ba7a AS build + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY *.go ./ +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/magpie . + +FROM alpine:3.22@sha256:2039be0c5ec6ce8566809626a252c930216a92109c043f282504accb5ee3c0c6 + +RUN addgroup -S magpie && adduser -S -G magpie magpie +RUN mkdir -p /var/lib/magpie && chown -R magpie:magpie /var/lib/magpie + +USER magpie +EXPOSE 8090 +ENV MAGPIE_ADDR=0.0.0.0:8090 +ENV MAGPIE_DATA_DIR=/var/lib/magpie + +COPY --from=build /out/magpie /usr/local/bin/magpie +ENTRYPOINT ["/usr/local/bin/magpie"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bc715cf --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +BINARY := magpie +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || printf dev) +COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || printf unknown) +DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) +LDFLAGS := -s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.date=$(DATE) +DIST := dist + +.PHONY: all fmt test build clean checksums release snapshot + +all: fmt test build + +fmt: + gofmt -w . + +test: + go test ./... + +build: + mkdir -p $(DIST) + CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY) . + +snapshot: clean + mkdir -p $(DIST) + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_linux_amd64 . + GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_linux_arm64 . + GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_darwin_amd64 . + GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_darwin_arm64 . + GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_windows_amd64.exe . + +checksums: + cd $(DIST) && sha256sum * > SHA256SUMS + +release: snapshot checksums + +clean: + rm -rf $(DIST) diff --git a/README.md b/README.md new file mode 100644 index 0000000..a62d79d --- /dev/null +++ b/README.md @@ -0,0 +1,271 @@ +# Magpie + +Magpie is a small object store for app-owned files. It stores objects on local disk and exposes a minimal S3-compatible path-style API. + +It is intended to run behind your application or edge proxy, on localhost, a private Docker network, or a VPN-only address. Do not expose write-capable endpoints directly to the public internet. + +## Status + +Magpie supports common application object-storage operations: + +- `PUT`, `GET`, `HEAD`, and `DELETE` objects +- path-style S3 routes: `/:bucket/:key` +- SigV4 header authentication for writes and private reads +- presigned `GET` and `HEAD` URLs +- prefix listing with continuation tokens +- batch delete +- copy object +- multipart upload +- bucket allowlist and opt-in public-read buckets +- local metadata index in SQLite +- async static-peer replication +- backup, restore, scrub, reindex, stats, and repair commands + +It does not implement bucket creation, ACLs, bucket policies, object versioning, lifecycle rules, quorum writes, or consensus replication. + +## Quick Start + +```sh +export MAGPIE_ADDR='127.0.0.1:8090' +export MAGPIE_DATA_DIR='/var/lib/magpie' +export MAGPIE_S3_ACCESS_KEY_ID='magpie' +export MAGPIE_S3_SECRET_ACCESS_KEY='replace-with-a-long-random-secret' +export MAGPIE_ALLOWED_BUCKETS='app-uploads' + +magpie serve +``` + +For local development from source: + +```sh +go run . serve +``` + +## Configuration + +Required authentication config: + +```env +MAGPIE_S3_ACCESS_KEY_ID=magpie +MAGPIE_S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret +``` + +Or define multiple scoped keys: + +```env +MAGPIE_S3_KEYS=app:secret1:read,write;auditor:secret2:read;admin:secret3:admin +``` + +Supported scopes are `read`, `write`, and `admin`. `admin` implies all scopes. + +Common runtime config: + +```env +MAGPIE_ADDR=127.0.0.1:8090 +MAGPIE_DATA_DIR=/var/lib/magpie +MAGPIE_ALLOWED_BUCKETS=app-uploads +# MAGPIE_PUBLIC_PREFIXES=app-uploads/avatars +MAGPIE_RATE_LIMIT_PER_MINUTE=600 +MAGPIE_TRUST_PROXY_HEADERS=false +MAGPIE_MAX_OBJECT_SIZE=1073741824 +MAGPIE_MIN_FREE_BYTES=1073741824 +MAGPIE_PRESIGNED_MAX_EXPIRY=24h +MAGPIE_MULTIPART_MAX_AGE=24h +MAGPIE_REINDEX_ON_START=false +MAGPIE_MAX_RESTORE_BYTES=10737418240 +``` + +`MAGPIE_ALLOWED_BUCKETS` should be set in production. Leave it empty only if you intentionally want any bucket name accepted. + +Buckets are private by default. Set `MAGPIE_PUBLIC_PREFIXES` to comma-separated `bucket/prefix` values, such as `app-uploads/avatars`, only when an app intentionally serves unsigned public media through an edge proxy. Set `MAGPIE_PUBLIC_PREFIXES=none` or leave it unset to require signed reads. + +Public reads are hardened against stored-XSS from uploader-controlled content types: responses carry `Content-Security-Policy: default-src 'none'; sandbox`, and any object that is not inert inline media (images, audio, video — SVG excluded) is served with `Content-Disposition: attachment` so browsers download it instead of rendering it. For untrusted user uploads, still prefer serving public objects from an origin separate from your application. + +## Application Integration + +Use any S3-compatible client that supports path-style endpoints and custom hosts. + +If Magpie runs on the same host as your application outside Docker: + +```env +S3_ACCESS_KEY_ID=magpie +S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret +S3_ENDPOINT=127.0.0.1:8090 +S3_BUCKET_NAME=app-uploads +S3_PUBLIC_URL=http://127.0.0.1:8090/app-uploads +S3_SCHEME=http:// +S3_PORT=8090 +``` + +If your app and Magpie are in the same Docker Compose network, use the Magpie service name: + +```env +S3_ENDPOINT=magpie:8090 +S3_BUCKET_NAME=app-uploads +S3_PUBLIC_URL=http://magpie:8090/app-uploads +S3_SCHEME=http:// +S3_PORT=8090 +``` + +Keep Magpie write access private. Your application should enforce user permissions before it writes to Magpie. Public buckets are intended for already-public objects such as avatars and timeline media. + +## API Notes + +S3 object routes use path-style URLs: + +```text +PUT /:bucket/:key +GET /:bucket/:key +HEAD /:bucket/:key +DELETE /:bucket/:key +POST /:bucket?delete +``` + +Objects are stored under: + +```text +MAGPIE_DATA_DIR// +``` + +Internal names are reserved and cannot be used as object keys: + +- `.multipart` +- `.magpie.db` +- `.magpie.db-*` +- `*.meta.json` + +Presigned URLs are read-only. Magpie accepts presigned `GET` and `HEAD` requests, but rejects presigned writes and deletes. + +Normal SigV4 write requests must include a real `X-Amz-Content-Sha256` payload hash. `UNSIGNED-PAYLOAD` is reserved for authenticated replication traffic. + +## Health And Metrics + +Health and metrics require authentication. + +```sh +curl -H "Authorization: Bearer $MAGPIE_AUTH_TOKEN" \ + http://127.0.0.1:8090/health +``` + +```sh +curl -H "Authorization: Bearer $MAGPIE_AUTH_TOKEN" \ + http://127.0.0.1:8090/metrics +``` + +The health endpoint returns `503` when free disk space is below `MAGPIE_MIN_FREE_BYTES`. Writes are rejected in that state. + +If you only use S3 credentials and do not set `MAGPIE_AUTH_TOKEN`, sign health and metrics requests with an admin S3 key instead. + +## CLI + +```sh +magpie serve +magpie stats +magpie scrub +magpie reindex +magpie repair +magpie backup /backup/magpie.tar.gz +magpie restore /backup/magpie.tar.gz +magpie smoke +magpie version +``` + +Command summary: + +- `stats`: object count, logical bytes, disk status, replication queue status +- `scrub`: verifies metadata against files and reports missing, corrupt, and orphaned objects +- `reindex`: rebuilds SQLite metadata from files on disk +- `repair`: queues known objects for replication to configured peers +- `backup`: creates a tar/gzip archive of the data directory +- `restore`: restores a tar/gzip archive into the data directory +- `smoke`: runs a local write/read/list/scrub/delete check + +Restore rejects unsafe archive paths, symlinks, unsupported archive entries, and archives whose expanded file content exceeds `MAGPIE_MAX_RESTORE_BYTES`. Backup archives are created with owner-only permissions. For consistent backups, stop Magpie or use a filesystem snapshot before running `magpie backup`. + +## Replication + +Replication is asynchronous and best-effort. Local writes complete before peers acknowledge them. + +Configure every node with matching S3 credentials and a shared replication secret: + +```env +MAGPIE_REPLICATION_PEERS=https://magpie-b.internal,https://magpie-c.internal +MAGPIE_REPLICATION_SECRET=replace-with-a-long-random-peer-secret +MAGPIE_REPLICATION_MAX_JOBS=10000 +MAGPIE_REPLICATION_MAX_ATTEMPTS=100 +``` + +`MAGPIE_REPLICATION_SECRET` is required when peers are configured. Replication-marked requests without the shared secret are rejected. + +Replication peer URLs must use `https://` by default because Magpie sends SigV4 authorization and the replication secret to peers. Set `MAGPIE_ALLOW_INSECURE_REPLICATION=true` only for trusted local/private networks where plaintext HTTP is acceptable. + +Use `magpie repair` after outages to queue a full object repair pass to all configured peers. + +## Deployment + +Docker Compose example: + +```sh +cp .env.production.example .env.production +``` + +```yaml +services: + magpie: + image: magpie:latest + container_name: magpie + build: . + restart: unless-stopped + env_file: + - .env.production + ports: + - "127.0.0.1:8090:8090" + volumes: + - /var/lib/magpie:/var/lib/magpie +``` + +The default Compose file is standalone and binds Magpie to localhost only. + +If another Compose project needs to reach Magpie by service name, use the optional network override. It attaches Magpie to an external shared network with the `magpie` network alias: + +```sh +docker network create elektrine-magpie-shared +docker compose -f docker-compose.yml -f docker-compose.network.yml up -d --build +``` + +Then attach your application container to the same external `elektrine-magpie-shared` network and use: + +```env +S3_ENDPOINT=magpie:8090 +S3_PUBLIC_URL=http://magpie:8090/app-uploads +S3_SCHEME=http:// +S3_PORT=8090 +``` + +You can rename the shared network without editing the override file: + +```sh +MAGPIE_DOCKER_NETWORK=my-existing-shared-network \ + docker compose -f docker-compose.yml -f docker-compose.network.yml up -d --build +``` + +Bare-metal installs can use `deploy/systemd/magpie.service`. The systemd unit is optional; use it only if you want Magpie managed directly by systemd. + +Back up `/var/lib/magpie` or whatever directory you set as `MAGPIE_DATA_DIR`. + +## Build And Test + +```sh +go test ./... +go build ./... +``` + +Release artifacts: + +```sh +make test +make build +make release +``` + +Artifacts are written to `dist/`, with checksums in `dist/SHA256SUMS`. diff --git a/backup.go b/backup.go new file mode 100644 index 0000000..952dbc3 --- /dev/null +++ b/backup.go @@ -0,0 +1,181 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +func BackupDir(src string, dst string) error { + out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + defer out.Close() + gz := gzip.NewWriter(out) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + root, err := filepath.Abs(src) + if err != nil { + return err + } + return filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil || entry.IsDir() { + return err + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing to back up symlink %q", path) + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + info, err := entry.Info() + if err != nil { + return err + } + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = filepath.ToSlash(rel) + if err := tw.WriteHeader(header); err != nil { + return err + } + file, err := os.Open(path) + if err != nil { + return err + } + fileInfo, statErr := file.Stat() + currentInfo, lstatErr := os.Lstat(path) + if statErr != nil || lstatErr != nil { + _ = file.Close() + if statErr != nil { + return statErr + } + return lstatErr + } + if currentInfo.Mode()&os.ModeSymlink != 0 || !os.SameFile(currentInfo, fileInfo) { + _ = file.Close() + return fmt.Errorf("refusing to back up replaced or symlinked file %q", path) + } + _, copyErr := io.Copy(tw, file) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + return closeErr + }) +} + +func RestoreDir(src string, dst string) error { + return RestoreDirWithLimit(src, dst, defaultMaxRestoreBytes) +} + +func RestoreDirWithLimit(src string, dst string, maxBytes int64) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + gz, err := gzip.NewReader(in) + if err != nil { + return err + } + defer gz.Close() + tr := tar.NewReader(gz) + root, err := filepath.Abs(dst) + if err != nil { + return err + } + if err := os.MkdirAll(root, 0o700); err != nil { + return err + } + var restoredBytes int64 + for { + header, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if strings.Contains(header.Name, "..") || strings.HasPrefix(header.Name, "/") { + return fmt.Errorf("unsafe archive path %q", header.Name) + } + if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA && header.Typeflag != tar.TypeDir { + return fmt.Errorf("unsupported archive entry %q", header.Name) + } + if header.Size < 0 { + return fmt.Errorf("invalid archive size for %q", header.Name) + } + if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeRegA { + restoredBytes += header.Size + if maxBytes > 0 && restoredBytes > maxBytes { + return fmt.Errorf("restore exceeds maximum size") + } + } + path := filepath.Join(root, filepath.FromSlash(header.Name)) + if !strings.HasPrefix(path, root+string(os.PathSeparator)) && path != root { + return fmt.Errorf("unsafe archive path %q", header.Name) + } + if err := rejectSymlinkParents(root, path); err != nil { + return err + } + if header.FileInfo().IsDir() { + if err := os.MkdirAll(path, 0o700); err != nil { + return err + } + continue + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + if info, err := os.Lstat(path); err == nil && info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing to restore through symlink %q", header.Name) + } + out, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + _, copyErr := io.CopyN(out, tr, header.Size) + closeErr := out.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + } +} + +func rejectSymlinkParents(root string, path string) error { + current := root + rel, err := filepath.Rel(root, filepath.Dir(path)) + if err != nil { + return err + } + if rel == "." { + return nil + } + for _, part := range strings.Split(rel, string(os.PathSeparator)) { + current = filepath.Join(current, part) + info, err := os.Lstat(current) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing to restore through symlink parent %q", current) + } + } + return nil +} diff --git a/deploy/systemd/magpie.service b/deploy/systemd/magpie.service new file mode 100644 index 0000000..8d2f34c --- /dev/null +++ b/deploy/systemd/magpie.service @@ -0,0 +1,21 @@ +[Unit] +Description=Magpie object storage service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=magpie +Group=magpie +EnvironmentFile=/etc/magpie/.env.production +ExecStart=/usr/local/bin/magpie +Restart=on-failure +RestartSec=5s +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/magpie + +[Install] +WantedBy=multi-user.target diff --git a/disk_unix.go b/disk_unix.go new file mode 100644 index 0000000..12d3846 --- /dev/null +++ b/disk_unix.go @@ -0,0 +1,21 @@ +//go:build !windows + +package main + +import ( + "fmt" + "syscall" +) + +func (s *FileStore) DiskStatus() (DiskStatus, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(s.root, &stat); err != nil { + return DiskStatus{}, err + } + if stat.Bsize <= 0 { + return DiskStatus{}, fmt.Errorf("invalid filesystem block size") + } + total := stat.Blocks * uint64(stat.Bsize) + free := stat.Bavail * uint64(stat.Bsize) + return DiskStatus{TotalBytes: total, FreeBytes: free, UsedBytes: total - free}, nil +} diff --git a/disk_windows.go b/disk_windows.go new file mode 100644 index 0000000..ecd0b7e --- /dev/null +++ b/disk_windows.go @@ -0,0 +1,7 @@ +//go:build windows + +package main + +func (s *FileStore) DiskStatus() (DiskStatus, error) { + return DiskStatus{}, nil +} diff --git a/docker-compose.network.yml b/docker-compose.network.yml new file mode 100644 index 0000000..e821a0e --- /dev/null +++ b/docker-compose.network.yml @@ -0,0 +1,11 @@ +services: + magpie: + networks: + magpie_shared: + aliases: + - magpie + +networks: + magpie_shared: + external: true + name: ${MAGPIE_DOCKER_NETWORK:-elektrine-magpie-shared} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..eaff37a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +services: + magpie: + image: magpie:latest + container_name: magpie + build: . + restart: unless-stopped + env_file: + - .env.production + ports: + - "127.0.0.1:8090:8090" + volumes: + - magpie-data:/var/lib/magpie + +volumes: + magpie-data: diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..499852d --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module magpie + +go 1.25.0 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.42.0 // indirect + modernc.org/libc v1.72.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.50.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..08ff8cd --- /dev/null +++ b/go.sum @@ -0,0 +1,21 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= +modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM= +modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= diff --git a/main.go b/main.go new file mode 100644 index 0000000..f423de9 --- /dev/null +++ b/main.go @@ -0,0 +1,415 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" +) + +const ( + defaultMaxObjectSize = 1024 * 1024 * 1024 + defaultMinFreeBytes = 1024 * 1024 * 1024 + defaultPresignedMaxExpiry = 24 * time.Hour + defaultMultipartMaxAge = 24 * time.Hour + defaultMaxRestoreBytes = 10 * 1024 * 1024 * 1024 + defaultReplicationMaxJobs = 10_000 + defaultReplicationMaxAttempts = 100 +) + +func main() { + cmd := "serve" + if len(os.Args) > 1 { + cmd = os.Args[1] + } + if cmd == "version" { + printJSON(versionInfo()) + return + } + if cmd == "help" || cmd == "-h" || cmd == "--help" { + printUsage() + return + } + + cfg := configFromEnv() + var store *FileStore + openStore := func() *FileStore { + if store != nil { + return store + } + opened, err := NewFileStore(cfg.DataDir) + if err != nil { + log.Fatalf("init store: %v", err) + } + store = opened + return store + } + + switch cmd { + case "serve": + serve(cfg, openStore()) + case "reindex": + mustJSON(openStore().Reindex()) + case "stats": + stats, err := openStore().Stats() + if err != nil { + log.Fatal(err) + } + printJSON(stats) + case "scrub": + report, err := openStore().Scrub() + if err != nil { + log.Fatal(err) + } + printJSON(report) + case "repair": + count, err := openStore().EnqueueRepair(cfg.ReplicationPeers) + if err != nil { + log.Fatal(err) + } + printJSON(map[string]any{"status": "queued", "jobs": count}) + case "backup": + if len(os.Args) < 3 { + log.Fatal("usage: magpie backup ") + } + mustJSON(BackupDir(cfg.DataDir, os.Args[2])) + case "restore": + if len(os.Args) < 3 { + log.Fatal("usage: magpie restore ") + } + mustJSON(RestoreDirWithLimit(os.Args[2], cfg.DataDir, cfg.MaxRestoreBytes)) + case "smoke": + mustJSON(runSmoke(openStore())) + default: + log.Fatalf("unknown command %q", sanitizeLogValue(cmd)) + } +} + +func serve(cfg Config, store *FileStore) { + if cfg.ReindexOnStart { + log.Printf("reindexing metadata from %s", cfg.DataDir) + if err := store.Reindex(); err != nil { + log.Fatalf("reindex store: %v", err) + } + } + + server := NewServerInstance(store, cfg) + srv := &http.Server{ + Addr: cfg.Addr, + Handler: server.Handler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 2 * time.Minute, + WriteTimeout: 2 * time.Minute, + IdleTimeout: 60 * time.Second, + MaxHeaderBytes: 16 * 1024, + } + + go func() { + log.Printf("object store listening on %s, data=%s", cfg.Addr, cfg.DataDir) + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("listen: %v", err) + } + }() + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + server.startMultipartCleanup(ctx) + server.startReplicationWorker(ctx) + <-ctx.Done() + + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + log.Printf("shutdown: %v", err) + } +} + +type Config struct { + Addr string + DataDir string + AuthToken string + S3AccessKeyID string + S3SecretAccessKey string + S3Keys []AccessKey + S3Region string + AllowUnsignedLocal bool + MaxObjectSize int64 + MinFreeBytes uint64 + PresignedMaxExpiry time.Duration + MultipartMaxAge time.Duration + ReindexOnStart bool + ReplicationPeers []string + ReplicationSecret string + AllowInsecureReplication bool + ReplicationMaxJobs int + ReplicationMaxAttempts int + MaxRestoreBytes int64 + AllowedBuckets map[string]bool + PublicPrefixes map[string]bool + RateLimitPerMinute int + TrustProxyHeaders bool +} + +type AccessKey struct { + ID string + Secret string + Permissions map[string]bool +} + +func configFromEnv() Config { + addr := getenv("MAGPIE_ADDR", "127.0.0.1:8090") + dataDir := getenv("MAGPIE_DATA_DIR", "./data") + authToken := os.Getenv("MAGPIE_AUTH_TOKEN") + s3AccessKeyID := os.Getenv("MAGPIE_S3_ACCESS_KEY_ID") + s3SecretAccessKey := os.Getenv("MAGPIE_S3_SECRET_ACCESS_KEY") + s3Keys := parseAccessKeys(os.Getenv("MAGPIE_S3_KEYS"), s3AccessKeyID, s3SecretAccessKey) + s3Region := getenv("MAGPIE_S3_REGION", "auto") + maxObjectSize := mustParseBytes("MAGPIE_MAX_OBJECT_SIZE", defaultMaxObjectSize) + minFreeBytes := mustParseUint("MAGPIE_MIN_FREE_BYTES", defaultMinFreeBytes) + presignedMaxExpiry := mustParseDuration("MAGPIE_PRESIGNED_MAX_EXPIRY", defaultPresignedMaxExpiry) + multipartMaxAge := mustParseDuration("MAGPIE_MULTIPART_MAX_AGE", defaultMultipartMaxAge) + reindexOnStart := mustParseBool("MAGPIE_REINDEX_ON_START", false) + allowInsecureReplication := mustParseBool("MAGPIE_ALLOW_INSECURE_REPLICATION", false) + replicationPeers := parseReplicationPeers(os.Getenv("MAGPIE_REPLICATION_PEERS"), allowInsecureReplication) + replicationSecret := os.Getenv("MAGPIE_REPLICATION_SECRET") + replicationMaxJobs := mustParseInt("MAGPIE_REPLICATION_MAX_JOBS", defaultReplicationMaxJobs) + replicationMaxAttempts := mustParseInt("MAGPIE_REPLICATION_MAX_ATTEMPTS", defaultReplicationMaxAttempts) + maxRestoreBytes := mustParseBytes("MAGPIE_MAX_RESTORE_BYTES", defaultMaxRestoreBytes) + allowedBuckets := parseSet(os.Getenv("MAGPIE_ALLOWED_BUCKETS")) + publicPrefixes := parsePublicPrefixes(os.Getenv("MAGPIE_PUBLIC_PREFIXES")) + rateLimitPerMinute := mustParseInt("MAGPIE_RATE_LIMIT_PER_MINUTE", 600) + trustProxyHeaders := mustParseBool("MAGPIE_TRUST_PROXY_HEADERS", false) + allowUnsignedLocal := mustParseBool("MAGPIE_ALLOW_UNSIGNED_PAYLOADS", false) + + if authToken == "" && len(s3Keys) == 0 { + log.Fatal("MAGPIE_AUTH_TOKEN or MAGPIE_S3_ACCESS_KEY_ID/MAGPIE_S3_SECRET_ACCESS_KEY or MAGPIE_S3_KEYS is required") + } + if len(replicationPeers) > 0 && replicationSecret == "" { + log.Fatal("MAGPIE_REPLICATION_SECRET is required when MAGPIE_REPLICATION_PEERS is set") + } + + if raw := os.Getenv("MAGPIE_MAX_PROCS"); raw != "" { + if _, err := strconv.Atoi(raw); err != nil { + log.Fatalf("MAGPIE_MAX_PROCS must be an integer: %v", err) + } + } + + return Config{ + Addr: addr, + DataDir: dataDir, + AuthToken: authToken, + S3AccessKeyID: s3AccessKeyID, + S3SecretAccessKey: s3SecretAccessKey, + S3Keys: s3Keys, + S3Region: s3Region, + AllowUnsignedLocal: allowUnsignedLocal, + MaxObjectSize: maxObjectSize, + MinFreeBytes: minFreeBytes, + PresignedMaxExpiry: presignedMaxExpiry, + MultipartMaxAge: multipartMaxAge, + ReindexOnStart: reindexOnStart, + ReplicationPeers: replicationPeers, + ReplicationSecret: replicationSecret, + AllowInsecureReplication: allowInsecureReplication, + ReplicationMaxJobs: replicationMaxJobs, + ReplicationMaxAttempts: replicationMaxAttempts, + MaxRestoreBytes: maxRestoreBytes, + AllowedBuckets: allowedBuckets, + PublicPrefixes: publicPrefixes, + RateLimitPerMinute: rateLimitPerMinute, + TrustProxyHeaders: trustProxyHeaders, + } +} + +func parseAccessKeys(raw string, legacyID string, legacySecret string) []AccessKey { + keys := make([]AccessKey, 0) + if legacyID != "" && legacySecret != "" { + keys = append(keys, AccessKey{ID: legacyID, Secret: legacySecret, Permissions: permissions("read,write,admin")}) + } + for _, entry := range strings.Split(raw, ";") { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + parts := strings.Split(entry, ":") + if len(parts) < 2 || parts[0] == "" || parts[1] == "" { + log.Fatal("MAGPIE_S3_KEYS entries must be id:secret[:read,write,admin]") + } + perms := "read,write" + if len(parts) >= 3 { + perms = parts[2] + } + keys = append(keys, AccessKey{ID: parts[0], Secret: parts[1], Permissions: permissions(perms)}) + } + return keys +} + +func permissions(raw string) map[string]bool { + result := map[string]bool{} + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(strings.ToLower(part)) + if part != "" { + result[part] = true + } + } + return result +} + +func parseCSV(raw string) []string { + parts := make([]string, 0) + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part != "" { + parts = append(parts, strings.TrimRight(part, "/")) + } + } + return parts +} + +func parseReplicationPeers(raw string, allowInsecure bool) []string { + peers := parseCSV(raw) + for _, peer := range peers { + if err := validateReplicationPeer(peer, allowInsecure); err != nil { + log.Fatal(err) + } + } + return peers +} + +func parseSet(raw string) map[string]bool { + values := map[string]bool{} + for _, value := range parseCSV(raw) { + values[value] = true + } + return values +} + +func parsePublicPrefixes(raw string) map[string]bool { + value := strings.TrimSpace(strings.ToLower(raw)) + if value == "none" || value == "false" || value == "0" { + return map[string]bool{} + } + if strings.TrimSpace(raw) != "" { + prefixes := parseSet(raw) + for prefix := range prefixes { + if err := ValidateKey(strings.TrimSuffix(prefix, "/")); err != nil || !strings.Contains(strings.Trim(prefix, "/"), "/") { + log.Fatal("MAGPIE_PUBLIC_PREFIXES entries must be bucket/prefix values") + } + } + return prefixes + } + return map[string]bool{} +} + +func printUsage() { + fmt.Println("usage: magpie [serve|reindex|stats|scrub|repair|version|backup|restore|smoke]") +} + +func mustJSON(err error) { + if err != nil { + log.Fatal(err) + } + printJSON(map[string]string{"status": "ok"}) +} + +func printJSON(value any) { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(value); err != nil { + log.Fatal(err) + } +} + +func getenv(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} + +func mustParseBytes(key string, fallback int64) int64 { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed < 0 { + log.Fatal(fmt.Sprintf("%s must be a non-negative integer byte count", key)) + } + return parsed +} + +func mustParseUint(key string, fallback uint64) uint64 { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := strconv.ParseUint(value, 10, 64) + if err != nil { + log.Fatal(fmt.Sprintf("%s must be a non-negative integer byte count", key)) + } + return parsed +} + +func mustParseDuration(key string, fallback time.Duration) time.Duration { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := time.ParseDuration(value) + if err != nil || parsed <= 0 { + log.Fatal(fmt.Sprintf("%s must be a positive duration like 1h or 24h", key)) + } + return parsed +} + +func mustParseBool(key string, fallback bool) bool { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := strconv.ParseBool(value) + if err != nil { + log.Fatal(fmt.Sprintf("%s must be true or false", key)) + } + return parsed +} + +func mustParseInt(key string, fallback int) int { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := strconv.Atoi(value) + if err != nil || parsed < 0 { + log.Fatal(fmt.Sprintf("%s must be a non-negative integer", key)) + } + return parsed +} + +func runSmoke(store *FileStore) error { + key := "smoke/test.txt" + if _, err := store.Put(key, strings.NewReader("ok"), "text/plain"); err != nil { + return err + } + obj, err := store.Open(key) + if err != nil { + return err + } + _ = obj.Close() + if _, err := store.List("smoke/", 10); err != nil { + return err + } + if _, err := store.Scrub(); err != nil { + return err + } + return store.Delete(key) +} diff --git a/replication.go b/replication.go new file mode 100644 index 0000000..bd6b4ba --- /dev/null +++ b/replication.go @@ -0,0 +1,330 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log" + "net/http" + "net/url" + "strings" + "time" +) + +const ( + replicationOpPut = "put" + replicationOpDelete = "delete" +) + +type ReplicationJob struct { + ID int64 + Peer string + Op string + Key string + Attempts int +} + +func (s *Server) enqueueReplicationPut(key string) { + s.enqueueReplication(replicationOpPut, key) +} + +func (s *Server) enqueueReplicationDelete(key string) { + s.enqueueReplication(replicationOpDelete, key) +} + +func (s *Server) enqueueReplication(op string, key string) { + if len(s.cfg.ReplicationPeers) == 0 || len(s.cfg.S3Keys) == 0 || s.cfg.ReplicationSecret == "" { + return + } + for _, peer := range s.cfg.ReplicationPeers { + if err := s.store.(*FileStore).EnqueueReplication(peer, op, key, s.cfg.ReplicationMaxJobs); err != nil { + log.Printf("replication enqueue op=%s key=%s peer=%s error=%v", op, sanitizeLogValue(key), sanitizeLogValue(peer), err) + } + } + go s.processReplicationQueue(context.Background()) +} + +func (s *Server) startReplicationWorker(ctx context.Context) { + if len(s.cfg.ReplicationPeers) == 0 || len(s.cfg.S3Keys) == 0 || s.cfg.ReplicationSecret == "" { + return + } + go func() { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.processReplicationQueue(ctx) + } + } + }() +} + +func (s *Server) processReplicationQueue(ctx context.Context) { + jobs, err := s.store.(*FileStore).DueReplicationJobs(25) + if err != nil { + log.Printf("replication due jobs error=%v", err) + return + } + for _, job := range jobs { + if ctx.Err() != nil { + return + } + if err := s.performReplicationJob(job); err != nil { + log.Printf("replication job id=%d op=%s key=%s peer=%s error=%v", job.ID, job.Op, sanitizeLogValue(job.Key), sanitizeLogValue(job.Peer), err) + if s.cfg.ReplicationMaxAttempts > 0 && job.Attempts+1 >= s.cfg.ReplicationMaxAttempts { + _ = s.store.(*FileStore).CompleteReplicationJob(job.ID) + continue + } + _ = s.store.(*FileStore).FailReplicationJob(job.ID, job.Attempts+1, err.Error()) + continue + } + _ = s.store.(*FileStore).CompleteReplicationJob(job.ID) + } +} + +func (s *Server) performReplicationJob(job ReplicationJob) error { + switch job.Op { + case replicationOpPut: + return s.performReplicationPut(job.Peer, job.Key) + case replicationOpDelete: + return s.performReplicationDelete(job.Peer, job.Key) + default: + return fmt.Errorf("unknown replication op %q", job.Op) + } +} + +func (s *Server) performReplicationPut(peer string, key string) error { + obj, err := s.store.Open(key) + if err != nil { + return err + } + defer obj.Close() + + peerURL, err := replicationObjectURL(peer, key, s.cfg.AllowInsecureReplication) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodPut, peerURL, obj) + if err != nil { + return err + } + req.Header.Set("Content-Type", obj.Info.ContentType) + req.Header.Set("X-Magpie-Replication", "1") + if s.cfg.ReplicationSecret != "" { + req.Header.Set("X-Magpie-Replication-Secret", s.cfg.ReplicationSecret) + } + s.signReplication(req) + resp, err := (&http.Client{Timeout: 5 * time.Minute}).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("peer status %d", resp.StatusCode) + } + return nil +} + +func (s *Server) performReplicationDelete(peer string, key string) error { + peerURL, err := replicationObjectURL(peer, key, s.cfg.AllowInsecureReplication) + if err != nil { + return err + } + req, err := http.NewRequest(http.MethodDelete, peerURL, nil) + if err != nil { + return err + } + req.Header.Set("X-Magpie-Replication", "1") + if s.cfg.ReplicationSecret != "" { + req.Header.Set("X-Magpie-Replication-Secret", s.cfg.ReplicationSecret) + } + s.signReplication(req) + resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("peer status %d", resp.StatusCode) + } + return nil +} + +func (s *Server) signReplication(req *http.Request) { + key := s.cfg.S3Keys[0] + amzDate := time.Now().UTC().Format("20060102T150405Z") + date := amzDate[:8] + scope := date + "/" + s.cfg.S3Region + "/s3/aws4_request" + signedHeaders := "host;x-amz-content-sha256;x-amz-date;x-magpie-replication" + if req.Header.Get("X-Magpie-Replication-Secret") != "" { + signedHeaders += ";x-magpie-replication-secret" + } + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD") + canonical := canonicalRequest(req, signedHeaders, "UNSIGNED-PAYLOAD", false) + sig := signature(key.Secret, amzDate, scope, canonical) + req.Header.Set("Authorization", awsAlgorithm+" Credential="+key.ID+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig) +} + +func replicationObjectURL(peer string, key string, allowInsecure bool) (string, error) { + if err := ValidateKey(key); err != nil { + return "", err + } + if err := validateReplicationPeer(peer, allowInsecure); err != nil { + return "", err + } + parsed, err := url.Parse(peer) + if err != nil { + return "", err + } + parsed.RawQuery = "" + parsed.Fragment = "" + parsed.Path = strings.TrimRight(parsed.Path, "/") + "/" + key + parsed.RawPath = "" + return parsed.String(), nil +} + +func validateReplicationPeer(peer string, allowInsecure bool) error { + parsed, err := url.Parse(peer) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return fmt.Errorf("invalid replication peer %q", peer) + } + if parsed.RawQuery != "" || parsed.Fragment != "" { + return fmt.Errorf("replication peer %q must not include query or fragment", peer) + } + if parsed.Scheme == "https" { + return nil + } + if allowInsecure && parsed.Scheme == "http" { + return nil + } + return fmt.Errorf("replication peer %q must use https or set MAGPIE_ALLOW_INSECURE_REPLICATION=true", peer) +} + +func (s *FileStore) EnqueueReplication(peer string, op string, key string, maxJobs int) error { + now := time.Now().UTC().Format(time.RFC3339Nano) + result, err := s.db.Exec(` +UPDATE replication_jobs +SET next_attempt_at = ?, updated_at = ? +WHERE peer = ? AND op = ? AND key = ? +`, now, now, peer, op, key) + if err != nil { + return err + } + if rows, err := result.RowsAffected(); err == nil && rows > 0 { + return nil + } + if maxJobs <= 0 { + maxJobs = defaultReplicationMaxJobs + } + var pending int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM replication_jobs`).Scan(&pending); err != nil { + return err + } + if pending >= maxJobs { + return fmt.Errorf("replication queue is full") + } + _, err = s.db.Exec(` +INSERT INTO replication_jobs (peer, op, key, next_attempt_at, created_at, updated_at) +VALUES (?, ?, ?, ?, ?, ?) +`, peer, op, key, now, now, now) + return err +} + +func (s *FileStore) DueReplicationJobs(limit int) ([]ReplicationJob, error) { + if limit <= 0 || limit > 100 { + limit = 25 + } + now := time.Now().UTC().Format(time.RFC3339Nano) + rows, err := s.db.Query(` +SELECT id, peer, op, key, attempts +FROM replication_jobs +WHERE next_attempt_at <= ? +ORDER BY id ASC +LIMIT ? +`, now, limit) + if err != nil { + return nil, err + } + defer rows.Close() + jobs := make([]ReplicationJob, 0) + for rows.Next() { + var job ReplicationJob + if err := rows.Scan(&job.ID, &job.Peer, &job.Op, &job.Key, &job.Attempts); err != nil { + return nil, err + } + jobs = append(jobs, job) + } + return jobs, rows.Err() +} + +func (s *FileStore) CompleteReplicationJob(id int64) error { + _, err := s.db.Exec(`DELETE FROM replication_jobs WHERE id = ?`, id) + return err +} + +func (s *FileStore) FailReplicationJob(id int64, attempts int, message string) error { + delay := time.Duration(attempts*attempts) * time.Second + if delay > 5*time.Minute { + delay = 5 * time.Minute + } + next := time.Now().UTC().Add(delay).Format(time.RFC3339Nano) + now := time.Now().UTC().Format(time.RFC3339Nano) + _, err := s.db.Exec(` +UPDATE replication_jobs +SET attempts = ?, next_attempt_at = ?, last_error = ?, updated_at = ? +WHERE id = ? +`, attempts, next, truncateError(message), now, id) + return err +} + +func (s *FileStore) EnqueueRepair(peers []string) (int, error) { + if len(peers) == 0 { + return 0, nil + } + count := 0 + after := "" + for { + page, err := s.ListPage("", after, 1000) + if err != nil { + return count, err + } + for _, object := range page.Objects { + for _, peer := range peers { + if err := s.EnqueueReplication(peer, replicationOpPut, object.Key, defaultReplicationMaxJobs); err != nil { + return count, err + } + count++ + } + } + if page.NextAfter == "" { + return count, nil + } + after = page.NextAfter + } +} + +func (s *FileStore) ReplicationQueueStats() (ReplicationQueueStats, error) { + var stats ReplicationQueueStats + err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(attempts), 0) FROM replication_jobs`).Scan(&stats.PendingJobs, &stats.MaxAttempts) + if errors.Is(err, sql.ErrNoRows) { + return stats, nil + } + return stats, err +} + +func truncateError(message string) string { + if len(message) <= 500 { + return message + } + return message[:500] +} + +type ReplicationQueueStats struct { + PendingJobs int64 `json:"pending_jobs"` + MaxAttempts int `json:"max_attempts"` +} diff --git a/s3.go b/s3.go new file mode 100644 index 0000000..2bd19eb --- /dev/null +++ b/s3.go @@ -0,0 +1,938 @@ +package main + +import ( + "context" + "crypto/md5" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +const ( + maxXMLRequestBytes = 1 << 20 + maxBatchDeleteItems = 1000 +) + +var errPayloadHashMismatch = errors.New("payload hash mismatch") + +func (s *Server) s3Object(w http.ResponseWriter, r *http.Request) { + if s.s3BucketList(w, r) { + return + } + if s.s3BucketPost(w, r) { + return + } + + bucket, key, ok := s.s3BucketKey(r.URL.Path) + if !ok { + s3Error(w, http.StatusNotFound, "NoSuchBucket", "bucket not found") + return + } + if !s.bucketAllowed(bucket) { + s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed") + return + } + + storeKey := bucket + "/" + key + if err := ValidateKey(storeKey); err != nil { + s3Error(w, http.StatusBadRequest, "InvalidObjectName", err.Error()) + return + } + publicRead := (r.Method == http.MethodGet || r.Method == http.MethodHead) && s.publicReadAllowed(storeKey) + if !publicRead && !s.authorizedS3(r) { + s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid") + return + } + if !s.replicationAllowed(r) { + s3Error(w, http.StatusForbidden, "AccessDenied", "replication secret is invalid") + return + } + + switch r.Method { + case http.MethodPost: + if _, ok := r.URL.Query()["uploads"]; ok { + s.s3CreateMultipartUpload(w, r, storeKey) + } else if r.URL.Query().Get("uploadId") != "" { + s.s3CompleteMultipartUpload(w, r, storeKey) + } else if _, ok := r.URL.Query()["delete"]; ok { + s.s3DeleteObjects(w, r, bucket) + } else { + s3Error(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed") + } + case http.MethodPut: + if r.URL.Query().Get("uploadId") != "" && r.URL.Query().Get("partNumber") != "" { + s.s3UploadPart(w, r, storeKey) + } else if r.Header.Get("X-Amz-Copy-Source") != "" { + s.s3CopyObject(w, r, storeKey) + } else { + s.s3Put(w, r, storeKey) + } + case http.MethodGet: + s.s3Get(w, r, storeKey, publicRead) + case http.MethodHead: + s.s3Head(w, r, storeKey, publicRead) + case http.MethodDelete: + if r.URL.Query().Get("uploadId") != "" { + s.s3AbortMultipartUpload(w, r, storeKey) + } else { + s.s3Delete(w, r, storeKey) + } + default: + w.Header().Set("Allow", "PUT, GET, HEAD, DELETE") + s3Error(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed") + } +} + +func (s *Server) s3BucketPost(w http.ResponseWriter, r *http.Request) bool { + path := strings.Trim(r.URL.Path, "/") + if r.Method != http.MethodPost || path == "" || strings.Contains(path, "/") { + return false + } + if _, ok := r.URL.Query()["delete"]; !ok { + return false + } + if err := ValidateKey(path); err != nil { + s3Error(w, http.StatusBadRequest, "InvalidBucketName", err.Error()) + return true + } + if !s.bucketAllowed(path) { + s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed") + return true + } + if !s.authorizedS3(r) { + s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid") + return true + } + if !s.replicationAllowed(r) { + s3Error(w, http.StatusForbidden, "AccessDenied", "replication secret is invalid") + return true + } + s.s3DeleteObjects(w, r, path) + return true +} + +func (s *Server) s3CreateMultipartUpload(w http.ResponseWriter, r *http.Request, key string) { + uploadID, err := randomID() + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "upload id failed") + return + } + dir := s.multipartDir(uploadID) + if err := os.MkdirAll(dir, 0o700); err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "upload init failed") + return + } + meta := multipartMeta{Key: key, ContentType: r.Header.Get("Content-Type")} + data, _ := xml.Marshal(meta) + if err := os.WriteFile(filepath.Join(dir, "meta.xml"), data, 0o600); err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "upload init failed") + return + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _ = xml.NewEncoder(w).Encode(createMultipartUploadResult{UploadID: uploadID}) +} + +func (s *Server) s3UploadPart(w http.ResponseWriter, r *http.Request, key string) { + release, ok := s.acceptWrite(w, r, true) + if !ok { + return + } + defer release() + uploadID := r.URL.Query().Get("uploadId") + partNumber, err := strconv.Atoi(r.URL.Query().Get("partNumber")) + if err != nil || partNumber <= 0 || partNumber > 10_000 { + s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid part number") + return + } + meta, err := s.multipartMeta(uploadID) + if err != nil || meta.Key != key { + s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found") + return + } + dir := s.multipartDir(uploadID) + if !s.acceptMultipartPartSize(w, dir, r.ContentLength) { + return + } + partPath := filepath.Join(dir, fmt.Sprintf("part-%05d", partNumber)) + body, verifyPayload, ok := s.s3StreamingBody(w, r) + if !ok { + return + } + tmp, err := os.CreateTemp(dir, ".part-*") + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed") + return + } + hash := md5Hash() + _, copyErr := io.Copy(io.MultiWriter(tmp, hash), http.MaxBytesReader(w, body, s.cfg.MaxObjectSize)) + closeErr := tmp.Close() + if copyErr != nil || closeErr != nil { + _ = os.Remove(tmp.Name()) + if errors.Is(copyErr, errPayloadHashMismatch) { + s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch") + return + } + s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed") + return + } + if !verifyPayload() { + _ = os.Remove(tmp.Name()) + s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch") + return + } + if err := os.Rename(tmp.Name(), partPath); err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed") + return + } + etag := fmt.Sprintf("%x", hash.Sum(nil)) + w.Header().Set("ETag", quoteETag(etag)) + w.WriteHeader(http.StatusOK) +} + +func (s *Server) acceptMultipartPartSize(w http.ResponseWriter, dir string, incomingSize int64) bool { + if incomingSize < 0 { + s3Error(w, http.StatusLengthRequired, "MissingContentLength", "content length is required") + return false + } + if s.cfg.MaxObjectSize <= 0 { + return true + } + + existingSize := int64(0) + parts, _ := filepath.Glob(filepath.Join(dir, "part-*")) + for _, part := range parts { + info, err := os.Stat(part) + if err == nil && !info.IsDir() { + existingSize += info.Size() + } + } + + if existingSize+incomingSize > s.cfg.MaxObjectSize { + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "multipart upload exceeds object size limit") + return false + } + + return true +} + +func (s *Server) s3CompleteMultipartUpload(w http.ResponseWriter, r *http.Request, key string) { + uploadID := r.URL.Query().Get("uploadId") + meta, err := s.multipartMeta(uploadID) + if err != nil || meta.Key != key { + s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found") + return + } + var complete completeMultipartUpload + completeBody, ok := s.s3BoundedBody(w, r, maxXMLRequestBytes) + if !ok { + return + } + if err := xml.Unmarshal(completeBody, &complete); err != nil && len(completeBody) > 0 { + s3Error(w, http.StatusBadRequest, "MalformedXML", "invalid complete request") + return + } + parts := complete.Parts + if len(parts) == 0 { + entries, _ := filepath.Glob(filepath.Join(s.multipartDir(uploadID), "part-*")) + for _, entry := range entries { + base := filepath.Base(entry) + n, _ := strconv.Atoi(strings.TrimPrefix(base, "part-")) + parts = append(parts, completedPart{PartNumber: n}) + } + } + if len(parts) > 10_000 { + s3Error(w, http.StatusBadRequest, "InvalidRequest", "too many parts") + return + } + sortParts(parts) + readers := make([]io.Reader, 0, len(parts)) + files := make([]*os.File, 0, len(parts)) + defer func() { + for _, file := range files { + _ = file.Close() + } + }() + var totalSize int64 + for _, part := range parts { + if part.PartNumber <= 0 || part.PartNumber > 10_000 { + s3Error(w, http.StatusBadRequest, "InvalidPart", "invalid part number") + return + } + file, err := os.Open(filepath.Join(s.multipartDir(uploadID), fmt.Sprintf("part-%05d", part.PartNumber))) + if err != nil { + s3Error(w, http.StatusBadRequest, "InvalidPart", "part missing") + return + } + stat, err := file.Stat() + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "part stat failed") + return + } + totalSize += stat.Size() + if s.cfg.MaxObjectSize > 0 && totalSize > s.cfg.MaxObjectSize { + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large") + return + } + files = append(files, file) + readers = append(readers, file) + } + release, ok := s.acceptWriteSize(w, totalSize, true) + if !ok { + return + } + defer release() + info, err := s.store.Put(key, io.MultiReader(readers...), meta.ContentType) + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "complete failed") + return + } + _ = os.RemoveAll(s.multipartDir(uploadID)) + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _ = xml.NewEncoder(w).Encode(completeMultipartUploadResult{Key: strings.TrimPrefix(key, strings.SplitN(key, "/", 2)[0]+"/"), ETag: quoteETag(info.ETag)}) + if r.Header.Get("X-Magpie-Replication") != "1" { + s.enqueueReplicationPut(key) + } +} + +func (s *Server) s3AbortMultipartUpload(w http.ResponseWriter, r *http.Request, _ string) { + uploadID := r.URL.Query().Get("uploadId") + if !validUploadID(uploadID) { + s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found") + return + } + _ = os.RemoveAll(s.multipartDir(uploadID)) + w.WriteHeader(http.StatusNoContent) +} + +func (s *Server) s3CopyObject(w http.ResponseWriter, r *http.Request, destinationKey string) { + if !signedHeaderIncludes(r, "x-amz-copy-source") { + s3Error(w, http.StatusForbidden, "AccessDenied", "copy source header must be signed") + return + } + sourceKey, err := copySourceKey(r.Header.Get("X-Amz-Copy-Source")) + if err != nil { + s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid copy source") + return + } + sourceBucket, _, _ := strings.Cut(sourceKey, "/") + if !s.bucketAllowed(sourceBucket) { + s3Error(w, http.StatusForbidden, "AccessDenied", "copy source bucket is not allowed") + return + } + if len(s.cfg.S3Keys) > 0 { + key, ok := s.requestAccessKey(r) + if !ok || !key.can("read") || !key.can("write") { + s3Error(w, http.StatusForbidden, "AccessDenied", "copy requires read and write permission") + return + } + } + obj, err := s.store.Open(sourceKey) + if err != nil { + if errors.Is(err, ErrNotFound) { + s3Error(w, http.StatusNotFound, "NoSuchKey", "copy source not found") + return + } + s3Error(w, http.StatusInternalServerError, "InternalError", "copy source failed") + return + } + defer obj.Close() + if s.cfg.MaxObjectSize > 0 && obj.Info.Size > s.cfg.MaxObjectSize { + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large") + return + } + release, ok := s.acceptWriteSize(w, obj.Info.Size, true) + if !ok { + return + } + defer release() + contentType := obj.Info.ContentType + if override := r.Header.Get("Content-Type"); override != "" && strings.EqualFold(r.Header.Get("X-Amz-Metadata-Directive"), "REPLACE") { + contentType = override + } + info, err := s.store.Put(destinationKey, obj, contentType) + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "copy failed") + return + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _ = xml.NewEncoder(w).Encode(copyObjectResult{ETag: quoteETag(info.ETag), LastModified: info.ModTime.UTC().Format(time.RFC3339)}) + if r.Header.Get("X-Magpie-Replication") != "1" { + s.enqueueReplicationPut(destinationKey) + } +} + +func (s *Server) s3DeleteObjects(w http.ResponseWriter, r *http.Request, bucket string) { + var request deleteObjectsRequest + deleteBody, ok := s.s3BoundedBody(w, r, maxXMLRequestBytes) + if !ok { + return + } + if err := xml.Unmarshal(deleteBody, &request); err != nil { + s3Error(w, http.StatusBadRequest, "MalformedXML", "invalid delete request") + return + } + if len(request.Objects) > maxBatchDeleteItems { + s3Error(w, http.StatusBadRequest, "InvalidRequest", "too many objects in delete request") + return + } + result := deleteObjectsResult{Deleted: make([]deletedObject, 0, len(request.Objects))} + for _, object := range request.Objects { + key := strings.TrimPrefix(object.Key, "/") + storeKey := bucket + "/" + key + if err := ValidateKey(storeKey); err != nil { + result.Errors = append(result.Errors, deleteObjectError{Key: object.Key, Code: "InvalidObjectName", Message: err.Error()}) + continue + } + if err := s.store.Delete(storeKey); err != nil && !errors.Is(err, ErrNotFound) { + result.Errors = append(result.Errors, deleteObjectError{Key: object.Key, Code: "InternalError", Message: "delete failed"}) + continue + } + result.Deleted = append(result.Deleted, deletedObject{Key: object.Key}) + if r.Header.Get("X-Magpie-Replication") != "1" { + s.enqueueReplicationDelete(storeKey) + } + } + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _ = xml.NewEncoder(w).Encode(result) +} + +func (s *Server) s3BucketList(w http.ResponseWriter, r *http.Request) bool { + if r.Method != http.MethodGet || strings.Trim(r.URL.Path, "/") == "" || strings.Contains(strings.Trim(r.URL.Path, "/"), "/") { + return false + } + if r.URL.RawQuery == "" && r.URL.Query().Get("prefix") == "" && r.URL.Query().Get("list-type") == "" { + return false + } + + bucket := strings.Trim(r.URL.Path, "/") + if err := ValidateKey(bucket); err != nil { + s3Error(w, http.StatusBadRequest, "InvalidBucketName", err.Error()) + return true + } + if !s.bucketAllowed(bucket) { + s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed") + return true + } + if !s.authorizedS3(r) { + s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid") + return true + } + + prefix := r.URL.Query().Get("prefix") + storePrefix := bucket + "/" + if prefix != "" { + storePrefix = bucket + "/" + strings.TrimPrefix(prefix, "/") + } + maxKeys := parseMaxKeys(r.URL.Query().Get("max-keys")) + continuationToken := r.URL.Query().Get("continuation-token") + after, err := decodeContinuationToken(continuationToken) + if err != nil { + s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid continuation token") + return true + } + page, err := s.store.ListPage(storePrefix, after, maxKeys) + if err != nil { + s3Error(w, http.StatusInternalServerError, "InternalError", "list failed") + return true + } + + contents := make([]s3ListContent, 0, len(page.Objects)) + for _, object := range page.Objects { + key := strings.TrimPrefix(object.Key, bucket+"/") + contents = append(contents, s3ListContent{ + Key: key, + LastModified: object.ModTime.UTC().Format(time.RFC3339), + ETag: quoteETag(object.ETag), + Size: object.Size, + StorageClass: "STANDARD", + }) + } + + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusOK) + _ = xml.NewEncoder(w).Encode(s3ListBucketResult{ + Name: bucket, + Prefix: prefix, + ContinuationToken: continuationToken, + NextContinuationToken: encodeContinuationToken(page.NextAfter), + KeyCount: len(contents), + MaxKeys: maxKeys, + IsTruncated: page.NextAfter != "", + Contents: contents, + }) + return true +} + +func encodeContinuationToken(key string) string { + if key == "" { + return "" + } + return base64.RawURLEncoding.EncodeToString([]byte(key)) +} + +func decodeContinuationToken(token string) (string, error) { + if token == "" { + return "", nil + } + decoded, err := base64.RawURLEncoding.DecodeString(token) + if err != nil { + return "", err + } + key := string(decoded) + if err := ValidateKey(key); err != nil { + return "", err + } + return key, nil +} + +func copySourceKey(value string) (string, error) { + value = strings.TrimPrefix(value, "/") + if value == "" { + return "", fmt.Errorf("missing copy source") + } + if decoded, err := url.PathUnescape(value); err == nil { + value = decoded + } + if err := ValidateKey(value); err != nil { + return "", err + } + return value, nil +} + +func (s *Server) s3BucketKey(path string) (string, string, bool) { + path = strings.TrimPrefix(path, "/") + parts := strings.SplitN(path, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", false + } + if err := ValidateKey(parts[0]); err != nil { + return "", "", false + } + if err := ValidateKey(parts[1]); err != nil { + return "", "", false + } + return parts[0], parts[1], true +} + +func (s *Server) s3Put(w http.ResponseWriter, r *http.Request, key string) { + release, ok := s.acceptWrite(w, r, true) + if !ok { + return + } + defer release() + contentType := r.Header.Get("Content-Type") + body, verifyPayload, ok := s.s3StreamingBody(w, r) + if !ok { + return + } + store, ok := s.store.(*FileStore) + if !ok { + s3Error(w, http.StatusInternalServerError, "InternalError", "store failed") + return + } + info, err := store.PutVerified(key, http.MaxBytesReader(w, body, s.cfg.MaxObjectSize), contentType, func() error { + if !verifyPayload() { + return errPayloadHashMismatch + } + return nil + }) + if err != nil { + if maxBytesError(err) { + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large") + return + } + if errors.Is(err, errPayloadHashMismatch) { + s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch") + return + } + s3Error(w, http.StatusInternalServerError, "InternalError", "store failed") + return + } + w.Header().Set("ETag", quoteETag(info.ETag)) + w.WriteHeader(http.StatusOK) + if r.Header.Get("X-Magpie-Replication") != "1" { + s.enqueueReplicationPut(key) + } +} + +func (s *Server) s3Get(w http.ResponseWriter, r *http.Request, key string, publicRead bool) { + obj, err := s.store.Open(key) + if err != nil { + if errors.Is(err, ErrNotFound) { + s3Error(w, http.StatusNotFound, "NoSuchKey", "object not found") + return + } + s3Error(w, http.StatusInternalServerError, "InternalError", "read failed") + return + } + defer obj.Close() + + s3ObjectHeaders(w, obj.Info) + if publicRead { + hardenPublicRead(w, obj.Info.ContentType) + } else if disposition := r.URL.Query().Get("response-content-disposition"); disposition != "" { + w.Header().Set("Content-Disposition", disposition) + } + http.ServeContent(w, r, obj.Info.Key, obj.Info.ModTime, obj) +} + +func (s *Server) s3Head(w http.ResponseWriter, _ *http.Request, key string, publicRead bool) { + info, err := s.store.Stat(key) + if err != nil { + if errors.Is(err, ErrNotFound) { + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(http.StatusInternalServerError) + return + } + s3ObjectHeaders(w, info) + if publicRead { + hardenPublicRead(w, info.ContentType) + } + w.WriteHeader(http.StatusOK) +} + +// hardenPublicRead applies defense-in-depth headers to unauthenticated public +// reads. Objects carry an uploader-supplied Content-Type, so a public prefix +// could otherwise serve active content (HTML, SVG) that executes script in +// Magpie's own origin. The CSP sandbox disables scripting even when a browser +// navigates directly to such an object, and any content type that is not inert +// inline media is forced to download rather than render. The attacker-influenced +// response-content-disposition override is intentionally not honored here. +func hardenPublicRead(w http.ResponseWriter, contentType string) { + w.Header().Set("Content-Security-Policy", "default-src 'none'; sandbox") + if !inlineSafeContentType(contentType) { + w.Header().Set("Content-Disposition", "attachment") + } +} + +// inlineSafeContentType reports whether a content type is inert raster/media +// that browsers render without executing script. SVG is excluded because it can +// carry script when loaded as a top-level document. +func inlineSafeContentType(contentType string) bool { + mediaType := contentType + if i := strings.IndexByte(mediaType, ';'); i >= 0 { + mediaType = mediaType[:i] + } + mediaType = strings.ToLower(strings.TrimSpace(mediaType)) + if mediaType == "image/svg+xml" || mediaType == "image/svg" { + return false + } + return strings.HasPrefix(mediaType, "image/") || + strings.HasPrefix(mediaType, "audio/") || + strings.HasPrefix(mediaType, "video/") +} + +func (s *Server) s3Delete(w http.ResponseWriter, r *http.Request, key string) { + if err := s.store.Delete(key); err != nil && !errors.Is(err, ErrNotFound) { + s3Error(w, http.StatusInternalServerError, "InternalError", "delete failed") + return + } + w.WriteHeader(http.StatusNoContent) + if r.Header.Get("X-Magpie-Replication") != "1" { + s.enqueueReplicationDelete(key) + } +} + +func s3ObjectHeaders(w http.ResponseWriter, info ObjectInfo) { + contentType := info.ContentType + if contentType == "" { + contentType = "application/octet-stream" + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size)) + w.Header().Set("ETag", quoteETag(info.ETag)) + w.Header().Set("Last-Modified", info.ModTime.UTC().Format(http.TimeFormat)) + w.Header().Set("Accept-Ranges", "bytes") + w.Header().Set("X-Content-Type-Options", "nosniff") +} + +func quoteETag(etag string) string { + if etag == "" { + return "\"\"" + } + if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") { + return etag + } + return "\"" + etag + "\"" +} + +type s3ErrorBody struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` +} + +func s3Error(w http.ResponseWriter, status int, code string, message string) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(status) + _ = xml.NewEncoder(w).Encode(s3ErrorBody{Code: code, Message: message}) +} + +func parseMaxKeys(value string) int { + if value == "" { + return 1000 + } + parsed, err := strconv.Atoi(value) + if err != nil || parsed <= 0 { + return 1000 + } + if parsed > 1000 { + return 1000 + } + return parsed +} + +func randomID() (string, error) { + buf := make([]byte, 18) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(buf), nil +} + +func md5Hash() hashHash { + return md5.New() +} + +type hashHash interface { + io.Writer + Sum([]byte) []byte +} + +func (s *Server) multipartDir(uploadID string) string { + if !validUploadID(uploadID) { + return filepath.Join(s.store.(*FileStore).root, ".multipart", "invalid") + } + return filepath.Join(s.store.(*FileStore).root, ".multipart", uploadID) +} + +func validUploadID(uploadID string) bool { + if uploadID == "" || len(uploadID) > 128 { + return false + } + for _, ch := range uploadID { + if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '-' || ch == '_' { + continue + } + return false + } + return true +} + +func (s *Server) startMultipartCleanup(ctx context.Context) { + store, ok := s.store.(*FileStore) + if !ok || s.cfg.MultipartMaxAge <= 0 { + return + } + if err := store.CleanupMultipart(s.cfg.MultipartMaxAge); err != nil { + log.Printf("multipart cleanup error=%v", err) + } + go func() { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := store.CleanupMultipart(s.cfg.MultipartMaxAge); err != nil { + log.Printf("multipart cleanup error=%v", err) + } + } + } + }() +} + +func (s *Server) s3BoundedBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, bool) { + body, verifyPayload, ok := s.s3StreamingBody(w, r) + if !ok { + return nil, false + } + data, err := io.ReadAll(http.MaxBytesReader(w, body, limit)) + if err != nil { + if errors.Is(err, errPayloadHashMismatch) { + s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch") + return nil, false + } + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "request body too large") + return nil, false + } + if !verifyPayload() { + s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch") + return nil, false + } + return data, true +} + +func (s *Server) s3StreamingBody(w http.ResponseWriter, r *http.Request) (io.ReadCloser, func() bool, bool) { + payloadHash := r.Header.Get("X-Amz-Content-Sha256") + if payloadHash == "" || payloadHash == "UNSIGNED-PAYLOAD" { + return r.Body, func() bool { return true }, true + } + if len(payloadHash) != sha256.Size*2 { + s3Error(w, http.StatusBadRequest, "InvalidDigest", "unsupported payload hash") + return nil, nil, false + } + if _, err := hex.DecodeString(payloadHash); err != nil { + s3Error(w, http.StatusBadRequest, "InvalidDigest", "invalid payload hash") + return nil, nil, false + } + hash := sha256.New() + verified := false + return hashVerifyingReadCloser{reader: r.Body, hash: hash, expected: strings.ToLower(payloadHash)}, func() bool { + if verified { + return true + } + verified = hex.EncodeToString(hash.Sum(nil)) == strings.ToLower(payloadHash) + return verified + }, true +} + +func signedHeaderIncludes(r *http.Request, header string) bool { + signedHeaders := r.URL.Query().Get("X-Amz-SignedHeaders") + if signedHeaders == "" { + auth := r.Header.Get("Authorization") + if strings.HasPrefix(auth, awsAlgorithm+" ") { + signedHeaders = parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" "))["SignedHeaders"] + } + } + for _, signed := range strings.Split(signedHeaders, ";") { + if strings.EqualFold(strings.TrimSpace(signed), header) { + return true + } + } + return false +} + +type hashVerifyingReadCloser struct { + reader io.ReadCloser + hash hashHash + expected string +} + +func (r hashVerifyingReadCloser) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + if n > 0 { + _, _ = r.hash.Write(p[:n]) + } + if errors.Is(err, io.EOF) && hex.EncodeToString(r.hash.Sum(nil)) != r.expected { + return n, errPayloadHashMismatch + } + return n, err +} + +func (r hashVerifyingReadCloser) Close() error { + return r.reader.Close() +} + +func (s *Server) multipartMeta(uploadID string) (multipartMeta, error) { + data, err := os.ReadFile(filepath.Join(s.multipartDir(uploadID), "meta.xml")) + if err != nil { + return multipartMeta{}, err + } + var meta multipartMeta + if err := xml.Unmarshal(data, &meta); err != nil { + return multipartMeta{}, err + } + return meta, nil +} + +func sortParts(parts []completedPart) { + sort.Slice(parts, func(i, j int) bool { return parts[i].PartNumber < parts[j].PartNumber }) +} + +type s3ListBucketResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Xmlns string `xml:"xmlns,attr,omitempty"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + ContinuationToken string `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + KeyCount int `xml:"KeyCount"` + MaxKeys int `xml:"MaxKeys"` + IsTruncated bool `xml:"IsTruncated"` + Contents []s3ListContent `xml:"Contents"` +} + +type s3ListContent struct { + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` + StorageClass string `xml:"StorageClass"` +} + +type multipartMeta struct { + Key string `xml:"Key"` + ContentType string `xml:"ContentType"` +} + +type createMultipartUploadResult struct { + XMLName xml.Name `xml:"InitiateMultipartUploadResult"` + UploadID string `xml:"UploadId"` +} + +type completeMultipartUpload struct { + Parts []completedPart `xml:"Part"` +} + +type completedPart struct { + PartNumber int `xml:"PartNumber"` + ETag string `xml:"ETag"` +} + +type completeMultipartUploadResult struct { + XMLName xml.Name `xml:"CompleteMultipartUploadResult"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` +} + +type copyObjectResult struct { + XMLName xml.Name `xml:"CopyObjectResult"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` +} + +type deleteObjectsRequest struct { + Objects []deleteObjectRequest `xml:"Object"` +} + +type deleteObjectRequest struct { + Key string `xml:"Key"` +} + +type deleteObjectsResult struct { + XMLName xml.Name `xml:"DeleteResult"` + Deleted []deletedObject `xml:"Deleted"` + Errors []deleteObjectError `xml:"Error,omitempty"` +} + +type deletedObject struct { + Key string `xml:"Key"` +} + +type deleteObjectError struct { + Key string `xml:"Key"` + Code string `xml:"Code"` + Message string `xml:"Message"` +} diff --git a/s3auth.go b/s3auth.go new file mode 100644 index 0000000..c8fd6ee --- /dev/null +++ b/s3auth.go @@ -0,0 +1,312 @@ +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +const awsAlgorithm = "AWS4-HMAC-SHA256" + +func (s *Server) authorizedS3(r *http.Request) bool { + if len(s.cfg.S3Keys) == 0 { + return s.authorized(r) + } + + if r.URL.Query().Get("X-Amz-Algorithm") != "" { + return s.authorizedS3Presigned(r) + } + + return s.authorizedS3Header(r) +} + +func (s *Server) authorizedS3Presigned(r *http.Request) bool { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + return false + } + query := r.URL.Query() + if query.Get("X-Amz-Algorithm") != awsAlgorithm { + return false + } + if query.Get("X-Amz-Signature") == "" { + return false + } + + credential := query.Get("X-Amz-Credential") + accessKey, scope, ok := splitCredential(credential) + key, ok := s.accessKey(accessKey) + if !ok || !key.can(permissionForMethod(r.Method)) { + return false + } + + amzDate := query.Get("X-Amz-Date") + if !s.presignedTimeValid(amzDate, query.Get("X-Amz-Expires")) { + return false + } + if !s.scopeAllowed(scope, amzDate) { + return false + } + + signedHeaders := query.Get("X-Amz-SignedHeaders") + canonicalRequest := canonicalRequest(r, signedHeaders, "UNSIGNED-PAYLOAD", true) + expected := signature(key.Secret, amzDate, scope, canonicalRequest) + return hmac.Equal([]byte(expected), []byte(query.Get("X-Amz-Signature"))) +} + +func (s *Server) authorizedS3Header(r *http.Request) bool { + return s.authorizedS3HeaderPermission(r, permissionForMethod(r.Method)) +} + +func (s *Server) authorizedS3HeaderPermission(r *http.Request, permission string) bool { + auth := r.Header.Get("Authorization") + if !strings.HasPrefix(auth, awsAlgorithm+" ") { + return false + } + parts := parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" ")) + credential := parts["Credential"] + signedHeaders := parts["SignedHeaders"] + providedSignature := parts["Signature"] + accessKey, scope, ok := splitCredential(credential) + key, keyOK := s.accessKey(accessKey) + if !ok || !keyOK || !key.can(permission) || signedHeaders == "" || providedSignature == "" { + return false + } + + amzDate := r.Header.Get("X-Amz-Date") + if !headerTimeValid(amzDate) { + return false + } + if !s.scopeAllowed(scope, amzDate) { + return false + } + payloadHash := r.Header.Get("X-Amz-Content-Sha256") + if payloadHash == "" { + payloadHash = "UNSIGNED-PAYLOAD" + } + if permission == "write" && r.Header.Get("X-Magpie-Replication") != "1" && payloadHash == "UNSIGNED-PAYLOAD" && !s.cfg.AllowUnsignedLocal { + return false + } + + canonicalRequest := canonicalRequest(r, signedHeaders, payloadHash, false) + expected := signature(key.Secret, amzDate, scope, canonicalRequest) + return hmac.Equal([]byte(expected), []byte(providedSignature)) +} + +func (s *Server) accessKey(id string) (AccessKey, bool) { + for _, key := range s.cfg.S3Keys { + if subtle.ConstantTimeCompare([]byte(key.ID), []byte(id)) == 1 { + return key, true + } + } + return AccessKey{}, false +} + +func (s *Server) requestAccessKey(r *http.Request) (AccessKey, bool) { + credential := r.URL.Query().Get("X-Amz-Credential") + if credential == "" { + auth := r.Header.Get("Authorization") + if !strings.HasPrefix(auth, awsAlgorithm+" ") { + return AccessKey{}, false + } + credential = parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" "))["Credential"] + } + accessKey, _, ok := splitCredential(credential) + if !ok { + return AccessKey{}, false + } + return s.accessKey(accessKey) +} + +func (key AccessKey) can(permission string) bool { + return key.Permissions["admin"] || key.Permissions[permission] +} + +func permissionForMethod(method string) string { + switch method { + case http.MethodGet, http.MethodHead: + return "read" + case http.MethodPut, http.MethodPost, http.MethodDelete: + return "write" + default: + return "admin" + } +} + +func signature(secret, amzDate, scope, canonicalRequest string) string { + date := strings.Split(scope, "/")[0] + stringToSign := strings.Join([]string{ + awsAlgorithm, + amzDate, + scope, + hexSHA256(canonicalRequest), + }, "\n") + key := signingKey(secret, date, scope) + sig := hmacSHA256(key, []byte(stringToSign)) + return hex.EncodeToString(sig) +} + +func signingKey(secret, date, scope string) []byte { + parts := strings.Split(scope, "/") + region := "auto" + service := "s3" + if len(parts) >= 3 { + region = parts[1] + service = parts[2] + } + kDate := hmacSHA256([]byte("AWS4"+secret), []byte(date)) + kRegion := hmacSHA256(kDate, []byte(region)) + kService := hmacSHA256(kRegion, []byte(service)) + return hmacSHA256(kService, []byte("aws4_request")) +} + +func (s *Server) scopeAllowed(scope string, amzDate string) bool { + parts := strings.Split(scope, "/") + if len(parts) != 4 { + return false + } + if len(amzDate) < 8 || parts[0] != amzDate[:8] { + return false + } + if parts[1] != s.cfg.S3Region || parts[2] != "s3" || parts[3] != "aws4_request" { + return false + } + return true +} + +func canonicalRequest(r *http.Request, signedHeaders, payloadHash string, presigned bool) string { + return strings.Join([]string{ + r.Method, + canonicalURI(r.URL.Path), + canonicalQuery(r.URL.Query(), presigned), + canonicalHeaders(r, signedHeaders), + signedHeaders, + payloadHash, + }, "\n") +} + +func canonicalURI(path string) string { + if path == "" { + return "/" + } + segments := strings.Split(path, "/") + for i, segment := range segments { + segments[i] = awsEscape(segment) + } + return strings.Join(segments, "/") +} + +func canonicalQuery(values url.Values, presigned bool) string { + type pair struct{ key, value string } + pairs := make([]pair, 0) + for key, vals := range values { + if presigned && key == "X-Amz-Signature" { + continue + } + for _, value := range vals { + pairs = append(pairs, pair{awsEscape(key), awsEscape(value)}) + } + } + sort.Slice(pairs, func(i, j int) bool { + if pairs[i].key == pairs[j].key { + return pairs[i].value < pairs[j].value + } + return pairs[i].key < pairs[j].key + }) + parts := make([]string, len(pairs)) + for i, pair := range pairs { + parts[i] = pair.key + "=" + pair.value + } + return strings.Join(parts, "&") +} + +func canonicalHeaders(r *http.Request, signedHeaders string) string { + headers := strings.Split(signedHeaders, ";") + var b strings.Builder + for _, header := range headers { + name := strings.ToLower(strings.TrimSpace(header)) + value := "" + if name == "host" { + value = r.Host + } else { + value = strings.Join(r.Header.Values(name), ",") + } + b.WriteString(name) + b.WriteString(":") + b.WriteString(compressSpaces(value)) + b.WriteString("\n") + } + return b.String() +} + +func splitCredential(credential string) (string, string, bool) { + parts := strings.SplitN(credential, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", false + } + scopeParts := strings.Split(parts[1], "/") + if len(scopeParts) != 4 || scopeParts[3] != "aws4_request" { + return "", "", false + } + return parts[0], parts[1], true +} + +func parseAuthParams(value string) map[string]string { + params := map[string]string{} + for _, part := range strings.Split(value, ",") { + keyValue := strings.SplitN(strings.TrimSpace(part), "=", 2) + if len(keyValue) == 2 { + params[keyValue[0]] = keyValue[1] + } + } + return params +} + +func (s *Server) presignedTimeValid(amzDate, expiresRaw string) bool { + created, err := time.Parse("20060102T150405Z", amzDate) + if err != nil { + return false + } + expires, err := time.ParseDuration(expiresRaw + "s") + if err != nil || expires <= 0 || expires > s.cfg.PresignedMaxExpiry { + return false + } + now := time.Now().UTC() + return now.After(created.Add(-15*time.Minute)) && now.Before(created.Add(expires)) +} + +func headerTimeValid(amzDate string) bool { + created, err := time.Parse("20060102T150405Z", amzDate) + if err != nil { + return false + } + now := time.Now().UTC() + return now.After(created.Add(-15*time.Minute)) && now.Before(created.Add(15*time.Minute)) +} + +func awsEscape(value string) string { + escaped := url.QueryEscape(value) + escaped = strings.ReplaceAll(escaped, "+", "%20") + escaped = strings.ReplaceAll(escaped, "%7E", "~") + return escaped +} + +func compressSpaces(value string) string { + return strings.Join(strings.Fields(strings.TrimSpace(value)), " ") +} + +func hexSHA256(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +func hmacSHA256(key []byte, value []byte) []byte { + mac := hmac.New(sha256.New, key) + _, _ = mac.Write(value) + return mac.Sum(nil) +} diff --git a/server.go b/server.go new file mode 100644 index 0000000..896b808 --- /dev/null +++ b/server.go @@ -0,0 +1,530 @@ +package main + +import ( + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "runtime/debug" + "strings" + "sync" + "sync/atomic" + "time" +) + +const objectPrefix = "/objects/" + +type ObjectStore interface { + Put(key string, body io.Reader, contentType string) (ObjectInfo, error) + Open(key string) (*ObjectRead, error) + Stat(key string) (ObjectInfo, error) + Delete(key string) error + DiskStatus() (DiskStatus, error) + List(prefix string, maxKeys int) ([]ObjectInfo, error) + ListPage(prefix string, after string, maxKeys int) (ListPage, error) +} + +type Server struct { + store ObjectStore + cfg Config + rateLimiter *RateLimiter + metrics *Metrics + writeMu sync.Mutex + reservedBytes uint64 +} + +func NewServer(store ObjectStore, cfg Config) http.Handler { + return NewServerInstance(store, cfg).Handler() +} + +func NewServerInstance(store ObjectStore, cfg Config) *Server { + cfg = cfg.withDefaults() + return &Server{store: store, cfg: cfg, rateLimiter: NewRateLimiter(cfg.RateLimitPerMinute), metrics: &Metrics{}} +} + +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/health", s.health) + mux.HandleFunc("/metrics", s.metricsHandler) + mux.HandleFunc(objectPrefix, s.object) + mux.HandleFunc("/", s.s3Object) + return s.recoverPanics(s.logRequests(s.rateLimit(mux))) +} + +func (s *Server) recoverPanics(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if recovered := recover(); recovered != nil { + log.Printf("panic method=%s path=%s remote=%s error=%v stack=%s", r.Method, safeLogPath(r.URL.Path), sanitizeLogValue(r.RemoteAddr), recovered, debug.Stack()) + s3Error(w, http.StatusInternalServerError, "InternalError", "internal server error") + } + }() + next.ServeHTTP(w, r) + }) +} + +func (s *Server) metricsHandler(w http.ResponseWriter, r *http.Request) { + if !s.opsAuthorized(r) { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"}) + return + } + w.Header().Set("Content-Type", "text/plain; version=0.0.4") + fmt.Fprintf(w, "magpie_requests_total %d\n", atomic.LoadUint64(&s.metrics.Requests)) + fmt.Fprintf(w, "magpie_errors_total %d\n", atomic.LoadUint64(&s.metrics.Errors)) + fmt.Fprintf(w, "magpie_response_bytes_total %d\n", atomic.LoadUint64(&s.metrics.ResponseBytes)) +} + +func (s *Server) opsAuthorized(r *http.Request) bool { + return s.authorized(r) || s.authorizedS3HeaderPermission(r, "admin") +} + +func (cfg Config) withDefaults() Config { + if len(cfg.S3Keys) == 0 && cfg.S3AccessKeyID != "" && cfg.S3SecretAccessKey != "" { + cfg.S3Keys = []AccessKey{{ID: cfg.S3AccessKeyID, Secret: cfg.S3SecretAccessKey, Permissions: permissions("read,write,admin")}} + } + if cfg.S3Region == "" { + cfg.S3Region = "auto" + } + if cfg.MaxObjectSize <= 0 { + cfg.MaxObjectSize = defaultMaxObjectSize + } + if cfg.PresignedMaxExpiry <= 0 { + cfg.PresignedMaxExpiry = defaultPresignedMaxExpiry + } + if cfg.MultipartMaxAge <= 0 { + cfg.MultipartMaxAge = defaultMultipartMaxAge + } + if cfg.PublicPrefixes == nil { + cfg.PublicPrefixes = map[string]bool{} + } + if cfg.ReplicationMaxJobs <= 0 { + cfg.ReplicationMaxJobs = defaultReplicationMaxJobs + } + if cfg.ReplicationMaxAttempts <= 0 { + cfg.ReplicationMaxAttempts = defaultReplicationMaxAttempts + } + if cfg.MaxRestoreBytes <= 0 { + cfg.MaxRestoreBytes = defaultMaxRestoreBytes + } + return cfg +} + +func (s *Server) health(w http.ResponseWriter, r *http.Request) { + if !s.opsAuthorized(r) { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"}) + return + } + disk, err := s.store.DiskStatus() + if err != nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"status": "error", "error": err.Error()}) + return + } + status := "ok" + code := http.StatusOK + if disk.FreeBytes < s.cfg.MinFreeBytes { + status = "low_disk" + code = http.StatusServiceUnavailable + } + writeJSON(w, code, map[string]any{"status": status, "disk": disk}) +} + +func (s *Server) object(w http.ResponseWriter, r *http.Request) { + if !s.authorized(r) { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"}) + return + } + + key := strings.TrimPrefix(r.URL.Path, objectPrefix) + if err := ValidateKey(key); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) + return + } + bucket, _, _ := strings.Cut(key, "/") + if !s.bucketAllowed(bucket) { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "bucket is not allowed"}) + return + } + + switch r.Method { + case http.MethodPut: + s.put(w, r, key) + case http.MethodGet: + s.get(w, r, key) + case http.MethodHead: + s.head(w, r, key) + case http.MethodDelete: + s.delete(w, key) + default: + w.Header().Set("Allow", "PUT, GET, HEAD, DELETE") + writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"}) + } +} + +func (s *Server) put(w http.ResponseWriter, r *http.Request, key string) { + release, ok := s.acceptWrite(w, r, false) + if !ok { + return + } + defer release() + contentType := r.Header.Get("Content-Type") + info, err := s.store.Put(key, http.MaxBytesReader(w, r.Body, s.cfg.MaxObjectSize), contentType) + if err != nil { + if maxBytesError(err) { + writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "object too large"}) + return + } + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "store failed"}) + return + } + writeJSON(w, http.StatusCreated, info) +} + +func (s *Server) acceptWrite(w http.ResponseWriter, r *http.Request, s3 bool) (func(), bool) { + size := r.ContentLength + if r.ContentLength < 0 { + if s.cfg.MaxObjectSize <= 0 { + disk, err := s.store.DiskStatus() + if err != nil || disk.FreeBytes < s.cfg.MinFreeBytes { + if s3 { + s3Error(w, http.StatusInsufficientStorage, "InsufficientStorage", "insufficient free disk space for unknown length write") + } else { + writeJSON(w, http.StatusInsufficientStorage, map[string]string{"error": "insufficient free disk space for unknown length write"}) + } + return nil, false + } + return func() {}, true + } + size = s.cfg.MaxObjectSize + } + if s.cfg.MaxObjectSize > 0 && r.ContentLength > s.cfg.MaxObjectSize { + if s3 { + s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large") + } else { + writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "object too large"}) + } + return nil, false + } + return s.acceptWriteSize(w, size, s3) +} + +func (s *Server) acceptWriteSize(w http.ResponseWriter, size int64, s3 bool) (func(), bool) { + if size < 0 { + return func() {}, true + } + disk, err := s.store.DiskStatus() + if err != nil { + if s3 { + s3Error(w, http.StatusInternalServerError, "InternalError", "disk status unavailable") + } else { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "disk status unavailable"}) + } + return nil, false + } + needed := uint64(size) + s.writeMu.Lock() + defer s.writeMu.Unlock() + reserved := s.reservedBytes + available := uint64(0) + if disk.FreeBytes > reserved { + available = disk.FreeBytes - reserved + } + if available < s.cfg.MinFreeBytes || needed > available || available-needed < s.cfg.MinFreeBytes { + if s3 { + s3Error(w, http.StatusInsufficientStorage, "InsufficientStorage", "insufficient free disk space") + } else { + writeJSON(w, http.StatusInsufficientStorage, map[string]string{"error": "insufficient free disk space"}) + } + return nil, false + } + s.reservedBytes += needed + released := false + return func() { + s.writeMu.Lock() + defer s.writeMu.Unlock() + if released { + return + } + released = true + if s.reservedBytes >= needed { + s.reservedBytes -= needed + } else { + s.reservedBytes = 0 + } + }, true +} + +func maxBytesError(err error) bool { + var maxErr *http.MaxBytesError + return errors.As(err, &maxErr) +} + +func (s *Server) get(w http.ResponseWriter, r *http.Request, key string) { + obj, err := s.store.Open(key) + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, ErrNotFound) { + status = http.StatusNotFound + } + writeJSON(w, status, map[string]string{"error": "not found"}) + return + } + defer obj.Close() + + contentType := obj.Info.ContentType + if contentType == "" { + contentType = "application/octet-stream" + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("X-Object-Key", obj.Info.Key) + w.Header().Set("X-Object-Size", fmt.Sprintf("%d", obj.Info.Size)) + w.Header().Set("X-Content-Type-Options", "nosniff") + http.ServeContent(w, r, obj.Info.Key, obj.Info.ModTime, obj) +} + +func (s *Server) head(w http.ResponseWriter, r *http.Request, key string) { + info, err := s.store.Stat(key) + if err != nil { + if errors.Is(err, ErrNotFound) { + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(http.StatusInternalServerError) + return + } + + contentType := info.ContentType + if contentType == "" { + contentType = "application/octet-stream" + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size)) + w.Header().Set("X-Object-Key", info.Key) + w.Header().Set("X-Object-Size", fmt.Sprintf("%d", info.Size)) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + _ = r.Body.Close() +} + +func (s *Server) delete(w http.ResponseWriter, key string) { + err := s.store.Delete(key) + if err != nil && !errors.Is(err, ErrNotFound) { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "delete failed"}) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (s *Server) authorized(r *http.Request) bool { + const prefix = "Bearer " + value := r.Header.Get("Authorization") + if !strings.HasPrefix(value, prefix) { + return false + } + provided := strings.TrimPrefix(value, prefix) + return s.cfg.AuthToken != "" && subtle.ConstantTimeCompare([]byte(provided), []byte(s.cfg.AuthToken)) == 1 +} + +func (s *Server) bucketAllowed(bucket string) bool { + return len(s.cfg.AllowedBuckets) == 0 || s.cfg.AllowedBuckets[bucket] +} + +func (s *Server) publicReadAllowed(key string) bool { + for prefix := range s.cfg.PublicPrefixes { + prefix = strings.TrimSuffix(prefix, "/") + if !strings.Contains(prefix, "/") { + continue + } + if key == prefix || strings.HasPrefix(key, prefix+"/") { + return true + } + } + return false +} + +func (s *Server) replicationAllowed(r *http.Request) bool { + if r.Header.Get("X-Magpie-Replication") != "1" { + return true + } + if s.cfg.ReplicationSecret == "" { + return false + } + return subtle.ConstantTimeCompare([]byte(r.Header.Get("X-Magpie-Replication-Secret")), []byte(s.cfg.ReplicationSecret)) == 1 +} + +func writeJSON(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(body) +} + +type statusWriter struct { + http.ResponseWriter + status int + bytes int +} + +func (w *statusWriter) WriteHeader(status int) { + w.status = status + w.ResponseWriter.WriteHeader(status) +} + +func (w *statusWriter) Write(b []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + n, err := w.ResponseWriter.Write(b) + w.bytes += n + return n, err +} + +func (s *Server) logRequests(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + recorder := &statusWriter{ResponseWriter: w} + next.ServeHTTP(recorder, r) + status := recorder.status + if status == 0 { + status = http.StatusOK + } + atomic.AddUint64(&s.metrics.Requests, 1) + if recorder.bytes > 0 { + atomic.AddUint64(&s.metrics.ResponseBytes, uint64(recorder.bytes)) + } + if status >= 400 { + atomic.AddUint64(&s.metrics.Errors, 1) + } + log.Printf( + "method=%s path=%s status=%d bytes=%d duration_ms=%d remote=%s", + r.Method, + safeLogPath(r.URL.Path), + status, + recorder.bytes, + time.Since(start).Milliseconds(), + sanitizeLogValue(r.RemoteAddr), + ) + }) +} + +func (s *Server) rateLimit(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !s.rateLimiter.Allow(s.rateLimitKey(r)) { + writeJSON(w, http.StatusTooManyRequests, map[string]string{"error": "rate limited"}) + return + } + next.ServeHTTP(w, r) + }) +} + +func (s *Server) rateLimitKey(r *http.Request) string { + if s.cfg.TrustProxyHeaders { + if forwarded := r.Header.Get("X-Forwarded-For"); forwarded != "" { + candidate := strings.TrimSpace(strings.Split(forwarded, ",")[0]) + if net.ParseIP(candidate) != nil { + return candidate + } + } + if realIP := strings.TrimSpace(r.Header.Get("X-Real-IP")); net.ParseIP(realIP) != nil { + return realIP + } + } + return r.RemoteAddr +} + +type Metrics struct { + Requests uint64 + Errors uint64 + ResponseBytes uint64 +} + +type RateLimiter struct { + limit int + mu sync.Mutex + buckets map[string]rateBucket +} + +type rateBucket struct { + minute int64 + count int +} + +func NewRateLimiter(limit int) *RateLimiter { + return &RateLimiter{limit: limit, buckets: map[string]rateBucket{}} +} + +func (r *RateLimiter) Allow(remote string) bool { + if r.limit <= 0 { + return true + } + ip, _, err := net.SplitHostPort(remote) + if err != nil { + ip = remote + } + minute := time.Now().Unix() / 60 + r.mu.Lock() + defer r.mu.Unlock() + for candidate, bucket := range r.buckets { + if bucket.minute < minute-1 { + delete(r.buckets, candidate) + } + } + bucket := r.buckets[ip] + if bucket.minute != minute { + bucket = rateBucket{minute: minute} + } + bucket.count++ + r.buckets[ip] = bucket + return bucket.count <= r.limit +} + +func safeLogPath(path string) string { + if len(path) <= 256 { + return sanitizeLogValue(path) + } + return sanitizeLogValue(path[:256]) + "..." +} + +func sanitizeLogValue(value string) string { + return strings.Map(func(r rune) rune { + if r < 0x20 || r == 0x7f { + return ' ' + } + return r + }, value) +} + +type ObjectInfo struct { + Key string `json:"key"` + Size int64 `json:"size"` + ContentType string `json:"content_type"` + ModTime time.Time `json:"mod_time"` + ETag string `json:"etag"` +} + +type DiskStatus struct { + TotalBytes uint64 `json:"total_bytes"` + FreeBytes uint64 `json:"free_bytes"` + UsedBytes uint64 `json:"used_bytes"` +} + +type StoreStats struct { + ObjectCount int64 `json:"object_count"` + TotalBytes int64 `json:"total_bytes"` + Disk DiskStatus `json:"disk"` + ReplicationQueue ReplicationQueueStats `json:"replication_queue"` +} + +type ScrubReport struct { + CheckedObjects int `json:"checked_objects"` + MissingKeys []string `json:"missing_keys"` + CorruptKeys []string `json:"corrupt_keys"` + OrphanKeys []string `json:"orphan_keys"` +} + +type ListPage struct { + Objects []ObjectInfo + NextAfter string +} diff --git a/server_test.go b/server_test.go new file mode 100644 index 0000000..ee685cb --- /dev/null +++ b/server_test.go @@ -0,0 +1,1737 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +const testAccessKey = "test-access" +const testSecretKey = "test-secret" + +func TestObjectLifecycle(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret"})) + defer srv.Close() + + putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/objects/attachments/test.txt", strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + putReq.Header.Set("Authorization", "Bearer secret") + putReq.Header.Set("Content-Type", "text/plain") + putResp, err := http.DefaultClient.Do(putReq) + if err != nil { + t.Fatal(err) + } + if putResp.StatusCode != http.StatusCreated { + t.Fatalf("PUT status = %d", putResp.StatusCode) + } + _ = putResp.Body.Close() + + getReq, err := http.NewRequest(http.MethodGet, srv.URL+"/objects/attachments/test.txt", nil) + if err != nil { + t.Fatal(err) + } + getReq.Header.Set("Authorization", "Bearer secret") + getResp, err := http.DefaultClient.Do(getReq) + if err != nil { + t.Fatal(err) + } + defer getResp.Body.Close() + if getResp.StatusCode != http.StatusOK { + t.Fatalf("GET status = %d", getResp.StatusCode) + } + body, err := io.ReadAll(getResp.Body) + if err != nil { + t.Fatal(err) + } + if string(body) != "hello" { + t.Fatalf("GET body = %q", string(body)) + } + if got := getResp.Header.Get("Content-Type"); !strings.HasPrefix(got, "text/plain") { + t.Fatalf("Content-Type = %q", got) + } + + delReq, err := http.NewRequest(http.MethodDelete, srv.URL+"/objects/attachments/test.txt", nil) + if err != nil { + t.Fatal(err) + } + delReq.Header.Set("Authorization", "Bearer secret") + delResp, err := http.DefaultClient.Do(delReq) + if err != nil { + t.Fatal(err) + } + if delResp.StatusCode != http.StatusNoContent { + t.Fatalf("DELETE status = %d", delResp.StatusCode) + } + _ = delResp.Body.Close() +} + +func TestUnauthorized(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret"})) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/objects/test.txt") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("status = %d", resp.StatusCode) + } +} + +func TestRejectsBadKeys(t *testing.T) { + bad := []string{"", "/absolute", "../escape", "a/../b", "a//b", "a\\b", "bucket/a.meta.json", ".multipart/a", ".magpie.db"} + for _, key := range bad { + if err := ValidateKey(key); err == nil { + t.Fatalf("ValidateKey(%q) returned nil", key) + } + } +} + +func TestRangeRequest(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("video/sample.txt", strings.NewReader("abcdef"), "text/plain"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret"})) + defer srv.Close() + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/objects/video/sample.txt", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer secret") + req.Header.Set("Range", "bytes=1-3") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusPartialContent { + t.Fatalf("status = %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if string(body) != "bcd" { + t.Fatalf("range body = %q", string(body)) + } +} + +func TestHealthIncludesDiskStatus(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret"})) + defer srv.Close() + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/health", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer secret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("health status = %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(body), "free_bytes") { + t.Fatalf("health body missing disk status: %s", string(body)) + } +} + +func TestMetricsEndpoint(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret"})) + defer srv.Close() + req, err := http.NewRequest(http.MethodGet, srv.URL+"/metrics", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer secret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body := readResponseBody(t, resp, http.StatusOK) + if !strings.Contains(string(body), "magpie_requests_total") { + t.Fatalf("metrics body = %s", string(body)) + } +} + +func TestRateLimit(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret", RateLimitPerMinute: 1})) + defer srv.Close() + firstReq, err := http.NewRequest(http.MethodGet, srv.URL+"/health", nil) + if err != nil { + t.Fatal(err) + } + firstReq.Header.Set("Authorization", "Bearer secret") + first, err := http.DefaultClient.Do(firstReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, first, http.StatusOK) + secondReq, err := http.NewRequest(http.MethodGet, srv.URL+"/health", nil) + if err != nil { + t.Fatal(err) + } + secondReq.Header.Set("Authorization", "Bearer secret") + second, err := http.DefaultClient.Do(secondReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, second, http.StatusTooManyRequests) +} + +func TestRejectsOversizedBearerUpload(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, Config{AuthToken: "secret", MaxObjectSize: 4})) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/objects/attachments/big.txt", strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer secret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusRequestEntityTooLarge { + t.Fatalf("oversized status = %d", resp.StatusCode) + } +} + +func TestS3LifecycleWithHeaderAuth(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + putReq, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/attachments/test.txt", strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + putReq.Header.Set("Content-Type", "text/plain") + signS3Header(t, putReq) + putResp, err := http.DefaultClient.Do(putReq) + if err != nil { + t.Fatal(err) + } + if putResp.StatusCode != http.StatusOK { + t.Fatalf("S3 PUT status = %d", putResp.StatusCode) + } + _ = putResp.Body.Close() + + headReq, err := http.NewRequest(http.MethodHead, srv.URL+"/bucket/attachments/test.txt", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, headReq) + headResp, err := http.DefaultClient.Do(headReq) + if err != nil { + t.Fatal(err) + } + if headResp.StatusCode != http.StatusOK { + t.Fatalf("S3 HEAD status = %d", headResp.StatusCode) + } + if got := headResp.Header.Get("ETag"); got == "" || got == "\"\"" { + t.Fatalf("S3 HEAD ETag = %q", got) + } + _ = headResp.Body.Close() + + getReq, err := http.NewRequest(http.MethodGet, srv.URL+"/bucket/attachments/test.txt", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, getReq) + getResp, err := http.DefaultClient.Do(getReq) + if err != nil { + t.Fatal(err) + } + defer getResp.Body.Close() + if getResp.StatusCode != http.StatusOK { + t.Fatalf("S3 GET status = %d", getResp.StatusCode) + } + body, err := io.ReadAll(getResp.Body) + if err != nil { + t.Fatal(err) + } + if string(body) != "hello" { + t.Fatalf("S3 GET body = %q", string(body)) + } +} + +func TestBucketAllowlist(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.AllowedBuckets = map[string]bool{"allowed": true} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + req, err := http.NewRequest(http.MethodPut, srv.URL+"/blocked/test.txt", strings.NewReader("x")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3ReadOnlyKeyCannotWrite(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := Config{S3Keys: []AccessKey{{ID: testAccessKey, Secret: testSecretKey, Permissions: permissions("read")}}} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/attachments/test.txt", strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("read-only PUT status = %d", resp.StatusCode) + } +} + +func TestS3MultipartUpload(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + initReq, err := http.NewRequest(http.MethodPost, srv.URL+"/bucket/large/object.txt?uploads", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, initReq) + initResp, err := http.DefaultClient.Do(initReq) + if err != nil { + t.Fatal(err) + } + initBody := readResponseBody(t, initResp, http.StatusOK) + var initResult createMultipartUploadResult + if err := xml.Unmarshal(initBody, &initResult); err != nil { + t.Fatal(err) + } + if initResult.UploadID == "" { + t.Fatalf("empty upload id: %s", string(initBody)) + } + + for partNumber, body := range map[int]string{1: "hello ", 2: "world"} { + partReq, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/large/object.txt?uploadId="+url.QueryEscape(initResult.UploadID)+"&partNumber="+strconv.Itoa(partNumber), strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + signS3Header(t, partReq) + partResp, err := http.DefaultClient.Do(partReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, partResp, http.StatusOK) + } + + completeXML := `12` + completeReq, err := http.NewRequest(http.MethodPost, srv.URL+"/bucket/large/object.txt?uploadId="+url.QueryEscape(initResult.UploadID), strings.NewReader(completeXML)) + if err != nil { + t.Fatal(err) + } + signS3Header(t, completeReq) + completeResp, err := http.DefaultClient.Do(completeReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, completeResp, http.StatusOK) + + getReq, err := http.NewRequest(http.MethodGet, srv.URL+"/bucket/large/object.txt", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, getReq) + getResp, err := http.DefaultClient.Do(getReq) + if err != nil { + t.Fatal(err) + } + body := readResponseBody(t, getResp, http.StatusOK) + if string(body) != "hello world" { + t.Fatalf("multipart body = %q", string(body)) + } +} + +func TestS3MultipartUploadIDCannotEscapeRoot(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/kept.txt", strings.NewReader("keep"), "text/plain"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodDelete, srv.URL+"/bucket/kept.txt?uploadId=..", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusNotFound) + + obj, err := store.Open("bucket/kept.txt") + if err != nil { + t.Fatalf("stored object was removed by invalid upload id: %v", err) + } + _ = obj.Close() +} + +func TestS3CopyObject(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/source.txt", strings.NewReader("copy body"), "text/plain"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + copyReq, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/dest.txt", nil) + if err != nil { + t.Fatal(err) + } + copyReq.Header.Set("X-Amz-Copy-Source", "/bucket/source.txt") + signS3Header(t, copyReq) + copyResp, err := http.DefaultClient.Do(copyReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, copyResp, http.StatusOK) + + obj, err := store.Open("bucket/dest.txt") + if err != nil { + t.Fatal(err) + } + defer obj.Close() + body, err := io.ReadAll(obj) + if err != nil { + t.Fatal(err) + } + if string(body) != "copy body" { + t.Fatalf("copied body = %q", string(body)) + } +} + +func TestS3DeleteObjects(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"bucket/a.txt", "bucket/b.txt", "bucket/c.txt"} { + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + deleteXML := `a.txtb.txt` + deleteReq, err := http.NewRequest(http.MethodPost, srv.URL+"/bucket?delete", strings.NewReader(deleteXML)) + if err != nil { + t.Fatal(err) + } + signS3Header(t, deleteReq) + deleteResp, err := http.DefaultClient.Do(deleteReq) + if err != nil { + t.Fatal(err) + } + body := readResponseBody(t, deleteResp, http.StatusOK) + if !strings.Contains(string(body), "a.txt") || !strings.Contains(string(body), "b.txt") { + t.Fatalf("delete response = %s", string(body)) + } + for _, key := range []string{"bucket/a.txt", "bucket/b.txt"} { + if _, err := store.Stat(key); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected %s deleted, err=%v", key, err) + } + } + if _, err := store.Stat("bucket/c.txt"); err != nil { + t.Fatalf("expected c.txt preserved: %v", err) + } +} + +func TestStaticPeerReplicationPutAndDelete(t *testing.T) { + peerStore, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + peerCfg := s3TestConfig() + peerCfg.ReplicationSecret = "replication-secret" + peer := httptest.NewServer(NewServer(peerStore, peerCfg)) + defer peer.Close() + + primaryStore, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.ReplicationPeers = []string{peer.URL} + cfg.ReplicationSecret = "replication-secret" + cfg.AllowInsecureReplication = true + primary := httptest.NewServer(NewServer(primaryStore, cfg)) + defer primary.Close() + + putReq, err := http.NewRequest(http.MethodPut, primary.URL+"/bucket/replicated.txt", strings.NewReader("copy me")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, putReq) + putResp, err := http.DefaultClient.Do(putReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, putResp, http.StatusOK) + + assertEventually(t, func() bool { + info, err := peerStore.Stat("bucket/replicated.txt") + return err == nil && info.Size == int64(len("copy me")) + }) + + deleteReq, err := http.NewRequest(http.MethodDelete, primary.URL+"/bucket/replicated.txt", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, deleteReq) + deleteResp, err := http.DefaultClient.Do(deleteReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, deleteResp, http.StatusNoContent) + + assertEventually(t, func() bool { + _, err := peerStore.Stat("bucket/replicated.txt") + return errors.Is(err, ErrNotFound) + }) + + specialName := "a?b#c%.txt" + specialReq, err := http.NewRequest(http.MethodPut, primary.URL+"/bucket/"+url.PathEscape(specialName), strings.NewReader("special")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, specialReq) + specialResp, err := http.DefaultClient.Do(specialReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, specialResp, http.StatusOK) + + assertEventually(t, func() bool { + info, err := peerStore.Stat("bucket/" + specialName) + return err == nil && info.Size == int64(len("special")) + }) +} + +func TestReplicationQueueRetriesFailedPeer(t *testing.T) { + primaryStore, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.ReplicationPeers = []string{"http://127.0.0.1:1"} + cfg.ReplicationSecret = "replication-secret" + cfg.AllowInsecureReplication = true + primary := httptest.NewServer(NewServer(primaryStore, cfg)) + defer primary.Close() + + putReq, err := http.NewRequest(http.MethodPut, primary.URL+"/bucket/retry.txt", strings.NewReader("retry")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, putReq) + putResp, err := http.DefaultClient.Do(putReq) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, putResp, http.StatusOK) + + assertEventually(t, func() bool { + stats, err := primaryStore.ReplicationQueueStats() + return err == nil && stats.PendingJobs == 1 && stats.MaxAttempts > 0 + }) +} + +func TestRepairQueuesAllObjectsForPeers(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"bucket/a.txt", "bucket/b.txt"} { + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 1001; i++ { + key := fmt.Sprintf("many/%04d.txt", i) + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + count, err := store.EnqueueRepair([]string{"http://peer-a", "http://peer-b"}) + if err != nil { + t.Fatal(err) + } + if count != 2006 { + t.Fatalf("repair jobs = %d", count) + } + stats, err := store.ReplicationQueueStats() + if err != nil { + t.Fatal(err) + } + if stats.PendingJobs != 2006 { + t.Fatalf("pending jobs = %d", stats.PendingJobs) + } +} + +func TestBackupAndRestore(t *testing.T) { + source := t.TempDir() + store, err := NewFileStore(source) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/object.txt", strings.NewReader("backup"), "text/plain"); err != nil { + t.Fatal(err) + } + archive := filepath.Join(t.TempDir(), "backup.tar.gz") + if err := BackupDir(source, archive); err != nil { + t.Fatal(err) + } + info, err := os.Stat(archive) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("backup mode = %o", info.Mode().Perm()) + } + destination := t.TempDir() + if err := RestoreDir(archive, destination); err != nil { + t.Fatal(err) + } + restored, err := NewFileStore(destination) + if err != nil { + t.Fatal(err) + } + obj, err := restored.Open("bucket/object.txt") + if err != nil { + t.Fatal(err) + } + defer obj.Close() + body, err := io.ReadAll(obj) + if err != nil { + t.Fatal(err) + } + if string(body) != "backup" { + t.Fatalf("restored body = %q", string(body)) + } +} + +func TestBackupRefusesSymlink(t *testing.T) { + source := t.TempDir() + if err := os.MkdirAll(filepath.Join(source, "bucket"), 0o700); err != nil { + t.Fatal(err) + } + outside := filepath.Join(t.TempDir(), "outside.txt") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(source, "bucket", "link.txt")); err != nil { + t.Fatal(err) + } + archive := filepath.Join(t.TempDir(), "backup.tar.gz") + if err := BackupDir(source, archive); err == nil { + t.Fatal("expected backup with symlink to fail") + } +} + +func TestRestoreEnforcesMaxBytes(t *testing.T) { + archive := filepath.Join(t.TempDir(), "archive.tar.gz") + if err := writeTestTarGz(archive, "bucket/object.txt", "body"); err != nil { + t.Fatal(err) + } + if err := RestoreDirWithLimit(archive, t.TempDir(), 3); err == nil { + t.Fatal("expected restore above max bytes to fail") + } +} + +func TestPutVerifiedRejectsBeforeObjectVisible(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + _, err = store.PutVerified("bucket/object.txt", strings.NewReader("body"), "text/plain", func() error { + return errPayloadHashMismatch + }) + if !errors.Is(err, errPayloadHashMismatch) { + t.Fatalf("PutVerified error = %v", err) + } + if _, err := store.Stat("bucket/object.txt"); !errors.Is(err, ErrNotFound) { + t.Fatalf("object became visible, err=%v", err) + } +} + +func TestS3PresignedGet(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/attachments/test.txt", strings.NewReader("hello"), "text/plain"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + presignedURL := presignS3URL(t, http.MethodGet, srv.URL+"/bucket/attachments/test.txt", map[string]string{ + "response-content-disposition": "attachment", + }) + resp, err := http.Get(presignedURL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("presigned GET status = %d", resp.StatusCode) + } + if got := resp.Header.Get("Content-Disposition"); got != "attachment" { + t.Fatalf("Content-Disposition = %q", got) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if string(body) != "hello" { + t.Fatalf("presigned GET body = %q", string(body)) + } +} + +func TestS3AllowedBucketsArePrivateByDefault(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/avatars/test.txt", strings.NewReader("hello"), "text/plain"); err != nil { + t.Fatal(err) + } + + cfg := s3TestConfig() + cfg.AllowedBuckets = map[string]bool{"app-uploads": true} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/app-uploads/avatars/test.txt") + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3PublicPrefixReadCanBeEnabled(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/avatars/test.txt", strings.NewReader("hello"), "text/plain"); err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/private/test.txt", strings.NewReader("secret"), "text/plain"); err != nil { + t.Fatal(err) + } + + cfg := s3TestConfig() + cfg.AllowedBuckets = map[string]bool{"app-uploads": true} + cfg.PublicPrefixes = map[string]bool{"app-uploads/avatars": true, "app-uploads": true} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/app-uploads/avatars/test.txt") + if err != nil { + t.Fatal(err) + } + body := readResponseBody(t, resp, http.StatusOK) + if string(body) != "hello" { + t.Fatalf("public GET body = %q", string(body)) + } + + privateResp, err := http.Get(srv.URL + "/app-uploads/private/test.txt") + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, privateResp, http.StatusForbidden) +} + +func TestS3PublicReadHardensActiveContent(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/avatars/evil.html", strings.NewReader(""), "text/html"); err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/avatars/evil.svg", strings.NewReader(""), "image/svg+xml"); err != nil { + t.Fatal(err) + } + if _, err := store.Put("app-uploads/avatars/pic.png", strings.NewReader("\x89PNG"), "image/png"); err != nil { + t.Fatal(err) + } + + cfg := s3TestConfig() + cfg.AllowedBuckets = map[string]bool{"app-uploads": true} + cfg.PublicPrefixes = map[string]bool{"app-uploads/avatars": true} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + // HTML must be served with a restrictive CSP and forced to download. + htmlResp, err := http.Get(srv.URL + "/app-uploads/avatars/evil.html") + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, htmlResp, http.StatusOK) + if got := htmlResp.Header.Get("Content-Security-Policy"); got != "default-src 'none'; sandbox" { + t.Fatalf("html Content-Security-Policy = %q", got) + } + if got := htmlResp.Header.Get("Content-Disposition"); got != "attachment" { + t.Fatalf("html Content-Disposition = %q", got) + } + + // SVG is an image type but scriptable, so it must also be forced to download. + svgResp, err := http.Get(srv.URL + "/app-uploads/avatars/evil.svg") + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, svgResp, http.StatusOK) + if got := svgResp.Header.Get("Content-Disposition"); got != "attachment" { + t.Fatalf("svg Content-Disposition = %q", got) + } + + // Inert raster media stays inline so avatars/images still render. + pngResp, err := http.Get(srv.URL + "/app-uploads/avatars/pic.png") + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, pngResp, http.StatusOK) + if got := pngResp.Header.Get("Content-Disposition"); got != "" { + t.Fatalf("png Content-Disposition = %q, want inline (empty)", got) + } + if got := pngResp.Header.Get("Content-Security-Policy"); got != "default-src 'none'; sandbox" { + t.Fatalf("png Content-Security-Policy = %q", got) + } +} + +func TestS3AuthenticatedReadKeepsDispositionOverride(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/attachments/page.html", strings.NewReader("

ok

"), "text/html"); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + presignedURL := presignS3URL(t, http.MethodGet, srv.URL+"/bucket/attachments/page.html", map[string]string{ + "response-content-disposition": "inline", + }) + resp, err := http.Get(presignedURL) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusOK) + // Authenticated reads are not public, so the caller's override is honored and + // no public-read hardening is applied. + if got := resp.Header.Get("Content-Disposition"); got != "inline" { + t.Fatalf("authenticated Content-Disposition = %q", got) + } + if got := resp.Header.Get("Content-Security-Policy"); got != "" { + t.Fatalf("authenticated Content-Security-Policy = %q, want none", got) + } +} + +func TestS3RejectsTamperedPresignedPath(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + presignedURL := presignS3URL(t, http.MethodGet, srv.URL+"/bucket/attachments/test.txt", nil) + tampered := strings.Replace(presignedURL, "test.txt", "other.txt", 1) + resp, err := http.Get(tampered) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("tampered status = %d", resp.StatusCode) + } +} + +func TestS3RejectsOversizedUpload(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.MaxObjectSize = 4 + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/attachments/big.txt", strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusRequestEntityTooLarge { + t.Fatalf("S3 oversized status = %d", resp.StatusCode) + } +} + +func TestS3RejectsTooLongPresignedExpiry(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.PresignedMaxExpiry = time.Hour + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + presignedURL := presignS3URLWithExpiry(t, http.MethodGet, srv.URL+"/bucket/attachments/test.txt", nil, "7200") + resp, err := http.Get(presignedURL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("long-expiry status = %d", resp.StatusCode) + } +} + +func TestS3RejectsWrongCredentialScope(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/scope.txt", strings.NewReader("body")) + if err != nil { + t.Fatal(err) + } + signS3HeaderWithScope(t, req, "wrong", "s3") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3ReadOnlyPresignedURLCannotWrite(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := Config{S3Keys: []AccessKey{{ID: testAccessKey, Secret: testSecretKey, Permissions: permissions("read")}}} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + presignedURL := presignS3URL(t, http.MethodPut, srv.URL+"/bucket/attachments/test.txt", nil) + req, err := http.NewRequest(http.MethodPut, presignedURL, strings.NewReader("nope")) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3PresignedURLCannotWrite(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + presignedURL := presignS3URL(t, http.MethodPut, srv.URL+"/bucket/attachments/test.txt", nil) + req, err := http.NewRequest(http.MethodPut, presignedURL, strings.NewReader("nope")) + if err != nil { + t.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3RejectsStaleHeaderSignature(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/replay.txt", strings.NewReader("replay")) + if err != nil { + t.Fatal(err) + } + signS3HeaderAt(t, req, time.Now().UTC().Add(-time.Hour)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3CopyObjectRequiresAllowedSourceBucket(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("blocked/secret.txt", strings.NewReader("secret"), "text/plain"); err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.AllowedBuckets = map[string]bool{"allowed": true} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/allowed/leak.txt", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Amz-Copy-Source", "/blocked/secret.txt") + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3RejectsPayloadHashMismatch(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/tampered.txt", strings.NewReader("tampered")) + if err != nil { + t.Fatal(err) + } + signS3HeaderWithPayloadHash(t, req, sha256Hex("original")) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) + if _, err := store.Stat("bucket/tampered.txt"); !errors.Is(err, ErrNotFound) { + t.Fatalf("tampered object persisted, err=%v", err) + } +} + +func TestS3CopyObjectRequiresReadAndWritePermission(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/source.txt", strings.NewReader("secret"), "text/plain"); err != nil { + t.Fatal(err) + } + cfg := Config{S3Keys: []AccessKey{{ID: testAccessKey, Secret: testSecretKey, Permissions: permissions("write")}}} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/copy.txt", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Amz-Copy-Source", "/bucket/source.txt") + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestBearerAPIHonorsBucketAllowlist(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := Config{AuthToken: "secret", AllowedBuckets: map[string]bool{"allowed": true}} + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/objects/blocked/file.txt", strings.NewReader("x")) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer secret") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestCleanupMultipartRemovesStaleUploads(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + dir := filepath.Join(store.root, ".multipart", "old-upload") + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + old := time.Now().Add(-2 * time.Hour) + if err := os.Chtimes(dir, old, old); err != nil { + t.Fatal(err) + } + if err := store.CleanupMultipart(time.Hour); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("multipart dir still exists, err=%v", err) + } +} + +func TestRestoreRefusesExistingSymlinkTarget(t *testing.T) { + archive := filepath.Join(t.TempDir(), "archive.tar.gz") + if err := writeTestTarGz(archive, "bucket/object.txt", "body"); err != nil { + t.Fatal(err) + } + destination := t.TempDir() + if err := os.MkdirAll(filepath.Join(destination, "bucket"), 0o700); err != nil { + t.Fatal(err) + } + outside := filepath.Join(t.TempDir(), "outside.txt") + if err := os.Symlink(outside, filepath.Join(destination, "bucket", "object.txt")); err != nil { + t.Fatal(err) + } + if err := RestoreDir(archive, destination); err == nil { + t.Fatal("expected restore through symlink to fail") + } +} + +func TestRestoreRefusesSymlinkParent(t *testing.T) { + archive := filepath.Join(t.TempDir(), "archive.tar.gz") + if err := writeTestTarGz(archive, "bucket/object.txt", "body"); err != nil { + t.Fatal(err) + } + destination := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(destination, "bucket")); err != nil { + t.Fatal(err) + } + if err := RestoreDir(archive, destination); err == nil { + t.Fatal("expected restore through symlink parent to fail") + } +} + +func TestS3MultipartCompleteEnforcesAggregateMaxSize(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cfg := s3TestConfig() + cfg.MaxObjectSize = 6 + srv := httptest.NewServer(NewServer(store, cfg)) + defer srv.Close() + + initReq, err := http.NewRequest(http.MethodPost, srv.URL+"/bucket/large.txt?uploads", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, initReq) + initResp, err := http.DefaultClient.Do(initReq) + if err != nil { + t.Fatal(err) + } + initBody := readResponseBody(t, initResp, http.StatusOK) + var initResult createMultipartUploadResult + if err := xml.Unmarshal(initBody, &initResult); err != nil { + t.Fatal(err) + } + + parts := []struct { + number int + body string + }{ + {number: 1, body: "abcd"}, + {number: 2, body: "efgh"}, + } + + for _, part := range parts { + partNumber := part.number + body := part.body + partReq, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/large.txt?uploadId="+url.QueryEscape(initResult.UploadID)+"&partNumber="+strconv.Itoa(partNumber), strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + signS3Header(t, partReq) + partResp, err := http.DefaultClient.Do(partReq) + if err != nil { + t.Fatal(err) + } + wantStatus := http.StatusOK + if partNumber == 2 { + wantStatus = http.StatusRequestEntityTooLarge + } + _ = readResponseBody(t, partResp, wantStatus) + } +} + +func TestS3RejectsReplicationHeaderWithoutSecret(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodPut, srv.URL+"/bucket/suppressed.txt", strings.NewReader("x")) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Magpie-Replication", "1") + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusForbidden) +} + +func TestS3RejectsOversizedBatchDelete(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + var body strings.Builder + body.WriteString("") + for i := 0; i <= maxBatchDeleteItems; i++ { + body.WriteString("key-") + body.WriteString(strconv.Itoa(i)) + body.WriteString("") + } + body.WriteString("") + req, err := http.NewRequest(http.MethodPost, srv.URL+"/bucket?delete", strings.NewReader(body.String())) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = readResponseBody(t, resp, http.StatusBadRequest) +} + +func TestS3ListBucketWithPrefix(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + fixtures := []string{ + "bucket/attachments/a.txt", + "bucket/attachments/b.txt", + "bucket/avatars/c.txt", + "otherbucket/attachments/d.txt", + } + for _, key := range fixtures { + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/bucket?list-type=2&prefix=attachments/", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("list status = %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + text := string(body) + if !strings.Contains(text, "attachments/a.txt") { + t.Fatalf("list missing a.txt: %s", text) + } + if !strings.Contains(text, "attachments/b.txt") { + t.Fatalf("list missing b.txt: %s", text) + } + if strings.Contains(text, "avatars/c.txt") || strings.Contains(text, "otherbucket") { + t.Fatalf("list included wrong keys: %s", text) + } + if !strings.Contains(text, "2") { + t.Fatalf("list key count wrong: %s", text) + } +} + +func TestS3ListBucketMaxKeys(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"bucket/a.txt", "bucket/b.txt", "bucket/c.txt"} { + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/bucket?list-type=2&max-keys=2", nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("max-keys list status = %d body=%s", resp.StatusCode, string(body)) + } + if !strings.Contains(string(body), "2") { + t.Fatalf("max-keys not honored: %s", string(body)) + } +} + +func TestS3ListBucketContinuationToken(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"bucket/a.txt", "bucket/b.txt", "bucket/c.txt"} { + if _, err := store.Put(key, strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + first := signedGET(t, srv.URL+"/bucket?list-type=2&max-keys=2") + firstBody := readResponseBody(t, first, http.StatusOK) + var firstPage s3ListBucketResult + if err := xml.Unmarshal(firstBody, &firstPage); err != nil { + t.Fatal(err) + } + if !firstPage.IsTruncated || firstPage.NextContinuationToken == "" || firstPage.KeyCount != 2 { + t.Fatalf("first page = %#v body=%s", firstPage, string(firstBody)) + } + + second := signedGET(t, srv.URL+"/bucket?list-type=2&max-keys=2&continuation-token="+url.QueryEscape(firstPage.NextContinuationToken)) + secondBody := readResponseBody(t, second, http.StatusOK) + var secondPage s3ListBucketResult + if err := xml.Unmarshal(secondBody, &secondPage); err != nil { + t.Fatal(err) + } + if secondPage.IsTruncated || secondPage.NextContinuationToken != "" || secondPage.KeyCount != 1 { + t.Fatalf("second page = %#v body=%s", secondPage, string(secondBody)) + } + if len(secondPage.Contents) != 1 || secondPage.Contents[0].Key != "c.txt" { + t.Fatalf("second page contents = %#v", secondPage.Contents) + } +} + +func TestS3ListBucketRejectsInvalidContinuationToken(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(NewServer(store, s3TestConfig())) + defer srv.Close() + + resp := signedGET(t, srv.URL+"/bucket?list-type=2&continuation-token=not-base64!!!") + _ = readResponseBody(t, resp, http.StatusBadRequest) +} + +func TestMetadataPersistsAcrossStoreRestart(t *testing.T) { + dir := t.TempDir() + store, err := NewFileStore(dir) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/attachments/a.txt", strings.NewReader("x"), "text/plain"); err != nil { + t.Fatal(err) + } + + reopened, err := NewFileStore(dir) + if err != nil { + t.Fatal(err) + } + objects, err := reopened.List("bucket/attachments/", 1000) + if err != nil { + t.Fatal(err) + } + if len(objects) != 1 || objects[0].Key != "bucket/attachments/a.txt" { + t.Fatalf("persisted objects = %#v", objects) + } +} + +func TestReindexBackfillsMetadataFromFilesystem(t *testing.T) { + dir := t.TempDir() + objectPath := filepath.Join(dir, "bucket", "attachments", "manual.txt") + if err := os.MkdirAll(filepath.Dir(objectPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(objectPath, []byte("manual"), 0o600); err != nil { + t.Fatal(err) + } + + store, err := NewFileStore(dir) + if err != nil { + t.Fatal(err) + } + objects, err := store.List("bucket/attachments/", 1000) + if err != nil { + t.Fatal(err) + } + if len(objects) != 0 { + t.Fatalf("objects before reindex = %#v", objects) + } + if err := store.Reindex(); err != nil { + t.Fatal(err) + } + objects, err = store.List("bucket/attachments/", 1000) + if err != nil { + t.Fatal(err) + } + if len(objects) != 1 || objects[0].Key != "bucket/attachments/manual.txt" || objects[0].ETag == "" { + t.Fatalf("objects after reindex = %#v", objects) + } +} + +func TestScrubReportsMissingCorruptAndOrphanedObjects(t *testing.T) { + dir := t.TempDir() + store, err := NewFileStore(dir) + if err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/good.txt", strings.NewReader("good"), "text/plain"); err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/missing.txt", strings.NewReader("missing"), "text/plain"); err != nil { + t.Fatal(err) + } + if _, err := store.Put("bucket/corrupt.txt", strings.NewReader("before"), "text/plain"); err != nil { + t.Fatal(err) + } + missingPath, _ := store.pathFor("bucket/missing.txt") + _ = os.Remove(missingPath) + corruptPath, _ := store.pathFor("bucket/corrupt.txt") + if err := os.WriteFile(corruptPath, []byte("after"), 0o600); err != nil { + t.Fatal(err) + } + orphanPath := filepath.Join(dir, "bucket", "orphan.txt") + if err := os.WriteFile(orphanPath, []byte("orphan"), 0o600); err != nil { + t.Fatal(err) + } + + report, err := store.Scrub() + if err != nil { + t.Fatal(err) + } + if !contains(report.MissingKeys, "bucket/missing.txt") { + t.Fatalf("missing keys = %#v", report.MissingKeys) + } + if !contains(report.CorruptKeys, "bucket/corrupt.txt") { + t.Fatalf("corrupt keys = %#v", report.CorruptKeys) + } + if !contains(report.OrphanKeys, "bucket/orphan.txt") { + t.Fatalf("orphan keys = %#v", report.OrphanKeys) + } +} + +func contains(values []string, value string) bool { + for _, candidate := range values { + if candidate == value { + return true + } + } + return false +} + +func assertEventually(t *testing.T, condition func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if condition() { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("condition not met before timeout") +} + +func s3TestConfig() Config { + return Config{S3AccessKeyID: testAccessKey, S3SecretAccessKey: testSecretKey, S3Region: "auto"} +} + +func signedGET(t *testing.T, rawURL string) *http.Response { + t.Helper() + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + t.Fatal(err) + } + signS3Header(t, req) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + return resp +} + +func readResponseBody(t *testing.T, resp *http.Response, wantStatus int) []byte { + t.Helper() + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != wantStatus { + t.Fatalf("status = %d want=%d body=%s", resp.StatusCode, wantStatus, string(body)) + } + return body +} + +func signS3Header(t *testing.T, req *http.Request) { + t.Helper() + signS3HeaderAt(t, req, time.Now().UTC()) +} + +func signS3HeaderAt(t *testing.T, req *http.Request, at time.Time) { + t.Helper() + amzDate := at.UTC().Format("20060102T150405Z") + date := amzDate[:8] + scope := date + "/auto/s3/aws4_request" + signedHeaders := "host;x-amz-content-sha256;x-amz-date" + if req.Header.Get("X-Amz-Copy-Source") != "" { + signedHeaders += ";x-amz-copy-source" + } + payloadHash := requestPayloadHash(t, req) + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + canonical := canonicalRequest(req, signedHeaders, payloadHash, false) + sig := signature(testSecretKey, amzDate, scope, canonical) + req.Header.Set( + "Authorization", + awsAlgorithm+" Credential="+testAccessKey+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig, + ) +} + +func requestPayloadHash(t *testing.T, req *http.Request) string { + t.Helper() + if req.Body == nil { + return sha256Hex("") + } + if req.GetBody == nil { + return "UNSIGNED-PAYLOAD" + } + body, err := req.GetBody() + if err != nil { + t.Fatal(err) + } + defer body.Close() + hash := sha256.New() + if _, err := io.Copy(hash, body); err != nil { + t.Fatal(err) + } + return hex.EncodeToString(hash.Sum(nil)) +} + +func signS3HeaderWithPayloadHash(t *testing.T, req *http.Request, payloadHash string) { + t.Helper() + amzDate := time.Now().UTC().Format("20060102T150405Z") + date := amzDate[:8] + scope := date + "/auto/s3/aws4_request" + signedHeaders := "host;x-amz-content-sha256;x-amz-date" + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + canonical := canonicalRequest(req, signedHeaders, payloadHash, false) + sig := signature(testSecretKey, amzDate, scope, canonical) + req.Header.Set( + "Authorization", + awsAlgorithm+" Credential="+testAccessKey+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig, + ) +} + +func signS3HeaderWithScope(t *testing.T, req *http.Request, region string, service string) { + t.Helper() + amzDate := time.Now().UTC().Format("20060102T150405Z") + date := amzDate[:8] + scope := date + "/" + region + "/" + service + "/aws4_request" + signedHeaders := "host;x-amz-content-sha256;x-amz-date" + payloadHash := requestPayloadHash(t, req) + req.Header.Set("X-Amz-Date", amzDate) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + canonical := canonicalRequest(req, signedHeaders, payloadHash, false) + sig := signature(testSecretKey, amzDate, scope, canonical) + req.Header.Set( + "Authorization", + awsAlgorithm+" Credential="+testAccessKey+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig, + ) +} + +func presignS3URL(t *testing.T, method string, rawURL string, extra map[string]string) string { + return presignS3URLWithExpiry(t, method, rawURL, extra, "3600") +} + +func presignS3URLWithExpiry(t *testing.T, method string, rawURL string, extra map[string]string, expires string) string { + t.Helper() + parsed, err := url.Parse(rawURL) + if err != nil { + t.Fatal(err) + } + amzDate := time.Now().UTC().Format("20060102T150405Z") + date := amzDate[:8] + scope := date + "/auto/s3/aws4_request" + query := parsed.Query() + for key, value := range extra { + query.Set(key, value) + } + query.Set("X-Amz-Algorithm", awsAlgorithm) + query.Set("X-Amz-Credential", testAccessKey+"/"+scope) + query.Set("X-Amz-Date", amzDate) + query.Set("X-Amz-Expires", expires) + query.Set("X-Amz-SignedHeaders", "host") + parsed.RawQuery = query.Encode() + + req, err := http.NewRequest(method, parsed.String(), nil) + if err != nil { + t.Fatal(err) + } + canonical := canonicalRequest(req, "host", "UNSIGNED-PAYLOAD", true) + query.Set("X-Amz-Signature", signature(testSecretKey, amzDate, scope, canonical)) + parsed.RawQuery = query.Encode() + return parsed.String() +} + +func sha256Hex(value string) string { + sum := sha256.Sum256([]byte(value)) + return fmt.Sprintf("%x", sum[:]) +} + +func writeTestTarGz(path string, name string, body string) error { + out, err := os.Create(path) + if err != nil { + return err + } + defer out.Close() + gz := gzip.NewWriter(out) + defer gz.Close() + tw := tar.NewWriter(gz) + defer tw.Close() + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(body))}); err != nil { + return err + } + _, err = tw.Write([]byte(body)) + return err +} diff --git a/store.go b/store.go new file mode 100644 index 0000000..fd14fd6 --- /dev/null +++ b/store.go @@ -0,0 +1,563 @@ +package main + +import ( + "crypto/md5" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + _ "modernc.org/sqlite" +) + +var ErrNotFound = errors.New("object not found") + +type FileStore struct { + root string + db *sql.DB +} + +type ObjectRead struct { + *os.File + Info ObjectInfo +} + +func NewFileStore(root string) (*FileStore, error) { + abs, err := filepath.Abs(root) + if err != nil { + return nil, err + } + if err := os.MkdirAll(abs, 0o700); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", filepath.Join(abs, ".magpie.db")) + if err != nil { + return nil, err + } + store := &FileStore{root: abs, db: db} + if err := store.initMetadata(); err != nil { + _ = db.Close() + return nil, err + } + return store, nil +} + +func (s *FileStore) Put(key string, body io.Reader, contentType string) (ObjectInfo, error) { + return s.PutVerified(key, body, contentType, nil) +} + +func (s *FileStore) PutVerified(key string, body io.Reader, contentType string, verify func() error) (ObjectInfo, error) { + if err := ValidateKey(key); err != nil { + return ObjectInfo{}, err + } + + path, err := s.pathFor(key) + if err != nil { + return ObjectInfo{}, err + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return ObjectInfo{}, err + } + + tmp, err := os.CreateTemp(filepath.Dir(path), ".upload-*") + if err != nil { + return ObjectInfo{}, err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + + hash := md5.New() + size, copyErr := io.Copy(io.MultiWriter(tmp, hash), body) + closeErr := tmp.Close() + if copyErr != nil { + return ObjectInfo{}, copyErr + } + if closeErr != nil { + return ObjectInfo{}, closeErr + } + if verify != nil { + if err := verify(); err != nil { + return ObjectInfo{}, err + } + } + + info := ObjectInfo{ + Key: key, + Size: size, + ContentType: contentType, + ModTime: time.Now().UTC(), + ETag: fmt.Sprintf("%x", hash.Sum(nil)), + } + if err := s.writeMeta(key, info); err != nil { + return ObjectInfo{}, err + } + if err := s.upsertMetadata(info); err != nil { + _ = os.Remove(path + ".meta.json") + return ObjectInfo{}, err + } + if err := os.Rename(tmpName, path); err != nil { + _ = os.Remove(path + ".meta.json") + return ObjectInfo{}, err + } + return info, nil +} + +func (s *FileStore) Open(key string) (*ObjectRead, error) { + if err := ValidateKey(key); err != nil { + return nil, err + } + path, err := s.pathFor(key) + if err != nil { + return nil, err + } + file, err := os.Open(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, ErrNotFound + } + return nil, err + } + info, err := s.Stat(key) + if err != nil { + _ = file.Close() + return nil, err + } + return &ObjectRead{File: file, Info: info}, nil +} + +func (s *FileStore) Stat(key string) (ObjectInfo, error) { + if err := ValidateKey(key); err != nil { + return ObjectInfo{}, err + } + path, err := s.pathFor(key) + if err != nil { + return ObjectInfo{}, err + } + stat, err := os.Stat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return ObjectInfo{}, ErrNotFound + } + return ObjectInfo{}, err + } + + if info, err := s.getMetadata(key); err == nil { + info.Size = stat.Size() + return info, nil + } + + info := ObjectInfo{Key: key, Size: stat.Size(), ModTime: stat.ModTime().UTC()} + if meta, err := s.readMeta(key); err == nil { + info.ContentType = meta.ContentType + info.ETag = meta.ETag + if !meta.ModTime.IsZero() { + info.ModTime = meta.ModTime + } + } + _ = s.upsertMetadata(info) + return info, nil +} + +func (s *FileStore) Delete(key string) error { + if err := ValidateKey(key); err != nil { + return err + } + path, err := s.pathFor(key) + if err != nil { + return err + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + _ = os.Remove(path + ".meta.json") + _, _ = s.db.Exec(`DELETE FROM objects WHERE key = ?`, key) + return nil +} + +func (s *FileStore) List(prefix string, maxKeys int) ([]ObjectInfo, error) { + page, err := s.ListPage(prefix, "", maxKeys) + if err != nil { + return nil, err + } + return page.Objects, nil +} + +func (s *FileStore) ListPage(prefix string, after string, maxKeys int) (ListPage, error) { + validatePrefix := strings.TrimSuffix(prefix, "/") + if validatePrefix != "" { + if err := ValidateKey(validatePrefix); err != nil { + return ListPage{}, err + } + } + if after != "" { + if err := ValidateKey(after); err != nil { + return ListPage{}, err + } + } + if maxKeys <= 0 || maxKeys > 1000 { + maxKeys = 1000 + } + + return s.listMetadata(prefix, after, maxKeys) +} + +func (s *FileStore) Reindex() error { + return filepath.WalkDir(s.root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + if entry.Name() == ".magpie.db" || strings.HasPrefix(entry.Name(), ".magpie.db-") || strings.HasSuffix(entry.Name(), ".meta.json") { + return nil + } + + rel, err := filepath.Rel(s.root, path) + if err != nil { + return err + } + key := filepath.ToSlash(rel) + if err := ValidateKey(key); err != nil { + return nil + } + info, err := s.statFromFilesystem(key) + if err != nil { + return err + } + return s.upsertMetadata(info) + }) +} + +func (s *FileStore) Stats() (StoreStats, error) { + var stats StoreStats + if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(size), 0) FROM objects`).Scan(&stats.ObjectCount, &stats.TotalBytes); err != nil { + return StoreStats{}, err + } + queue, err := s.ReplicationQueueStats() + if err != nil { + return StoreStats{}, err + } + stats.ReplicationQueue = queue + disk, err := s.DiskStatus() + if err != nil { + return StoreStats{}, err + } + stats.Disk = disk + return stats, nil +} + +func (s *FileStore) Scrub() (ScrubReport, error) { + report := ScrubReport{} + rows, err := s.db.Query(`SELECT key, size, etag FROM objects ORDER BY key ASC`) + if err != nil { + return report, err + } + defer rows.Close() + + metadataKeys := map[string]bool{} + for rows.Next() { + var key string + var size int64 + var etag string + if err := rows.Scan(&key, &size, &etag); err != nil { + return report, err + } + metadataKeys[key] = true + report.CheckedObjects++ + path, err := s.pathFor(key) + if err != nil { + report.CorruptKeys = append(report.CorruptKeys, key) + continue + } + stat, err := os.Stat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + report.MissingKeys = append(report.MissingKeys, key) + continue + } + return report, err + } + if stat.Size() != size { + report.CorruptKeys = append(report.CorruptKeys, key) + continue + } + computed, err := fileETag(path) + if err != nil { + return report, err + } + if etag != "" && computed != etag { + report.CorruptKeys = append(report.CorruptKeys, key) + } + } + if err := rows.Err(); err != nil { + return report, err + } + + if err := filepath.WalkDir(s.root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || entry.Name() == ".magpie.db" || strings.HasPrefix(entry.Name(), ".magpie.db-") || strings.HasSuffix(entry.Name(), ".meta.json") || strings.Contains(path, string(os.PathSeparator)+".multipart"+string(os.PathSeparator)) { + return nil + } + rel, err := filepath.Rel(s.root, path) + if err != nil { + return err + } + key := filepath.ToSlash(rel) + if !metadataKeys[key] { + report.OrphanKeys = append(report.OrphanKeys, key) + } + return nil + }); err != nil { + return report, err + } + sort.Strings(report.MissingKeys) + sort.Strings(report.CorruptKeys) + sort.Strings(report.OrphanKeys) + return report, nil +} + +func (s *FileStore) CleanupMultipart(maxAge time.Duration) error { + if maxAge <= 0 { + return nil + } + root := filepath.Join(s.root, ".multipart") + entries, err := os.ReadDir(root) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + cutoff := time.Now().Add(-maxAge) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + info, err := entry.Info() + if err != nil { + return err + } + if info.ModTime().Before(cutoff) { + if err := os.RemoveAll(filepath.Join(root, entry.Name())); err != nil { + return err + } + } + } + return nil +} + +func (s *FileStore) pathFor(key string) (string, error) { + clean := filepath.Clean(filepath.FromSlash(key)) + path := filepath.Join(s.root, clean) + if path != s.root && !strings.HasPrefix(path, s.root+string(os.PathSeparator)) { + return "", fmt.Errorf("invalid object key") + } + return path, nil +} + +func (s *FileStore) writeMeta(key string, info ObjectInfo) error { + path, err := s.pathFor(key) + if err != nil { + return err + } + data, err := json.Marshal(info) + if err != nil { + return err + } + return os.WriteFile(path+".meta.json", data, 0o600) +} + +func (s *FileStore) readMeta(key string) (ObjectInfo, error) { + path, err := s.pathFor(key) + if err != nil { + return ObjectInfo{}, err + } + data, err := os.ReadFile(path + ".meta.json") + if err != nil { + return ObjectInfo{}, err + } + var info ObjectInfo + if err := json.Unmarshal(data, &info); err != nil { + return ObjectInfo{}, err + } + return info, nil +} + +func (s *FileStore) statFromFilesystem(key string) (ObjectInfo, error) { + path, err := s.pathFor(key) + if err != nil { + return ObjectInfo{}, err + } + stat, err := os.Stat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return ObjectInfo{}, ErrNotFound + } + return ObjectInfo{}, err + } + info := ObjectInfo{Key: key, Size: stat.Size(), ModTime: stat.ModTime().UTC()} + if meta, err := s.readMeta(key); err == nil { + info.ContentType = meta.ContentType + info.ETag = meta.ETag + if !meta.ModTime.IsZero() { + info.ModTime = meta.ModTime + } + } + if info.ETag == "" { + etag, err := fileETag(path) + if err != nil { + return ObjectInfo{}, err + } + info.ETag = etag + } + return info, nil +} + +func fileETag(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := md5.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func (s *FileStore) initMetadata() error { + _, err := s.db.Exec(` +PRAGMA journal_mode = WAL; +PRAGMA synchronous = NORMAL; +CREATE TABLE IF NOT EXISTS objects ( + key TEXT PRIMARY KEY, + size INTEGER NOT NULL, + content_type TEXT NOT NULL DEFAULT '', + etag TEXT NOT NULL DEFAULT '', + mod_time TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS objects_key_idx ON objects(key); +CREATE TABLE IF NOT EXISTS replication_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + peer TEXT NOT NULL, + op TEXT NOT NULL, + key TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + next_attempt_at TEXT NOT NULL, + last_error TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS replication_jobs_due_idx ON replication_jobs(next_attempt_at, id); +`) + return err +} + +func (s *FileStore) upsertMetadata(info ObjectInfo) error { + _, err := s.db.Exec(` +INSERT INTO objects (key, size, content_type, etag, mod_time) +VALUES (?, ?, ?, ?, ?) +ON CONFLICT(key) DO UPDATE SET + size = excluded.size, + content_type = excluded.content_type, + etag = excluded.etag, + mod_time = excluded.mod_time +`, info.Key, info.Size, info.ContentType, info.ETag, info.ModTime.UTC().Format(time.RFC3339Nano)) + return err +} + +func (s *FileStore) getMetadata(key string) (ObjectInfo, error) { + var info ObjectInfo + var modTime string + err := s.db.QueryRow(`SELECT key, size, content_type, etag, mod_time FROM objects WHERE key = ?`, key). + Scan(&info.Key, &info.Size, &info.ContentType, &info.ETag, &modTime) + if err != nil { + return ObjectInfo{}, err + } + parsed, err := time.Parse(time.RFC3339Nano, modTime) + if err != nil { + return ObjectInfo{}, err + } + info.ModTime = parsed + return info, nil +} + +func (s *FileStore) listMetadata(prefix string, after string, maxKeys int) (ListPage, error) { + rows, err := s.db.Query(` +SELECT key, size, content_type, etag, mod_time +FROM objects +WHERE key LIKE ? ESCAPE '\' AND key > ? +ORDER BY key ASC +LIMIT ? +`, escapeLike(prefix)+"%", after, maxKeys+1) + if err != nil { + return ListPage{}, err + } + defer rows.Close() + + objects := make([]ObjectInfo, 0) + for rows.Next() { + var info ObjectInfo + var modTime string + if err := rows.Scan(&info.Key, &info.Size, &info.ContentType, &info.ETag, &modTime); err != nil { + return ListPage{}, err + } + parsed, err := time.Parse(time.RFC3339Nano, modTime) + if err != nil { + return ListPage{}, err + } + info.ModTime = parsed + objects = append(objects, info) + } + if err := rows.Err(); err != nil { + return ListPage{}, err + } + + page := ListPage{Objects: objects} + if len(objects) > maxKeys { + page.NextAfter = objects[maxKeys-1].Key + page.Objects = objects[:maxKeys] + } + return page, nil +} + +func escapeLike(value string) string { + value = strings.ReplaceAll(value, `\`, `\\`) + value = strings.ReplaceAll(value, `%`, `\%`) + value = strings.ReplaceAll(value, `_`, `\_`) + return value +} + +func ValidateKey(key string) error { + if key == "" { + return fmt.Errorf("object key is required") + } + if len(key) > 1024 { + return fmt.Errorf("object key is too long") + } + if strings.HasPrefix(key, "/") || strings.HasPrefix(key, "\\") { + return fmt.Errorf("object key cannot be absolute") + } + if strings.Contains(key, "\x00") || strings.Contains(key, "\\") || strings.Contains(key, "//") { + return fmt.Errorf("object key contains invalid characters") + } + for _, part := range strings.Split(key, "/") { + if part == "" || part == "." || part == ".." { + return fmt.Errorf("object key contains invalid path segment") + } + if part == ".multipart" || part == ".magpie.db" || strings.HasPrefix(part, ".magpie.db-") || strings.HasSuffix(part, ".meta.json") { + return fmt.Errorf("object key uses reserved internal name") + } + } + return nil +} diff --git a/version.go b/version.go new file mode 100644 index 0000000..4b4e477 --- /dev/null +++ b/version.go @@ -0,0 +1,29 @@ +package main + +import "runtime" + +var ( + version = "dev" + commit = "unknown" + date = "unknown" +) + +type VersionInfo struct { + Version string `json:"version"` + Commit string `json:"commit"` + Date string `json:"date"` + Go string `json:"go"` + OS string `json:"os"` + Arch string `json:"arch"` +} + +func versionInfo() VersionInfo { + return VersionInfo{ + Version: version, + Commit: commit, + Date: date, + Go: runtime.Version(), + OS: runtime.GOOS, + Arch: runtime.GOARCH, + } +}