Initial commit on Forgejo

Fresh repository history for elektrine/magpie hosted at
https://git.elektrine.com/elektrine/magpie.

.dockerignore

@@ -0,0 +1,19 @@
+.git/
+.env
+.env.*
+!.env.production.example
+data/
+.multipart/
+dist/
+tmp/
+*.db
+*.db-*
+*.sqlite
+*.sqlite-*
+*.meta.json
+*.tar
+*.tar.gz
+*.tgz
+*.log
+coverage.out
+magpie

.env.production.example

@@ -0,0 +1,62 @@
+MAGPIE_ADDR=127.0.0.1:8090
+MAGPIE_DATA_DIR=/var/lib/magpie
+
+# Optional bearer-token API for /objects/* routes.
+MAGPIE_AUTH_TOKEN=change-this-long-random-token
+
+# S3-compatible API credentials for /:bucket/:key routes.
+MAGPIE_S3_ACCESS_KEY_ID=magpie
+MAGPIE_S3_SECRET_ACCESS_KEY=change-this-long-random-s3-secret
+# Optional multi-key form: id:secret:read,write,admin;id2:secret2:read
+MAGPIE_S3_KEYS=
+MAGPIE_S3_REGION=auto
+
+# Temporarily set true only for trusted private migrations with clients that
+# upload using SigV4 UNSIGNED-PAYLOAD, such as some rclone versions.
+MAGPIE_ALLOW_UNSIGNED_PAYLOADS=false
+
+# Comma-separated peer base URLs for async write/delete replication.
+# MAGPIE_REPLICATION_SECRET is required when peers are set.
+MAGPIE_REPLICATION_PEERS=
+MAGPIE_REPLICATION_SECRET=
+# Replication peers must use https unless this is explicitly enabled for a
+# trusted local/private network.
+MAGPIE_ALLOW_INSECURE_REPLICATION=false
+MAGPIE_REPLICATION_MAX_JOBS=10000
+MAGPIE_REPLICATION_MAX_ATTEMPTS=100
+
+# Comma-separated bucket allowlist. Empty allows any bucket name.
+MAGPIE_ALLOWED_BUCKETS=app-uploads
+
+# Comma-separated bucket/prefix values that allow unsigned public GET/HEAD requests.
+# Empty/private by default. Set to `app-uploads/avatars` only for public media.
+# MAGPIE_PUBLIC_PREFIXES=app-uploads/avatars
+
+# Per-IP request limit per minute. Set 0 to disable.
+MAGPIE_RATE_LIMIT_PER_MINUTE=600
+# Trust X-Forwarded-For/X-Real-IP for rate limiting only behind a trusted proxy.
+MAGPIE_TRUST_PROXY_HEADERS=false
+
+# Upload safety limits.
+# Default: 1073741824 (1 GiB)
+MAGPIE_MAX_OBJECT_SIZE=1073741824
+
+# Reject writes if disk free space would fall below this value.
+# Default: 1073741824 (1 GiB)
+MAGPIE_MIN_FREE_BYTES=1073741824
+
+# Maximum accepted SigV4 presigned URL lifetime.
+# Default: 24h
+MAGPIE_PRESIGNED_MAX_EXPIRY=24h
+
+# Remove incomplete multipart uploads older than this age.
+# Default: 24h
+MAGPIE_MULTIPART_MAX_AGE=24h
+
+# Maximum total expanded bytes accepted by restore.
+# Default: 10737418240 (10 GiB)
+MAGPIE_MAX_RESTORE_BYTES=10737418240
+
+# Rebuild SQLite metadata from existing files on startup.
+# Useful after manual restores/migrations; usually false for normal startup.
+MAGPIE_REINDEX_ON_START=false

.github/workflows/release.yml

@@ -0,0 +1,51 @@
+name: release
+
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
+
+      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Test
+        run: go test ./...
+
+      - name: Build release artifacts
+        run: make release VERSION="$GITHUB_REF_NAME" COMMIT="${GITHUB_SHA::7}"
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: magpie-${{ github.ref_name }}
+          path: dist/*
+
+  publish:
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
+        with:
+          name: magpie-${{ github.ref_name }}
+          path: dist
+
+      - name: Publish GitHub release
+        uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65
+        with:
+          files: dist/*

.gitignore

@@ -0,0 +1,30 @@
+# Local binaries and build output
+magpie
+dist/
+coverage.out
+
+# Runtime data
+data/
+*.db
+*.db-*
+*.sqlite
+*.sqlite-*
+*.meta.json
+.multipart/
+
+# Local environment files
+.env
+.env.*
+!.env.production.example
+
+# Logs and temporary files
+*.log
+*.tmp
+tmp/
+
+# Editor and OS files
+.DS_Store
+.idea/
+.vscode/
+*.swp
+*.swo

Dockerfile

@@ -0,0 +1,20 @@
+FROM golang:1.26-alpine@sha256:f44b851aa23dfa219d18db6eab743203245429d355cb619cf96a2ffe2a84ba7a AS build
+
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY *.go ./
+RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/magpie .
+
+FROM alpine:3.22@sha256:2039be0c5ec6ce8566809626a252c930216a92109c043f282504accb5ee3c0c6
+
+RUN addgroup -S magpie && adduser -S -G magpie magpie
+RUN mkdir -p /var/lib/magpie && chown -R magpie:magpie /var/lib/magpie
+
+USER magpie
+EXPOSE 8090
+ENV MAGPIE_ADDR=0.0.0.0:8090
+ENV MAGPIE_DATA_DIR=/var/lib/magpie
+
+COPY --from=build /out/magpie /usr/local/bin/magpie
+ENTRYPOINT ["/usr/local/bin/magpie"]

Makefile

@@ -0,0 +1,36 @@
+BINARY := magpie
+VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || printf dev)
+COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || printf unknown)
+DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
+LDFLAGS := -s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.date=$(DATE)
+DIST := dist
+
+.PHONY: all fmt test build clean checksums release snapshot
+
+all: fmt test build
+
+fmt:
+	gofmt -w .
+
+test:
+	go test ./...
+
+build:
+	mkdir -p $(DIST)
+	CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY) .
+
+snapshot: clean
+	mkdir -p $(DIST)
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_linux_amd64 .
+	GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_linux_arm64 .
+	GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_darwin_amd64 .
+	GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_darwin_arm64 .
+	GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_windows_amd64.exe .
+
+checksums:
+	cd $(DIST) && sha256sum * > SHA256SUMS
+
+release: snapshot checksums
+
+clean:
+	rm -rf $(DIST)

README.md

@@ -0,0 +1,271 @@
+# Magpie
+
+Magpie is a small object store for app-owned files. It stores objects on local disk and exposes a minimal S3-compatible path-style API.
+
+It is intended to run behind your application or edge proxy, on localhost, a private Docker network, or a VPN-only address. Do not expose write-capable endpoints directly to the public internet.
+
+## Status
+
+Magpie supports common application object-storage operations:
+
+- `PUT`, `GET`, `HEAD`, and `DELETE` objects
+- path-style S3 routes: `/:bucket/:key`
+- SigV4 header authentication for writes and private reads
+- presigned `GET` and `HEAD` URLs
+- prefix listing with continuation tokens
+- batch delete
+- copy object
+- multipart upload
+- bucket allowlist and opt-in public-read buckets
+- local metadata index in SQLite
+- async static-peer replication
+- backup, restore, scrub, reindex, stats, and repair commands
+
+It does not implement bucket creation, ACLs, bucket policies, object versioning, lifecycle rules, quorum writes, or consensus replication.
+
+## Quick Start
+
+```sh
+export MAGPIE_ADDR='127.0.0.1:8090'
+export MAGPIE_DATA_DIR='/var/lib/magpie'
+export MAGPIE_S3_ACCESS_KEY_ID='magpie'
+export MAGPIE_S3_SECRET_ACCESS_KEY='replace-with-a-long-random-secret'
+export MAGPIE_ALLOWED_BUCKETS='app-uploads'
+
+magpie serve
+```
+
+For local development from source:
+
+```sh
+go run . serve
+```
+
+## Configuration
+
+Required authentication config:
+
+```env
+MAGPIE_S3_ACCESS_KEY_ID=magpie
+MAGPIE_S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret
+```
+
+Or define multiple scoped keys:
+
+```env
+MAGPIE_S3_KEYS=app:secret1:read,write;auditor:secret2:read;admin:secret3:admin
+```
+
+Supported scopes are `read`, `write`, and `admin`. `admin` implies all scopes.
+
+Common runtime config:
+
+```env
+MAGPIE_ADDR=127.0.0.1:8090
+MAGPIE_DATA_DIR=/var/lib/magpie
+MAGPIE_ALLOWED_BUCKETS=app-uploads
+# MAGPIE_PUBLIC_PREFIXES=app-uploads/avatars
+MAGPIE_RATE_LIMIT_PER_MINUTE=600
+MAGPIE_TRUST_PROXY_HEADERS=false
+MAGPIE_MAX_OBJECT_SIZE=1073741824
+MAGPIE_MIN_FREE_BYTES=1073741824
+MAGPIE_PRESIGNED_MAX_EXPIRY=24h
+MAGPIE_MULTIPART_MAX_AGE=24h
+MAGPIE_REINDEX_ON_START=false
+MAGPIE_MAX_RESTORE_BYTES=10737418240
+```
+
+`MAGPIE_ALLOWED_BUCKETS` should be set in production. Leave it empty only if you intentionally want any bucket name accepted.
+
+Buckets are private by default. Set `MAGPIE_PUBLIC_PREFIXES` to comma-separated `bucket/prefix` values, such as `app-uploads/avatars`, only when an app intentionally serves unsigned public media through an edge proxy. Set `MAGPIE_PUBLIC_PREFIXES=none` or leave it unset to require signed reads.
+
+Public reads are hardened against stored-XSS from uploader-controlled content types: responses carry `Content-Security-Policy: default-src 'none'; sandbox`, and any object that is not inert inline media (images, audio, video — SVG excluded) is served with `Content-Disposition: attachment` so browsers download it instead of rendering it. For untrusted user uploads, still prefer serving public objects from an origin separate from your application.
+
+## Application Integration
+
+Use any S3-compatible client that supports path-style endpoints and custom hosts.
+
+If Magpie runs on the same host as your application outside Docker:
+
+```env
+S3_ACCESS_KEY_ID=magpie
+S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret
+S3_ENDPOINT=127.0.0.1:8090
+S3_BUCKET_NAME=app-uploads
+S3_PUBLIC_URL=http://127.0.0.1:8090/app-uploads
+S3_SCHEME=http://
+S3_PORT=8090
+```
+
+If your app and Magpie are in the same Docker Compose network, use the Magpie service name:
+
+```env
+S3_ENDPOINT=magpie:8090
+S3_BUCKET_NAME=app-uploads
+S3_PUBLIC_URL=http://magpie:8090/app-uploads
+S3_SCHEME=http://
+S3_PORT=8090
+```
+
+Keep Magpie write access private. Your application should enforce user permissions before it writes to Magpie. Public buckets are intended for already-public objects such as avatars and timeline media.
+
+## API Notes
+
+S3 object routes use path-style URLs:
+
+```text
+PUT    /:bucket/:key
+GET    /:bucket/:key
+HEAD   /:bucket/:key
+DELETE /:bucket/:key
+POST   /:bucket?delete
+```
+
+Objects are stored under:
+
+```text
+MAGPIE_DATA_DIR/<bucket>/<key>
+```
+
+Internal names are reserved and cannot be used as object keys:
+
+- `.multipart`
+- `.magpie.db`
+- `.magpie.db-*`
+- `*.meta.json`
+
+Presigned URLs are read-only. Magpie accepts presigned `GET` and `HEAD` requests, but rejects presigned writes and deletes.
+
+Normal SigV4 write requests must include a real `X-Amz-Content-Sha256` payload hash. `UNSIGNED-PAYLOAD` is reserved for authenticated replication traffic.
+
+## Health And Metrics
+
+Health and metrics require authentication.
+
+```sh
+curl -H "Authorization: Bearer $MAGPIE_AUTH_TOKEN" \
+  http://127.0.0.1:8090/health
+```
+
+```sh
+curl -H "Authorization: Bearer $MAGPIE_AUTH_TOKEN" \
+  http://127.0.0.1:8090/metrics
+```
+
+The health endpoint returns `503` when free disk space is below `MAGPIE_MIN_FREE_BYTES`. Writes are rejected in that state.
+
+If you only use S3 credentials and do not set `MAGPIE_AUTH_TOKEN`, sign health and metrics requests with an admin S3 key instead.
+
+## CLI
+
+```sh
+magpie serve
+magpie stats
+magpie scrub
+magpie reindex
+magpie repair
+magpie backup /backup/magpie.tar.gz
+magpie restore /backup/magpie.tar.gz
+magpie smoke
+magpie version
+```
+
+Command summary:
+
+- `stats`: object count, logical bytes, disk status, replication queue status
+- `scrub`: verifies metadata against files and reports missing, corrupt, and orphaned objects
+- `reindex`: rebuilds SQLite metadata from files on disk
+- `repair`: queues known objects for replication to configured peers
+- `backup`: creates a tar/gzip archive of the data directory
+- `restore`: restores a tar/gzip archive into the data directory
+- `smoke`: runs a local write/read/list/scrub/delete check
+
+Restore rejects unsafe archive paths, symlinks, unsupported archive entries, and archives whose expanded file content exceeds `MAGPIE_MAX_RESTORE_BYTES`. Backup archives are created with owner-only permissions. For consistent backups, stop Magpie or use a filesystem snapshot before running `magpie backup`.
+
+## Replication
+
+Replication is asynchronous and best-effort. Local writes complete before peers acknowledge them.
+
+Configure every node with matching S3 credentials and a shared replication secret:
+
+```env
+MAGPIE_REPLICATION_PEERS=https://magpie-b.internal,https://magpie-c.internal
+MAGPIE_REPLICATION_SECRET=replace-with-a-long-random-peer-secret
+MAGPIE_REPLICATION_MAX_JOBS=10000
+MAGPIE_REPLICATION_MAX_ATTEMPTS=100
+```
+
+`MAGPIE_REPLICATION_SECRET` is required when peers are configured. Replication-marked requests without the shared secret are rejected.
+
+Replication peer URLs must use `https://` by default because Magpie sends SigV4 authorization and the replication secret to peers. Set `MAGPIE_ALLOW_INSECURE_REPLICATION=true` only for trusted local/private networks where plaintext HTTP is acceptable.
+
+Use `magpie repair` after outages to queue a full object repair pass to all configured peers.
+
+## Deployment
+
+Docker Compose example:
+
+```sh
+cp .env.production.example .env.production
+```
+
+```yaml
+services:
+  magpie:
+    image: magpie:latest
+    container_name: magpie
+    build: .
+    restart: unless-stopped
+    env_file:
+      - .env.production
+    ports:
+      - "127.0.0.1:8090:8090"
+    volumes:
+      - /var/lib/magpie:/var/lib/magpie
+```
+
+The default Compose file is standalone and binds Magpie to localhost only.
+
+If another Compose project needs to reach Magpie by service name, use the optional network override. It attaches Magpie to an external shared network with the `magpie` network alias:
+
+```sh
+docker network create elektrine-magpie-shared
+docker compose -f docker-compose.yml -f docker-compose.network.yml up -d --build
+```
+
+Then attach your application container to the same external `elektrine-magpie-shared` network and use:
+
+```env
+S3_ENDPOINT=magpie:8090
+S3_PUBLIC_URL=http://magpie:8090/app-uploads
+S3_SCHEME=http://
+S3_PORT=8090
+```
+
+You can rename the shared network without editing the override file:
+
+```sh
+MAGPIE_DOCKER_NETWORK=my-existing-shared-network \
+  docker compose -f docker-compose.yml -f docker-compose.network.yml up -d --build
+```
+
+Bare-metal installs can use `deploy/systemd/magpie.service`. The systemd unit is optional; use it only if you want Magpie managed directly by systemd.
+
+Back up `/var/lib/magpie` or whatever directory you set as `MAGPIE_DATA_DIR`.
+
+## Build And Test
+
+```sh
+go test ./...
+go build ./...
+```
+
+Release artifacts:
+
+```sh
+make test
+make build
+make release
+```
+
+Artifacts are written to `dist/`, with checksums in `dist/SHA256SUMS`.

backup.go

@@ -0,0 +1,181 @@
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func BackupDir(src string, dst string) error {
+	out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+	gz := gzip.NewWriter(out)
+	defer gz.Close()
+	tw := tar.NewWriter(gz)
+	defer tw.Close()
+	root, err := filepath.Abs(src)
+	if err != nil {
+		return err
+	}
+	return filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
+		if err != nil || entry.IsDir() {
+			return err
+		}
+		if entry.Type()&os.ModeSymlink != 0 {
+			return fmt.Errorf("refusing to back up symlink %q", path)
+		}
+		rel, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		info, err := entry.Info()
+		if err != nil {
+			return err
+		}
+		header, err := tar.FileInfoHeader(info, "")
+		if err != nil {
+			return err
+		}
+		header.Name = filepath.ToSlash(rel)
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		fileInfo, statErr := file.Stat()
+		currentInfo, lstatErr := os.Lstat(path)
+		if statErr != nil || lstatErr != nil {
+			_ = file.Close()
+			if statErr != nil {
+				return statErr
+			}
+			return lstatErr
+		}
+		if currentInfo.Mode()&os.ModeSymlink != 0 || !os.SameFile(currentInfo, fileInfo) {
+			_ = file.Close()
+			return fmt.Errorf("refusing to back up replaced or symlinked file %q", path)
+		}
+		_, copyErr := io.Copy(tw, file)
+		closeErr := file.Close()
+		if copyErr != nil {
+			return copyErr
+		}
+		return closeErr
+	})
+}
+
+func RestoreDir(src string, dst string) error {
+	return RestoreDirWithLimit(src, dst, defaultMaxRestoreBytes)
+}
+
+func RestoreDirWithLimit(src string, dst string, maxBytes int64) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	gz, err := gzip.NewReader(in)
+	if err != nil {
+		return err
+	}
+	defer gz.Close()
+	tr := tar.NewReader(gz)
+	root, err := filepath.Abs(dst)
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(root, 0o700); err != nil {
+		return err
+	}
+	var restoredBytes int64
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		if strings.Contains(header.Name, "..") || strings.HasPrefix(header.Name, "/") {
+			return fmt.Errorf("unsafe archive path %q", header.Name)
+		}
+		if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA && header.Typeflag != tar.TypeDir {
+			return fmt.Errorf("unsupported archive entry %q", header.Name)
+		}
+		if header.Size < 0 {
+			return fmt.Errorf("invalid archive size for %q", header.Name)
+		}
+		if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeRegA {
+			restoredBytes += header.Size
+			if maxBytes > 0 && restoredBytes > maxBytes {
+				return fmt.Errorf("restore exceeds maximum size")
+			}
+		}
+		path := filepath.Join(root, filepath.FromSlash(header.Name))
+		if !strings.HasPrefix(path, root+string(os.PathSeparator)) && path != root {
+			return fmt.Errorf("unsafe archive path %q", header.Name)
+		}
+		if err := rejectSymlinkParents(root, path); err != nil {
+			return err
+		}
+		if header.FileInfo().IsDir() {
+			if err := os.MkdirAll(path, 0o700); err != nil {
+				return err
+			}
+			continue
+		}
+		if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+			return err
+		}
+		if info, err := os.Lstat(path); err == nil && info.Mode()&os.ModeSymlink != 0 {
+			return fmt.Errorf("refusing to restore through symlink %q", header.Name)
+		}
+		out, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
+		if err != nil {
+			return err
+		}
+		_, copyErr := io.CopyN(out, tr, header.Size)
+		closeErr := out.Close()
+		if copyErr != nil {
+			return copyErr
+		}
+		if closeErr != nil {
+			return closeErr
+		}
+	}
+}
+
+func rejectSymlinkParents(root string, path string) error {
+	current := root
+	rel, err := filepath.Rel(root, filepath.Dir(path))
+	if err != nil {
+		return err
+	}
+	if rel == "." {
+		return nil
+	}
+	for _, part := range strings.Split(rel, string(os.PathSeparator)) {
+		current = filepath.Join(current, part)
+		info, err := os.Lstat(current)
+		if errors.Is(err, os.ErrNotExist) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		if info.Mode()&os.ModeSymlink != 0 {
+			return fmt.Errorf("refusing to restore through symlink parent %q", current)
+		}
+	}
+	return nil
+}

deploy/systemd/magpie.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=Magpie object storage service
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=magpie
+Group=magpie
+EnvironmentFile=/etc/magpie/.env.production
+ExecStart=/usr/local/bin/magpie
+Restart=on-failure
+RestartSec=5s
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths=/var/lib/magpie
+
+[Install]
+WantedBy=multi-user.target

disk_unix.go

@@ -0,0 +1,21 @@
+//go:build !windows
+
+package main
+
+import (
+	"fmt"
+	"syscall"
+)
+
+func (s *FileStore) DiskStatus() (DiskStatus, error) {
+	var stat syscall.Statfs_t
+	if err := syscall.Statfs(s.root, &stat); err != nil {
+		return DiskStatus{}, err
+	}
+	if stat.Bsize <= 0 {
+		return DiskStatus{}, fmt.Errorf("invalid filesystem block size")
+	}
+	total := stat.Blocks * uint64(stat.Bsize)
+	free := stat.Bavail * uint64(stat.Bsize)
+	return DiskStatus{TotalBytes: total, FreeBytes: free, UsedBytes: total - free}, nil
+}

disk_windows.go

@@ -0,0 +1,7 @@
+//go:build windows
+
+package main
+
+func (s *FileStore) DiskStatus() (DiskStatus, error) {
+	return DiskStatus{}, nil
+}

docker-compose.network.yml

@@ -0,0 +1,11 @@
+services:
+  magpie:
+    networks:
+      magpie_shared:
+        aliases:
+          - magpie
+
+networks:
+  magpie_shared:
+    external: true
+    name: ${MAGPIE_DOCKER_NETWORK:-elektrine-magpie-shared}

docker-compose.yml

@@ -0,0 +1,15 @@
+services:
+  magpie:
+    image: magpie:latest
+    container_name: magpie
+    build: .
+    restart: unless-stopped
+    env_file:
+      - .env.production
+    ports:
+      - "127.0.0.1:8090:8090"
+    volumes:
+      - magpie-data:/var/lib/magpie
+
+volumes:
+  magpie-data:

go.mod

@@ -0,0 +1,16 @@
+module magpie
+
+go 1.25.0
+
+require (
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	modernc.org/libc v1.72.0 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
+	modernc.org/sqlite v1.50.0 // indirect
+)

go.sum

@@ -0,0 +1,21 @@
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c=
+modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM=
+modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=

main.go

@@ -0,0 +1,415 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+)
+
+const (
+	defaultMaxObjectSize          = 1024 * 1024 * 1024
+	defaultMinFreeBytes           = 1024 * 1024 * 1024
+	defaultPresignedMaxExpiry     = 24 * time.Hour
+	defaultMultipartMaxAge        = 24 * time.Hour
+	defaultMaxRestoreBytes        = 10 * 1024 * 1024 * 1024
+	defaultReplicationMaxJobs     = 10_000
+	defaultReplicationMaxAttempts = 100
+)
+
+func main() {
+	cmd := "serve"
+	if len(os.Args) > 1 {
+		cmd = os.Args[1]
+	}
+	if cmd == "version" {
+		printJSON(versionInfo())
+		return
+	}
+	if cmd == "help" || cmd == "-h" || cmd == "--help" {
+		printUsage()
+		return
+	}
+
+	cfg := configFromEnv()
+	var store *FileStore
+	openStore := func() *FileStore {
+		if store != nil {
+			return store
+		}
+		opened, err := NewFileStore(cfg.DataDir)
+		if err != nil {
+			log.Fatalf("init store: %v", err)
+		}
+		store = opened
+		return store
+	}
+
+	switch cmd {
+	case "serve":
+		serve(cfg, openStore())
+	case "reindex":
+		mustJSON(openStore().Reindex())
+	case "stats":
+		stats, err := openStore().Stats()
+		if err != nil {
+			log.Fatal(err)
+		}
+		printJSON(stats)
+	case "scrub":
+		report, err := openStore().Scrub()
+		if err != nil {
+			log.Fatal(err)
+		}
+		printJSON(report)
+	case "repair":
+		count, err := openStore().EnqueueRepair(cfg.ReplicationPeers)
+		if err != nil {
+			log.Fatal(err)
+		}
+		printJSON(map[string]any{"status": "queued", "jobs": count})
+	case "backup":
+		if len(os.Args) < 3 {
+			log.Fatal("usage: magpie backup <archive.tar.gz>")
+		}
+		mustJSON(BackupDir(cfg.DataDir, os.Args[2]))
+	case "restore":
+		if len(os.Args) < 3 {
+			log.Fatal("usage: magpie restore <archive.tar.gz>")
+		}
+		mustJSON(RestoreDirWithLimit(os.Args[2], cfg.DataDir, cfg.MaxRestoreBytes))
+	case "smoke":
+		mustJSON(runSmoke(openStore()))
+	default:
+		log.Fatalf("unknown command %q", sanitizeLogValue(cmd))
+	}
+}
+
+func serve(cfg Config, store *FileStore) {
+	if cfg.ReindexOnStart {
+		log.Printf("reindexing metadata from %s", cfg.DataDir)
+		if err := store.Reindex(); err != nil {
+			log.Fatalf("reindex store: %v", err)
+		}
+	}
+
+	server := NewServerInstance(store, cfg)
+	srv := &http.Server{
+		Addr:              cfg.Addr,
+		Handler:           server.Handler(),
+		ReadHeaderTimeout: 5 * time.Second,
+		ReadTimeout:       2 * time.Minute,
+		WriteTimeout:      2 * time.Minute,
+		IdleTimeout:       60 * time.Second,
+		MaxHeaderBytes:    16 * 1024,
+	}
+
+	go func() {
+		log.Printf("object store listening on %s, data=%s", cfg.Addr, cfg.DataDir)
+		if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			log.Fatalf("listen: %v", err)
+		}
+	}()
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	server.startMultipartCleanup(ctx)
+	server.startReplicationWorker(ctx)
+	<-ctx.Done()
+
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if err := srv.Shutdown(shutdownCtx); err != nil {
+		log.Printf("shutdown: %v", err)
+	}
+}
+
+type Config struct {
+	Addr                     string
+	DataDir                  string
+	AuthToken                string
+	S3AccessKeyID            string
+	S3SecretAccessKey        string
+	S3Keys                   []AccessKey
+	S3Region                 string
+	AllowUnsignedLocal       bool
+	MaxObjectSize            int64
+	MinFreeBytes             uint64
+	PresignedMaxExpiry       time.Duration
+	MultipartMaxAge          time.Duration
+	ReindexOnStart           bool
+	ReplicationPeers         []string
+	ReplicationSecret        string
+	AllowInsecureReplication bool
+	ReplicationMaxJobs       int
+	ReplicationMaxAttempts   int
+	MaxRestoreBytes          int64
+	AllowedBuckets           map[string]bool
+	PublicPrefixes           map[string]bool
+	RateLimitPerMinute       int
+	TrustProxyHeaders        bool
+}
+
+type AccessKey struct {
+	ID          string
+	Secret      string
+	Permissions map[string]bool
+}
+
+func configFromEnv() Config {
+	addr := getenv("MAGPIE_ADDR", "127.0.0.1:8090")
+	dataDir := getenv("MAGPIE_DATA_DIR", "./data")
+	authToken := os.Getenv("MAGPIE_AUTH_TOKEN")
+	s3AccessKeyID := os.Getenv("MAGPIE_S3_ACCESS_KEY_ID")
+	s3SecretAccessKey := os.Getenv("MAGPIE_S3_SECRET_ACCESS_KEY")
+	s3Keys := parseAccessKeys(os.Getenv("MAGPIE_S3_KEYS"), s3AccessKeyID, s3SecretAccessKey)
+	s3Region := getenv("MAGPIE_S3_REGION", "auto")
+	maxObjectSize := mustParseBytes("MAGPIE_MAX_OBJECT_SIZE", defaultMaxObjectSize)
+	minFreeBytes := mustParseUint("MAGPIE_MIN_FREE_BYTES", defaultMinFreeBytes)
+	presignedMaxExpiry := mustParseDuration("MAGPIE_PRESIGNED_MAX_EXPIRY", defaultPresignedMaxExpiry)
+	multipartMaxAge := mustParseDuration("MAGPIE_MULTIPART_MAX_AGE", defaultMultipartMaxAge)
+	reindexOnStart := mustParseBool("MAGPIE_REINDEX_ON_START", false)
+	allowInsecureReplication := mustParseBool("MAGPIE_ALLOW_INSECURE_REPLICATION", false)
+	replicationPeers := parseReplicationPeers(os.Getenv("MAGPIE_REPLICATION_PEERS"), allowInsecureReplication)
+	replicationSecret := os.Getenv("MAGPIE_REPLICATION_SECRET")
+	replicationMaxJobs := mustParseInt("MAGPIE_REPLICATION_MAX_JOBS", defaultReplicationMaxJobs)
+	replicationMaxAttempts := mustParseInt("MAGPIE_REPLICATION_MAX_ATTEMPTS", defaultReplicationMaxAttempts)
+	maxRestoreBytes := mustParseBytes("MAGPIE_MAX_RESTORE_BYTES", defaultMaxRestoreBytes)
+	allowedBuckets := parseSet(os.Getenv("MAGPIE_ALLOWED_BUCKETS"))
+	publicPrefixes := parsePublicPrefixes(os.Getenv("MAGPIE_PUBLIC_PREFIXES"))
+	rateLimitPerMinute := mustParseInt("MAGPIE_RATE_LIMIT_PER_MINUTE", 600)
+	trustProxyHeaders := mustParseBool("MAGPIE_TRUST_PROXY_HEADERS", false)
+	allowUnsignedLocal := mustParseBool("MAGPIE_ALLOW_UNSIGNED_PAYLOADS", false)
+
+	if authToken == "" && len(s3Keys) == 0 {
+		log.Fatal("MAGPIE_AUTH_TOKEN or MAGPIE_S3_ACCESS_KEY_ID/MAGPIE_S3_SECRET_ACCESS_KEY or MAGPIE_S3_KEYS is required")
+	}
+	if len(replicationPeers) > 0 && replicationSecret == "" {
+		log.Fatal("MAGPIE_REPLICATION_SECRET is required when MAGPIE_REPLICATION_PEERS is set")
+	}
+
+	if raw := os.Getenv("MAGPIE_MAX_PROCS"); raw != "" {
+		if _, err := strconv.Atoi(raw); err != nil {
+			log.Fatalf("MAGPIE_MAX_PROCS must be an integer: %v", err)
+		}
+	}
+
+	return Config{
+		Addr:                     addr,
+		DataDir:                  dataDir,
+		AuthToken:                authToken,
+		S3AccessKeyID:            s3AccessKeyID,
+		S3SecretAccessKey:        s3SecretAccessKey,
+		S3Keys:                   s3Keys,
+		S3Region:                 s3Region,
+		AllowUnsignedLocal:       allowUnsignedLocal,
+		MaxObjectSize:            maxObjectSize,
+		MinFreeBytes:             minFreeBytes,
+		PresignedMaxExpiry:       presignedMaxExpiry,
+		MultipartMaxAge:          multipartMaxAge,
+		ReindexOnStart:           reindexOnStart,
+		ReplicationPeers:         replicationPeers,
+		ReplicationSecret:        replicationSecret,
+		AllowInsecureReplication: allowInsecureReplication,
+		ReplicationMaxJobs:       replicationMaxJobs,
+		ReplicationMaxAttempts:   replicationMaxAttempts,
+		MaxRestoreBytes:          maxRestoreBytes,
+		AllowedBuckets:           allowedBuckets,
+		PublicPrefixes:           publicPrefixes,
+		RateLimitPerMinute:       rateLimitPerMinute,
+		TrustProxyHeaders:        trustProxyHeaders,
+	}
+}
+
+func parseAccessKeys(raw string, legacyID string, legacySecret string) []AccessKey {
+	keys := make([]AccessKey, 0)
+	if legacyID != "" && legacySecret != "" {
+		keys = append(keys, AccessKey{ID: legacyID, Secret: legacySecret, Permissions: permissions("read,write,admin")})
+	}
+	for _, entry := range strings.Split(raw, ";") {
+		entry = strings.TrimSpace(entry)
+		if entry == "" {
+			continue
+		}
+		parts := strings.Split(entry, ":")
+		if len(parts) < 2 || parts[0] == "" || parts[1] == "" {
+			log.Fatal("MAGPIE_S3_KEYS entries must be id:secret[:read,write,admin]")
+		}
+		perms := "read,write"
+		if len(parts) >= 3 {
+			perms = parts[2]
+		}
+		keys = append(keys, AccessKey{ID: parts[0], Secret: parts[1], Permissions: permissions(perms)})
+	}
+	return keys
+}
+
+func permissions(raw string) map[string]bool {
+	result := map[string]bool{}
+	for _, part := range strings.Split(raw, ",") {
+		part = strings.TrimSpace(strings.ToLower(part))
+		if part != "" {
+			result[part] = true
+		}
+	}
+	return result
+}
+
+func parseCSV(raw string) []string {
+	parts := make([]string, 0)
+	for _, part := range strings.Split(raw, ",") {
+		part = strings.TrimSpace(part)
+		if part != "" {
+			parts = append(parts, strings.TrimRight(part, "/"))
+		}
+	}
+	return parts
+}
+
+func parseReplicationPeers(raw string, allowInsecure bool) []string {
+	peers := parseCSV(raw)
+	for _, peer := range peers {
+		if err := validateReplicationPeer(peer, allowInsecure); err != nil {
+			log.Fatal(err)
+		}
+	}
+	return peers
+}
+
+func parseSet(raw string) map[string]bool {
+	values := map[string]bool{}
+	for _, value := range parseCSV(raw) {
+		values[value] = true
+	}
+	return values
+}
+
+func parsePublicPrefixes(raw string) map[string]bool {
+	value := strings.TrimSpace(strings.ToLower(raw))
+	if value == "none" || value == "false" || value == "0" {
+		return map[string]bool{}
+	}
+	if strings.TrimSpace(raw) != "" {
+		prefixes := parseSet(raw)
+		for prefix := range prefixes {
+			if err := ValidateKey(strings.TrimSuffix(prefix, "/")); err != nil || !strings.Contains(strings.Trim(prefix, "/"), "/") {
+				log.Fatal("MAGPIE_PUBLIC_PREFIXES entries must be bucket/prefix values")
+			}
+		}
+		return prefixes
+	}
+	return map[string]bool{}
+}
+
+func printUsage() {
+	fmt.Println("usage: magpie [serve|reindex|stats|scrub|repair|version|backup|restore|smoke]")
+}
+
+func mustJSON(err error) {
+	if err != nil {
+		log.Fatal(err)
+	}
+	printJSON(map[string]string{"status": "ok"})
+}
+
+func printJSON(value any) {
+	encoder := json.NewEncoder(os.Stdout)
+	encoder.SetIndent("", "  ")
+	if err := encoder.Encode(value); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func getenv(key, fallback string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return fallback
+}
+
+func mustParseBytes(key string, fallback int64) int64 {
+	value := os.Getenv(key)
+	if value == "" {
+		return fallback
+	}
+	parsed, err := strconv.ParseInt(value, 10, 64)
+	if err != nil || parsed < 0 {
+		log.Fatal(fmt.Sprintf("%s must be a non-negative integer byte count", key))
+	}
+	return parsed
+}
+
+func mustParseUint(key string, fallback uint64) uint64 {
+	value := os.Getenv(key)
+	if value == "" {
+		return fallback
+	}
+	parsed, err := strconv.ParseUint(value, 10, 64)
+	if err != nil {
+		log.Fatal(fmt.Sprintf("%s must be a non-negative integer byte count", key))
+	}
+	return parsed
+}
+
+func mustParseDuration(key string, fallback time.Duration) time.Duration {
+	value := os.Getenv(key)
+	if value == "" {
+		return fallback
+	}
+	parsed, err := time.ParseDuration(value)
+	if err != nil || parsed <= 0 {
+		log.Fatal(fmt.Sprintf("%s must be a positive duration like 1h or 24h", key))
+	}
+	return parsed
+}
+
+func mustParseBool(key string, fallback bool) bool {
+	value := os.Getenv(key)
+	if value == "" {
+		return fallback
+	}
+	parsed, err := strconv.ParseBool(value)
+	if err != nil {
+		log.Fatal(fmt.Sprintf("%s must be true or false", key))
+	}
+	return parsed
+}
+
+func mustParseInt(key string, fallback int) int {
+	value := os.Getenv(key)
+	if value == "" {
+		return fallback
+	}
+	parsed, err := strconv.Atoi(value)
+	if err != nil || parsed < 0 {
+		log.Fatal(fmt.Sprintf("%s must be a non-negative integer", key))
+	}
+	return parsed
+}
+
+func runSmoke(store *FileStore) error {
+	key := "smoke/test.txt"
+	if _, err := store.Put(key, strings.NewReader("ok"), "text/plain"); err != nil {
+		return err
+	}
+	obj, err := store.Open(key)
+	if err != nil {
+		return err
+	}
+	_ = obj.Close()
+	if _, err := store.List("smoke/", 10); err != nil {
+		return err
+	}
+	if _, err := store.Scrub(); err != nil {
+		return err
+	}
+	return store.Delete(key)
+}

replication.go

@@ -0,0 +1,330 @@
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+const (
+	replicationOpPut    = "put"
+	replicationOpDelete = "delete"
+)
+
+type ReplicationJob struct {
+	ID       int64
+	Peer     string
+	Op       string
+	Key      string
+	Attempts int
+}
+
+func (s *Server) enqueueReplicationPut(key string) {
+	s.enqueueReplication(replicationOpPut, key)
+}
+
+func (s *Server) enqueueReplicationDelete(key string) {
+	s.enqueueReplication(replicationOpDelete, key)
+}
+
+func (s *Server) enqueueReplication(op string, key string) {
+	if len(s.cfg.ReplicationPeers) == 0 || len(s.cfg.S3Keys) == 0 || s.cfg.ReplicationSecret == "" {
+		return
+	}
+	for _, peer := range s.cfg.ReplicationPeers {
+		if err := s.store.(*FileStore).EnqueueReplication(peer, op, key, s.cfg.ReplicationMaxJobs); err != nil {
+			log.Printf("replication enqueue op=%s key=%s peer=%s error=%v", op, sanitizeLogValue(key), sanitizeLogValue(peer), err)
+		}
+	}
+	go s.processReplicationQueue(context.Background())
+}
+
+func (s *Server) startReplicationWorker(ctx context.Context) {
+	if len(s.cfg.ReplicationPeers) == 0 || len(s.cfg.S3Keys) == 0 || s.cfg.ReplicationSecret == "" {
+		return
+	}
+	go func() {
+		ticker := time.NewTicker(2 * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				s.processReplicationQueue(ctx)
+			}
+		}
+	}()
+}
+
+func (s *Server) processReplicationQueue(ctx context.Context) {
+	jobs, err := s.store.(*FileStore).DueReplicationJobs(25)
+	if err != nil {
+		log.Printf("replication due jobs error=%v", err)
+		return
+	}
+	for _, job := range jobs {
+		if ctx.Err() != nil {
+			return
+		}
+		if err := s.performReplicationJob(job); err != nil {
+			log.Printf("replication job id=%d op=%s key=%s peer=%s error=%v", job.ID, job.Op, sanitizeLogValue(job.Key), sanitizeLogValue(job.Peer), err)
+			if s.cfg.ReplicationMaxAttempts > 0 && job.Attempts+1 >= s.cfg.ReplicationMaxAttempts {
+				_ = s.store.(*FileStore).CompleteReplicationJob(job.ID)
+				continue
+			}
+			_ = s.store.(*FileStore).FailReplicationJob(job.ID, job.Attempts+1, err.Error())
+			continue
+		}
+		_ = s.store.(*FileStore).CompleteReplicationJob(job.ID)
+	}
+}
+
+func (s *Server) performReplicationJob(job ReplicationJob) error {
+	switch job.Op {
+	case replicationOpPut:
+		return s.performReplicationPut(job.Peer, job.Key)
+	case replicationOpDelete:
+		return s.performReplicationDelete(job.Peer, job.Key)
+	default:
+		return fmt.Errorf("unknown replication op %q", job.Op)
+	}
+}
+
+func (s *Server) performReplicationPut(peer string, key string) error {
+	obj, err := s.store.Open(key)
+	if err != nil {
+		return err
+	}
+	defer obj.Close()
+
+	peerURL, err := replicationObjectURL(peer, key, s.cfg.AllowInsecureReplication)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequest(http.MethodPut, peerURL, obj)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", obj.Info.ContentType)
+	req.Header.Set("X-Magpie-Replication", "1")
+	if s.cfg.ReplicationSecret != "" {
+		req.Header.Set("X-Magpie-Replication-Secret", s.cfg.ReplicationSecret)
+	}
+	s.signReplication(req)
+	resp, err := (&http.Client{Timeout: 5 * time.Minute}).Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("peer status %d", resp.StatusCode)
+	}
+	return nil
+}
+
+func (s *Server) performReplicationDelete(peer string, key string) error {
+	peerURL, err := replicationObjectURL(peer, key, s.cfg.AllowInsecureReplication)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequest(http.MethodDelete, peerURL, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("X-Magpie-Replication", "1")
+	if s.cfg.ReplicationSecret != "" {
+		req.Header.Set("X-Magpie-Replication-Secret", s.cfg.ReplicationSecret)
+	}
+	s.signReplication(req)
+	resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("peer status %d", resp.StatusCode)
+	}
+	return nil
+}
+
+func (s *Server) signReplication(req *http.Request) {
+	key := s.cfg.S3Keys[0]
+	amzDate := time.Now().UTC().Format("20060102T150405Z")
+	date := amzDate[:8]
+	scope := date + "/" + s.cfg.S3Region + "/s3/aws4_request"
+	signedHeaders := "host;x-amz-content-sha256;x-amz-date;x-magpie-replication"
+	if req.Header.Get("X-Magpie-Replication-Secret") != "" {
+		signedHeaders += ";x-magpie-replication-secret"
+	}
+	req.Header.Set("X-Amz-Date", amzDate)
+	req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
+	canonical := canonicalRequest(req, signedHeaders, "UNSIGNED-PAYLOAD", false)
+	sig := signature(key.Secret, amzDate, scope, canonical)
+	req.Header.Set("Authorization", awsAlgorithm+" Credential="+key.ID+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
+}
+
+func replicationObjectURL(peer string, key string, allowInsecure bool) (string, error) {
+	if err := ValidateKey(key); err != nil {
+		return "", err
+	}
+	if err := validateReplicationPeer(peer, allowInsecure); err != nil {
+		return "", err
+	}
+	parsed, err := url.Parse(peer)
+	if err != nil {
+		return "", err
+	}
+	parsed.RawQuery = ""
+	parsed.Fragment = ""
+	parsed.Path = strings.TrimRight(parsed.Path, "/") + "/" + key
+	parsed.RawPath = ""
+	return parsed.String(), nil
+}
+
+func validateReplicationPeer(peer string, allowInsecure bool) error {
+	parsed, err := url.Parse(peer)
+	if err != nil || parsed.Scheme == "" || parsed.Host == "" {
+		return fmt.Errorf("invalid replication peer %q", peer)
+	}
+	if parsed.RawQuery != "" || parsed.Fragment != "" {
+		return fmt.Errorf("replication peer %q must not include query or fragment", peer)
+	}
+	if parsed.Scheme == "https" {
+		return nil
+	}
+	if allowInsecure && parsed.Scheme == "http" {
+		return nil
+	}
+	return fmt.Errorf("replication peer %q must use https or set MAGPIE_ALLOW_INSECURE_REPLICATION=true", peer)
+}
+
+func (s *FileStore) EnqueueReplication(peer string, op string, key string, maxJobs int) error {
+	now := time.Now().UTC().Format(time.RFC3339Nano)
+	result, err := s.db.Exec(`
+UPDATE replication_jobs
+SET next_attempt_at = ?, updated_at = ?
+WHERE peer = ? AND op = ? AND key = ?
+`, now, now, peer, op, key)
+	if err != nil {
+		return err
+	}
+	if rows, err := result.RowsAffected(); err == nil && rows > 0 {
+		return nil
+	}
+	if maxJobs <= 0 {
+		maxJobs = defaultReplicationMaxJobs
+	}
+	var pending int
+	if err := s.db.QueryRow(`SELECT COUNT(*) FROM replication_jobs`).Scan(&pending); err != nil {
+		return err
+	}
+	if pending >= maxJobs {
+		return fmt.Errorf("replication queue is full")
+	}
+	_, err = s.db.Exec(`
+INSERT INTO replication_jobs (peer, op, key, next_attempt_at, created_at, updated_at)
+VALUES (?, ?, ?, ?, ?, ?)
+`, peer, op, key, now, now, now)
+	return err
+}
+
+func (s *FileStore) DueReplicationJobs(limit int) ([]ReplicationJob, error) {
+	if limit <= 0 || limit > 100 {
+		limit = 25
+	}
+	now := time.Now().UTC().Format(time.RFC3339Nano)
+	rows, err := s.db.Query(`
+SELECT id, peer, op, key, attempts
+FROM replication_jobs
+WHERE next_attempt_at <= ?
+ORDER BY id ASC
+LIMIT ?
+`, now, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	jobs := make([]ReplicationJob, 0)
+	for rows.Next() {
+		var job ReplicationJob
+		if err := rows.Scan(&job.ID, &job.Peer, &job.Op, &job.Key, &job.Attempts); err != nil {
+			return nil, err
+		}
+		jobs = append(jobs, job)
+	}
+	return jobs, rows.Err()
+}
+
+func (s *FileStore) CompleteReplicationJob(id int64) error {
+	_, err := s.db.Exec(`DELETE FROM replication_jobs WHERE id = ?`, id)
+	return err
+}
+
+func (s *FileStore) FailReplicationJob(id int64, attempts int, message string) error {
+	delay := time.Duration(attempts*attempts) * time.Second
+	if delay > 5*time.Minute {
+		delay = 5 * time.Minute
+	}
+	next := time.Now().UTC().Add(delay).Format(time.RFC3339Nano)
+	now := time.Now().UTC().Format(time.RFC3339Nano)
+	_, err := s.db.Exec(`
+UPDATE replication_jobs
+SET attempts = ?, next_attempt_at = ?, last_error = ?, updated_at = ?
+WHERE id = ?
+`, attempts, next, truncateError(message), now, id)
+	return err
+}
+
+func (s *FileStore) EnqueueRepair(peers []string) (int, error) {
+	if len(peers) == 0 {
+		return 0, nil
+	}
+	count := 0
+	after := ""
+	for {
+		page, err := s.ListPage("", after, 1000)
+		if err != nil {
+			return count, err
+		}
+		for _, object := range page.Objects {
+			for _, peer := range peers {
+				if err := s.EnqueueReplication(peer, replicationOpPut, object.Key, defaultReplicationMaxJobs); err != nil {
+					return count, err
+				}
+				count++
+			}
+		}
+		if page.NextAfter == "" {
+			return count, nil
+		}
+		after = page.NextAfter
+	}
+}
+
+func (s *FileStore) ReplicationQueueStats() (ReplicationQueueStats, error) {
+	var stats ReplicationQueueStats
+	err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(attempts), 0) FROM replication_jobs`).Scan(&stats.PendingJobs, &stats.MaxAttempts)
+	if errors.Is(err, sql.ErrNoRows) {
+		return stats, nil
+	}
+	return stats, err
+}
+
+func truncateError(message string) string {
+	if len(message) <= 500 {
+		return message
+	}
+	return message[:500]
+}
+
+type ReplicationQueueStats struct {
+	PendingJobs int64 `json:"pending_jobs"`
+	MaxAttempts int   `json:"max_attempts"`
+}

s3.go

@@ -0,0 +1,938 @@
+package main
+
+import (
+	"context"
+	"crypto/md5"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	maxXMLRequestBytes  = 1 << 20
+	maxBatchDeleteItems = 1000
+)
+
+var errPayloadHashMismatch = errors.New("payload hash mismatch")
+
+func (s *Server) s3Object(w http.ResponseWriter, r *http.Request) {
+	if s.s3BucketList(w, r) {
+		return
+	}
+	if s.s3BucketPost(w, r) {
+		return
+	}
+
+	bucket, key, ok := s.s3BucketKey(r.URL.Path)
+	if !ok {
+		s3Error(w, http.StatusNotFound, "NoSuchBucket", "bucket not found")
+		return
+	}
+	if !s.bucketAllowed(bucket) {
+		s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed")
+		return
+	}
+
+	storeKey := bucket + "/" + key
+	if err := ValidateKey(storeKey); err != nil {
+		s3Error(w, http.StatusBadRequest, "InvalidObjectName", err.Error())
+		return
+	}
+	publicRead := (r.Method == http.MethodGet || r.Method == http.MethodHead) && s.publicReadAllowed(storeKey)
+	if !publicRead && !s.authorizedS3(r) {
+		s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid")
+		return
+	}
+	if !s.replicationAllowed(r) {
+		s3Error(w, http.StatusForbidden, "AccessDenied", "replication secret is invalid")
+		return
+	}
+
+	switch r.Method {
+	case http.MethodPost:
+		if _, ok := r.URL.Query()["uploads"]; ok {
+			s.s3CreateMultipartUpload(w, r, storeKey)
+		} else if r.URL.Query().Get("uploadId") != "" {
+			s.s3CompleteMultipartUpload(w, r, storeKey)
+		} else if _, ok := r.URL.Query()["delete"]; ok {
+			s.s3DeleteObjects(w, r, bucket)
+		} else {
+			s3Error(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed")
+		}
+	case http.MethodPut:
+		if r.URL.Query().Get("uploadId") != "" && r.URL.Query().Get("partNumber") != "" {
+			s.s3UploadPart(w, r, storeKey)
+		} else if r.Header.Get("X-Amz-Copy-Source") != "" {
+			s.s3CopyObject(w, r, storeKey)
+		} else {
+			s.s3Put(w, r, storeKey)
+		}
+	case http.MethodGet:
+		s.s3Get(w, r, storeKey, publicRead)
+	case http.MethodHead:
+		s.s3Head(w, r, storeKey, publicRead)
+	case http.MethodDelete:
+		if r.URL.Query().Get("uploadId") != "" {
+			s.s3AbortMultipartUpload(w, r, storeKey)
+		} else {
+			s.s3Delete(w, r, storeKey)
+		}
+	default:
+		w.Header().Set("Allow", "PUT, GET, HEAD, DELETE")
+		s3Error(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed")
+	}
+}
+
+func (s *Server) s3BucketPost(w http.ResponseWriter, r *http.Request) bool {
+	path := strings.Trim(r.URL.Path, "/")
+	if r.Method != http.MethodPost || path == "" || strings.Contains(path, "/") {
+		return false
+	}
+	if _, ok := r.URL.Query()["delete"]; !ok {
+		return false
+	}
+	if err := ValidateKey(path); err != nil {
+		s3Error(w, http.StatusBadRequest, "InvalidBucketName", err.Error())
+		return true
+	}
+	if !s.bucketAllowed(path) {
+		s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed")
+		return true
+	}
+	if !s.authorizedS3(r) {
+		s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid")
+		return true
+	}
+	if !s.replicationAllowed(r) {
+		s3Error(w, http.StatusForbidden, "AccessDenied", "replication secret is invalid")
+		return true
+	}
+	s.s3DeleteObjects(w, r, path)
+	return true
+}
+
+func (s *Server) s3CreateMultipartUpload(w http.ResponseWriter, r *http.Request, key string) {
+	uploadID, err := randomID()
+	if err != nil {
+		s3Error(w, http.StatusInternalServerError, "InternalError", "upload id failed")
+		return
+	}
+	dir := s.multipartDir(uploadID)
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		s3Error(w, http.StatusInternalServerError, "InternalError", "upload init failed")
+		return
+	}
+	meta := multipartMeta{Key: key, ContentType: r.Header.Get("Content-Type")}
+	data, _ := xml.Marshal(meta)
+	if err := os.WriteFile(filepath.Join(dir, "meta.xml"), data, 0o600); err != nil {
+		s3Error(w, http.StatusInternalServerError, "InternalError", "upload init failed")
+		return
+	}
+	w.Header().Set("Content-Type", "application/xml")
+	w.WriteHeader(http.StatusOK)
+	_ = xml.NewEncoder(w).Encode(createMultipartUploadResult{UploadID: uploadID})
+}
+
+func (s *Server) s3UploadPart(w http.ResponseWriter, r *http.Request, key string) {
+	release, ok := s.acceptWrite(w, r, true)
+	if !ok {
+		return
+	}
+	defer release()
+	uploadID := r.URL.Query().Get("uploadId")
+	partNumber, err := strconv.Atoi(r.URL.Query().Get("partNumber"))
+	if err != nil || partNumber <= 0 || partNumber > 10_000 {
+		s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid part number")
+		return
+	}
+	meta, err := s.multipartMeta(uploadID)
+	if err != nil || meta.Key != key {
+		s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found")
+		return
+	}
+	dir := s.multipartDir(uploadID)
+	if !s.acceptMultipartPartSize(w, dir, r.ContentLength) {
+		return
+	}
+	partPath := filepath.Join(dir, fmt.Sprintf("part-%05d", partNumber))
+	body, verifyPayload, ok := s.s3StreamingBody(w, r)
+	if !ok {
+		return
+	}
+	tmp, err := os.CreateTemp(dir, ".part-*")
+	if err != nil {
+		s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed")
+		return
+	}
+	hash := md5Hash()
+	_, copyErr := io.Copy(io.MultiWriter(tmp, hash), http.MaxBytesReader(w, body, s.cfg.MaxObjectSize))
+	closeErr := tmp.Close()
+	if copyErr != nil || closeErr != nil {
+		_ = os.Remove(tmp.Name())
+		if errors.Is(copyErr, errPayloadHashMismatch) {
+			s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
+			return
+		}
+		s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed")
+		return
+	}
+	if !verifyPayload() {
+		_ = os.Remove(tmp.Name())
+		s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
+		return
+	}
+	if err := os.Rename(tmp.Name(), partPath); err != nil {
+		s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed")
+		return
+	}
+	etag := fmt.Sprintf("%x", hash.Sum(nil))
+	w.Header().Set("ETag", quoteETag(etag))
+	w.WriteHeader(http.StatusOK)
+}
+
+func (s *Server) acceptMultipartPartSize(w http.ResponseWriter, dir string, incomingSize int64) bool {
+	if incomingSize < 0 {
+		s3Error(w, http.StatusLengthRequired, "MissingContentLength", "content length is required")
+		return false
+	}
+	if s.cfg.MaxObjectSize <= 0 {
+		return true
+	}
+
+	existingSize := int64(0)
+	parts, _ := filepath.Glob(filepath.Join(dir, "part-*"))
+	for _, part := range parts {
+		info, err := os.Stat(part)
+		if err == nil && !info.IsDir() {
+			existingSize += info.Size()
+		}
+	}
+
+	if existingSize+incomingSize > s.cfg.MaxObjectSize {
+		s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "multipart upload exceeds object size limit")
+		return false
+	}
+
+	return true
+}
+
+func (s *Server) s3CompleteMultipartUpload(w http.ResponseWriter, r *http.Request, key string) {
+	uploadID := r.URL.Query().Get("uploadId")
+	meta, err := s.multipartMeta(uploadID)
+	if err != nil || meta.Key != key {
+		s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found")
+		return
+	}
+	var complete completeMultipartUpload
+	completeBody, ok := s.s3BoundedBody(w, r, maxXMLRequestBytes)
+	if !ok {
+		return
+	}
+	if err := xml.Unmarshal(completeBody, &complete); err != nil && len(completeBody) > 0 {
+		s3Error(w, http.StatusBadRequest, "MalformedXML", "invalid complete request")
+		return
+	}
+	parts := complete.Parts
+	if len(parts) == 0 {
+		entries, _ := filepath.Glob(filepath.Join(s.multipartDir(uploadID), "part-*"))
+		for _, entry := range entries {
+			base := filepath.Base(entry)
+			n, _ := strconv.Atoi(strings.TrimPrefix(base, "part-"))
+			parts = append(parts, completedPart{PartNumber: n})
+		}
+	}
+	if len(parts) > 10_000 {
+		s3Error(w, http.StatusBadRequest, "InvalidRequest", "too many parts")
+		return
+	}
+	sortParts(parts)
+	readers := make([]io.Reader, 0, len(parts))
+	files := make([]*os.File, 0, len(parts))
+	defer func() {
+		for _, file := range files {
+			_ = file.Close()
+		}
+	}()
+	var totalSize int64
+	for _, part := range parts {
+		if part.PartNumber <= 0 || part.PartNumber > 10_000 {
+			s3Error(w, http.StatusBadRequest, "InvalidPart", "invalid part number")
+			return
+		}
+		file, err := os.Open(filepath.Join(s.multipartDir(uploadID), fmt.Sprintf("part-%05d", part.PartNumber)))
+		if err != nil {
+			s3Error(w, http.StatusBadRequest, "InvalidPart", "part missing")
+			return
+		}
+		stat, err := file.Stat()
+		if err != nil {
+			s3Error(w, http.StatusInternalServerError, "InternalError", "part stat failed")
+			return
+		}
+		totalSize += stat.Size()
+		if s.cfg.MaxObjectSize > 0 && totalSize > s.cfg.MaxObjectSize {
+			s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large")
+			return
+		}
+		files = append(files, file)
+		readers = append(readers, file)
+	}
+	release, ok := s.acceptWriteSize(w, totalSize, true)
+	if !ok {
+		return
+	}
+	defer release()
+	info, err := s.store.Put(key, io.MultiReader(readers...), meta.ContentType)
+	if err != nil {
+		s3Error(w, http.StatusInternalServerError, "InternalError", "complete failed")
+		return
+	}
+	_ = os.RemoveAll(s.multipartDir(uploadID))
+	w.Header().Set("Content-Type", "application/xml")
+	w.WriteHeader(http.StatusOK)
+	_ = xml.NewEncoder(w).Encode(completeMultipartUploadResult{Key: strings.TrimPrefix(key, strings.SplitN(key, "/", 2)[0]+"/"), ETag: quoteETag(info.ETag)})
+	if r.Header.Get("X-Magpie-Replication") != "1" {
+		s.enqueueReplicationPut(key)
+	}
+}
+
+func (s *Server) s3AbortMultipartUpload(w http.ResponseWriter, r *http.Request, _ string) {
+	uploadID := r.URL.Query().Get("uploadId")
+	if !validUploadID(uploadID) {
+		s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found")
+		return
+	}
+	_ = os.RemoveAll(s.multipartDir(uploadID))
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func (s *Server) s3CopyObject(w http.ResponseWriter, r *http.Request, destinationKey string) {
+	if !signedHeaderIncludes(r, "x-amz-copy-source") {
+		s3Error(w, http.StatusForbidden, "AccessDenied", "copy source header must be signed")
+		return
+	}
+	sourceKey, err := copySourceKey(r.Header.Get("X-Amz-Copy-Source"))
+	if err != nil {
+		s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid copy source")
+		return
+	}
+	sourceBucket, _, _ := strings.Cut(sourceKey, "/")
+	if !s.bucketAllowed(sourceBucket) {
+		s3Error(w, http.StatusForbidden, "AccessDenied", "copy source bucket is not allowed")
+		return
+	}
+	if len(s.cfg.S3Keys) > 0 {
+		key, ok := s.requestAccessKey(r)
+		if !ok || !key.can("read") || !key.can("write") {
+			s3Error(w, http.StatusForbidden, "AccessDenied", "copy requires read and write permission")
+			return
+		}
+	}
+	obj, err := s.store.Open(sourceKey)
+	if err != nil {
+		if errors.Is(err, ErrNotFound) {
+			s3Error(w, http.StatusNotFound, "NoSuchKey", "copy source not found")
+			return
+		}
+		s3Error(w, http.StatusInternalServerError, "InternalError", "copy source failed")
+		return
+	}
+	defer obj.Close()
+	if s.cfg.MaxObjectSize > 0 && obj.Info.Size > s.cfg.MaxObjectSize {
+		s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large")
+		return
+	}
+	release, ok := s.acceptWriteSize(w, obj.Info.Size, true)
+	if !ok {
+		return
+	}
+	defer release()
+	contentType := obj.Info.ContentType
+	if override := r.Header.Get("Content-Type"); override != "" && strings.EqualFold(r.Header.Get("X-Amz-Metadata-Directive"), "REPLACE") {
+		contentType = override
+	}
+	info, err := s.store.Put(destinationKey, obj, contentType)
+	if err != nil {
+		s3Error(w, http.StatusInternalServerError, "InternalError", "copy failed")
+		return
+	}
+	w.Header().Set("Content-Type", "application/xml")
+	w.WriteHeader(http.StatusOK)
+	_ = xml.NewEncoder(w).Encode(copyObjectResult{ETag: quoteETag(info.ETag), LastModified: info.ModTime.UTC().Format(time.RFC3339)})
+	if r.Header.Get("X-Magpie-Replication") != "1" {
+		s.enqueueReplicationPut(destinationKey)
+	}
+}
+
+func (s *Server) s3DeleteObjects(w http.ResponseWriter, r *http.Request, bucket string) {
+	var request deleteObjectsRequest
+	deleteBody, ok := s.s3BoundedBody(w, r, maxXMLRequestBytes)
+	if !ok {
+		return
+	}
+	if err := xml.Unmarshal(deleteBody, &request); err != nil {
+		s3Error(w, http.StatusBadRequest, "MalformedXML", "invalid delete request")
+		return
+	}
+	if len(request.Objects) > maxBatchDeleteItems {
+		s3Error(w, http.StatusBadRequest, "InvalidRequest", "too many objects in delete request")
+		return
+	}
+	result := deleteObjectsResult{Deleted: make([]deletedObject, 0, len(request.Objects))}
+	for _, object := range request.Objects {
+		key := strings.TrimPrefix(object.Key, "/")
+		storeKey := bucket + "/" + key
+		if err := ValidateKey(storeKey); err != nil {
+			result.Errors = append(result.Errors, deleteObjectError{Key: object.Key, Code: "InvalidObjectName", Message: err.Error()})
+			continue
+		}
+		if err := s.store.Delete(storeKey); err != nil && !errors.Is(err, ErrNotFound) {
+			result.Errors = append(result.Errors, deleteObjectError{Key: object.Key, Code: "InternalError", Message: "delete failed"})
+			continue
+		}
+		result.Deleted = append(result.Deleted, deletedObject{Key: object.Key})
+		if r.Header.Get("X-Magpie-Replication") != "1" {
+			s.enqueueReplicationDelete(storeKey)
+		}
+	}
+	w.Header().Set("Content-Type", "application/xml")
+	w.WriteHeader(http.StatusOK)
+	_ = xml.NewEncoder(w).Encode(result)
+}
+
+func (s *Server) s3BucketList(w http.ResponseWriter, r *http.Request) bool {
+	if r.Method != http.MethodGet || strings.Trim(r.URL.Path, "/") == "" || strings.Contains(strings.Trim(r.URL.Path, "/"), "/") {
+		return false
+	}
+	if r.URL.RawQuery == "" && r.URL.Query().Get("prefix") == "" && r.URL.Query().Get("list-type") == "" {
+		return false
+	}
+
+	bucket := strings.Trim(r.URL.Path, "/")
+	if err := ValidateKey(bucket); err != nil {
+		s3Error(w, http.StatusBadRequest, "InvalidBucketName", err.Error())
+		return true
+	}
+	if !s.bucketAllowed(bucket) {
+		s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed")
+		return true
+	}
+	if !s.authorizedS3(r) {
+		s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid")
+		return true
+	}
+
+	prefix := r.URL.Query().Get("prefix")
+	storePrefix := bucket + "/"
+	if prefix != "" {
+		storePrefix = bucket + "/" + strings.TrimPrefix(prefix, "/")
+	}
+	maxKeys := parseMaxKeys(r.URL.Query().Get("max-keys"))
+	continuationToken := r.URL.Query().Get("continuation-token")
+	after, err := decodeContinuationToken(continuationToken)
+	if err != nil {
+		s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid continuation token")
+		return true
+	}
+	page, err := s.store.ListPage(storePrefix, after, maxKeys)
+	if err != nil {
+		s3Error(w, http.StatusInternalServerError, "InternalError", "list failed")
+		return true
+	}
+
+	contents := make([]s3ListContent, 0, len(page.Objects))
+	for _, object := range page.Objects {
+		key := strings.TrimPrefix(object.Key, bucket+"/")
+		contents = append(contents, s3ListContent{
+			Key:          key,
+			LastModified: object.ModTime.UTC().Format(time.RFC3339),
+			ETag:         quoteETag(object.ETag),
+			Size:         object.Size,
+			StorageClass: "STANDARD",
+		})
+	}
+
+	w.Header().Set("Content-Type", "application/xml")
+	w.WriteHeader(http.StatusOK)
+	_ = xml.NewEncoder(w).Encode(s3ListBucketResult{
+		Name:                  bucket,
+		Prefix:                prefix,
+		ContinuationToken:     continuationToken,
+		NextContinuationToken: encodeContinuationToken(page.NextAfter),
+		KeyCount:              len(contents),
+		MaxKeys:               maxKeys,
+		IsTruncated:           page.NextAfter != "",
+		Contents:              contents,
+	})
+	return true
+}
+
+func encodeContinuationToken(key string) string {
+	if key == "" {
+		return ""
+	}
+	return base64.RawURLEncoding.EncodeToString([]byte(key))
+}
+
+func decodeContinuationToken(token string) (string, error) {
+	if token == "" {
+		return "", nil
+	}
+	decoded, err := base64.RawURLEncoding.DecodeString(token)
+	if err != nil {
+		return "", err
+	}
+	key := string(decoded)
+	if err := ValidateKey(key); err != nil {
+		return "", err
+	}
+	return key, nil
+}
+
+func copySourceKey(value string) (string, error) {
+	value = strings.TrimPrefix(value, "/")
+	if value == "" {
+		return "", fmt.Errorf("missing copy source")
+	}
+	if decoded, err := url.PathUnescape(value); err == nil {
+		value = decoded
+	}
+	if err := ValidateKey(value); err != nil {
+		return "", err
+	}
+	return value, nil
+}
+
+func (s *Server) s3BucketKey(path string) (string, string, bool) {
+	path = strings.TrimPrefix(path, "/")
+	parts := strings.SplitN(path, "/", 2)
+	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
+		return "", "", false
+	}
+	if err := ValidateKey(parts[0]); err != nil {
+		return "", "", false
+	}
+	if err := ValidateKey(parts[1]); err != nil {
+		return "", "", false
+	}
+	return parts[0], parts[1], true
+}
+
+func (s *Server) s3Put(w http.ResponseWriter, r *http.Request, key string) {
+	release, ok := s.acceptWrite(w, r, true)
+	if !ok {
+		return
+	}
+	defer release()
+	contentType := r.Header.Get("Content-Type")
+	body, verifyPayload, ok := s.s3StreamingBody(w, r)
+	if !ok {
+		return
+	}
+	store, ok := s.store.(*FileStore)
+	if !ok {
+		s3Error(w, http.StatusInternalServerError, "InternalError", "store failed")
+		return
+	}
+	info, err := store.PutVerified(key, http.MaxBytesReader(w, body, s.cfg.MaxObjectSize), contentType, func() error {
+		if !verifyPayload() {
+			return errPayloadHashMismatch
+		}
+		return nil
+	})
+	if err != nil {
+		if maxBytesError(err) {
+			s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large")
+			return
+		}
+		if errors.Is(err, errPayloadHashMismatch) {
+			s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
+			return
+		}
+		s3Error(w, http.StatusInternalServerError, "InternalError", "store failed")
+		return
+	}
+	w.Header().Set("ETag", quoteETag(info.ETag))
+	w.WriteHeader(http.StatusOK)
+	if r.Header.Get("X-Magpie-Replication") != "1" {
+		s.enqueueReplicationPut(key)
+	}
+}
+
+func (s *Server) s3Get(w http.ResponseWriter, r *http.Request, key string, publicRead bool) {
+	obj, err := s.store.Open(key)
+	if err != nil {
+		if errors.Is(err, ErrNotFound) {
+			s3Error(w, http.StatusNotFound, "NoSuchKey", "object not found")
+			return
+		}
+		s3Error(w, http.StatusInternalServerError, "InternalError", "read failed")
+		return
+	}
+	defer obj.Close()
+
+	s3ObjectHeaders(w, obj.Info)
+	if publicRead {
+		hardenPublicRead(w, obj.Info.ContentType)
+	} else if disposition := r.URL.Query().Get("response-content-disposition"); disposition != "" {
+		w.Header().Set("Content-Disposition", disposition)
+	}
+	http.ServeContent(w, r, obj.Info.Key, obj.Info.ModTime, obj)
+}
+
+func (s *Server) s3Head(w http.ResponseWriter, _ *http.Request, key string, publicRead bool) {
+	info, err := s.store.Stat(key)
+	if err != nil {
+		if errors.Is(err, ErrNotFound) {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	s3ObjectHeaders(w, info)
+	if publicRead {
+		hardenPublicRead(w, info.ContentType)
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+// hardenPublicRead applies defense-in-depth headers to unauthenticated public
+// reads. Objects carry an uploader-supplied Content-Type, so a public prefix
+// could otherwise serve active content (HTML, SVG) that executes script in
+// Magpie's own origin. The CSP sandbox disables scripting even when a browser
+// navigates directly to such an object, and any content type that is not inert
+// inline media is forced to download rather than render. The attacker-influenced
+// response-content-disposition override is intentionally not honored here.
+func hardenPublicRead(w http.ResponseWriter, contentType string) {
+	w.Header().Set("Content-Security-Policy", "default-src 'none'; sandbox")
+	if !inlineSafeContentType(contentType) {
+		w.Header().Set("Content-Disposition", "attachment")
+	}
+}
+
+// inlineSafeContentType reports whether a content type is inert raster/media
+// that browsers render without executing script. SVG is excluded because it can
+// carry script when loaded as a top-level document.
+func inlineSafeContentType(contentType string) bool {
+	mediaType := contentType
+	if i := strings.IndexByte(mediaType, ';'); i >= 0 {
+		mediaType = mediaType[:i]
+	}
+	mediaType = strings.ToLower(strings.TrimSpace(mediaType))
+	if mediaType == "image/svg+xml" || mediaType == "image/svg" {
+		return false
+	}
+	return strings.HasPrefix(mediaType, "image/") ||
+		strings.HasPrefix(mediaType, "audio/") ||
+		strings.HasPrefix(mediaType, "video/")
+}
+
+func (s *Server) s3Delete(w http.ResponseWriter, r *http.Request, key string) {
+	if err := s.store.Delete(key); err != nil && !errors.Is(err, ErrNotFound) {
+		s3Error(w, http.StatusInternalServerError, "InternalError", "delete failed")
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+	if r.Header.Get("X-Magpie-Replication") != "1" {
+		s.enqueueReplicationDelete(key)
+	}
+}
+
+func s3ObjectHeaders(w http.ResponseWriter, info ObjectInfo) {
+	contentType := info.ContentType
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	w.Header().Set("Content-Type", contentType)
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
+	w.Header().Set("ETag", quoteETag(info.ETag))
+	w.Header().Set("Last-Modified", info.ModTime.UTC().Format(http.TimeFormat))
+	w.Header().Set("Accept-Ranges", "bytes")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+}
+
+func quoteETag(etag string) string {
+	if etag == "" {
+		return "\"\""
+	}
+	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
+		return etag
+	}
+	return "\"" + etag + "\""
+}
+
+type s3ErrorBody struct {
+	XMLName xml.Name `xml:"Error"`
+	Code    string   `xml:"Code"`
+	Message string   `xml:"Message"`
+}
+
+func s3Error(w http.ResponseWriter, status int, code string, message string) {
+	w.Header().Set("Content-Type", "application/xml")
+	w.WriteHeader(status)
+	_ = xml.NewEncoder(w).Encode(s3ErrorBody{Code: code, Message: message})
+}
+
+func parseMaxKeys(value string) int {
+	if value == "" {
+		return 1000
+	}
+	parsed, err := strconv.Atoi(value)
+	if err != nil || parsed <= 0 {
+		return 1000
+	}
+	if parsed > 1000 {
+		return 1000
+	}
+	return parsed
+}
+
+func randomID() (string, error) {
+	buf := make([]byte, 18)
+	if _, err := rand.Read(buf); err != nil {
+		return "", err
+	}
+	return base64.RawURLEncoding.EncodeToString(buf), nil
+}
+
+func md5Hash() hashHash {
+	return md5.New()
+}
+
+type hashHash interface {
+	io.Writer
+	Sum([]byte) []byte
+}
+
+func (s *Server) multipartDir(uploadID string) string {
+	if !validUploadID(uploadID) {
+		return filepath.Join(s.store.(*FileStore).root, ".multipart", "invalid")
+	}
+	return filepath.Join(s.store.(*FileStore).root, ".multipart", uploadID)
+}
+
+func validUploadID(uploadID string) bool {
+	if uploadID == "" || len(uploadID) > 128 {
+		return false
+	}
+	for _, ch := range uploadID {
+		if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '-' || ch == '_' {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
+func (s *Server) startMultipartCleanup(ctx context.Context) {
+	store, ok := s.store.(*FileStore)
+	if !ok || s.cfg.MultipartMaxAge <= 0 {
+		return
+	}
+	if err := store.CleanupMultipart(s.cfg.MultipartMaxAge); err != nil {
+		log.Printf("multipart cleanup error=%v", err)
+	}
+	go func() {
+		ticker := time.NewTicker(time.Hour)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				if err := store.CleanupMultipart(s.cfg.MultipartMaxAge); err != nil {
+					log.Printf("multipart cleanup error=%v", err)
+				}
+			}
+		}
+	}()
+}
+
+func (s *Server) s3BoundedBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, bool) {
+	body, verifyPayload, ok := s.s3StreamingBody(w, r)
+	if !ok {
+		return nil, false
+	}
+	data, err := io.ReadAll(http.MaxBytesReader(w, body, limit))
+	if err != nil {
+		if errors.Is(err, errPayloadHashMismatch) {
+			s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
+			return nil, false
+		}
+		s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "request body too large")
+		return nil, false
+	}
+	if !verifyPayload() {
+		s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
+		return nil, false
+	}
+	return data, true
+}
+
+func (s *Server) s3StreamingBody(w http.ResponseWriter, r *http.Request) (io.ReadCloser, func() bool, bool) {
+	payloadHash := r.Header.Get("X-Amz-Content-Sha256")
+	if payloadHash == "" || payloadHash == "UNSIGNED-PAYLOAD" {
+		return r.Body, func() bool { return true }, true
+	}
+	if len(payloadHash) != sha256.Size*2 {
+		s3Error(w, http.StatusBadRequest, "InvalidDigest", "unsupported payload hash")
+		return nil, nil, false
+	}
+	if _, err := hex.DecodeString(payloadHash); err != nil {
+		s3Error(w, http.StatusBadRequest, "InvalidDigest", "invalid payload hash")
+		return nil, nil, false
+	}
+	hash := sha256.New()
+	verified := false
+	return hashVerifyingReadCloser{reader: r.Body, hash: hash, expected: strings.ToLower(payloadHash)}, func() bool {
+		if verified {
+			return true
+		}
+		verified = hex.EncodeToString(hash.Sum(nil)) == strings.ToLower(payloadHash)
+		return verified
+	}, true
+}
+
+func signedHeaderIncludes(r *http.Request, header string) bool {
+	signedHeaders := r.URL.Query().Get("X-Amz-SignedHeaders")
+	if signedHeaders == "" {
+		auth := r.Header.Get("Authorization")
+		if strings.HasPrefix(auth, awsAlgorithm+" ") {
+			signedHeaders = parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" "))["SignedHeaders"]
+		}
+	}
+	for _, signed := range strings.Split(signedHeaders, ";") {
+		if strings.EqualFold(strings.TrimSpace(signed), header) {
+			return true
+		}
+	}
+	return false
+}
+
+type hashVerifyingReadCloser struct {
+	reader   io.ReadCloser
+	hash     hashHash
+	expected string
+}
+
+func (r hashVerifyingReadCloser) Read(p []byte) (int, error) {
+	n, err := r.reader.Read(p)
+	if n > 0 {
+		_, _ = r.hash.Write(p[:n])
+	}
+	if errors.Is(err, io.EOF) && hex.EncodeToString(r.hash.Sum(nil)) != r.expected {
+		return n, errPayloadHashMismatch
+	}
+	return n, err
+}
+
+func (r hashVerifyingReadCloser) Close() error {
+	return r.reader.Close()
+}
+
+func (s *Server) multipartMeta(uploadID string) (multipartMeta, error) {
+	data, err := os.ReadFile(filepath.Join(s.multipartDir(uploadID), "meta.xml"))
+	if err != nil {
+		return multipartMeta{}, err
+	}
+	var meta multipartMeta
+	if err := xml.Unmarshal(data, &meta); err != nil {
+		return multipartMeta{}, err
+	}
+	return meta, nil
+}
+
+func sortParts(parts []completedPart) {
+	sort.Slice(parts, func(i, j int) bool { return parts[i].PartNumber < parts[j].PartNumber })
+}
+
+type s3ListBucketResult struct {
+	XMLName               xml.Name        `xml:"ListBucketResult"`
+	Xmlns                 string          `xml:"xmlns,attr,omitempty"`
+	Name                  string          `xml:"Name"`
+	Prefix                string          `xml:"Prefix"`
+	ContinuationToken     string          `xml:"ContinuationToken,omitempty"`
+	NextContinuationToken string          `xml:"NextContinuationToken,omitempty"`
+	KeyCount              int             `xml:"KeyCount"`
+	MaxKeys               int             `xml:"MaxKeys"`
+	IsTruncated           bool            `xml:"IsTruncated"`
+	Contents              []s3ListContent `xml:"Contents"`
+}
+
+type s3ListContent struct {
+	Key          string `xml:"Key"`
+	LastModified string `xml:"LastModified"`
+	ETag         string `xml:"ETag"`
+	Size         int64  `xml:"Size"`
+	StorageClass string `xml:"StorageClass"`
+}
+
+type multipartMeta struct {
+	Key         string `xml:"Key"`
+	ContentType string `xml:"ContentType"`
+}
+
+type createMultipartUploadResult struct {
+	XMLName  xml.Name `xml:"InitiateMultipartUploadResult"`
+	UploadID string   `xml:"UploadId"`
+}
+
+type completeMultipartUpload struct {
+	Parts []completedPart `xml:"Part"`
+}
+
+type completedPart struct {
+	PartNumber int    `xml:"PartNumber"`
+	ETag       string `xml:"ETag"`
+}
+
+type completeMultipartUploadResult struct {
+	XMLName xml.Name `xml:"CompleteMultipartUploadResult"`
+	Key     string   `xml:"Key"`
+	ETag    string   `xml:"ETag"`
+}
+
+type copyObjectResult struct {
+	XMLName      xml.Name `xml:"CopyObjectResult"`
+	LastModified string   `xml:"LastModified"`
+	ETag         string   `xml:"ETag"`
+}
+
+type deleteObjectsRequest struct {
+	Objects []deleteObjectRequest `xml:"Object"`
+}
+
+type deleteObjectRequest struct {
+	Key string `xml:"Key"`
+}
+
+type deleteObjectsResult struct {
+	XMLName xml.Name            `xml:"DeleteResult"`
+	Deleted []deletedObject     `xml:"Deleted"`
+	Errors  []deleteObjectError `xml:"Error,omitempty"`
+}
+
+type deletedObject struct {
+	Key string `xml:"Key"`
+}
+
+type deleteObjectError struct {
+	Key     string `xml:"Key"`
+	Code    string `xml:"Code"`
+	Message string `xml:"Message"`
+}

s3auth.go

@@ -0,0 +1,312 @@
+package main
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/hex"
+	"net/http"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+)
+
+const awsAlgorithm = "AWS4-HMAC-SHA256"
+
+func (s *Server) authorizedS3(r *http.Request) bool {
+	if len(s.cfg.S3Keys) == 0 {
+		return s.authorized(r)
+	}
+
+	if r.URL.Query().Get("X-Amz-Algorithm") != "" {
+		return s.authorizedS3Presigned(r)
+	}
+
+	return s.authorizedS3Header(r)
+}
+
+func (s *Server) authorizedS3Presigned(r *http.Request) bool {
+	if r.Method != http.MethodGet && r.Method != http.MethodHead {
+		return false
+	}
+	query := r.URL.Query()
+	if query.Get("X-Amz-Algorithm") != awsAlgorithm {
+		return false
+	}
+	if query.Get("X-Amz-Signature") == "" {
+		return false
+	}
+
+	credential := query.Get("X-Amz-Credential")
+	accessKey, scope, ok := splitCredential(credential)
+	key, ok := s.accessKey(accessKey)
+	if !ok || !key.can(permissionForMethod(r.Method)) {
+		return false
+	}
+
+	amzDate := query.Get("X-Amz-Date")
+	if !s.presignedTimeValid(amzDate, query.Get("X-Amz-Expires")) {
+		return false
+	}
+	if !s.scopeAllowed(scope, amzDate) {
+		return false
+	}
+
+	signedHeaders := query.Get("X-Amz-SignedHeaders")
+	canonicalRequest := canonicalRequest(r, signedHeaders, "UNSIGNED-PAYLOAD", true)
+	expected := signature(key.Secret, amzDate, scope, canonicalRequest)
+	return hmac.Equal([]byte(expected), []byte(query.Get("X-Amz-Signature")))
+}
+
+func (s *Server) authorizedS3Header(r *http.Request) bool {
+	return s.authorizedS3HeaderPermission(r, permissionForMethod(r.Method))
+}
+
+func (s *Server) authorizedS3HeaderPermission(r *http.Request, permission string) bool {
+	auth := r.Header.Get("Authorization")
+	if !strings.HasPrefix(auth, awsAlgorithm+" ") {
+		return false
+	}
+	parts := parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" "))
+	credential := parts["Credential"]
+	signedHeaders := parts["SignedHeaders"]
+	providedSignature := parts["Signature"]
+	accessKey, scope, ok := splitCredential(credential)
+	key, keyOK := s.accessKey(accessKey)
+	if !ok || !keyOK || !key.can(permission) || signedHeaders == "" || providedSignature == "" {
+		return false
+	}
+
+	amzDate := r.Header.Get("X-Amz-Date")
+	if !headerTimeValid(amzDate) {
+		return false
+	}
+	if !s.scopeAllowed(scope, amzDate) {
+		return false
+	}
+	payloadHash := r.Header.Get("X-Amz-Content-Sha256")
+	if payloadHash == "" {
+		payloadHash = "UNSIGNED-PAYLOAD"
+	}
+	if permission == "write" && r.Header.Get("X-Magpie-Replication") != "1" && payloadHash == "UNSIGNED-PAYLOAD" && !s.cfg.AllowUnsignedLocal {
+		return false
+	}
+
+	canonicalRequest := canonicalRequest(r, signedHeaders, payloadHash, false)
+	expected := signature(key.Secret, amzDate, scope, canonicalRequest)
+	return hmac.Equal([]byte(expected), []byte(providedSignature))
+}
+
+func (s *Server) accessKey(id string) (AccessKey, bool) {
+	for _, key := range s.cfg.S3Keys {
+		if subtle.ConstantTimeCompare([]byte(key.ID), []byte(id)) == 1 {
+			return key, true
+		}
+	}
+	return AccessKey{}, false
+}
+
+func (s *Server) requestAccessKey(r *http.Request) (AccessKey, bool) {
+	credential := r.URL.Query().Get("X-Amz-Credential")
+	if credential == "" {
+		auth := r.Header.Get("Authorization")
+		if !strings.HasPrefix(auth, awsAlgorithm+" ") {
+			return AccessKey{}, false
+		}
+		credential = parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" "))["Credential"]
+	}
+	accessKey, _, ok := splitCredential(credential)
+	if !ok {
+		return AccessKey{}, false
+	}
+	return s.accessKey(accessKey)
+}
+
+func (key AccessKey) can(permission string) bool {
+	return key.Permissions["admin"] || key.Permissions[permission]
+}
+
+func permissionForMethod(method string) string {
+	switch method {
+	case http.MethodGet, http.MethodHead:
+		return "read"
+	case http.MethodPut, http.MethodPost, http.MethodDelete:
+		return "write"
+	default:
+		return "admin"
+	}
+}
+
+func signature(secret, amzDate, scope, canonicalRequest string) string {
+	date := strings.Split(scope, "/")[0]
+	stringToSign := strings.Join([]string{
+		awsAlgorithm,
+		amzDate,
+		scope,
+		hexSHA256(canonicalRequest),
+	}, "\n")
+	key := signingKey(secret, date, scope)
+	sig := hmacSHA256(key, []byte(stringToSign))
+	return hex.EncodeToString(sig)
+}
+
+func signingKey(secret, date, scope string) []byte {
+	parts := strings.Split(scope, "/")
+	region := "auto"
+	service := "s3"
+	if len(parts) >= 3 {
+		region = parts[1]
+		service = parts[2]
+	}
+	kDate := hmacSHA256([]byte("AWS4"+secret), []byte(date))
+	kRegion := hmacSHA256(kDate, []byte(region))
+	kService := hmacSHA256(kRegion, []byte(service))
+	return hmacSHA256(kService, []byte("aws4_request"))
+}
+
+func (s *Server) scopeAllowed(scope string, amzDate string) bool {
+	parts := strings.Split(scope, "/")
+	if len(parts) != 4 {
+		return false
+	}
+	if len(amzDate) < 8 || parts[0] != amzDate[:8] {
+		return false
+	}
+	if parts[1] != s.cfg.S3Region || parts[2] != "s3" || parts[3] != "aws4_request" {
+		return false
+	}
+	return true
+}
+
+func canonicalRequest(r *http.Request, signedHeaders, payloadHash string, presigned bool) string {
+	return strings.Join([]string{
+		r.Method,
+		canonicalURI(r.URL.Path),
+		canonicalQuery(r.URL.Query(), presigned),
+		canonicalHeaders(r, signedHeaders),
+		signedHeaders,
+		payloadHash,
+	}, "\n")
+}
+
+func canonicalURI(path string) string {
+	if path == "" {
+		return "/"
+	}
+	segments := strings.Split(path, "/")
+	for i, segment := range segments {
+		segments[i] = awsEscape(segment)
+	}
+	return strings.Join(segments, "/")
+}
+
+func canonicalQuery(values url.Values, presigned bool) string {
+	type pair struct{ key, value string }
+	pairs := make([]pair, 0)
+	for key, vals := range values {
+		if presigned && key == "X-Amz-Signature" {
+			continue
+		}
+		for _, value := range vals {
+			pairs = append(pairs, pair{awsEscape(key), awsEscape(value)})
+		}
+	}
+	sort.Slice(pairs, func(i, j int) bool {
+		if pairs[i].key == pairs[j].key {
+			return pairs[i].value < pairs[j].value
+		}
+		return pairs[i].key < pairs[j].key
+	})
+	parts := make([]string, len(pairs))
+	for i, pair := range pairs {
+		parts[i] = pair.key + "=" + pair.value
+	}
+	return strings.Join(parts, "&")
+}
+
+func canonicalHeaders(r *http.Request, signedHeaders string) string {
+	headers := strings.Split(signedHeaders, ";")
+	var b strings.Builder
+	for _, header := range headers {
+		name := strings.ToLower(strings.TrimSpace(header))
+		value := ""
+		if name == "host" {
+			value = r.Host
+		} else {
+			value = strings.Join(r.Header.Values(name), ",")
+		}
+		b.WriteString(name)
+		b.WriteString(":")
+		b.WriteString(compressSpaces(value))
+		b.WriteString("\n")
+	}
+	return b.String()
+}
+
+func splitCredential(credential string) (string, string, bool) {
+	parts := strings.SplitN(credential, "/", 2)
+	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
+		return "", "", false
+	}
+	scopeParts := strings.Split(parts[1], "/")
+	if len(scopeParts) != 4 || scopeParts[3] != "aws4_request" {
+		return "", "", false
+	}
+	return parts[0], parts[1], true
+}
+
+func parseAuthParams(value string) map[string]string {
+	params := map[string]string{}
+	for _, part := range strings.Split(value, ",") {
+		keyValue := strings.SplitN(strings.TrimSpace(part), "=", 2)
+		if len(keyValue) == 2 {
+			params[keyValue[0]] = keyValue[1]
+		}
+	}
+	return params
+}
+
+func (s *Server) presignedTimeValid(amzDate, expiresRaw string) bool {
+	created, err := time.Parse("20060102T150405Z", amzDate)
+	if err != nil {
+		return false
+	}
+	expires, err := time.ParseDuration(expiresRaw + "s")
+	if err != nil || expires <= 0 || expires > s.cfg.PresignedMaxExpiry {
+		return false
+	}
+	now := time.Now().UTC()
+	return now.After(created.Add(-15*time.Minute)) && now.Before(created.Add(expires))
+}
+
+func headerTimeValid(amzDate string) bool {
+	created, err := time.Parse("20060102T150405Z", amzDate)
+	if err != nil {
+		return false
+	}
+	now := time.Now().UTC()
+	return now.After(created.Add(-15*time.Minute)) && now.Before(created.Add(15*time.Minute))
+}
+
+func awsEscape(value string) string {
+	escaped := url.QueryEscape(value)
+	escaped = strings.ReplaceAll(escaped, "+", "%20")
+	escaped = strings.ReplaceAll(escaped, "%7E", "~")
+	return escaped
+}
+
+func compressSpaces(value string) string {
+	return strings.Join(strings.Fields(strings.TrimSpace(value)), " ")
+}
+
+func hexSHA256(value string) string {
+	sum := sha256.Sum256([]byte(value))
+	return hex.EncodeToString(sum[:])
+}
+
+func hmacSHA256(key []byte, value []byte) []byte {
+	mac := hmac.New(sha256.New, key)
+	_, _ = mac.Write(value)
+	return mac.Sum(nil)
+}

server.go

@@ -0,0 +1,530 @@
+package main
+
+import (
+	"crypto/subtle"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"runtime/debug"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+const objectPrefix = "/objects/"
+
+type ObjectStore interface {
+	Put(key string, body io.Reader, contentType string) (ObjectInfo, error)
+	Open(key string) (*ObjectRead, error)
+	Stat(key string) (ObjectInfo, error)
+	Delete(key string) error
+	DiskStatus() (DiskStatus, error)
+	List(prefix string, maxKeys int) ([]ObjectInfo, error)
+	ListPage(prefix string, after string, maxKeys int) (ListPage, error)
+}
+
+type Server struct {
+	store         ObjectStore
+	cfg           Config
+	rateLimiter   *RateLimiter
+	metrics       *Metrics
+	writeMu       sync.Mutex
+	reservedBytes uint64
+}
+
+func NewServer(store ObjectStore, cfg Config) http.Handler {
+	return NewServerInstance(store, cfg).Handler()
+}
+
+func NewServerInstance(store ObjectStore, cfg Config) *Server {
+	cfg = cfg.withDefaults()
+	return &Server{store: store, cfg: cfg, rateLimiter: NewRateLimiter(cfg.RateLimitPerMinute), metrics: &Metrics{}}
+}
+
+func (s *Server) Handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/health", s.health)
+	mux.HandleFunc("/metrics", s.metricsHandler)
+	mux.HandleFunc(objectPrefix, s.object)
+	mux.HandleFunc("/", s.s3Object)
+	return s.recoverPanics(s.logRequests(s.rateLimit(mux)))
+}
+
+func (s *Server) recoverPanics(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if recovered := recover(); recovered != nil {
+				log.Printf("panic method=%s path=%s remote=%s error=%v stack=%s", r.Method, safeLogPath(r.URL.Path), sanitizeLogValue(r.RemoteAddr), recovered, debug.Stack())
+				s3Error(w, http.StatusInternalServerError, "InternalError", "internal server error")
+			}
+		}()
+		next.ServeHTTP(w, r)
+	})
+}
+
+func (s *Server) metricsHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.opsAuthorized(r) {
+		writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain; version=0.0.4")
+	fmt.Fprintf(w, "magpie_requests_total %d\n", atomic.LoadUint64(&s.metrics.Requests))
+	fmt.Fprintf(w, "magpie_errors_total %d\n", atomic.LoadUint64(&s.metrics.Errors))
+	fmt.Fprintf(w, "magpie_response_bytes_total %d\n", atomic.LoadUint64(&s.metrics.ResponseBytes))
+}
+
+func (s *Server) opsAuthorized(r *http.Request) bool {
+	return s.authorized(r) || s.authorizedS3HeaderPermission(r, "admin")
+}
+
+func (cfg Config) withDefaults() Config {
+	if len(cfg.S3Keys) == 0 && cfg.S3AccessKeyID != "" && cfg.S3SecretAccessKey != "" {
+		cfg.S3Keys = []AccessKey{{ID: cfg.S3AccessKeyID, Secret: cfg.S3SecretAccessKey, Permissions: permissions("read,write,admin")}}
+	}
+	if cfg.S3Region == "" {
+		cfg.S3Region = "auto"
+	}
+	if cfg.MaxObjectSize <= 0 {
+		cfg.MaxObjectSize = defaultMaxObjectSize
+	}
+	if cfg.PresignedMaxExpiry <= 0 {
+		cfg.PresignedMaxExpiry = defaultPresignedMaxExpiry
+	}
+	if cfg.MultipartMaxAge <= 0 {
+		cfg.MultipartMaxAge = defaultMultipartMaxAge
+	}
+	if cfg.PublicPrefixes == nil {
+		cfg.PublicPrefixes = map[string]bool{}
+	}
+	if cfg.ReplicationMaxJobs <= 0 {
+		cfg.ReplicationMaxJobs = defaultReplicationMaxJobs
+	}
+	if cfg.ReplicationMaxAttempts <= 0 {
+		cfg.ReplicationMaxAttempts = defaultReplicationMaxAttempts
+	}
+	if cfg.MaxRestoreBytes <= 0 {
+		cfg.MaxRestoreBytes = defaultMaxRestoreBytes
+	}
+	return cfg
+}
+
+func (s *Server) health(w http.ResponseWriter, r *http.Request) {
+	if !s.opsAuthorized(r) {
+		writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
+		return
+	}
+	disk, err := s.store.DiskStatus()
+	if err != nil {
+		writeJSON(w, http.StatusServiceUnavailable, map[string]string{"status": "error", "error": err.Error()})
+		return
+	}
+	status := "ok"
+	code := http.StatusOK
+	if disk.FreeBytes < s.cfg.MinFreeBytes {
+		status = "low_disk"
+		code = http.StatusServiceUnavailable
+	}
+	writeJSON(w, code, map[string]any{"status": status, "disk": disk})
+}
+
+func (s *Server) object(w http.ResponseWriter, r *http.Request) {
+	if !s.authorized(r) {
+		writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
+		return
+	}
+
+	key := strings.TrimPrefix(r.URL.Path, objectPrefix)
+	if err := ValidateKey(key); err != nil {
+		writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
+		return
+	}
+	bucket, _, _ := strings.Cut(key, "/")
+	if !s.bucketAllowed(bucket) {
+		writeJSON(w, http.StatusForbidden, map[string]string{"error": "bucket is not allowed"})
+		return
+	}
+
+	switch r.Method {
+	case http.MethodPut:
+		s.put(w, r, key)
+	case http.MethodGet:
+		s.get(w, r, key)
+	case http.MethodHead:
+		s.head(w, r, key)
+	case http.MethodDelete:
+		s.delete(w, key)
+	default:
+		w.Header().Set("Allow", "PUT, GET, HEAD, DELETE")
+		writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
+	}
+}
+
+func (s *Server) put(w http.ResponseWriter, r *http.Request, key string) {
+	release, ok := s.acceptWrite(w, r, false)
+	if !ok {
+		return
+	}
+	defer release()
+	contentType := r.Header.Get("Content-Type")
+	info, err := s.store.Put(key, http.MaxBytesReader(w, r.Body, s.cfg.MaxObjectSize), contentType)
+	if err != nil {
+		if maxBytesError(err) {
+			writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "object too large"})
+			return
+		}
+		writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "store failed"})
+		return
+	}
+	writeJSON(w, http.StatusCreated, info)
+}
+
+func (s *Server) acceptWrite(w http.ResponseWriter, r *http.Request, s3 bool) (func(), bool) {
+	size := r.ContentLength
+	if r.ContentLength < 0 {
+		if s.cfg.MaxObjectSize <= 0 {
+			disk, err := s.store.DiskStatus()
+			if err != nil || disk.FreeBytes < s.cfg.MinFreeBytes {
+				if s3 {
+					s3Error(w, http.StatusInsufficientStorage, "InsufficientStorage", "insufficient free disk space for unknown length write")
+				} else {
+					writeJSON(w, http.StatusInsufficientStorage, map[string]string{"error": "insufficient free disk space for unknown length write"})
+				}
+				return nil, false
+			}
+			return func() {}, true
+		}
+		size = s.cfg.MaxObjectSize
+	}
+	if s.cfg.MaxObjectSize > 0 && r.ContentLength > s.cfg.MaxObjectSize {
+		if s3 {
+			s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large")
+		} else {
+			writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "object too large"})
+		}
+		return nil, false
+	}
+	return s.acceptWriteSize(w, size, s3)
+}
+
+func (s *Server) acceptWriteSize(w http.ResponseWriter, size int64, s3 bool) (func(), bool) {
+	if size < 0 {
+		return func() {}, true
+	}
+	disk, err := s.store.DiskStatus()
+	if err != nil {
+		if s3 {
+			s3Error(w, http.StatusInternalServerError, "InternalError", "disk status unavailable")
+		} else {
+			writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "disk status unavailable"})
+		}
+		return nil, false
+	}
+	needed := uint64(size)
+	s.writeMu.Lock()
+	defer s.writeMu.Unlock()
+	reserved := s.reservedBytes
+	available := uint64(0)
+	if disk.FreeBytes > reserved {
+		available = disk.FreeBytes - reserved
+	}
+	if available < s.cfg.MinFreeBytes || needed > available || available-needed < s.cfg.MinFreeBytes {
+		if s3 {
+			s3Error(w, http.StatusInsufficientStorage, "InsufficientStorage", "insufficient free disk space")
+		} else {
+			writeJSON(w, http.StatusInsufficientStorage, map[string]string{"error": "insufficient free disk space"})
+		}
+		return nil, false
+	}
+	s.reservedBytes += needed
+	released := false
+	return func() {
+		s.writeMu.Lock()
+		defer s.writeMu.Unlock()
+		if released {
+			return
+		}
+		released = true
+		if s.reservedBytes >= needed {
+			s.reservedBytes -= needed
+		} else {
+			s.reservedBytes = 0
+		}
+	}, true
+}
+
+func maxBytesError(err error) bool {
+	var maxErr *http.MaxBytesError
+	return errors.As(err, &maxErr)
+}
+
+func (s *Server) get(w http.ResponseWriter, r *http.Request, key string) {
+	obj, err := s.store.Open(key)
+	if err != nil {
+		status := http.StatusInternalServerError
+		if errors.Is(err, ErrNotFound) {
+			status = http.StatusNotFound
+		}
+		writeJSON(w, status, map[string]string{"error": "not found"})
+		return
+	}
+	defer obj.Close()
+
+	contentType := obj.Info.ContentType
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	w.Header().Set("Content-Type", contentType)
+	w.Header().Set("X-Object-Key", obj.Info.Key)
+	w.Header().Set("X-Object-Size", fmt.Sprintf("%d", obj.Info.Size))
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	http.ServeContent(w, r, obj.Info.Key, obj.Info.ModTime, obj)
+}
+
+func (s *Server) head(w http.ResponseWriter, r *http.Request, key string) {
+	info, err := s.store.Stat(key)
+	if err != nil {
+		if errors.Is(err, ErrNotFound) {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	contentType := info.ContentType
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	}
+	w.Header().Set("Content-Type", contentType)
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
+	w.Header().Set("X-Object-Key", info.Key)
+	w.Header().Set("X-Object-Size", fmt.Sprintf("%d", info.Size))
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.WriteHeader(http.StatusOK)
+	_ = r.Body.Close()
+}
+
+func (s *Server) delete(w http.ResponseWriter, key string) {
+	err := s.store.Delete(key)
+	if err != nil && !errors.Is(err, ErrNotFound) {
+		writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "delete failed"})
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func (s *Server) authorized(r *http.Request) bool {
+	const prefix = "Bearer "
+	value := r.Header.Get("Authorization")
+	if !strings.HasPrefix(value, prefix) {
+		return false
+	}
+	provided := strings.TrimPrefix(value, prefix)
+	return s.cfg.AuthToken != "" && subtle.ConstantTimeCompare([]byte(provided), []byte(s.cfg.AuthToken)) == 1
+}
+
+func (s *Server) bucketAllowed(bucket string) bool {
+	return len(s.cfg.AllowedBuckets) == 0 || s.cfg.AllowedBuckets[bucket]
+}
+
+func (s *Server) publicReadAllowed(key string) bool {
+	for prefix := range s.cfg.PublicPrefixes {
+		prefix = strings.TrimSuffix(prefix, "/")
+		if !strings.Contains(prefix, "/") {
+			continue
+		}
+		if key == prefix || strings.HasPrefix(key, prefix+"/") {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *Server) replicationAllowed(r *http.Request) bool {
+	if r.Header.Get("X-Magpie-Replication") != "1" {
+		return true
+	}
+	if s.cfg.ReplicationSecret == "" {
+		return false
+	}
+	return subtle.ConstantTimeCompare([]byte(r.Header.Get("X-Magpie-Replication-Secret")), []byte(s.cfg.ReplicationSecret)) == 1
+}
+
+func writeJSON(w http.ResponseWriter, status int, body any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(body)
+}
+
+type statusWriter struct {
+	http.ResponseWriter
+	status int
+	bytes  int
+}
+
+func (w *statusWriter) WriteHeader(status int) {
+	w.status = status
+	w.ResponseWriter.WriteHeader(status)
+}
+
+func (w *statusWriter) Write(b []byte) (int, error) {
+	if w.status == 0 {
+		w.status = http.StatusOK
+	}
+	n, err := w.ResponseWriter.Write(b)
+	w.bytes += n
+	return n, err
+}
+
+func (s *Server) logRequests(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		recorder := &statusWriter{ResponseWriter: w}
+		next.ServeHTTP(recorder, r)
+		status := recorder.status
+		if status == 0 {
+			status = http.StatusOK
+		}
+		atomic.AddUint64(&s.metrics.Requests, 1)
+		if recorder.bytes > 0 {
+			atomic.AddUint64(&s.metrics.ResponseBytes, uint64(recorder.bytes))
+		}
+		if status >= 400 {
+			atomic.AddUint64(&s.metrics.Errors, 1)
+		}
+		log.Printf(
+			"method=%s path=%s status=%d bytes=%d duration_ms=%d remote=%s",
+			r.Method,
+			safeLogPath(r.URL.Path),
+			status,
+			recorder.bytes,
+			time.Since(start).Milliseconds(),
+			sanitizeLogValue(r.RemoteAddr),
+		)
+	})
+}
+
+func (s *Server) rateLimit(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !s.rateLimiter.Allow(s.rateLimitKey(r)) {
+			writeJSON(w, http.StatusTooManyRequests, map[string]string{"error": "rate limited"})
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+func (s *Server) rateLimitKey(r *http.Request) string {
+	if s.cfg.TrustProxyHeaders {
+		if forwarded := r.Header.Get("X-Forwarded-For"); forwarded != "" {
+			candidate := strings.TrimSpace(strings.Split(forwarded, ",")[0])
+			if net.ParseIP(candidate) != nil {
+				return candidate
+			}
+		}
+		if realIP := strings.TrimSpace(r.Header.Get("X-Real-IP")); net.ParseIP(realIP) != nil {
+			return realIP
+		}
+	}
+	return r.RemoteAddr
+}
+
+type Metrics struct {
+	Requests      uint64
+	Errors        uint64
+	ResponseBytes uint64
+}
+
+type RateLimiter struct {
+	limit   int
+	mu      sync.Mutex
+	buckets map[string]rateBucket
+}
+
+type rateBucket struct {
+	minute int64
+	count  int
+}
+
+func NewRateLimiter(limit int) *RateLimiter {
+	return &RateLimiter{limit: limit, buckets: map[string]rateBucket{}}
+}
+
+func (r *RateLimiter) Allow(remote string) bool {
+	if r.limit <= 0 {
+		return true
+	}
+	ip, _, err := net.SplitHostPort(remote)
+	if err != nil {
+		ip = remote
+	}
+	minute := time.Now().Unix() / 60
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for candidate, bucket := range r.buckets {
+		if bucket.minute < minute-1 {
+			delete(r.buckets, candidate)
+		}
+	}
+	bucket := r.buckets[ip]
+	if bucket.minute != minute {
+		bucket = rateBucket{minute: minute}
+	}
+	bucket.count++
+	r.buckets[ip] = bucket
+	return bucket.count <= r.limit
+}
+
+func safeLogPath(path string) string {
+	if len(path) <= 256 {
+		return sanitizeLogValue(path)
+	}
+	return sanitizeLogValue(path[:256]) + "..."
+}
+
+func sanitizeLogValue(value string) string {
+	return strings.Map(func(r rune) rune {
+		if r < 0x20 || r == 0x7f {
+			return ' '
+		}
+		return r
+	}, value)
+}
+
+type ObjectInfo struct {
+	Key         string    `json:"key"`
+	Size        int64     `json:"size"`
+	ContentType string    `json:"content_type"`
+	ModTime     time.Time `json:"mod_time"`
+	ETag        string    `json:"etag"`
+}
+
+type DiskStatus struct {
+	TotalBytes uint64 `json:"total_bytes"`
+	FreeBytes  uint64 `json:"free_bytes"`
+	UsedBytes  uint64 `json:"used_bytes"`
+}
+
+type StoreStats struct {
+	ObjectCount      int64                 `json:"object_count"`
+	TotalBytes       int64                 `json:"total_bytes"`
+	Disk             DiskStatus            `json:"disk"`
+	ReplicationQueue ReplicationQueueStats `json:"replication_queue"`
+}
+
+type ScrubReport struct {
+	CheckedObjects int      `json:"checked_objects"`
+	MissingKeys    []string `json:"missing_keys"`
+	CorruptKeys    []string `json:"corrupt_keys"`
+	OrphanKeys     []string `json:"orphan_keys"`
+}
+
+type ListPage struct {
+	Objects   []ObjectInfo
+	NextAfter string
+}

store.go

@@ -0,0 +1,563 @@
+package main
+
+import (
+	"crypto/md5"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+var ErrNotFound = errors.New("object not found")
+
+type FileStore struct {
+	root string
+	db   *sql.DB
+}
+
+type ObjectRead struct {
+	*os.File
+	Info ObjectInfo
+}
+
+func NewFileStore(root string) (*FileStore, error) {
+	abs, err := filepath.Abs(root)
+	if err != nil {
+		return nil, err
+	}
+	if err := os.MkdirAll(abs, 0o700); err != nil {
+		return nil, err
+	}
+	db, err := sql.Open("sqlite", filepath.Join(abs, ".magpie.db"))
+	if err != nil {
+		return nil, err
+	}
+	store := &FileStore{root: abs, db: db}
+	if err := store.initMetadata(); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	return store, nil
+}
+
+func (s *FileStore) Put(key string, body io.Reader, contentType string) (ObjectInfo, error) {
+	return s.PutVerified(key, body, contentType, nil)
+}
+
+func (s *FileStore) PutVerified(key string, body io.Reader, contentType string, verify func() error) (ObjectInfo, error) {
+	if err := ValidateKey(key); err != nil {
+		return ObjectInfo{}, err
+	}
+
+	path, err := s.pathFor(key)
+	if err != nil {
+		return ObjectInfo{}, err
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return ObjectInfo{}, err
+	}
+
+	tmp, err := os.CreateTemp(filepath.Dir(path), ".upload-*")
+	if err != nil {
+		return ObjectInfo{}, err
+	}
+	tmpName := tmp.Name()
+	defer os.Remove(tmpName)
+
+	hash := md5.New()
+	size, copyErr := io.Copy(io.MultiWriter(tmp, hash), body)
+	closeErr := tmp.Close()
+	if copyErr != nil {
+		return ObjectInfo{}, copyErr
+	}
+	if closeErr != nil {
+		return ObjectInfo{}, closeErr
+	}
+	if verify != nil {
+		if err := verify(); err != nil {
+			return ObjectInfo{}, err
+		}
+	}
+
+	info := ObjectInfo{
+		Key:         key,
+		Size:        size,
+		ContentType: contentType,
+		ModTime:     time.Now().UTC(),
+		ETag:        fmt.Sprintf("%x", hash.Sum(nil)),
+	}
+	if err := s.writeMeta(key, info); err != nil {
+		return ObjectInfo{}, err
+	}
+	if err := s.upsertMetadata(info); err != nil {
+		_ = os.Remove(path + ".meta.json")
+		return ObjectInfo{}, err
+	}
+	if err := os.Rename(tmpName, path); err != nil {
+		_ = os.Remove(path + ".meta.json")
+		return ObjectInfo{}, err
+	}
+	return info, nil
+}
+
+func (s *FileStore) Open(key string) (*ObjectRead, error) {
+	if err := ValidateKey(key); err != nil {
+		return nil, err
+	}
+	path, err := s.pathFor(key)
+	if err != nil {
+		return nil, err
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil, ErrNotFound
+		}
+		return nil, err
+	}
+	info, err := s.Stat(key)
+	if err != nil {
+		_ = file.Close()
+		return nil, err
+	}
+	return &ObjectRead{File: file, Info: info}, nil
+}
+
+func (s *FileStore) Stat(key string) (ObjectInfo, error) {
+	if err := ValidateKey(key); err != nil {
+		return ObjectInfo{}, err
+	}
+	path, err := s.pathFor(key)
+	if err != nil {
+		return ObjectInfo{}, err
+	}
+	stat, err := os.Stat(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return ObjectInfo{}, ErrNotFound
+		}
+		return ObjectInfo{}, err
+	}
+
+	if info, err := s.getMetadata(key); err == nil {
+		info.Size = stat.Size()
+		return info, nil
+	}
+
+	info := ObjectInfo{Key: key, Size: stat.Size(), ModTime: stat.ModTime().UTC()}
+	if meta, err := s.readMeta(key); err == nil {
+		info.ContentType = meta.ContentType
+		info.ETag = meta.ETag
+		if !meta.ModTime.IsZero() {
+			info.ModTime = meta.ModTime
+		}
+	}
+	_ = s.upsertMetadata(info)
+	return info, nil
+}
+
+func (s *FileStore) Delete(key string) error {
+	if err := ValidateKey(key); err != nil {
+		return err
+	}
+	path, err := s.pathFor(key)
+	if err != nil {
+		return err
+	}
+	if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	_ = os.Remove(path + ".meta.json")
+	_, _ = s.db.Exec(`DELETE FROM objects WHERE key = ?`, key)
+	return nil
+}
+
+func (s *FileStore) List(prefix string, maxKeys int) ([]ObjectInfo, error) {
+	page, err := s.ListPage(prefix, "", maxKeys)
+	if err != nil {
+		return nil, err
+	}
+	return page.Objects, nil
+}
+
+func (s *FileStore) ListPage(prefix string, after string, maxKeys int) (ListPage, error) {
+	validatePrefix := strings.TrimSuffix(prefix, "/")
+	if validatePrefix != "" {
+		if err := ValidateKey(validatePrefix); err != nil {
+			return ListPage{}, err
+		}
+	}
+	if after != "" {
+		if err := ValidateKey(after); err != nil {
+			return ListPage{}, err
+		}
+	}
+	if maxKeys <= 0 || maxKeys > 1000 {
+		maxKeys = 1000
+	}
+
+	return s.listMetadata(prefix, after, maxKeys)
+}
+
+func (s *FileStore) Reindex() error {
+	return filepath.WalkDir(s.root, func(path string, entry os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if entry.IsDir() {
+			return nil
+		}
+		if entry.Name() == ".magpie.db" || strings.HasPrefix(entry.Name(), ".magpie.db-") || strings.HasSuffix(entry.Name(), ".meta.json") {
+			return nil
+		}
+
+		rel, err := filepath.Rel(s.root, path)
+		if err != nil {
+			return err
+		}
+		key := filepath.ToSlash(rel)
+		if err := ValidateKey(key); err != nil {
+			return nil
+		}
+		info, err := s.statFromFilesystem(key)
+		if err != nil {
+			return err
+		}
+		return s.upsertMetadata(info)
+	})
+}
+
+func (s *FileStore) Stats() (StoreStats, error) {
+	var stats StoreStats
+	if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(size), 0) FROM objects`).Scan(&stats.ObjectCount, &stats.TotalBytes); err != nil {
+		return StoreStats{}, err
+	}
+	queue, err := s.ReplicationQueueStats()
+	if err != nil {
+		return StoreStats{}, err
+	}
+	stats.ReplicationQueue = queue
+	disk, err := s.DiskStatus()
+	if err != nil {
+		return StoreStats{}, err
+	}
+	stats.Disk = disk
+	return stats, nil
+}
+
+func (s *FileStore) Scrub() (ScrubReport, error) {
+	report := ScrubReport{}
+	rows, err := s.db.Query(`SELECT key, size, etag FROM objects ORDER BY key ASC`)
+	if err != nil {
+		return report, err
+	}
+	defer rows.Close()
+
+	metadataKeys := map[string]bool{}
+	for rows.Next() {
+		var key string
+		var size int64
+		var etag string
+		if err := rows.Scan(&key, &size, &etag); err != nil {
+			return report, err
+		}
+		metadataKeys[key] = true
+		report.CheckedObjects++
+		path, err := s.pathFor(key)
+		if err != nil {
+			report.CorruptKeys = append(report.CorruptKeys, key)
+			continue
+		}
+		stat, err := os.Stat(path)
+		if err != nil {
+			if errors.Is(err, os.ErrNotExist) {
+				report.MissingKeys = append(report.MissingKeys, key)
+				continue
+			}
+			return report, err
+		}
+		if stat.Size() != size {
+			report.CorruptKeys = append(report.CorruptKeys, key)
+			continue
+		}
+		computed, err := fileETag(path)
+		if err != nil {
+			return report, err
+		}
+		if etag != "" && computed != etag {
+			report.CorruptKeys = append(report.CorruptKeys, key)
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return report, err
+	}
+
+	if err := filepath.WalkDir(s.root, func(path string, entry os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if entry.IsDir() || entry.Name() == ".magpie.db" || strings.HasPrefix(entry.Name(), ".magpie.db-") || strings.HasSuffix(entry.Name(), ".meta.json") || strings.Contains(path, string(os.PathSeparator)+".multipart"+string(os.PathSeparator)) {
+			return nil
+		}
+		rel, err := filepath.Rel(s.root, path)
+		if err != nil {
+			return err
+		}
+		key := filepath.ToSlash(rel)
+		if !metadataKeys[key] {
+			report.OrphanKeys = append(report.OrphanKeys, key)
+		}
+		return nil
+	}); err != nil {
+		return report, err
+	}
+	sort.Strings(report.MissingKeys)
+	sort.Strings(report.CorruptKeys)
+	sort.Strings(report.OrphanKeys)
+	return report, nil
+}
+
+func (s *FileStore) CleanupMultipart(maxAge time.Duration) error {
+	if maxAge <= 0 {
+		return nil
+	}
+	root := filepath.Join(s.root, ".multipart")
+	entries, err := os.ReadDir(root)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	cutoff := time.Now().Add(-maxAge)
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+		info, err := entry.Info()
+		if err != nil {
+			return err
+		}
+		if info.ModTime().Before(cutoff) {
+			if err := os.RemoveAll(filepath.Join(root, entry.Name())); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (s *FileStore) pathFor(key string) (string, error) {
+	clean := filepath.Clean(filepath.FromSlash(key))
+	path := filepath.Join(s.root, clean)
+	if path != s.root && !strings.HasPrefix(path, s.root+string(os.PathSeparator)) {
+		return "", fmt.Errorf("invalid object key")
+	}
+	return path, nil
+}
+
+func (s *FileStore) writeMeta(key string, info ObjectInfo) error {
+	path, err := s.pathFor(key)
+	if err != nil {
+		return err
+	}
+	data, err := json.Marshal(info)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path+".meta.json", data, 0o600)
+}
+
+func (s *FileStore) readMeta(key string) (ObjectInfo, error) {
+	path, err := s.pathFor(key)
+	if err != nil {
+		return ObjectInfo{}, err
+	}
+	data, err := os.ReadFile(path + ".meta.json")
+	if err != nil {
+		return ObjectInfo{}, err
+	}
+	var info ObjectInfo
+	if err := json.Unmarshal(data, &info); err != nil {
+		return ObjectInfo{}, err
+	}
+	return info, nil
+}
+
+func (s *FileStore) statFromFilesystem(key string) (ObjectInfo, error) {
+	path, err := s.pathFor(key)
+	if err != nil {
+		return ObjectInfo{}, err
+	}
+	stat, err := os.Stat(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return ObjectInfo{}, ErrNotFound
+		}
+		return ObjectInfo{}, err
+	}
+	info := ObjectInfo{Key: key, Size: stat.Size(), ModTime: stat.ModTime().UTC()}
+	if meta, err := s.readMeta(key); err == nil {
+		info.ContentType = meta.ContentType
+		info.ETag = meta.ETag
+		if !meta.ModTime.IsZero() {
+			info.ModTime = meta.ModTime
+		}
+	}
+	if info.ETag == "" {
+		etag, err := fileETag(path)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		info.ETag = etag
+	}
+	return info, nil
+}
+
+func fileETag(path string) (string, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+	hash := md5.New()
+	if _, err := io.Copy(hash, file); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%x", hash.Sum(nil)), nil
+}
+
+func (s *FileStore) initMetadata() error {
+	_, err := s.db.Exec(`
+PRAGMA journal_mode = WAL;
+PRAGMA synchronous = NORMAL;
+CREATE TABLE IF NOT EXISTS objects (
+  key TEXT PRIMARY KEY,
+  size INTEGER NOT NULL,
+  content_type TEXT NOT NULL DEFAULT '',
+  etag TEXT NOT NULL DEFAULT '',
+  mod_time TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS objects_key_idx ON objects(key);
+CREATE TABLE IF NOT EXISTS replication_jobs (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  peer TEXT NOT NULL,
+  op TEXT NOT NULL,
+  key TEXT NOT NULL,
+  attempts INTEGER NOT NULL DEFAULT 0,
+  next_attempt_at TEXT NOT NULL,
+  last_error TEXT NOT NULL DEFAULT '',
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS replication_jobs_due_idx ON replication_jobs(next_attempt_at, id);
+`)
+	return err
+}
+
+func (s *FileStore) upsertMetadata(info ObjectInfo) error {
+	_, err := s.db.Exec(`
+INSERT INTO objects (key, size, content_type, etag, mod_time)
+VALUES (?, ?, ?, ?, ?)
+ON CONFLICT(key) DO UPDATE SET
+  size = excluded.size,
+  content_type = excluded.content_type,
+  etag = excluded.etag,
+  mod_time = excluded.mod_time
+`, info.Key, info.Size, info.ContentType, info.ETag, info.ModTime.UTC().Format(time.RFC3339Nano))
+	return err
+}
+
+func (s *FileStore) getMetadata(key string) (ObjectInfo, error) {
+	var info ObjectInfo
+	var modTime string
+	err := s.db.QueryRow(`SELECT key, size, content_type, etag, mod_time FROM objects WHERE key = ?`, key).
+		Scan(&info.Key, &info.Size, &info.ContentType, &info.ETag, &modTime)
+	if err != nil {
+		return ObjectInfo{}, err
+	}
+	parsed, err := time.Parse(time.RFC3339Nano, modTime)
+	if err != nil {
+		return ObjectInfo{}, err
+	}
+	info.ModTime = parsed
+	return info, nil
+}
+
+func (s *FileStore) listMetadata(prefix string, after string, maxKeys int) (ListPage, error) {
+	rows, err := s.db.Query(`
+SELECT key, size, content_type, etag, mod_time
+FROM objects
+WHERE key LIKE ? ESCAPE '\' AND key > ?
+ORDER BY key ASC
+LIMIT ?
+`, escapeLike(prefix)+"%", after, maxKeys+1)
+	if err != nil {
+		return ListPage{}, err
+	}
+	defer rows.Close()
+
+	objects := make([]ObjectInfo, 0)
+	for rows.Next() {
+		var info ObjectInfo
+		var modTime string
+		if err := rows.Scan(&info.Key, &info.Size, &info.ContentType, &info.ETag, &modTime); err != nil {
+			return ListPage{}, err
+		}
+		parsed, err := time.Parse(time.RFC3339Nano, modTime)
+		if err != nil {
+			return ListPage{}, err
+		}
+		info.ModTime = parsed
+		objects = append(objects, info)
+	}
+	if err := rows.Err(); err != nil {
+		return ListPage{}, err
+	}
+
+	page := ListPage{Objects: objects}
+	if len(objects) > maxKeys {
+		page.NextAfter = objects[maxKeys-1].Key
+		page.Objects = objects[:maxKeys]
+	}
+	return page, nil
+}
+
+func escapeLike(value string) string {
+	value = strings.ReplaceAll(value, `\`, `\\`)
+	value = strings.ReplaceAll(value, `%`, `\%`)
+	value = strings.ReplaceAll(value, `_`, `\_`)
+	return value
+}
+
+func ValidateKey(key string) error {
+	if key == "" {
+		return fmt.Errorf("object key is required")
+	}
+	if len(key) > 1024 {
+		return fmt.Errorf("object key is too long")
+	}
+	if strings.HasPrefix(key, "/") || strings.HasPrefix(key, "\\") {
+		return fmt.Errorf("object key cannot be absolute")
+	}
+	if strings.Contains(key, "\x00") || strings.Contains(key, "\\") || strings.Contains(key, "//") {
+		return fmt.Errorf("object key contains invalid characters")
+	}
+	for _, part := range strings.Split(key, "/") {
+		if part == "" || part == "." || part == ".." {
+			return fmt.Errorf("object key contains invalid path segment")
+		}
+		if part == ".multipart" || part == ".magpie.db" || strings.HasPrefix(part, ".magpie.db-") || strings.HasSuffix(part, ".meta.json") {
+			return fmt.Errorf("object key uses reserved internal name")
+		}
+	}
+	return nil
+}

version.go

@@ -0,0 +1,29 @@
+package main
+
+import "runtime"
+
+var (
+	version = "dev"
+	commit  = "unknown"
+	date    = "unknown"
+)
+
+type VersionInfo struct {
+	Version string `json:"version"`
+	Commit  string `json:"commit"`
+	Date    string `json:"date"`
+	Go      string `json:"go"`
+	OS      string `json:"os"`
+	Arch    string `json:"arch"`
+}
+
+func versionInfo() VersionInfo {
+	return VersionInfo{
+		Version: version,
+		Commit:  commit,
+		Date:    date,
+		Go:      runtime.Version(),
+		OS:      runtime.GOOS,
+		Arch:    runtime.GOARCH,
+	}
+}