Initial commit on Forgejo
Fresh repository history for elektrine/magpie hosted at https://git.elektrine.com/elektrine/magpie.
This commit is contained in:
commit
ddbd19f153
23 changed files with 5636 additions and 0 deletions
19
.dockerignore
Normal file
19
.dockerignore
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
.git/
|
||||
.env
|
||||
.env.*
|
||||
!.env.production.example
|
||||
data/
|
||||
.multipart/
|
||||
dist/
|
||||
tmp/
|
||||
*.db
|
||||
*.db-*
|
||||
*.sqlite
|
||||
*.sqlite-*
|
||||
*.meta.json
|
||||
*.tar
|
||||
*.tar.gz
|
||||
*.tgz
|
||||
*.log
|
||||
coverage.out
|
||||
magpie
|
||||
62
.env.production.example
Normal file
62
.env.production.example
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
MAGPIE_ADDR=127.0.0.1:8090
|
||||
MAGPIE_DATA_DIR=/var/lib/magpie
|
||||
|
||||
# Optional bearer-token API for /objects/* routes.
|
||||
MAGPIE_AUTH_TOKEN=change-this-long-random-token
|
||||
|
||||
# S3-compatible API credentials for /:bucket/:key routes.
|
||||
MAGPIE_S3_ACCESS_KEY_ID=magpie
|
||||
MAGPIE_S3_SECRET_ACCESS_KEY=change-this-long-random-s3-secret
|
||||
# Optional multi-key form: id:secret:read,write,admin;id2:secret2:read
|
||||
MAGPIE_S3_KEYS=
|
||||
MAGPIE_S3_REGION=auto
|
||||
|
||||
# Temporarily set true only for trusted private migrations with clients that
|
||||
# upload using SigV4 UNSIGNED-PAYLOAD, such as some rclone versions.
|
||||
MAGPIE_ALLOW_UNSIGNED_PAYLOADS=false
|
||||
|
||||
# Comma-separated peer base URLs for async write/delete replication.
|
||||
# MAGPIE_REPLICATION_SECRET is required when peers are set.
|
||||
MAGPIE_REPLICATION_PEERS=
|
||||
MAGPIE_REPLICATION_SECRET=
|
||||
# Replication peers must use https unless this is explicitly enabled for a
|
||||
# trusted local/private network.
|
||||
MAGPIE_ALLOW_INSECURE_REPLICATION=false
|
||||
MAGPIE_REPLICATION_MAX_JOBS=10000
|
||||
MAGPIE_REPLICATION_MAX_ATTEMPTS=100
|
||||
|
||||
# Comma-separated bucket allowlist. Empty allows any bucket name.
|
||||
MAGPIE_ALLOWED_BUCKETS=app-uploads
|
||||
|
||||
# Comma-separated bucket/prefix values that allow unsigned public GET/HEAD requests.
|
||||
# Empty/private by default. Set to `app-uploads/avatars` only for public media.
|
||||
# MAGPIE_PUBLIC_PREFIXES=app-uploads/avatars
|
||||
|
||||
# Per-IP request limit per minute. Set 0 to disable.
|
||||
MAGPIE_RATE_LIMIT_PER_MINUTE=600
|
||||
# Trust X-Forwarded-For/X-Real-IP for rate limiting only behind a trusted proxy.
|
||||
MAGPIE_TRUST_PROXY_HEADERS=false
|
||||
|
||||
# Upload safety limits.
|
||||
# Default: 1073741824 (1 GiB)
|
||||
MAGPIE_MAX_OBJECT_SIZE=1073741824
|
||||
|
||||
# Reject writes if disk free space would fall below this value.
|
||||
# Default: 1073741824 (1 GiB)
|
||||
MAGPIE_MIN_FREE_BYTES=1073741824
|
||||
|
||||
# Maximum accepted SigV4 presigned URL lifetime.
|
||||
# Default: 24h
|
||||
MAGPIE_PRESIGNED_MAX_EXPIRY=24h
|
||||
|
||||
# Remove incomplete multipart uploads older than this age.
|
||||
# Default: 24h
|
||||
MAGPIE_MULTIPART_MAX_AGE=24h
|
||||
|
||||
# Maximum total expanded bytes accepted by restore.
|
||||
# Default: 10737418240 (10 GiB)
|
||||
MAGPIE_MAX_RESTORE_BYTES=10737418240
|
||||
|
||||
# Rebuild SQLite metadata from existing files on startup.
|
||||
# Useful after manual restores/migrations; usually false for normal startup.
|
||||
MAGPIE_REINDEX_ON_START=false
|
||||
51
.github/workflows/release.yml
vendored
Normal file
51
.github/workflows/release.yml
vendored
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
|
||||
- uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Test
|
||||
run: go test ./...
|
||||
|
||||
- name: Build release artifacts
|
||||
run: make release VERSION="$GITHUB_REF_NAME" COMMIT="${GITHUB_SHA::7}"
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
|
||||
with:
|
||||
name: magpie-${{ github.ref_name }}
|
||||
path: dist/*
|
||||
|
||||
publish:
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093
|
||||
with:
|
||||
name: magpie-${{ github.ref_name }}
|
||||
path: dist
|
||||
|
||||
- name: Publish GitHub release
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65
|
||||
with:
|
||||
files: dist/*
|
||||
30
.gitignore
vendored
Normal file
30
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
# Local binaries and build output
|
||||
magpie
|
||||
dist/
|
||||
coverage.out
|
||||
|
||||
# Runtime data
|
||||
data/
|
||||
*.db
|
||||
*.db-*
|
||||
*.sqlite
|
||||
*.sqlite-*
|
||||
*.meta.json
|
||||
.multipart/
|
||||
|
||||
# Local environment files
|
||||
.env
|
||||
.env.*
|
||||
!.env.production.example
|
||||
|
||||
# Logs and temporary files
|
||||
*.log
|
||||
*.tmp
|
||||
tmp/
|
||||
|
||||
# Editor and OS files
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
20
Dockerfile
Normal file
20
Dockerfile
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
FROM golang:1.26-alpine@sha256:f44b851aa23dfa219d18db6eab743203245429d355cb619cf96a2ffe2a84ba7a AS build
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY *.go ./
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/magpie .
|
||||
|
||||
FROM alpine:3.22@sha256:2039be0c5ec6ce8566809626a252c930216a92109c043f282504accb5ee3c0c6
|
||||
|
||||
RUN addgroup -S magpie && adduser -S -G magpie magpie
|
||||
RUN mkdir -p /var/lib/magpie && chown -R magpie:magpie /var/lib/magpie
|
||||
|
||||
USER magpie
|
||||
EXPOSE 8090
|
||||
ENV MAGPIE_ADDR=0.0.0.0:8090
|
||||
ENV MAGPIE_DATA_DIR=/var/lib/magpie
|
||||
|
||||
COPY --from=build /out/magpie /usr/local/bin/magpie
|
||||
ENTRYPOINT ["/usr/local/bin/magpie"]
|
||||
36
Makefile
Normal file
36
Makefile
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
BINARY := magpie
|
||||
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || printf dev)
|
||||
COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || printf unknown)
|
||||
DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
LDFLAGS := -s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.date=$(DATE)
|
||||
DIST := dist
|
||||
|
||||
.PHONY: all fmt test build clean checksums release snapshot
|
||||
|
||||
all: fmt test build
|
||||
|
||||
fmt:
|
||||
gofmt -w .
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
build:
|
||||
mkdir -p $(DIST)
|
||||
CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY) .
|
||||
|
||||
snapshot: clean
|
||||
mkdir -p $(DIST)
|
||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_linux_amd64 .
|
||||
GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_linux_arm64 .
|
||||
GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_darwin_amd64 .
|
||||
GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_darwin_arm64 .
|
||||
GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o $(DIST)/$(BINARY)_windows_amd64.exe .
|
||||
|
||||
checksums:
|
||||
cd $(DIST) && sha256sum * > SHA256SUMS
|
||||
|
||||
release: snapshot checksums
|
||||
|
||||
clean:
|
||||
rm -rf $(DIST)
|
||||
271
README.md
Normal file
271
README.md
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
# Magpie
|
||||
|
||||
Magpie is a small object store for app-owned files. It stores objects on local disk and exposes a minimal S3-compatible path-style API.
|
||||
|
||||
It is intended to run behind your application or edge proxy, on localhost, a private Docker network, or a VPN-only address. Do not expose write-capable endpoints directly to the public internet.
|
||||
|
||||
## Status
|
||||
|
||||
Magpie supports common application object-storage operations:
|
||||
|
||||
- `PUT`, `GET`, `HEAD`, and `DELETE` objects
|
||||
- path-style S3 routes: `/:bucket/:key`
|
||||
- SigV4 header authentication for writes and private reads
|
||||
- presigned `GET` and `HEAD` URLs
|
||||
- prefix listing with continuation tokens
|
||||
- batch delete
|
||||
- copy object
|
||||
- multipart upload
|
||||
- bucket allowlist and opt-in public-read buckets
|
||||
- local metadata index in SQLite
|
||||
- async static-peer replication
|
||||
- backup, restore, scrub, reindex, stats, and repair commands
|
||||
|
||||
It does not implement bucket creation, ACLs, bucket policies, object versioning, lifecycle rules, quorum writes, or consensus replication.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```sh
|
||||
export MAGPIE_ADDR='127.0.0.1:8090'
|
||||
export MAGPIE_DATA_DIR='/var/lib/magpie'
|
||||
export MAGPIE_S3_ACCESS_KEY_ID='magpie'
|
||||
export MAGPIE_S3_SECRET_ACCESS_KEY='replace-with-a-long-random-secret'
|
||||
export MAGPIE_ALLOWED_BUCKETS='app-uploads'
|
||||
|
||||
magpie serve
|
||||
```
|
||||
|
||||
For local development from source:
|
||||
|
||||
```sh
|
||||
go run . serve
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Required authentication config:
|
||||
|
||||
```env
|
||||
MAGPIE_S3_ACCESS_KEY_ID=magpie
|
||||
MAGPIE_S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret
|
||||
```
|
||||
|
||||
Or define multiple scoped keys:
|
||||
|
||||
```env
|
||||
MAGPIE_S3_KEYS=app:secret1:read,write;auditor:secret2:read;admin:secret3:admin
|
||||
```
|
||||
|
||||
Supported scopes are `read`, `write`, and `admin`. `admin` implies all scopes.
|
||||
|
||||
Common runtime config:
|
||||
|
||||
```env
|
||||
MAGPIE_ADDR=127.0.0.1:8090
|
||||
MAGPIE_DATA_DIR=/var/lib/magpie
|
||||
MAGPIE_ALLOWED_BUCKETS=app-uploads
|
||||
# MAGPIE_PUBLIC_PREFIXES=app-uploads/avatars
|
||||
MAGPIE_RATE_LIMIT_PER_MINUTE=600
|
||||
MAGPIE_TRUST_PROXY_HEADERS=false
|
||||
MAGPIE_MAX_OBJECT_SIZE=1073741824
|
||||
MAGPIE_MIN_FREE_BYTES=1073741824
|
||||
MAGPIE_PRESIGNED_MAX_EXPIRY=24h
|
||||
MAGPIE_MULTIPART_MAX_AGE=24h
|
||||
MAGPIE_REINDEX_ON_START=false
|
||||
MAGPIE_MAX_RESTORE_BYTES=10737418240
|
||||
```
|
||||
|
||||
`MAGPIE_ALLOWED_BUCKETS` should be set in production. Leave it empty only if you intentionally want any bucket name accepted.
|
||||
|
||||
Buckets are private by default. Set `MAGPIE_PUBLIC_PREFIXES` to comma-separated `bucket/prefix` values, such as `app-uploads/avatars`, only when an app intentionally serves unsigned public media through an edge proxy. Set `MAGPIE_PUBLIC_PREFIXES=none` or leave it unset to require signed reads.
|
||||
|
||||
Public reads are hardened against stored-XSS from uploader-controlled content types: responses carry `Content-Security-Policy: default-src 'none'; sandbox`, and any object that is not inert inline media (images, audio, video — SVG excluded) is served with `Content-Disposition: attachment` so browsers download it instead of rendering it. For untrusted user uploads, still prefer serving public objects from an origin separate from your application.
|
||||
|
||||
## Application Integration
|
||||
|
||||
Use any S3-compatible client that supports path-style endpoints and custom hosts.
|
||||
|
||||
If Magpie runs on the same host as your application outside Docker:
|
||||
|
||||
```env
|
||||
S3_ACCESS_KEY_ID=magpie
|
||||
S3_SECRET_ACCESS_KEY=replace-with-a-long-random-secret
|
||||
S3_ENDPOINT=127.0.0.1:8090
|
||||
S3_BUCKET_NAME=app-uploads
|
||||
S3_PUBLIC_URL=http://127.0.0.1:8090/app-uploads
|
||||
S3_SCHEME=http://
|
||||
S3_PORT=8090
|
||||
```
|
||||
|
||||
If your app and Magpie are in the same Docker Compose network, use the Magpie service name:
|
||||
|
||||
```env
|
||||
S3_ENDPOINT=magpie:8090
|
||||
S3_BUCKET_NAME=app-uploads
|
||||
S3_PUBLIC_URL=http://magpie:8090/app-uploads
|
||||
S3_SCHEME=http://
|
||||
S3_PORT=8090
|
||||
```
|
||||
|
||||
Keep Magpie write access private. Your application should enforce user permissions before it writes to Magpie. Public buckets are intended for already-public objects such as avatars and timeline media.
|
||||
|
||||
## API Notes
|
||||
|
||||
S3 object routes use path-style URLs:
|
||||
|
||||
```text
|
||||
PUT /:bucket/:key
|
||||
GET /:bucket/:key
|
||||
HEAD /:bucket/:key
|
||||
DELETE /:bucket/:key
|
||||
POST /:bucket?delete
|
||||
```
|
||||
|
||||
Objects are stored under:
|
||||
|
||||
```text
|
||||
MAGPIE_DATA_DIR/<bucket>/<key>
|
||||
```
|
||||
|
||||
Internal names are reserved and cannot be used as object keys:
|
||||
|
||||
- `.multipart`
|
||||
- `.magpie.db`
|
||||
- `.magpie.db-*`
|
||||
- `*.meta.json`
|
||||
|
||||
Presigned URLs are read-only. Magpie accepts presigned `GET` and `HEAD` requests, but rejects presigned writes and deletes.
|
||||
|
||||
Normal SigV4 write requests must include a real `X-Amz-Content-Sha256` payload hash. `UNSIGNED-PAYLOAD` is reserved for authenticated replication traffic.
|
||||
|
||||
## Health And Metrics
|
||||
|
||||
Health and metrics require authentication.
|
||||
|
||||
```sh
|
||||
curl -H "Authorization: Bearer $MAGPIE_AUTH_TOKEN" \
|
||||
http://127.0.0.1:8090/health
|
||||
```
|
||||
|
||||
```sh
|
||||
curl -H "Authorization: Bearer $MAGPIE_AUTH_TOKEN" \
|
||||
http://127.0.0.1:8090/metrics
|
||||
```
|
||||
|
||||
The health endpoint returns `503` when free disk space is below `MAGPIE_MIN_FREE_BYTES`. Writes are rejected in that state.
|
||||
|
||||
If you only use S3 credentials and do not set `MAGPIE_AUTH_TOKEN`, sign health and metrics requests with an admin S3 key instead.
|
||||
|
||||
## CLI
|
||||
|
||||
```sh
|
||||
magpie serve
|
||||
magpie stats
|
||||
magpie scrub
|
||||
magpie reindex
|
||||
magpie repair
|
||||
magpie backup /backup/magpie.tar.gz
|
||||
magpie restore /backup/magpie.tar.gz
|
||||
magpie smoke
|
||||
magpie version
|
||||
```
|
||||
|
||||
Command summary:
|
||||
|
||||
- `stats`: object count, logical bytes, disk status, replication queue status
|
||||
- `scrub`: verifies metadata against files and reports missing, corrupt, and orphaned objects
|
||||
- `reindex`: rebuilds SQLite metadata from files on disk
|
||||
- `repair`: queues known objects for replication to configured peers
|
||||
- `backup`: creates a tar/gzip archive of the data directory
|
||||
- `restore`: restores a tar/gzip archive into the data directory
|
||||
- `smoke`: runs a local write/read/list/scrub/delete check
|
||||
|
||||
Restore rejects unsafe archive paths, symlinks, unsupported archive entries, and archives whose expanded file content exceeds `MAGPIE_MAX_RESTORE_BYTES`. Backup archives are created with owner-only permissions. For consistent backups, stop Magpie or use a filesystem snapshot before running `magpie backup`.
|
||||
|
||||
## Replication
|
||||
|
||||
Replication is asynchronous and best-effort. Local writes complete before peers acknowledge them.
|
||||
|
||||
Configure every node with matching S3 credentials and a shared replication secret:
|
||||
|
||||
```env
|
||||
MAGPIE_REPLICATION_PEERS=https://magpie-b.internal,https://magpie-c.internal
|
||||
MAGPIE_REPLICATION_SECRET=replace-with-a-long-random-peer-secret
|
||||
MAGPIE_REPLICATION_MAX_JOBS=10000
|
||||
MAGPIE_REPLICATION_MAX_ATTEMPTS=100
|
||||
```
|
||||
|
||||
`MAGPIE_REPLICATION_SECRET` is required when peers are configured. Replication-marked requests without the shared secret are rejected.
|
||||
|
||||
Replication peer URLs must use `https://` by default because Magpie sends SigV4 authorization and the replication secret to peers. Set `MAGPIE_ALLOW_INSECURE_REPLICATION=true` only for trusted local/private networks where plaintext HTTP is acceptable.
|
||||
|
||||
Use `magpie repair` after outages to queue a full object repair pass to all configured peers.
|
||||
|
||||
## Deployment
|
||||
|
||||
Docker Compose example:
|
||||
|
||||
```sh
|
||||
cp .env.production.example .env.production
|
||||
```
|
||||
|
||||
```yaml
|
||||
services:
|
||||
magpie:
|
||||
image: magpie:latest
|
||||
container_name: magpie
|
||||
build: .
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- .env.production
|
||||
ports:
|
||||
- "127.0.0.1:8090:8090"
|
||||
volumes:
|
||||
- /var/lib/magpie:/var/lib/magpie
|
||||
```
|
||||
|
||||
The default Compose file is standalone and binds Magpie to localhost only.
|
||||
|
||||
If another Compose project needs to reach Magpie by service name, use the optional network override. It attaches Magpie to an external shared network with the `magpie` network alias:
|
||||
|
||||
```sh
|
||||
docker network create elektrine-magpie-shared
|
||||
docker compose -f docker-compose.yml -f docker-compose.network.yml up -d --build
|
||||
```
|
||||
|
||||
Then attach your application container to the same external `elektrine-magpie-shared` network and use:
|
||||
|
||||
```env
|
||||
S3_ENDPOINT=magpie:8090
|
||||
S3_PUBLIC_URL=http://magpie:8090/app-uploads
|
||||
S3_SCHEME=http://
|
||||
S3_PORT=8090
|
||||
```
|
||||
|
||||
You can rename the shared network without editing the override file:
|
||||
|
||||
```sh
|
||||
MAGPIE_DOCKER_NETWORK=my-existing-shared-network \
|
||||
docker compose -f docker-compose.yml -f docker-compose.network.yml up -d --build
|
||||
```
|
||||
|
||||
Bare-metal installs can use `deploy/systemd/magpie.service`. The systemd unit is optional; use it only if you want Magpie managed directly by systemd.
|
||||
|
||||
Back up `/var/lib/magpie` or whatever directory you set as `MAGPIE_DATA_DIR`.
|
||||
|
||||
## Build And Test
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go build ./...
|
||||
```
|
||||
|
||||
Release artifacts:
|
||||
|
||||
```sh
|
||||
make test
|
||||
make build
|
||||
make release
|
||||
```
|
||||
|
||||
Artifacts are written to `dist/`, with checksums in `dist/SHA256SUMS`.
|
||||
181
backup.go
Normal file
181
backup.go
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func BackupDir(src string, dst string) error {
|
||||
out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
gz := gzip.NewWriter(out)
|
||||
defer gz.Close()
|
||||
tw := tar.NewWriter(gz)
|
||||
defer tw.Close()
|
||||
root, err := filepath.Abs(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil || entry.IsDir() {
|
||||
return err
|
||||
}
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("refusing to back up symlink %q", path)
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header, err := tar.FileInfoHeader(info, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header.Name = filepath.ToSlash(rel)
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileInfo, statErr := file.Stat()
|
||||
currentInfo, lstatErr := os.Lstat(path)
|
||||
if statErr != nil || lstatErr != nil {
|
||||
_ = file.Close()
|
||||
if statErr != nil {
|
||||
return statErr
|
||||
}
|
||||
return lstatErr
|
||||
}
|
||||
if currentInfo.Mode()&os.ModeSymlink != 0 || !os.SameFile(currentInfo, fileInfo) {
|
||||
_ = file.Close()
|
||||
return fmt.Errorf("refusing to back up replaced or symlinked file %q", path)
|
||||
}
|
||||
_, copyErr := io.Copy(tw, file)
|
||||
closeErr := file.Close()
|
||||
if copyErr != nil {
|
||||
return copyErr
|
||||
}
|
||||
return closeErr
|
||||
})
|
||||
}
|
||||
|
||||
func RestoreDir(src string, dst string) error {
|
||||
return RestoreDirWithLimit(src, dst, defaultMaxRestoreBytes)
|
||||
}
|
||||
|
||||
func RestoreDirWithLimit(src string, dst string, maxBytes int64) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
gz, err := gzip.NewReader(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gz.Close()
|
||||
tr := tar.NewReader(gz)
|
||||
root, err := filepath.Abs(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(root, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
var restoredBytes int64
|
||||
for {
|
||||
header, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.Contains(header.Name, "..") || strings.HasPrefix(header.Name, "/") {
|
||||
return fmt.Errorf("unsafe archive path %q", header.Name)
|
||||
}
|
||||
if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA && header.Typeflag != tar.TypeDir {
|
||||
return fmt.Errorf("unsupported archive entry %q", header.Name)
|
||||
}
|
||||
if header.Size < 0 {
|
||||
return fmt.Errorf("invalid archive size for %q", header.Name)
|
||||
}
|
||||
if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeRegA {
|
||||
restoredBytes += header.Size
|
||||
if maxBytes > 0 && restoredBytes > maxBytes {
|
||||
return fmt.Errorf("restore exceeds maximum size")
|
||||
}
|
||||
}
|
||||
path := filepath.Join(root, filepath.FromSlash(header.Name))
|
||||
if !strings.HasPrefix(path, root+string(os.PathSeparator)) && path != root {
|
||||
return fmt.Errorf("unsafe archive path %q", header.Name)
|
||||
}
|
||||
if err := rejectSymlinkParents(root, path); err != nil {
|
||||
return err
|
||||
}
|
||||
if header.FileInfo().IsDir() {
|
||||
if err := os.MkdirAll(path, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
if info, err := os.Lstat(path); err == nil && info.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("refusing to restore through symlink %q", header.Name)
|
||||
}
|
||||
out, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, copyErr := io.CopyN(out, tr, header.Size)
|
||||
closeErr := out.Close()
|
||||
if copyErr != nil {
|
||||
return copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func rejectSymlinkParents(root string, path string) error {
|
||||
current := root
|
||||
rel, err := filepath.Rel(root, filepath.Dir(path))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
for _, part := range strings.Split(rel, string(os.PathSeparator)) {
|
||||
current = filepath.Join(current, part)
|
||||
info, err := os.Lstat(current)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("refusing to restore through symlink parent %q", current)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
21
deploy/systemd/magpie.service
Normal file
21
deploy/systemd/magpie.service
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
[Unit]
|
||||
Description=Magpie object storage service
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=magpie
|
||||
Group=magpie
|
||||
EnvironmentFile=/etc/magpie/.env.production
|
||||
ExecStart=/usr/local/bin/magpie
|
||||
Restart=on-failure
|
||||
RestartSec=5s
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/var/lib/magpie
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
21
disk_unix.go
Normal file
21
disk_unix.go
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
//go:build !windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func (s *FileStore) DiskStatus() (DiskStatus, error) {
|
||||
var stat syscall.Statfs_t
|
||||
if err := syscall.Statfs(s.root, &stat); err != nil {
|
||||
return DiskStatus{}, err
|
||||
}
|
||||
if stat.Bsize <= 0 {
|
||||
return DiskStatus{}, fmt.Errorf("invalid filesystem block size")
|
||||
}
|
||||
total := stat.Blocks * uint64(stat.Bsize)
|
||||
free := stat.Bavail * uint64(stat.Bsize)
|
||||
return DiskStatus{TotalBytes: total, FreeBytes: free, UsedBytes: total - free}, nil
|
||||
}
|
||||
7
disk_windows.go
Normal file
7
disk_windows.go
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
//go:build windows
|
||||
|
||||
package main
|
||||
|
||||
func (s *FileStore) DiskStatus() (DiskStatus, error) {
|
||||
return DiskStatus{}, nil
|
||||
}
|
||||
11
docker-compose.network.yml
Normal file
11
docker-compose.network.yml
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
services:
|
||||
magpie:
|
||||
networks:
|
||||
magpie_shared:
|
||||
aliases:
|
||||
- magpie
|
||||
|
||||
networks:
|
||||
magpie_shared:
|
||||
external: true
|
||||
name: ${MAGPIE_DOCKER_NETWORK:-elektrine-magpie-shared}
|
||||
15
docker-compose.yml
Normal file
15
docker-compose.yml
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
services:
|
||||
magpie:
|
||||
image: magpie:latest
|
||||
container_name: magpie
|
||||
build: .
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- .env.production
|
||||
ports:
|
||||
- "127.0.0.1:8090:8090"
|
||||
volumes:
|
||||
- magpie-data:/var/lib/magpie
|
||||
|
||||
volumes:
|
||||
magpie-data:
|
||||
16
go.mod
Normal file
16
go.mod
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
module magpie
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
modernc.org/libc v1.72.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.50.0 // indirect
|
||||
)
|
||||
21
go.sum
Normal file
21
go.sum
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c=
|
||||
modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM=
|
||||
modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
|
||||
415
main.go
Normal file
415
main.go
Normal file
|
|
@ -0,0 +1,415 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultMaxObjectSize = 1024 * 1024 * 1024
|
||||
defaultMinFreeBytes = 1024 * 1024 * 1024
|
||||
defaultPresignedMaxExpiry = 24 * time.Hour
|
||||
defaultMultipartMaxAge = 24 * time.Hour
|
||||
defaultMaxRestoreBytes = 10 * 1024 * 1024 * 1024
|
||||
defaultReplicationMaxJobs = 10_000
|
||||
defaultReplicationMaxAttempts = 100
|
||||
)
|
||||
|
||||
func main() {
|
||||
cmd := "serve"
|
||||
if len(os.Args) > 1 {
|
||||
cmd = os.Args[1]
|
||||
}
|
||||
if cmd == "version" {
|
||||
printJSON(versionInfo())
|
||||
return
|
||||
}
|
||||
if cmd == "help" || cmd == "-h" || cmd == "--help" {
|
||||
printUsage()
|
||||
return
|
||||
}
|
||||
|
||||
cfg := configFromEnv()
|
||||
var store *FileStore
|
||||
openStore := func() *FileStore {
|
||||
if store != nil {
|
||||
return store
|
||||
}
|
||||
opened, err := NewFileStore(cfg.DataDir)
|
||||
if err != nil {
|
||||
log.Fatalf("init store: %v", err)
|
||||
}
|
||||
store = opened
|
||||
return store
|
||||
}
|
||||
|
||||
switch cmd {
|
||||
case "serve":
|
||||
serve(cfg, openStore())
|
||||
case "reindex":
|
||||
mustJSON(openStore().Reindex())
|
||||
case "stats":
|
||||
stats, err := openStore().Stats()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
printJSON(stats)
|
||||
case "scrub":
|
||||
report, err := openStore().Scrub()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
printJSON(report)
|
||||
case "repair":
|
||||
count, err := openStore().EnqueueRepair(cfg.ReplicationPeers)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
printJSON(map[string]any{"status": "queued", "jobs": count})
|
||||
case "backup":
|
||||
if len(os.Args) < 3 {
|
||||
log.Fatal("usage: magpie backup <archive.tar.gz>")
|
||||
}
|
||||
mustJSON(BackupDir(cfg.DataDir, os.Args[2]))
|
||||
case "restore":
|
||||
if len(os.Args) < 3 {
|
||||
log.Fatal("usage: magpie restore <archive.tar.gz>")
|
||||
}
|
||||
mustJSON(RestoreDirWithLimit(os.Args[2], cfg.DataDir, cfg.MaxRestoreBytes))
|
||||
case "smoke":
|
||||
mustJSON(runSmoke(openStore()))
|
||||
default:
|
||||
log.Fatalf("unknown command %q", sanitizeLogValue(cmd))
|
||||
}
|
||||
}
|
||||
|
||||
func serve(cfg Config, store *FileStore) {
|
||||
if cfg.ReindexOnStart {
|
||||
log.Printf("reindexing metadata from %s", cfg.DataDir)
|
||||
if err := store.Reindex(); err != nil {
|
||||
log.Fatalf("reindex store: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
server := NewServerInstance(store, cfg)
|
||||
srv := &http.Server{
|
||||
Addr: cfg.Addr,
|
||||
Handler: server.Handler(),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 2 * time.Minute,
|
||||
WriteTimeout: 2 * time.Minute,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
MaxHeaderBytes: 16 * 1024,
|
||||
}
|
||||
|
||||
go func() {
|
||||
log.Printf("object store listening on %s, data=%s", cfg.Addr, cfg.DataDir)
|
||||
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Fatalf("listen: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
server.startMultipartCleanup(ctx)
|
||||
server.startReplicationWorker(ctx)
|
||||
<-ctx.Done()
|
||||
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := srv.Shutdown(shutdownCtx); err != nil {
|
||||
log.Printf("shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Addr string
|
||||
DataDir string
|
||||
AuthToken string
|
||||
S3AccessKeyID string
|
||||
S3SecretAccessKey string
|
||||
S3Keys []AccessKey
|
||||
S3Region string
|
||||
AllowUnsignedLocal bool
|
||||
MaxObjectSize int64
|
||||
MinFreeBytes uint64
|
||||
PresignedMaxExpiry time.Duration
|
||||
MultipartMaxAge time.Duration
|
||||
ReindexOnStart bool
|
||||
ReplicationPeers []string
|
||||
ReplicationSecret string
|
||||
AllowInsecureReplication bool
|
||||
ReplicationMaxJobs int
|
||||
ReplicationMaxAttempts int
|
||||
MaxRestoreBytes int64
|
||||
AllowedBuckets map[string]bool
|
||||
PublicPrefixes map[string]bool
|
||||
RateLimitPerMinute int
|
||||
TrustProxyHeaders bool
|
||||
}
|
||||
|
||||
type AccessKey struct {
|
||||
ID string
|
||||
Secret string
|
||||
Permissions map[string]bool
|
||||
}
|
||||
|
||||
func configFromEnv() Config {
|
||||
addr := getenv("MAGPIE_ADDR", "127.0.0.1:8090")
|
||||
dataDir := getenv("MAGPIE_DATA_DIR", "./data")
|
||||
authToken := os.Getenv("MAGPIE_AUTH_TOKEN")
|
||||
s3AccessKeyID := os.Getenv("MAGPIE_S3_ACCESS_KEY_ID")
|
||||
s3SecretAccessKey := os.Getenv("MAGPIE_S3_SECRET_ACCESS_KEY")
|
||||
s3Keys := parseAccessKeys(os.Getenv("MAGPIE_S3_KEYS"), s3AccessKeyID, s3SecretAccessKey)
|
||||
s3Region := getenv("MAGPIE_S3_REGION", "auto")
|
||||
maxObjectSize := mustParseBytes("MAGPIE_MAX_OBJECT_SIZE", defaultMaxObjectSize)
|
||||
minFreeBytes := mustParseUint("MAGPIE_MIN_FREE_BYTES", defaultMinFreeBytes)
|
||||
presignedMaxExpiry := mustParseDuration("MAGPIE_PRESIGNED_MAX_EXPIRY", defaultPresignedMaxExpiry)
|
||||
multipartMaxAge := mustParseDuration("MAGPIE_MULTIPART_MAX_AGE", defaultMultipartMaxAge)
|
||||
reindexOnStart := mustParseBool("MAGPIE_REINDEX_ON_START", false)
|
||||
allowInsecureReplication := mustParseBool("MAGPIE_ALLOW_INSECURE_REPLICATION", false)
|
||||
replicationPeers := parseReplicationPeers(os.Getenv("MAGPIE_REPLICATION_PEERS"), allowInsecureReplication)
|
||||
replicationSecret := os.Getenv("MAGPIE_REPLICATION_SECRET")
|
||||
replicationMaxJobs := mustParseInt("MAGPIE_REPLICATION_MAX_JOBS", defaultReplicationMaxJobs)
|
||||
replicationMaxAttempts := mustParseInt("MAGPIE_REPLICATION_MAX_ATTEMPTS", defaultReplicationMaxAttempts)
|
||||
maxRestoreBytes := mustParseBytes("MAGPIE_MAX_RESTORE_BYTES", defaultMaxRestoreBytes)
|
||||
allowedBuckets := parseSet(os.Getenv("MAGPIE_ALLOWED_BUCKETS"))
|
||||
publicPrefixes := parsePublicPrefixes(os.Getenv("MAGPIE_PUBLIC_PREFIXES"))
|
||||
rateLimitPerMinute := mustParseInt("MAGPIE_RATE_LIMIT_PER_MINUTE", 600)
|
||||
trustProxyHeaders := mustParseBool("MAGPIE_TRUST_PROXY_HEADERS", false)
|
||||
allowUnsignedLocal := mustParseBool("MAGPIE_ALLOW_UNSIGNED_PAYLOADS", false)
|
||||
|
||||
if authToken == "" && len(s3Keys) == 0 {
|
||||
log.Fatal("MAGPIE_AUTH_TOKEN or MAGPIE_S3_ACCESS_KEY_ID/MAGPIE_S3_SECRET_ACCESS_KEY or MAGPIE_S3_KEYS is required")
|
||||
}
|
||||
if len(replicationPeers) > 0 && replicationSecret == "" {
|
||||
log.Fatal("MAGPIE_REPLICATION_SECRET is required when MAGPIE_REPLICATION_PEERS is set")
|
||||
}
|
||||
|
||||
if raw := os.Getenv("MAGPIE_MAX_PROCS"); raw != "" {
|
||||
if _, err := strconv.Atoi(raw); err != nil {
|
||||
log.Fatalf("MAGPIE_MAX_PROCS must be an integer: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return Config{
|
||||
Addr: addr,
|
||||
DataDir: dataDir,
|
||||
AuthToken: authToken,
|
||||
S3AccessKeyID: s3AccessKeyID,
|
||||
S3SecretAccessKey: s3SecretAccessKey,
|
||||
S3Keys: s3Keys,
|
||||
S3Region: s3Region,
|
||||
AllowUnsignedLocal: allowUnsignedLocal,
|
||||
MaxObjectSize: maxObjectSize,
|
||||
MinFreeBytes: minFreeBytes,
|
||||
PresignedMaxExpiry: presignedMaxExpiry,
|
||||
MultipartMaxAge: multipartMaxAge,
|
||||
ReindexOnStart: reindexOnStart,
|
||||
ReplicationPeers: replicationPeers,
|
||||
ReplicationSecret: replicationSecret,
|
||||
AllowInsecureReplication: allowInsecureReplication,
|
||||
ReplicationMaxJobs: replicationMaxJobs,
|
||||
ReplicationMaxAttempts: replicationMaxAttempts,
|
||||
MaxRestoreBytes: maxRestoreBytes,
|
||||
AllowedBuckets: allowedBuckets,
|
||||
PublicPrefixes: publicPrefixes,
|
||||
RateLimitPerMinute: rateLimitPerMinute,
|
||||
TrustProxyHeaders: trustProxyHeaders,
|
||||
}
|
||||
}
|
||||
|
||||
func parseAccessKeys(raw string, legacyID string, legacySecret string) []AccessKey {
|
||||
keys := make([]AccessKey, 0)
|
||||
if legacyID != "" && legacySecret != "" {
|
||||
keys = append(keys, AccessKey{ID: legacyID, Secret: legacySecret, Permissions: permissions("read,write,admin")})
|
||||
}
|
||||
for _, entry := range strings.Split(raw, ";") {
|
||||
entry = strings.TrimSpace(entry)
|
||||
if entry == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(entry, ":")
|
||||
if len(parts) < 2 || parts[0] == "" || parts[1] == "" {
|
||||
log.Fatal("MAGPIE_S3_KEYS entries must be id:secret[:read,write,admin]")
|
||||
}
|
||||
perms := "read,write"
|
||||
if len(parts) >= 3 {
|
||||
perms = parts[2]
|
||||
}
|
||||
keys = append(keys, AccessKey{ID: parts[0], Secret: parts[1], Permissions: permissions(perms)})
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func permissions(raw string) map[string]bool {
|
||||
result := map[string]bool{}
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
part = strings.TrimSpace(strings.ToLower(part))
|
||||
if part != "" {
|
||||
result[part] = true
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func parseCSV(raw string) []string {
|
||||
parts := make([]string, 0)
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
parts = append(parts, strings.TrimRight(part, "/"))
|
||||
}
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func parseReplicationPeers(raw string, allowInsecure bool) []string {
|
||||
peers := parseCSV(raw)
|
||||
for _, peer := range peers {
|
||||
if err := validateReplicationPeer(peer, allowInsecure); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
return peers
|
||||
}
|
||||
|
||||
func parseSet(raw string) map[string]bool {
|
||||
values := map[string]bool{}
|
||||
for _, value := range parseCSV(raw) {
|
||||
values[value] = true
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func parsePublicPrefixes(raw string) map[string]bool {
|
||||
value := strings.TrimSpace(strings.ToLower(raw))
|
||||
if value == "none" || value == "false" || value == "0" {
|
||||
return map[string]bool{}
|
||||
}
|
||||
if strings.TrimSpace(raw) != "" {
|
||||
prefixes := parseSet(raw)
|
||||
for prefix := range prefixes {
|
||||
if err := ValidateKey(strings.TrimSuffix(prefix, "/")); err != nil || !strings.Contains(strings.Trim(prefix, "/"), "/") {
|
||||
log.Fatal("MAGPIE_PUBLIC_PREFIXES entries must be bucket/prefix values")
|
||||
}
|
||||
}
|
||||
return prefixes
|
||||
}
|
||||
return map[string]bool{}
|
||||
}
|
||||
|
||||
func printUsage() {
|
||||
fmt.Println("usage: magpie [serve|reindex|stats|scrub|repair|version|backup|restore|smoke]")
|
||||
}
|
||||
|
||||
func mustJSON(err error) {
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
printJSON(map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
func printJSON(value any) {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
if err := encoder.Encode(value); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getenv(key, fallback string) string {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func mustParseBytes(key string, fallback int64) int64 {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || parsed < 0 {
|
||||
log.Fatal(fmt.Sprintf("%s must be a non-negative integer byte count", key))
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func mustParseUint(key string, fallback uint64) uint64 {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.ParseUint(value, 10, 64)
|
||||
if err != nil {
|
||||
log.Fatal(fmt.Sprintf("%s must be a non-negative integer byte count", key))
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func mustParseDuration(key string, fallback time.Duration) time.Duration {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := time.ParseDuration(value)
|
||||
if err != nil || parsed <= 0 {
|
||||
log.Fatal(fmt.Sprintf("%s must be a positive duration like 1h or 24h", key))
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func mustParseBool(key string, fallback bool) bool {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
log.Fatal(fmt.Sprintf("%s must be true or false", key))
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func mustParseInt(key string, fallback int) int {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed < 0 {
|
||||
log.Fatal(fmt.Sprintf("%s must be a non-negative integer", key))
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func runSmoke(store *FileStore) error {
|
||||
key := "smoke/test.txt"
|
||||
if _, err := store.Put(key, strings.NewReader("ok"), "text/plain"); err != nil {
|
||||
return err
|
||||
}
|
||||
obj, err := store.Open(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = obj.Close()
|
||||
if _, err := store.List("smoke/", 10); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := store.Scrub(); err != nil {
|
||||
return err
|
||||
}
|
||||
return store.Delete(key)
|
||||
}
|
||||
330
replication.go
Normal file
330
replication.go
Normal file
|
|
@ -0,0 +1,330 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
replicationOpPut = "put"
|
||||
replicationOpDelete = "delete"
|
||||
)
|
||||
|
||||
type ReplicationJob struct {
|
||||
ID int64
|
||||
Peer string
|
||||
Op string
|
||||
Key string
|
||||
Attempts int
|
||||
}
|
||||
|
||||
func (s *Server) enqueueReplicationPut(key string) {
|
||||
s.enqueueReplication(replicationOpPut, key)
|
||||
}
|
||||
|
||||
func (s *Server) enqueueReplicationDelete(key string) {
|
||||
s.enqueueReplication(replicationOpDelete, key)
|
||||
}
|
||||
|
||||
func (s *Server) enqueueReplication(op string, key string) {
|
||||
if len(s.cfg.ReplicationPeers) == 0 || len(s.cfg.S3Keys) == 0 || s.cfg.ReplicationSecret == "" {
|
||||
return
|
||||
}
|
||||
for _, peer := range s.cfg.ReplicationPeers {
|
||||
if err := s.store.(*FileStore).EnqueueReplication(peer, op, key, s.cfg.ReplicationMaxJobs); err != nil {
|
||||
log.Printf("replication enqueue op=%s key=%s peer=%s error=%v", op, sanitizeLogValue(key), sanitizeLogValue(peer), err)
|
||||
}
|
||||
}
|
||||
go s.processReplicationQueue(context.Background())
|
||||
}
|
||||
|
||||
func (s *Server) startReplicationWorker(ctx context.Context) {
|
||||
if len(s.cfg.ReplicationPeers) == 0 || len(s.cfg.S3Keys) == 0 || s.cfg.ReplicationSecret == "" {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(2 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.processReplicationQueue(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *Server) processReplicationQueue(ctx context.Context) {
|
||||
jobs, err := s.store.(*FileStore).DueReplicationJobs(25)
|
||||
if err != nil {
|
||||
log.Printf("replication due jobs error=%v", err)
|
||||
return
|
||||
}
|
||||
for _, job := range jobs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err := s.performReplicationJob(job); err != nil {
|
||||
log.Printf("replication job id=%d op=%s key=%s peer=%s error=%v", job.ID, job.Op, sanitizeLogValue(job.Key), sanitizeLogValue(job.Peer), err)
|
||||
if s.cfg.ReplicationMaxAttempts > 0 && job.Attempts+1 >= s.cfg.ReplicationMaxAttempts {
|
||||
_ = s.store.(*FileStore).CompleteReplicationJob(job.ID)
|
||||
continue
|
||||
}
|
||||
_ = s.store.(*FileStore).FailReplicationJob(job.ID, job.Attempts+1, err.Error())
|
||||
continue
|
||||
}
|
||||
_ = s.store.(*FileStore).CompleteReplicationJob(job.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) performReplicationJob(job ReplicationJob) error {
|
||||
switch job.Op {
|
||||
case replicationOpPut:
|
||||
return s.performReplicationPut(job.Peer, job.Key)
|
||||
case replicationOpDelete:
|
||||
return s.performReplicationDelete(job.Peer, job.Key)
|
||||
default:
|
||||
return fmt.Errorf("unknown replication op %q", job.Op)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) performReplicationPut(peer string, key string) error {
|
||||
obj, err := s.store.Open(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer obj.Close()
|
||||
|
||||
peerURL, err := replicationObjectURL(peer, key, s.cfg.AllowInsecureReplication)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodPut, peerURL, obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", obj.Info.ContentType)
|
||||
req.Header.Set("X-Magpie-Replication", "1")
|
||||
if s.cfg.ReplicationSecret != "" {
|
||||
req.Header.Set("X-Magpie-Replication-Secret", s.cfg.ReplicationSecret)
|
||||
}
|
||||
s.signReplication(req)
|
||||
resp, err := (&http.Client{Timeout: 5 * time.Minute}).Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("peer status %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) performReplicationDelete(peer string, key string) error {
|
||||
peerURL, err := replicationObjectURL(peer, key, s.cfg.AllowInsecureReplication)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodDelete, peerURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("X-Magpie-Replication", "1")
|
||||
if s.cfg.ReplicationSecret != "" {
|
||||
req.Header.Set("X-Magpie-Replication-Secret", s.cfg.ReplicationSecret)
|
||||
}
|
||||
s.signReplication(req)
|
||||
resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("peer status %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) signReplication(req *http.Request) {
|
||||
key := s.cfg.S3Keys[0]
|
||||
amzDate := time.Now().UTC().Format("20060102T150405Z")
|
||||
date := amzDate[:8]
|
||||
scope := date + "/" + s.cfg.S3Region + "/s3/aws4_request"
|
||||
signedHeaders := "host;x-amz-content-sha256;x-amz-date;x-magpie-replication"
|
||||
if req.Header.Get("X-Magpie-Replication-Secret") != "" {
|
||||
signedHeaders += ";x-magpie-replication-secret"
|
||||
}
|
||||
req.Header.Set("X-Amz-Date", amzDate)
|
||||
req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
|
||||
canonical := canonicalRequest(req, signedHeaders, "UNSIGNED-PAYLOAD", false)
|
||||
sig := signature(key.Secret, amzDate, scope, canonical)
|
||||
req.Header.Set("Authorization", awsAlgorithm+" Credential="+key.ID+"/"+scope+", SignedHeaders="+signedHeaders+", Signature="+sig)
|
||||
}
|
||||
|
||||
func replicationObjectURL(peer string, key string, allowInsecure bool) (string, error) {
|
||||
if err := ValidateKey(key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := validateReplicationPeer(peer, allowInsecure); err != nil {
|
||||
return "", err
|
||||
}
|
||||
parsed, err := url.Parse(peer)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parsed.RawQuery = ""
|
||||
parsed.Fragment = ""
|
||||
parsed.Path = strings.TrimRight(parsed.Path, "/") + "/" + key
|
||||
parsed.RawPath = ""
|
||||
return parsed.String(), nil
|
||||
}
|
||||
|
||||
func validateReplicationPeer(peer string, allowInsecure bool) error {
|
||||
parsed, err := url.Parse(peer)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return fmt.Errorf("invalid replication peer %q", peer)
|
||||
}
|
||||
if parsed.RawQuery != "" || parsed.Fragment != "" {
|
||||
return fmt.Errorf("replication peer %q must not include query or fragment", peer)
|
||||
}
|
||||
if parsed.Scheme == "https" {
|
||||
return nil
|
||||
}
|
||||
if allowInsecure && parsed.Scheme == "http" {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("replication peer %q must use https or set MAGPIE_ALLOW_INSECURE_REPLICATION=true", peer)
|
||||
}
|
||||
|
||||
func (s *FileStore) EnqueueReplication(peer string, op string, key string, maxJobs int) error {
|
||||
now := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
result, err := s.db.Exec(`
|
||||
UPDATE replication_jobs
|
||||
SET next_attempt_at = ?, updated_at = ?
|
||||
WHERE peer = ? AND op = ? AND key = ?
|
||||
`, now, now, peer, op, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rows, err := result.RowsAffected(); err == nil && rows > 0 {
|
||||
return nil
|
||||
}
|
||||
if maxJobs <= 0 {
|
||||
maxJobs = defaultReplicationMaxJobs
|
||||
}
|
||||
var pending int
|
||||
if err := s.db.QueryRow(`SELECT COUNT(*) FROM replication_jobs`).Scan(&pending); err != nil {
|
||||
return err
|
||||
}
|
||||
if pending >= maxJobs {
|
||||
return fmt.Errorf("replication queue is full")
|
||||
}
|
||||
_, err = s.db.Exec(`
|
||||
INSERT INTO replication_jobs (peer, op, key, next_attempt_at, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`, peer, op, key, now, now, now)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *FileStore) DueReplicationJobs(limit int) ([]ReplicationJob, error) {
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 25
|
||||
}
|
||||
now := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
rows, err := s.db.Query(`
|
||||
SELECT id, peer, op, key, attempts
|
||||
FROM replication_jobs
|
||||
WHERE next_attempt_at <= ?
|
||||
ORDER BY id ASC
|
||||
LIMIT ?
|
||||
`, now, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
jobs := make([]ReplicationJob, 0)
|
||||
for rows.Next() {
|
||||
var job ReplicationJob
|
||||
if err := rows.Scan(&job.ID, &job.Peer, &job.Op, &job.Key, &job.Attempts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
return jobs, rows.Err()
|
||||
}
|
||||
|
||||
func (s *FileStore) CompleteReplicationJob(id int64) error {
|
||||
_, err := s.db.Exec(`DELETE FROM replication_jobs WHERE id = ?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *FileStore) FailReplicationJob(id int64, attempts int, message string) error {
|
||||
delay := time.Duration(attempts*attempts) * time.Second
|
||||
if delay > 5*time.Minute {
|
||||
delay = 5 * time.Minute
|
||||
}
|
||||
next := time.Now().UTC().Add(delay).Format(time.RFC3339Nano)
|
||||
now := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
_, err := s.db.Exec(`
|
||||
UPDATE replication_jobs
|
||||
SET attempts = ?, next_attempt_at = ?, last_error = ?, updated_at = ?
|
||||
WHERE id = ?
|
||||
`, attempts, next, truncateError(message), now, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *FileStore) EnqueueRepair(peers []string) (int, error) {
|
||||
if len(peers) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
count := 0
|
||||
after := ""
|
||||
for {
|
||||
page, err := s.ListPage("", after, 1000)
|
||||
if err != nil {
|
||||
return count, err
|
||||
}
|
||||
for _, object := range page.Objects {
|
||||
for _, peer := range peers {
|
||||
if err := s.EnqueueReplication(peer, replicationOpPut, object.Key, defaultReplicationMaxJobs); err != nil {
|
||||
return count, err
|
||||
}
|
||||
count++
|
||||
}
|
||||
}
|
||||
if page.NextAfter == "" {
|
||||
return count, nil
|
||||
}
|
||||
after = page.NextAfter
|
||||
}
|
||||
}
|
||||
|
||||
func (s *FileStore) ReplicationQueueStats() (ReplicationQueueStats, error) {
|
||||
var stats ReplicationQueueStats
|
||||
err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(MAX(attempts), 0) FROM replication_jobs`).Scan(&stats.PendingJobs, &stats.MaxAttempts)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return stats, nil
|
||||
}
|
||||
return stats, err
|
||||
}
|
||||
|
||||
func truncateError(message string) string {
|
||||
if len(message) <= 500 {
|
||||
return message
|
||||
}
|
||||
return message[:500]
|
||||
}
|
||||
|
||||
type ReplicationQueueStats struct {
|
||||
PendingJobs int64 `json:"pending_jobs"`
|
||||
MaxAttempts int `json:"max_attempts"`
|
||||
}
|
||||
938
s3.go
Normal file
938
s3.go
Normal file
|
|
@ -0,0 +1,938 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
maxXMLRequestBytes = 1 << 20
|
||||
maxBatchDeleteItems = 1000
|
||||
)
|
||||
|
||||
var errPayloadHashMismatch = errors.New("payload hash mismatch")
|
||||
|
||||
func (s *Server) s3Object(w http.ResponseWriter, r *http.Request) {
|
||||
if s.s3BucketList(w, r) {
|
||||
return
|
||||
}
|
||||
if s.s3BucketPost(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
bucket, key, ok := s.s3BucketKey(r.URL.Path)
|
||||
if !ok {
|
||||
s3Error(w, http.StatusNotFound, "NoSuchBucket", "bucket not found")
|
||||
return
|
||||
}
|
||||
if !s.bucketAllowed(bucket) {
|
||||
s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed")
|
||||
return
|
||||
}
|
||||
|
||||
storeKey := bucket + "/" + key
|
||||
if err := ValidateKey(storeKey); err != nil {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidObjectName", err.Error())
|
||||
return
|
||||
}
|
||||
publicRead := (r.Method == http.MethodGet || r.Method == http.MethodHead) && s.publicReadAllowed(storeKey)
|
||||
if !publicRead && !s.authorizedS3(r) {
|
||||
s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid")
|
||||
return
|
||||
}
|
||||
if !s.replicationAllowed(r) {
|
||||
s3Error(w, http.StatusForbidden, "AccessDenied", "replication secret is invalid")
|
||||
return
|
||||
}
|
||||
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
if _, ok := r.URL.Query()["uploads"]; ok {
|
||||
s.s3CreateMultipartUpload(w, r, storeKey)
|
||||
} else if r.URL.Query().Get("uploadId") != "" {
|
||||
s.s3CompleteMultipartUpload(w, r, storeKey)
|
||||
} else if _, ok := r.URL.Query()["delete"]; ok {
|
||||
s.s3DeleteObjects(w, r, bucket)
|
||||
} else {
|
||||
s3Error(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed")
|
||||
}
|
||||
case http.MethodPut:
|
||||
if r.URL.Query().Get("uploadId") != "" && r.URL.Query().Get("partNumber") != "" {
|
||||
s.s3UploadPart(w, r, storeKey)
|
||||
} else if r.Header.Get("X-Amz-Copy-Source") != "" {
|
||||
s.s3CopyObject(w, r, storeKey)
|
||||
} else {
|
||||
s.s3Put(w, r, storeKey)
|
||||
}
|
||||
case http.MethodGet:
|
||||
s.s3Get(w, r, storeKey, publicRead)
|
||||
case http.MethodHead:
|
||||
s.s3Head(w, r, storeKey, publicRead)
|
||||
case http.MethodDelete:
|
||||
if r.URL.Query().Get("uploadId") != "" {
|
||||
s.s3AbortMultipartUpload(w, r, storeKey)
|
||||
} else {
|
||||
s.s3Delete(w, r, storeKey)
|
||||
}
|
||||
default:
|
||||
w.Header().Set("Allow", "PUT, GET, HEAD, DELETE")
|
||||
s3Error(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) s3BucketPost(w http.ResponseWriter, r *http.Request) bool {
|
||||
path := strings.Trim(r.URL.Path, "/")
|
||||
if r.Method != http.MethodPost || path == "" || strings.Contains(path, "/") {
|
||||
return false
|
||||
}
|
||||
if _, ok := r.URL.Query()["delete"]; !ok {
|
||||
return false
|
||||
}
|
||||
if err := ValidateKey(path); err != nil {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidBucketName", err.Error())
|
||||
return true
|
||||
}
|
||||
if !s.bucketAllowed(path) {
|
||||
s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed")
|
||||
return true
|
||||
}
|
||||
if !s.authorizedS3(r) {
|
||||
s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid")
|
||||
return true
|
||||
}
|
||||
if !s.replicationAllowed(r) {
|
||||
s3Error(w, http.StatusForbidden, "AccessDenied", "replication secret is invalid")
|
||||
return true
|
||||
}
|
||||
s.s3DeleteObjects(w, r, path)
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Server) s3CreateMultipartUpload(w http.ResponseWriter, r *http.Request, key string) {
|
||||
uploadID, err := randomID()
|
||||
if err != nil {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "upload id failed")
|
||||
return
|
||||
}
|
||||
dir := s.multipartDir(uploadID)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "upload init failed")
|
||||
return
|
||||
}
|
||||
meta := multipartMeta{Key: key, ContentType: r.Header.Get("Content-Type")}
|
||||
data, _ := xml.Marshal(meta)
|
||||
if err := os.WriteFile(filepath.Join(dir, "meta.xml"), data, 0o600); err != nil {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "upload init failed")
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = xml.NewEncoder(w).Encode(createMultipartUploadResult{UploadID: uploadID})
|
||||
}
|
||||
|
||||
func (s *Server) s3UploadPart(w http.ResponseWriter, r *http.Request, key string) {
|
||||
release, ok := s.acceptWrite(w, r, true)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
uploadID := r.URL.Query().Get("uploadId")
|
||||
partNumber, err := strconv.Atoi(r.URL.Query().Get("partNumber"))
|
||||
if err != nil || partNumber <= 0 || partNumber > 10_000 {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid part number")
|
||||
return
|
||||
}
|
||||
meta, err := s.multipartMeta(uploadID)
|
||||
if err != nil || meta.Key != key {
|
||||
s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found")
|
||||
return
|
||||
}
|
||||
dir := s.multipartDir(uploadID)
|
||||
if !s.acceptMultipartPartSize(w, dir, r.ContentLength) {
|
||||
return
|
||||
}
|
||||
partPath := filepath.Join(dir, fmt.Sprintf("part-%05d", partNumber))
|
||||
body, verifyPayload, ok := s.s3StreamingBody(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, ".part-*")
|
||||
if err != nil {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed")
|
||||
return
|
||||
}
|
||||
hash := md5Hash()
|
||||
_, copyErr := io.Copy(io.MultiWriter(tmp, hash), http.MaxBytesReader(w, body, s.cfg.MaxObjectSize))
|
||||
closeErr := tmp.Close()
|
||||
if copyErr != nil || closeErr != nil {
|
||||
_ = os.Remove(tmp.Name())
|
||||
if errors.Is(copyErr, errPayloadHashMismatch) {
|
||||
s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
|
||||
return
|
||||
}
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed")
|
||||
return
|
||||
}
|
||||
if !verifyPayload() {
|
||||
_ = os.Remove(tmp.Name())
|
||||
s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
|
||||
return
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), partPath); err != nil {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed")
|
||||
return
|
||||
}
|
||||
etag := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
w.Header().Set("ETag", quoteETag(etag))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (s *Server) acceptMultipartPartSize(w http.ResponseWriter, dir string, incomingSize int64) bool {
|
||||
if incomingSize < 0 {
|
||||
s3Error(w, http.StatusLengthRequired, "MissingContentLength", "content length is required")
|
||||
return false
|
||||
}
|
||||
if s.cfg.MaxObjectSize <= 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
existingSize := int64(0)
|
||||
parts, _ := filepath.Glob(filepath.Join(dir, "part-*"))
|
||||
for _, part := range parts {
|
||||
info, err := os.Stat(part)
|
||||
if err == nil && !info.IsDir() {
|
||||
existingSize += info.Size()
|
||||
}
|
||||
}
|
||||
|
||||
if existingSize+incomingSize > s.cfg.MaxObjectSize {
|
||||
s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "multipart upload exceeds object size limit")
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Server) s3CompleteMultipartUpload(w http.ResponseWriter, r *http.Request, key string) {
|
||||
uploadID := r.URL.Query().Get("uploadId")
|
||||
meta, err := s.multipartMeta(uploadID)
|
||||
if err != nil || meta.Key != key {
|
||||
s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found")
|
||||
return
|
||||
}
|
||||
var complete completeMultipartUpload
|
||||
completeBody, ok := s.s3BoundedBody(w, r, maxXMLRequestBytes)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := xml.Unmarshal(completeBody, &complete); err != nil && len(completeBody) > 0 {
|
||||
s3Error(w, http.StatusBadRequest, "MalformedXML", "invalid complete request")
|
||||
return
|
||||
}
|
||||
parts := complete.Parts
|
||||
if len(parts) == 0 {
|
||||
entries, _ := filepath.Glob(filepath.Join(s.multipartDir(uploadID), "part-*"))
|
||||
for _, entry := range entries {
|
||||
base := filepath.Base(entry)
|
||||
n, _ := strconv.Atoi(strings.TrimPrefix(base, "part-"))
|
||||
parts = append(parts, completedPart{PartNumber: n})
|
||||
}
|
||||
}
|
||||
if len(parts) > 10_000 {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidRequest", "too many parts")
|
||||
return
|
||||
}
|
||||
sortParts(parts)
|
||||
readers := make([]io.Reader, 0, len(parts))
|
||||
files := make([]*os.File, 0, len(parts))
|
||||
defer func() {
|
||||
for _, file := range files {
|
||||
_ = file.Close()
|
||||
}
|
||||
}()
|
||||
var totalSize int64
|
||||
for _, part := range parts {
|
||||
if part.PartNumber <= 0 || part.PartNumber > 10_000 {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidPart", "invalid part number")
|
||||
return
|
||||
}
|
||||
file, err := os.Open(filepath.Join(s.multipartDir(uploadID), fmt.Sprintf("part-%05d", part.PartNumber)))
|
||||
if err != nil {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidPart", "part missing")
|
||||
return
|
||||
}
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "part stat failed")
|
||||
return
|
||||
}
|
||||
totalSize += stat.Size()
|
||||
if s.cfg.MaxObjectSize > 0 && totalSize > s.cfg.MaxObjectSize {
|
||||
s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large")
|
||||
return
|
||||
}
|
||||
files = append(files, file)
|
||||
readers = append(readers, file)
|
||||
}
|
||||
release, ok := s.acceptWriteSize(w, totalSize, true)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
info, err := s.store.Put(key, io.MultiReader(readers...), meta.ContentType)
|
||||
if err != nil {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "complete failed")
|
||||
return
|
||||
}
|
||||
_ = os.RemoveAll(s.multipartDir(uploadID))
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = xml.NewEncoder(w).Encode(completeMultipartUploadResult{Key: strings.TrimPrefix(key, strings.SplitN(key, "/", 2)[0]+"/"), ETag: quoteETag(info.ETag)})
|
||||
if r.Header.Get("X-Magpie-Replication") != "1" {
|
||||
s.enqueueReplicationPut(key)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) s3AbortMultipartUpload(w http.ResponseWriter, r *http.Request, _ string) {
|
||||
uploadID := r.URL.Query().Get("uploadId")
|
||||
if !validUploadID(uploadID) {
|
||||
s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found")
|
||||
return
|
||||
}
|
||||
_ = os.RemoveAll(s.multipartDir(uploadID))
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (s *Server) s3CopyObject(w http.ResponseWriter, r *http.Request, destinationKey string) {
|
||||
if !signedHeaderIncludes(r, "x-amz-copy-source") {
|
||||
s3Error(w, http.StatusForbidden, "AccessDenied", "copy source header must be signed")
|
||||
return
|
||||
}
|
||||
sourceKey, err := copySourceKey(r.Header.Get("X-Amz-Copy-Source"))
|
||||
if err != nil {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid copy source")
|
||||
return
|
||||
}
|
||||
sourceBucket, _, _ := strings.Cut(sourceKey, "/")
|
||||
if !s.bucketAllowed(sourceBucket) {
|
||||
s3Error(w, http.StatusForbidden, "AccessDenied", "copy source bucket is not allowed")
|
||||
return
|
||||
}
|
||||
if len(s.cfg.S3Keys) > 0 {
|
||||
key, ok := s.requestAccessKey(r)
|
||||
if !ok || !key.can("read") || !key.can("write") {
|
||||
s3Error(w, http.StatusForbidden, "AccessDenied", "copy requires read and write permission")
|
||||
return
|
||||
}
|
||||
}
|
||||
obj, err := s.store.Open(sourceKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
s3Error(w, http.StatusNotFound, "NoSuchKey", "copy source not found")
|
||||
return
|
||||
}
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "copy source failed")
|
||||
return
|
||||
}
|
||||
defer obj.Close()
|
||||
if s.cfg.MaxObjectSize > 0 && obj.Info.Size > s.cfg.MaxObjectSize {
|
||||
s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large")
|
||||
return
|
||||
}
|
||||
release, ok := s.acceptWriteSize(w, obj.Info.Size, true)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
contentType := obj.Info.ContentType
|
||||
if override := r.Header.Get("Content-Type"); override != "" && strings.EqualFold(r.Header.Get("X-Amz-Metadata-Directive"), "REPLACE") {
|
||||
contentType = override
|
||||
}
|
||||
info, err := s.store.Put(destinationKey, obj, contentType)
|
||||
if err != nil {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "copy failed")
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = xml.NewEncoder(w).Encode(copyObjectResult{ETag: quoteETag(info.ETag), LastModified: info.ModTime.UTC().Format(time.RFC3339)})
|
||||
if r.Header.Get("X-Magpie-Replication") != "1" {
|
||||
s.enqueueReplicationPut(destinationKey)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) s3DeleteObjects(w http.ResponseWriter, r *http.Request, bucket string) {
|
||||
var request deleteObjectsRequest
|
||||
deleteBody, ok := s.s3BoundedBody(w, r, maxXMLRequestBytes)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := xml.Unmarshal(deleteBody, &request); err != nil {
|
||||
s3Error(w, http.StatusBadRequest, "MalformedXML", "invalid delete request")
|
||||
return
|
||||
}
|
||||
if len(request.Objects) > maxBatchDeleteItems {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidRequest", "too many objects in delete request")
|
||||
return
|
||||
}
|
||||
result := deleteObjectsResult{Deleted: make([]deletedObject, 0, len(request.Objects))}
|
||||
for _, object := range request.Objects {
|
||||
key := strings.TrimPrefix(object.Key, "/")
|
||||
storeKey := bucket + "/" + key
|
||||
if err := ValidateKey(storeKey); err != nil {
|
||||
result.Errors = append(result.Errors, deleteObjectError{Key: object.Key, Code: "InvalidObjectName", Message: err.Error()})
|
||||
continue
|
||||
}
|
||||
if err := s.store.Delete(storeKey); err != nil && !errors.Is(err, ErrNotFound) {
|
||||
result.Errors = append(result.Errors, deleteObjectError{Key: object.Key, Code: "InternalError", Message: "delete failed"})
|
||||
continue
|
||||
}
|
||||
result.Deleted = append(result.Deleted, deletedObject{Key: object.Key})
|
||||
if r.Header.Get("X-Magpie-Replication") != "1" {
|
||||
s.enqueueReplicationDelete(storeKey)
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = xml.NewEncoder(w).Encode(result)
|
||||
}
|
||||
|
||||
func (s *Server) s3BucketList(w http.ResponseWriter, r *http.Request) bool {
|
||||
if r.Method != http.MethodGet || strings.Trim(r.URL.Path, "/") == "" || strings.Contains(strings.Trim(r.URL.Path, "/"), "/") {
|
||||
return false
|
||||
}
|
||||
if r.URL.RawQuery == "" && r.URL.Query().Get("prefix") == "" && r.URL.Query().Get("list-type") == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
bucket := strings.Trim(r.URL.Path, "/")
|
||||
if err := ValidateKey(bucket); err != nil {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidBucketName", err.Error())
|
||||
return true
|
||||
}
|
||||
if !s.bucketAllowed(bucket) {
|
||||
s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed")
|
||||
return true
|
||||
}
|
||||
if !s.authorizedS3(r) {
|
||||
s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid")
|
||||
return true
|
||||
}
|
||||
|
||||
prefix := r.URL.Query().Get("prefix")
|
||||
storePrefix := bucket + "/"
|
||||
if prefix != "" {
|
||||
storePrefix = bucket + "/" + strings.TrimPrefix(prefix, "/")
|
||||
}
|
||||
maxKeys := parseMaxKeys(r.URL.Query().Get("max-keys"))
|
||||
continuationToken := r.URL.Query().Get("continuation-token")
|
||||
after, err := decodeContinuationToken(continuationToken)
|
||||
if err != nil {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid continuation token")
|
||||
return true
|
||||
}
|
||||
page, err := s.store.ListPage(storePrefix, after, maxKeys)
|
||||
if err != nil {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "list failed")
|
||||
return true
|
||||
}
|
||||
|
||||
contents := make([]s3ListContent, 0, len(page.Objects))
|
||||
for _, object := range page.Objects {
|
||||
key := strings.TrimPrefix(object.Key, bucket+"/")
|
||||
contents = append(contents, s3ListContent{
|
||||
Key: key,
|
||||
LastModified: object.ModTime.UTC().Format(time.RFC3339),
|
||||
ETag: quoteETag(object.ETag),
|
||||
Size: object.Size,
|
||||
StorageClass: "STANDARD",
|
||||
})
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = xml.NewEncoder(w).Encode(s3ListBucketResult{
|
||||
Name: bucket,
|
||||
Prefix: prefix,
|
||||
ContinuationToken: continuationToken,
|
||||
NextContinuationToken: encodeContinuationToken(page.NextAfter),
|
||||
KeyCount: len(contents),
|
||||
MaxKeys: maxKeys,
|
||||
IsTruncated: page.NextAfter != "",
|
||||
Contents: contents,
|
||||
})
|
||||
return true
|
||||
}
|
||||
|
||||
func encodeContinuationToken(key string) string {
|
||||
if key == "" {
|
||||
return ""
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(key))
|
||||
}
|
||||
|
||||
func decodeContinuationToken(token string) (string, error) {
|
||||
if token == "" {
|
||||
return "", nil
|
||||
}
|
||||
decoded, err := base64.RawURLEncoding.DecodeString(token)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
key := string(decoded)
|
||||
if err := ValidateKey(key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func copySourceKey(value string) (string, error) {
|
||||
value = strings.TrimPrefix(value, "/")
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("missing copy source")
|
||||
}
|
||||
if decoded, err := url.PathUnescape(value); err == nil {
|
||||
value = decoded
|
||||
}
|
||||
if err := ValidateKey(value); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (s *Server) s3BucketKey(path string) (string, string, bool) {
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
parts := strings.SplitN(path, "/", 2)
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", false
|
||||
}
|
||||
if err := ValidateKey(parts[0]); err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
if err := ValidateKey(parts[1]); err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[0], parts[1], true
|
||||
}
|
||||
|
||||
func (s *Server) s3Put(w http.ResponseWriter, r *http.Request, key string) {
|
||||
release, ok := s.acceptWrite(w, r, true)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
body, verifyPayload, ok := s.s3StreamingBody(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
store, ok := s.store.(*FileStore)
|
||||
if !ok {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "store failed")
|
||||
return
|
||||
}
|
||||
info, err := store.PutVerified(key, http.MaxBytesReader(w, body, s.cfg.MaxObjectSize), contentType, func() error {
|
||||
if !verifyPayload() {
|
||||
return errPayloadHashMismatch
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if maxBytesError(err) {
|
||||
s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, errPayloadHashMismatch) {
|
||||
s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
|
||||
return
|
||||
}
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "store failed")
|
||||
return
|
||||
}
|
||||
w.Header().Set("ETag", quoteETag(info.ETag))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if r.Header.Get("X-Magpie-Replication") != "1" {
|
||||
s.enqueueReplicationPut(key)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) s3Get(w http.ResponseWriter, r *http.Request, key string, publicRead bool) {
|
||||
obj, err := s.store.Open(key)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
s3Error(w, http.StatusNotFound, "NoSuchKey", "object not found")
|
||||
return
|
||||
}
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "read failed")
|
||||
return
|
||||
}
|
||||
defer obj.Close()
|
||||
|
||||
s3ObjectHeaders(w, obj.Info)
|
||||
if publicRead {
|
||||
hardenPublicRead(w, obj.Info.ContentType)
|
||||
} else if disposition := r.URL.Query().Get("response-content-disposition"); disposition != "" {
|
||||
w.Header().Set("Content-Disposition", disposition)
|
||||
}
|
||||
http.ServeContent(w, r, obj.Info.Key, obj.Info.ModTime, obj)
|
||||
}
|
||||
|
||||
func (s *Server) s3Head(w http.ResponseWriter, _ *http.Request, key string, publicRead bool) {
|
||||
info, err := s.store.Stat(key)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
s3ObjectHeaders(w, info)
|
||||
if publicRead {
|
||||
hardenPublicRead(w, info.ContentType)
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
// hardenPublicRead applies defense-in-depth headers to unauthenticated public
|
||||
// reads. Objects carry an uploader-supplied Content-Type, so a public prefix
|
||||
// could otherwise serve active content (HTML, SVG) that executes script in
|
||||
// Magpie's own origin. The CSP sandbox disables scripting even when a browser
|
||||
// navigates directly to such an object, and any content type that is not inert
|
||||
// inline media is forced to download rather than render. The attacker-influenced
|
||||
// response-content-disposition override is intentionally not honored here.
|
||||
func hardenPublicRead(w http.ResponseWriter, contentType string) {
|
||||
w.Header().Set("Content-Security-Policy", "default-src 'none'; sandbox")
|
||||
if !inlineSafeContentType(contentType) {
|
||||
w.Header().Set("Content-Disposition", "attachment")
|
||||
}
|
||||
}
|
||||
|
||||
// inlineSafeContentType reports whether a content type is inert raster/media
|
||||
// that browsers render without executing script. SVG is excluded because it can
|
||||
// carry script when loaded as a top-level document.
|
||||
func inlineSafeContentType(contentType string) bool {
|
||||
mediaType := contentType
|
||||
if i := strings.IndexByte(mediaType, ';'); i >= 0 {
|
||||
mediaType = mediaType[:i]
|
||||
}
|
||||
mediaType = strings.ToLower(strings.TrimSpace(mediaType))
|
||||
if mediaType == "image/svg+xml" || mediaType == "image/svg" {
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(mediaType, "image/") ||
|
||||
strings.HasPrefix(mediaType, "audio/") ||
|
||||
strings.HasPrefix(mediaType, "video/")
|
||||
}
|
||||
|
||||
func (s *Server) s3Delete(w http.ResponseWriter, r *http.Request, key string) {
|
||||
if err := s.store.Delete(key); err != nil && !errors.Is(err, ErrNotFound) {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "delete failed")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
if r.Header.Get("X-Magpie-Replication") != "1" {
|
||||
s.enqueueReplicationDelete(key)
|
||||
}
|
||||
}
|
||||
|
||||
func s3ObjectHeaders(w http.ResponseWriter, info ObjectInfo) {
|
||||
contentType := info.ContentType
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
w.Header().Set("ETag", quoteETag(info.ETag))
|
||||
w.Header().Set("Last-Modified", info.ModTime.UTC().Format(http.TimeFormat))
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
}
|
||||
|
||||
func quoteETag(etag string) string {
|
||||
if etag == "" {
|
||||
return "\"\""
|
||||
}
|
||||
if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
|
||||
return etag
|
||||
}
|
||||
return "\"" + etag + "\""
|
||||
}
|
||||
|
||||
type s3ErrorBody struct {
|
||||
XMLName xml.Name `xml:"Error"`
|
||||
Code string `xml:"Code"`
|
||||
Message string `xml:"Message"`
|
||||
}
|
||||
|
||||
func s3Error(w http.ResponseWriter, status int, code string, message string) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(status)
|
||||
_ = xml.NewEncoder(w).Encode(s3ErrorBody{Code: code, Message: message})
|
||||
}
|
||||
|
||||
func parseMaxKeys(value string) int {
|
||||
if value == "" {
|
||||
return 1000
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed <= 0 {
|
||||
return 1000
|
||||
}
|
||||
if parsed > 1000 {
|
||||
return 1000
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func randomID() (string, error) {
|
||||
buf := make([]byte, 18)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(buf), nil
|
||||
}
|
||||
|
||||
func md5Hash() hashHash {
|
||||
return md5.New()
|
||||
}
|
||||
|
||||
type hashHash interface {
|
||||
io.Writer
|
||||
Sum([]byte) []byte
|
||||
}
|
||||
|
||||
func (s *Server) multipartDir(uploadID string) string {
|
||||
if !validUploadID(uploadID) {
|
||||
return filepath.Join(s.store.(*FileStore).root, ".multipart", "invalid")
|
||||
}
|
||||
return filepath.Join(s.store.(*FileStore).root, ".multipart", uploadID)
|
||||
}
|
||||
|
||||
func validUploadID(uploadID string) bool {
|
||||
if uploadID == "" || len(uploadID) > 128 {
|
||||
return false
|
||||
}
|
||||
for _, ch := range uploadID {
|
||||
if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '-' || ch == '_' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Server) startMultipartCleanup(ctx context.Context) {
|
||||
store, ok := s.store.(*FileStore)
|
||||
if !ok || s.cfg.MultipartMaxAge <= 0 {
|
||||
return
|
||||
}
|
||||
if err := store.CleanupMultipart(s.cfg.MultipartMaxAge); err != nil {
|
||||
log.Printf("multipart cleanup error=%v", err)
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(time.Hour)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := store.CleanupMultipart(s.cfg.MultipartMaxAge); err != nil {
|
||||
log.Printf("multipart cleanup error=%v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (s *Server) s3BoundedBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, bool) {
|
||||
body, verifyPayload, ok := s.s3StreamingBody(w, r)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
data, err := io.ReadAll(http.MaxBytesReader(w, body, limit))
|
||||
if err != nil {
|
||||
if errors.Is(err, errPayloadHashMismatch) {
|
||||
s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
|
||||
return nil, false
|
||||
}
|
||||
s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "request body too large")
|
||||
return nil, false
|
||||
}
|
||||
if !verifyPayload() {
|
||||
s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
|
||||
return nil, false
|
||||
}
|
||||
return data, true
|
||||
}
|
||||
|
||||
func (s *Server) s3StreamingBody(w http.ResponseWriter, r *http.Request) (io.ReadCloser, func() bool, bool) {
|
||||
payloadHash := r.Header.Get("X-Amz-Content-Sha256")
|
||||
if payloadHash == "" || payloadHash == "UNSIGNED-PAYLOAD" {
|
||||
return r.Body, func() bool { return true }, true
|
||||
}
|
||||
if len(payloadHash) != sha256.Size*2 {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidDigest", "unsupported payload hash")
|
||||
return nil, nil, false
|
||||
}
|
||||
if _, err := hex.DecodeString(payloadHash); err != nil {
|
||||
s3Error(w, http.StatusBadRequest, "InvalidDigest", "invalid payload hash")
|
||||
return nil, nil, false
|
||||
}
|
||||
hash := sha256.New()
|
||||
verified := false
|
||||
return hashVerifyingReadCloser{reader: r.Body, hash: hash, expected: strings.ToLower(payloadHash)}, func() bool {
|
||||
if verified {
|
||||
return true
|
||||
}
|
||||
verified = hex.EncodeToString(hash.Sum(nil)) == strings.ToLower(payloadHash)
|
||||
return verified
|
||||
}, true
|
||||
}
|
||||
|
||||
func signedHeaderIncludes(r *http.Request, header string) bool {
|
||||
signedHeaders := r.URL.Query().Get("X-Amz-SignedHeaders")
|
||||
if signedHeaders == "" {
|
||||
auth := r.Header.Get("Authorization")
|
||||
if strings.HasPrefix(auth, awsAlgorithm+" ") {
|
||||
signedHeaders = parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" "))["SignedHeaders"]
|
||||
}
|
||||
}
|
||||
for _, signed := range strings.Split(signedHeaders, ";") {
|
||||
if strings.EqualFold(strings.TrimSpace(signed), header) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type hashVerifyingReadCloser struct {
|
||||
reader io.ReadCloser
|
||||
hash hashHash
|
||||
expected string
|
||||
}
|
||||
|
||||
func (r hashVerifyingReadCloser) Read(p []byte) (int, error) {
|
||||
n, err := r.reader.Read(p)
|
||||
if n > 0 {
|
||||
_, _ = r.hash.Write(p[:n])
|
||||
}
|
||||
if errors.Is(err, io.EOF) && hex.EncodeToString(r.hash.Sum(nil)) != r.expected {
|
||||
return n, errPayloadHashMismatch
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r hashVerifyingReadCloser) Close() error {
|
||||
return r.reader.Close()
|
||||
}
|
||||
|
||||
func (s *Server) multipartMeta(uploadID string) (multipartMeta, error) {
|
||||
data, err := os.ReadFile(filepath.Join(s.multipartDir(uploadID), "meta.xml"))
|
||||
if err != nil {
|
||||
return multipartMeta{}, err
|
||||
}
|
||||
var meta multipartMeta
|
||||
if err := xml.Unmarshal(data, &meta); err != nil {
|
||||
return multipartMeta{}, err
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
func sortParts(parts []completedPart) {
|
||||
sort.Slice(parts, func(i, j int) bool { return parts[i].PartNumber < parts[j].PartNumber })
|
||||
}
|
||||
|
||||
type s3ListBucketResult struct {
|
||||
XMLName xml.Name `xml:"ListBucketResult"`
|
||||
Xmlns string `xml:"xmlns,attr,omitempty"`
|
||||
Name string `xml:"Name"`
|
||||
Prefix string `xml:"Prefix"`
|
||||
ContinuationToken string `xml:"ContinuationToken,omitempty"`
|
||||
NextContinuationToken string `xml:"NextContinuationToken,omitempty"`
|
||||
KeyCount int `xml:"KeyCount"`
|
||||
MaxKeys int `xml:"MaxKeys"`
|
||||
IsTruncated bool `xml:"IsTruncated"`
|
||||
Contents []s3ListContent `xml:"Contents"`
|
||||
}
|
||||
|
||||
type s3ListContent struct {
|
||||
Key string `xml:"Key"`
|
||||
LastModified string `xml:"LastModified"`
|
||||
ETag string `xml:"ETag"`
|
||||
Size int64 `xml:"Size"`
|
||||
StorageClass string `xml:"StorageClass"`
|
||||
}
|
||||
|
||||
type multipartMeta struct {
|
||||
Key string `xml:"Key"`
|
||||
ContentType string `xml:"ContentType"`
|
||||
}
|
||||
|
||||
type createMultipartUploadResult struct {
|
||||
XMLName xml.Name `xml:"InitiateMultipartUploadResult"`
|
||||
UploadID string `xml:"UploadId"`
|
||||
}
|
||||
|
||||
type completeMultipartUpload struct {
|
||||
Parts []completedPart `xml:"Part"`
|
||||
}
|
||||
|
||||
type completedPart struct {
|
||||
PartNumber int `xml:"PartNumber"`
|
||||
ETag string `xml:"ETag"`
|
||||
}
|
||||
|
||||
type completeMultipartUploadResult struct {
|
||||
XMLName xml.Name `xml:"CompleteMultipartUploadResult"`
|
||||
Key string `xml:"Key"`
|
||||
ETag string `xml:"ETag"`
|
||||
}
|
||||
|
||||
type copyObjectResult struct {
|
||||
XMLName xml.Name `xml:"CopyObjectResult"`
|
||||
LastModified string `xml:"LastModified"`
|
||||
ETag string `xml:"ETag"`
|
||||
}
|
||||
|
||||
type deleteObjectsRequest struct {
|
||||
Objects []deleteObjectRequest `xml:"Object"`
|
||||
}
|
||||
|
||||
type deleteObjectRequest struct {
|
||||
Key string `xml:"Key"`
|
||||
}
|
||||
|
||||
type deleteObjectsResult struct {
|
||||
XMLName xml.Name `xml:"DeleteResult"`
|
||||
Deleted []deletedObject `xml:"Deleted"`
|
||||
Errors []deleteObjectError `xml:"Error,omitempty"`
|
||||
}
|
||||
|
||||
type deletedObject struct {
|
||||
Key string `xml:"Key"`
|
||||
}
|
||||
|
||||
type deleteObjectError struct {
|
||||
Key string `xml:"Key"`
|
||||
Code string `xml:"Code"`
|
||||
Message string `xml:"Message"`
|
||||
}
|
||||
312
s3auth.go
Normal file
312
s3auth.go
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const awsAlgorithm = "AWS4-HMAC-SHA256"
|
||||
|
||||
func (s *Server) authorizedS3(r *http.Request) bool {
|
||||
if len(s.cfg.S3Keys) == 0 {
|
||||
return s.authorized(r)
|
||||
}
|
||||
|
||||
if r.URL.Query().Get("X-Amz-Algorithm") != "" {
|
||||
return s.authorizedS3Presigned(r)
|
||||
}
|
||||
|
||||
return s.authorizedS3Header(r)
|
||||
}
|
||||
|
||||
func (s *Server) authorizedS3Presigned(r *http.Request) bool {
|
||||
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
||||
return false
|
||||
}
|
||||
query := r.URL.Query()
|
||||
if query.Get("X-Amz-Algorithm") != awsAlgorithm {
|
||||
return false
|
||||
}
|
||||
if query.Get("X-Amz-Signature") == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
credential := query.Get("X-Amz-Credential")
|
||||
accessKey, scope, ok := splitCredential(credential)
|
||||
key, ok := s.accessKey(accessKey)
|
||||
if !ok || !key.can(permissionForMethod(r.Method)) {
|
||||
return false
|
||||
}
|
||||
|
||||
amzDate := query.Get("X-Amz-Date")
|
||||
if !s.presignedTimeValid(amzDate, query.Get("X-Amz-Expires")) {
|
||||
return false
|
||||
}
|
||||
if !s.scopeAllowed(scope, amzDate) {
|
||||
return false
|
||||
}
|
||||
|
||||
signedHeaders := query.Get("X-Amz-SignedHeaders")
|
||||
canonicalRequest := canonicalRequest(r, signedHeaders, "UNSIGNED-PAYLOAD", true)
|
||||
expected := signature(key.Secret, amzDate, scope, canonicalRequest)
|
||||
return hmac.Equal([]byte(expected), []byte(query.Get("X-Amz-Signature")))
|
||||
}
|
||||
|
||||
func (s *Server) authorizedS3Header(r *http.Request) bool {
|
||||
return s.authorizedS3HeaderPermission(r, permissionForMethod(r.Method))
|
||||
}
|
||||
|
||||
func (s *Server) authorizedS3HeaderPermission(r *http.Request, permission string) bool {
|
||||
auth := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(auth, awsAlgorithm+" ") {
|
||||
return false
|
||||
}
|
||||
parts := parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" "))
|
||||
credential := parts["Credential"]
|
||||
signedHeaders := parts["SignedHeaders"]
|
||||
providedSignature := parts["Signature"]
|
||||
accessKey, scope, ok := splitCredential(credential)
|
||||
key, keyOK := s.accessKey(accessKey)
|
||||
if !ok || !keyOK || !key.can(permission) || signedHeaders == "" || providedSignature == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
amzDate := r.Header.Get("X-Amz-Date")
|
||||
if !headerTimeValid(amzDate) {
|
||||
return false
|
||||
}
|
||||
if !s.scopeAllowed(scope, amzDate) {
|
||||
return false
|
||||
}
|
||||
payloadHash := r.Header.Get("X-Amz-Content-Sha256")
|
||||
if payloadHash == "" {
|
||||
payloadHash = "UNSIGNED-PAYLOAD"
|
||||
}
|
||||
if permission == "write" && r.Header.Get("X-Magpie-Replication") != "1" && payloadHash == "UNSIGNED-PAYLOAD" && !s.cfg.AllowUnsignedLocal {
|
||||
return false
|
||||
}
|
||||
|
||||
canonicalRequest := canonicalRequest(r, signedHeaders, payloadHash, false)
|
||||
expected := signature(key.Secret, amzDate, scope, canonicalRequest)
|
||||
return hmac.Equal([]byte(expected), []byte(providedSignature))
|
||||
}
|
||||
|
||||
func (s *Server) accessKey(id string) (AccessKey, bool) {
|
||||
for _, key := range s.cfg.S3Keys {
|
||||
if subtle.ConstantTimeCompare([]byte(key.ID), []byte(id)) == 1 {
|
||||
return key, true
|
||||
}
|
||||
}
|
||||
return AccessKey{}, false
|
||||
}
|
||||
|
||||
func (s *Server) requestAccessKey(r *http.Request) (AccessKey, bool) {
|
||||
credential := r.URL.Query().Get("X-Amz-Credential")
|
||||
if credential == "" {
|
||||
auth := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(auth, awsAlgorithm+" ") {
|
||||
return AccessKey{}, false
|
||||
}
|
||||
credential = parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" "))["Credential"]
|
||||
}
|
||||
accessKey, _, ok := splitCredential(credential)
|
||||
if !ok {
|
||||
return AccessKey{}, false
|
||||
}
|
||||
return s.accessKey(accessKey)
|
||||
}
|
||||
|
||||
func (key AccessKey) can(permission string) bool {
|
||||
return key.Permissions["admin"] || key.Permissions[permission]
|
||||
}
|
||||
|
||||
func permissionForMethod(method string) string {
|
||||
switch method {
|
||||
case http.MethodGet, http.MethodHead:
|
||||
return "read"
|
||||
case http.MethodPut, http.MethodPost, http.MethodDelete:
|
||||
return "write"
|
||||
default:
|
||||
return "admin"
|
||||
}
|
||||
}
|
||||
|
||||
func signature(secret, amzDate, scope, canonicalRequest string) string {
|
||||
date := strings.Split(scope, "/")[0]
|
||||
stringToSign := strings.Join([]string{
|
||||
awsAlgorithm,
|
||||
amzDate,
|
||||
scope,
|
||||
hexSHA256(canonicalRequest),
|
||||
}, "\n")
|
||||
key := signingKey(secret, date, scope)
|
||||
sig := hmacSHA256(key, []byte(stringToSign))
|
||||
return hex.EncodeToString(sig)
|
||||
}
|
||||
|
||||
func signingKey(secret, date, scope string) []byte {
|
||||
parts := strings.Split(scope, "/")
|
||||
region := "auto"
|
||||
service := "s3"
|
||||
if len(parts) >= 3 {
|
||||
region = parts[1]
|
||||
service = parts[2]
|
||||
}
|
||||
kDate := hmacSHA256([]byte("AWS4"+secret), []byte(date))
|
||||
kRegion := hmacSHA256(kDate, []byte(region))
|
||||
kService := hmacSHA256(kRegion, []byte(service))
|
||||
return hmacSHA256(kService, []byte("aws4_request"))
|
||||
}
|
||||
|
||||
func (s *Server) scopeAllowed(scope string, amzDate string) bool {
|
||||
parts := strings.Split(scope, "/")
|
||||
if len(parts) != 4 {
|
||||
return false
|
||||
}
|
||||
if len(amzDate) < 8 || parts[0] != amzDate[:8] {
|
||||
return false
|
||||
}
|
||||
if parts[1] != s.cfg.S3Region || parts[2] != "s3" || parts[3] != "aws4_request" {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func canonicalRequest(r *http.Request, signedHeaders, payloadHash string, presigned bool) string {
|
||||
return strings.Join([]string{
|
||||
r.Method,
|
||||
canonicalURI(r.URL.Path),
|
||||
canonicalQuery(r.URL.Query(), presigned),
|
||||
canonicalHeaders(r, signedHeaders),
|
||||
signedHeaders,
|
||||
payloadHash,
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
func canonicalURI(path string) string {
|
||||
if path == "" {
|
||||
return "/"
|
||||
}
|
||||
segments := strings.Split(path, "/")
|
||||
for i, segment := range segments {
|
||||
segments[i] = awsEscape(segment)
|
||||
}
|
||||
return strings.Join(segments, "/")
|
||||
}
|
||||
|
||||
func canonicalQuery(values url.Values, presigned bool) string {
|
||||
type pair struct{ key, value string }
|
||||
pairs := make([]pair, 0)
|
||||
for key, vals := range values {
|
||||
if presigned && key == "X-Amz-Signature" {
|
||||
continue
|
||||
}
|
||||
for _, value := range vals {
|
||||
pairs = append(pairs, pair{awsEscape(key), awsEscape(value)})
|
||||
}
|
||||
}
|
||||
sort.Slice(pairs, func(i, j int) bool {
|
||||
if pairs[i].key == pairs[j].key {
|
||||
return pairs[i].value < pairs[j].value
|
||||
}
|
||||
return pairs[i].key < pairs[j].key
|
||||
})
|
||||
parts := make([]string, len(pairs))
|
||||
for i, pair := range pairs {
|
||||
parts[i] = pair.key + "=" + pair.value
|
||||
}
|
||||
return strings.Join(parts, "&")
|
||||
}
|
||||
|
||||
func canonicalHeaders(r *http.Request, signedHeaders string) string {
|
||||
headers := strings.Split(signedHeaders, ";")
|
||||
var b strings.Builder
|
||||
for _, header := range headers {
|
||||
name := strings.ToLower(strings.TrimSpace(header))
|
||||
value := ""
|
||||
if name == "host" {
|
||||
value = r.Host
|
||||
} else {
|
||||
value = strings.Join(r.Header.Values(name), ",")
|
||||
}
|
||||
b.WriteString(name)
|
||||
b.WriteString(":")
|
||||
b.WriteString(compressSpaces(value))
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func splitCredential(credential string) (string, string, bool) {
|
||||
parts := strings.SplitN(credential, "/", 2)
|
||||
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
|
||||
return "", "", false
|
||||
}
|
||||
scopeParts := strings.Split(parts[1], "/")
|
||||
if len(scopeParts) != 4 || scopeParts[3] != "aws4_request" {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[0], parts[1], true
|
||||
}
|
||||
|
||||
func parseAuthParams(value string) map[string]string {
|
||||
params := map[string]string{}
|
||||
for _, part := range strings.Split(value, ",") {
|
||||
keyValue := strings.SplitN(strings.TrimSpace(part), "=", 2)
|
||||
if len(keyValue) == 2 {
|
||||
params[keyValue[0]] = keyValue[1]
|
||||
}
|
||||
}
|
||||
return params
|
||||
}
|
||||
|
||||
func (s *Server) presignedTimeValid(amzDate, expiresRaw string) bool {
|
||||
created, err := time.Parse("20060102T150405Z", amzDate)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
expires, err := time.ParseDuration(expiresRaw + "s")
|
||||
if err != nil || expires <= 0 || expires > s.cfg.PresignedMaxExpiry {
|
||||
return false
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
return now.After(created.Add(-15*time.Minute)) && now.Before(created.Add(expires))
|
||||
}
|
||||
|
||||
func headerTimeValid(amzDate string) bool {
|
||||
created, err := time.Parse("20060102T150405Z", amzDate)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
return now.After(created.Add(-15*time.Minute)) && now.Before(created.Add(15*time.Minute))
|
||||
}
|
||||
|
||||
func awsEscape(value string) string {
|
||||
escaped := url.QueryEscape(value)
|
||||
escaped = strings.ReplaceAll(escaped, "+", "%20")
|
||||
escaped = strings.ReplaceAll(escaped, "%7E", "~")
|
||||
return escaped
|
||||
}
|
||||
|
||||
func compressSpaces(value string) string {
|
||||
return strings.Join(strings.Fields(strings.TrimSpace(value)), " ")
|
||||
}
|
||||
|
||||
func hexSHA256(value string) string {
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func hmacSHA256(key []byte, value []byte) []byte {
|
||||
mac := hmac.New(sha256.New, key)
|
||||
_, _ = mac.Write(value)
|
||||
return mac.Sum(nil)
|
||||
}
|
||||
530
server.go
Normal file
530
server.go
Normal file
|
|
@ -0,0 +1,530 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const objectPrefix = "/objects/"
|
||||
|
||||
type ObjectStore interface {
|
||||
Put(key string, body io.Reader, contentType string) (ObjectInfo, error)
|
||||
Open(key string) (*ObjectRead, error)
|
||||
Stat(key string) (ObjectInfo, error)
|
||||
Delete(key string) error
|
||||
DiskStatus() (DiskStatus, error)
|
||||
List(prefix string, maxKeys int) ([]ObjectInfo, error)
|
||||
ListPage(prefix string, after string, maxKeys int) (ListPage, error)
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
store ObjectStore
|
||||
cfg Config
|
||||
rateLimiter *RateLimiter
|
||||
metrics *Metrics
|
||||
writeMu sync.Mutex
|
||||
reservedBytes uint64
|
||||
}
|
||||
|
||||
func NewServer(store ObjectStore, cfg Config) http.Handler {
|
||||
return NewServerInstance(store, cfg).Handler()
|
||||
}
|
||||
|
||||
func NewServerInstance(store ObjectStore, cfg Config) *Server {
|
||||
cfg = cfg.withDefaults()
|
||||
return &Server{store: store, cfg: cfg, rateLimiter: NewRateLimiter(cfg.RateLimitPerMinute), metrics: &Metrics{}}
|
||||
}
|
||||
|
||||
func (s *Server) Handler() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/health", s.health)
|
||||
mux.HandleFunc("/metrics", s.metricsHandler)
|
||||
mux.HandleFunc(objectPrefix, s.object)
|
||||
mux.HandleFunc("/", s.s3Object)
|
||||
return s.recoverPanics(s.logRequests(s.rateLimit(mux)))
|
||||
}
|
||||
|
||||
func (s *Server) recoverPanics(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
log.Printf("panic method=%s path=%s remote=%s error=%v stack=%s", r.Method, safeLogPath(r.URL.Path), sanitizeLogValue(r.RemoteAddr), recovered, debug.Stack())
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "internal server error")
|
||||
}
|
||||
}()
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) metricsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.opsAuthorized(r) {
|
||||
writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
|
||||
fmt.Fprintf(w, "magpie_requests_total %d\n", atomic.LoadUint64(&s.metrics.Requests))
|
||||
fmt.Fprintf(w, "magpie_errors_total %d\n", atomic.LoadUint64(&s.metrics.Errors))
|
||||
fmt.Fprintf(w, "magpie_response_bytes_total %d\n", atomic.LoadUint64(&s.metrics.ResponseBytes))
|
||||
}
|
||||
|
||||
func (s *Server) opsAuthorized(r *http.Request) bool {
|
||||
return s.authorized(r) || s.authorizedS3HeaderPermission(r, "admin")
|
||||
}
|
||||
|
||||
func (cfg Config) withDefaults() Config {
|
||||
if len(cfg.S3Keys) == 0 && cfg.S3AccessKeyID != "" && cfg.S3SecretAccessKey != "" {
|
||||
cfg.S3Keys = []AccessKey{{ID: cfg.S3AccessKeyID, Secret: cfg.S3SecretAccessKey, Permissions: permissions("read,write,admin")}}
|
||||
}
|
||||
if cfg.S3Region == "" {
|
||||
cfg.S3Region = "auto"
|
||||
}
|
||||
if cfg.MaxObjectSize <= 0 {
|
||||
cfg.MaxObjectSize = defaultMaxObjectSize
|
||||
}
|
||||
if cfg.PresignedMaxExpiry <= 0 {
|
||||
cfg.PresignedMaxExpiry = defaultPresignedMaxExpiry
|
||||
}
|
||||
if cfg.MultipartMaxAge <= 0 {
|
||||
cfg.MultipartMaxAge = defaultMultipartMaxAge
|
||||
}
|
||||
if cfg.PublicPrefixes == nil {
|
||||
cfg.PublicPrefixes = map[string]bool{}
|
||||
}
|
||||
if cfg.ReplicationMaxJobs <= 0 {
|
||||
cfg.ReplicationMaxJobs = defaultReplicationMaxJobs
|
||||
}
|
||||
if cfg.ReplicationMaxAttempts <= 0 {
|
||||
cfg.ReplicationMaxAttempts = defaultReplicationMaxAttempts
|
||||
}
|
||||
if cfg.MaxRestoreBytes <= 0 {
|
||||
cfg.MaxRestoreBytes = defaultMaxRestoreBytes
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func (s *Server) health(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.opsAuthorized(r) {
|
||||
writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
|
||||
return
|
||||
}
|
||||
disk, err := s.store.DiskStatus()
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"status": "error", "error": err.Error()})
|
||||
return
|
||||
}
|
||||
status := "ok"
|
||||
code := http.StatusOK
|
||||
if disk.FreeBytes < s.cfg.MinFreeBytes {
|
||||
status = "low_disk"
|
||||
code = http.StatusServiceUnavailable
|
||||
}
|
||||
writeJSON(w, code, map[string]any{"status": status, "disk": disk})
|
||||
}
|
||||
|
||||
func (s *Server) object(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.authorized(r) {
|
||||
writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"})
|
||||
return
|
||||
}
|
||||
|
||||
key := strings.TrimPrefix(r.URL.Path, objectPrefix)
|
||||
if err := ValidateKey(key); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
bucket, _, _ := strings.Cut(key, "/")
|
||||
if !s.bucketAllowed(bucket) {
|
||||
writeJSON(w, http.StatusForbidden, map[string]string{"error": "bucket is not allowed"})
|
||||
return
|
||||
}
|
||||
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
s.put(w, r, key)
|
||||
case http.MethodGet:
|
||||
s.get(w, r, key)
|
||||
case http.MethodHead:
|
||||
s.head(w, r, key)
|
||||
case http.MethodDelete:
|
||||
s.delete(w, key)
|
||||
default:
|
||||
w.Header().Set("Allow", "PUT, GET, HEAD, DELETE")
|
||||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) put(w http.ResponseWriter, r *http.Request, key string) {
|
||||
release, ok := s.acceptWrite(w, r, false)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
info, err := s.store.Put(key, http.MaxBytesReader(w, r.Body, s.cfg.MaxObjectSize), contentType)
|
||||
if err != nil {
|
||||
if maxBytesError(err) {
|
||||
writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "object too large"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "store failed"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, info)
|
||||
}
|
||||
|
||||
func (s *Server) acceptWrite(w http.ResponseWriter, r *http.Request, s3 bool) (func(), bool) {
|
||||
size := r.ContentLength
|
||||
if r.ContentLength < 0 {
|
||||
if s.cfg.MaxObjectSize <= 0 {
|
||||
disk, err := s.store.DiskStatus()
|
||||
if err != nil || disk.FreeBytes < s.cfg.MinFreeBytes {
|
||||
if s3 {
|
||||
s3Error(w, http.StatusInsufficientStorage, "InsufficientStorage", "insufficient free disk space for unknown length write")
|
||||
} else {
|
||||
writeJSON(w, http.StatusInsufficientStorage, map[string]string{"error": "insufficient free disk space for unknown length write"})
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
return func() {}, true
|
||||
}
|
||||
size = s.cfg.MaxObjectSize
|
||||
}
|
||||
if s.cfg.MaxObjectSize > 0 && r.ContentLength > s.cfg.MaxObjectSize {
|
||||
if s3 {
|
||||
s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large")
|
||||
} else {
|
||||
writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{"error": "object too large"})
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
return s.acceptWriteSize(w, size, s3)
|
||||
}
|
||||
|
||||
func (s *Server) acceptWriteSize(w http.ResponseWriter, size int64, s3 bool) (func(), bool) {
|
||||
if size < 0 {
|
||||
return func() {}, true
|
||||
}
|
||||
disk, err := s.store.DiskStatus()
|
||||
if err != nil {
|
||||
if s3 {
|
||||
s3Error(w, http.StatusInternalServerError, "InternalError", "disk status unavailable")
|
||||
} else {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "disk status unavailable"})
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
needed := uint64(size)
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
reserved := s.reservedBytes
|
||||
available := uint64(0)
|
||||
if disk.FreeBytes > reserved {
|
||||
available = disk.FreeBytes - reserved
|
||||
}
|
||||
if available < s.cfg.MinFreeBytes || needed > available || available-needed < s.cfg.MinFreeBytes {
|
||||
if s3 {
|
||||
s3Error(w, http.StatusInsufficientStorage, "InsufficientStorage", "insufficient free disk space")
|
||||
} else {
|
||||
writeJSON(w, http.StatusInsufficientStorage, map[string]string{"error": "insufficient free disk space"})
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
s.reservedBytes += needed
|
||||
released := false
|
||||
return func() {
|
||||
s.writeMu.Lock()
|
||||
defer s.writeMu.Unlock()
|
||||
if released {
|
||||
return
|
||||
}
|
||||
released = true
|
||||
if s.reservedBytes >= needed {
|
||||
s.reservedBytes -= needed
|
||||
} else {
|
||||
s.reservedBytes = 0
|
||||
}
|
||||
}, true
|
||||
}
|
||||
|
||||
func maxBytesError(err error) bool {
|
||||
var maxErr *http.MaxBytesError
|
||||
return errors.As(err, &maxErr)
|
||||
}
|
||||
|
||||
func (s *Server) get(w http.ResponseWriter, r *http.Request, key string) {
|
||||
obj, err := s.store.Open(key)
|
||||
if err != nil {
|
||||
status := http.StatusInternalServerError
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
status = http.StatusNotFound
|
||||
}
|
||||
writeJSON(w, status, map[string]string{"error": "not found"})
|
||||
return
|
||||
}
|
||||
defer obj.Close()
|
||||
|
||||
contentType := obj.Info.ContentType
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("X-Object-Key", obj.Info.Key)
|
||||
w.Header().Set("X-Object-Size", fmt.Sprintf("%d", obj.Info.Size))
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
http.ServeContent(w, r, obj.Info.Key, obj.Info.ModTime, obj)
|
||||
}
|
||||
|
||||
func (s *Server) head(w http.ResponseWriter, r *http.Request, key string) {
|
||||
info, err := s.store.Stat(key)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := info.ContentType
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
|
||||
w.Header().Set("X-Object-Key", info.Key)
|
||||
w.Header().Set("X-Object-Size", fmt.Sprintf("%d", info.Size))
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = r.Body.Close()
|
||||
}
|
||||
|
||||
func (s *Server) delete(w http.ResponseWriter, key string) {
|
||||
err := s.store.Delete(key)
|
||||
if err != nil && !errors.Is(err, ErrNotFound) {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "delete failed"})
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (s *Server) authorized(r *http.Request) bool {
|
||||
const prefix = "Bearer "
|
||||
value := r.Header.Get("Authorization")
|
||||
if !strings.HasPrefix(value, prefix) {
|
||||
return false
|
||||
}
|
||||
provided := strings.TrimPrefix(value, prefix)
|
||||
return s.cfg.AuthToken != "" && subtle.ConstantTimeCompare([]byte(provided), []byte(s.cfg.AuthToken)) == 1
|
||||
}
|
||||
|
||||
func (s *Server) bucketAllowed(bucket string) bool {
|
||||
return len(s.cfg.AllowedBuckets) == 0 || s.cfg.AllowedBuckets[bucket]
|
||||
}
|
||||
|
||||
func (s *Server) publicReadAllowed(key string) bool {
|
||||
for prefix := range s.cfg.PublicPrefixes {
|
||||
prefix = strings.TrimSuffix(prefix, "/")
|
||||
if !strings.Contains(prefix, "/") {
|
||||
continue
|
||||
}
|
||||
if key == prefix || strings.HasPrefix(key, prefix+"/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Server) replicationAllowed(r *http.Request) bool {
|
||||
if r.Header.Get("X-Magpie-Replication") != "1" {
|
||||
return true
|
||||
}
|
||||
if s.cfg.ReplicationSecret == "" {
|
||||
return false
|
||||
}
|
||||
return subtle.ConstantTimeCompare([]byte(r.Header.Get("X-Magpie-Replication-Secret")), []byte(s.cfg.ReplicationSecret)) == 1
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, body any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(body)
|
||||
}
|
||||
|
||||
type statusWriter struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
bytes int
|
||||
}
|
||||
|
||||
func (w *statusWriter) WriteHeader(status int) {
|
||||
w.status = status
|
||||
w.ResponseWriter.WriteHeader(status)
|
||||
}
|
||||
|
||||
func (w *statusWriter) Write(b []byte) (int, error) {
|
||||
if w.status == 0 {
|
||||
w.status = http.StatusOK
|
||||
}
|
||||
n, err := w.ResponseWriter.Write(b)
|
||||
w.bytes += n
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (s *Server) logRequests(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
recorder := &statusWriter{ResponseWriter: w}
|
||||
next.ServeHTTP(recorder, r)
|
||||
status := recorder.status
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
atomic.AddUint64(&s.metrics.Requests, 1)
|
||||
if recorder.bytes > 0 {
|
||||
atomic.AddUint64(&s.metrics.ResponseBytes, uint64(recorder.bytes))
|
||||
}
|
||||
if status >= 400 {
|
||||
atomic.AddUint64(&s.metrics.Errors, 1)
|
||||
}
|
||||
log.Printf(
|
||||
"method=%s path=%s status=%d bytes=%d duration_ms=%d remote=%s",
|
||||
r.Method,
|
||||
safeLogPath(r.URL.Path),
|
||||
status,
|
||||
recorder.bytes,
|
||||
time.Since(start).Milliseconds(),
|
||||
sanitizeLogValue(r.RemoteAddr),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) rateLimit(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.rateLimiter.Allow(s.rateLimitKey(r)) {
|
||||
writeJSON(w, http.StatusTooManyRequests, map[string]string{"error": "rate limited"})
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) rateLimitKey(r *http.Request) string {
|
||||
if s.cfg.TrustProxyHeaders {
|
||||
if forwarded := r.Header.Get("X-Forwarded-For"); forwarded != "" {
|
||||
candidate := strings.TrimSpace(strings.Split(forwarded, ",")[0])
|
||||
if net.ParseIP(candidate) != nil {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
if realIP := strings.TrimSpace(r.Header.Get("X-Real-IP")); net.ParseIP(realIP) != nil {
|
||||
return realIP
|
||||
}
|
||||
}
|
||||
return r.RemoteAddr
|
||||
}
|
||||
|
||||
type Metrics struct {
|
||||
Requests uint64
|
||||
Errors uint64
|
||||
ResponseBytes uint64
|
||||
}
|
||||
|
||||
type RateLimiter struct {
|
||||
limit int
|
||||
mu sync.Mutex
|
||||
buckets map[string]rateBucket
|
||||
}
|
||||
|
||||
type rateBucket struct {
|
||||
minute int64
|
||||
count int
|
||||
}
|
||||
|
||||
func NewRateLimiter(limit int) *RateLimiter {
|
||||
return &RateLimiter{limit: limit, buckets: map[string]rateBucket{}}
|
||||
}
|
||||
|
||||
func (r *RateLimiter) Allow(remote string) bool {
|
||||
if r.limit <= 0 {
|
||||
return true
|
||||
}
|
||||
ip, _, err := net.SplitHostPort(remote)
|
||||
if err != nil {
|
||||
ip = remote
|
||||
}
|
||||
minute := time.Now().Unix() / 60
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for candidate, bucket := range r.buckets {
|
||||
if bucket.minute < minute-1 {
|
||||
delete(r.buckets, candidate)
|
||||
}
|
||||
}
|
||||
bucket := r.buckets[ip]
|
||||
if bucket.minute != minute {
|
||||
bucket = rateBucket{minute: minute}
|
||||
}
|
||||
bucket.count++
|
||||
r.buckets[ip] = bucket
|
||||
return bucket.count <= r.limit
|
||||
}
|
||||
|
||||
func safeLogPath(path string) string {
|
||||
if len(path) <= 256 {
|
||||
return sanitizeLogValue(path)
|
||||
}
|
||||
return sanitizeLogValue(path[:256]) + "..."
|
||||
}
|
||||
|
||||
func sanitizeLogValue(value string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r < 0x20 || r == 0x7f {
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, value)
|
||||
}
|
||||
|
||||
type ObjectInfo struct {
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size"`
|
||||
ContentType string `json:"content_type"`
|
||||
ModTime time.Time `json:"mod_time"`
|
||||
ETag string `json:"etag"`
|
||||
}
|
||||
|
||||
type DiskStatus struct {
|
||||
TotalBytes uint64 `json:"total_bytes"`
|
||||
FreeBytes uint64 `json:"free_bytes"`
|
||||
UsedBytes uint64 `json:"used_bytes"`
|
||||
}
|
||||
|
||||
type StoreStats struct {
|
||||
ObjectCount int64 `json:"object_count"`
|
||||
TotalBytes int64 `json:"total_bytes"`
|
||||
Disk DiskStatus `json:"disk"`
|
||||
ReplicationQueue ReplicationQueueStats `json:"replication_queue"`
|
||||
}
|
||||
|
||||
type ScrubReport struct {
|
||||
CheckedObjects int `json:"checked_objects"`
|
||||
MissingKeys []string `json:"missing_keys"`
|
||||
CorruptKeys []string `json:"corrupt_keys"`
|
||||
OrphanKeys []string `json:"orphan_keys"`
|
||||
}
|
||||
|
||||
type ListPage struct {
|
||||
Objects []ObjectInfo
|
||||
NextAfter string
|
||||
}
|
||||
1737
server_test.go
Normal file
1737
server_test.go
Normal file
File diff suppressed because it is too large
Load diff
563
store.go
Normal file
563
store.go
Normal file
|
|
@ -0,0 +1,563 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
var ErrNotFound = errors.New("object not found")
|
||||
|
||||
type FileStore struct {
|
||||
root string
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
type ObjectRead struct {
|
||||
*os.File
|
||||
Info ObjectInfo
|
||||
}
|
||||
|
||||
func NewFileStore(root string) (*FileStore, error) {
|
||||
abs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.MkdirAll(abs, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db, err := sql.Open("sqlite", filepath.Join(abs, ".magpie.db"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store := &FileStore{root: abs, db: db}
|
||||
if err := store.initMetadata(); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) Put(key string, body io.Reader, contentType string) (ObjectInfo, error) {
|
||||
return s.PutVerified(key, body, contentType, nil)
|
||||
}
|
||||
|
||||
func (s *FileStore) PutVerified(key string, body io.Reader, contentType string, verify func() error) (ObjectInfo, error) {
|
||||
if err := ValidateKey(key); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
path, err := s.pathFor(key)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp(filepath.Dir(path), ".upload-*")
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
defer os.Remove(tmpName)
|
||||
|
||||
hash := md5.New()
|
||||
size, copyErr := io.Copy(io.MultiWriter(tmp, hash), body)
|
||||
closeErr := tmp.Close()
|
||||
if copyErr != nil {
|
||||
return ObjectInfo{}, copyErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return ObjectInfo{}, closeErr
|
||||
}
|
||||
if verify != nil {
|
||||
if err := verify(); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
}
|
||||
|
||||
info := ObjectInfo{
|
||||
Key: key,
|
||||
Size: size,
|
||||
ContentType: contentType,
|
||||
ModTime: time.Now().UTC(),
|
||||
ETag: fmt.Sprintf("%x", hash.Sum(nil)),
|
||||
}
|
||||
if err := s.writeMeta(key, info); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if err := s.upsertMetadata(info); err != nil {
|
||||
_ = os.Remove(path + ".meta.json")
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
if err := os.Rename(tmpName, path); err != nil {
|
||||
_ = os.Remove(path + ".meta.json")
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) Open(key string) (*ObjectRead, error) {
|
||||
if err := ValidateKey(key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
path, err := s.pathFor(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
info, err := s.Stat(key)
|
||||
if err != nil {
|
||||
_ = file.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &ObjectRead{File: file, Info: info}, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) Stat(key string) (ObjectInfo, error) {
|
||||
if err := ValidateKey(key); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
path, err := s.pathFor(key)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
stat, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return ObjectInfo{}, ErrNotFound
|
||||
}
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
if info, err := s.getMetadata(key); err == nil {
|
||||
info.Size = stat.Size()
|
||||
return info, nil
|
||||
}
|
||||
|
||||
info := ObjectInfo{Key: key, Size: stat.Size(), ModTime: stat.ModTime().UTC()}
|
||||
if meta, err := s.readMeta(key); err == nil {
|
||||
info.ContentType = meta.ContentType
|
||||
info.ETag = meta.ETag
|
||||
if !meta.ModTime.IsZero() {
|
||||
info.ModTime = meta.ModTime
|
||||
}
|
||||
}
|
||||
_ = s.upsertMetadata(info)
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) Delete(key string) error {
|
||||
if err := ValidateKey(key); err != nil {
|
||||
return err
|
||||
}
|
||||
path, err := s.pathFor(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(path + ".meta.json")
|
||||
_, _ = s.db.Exec(`DELETE FROM objects WHERE key = ?`, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *FileStore) List(prefix string, maxKeys int) ([]ObjectInfo, error) {
|
||||
page, err := s.ListPage(prefix, "", maxKeys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return page.Objects, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) ListPage(prefix string, after string, maxKeys int) (ListPage, error) {
|
||||
validatePrefix := strings.TrimSuffix(prefix, "/")
|
||||
if validatePrefix != "" {
|
||||
if err := ValidateKey(validatePrefix); err != nil {
|
||||
return ListPage{}, err
|
||||
}
|
||||
}
|
||||
if after != "" {
|
||||
if err := ValidateKey(after); err != nil {
|
||||
return ListPage{}, err
|
||||
}
|
||||
}
|
||||
if maxKeys <= 0 || maxKeys > 1000 {
|
||||
maxKeys = 1000
|
||||
}
|
||||
|
||||
return s.listMetadata(prefix, after, maxKeys)
|
||||
}
|
||||
|
||||
func (s *FileStore) Reindex() error {
|
||||
return filepath.WalkDir(s.root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if entry.Name() == ".magpie.db" || strings.HasPrefix(entry.Name(), ".magpie.db-") || strings.HasSuffix(entry.Name(), ".meta.json") {
|
||||
return nil
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(s.root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := filepath.ToSlash(rel)
|
||||
if err := ValidateKey(key); err != nil {
|
||||
return nil
|
||||
}
|
||||
info, err := s.statFromFilesystem(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.upsertMetadata(info)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *FileStore) Stats() (StoreStats, error) {
|
||||
var stats StoreStats
|
||||
if err := s.db.QueryRow(`SELECT COUNT(*), COALESCE(SUM(size), 0) FROM objects`).Scan(&stats.ObjectCount, &stats.TotalBytes); err != nil {
|
||||
return StoreStats{}, err
|
||||
}
|
||||
queue, err := s.ReplicationQueueStats()
|
||||
if err != nil {
|
||||
return StoreStats{}, err
|
||||
}
|
||||
stats.ReplicationQueue = queue
|
||||
disk, err := s.DiskStatus()
|
||||
if err != nil {
|
||||
return StoreStats{}, err
|
||||
}
|
||||
stats.Disk = disk
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) Scrub() (ScrubReport, error) {
|
||||
report := ScrubReport{}
|
||||
rows, err := s.db.Query(`SELECT key, size, etag FROM objects ORDER BY key ASC`)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
metadataKeys := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var key string
|
||||
var size int64
|
||||
var etag string
|
||||
if err := rows.Scan(&key, &size, &etag); err != nil {
|
||||
return report, err
|
||||
}
|
||||
metadataKeys[key] = true
|
||||
report.CheckedObjects++
|
||||
path, err := s.pathFor(key)
|
||||
if err != nil {
|
||||
report.CorruptKeys = append(report.CorruptKeys, key)
|
||||
continue
|
||||
}
|
||||
stat, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
report.MissingKeys = append(report.MissingKeys, key)
|
||||
continue
|
||||
}
|
||||
return report, err
|
||||
}
|
||||
if stat.Size() != size {
|
||||
report.CorruptKeys = append(report.CorruptKeys, key)
|
||||
continue
|
||||
}
|
||||
computed, err := fileETag(path)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
if etag != "" && computed != etag {
|
||||
report.CorruptKeys = append(report.CorruptKeys, key)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
if err := filepath.WalkDir(s.root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() || entry.Name() == ".magpie.db" || strings.HasPrefix(entry.Name(), ".magpie.db-") || strings.HasSuffix(entry.Name(), ".meta.json") || strings.Contains(path, string(os.PathSeparator)+".multipart"+string(os.PathSeparator)) {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(s.root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
key := filepath.ToSlash(rel)
|
||||
if !metadataKeys[key] {
|
||||
report.OrphanKeys = append(report.OrphanKeys, key)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return report, err
|
||||
}
|
||||
sort.Strings(report.MissingKeys)
|
||||
sort.Strings(report.CorruptKeys)
|
||||
sort.Strings(report.OrphanKeys)
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) CleanupMultipart(maxAge time.Duration) error {
|
||||
if maxAge <= 0 {
|
||||
return nil
|
||||
}
|
||||
root := filepath.Join(s.root, ".multipart")
|
||||
entries, err := os.ReadDir(root)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cutoff := time.Now().Add(-maxAge)
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.ModTime().Before(cutoff) {
|
||||
if err := os.RemoveAll(filepath.Join(root, entry.Name())); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *FileStore) pathFor(key string) (string, error) {
|
||||
clean := filepath.Clean(filepath.FromSlash(key))
|
||||
path := filepath.Join(s.root, clean)
|
||||
if path != s.root && !strings.HasPrefix(path, s.root+string(os.PathSeparator)) {
|
||||
return "", fmt.Errorf("invalid object key")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) writeMeta(key string, info ObjectInfo) error {
|
||||
path, err := s.pathFor(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path+".meta.json", data, 0o600)
|
||||
}
|
||||
|
||||
func (s *FileStore) readMeta(key string) (ObjectInfo, error) {
|
||||
path, err := s.pathFor(key)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
data, err := os.ReadFile(path + ".meta.json")
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
var info ObjectInfo
|
||||
if err := json.Unmarshal(data, &info); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) statFromFilesystem(key string) (ObjectInfo, error) {
|
||||
path, err := s.pathFor(key)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
stat, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return ObjectInfo{}, ErrNotFound
|
||||
}
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
info := ObjectInfo{Key: key, Size: stat.Size(), ModTime: stat.ModTime().UTC()}
|
||||
if meta, err := s.readMeta(key); err == nil {
|
||||
info.ContentType = meta.ContentType
|
||||
info.ETag = meta.ETag
|
||||
if !meta.ModTime.IsZero() {
|
||||
info.ModTime = meta.ModTime
|
||||
}
|
||||
}
|
||||
if info.ETag == "" {
|
||||
etag, err := fileETag(path)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
info.ETag = etag
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func fileETag(path string) (string, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
hash := md5.New()
|
||||
if _, err := io.Copy(hash, file); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%x", hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func (s *FileStore) initMetadata() error {
|
||||
_, err := s.db.Exec(`
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
CREATE TABLE IF NOT EXISTS objects (
|
||||
key TEXT PRIMARY KEY,
|
||||
size INTEGER NOT NULL,
|
||||
content_type TEXT NOT NULL DEFAULT '',
|
||||
etag TEXT NOT NULL DEFAULT '',
|
||||
mod_time TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS objects_key_idx ON objects(key);
|
||||
CREATE TABLE IF NOT EXISTS replication_jobs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
peer TEXT NOT NULL,
|
||||
op TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt_at TEXT NOT NULL,
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS replication_jobs_due_idx ON replication_jobs(next_attempt_at, id);
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *FileStore) upsertMetadata(info ObjectInfo) error {
|
||||
_, err := s.db.Exec(`
|
||||
INSERT INTO objects (key, size, content_type, etag, mod_time)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
size = excluded.size,
|
||||
content_type = excluded.content_type,
|
||||
etag = excluded.etag,
|
||||
mod_time = excluded.mod_time
|
||||
`, info.Key, info.Size, info.ContentType, info.ETag, info.ModTime.UTC().Format(time.RFC3339Nano))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *FileStore) getMetadata(key string) (ObjectInfo, error) {
|
||||
var info ObjectInfo
|
||||
var modTime string
|
||||
err := s.db.QueryRow(`SELECT key, size, content_type, etag, mod_time FROM objects WHERE key = ?`, key).
|
||||
Scan(&info.Key, &info.Size, &info.ContentType, &info.ETag, &modTime)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339Nano, modTime)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
info.ModTime = parsed
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (s *FileStore) listMetadata(prefix string, after string, maxKeys int) (ListPage, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT key, size, content_type, etag, mod_time
|
||||
FROM objects
|
||||
WHERE key LIKE ? ESCAPE '\' AND key > ?
|
||||
ORDER BY key ASC
|
||||
LIMIT ?
|
||||
`, escapeLike(prefix)+"%", after, maxKeys+1)
|
||||
if err != nil {
|
||||
return ListPage{}, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
objects := make([]ObjectInfo, 0)
|
||||
for rows.Next() {
|
||||
var info ObjectInfo
|
||||
var modTime string
|
||||
if err := rows.Scan(&info.Key, &info.Size, &info.ContentType, &info.ETag, &modTime); err != nil {
|
||||
return ListPage{}, err
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339Nano, modTime)
|
||||
if err != nil {
|
||||
return ListPage{}, err
|
||||
}
|
||||
info.ModTime = parsed
|
||||
objects = append(objects, info)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return ListPage{}, err
|
||||
}
|
||||
|
||||
page := ListPage{Objects: objects}
|
||||
if len(objects) > maxKeys {
|
||||
page.NextAfter = objects[maxKeys-1].Key
|
||||
page.Objects = objects[:maxKeys]
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func escapeLike(value string) string {
|
||||
value = strings.ReplaceAll(value, `\`, `\\`)
|
||||
value = strings.ReplaceAll(value, `%`, `\%`)
|
||||
value = strings.ReplaceAll(value, `_`, `\_`)
|
||||
return value
|
||||
}
|
||||
|
||||
func ValidateKey(key string) error {
|
||||
if key == "" {
|
||||
return fmt.Errorf("object key is required")
|
||||
}
|
||||
if len(key) > 1024 {
|
||||
return fmt.Errorf("object key is too long")
|
||||
}
|
||||
if strings.HasPrefix(key, "/") || strings.HasPrefix(key, "\\") {
|
||||
return fmt.Errorf("object key cannot be absolute")
|
||||
}
|
||||
if strings.Contains(key, "\x00") || strings.Contains(key, "\\") || strings.Contains(key, "//") {
|
||||
return fmt.Errorf("object key contains invalid characters")
|
||||
}
|
||||
for _, part := range strings.Split(key, "/") {
|
||||
if part == "" || part == "." || part == ".." {
|
||||
return fmt.Errorf("object key contains invalid path segment")
|
||||
}
|
||||
if part == ".multipart" || part == ".magpie.db" || strings.HasPrefix(part, ".magpie.db-") || strings.HasSuffix(part, ".meta.json") {
|
||||
return fmt.Errorf("object key uses reserved internal name")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
29
version.go
Normal file
29
version.go
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
package main
|
||||
|
||||
import "runtime"
|
||||
|
||||
var (
|
||||
version = "dev"
|
||||
commit = "unknown"
|
||||
date = "unknown"
|
||||
)
|
||||
|
||||
type VersionInfo struct {
|
||||
Version string `json:"version"`
|
||||
Commit string `json:"commit"`
|
||||
Date string `json:"date"`
|
||||
Go string `json:"go"`
|
||||
OS string `json:"os"`
|
||||
Arch string `json:"arch"`
|
||||
}
|
||||
|
||||
func versionInfo() VersionInfo {
|
||||
return VersionInfo{
|
||||
Version: version,
|
||||
Commit: commit,
|
||||
Date: date,
|
||||
Go: runtime.Version(),
|
||||
OS: runtime.GOOS,
|
||||
Arch: runtime.GOARCH,
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue