magpie/s3.go

package main

import (
	"context"
	"crypto/md5"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	maxXMLRequestBytes  = 1 << 20
	maxBatchDeleteItems = 1000
)

var errPayloadHashMismatch = errors.New("payload hash mismatch")

func (s *Server) s3Object(w http.ResponseWriter, r *http.Request) {
	if s.s3BucketList(w, r) {
		return
	}
	if s.s3BucketPost(w, r) {
		return
	}

	bucket, key, ok := s.s3BucketKey(r.URL.Path)
	if !ok {
		s3Error(w, http.StatusNotFound, "NoSuchBucket", "bucket not found")
		return
	}
	if !s.bucketAllowed(bucket) {
		s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed")
		return
	}

	storeKey := bucket + "/" + key
	if err := ValidateKey(storeKey); err != nil {
		s3Error(w, http.StatusBadRequest, "InvalidObjectName", err.Error())
		return
	}
	publicRead := (r.Method == http.MethodGet || r.Method == http.MethodHead) && s.publicReadAllowed(storeKey)
	if !publicRead && !s.authorizedS3(r) {
		s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid")
		return
	}
	if !s.replicationAllowed(r) {
		s3Error(w, http.StatusForbidden, "AccessDenied", "replication secret is invalid")
		return
	}

	switch r.Method {
	case http.MethodPost:
		if _, ok := r.URL.Query()["uploads"]; ok {
			s.s3CreateMultipartUpload(w, r, storeKey)
		} else if r.URL.Query().Get("uploadId") != "" {
			s.s3CompleteMultipartUpload(w, r, storeKey)
		} else if _, ok := r.URL.Query()["delete"]; ok {
			s.s3DeleteObjects(w, r, bucket)
		} else {
			s3Error(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed")
		}
	case http.MethodPut:
		if r.URL.Query().Get("uploadId") != "" && r.URL.Query().Get("partNumber") != "" {
			s.s3UploadPart(w, r, storeKey)
		} else if r.Header.Get("X-Amz-Copy-Source") != "" {
			s.s3CopyObject(w, r, storeKey)
		} else {
			s.s3Put(w, r, storeKey)
		}
	case http.MethodGet:
		s.s3Get(w, r, storeKey, publicRead)
	case http.MethodHead:
		s.s3Head(w, r, storeKey, publicRead)
	case http.MethodDelete:
		if r.URL.Query().Get("uploadId") != "" {
			s.s3AbortMultipartUpload(w, r, storeKey)
		} else {
			s.s3Delete(w, r, storeKey)
		}
	default:
		w.Header().Set("Allow", "PUT, GET, HEAD, DELETE")
		s3Error(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "method not allowed")
	}
}

func (s *Server) s3BucketPost(w http.ResponseWriter, r *http.Request) bool {
	path := strings.Trim(r.URL.Path, "/")
	if r.Method != http.MethodPost || path == "" || strings.Contains(path, "/") {
		return false
	}
	if _, ok := r.URL.Query()["delete"]; !ok {
		return false
	}
	if err := ValidateKey(path); err != nil {
		s3Error(w, http.StatusBadRequest, "InvalidBucketName", err.Error())
		return true
	}
	if !s.bucketAllowed(path) {
		s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed")
		return true
	}
	if !s.authorizedS3(r) {
		s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid")
		return true
	}
	if !s.replicationAllowed(r) {
		s3Error(w, http.StatusForbidden, "AccessDenied", "replication secret is invalid")
		return true
	}
	s.s3DeleteObjects(w, r, path)
	return true
}

func (s *Server) s3CreateMultipartUpload(w http.ResponseWriter, r *http.Request, key string) {
	uploadID, err := randomID()
	if err != nil {
		s3Error(w, http.StatusInternalServerError, "InternalError", "upload id failed")
		return
	}
	dir := s.multipartDir(uploadID)
	if err := os.MkdirAll(dir, 0o700); err != nil {
		s3Error(w, http.StatusInternalServerError, "InternalError", "upload init failed")
		return
	}
	meta := multipartMeta{Key: key, ContentType: r.Header.Get("Content-Type")}
	data, _ := xml.Marshal(meta)
	if err := os.WriteFile(filepath.Join(dir, "meta.xml"), data, 0o600); err != nil {
		s3Error(w, http.StatusInternalServerError, "InternalError", "upload init failed")
		return
	}
	w.Header().Set("Content-Type", "application/xml")
	w.WriteHeader(http.StatusOK)
	_ = xml.NewEncoder(w).Encode(createMultipartUploadResult{UploadID: uploadID})
}

func (s *Server) s3UploadPart(w http.ResponseWriter, r *http.Request, key string) {
	release, ok := s.acceptWrite(w, r, true)
	if !ok {
		return
	}
	defer release()
	uploadID := r.URL.Query().Get("uploadId")
	partNumber, err := strconv.Atoi(r.URL.Query().Get("partNumber"))
	if err != nil || partNumber <= 0 || partNumber > 10_000 {
		s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid part number")
		return
	}
	meta, err := s.multipartMeta(uploadID)
	if err != nil || meta.Key != key {
		s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found")
		return
	}
	dir := s.multipartDir(uploadID)
	if !s.acceptMultipartPartSize(w, dir, r.ContentLength) {
		return
	}
	partPath := filepath.Join(dir, fmt.Sprintf("part-%05d", partNumber))
	body, verifyPayload, ok := s.s3StreamingBody(w, r)
	if !ok {
		return
	}
	tmp, err := os.CreateTemp(dir, ".part-*")
	if err != nil {
		s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed")
		return
	}
	hash := md5Hash()
	_, copyErr := io.Copy(io.MultiWriter(tmp, hash), http.MaxBytesReader(w, body, s.cfg.MaxObjectSize))
	closeErr := tmp.Close()
	if copyErr != nil || closeErr != nil {
		_ = os.Remove(tmp.Name())
		if errors.Is(copyErr, errPayloadHashMismatch) {
			s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
			return
		}
		s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed")
		return
	}
	if !verifyPayload() {
		_ = os.Remove(tmp.Name())
		s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
		return
	}
	if err := os.Rename(tmp.Name(), partPath); err != nil {
		s3Error(w, http.StatusInternalServerError, "InternalError", "part upload failed")
		return
	}
	etag := fmt.Sprintf("%x", hash.Sum(nil))
	w.Header().Set("ETag", quoteETag(etag))
	w.WriteHeader(http.StatusOK)
}

func (s *Server) acceptMultipartPartSize(w http.ResponseWriter, dir string, incomingSize int64) bool {
	if incomingSize < 0 {
		s3Error(w, http.StatusLengthRequired, "MissingContentLength", "content length is required")
		return false
	}
	if s.cfg.MaxObjectSize <= 0 {
		return true
	}

	existingSize := int64(0)
	parts, _ := filepath.Glob(filepath.Join(dir, "part-*"))
	for _, part := range parts {
		info, err := os.Stat(part)
		if err == nil && !info.IsDir() {
			existingSize += info.Size()
		}
	}

	if existingSize+incomingSize > s.cfg.MaxObjectSize {
		s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "multipart upload exceeds object size limit")
		return false
	}

	return true
}

func (s *Server) s3CompleteMultipartUpload(w http.ResponseWriter, r *http.Request, key string) {
	uploadID := r.URL.Query().Get("uploadId")
	meta, err := s.multipartMeta(uploadID)
	if err != nil || meta.Key != key {
		s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found")
		return
	}
	var complete completeMultipartUpload
	completeBody, ok := s.s3BoundedBody(w, r, maxXMLRequestBytes)
	if !ok {
		return
	}
	if err := xml.Unmarshal(completeBody, &complete); err != nil && len(completeBody) > 0 {
		s3Error(w, http.StatusBadRequest, "MalformedXML", "invalid complete request")
		return
	}
	parts := complete.Parts
	if len(parts) == 0 {
		entries, _ := filepath.Glob(filepath.Join(s.multipartDir(uploadID), "part-*"))
		for _, entry := range entries {
			base := filepath.Base(entry)
			n, _ := strconv.Atoi(strings.TrimPrefix(base, "part-"))
			parts = append(parts, completedPart{PartNumber: n})
		}
	}
	if len(parts) > 10_000 {
		s3Error(w, http.StatusBadRequest, "InvalidRequest", "too many parts")
		return
	}
	sortParts(parts)
	readers := make([]io.Reader, 0, len(parts))
	files := make([]*os.File, 0, len(parts))
	defer func() {
		for _, file := range files {
			_ = file.Close()
		}
	}()
	var totalSize int64
	for _, part := range parts {
		if part.PartNumber <= 0 || part.PartNumber > 10_000 {
			s3Error(w, http.StatusBadRequest, "InvalidPart", "invalid part number")
			return
		}
		file, err := os.Open(filepath.Join(s.multipartDir(uploadID), fmt.Sprintf("part-%05d", part.PartNumber)))
		if err != nil {
			s3Error(w, http.StatusBadRequest, "InvalidPart", "part missing")
			return
		}
		stat, err := file.Stat()
		if err != nil {
			s3Error(w, http.StatusInternalServerError, "InternalError", "part stat failed")
			return
		}
		totalSize += stat.Size()
		if s.cfg.MaxObjectSize > 0 && totalSize > s.cfg.MaxObjectSize {
			s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large")
			return
		}
		files = append(files, file)
		readers = append(readers, file)
	}
	release, ok := s.acceptWriteSize(w, totalSize, true)
	if !ok {
		return
	}
	defer release()
	info, err := s.store.Put(key, io.MultiReader(readers...), meta.ContentType)
	if err != nil {
		s3Error(w, http.StatusInternalServerError, "InternalError", "complete failed")
		return
	}
	_ = os.RemoveAll(s.multipartDir(uploadID))
	w.Header().Set("Content-Type", "application/xml")
	w.WriteHeader(http.StatusOK)
	_ = xml.NewEncoder(w).Encode(completeMultipartUploadResult{Key: strings.TrimPrefix(key, strings.SplitN(key, "/", 2)[0]+"/"), ETag: quoteETag(info.ETag)})
	if r.Header.Get("X-Magpie-Replication") != "1" {
		s.enqueueReplicationPut(key)
	}
}

func (s *Server) s3AbortMultipartUpload(w http.ResponseWriter, r *http.Request, _ string) {
	uploadID := r.URL.Query().Get("uploadId")
	if !validUploadID(uploadID) {
		s3Error(w, http.StatusNotFound, "NoSuchUpload", "multipart upload not found")
		return
	}
	_ = os.RemoveAll(s.multipartDir(uploadID))
	w.WriteHeader(http.StatusNoContent)
}

func (s *Server) s3CopyObject(w http.ResponseWriter, r *http.Request, destinationKey string) {
	if !signedHeaderIncludes(r, "x-amz-copy-source") {
		s3Error(w, http.StatusForbidden, "AccessDenied", "copy source header must be signed")
		return
	}
	sourceKey, err := copySourceKey(r.Header.Get("X-Amz-Copy-Source"))
	if err != nil {
		s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid copy source")
		return
	}
	sourceBucket, _, _ := strings.Cut(sourceKey, "/")
	if !s.bucketAllowed(sourceBucket) {
		s3Error(w, http.StatusForbidden, "AccessDenied", "copy source bucket is not allowed")
		return
	}
	if len(s.cfg.S3Keys) > 0 {
		key, ok := s.requestAccessKey(r)
		if !ok || !key.can("read") || !key.can("write") {
			s3Error(w, http.StatusForbidden, "AccessDenied", "copy requires read and write permission")
			return
		}
	}
	obj, err := s.store.Open(sourceKey)
	if err != nil {
		if errors.Is(err, ErrNotFound) {
			s3Error(w, http.StatusNotFound, "NoSuchKey", "copy source not found")
			return
		}
		s3Error(w, http.StatusInternalServerError, "InternalError", "copy source failed")
		return
	}
	defer obj.Close()
	if s.cfg.MaxObjectSize > 0 && obj.Info.Size > s.cfg.MaxObjectSize {
		s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large")
		return
	}
	release, ok := s.acceptWriteSize(w, obj.Info.Size, true)
	if !ok {
		return
	}
	defer release()
	contentType := obj.Info.ContentType
	if override := r.Header.Get("Content-Type"); override != "" && strings.EqualFold(r.Header.Get("X-Amz-Metadata-Directive"), "REPLACE") {
		contentType = override
	}
	info, err := s.store.Put(destinationKey, obj, contentType)
	if err != nil {
		s3Error(w, http.StatusInternalServerError, "InternalError", "copy failed")
		return
	}
	w.Header().Set("Content-Type", "application/xml")
	w.WriteHeader(http.StatusOK)
	_ = xml.NewEncoder(w).Encode(copyObjectResult{ETag: quoteETag(info.ETag), LastModified: info.ModTime.UTC().Format(time.RFC3339)})
	if r.Header.Get("X-Magpie-Replication") != "1" {
		s.enqueueReplicationPut(destinationKey)
	}
}

func (s *Server) s3DeleteObjects(w http.ResponseWriter, r *http.Request, bucket string) {
	var request deleteObjectsRequest
	deleteBody, ok := s.s3BoundedBody(w, r, maxXMLRequestBytes)
	if !ok {
		return
	}
	if err := xml.Unmarshal(deleteBody, &request); err != nil {
		s3Error(w, http.StatusBadRequest, "MalformedXML", "invalid delete request")
		return
	}
	if len(request.Objects) > maxBatchDeleteItems {
		s3Error(w, http.StatusBadRequest, "InvalidRequest", "too many objects in delete request")
		return
	}
	result := deleteObjectsResult{Deleted: make([]deletedObject, 0, len(request.Objects))}
	for _, object := range request.Objects {
		key := strings.TrimPrefix(object.Key, "/")
		storeKey := bucket + "/" + key
		if err := ValidateKey(storeKey); err != nil {
			result.Errors = append(result.Errors, deleteObjectError{Key: object.Key, Code: "InvalidObjectName", Message: err.Error()})
			continue
		}
		if err := s.store.Delete(storeKey); err != nil && !errors.Is(err, ErrNotFound) {
			result.Errors = append(result.Errors, deleteObjectError{Key: object.Key, Code: "InternalError", Message: "delete failed"})
			continue
		}
		result.Deleted = append(result.Deleted, deletedObject{Key: object.Key})
		if r.Header.Get("X-Magpie-Replication") != "1" {
			s.enqueueReplicationDelete(storeKey)
		}
	}
	w.Header().Set("Content-Type", "application/xml")
	w.WriteHeader(http.StatusOK)
	_ = xml.NewEncoder(w).Encode(result)
}

func (s *Server) s3BucketList(w http.ResponseWriter, r *http.Request) bool {
	if r.Method != http.MethodGet || strings.Trim(r.URL.Path, "/") == "" || strings.Contains(strings.Trim(r.URL.Path, "/"), "/") {
		return false
	}
	if r.URL.RawQuery == "" && r.URL.Query().Get("prefix") == "" && r.URL.Query().Get("list-type") == "" {
		return false
	}

	bucket := strings.Trim(r.URL.Path, "/")
	if err := ValidateKey(bucket); err != nil {
		s3Error(w, http.StatusBadRequest, "InvalidBucketName", err.Error())
		return true
	}
	if !s.bucketAllowed(bucket) {
		s3Error(w, http.StatusForbidden, "AccessDenied", "bucket is not allowed")
		return true
	}
	if !s.authorizedS3(r) {
		s3Error(w, http.StatusForbidden, "SignatureDoesNotMatch", "request signature is invalid")
		return true
	}

	prefix := r.URL.Query().Get("prefix")
	storePrefix := bucket + "/"
	if prefix != "" {
		storePrefix = bucket + "/" + strings.TrimPrefix(prefix, "/")
	}
	maxKeys := parseMaxKeys(r.URL.Query().Get("max-keys"))
	continuationToken := r.URL.Query().Get("continuation-token")
	after, err := decodeContinuationToken(continuationToken)
	if err != nil {
		s3Error(w, http.StatusBadRequest, "InvalidArgument", "invalid continuation token")
		return true
	}
	page, err := s.store.ListPage(storePrefix, after, maxKeys)
	if err != nil {
		s3Error(w, http.StatusInternalServerError, "InternalError", "list failed")
		return true
	}

	contents := make([]s3ListContent, 0, len(page.Objects))
	for _, object := range page.Objects {
		key := strings.TrimPrefix(object.Key, bucket+"/")
		contents = append(contents, s3ListContent{
			Key:          key,
			LastModified: object.ModTime.UTC().Format(time.RFC3339),
			ETag:         quoteETag(object.ETag),
			Size:         object.Size,
			StorageClass: "STANDARD",
		})
	}

	w.Header().Set("Content-Type", "application/xml")
	w.WriteHeader(http.StatusOK)
	_ = xml.NewEncoder(w).Encode(s3ListBucketResult{
		Name:                  bucket,
		Prefix:                prefix,
		ContinuationToken:     continuationToken,
		NextContinuationToken: encodeContinuationToken(page.NextAfter),
		KeyCount:              len(contents),
		MaxKeys:               maxKeys,
		IsTruncated:           page.NextAfter != "",
		Contents:              contents,
	})
	return true
}

func encodeContinuationToken(key string) string {
	if key == "" {
		return ""
	}
	return base64.RawURLEncoding.EncodeToString([]byte(key))
}

func decodeContinuationToken(token string) (string, error) {
	if token == "" {
		return "", nil
	}
	decoded, err := base64.RawURLEncoding.DecodeString(token)
	if err != nil {
		return "", err
	}
	key := string(decoded)
	if err := ValidateKey(key); err != nil {
		return "", err
	}
	return key, nil
}

func copySourceKey(value string) (string, error) {
	value = strings.TrimPrefix(value, "/")
	if value == "" {
		return "", fmt.Errorf("missing copy source")
	}
	if decoded, err := url.PathUnescape(value); err == nil {
		value = decoded
	}
	if err := ValidateKey(value); err != nil {
		return "", err
	}
	return value, nil
}

func (s *Server) s3BucketKey(path string) (string, string, bool) {
	path = strings.TrimPrefix(path, "/")
	parts := strings.SplitN(path, "/", 2)
	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
		return "", "", false
	}
	if err := ValidateKey(parts[0]); err != nil {
		return "", "", false
	}
	if err := ValidateKey(parts[1]); err != nil {
		return "", "", false
	}
	return parts[0], parts[1], true
}

func (s *Server) s3Put(w http.ResponseWriter, r *http.Request, key string) {
	release, ok := s.acceptWrite(w, r, true)
	if !ok {
		return
	}
	defer release()
	contentType := r.Header.Get("Content-Type")
	body, verifyPayload, ok := s.s3StreamingBody(w, r)
	if !ok {
		return
	}
	store, ok := s.store.(*FileStore)
	if !ok {
		s3Error(w, http.StatusInternalServerError, "InternalError", "store failed")
		return
	}
	info, err := store.PutVerified(key, http.MaxBytesReader(w, body, s.cfg.MaxObjectSize), contentType, func() error {
		if !verifyPayload() {
			return errPayloadHashMismatch
		}
		return nil
	})
	if err != nil {
		if maxBytesError(err) {
			s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "object too large")
			return
		}
		if errors.Is(err, errPayloadHashMismatch) {
			s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
			return
		}
		s3Error(w, http.StatusInternalServerError, "InternalError", "store failed")
		return
	}
	w.Header().Set("ETag", quoteETag(info.ETag))
	w.WriteHeader(http.StatusOK)
	if r.Header.Get("X-Magpie-Replication") != "1" {
		s.enqueueReplicationPut(key)
	}
}

func (s *Server) s3Get(w http.ResponseWriter, r *http.Request, key string, publicRead bool) {
	obj, err := s.store.Open(key)
	if err != nil {
		if errors.Is(err, ErrNotFound) {
			s3Error(w, http.StatusNotFound, "NoSuchKey", "object not found")
			return
		}
		s3Error(w, http.StatusInternalServerError, "InternalError", "read failed")
		return
	}
	defer obj.Close()

	s3ObjectHeaders(w, obj.Info)
	if publicRead {
		hardenPublicRead(w, obj.Info.ContentType)
	} else if disposition := r.URL.Query().Get("response-content-disposition"); disposition != "" {
		w.Header().Set("Content-Disposition", disposition)
	}
	http.ServeContent(w, r, obj.Info.Key, obj.Info.ModTime, obj)
}

func (s *Server) s3Head(w http.ResponseWriter, _ *http.Request, key string, publicRead bool) {
	info, err := s.store.Stat(key)
	if err != nil {
		if errors.Is(err, ErrNotFound) {
			w.WriteHeader(http.StatusNotFound)
			return
		}
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	s3ObjectHeaders(w, info)
	if publicRead {
		hardenPublicRead(w, info.ContentType)
	}
	w.WriteHeader(http.StatusOK)
}

// hardenPublicRead applies defense-in-depth headers to unauthenticated public
// reads. Objects carry an uploader-supplied Content-Type, so a public prefix
// could otherwise serve active content (HTML, SVG) that executes script in
// Magpie's own origin. The CSP sandbox disables scripting even when a browser
// navigates directly to such an object, and any content type that is not inert
// inline media is forced to download rather than render. The attacker-influenced
// response-content-disposition override is intentionally not honored here.
func hardenPublicRead(w http.ResponseWriter, contentType string) {
	w.Header().Set("Content-Security-Policy", "default-src 'none'; sandbox")
	if !inlineSafeContentType(contentType) {
		w.Header().Set("Content-Disposition", "attachment")
	}
}

// inlineSafeContentType reports whether a content type is inert raster/media
// that browsers render without executing script. SVG is excluded because it can
// carry script when loaded as a top-level document.
func inlineSafeContentType(contentType string) bool {
	mediaType := contentType
	if i := strings.IndexByte(mediaType, ';'); i >= 0 {
		mediaType = mediaType[:i]
	}
	mediaType = strings.ToLower(strings.TrimSpace(mediaType))
	if mediaType == "image/svg+xml" || mediaType == "image/svg" {
		return false
	}
	return strings.HasPrefix(mediaType, "image/") ||
		strings.HasPrefix(mediaType, "audio/") ||
		strings.HasPrefix(mediaType, "video/")
}

func (s *Server) s3Delete(w http.ResponseWriter, r *http.Request, key string) {
	if err := s.store.Delete(key); err != nil && !errors.Is(err, ErrNotFound) {
		s3Error(w, http.StatusInternalServerError, "InternalError", "delete failed")
		return
	}
	w.WriteHeader(http.StatusNoContent)
	if r.Header.Get("X-Magpie-Replication") != "1" {
		s.enqueueReplicationDelete(key)
	}
}

func s3ObjectHeaders(w http.ResponseWriter, info ObjectInfo) {
	contentType := info.ContentType
	if contentType == "" {
		contentType = "application/octet-stream"
	}
	w.Header().Set("Content-Type", contentType)
	w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
	w.Header().Set("ETag", quoteETag(info.ETag))
	w.Header().Set("Last-Modified", info.ModTime.UTC().Format(http.TimeFormat))
	w.Header().Set("Accept-Ranges", "bytes")
	w.Header().Set("X-Content-Type-Options", "nosniff")
}

func quoteETag(etag string) string {
	if etag == "" {
		return "\"\""
	}
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		return etag
	}
	return "\"" + etag + "\""
}

type s3ErrorBody struct {
	XMLName xml.Name `xml:"Error"`
	Code    string   `xml:"Code"`
	Message string   `xml:"Message"`
}

func s3Error(w http.ResponseWriter, status int, code string, message string) {
	w.Header().Set("Content-Type", "application/xml")
	w.WriteHeader(status)
	_ = xml.NewEncoder(w).Encode(s3ErrorBody{Code: code, Message: message})
}

func parseMaxKeys(value string) int {
	if value == "" {
		return 1000
	}
	parsed, err := strconv.Atoi(value)
	if err != nil || parsed <= 0 {
		return 1000
	}
	if parsed > 1000 {
		return 1000
	}
	return parsed
}

func randomID() (string, error) {
	buf := make([]byte, 18)
	if _, err := rand.Read(buf); err != nil {
		return "", err
	}
	return base64.RawURLEncoding.EncodeToString(buf), nil
}

func md5Hash() hashHash {
	return md5.New()
}

type hashHash interface {
	io.Writer
	Sum([]byte) []byte
}

func (s *Server) multipartDir(uploadID string) string {
	if !validUploadID(uploadID) {
		return filepath.Join(s.store.(*FileStore).root, ".multipart", "invalid")
	}
	return filepath.Join(s.store.(*FileStore).root, ".multipart", uploadID)
}

func validUploadID(uploadID string) bool {
	if uploadID == "" || len(uploadID) > 128 {
		return false
	}
	for _, ch := range uploadID {
		if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '-' || ch == '_' {
			continue
		}
		return false
	}
	return true
}

func (s *Server) startMultipartCleanup(ctx context.Context) {
	store, ok := s.store.(*FileStore)
	if !ok || s.cfg.MultipartMaxAge <= 0 {
		return
	}
	if err := store.CleanupMultipart(s.cfg.MultipartMaxAge); err != nil {
		log.Printf("multipart cleanup error=%v", err)
	}
	go func() {
		ticker := time.NewTicker(time.Hour)
		defer ticker.Stop()
		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				if err := store.CleanupMultipart(s.cfg.MultipartMaxAge); err != nil {
					log.Printf("multipart cleanup error=%v", err)
				}
			}
		}
	}()
}

func (s *Server) s3BoundedBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, bool) {
	body, verifyPayload, ok := s.s3StreamingBody(w, r)
	if !ok {
		return nil, false
	}
	data, err := io.ReadAll(http.MaxBytesReader(w, body, limit))
	if err != nil {
		if errors.Is(err, errPayloadHashMismatch) {
			s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
			return nil, false
		}
		s3Error(w, http.StatusRequestEntityTooLarge, "EntityTooLarge", "request body too large")
		return nil, false
	}
	if !verifyPayload() {
		s3Error(w, http.StatusForbidden, "XAmzContentSHA256Mismatch", "payload hash mismatch")
		return nil, false
	}
	return data, true
}

func (s *Server) s3StreamingBody(w http.ResponseWriter, r *http.Request) (io.ReadCloser, func() bool, bool) {
	payloadHash := r.Header.Get("X-Amz-Content-Sha256")
	if payloadHash == "" || payloadHash == "UNSIGNED-PAYLOAD" {
		return r.Body, func() bool { return true }, true
	}
	if len(payloadHash) != sha256.Size*2 {
		s3Error(w, http.StatusBadRequest, "InvalidDigest", "unsupported payload hash")
		return nil, nil, false
	}
	if _, err := hex.DecodeString(payloadHash); err != nil {
		s3Error(w, http.StatusBadRequest, "InvalidDigest", "invalid payload hash")
		return nil, nil, false
	}
	hash := sha256.New()
	verified := false
	return hashVerifyingReadCloser{reader: r.Body, hash: hash, expected: strings.ToLower(payloadHash)}, func() bool {
		if verified {
			return true
		}
		verified = hex.EncodeToString(hash.Sum(nil)) == strings.ToLower(payloadHash)
		return verified
	}, true
}

func signedHeaderIncludes(r *http.Request, header string) bool {
	signedHeaders := r.URL.Query().Get("X-Amz-SignedHeaders")
	if signedHeaders == "" {
		auth := r.Header.Get("Authorization")
		if strings.HasPrefix(auth, awsAlgorithm+" ") {
			signedHeaders = parseAuthParams(strings.TrimPrefix(auth, awsAlgorithm+" "))["SignedHeaders"]
		}
	}
	for _, signed := range strings.Split(signedHeaders, ";") {
		if strings.EqualFold(strings.TrimSpace(signed), header) {
			return true
		}
	}
	return false
}

type hashVerifyingReadCloser struct {
	reader   io.ReadCloser
	hash     hashHash
	expected string
}

func (r hashVerifyingReadCloser) Read(p []byte) (int, error) {
	n, err := r.reader.Read(p)
	if n > 0 {
		_, _ = r.hash.Write(p[:n])
	}
	if errors.Is(err, io.EOF) && hex.EncodeToString(r.hash.Sum(nil)) != r.expected {
		return n, errPayloadHashMismatch
	}
	return n, err
}

func (r hashVerifyingReadCloser) Close() error {
	return r.reader.Close()
}

func (s *Server) multipartMeta(uploadID string) (multipartMeta, error) {
	data, err := os.ReadFile(filepath.Join(s.multipartDir(uploadID), "meta.xml"))
	if err != nil {
		return multipartMeta{}, err
	}
	var meta multipartMeta
	if err := xml.Unmarshal(data, &meta); err != nil {
		return multipartMeta{}, err
	}
	return meta, nil
}

func sortParts(parts []completedPart) {
	sort.Slice(parts, func(i, j int) bool { return parts[i].PartNumber < parts[j].PartNumber })
}

type s3ListBucketResult struct {
	XMLName               xml.Name        `xml:"ListBucketResult"`
	Xmlns                 string          `xml:"xmlns,attr,omitempty"`
	Name                  string          `xml:"Name"`
	Prefix                string          `xml:"Prefix"`
	ContinuationToken     string          `xml:"ContinuationToken,omitempty"`
	NextContinuationToken string          `xml:"NextContinuationToken,omitempty"`
	KeyCount              int             `xml:"KeyCount"`
	MaxKeys               int             `xml:"MaxKeys"`
	IsTruncated           bool            `xml:"IsTruncated"`
	Contents              []s3ListContent `xml:"Contents"`
}

type s3ListContent struct {
	Key          string `xml:"Key"`
	LastModified string `xml:"LastModified"`
	ETag         string `xml:"ETag"`
	Size         int64  `xml:"Size"`
	StorageClass string `xml:"StorageClass"`
}

type multipartMeta struct {
	Key         string `xml:"Key"`
	ContentType string `xml:"ContentType"`
}

type createMultipartUploadResult struct {
	XMLName  xml.Name `xml:"InitiateMultipartUploadResult"`
	UploadID string   `xml:"UploadId"`
}

type completeMultipartUpload struct {
	Parts []completedPart `xml:"Part"`
}

type completedPart struct {
	PartNumber int    `xml:"PartNumber"`
	ETag       string `xml:"ETag"`
}

type completeMultipartUploadResult struct {
	XMLName xml.Name `xml:"CompleteMultipartUploadResult"`
	Key     string   `xml:"Key"`
	ETag    string   `xml:"ETag"`
}

type copyObjectResult struct {
	XMLName      xml.Name `xml:"CopyObjectResult"`
	LastModified string   `xml:"LastModified"`
	ETag         string   `xml:"ETag"`
}

type deleteObjectsRequest struct {
	Objects []deleteObjectRequest `xml:"Object"`
}

type deleteObjectRequest struct {
	Key string `xml:"Key"`
}

type deleteObjectsResult struct {
	XMLName xml.Name            `xml:"DeleteResult"`
	Deleted []deletedObject     `xml:"Deleted"`
	Errors  []deleteObjectError `xml:"Error,omitempty"`
}

type deletedObject struct {
	Key string `xml:"Key"`
}

type deleteObjectError struct {
	Key     string `xml:"Key"`
	Code    string `xml:"Code"`
	Message string `xml:"Message"`
}