spacefs-s3sdk 0.1.0__tar.gz

spacefs_s3sdk-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+.venv/
+dist/
+build/
+__pycache__/
+*.egg-info/
+.pytest_cache/

spacefs_s3sdk-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Space Computer Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

spacefs_s3sdk-0.1.0/PKG-INFO

@@ -0,0 +1,75 @@
+Metadata-Version: 2.5
+Name: spacefs-s3sdk
+Version: 0.1.0
+Summary: s3sdk SDK: S3-compatible access to Space drives plus partial writes, copy-on-write forks, and version history.
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: boto3>=1.36
+Provides-Extra: test
+Requires-Dist: pytest; extra == 'test'
+Description-Content-Type: text/markdown
+
+# s3sdk — Python SDK
+
+s3sdk is the S3 API over Space drives, plus extensions S3 can't express:
+writes at an offset, batched patches, copy-on-write drive forks, and per-object version
+history with restore. Wire protocol: [`../spec/PROTOCOL.md`](../spec/PROTOCOL.md).
+
+## Install
+
+```sh
+pip install spacefs-s3sdk        # Python >= 3.10; pulls in boto3; import s3sdk
+```
+
+## Usage
+
+```python
+from s3sdk import S3SDKClient, S3SDKError, Edit
+
+c = S3SDKClient("S3SDK...", "secret...")          # endpoint defaults to https://s3sdk.spacefs.com
+
+c.create_drive("footage", display_name="Footage")
+v1 = c.put_object("footage", "cuts/a.txt", "hello world")
+c.write_at("footage", "cuts/a.txt", 6, b"WORLD")               # pwrite, one new version
+c.patch("footage", "cuts/a.txt", [Edit(0, b"H"), Edit(11, b"!")])
+c.truncate("footage", "cuts/a.txt", 5)
+print(c.read_range("footage", "cuts/a.txt", 1, 3))            # b"ell"
+
+# Optimistic concurrency
+try:
+    c.write_at("footage", "cuts/a.txt", 0, b"x", if_version=v1.version_id)
+except S3SDKError as e:
+    if e.status == 412:
+        print("head is now", e.current_version_id)
+
+# History
+versions = c.list_versions("footage", "cuts/a.txt")           # oldest first
+old = c.get_object("footage", "cuts/a.txt", version_id=versions[0].version_id)
+then = c.get_object("footage", "cuts/a.txt", as_of=versions[0].last_modified_raw)
+c.restore_version("footage", "cuts/a.txt", versions[0].version_id)
+
+# Copy-on-write forks
+fork = c.fork_drive("footage", "footage-experiment")
+print(c.describe_drive("footage-experiment").fork_of)
+c.delete_drive("footage-experiment", hard=True)
+
+# Any standard S3 call: c.s3 is a boto3 client (path-style)
+c.s3.head_object(Bucket="footage", Key="cuts/a.txt")
+```
+
+Every error — gateway extensions and boto3 calls alike — is raised as
+`S3SDKError(status, code, message, current_version_id, request_id)`.
+
+## Development
+
+```sh
+python3 -m venv .venv && .venv/bin/pip install -e '.[test]'
+.venv/bin/python -m pytest tests
+
+# Conformance suite against a gateway (see ../spec/conformance/README.md)
+S3SDK_ENDPOINT=http://127.0.0.1:19000 \
+S3SDK_ACCESS_KEY_ID=S3SDKCONFORMANCE0001 \
+S3SDK_SECRET_ACCESS_KEY=conformance-secret-0000000000000000000000 \
+  .venv/bin/python -m s3sdk.conformance ../spec/conformance/cases.json
+```

spacefs_s3sdk-0.1.0/README.md

@@ -0,0 +1,63 @@
+# s3sdk — Python SDK
+
+s3sdk is the S3 API over Space drives, plus extensions S3 can't express:
+writes at an offset, batched patches, copy-on-write drive forks, and per-object version
+history with restore. Wire protocol: [`../spec/PROTOCOL.md`](../spec/PROTOCOL.md).
+
+## Install
+
+```sh
+pip install spacefs-s3sdk        # Python >= 3.10; pulls in boto3; import s3sdk
+```
+
+## Usage
+
+```python
+from s3sdk import S3SDKClient, S3SDKError, Edit
+
+c = S3SDKClient("S3SDK...", "secret...")          # endpoint defaults to https://s3sdk.spacefs.com
+
+c.create_drive("footage", display_name="Footage")
+v1 = c.put_object("footage", "cuts/a.txt", "hello world")
+c.write_at("footage", "cuts/a.txt", 6, b"WORLD")               # pwrite, one new version
+c.patch("footage", "cuts/a.txt", [Edit(0, b"H"), Edit(11, b"!")])
+c.truncate("footage", "cuts/a.txt", 5)
+print(c.read_range("footage", "cuts/a.txt", 1, 3))            # b"ell"
+
+# Optimistic concurrency
+try:
+    c.write_at("footage", "cuts/a.txt", 0, b"x", if_version=v1.version_id)
+except S3SDKError as e:
+    if e.status == 412:
+        print("head is now", e.current_version_id)
+
+# History
+versions = c.list_versions("footage", "cuts/a.txt")           # oldest first
+old = c.get_object("footage", "cuts/a.txt", version_id=versions[0].version_id)
+then = c.get_object("footage", "cuts/a.txt", as_of=versions[0].last_modified_raw)
+c.restore_version("footage", "cuts/a.txt", versions[0].version_id)
+
+# Copy-on-write forks
+fork = c.fork_drive("footage", "footage-experiment")
+print(c.describe_drive("footage-experiment").fork_of)
+c.delete_drive("footage-experiment", hard=True)
+
+# Any standard S3 call: c.s3 is a boto3 client (path-style)
+c.s3.head_object(Bucket="footage", Key="cuts/a.txt")
+```
+
+Every error — gateway extensions and boto3 calls alike — is raised as
+`S3SDKError(status, code, message, current_version_id, request_id)`.
+
+## Development
+
+```sh
+python3 -m venv .venv && .venv/bin/pip install -e '.[test]'
+.venv/bin/python -m pytest tests
+
+# Conformance suite against a gateway (see ../spec/conformance/README.md)
+S3SDK_ENDPOINT=http://127.0.0.1:19000 \
+S3SDK_ACCESS_KEY_ID=S3SDKCONFORMANCE0001 \
+S3SDK_SECRET_ACCESS_KEY=conformance-secret-0000000000000000000000 \
+  .venv/bin/python -m s3sdk.conformance ../spec/conformance/cases.json
+```

spacefs_s3sdk-0.1.0/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "spacefs-s3sdk"
+version = "0.1.0"
+description = "s3sdk SDK: S3-compatible access to Space drives plus partial writes, copy-on-write forks, and version history."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+dependencies = ["boto3>=1.36"]
+
+[project.optional-dependencies]
+test = ["pytest"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/s3sdk"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

spacefs_s3sdk-0.1.0/src/s3sdk/__init__.py

@@ -0,0 +1,41 @@
+"""s3sdk SDK: S3-compatible access to Space drives plus partial writes,
+copy-on-write forks, and per-object version history. See spec/PROTOCOL.md."""
+
+from .client import (
+    DEFAULT_ENDPOINT,
+    CreateDriveResult,
+    DriveInfo,
+    DriveSummary,
+    ForkOf,
+    ForkResult,
+    ListResult,
+    ObjectResult,
+    RestoreResult,
+    S3SDKClient,
+    S3SDKError,
+    VersionEntry,
+    WriteResult,
+)
+from .patch import Edit, PatchError, decode_patch, encode_patch
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "DEFAULT_ENDPOINT",
+    "CreateDriveResult",
+    "DriveInfo",
+    "DriveSummary",
+    "Edit",
+    "ForkOf",
+    "ForkResult",
+    "ListResult",
+    "ObjectResult",
+    "PatchError",
+    "RestoreResult",
+    "S3SDKClient",
+    "S3SDKError",
+    "VersionEntry",
+    "WriteResult",
+    "decode_patch",
+    "encode_patch",
+]

spacefs_s3sdk-0.1.0/src/s3sdk/client.py

@@ -0,0 +1,637 @@
+"""S3SDK client.
+
+Standard S3 calls go through the bundled boto3 client (``client.s3``). The S3SDK extensions
+(partial writes, forks, version history) are plain SigV4-signed requests; see spec/PROTOCOL.md.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from email.utils import parsedate_to_datetime
+from typing import Any, Literal, Mapping, Optional, Sequence, Union
+from urllib.parse import quote, urlsplit
+
+import boto3
+import urllib3
+from botocore.auth import S3SigV4Auth
+from botocore.awsrequest import AWSRequest
+from botocore.config import Config
+from botocore.credentials import Credentials
+from botocore.exceptions import ClientError
+
+from .patch import CONTENT_TYPE as PATCH_CONTENT_TYPE
+from .patch import Edit, encode_patch
+
+__all__ = [
+    "DEFAULT_ENDPOINT",
+    "S3SDKClient",
+    "S3SDKError",
+    "WriteResult",
+    "RestoreResult",
+    "ObjectResult",
+    "VersionEntry",
+    "DriveInfo",
+    "ForkOf",
+    "DriveSummary",
+    "ForkResult",
+    "ListResult",
+    "CreateDriveResult",
+]
+
+DEFAULT_ENDPOINT = "https://s3sdk.spacefs.com"
+
+Body = Union[bytes, bytearray, memoryview, str]
+Operation = Literal["put", "write", "copy", "restore", "rename", "other"]
+
+
+class S3SDKError(Exception):
+    """An S3 or S3SDK error response."""
+
+    def __init__(
+        self,
+        status: int,
+        code: str,
+        message: Optional[str] = None,
+        current_version_id: Optional[str] = None,
+        request_id: Optional[str] = None,
+    ) -> None:
+        self.status = status
+        self.code = code
+        self.message = message or f"{code} (HTTP {status})"
+        #: On 412: the object's current head version, when it exists.
+        self.current_version_id = current_version_id
+        self.request_id = request_id
+        super().__init__(self.message)
+
+    def __repr__(self) -> str:
+        return (
+            f"S3SDKError(status={self.status!r}, code={self.code!r}, message={self.message!r}, "
+            f"current_version_id={self.current_version_id!r}, request_id={self.request_id!r})"
+        )
+
+
+# ---- results ------------------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class WriteResult:
+    version_id: str
+    etag: str
+    #: Object size after the write.
+    size: int
+
+
+@dataclass(frozen=True)
+class RestoreResult(WriteResult):
+    restored_from: str = ""
+
+
+@dataclass(frozen=True)
+class ObjectResult:
+    body: bytes
+    version_id: str
+    etag: str
+    #: Full object size.
+    size: int
+    content_type: Optional[str] = None
+    last_modified: Optional[datetime] = None
+
+
+@dataclass(frozen=True)
+class VersionEntry:
+    version_id: str
+    is_latest: bool
+    size: int
+    etag: str
+    last_modified: datetime
+    #: The exact RFC 3339 timestamp from the server (sub-millisecond); use it for ``as_of`` reads.
+    last_modified_raw: str
+    operation: str
+    restored_from: Optional[str] = None
+
+
+@dataclass(frozen=True)
+class ForkOf:
+    drive_id: str
+    fork_point: datetime
+
+
+@dataclass(frozen=True)
+class DriveInfo:
+    drive_id: str
+    alias: str
+    display_name: str
+    created_at: datetime
+    fork_of: Optional[ForkOf]
+    forks: list[str]
+    usage_bytes: Optional[int]
+
+
+@dataclass(frozen=True)
+class DriveSummary:
+    name: str
+    created_at: Optional[datetime] = None
+
+
+@dataclass(frozen=True)
+class CreateDriveResult:
+    drive_id: str
+
+
+@dataclass(frozen=True)
+class ForkResult:
+    drive_id: str
+    source_drive_id: str
+    fork_point: datetime
+
+
+@dataclass(frozen=True)
+class ListResult:
+    keys: list[str]
+    prefixes: list[str]
+
+
+# ---- helpers ------------------------------------------------------------------------------------
+
+_RFC3339 = re.compile(
+    r"^(\d{4})-(\d{2})-(\d{2})[Tt ](\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|z|[+-]\d{2}:\d{2})$"
+)
+
+
+def parse_timestamp(value: str) -> datetime:
+    """Parses an RFC 3339 timestamp (any fractional precision; truncated to microseconds)."""
+    m = _RFC3339.match(value.strip())
+    if not m:
+        raise ValueError(f"not an RFC 3339 timestamp: {value!r}")
+    y, mo, d, h, mi, s, frac, tz = m.groups()
+    micro = int((frac or "0")[:6].ljust(6, "0"))
+    if tz in ("Z", "z"):
+        tzinfo = timezone.utc
+    else:
+        sign = 1 if tz[0] == "+" else -1
+        tzinfo = timezone(sign * timedelta(hours=int(tz[1:3]), minutes=int(tz[4:6])))
+    return datetime(int(y), int(mo), int(d), int(h), int(mi), int(s), micro, tzinfo=tzinfo)
+
+
+def format_timestamp(value: datetime) -> str:
+    """Formats a datetime as RFC 3339 UTC. Naive datetimes are taken to be UTC."""
+    if value.tzinfo is None:
+        value = value.replace(tzinfo=timezone.utc)
+    return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
+
+def _to_bytes(body: Body) -> bytes:
+    return body.encode("utf-8") if isinstance(body, str) else bytes(body)
+
+
+def _encode_key(key: str) -> str:
+    """RFC 3986 encoding of each path segment (S3 canonical form); ``/`` separators are kept."""
+    return "/".join(quote(seg, safe="") for seg in key.split("/"))
+
+
+def _unquote_etag(etag: str) -> str:
+    if etag.startswith("W/"):
+        etag = etag[2:]
+    if len(etag) >= 2 and etag.startswith('"') and etag.endswith('"'):
+        etag = etag[1:-1]
+    return etag
+
+
+def _xml_fields(text: str) -> dict[str, str]:
+    if not text:
+        return {}
+    try:
+        root = ET.fromstring(text)
+    except ET.ParseError:
+        out = {}
+        for tag in ("Code", "Message", "RequestId"):
+            m = re.search(rf"<{tag}>([\s\S]*?)</{tag}>", text)
+            if m:
+                out[tag] = m.group(1)
+        return out
+    return {child.tag.split("}")[-1]: (child.text or "") for child in root}
+
+
+def _error_from_response(res: urllib3.BaseHTTPResponse) -> S3SDKError:
+    text = "" if res.status == 304 else res.data.decode("utf-8", "replace")
+    fields = _xml_fields(text)
+    return S3SDKError(
+        status=res.status,
+        code=fields.get("Code") or ("NotModified" if res.status == 304 else f"Http{res.status}"),
+        message=fields.get("Message"),
+        current_version_id=res.headers.get("x-amz-version-id"),
+        request_id=fields.get("RequestId") or res.headers.get("x-amz-request-id"),
+    )
+
+
+def _error_from_client_error(err: ClientError) -> S3SDKError:
+    resp = err.response or {}
+    meta = resp.get("ResponseMetadata", {})
+    headers = meta.get("HTTPHeaders", {}) or {}
+    error = resp.get("Error", {}) or {}
+    status = meta.get("HTTPStatusCode") or 0
+    return S3SDKError(
+        status=status,
+        code=error.get("Code") or f"Http{status}",
+        message=error.get("Message") or str(err),
+        current_version_id=headers.get("x-amz-version-id"),
+        request_id=meta.get("RequestId") or error.get("RequestId"),
+    )
+
+
+def _header(res: urllib3.BaseHTTPResponse, name: str) -> str:
+    v = res.headers.get(name)
+    if v is None:
+        raise S3SDKError(res.status, "InvalidResponse", f"missing {name} header")
+    return v
+
+
+def _write_result(res: urllib3.BaseHTTPResponse) -> WriteResult:
+    return WriteResult(
+        version_id=_header(res, "x-amz-version-id"),
+        etag=_unquote_etag(_header(res, "etag")),
+        size=int(_header(res, "x-s3sdk-size")),
+    )
+
+
+def _precondition_headers(if_version: Optional[str], if_match: Optional[str]) -> dict[str, str]:
+    h: dict[str, str] = {}
+    if if_version is not None:
+        h["x-s3sdk-if-version"] = if_version
+    if if_match is not None:
+        h["If-Match"] = "*" if if_match == "*" else f'"{_unquote_etag(if_match)}"'
+    return h
+
+
+class _AwsErrors:
+    """Context manager translating botocore ClientError into S3SDKError."""
+
+    def __enter__(self) -> None:
+        return None
+
+    def __exit__(self, exc_type, exc, tb) -> bool:
+        if isinstance(exc, ClientError):
+            raise _error_from_client_error(exc) from exc
+        return False
+
+
+_aws = _AwsErrors()
+
+
+# ---- client -------------------------------------------------------------------------------------
+
+
+class S3SDKClient:
+    """S3SDK client. ``.s3`` is a plain boto3 S3 client for any standard S3 call."""
+
+    def __init__(
+        self,
+        access_key_id: str,
+        secret_access_key: str,
+        endpoint: str = DEFAULT_ENDPOINT,
+        region: str = "us-east-1",
+        *,
+        force_path_style: bool = True,
+        boto_config: Optional[Config] = None,
+        http: Optional[urllib3.PoolManager] = None,
+        timeout: Union[float, urllib3.Timeout, None] = 60.0,
+    ) -> None:
+        parts = urlsplit(endpoint)
+        if parts.scheme not in ("http", "https") or not parts.netloc:
+            raise ValueError(f"invalid endpoint: {endpoint!r}")
+        self._scheme = parts.scheme
+        self._netloc = parts.netloc
+        self._base_path = parts.path.rstrip("/")
+        self._credentials = Credentials(access_key_id, secret_access_key)
+        self._region = region
+        self._http = http or urllib3.PoolManager(retries=False)
+        self._timeout = timeout
+
+        config = Config(
+            s3={"addressing_style": "path" if force_path_style else "auto"},
+            # The gateway returns its own ETags; SDK-side checksum validation doesn't apply.
+            request_checksum_calculation="when_required",
+            response_checksum_validation="when_required",
+        )
+        if boto_config is not None:
+            config = config.merge(boto_config)
+        #: The underlying boto3 S3 client, for any standard S3 call.
+        self.s3 = boto3.client(
+            "s3",
+            endpoint_url=endpoint,
+            region_name=region,
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            config=config,
+        )
+
+    # ---- drives ---------------------------------------------------------------------------------
+
+    def list_drives(self) -> list[DriveSummary]:
+        with _aws:
+            out = self.s3.list_buckets()
+        return [DriveSummary(b.get("Name", ""), b.get("CreationDate")) for b in out.get("Buckets", [])]
+
+    def create_drive(self, name: str, display_name: Optional[str] = None) -> CreateDriveResult:
+        headers: dict[str, str] = {}
+        if display_name is not None:
+            headers["x-s3sdk-display-name"] = quote(display_name, safe="")
+        res = self._send("PUT", name, headers=headers)
+        return CreateDriveResult(_header(res, "x-s3sdk-drive-id"))
+
+    def fork_drive(self, source: str, name: str) -> ForkResult:
+        """Copy-on-write fork of ``source``'s current state into a new drive ``name``."""
+        res = self._send("PUT", name, headers={"x-s3sdk-fork-source": source})
+        return ForkResult(
+            drive_id=_header(res, "x-s3sdk-drive-id"),
+            source_drive_id=_header(res, "x-s3sdk-fork-source-id"),
+            fork_point=parse_timestamp(_header(res, "x-s3sdk-fork-point")),
+        )
+
+    def delete_drive(self, name: str, hard: bool = False) -> None:
+        """Soft delete (recoverable) unless ``hard``. A hard delete fails while forks exist."""
+        if hard:
+            self._send("DELETE", name, headers={"x-s3sdk-hard-delete": "true"})
+        else:
+            with _aws:
+                self.s3.delete_bucket(Bucket=name)
+
+    def describe_drive(self, name: str) -> DriveInfo:
+        res = self._send("GET", name, query=[("x-s3sdk-drive", "")])
+        j = json.loads(res.data)
+        fork_of = j.get("forkOf")
+        return DriveInfo(
+            drive_id=j["driveId"],
+            alias=j["alias"],
+            display_name=j["displayName"],
+            created_at=parse_timestamp(j["createdAt"]),
+            fork_of=ForkOf(fork_of["driveId"], parse_timestamp(fork_of["forkPoint"])) if fork_of else None,
+            forks=list(j.get("forks") or []),
+            usage_bytes=j.get("usageBytes"),
+        )
+
+    # ---- objects --------------------------------------------------------------------------------
+
+    def put_object(
+        self,
+        drive: str,
+        key: str,
+        body: Body,
+        content_type: Optional[str] = None,
+        if_version: Optional[str] = None,
+        if_match: Optional[str] = None,
+        if_none_match: Optional[Literal["*"]] = None,
+    ) -> WriteResult:
+        data = _to_bytes(body)
+        if if_version is not None:
+            # `x-s3sdk-if-version` isn't a boto3 parameter; send the PUT ourselves.
+            headers = _precondition_headers(if_version, if_match)
+            if content_type:
+                headers["Content-Type"] = content_type
+            if if_none_match:
+                headers["If-None-Match"] = if_none_match
+            res = self._send("PUT", drive, key, headers=headers, body=data)
+            return WriteResult(_header(res, "x-amz-version-id"), _unquote_etag(_header(res, "etag")), len(data))
+        params: dict[str, Any] = {"Bucket": drive, "Key": key, "Body": data, "ContentLength": len(data)}
+        if content_type is not None:
+            params["ContentType"] = content_type
+        if if_match is not None:
+            params["IfMatch"] = if_match
+        if if_none_match is not None:
+            params["IfNoneMatch"] = if_none_match
+        with _aws:
+            out = self.s3.put_object(**params)
+        return WriteResult(out.get("VersionId", ""), _unquote_etag(out.get("ETag", "")), len(data))
+
+    def get_object(
+        self,
+        drive: str,
+        key: str,
+        version_id: Optional[str] = None,
+        as_of: Union[datetime, str, None] = None,
+    ) -> ObjectResult:
+        """Reads a whole object: the head, ``version_id``, or the version current at ``as_of``."""
+        if as_of is not None:
+            if version_id is not None:
+                raise ValueError("version_id and as_of are mutually exclusive")
+            stamp = as_of if isinstance(as_of, str) else format_timestamp(as_of)
+            res = self._send("GET", drive, key, headers={"x-s3sdk-as-of": stamp})
+            data = res.data
+            lm = res.headers.get("last-modified")
+            return ObjectResult(
+                body=data,
+                version_id=_header(res, "x-amz-version-id"),
+                etag=_unquote_etag(_header(res, "etag")),
+                size=len(data),
+                content_type=res.headers.get("content-type"),
+                last_modified=parsedate_to_datetime(lm) if lm else None,
+            )
+        params: dict[str, Any] = {"Bucket": drive, "Key": key}
+        if version_id is not None:
+            params["VersionId"] = version_id
+        with _aws:
+            out = self.s3.get_object(**params)
+            data = out["Body"].read()
+        return ObjectResult(
+            body=data,
+            version_id=out.get("VersionId", ""),
+            etag=_unquote_etag(out.get("ETag", "")),
+            size=out.get("ContentLength", len(data)),
+            content_type=out.get("ContentType"),
+            last_modified=out.get("LastModified"),
+        )
+
+    def read_range(
+        self,
+        drive: str,
+        key: str,
+        offset: int,
+        length: Optional[int] = None,
+        version_id: Optional[str] = None,
+    ) -> bytes:
+        """Reads ``[offset, offset+length)`` (clamped to the object's end); ``length=None`` = to the end."""
+        if length is not None and length <= 0:
+            return b""
+        rng = f"bytes={offset}-" if length is None else f"bytes={offset}-{offset + length - 1}"
+        params: dict[str, Any] = {"Bucket": drive, "Key": key, "Range": rng}
+        if version_id is not None:
+            params["VersionId"] = version_id
+        with _aws:
+            out = self.s3.get_object(**params)
+            return out["Body"].read()
+
+    def write_at(
+        self,
+        drive: str,
+        key: str,
+        offset: int,
+        data: Body,
+        size: Optional[int] = None,
+        if_version: Optional[str] = None,
+        if_match: Optional[str] = None,
+    ) -> WriteResult:
+        """``pwrite``: writes ``data`` at ``offset`` as one new version. Creates the object if absent."""
+        headers = _precondition_headers(if_version, if_match)
+        headers["x-s3sdk-offset"] = str(offset)
+        if size is not None:
+            headers["x-s3sdk-size"] = str(size)
+        res = self._send("PUT", drive, key, query=[("x-s3sdk-write", "")], headers=headers, body=_to_bytes(data))
+        return _write_result(res)
+
+    def patch(
+        self,
+        drive: str,
+        key: str,
+        edits: Sequence[Edit],
+        size: Optional[int] = None,
+        if_version: Optional[str] = None,
+        if_match: Optional[str] = None,
+    ) -> WriteResult:
+        """Applies several edits atomically as one new version."""
+        return self.raw_patch(drive, key, encode_patch(edits), size=size, if_version=if_version, if_match=if_match)
+
+    def raw_patch(
+        self,
+        drive: str,
+        key: str,
+        body: Union[bytes, bytearray, memoryview],
+        size: Optional[int] = None,
+        if_version: Optional[str] = None,
+        if_match: Optional[str] = None,
+    ) -> WriteResult:
+        """Sends an already-encoded ``application/vnd.s3sdk.patch`` body."""
+        headers = _precondition_headers(if_version, if_match)
+        headers["Content-Type"] = PATCH_CONTENT_TYPE
+        if size is not None:
+            headers["x-s3sdk-size"] = str(size)
+        res = self._send("POST", drive, key, query=[("x-s3sdk-patch", "")], headers=headers, body=bytes(body))
+        return _write_result(res)
+
+    def truncate(
+        self,
+        drive: str,
+        key: str,
+        size: int,
+        if_version: Optional[str] = None,
+        if_match: Optional[str] = None,
+    ) -> WriteResult:
+        """Sets the object's size (truncating, or extending with zeros)."""
+        headers = _precondition_headers(if_version, if_match)
+        headers["x-s3sdk-offset"] = "0"
+        headers["x-s3sdk-size"] = str(size)
+        res = self._send("PUT", drive, key, query=[("x-s3sdk-write", "")], headers=headers, body=b"")
+        return _write_result(res)
+
+    def delete_object(self, drive: str, key: str) -> None:
+        with _aws:
+            self.s3.delete_object(Bucket=drive, Key=key)
+
+    def list_objects(self, drive: str, prefix: Optional[str] = None, delimiter: Optional[str] = None) -> ListResult:
+        """Lists every key and common prefix (all pages)."""
+        keys: list[str] = []
+        prefixes: list[str] = []
+        token: Optional[str] = None
+        while True:
+            params: dict[str, Any] = {"Bucket": drive}
+            if prefix is not None:
+                params["Prefix"] = prefix
+            if delimiter is not None:
+                params["Delimiter"] = delimiter
+            if token:
+                params["ContinuationToken"] = token
+            with _aws:
+                out = self.s3.list_objects_v2(**params)
+            keys.extend(o["Key"] for o in out.get("Contents", []) if "Key" in o)
+            prefixes.extend(p["Prefix"] for p in out.get("CommonPrefixes", []) if "Prefix" in p)
+            token = out.get("NextContinuationToken") if out.get("IsTruncated") else None
+            if not token:
+                return ListResult(keys, prefixes)
+
+    # ---- history --------------------------------------------------------------------------------
+
+    def list_versions(self, drive: str, key: str) -> list[VersionEntry]:
+        """Full history of one object, oldest first."""
+        res = self._send("GET", drive, key, query=[("x-s3sdk-versions", "")])
+        j = json.loads(res.data)
+        return [
+            VersionEntry(
+                version_id=v["versionId"],
+                is_latest=bool(v["isLatest"]),
+                size=int(v["size"]),
+                etag=_unquote_etag(v["etag"]),
+                last_modified=parse_timestamp(v["lastModified"]),
+                last_modified_raw=v["lastModified"],
+                operation=v["operation"],
+                restored_from=v.get("restoredFrom"),
+            )
+            for v in j["versions"]
+        ]
+
+    def restore_version(
+        self,
+        drive: str,
+        key: str,
+        version_id: str,
+        if_version: Optional[str] = None,
+        if_match: Optional[str] = None,
+    ) -> RestoreResult:
+        """Makes ``version_id``'s content the new head (history is kept)."""
+        res = self._send(
+            "POST",
+            drive,
+            key,
+            query=[("x-s3sdk-restore", ""), ("versionId", version_id)],
+            headers=_precondition_headers(if_version, if_match),
+        )
+        w = _write_result(res)
+        return RestoreResult(w.version_id, w.etag, w.size, restored_from=_header(res, "x-s3sdk-restored-from"))
+
+    # ---- transport ------------------------------------------------------------------------------
+
+    def _send(
+        self,
+        method: str,
+        drive: str,
+        key: str = "",
+        *,
+        query: Sequence[tuple[str, str]] = (),
+        headers: Optional[Mapping[str, str]] = None,
+        body: bytes = b"",
+    ) -> urllib3.BaseHTTPResponse:
+        """Sends a SigV4-signed path-style request; raises S3SDKError on a non-2xx response."""
+        path = f"{self._base_path}/{quote(drive, safe='')}"
+        if key:
+            path += "/" + _encode_key(key)
+        qs = "&".join(
+            quote(k, safe="") if v == "" else f"{quote(k, safe='')}={quote(v, safe='')}" for k, v in query
+        )
+        url = f"{self._scheme}://{self._netloc}{path}"
+        if qs:
+            url += "?" + qs
+
+        req_headers: dict[str, str] = {"X-Amz-Content-SHA256": hashlib.sha256(body).hexdigest()}
+        req_headers.update(headers or {})
+        if body or method in ("PUT", "POST"):
+            req_headers["Content-Length"] = str(len(body))
+        request = AWSRequest(method=method, url=url, headers=req_headers, data=body)
+        # S3SigV4Auth leaves the (already RFC 3986-encoded) path as-is instead of double-encoding
+        # it, matching S3's canonical form, and signs the payload hash.
+        S3SigV4Auth(self._credentials, "s3", self._region).add_auth(request)
+
+        res = self._http.request(
+            method,
+            url,
+            body=None if method in ("GET", "HEAD") else body,
+            headers=dict(request.headers.items()),
+            timeout=self._timeout,
+            redirect=False,
+            retries=False,
+        )
+        if not 200 <= res.status < 300:
+            raise _error_from_response(res)
+        return res

spacefs_s3sdk-0.1.0/src/s3sdk/conformance.py

@@ -0,0 +1,253 @@
+"""Runs spec/conformance/cases.json against a live gateway.
+
+    S3SDK_ENDPOINT, S3SDK_ACCESS_KEY_ID, S3SDK_SECRET_ACCESS_KEY
+    python -m s3sdk.conformance ../spec/conformance/cases.json
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import time
+from dataclasses import asdict
+from typing import Any, Optional
+
+from .client import S3SDKClient, S3SDKError
+from .patch import Edit
+
+_VAR = re.compile(r"\$\{(\w+)\}")
+
+
+def _base36(n: int) -> str:
+    digits = "0123456789abcdefghijklmnopqrstuvwxyz"
+    out = ""
+    while True:
+        n, r = divmod(n, 36)
+        out = digits[r] + out
+        if n == 0:
+            return out
+
+
+def _env(name: str) -> str:
+    v = os.environ.get(name)
+    if not v:
+        raise SystemExit(f"{name} is required")
+    return v
+
+
+def subst(v: Any, vars: dict[str, str]) -> Any:
+    if isinstance(v, str):
+
+        def repl(m: re.Match[str]) -> str:
+            name = m.group(1)
+            if name not in vars:
+                raise ValueError(f"undefined variable {name}")
+            return vars[name]
+
+        return _VAR.sub(repl, v)
+    if isinstance(v, list):
+        return [subst(x, vars) for x in v]
+    if isinstance(v, dict):
+        return {k: subst(x, vars) for k, x in v.items()}
+    return v
+
+
+def _body_arg(a: dict[str, Any]) -> bytes:
+    if isinstance(a.get("bodyHex"), str):
+        return bytes.fromhex(a["bodyHex"])
+    body = a.get("body")
+    return body.encode("utf-8") if isinstance(body, str) else b""
+
+
+def _str(v: Any) -> Optional[str]:
+    return v if isinstance(v, str) else None
+
+
+def _num(v: Any) -> Optional[int]:
+    return v if isinstance(v, int) and not isinstance(v, bool) else None
+
+
+def _write(r: Any) -> dict[str, Any]:
+    return {"versionId": r.version_id, "etag": r.etag, "size": r.size}
+
+
+def run(client: S3SDKClient, a: dict[str, Any]) -> dict[str, Any]:
+    op = a.get("op")
+    drive = _str(a.get("drive")) or ""
+    key = _str(a.get("key")) or ""
+    pre = {"if_version": _str(a.get("ifVersion")), "if_match": _str(a.get("ifMatch"))}
+    if op == "createDrive":
+        r = client.create_drive(drive, display_name=_str(a.get("displayName")))
+        return {"driveId": r.drive_id}
+    if op == "forkDrive":
+        f = client.fork_drive(_str(a.get("source")) or "", drive)
+        return {"driveId": f.drive_id, "sourceDriveId": f.source_drive_id, "forkPoint": f.fork_point.isoformat()}
+    if op == "describeDrive":
+        d = client.describe_drive(drive)
+        return {
+            "driveId": d.drive_id,
+            "alias": d.alias,
+            "displayName": d.display_name,
+            "forkOf": d.fork_of.drive_id if d.fork_of else None,
+            "forks": d.forks,
+        }
+    if op == "listDrives":
+        return {"names": [d.name for d in client.list_drives()]}
+    if op == "deleteDrive":
+        client.delete_drive(drive, hard=a.get("hard") is True)
+        return {}
+    if op == "putObject":
+        r = client.put_object(
+            drive, key, _body_arg(a), if_none_match="*" if a.get("ifNoneMatch") == "*" else None, **pre
+        )
+        return _write(r)
+    if op == "getObject":
+        o = client.get_object(drive, key, version_id=_str(a.get("versionId")), as_of=_str(a.get("asOf")))
+        return {"body": o.body, "versionId": o.version_id, "etag": o.etag, "size": o.size}
+    if op == "readRange":
+        body = client.read_range(
+            drive, key, _num(a.get("offset")) or 0, length=_num(a.get("length")), version_id=_str(a.get("versionId"))
+        )
+        return {"body": body}
+    if op == "writeAt":
+        return _write(client.write_at(drive, key, _num(a.get("offset")) or 0, _body_arg(a), size=_num(a.get("size")), **pre))
+    if op == "patch":
+        edits = [Edit(_num(e.get("offset")) or 0, _body_arg(e)) for e in a["edits"]]
+        return _write(client.patch(drive, key, edits, size=_num(a.get("size")), **pre))
+    if op == "rawPatch":
+        return _write(client.raw_patch(drive, key, _body_arg(a), size=_num(a.get("size")), **pre))
+    if op == "truncate":
+        return _write(client.truncate(drive, key, _num(a.get("size")) or 0, **pre))
+    if op == "listVersions":
+        vs = client.list_versions(drive, key)
+        latest = next((v for v in vs if v.is_latest), None)
+        return {
+            "count": len(vs),
+            "operations": [v.operation for v in vs],
+            "versionIds": [v.version_id for v in vs],
+            "latest": latest.version_id if latest else None,
+            "latestTime": latest.last_modified_raw if latest else None,
+        }
+    if op == "restoreVersion":
+        r = client.restore_version(drive, key, _str(a.get("versionId")) or "", **pre)
+        return {**_write(r), "restoredFrom": r.restored_from}
+    if op == "listObjects":
+        lr = client.list_objects(drive, prefix=_str(a.get("prefix")), delimiter=_str(a.get("delimiter")))
+        return asdict(lr)
+    if op == "deleteObject":
+        client.delete_object(drive, key)
+        return {}
+    if op == "sleep":
+        time.sleep((_num(a.get("ms")) or 0) / 1000)
+        return {}
+    raise ValueError(f"unknown op {op!r}")
+
+
+def check(result: dict[str, Any], expect: dict[str, Any]) -> None:
+    for field, want in expect.items():
+        if field == "body":
+            got: Any = result["body"].decode("utf-8", "replace")
+        elif field == "bodyHex":
+            got = result["body"].hex()
+        elif field == "contains":
+            if want not in result["names"]:
+                raise AssertionError(f"names does not contain {want!r}")
+            continue
+        else:
+            if field not in result:
+                raise AssertionError(f"{field}: missing from result")
+            got = result[field]
+        if got != want:
+            raise AssertionError(f"{field}: expected {want!r}, got {got!r}")
+
+
+def _save_value(v: Any) -> str:
+    if v is None:
+        return "null"
+    if isinstance(v, bool):
+        return "true" if v else "false"
+    return str(v)
+
+
+def run_case(client: S3SDKClient, steps: list[dict[str, Any]], vars: dict[str, str]) -> None:
+    for si, raw in enumerate(steps):
+        step = subst(raw, vars)
+        where = f"step {si} ({step.get('op')})"
+        result: Optional[dict[str, Any]] = None
+        error: Optional[BaseException] = None
+        try:
+            result = run(client, step)
+        except Exception as e:  # noqa: BLE001 - every failure is reported against the step
+            error = e
+        want_err = step.get("expectError")
+        if want_err is not None:
+            if not isinstance(error, S3SDKError):
+                raise AssertionError(f"{where}: expected {json.dumps(want_err)}, got {repr(error) if error else 'success'}")
+            got: dict[str, Any] = {"status": error.status, "code": error.code}
+            if "currentVersionId" in want_err:
+                got["currentVersionId"] = error.current_version_id
+            if got != want_err:
+                raise AssertionError(
+                    f"{where}: expected error {json.dumps(want_err)}, got {json.dumps(got)} ({error.message})"
+                )
+            continue
+        if error is not None:
+            raise AssertionError(f"{where}: {error!r}") from error
+        assert result is not None
+        try:
+            if step.get("expect"):
+                check(result, step["expect"])
+        except AssertionError as e:
+            raise AssertionError(f"{where}: {e}") from None
+        for name, field in (step.get("save") or {}).items():
+            if field not in result:
+                raise AssertionError(f"{where}: cannot save {name}: result has no field {field}")
+            vars[name] = _save_value(result[field])
+
+
+def cleanup(client: S3SDKClient, vars: dict[str, str]) -> None:
+    # Forks (${drive2}) before parents.
+    for d in (vars["drive2"], vars["drive"]):
+        try:
+            client.delete_drive(d, hard=True)
+        except S3SDKError as e:
+            if e.code != "NoSuchBucket":
+                print(f"     cleanup of {d} failed: {e.status} {e.code} {e.message}", file=sys.stderr)
+        except Exception as e:  # noqa: BLE001
+            print(f"     cleanup of {d} failed: {e!r}", file=sys.stderr)
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    argv = sys.argv[1:] if argv is None else argv
+    path = argv[0] if argv else "../spec/conformance/cases.json"
+    with open(path, encoding="utf-8") as f:
+        doc = json.load(f)
+    client = S3SDKClient(
+        endpoint=_env("S3SDK_ENDPOINT"),
+        access_key_id=_env("S3SDK_ACCESS_KEY_ID"),
+        secret_access_key=_env("S3SDK_SECRET_ACCESS_KEY"),
+    )
+    stamp = _base36(int(time.time() * 1000))
+    cases = doc["cases"]
+    failed = 0
+    for ci, case in enumerate(cases):
+        vars = {"drive": f"p{stamp}-{ci}-1", "drive2": f"p{stamp}-{ci}-2"}
+        try:
+            run_case(client, case["steps"], vars)
+            print(f"ok   {case['name']}", flush=True)
+        except Exception as e:  # noqa: BLE001
+            failed += 1
+            print(f"FAIL {case['name']}: {e}", flush=True)
+        finally:
+            cleanup(client, vars)
+    if failed:
+        print(f"{failed} of {len(cases)} cases failed")
+    else:
+        print(f"all {len(cases)} cases passed")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

spacefs_s3sdk-0.1.0/src/s3sdk/patch.py

@@ -0,0 +1,91 @@
+"""The ``application/vnd.s3sdk.patch`` body codec (spec/PROTOCOL.md §4.2).
+
+Wire format (all integers big-endian)::
+
+    magic    4 bytes  "S3SP"
+    version  u8       1
+    reserved 3 bytes  0
+    count    u32      number of edits (1..=10_000)
+    repeat count times:
+      offset u64
+      length u64
+      bytes  [length]
+"""
+
+from __future__ import annotations
+
+import struct
+from dataclasses import dataclass
+from typing import Sequence, Union
+
+__all__ = ["Edit", "PatchError", "encode_patch", "decode_patch", "CONTENT_TYPE", "MAX_EDITS"]
+
+CONTENT_TYPE = "application/vnd.s3sdk.patch"
+MAGIC = b"S3SP"
+VERSION = 1
+MAX_EDITS = 10_000
+_U64_MAX = (1 << 64) - 1
+
+_HEADER = struct.Struct(">4sB3xI")
+_EDIT = struct.Struct(">QQ")
+
+
+@dataclass(frozen=True)
+class Edit:
+    """Write ``data`` at byte ``offset``. ``str`` data is encoded as UTF-8."""
+
+    offset: int
+    data: Union[bytes, bytearray, memoryview, str]
+
+    def payload(self) -> bytes:
+        return self.data.encode("utf-8") if isinstance(self.data, str) else bytes(self.data)
+
+
+class PatchError(ValueError):
+    """A patch that cannot be encoded, or a body that is not a valid patch."""
+
+
+def encode_patch(edits: Sequence[Edit]) -> bytes:
+    """Encodes ``edits`` (applied in order; later edits win) into a patch body."""
+    if not 1 <= len(edits) <= MAX_EDITS:
+        raise PatchError(f"a patch needs 1..={MAX_EDITS} edits")
+    parts = [_HEADER.pack(MAGIC, VERSION, len(edits))]
+    for e in edits:
+        if not isinstance(e.offset, int) or isinstance(e.offset, bool):
+            raise PatchError("edit offset must be an integer")
+        if e.offset < 0:
+            raise PatchError("edit offset must be non-negative")
+        if e.offset > _U64_MAX:
+            raise PatchError("edit offset exceeds u64")
+        data = e.payload()
+        parts.append(_EDIT.pack(e.offset, len(data)))
+        parts.append(data)
+    return b"".join(parts)
+
+
+def decode_patch(body: Union[bytes, bytearray, memoryview]) -> list[Edit]:
+    """Decodes a patch body; raises :class:`PatchError` if it is malformed."""
+    body = bytes(body)
+    if len(body) < _HEADER.size:
+        raise PatchError("invalid patch: body shorter than header")
+    magic, version, count = _HEADER.unpack_from(body, 0)
+    if magic != MAGIC:
+        raise PatchError("invalid patch: bad magic")
+    if version != VERSION:
+        raise PatchError("invalid patch: unsupported version")
+    if not 1 <= count <= MAX_EDITS:
+        raise PatchError("invalid patch: edit count out of range")
+    edits: list[Edit] = []
+    pos = _HEADER.size
+    for _ in range(count):
+        if len(body) - pos < _EDIT.size:
+            raise PatchError("invalid patch: truncated edit header")
+        offset, length = _EDIT.unpack_from(body, pos)
+        pos += _EDIT.size
+        if length > len(body) - pos:
+            raise PatchError("invalid patch: truncated edit payload")
+        edits.append(Edit(offset, body[pos : pos + length]))
+        pos += length
+    if pos != len(body):
+        raise PatchError("invalid patch: trailing bytes")
+    return edits

spacefs_s3sdk-0.1.0/tests/test_patch.py

@@ -0,0 +1,77 @@
+import struct
+
+import pytest
+
+from s3sdk import Edit, PatchError, decode_patch, encode_patch
+
+
+def test_roundtrip():
+    body = encode_patch([Edit(0, "hello"), Edit(2**40, b""), Edit(7, b"\xff")])
+    assert [(e.offset, e.data) for e in decode_patch(body)] == [
+        (0, b"hello"),
+        (2**40, b""),
+        (7, b"\xff"),
+    ]
+
+
+def test_wire_format_matches_spec():
+    body = encode_patch([Edit(3, b"ab")])
+    assert body.hex() == (
+        "53335350" "01000000" "00000001" "0000000000000003" "0000000000000002" "6162"
+    )
+
+
+def test_max_offset():
+    body = encode_patch([Edit(2**64 - 1, b"x")])
+    assert decode_patch(body)[0].offset == 2**64 - 1
+
+
+def _header(count: int, magic: bytes = b"S3SP", version: int = 1) -> bytes:
+    return struct.pack(">4sB3xI", magic, version, count)
+
+
+def test_rejects_truncated_bodies():
+    good = encode_patch([Edit(3, b"abc")])
+    for n in range(len(good)):
+        with pytest.raises(PatchError):
+            decode_patch(good[:n])
+
+
+def test_rejects_trailing_bytes():
+    good = encode_patch([Edit(3, b"abc")])
+    with pytest.raises(PatchError, match="trailing"):
+        decode_patch(good + b"\x00")
+
+
+def test_rejects_bad_magic_and_version():
+    tail = struct.pack(">QQ", 0, 1) + b"x"
+    with pytest.raises(PatchError, match="magic"):
+        decode_patch(_header(1, magic=b"S3SDKQ") + tail)
+    with pytest.raises(PatchError, match="version"):
+        decode_patch(_header(1, version=2) + tail)
+    # The conformance suite's malformed body (version 2).
+    with pytest.raises(PatchError):
+        decode_patch(bytes.fromhex("53335350020000000000000100"))
+
+
+def test_rejects_edit_count_out_of_range():
+    with pytest.raises(PatchError, match="count"):
+        decode_patch(_header(0))
+    with pytest.raises(PatchError, match="count"):
+        decode_patch(_header(10_001) + (struct.pack(">QQ", 0, 0) * 10_001))
+
+
+def test_accepts_max_edit_count():
+    body = encode_patch([Edit(i, b"") for i in range(10_000)])
+    assert len(decode_patch(body)) == 10_000
+
+
+def test_encode_rejects_bad_input():
+    with pytest.raises(PatchError):
+        encode_patch([])
+    with pytest.raises(PatchError):
+        encode_patch([Edit(0, b"")] * 10_001)
+    with pytest.raises(PatchError):
+        encode_patch([Edit(-1, b"x")])
+    with pytest.raises(PatchError):
+        encode_patch([Edit(2**64, b"x")])