floe-python 0.8.0__py3-none-any.whl

floe/storage/local.py

@@ -0,0 +1,175 @@
+"""Local storage client, ported from `io/storage/providers/local.rs` (`LocalClient`)."""
+
+import hashlib
+import os
+import shutil
+import time
+from collections.abc import Generator
+from contextlib import contextmanager
+from pathlib import Path
+
+from floe.errors import FloeStorageError
+from floe.storage.paths import normalize_local_path
+from floe.storage.types import ObjectRef, StoredObject
+
+_LOCK_STALE_TTL_S = 5 * 60
+_LOCK_TIMEOUT_S = 10
+
+
+def _path(uri: str) -> Path:
+    return Path(uri.removeprefix("local://"))
+
+
+def _version(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _lock_is_stale(lock: Path) -> bool:
+    try:
+        acquired_ms = int(lock.read_text().split(":")[0].strip())
+        age = time.time() - acquired_ms / 1000
+    except (ValueError, OSError):
+        try:
+            age = time.time() - lock.stat().st_mtime
+        except OSError:
+            return False
+    return age > _LOCK_STALE_TTL_S
+
+
+def _break_stale_lock(lock: Path) -> bool:
+    """Single-winner break: a hard link to a per-inode claim name succeeds once."""
+    try:
+        ino = lock.stat().st_ino
+        claim = Path(f"{lock}.{ino}.742.claim")
+        os.link(lock, claim)
+    except OSError:  # lock vanished, or another waiter already holds the claim
+        return False
+    try:
+        # the lock may have been released and retaken between the check and the link
+        if claim.stat().st_ino == ino and _lock_is_stale(claim):
+            lock.unlink(missing_ok=True)
+            return True
+        return False
+    finally:
+        # ponytail: a crash before this unlink leaves the claim; the lock then times out
+        claim.unlink(missing_ok=True)
+
+
+@contextmanager
+def _file_lock(base: Path) -> Generator[None]:
+    lock = Path(f"{base}.lock")
+    deadline = time.monotonic() + _LOCK_TIMEOUT_S
+    while True:
+        try:
+            fd = os.open(lock, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
+        except FileExistsError:
+            if _lock_is_stale(lock) and _break_stale_lock(lock):
+                continue
+            if time.monotonic() >= deadline:
+                raise FloeStorageError(
+                    f"timed out acquiring local state lock {lock}"
+                ) from None
+            time.sleep(0.01)
+            continue
+        with os.fdopen(fd, "w") as f:
+            f.write(f"{time.time_ns() // 1_000_000}:{os.getpid()}")
+        break
+    try:
+        yield
+    finally:
+        lock.unlink(missing_ok=True)
+
+
+class LocalClient:
+    def resolve_uri(self, path: str) -> str:
+        return f"local://{normalize_local_path(Path.cwd() / path)}"
+
+    def list(self, prefix: str, start_after: str = "") -> list[ObjectRef]:
+        p = Path(prefix)
+        if p.is_file():
+            return [ObjectRef(self.resolve_uri(prefix), prefix, None, None)]
+        if not p.exists():
+            return []
+        keys = [str(c) for c in p.iterdir() if c.is_file() and str(c) > start_after]
+        return sorted(
+            (ObjectRef(self.resolve_uri(k), k, None, None) for k in keys),
+            key=lambda r: r.uri,
+        )
+
+    def copy(self, src_uri: str, dst_uri: str) -> None:
+        src, dst = _path(src_uri), _path(dst_uri)
+        try:
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            mode = src.stat().st_mode & 0o777
+            fd = os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
+            os.fchmod(fd, mode)  # #779: the open mode applies on create only
+            with src.open("rb") as fin, os.fdopen(fd, "wb") as fout:
+                shutil.copyfileobj(fin, fout)
+        except OSError as e:
+            raise FloeStorageError(f"local copy failed from {src} to {dst}: {e}") from e
+
+    def write(self, uri: str, body: bytes) -> None:
+        path = _path(uri)
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(body)
+        except OSError as e:
+            raise FloeStorageError(f"local write failed for {path}: {e}") from e
+
+    def delete(self, uri: str) -> None:
+        path = _path(uri)
+        try:
+            path.unlink(missing_ok=True)
+        except OSError as e:
+            raise FloeStorageError(f"local delete failed for {path}: {e}") from e
+
+    def read(self, uri: str) -> StoredObject | None:
+        path = _path(uri)
+        try:
+            if not path.exists():
+                return None
+            with _file_lock(path):
+                if not path.exists():
+                    return None
+                return StoredObject(path.read_bytes(), _version(path))
+        except OSError as e:
+            raise FloeStorageError(f"local read failed for {path}: {e}") from e
+
+    def write_conditional(
+        self, uri: str, expected_version: str | None, body: bytes
+    ) -> str | None:
+        path = _path(uri)
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            with _file_lock(path):
+                current = _version(path) if path.exists() else None
+                if current != expected_version:
+                    return None
+                tmp = path.with_name(f"{path.name}.{os.getpid()}.742.tmp")
+                try:
+                    with tmp.open("wb") as f:
+                        if current is not None:
+                            shutil.copymode(path, tmp)
+                        f.write(body)
+                        f.flush()
+                        os.fsync(f.fileno())
+                    os.replace(tmp, path)
+                finally:
+                    tmp.unlink(missing_ok=True)
+                return _version(path)
+        except OSError as e:
+            raise FloeStorageError(f"local write failed for {path}: {e}") from e
+
+    def delete_conditional(self, uri: str, expected_version: str | None) -> bool:
+        """False on a version conflict; `None` expects no file."""
+        path = _path(uri)
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            with _file_lock(path):
+                current = _version(path) if path.exists() else None
+                if current != expected_version:
+                    return False
+                path.unlink(missing_ok=True)
+                return True
+        except OSError as e:
+            raise FloeStorageError(f"local delete failed for {path}: {e}") from e

floe/storage/partitioning.py

@@ -0,0 +1,116 @@
+"""D28: `source.partitioning`, the date and batch of each input from its directory, and the load order."""
+
+import re
+from datetime import date, datetime, timedelta
+from typing import NamedTuple
+
+from floe.config.types import SourcePartitioning
+from floe.errors import FloeRunError
+from floe.state import EntityState
+from floe.storage.inputs import InputFile, ResolvedInputs
+
+_TOKEN = re.compile(r"\{(yyyy|MM|dd|HH|batch)\}")  # validated: no other token
+_GROUP = {"yyyy": r"\d{4}", "MM": r"\d{2}", "dd": r"\d{2}", "HH": r"\d{2}", "batch": r"[^/]+"}  # fmt: skip
+_FRACTION = re.compile(r"(?:\.(\d+))?Z$")
+_ARCHIVED = re.compile(r"__run-(?!.*__run-).*__src-[0-9a-f]{16}(?=(?:\.[^.]*)?$)")  # D29 suffix  # fmt: skip
+
+
+class Partition(NamedTuple):
+    when: date | datetime  # a datetime when the pattern has `{HH}`
+    batch: str | None
+
+
+def source_root(uri: str) -> str:
+    """The directory partition paths are relative to: the source URI, cut before its first glob."""
+    cut = next((i for i, c in enumerate(uri) if c in "*?["), None)
+    return (uri if cut is None else uri[: uri.rfind("/", 0, cut)]).rstrip("/")
+
+
+def partition_of(p: SourcePartitioning, entity: str, root: str, uri: str) -> Partition:  # fmt: skip
+    """Match the input's directory, relative to `root`, against the whole pattern."""
+    parts = _TOKEN.split(p.pattern.strip("/"))
+    regex = "".join(re.escape(s) if i % 2 == 0 else f"(?P<{s}>{_GROUP[s]})" for i, s in enumerate(parts))  # fmt: skip
+    folder = uri.removeprefix(f"{root}/").rpartition("/")[0] if uri.startswith(f"{root}/") else None  # fmt: skip
+    match = None if folder is None else re.fullmatch(regex, folder)
+    if match is None:
+        raise FloeRunError(f"entity.name={entity} input {uri} does not match source.partitioning.pattern={p.pattern}")  # fmt: skip
+    hour = match.groupdict().get("HH")
+    try:
+        when = datetime(int(match["yyyy"]), int(match["MM"]), int(match["dd"]), int(hour or 0))  # fmt: skip
+    except (
+        ValueError
+    ) as err:  # the date as written: Python's own wording varies by version
+        written = f"{match['yyyy']}-{match['MM']}-{match['dd']}" + (f"T{hour}" if hour is not None else "")  # fmt: skip
+        raise FloeRunError(f"entity.name={entity} input {uri} has an invalid source.partitioning date: {written}") from err  # fmt: skip
+    return Partition(when if hour is not None else when.date(), match.groupdict().get("batch"))  # fmt: skip
+
+
+def _mtime(file: InputFile) -> str:
+    """RFC 3339 with nine fraction digits, so the strings sort in time order."""
+    return _FRACTION.sub(lambda m: f".{(m[1] or '').ljust(9, '0')}Z", file.source_mtime or "")  # fmt: skip
+
+
+def in_load_order(p: SourcePartitioning, entity: str, root: str, inputs: ResolvedInputs, archived: bool = False, start: date | None = None) -> ResolvedInputs:  # fmt: skip
+    """By date, then batch (earliest modification time, then name), then file name; hash batches don't sort by name.
+
+    D29: `archived` inputs sort by their original name, so a full refresh replays the files in their first order.
+    D30: a file dated before `start` is dropped, whatever the listing returned.
+    """
+    of = {f.source_uri: partition_of(p, entity, root, f.source_uri) for f in inputs.files}  # fmt: skip
+    if start is not None:
+        inputs = ResolvedInputs([f for f in inputs.files if _day(of[f.source_uri].when) >= start], [], inputs.mode)  # fmt: skip
+    first: dict[Partition, str] = {}
+    for f in inputs.files:
+        first[of[f.source_uri]] = min(first.get(of[f.source_uri], "~"), _mtime(f))
+
+    def key(f: InputFile) -> tuple[str, str, str, str]:
+        part = of[f.source_uri]
+        name = _ARCHIVED.sub("", f.source_name) if archived else f.source_name
+        return (part.when.isoformat(), first[part], part.batch or "", name)
+
+    files = sorted(inputs.files, key=key)
+    return ResolvedInputs(files, [f.source_uri for f in files], inputs.mode)
+
+
+def _day(when: date | datetime) -> date:
+    return when.date() if isinstance(when, datetime) else when
+
+
+def _dated(p: SourcePartitioning, entity: str, root: str, uri: str) -> date | None:
+    """The partition date of a state entry; None for one outside the pattern (another source path)."""
+    try:
+        return _day(partition_of(p, entity, root, uri).when)
+    except FloeRunError:
+        return None
+
+
+def window(p: SourcePartitioning, entity: str, root: str, state: EntityState | None) -> date | None:  # fmt: skip
+    """D30: the first date a file-mode run lists, the watermark minus `lookback_days`.
+
+    An outstanding claim holds it back to the claim's date, so a crashed run's files stay listed until it is retried.
+    """
+    if state is None or state.watermark is None:
+        return None
+    start = date.fromisoformat(state.watermark) - timedelta(days=p.lookback_days)
+    return min([start, *(d for u in state.claims if (d := _dated(p, entity, root, u)))])  # fmt: skip
+
+
+def listing_start(p: SourcePartitioning, start: date) -> str | None:
+    """D30: the pattern up to `{dd}` for the day before `start`, which `start` and every later day sort after; None when the date does not lead the pattern.
+
+    The day before, not `start`: SeaweedFS treats `StartAfter` as a directory marker and skips that whole directory.
+    """
+    head, start = p.pattern.strip("/").partition("{dd}")[0], start - timedelta(days=1)
+    if "{batch}" in head or "{HH}" in head or not 0 <= head.find("{yyyy}") < head.find("{MM}"):  # fmt: skip
+        return None
+    return head.replace("{yyyy}", f"{start.year:04d}").replace("{MM}", f"{start.month:02d}") + f"{start.day:02d}"  # fmt: skip
+
+
+def advance(p: SourcePartitioning, entity: str, root: str, state: EntityState, promoted: list[str]) -> None:  # fmt: skip
+    """D30, on promote: the watermark moves to the latest date promoted, never back (#949 will prune `files`).
+
+    Without one (a first run, or after a failed full refresh), every file in the state counts.
+    """
+    known = promoted if state.watermark else [*promoted, *state.files]
+    days = [d.isoformat() for u in known if (d := _dated(p, entity, root, u))]
+    state.watermark = max([state.watermark or "", *days]) or None

floe/storage/paths.py

@@ -0,0 +1,69 @@
+"""Path and URI helpers, ported from `io/storage/core/{paths,uri}.rs`."""
+
+from pathlib import Path
+
+from floe.errors import FloeConfigError
+
+
+def parse_s3_uri(uri: str) -> tuple[str, str]:
+    if not uri.startswith("s3://"):
+        raise FloeConfigError(f"expected s3 uri, got {uri}")
+    bucket, _, key = uri.removeprefix("s3://").partition("/")
+    if not bucket:
+        raise FloeConfigError(f"missing bucket in s3 uri: {uri}")
+    return bucket, key
+
+
+def normalize_local_path(path: Path) -> Path:
+    """Collapse `.` and `..` lexically, but keep a `..` that follows a symlink."""
+    out = Path(path.anchor)
+    for part in path.parts[1 if path.anchor else 0 :]:
+        if part != "..":
+            out /= part
+        elif out.name not in ("", "..") and not out.is_symlink():
+            out = out.parent
+        elif not (path.is_absolute() and out == Path(out.anchor)):
+            out /= ".."
+    if out == Path():
+        return Path(path.anchor or ".")
+    return out
+
+
+_MAX_FILENAME_COMPONENT_BYTES = 255
+_MAX_ARCHIVE_RUN_COMPONENT_BYTES = 48
+
+
+def fnv1a_hex(value: str) -> str:
+    h = 0xCBF29CE484222325
+    for byte in value.encode():
+        h = ((h ^ byte) * 0x100000001B3) & 0xFFFFFFFFFFFFFFFF
+    return f"{h:016x}"
+
+
+def _truncate_utf8(value: str, max_bytes: int) -> str:
+    return value.encode()[:max_bytes].decode(errors="ignore")
+
+
+def _run_component(run_id: str) -> str:
+    safe = "".join(
+        c if c.isascii() and (c.isalnum() or c in "-_.") else "_" for c in run_id
+    )
+    safe = safe.strip("_") or "run"
+    if len(safe) <= _MAX_ARCHIVE_RUN_COMPONENT_BYTES:
+        return safe
+    return f"{_truncate_utf8(safe, _MAX_ARCHIVE_RUN_COMPONENT_BYTES)}_{fnv1a_hex(run_id)[:8]}"
+
+
+def archive_relative_path_for_run(
+    entity: str, filename: str, run_id: str, source_uri: str
+) -> str:
+    name = filename.rsplit("/", 1)[-1] or filename
+    stem, _, ext = name.rpartition(".")
+    if not ext or not stem:
+        stem, ext = name, ""
+    suffix = f"__run-{_run_component(run_id)}__src-{fnv1a_hex(source_uri)}"
+    ext_suffix = f".{ext}" if ext else ""
+    available = _MAX_FILENAME_COMPONENT_BYTES - len(suffix) - len(ext_suffix.encode())
+    archived = f"{_truncate_utf8(stem, max(available, 0))}{suffix}{ext_suffix}"
+    entity = entity.strip("/")
+    return f"{entity}/{archived}" if entity else archived

floe/storage/resolve.py

@@ -0,0 +1,55 @@
+"""Storage reference resolution: name, else `storages.default`, else local (Rust `StorageResolver::resolve_path`)."""
+
+from pathlib import Path
+
+from floe.config.storage import StorageDefinition, StoragesConfig
+from floe.errors import FloeConfigError, FloeRunError
+from floe.storage.local import LocalClient
+from floe.storage.paths import normalize_local_path, parse_s3_uri
+from floe.storage.s3 import S3Client
+
+
+def _definition(
+    storages: StoragesConfig | None, name: str | None, field: str
+) -> StorageDefinition | None:
+    """`None` is the implicit local storage; unknown names raise, never fall back to local."""
+    where = name or (storages and storages.default) or "local"
+    if storages is None and where == "local":
+        return None
+    found = storages and next((d for d in storages.definitions if d.name == where), None)  # fmt: skip
+    if not found:
+        raise FloeRunError(f"{field}={where} references unknown storage")
+    return found
+
+
+def resolve_path(
+    storages: StoragesConfig | None,
+    config_dir: Path,
+    name: str | None,
+    raw: str,
+    field: str,
+) -> tuple[str, StorageDefinition | None]:
+    """`raw` as a `local://` or `s3://` URI, with its storage definition."""
+    definition = _definition(storages, name, field)
+    if definition is None or definition.type == "local":
+        if "://" in raw:
+            raise FloeConfigError(f"{field} must be a local path (got {raw})")
+        return f"local://{normalize_local_path(config_dir / raw)}", definition
+    if definition.type != "s3":
+        raise FloeRunError(f"{field}={definition.name} (type {definition.type}) is not supported")  # fmt: skip
+    if definition.bucket is None:
+        raise FloeConfigError(f"storage {definition.name} requires bucket for type s3")  # fmt: skip
+    if raw.startswith("s3://"):
+        bucket, key = parse_s3_uri(raw)
+        if bucket != definition.bucket:
+            raise FloeConfigError(f"storage {definition.name} bucket mismatch: {bucket}")  # fmt: skip
+    else:
+        key = "/".join(p for p in ((definition.prefix or "").strip("/"), raw.lstrip("/")) if p)  # fmt: skip
+    return f"s3://{definition.bucket}/{key}" if key else f"s3://{definition.bucket}", definition  # fmt: skip
+
+
+def storage_client(definition: StorageDefinition | None) -> LocalClient | S3Client:
+    """S3 credentials come from the environment, as in Rust."""
+    if definition is None or definition.type == "local":
+        return LocalClient()
+    return S3Client(definition.bucket or "", definition.region, definition.endpoint, definition.path_style_access)  # fmt: skip

floe/storage/s3.py

@@ -0,0 +1,182 @@
+"""S3 storage client, ported from `io/storage/providers/s3.rs` (`S3Client`)."""
+
+import os
+import shutil
+from datetime import UTC, datetime
+from pathlib import Path
+
+import boto3
+from botocore.config import Config
+from botocore.exceptions import BotoCoreError, ClientError
+
+from floe.errors import FloeStorageError
+from floe.storage.paths import fnv1a_hex, parse_s3_uri
+from floe.storage.types import ObjectRef, StoredObject
+
+_CONFLICT = ("PreconditionFailed", "ConditionalRequestConflict", "412", "409")
+_NOT_FOUND = ("NoSuchKey", "NotFound", "404")
+
+
+def resolve_path_style_access(configured: bool | None) -> bool:
+    if configured is not None:
+        return configured
+    env = os.environ.get("AWS_S3_FORCE_PATH_STYLE", "").strip()
+    return env == "1" or env.lower() == "true"
+
+
+def _code(err: ClientError) -> str:
+    return str(err.response.get("Error", {}).get("Code", ""))
+
+
+def _is(err: ClientError, codes: tuple[str, ...]) -> bool:
+    status = str(err.response.get("ResponseMetadata", {}).get("HTTPStatusCode", ""))
+    return _code(err) in codes or status in codes
+
+
+def _iso(ts: datetime | None) -> str | None:
+    return ts and ts.astimezone(UTC).isoformat().replace("+00:00", "Z")
+
+
+def temp_path_for_key(temp_dir: Path, key: str) -> Path:
+    name = key.rsplit("/", 1)[-1] or "object"
+    safe = "".join(
+        c if c.isascii() and (c.isalnum() or c in "-_.") else "_" for c in name
+    )
+    return temp_dir / f"{fnv1a_hex(key)}_{safe}"
+
+
+class S3Client:
+    def __init__(
+        self,
+        bucket: str,
+        region: str | None = None,
+        endpoint: str | None = None,
+        path_style_access: bool | None = None,
+    ) -> None:
+        self.bucket = bucket
+        self._client = boto3.session.Session().client(  # pyright: ignore[reportUnknownMemberType]  # stubs cover s3 only
+            "s3",
+            region_name=region,
+            endpoint_url=endpoint,
+            config=Config(
+                s3={"addressing_style": "path"}
+                if resolve_path_style_access(path_style_access)
+                else None,
+                # boto3 >= 1.36 adds CRC checksums that S3-compatible services may reject.
+                request_checksum_calculation="when_required",
+                response_checksum_validation="when_required",
+            ),
+        )
+        self.endpoint: str = self._client.meta.endpoint_url.rstrip("/")  # pyright: ignore[reportUnknownMemberType]  # identifies the store behind a bucket name
+
+    def resolve_uri(self, path: str) -> str:
+        key = path.lstrip("/")
+        return f"s3://{self.bucket}/{key}" if key else f"s3://{self.bucket}"
+
+    def list(self, prefix: str, start_after: str = "") -> list[ObjectRef]:
+        """Keys under `prefix`; D30: only those after `start_after`."""
+        try:
+            paginator = self._client.get_paginator("list_objects_v2")
+            pages = (
+                paginator.paginate(Bucket=self.bucket, Prefix=prefix, StartAfter=start_after)
+                if start_after
+                else paginator.paginate(Bucket=self.bucket, Prefix=prefix)
+            )  # fmt: skip
+            return [
+                ObjectRef(
+                    f"s3://{self.bucket}/{o['Key']}",
+                    o["Key"],
+                    _iso(o.get("LastModified")),
+                    o.get("Size"),
+                )
+                for page in pages
+                for o in page.get("Contents", [])
+                if "Key" in o
+            ]
+        except (ClientError, BotoCoreError) as e:
+            raise FloeStorageError(
+                f"s3 list objects failed for bucket {self.bucket}: {e}"
+            ) from e
+
+    def download(self, uri: str, temp_dir: Path) -> Path:
+        bucket, key = parse_s3_uri(uri)
+        dest = temp_path_for_key(temp_dir, key)
+        try:
+            body = self._client.get_object(Bucket=bucket, Key=key)["Body"]
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            with dest.open("wb") as f:
+                shutil.copyfileobj(body, f)
+        except (ClientError, BotoCoreError, OSError) as e:
+            raise FloeStorageError(f"s3 get object failed: {e}") from e
+        return dest
+
+    def write(self, uri: str, body: bytes) -> None:
+        bucket, key = parse_s3_uri(uri)
+        try:
+            self._client.put_object(Bucket=bucket, Key=key, Body=body)
+        except (ClientError, BotoCoreError) as e:
+            raise FloeStorageError(f"s3 put object failed: {e}") from e
+
+    def copy(self, src_uri: str, dst_uri: str) -> None:
+        (sb, sk), (db, dk) = parse_s3_uri(src_uri), parse_s3_uri(dst_uri)
+        try:
+            self._client.copy_object(
+                Bucket=db, Key=dk, CopySource={"Bucket": sb, "Key": sk}
+            )
+        except (ClientError, BotoCoreError) as e:
+            raise FloeStorageError(f"s3 copy object failed: {e}") from e
+
+    def delete(self, uri: str) -> None:
+        bucket, key = parse_s3_uri(uri)
+        try:
+            self._client.delete_object(Bucket=bucket, Key=key)
+        except (ClientError, BotoCoreError) as e:
+            raise FloeStorageError(f"s3 delete object failed: {e}") from e
+
+    def read(self, uri: str) -> StoredObject | None:
+        bucket, key = parse_s3_uri(uri)
+        try:
+            resp = self._client.get_object(Bucket=bucket, Key=key)
+            return StoredObject(resp["Body"].read(), resp.get("ETag", ""))
+        except ClientError as e:
+            if _is(e, _NOT_FOUND):
+                return None
+            raise FloeStorageError(f"s3 get object failed: {e}") from e
+        except BotoCoreError as e:
+            raise FloeStorageError(f"s3 get object failed: {e}") from e
+
+    def write_conditional(
+        self, uri: str, expected_version: str | None, body: bytes
+    ) -> str | None:
+        bucket, key = parse_s3_uri(uri)
+        try:
+            if expected_version is None:
+                resp = self._client.put_object(
+                    Bucket=bucket, Key=key, Body=body, IfNoneMatch="*"
+                )
+            else:
+                resp = self._client.put_object(
+                    Bucket=bucket, Key=key, Body=body, IfMatch=expected_version
+                )
+            return resp.get("ETag", "")
+        except ClientError as e:
+            if _is(e, _CONFLICT):
+                return None
+            raise FloeStorageError(f"s3 put object failed: {e}") from e
+        except BotoCoreError as e:
+            raise FloeStorageError(f"s3 put object failed: {e}") from e
+
+    def delete_conditional(self, uri: str, expected_version: str | None) -> bool:
+        """An If-Match delete; False on a conflict. `None` deletes nothing, as Rust: an object there is a concurrent runner's."""
+        if expected_version is None:
+            return True
+        bucket, key = parse_s3_uri(uri)
+        try:
+            self._client.delete_object(Bucket=bucket, Key=key, IfMatch=expected_version)
+            return True
+        except ClientError as e:
+            if _is(e, _CONFLICT):
+                return False
+            raise FloeStorageError(f"s3 delete object failed: {e}") from e
+        except BotoCoreError as e:
+            raise FloeStorageError(f"s3 delete object failed: {e}") from e

floe/storage/types.py

@@ -0,0 +1,33 @@
+"""Storage client types, ported from `io/storage/mod.rs` and `planner.rs`."""
+
+from pathlib import Path
+from typing import NamedTuple, Protocol
+
+
+class ObjectRef(NamedTuple):
+    uri: str
+    key: str
+    last_modified: str | None
+    size: int | None
+
+
+class StoredObject(NamedTuple):
+    body: bytes
+    version: str
+
+
+class StorageClient(Protocol):
+    def resolve_uri(self, path: str) -> str: ...
+    def list(self, prefix: str, start_after: str = "") -> list[ObjectRef]: ...
+    def write(self, uri: str, body: bytes) -> None: ...
+    def copy(self, src_uri: str, dst_uri: str) -> None: ...
+    def delete(self, uri: str) -> None: ...
+    def read(self, uri: str) -> StoredObject | None: ...
+    def write_conditional(
+        self, uri: str, expected_version: str | None, body: bytes
+    ) -> str | None: ...
+    def delete_conditional(self, uri: str, expected_version: str | None) -> bool: ...
+
+
+class Downloader(Protocol):
+    def download(self, uri: str, temp_dir: Path) -> Path: ...