mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/fill_staging.py

@@ -0,0 +1,1105 @@
+"""Descriptor-rooted private immutable staging for resumable catalog fills."""
+
+from __future__ import annotations
+
+import contextlib
+import fcntl
+import os
+import secrets
+import stat
+import tempfile
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    canonical_sha256,
+    parse_canonical_json,
+    sha256_bytes,
+)
+from mostlyright.data_harness.sources.contracts import SourceContractError
+
+STATE_SCHEMA = "harness-catalog-fill-state.v2"
+SHARD_SCHEMA = "harness-catalog-fill-shard.v1"
+MAX_STATE_BYTES = 64 * 1024
+MAX_SHARD_BYTES = 64 * 1024 * 1024
+MAX_INDEX_SEGMENT_BYTES = 1024 * 1024
+MAX_INDEX_SEGMENT_ITEMS = 4_096
+MAX_BLOB_BYTES = 16 * 1024 * 1024
+MAX_HISTORY_DIGESTS = 1_000_000
+MAX_INDEX_ITEMS = 20_000_000
+_OPEN_DIRECTORY = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
+_OPEN_MEMBER = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
+_SHARD_KINDS = frozenset(
+    {
+        "classification",
+        "history",
+        "index",
+        "loop_set",
+        "normalized",
+        "page",
+        "partition_run",
+        "pass_map",
+        "response",
+    }
+)
+_BLOB_KINDS = frozenset({"raw_response"})
+
+
+class CatalogFillRefused(SourceContractError):
+    """A stable local fill/configuration/staging refusal."""
+
+
+def shard_byte_limit(kind: str) -> int:
+    """Return the live immutable-shard byte ceiling for preflight composition."""
+
+    return MAX_INDEX_SEGMENT_BYTES if kind == "index" else MAX_SHARD_BYTES
+
+
+class FillStaging:
+    """One private staging root retained by descriptor for the whole locked operation."""
+
+    def __init__(self, root: Path, *, root_descriptor: int | None = None) -> None:
+        self.root = Path(root)
+        self._retained_root_descriptor = root_descriptor
+        self.state_path = self.root / "state.json"
+        self.lock_path = self.root / ".lock"
+        self.shard_root = self.root / "shards"
+        self._root_fd = -1
+        self._shards_fd = -1
+        self._kind_fds: dict[str, int] = {}
+        self._chain: list[tuple[int, str, int, tuple[int, int, int]]] = []
+        self._owned_fds: list[int] = []
+        self._active_histories: dict[str, list[str]] = {}
+
+    @contextlib.contextmanager
+    def locked(self) -> Iterator[None]:
+        if self._root_fd >= 0:
+            raise CatalogFillRefused(
+                "FILL_STAGING_LOCK", "fill.staging", "staging lock is not reentrant"
+            )
+        lock_fd = -1
+        try:
+            self._open_retained_tree()
+            flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)
+            lock_fd = os.open(".lock", flags, 0o600, dir_fd=self._root_fd)
+            info = os.fstat(lock_fd)
+            if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
+                raise CatalogFillRefused(
+                    "FILL_STAGING_PATH",
+                    "fill.staging.lock",
+                    "lock must be a single-link regular file",
+                )
+            fcntl.flock(lock_fd, fcntl.LOCK_EX)
+            self._verify_tree()
+            yield
+            self._verify_tree()
+        finally:
+            if lock_fd >= 0:
+                os.close(lock_fd)
+            for descriptor in reversed(self._owned_fds):
+                os.close(descriptor)
+            self._owned_fds = []
+            self._chain = []
+            self._kind_fds = {}
+            self._shards_fd = -1
+            self._root_fd = -1
+            self._active_histories = {}
+
+    def load(self) -> dict[str, Any] | None:
+        self._require_locked()
+        return self._load_state(validate_shards=True)
+
+    def write_shard(self, kind: str, payload: dict[str, Any]) -> str:
+        self._require_locked()
+        if kind not in _SHARD_KINDS:
+            raise CatalogFillRefused("FILL_STAGING_KIND", "fill.staging.kind", "unknown shard")
+        document = {"schema_version": SHARD_SCHEMA, "kind": kind, "payload": payload}
+        raw = canonical_json_bytes(document)
+        maximum = shard_byte_limit(kind)
+        if len(raw) > maximum:
+            raise CatalogFillRefused(
+                "FILL_SHARD_LIMIT", "fill.staging.shard", "shard exceeds the byte limit"
+            )
+        digest = sha256_bytes(raw)
+        name = f"{digest}.json"
+        directory_fd = self._kind_fds[kind]
+        self._publish_immutable(
+            directory_fd,
+            name,
+            raw,
+            maximum=len(raw),
+            field="fill.staging.shard",
+        )
+        self._verify_tree()
+        return digest
+
+    def read_shard(self, kind: str, digest: str) -> dict[str, Any]:
+        self._require_locked()
+        if kind not in _SHARD_KINDS or not _is_digest(digest):
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.shard", "shard coordinate is invalid"
+            )
+        try:
+            raw = self._read_regular_at(
+                self._kind_fds[kind],
+                f"{digest}.json",
+                maximum=MAX_INDEX_SEGMENT_BYTES if kind == "index" else MAX_SHARD_BYTES,
+            )
+        except FileNotFoundError as error:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.shard", "referenced shard is absent"
+            ) from error
+        if sha256_bytes(raw) != digest:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.shard", "shard digest mismatch"
+            )
+        try:
+            document = parse_canonical_json(raw)
+        except CanonicalJSONError:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.shard", "shard is not canonical"
+            ) from None
+        if (
+            not isinstance(document, dict)
+            or set(document) != {"schema_version", "kind", "payload"}
+            or document["schema_version"] != SHARD_SCHEMA
+            or document["kind"] != kind
+            or not isinstance(document["payload"], dict)
+        ):
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.shard", "shard envelope is invalid"
+            )
+        return document["payload"]
+
+    def write_blob(self, kind: str, content: bytes) -> str:
+        """Install exact bounded bytes under their digest without a JSON/base64 rewrite."""
+
+        self._require_locked()
+        if kind not in _BLOB_KINDS or not isinstance(content, bytes):
+            raise CatalogFillRefused("FILL_STAGING_KIND", "fill.staging.kind", "unknown blob")
+        if len(content) > MAX_BLOB_BYTES:
+            raise CatalogFillRefused(
+                "FILL_SHARD_LIMIT", "fill.staging.blob", "blob exceeds the byte limit"
+            )
+        digest = sha256_bytes(content)
+        name = f"{digest}.bin"
+        directory_fd = self._kind_fds[kind]
+        self._publish_immutable(
+            directory_fd,
+            name,
+            content,
+            maximum=len(content),
+            field="fill.staging.blob",
+        )
+        return digest
+
+    def _publish_immutable(
+        self,
+        directory_fd: int,
+        name: str,
+        content: bytes,
+        *,
+        maximum: int,
+        field: str,
+    ) -> None:
+        """Atomically install immutable bytes without ever exposing a partial final name."""
+
+        try:
+            existing = self._read_regular_at(directory_fd, name, maximum=maximum)
+        except FileNotFoundError:
+            existing = None
+        except OSError as error:
+            raise CatalogFillRefused(
+                "FILL_STAGING_IO", field, "immutable object lookup failed"
+            ) from error
+        if existing is not None:
+            if existing != content:
+                raise CatalogFillRefused("FILL_SHARD_COLLISION", field, "digest path bytes differ")
+            return
+
+        temporary = f".publish.{os.getpid()}.{secrets.token_hex(16)}.tmp"
+        descriptor = -1
+        installed = False
+        try:
+            flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
+            descriptor = os.open(temporary, flags, 0o600, dir_fd=directory_fd)
+            _write_all(descriptor, content)
+            os.fsync(descriptor)
+            os.close(descriptor)
+            descriptor = -1
+            try:
+                os.link(
+                    temporary,
+                    name,
+                    src_dir_fd=directory_fd,
+                    dst_dir_fd=directory_fd,
+                    follow_symlinks=False,
+                )
+                installed = True
+            except FileExistsError:
+                pass
+        except OSError as error:
+            raise CatalogFillRefused(
+                "FILL_STAGING_IO", field, "immutable object publication failed"
+            ) from error
+        finally:
+            if descriptor >= 0:
+                os.close(descriptor)
+            try:
+                os.unlink(temporary, dir_fd=directory_fd)
+            except FileNotFoundError:
+                pass
+            except OSError as error:
+                raise CatalogFillRefused(
+                    "FILL_STAGING_IO", field, "temporary object cleanup failed"
+                ) from error
+            try:
+                os.fsync(directory_fd)
+            except OSError as error:
+                raise CatalogFillRefused(
+                    "FILL_STAGING_IO", field, "immutable object directory sync failed"
+                ) from error
+
+        try:
+            observed = self._read_regular_at(directory_fd, name, maximum=maximum)
+        except OSError as error:
+            raise CatalogFillRefused(
+                "FILL_STAGING_IO", field, "immutable object readback failed"
+            ) from error
+        if observed != content:
+            code = "FILL_SHARD_READBACK" if installed else "FILL_SHARD_COLLISION"
+            raise CatalogFillRefused(code, field, "immutable object readback differs")
+
+    def read_blob(self, kind: str, digest: str) -> bytes:
+        self._require_locked()
+        if kind not in _BLOB_KINDS or not _is_digest(digest):
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.blob", "blob coordinate is invalid"
+            )
+        try:
+            content = self._read_regular_at(
+                self._kind_fds[kind], f"{digest}.bin", maximum=MAX_BLOB_BYTES
+            )
+        except FileNotFoundError as error:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.blob", "referenced blob is absent"
+            ) from error
+        if sha256_bytes(content) != digest:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.blob", "blob digest mismatch"
+            )
+        return content
+
+    @staticmethod
+    def empty_index_head() -> dict[str, Any]:
+        return {
+            "root_sha256": None,
+            "count": 0,
+            "tail_sha256": None,
+            "segment_count": 0,
+            "descriptor_count": 0,
+        }
+
+    def append_index(
+        self,
+        head: dict[str, Any],
+        *,
+        stream: str,
+        items: list[dict[str, Any]],
+    ) -> dict[str, Any]:
+        """Append one immutable <=4096-item/1-MiB segment to a compact Merkle-style head."""
+
+        self._require_locked()
+        _validate_index_head(head)
+        if (
+            not isinstance(stream, str)
+            or not stream
+            or not isinstance(items, list)
+            or not 1 <= len(items) <= MAX_INDEX_SEGMENT_ITEMS
+            or any(not isinstance(item, dict) for item in items)
+        ):
+            raise CatalogFillRefused(
+                "FILL_INDEX_SEGMENT", "fill.staging.index", "index segment is invalid"
+            )
+        if head["count"] + len(items) > MAX_INDEX_ITEMS:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_LIMIT",
+                "fill.staging.index",
+                "prospective index count exceeds the segmented cap",
+            )
+        payload = {
+            "stream": stream,
+            "previous_sha256": head["tail_sha256"],
+            "start": head["count"],
+            "items": items,
+        }
+        digest = self.write_shard("index", payload)
+        root = canonical_index_root(
+            previous_root=head["root_sha256"],
+            segment_sha256=digest,
+            start=head["count"],
+            item_count=len(items),
+        )
+        return {
+            "root_sha256": root,
+            "count": head["count"] + len(items),
+            "tail_sha256": digest,
+            "segment_count": head["segment_count"] + 1,
+            # Segment descriptors live only in the immutable chain. The bounded head embeds none.
+            "descriptor_count": 0,
+        }
+
+    def iter_index(
+        self,
+        head: dict[str, Any],
+        *,
+        stream: str,
+        start: int = 0,
+    ) -> Iterator[dict[str, Any]]:
+        self._require_locked()
+        _validate_index_head(head)
+        if type(start) is not int or not 0 <= start <= head["count"]:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.index", "index start is invalid"
+            )
+        traversed = 0
+        current = head["tail_sha256"]
+        with tempfile.TemporaryFile(mode="w+b") as coordinates:
+            while current is not None:
+                if traversed >= head["segment_count"] + 1:
+                    raise CatalogFillRefused(
+                        "FILL_CHECKPOINT_CORRUPT", "fill.staging.index", "index chain loops"
+                    )
+                traversed += 1
+                node = self.read_shard("index", current)
+                if (
+                    set(node) != {"stream", "previous_sha256", "start", "items"}
+                    or node["stream"] != stream
+                    or type(node["start"]) is not int
+                    or not isinstance(node["items"], list)
+                    or not 1 <= len(node["items"]) <= MAX_INDEX_SEGMENT_ITEMS
+                    or any(not isinstance(item, dict) for item in node["items"])
+                    or (
+                        node["previous_sha256"] is not None
+                        and not _is_digest(node["previous_sha256"])
+                    )
+                ):
+                    raise CatalogFillRefused(
+                        "FILL_CHECKPOINT_CORRUPT", "fill.staging.index", "index node is invalid"
+                    )
+                coordinates.write(current.encode("ascii") + b"\n")
+                current = node["previous_sha256"]
+            if traversed != head["segment_count"]:
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_CORRUPT", "fill.staging.index", "segment count differs"
+                )
+            count = 0
+            root: str | None = None
+            for position in range(head["segment_count"] - 1, -1, -1):
+                coordinates.seek(position * 65)
+                digest = coordinates.read(64).decode("ascii")
+                node = self.read_shard("index", digest)
+                if node["start"] != count:
+                    raise CatalogFillRefused(
+                        "FILL_CHECKPOINT_CORRUPT",
+                        "fill.staging.index",
+                        "index is not contiguous",
+                    )
+                root = canonical_index_root(
+                    previous_root=root,
+                    segment_sha256=digest,
+                    start=count,
+                    item_count=len(node["items"]),
+                )
+                for offset, item in enumerate(node["items"]):
+                    if count + offset >= start:
+                        yield item
+                count += len(node["items"])
+        if count != head["count"] or root != head["root_sha256"]:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.index", "index head differs"
+            )
+
+    def validate_index(self, head: dict[str, Any], *, stream: str) -> None:
+        """Read and authenticate every bounded segment behind one compact head."""
+
+        for _item in self.iter_index(head, stream=stream):
+            pass
+
+    def integrity_epoch(self) -> tuple[tuple[str, int, int, int, int], ...]:
+        """Return a cheap process-local mutation coordinate for the retained tree."""
+
+        self._require_locked()
+        descriptors = {
+            "$root": self._root_fd,
+            "$shards": self._shards_fd,
+            **self._kind_fds,
+        }
+        coordinates: list[tuple[str, int, int, int, int]] = []
+        for name, descriptor in sorted(descriptors.items()):
+            info = os.fstat(descriptor)
+            coordinates.append((name, info.st_dev, info.st_ino, info.st_mtime_ns, info.st_ctime_ns))
+        return tuple(coordinates)
+
+    def index_tail_items(self, head: dict[str, Any], *, stream: str) -> tuple[dict[str, Any], ...]:
+        """Authenticate only the newly appended immutable tail segment."""
+
+        self._require_locked()
+        _validate_index_head(head)
+        if head["count"] == 0:
+            if (
+                head["tail_sha256"] is not None
+                or head["root_sha256"] is not None
+                or head["segment_count"] != 0
+            ):
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_CORRUPT",
+                    "fill.staging.index",
+                    "empty index head has a tail",
+                )
+            return ()
+        tail = head["tail_sha256"]
+        if not _is_digest(tail) or head["segment_count"] < 1:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.index", "index tail is absent"
+            )
+        node = self.read_shard("index", tail)
+        if (
+            set(node) != {"stream", "previous_sha256", "start", "items"}
+            or node["stream"] != stream
+            or type(node["start"]) is not int
+            or not isinstance(node["items"], list)
+            or not 1 <= len(node["items"]) <= MAX_INDEX_SEGMENT_ITEMS
+            or any(not isinstance(item, dict) for item in node["items"])
+            or node["start"] + len(node["items"]) != head["count"]
+            or (head["segment_count"] == 1) != (node["previous_sha256"] is None)
+            or (node["previous_sha256"] is not None and not _is_digest(node["previous_sha256"]))
+        ):
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.index", "index tail is invalid"
+            )
+        return tuple(node["items"])
+
+    def commit(
+        self,
+        state: dict[str, Any],
+        *,
+        expected_generation: int,
+        expected_checkpoint_digest: str | None,
+    ) -> dict[str, Any]:
+        """CAS one state generation against both generation and exact live bytes."""
+
+        self._require_locked()
+        live_document, live_raw = self._read_state_document()
+        live_generation = -1 if live_document is None else live_document["generation"]
+        live_digest = None if live_raw is None else sha256_bytes(live_raw)
+        if live_generation != expected_generation or live_digest != expected_checkpoint_digest:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_STALE",
+                "fill.staging.state",
+                "live checkpoint generation or digest changed",
+            )
+        next_state = dict(state)
+        next_state["schema_version"] = STATE_SCHEMA
+        next_state["generation"] = expected_generation + 1
+        compact = dict(next_state)
+        if "evidence_indexes" in next_state:
+            if "page_shards" in next_state or "response_shards" in next_state:
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_CORRUPT",
+                    "fill.staging.state",
+                    "segmented state may not embed growing shard histories",
+                )
+        else:
+            for stream in ("page", "response"):
+                values = next_state[f"{stream}_shards"]
+                compact.pop(f"{stream}_shards")
+                compact[f"{stream}_history"] = self._advance_history(
+                    stream,
+                    values,
+                    live_document=live_document,
+                )
+        raw = canonical_json_bytes(compact)
+        if len(raw) > MAX_STATE_BYTES:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_LIMIT", "fill.staging.state", "state exceeds the byte limit"
+            )
+        temporary = f".state.{os.getpid()}.{expected_generation + 1}.{secrets.token_hex(8)}.tmp"
+        flags = os.O_RDWR | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
+        replaced = False
+        try:
+            descriptor = os.open(temporary, flags, 0o600, dir_fd=self._root_fd)
+            try:
+                _write_all(descriptor, raw)
+                os.fsync(descriptor)
+                os.lseek(descriptor, 0, os.SEEK_SET)
+                if _read_open(descriptor, maximum=len(raw)) != raw:
+                    raise CatalogFillRefused(
+                        "FILL_CHECKPOINT_READBACK",
+                        "fill.staging.state",
+                        "temporary readback differs",
+                    )
+            finally:
+                os.close(descriptor)
+            self._verify_tree()
+            os.replace(
+                temporary,
+                "state.json",
+                src_dir_fd=self._root_fd,
+                dst_dir_fd=self._root_fd,
+            )
+            replaced = True
+            os.fsync(self._root_fd)
+            if self._read_regular_at(self._root_fd, "state.json", maximum=len(raw)) != raw:
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_READBACK", "fill.staging.state", "state readback differs"
+                )
+            self._verify_tree()
+        except BaseException:
+            if replaced:
+                self._restore_checkpoint(live_raw)
+            raise
+        finally:
+            try:
+                os.unlink(temporary, dir_fd=self._root_fd)
+            except FileNotFoundError:
+                pass
+        if "page_shards" in next_state:
+            self._active_histories = {
+                "page": next_state["page_shards"],
+                "response": next_state["response_shards"],
+            }
+        return next_state
+
+    def _restore_checkpoint(self, predecessor: bytes | None) -> None:
+        """Restore the authorized head after a post-replace durability failure."""
+
+        if predecessor is None:
+            try:
+                os.unlink("state.json", dir_fd=self._root_fd)
+            except FileNotFoundError:
+                pass
+            os.fsync(self._root_fd)
+            return
+        temporary = f".state.recover.{os.getpid()}.{secrets.token_hex(8)}.tmp"
+        flags = os.O_RDWR | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
+        try:
+            descriptor = os.open(temporary, flags, 0o600, dir_fd=self._root_fd)
+            try:
+                _write_all(descriptor, predecessor)
+                os.fsync(descriptor)
+            finally:
+                os.close(descriptor)
+            os.replace(
+                temporary,
+                "state.json",
+                src_dir_fd=self._root_fd,
+                dst_dir_fd=self._root_fd,
+            )
+            os.fsync(self._root_fd)
+            if (
+                self._read_regular_at(self._root_fd, "state.json", maximum=len(predecessor))
+                != predecessor
+            ):
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_RECOVERY",
+                    "fill.staging.state",
+                    "predecessor checkpoint recovery readback differs",
+                )
+        finally:
+            try:
+                os.unlink(temporary, dir_fd=self._root_fd)
+            except FileNotFoundError:
+                pass
+
+    def state_digest(self) -> str:
+        self._require_locked()
+        try:
+            raw = self._read_regular_at(self._root_fd, "state.json", maximum=MAX_STATE_BYTES)
+        except FileNotFoundError as error:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.state", "state is absent"
+            ) from error
+        return sha256_bytes(raw)
+
+    def _load_state(self, *, validate_shards: bool) -> dict[str, Any] | None:
+        state, _raw = self._read_state_document()
+        if state is None:
+            return None
+        if type(state.get("generation")) is not int or state["generation"] < 0:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.generation", "generation is invalid"
+            )
+        budget = state.get("budget")
+        budget_keys = {
+            "requests",
+            "responses",
+            "pages",
+            "records",
+            "network_bytes",
+            "elapsed_ns",
+            "skipped_records",
+        }
+        if (
+            not isinstance(budget, dict)
+            or set(budget) != budget_keys
+            or any(type(value) is not int or value < 0 for value in budget.values())
+        ):
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.budget", "budget is invalid"
+            )
+        if "evidence_indexes" in state:
+            if "page_history" in state or "response_history" in state:
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_CORRUPT",
+                    "fill.staging.state",
+                    "segmented state may not carry legacy histories",
+                )
+            indexes = state["evidence_indexes"]
+            if not isinstance(indexes, dict) or not indexes:
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_CORRUPT", "fill.staging.indexes", "indexes are invalid"
+                )
+            for stream, head in indexes.items():
+                if not isinstance(stream, str) or not stream:
+                    raise CatalogFillRefused(
+                        "FILL_CHECKPOINT_CORRUPT",
+                        "fill.staging.indexes",
+                        "index stream is invalid",
+                    )
+                _validate_index_head(head)
+                if validate_shards:
+                    # Segmented checkpoints carry a producer-authenticated tail seal. Ordinary
+                    # resumes authenticate the immutable appended tail here; eligibility runs
+                    # the full streaming chain audit explicitly.
+                    self.index_tail_items(head, stream=stream)
+            return state
+        pages = self._expand_history("page", state.get("page_history"))
+        responses = self._expand_history("response", state.get("response_history"))
+        if budget["pages"] != len(pages) or budget["responses"] < len(responses):
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT",
+                "fill.staging.budget",
+                "budget does not bind the staged shards",
+            )
+        if validate_shards:
+            for digest in pages:
+                self.read_shard("page", digest)
+            for digest in responses:
+                self.read_shard("response", digest)
+        expanded = dict(state)
+        del expanded["page_history"]
+        del expanded["response_history"]
+        expanded["page_shards"] = pages
+        expanded["response_shards"] = responses
+        self._active_histories = {"page": pages, "response": responses}
+        return expanded
+
+    def _read_state_document(self) -> tuple[dict[str, Any] | None, bytes | None]:
+        try:
+            raw = self._read_regular_at(self._root_fd, "state.json", maximum=MAX_STATE_BYTES)
+        except FileNotFoundError:
+            return None, None
+        try:
+            state = parse_canonical_json(raw)
+        except CanonicalJSONError as error:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT",
+                "fill.staging.state",
+                f"state is not canonical: {error.code}",
+            ) from None
+        if not isinstance(state, dict) or state.get("schema_version") != STATE_SCHEMA:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.state", "state schema is invalid"
+            )
+        if "page_shards" in state or "response_shards" in state:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT",
+                "fill.staging.state",
+                "growing shard history may not be embedded in the checkpoint head",
+            )
+        return state, raw
+
+    def _advance_history(
+        self,
+        stream: str,
+        values: Any,
+        *,
+        live_document: dict[str, Any] | None,
+    ) -> dict[str, Any]:
+        if not isinstance(values, list) or any(not _is_digest(value) for value in values):
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT",
+                f"fill.staging.{stream}_shards",
+                "history values must be SHA-256 digests",
+            )
+        prior = (
+            {"head_sha256": None, "count": 0}
+            if live_document is None
+            else live_document.get(f"{stream}_history")
+        )
+        if not _valid_history_head(prior):
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT",
+                f"fill.staging.{stream}_history",
+                "history head is invalid",
+            )
+        prior_count = prior["count"]
+        active = self._active_histories.get(stream)
+        if live_document is not None and values is not active:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_STALE",
+                f"fill.staging.{stream}_history",
+                "history must extend the exact list returned by the retained staging load",
+            )
+        if not prior_count <= len(values) <= MAX_HISTORY_DIGESTS:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_LIMIT",
+                f"fill.staging.{stream}_history",
+                "history count is outside its bound",
+            )
+        suffix = values[prior_count:]
+        if not suffix:
+            return dict(prior)
+        head = self.write_shard(
+            "history",
+            {
+                "stream": stream,
+                "previous_sha256": prior["head_sha256"],
+                "start": prior_count,
+                "digests": suffix,
+            },
+        )
+        return {"head_sha256": head, "count": len(values)}
+
+    def _expand_history(self, stream: str, head: Any) -> list[str]:
+        if not _valid_history_head(head):
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT",
+                f"fill.staging.{stream}_history",
+                "history head is invalid",
+            )
+        count = head["count"]
+        if count > MAX_HISTORY_DIGESTS:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_LIMIT",
+                f"fill.staging.{stream}_history",
+                "history exceeds the digest-count limit",
+            )
+        chunks: list[tuple[int, list[str]]] = []
+        seen: set[str] = set()
+        current = head["head_sha256"]
+        while current is not None:
+            if current in seen or len(seen) > count:
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_CORRUPT",
+                    f"fill.staging.{stream}_history",
+                    "history chain loops or exceeds its count",
+                )
+            seen.add(current)
+            node = self.read_shard("history", current)
+            if (
+                set(node) != {"stream", "previous_sha256", "start", "digests"}
+                or node["stream"] != stream
+                or type(node["start"]) is not int
+                or not isinstance(node["digests"], list)
+                or any(not _is_digest(value) for value in node["digests"])
+                or (node["previous_sha256"] is not None and not _is_digest(node["previous_sha256"]))
+            ):
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_CORRUPT",
+                    f"fill.staging.{stream}_history",
+                    "history node is invalid",
+                )
+            chunks.append((node["start"], node["digests"]))
+            current = node["previous_sha256"]
+        result: list[str] = []
+        for start, digests in reversed(chunks):
+            if start != len(result):
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_CORRUPT",
+                    f"fill.staging.{stream}_history",
+                    "history nodes are not contiguous",
+                )
+            result.extend(digests)
+        if len(result) != count:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT",
+                f"fill.staging.{stream}_history",
+                "history count differs from its chain",
+            )
+        return result
+
+    def _open_retained_tree(self) -> None:
+        if self._retained_root_descriptor is not None:
+            try:
+                descriptor = os.dup(self._retained_root_descriptor)
+                opened = os.fstat(descriptor)
+            except OSError as error:
+                raise CatalogFillRefused(
+                    "FILL_STAGING_PATH",
+                    "fill.staging",
+                    "the retained staging root is unusable",
+                ) from error
+            self._owned_fds.append(descriptor)
+            if not stat.S_ISDIR(opened.st_mode) or stat.S_IMODE(opened.st_mode) & 0o077:
+                raise CatalogFillRefused(
+                    "FILL_STAGING_PERMISSIONS",
+                    "fill.staging",
+                    "staging root must be a retained private directory",
+                )
+            self._root_fd = descriptor
+            self._shards_fd = self._open_or_create_directory(self._root_fd, "shards")
+            for kind in sorted(_SHARD_KINDS | _BLOB_KINDS):
+                self._kind_fds[kind] = self._open_or_create_directory(self._shards_fd, kind)
+            return
+        absolute = Path(os.path.abspath(self.root))
+        descriptor = os.open("/", _OPEN_DIRECTORY)
+        self._owned_fds.append(descriptor)
+        for part in absolute.parts[1:]:
+            try:
+                named = os.stat(part, dir_fd=descriptor, follow_symlinks=False)
+            except FileNotFoundError:
+                os.mkdir(part, mode=0o700, dir_fd=descriptor)
+                os.fsync(descriptor)
+                named = os.stat(part, dir_fd=descriptor, follow_symlinks=False)
+            child = os.open(part, _OPEN_DIRECTORY, dir_fd=descriptor)
+            self._owned_fds.append(child)
+            opened = os.fstat(child)
+            if not stat.S_ISDIR(named.st_mode) or _directory_identity(named) != _directory_identity(
+                opened
+            ):
+                raise CatalogFillRefused(
+                    "FILL_STAGING_RACE", "fill.staging", "named and opened directory differ"
+                )
+            self._chain.append((descriptor, part, child, _directory_identity(opened)))
+            descriptor = child
+        self._root_fd = descriptor
+        root_info = os.fstat(self._root_fd)
+        if stat.S_IMODE(root_info.st_mode) & 0o077:
+            raise CatalogFillRefused(
+                "FILL_STAGING_PERMISSIONS", "fill.staging", "staging root must be mode 0700"
+            )
+        self._shards_fd = self._open_or_create_directory(self._root_fd, "shards")
+        for kind in sorted(_SHARD_KINDS | _BLOB_KINDS):
+            self._kind_fds[kind] = self._open_or_create_directory(self._shards_fd, kind)
+
+    def _open_or_create_directory(self, parent_fd: int, name: str) -> int:
+        try:
+            os.mkdir(name, mode=0o700, dir_fd=parent_fd)
+            os.fsync(parent_fd)
+        except FileExistsError:
+            pass
+        named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+        descriptor = os.open(name, _OPEN_DIRECTORY, dir_fd=parent_fd)
+        self._owned_fds.append(descriptor)
+        opened = os.fstat(descriptor)
+        if (
+            not stat.S_ISDIR(named.st_mode)
+            or _directory_identity(named) != _directory_identity(opened)
+            or stat.S_IMODE(opened.st_mode) & 0o077
+        ):
+            raise CatalogFillRefused(
+                "FILL_STAGING_PATH",
+                "fill.staging",
+                "staging components must be retained private directories",
+            )
+        return descriptor
+
+    def _verify_tree(self) -> None:
+        for parent_fd, name, child_fd, identity in self._chain:
+            try:
+                named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+            except OSError as error:
+                raise CatalogFillRefused(
+                    "FILL_STAGING_RACE", "fill.staging", "staging ancestor was replaced"
+                ) from error
+            if (
+                _directory_identity(named) != identity
+                or _directory_identity(os.fstat(child_fd)) != identity
+            ):
+                raise CatalogFillRefused(
+                    "FILL_STAGING_RACE", "fill.staging", "staging ancestor was replaced"
+                )
+        for parent_fd, name, child_fd in (
+            (self._root_fd, "shards", self._shards_fd),
+            *((self._shards_fd, kind, fd) for kind, fd in self._kind_fds.items()),
+        ):
+            named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+            if _directory_identity(named) != _directory_identity(os.fstat(child_fd)):
+                raise CatalogFillRefused(
+                    "FILL_STAGING_RACE", "fill.staging", "staging directory was replaced"
+                )
+
+    def _read_regular_at(self, parent_fd: int, name: str, *, maximum: int) -> bytes:
+        descriptor = -1
+        try:
+            named_before = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+            descriptor = os.open(name, _OPEN_MEMBER, dir_fd=parent_fd)
+        except FileNotFoundError:
+            if descriptor >= 0:
+                os.close(descriptor)
+            raise
+        except OSError as error:
+            if descriptor >= 0:
+                os.close(descriptor)
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.member", "member cannot be opened"
+            ) from error
+        try:
+            opened_before = os.fstat(descriptor)
+            if (
+                not stat.S_ISREG(named_before.st_mode)
+                or not stat.S_ISREG(opened_before.st_mode)
+                or named_before.st_nlink != 1
+                or opened_before.st_nlink != 1
+                or opened_before.st_size > maximum
+                or _identity(named_before) != _identity(opened_before)
+            ):
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_CORRUPT",
+                    "fill.staging.member",
+                    "member is not a bounded single-link regular file",
+                )
+            raw = _read_open(descriptor, maximum=maximum)
+            opened_after = os.fstat(descriptor)
+            named_after = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+            if (
+                _identity(opened_before) != _identity(opened_after)
+                or _identity(opened_after) != _identity(named_after)
+                or len(raw) != opened_after.st_size
+            ):
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_CORRUPT", "fill.staging.member", "member changed during read"
+                )
+            return raw
+        finally:
+            os.close(descriptor)
+
+    def _require_locked(self) -> None:
+        if self._root_fd < 0:
+            raise CatalogFillRefused(
+                "FILL_STAGING_LOCK", "fill.staging", "operation requires the retained lock"
+            )
+
+
+def _read_open(descriptor: int, *, maximum: int) -> bytes:
+    chunks: list[bytes] = []
+    observed = 0
+    while True:
+        chunk = os.read(descriptor, min(1024 * 1024, maximum - observed + 1))
+        if not chunk:
+            break
+        observed += len(chunk)
+        if observed > maximum:
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_CORRUPT", "fill.staging.member", "member exceeds byte limit"
+            )
+        chunks.append(chunk)
+    return b"".join(chunks)
+
+
+def _write_all(descriptor: int, raw: bytes) -> None:
+    written = 0
+    while written < len(raw):
+        count = os.write(descriptor, raw[written:])
+        if count < 1:
+            raise CatalogFillRefused(
+                "FILL_STAGING_WRITE", "fill.staging.member", "write made no progress"
+            )
+        written += count
+
+
+def _identity(info: os.stat_result) -> tuple[int, ...]:
+    return (
+        info.st_dev,
+        info.st_ino,
+        info.st_mode,
+        info.st_nlink,
+        info.st_size,
+        info.st_mtime_ns,
+        info.st_ctime_ns,
+    )
+
+
+def _directory_identity(info: os.stat_result) -> tuple[int, int, int]:
+    return (info.st_dev, info.st_ino, stat.S_IFMT(info.st_mode))
+
+
+def _is_digest(value: Any) -> bool:
+    return (
+        isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value)
+    )
+
+
+def _valid_history_head(value: Any) -> bool:
+    return (
+        isinstance(value, dict)
+        and set(value) == {"head_sha256", "count"}
+        and type(value["count"]) is int
+        and value["count"] >= 0
+        and (
+            (value["count"] == 0 and value["head_sha256"] is None)
+            or (value["count"] > 0 and _is_digest(value["head_sha256"]))
+        )
+    )
+
+
+def canonical_index_root(
+    *,
+    previous_root: str | None,
+    segment_sha256: str,
+    start: int,
+    item_count: int,
+) -> str:
+    return canonical_sha256(
+        {
+            "previous_root_sha256": previous_root,
+            "segment_sha256": segment_sha256,
+            "start": start,
+            "item_count": item_count,
+        }
+    )
+
+
+def _validate_index_head(value: Any) -> None:
+    if (
+        not isinstance(value, dict)
+        or set(value)
+        != {
+            "root_sha256",
+            "count",
+            "tail_sha256",
+            "segment_count",
+            "descriptor_count",
+        }
+        or type(value["count"]) is not int
+        or type(value["segment_count"]) is not int
+        or value["count"] < 0
+        or value["segment_count"] < 0
+        or value["count"] > MAX_INDEX_ITEMS
+        or value["segment_count"] > value["count"]
+        or value["descriptor_count"] != 0
+        or (
+            value["count"] == 0
+            and (
+                value["root_sha256"] is not None
+                or value["tail_sha256"] is not None
+                or value["segment_count"] != 0
+            )
+        )
+        or (
+            value["count"] > 0
+            and (
+                not _is_digest(value["root_sha256"])
+                or not _is_digest(value["tail_sha256"])
+                or value["segment_count"] < 1
+            )
+        )
+    ):
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CORRUPT", "fill.staging.index", "index head is invalid"
+        )