mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/local_search/build_index.py

@@ -0,0 +1,1087 @@
+"""Build the private local index from bounded, exact-verified sealed Builds.
+
+Discovery and verification are deliberately separate operations.  Discovery retains the supplied
+root descriptor, follows no names, and returns only root-relative candidate coordinates plus
+sanitized diagnostics.  It never reads a dataset member.  Publication later opens each coordinate
+through the existing exact verifier and writes only beneath the explicit index root.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import errno
+import heapq
+import json
+import os
+import stat
+import sys
+import time
+import uuid
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from mostlyright.data_harness import pipeline
+from mostlyright.data_harness.canonical import canonical_json_bytes, canonical_sha256, sha256_bytes
+from mostlyright.data_harness.local_search.contracts import (
+    BuildDiagnosticDocument,
+    BuildField,
+    BuildIndexDiagnostic,
+    BuildIndexEntry,
+    BuildIndexManifest,
+    BuildLocator,
+    BuildLocatorDocument,
+    BuildRecipeLineage,
+    BuildSourceFacts,
+    BuildTimeCoverage,
+    LocalSearchContractError,
+    SealedProfileFactsV1,
+    SnapshotMember,
+    manifest_aggregate_sha256,
+)
+from mostlyright.data_harness.local_search.sealed import capture_build_index_snapshot
+
+_OPEN_DIRECTORY = (
+    os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
+)
+_RENAME_ARGTYPES = [
+    ctypes.c_int,
+    ctypes.c_char_p,
+    ctypes.c_int,
+    ctypes.c_char_p,
+    ctypes.c_uint,
+]
+
+
+class LocalSearchIndexError(RuntimeError):
+    """A stable private-index refusal without raw filesystem error text."""
+
+    def __init__(self, code: str, path: str, detail: str) -> None:
+        self.code = code
+        self.path = path
+        self.detail = detail
+        super().__init__(f"{path}: {detail} [{code}]")
+
+
+@dataclass(frozen=True)
+class BuildDiscoveryLimits:
+    """Aggregate ceilings for one discovery and indexing operation."""
+
+    max_depth: int = 6
+    max_entries: int = 100_000
+    max_builds: int = 10_000
+    max_receipt_bytes: int = 2 * 1024 * 1024
+    max_member_bytes: int = pipeline.MAX_CANDIDATE_MEMBER_BYTES
+    max_aggregate_bytes: int = pipeline.MAX_CANDIDATE_BYTES
+    max_elapsed_seconds: float = 60.0
+    max_diagnostics: int = 100_000
+
+    def __post_init__(self) -> None:
+        integer_bounds = (
+            ("max_depth", self.max_depth, 32),
+            ("max_entries", self.max_entries, 1_000_000),
+            ("max_builds", self.max_builds, 100_000),
+            ("max_receipt_bytes", self.max_receipt_bytes, 64 * 1024 * 1024),
+            # Bounded by the candidate ceilings in ``pipeline``: a member may be one
+            # year-scale normalized source and the aggregate may hold the sources a
+            # graph execution retains, so the search index admits what a build seals.
+            ("max_member_bytes", self.max_member_bytes, pipeline.MAX_CANDIDATE_MEMBER_BYTES),
+            ("max_aggregate_bytes", self.max_aggregate_bytes, pipeline.MAX_CANDIDATE_BYTES),
+            ("max_diagnostics", self.max_diagnostics, 100_000),
+        )
+        for name, value, maximum in integer_bounds:
+            if type(value) is not int or not 1 <= value <= maximum:
+                raise LocalSearchIndexError(
+                    "BUILD_LIMIT", f"limits.{name}", f"must be an integer in [1, {maximum}]"
+                )
+        if (
+            isinstance(self.max_elapsed_seconds, bool)
+            or not isinstance(self.max_elapsed_seconds, (int, float))
+            or not 0 < self.max_elapsed_seconds <= 3600
+        ):
+            raise LocalSearchIndexError(
+                "BUILD_LIMIT",
+                "limits.max_elapsed_seconds",
+                "must be a number in (0, 3600]",
+            )
+        if self.max_receipt_bytes > self.max_member_bytes:
+            raise LocalSearchIndexError(
+                "BUILD_LIMIT",
+                "limits.max_receipt_bytes",
+                "cannot exceed max_member_bytes",
+            )
+        if self.max_member_bytes > self.max_aggregate_bytes:
+            raise LocalSearchIndexError(
+                "BUILD_LIMIT",
+                "limits.max_member_bytes",
+                "cannot exceed max_aggregate_bytes",
+            )
+
+
+@dataclass(frozen=True)
+class BuildDiscovery:
+    """Deterministic candidate coordinates and bounded diagnostics from one root."""
+
+    candidates: tuple[str, ...]
+    diagnostics: tuple[BuildIndexDiagnostic, ...]
+    entries_seen: int
+    builds_seen: int
+
+
+@dataclass(frozen=True)
+class PublishedBuildIndex:
+    """One completely published private index successor."""
+
+    index_root: Path
+    manifest: BuildIndexManifest
+    manifest_sha256: str
+
+
+@dataclass(frozen=True)
+class _RootHandle:
+    descriptor: int
+    identity: tuple[int, int]
+
+
+@dataclass
+class _WalkState:
+    entries_seen: int = 0
+    builds_seen: int = 0
+    stopped: bool = False
+
+
+@dataclass(frozen=True)
+class _DescendingName:
+    """Heap item whose smallest value is the lexicographically greatest retained name."""
+
+    name: str
+
+    def __lt__(self, other: _DescendingName) -> bool:
+        return self.name > other.name
+
+
+def discover_build_candidates(
+    root: Path,
+    *,
+    limits: BuildDiscoveryLimits | None = None,
+) -> BuildDiscovery:
+    """Find possible sealed Builds below one retained, no-follow directory root.
+
+    A candidate is only a coordinate whose ``candidate/manifest.json`` entry is a bounded,
+    single-link regular file.  No receipt bytes or dataset rows are read here.  Exact admission is
+    exclusively the job of :func:`pipeline.open_verified_snapshot` during publication.
+    """
+
+    if not isinstance(root, Path):
+        raise LocalSearchIndexError("BUILD_INVALID", "root", "must be a pathlib.Path")
+    selected = limits or BuildDiscoveryLimits()
+    if not isinstance(selected, BuildDiscoveryLimits):
+        raise LocalSearchIndexError("BUILD_LIMIT", "limits", "must be BuildDiscoveryLimits")
+
+    opened = _open_root(root)
+    try:
+        discovered = _discover_from_open_root(opened, selected, started=time.monotonic())
+        _validate_root_binding(root, opened)
+        return discovered
+    finally:
+        os.close(opened.descriptor)
+
+
+def _discover_from_open_root(
+    opened: _RootHandle,
+    limits: BuildDiscoveryLimits,
+    *,
+    started: float,
+) -> BuildDiscovery:
+    state = _WalkState()
+    candidates: list[str] = []
+    diagnostics: list[BuildIndexDiagnostic] = []
+    pending: list[tuple[tuple[str, ...], int]] = [((), 0)]
+    while pending and not state.stopped:
+        relative, depth = heapq.heappop(pending)
+        if time.monotonic() - started > limits.max_elapsed_seconds:
+            _diagnose(
+                diagnostics,
+                limits,
+                "BUILD_LIMIT",
+                _diagnostic_path(relative),
+            )
+            break
+        try:
+            directory = _open_directory_beneath(opened.descriptor, relative)
+        except OSError:
+            _diagnose(
+                diagnostics,
+                limits,
+                "BUILD_UNREADABLE",
+                _diagnostic_path(relative),
+            )
+            continue
+        try:
+            candidate = _candidate_disposition(directory, relative, limits)
+            if candidate is not None:
+                disposition, candidate_entries = candidate
+                if state.entries_seen + candidate_entries > limits.max_entries:
+                    state.stopped = True
+                    _diagnose(
+                        diagnostics,
+                        limits,
+                        "BUILD_LIMIT",
+                        _diagnostic_path(relative),
+                    )
+                    continue
+                state.entries_seen += candidate_entries
+                if disposition != "BUILD_CANDIDATE":
+                    _diagnose(
+                        diagnostics,
+                        limits,
+                        disposition,
+                        _diagnostic_path(relative),
+                    )
+                    continue
+                if state.builds_seen >= limits.max_builds:
+                    _diagnose(
+                        diagnostics,
+                        limits,
+                        "BUILD_LIMIT",
+                        _diagnostic_path(relative),
+                    )
+                    state.stopped = True
+                    continue
+                state.builds_seen += 1
+                candidates.append(_relative_path(relative))
+                continue
+
+            remaining_entries = limits.max_entries - state.entries_seen
+            children, incomplete, first_excluded = _directory_children(
+                directory, maximum=remaining_entries
+            )
+            for name, kind in children:
+                _consume_entry(state, limits)
+                if kind != "directory":
+                    if kind != "regular":
+                        _diagnose(diagnostics, limits, kind, _join(relative, name))
+                    continue
+                child_relative = (*relative, name)
+                if depth >= limits.max_depth:
+                    _diagnose(
+                        diagnostics,
+                        limits,
+                        "BUILD_LIMIT",
+                        _relative_path(child_relative),
+                    )
+                else:
+                    heapq.heappush(pending, (child_relative, depth + 1))
+            if first_excluded is not None:
+                state.stopped = True
+                _diagnose(
+                    diagnostics,
+                    limits,
+                    "BUILD_LIMIT",
+                    _join(relative, first_excluded),
+                )
+            if incomplete:
+                _diagnose(
+                    diagnostics,
+                    limits,
+                    "BUILD_UNREADABLE",
+                    _diagnostic_path(relative),
+                )
+        finally:
+            os.close(directory)
+    return BuildDiscovery(
+        candidates=tuple(sorted(candidates)),
+        diagnostics=tuple(
+            sorted(set(diagnostics), key=lambda item: (item.relative_path, item.code))
+        ),
+        entries_seen=state.entries_seen,
+        builds_seen=state.builds_seen,
+    )
+
+
+def _validate_root_binding(root: Path, opened: _RootHandle) -> None:
+    after = _open_root(root)
+    try:
+        if after.identity != opened.identity:
+            raise LocalSearchIndexError(
+                "BUILD_RACE", "root", "changed while Build discovery was in progress"
+            )
+    finally:
+        os.close(after.descriptor)
+
+
+def _open_root(root: Path) -> _RootHandle:
+    try:
+        named = os.lstat(root)
+    except OSError as error:
+        raise LocalSearchIndexError(
+            "BUILD_UNREADABLE", "root", "cannot inspect the discovery root"
+        ) from error
+    if stat.S_ISLNK(named.st_mode):
+        raise LocalSearchIndexError(
+            "BUILD_SYMLINK", "root", "the discovery root must not be a link"
+        )
+    try:
+        descriptor = os.open(root, _OPEN_DIRECTORY)
+    except OSError as error:
+        raise LocalSearchIndexError(
+            "BUILD_UNREADABLE", "root", "cannot open a retained no-follow directory"
+        ) from error
+    try:
+        opened = os.fstat(descriptor)
+        if not stat.S_ISDIR(named.st_mode) or not stat.S_ISDIR(opened.st_mode):
+            raise LocalSearchIndexError(
+                "BUILD_SPECIAL", "root", "the discovery root must be a directory"
+            )
+        identity = (opened.st_dev, opened.st_ino)
+        if identity != (named.st_dev, named.st_ino):
+            raise LocalSearchIndexError(
+                "BUILD_RACE", "root", "named and opened discovery roots differ"
+            )
+        return _RootHandle(descriptor, identity)
+    except BaseException:
+        os.close(descriptor)
+        raise
+
+
+def _open_directory_beneath(root_fd: int, relative: tuple[str, ...]) -> int:
+    descriptor = os.dup(root_fd)
+    try:
+        for name in relative:
+            named = os.stat(name, dir_fd=descriptor, follow_symlinks=False)
+            child = os.open(name, _OPEN_DIRECTORY, dir_fd=descriptor)
+            opened = os.fstat(child)
+            if (
+                not stat.S_ISDIR(named.st_mode)
+                or not stat.S_ISDIR(opened.st_mode)
+                or (named.st_dev, named.st_ino) != (opened.st_dev, opened.st_ino)
+            ):
+                os.close(child)
+                raise OSError(errno.ESTALE, "descendant directory changed")
+            os.close(descriptor)
+            descriptor = child
+        return descriptor
+    except BaseException:
+        os.close(descriptor)
+        raise
+
+
+def _candidate_disposition(
+    directory_fd: int,
+    relative: tuple[str, ...],
+    limits: BuildDiscoveryLimits,
+) -> tuple[str, int] | None:
+    try:
+        candidate = os.stat("candidate", dir_fd=directory_fd, follow_symlinks=False)
+    except FileNotFoundError:
+        return None
+    except OSError:
+        return "BUILD_UNREADABLE", 1
+    if stat.S_ISLNK(candidate.st_mode):
+        return "BUILD_SYMLINK", 1
+    if not stat.S_ISDIR(candidate.st_mode):
+        return "BUILD_SPECIAL", 1
+
+    candidate_fd = -1
+    try:
+        candidate_fd = os.open("candidate", _OPEN_DIRECTORY, dir_fd=directory_fd)
+        opened = os.fstat(candidate_fd)
+        if (candidate.st_dev, candidate.st_ino) != (opened.st_dev, opened.st_ino):
+            return "BUILD_RACE", 1
+        try:
+            receipt = os.stat("manifest.json", dir_fd=candidate_fd, follow_symlinks=False)
+        except FileNotFoundError:
+            return "BUILD_INVALID", 1
+        except OSError:
+            return "BUILD_UNREADABLE", 2
+        if stat.S_ISLNK(receipt.st_mode):
+            return "BUILD_SYMLINK", 2
+        if not stat.S_ISREG(receipt.st_mode):
+            return "BUILD_SPECIAL", 2
+        if receipt.st_nlink != 1:
+            return "BUILD_HARDLINK", 2
+        if receipt.st_size > limits.max_receipt_bytes:
+            return "BUILD_LIMIT", 2
+        if not relative:
+            return "BUILD_INVALID", 2
+        return "BUILD_CANDIDATE", 2
+    except OSError:
+        return "BUILD_RACE", 1
+    finally:
+        if candidate_fd >= 0:
+            os.close(candidate_fd)
+
+
+def _directory_children(
+    directory_fd: int, *, maximum: int
+) -> tuple[list[tuple[str, str]], bool, str | None]:
+    """Return only the lexically first bounded child set from an incremental enumeration."""
+
+    retained: list[_DescendingName] = []
+    incomplete = False
+    try:
+        with os.scandir(directory_fd) as entries:
+            iterator = iter(entries)
+            while True:
+                try:
+                    entry = next(iterator)
+                except StopIteration:
+                    break
+                except OSError:
+                    incomplete = True
+                    break
+                if entry.name == "candidate":
+                    continue
+                item = _DescendingName(entry.name)
+                if len(retained) < maximum + 1:
+                    heapq.heappush(retained, item)
+                elif entry.name < retained[0].name:
+                    heapq.heapreplace(retained, item)
+    except OSError:
+        incomplete = True
+
+    selected_names = sorted(item.name for item in retained)
+    first_excluded = selected_names[maximum] if len(selected_names) > maximum else None
+    children: list[tuple[str, str]] = []
+    for name in selected_names[:maximum]:
+        try:
+            observed = os.stat(name, dir_fd=directory_fd, follow_symlinks=False)
+        except OSError:
+            children.append((name, "BUILD_UNREADABLE"))
+            continue
+        if stat.S_ISLNK(observed.st_mode):
+            children.append((name, "BUILD_SYMLINK"))
+        elif stat.S_ISDIR(observed.st_mode):
+            children.append((name, "directory"))
+        elif not stat.S_ISREG(observed.st_mode):
+            children.append((name, "BUILD_SPECIAL"))
+        else:
+            children.append((name, "regular"))
+    return children, incomplete, first_excluded
+
+
+def _consume_entry(state: _WalkState, limits: BuildDiscoveryLimits) -> bool:
+    if state.entries_seen >= limits.max_entries:
+        state.stopped = True
+        return False
+    state.entries_seen += 1
+    return True
+
+
+def _diagnose(
+    diagnostics: list[BuildIndexDiagnostic],
+    limits: BuildDiscoveryLimits,
+    code: str,
+    relative_path: str,
+) -> None:
+    if len(diagnostics) >= limits.max_diagnostics:
+        return
+    diagnostics.append(BuildIndexDiagnostic(code, relative_path))
+
+
+def _diagnostic_path(relative: tuple[str, ...]) -> str:
+    return _relative_path(relative) if relative else "root"
+
+
+def _join(relative: tuple[str, ...], name: str) -> str:
+    return _relative_path((*relative, name))
+
+
+def _relative_path(relative: tuple[str, ...]) -> str:
+    return PurePosixPath(*relative).as_posix()
+
+
+def build_private_index(
+    build_root: Path,
+    index_root: Path,
+    *,
+    limits: BuildDiscoveryLimits | None = None,
+    fault_hook: Callable[[str], None] | None = None,
+) -> PublishedBuildIndex:
+    """Verify local Builds once and atomically publish a complete private index successor."""
+
+    if not isinstance(build_root, Path) or not isinstance(index_root, Path):
+        raise LocalSearchIndexError(
+            "BUILD_INVALID", "root", "Build and index roots must be pathlib.Path values"
+        )
+    selected = limits or BuildDiscoveryLimits()
+    if not isinstance(selected, BuildDiscoveryLimits):
+        raise LocalSearchIndexError("BUILD_LIMIT", "limits", "must be BuildDiscoveryLimits")
+    if fault_hook is not None and not callable(fault_hook):
+        raise LocalSearchIndexError("BUILD_INVALID", "fault_hook", "must be callable")
+
+    build_root = Path(os.path.abspath(os.fspath(build_root)))
+    index_root = Path(os.path.abspath(os.fspath(index_root)))
+    try:
+        common = Path(os.path.commonpath((build_root, index_root)))
+    except ValueError as error:
+        raise LocalSearchIndexError(
+            "BUILD_INVALID", "index_root", "must share a local filesystem coordinate"
+        ) from error
+    if common == build_root:
+        raise LocalSearchIndexError(
+            "BUILD_INVALID", "index_root", "must be outside the Build discovery root"
+        )
+
+    opened = _open_root(build_root)
+    started = time.monotonic()
+    try:
+        discovered = _discover_from_open_root(opened, selected, started=started)
+        diagnostics = list(discovered.diagnostics)
+        entries_by_digest: dict[str, BuildIndexEntry] = {}
+        locators: list[BuildLocator] = []
+        aggregate_bytes = 0
+
+        for relative_path in discovered.candidates:
+            if time.monotonic() - started > selected.max_elapsed_seconds:
+                _diagnose(diagnostics, selected, "BUILD_LIMIT", relative_path)
+                break
+            relative = tuple(PurePosixPath(relative_path).parts)
+            try:
+                run_fd = _open_directory_beneath(opened.descriptor, relative)
+            except OSError:
+                _diagnose(diagnostics, selected, "BUILD_RACE", relative_path)
+                continue
+            try:
+                remaining_aggregate_bytes = selected.max_aggregate_bytes - aggregate_bytes
+                if remaining_aggregate_bytes < 1:
+                    _diagnose(diagnostics, selected, "BUILD_LIMIT", relative_path)
+                    continue
+                with pipeline.open_verified_snapshot(
+                    build_root.joinpath(*relative),
+                    _run_fd=run_fd,
+                    _ancestor_validator=lambda: _validate_retained_root(build_root, opened),
+                    _max_manifest_bytes=min(selected.max_receipt_bytes, remaining_aggregate_bytes),
+                    _max_member_bytes=min(selected.max_member_bytes, remaining_aggregate_bytes),
+                    _max_aggregate_bytes=remaining_aggregate_bytes,
+                ) as snapshot:
+                    verified = snapshot.verified
+                    manifest = verified.manifest
+                    member_sizes = _verified_member_sizes(manifest)
+                    if any(size > selected.max_member_bytes for size in member_sizes):
+                        _diagnose(diagnostics, selected, "BUILD_LIMIT", relative_path)
+                        continue
+                    build_bytes = sum(member_sizes) + len(
+                        pipeline.canonical_json_line_bytes(manifest)
+                    )
+                    if aggregate_bytes + build_bytes > selected.max_aggregate_bytes:
+                        _diagnose(diagnostics, selected, "BUILD_LIMIT", relative_path)
+                        continue
+                    entry = _entry_from_verified_snapshot(snapshot)
+                    if time.monotonic() - started > selected.max_elapsed_seconds:
+                        _diagnose(diagnostics, selected, "BUILD_LIMIT", relative_path)
+                        continue
+                    before_digest = verified.candidate_digest
+                    snapshot.validate()
+                    after_digest = snapshot.verified.candidate_digest
+                    if after_digest != before_digest or entry.candidate_sha256 != before_digest:
+                        _diagnose(diagnostics, selected, "BUILD_RACE", relative_path)
+                        continue
+                    aggregate_bytes += build_bytes
+                    entries_by_digest.setdefault(entry.facts_sha256, entry)
+                    locators.append(BuildLocator(entry.facts_sha256, relative_path))
+            except pipeline.BuildError as error:
+                _diagnose(
+                    diagnostics,
+                    selected,
+                    _diagnostic_for_build_error(error),
+                    relative_path,
+                )
+            except LocalSearchContractError:
+                _diagnose(diagnostics, selected, "BUILD_INVALID", relative_path)
+            finally:
+                os.close(run_fd)
+
+        _validate_retained_root(build_root, opened)
+    finally:
+        os.close(opened.descriptor)
+
+    ordered_entries = tuple(entries_by_digest[key] for key in sorted(entries_by_digest))
+    locator_document = BuildLocatorDocument(
+        tuple(sorted(set(locators), key=lambda item: (item.facts_sha256, item.relative_path)))
+    )
+    diagnostic_document = BuildDiagnosticDocument(
+        tuple(sorted(set(diagnostics), key=lambda item: (item.relative_path, item.code)))
+    )
+    return _publish_index(
+        index_root,
+        ordered_entries,
+        locator_document,
+        diagnostic_document,
+        fault_hook=fault_hook,
+    )
+
+
+def _validate_retained_root(root: Path, opened: _RootHandle) -> None:
+    observed = os.fstat(opened.descriptor)
+    if (observed.st_dev, observed.st_ino) != opened.identity:
+        raise LocalSearchIndexError("BUILD_RACE", "root", "retained Build root changed")
+    _validate_root_binding(root, opened)
+
+
+def _verified_member_sizes(manifest: Mapping[str, Any]) -> tuple[int, ...]:
+    members = manifest.get("members")
+    if not isinstance(members, list):
+        raise pipeline.BuildError("MANIFEST_INVALID", "candidate members are invalid")
+    sizes: list[int] = []
+    for member in members:
+        if not isinstance(member, Mapping) or type(member.get("bytes")) is not int:
+            raise pipeline.BuildError("MANIFEST_INVALID", "candidate member size is invalid")
+        sizes.append(member["bytes"])
+    return tuple(sizes)
+
+
+def _entry_from_verified_snapshot(snapshot: pipeline.VerifiedSnapshot) -> BuildIndexEntry:
+    """Derive every index field from the one retained verified observation."""
+
+    verified = snapshot.verified
+    inspection = snapshot.inspection()
+    profile_raw = snapshot.member_bytes("evidence/profile.json")
+    profile = _json_object(profile_raw, "evidence/profile.json")
+    lineage = _json_object(snapshot.member_bytes("evidence/lineage.json"), "evidence/lineage.json")
+    sources_value = _json_value(
+        snapshot.member_bytes("evidence/sources.json"), "evidence/sources.json"
+    )
+    if not isinstance(sources_value, list):
+        raise pipeline.BuildError("CANDIDATE_EVIDENCE_SCHEMA", "sources evidence is invalid")
+
+    columns = profile.get("columns")
+    types = profile.get("types")
+    null_counts = profile.get("null_counts")
+    row_count = profile.get("row_count")
+    if (
+        profile.get("schema_version") != pipeline.PROFILE_EVIDENCE_VERSION
+        or not isinstance(columns, list)
+        or not isinstance(types, Mapping)
+        or not isinstance(null_counts, Mapping)
+        or type(row_count) is not int
+        or set(types) != set(columns)
+        or set(null_counts) != set(columns)
+    ):
+        raise pipeline.BuildError("CANDIDATE_EVIDENCE_SCHEMA", "profile evidence is invalid")
+
+    requirements_raw = _optional_snapshot_member(snapshot, "requirements.json")
+    requirements = (
+        _json_object(requirements_raw, "requirements.json")
+        if requirements_raw is not None
+        else None
+    )
+    manifest = verified.manifest
+    requirements_sha256 = (
+        sha256_bytes(requirements_raw)
+        if requirements_raw is not None
+        else _manifest_digest(manifest, "plan_sha256")
+    )
+    time_coverage = _time_coverage(requirements)
+
+    source_facts: list[BuildSourceFacts] = []
+    for source in sources_value:
+        if not isinstance(source, Mapping):
+            raise pipeline.BuildError("CANDIDATE_EVIDENCE_SCHEMA", "source evidence is invalid")
+        rights = source.get("rights")
+        if not isinstance(rights, Mapping):
+            raise pipeline.BuildError("CANDIDATE_EVIDENCE_SCHEMA", "source rights are invalid")
+        source_facts.append(
+            BuildSourceFacts(
+                source_id=source.get("source_id"),
+                content_sha256=source.get("sha256"),
+                rights_status=rights.get("status"),
+                rights_evidence_sha256=sha256_bytes(canonical_json_bytes(dict(rights))),
+            )
+        )
+
+    recipe_lineage = _recipe_lineage(snapshot)
+    return BuildIndexEntry(
+        question=inspection.question,
+        question_sha256=sha256_bytes(inspection.question.encode("utf-8")),
+        requirements_sha256=requirements_sha256,
+        grain=inspection.grain,
+        time_coverage=time_coverage,
+        fields=tuple(BuildField(name, str(types[name])) for name in columns),
+        sources=tuple(sorted(source_facts, key=lambda item: item.source_id)),
+        profile=SealedProfileFactsV1(
+            evidence_sha256=sha256_bytes(profile_raw),
+            row_count=row_count,
+            columns=tuple(columns),
+            logical_types=tuple(str(types[name]) for name in columns),
+            null_counts=tuple(null_counts[name] for name in columns),
+        ),
+        recipe_lineage=recipe_lineage,
+        candidate_sha256=verified.candidate_digest,
+        table_sha256=verified.table_sha256,
+        validation_policy_sha256=verified.validation_policy_digest,
+        lineage_sha256=sha256_bytes(canonical_json_bytes(lineage)),
+    )
+
+
+def _optional_snapshot_member(snapshot: pipeline.VerifiedSnapshot, path: str) -> bytes | None:
+    member_paths = {
+        item.get("path")
+        for item in snapshot.verified.manifest.get("members", ())
+        if isinstance(item, Mapping)
+    }
+    if path not in member_paths:
+        return None
+    return snapshot.member_bytes(path)
+
+
+def _recipe_lineage(snapshot: pipeline.VerifiedSnapshot) -> BuildRecipeLineage | None:
+    recipe_raw = _optional_snapshot_member(snapshot, "recipe.json")
+    if recipe_raw is None:
+        return None
+    execution_raw = _optional_snapshot_member(snapshot, "recipe-execution.json")
+    if execution_raw is None:
+        raise pipeline.BuildError("CANDIDATE_MEMBER_MISMATCH", "recipe execution is absent")
+    recipe = _json_object(recipe_raw, "recipe.json")
+    execution = _json_object(execution_raw, "recipe-execution.json")
+    recipe_sha256 = canonical_sha256(recipe)
+    if execution.get("recipe_digest") != recipe_sha256:
+        raise pipeline.BuildError("CANDIDATE_MEMBER_INVALID", "recipe lineage is invalid")
+    return BuildRecipeLineage(
+        recipe_id=recipe.get("recipe_id"),
+        recipe_version=recipe.get("recipe_version"),
+        recipe_sha256=recipe_sha256,
+        predecessor_candidate_sha256=execution.get("predecessor_candidate_digest"),
+    )
+
+
+def _time_coverage(requirements: Mapping[str, Any] | None) -> BuildTimeCoverage | None:
+    if requirements is None:
+        return None
+    value = requirements.get("time_range")
+    if not isinstance(value, Mapping):
+        raise pipeline.BuildError("CANDIDATE_MEMBER_INVALID", "requirements time range is invalid")
+    start = value.get("start_inclusive")
+    end = value.get("end_exclusive")
+    if not isinstance(start, str) or not isinstance(end, str):
+        raise pipeline.BuildError("CANDIDATE_MEMBER_INVALID", "requirements time range is invalid")
+    return BuildTimeCoverage(start[:10], end[:10])
+
+
+def _manifest_digest(manifest: Mapping[str, Any], name: str) -> str:
+    value = manifest.get(name)
+    if not isinstance(value, str):
+        raise pipeline.BuildError("MANIFEST_INVALID", f"candidate {name} is invalid")
+    return value
+
+
+def _json_value(raw: bytes, path: str) -> Any:
+    try:
+        return json.loads(raw)
+    except (UnicodeError, json.JSONDecodeError) as error:
+        raise pipeline.BuildError("CANDIDATE_MEMBER_INVALID", f"{path} is invalid") from error
+
+
+def _json_object(raw: bytes, path: str) -> Mapping[str, Any]:
+    value = _json_value(raw, path)
+    if not isinstance(value, Mapping):
+        raise pipeline.BuildError("CANDIDATE_MEMBER_INVALID", f"{path} is not an object")
+    return value
+
+
+def _diagnostic_for_build_error(error: pipeline.BuildError) -> str:
+    finding = error.finding_id
+    if "RACE" in finding:
+        return "BUILD_RACE"
+    if "TOO_LARGE" in finding or "LIMIT" in finding:
+        return "BUILD_LIMIT"
+    if "SYMLINK" in finding:
+        return "BUILD_SYMLINK"
+    if "HARDLINK" in finding:
+        return "BUILD_HARDLINK"
+    return "BUILD_INVALID"
+
+
+def _publish_index(
+    index_root: Path,
+    entries: tuple[BuildIndexEntry, ...],
+    locators: BuildLocatorDocument,
+    diagnostics: BuildDiagnosticDocument,
+    *,
+    fault_hook: Callable[[str], None] | None,
+) -> PublishedBuildIndex:
+    parent = index_root.parent
+    parent_fd = _open_publish_parent(parent)
+    staging_name = f".{index_root.name}.tmp-{uuid.uuid4().hex}"
+    staging = parent / staging_name
+    published = False
+    staging_exists = False
+    staging_fd = -1
+    cleanup_fd = -1
+    cleanup_identity: tuple[int, int] | None = None
+    try:
+        try:
+            os.mkdir(staging_name, mode=0o700, dir_fd=parent_fd)
+        except FileExistsError as error:  # pragma: no cover - UUID collision
+            raise LocalSearchIndexError(
+                "INDEX_DURABILITY", "index_root", "staging coordinate already exists"
+            ) from error
+        staging_exists = True
+        staging_fd = os.open(staging_name, _OPEN_DIRECTORY, dir_fd=parent_fd)
+        staging_named = os.stat(staging_name, dir_fd=parent_fd, follow_symlinks=False)
+        staging_opened = os.fstat(staging_fd)
+        cleanup_identity = _directory_identity(staging_opened)
+        if (
+            not stat.S_ISDIR(staging_named.st_mode)
+            or _directory_identity(staging_named) != cleanup_identity
+        ):
+            raise LocalSearchIndexError(
+                "INDEX_DURABILITY", "index_root", "staging root identity changed"
+            )
+        cleanup_fd = staging_fd
+        manifest = _write_index_snapshot(staging_fd, entries, locators, diagnostics)
+        manifest_bytes = manifest.canonical_bytes
+        manifest_sha256 = sha256_bytes(manifest_bytes)
+        os.fsync(staging_fd)
+
+        capture_build_index_snapshot(staging, expected_manifest_sha256=manifest_sha256)
+        if fault_hook is not None:
+            fault_hook("after_staging")
+
+        existing = _entry_stat(parent_fd, index_root.name)
+        if existing is None:
+            os.rename(staging_name, index_root.name, src_dir_fd=parent_fd, dst_dir_fd=parent_fd)
+            staging_exists = False
+        elif stat.S_ISDIR(existing.st_mode):
+            previous_fd = os.open(index_root.name, _OPEN_DIRECTORY, dir_fd=parent_fd)
+            previous_opened = os.fstat(previous_fd)
+            if _directory_identity(existing) != _directory_identity(previous_opened):
+                os.close(previous_fd)
+                raise LocalSearchIndexError(
+                    "INDEX_DURABILITY", "index_root", "existing index root identity changed"
+                )
+            try:
+                _rename_exchange(parent_fd, staging_name, index_root.name)
+            except BaseException:
+                os.close(previous_fd)
+                raise
+            cleanup_fd = previous_fd
+            cleanup_identity = _directory_identity(previous_opened)
+        else:
+            raise LocalSearchIndexError(
+                "INDEX_DURABILITY", "index_root", "existing index root is not a directory"
+            )
+        published = True
+        os.fsync(parent_fd)
+        if fault_hook is not None:
+            fault_hook("after_publish")
+        captured = capture_build_index_snapshot(
+            index_root, expected_manifest_sha256=manifest_sha256
+        )
+        if captured.manifest != manifest:
+            raise LocalSearchIndexError(
+                "INDEX_DURABILITY", "index_root", "published successor failed exact readback"
+            )
+        return PublishedBuildIndex(index_root, manifest, manifest_sha256)
+    finally:
+        try:
+            if staging_exists and cleanup_fd >= 0 and cleanup_identity is not None:
+                _remove_open_tree_at(
+                    parent_fd,
+                    staging_name,
+                    cleanup_fd,
+                    expected_identity=cleanup_identity,
+                    ignore_errors=published,
+                )
+        finally:
+            if cleanup_fd >= 0 and cleanup_fd != staging_fd:
+                os.close(cleanup_fd)
+            if staging_fd >= 0:
+                os.close(staging_fd)
+            os.close(parent_fd)
+
+
+def _open_publish_parent(parent: Path) -> int:
+    try:
+        named = os.lstat(parent)
+        descriptor = os.open(parent, _OPEN_DIRECTORY)
+        opened = os.fstat(descriptor)
+    except OSError as error:
+        raise LocalSearchIndexError(
+            "INDEX_DURABILITY", "index_root", "cannot open the retained publication parent"
+        ) from error
+    if not stat.S_ISDIR(named.st_mode) or (named.st_dev, named.st_ino) != (
+        opened.st_dev,
+        opened.st_ino,
+    ):
+        os.close(descriptor)
+        raise LocalSearchIndexError(
+            "INDEX_DURABILITY", "index_root", "publication parent identity changed"
+        )
+    return descriptor
+
+
+def _write_index_snapshot(
+    root_fd: int,
+    entries: tuple[BuildIndexEntry, ...],
+    locators: BuildLocatorDocument,
+    diagnostics: BuildDiagnosticDocument,
+) -> BuildIndexManifest:
+    os.mkdir("entries", mode=0o700, dir_fd=root_fd)
+    entries_fd = os.open("entries", _OPEN_DIRECTORY, dir_fd=root_fd)
+    entry_members: list[SnapshotMember] = []
+    try:
+        for entry in sorted(entries, key=lambda item: item.facts_sha256):
+            raw = entry.canonical_bytes
+            name = f"{entry.facts_sha256}.json"
+            _create_file_at(entries_fd, name, raw, mode=0o444)
+            entry_members.append(SnapshotMember(f"entries/{name}", len(raw), sha256_bytes(raw)))
+        os.fsync(entries_fd)
+    finally:
+        os.close(entries_fd)
+
+    locator_raw = locators.canonical_bytes
+    diagnostic_raw = diagnostics.canonical_bytes
+    _create_file_at(root_fd, "locators.json", locator_raw, mode=0o600)
+    _create_file_at(root_fd, "diagnostics.json", diagnostic_raw, mode=0o600)
+    locator_member = SnapshotMember("locators.json", len(locator_raw), sha256_bytes(locator_raw))
+    diagnostic_member = SnapshotMember(
+        "diagnostics.json", len(diagnostic_raw), sha256_bytes(diagnostic_raw)
+    )
+    members = (*entry_members, locator_member, diagnostic_member)
+    manifest = BuildIndexManifest(
+        entries=tuple(entry_members),
+        locators=locator_member,
+        diagnostics=diagnostic_member,
+        entry_count=len(entry_members),
+        locator_count=len(locators.locators),
+        diagnostic_count=len(diagnostics.diagnostics),
+        total_bytes=sum(item.bytes for item in members),
+        aggregate_sha256=manifest_aggregate_sha256(members),
+    )
+    _create_file_at(root_fd, "manifest.json", manifest.canonical_bytes, mode=0o600)
+    return manifest
+
+
+def _create_file_at(parent_fd: int, name: str, raw: bytes, *, mode: int) -> None:
+    flags = (
+        os.O_RDWR
+        | os.O_CREAT
+        | os.O_EXCL
+        | getattr(os, "O_CLOEXEC", 0)
+        | getattr(os, "O_NOFOLLOW", 0)
+    )
+    descriptor = os.open(name, flags, 0o600, dir_fd=parent_fd)
+    try:
+        view = memoryview(raw)
+        while view:
+            written = os.write(descriptor, view)
+            if written < 1:
+                raise LocalSearchIndexError(
+                    "INDEX_DURABILITY", name, "member write stopped making progress"
+                )
+            view = view[written:]
+        os.fsync(descriptor)
+        observed = os.pread(descriptor, len(raw) + 1, 0)
+        if observed != raw:
+            raise LocalSearchIndexError(
+                "INDEX_DURABILITY", name, "member readback differs from committed bytes"
+            )
+        os.fchmod(descriptor, mode)
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def _entry_stat(parent_fd: int, name: str) -> os.stat_result | None:
+    try:
+        return os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+    except FileNotFoundError:
+        return None
+
+
+def _directory_identity(observed: os.stat_result) -> tuple[int, int]:
+    return observed.st_dev, observed.st_ino
+
+
+def _remove_open_tree_at(
+    parent_fd: int,
+    name: str,
+    descriptor: int,
+    *,
+    expected_identity: tuple[int, int],
+    ignore_errors: bool,
+) -> None:
+    """Remove only the still-bound descriptor-rooted staging tree."""
+
+    try:
+        named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+        opened = os.fstat(descriptor)
+        if (
+            not stat.S_ISDIR(named.st_mode)
+            or not stat.S_ISDIR(opened.st_mode)
+            or _directory_identity(named) != expected_identity
+            or _directory_identity(opened) != expected_identity
+        ):
+            return
+        _clear_open_directory(descriptor)
+        after = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+        if _directory_identity(after) != expected_identity:
+            return
+        os.rmdir(name, dir_fd=parent_fd)
+    except FileNotFoundError:
+        return
+    except OSError as error:
+        if not ignore_errors:
+            raise LocalSearchIndexError(
+                "INDEX_DURABILITY", "index_root", "staging cleanup failed safely"
+            ) from error
+
+
+def _clear_open_directory(directory_fd: int) -> None:
+    with os.scandir(directory_fd) as iterator:
+        names = sorted(entry.name for entry in iterator)
+    for name in names:
+        named = os.stat(name, dir_fd=directory_fd, follow_symlinks=False)
+        if stat.S_ISDIR(named.st_mode):
+            child_fd = os.open(name, _OPEN_DIRECTORY, dir_fd=directory_fd)
+            try:
+                opened = os.fstat(child_fd)
+                identity = _directory_identity(opened)
+                if _directory_identity(named) != identity:
+                    raise OSError(errno.ESTALE, "cleanup child identity changed")
+                _clear_open_directory(child_fd)
+                after = os.stat(name, dir_fd=directory_fd, follow_symlinks=False)
+                if _directory_identity(after) != identity:
+                    raise OSError(errno.ESTALE, "cleanup child identity changed")
+                os.rmdir(name, dir_fd=directory_fd)
+            finally:
+                os.close(child_fd)
+        else:
+            os.unlink(name, dir_fd=directory_fd)
+
+
+def _rename_exchange(parent_fd: int, left: str, right: str) -> None:
+    libc = ctypes.CDLL(None, use_errno=True)
+    left_raw = os.fsencode(left)
+    right_raw = os.fsencode(right)
+    if sys.platform == "darwin" and hasattr(libc, "renameatx_np"):
+        rename = libc.renameatx_np
+        rename.argtypes = _RENAME_ARGTYPES
+        rename.restype = ctypes.c_int
+        result = rename(parent_fd, left_raw, parent_fd, right_raw, 0x00000002)
+    elif hasattr(libc, "renameat2"):
+        rename = libc.renameat2
+        rename.argtypes = _RENAME_ARGTYPES
+        rename.restype = ctypes.c_int
+        result = rename(parent_fd, left_raw, parent_fd, right_raw, 0x00000002)
+    else:  # pragma: no cover - supported product platforms provide one primitive
+        raise LocalSearchIndexError(
+            "INDEX_DURABILITY", "index_root", "atomic directory exchange is unavailable"
+        )
+    if result != 0:
+        error_number = ctypes.get_errno()
+        raise LocalSearchIndexError(
+            "INDEX_DURABILITY", "index_root", "atomic directory exchange failed"
+        ) from OSError(error_number, os.strerror(error_number))
+
+
+__all__ = [
+    "BuildDiscovery",
+    "BuildDiscoveryLimits",
+    "LocalSearchIndexError",
+    "PublishedBuildIndex",
+    "build_private_index",
+    "discover_build_candidates",
+]