mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/local_search/sealed.py

@@ -0,0 +1,474 @@
+"""Descriptor-rooted, bounded capture of one private Build-index snapshot."""
+
+from __future__ import annotations
+
+import os
+import stat
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    parse_canonical_json,
+    sha256_bytes,
+)
+from mostlyright.data_harness.local_search.contracts import (
+    MAX_ENTRIES,
+    BuildDiagnosticDocument,
+    BuildIndexEntry,
+    BuildIndexManifest,
+    BuildLocatorDocument,
+    LocalSearchContractError,
+    SnapshotMember,
+    build_diagnostic_document_from_dict,
+    build_index_entry_from_dict,
+    build_index_manifest_from_dict,
+    build_locator_document_from_dict,
+)
+
+_OPEN_DIRECTORY = os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_CLOEXEC", 0)
+_OPEN_MEMBER = (
+    os.O_RDONLY
+    | getattr(os, "O_CLOEXEC", 0)
+    | getattr(os, "O_NOFOLLOW", 0)
+    | getattr(os, "O_NONBLOCK", 0)
+)
+_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
+_READ_CHUNK = 1024 * 1024
+
+
+@dataclass(frozen=True)
+class SnapshotLimits:
+    """Closed capture budgets applied before unbounded allocation."""
+
+    max_member_bytes: int = 16 * 1024 * 1024
+    max_total_bytes: int = 256 * 1024 * 1024
+    max_members: int = MAX_ENTRIES + 2
+
+    def __post_init__(self) -> None:
+        for name, value, maximum in (
+            ("max_member_bytes", self.max_member_bytes, 256 * 1024 * 1024),
+            ("max_total_bytes", self.max_total_bytes, 1024 * 1024 * 1024),
+            ("max_members", self.max_members, MAX_ENTRIES + 2),
+        ):
+            if type(value) is not int or not 1 <= value <= maximum:
+                raise LocalSearchContractError(
+                    "LOCAL_SEARCH_SNAPSHOT_LIMIT",
+                    f"limits.{name}",
+                    f"must be an integer in [1, {maximum}]",
+                )
+
+
+@dataclass(frozen=True)
+class CapturedBuildIndexSnapshot:
+    """A fully verified private index plus the exact bytes that produced it."""
+
+    manifest: BuildIndexManifest
+    manifest_sha256: str
+    entries: tuple[BuildIndexEntry, ...]
+    locators: BuildLocatorDocument
+    diagnostics: BuildDiagnosticDocument
+    manifest_bytes: bytes
+    member_bytes: tuple[tuple[str, bytes], ...]
+
+
+def capture_build_index_snapshot(
+    directory: Path,
+    *,
+    expected_manifest_sha256: str,
+    limits: SnapshotLimits | None = None,
+) -> CapturedBuildIndexSnapshot:
+    """Capture and verify a private index from one retained root identity.
+
+    ``expected_manifest_sha256`` is deliberately mandatory: the live directory cannot nominate
+    its own authority coordinate merely by replacing all of its files consistently.
+    """
+
+    _expected_digest(expected_manifest_sha256)
+    if not isinstance(directory, Path):
+        raise LocalSearchContractError("LOCAL_SEARCH_TYPE", "directory", "must be a pathlib.Path")
+    selected_limits = limits or SnapshotLimits()
+    if not isinstance(selected_limits, SnapshotLimits):
+        raise LocalSearchContractError("LOCAL_SEARCH_TYPE", "limits", "must be SnapshotLimits")
+
+    root_fd = _open_root(directory)
+    try:
+        root_identity = _directory_identity(root_fd, "root")
+        root_names_before = _directory_names(root_fd, "root", maximum=5)
+        expected_root_names = {"manifest.json", "entries", "locators.json", "diagnostics.json"}
+        _exact_membership(root_names_before, expected_root_names, "root")
+
+        manifest_bytes = _read_plain_file(
+            root_fd,
+            "manifest.json",
+            maximum=min(selected_limits.max_member_bytes, selected_limits.max_total_bytes),
+        )
+        manifest_sha256 = sha256_bytes(manifest_bytes)
+        if manifest_sha256 != expected_manifest_sha256:
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_MANIFEST_MISMATCH",
+                "manifest.json",
+                "does not match the caller's expected manifest digest",
+            )
+        manifest = build_index_manifest_from_dict(_canonical_mapping(manifest_bytes, "manifest"))
+        if len(manifest.members) > selected_limits.max_members:
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_SNAPSHOT_LIMIT",
+                "manifest.entries",
+                "snapshot exceeds the member-count limit",
+            )
+        if manifest.total_bytes + len(manifest_bytes) > selected_limits.max_total_bytes:
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_SNAPSHOT_LIMIT",
+                "manifest.total_bytes",
+                "snapshot exceeds the aggregate byte limit",
+            )
+        for member in manifest.members:
+            if member.bytes > selected_limits.max_member_bytes:
+                raise LocalSearchContractError(
+                    "LOCAL_SEARCH_SNAPSHOT_LIMIT",
+                    member.path,
+                    "member exceeds the byte limit",
+                )
+
+        entries_fd = _open_child_directory(root_fd, "entries")
+        try:
+            entries_identity = _directory_identity(entries_fd, "entries")
+            expected_entry_names = {
+                member.path.removeprefix("entries/") for member in manifest.entries
+            }
+            _exact_membership(
+                _directory_names(
+                    entries_fd,
+                    "entries",
+                    maximum=selected_limits.max_members + 1,
+                ),
+                expected_entry_names,
+                "entries",
+            )
+
+            captured_members: list[tuple[str, bytes]] = []
+            parsed_entries: list[BuildIndexEntry] = []
+            for member in manifest.entries:
+                name = member.path.removeprefix("entries/")
+                raw = _capture_bound_member(entries_fd, name, member, selected_limits)
+                entry = build_index_entry_from_dict(_canonical_mapping(raw, member.path))
+                if entry.facts_sha256 != member.sha256 or name != f"{entry.facts_sha256}.json":
+                    raise LocalSearchContractError(
+                        "LOCAL_SEARCH_ENTRY_BINDING",
+                        member.path,
+                        "filename and member digest must equal the entry facts digest",
+                    )
+                parsed_entries.append(entry)
+                captured_members.append((member.path, raw))
+
+            _validate_directory(entries_fd, entries_identity, "entries")
+            _validate_named_directory(root_fd, "entries", entries_identity)
+            _exact_membership(
+                _directory_names(
+                    entries_fd,
+                    "entries",
+                    maximum=selected_limits.max_members + 1,
+                ),
+                expected_entry_names,
+                "entries",
+            )
+        finally:
+            os.close(entries_fd)
+
+        locator_bytes = _capture_bound_member(
+            root_fd, "locators.json", manifest.locators, selected_limits
+        )
+        diagnostic_bytes = _capture_bound_member(
+            root_fd, "diagnostics.json", manifest.diagnostics, selected_limits
+        )
+        locators = build_locator_document_from_dict(
+            _canonical_mapping(locator_bytes, "locators.json")
+        )
+        diagnostics = build_diagnostic_document_from_dict(
+            _canonical_mapping(diagnostic_bytes, "diagnostics.json")
+        )
+        if len(locators.locators) != manifest.locator_count:
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_MANIFEST",
+                "manifest.locator_count",
+                "does not match locators.json",
+            )
+        if len(diagnostics.diagnostics) != manifest.diagnostic_count:
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_MANIFEST",
+                "manifest.diagnostic_count",
+                "does not match diagnostics.json",
+            )
+        entry_digests = {entry.facts_sha256 for entry in parsed_entries}
+        dangling = sorted(
+            locator.facts_sha256
+            for locator in locators.locators
+            if locator.facts_sha256 not in entry_digests
+        )
+        if dangling:
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_LOCATOR_BINDING",
+                "locators.json",
+                "a locator names facts absent from this snapshot",
+            )
+
+        captured_members.extend(
+            (("locators.json", locator_bytes), ("diagnostics.json", diagnostic_bytes))
+        )
+        _validate_directory(root_fd, root_identity, "root")
+        _exact_membership(_directory_names(root_fd, "root", maximum=5), expected_root_names, "root")
+        return CapturedBuildIndexSnapshot(
+            manifest=manifest,
+            manifest_sha256=manifest_sha256,
+            entries=tuple(parsed_entries),
+            locators=locators,
+            diagnostics=diagnostics,
+            manifest_bytes=manifest_bytes,
+            member_bytes=tuple(captured_members),
+        )
+    finally:
+        os.close(root_fd)
+
+
+def _open_root(directory: Path) -> int:
+    try:
+        descriptor = os.open(directory, _OPEN_DIRECTORY | _NOFOLLOW)
+    except OSError as error:
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_SNAPSHOT_ROOT",
+            "root",
+            "cannot open a retained no-follow directory",
+        ) from error
+    try:
+        _directory_identity(descriptor, "root")
+    except BaseException:
+        os.close(descriptor)
+        raise
+    return descriptor
+
+
+def _open_child_directory(parent_fd: int, name: str) -> int:
+    try:
+        named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+        descriptor = os.open(name, _OPEN_DIRECTORY | _NOFOLLOW, dir_fd=parent_fd)
+    except OSError as error:
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_SNAPSHOT_MEMBER",
+            name,
+            "cannot open a retained no-follow directory",
+        ) from error
+    try:
+        opened = os.fstat(descriptor)
+        if not stat.S_ISDIR(named.st_mode) or not stat.S_ISDIR(opened.st_mode):
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_SNAPSHOT_MEMBER", name, "must be a directory"
+            )
+        if _identity(named) != _identity(opened):
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_SNAPSHOT_RACE", name, "named and opened identities differ"
+            )
+    except BaseException:
+        os.close(descriptor)
+        raise
+    return descriptor
+
+
+def _capture_bound_member(
+    parent_fd: int,
+    name: str,
+    expected: SnapshotMember,
+    limits: SnapshotLimits,
+) -> bytes:
+    raw = _read_plain_file(parent_fd, name, maximum=limits.max_member_bytes)
+    if len(raw) != expected.bytes or sha256_bytes(raw) != expected.sha256:
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_SNAPSHOT_MEMBER",
+            expected.path,
+            "size or digest differs from the manifest",
+        )
+    return raw
+
+
+def _read_plain_file(parent_fd: int, name: str, *, maximum: int) -> bytes:
+    """Read one named file once, checking opened and named identity on both sides."""
+
+    descriptor = -1
+    try:
+        named_before = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+        descriptor = os.open(name, _OPEN_MEMBER, dir_fd=parent_fd)
+        opened_before = os.fstat(descriptor)
+    except OSError as error:
+        if descriptor >= 0:
+            os.close(descriptor)
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_SNAPSHOT_MEMBER", name, "cannot open a plain no-follow member"
+        ) from error
+    try:
+        if (
+            not stat.S_ISREG(named_before.st_mode)
+            or not stat.S_ISREG(opened_before.st_mode)
+            or named_before.st_nlink != 1
+            or opened_before.st_nlink != 1
+        ):
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_SNAPSHOT_MEMBER",
+                name,
+                "must be a single-link regular file",
+            )
+        if _identity(named_before) != _identity(opened_before):
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_SNAPSHOT_RACE", name, "named and opened identities differ"
+            )
+        if opened_before.st_size > maximum:
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_SNAPSHOT_LIMIT", name, "member exceeds the byte limit"
+            )
+        chunks: list[bytes] = []
+        observed = 0
+        while True:
+            chunk = os.read(descriptor, min(_READ_CHUNK, maximum - observed + 1))
+            if not chunk:
+                break
+            observed += len(chunk)
+            if observed > maximum:
+                raise LocalSearchContractError(
+                    "LOCAL_SEARCH_SNAPSHOT_LIMIT", name, "member exceeds the byte limit"
+                )
+            chunks.append(chunk)
+        opened_after = os.fstat(descriptor)
+        try:
+            named_after = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+        except OSError as error:
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_SNAPSHOT_RACE", name, "named member disappeared during capture"
+            ) from error
+        raw = b"".join(chunks)
+        if (
+            _identity(opened_before) != _identity(opened_after)
+            or _identity(opened_after) != _identity(named_after)
+            or opened_after.st_nlink != 1
+            or len(raw) != opened_after.st_size
+        ):
+            raise LocalSearchContractError(
+                "LOCAL_SEARCH_SNAPSHOT_RACE", name, "member changed during capture"
+            )
+        return raw
+    finally:
+        os.close(descriptor)
+
+
+def _canonical_mapping(raw: bytes, path: str) -> Mapping[str, Any]:
+    try:
+        value = parse_canonical_json(raw)
+    except CanonicalJSONError as error:
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_CANONICAL", path, "member is not exact canonical JSON"
+        ) from error
+    if not isinstance(value, Mapping):
+        raise LocalSearchContractError("LOCAL_SEARCH_TYPE", path, "must contain an object")
+    return value
+
+
+def _expected_digest(value: Any) -> str:
+    if (
+        not isinstance(value, str)
+        or len(value) != 64
+        or any(char not in "0123456789abcdef" for char in value)
+    ):
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_SHA256",
+            "expected_manifest_sha256",
+            "must be a lowercase 64-character SHA-256",
+        )
+    return value
+
+
+def _directory_names(descriptor: int, path: str, *, maximum: int) -> set[str]:
+    """Read at most ``maximum`` names, refusing before an attacker-sized allocation."""
+
+    names: set[str] = set()
+    try:
+        with os.scandir(descriptor) as entries:
+            for entry in entries:
+                if len(names) >= maximum:
+                    raise LocalSearchContractError(
+                        "LOCAL_SEARCH_SNAPSHOT_MEMBERSHIP",
+                        path,
+                        "directory contains more names than the bounded snapshot permits",
+                    )
+                if entry.name in names:
+                    raise LocalSearchContractError(
+                        "LOCAL_SEARCH_SNAPSHOT_MEMBERSHIP",
+                        path,
+                        "directory contains a duplicate name",
+                    )
+                names.add(entry.name)
+    except LocalSearchContractError:
+        raise
+    except OSError as error:
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_SNAPSHOT_RACE", path, "directory cannot be listed"
+        ) from error
+    return names
+
+
+def _exact_membership(actual: set[str], expected: set[str], path: str) -> None:
+    if actual != expected:
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_SNAPSHOT_MEMBERSHIP",
+            path,
+            "membership is not exact; "
+            f"missing_count={len(expected - actual)}, extra_count={len(actual - expected)}",
+        )
+
+
+def _directory_identity(descriptor: int, path: str) -> tuple[int, int, int, int, int, int]:
+    try:
+        observed = os.fstat(descriptor)
+    except OSError as error:
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_SNAPSHOT_RACE", path, "directory identity cannot be read"
+        ) from error
+    if not stat.S_ISDIR(observed.st_mode):
+        raise LocalSearchContractError("LOCAL_SEARCH_SNAPSHOT_MEMBER", path, "must be a directory")
+    return _identity(observed)
+
+
+def _validate_directory(
+    descriptor: int, expected: tuple[int, int, int, int, int, int], path: str
+) -> None:
+    if _directory_identity(descriptor, path) != expected:
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_SNAPSHOT_RACE", path, "directory changed during capture"
+        )
+
+
+def _validate_named_directory(
+    parent_fd: int,
+    name: str,
+    expected: tuple[int, int, int, int, int, int],
+) -> None:
+    try:
+        named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+    except OSError as error:
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_SNAPSHOT_RACE", name, "directory disappeared during capture"
+        ) from error
+    if not stat.S_ISDIR(named.st_mode) or _identity(named) != expected:
+        raise LocalSearchContractError(
+            "LOCAL_SEARCH_SNAPSHOT_RACE", name, "named directory changed during capture"
+        )
+
+
+def _identity(value: os.stat_result) -> tuple[int, int, int, int, int, int]:
+    return (
+        value.st_dev,
+        value.st_ino,
+        value.st_mode,
+        value.st_size,
+        value.st_mtime_ns,
+        value.st_ctime_ns,
+    )