mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/registry.py

@@ -0,0 +1,514 @@
+"""Allowlisted source-adapter registry."""
+
+from __future__ import annotations
+
+import errno
+import hashlib
+import os
+import stat
+from dataclasses import dataclass, field, replace
+from pathlib import Path
+from typing import Protocol
+
+from mostlyright.data_harness.acquisition.http import ProbeTransport
+from mostlyright.data_harness.sources.contracts import (
+    AcquisitionReceipt,
+    AcquisitionReceiptV2,
+    AcquisitionReceiptV3,
+    AcquisitionRequest,
+    AdapterDescriptor,
+    SourceContractError,
+    SourceObservation,
+)
+
+
+@dataclass(frozen=True)
+class AcquisitionContext:
+    """Trusted coordinator inputs that source content cannot override."""
+
+    snapshot_root: Path
+    trusted_clock: str
+    attempt_id: str
+    max_source_bytes: int
+    max_rows: int
+    max_columns: int
+    crawler_sandbox_attestation_digest: str | None = None
+    # The decode/normalize output budget, distinct from the fetched-byte budget above. It
+    # narrows Reader decode output and bounds the sealed snapshot; ``None`` means "no separate
+    # budget was stated" and every consumer falls back to ``max_source_bytes``, which is the
+    # historical behaviour.
+    max_normalized_bytes: int | None = None
+
+    def __post_init__(self) -> None:
+        if not self.snapshot_root.is_absolute():
+            raise SourceContractError(
+                "TRUSTED_ROOT",
+                "context.snapshot_root",
+                "trusted snapshot root must be absolute",
+            )
+        if type(self.max_source_bytes) is not int or not 1 <= self.max_source_bytes <= 1 << 34:
+            raise SourceContractError(
+                "LIMIT",
+                "context.max_source_bytes",
+                "must be a positive bounded integer",
+            )
+        if self.max_normalized_bytes is not None and (
+            type(self.max_normalized_bytes) is not int
+            or not 1 <= self.max_normalized_bytes <= 1 << 34
+        ):
+            raise SourceContractError(
+                "LIMIT",
+                "context.max_normalized_bytes",
+                "must be a positive bounded integer",
+            )
+        if type(self.max_rows) is not int or not 1 <= self.max_rows <= 10_000_000:
+            raise SourceContractError(
+                "LIMIT",
+                "context.max_rows",
+                "must be a positive bounded integer",
+            )
+        if type(self.max_columns) is not int or not 1 <= self.max_columns <= 10_000:
+            raise SourceContractError(
+                "LIMIT",
+                "context.max_columns",
+                "must be a positive bounded integer",
+            )
+
+
+@dataclass(frozen=True)
+class AcquisitionOutput:
+    """Immutable adapter result binding one receipt, its observation, and the sealed snapshot."""
+
+    receipt: AcquisitionReceipt | AcquisitionReceiptV2 | AcquisitionReceiptV3
+    observation: SourceObservation
+    parsed_schema_digest: str | None = None
+    snapshot_path: Path | None = None
+    next_watermark: str | None = None
+    snapshot_content: bytes | None = field(default=None, repr=False)
+    # What the terminal response stated, sealed inside the Clean room and carried out hashed.
+    # It is not on the receipt: a receipt is sealed evidence whose digest is bound across the
+    # repository, and this fact belongs to the empirical observation the probe produces rather
+    # than to the artifact it acquired. ``None`` for every adapter that settles on no single
+    # response, which is every adapter but the whole-document public HTTPS one.
+    probe_transport: ProbeTransport | None = None
+    # The parsed geometry an empirical observation states beside the schema digest. Both are
+    # facts about the same parse ``parsed_schema_digest`` already comes from, carried rather
+    # than recomputed so a consumer cannot derive a second answer from a second parse.
+    parsed_row_count: int | None = None
+    parsed_row_digest: str | None = None
+
+    def __post_init__(self) -> None:
+        if self.receipt.source_id != self.observation.source_id:
+            raise SourceContractError(
+                "SOURCE_BINDING",
+                "output",
+                "receipt and observation source IDs must match",
+            )
+        if self.receipt.observation_digest != self.observation.digest:
+            raise SourceContractError(
+                "OBSERVATION_BINDING",
+                "output.receipt.observation_digest",
+                "receipt does not bind the exact source observation",
+            )
+        if self.receipt.observed_at != self.observation.observed_at:
+            raise SourceContractError(
+                "OBSERVATION_TIME_BINDING",
+                "output.receipt.observed_at",
+                "receipt and observation times differ",
+            )
+        if self.receipt.event_time_field != self.observation.event_time_field:
+            raise SourceContractError(
+                "EVENT_TIME_BINDING",
+                "output.receipt.event_time_field",
+                "receipt and observation event-time fields differ",
+            )
+        if self.parsed_schema_digest is not None and (
+            len(self.parsed_schema_digest) != 64
+            or any(char not in "0123456789abcdef" for char in self.parsed_schema_digest)
+        ):
+            raise SourceContractError(
+                "PARSED_SCHEMA_DIGEST",
+                "output.parsed_schema_digest",
+                "must be a lowercase SHA-256",
+            )
+        if self.snapshot_path is not None and not self.snapshot_path.is_absolute():
+            raise SourceContractError(
+                "SNAPSHOT_PATH",
+                "output.snapshot_path",
+                "must be an absolute coordinator-owned path",
+            )
+        if self.snapshot_content is not None:
+            if not isinstance(self.snapshot_content, bytes):
+                raise SourceContractError(
+                    "SNAPSHOT_CONTENT",
+                    "output.snapshot_content",
+                    "must be immutable bytes",
+                )
+            if (
+                len(self.snapshot_content) != self.receipt.snapshot.size_bytes
+                or hashlib.sha256(self.snapshot_content).hexdigest()
+                != self.receipt.snapshot.content_sha256
+            ):
+                raise SourceContractError(
+                    "SNAPSHOT_CONTENT",
+                    "output.snapshot_content",
+                    "must match the exact acquisition receipt",
+                )
+        if self.probe_transport is not None and not isinstance(
+            self.probe_transport, ProbeTransport
+        ):
+            raise SourceContractError(
+                "PROBE_TRANSPORT",
+                "output.probe_transport",
+                "must be sealed transport evidence",
+            )
+        row_facts = (self.parsed_row_count, self.parsed_row_digest)
+        if any(item is None for item in row_facts) and not all(item is None for item in row_facts):
+            raise SourceContractError(
+                "PARSED_ROW_FACTS",
+                "output.parsed_row_count",
+                "row count and row digest are both present or both absent",
+            )
+        if self.parsed_row_digest is not None and (
+            len(self.parsed_row_digest) != 64
+            or any(char not in "0123456789abcdef" for char in self.parsed_row_digest)
+        ):
+            raise SourceContractError(
+                "PARSED_ROW_FACTS",
+                "output.parsed_row_digest",
+                "must be a lowercase SHA-256",
+            )
+        if self.next_watermark is not None and (
+            not isinstance(self.next_watermark, str)
+            or not 1 <= len(self.next_watermark.encode("utf-8")) <= 512
+        ):
+            raise SourceContractError(
+                "WATERMARK",
+                "output.next_watermark",
+                "must be bounded nonempty text",
+            )
+
+
+class SourceAdapter(Protocol):
+    @property
+    def descriptor(self) -> AdapterDescriptor:
+        """Return the immutable adapter identity."""
+
+    def acquire(
+        self,
+        request: AcquisitionRequest,
+        context: AcquisitionContext,
+    ) -> AcquisitionOutput:
+        """Acquire through the adapter's deterministic boundary."""
+
+
+class AdapterRegistry:
+    """Coordinator-owned adapter allowlist."""
+
+    def __init__(self, adapters: tuple[SourceAdapter, ...] = ()) -> None:
+        self._adapters: dict[tuple[str, str], SourceAdapter] = {}
+        for adapter in adapters:
+            self.register(adapter)
+
+    def register(self, adapter: SourceAdapter) -> None:
+        descriptor = adapter.descriptor
+        if not isinstance(descriptor, AdapterDescriptor):
+            raise SourceContractError(
+                "ADAPTER_DESCRIPTOR",
+                "adapter.descriptor",
+                "adapter descriptor must use the strict contract",
+            )
+        key = (descriptor.adapter_id, descriptor.adapter_version)
+        if key in self._adapters:
+            coordinate = f"{descriptor.adapter_id}@{descriptor.adapter_version}"
+            raise SourceContractError(
+                "ADAPTER_DUPLICATE",
+                "registry",
+                f"adapter {coordinate} is already registered",
+            )
+        self._adapters[key] = adapter
+
+    def resolve(self, adapter_id: str, adapter_version: str) -> SourceAdapter:
+        try:
+            return self._adapters[(adapter_id, adapter_version)]
+        except KeyError:
+            raise SourceContractError(
+                "ADAPTER_UNAVAILABLE",
+                "request.adapter_id",
+                f"adapter {adapter_id}@{adapter_version} is not allowlisted",
+            ) from None
+
+    def acquire(
+        self,
+        request: AcquisitionRequest,
+        context: AcquisitionContext,
+    ) -> AcquisitionOutput:
+        adapter = self.resolve(request.adapter_id, request.adapter_version)
+        output = adapter.acquire(request, context)
+        descriptor = adapter.descriptor
+        if output.receipt.request_digest != request.digest:
+            raise SourceContractError(
+                "REQUEST_BINDING",
+                "output.receipt.request_digest",
+                "adapter receipt does not bind the exact request",
+            )
+        if output.receipt.query_digest != request.query_digest:
+            raise SourceContractError(
+                "QUERY_BINDING",
+                "output.receipt.query_digest",
+                "adapter receipt does not bind the exact query",
+            )
+        if output.receipt.adapter != descriptor:
+            raise SourceContractError(
+                "ADAPTER_BINDING",
+                "output.receipt.adapter",
+                "adapter receipt identity differs from the registered adapter",
+            )
+        trusted_path, trusted_content = _authenticate_acquisition_snapshot(context, output)
+        return replace(
+            output,
+            snapshot_path=trusted_path,
+            snapshot_content=trusted_content,
+        )
+
+    def inventory(self) -> tuple[AdapterDescriptor, ...]:
+        return tuple(self._adapters[key].descriptor for key in sorted(self._adapters))
+
+
+def _authenticate_acquisition_snapshot(
+    context: AcquisitionContext,
+    output: AcquisitionOutput,
+) -> tuple[Path, bytes]:
+    """Return receipt-addressed bytes authenticated beneath the exact trusted root."""
+
+    reference = output.receipt.snapshot
+    expected_name = f"{reference.content_sha256}.{reference.data_format}"
+    if reference.storage_name != expected_name:
+        raise SourceContractError(
+            "ACQUISITION_SNAPSHOT",
+            "output.receipt.snapshot.storage_name",
+            "must be the exact receipt-addressed snapshot name",
+        )
+    trusted_path = context.snapshot_root / expected_name
+    if output.snapshot_path is not None and output.snapshot_path != trusted_path:
+        raise SourceContractError(
+            "ACQUISITION_SNAPSHOT",
+            "output.snapshot_path",
+            "adapter path differs from the receipt-addressed trusted-root path",
+        )
+    # The sealed snapshot is the normalized acquisition output, so its bound is the
+    # normalized-output budget when one was stated, falling back to the fetched-byte
+    # budget exactly as the hosted query composition does.
+    snapshot_limit = (
+        context.max_source_bytes
+        if context.max_normalized_bytes is None
+        else context.max_normalized_bytes
+    )
+    if reference.size_bytes > snapshot_limit:
+        raise SourceContractError(
+            "ACQUISITION_SNAPSHOT",
+            "output.receipt.snapshot.size_bytes",
+            "snapshot exceeds the trusted acquisition byte limit",
+        )
+    directory_flags = (
+        os.O_RDONLY
+        | getattr(os, "O_DIRECTORY", 0)
+        | getattr(os, "O_NOFOLLOW", 0)
+        | getattr(os, "O_CLOEXEC", 0)
+    )
+    try:
+        root_fd = os.open(context.snapshot_root, directory_flags)
+    except OSError as error:
+        raise SourceContractError(
+            "ACQUISITION_SNAPSHOT",
+            "context.snapshot_root",
+            "trusted snapshot root is unavailable or unsafe"
+            f" (open refused: {_describe_os_error(error)})",
+        ) from None
+    descriptor: int | None = None
+    try:
+        root_info = os.fstat(root_fd)
+        flags = (
+            os.O_RDONLY
+            | getattr(os, "O_NOFOLLOW", 0)
+            | getattr(os, "O_NONBLOCK", 0)
+            | getattr(os, "O_CLOEXEC", 0)
+        )
+        try:
+            descriptor = os.open(expected_name, flags, dir_fd=root_fd)
+        except OSError as error:
+            raise SourceContractError(
+                "ACQUISITION_SNAPSHOT",
+                "output.receipt.snapshot.storage_name",
+                "receipt-addressed snapshot is unavailable or unsafe"
+                f" (open refused: {_describe_os_error(error)})",
+            ) from None
+        opened = os.fstat(descriptor)
+        _require_authenticated_snapshot_stat(
+            opened,
+            root_info=root_info,
+            expected_size=reference.size_bytes,
+        )
+        try:
+            named = os.stat(expected_name, dir_fd=root_fd, follow_symlinks=False)
+        except OSError as error:
+            raise SourceContractError(
+                "ACQUISITION_SNAPSHOT",
+                "output.receipt.snapshot.storage_name",
+                "receipt-addressed snapshot changed during authentication"
+                f" (before the read: {_describe_os_error(error)})",
+            ) from None
+        _require_stable_snapshot_identity(opened, named, observed_between=_OPEN_VERSUS_NAME)
+        raw = bytearray()
+        while len(raw) <= reference.size_bytes:
+            chunk = os.read(
+                descriptor,
+                min(65_536, reference.size_bytes + 1 - len(raw)),
+            )
+            if not chunk:
+                break
+            raw.extend(chunk)
+        _require_stable_snapshot_identity(
+            opened,
+            os.fstat(descriptor),
+            observed_between=_OPEN_VERSUS_READ,
+        )
+        os.close(descriptor)
+        descriptor = None
+        try:
+            after = os.stat(expected_name, dir_fd=root_fd, follow_symlinks=False)
+        except OSError as error:
+            raise SourceContractError(
+                "ACQUISITION_SNAPSHOT",
+                "output.receipt.snapshot.storage_name",
+                "receipt-addressed snapshot changed during authentication"
+                f" (after the read: {_describe_os_error(error)})",
+            ) from None
+        _require_stable_snapshot_identity(named, after, observed_between=_NAME_VERSUS_READ)
+    finally:
+        if descriptor is not None:
+            os.close(descriptor)
+        os.close(root_fd)
+    content = bytes(raw)
+    if (
+        len(content) != reference.size_bytes
+        or hashlib.sha256(content).hexdigest() != reference.content_sha256
+    ):
+        raise SourceContractError(
+            "ACQUISITION_SNAPSHOT",
+            "output.receipt.snapshot",
+            "authenticated bytes do not match the exact acquisition receipt",
+        )
+    return trusted_path, content
+
+
+def _describe_os_error(error: OSError) -> str:
+    """Name an errno without putting a path or any other caller string in the message."""
+
+    return f"{errno.errorcode.get(error.errno or 0, 'errno')} {error.strerror or ''}".strip()
+
+
+def _require_authenticated_snapshot_stat(
+    info: os.stat_result,
+    *,
+    root_info: os.stat_result,
+    expected_size: int,
+) -> None:
+    """Refuse an entry that is not one immutable, singly-linked, root-owned regular file.
+
+    Every field is evaluated rather than short-circuited so the refusal can name each one that
+    failed with its observed and its required value.  This is the reader's half of the check
+    ``acquisition/http.py`` runs when it seals the snapshot, and it carried the same defect that
+    half did before #229: six distinct causes sharing one sentence, so an ``ACQUISITION_SNAPSHOT``
+    read off a machine nobody can attach a debugger to said only that the entry was unsafe.
+    ``st_nlink`` is the one a concurrent same-digest sealer can move, which is exactly why #209's
+    first diagnosis named the wrong gate.
+    """
+
+    faults: list[str] = []
+    if not stat.S_ISREG(info.st_mode):
+        faults.append(f"kind {stat.S_IFMT(info.st_mode):#o} is not a regular file")
+    if stat.S_IMODE(info.st_mode) != 0o444:
+        faults.append(f"st_mode {stat.S_IMODE(info.st_mode):#o} != 0o444")
+    if info.st_nlink != 1:
+        faults.append(f"st_nlink {info.st_nlink} != 1")
+    if info.st_size != expected_size:
+        faults.append(f"st_size {info.st_size} != {expected_size}")
+    if info.st_uid != root_info.st_uid:
+        faults.append(f"st_uid {info.st_uid} != {root_info.st_uid}")
+    if info.st_gid != root_info.st_gid:
+        faults.append(f"st_gid {info.st_gid} != {root_info.st_gid}")
+    if faults:
+        raise SourceContractError(
+            "ACQUISITION_SNAPSHOT",
+            "output.receipt.snapshot.storage_name",
+            "snapshot is not a root-owned single-link mode-0444 regular file of the exact size"
+            f" ({', '.join(faults)})",
+        )
+
+
+# The three points at which the bound stat signature is re-observed.  They are named in the
+# refusal so a field report says which observation drifted rather than only that one did.
+_OPEN_VERSUS_NAME = "the opened descriptor and the named entry"
+_OPEN_VERSUS_READ = "the opened descriptor before and after the read"
+_NAME_VERSUS_READ = "the named entry before and after the read"
+
+# The one drift with a single cause, said in words rather than left to be read off a field.  Kept
+# byte-identical to its counterpart in ``acquisition/http.py``: two copies of one guard that say
+# different things about one field is the drift #237 was about.
+_LAST_NAME_REMOVED = "; the last name for it was removed while it was open"
+
+#: The anti-TOCTOU binding, held field by field so a refusal can name what moved.  It is the same
+#: set ``acquisition/http.py`` binds at seal time; ``tests/h3/test_integration.py`` holds the two
+#: to each other so neither can quietly narrow.
+_SNAPSHOT_STAT_FIELDS = (
+    "st_dev",
+    "st_ino",
+    "st_uid",
+    "st_gid",
+    "st_mode",
+    "st_nlink",
+    "st_size",
+    "st_mtime_ns",
+    "st_ctime_ns",
+)
+
+
+def _require_stable_snapshot_identity(
+    before: os.stat_result,
+    after: os.stat_result,
+    *,
+    observed_between: str,
+) -> None:
+    """Refuse a snapshot whose bound identity moved, naming where it moved and by what.
+
+    The signature is a deliberate anti-TOCTOU binding, so it stays exactly as wide as it was.
+    What changes is only what the refusal says: which of the three observation pairs disagreed,
+    and every field that differs with its old and its new value.  ``st_nlink`` moving means
+    another name was attached to or detached from this inode -- an unlink of the published name
+    reads as ``st_nlink 1 -> 0`` -- ``st_ino`` or ``st_dev`` moving means the name was rebound to
+    a different file, and ``st_ctime_ns`` alone means metadata was touched.  Those are different
+    incidents and a field recurrence has to be able to tell them apart.
+
+    ``st_nlink`` reaching zero is the one of them with a single cause, so it is named rather than
+    left to be inferred: the last name for this inode was removed while this descriptor held it
+    open, which is a deletion of the snapshot rather than a change to it.  The remover is not
+    thereby identified.  Governed deletion takes the sealer's per-name lock, but this reader holds
+    no lock at all, so the harness's own retention sweep can be the remover here just as an
+    unowned process can -- the sealer's copy of this guard is the one whose refusal rules the
+    harness out, because the sealer reads inside the held name.
+    """
+
+    moved = tuple(
+        f"{field} {getattr(before, field)} -> {getattr(after, field)}"
+        for field in _SNAPSHOT_STAT_FIELDS
+        if getattr(before, field) != getattr(after, field)
+    )
+    if moved:
+        raise SourceContractError(
+            "ACQUISITION_SNAPSHOT",
+            "output.receipt.snapshot.storage_name",
+            "snapshot changed during authentication"
+            f" (between {observed_between}: {', '.join(moved)})"
+            f"{_LAST_NAME_REMOVED if before.st_nlink and not after.st_nlink else ''}",
+        )

mostlyright/data_harness/sources/rights_rule.py

@@ -0,0 +1,59 @@
+"""The single repository-wide rights-status disposition authority.
+
+One rule decides what a rights status *means*: admit, escalate, or refuse. The fitness
+adjudicator (:mod:`mostlyright.data_harness.sources.fitness`) and the source-catalog gate both
+read this module, so the two cannot drift into disagreeing adjudicators.
+
+This module deliberately imports almost nothing. It depends on ``SourceContractError`` and
+``RIGHTS_STATUSES`` from :mod:`mostlyright.data_harness.sources.contracts` and on nothing else --
+not ``fitness``, not ``local_contracts``, not anything under ``catalog``. That import poverty is
+what lets the catalog gate consume the rule without dragging the tenant proposal layer in.
+
+The mapping is exhaustive by assertion at import time. A rights status added to the vocabulary
+without a disposition is an import-time failure, never a silent admit.
+"""
+
+from __future__ import annotations
+
+from mostlyright.data_harness.sources.contracts import RIGHTS_STATUSES, SourceContractError
+
+RIGHTS_DISPOSITIONS: frozenset[str] = frozenset({"admit", "escalate", "refuse"})
+
+_DISPOSITION_BY_STATUS: dict[str, str] = {
+    "approved": "admit",
+    "conditional": "admit",
+    "unclear": "escalate",
+    "prohibited": "refuse",
+}
+
+if set(_DISPOSITION_BY_STATUS) != set(RIGHTS_STATUSES):  # pragma: no cover - import-time guard
+    _missing = sorted(set(RIGHTS_STATUSES) - set(_DISPOSITION_BY_STATUS))
+    _extra = sorted(set(_DISPOSITION_BY_STATUS) - set(RIGHTS_STATUSES))
+    raise SourceContractError(
+        "RIGHTS_DISPOSITION_COVERAGE",
+        "rights_rule",
+        f"every rights status requires a disposition; missing={_missing}, extra={_extra}",
+    )
+
+if not set(_DISPOSITION_BY_STATUS.values()) <= RIGHTS_DISPOSITIONS:  # pragma: no cover
+    raise SourceContractError(
+        "RIGHTS_DISPOSITION_VALUE",
+        "rights_rule",
+        f"dispositions must be drawn from {sorted(RIGHTS_DISPOSITIONS)}",
+    )
+
+
+def rights_disposition(status: str) -> str:
+    """Return ``admit``, ``escalate``, or ``refuse`` for one rights status.
+
+    Raises ``SourceContractError`` with code ``RIGHTS_STATUS`` for anything outside
+    ``RIGHTS_STATUSES``; an unknown status is never admitted.
+    """
+
+    if not isinstance(status, str) or status not in _DISPOSITION_BY_STATUS:
+        raise SourceContractError(
+            "RIGHTS_STATUS",
+            "rights.status",
+            f"must be one of {sorted(RIGHTS_STATUSES)}",
+        )
+    return _DISPOSITION_BY_STATUS[status]