mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/cadence_history.py

@@ -0,0 +1,546 @@
+"""The local append-only empirical observation history one operator retains for one source.
+
+Hosted probes append to a chain Studio authenticates and persists.  A local probe has no Studio,
+so this module is the local coordinator's equivalent: one folder per source epoch, one immutable
+canonical record per probe, and the same digest chain the contract validates.
+
+Three properties are the point, and each is enforced rather than described.  The history is
+append-only, because a probe record is created with ``O_EXCL`` and a sequence that already exists
+is a refusal rather than an overwrite.  It is canonical, because every record is the exact
+canonical JSON bytes of its document and is re-parsed through the contract on the way back in.
+It is digest-bound, because each record carries the observation, the observation's digest, and
+the probe receipt whose digest that observation names -- so a record that was edited no longer
+verifies against itself, and a record moved between histories no longer verifies against its
+predecessor.
+
+The stored authority is what fixes the epoch.  The contract requires a new inference epoch when
+the connector configuration, locator, program, source authority, Reader, Reader options, frontier
+derivation, time interpretation, or algorithm changes, so a history whose authority no longer
+matches the one this probe was given is refused whole rather than continued.  Two epochs never
+enter one inference.
+
+This module reads no clock and no network.  It grants nothing: a local history is evidence an
+operator retained, not a release, an approval, or a schedule.
+"""
+
+from __future__ import annotations
+
+import os
+import stat
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    canonical_sha256,
+    parse_canonical_json,
+)
+from mostlyright.data_harness.sources.cadence import (
+    CadenceContractError,
+    EmpiricalSourceObservation,
+    parse_empirical_source_observation,
+    validate_observation_history,
+)
+from mostlyright.data_harness.sources.cadence_emission import (
+    SOURCE_CADENCE_AUTHORITY_FIELDS,
+    SourceCadenceAuthority,
+    SourceCadenceEmissionError,
+)
+from mostlyright.data_harness.sources.contracts import (
+    AcquisitionRequest,
+    decode_options_digest,
+)
+from mostlyright.data_harness.ux.plain_file import (
+    MISSING,
+    PlainFileRefusal,
+    read_plain_file,
+)
+
+LOCAL_CADENCE_AUTHORITY_SCHEMA = "harness-local-source-cadence-authority.v1"
+LOCAL_CADENCE_RECORD_SCHEMA = "harness-local-source-cadence-record.v1"
+LOCAL_CADENCE_PROBE_REFUSAL_SCHEMA = "harness-local-source-probe-refusal.v1"
+
+#: The contract's own ceiling on one inference history.  Refusing at the same number keeps a
+#: folder that can still be read by ``infer_source_cadence`` from being written past the point
+#: where it cannot.
+MAX_LOCAL_OBSERVATIONS = 4_096
+_AUTHORITY_NAME = "authority.json"
+_OBSERVATIONS_NAME = "observations"
+_RECORD_KEYS = frozenset({"schema_version", "observation", "observation_digest", "probe_receipt"})
+# A record is small: one observation, one receipt.  The ceiling is a bound on what this reads
+# from a folder another local user may also write to, not a size anything here aims for.
+_MAX_RECORD_BYTES = 1 << 20
+
+
+class LocalCadenceHistoryError(ValueError):
+    """Typed refusal raised by the local observation history."""
+
+    def __init__(self, code: str, detail: str) -> None:
+        self.code = code
+        self.detail = detail
+        super().__init__(f"{detail} [{code}]")
+
+
+@dataclass(frozen=True)
+class LocalObservationHistory:
+    """One epoch's stored authority and its complete validated observation chain."""
+
+    directory: Path
+    authority: SourceCadenceAuthority
+    observations: tuple[EmpiricalSourceObservation, ...]
+
+    @property
+    def next_sequence(self) -> int:
+        return len(self.observations)
+
+    @property
+    def previous_observation_digest(self) -> str | None:
+        return self.observations[-1].digest if self.observations else None
+
+    @property
+    def latest_acquired(self) -> EmpiricalSourceObservation | None:
+        """Return the newest observation that acquired bytes, which is the change baseline.
+
+        A probe that acquired nothing proves nothing about the content, so it can neither be a
+        baseline nor erase one.  Reading backwards for the newest acquired fact is what lets a
+        failed probe sit between two acquisitions without turning the next one into a change.
+        """
+
+        for observation in reversed(self.observations):
+            if observation.content_digest is not None:
+                return observation
+        return None
+
+
+def parse_local_cadence_authority(value: Any) -> SourceCadenceAuthority:
+    """Parse one operator-stated local cadence authority document."""
+
+    try:
+        return SourceCadenceAuthority.from_mapping(
+            value,
+            schema_version=LOCAL_CADENCE_AUTHORITY_SCHEMA,
+        )
+    except SourceCadenceEmissionError as error:
+        raise LocalCadenceHistoryError("CADENCE_AUTHORITY_INVALID", error.detail) from error
+
+
+def require_authority_binds_request(
+    authority: SourceCadenceAuthority,
+    request: AcquisitionRequest,
+) -> None:
+    """Refuse an authority that does not describe the acquisition it is recording.
+
+    A local coordinator states the epoch identity, exactly as Studio states it hosted.  Two of
+    its eleven fields are not statements at all locally, because this side holds the acquisition
+    document and can check them.  The locator is the canonical query digest: an authority whose
+    locator names another address would splice two sources into one history, and the inference
+    would read a cadence neither publisher has.  The Reader coordinate is the pinned decoder:
+    an authority naming a Reader the request did not entitle would attribute this epoch's parse
+    to code that never ran.
+
+    The remaining fields stay statements this side takes as given, the way the hosted producer
+    takes Studio's.  Recording that difference is the point of checking the two that can be
+    checked rather than none.
+    """
+
+    if authority.locator_digest != request.query_digest:
+        raise LocalCadenceHistoryError(
+            "CADENCE_AUTHORITY_LOCATOR",
+            "the cadence authority names a locator other than this request's exact query",
+        )
+    pin = request.reader_pin
+    if pin is None:
+        return
+    if (
+        authority.reader_family_id != pin["family_id"]
+        or authority.reader_family_version != pin["family_version"]
+        or authority.reader_options_digest != decode_options_digest(pin["decode_options"])
+    ):
+        raise LocalCadenceHistoryError(
+            "CADENCE_AUTHORITY_READER",
+            "the cadence authority names a Reader coordinate this request did not entitle",
+        )
+
+
+def probe_refusal_document(
+    *,
+    source_id: str,
+    request_digest: str,
+    query_digest: str,
+    acquisition_environment: str,
+    clean_room_attestation_digest: str,
+    external_network_policy_attestation: str,
+    refusal_code: str,
+    probed_at: str,
+) -> dict[str, Any]:
+    """Seal what a refused probe proves, so its observation has a receipt to bind.
+
+    Every observation names a probe receipt, because evidence that a probe happened is what
+    separates a recorded absence from an assumption.  A refused probe acquired nothing, so there
+    is no acquisition receipt to name and this document is the receipt: the exact request, the
+    confinement it ran in, and the typed code the source interaction returned.  It holds no
+    response body, no header text, and no acquired fact, because the probe obtained none.
+    """
+
+    return {
+        "schema_version": LOCAL_CADENCE_PROBE_REFUSAL_SCHEMA,
+        "source_id": source_id,
+        "request_digest": request_digest,
+        "query_digest": query_digest,
+        "acquisition_environment": acquisition_environment,
+        "clean_room_attestation_digest": clean_room_attestation_digest,
+        "external_network_policy_attestation": external_network_policy_attestation,
+        "refusal_code": refusal_code,
+        "probed_at": probed_at,
+    }
+
+
+def open_local_history(
+    directory: Path,
+    *,
+    authority: SourceCadenceAuthority | None,
+) -> LocalObservationHistory:
+    """Open or start one local history and return its complete validated chain.
+
+    ``authority`` is the authority this probe was given.  A history that already holds one keeps
+    it: the stored bytes decide, and a supplied authority that differs is refused rather than
+    merged, because a differing authority is a different epoch and the contract never combines
+    two epochs into one inference.
+    """
+
+    root = _require_history_directory(directory)
+    stored = _read_authority(root)
+    if stored is None:
+        if authority is None:
+            raise LocalCadenceHistoryError(
+                "CADENCE_AUTHORITY_REQUIRED",
+                "this history holds no authority yet, so the first probe has to state one",
+            )
+        try:
+            _write_exclusive(root / _AUTHORITY_NAME, canonical_json_bytes(authority.to_dict()))
+        except FileExistsError as error:
+            raise LocalCadenceHistoryError(
+                "CADENCE_AUTHORITY_EXISTS",
+                "another probe stated this history's epoch identity first",
+            ) from error
+        stored = authority
+    elif authority is not None and authority.to_dict() != stored.to_dict():
+        raise LocalCadenceHistoryError(
+            "CADENCE_AUTHORITY_CHANGED",
+            "this history was recorded under a different epoch identity",
+        )
+    observations = _read_observations(root, stored)
+    return LocalObservationHistory(directory=root, authority=stored, observations=observations)
+
+
+def append_local_observation(
+    history: LocalObservationHistory,
+    observation: EmpiricalSourceObservation,
+    probe_receipt: dict[str, Any],
+) -> LocalObservationHistory:
+    """Append one probe record and return the extended, revalidated history."""
+
+    if observation.sequence != history.next_sequence:
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_SEQUENCE",
+            "the observation does not continue this history",
+        )
+    if len(history.observations) >= MAX_LOCAL_OBSERVATIONS:
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_FULL",
+            "this history already holds every observation one inference may read",
+        )
+    _require_no_record_beyond(history)
+    receipt_digest = canonical_sha256(probe_receipt)
+    if receipt_digest != observation.probe_receipt_digest:
+        raise LocalCadenceHistoryError(
+            "CADENCE_RECEIPT_BINDING",
+            "the observation does not name the probe receipt recorded beside it",
+        )
+    extended = (*history.observations, observation)
+    try:
+        validate_observation_history(extended)
+    except CadenceContractError as error:
+        raise LocalCadenceHistoryError("CADENCE_HISTORY_INVALID", error.detail) from error
+    record = {
+        "schema_version": LOCAL_CADENCE_RECORD_SCHEMA,
+        "observation": observation.to_dict(),
+        "observation_digest": observation.digest,
+        "probe_receipt": dict(probe_receipt),
+    }
+    observations_root = history.directory / _OBSERVATIONS_NAME
+    _make_directory(observations_root)
+    try:
+        _write_exclusive(
+            observations_root / _record_name(observation.sequence),
+            canonical_json_bytes(record),
+        )
+    except FileExistsError as error:
+        raise LocalCadenceHistoryError(
+            "CADENCE_RECORD_EXISTS",
+            "another probe was recorded into this history while this one ran",
+        ) from error
+    return LocalObservationHistory(
+        directory=history.directory,
+        authority=history.authority,
+        observations=extended,
+    )
+
+
+def _require_no_record_beyond(history: LocalObservationHistory) -> None:
+    """Refuse to append when the folder holds a record this history did not read.
+
+    A read stops at the first missing sequence, which leaves two ways the folder can hold more
+    than the history does, and both are refusals.
+
+    A record at exactly the next coordinate appeared after this history was read, which is a
+    second probe of the same source running at the same time.  A record *past* it means the read
+    stopped at a gap -- an entry removed or never written -- while later entries are still on
+    disk.  Appending into that gap would write a new sequence the survivors no longer bind, and
+    every read afterwards would refuse the chain permanently, because the predecessor they name
+    would never exist again.
+
+    The folder is read once rather than one coordinate probed, because the gap case is exactly
+    the one a single probe of the next name cannot see.
+    """
+
+    observations_root = history.directory / _OBSERVATIONS_NAME
+    try:
+        names = {item.name for item in observations_root.iterdir()}
+    except FileNotFoundError:
+        return
+    except OSError as error:
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_UNREADABLE",
+            f"the cadence history folder could not be read: {error.strerror or error}",
+        ) from error
+    beyond = [
+        sequence
+        for sequence in range(history.next_sequence, MAX_LOCAL_OBSERVATIONS + 1)
+        if _record_name(sequence) in names
+    ]
+    if not beyond:
+        return
+    if beyond == [history.next_sequence]:
+        raise LocalCadenceHistoryError(
+            "CADENCE_RECORD_EXISTS",
+            "another probe was recorded into this history while this one ran",
+        )
+    raise LocalCadenceHistoryError(
+        "CADENCE_HISTORY_INCOMPLETE",
+        "this history holds records past the point it can be read from",
+    )
+
+
+def _record_name(sequence: int) -> str:
+    return f"{sequence:010d}.json"
+
+
+def _require_history_directory(directory: Path) -> Path:
+    root = directory.absolute()
+    _make_directory(root)
+    return root
+
+
+def _make_directory(path: Path) -> None:
+    try:
+        path.mkdir(mode=0o700, parents=True, exist_ok=True)
+    except OSError as error:
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_UNWRITABLE",
+            f"the cadence history folder could not be created: {error.strerror or error}",
+        ) from error
+    # ``lstat`` rather than ``stat``: a link that points at a folder reports a folder through
+    # the link, and this has to be a folder this command created or already owns.  One test is
+    # enough -- a mode is a link or a folder and never both -- so a second ``S_ISLNK`` clause
+    # here would be a guard that can never fire.
+    if not stat.S_ISDIR(path.lstat().st_mode):
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_UNWRITABLE",
+            "the cadence history path is not a folder this command can own",
+        )
+
+
+def _write_exclusive(path: Path, payload: bytes) -> None:
+    """Create one file, or let the collision reach the caller that can name it.
+
+    ``O_EXCL`` is the append-only rule.  A name that already exists was written by somebody --
+    a second probe racing this one, or a second first probe of the same history -- and both are
+    refusals rather than overwrites, because what is already there is somebody's evidence.
+
+    ``FileExistsError`` is deliberately not turned into a typed refusal here.  An authority
+    written twice and a record written twice are different events with different repairs, so
+    each caller raises its own code, written as a literal at its own raise.  A code passed in as
+    an argument would be a code the repository's typed-code completeness gates cannot see, and
+    this repository has been caught by that shape twice already.
+    """
+
+    try:
+        descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    except OSError as error:
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_UNWRITABLE",
+            f"the cadence record could not be created: {error.strerror or error}",
+        ) from error
+    try:
+        with os.fdopen(descriptor, "wb") as handle:
+            handle.write(payload)
+            handle.flush()
+            os.fsync(handle.fileno())
+    except OSError as error:
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_UNWRITABLE",
+            f"the cadence record could not be written: {error.strerror or error}",
+        ) from error
+
+
+def _read_document(path: Path) -> Any:
+    """Read one stored record, under the open-then-fstat rule every named file is read by.
+
+    A history folder is an ordinary folder another local user may also write to, so a record is
+    read as a bounded plain file rather than by size-then-read, and the bytes have to be the exact
+    canonical encoding of what they decode to.  A record that is not is not evidence: canonical
+    bytes are what its digests were taken over.
+    """
+
+    try:
+        raw = read_plain_file(path, max_bytes=_MAX_RECORD_BYTES)
+    except PlainFileRefusal as refusal:
+        if refusal.reason == MISSING:
+            return None
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_UNREADABLE",
+            f"a cadence history entry is {refusal.reason} and cannot be read as a record",
+        ) from None
+    except OSError as error:
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_UNREADABLE",
+            f"a cadence history file could not be read: {error.strerror or error}",
+        ) from error
+    try:
+        value = parse_canonical_json(raw)
+    except CanonicalJSONError as error:
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_UNREADABLE",
+            "a cadence history file is not the exact canonical bytes it was written as",
+        ) from error
+    if not isinstance(value, dict):
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_UNREADABLE",
+            "a cadence history file does not hold a document",
+        )
+    return value
+
+
+def _read_authority(root: Path) -> SourceCadenceAuthority | None:
+    value = _read_document(root / _AUTHORITY_NAME)
+    if value is None:
+        return None
+    if set(value) != SOURCE_CADENCE_AUTHORITY_FIELDS:
+        raise LocalCadenceHistoryError(
+            "CADENCE_AUTHORITY_INVALID",
+            "the stored cadence authority does not have the exact versioned key set",
+        )
+    return parse_local_cadence_authority(value)
+
+
+def _read_observations(
+    root: Path,
+    authority: SourceCadenceAuthority,
+) -> tuple[EmpiricalSourceObservation, ...]:
+    """Re-admit every stored record in sequence and validate the chain it forms.
+
+    Names are derived rather than listed: a record is read at the coordinate it must occupy, so
+    a file dropped into the folder under some other name is not evidence and a gap ends the
+    history rather than being skipped over.
+    """
+
+    observations_root = root / _OBSERVATIONS_NAME
+    if not observations_root.exists():
+        return ()
+    observations: list[EmpiricalSourceObservation] = []
+    while len(observations) < MAX_LOCAL_OBSERVATIONS:
+        value = _read_document(observations_root / _record_name(len(observations)))
+        if value is None:
+            break
+        observations.append(_parse_record(value, sequence=len(observations), authority=authority))
+    if (observations_root / _record_name(len(observations))).exists():
+        raise LocalCadenceHistoryError(
+            "CADENCE_HISTORY_FULL",
+            "this history holds more observations than one inference may read",
+        )
+    try:
+        return validate_observation_history(tuple(observations))
+    except CadenceContractError as error:
+        raise LocalCadenceHistoryError("CADENCE_HISTORY_INVALID", error.detail) from error
+
+
+def _parse_record(
+    value: dict[str, Any],
+    *,
+    sequence: int,
+    authority: SourceCadenceAuthority,
+) -> EmpiricalSourceObservation:
+    if set(value) != _RECORD_KEYS or value["schema_version"] != LOCAL_CADENCE_RECORD_SCHEMA:
+        raise LocalCadenceHistoryError(
+            "CADENCE_RECORD_INVALID",
+            "a cadence record does not have the exact versioned key set",
+        )
+    receipt = value["probe_receipt"]
+    if not isinstance(receipt, dict):
+        raise LocalCadenceHistoryError(
+            "CADENCE_RECORD_INVALID",
+            "a cadence record carries no probe receipt",
+        )
+    try:
+        observation = parse_empirical_source_observation(value["observation"])
+    except (CadenceContractError, TypeError, ValueError) as error:
+        raise LocalCadenceHistoryError(
+            "CADENCE_RECORD_INVALID",
+            "a cadence record does not hold an exact observation",
+        ) from error
+    if observation.digest != value["observation_digest"] or observation.sequence != sequence:
+        raise LocalCadenceHistoryError(
+            "CADENCE_RECORD_INVALID",
+            "a cadence record does not bind the observation stored in it",
+        )
+    if canonical_sha256(receipt) != observation.probe_receipt_digest:
+        raise LocalCadenceHistoryError(
+            "CADENCE_RECEIPT_BINDING",
+            "a cadence record does not name the probe receipt stored beside it",
+        )
+    if (
+        observation.inference_epoch != authority.inference_epoch
+        or observation.connector_configuration_digest != authority.connector_configuration_digest
+        or observation.locator_digest != authority.locator_digest
+        or observation.recipe_digest != authority.recipe_digest
+        or observation.source_authority_digest != authority.source_authority_digest
+        or observation.reader_family_id != authority.reader_family_id
+        or observation.reader_family_version != authority.reader_family_version
+        or observation.reader_options_digest != authority.reader_options_digest
+        or observation.frontier_policy_digest != authority.frontier_policy_digest
+        or observation.time_interpretation_digest != authority.time_interpretation_digest
+        or observation.inference_algorithm_digest != authority.inference_algorithm_digest
+    ):
+        raise LocalCadenceHistoryError(
+            "CADENCE_RECORD_INVALID",
+            "a cadence record was recorded under a different epoch identity",
+        )
+    return observation
+
+
+__all__ = [
+    "LOCAL_CADENCE_AUTHORITY_SCHEMA",
+    "LOCAL_CADENCE_PROBE_REFUSAL_SCHEMA",
+    "LOCAL_CADENCE_RECORD_SCHEMA",
+    "MAX_LOCAL_OBSERVATIONS",
+    "LocalCadenceHistoryError",
+    "LocalObservationHistory",
+    "append_local_observation",
+    "open_local_history",
+    "parse_local_cadence_authority",
+    "probe_refusal_document",
+    "require_authority_binds_request",
+]

mostlyright/data_harness/sources/catalog/__init__.py

@@ -0,0 +1,17 @@
+"""Source-catalog subpackage: versioned facts about public sources.
+
+This package intentionally carries **no re-exports**. Consumers import the concrete module they
+need (``...sources.catalog.contracts``, and later ``retrieval`` and ``gating``). Two reasons:
+
+1. Retrieval and gating must never share a namespace. Similarity *finds* candidates; structured
+   facts *gate* decisions. If both were re-exported here, a gating caller would sit one attribute
+   access away from a similarity score, and the separation would become a convention rather than
+   a module boundary.
+2. A package ``__init__`` that names every module is a single contended file. Keeping it empty of
+   imports lets the catalog modules land independently.
+
+Everything in this package describes **public** sources only. No module here may carry a dataset,
+a recipe, a Reader, or anything learned from a tenant's private source.
+"""
+
+from __future__ import annotations