mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/cadence_emission.py

@@ -0,0 +1,453 @@
+"""The one empirical-observation emitter both governed producers share.
+
+Two producers probe the same kind of source.  The trusted Builder drives a hosted Courier
+session and reads Studio's append coordinate; ``mr-data acquire`` drives a local Clean room and
+reads an on-disk history.  What they may state about the publisher is identical, so the rule
+that turns one probe into one :class:`~mostlyright.data_harness.sources.cadence.
+EmpiricalSourceObservation` lives here once rather than twice.
+
+That is not tidiness.  The closure bar for empirical cadence is that local and hosted inference
+replay to the same result, and two restatements of the same mapping are two things that can
+drift apart between releases while every test on either side still passes.  One implementation
+makes byte-identity a property of the code rather than a coincidence a test has to keep
+re-checking.
+
+This module records evidence and infers nothing.  It reads no clock, no network, no environment,
+and no file: every timestamp, digest, and coordinate arrives from the caller that authenticated
+it.  Deciding *what to probe next* remains Studio's, and releasing anything remains Studio's.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Any
+
+from mostlyright.data_harness.sources.cadence import (
+    SOURCE_CADENCE_ALGORITHM_DIGEST,
+    CadenceContractError,
+    EmpiricalSourceObservation,
+    SourceFrontier,
+)
+
+#: The exact key set of a cadence authority document, whichever coordinator states it.
+#: ``schema_version`` names the stating authority -- Studio's contract hosted, the local
+#: acquisition contract locally -- and the remaining eleven fields are the epoch identity the
+#: observation contract fixes.  A producer that accepted a different key set could bind an
+#: observation to an identity its coordinator never stated.
+SOURCE_CADENCE_AUTHORITY_FIELDS = frozenset(
+    {
+        "schema_version",
+        "inference_epoch",
+        "connector_configuration_digest",
+        "locator_digest",
+        "recipe_digest",
+        "source_authority_digest",
+        "reader_family_id",
+        "reader_family_version",
+        "reader_options_digest",
+        "frontier_policy_digest",
+        "time_interpretation_digest",
+        "inference_algorithm_digest",
+    }
+)
+
+# A probe that reached the source and refused reports one closed failure code.  These are the
+# only cadence outcomes that code alone can prove.  The map is deliberately narrow: a refusal
+# carries no HTTP status, so nothing here may claim ``http_4xx``, ``http_5xx``, ``rate_limited``,
+# or ``source_unavailable`` -- each asserts a response no producer on this path saw.
+# ``HTTP_NOT_FOUND`` is the one code the acquisition layer defines as "this address is not
+# published" (``acquisition/http.py``), so it is the only code that becomes evidence about the
+# publisher rather than about the probe.  Every other code stays a ``failed`` probe, which the
+# inference is required to treat as proving nothing about whether an edition exists.
+SOURCE_REACHED_FAILURE_OUTCOMES: Mapping[str, tuple[str, str | None]] = MappingProxyType(
+    {
+        "HTTP_NOT_FOUND": ("not_published", None),
+        "HTTP_STATUS": ("failed", "unknown_failure"),
+        "TOTAL_TIMEOUT": ("failed", "timeout"),
+        "CONNECT_TIMEOUT": ("failed", "timeout"),
+        "READ_TIMEOUT": ("failed", "timeout"),
+        "TLS_FAILED": ("failed", "tls_failure"),
+        "CONNECT_FAILED": ("failed", "transport_failure"),
+        "DNS_FAILED": ("failed", "transport_failure"),
+        "MEDIA_TYPE": ("failed", "contract_refused"),
+        "BYTE_LIMIT": ("failed", "contract_refused"),
+        "REDIRECT_LIMIT": ("failed", "contract_refused"),
+        "REDIRECT_POLICY": ("failed", "contract_refused"),
+        "CRAWLER_ENDPOINT_MISMATCH": ("failed", "contract_refused"),
+        "CRAWLER_MEDIA_TYPE_MISMATCH": ("failed", "contract_refused"),
+        "CRAWLER_RESPONSE_INVALID": ("failed", "contract_refused"),
+        "CRAWLER_NORMALIZATION_INPUT_INVALID": ("failed", "contract_refused"),
+        "READER_FAMILY_UNKNOWN": ("failed", "reader_failure"),
+        "READER_DEPENDENCY_UNAVAILABLE": ("failed", "reader_failure"),
+        "READER_CONTRACT": ("failed", "reader_failure"),
+        "READER_OPTIONS": ("failed", "reader_failure"),
+        "READER_ADMISSION": ("failed", "reader_failure"),
+        "READER_BUDGET": ("failed", "reader_failure"),
+        "READER_MEMBER_MISSING": ("failed", "reader_failure"),
+        "READER_MEMBER_AMBIGUOUS": ("failed", "reader_failure"),
+        "READER_MEMBER_TYPE": ("failed", "reader_failure"),
+        "READER_NESTED_CONTAINER": ("failed", "reader_failure"),
+        "READER_SHEET_MISSING": ("failed", "reader_failure"),
+        "READER_RANGE": ("failed", "reader_failure"),
+        "READER_GRID_UNKNOWN": ("failed", "reader_failure"),
+        "READER_PACKING_UNKNOWN": ("failed", "reader_failure"),
+        "READER_DECODE": ("failed", "reader_failure"),
+        "READER_OUTPUT": ("failed", "reader_failure"),
+    }
+)
+UNCLASSIFIED_FAILURE_OUTCOME: tuple[str, str | None] = ("failed", "unknown_failure")
+# These two refusals provably precede any source byte.  Both producers run the Reader
+# memory-boundary assertion and the packaged-sample warm-up before the fetch, and the warm-up
+# decodes the packaged known sample rather than the source.  An observation built from either
+# would charge the publisher for a probe of this source that never happened, so they record
+# nothing at all.
+PRE_FETCH_FAILURE_CODES = frozenset({"CRAWLER_MEMORY_BOUNDARY", "READER_WARMUP"})
+
+_DIGEST = re.compile(r"^[0-9a-f]{64}$")
+# The frontier every whole-document HTTPS probe records.  Neither producer's query contract
+# declares an approved typed event-time field or partition key, so there is no deterministic
+# derivation to record and the contract refuses a frontier on an absent result outright.
+_NO_FRONTIER = SourceFrontier("none", None, "none")
+
+
+class SourceCadenceEmissionError(ValueError):
+    """Typed refusal raised when a probe cannot be recorded as an exact observation."""
+
+    def __init__(self, code: str, detail: str) -> None:
+        self.code = code
+        self.detail = detail
+        super().__init__(f"{detail} [{code}]")
+
+
+def sealed_refusal_absent_outcome(failure_code: str) -> tuple[str, str | None] | None:
+    """Return the absent result a *sealed* refusal supports, or ``None`` for none.
+
+    A sealed refusal is a completed result: the Courier reached the source, refused, and bound
+    that refusal into a document it returned.  Reaching the source is therefore proven even for
+    a code this map does not classify, so an unclassified code is still a failed probe and
+    records ``unknown_failure`` rather than a classification the producer cannot support.  It
+    can never become ``not_published``.
+    """
+
+    if failure_code in PRE_FETCH_FAILURE_CODES:
+        return None
+    return SOURCE_REACHED_FAILURE_OUTCOMES.get(failure_code, UNCLASSIFIED_FAILURE_OUTCOME)
+
+
+def local_refusal_absent_outcome(failure_code: str) -> tuple[str, str | None] | None:
+    """Return the absent result a *local* refusal supports, or ``None`` for none.
+
+    The map and the pre-fetch set are the hosted path's own; one line differs, and the reason is
+    evidence rather than preference.  A hosted refusal arrives inside a document the Courier
+    sealed around its own failure, which is what proves the source was reached.  A local
+    acquisition raises its refusal as an exception with no such seal, so an unclassified code --
+    a mismatched attestation, a malformed acquisition document, a snapshot that could not be
+    sealed -- proves only that this machine stopped, and says nothing about the publisher.
+    Recording it would charge the source for a probe it never answered, so it records nothing.
+    """
+
+    if failure_code in PRE_FETCH_FAILURE_CODES:
+        return None
+    return SOURCE_REACHED_FAILURE_OUTCOMES.get(failure_code)
+
+
+@dataclass(frozen=True)
+class SourceCadenceAuthority:
+    """The epoch identity one coordinator fixed for one source, as stated by that coordinator."""
+
+    inference_epoch: str
+    connector_configuration_digest: str
+    locator_digest: str
+    recipe_digest: str
+    source_authority_digest: str
+    reader_family_id: str
+    reader_family_version: str
+    reader_options_digest: str
+    frontier_policy_digest: str
+    time_interpretation_digest: str
+    inference_algorithm_digest: str
+    schema_version: str
+
+    @classmethod
+    def from_mapping(cls, value: Any, *, schema_version: str) -> SourceCadenceAuthority:
+        """Admit exactly the stated authority contract, or refuse it whole.
+
+        ``inference_algorithm_digest`` must be this build's algorithm digest.  A coordinator
+        naming any other algorithm is naming an inference this producer cannot replay, and an
+        observation bound to it would enter a history no consumer here can validate.
+        """
+
+        if not isinstance(value, Mapping) or set(value) != SOURCE_CADENCE_AUTHORITY_FIELDS:
+            raise SourceCadenceEmissionError(
+                "SOURCE_CADENCE_AUTHORITY_INVALID",
+                "the cadence authority does not have the exact versioned key set",
+            )
+        payload = dict(value)
+        if payload["schema_version"] != schema_version:
+            raise SourceCadenceEmissionError(
+                "SOURCE_CADENCE_AUTHORITY_INVALID",
+                "the cadence authority states another contract version",
+            )
+        if payload["inference_algorithm_digest"] != SOURCE_CADENCE_ALGORITHM_DIGEST:
+            raise SourceCadenceEmissionError(
+                "SOURCE_CADENCE_AUTHORITY_INVALID",
+                "the cadence authority names an inference algorithm this build cannot replay",
+            )
+        try:
+            return cls(**payload)
+        except TypeError as error:  # pragma: no cover - key set already checked above
+            raise SourceCadenceEmissionError(
+                "SOURCE_CADENCE_AUTHORITY_INVALID",
+                "the cadence authority is not the exact authority document",
+            ) from error
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "inference_epoch": self.inference_epoch,
+            "connector_configuration_digest": self.connector_configuration_digest,
+            "locator_digest": self.locator_digest,
+            "recipe_digest": self.recipe_digest,
+            "source_authority_digest": self.source_authority_digest,
+            "reader_family_id": self.reader_family_id,
+            "reader_family_version": self.reader_family_version,
+            "reader_options_digest": self.reader_options_digest,
+            "frontier_policy_digest": self.frontier_policy_digest,
+            "time_interpretation_digest": self.time_interpretation_digest,
+            "inference_algorithm_digest": self.inference_algorithm_digest,
+        }
+
+
+@dataclass(frozen=True)
+class SourceProbeCoordinate:
+    """Where in one epoch's chain this probe lands, and what bounded it in time."""
+
+    sequence: int
+    previous_observation_digest: str | None
+    latest_acquired: EmpiricalSourceObservation | None
+    started_at: str
+    completed_at: str
+    probe_receipt_digest: str
+
+
+@dataclass(frozen=True)
+class AcquiredProbeFacts:
+    """What one probe that acquired bytes independently proved about them.
+
+    Every field is a fact about the acquired artifact or the response that carried it.  The
+    validator headers are digests rather than values because they are untrusted publisher bytes;
+    the contract stores them hashed for the same reason.
+    """
+
+    acquired_size_bytes: int
+    content_digest: str
+    row_digest: str
+    schema_digest: str
+    row_count: int
+    http_status_code: int | None
+    etag_digest: str | None
+    last_modified_digest: str | None
+
+
+def acquired_observation(
+    *,
+    source_id: str,
+    authority: SourceCadenceAuthority,
+    coordinate: SourceProbeCoordinate,
+    facts: AcquiredProbeFacts,
+) -> EmpiricalSourceObservation:
+    """Build the exact observation for one probe that acquired bytes.
+
+    The result kind comes from one comparison and nothing else: the acquired content digest
+    against the predecessor acquired observation's.  Equal content that arrives with any other
+    parsed fact changed is not an unchanged edition and is not a changed one either -- it is a
+    contradiction, and it is refused rather than recorded as either.
+    """
+
+    latest_acquired = coordinate.latest_acquired
+    if latest_acquired is None or latest_acquired.content_digest != facts.content_digest:
+        result_kind = "changed"
+    else:
+        if (
+            facts.acquired_size_bytes != latest_acquired.acquired_size_bytes
+            or facts.row_digest != latest_acquired.row_digest
+            or facts.schema_digest != latest_acquired.schema_digest
+            or facts.row_count != latest_acquired.row_count
+            or latest_acquired.frontier != _NO_FRONTIER
+            or latest_acquired.max_event_time is not None
+            or latest_acquired.partition_digests
+            or latest_acquired.next_watermark_candidate_digest is not None
+            or latest_acquired.publication_reference_time is not None
+            or latest_acquired.edition_sequence is not None
+        ):
+            raise SourceCadenceEmissionError(
+                "SOURCE_CADENCE_RESULT_INCONSISTENT",
+                "identical acquired content has contradictory parsed evidence",
+            )
+        result_kind = "unchanged"
+    return _observation(
+        source_id=source_id,
+        authority=authority,
+        coordinate=coordinate,
+        result_kind=result_kind,
+        failure_code=None,
+        http_status_code=facts.http_status_code,
+        etag_digest=facts.etag_digest,
+        last_modified_digest=facts.last_modified_digest,
+        acquired_size_bytes=facts.acquired_size_bytes,
+        content_digest=facts.content_digest,
+        row_digest=facts.row_digest,
+        schema_digest=facts.schema_digest,
+        row_count=facts.row_count,
+        acquisition_receipt_digest=coordinate.probe_receipt_digest,
+    )
+
+
+def absent_observation(
+    *,
+    source_id: str,
+    authority: SourceCadenceAuthority,
+    coordinate: SourceProbeCoordinate,
+    outcome: tuple[str, str | None],
+) -> EmpiricalSourceObservation:
+    """Build the exact observation for one probe that reached the source and acquired nothing.
+
+    The result kind comes only from the closed failure code the probe reported.  No bytes were
+    acquired, so the observation reports no status code, validator digest, acquired fact,
+    frontier, or publication coordinate -- it holds none of them, and the contract refuses an
+    absent result that claims any of them.
+    """
+
+    result_kind, failure_code = outcome
+    return _observation(
+        source_id=source_id,
+        authority=authority,
+        coordinate=coordinate,
+        result_kind=result_kind,
+        failure_code=failure_code,
+        # A refusal carries no HTTP status, so the observation claims none.  The contract
+        # admits a null status for both absent result kinds.
+        http_status_code=None,
+        etag_digest=None,
+        last_modified_digest=None,
+        acquired_size_bytes=None,
+        content_digest=None,
+        row_digest=None,
+        schema_digest=None,
+        row_count=None,
+        acquisition_receipt_digest=None,
+    )
+
+
+def _observation(
+    *,
+    source_id: str,
+    authority: SourceCadenceAuthority,
+    coordinate: SourceProbeCoordinate,
+    result_kind: str,
+    failure_code: str | None,
+    http_status_code: int | None,
+    etag_digest: str | None,
+    last_modified_digest: str | None,
+    acquired_size_bytes: int | None,
+    content_digest: str | None,
+    row_digest: str | None,
+    schema_digest: str | None,
+    row_count: int | None,
+    acquisition_receipt_digest: str | None,
+) -> EmpiricalSourceObservation:
+    """Assemble one observation from an authority, a coordinate, and one probe's facts.
+
+    Every field either comes from the coordinator that authenticated it or is absent, and the
+    contract validates the assembled result.  A refusal here is typed rather than a raw
+    ``ValueError`` so both producers can report it with their own code without inspecting the
+    exception's text.
+    """
+
+    try:
+        return EmpiricalSourceObservation(
+            source_id=source_id,
+            sequence=coordinate.sequence,
+            inference_epoch=authority.inference_epoch,
+            previous_observation_digest=coordinate.previous_observation_digest,
+            connector_configuration_digest=authority.connector_configuration_digest,
+            locator_digest=authority.locator_digest,
+            recipe_digest=authority.recipe_digest,
+            source_authority_digest=authority.source_authority_digest,
+            reader_family_id=authority.reader_family_id,
+            reader_family_version=authority.reader_family_version,
+            reader_options_digest=authority.reader_options_digest,
+            frontier_policy_digest=authority.frontier_policy_digest,
+            time_interpretation_digest=authority.time_interpretation_digest,
+            inference_algorithm_digest=authority.inference_algorithm_digest,
+            started_at=coordinate.started_at,
+            completed_at=coordinate.completed_at,
+            result_kind=result_kind,
+            failure_code=failure_code,
+            http_status_class=(
+                "none" if http_status_code is None else f"{http_status_code // 100}xx"
+            ),
+            http_status_code=http_status_code,
+            # Neither producer issues a conditional request on this transport, so no probe here
+            # can report the no-body HTTP 304 shape the contract also admits.
+            conditional_request_digest=None,
+            probe_receipt_digest=coordinate.probe_receipt_digest,
+            etag_digest=etag_digest,
+            last_modified_digest=last_modified_digest,
+            acquired_size_bytes=acquired_size_bytes,
+            content_digest=content_digest,
+            row_digest=row_digest,
+            schema_digest=schema_digest,
+            row_count=row_count,
+            # A whole-document HTTPS fetch observes no approved typed frontier field, so this
+            # path records no frontier, no maximum event time, and no partition digests rather
+            # than deriving them from byte order.  The publication coordinate is absent for two
+            # independent reasons.  The receipt seals only ``last_modified_digest``, a one-way
+            # hash, so there is no publication reference time here to record.  And even with the
+            # value, the contract admits that time only paired with an edition sequence, which
+            # this transport cannot prove: it cannot tell how many editions a publisher released
+            # between two probes, and counting observed changes would assert exactly that.
+            frontier=_NO_FRONTIER,
+            max_event_time=None,
+            partition_digests=(),
+            acquisition_receipt_digest=acquisition_receipt_digest,
+            next_watermark_candidate_digest=None,
+            publication_reference_time=None,
+            edition_sequence=None,
+        )
+    except CadenceContractError as error:
+        raise SourceCadenceEmissionError(
+            "SOURCE_CADENCE_OBSERVATION_INVALID",
+            f"the probe does not form an exact observation: {error.detail}",
+        ) from error
+
+
+def require_probe_receipt_digest(value: Any) -> str:
+    """Admit one bare lowercase SHA-256 probe receipt digest."""
+
+    if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
+        raise SourceCadenceEmissionError(
+            "SOURCE_CADENCE_CONTEXT_INVALID",
+            "the probe receipt digest is not a bare lowercase SHA-256 digest",
+        )
+    return value
+
+
+__all__ = [
+    "PRE_FETCH_FAILURE_CODES",
+    "SOURCE_CADENCE_AUTHORITY_FIELDS",
+    "SOURCE_REACHED_FAILURE_OUTCOMES",
+    "UNCLASSIFIED_FAILURE_OUTCOME",
+    "AcquiredProbeFacts",
+    "SourceCadenceAuthority",
+    "SourceCadenceEmissionError",
+    "SourceProbeCoordinate",
+    "absent_observation",
+    "acquired_observation",
+    "local_refusal_absent_outcome",
+    "require_probe_receipt_digest",
+    "sealed_refusal_absent_outcome",
+]