mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/integration.py

@@ -0,0 +1,218 @@
+"""Strict source-independent integration seam for acquired tabular snapshots."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from mostlyright.data_harness.acquisition.parsing import ParsedTable
+from mostlyright.data_harness.canonical import canonical_sha256, sha256_bytes
+from mostlyright.data_harness.sources.contracts import SourceContractError
+from mostlyright.data_harness.sources.registry import AcquisitionOutput
+
+
+@dataclass(frozen=True)
+class OneToOneJoinEvidence:
+    left_input_sha256: str
+    right_input_sha256: str
+    join_key: str
+    left_rows: int
+    right_rows: int
+    output_rows: int
+    unmatched_left_rows: int
+    row_multiplier: float
+    output_columns: tuple[str, ...]
+    output_rows_value: tuple[tuple[Any, ...], ...]
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(
+            {
+                "left_input_sha256": self.left_input_sha256,
+                "right_input_sha256": self.right_input_sha256,
+                "join_key": self.join_key,
+                "left_rows": self.left_rows,
+                "right_rows": self.right_rows,
+                "output_rows": self.output_rows,
+                "unmatched_left_rows": self.unmatched_left_rows,
+                "row_multiplier": {
+                    "numerator": self.output_rows,
+                    "denominator": self.left_rows,
+                },
+                "output_columns": list(self.output_columns),
+                "output_rows_value": [list(row) for row in self.output_rows_value],
+            }
+        )
+
+
+@dataclass(frozen=True)
+class AcquiredOneToOneJoinEvidence:
+    """A one-to-one join bound to both exact acquisition receipts and observations."""
+
+    left_receipt_digest: str
+    right_receipt_digest: str
+    left_observation_digest: str
+    right_observation_digest: str
+    join: OneToOneJoinEvidence
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(
+            {
+                "left_receipt_digest": self.left_receipt_digest,
+                "right_receipt_digest": self.right_receipt_digest,
+                "left_observation_digest": self.left_observation_digest,
+                "right_observation_digest": self.right_observation_digest,
+                "join_digest": self.join.digest,
+            }
+        )
+
+
+def join_acquired_sources_one_to_one(
+    left_output: AcquisitionOutput,
+    left_table: ParsedTable,
+    right_output: AcquisitionOutput,
+    right_table: ParsedTable,
+    *,
+    join_key: str,
+) -> AcquiredOneToOneJoinEvidence:
+    """Join only tables proven to be parsed from their exact admitted snapshots."""
+
+    _require_acquired_table_binding(left_output, left_table, "left")
+    _require_acquired_table_binding(right_output, right_table, "right")
+    return AcquiredOneToOneJoinEvidence(
+        left_receipt_digest=left_output.receipt.digest,
+        right_receipt_digest=right_output.receipt.digest,
+        left_observation_digest=left_output.observation.digest,
+        right_observation_digest=right_output.observation.digest,
+        join=join_source_tables_one_to_one(left_table, right_table, join_key=join_key),
+    )
+
+
+def join_source_tables_one_to_one(
+    left: ParsedTable,
+    right: ParsedTable,
+    *,
+    join_key: str,
+) -> OneToOneJoinEvidence:
+    """Join two acquired tables only when the exact key relation is one-to-one and complete."""
+
+    if not isinstance(join_key, str) or not join_key:
+        raise SourceContractError("JOIN_KEY", "join_key", "must be non-empty text")
+    try:
+        left_key_index = left.columns.index(join_key)
+        right_key_index = right.columns.index(join_key)
+    except ValueError:
+        raise SourceContractError(
+            "JOIN_KEY",
+            "join_key",
+            "must exist in both acquired tables",
+        ) from None
+    right_extra_columns = tuple(column for column in right.columns if column != join_key)
+    overlaps = sorted(set(left.columns) & set(right_extra_columns))
+    if overlaps:
+        raise SourceContractError(
+            "JOIN_COLUMN_COLLISION",
+            "right.columns",
+            f"non-key columns collide with the left table: {overlaps}",
+        )
+    right_extra_indexes = tuple(
+        index for index, column in enumerate(right.columns) if column != join_key
+    )
+    left_keys = _unique_non_null_keys(left.rows, left_key_index, "left")
+    right_keys = _unique_non_null_keys(right.rows, right_key_index, "right")
+    right_by_key = {
+        row[right_key_index]: tuple(row[index] for index in right_extra_indexes)
+        for row in right.rows
+    }
+    unmatched = tuple(key for key in left_keys if key not in right_keys)
+    if unmatched:
+        raise SourceContractError(
+            "JOIN_UNMATCHED_LEFT",
+            "right.rows",
+            f"right table is missing {len(unmatched)} left keys",
+        )
+    output_rows = tuple((*row, *right_by_key[row[left_key_index]]) for row in left.rows)
+    if len(output_rows) != len(left.rows):
+        raise SourceContractError(
+            "JOIN_ROW_MULTIPLICATION",
+            "output_rows",
+            "one-to-one join changed the left row count",
+        )
+    return OneToOneJoinEvidence(
+        left_input_sha256=left.input_sha256,
+        right_input_sha256=right.input_sha256,
+        join_key=join_key,
+        left_rows=len(left.rows),
+        right_rows=len(right.rows),
+        output_rows=len(output_rows),
+        unmatched_left_rows=0,
+        row_multiplier=1.0,
+        output_columns=(*left.columns, *right_extra_columns),
+        output_rows_value=output_rows,
+    )
+
+
+def _require_acquired_table_binding(
+    output: AcquisitionOutput,
+    table: ParsedTable,
+    side: str,
+) -> None:
+    if (
+        output.parsed_schema_digest != table.schema_digest
+        or output.receipt.snapshot.content_sha256 != table.input_sha256
+        or output.receipt.observation_digest != output.observation.digest
+    ):
+        raise SourceContractError(
+            "ACQUIRED_TABLE_BINDING",
+            f"{side}.acquisition",
+            "parsed table is not bound to the exact receipt and observation",
+        )
+    content = output.snapshot_content
+    if content is None:
+        raise SourceContractError(
+            "ACQUIRED_SNAPSHOT",
+            f"{side}.snapshot_content",
+            "registry-authenticated snapshot bytes are unavailable",
+        )
+    if (
+        sha256_bytes(content) != output.receipt.snapshot.content_sha256
+        or len(content) != output.receipt.snapshot.size_bytes
+    ):
+        raise SourceContractError(
+            "ACQUIRED_SNAPSHOT",
+            f"{side}.snapshot_content",
+            "authenticated snapshot bytes do not match the exact acquisition receipt",
+        )
+
+
+def _unique_non_null_keys(
+    rows: tuple[tuple[Any, ...], ...],
+    key_index: int,
+    side: str,
+) -> frozenset[Any]:
+    keys: list[Any] = []
+    for row in rows:
+        key = row[key_index]
+        if key is None or key == "":
+            raise SourceContractError(
+                "JOIN_NULL_KEY",
+                f"{side}.rows",
+                "join keys must be non-null and non-empty",
+            )
+        keys.append(key)
+    try:
+        unique_keys = frozenset(keys)
+    except TypeError:
+        raise SourceContractError(
+            "JOIN_KEY_TYPE",
+            f"{side}.rows",
+            "join keys must be scalar hashable values",
+        ) from None
+    if len(unique_keys) != len(keys):
+        raise SourceContractError(
+            "JOIN_CARDINALITY",
+            f"{side}.rows",
+            "join keys must be unique for a one-to-one integration",
+        )
+    return unique_keys

mostlyright/data_harness/sources/range_reader.py

@@ -0,0 +1,418 @@
+"""Compose a range acquisition with a pinned Reader and immutable revision evidence.
+
+This module owns the public seam that neither the range crawler nor a Reader can own alone.
+The crawler returns exact selected messages and source-relative accounting; the Reader turns one
+message into canonical table bytes.  The seam binds those two outcomes before returning anything
+a recipe may seal.
+
+It is intentionally source-general.  HRRR supplies the first task, but no point, variable, host,
+cycle cadence, or expected value is embedded here.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from mostlyright.data_harness.acquisition.http import RetrievalLimits
+from mostlyright.data_harness.acquisition.parsing import ParseLimits
+from mostlyright.data_harness.acquisition.sandbox import (
+    CrawlerSandbox,
+    FetchedMember,
+    SandboxResult,
+)
+from mostlyright.data_harness.acquisition.url_policy import AcquisitionSecurityError
+from mostlyright.data_harness.canonical import canonical_sha256, sha256_bytes
+from mostlyright.data_harness.readers.contracts import ReaderBudgets, ReaderPin
+
+MAX_SOURCE_OBJECT_BYTES = 1 << 44
+_DIGEST = re.compile(r"^[0-9a-f]{64}$")
+
+
+class _RangeReaderSandbox(Protocol):
+    def fetch_ranges(self, **kwargs: Any) -> SandboxResult: ...
+
+    def decode_and_parse(self, **kwargs: Any) -> SandboxResult: ...
+
+
+def _digest(value: str, subject: str) -> str:
+    if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
+        raise AcquisitionSecurityError("RANGE_READER_LOCK", f"{subject} is not a SHA-256")
+    return value
+
+
+def _positive(value: int, subject: str, *, maximum: int = MAX_SOURCE_OBJECT_BYTES) -> int:
+    if type(value) is not int or not 1 <= value <= maximum:
+        raise AcquisitionSecurityError(
+            "RANGE_READER_LOCK",
+            f"{subject} must be a positive bounded integer",
+        )
+    return value
+
+
+@dataclass(frozen=True)
+class LockedSlice:
+    """One selected message in the operator-frozen source revision."""
+
+    descriptor: str
+    index: int
+    first_byte: int
+    last_byte: int
+    sha256: str
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.descriptor, str) or not self.descriptor:
+            raise AcquisitionSecurityError("RANGE_READER_LOCK", "slice descriptor is empty")
+        _positive(self.index, "slice index", maximum=1_000_000)
+        if (
+            type(self.first_byte) is not int
+            or type(self.last_byte) is not int
+            or self.first_byte < 0
+            or self.last_byte < self.first_byte
+            or self.last_byte >= MAX_SOURCE_OBJECT_BYTES
+        ):
+            raise AcquisitionSecurityError("RANGE_READER_LOCK", "slice byte range is invalid")
+        _digest(self.sha256, "slice digest")
+
+    @property
+    def size_bytes(self) -> int:
+        return self.last_byte - self.first_byte + 1
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "descriptor": self.descriptor,
+            "index": self.index,
+            "first_byte": self.first_byte,
+            "last_byte": self.last_byte,
+            "size_bytes": self.size_bytes,
+            "sha256": self.sha256,
+        }
+
+
+@dataclass(frozen=True)
+class RangeRevisionLock:
+    """The source revision the coordinator selected before contestant execution."""
+
+    cycle: str
+    sidecar_url: str
+    sidecar_final_url: str
+    object_url: str
+    sidecar_sha256: str
+    sidecar_size_bytes: int
+    sidecar_etag: str | None
+    sidecar_last_modified: str | None
+    object_etag: str | None
+    object_last_modified: str | None
+    full_object_size_bytes: int
+    slices: tuple[LockedSlice, ...]
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.cycle, str) or not self.cycle:
+            raise AcquisitionSecurityError("RANGE_READER_LOCK", "cycle is empty")
+        for subject, url in (
+            ("sidecar URL", self.sidecar_url),
+            ("sidecar final URL", self.sidecar_final_url),
+            ("object URL", self.object_url),
+        ):
+            if not isinstance(url, str) or not url.startswith("https://") or len(url) > 2_048:
+                raise AcquisitionSecurityError("RANGE_READER_LOCK", f"{subject} is invalid")
+        _digest(self.sidecar_sha256, "sidecar digest")
+        _positive(self.sidecar_size_bytes, "sidecar size", maximum=1 << 30)
+        _positive(self.full_object_size_bytes, "full object size")
+        for subject, pair in (
+            ("sidecar", (self.sidecar_etag, self.sidecar_last_modified)),
+            ("object", (self.object_etag, self.object_last_modified)),
+        ):
+            if pair == (None, None):
+                raise AcquisitionSecurityError(
+                    "RANGE_READER_LOCK", f"{subject} revision has no HTTP validator"
+                )
+            for value in pair:
+                if value is not None and (
+                    not isinstance(value, str) or not 1 <= len(value) <= 1_024
+                ):
+                    raise AcquisitionSecurityError(
+                        "RANGE_READER_LOCK", f"{subject} validator is invalid"
+                    )
+        if not isinstance(self.slices, tuple) or len(self.slices) != 1:
+            raise AcquisitionSecurityError(
+                "RANGE_READER_LOCK",
+                "this contract version requires exactly one selected message",
+            )
+        if self.slices[0].last_byte >= self.full_object_size_bytes:
+            raise AcquisitionSecurityError(
+                "RANGE_READER_LOCK",
+                "selected message runs past the locked object",
+            )
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(self.to_dict())
+
+    @property
+    def source_revision_digest(self) -> str:
+        return canonical_sha256(
+            {
+                "cycle": self.cycle,
+                "sidecar_final_url": self.sidecar_final_url,
+                "sidecar_sha256": self.sidecar_sha256,
+                "sidecar_size_bytes": self.sidecar_size_bytes,
+                "sidecar_etag": self.sidecar_etag,
+                "sidecar_last_modified": self.sidecar_last_modified,
+                "object_url": self.object_url,
+                "object_etag": self.object_etag,
+                "object_last_modified": self.object_last_modified,
+                "full_object_size_bytes": self.full_object_size_bytes,
+                "slices": [item.to_dict() for item in self.slices],
+            }
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "cycle": self.cycle,
+            "sidecar_url": self.sidecar_url,
+            "sidecar_final_url": self.sidecar_final_url,
+            "object_url": self.object_url,
+            "sidecar_sha256": self.sidecar_sha256,
+            "sidecar_size_bytes": self.sidecar_size_bytes,
+            "sidecar_etag": self.sidecar_etag,
+            "sidecar_last_modified": self.sidecar_last_modified,
+            "object_etag": self.object_etag,
+            "object_last_modified": self.object_last_modified,
+            "full_object_size_bytes": self.full_object_size_bytes,
+            "slices": [item.to_dict() for item in self.slices],
+        }
+
+    @classmethod
+    def from_dict(cls, value: Any) -> RangeRevisionLock:
+        if not isinstance(value, dict) or set(value) != {
+            "cycle",
+            "sidecar_url",
+            "sidecar_final_url",
+            "object_url",
+            "sidecar_sha256",
+            "sidecar_size_bytes",
+            "sidecar_etag",
+            "sidecar_last_modified",
+            "object_etag",
+            "object_last_modified",
+            "full_object_size_bytes",
+            "slices",
+        }:
+            raise AcquisitionSecurityError("RANGE_READER_LOCK", "revision lock shape is invalid")
+        raw_slices = value["slices"]
+        if not isinstance(raw_slices, list):
+            raise AcquisitionSecurityError("RANGE_READER_LOCK", "lock slices are not an array")
+        slices: list[LockedSlice] = []
+        for raw in raw_slices:
+            if not isinstance(raw, dict) or set(raw) != {
+                "descriptor",
+                "index",
+                "first_byte",
+                "last_byte",
+                "size_bytes",
+                "sha256",
+            }:
+                raise AcquisitionSecurityError("RANGE_READER_LOCK", "locked slice shape is invalid")
+            stated_size = raw["size_bytes"]
+            item = LockedSlice(
+                descriptor=raw["descriptor"],
+                index=raw["index"],
+                first_byte=raw["first_byte"],
+                last_byte=raw["last_byte"],
+                sha256=raw["sha256"],
+            )
+            if stated_size != item.size_bytes:
+                raise AcquisitionSecurityError("RANGE_READER_LOCK", "locked slice size is invalid")
+            slices.append(item)
+        kwargs = dict(value)
+        kwargs.pop("slices")
+        return cls(**kwargs, slices=tuple(slices))
+
+
+@dataclass(frozen=True)
+class RangeReaderOutput:
+    """Canonical Reader bytes plus the exact recipe-consumable evidence document."""
+
+    content: bytes
+    media_type: str
+    parsed_schema_digest: str
+    decode_flags: tuple[str, ...]
+    fetched_members: tuple[FetchedMember, ...]
+    transport_evidence_digest: str
+    efficiency_numerator: int
+
+
+def acquire_range_reader(
+    sandbox: _RangeReaderSandbox,
+    *,
+    request_id: str,
+    lock: RangeRevisionLock,
+    allowed_hostnames: tuple[str, ...],
+    retrieval_limits: RetrievalLimits,
+    reader_pin: ReaderPin,
+    reader_budgets: ReaderBudgets,
+    parse_limits: ParseLimits,
+    expected_fetch_policy_digest: str,
+    expected_decode_policy_digest: str,
+) -> RangeReaderOutput:
+    """Fetch, revision-check, decode and bind one selected message without a glue-script gap."""
+
+    _digest(expected_fetch_policy_digest, "fetch policy digest")
+    _digest(expected_decode_policy_digest, "decode policy digest")
+    fetched = sandbox.fetch_ranges(
+        request_id=request_id,
+        sidecar_url=lock.sidecar_url,
+        object_url=lock.object_url,
+        selectors=tuple(item.descriptor for item in lock.slices),
+        allowed_hostnames=allowed_hostnames,
+        retrieval_limits=retrieval_limits,
+        expected_reference_time=lock.cycle,
+    )
+    if fetched.policy_digest != expected_fetch_policy_digest:
+        raise AcquisitionSecurityError(
+            "RANGE_READER_POLICY",
+            "range result does not bind the coordinator-pinned fetch policy",
+        )
+    if (
+        fetched.ranges is None
+        or len(fetched.ranges) != len(lock.slices)
+        or fetched.sidecar_content_sha256 != lock.sidecar_sha256
+        or fetched.sidecar_size_bytes != lock.sidecar_size_bytes
+        or fetched.full_object_size_bytes != lock.full_object_size_bytes
+        or fetched.fetched_members is None
+        or fetched.transport_evidence_digest is None
+        or fetched.final_url is None
+    ):
+        raise AcquisitionSecurityError(
+            "RANGE_READER_REVISION",
+            "range result does not match the locked sidecar and object revision",
+        )
+    for delivered, expected in zip(fetched.ranges, lock.slices, strict=True):
+        if (
+            delivered.descriptor != expected.descriptor
+            or delivered.index != expected.index
+            or delivered.first_byte != expected.first_byte
+            or delivered.last_byte != expected.last_byte
+            or delivered.sha256 != expected.sha256
+            or len(delivered.content) != expected.size_bytes
+        ):
+            raise AcquisitionSecurityError(
+                "RANGE_READER_REVISION",
+                "delivered message differs from the locked selection",
+            )
+
+    members = fetched.fetched_members
+    roles = tuple(member.role for member in members)
+    terminal = {
+        role: members[max(index for index, value in enumerate(roles) if value == role)]
+        for role in ("cycle_probe", "sidecar_initial", "selected_slice", "sidecar_final")
+    }
+    sidecar_initial = terminal["sidecar_initial"]
+    sidecar_final = terminal["sidecar_final"]
+    object_probe = terminal["cycle_probe"]
+    object_slice = terminal["selected_slice"]
+    if (
+        fetched.final_url != lock.sidecar_final_url
+        or sidecar_initial.url != lock.sidecar_final_url
+        or sidecar_final.url != lock.sidecar_final_url
+        or sidecar_initial.response_body_sha256 != lock.sidecar_sha256
+        or sidecar_final.response_body_sha256 != lock.sidecar_sha256
+        or sidecar_initial.response_body_size_bytes != lock.sidecar_size_bytes
+        or sidecar_final.response_body_size_bytes != lock.sidecar_size_bytes
+        or (sidecar_initial.etag, sidecar_initial.last_modified)
+        != (lock.sidecar_etag, lock.sidecar_last_modified)
+        or (sidecar_final.etag, sidecar_final.last_modified)
+        != (lock.sidecar_etag, lock.sidecar_last_modified)
+        or object_probe.url != lock.object_url
+        or object_slice.url != lock.object_url
+        or (object_probe.etag, object_probe.last_modified)
+        != (lock.object_etag, lock.object_last_modified)
+        or (object_slice.etag, object_slice.last_modified)
+        != (lock.object_etag, lock.object_last_modified)
+    ):
+        raise AcquisitionSecurityError(
+            "RANGE_READER_REVISION",
+            "response validators or final member identities differ from the locked revision",
+        )
+
+    selected = fetched.ranges[0]
+    decoded = sandbox.decode_and_parse(
+        request_id=f"{request_id}.decode",
+        content=selected.content,
+        reader_pin={
+            "family_id": reader_pin.family_id,
+            "family_version": reader_pin.family_version,
+            "decode_options": dict(reader_pin.decode_options),
+        },
+        output_format="csv",
+        limits=parse_limits,
+        reader_budgets={
+            name: getattr(reader_budgets, name) for name in reader_budgets.__dataclass_fields__
+        },
+    )
+    if decoded.policy_digest != expected_decode_policy_digest:
+        raise AcquisitionSecurityError(
+            "RANGE_READER_POLICY",
+            "decode result does not bind the coordinator-pinned decode policy",
+        )
+    if (
+        decoded.content is None
+        or decoded.media_type is None
+        or decoded.parsed is None
+        or decoded.parsed.input_sha256 != sha256_bytes(decoded.content)
+        or decoded.decode_family_id != reader_pin.family_id
+        or decoded.decode_family_version != reader_pin.family_version
+        or decoded.decode_options_digest != reader_pin.options_digest
+        or decoded.decode_flags is None
+    ):
+        raise AcquisitionSecurityError(
+            "RANGE_READER_DECODE",
+            "decode result is incomplete or does not match the pinned Reader",
+        )
+
+    # Receipt v2 historically measures retained response bodies only. The HTTP and sandbox
+    # runtime enforce the stronger cumulative budget (including failed-peer partial bodies), but
+    # changing this sealed formula would invalidate already published v2 receipts.
+    numerator = sum(member.response_body_size_bytes for member in members)
+    if numerator * 100 > lock.full_object_size_bytes:
+        raise AcquisitionSecurityError(
+            "RANGE_READER_EFFICIENCY",
+            "source response bodies exceed one percent of the locked object",
+        )
+    return RangeReaderOutput(
+        content=decoded.content,
+        media_type=decoded.media_type,
+        parsed_schema_digest=decoded.parsed.schema_digest,
+        decode_flags=decoded.decode_flags,
+        fetched_members=members,
+        transport_evidence_digest=fetched.transport_evidence_digest,
+        efficiency_numerator=numerator,
+    )
+
+
+def default_policy_digests() -> tuple[str, str]:
+    """Expose the exact two sandbox authorities an operator must pin for this composition."""
+
+    from mostlyright.data_harness.acquisition.sandbox import sandbox_policy_digest
+
+    return sandbox_policy_digest("fetch_ranges"), sandbox_policy_digest("decode_and_parse")
+
+
+def range_reader_policy_digest() -> str:
+    """Coordinator pin for the exact two sandbox policies used by the composed adapter."""
+
+    fetch, decode = default_policy_digests()
+    return canonical_sha256({"fetch_ranges": fetch, "decode_and_parse": decode})
+
+
+def require_crawler_sandbox(value: Any) -> CrawlerSandbox:
+    """A narrow runtime guard for callers constructing the production path."""
+
+    if not isinstance(value, CrawlerSandbox):
+        raise AcquisitionSecurityError(
+            "RANGE_READER_SANDBOX",
+            "range Reader acquisition needs the shipped clean-room runner",
+        )
+    return value