mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/hosted_handoff.py

@@ -0,0 +1,62 @@
+"""The record one hosted Table handoff leaves behind, and the one way to read it.
+
+The record answers a question a folder full of verified bytes cannot: which released version, which
+hosted run, and which fingerprints these local bytes came from. Nothing is admitted anywhere on the
+strength of it -- the viewer verifies the Build and re-derives the notebook for itself, exactly as
+it does for a Build made here -- so this carries no authority and is never consulted by a decision.
+It exists so that what is already admitted can be attributed.
+
+It lives in its own module because two very different callers need it: the lane that writes it,
+which reaches Studio, and the viewer, which reaches nothing. A viewer that had to import the
+downloading lane to read one small local file would be importing a Studio client to render a
+notebook.
+"""
+
+from __future__ import annotations
+
+import os
+import stat
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness import canonical
+
+HANDOFF_RECEIPT_SCHEMA = "mostlyright-hosted-table-handoff-receipt.v2"
+HANDOFF_RELATIVE_PATH = Path("evidence") / "hosted-table" / "handoff.json"
+MAX_HANDOFF_RECEIPT_BYTES = 64 * 1024
+
+
+def read_handoff_receipt(
+    run_dir: Path, *, max_bytes: int = MAX_HANDOFF_RECEIPT_BYTES
+) -> dict[str, Any] | None:
+    """Return the handoff record under ``run_dir``, or nothing, without following a link."""
+
+    path = Path(run_dir) / HANDOFF_RELATIVE_PATH
+    try:
+        descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+    except OSError:
+        return None
+    try:
+        info = os.fstat(descriptor)
+        if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size > max_bytes:
+            return None
+        raw = os.read(descriptor, max_bytes)
+    except OSError:
+        return None
+    finally:
+        os.close(descriptor)
+    try:
+        value = canonical.parse_canonical_json(raw)
+    except canonical.CanonicalJSONError:
+        return None
+    if not isinstance(value, dict) or value.get("schema_version") != HANDOFF_RECEIPT_SCHEMA:
+        return None
+    return value
+
+
+__all__ = [
+    "HANDOFF_RECEIPT_SCHEMA",
+    "HANDOFF_RELATIVE_PATH",
+    "MAX_HANDOFF_RECEIPT_BYTES",
+    "read_handoff_receipt",
+]

mostlyright/data_harness/hosted_ingestion_contract.py

@@ -0,0 +1,504 @@
+"""Frozen, capability-free wire contracts for one hosted ingestion delivery.
+
+This module is intentionally independent from the generated Studio client.  It is the narrow
+contract a Cloud Run Job receives *after* redeeming its opaque dispatch id.  The document carries
+only identities, immutable source pins, GCS object coordinates and a deadline; it never carries a
+Studio bearer credential, signed URL, bucket authority, or source credential.  Studio owns the
+matching schema and translates its typed ``sha256:...`` values at its one adapter boundary; the
+Harness contracts deliberately use the existing bare 64-hex ingestion digests.
+
+The persisted delivery envelope binds a mutable, delivery-scoped manifest to the exact execution
+and attempt which may advance it.  Chunks deliberately remain content-addressed under a shared,
+reviewed prefix, but two deliveries can never CAS the same manifest object.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any, Self
+from uuid import UUID
+
+from mostlyright.data_harness.canonical import canonical_sha256
+from mostlyright.data_harness.ingestion.contracts import (
+    IngestionCorrupt,
+    IngestionError,
+    IngestionManifestV1,
+    SourceVersionPinV1,
+)
+
+INGESTION_MANIFEST_REF_SCHEMA = "mostlyright-ingestion-manifest-ref.v1"
+INGESTION_DELIVERY_BINDING_SCHEMA = "mostlyright-ingestion-delivery-binding.v1"
+INGESTION_DELIVERY_MANIFEST_SCHEMA = "mostlyright-ingestion-delivery-manifest.v1"
+HOSTED_INGESTION_DISPATCH_SCHEMA = "mostlyright-ingestion-dispatch.v1"
+PUBLIC_OFFSET_CONTINUATION_SCHEMA = "mostlyright-ingestion-public-offset.v1"
+
+MAX_MANIFEST_REF_BYTES = 16 * 1024 * 1024
+MAX_PREFIX_LENGTH = 512
+MAX_OBJECT_NAME_LENGTH = 1024
+MAX_SEGMENT_ID_LENGTH = 128
+MAX_DELIVERY_DEADLINE_SECONDS = 15 * 60
+
+_DIGEST = re.compile(r"[0-9a-f]{64}\Z")
+_GENERATION = re.compile(r"[1-9][0-9]*\Z")
+_PREFIX_SEGMENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z")
+_SEGMENT_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z")
+_RFC3339 = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?Z\Z")
+
+
+def _error(code: str, path: str, detail: str) -> IngestionError:
+    return IngestionError(code, path, detail)
+
+
+def _require_exact(value: object, keys: frozenset[str], path: str) -> dict[str, Any]:
+    if not isinstance(value, dict) or set(value) != keys:
+        raise _error("INGESTION_DELIVERY_DOCUMENT", path, "has unsupported or missing members")
+    return value
+
+
+def _require_digest(value: object, path: str) -> str:
+    if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
+        raise _error("INGESTION_DIGEST", path, "must be one lowercase SHA-256 digest")
+    return value
+
+
+def _require_uuid4(value: object, path: str) -> str:
+    if not isinstance(value, str):
+        raise _error("INGESTION_DELIVERY_ID", path, "must be one canonical UUID4")
+    try:
+        parsed = UUID(value)
+    except ValueError as error:
+        raise _error("INGESTION_DELIVERY_ID", path, "must be one canonical UUID4") from error
+    if parsed.version != 4 or str(parsed) != value:
+        raise _error("INGESTION_DELIVERY_ID", path, "must be one canonical UUID4")
+    return value
+
+
+def _require_positive_int(value: object, path: str, *, maximum: int = (1 << 63) - 1) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or not 0 < value <= maximum:
+        raise _error("INGESTION_DELIVERY_NUMBER", path, "must be one bounded positive integer")
+    return value
+
+
+def _require_nonnegative_int(value: object, path: str, *, maximum: int = (1 << 63) - 1) -> int:
+    if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= maximum:
+        raise _error("INGESTION_DELIVERY_NUMBER", path, "must be one bounded non-negative integer")
+    return value
+
+
+def _require_prefix(value: object, path: str) -> str:
+    if not isinstance(value, str) or not value or len(value) > MAX_PREFIX_LENGTH:
+        raise _error("INGESTION_DELIVERY_PREFIX", path, "must be one bounded object prefix")
+    if any(_PREFIX_SEGMENT.fullmatch(part) is None for part in value.split("/")):
+        raise _error("INGESTION_DELIVERY_PREFIX", path, "contains an unsafe object-prefix segment")
+    return value
+
+
+def _require_object_name(value: object, path: str) -> str:
+    """Validate a GCS object name without imposing the shorter prefix limit on its suffix."""
+
+    if not isinstance(value, str) or not value or len(value) > MAX_OBJECT_NAME_LENGTH:
+        raise _error("INGESTION_DELIVERY_OBJECT", path, "must be one bounded object name")
+    if any(_PREFIX_SEGMENT.fullmatch(part) is None for part in value.split("/")):
+        raise _error("INGESTION_DELIVERY_OBJECT", path, "contains an unsafe object-name segment")
+    return value
+
+
+def _require_delivery_prefix(value: object, path: str, delivery_id: str) -> str:
+    """A mutable manifest prefix must name precisely the delivery that may advance it."""
+
+    prefix = _require_prefix(value, path)
+    if prefix.rsplit("/", 1)[-1] != delivery_id:
+        raise _error(
+            "INGESTION_DELIVERY_PREFIX",
+            path,
+            "must end in this delivery's canonical identifier",
+        )
+    return prefix
+
+
+def _require_run_id(value: object, path: str) -> str:
+    if not isinstance(value, str) or _SEGMENT_ID.fullmatch(value) is None:
+        raise _error("INGESTION_RUN_ID", path, "must be one safe bounded run id")
+    return value
+
+
+def _require_segment_id(value: object, path: str) -> str:
+    if not isinstance(value, str) or _SEGMENT_ID.fullmatch(value) is None:
+        raise _error("INGESTION_SEGMENT_ID", path, "must be one safe bounded segment id")
+    return value
+
+
+def _require_deadline(value: object, path: str) -> str:
+    if not isinstance(value, str) or _RFC3339.fullmatch(value) is None:
+        raise _error("INGESTION_DELIVERY_DEADLINE", path, "must be one UTC RFC3339 timestamp")
+    try:
+        datetime.fromisoformat(value.removesuffix("Z") + "+00:00")
+    except ValueError as error:
+        raise _error("INGESTION_DELIVERY_DEADLINE", path, "is not a real UTC timestamp") from error
+    return value
+
+
+def parse_delivery_deadline(value: str) -> datetime:
+    """Return a UTC deadline after the frozen wire syntax has been validated."""
+
+    _require_deadline(value, "delivery.deadline")
+    return datetime.fromisoformat(value.removesuffix("Z") + "+00:00").astimezone(UTC)
+
+
+@dataclass(frozen=True)
+class IngestionManifestReferenceV1:
+    """An exact GCS generation and raw-document digest, never a bucket authority."""
+
+    object_name: str
+    generation: str
+    manifest_sha256: str
+    byte_length: int
+    schema_version: str = INGESTION_MANIFEST_REF_SCHEMA
+
+    def __post_init__(self) -> None:
+        if self.schema_version != INGESTION_MANIFEST_REF_SCHEMA:
+            raise _error(
+                "INGESTION_MANIFEST_REF_SCHEMA", "manifest_ref.schema_version", "is unsupported"
+            )
+        _require_object_name(self.object_name, "manifest_ref.object_name")
+        if _GENERATION.fullmatch(self.generation) is None:
+            raise _error(
+                "INGESTION_MANIFEST_REF_GENERATION", "manifest_ref.generation", "must be positive"
+            )
+        _require_digest(self.manifest_sha256, "manifest_ref.manifest_sha256")
+        if (
+            isinstance(self.byte_length, bool)
+            or not isinstance(self.byte_length, int)
+            or not 0 < self.byte_length <= MAX_MANIFEST_REF_BYTES
+        ):
+            raise _error(
+                "INGESTION_MANIFEST_REF_LENGTH",
+                "manifest_ref.byte_length",
+                "is outside the supported range",
+            )
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {
+            "schema_version": self.schema_version,
+            "object_name": self.object_name,
+            "generation": self.generation,
+            "manifest_sha256": self.manifest_sha256,
+            "byte_length": self.byte_length,
+        }
+
+    @classmethod
+    def from_dict(cls, value: object) -> Self:
+        document = _require_exact(
+            value,
+            frozenset(
+                {"schema_version", "object_name", "generation", "manifest_sha256", "byte_length"}
+            ),
+            "manifest_ref",
+        )
+        return cls(
+            schema_version=document["schema_version"],
+            object_name=document["object_name"],
+            generation=document["generation"],
+            manifest_sha256=document["manifest_sha256"],
+            byte_length=document["byte_length"],
+        )
+
+    def require_for_prefix(self, *, prefix: str, run_id: str) -> None:
+        """Reject a ref outside the exact reviewed delivery namespace and run coordinate."""
+
+        checked_prefix = _require_prefix(prefix, "manifest_prefix")
+        checked_run_id = _require_run_id(run_id, "run_id")
+        expected = f"{checked_prefix}/runs/{checked_run_id}.json"
+        if self.object_name != expected:
+            raise _error(
+                "INGESTION_MANIFEST_REF_PATH",
+                "manifest_ref.object_name",
+                "does not name this delivery's manifest object",
+            )
+
+
+@dataclass(frozen=True)
+class PublicOffsetContinuationV1:
+    """The only first-slice continuation: a deterministic offset in immutable public bytes."""
+
+    offset: int
+    schema_version: str = PUBLIC_OFFSET_CONTINUATION_SCHEMA
+
+    def __post_init__(self) -> None:
+        if self.schema_version != PUBLIC_OFFSET_CONTINUATION_SCHEMA:
+            raise _error(
+                "INGESTION_CONTINUATION_SCHEMA", "continuation.schema_version", "is unsupported"
+            )
+        _require_nonnegative_int(self.offset, "continuation.offset")
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {"schema_version": self.schema_version, "offset": self.offset}
+
+    @classmethod
+    def from_dict(cls, value: object) -> Self:
+        document = _require_exact(value, frozenset({"schema_version", "offset"}), "continuation")
+        return cls(schema_version=document["schema_version"], offset=document["offset"])
+
+
+@dataclass(frozen=True)
+class IngestionDeliveryBindingV1:
+    """Identity/fence context sealed around every delivery-scoped manifest document."""
+
+    delivery_id: str
+    execution_id: str
+    run_id: str
+    segment_id: str
+    source_pin_sha256: str
+    attempt_id: str
+    attempt_generation: int
+    attempt_fence: int
+    checkpoint_epoch: int
+    checkpoint_sequence: int
+    parent_checkpoint_digest: str | None
+    deadline: str
+    schema_version: str = INGESTION_DELIVERY_BINDING_SCHEMA
+
+    def __post_init__(self) -> None:
+        if self.schema_version != INGESTION_DELIVERY_BINDING_SCHEMA:
+            raise _error("INGESTION_DELIVERY_SCHEMA", "delivery.schema_version", "is unsupported")
+        _require_uuid4(self.delivery_id, "delivery.delivery_id")
+        _require_uuid4(self.execution_id, "delivery.execution_id")
+        _require_run_id(self.run_id, "delivery.run_id")
+        _require_segment_id(self.segment_id, "delivery.segment_id")
+        _require_digest(self.source_pin_sha256, "delivery.source_pin_sha256")
+        _require_uuid4(self.attempt_id, "delivery.attempt_id")
+        _require_positive_int(self.attempt_generation, "delivery.attempt_generation")
+        _require_positive_int(self.attempt_fence, "delivery.attempt_fence")
+        _require_nonnegative_int(self.checkpoint_epoch, "delivery.checkpoint_epoch")
+        _require_nonnegative_int(self.checkpoint_sequence, "delivery.checkpoint_sequence")
+        if self.parent_checkpoint_digest is not None:
+            _require_digest(self.parent_checkpoint_digest, "delivery.parent_checkpoint_digest")
+        _require_deadline(self.deadline, "delivery.deadline")
+
+    @property
+    def deadline_at(self) -> datetime:
+        return parse_delivery_deadline(self.deadline)
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(self.to_dict())
+
+    def to_dict(self) -> dict[str, str | int | None]:
+        return {
+            "schema_version": self.schema_version,
+            "delivery_id": self.delivery_id,
+            "execution_id": self.execution_id,
+            "run_id": self.run_id,
+            "segment_id": self.segment_id,
+            "source_pin_sha256": self.source_pin_sha256,
+            "attempt_id": self.attempt_id,
+            "attempt_generation": self.attempt_generation,
+            "attempt_fence": self.attempt_fence,
+            "checkpoint_epoch": self.checkpoint_epoch,
+            "checkpoint_sequence": self.checkpoint_sequence,
+            "parent_checkpoint_digest": self.parent_checkpoint_digest,
+            "deadline": self.deadline,
+        }
+
+    @classmethod
+    def from_dict(cls, value: object) -> Self:
+        document = _require_exact(
+            value,
+            frozenset(
+                {
+                    "schema_version",
+                    "delivery_id",
+                    "execution_id",
+                    "run_id",
+                    "segment_id",
+                    "source_pin_sha256",
+                    "attempt_id",
+                    "attempt_generation",
+                    "attempt_fence",
+                    "checkpoint_epoch",
+                    "checkpoint_sequence",
+                    "parent_checkpoint_digest",
+                    "deadline",
+                }
+            ),
+            "delivery",
+        )
+        return cls(**document)
+
+    def require_manifest(self, manifest: IngestionManifestV1) -> None:
+        if manifest.run_id != self.run_id:
+            raise IngestionCorrupt(
+                "INGESTION_DELIVERY_RUN", "ingestion_manifest.run_id", "does not match delivery"
+            )
+        if manifest.source_pin.digest != self.source_pin_sha256:
+            raise IngestionCorrupt(
+                "INGESTION_DELIVERY_SOURCE",
+                "ingestion_manifest.source_pin",
+                "does not match delivery",
+            )
+
+
+@dataclass(frozen=True)
+class IngestionDeliveryManifestV1:
+    """The only persisted mutable document in a delivery-specific manifest namespace."""
+
+    delivery: IngestionDeliveryBindingV1
+    manifest: IngestionManifestV1
+    schema_version: str = INGESTION_DELIVERY_MANIFEST_SCHEMA
+
+    def __post_init__(self) -> None:
+        if self.schema_version != INGESTION_DELIVERY_MANIFEST_SCHEMA:
+            raise _error(
+                "INGESTION_DELIVERY_MANIFEST_SCHEMA",
+                "delivery_manifest.schema_version",
+                "is unsupported",
+            )
+        if not isinstance(self.delivery, IngestionDeliveryBindingV1):
+            raise _error("INGESTION_DELIVERY_BINDING", "delivery_manifest.delivery", "is invalid")
+        if not isinstance(self.manifest, IngestionManifestV1):
+            raise _error("INGESTION_MANIFEST", "delivery_manifest.manifest", "is invalid")
+        self.delivery.require_manifest(self.manifest)
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "schema_version": self.schema_version,
+            "delivery": self.delivery.to_dict(),
+            "manifest": self.manifest.to_dict(),
+        }
+
+    @classmethod
+    def from_dict(cls, value: object) -> Self:
+        document = _require_exact(
+            value, frozenset({"schema_version", "delivery", "manifest"}), "delivery_manifest"
+        )
+        return cls(
+            schema_version=document["schema_version"],
+            delivery=IngestionDeliveryBindingV1.from_dict(document["delivery"]),
+            manifest=IngestionManifestV1.from_dict(document["manifest"]),
+        )
+
+
+@dataclass(frozen=True)
+class PriorManifestV1:
+    """The predecessor's namespace plus an exact document reference, or nothing at all."""
+
+    prefix: str
+    manifest_ref: IngestionManifestReferenceV1
+
+    def __post_init__(self) -> None:
+        _require_prefix(self.prefix, "prior_manifest.prefix")
+        if not isinstance(self.manifest_ref, IngestionManifestReferenceV1):
+            raise _error("INGESTION_MANIFEST_REF", "prior_manifest.manifest_ref", "is invalid")
+
+    def to_dict(self) -> dict[str, object]:
+        return {"prefix": self.prefix, "manifest_ref": self.manifest_ref.to_dict()}
+
+    @classmethod
+    def from_dict(cls, value: object) -> Self:
+        document = _require_exact(value, frozenset({"prefix", "manifest_ref"}), "prior_manifest")
+        return cls(
+            prefix=document["prefix"],
+            manifest_ref=IngestionManifestReferenceV1.from_dict(document["manifest_ref"]),
+        )
+
+
+@dataclass(frozen=True)
+class HostedIngestionDispatchV1:
+    """One authenticated dispatch exchange result with an entirely closed schema."""
+
+    delivery: IngestionDeliveryBindingV1
+    source_pin: SourceVersionPinV1
+    continuation: PublicOffsetContinuationV1
+    manifest_prefix: str
+    chunk_prefix: str
+    prior_manifest: PriorManifestV1 | None
+    schema_version: str = HOSTED_INGESTION_DISPATCH_SCHEMA
+
+    def __post_init__(self) -> None:
+        if self.schema_version != HOSTED_INGESTION_DISPATCH_SCHEMA:
+            raise _error("INGESTION_DISPATCH_SCHEMA", "dispatch.schema_version", "is unsupported")
+        if not isinstance(self.delivery, IngestionDeliveryBindingV1):
+            raise _error("INGESTION_DELIVERY_BINDING", "dispatch.delivery", "is invalid")
+        if not isinstance(self.source_pin, SourceVersionPinV1):
+            raise _error("INGESTION_SOURCE_PIN", "dispatch.source_pin", "is invalid")
+        if self.source_pin.digest != self.delivery.source_pin_sha256:
+            raise _error(
+                "INGESTION_DELIVERY_SOURCE", "dispatch.source_pin", "does not match delivery"
+            )
+        if not isinstance(self.continuation, PublicOffsetContinuationV1):
+            raise _error("INGESTION_CONTINUATION", "dispatch.continuation", "is invalid")
+        _require_prefix(self.manifest_prefix, "dispatch.manifest_prefix")
+        _require_delivery_prefix(
+            self.manifest_prefix, "dispatch.manifest_prefix", self.delivery.delivery_id
+        )
+        _require_prefix(self.chunk_prefix, "dispatch.chunk_prefix")
+        if self.prior_manifest is not None and not isinstance(self.prior_manifest, PriorManifestV1):
+            raise _error("INGESTION_PRIOR_MANIFEST", "dispatch.prior_manifest", "is invalid")
+        if self.prior_manifest is not None:
+            self.prior_manifest.manifest_ref.require_for_prefix(
+                prefix=self.prior_manifest.prefix, run_id=self.delivery.run_id
+            )
+
+    def to_dict(self) -> dict[str, object]:
+        return {
+            "schema_version": self.schema_version,
+            "delivery": self.delivery.to_dict(),
+            "source_pin": self.source_pin.to_dict(),
+            "continuation": self.continuation.to_dict(),
+            "manifest_prefix": self.manifest_prefix,
+            "chunk_prefix": self.chunk_prefix,
+            "prior_manifest": None
+            if self.prior_manifest is None
+            else self.prior_manifest.to_dict(),
+        }
+
+    @classmethod
+    def from_dict(cls, value: object) -> Self:
+        document = _require_exact(
+            value,
+            frozenset(
+                {
+                    "schema_version",
+                    "delivery",
+                    "source_pin",
+                    "continuation",
+                    "manifest_prefix",
+                    "chunk_prefix",
+                    "prior_manifest",
+                }
+            ),
+            "dispatch",
+        )
+        prior = document["prior_manifest"]
+        if prior is not None and not isinstance(prior, dict):
+            raise _error(
+                "INGESTION_PRIOR_MANIFEST", "dispatch.prior_manifest", "must be null or object"
+            )
+        return cls(
+            schema_version=document["schema_version"],
+            delivery=IngestionDeliveryBindingV1.from_dict(document["delivery"]),
+            source_pin=SourceVersionPinV1.from_dict(document["source_pin"]),
+            continuation=PublicOffsetContinuationV1.from_dict(document["continuation"]),
+            manifest_prefix=document["manifest_prefix"],
+            chunk_prefix=document["chunk_prefix"],
+            prior_manifest=None if prior is None else PriorManifestV1.from_dict(prior),
+        )
+
+
+__all__ = [
+    "HOSTED_INGESTION_DISPATCH_SCHEMA",
+    "INGESTION_DELIVERY_BINDING_SCHEMA",
+    "INGESTION_DELIVERY_MANIFEST_SCHEMA",
+    "INGESTION_MANIFEST_REF_SCHEMA",
+    "MAX_DELIVERY_DEADLINE_SECONDS",
+    "PUBLIC_OFFSET_CONTINUATION_SCHEMA",
+    "HostedIngestionDispatchV1",
+    "IngestionDeliveryBindingV1",
+    "IngestionDeliveryManifestV1",
+    "IngestionManifestReferenceV1",
+    "PriorManifestV1",
+    "PublicOffsetContinuationV1",
+    "parse_delivery_deadline",
+]