mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/ingestion/__init__.py

@@ -0,0 +1,56 @@
+"""Core contracts and local durable state for resumable engine-wide ingestion."""
+
+from mostlyright.data_harness.ingestion.contracts import (
+    CHUNK_REF_SCHEMA,
+    INGESTION_MANIFEST_SCHEMA,
+    SOURCE_VERSION_PIN_SCHEMA,
+    ChunkRef,
+    IngestionConflict,
+    IngestionCorrupt,
+    IngestionError,
+    IngestionManifestV1,
+    SourceVersionPinV1,
+    merkle_root_sha256,
+)
+from mostlyright.data_harness.ingestion.faults import DeterministicFaultInjector, FaultInjected
+from mostlyright.data_harness.ingestion.gcs_store import (
+    GCSIngestionStore,
+    GCSObjectNotFound,
+    GCSObjectRead,
+    GCSObjectTransport,
+    GCSObjectVersion,
+    GCSPreconditionFailed,
+    GCSStorageError,
+    GoogleCloudStorageJSONTransport,
+    workload_identity_access_token,
+)
+from mostlyright.data_harness.ingestion.spool import SpooledChunk, spool_hash_stream
+from mostlyright.data_harness.ingestion.store import IngestionStore, LocalIngestionStore
+
+__all__ = [
+    "CHUNK_REF_SCHEMA",
+    "INGESTION_MANIFEST_SCHEMA",
+    "SOURCE_VERSION_PIN_SCHEMA",
+    "ChunkRef",
+    "DeterministicFaultInjector",
+    "FaultInjected",
+    "GCSIngestionStore",
+    "GCSObjectNotFound",
+    "GCSObjectRead",
+    "GCSObjectTransport",
+    "GCSObjectVersion",
+    "GCSPreconditionFailed",
+    "GCSStorageError",
+    "GoogleCloudStorageJSONTransport",
+    "IngestionConflict",
+    "IngestionCorrupt",
+    "IngestionError",
+    "IngestionManifestV1",
+    "IngestionStore",
+    "LocalIngestionStore",
+    "SourceVersionPinV1",
+    "SpooledChunk",
+    "merkle_root_sha256",
+    "spool_hash_stream",
+    "workload_identity_access_token",
+]

mostlyright/data_harness/ingestion/contracts.py

@@ -0,0 +1,461 @@
+"""Immutable contracts for the resumable ingestion engine.
+
+The engine deliberately commits one source version at a time.  A source is pinned before
+the first byte is accepted, chunks are admitted in one gap-free ordinal sequence, and a
+finalized manifest is an immutable, order-sensitive Merkle commitment.  This leaves a
+small, portable contract for remote stores while keeping the local store implementation
+free to decide how it makes the individual state transitions durable.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass, replace
+from typing import Any, Self
+
+from mostlyright.data_harness.canonical import canonical_json_bytes, canonical_sha256
+
+SOURCE_VERSION_PIN_SCHEMA = "harness-source-version-pin.v1"
+CHUNK_REF_SCHEMA = "harness-ingestion-chunk-ref.v1"
+INGESTION_MANIFEST_SCHEMA = "harness-ingestion-manifest.v1"
+
+MAX_SOURCE_ID_LENGTH = 128
+MAX_SOURCE_VERSION_LENGTH = 512
+MAX_RUN_ID_LENGTH = 128
+MAX_CHUNK_BYTES = 1 << 40
+MAX_RECORDS_PER_CHUNK = (1 << 63) - 1
+MAX_MANIFEST_CHUNKS = 10_000
+
+_SOURCE_ID = re.compile(r"[a-z][a-z0-9._-]{0,127}\Z")
+_RUN_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z")
+_DIGEST = re.compile(r"[0-9a-f]{64}\Z")
+# A source version crosses the durable checkpoint boundary. It is not an opaque connector header:
+# ETags, cursors, signed URLs and bearer values have all appeared in such headers in practice.
+# Admit only a public immutable content identity or a monotonically assigned public generation.
+# The connector-specific canonical evidence remains in ``fence_sha256``; no raw response value is
+# needed in a manifest.
+_PUBLIC_SOURCE_VERSION = re.compile(
+    r"(?:generation:(?:0|[1-9][0-9]{0,31})|(?:etag|version)-sha256:[0-9a-f]{64})\Z"
+)
+
+
+class IngestionError(ValueError):
+    """A stable refusal or corruption error from the ingestion core."""
+
+    def __init__(self, code: str, path: str, detail: str) -> None:
+        self.code = code
+        self.path = path
+        self.detail = detail
+        super().__init__(f"{path}: {detail} [{code}]")
+
+
+class IngestionConflict(IngestionError):
+    """A fenced transition did not apply to the state the caller observed."""
+
+
+class IngestionCorrupt(IngestionError):
+    """Durable bytes no longer satisfy the ingestion contract."""
+
+
+def _require_digest(value: object, path: str) -> str:
+    if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
+        raise IngestionError("INGESTION_DIGEST", path, "must be a lowercase SHA-256 digest")
+    return value
+
+
+def _require_text(value: object, path: str, *, maximum: int) -> str:
+    if (
+        not isinstance(value, str)
+        or not value
+        or len(value) > maximum
+        or any(ord(character) < 0x20 or 0xD800 <= ord(character) <= 0xDFFF for character in value)
+    ):
+        raise IngestionError("INGESTION_TEXT", path, "must be a bounded non-control string")
+    return value
+
+
+def _require_exact_mapping(value: object, keys: frozenset[str], path: str) -> Mapping[str, Any]:
+    if not isinstance(value, Mapping) or set(value) != keys:
+        raise IngestionError("INGESTION_DOCUMENT", path, "has unsupported or missing members")
+    return value
+
+
+@dataclass(frozen=True)
+class SourceVersionPinV1:
+    """The exact source revision and acquisition policy a run is allowed to consume.
+
+    ``source_version`` is one non-secret immutable identity: a public generation or a digest of
+    a public ETag/version. ``fence_sha256`` binds the connector-specific version evidence after
+    canonicalization. ``acquisition_policy_sha256`` binds the reviewed connector policy. Neither
+    permits a raw header/cursor or a later source version to enter durable resume state.
+    """
+
+    source_id: str
+    source_version: str
+    fence_sha256: str
+    acquisition_policy_sha256: str
+    schema_version: str = SOURCE_VERSION_PIN_SCHEMA
+
+    def __post_init__(self) -> None:
+        if self.schema_version != SOURCE_VERSION_PIN_SCHEMA:
+            raise IngestionError(
+                "INGESTION_SOURCE_SCHEMA",
+                "source_version_pin.schema_version",
+                "is unsupported",
+            )
+        if not isinstance(self.source_id, str) or _SOURCE_ID.fullmatch(self.source_id) is None:
+            raise IngestionError(
+                "INGESTION_SOURCE_ID",
+                "source_version_pin.source_id",
+                "is invalid",
+            )
+        if (
+            not isinstance(self.source_version, str)
+            or len(self.source_version) > MAX_SOURCE_VERSION_LENGTH
+            or _PUBLIC_SOURCE_VERSION.fullmatch(self.source_version) is None
+        ):
+            raise IngestionError(
+                "INGESTION_SOURCE_VERSION",
+                "source_version_pin.source_version",
+                "must be a public generation or a SHA-256-pinned public ETag/version",
+            )
+        _require_digest(self.fence_sha256, "source_version_pin.fence_sha256")
+        _require_digest(
+            self.acquisition_policy_sha256,
+            "source_version_pin.acquisition_policy_sha256",
+        )
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(self.to_dict())
+
+    def to_dict(self) -> dict[str, str]:
+        return {
+            "schema_version": self.schema_version,
+            "source_id": self.source_id,
+            "source_version": self.source_version,
+            "fence_sha256": self.fence_sha256,
+            "acquisition_policy_sha256": self.acquisition_policy_sha256,
+        }
+
+    @classmethod
+    def from_dict(cls, value: object) -> Self:
+        document = _require_exact_mapping(
+            value,
+            frozenset(
+                {
+                    "schema_version",
+                    "source_id",
+                    "source_version",
+                    "fence_sha256",
+                    "acquisition_policy_sha256",
+                }
+            ),
+            "source_version_pin",
+        )
+        return cls(
+            schema_version=document["schema_version"],
+            source_id=document["source_id"],
+            source_version=document["source_version"],
+            fence_sha256=document["fence_sha256"],
+            acquisition_policy_sha256=document["acquisition_policy_sha256"],
+        )
+
+
+@dataclass(frozen=True)
+class ChunkRef:
+    """One immutable byte object, assigned exactly one position in a pinned run."""
+
+    ordinal: int
+    content_sha256: str
+    byte_length: int
+    record_count: int
+    source_pin_sha256: str
+    schema_version: str = CHUNK_REF_SCHEMA
+
+    def __post_init__(self) -> None:
+        if self.schema_version != CHUNK_REF_SCHEMA:
+            raise IngestionError(
+                "INGESTION_CHUNK_SCHEMA", "chunk_ref.schema_version", "is unsupported"
+            )
+        if isinstance(self.ordinal, bool) or not isinstance(self.ordinal, int) or self.ordinal < 0:
+            raise IngestionError(
+                "INGESTION_ORDINAL", "chunk_ref.ordinal", "must be a non-negative integer"
+            )
+        _require_digest(self.content_sha256, "chunk_ref.content_sha256")
+        _require_digest(self.source_pin_sha256, "chunk_ref.source_pin_sha256")
+        if (
+            isinstance(self.byte_length, bool)
+            or not isinstance(self.byte_length, int)
+            or not 0 <= self.byte_length <= MAX_CHUNK_BYTES
+        ):
+            raise IngestionError(
+                "INGESTION_CHUNK_BYTES",
+                "chunk_ref.byte_length",
+                "is outside the supported range",
+            )
+        if (
+            isinstance(self.record_count, bool)
+            or not isinstance(self.record_count, int)
+            or not 0 <= self.record_count <= MAX_RECORDS_PER_CHUNK
+        ):
+            raise IngestionError(
+                "INGESTION_RECORD_COUNT",
+                "chunk_ref.record_count",
+                "is outside the supported range",
+            )
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(self.to_dict())
+
+    def to_dict(self) -> dict[str, str | int]:
+        return {
+            "schema_version": self.schema_version,
+            "ordinal": self.ordinal,
+            "content_sha256": self.content_sha256,
+            "byte_length": self.byte_length,
+            "record_count": self.record_count,
+            "source_pin_sha256": self.source_pin_sha256,
+        }
+
+    @classmethod
+    def from_dict(cls, value: object) -> Self:
+        document = _require_exact_mapping(
+            value,
+            frozenset(
+                {
+                    "schema_version",
+                    "ordinal",
+                    "content_sha256",
+                    "byte_length",
+                    "record_count",
+                    "source_pin_sha256",
+                }
+            ),
+            "chunk_ref",
+        )
+        return cls(
+            schema_version=document["schema_version"],
+            ordinal=document["ordinal"],
+            content_sha256=document["content_sha256"],
+            byte_length=document["byte_length"],
+            record_count=document["record_count"],
+            source_pin_sha256=document["source_pin_sha256"],
+        )
+
+
+def merkle_root_sha256(chunks: tuple[ChunkRef, ...]) -> str:
+    """Return an order-sensitive, domain-separated Merkle root for ``chunks``."""
+
+    if not chunks:
+        return hashlib.sha256(b"mostlyright-ingestion-merkle-v1\x00empty").hexdigest()
+    level = [
+        hashlib.sha256(
+            b"mostlyright-ingestion-merkle-v1\x00leaf\x00" + canonical_json_bytes(chunk.to_dict())
+        ).digest()
+        for chunk in chunks
+    ]
+    while len(level) > 1:
+        if len(level) % 2:
+            level.append(level[-1])
+        level = [
+            hashlib.sha256(b"mostlyright-ingestion-merkle-v1\x00node\x00" + left + right).digest()
+            for left, right in zip(level[::2], level[1::2], strict=True)
+        ]
+    return level[0].hex()
+
+
+@dataclass(frozen=True)
+class IngestionManifestV1:
+    """An append-only source-pinned state machine.
+
+    Each persisted state points at its immediate predecessor digest.  That turns an
+    ambiguous successful replacement into a safe idempotent retry without allowing a
+    caller holding an arbitrary stale fence to overwrite unrelated work.
+    """
+
+    run_id: str
+    source_pin: SourceVersionPinV1
+    chunks: tuple[ChunkRef, ...]
+    state: str
+    previous_manifest_sha256: str | None
+    merkle_root_sha256: str | None
+    schema_version: str = INGESTION_MANIFEST_SCHEMA
+
+    def __post_init__(self) -> None:
+        if self.schema_version != INGESTION_MANIFEST_SCHEMA:
+            raise IngestionError(
+                "INGESTION_MANIFEST_SCHEMA", "ingestion_manifest.schema_version", "is unsupported"
+            )
+        if not isinstance(self.run_id, str) or _RUN_ID.fullmatch(self.run_id) is None:
+            raise IngestionError("INGESTION_RUN_ID", "ingestion_manifest.run_id", "is invalid")
+        if not isinstance(self.source_pin, SourceVersionPinV1):
+            raise IngestionError(
+                "INGESTION_SOURCE_PIN",
+                "ingestion_manifest.source_pin",
+                "must be SourceVersionPinV1",
+            )
+        if not isinstance(self.chunks, tuple) or len(self.chunks) > MAX_MANIFEST_CHUNKS:
+            raise IngestionError(
+                "INGESTION_CHUNK_LIMIT",
+                "ingestion_manifest.chunks",
+                "must be a bounded tuple",
+            )
+        expected_pin = self.source_pin.digest
+        for ordinal, chunk in enumerate(self.chunks):
+            if not isinstance(chunk, ChunkRef):
+                raise IngestionError(
+                    "INGESTION_CHUNK", "ingestion_manifest.chunks", "must contain ChunkRef values"
+                )
+            if chunk.ordinal != ordinal:
+                raise IngestionError(
+                    "INGESTION_GAP",
+                    "ingestion_manifest.chunks",
+                    "ordinals must be ordered and gap-free",
+                )
+            if chunk.source_pin_sha256 != expected_pin:
+                raise IngestionError(
+                    "INGESTION_SOURCE_DRIFT",
+                    "ingestion_manifest.chunks",
+                    "chunk belongs to a different source pin",
+                )
+        if self.previous_manifest_sha256 is not None:
+            _require_digest(
+                self.previous_manifest_sha256, "ingestion_manifest.previous_manifest_sha256"
+            )
+        if not self.chunks and self.state == "open" and self.previous_manifest_sha256 is not None:
+            raise IngestionError(
+                "INGESTION_INITIAL_PREDECESSOR",
+                "ingestion_manifest.previous_manifest_sha256",
+                "initial open state must not have a predecessor",
+            )
+        if self.chunks or self.state == "finalized":
+            if self.previous_manifest_sha256 is None:
+                raise IngestionError(
+                    "INGESTION_PREDECESSOR",
+                    "ingestion_manifest.previous_manifest_sha256",
+                    "non-initial state must name its predecessor",
+                )
+        if self.state == "open":
+            if self.merkle_root_sha256 is not None:
+                raise IngestionError(
+                    "INGESTION_OPEN_ROOT",
+                    "ingestion_manifest.merkle_root_sha256",
+                    "open state must not have a Merkle root",
+                )
+        elif self.state == "finalized":
+            if self.merkle_root_sha256 != merkle_root_sha256(self.chunks):
+                raise IngestionError(
+                    "INGESTION_MERKLE",
+                    "ingestion_manifest.merkle_root_sha256",
+                    "does not commit exactly the ordered chunk sequence",
+                )
+        else:
+            raise IngestionError("INGESTION_STATE", "ingestion_manifest.state", "is unsupported")
+
+    @classmethod
+    def initial(cls, *, run_id: str, source_pin: SourceVersionPinV1) -> Self:
+        return cls(
+            run_id=run_id,
+            source_pin=source_pin,
+            chunks=(),
+            state="open",
+            previous_manifest_sha256=None,
+            merkle_root_sha256=None,
+        )
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(self.to_dict())
+
+    @property
+    def is_finalized(self) -> bool:
+        return self.state == "finalized"
+
+    def append(self, chunk: ChunkRef) -> Self:
+        if self.is_finalized:
+            raise IngestionConflict(
+                "INGESTION_FINALIZED",
+                "ingestion_manifest.state",
+                "cannot append to a finalized run",
+            )
+        if chunk.source_pin_sha256 != self.source_pin.digest:
+            raise IngestionConflict(
+                "INGESTION_SOURCE_DRIFT",
+                "chunk_ref.source_pin_sha256",
+                "does not equal the manifest source pin",
+            )
+        if chunk.ordinal != len(self.chunks):
+            raise IngestionConflict(
+                "INGESTION_ORDINAL_FENCE",
+                "chunk_ref.ordinal",
+                "must equal the next gap-free manifest ordinal",
+            )
+        if len(self.chunks) >= MAX_MANIFEST_CHUNKS:
+            raise IngestionConflict(
+                "INGESTION_CHUNK_LIMIT", "ingestion_manifest.chunks", "cannot accept another chunk"
+            )
+        return replace(
+            self,
+            chunks=(*self.chunks, chunk),
+            previous_manifest_sha256=self.digest,
+        )
+
+    def finalize(self) -> Self:
+        if self.is_finalized:
+            return self
+        return replace(
+            self,
+            state="finalized",
+            previous_manifest_sha256=self.digest,
+            merkle_root_sha256=merkle_root_sha256(self.chunks),
+        )
+
+    def is_direct_successor_of(self, predecessor_digest: str) -> bool:
+        return self.previous_manifest_sha256 == predecessor_digest
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "run_id": self.run_id,
+            "source_pin": self.source_pin.to_dict(),
+            "chunks": [chunk.to_dict() for chunk in self.chunks],
+            "state": self.state,
+            "previous_manifest_sha256": self.previous_manifest_sha256,
+            "merkle_root_sha256": self.merkle_root_sha256,
+        }
+
+    @classmethod
+    def from_dict(cls, value: object) -> Self:
+        document = _require_exact_mapping(
+            value,
+            frozenset(
+                {
+                    "schema_version",
+                    "run_id",
+                    "source_pin",
+                    "chunks",
+                    "state",
+                    "previous_manifest_sha256",
+                    "merkle_root_sha256",
+                }
+            ),
+            "ingestion_manifest",
+        )
+        raw_chunks = document["chunks"]
+        if not isinstance(raw_chunks, list):
+            raise IngestionError(
+                "INGESTION_DOCUMENT", "ingestion_manifest.chunks", "must be a list"
+            )
+        return cls(
+            schema_version=document["schema_version"],
+            run_id=document["run_id"],
+            source_pin=SourceVersionPinV1.from_dict(document["source_pin"]),
+            chunks=tuple(ChunkRef.from_dict(item) for item in raw_chunks),
+            state=document["state"],
+            previous_manifest_sha256=document["previous_manifest_sha256"],
+            merkle_root_sha256=document["merkle_root_sha256"],
+        )

mostlyright/data_harness/ingestion/faults.py

@@ -0,0 +1,42 @@
+"""Deterministic crash simulation for ingestion durability tests."""
+
+from __future__ import annotations
+
+from collections import Counter
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+
+
+class FaultInjected(RuntimeError):
+    """Raised exactly at a configured durability checkpoint."""
+
+    def __init__(self, checkpoint: str, occurrence: int) -> None:
+        self.checkpoint = checkpoint
+        self.occurrence = occurrence
+        super().__init__(f"fault injected at {checkpoint!r} occurrence {occurrence}")
+
+
+@dataclass
+class DeterministicFaultInjector:
+    """Fail a named checkpoint on a configured occurrence, once per injector.
+
+    It is intentionally stateful and explicit rather than probabilistic so every crash
+    window can be regression-tested and replayed exactly.
+    """
+
+    fail_on: Mapping[str, int] = field(default_factory=dict)
+    occurrences: Counter[str] = field(default_factory=Counter, init=False)
+    _fired: set[str] = field(default_factory=set, init=False)
+
+    def __post_init__(self) -> None:
+        for checkpoint, occurrence in self.fail_on.items():
+            if not isinstance(checkpoint, str) or not checkpoint:
+                raise ValueError("fault checkpoint must be a non-empty string")
+            if isinstance(occurrence, bool) or not isinstance(occurrence, int) or occurrence <= 0:
+                raise ValueError("fault occurrence must be a positive integer")
+
+    def checkpoint(self, name: str) -> None:
+        self.occurrences[name] += 1
+        if self.fail_on.get(name) == self.occurrences[name] and name not in self._fired:
+            self._fired.add(name)
+            raise FaultInjected(name, self.occurrences[name])