mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/stream.py

@@ -0,0 +1,524 @@
+"""Bounded stream-batch adapter with durable replay-safe watermark evidence."""
+
+from __future__ import annotations
+
+import contextlib
+import fcntl
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Protocol
+
+from mostlyright.data_harness.acquisition.http import seal_content_addressed_snapshot
+from mostlyright.data_harness.acquisition.sandbox import CrawlerSandbox
+from mostlyright.data_harness.canonical import (
+    MAX_CANONICAL_MEMBERS,
+    CanonicalJSONError,
+    canonical_json_bytes,
+    canonical_sha256,
+    parse_json,
+    sha256_bytes,
+)
+from mostlyright.data_harness.sources._adapter_steps import (
+    _authorize_candidate,
+    _output,
+    _parse_limits,
+    _require_sandbox_binding,
+)
+from mostlyright.data_harness.sources.adapters import AdapterGovernance
+from mostlyright.data_harness.sources.contracts import (
+    AcquisitionRequest,
+    AdapterDescriptor,
+    SourceContractError,
+)
+from mostlyright.data_harness.sources.governance import (
+    classify_before_persistence,
+    require_admissible,
+)
+from mostlyright.data_harness.sources.registry import AcquisitionContext, AcquisitionOutput
+
+_STATE_VERSION = "harness-stream-watermark.v1"
+# The exact coordinate a Recipe pins and a registry resolves.  Named once here so a caller
+# that has to recognise a stream source reads the adapter's own identity rather than
+# repeating the string.
+STREAM_BATCH_ADAPTER_ID = "user.stream-batch"
+STREAM_BATCH_ADAPTER_VERSION = "1.0.0"
+# The one grammar an event name has to be in to enter the ledger.  Named once, here, because the
+# Recorder that assigns names and the ledger that admits them must not drift apart.
+EVENT_ID_GRAMMAR = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$")
+_EVENT_ID = EVENT_ID_GRAMMAR
+MAX_BATCH_EVENTS = 100_000
+MAX_STATE_BYTES = 64 * 1024 * 1024
+# The state object itself contributes eight canonical members. Keeping the ledger
+# inside the same parser budget is required in addition to the serialized byte cap.
+MAX_SEEN_EVENTS = MAX_CANONICAL_MEMBERS - 8
+
+
+@dataclass(frozen=True)
+class StreamBatch:
+    """One bounded provider batch from a StreamGateway, with the watermark it advances to."""
+
+    query_digest: str
+    previous_watermark: str | None
+    next_watermark: str
+    provider_event_ids: tuple[str, ...]
+    content: bytes
+    content_sha256: str
+    media_type: str
+    data_format: str
+    filename: str
+    observed_at: str
+    available_at: str
+
+    def __post_init__(self) -> None:
+        if sha256_bytes(self.content) != self.content_sha256:
+            raise SourceContractError(
+                "STREAM_CONTENT_DIGEST",
+                "stream_batch.content_sha256",
+                "does not match exact batch bytes",
+            )
+        if not self.provider_event_ids or len(self.provider_event_ids) > MAX_BATCH_EVENTS:
+            raise SourceContractError(
+                "STREAM_EVENT_LIMIT",
+                "stream_batch.provider_event_ids",
+                "must be non-empty and bounded",
+            )
+        if len(set(self.provider_event_ids)) != len(self.provider_event_ids) or any(
+            _EVENT_ID.fullmatch(value) is None for value in self.provider_event_ids
+        ):
+            raise SourceContractError(
+                "STREAM_EVENT_ID",
+                "stream_batch.provider_event_ids",
+                "provider event IDs must be valid and unique",
+            )
+        if not self.next_watermark or self.next_watermark == self.previous_watermark:
+            raise SourceContractError(
+                "STREAM_WATERMARK",
+                "stream_batch.next_watermark",
+                "must advance to a non-empty different value",
+            )
+        for name, value in (
+            ("previous_watermark", self.previous_watermark),
+            ("next_watermark", self.next_watermark),
+        ):
+            if value is not None and (
+                not isinstance(value, str)
+                or not value
+                or len(value) > 512
+                or any(0xD800 <= ord(char) <= 0xDFFF for char in value)
+            ):
+                raise SourceContractError(
+                    "STREAM_WATERMARK",
+                    f"stream_batch.{name}",
+                    "must be a bounded valid string or null",
+                )
+        if self.data_format not in {"json", "ndjson"}:
+            raise SourceContractError(
+                "STREAM_FORMAT",
+                "stream_batch.data_format",
+                "must be json or ndjson",
+            )
+
+
+class StreamGateway(Protocol):
+    def read_batch(self, request: AcquisitionRequest) -> StreamBatch:
+        """Return one coordinator-bounded provider batch."""
+
+
+@dataclass(frozen=True)
+class StreamState:
+    """Durable per-source watermark and admitted event-ID ledger; StreamWatermarkStore writes it."""
+
+    source_id: str
+    watermark: str
+    seen_event_ids: tuple[str, ...]
+    last_batch_content_sha256: str
+    last_batch_event_ids_digest: str
+    last_query_digest: str
+    updated_at: str
+    schema_version: str = _STATE_VERSION
+
+    def __post_init__(self) -> None:
+        if self.schema_version != _STATE_VERSION:
+            raise SourceContractError(
+                "STREAM_STATE_VERSION",
+                "stream_state.schema_version",
+                "is unsupported",
+            )
+        if not re.fullmatch(r"[a-z][a-z0-9._-]{0,127}", self.source_id):
+            raise SourceContractError("STREAM_STATE", "stream_state.source_id", "is invalid")
+        if not self.watermark or len(self.watermark) > 512:
+            raise SourceContractError("STREAM_STATE", "stream_state.watermark", "is invalid")
+        if (
+            len(self.seen_event_ids) > MAX_SEEN_EVENTS
+            or len(set(self.seen_event_ids)) != len(self.seen_event_ids)
+            or tuple(sorted(self.seen_event_ids)) != self.seen_event_ids
+            or any(_EVENT_ID.fullmatch(value) is None for value in self.seen_event_ids)
+        ):
+            raise SourceContractError(
+                "STREAM_STATE",
+                "stream_state.seen_event_ids",
+                "must be a bounded canonical unique event-ID ledger",
+            )
+        for name, value in (
+            ("last_batch_content_sha256", self.last_batch_content_sha256),
+            ("last_batch_event_ids_digest", self.last_batch_event_ids_digest),
+            ("last_query_digest", self.last_query_digest),
+        ):
+            if len(value) != 64 or any(char not in "0123456789abcdef" for char in value):
+                raise SourceContractError("STREAM_STATE", f"stream_state.{name}", "is invalid")
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(self.to_dict())
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "source_id": self.source_id,
+            "watermark": self.watermark,
+            "seen_event_ids": list(self.seen_event_ids),
+            "last_batch_content_sha256": self.last_batch_content_sha256,
+            "last_batch_event_ids_digest": self.last_batch_event_ids_digest,
+            "last_query_digest": self.last_query_digest,
+            "updated_at": self.updated_at,
+        }
+
+
+@dataclass(frozen=True)
+class StreamWatermarkStore:
+    root: Path
+
+    def __post_init__(self) -> None:
+        if not self.root.is_absolute() or self.root.is_symlink() or not self.root.is_dir():
+            raise SourceContractError(
+                "STREAM_STATE_ROOT",
+                "watermark_store.root",
+                "must be an existing absolute non-symlink directory",
+            )
+
+    def commit(
+        self,
+        *,
+        source_id: str,
+        batch: StreamBatch,
+        updated_at: str,
+    ) -> StreamState:
+        if not re.fullmatch(r"[a-z][a-z0-9._-]{0,127}", source_id):
+            raise SourceContractError("STREAM_SOURCE_ID", "source_id", "is invalid")
+        root_fd = os.open(
+            self.root,
+            os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0),
+        )
+        state_name = f"watermark-{source_id}.json"
+        try:
+            fcntl.flock(root_fd, fcntl.LOCK_EX)
+            current = _load_state(root_fd, state_name)
+            event_digest = canonical_sha256(list(batch.provider_event_ids))
+            if current is not None and (
+                current.watermark == batch.next_watermark
+                and current.last_batch_content_sha256 == batch.content_sha256
+                and current.last_batch_event_ids_digest == event_digest
+                and current.last_query_digest == batch.query_digest
+            ):
+                return current
+            current_watermark = current.watermark if current is not None else None
+            if batch.previous_watermark != current_watermark:
+                raise SourceContractError(
+                    "STREAM_WATERMARK_CONFLICT",
+                    "stream_batch.previous_watermark",
+                    "does not match durable coordinator state",
+                )
+            seen = set(current.seen_event_ids if current is not None else ())
+            duplicates = sorted(seen & set(batch.provider_event_ids))
+            if duplicates:
+                raise SourceContractError(
+                    "STREAM_EVENT_REPLAY",
+                    "stream_batch.provider_event_ids",
+                    f"provider event IDs were already admitted: {duplicates[:10]}",
+                )
+            combined = tuple(sorted((*seen, *batch.provider_event_ids)))
+            if len(combined) > MAX_SEEN_EVENTS:
+                raise SourceContractError(
+                    "STREAM_SEEN_EVENT_LIMIT",
+                    "stream_state.seen_event_ids",
+                    "durable event-ID ledger reached its V1 bound",
+                )
+            successor = StreamState(
+                source_id=source_id,
+                watermark=batch.next_watermark,
+                seen_event_ids=combined,
+                last_batch_content_sha256=batch.content_sha256,
+                last_batch_event_ids_digest=event_digest,
+                last_query_digest=batch.query_digest,
+                updated_at=updated_at,
+            )
+            _atomic_state_replace(root_fd, state_name, successor)
+            return successor
+        finally:
+            with contextlib.suppress(OSError):
+                fcntl.flock(root_fd, fcntl.LOCK_UN)
+            os.close(root_fd)
+
+
+def _stream_descriptor() -> AdapterDescriptor:
+    return AdapterDescriptor(
+        adapter_id=STREAM_BATCH_ADAPTER_ID,
+        adapter_version=STREAM_BATCH_ADAPTER_VERSION,
+        source_class="stream",
+        data_formats=("json", "ndjson"),
+        capabilities=("snapshot", "stream", "refresh"),
+    )
+
+
+@dataclass(frozen=True)
+class StreamBatchAdapter:
+    gateway: StreamGateway
+    sandbox: CrawlerSandbox
+    governance: AdapterGovernance
+    watermark_store: StreamWatermarkStore
+    descriptor: AdapterDescriptor = field(default_factory=_stream_descriptor)
+
+    def acquire(
+        self,
+        request: AcquisitionRequest,
+        context: AcquisitionContext,
+    ) -> AcquisitionOutput:
+        if (request.adapter_id, request.adapter_version) != (
+            self.descriptor.adapter_id,
+            self.descriptor.adapter_version,
+        ):
+            raise SourceContractError(
+                "ADAPTER_BINDING",
+                "request",
+                "request does not bind the exact stream adapter",
+            )
+        if request.credential_reference_id is not None:
+            raise SourceContractError(
+                "STREAM_CREDENTIAL_FORBIDDEN",
+                "request.credential_reference_id",
+                "stream gateway credentials remain in the trusted gateway",
+            )
+        _stream_query(request.query)
+        _authorize_candidate(request, self.governance)
+        batch = self.gateway.read_batch(request)
+        if batch.query_digest != request.query_digest:
+            raise SourceContractError(
+                "STREAM_QUERY_BINDING",
+                "stream_batch.query_digest",
+                "batch does not bind the exact canonical query",
+            )
+        if (
+            request.query["expected_watermark"] != batch.previous_watermark
+            or request.query["data_format"] != batch.data_format
+            or request.query["filename"] != batch.filename
+        ):
+            raise SourceContractError(
+                "STREAM_QUERY_BINDING",
+                "stream_batch",
+                "batch watermark, format, or filename differs from the exact request",
+            )
+        if (
+            batch.observed_at != self.governance.observation.observed_at
+            or batch.available_at != self.governance.observation.available_at
+        ):
+            raise SourceContractError(
+                "STREAM_TEMPORAL_BINDING",
+                "stream_batch",
+                "batch times differ from the recorded observation evidence",
+            )
+        if batch.data_format not in self.descriptor.data_formats:
+            raise SourceContractError(
+                "STREAM_FORMAT",
+                "stream_batch.data_format",
+                "batch format is not supported by the adapter",
+            )
+        classification = classify_before_persistence(
+            batch.content,
+            declared_classification=self.governance.declared_classification,
+            lawful_basis_declared=self.governance.lawful_basis_declared,
+        )
+        require_admissible(classification)
+        parsed = self.sandbox.parse(
+            request_id=request.request_id,
+            content=batch.content,
+            data_format=batch.data_format,
+            media_type=batch.media_type,
+            filename=batch.filename,
+            limits=_parse_limits(context),
+        )
+        _require_sandbox_binding(parsed.policy_digest, context)
+        if parsed.parsed is None or parsed.parsed.input_sha256 != batch.content_sha256:
+            raise SourceContractError(
+                "STREAM_CONTENT_BINDING",
+                "sandbox.parsed",
+                "parsed result does not bind the exact stream bytes",
+            )
+        snapshot_path = seal_content_addressed_snapshot(
+            context.snapshot_root,
+            content=batch.content,
+            suffix=batch.data_format,
+        )
+        state = self.watermark_store.commit(
+            source_id=request.source_id,
+            batch=batch,
+            updated_at=request.requested_at,
+        )
+        content_urn = f"urn:sha256:{batch.content_sha256}"
+        return _output(
+            request=request,
+            descriptor=self.descriptor,
+            governance=self.governance,
+            snapshot_path=snapshot_path,
+            content=batch.content,
+            data_format=batch.data_format,
+            media_type=batch.media_type,
+            source_uri=content_urn,
+            final_uri=content_urn,
+            classification=classification.classification,
+            transport_evidence_digest=state.digest,
+            parsed_schema_digest=parsed.parsed.schema_digest,
+            next_watermark=state.watermark,
+        )
+
+
+def _stream_query(value: Any) -> None:
+    if not isinstance(value, dict) or set(value) != {
+        "stream_id",
+        "data_format",
+        "filename",
+        "expected_watermark",
+    }:
+        raise SourceContractError(
+            "STREAM_QUERY",
+            "request.query",
+            "must contain the exact stream batch query fields",
+        )
+    if value["data_format"] not in {"json", "ndjson"}:
+        raise SourceContractError(
+            "STREAM_QUERY",
+            "request.query.data_format",
+            "must be json or ndjson",
+        )
+    if value["expected_watermark"] is not None and not isinstance(value["expected_watermark"], str):
+        raise SourceContractError(
+            "STREAM_QUERY",
+            "request.query.expected_watermark",
+            "must be a string or null",
+        )
+    for name in ("stream_id", "filename"):
+        if not isinstance(value[name], str) or not value[name] or len(value[name]) > 512:
+            raise SourceContractError(
+                "STREAM_QUERY",
+                f"request.query.{name}",
+                "must be a non-empty bounded string",
+            )
+
+
+def _load_state(root_fd: int, name: str) -> StreamState | None:
+    try:
+        descriptor = os.open(
+            name,
+            os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
+            dir_fd=root_fd,
+        )
+    except FileNotFoundError:
+        return None
+    try:
+        raw = _read_bounded(descriptor, MAX_STATE_BYTES)
+    finally:
+        os.close(descriptor)
+    value = parse_json(raw)
+    if not isinstance(value, dict) or set(value) != {
+        "schema_version",
+        "source_id",
+        "watermark",
+        "seen_event_ids",
+        "last_batch_content_sha256",
+        "last_batch_event_ids_digest",
+        "last_query_digest",
+        "updated_at",
+    }:
+        raise SourceContractError("STREAM_STATE", "watermark_state", "shape is invalid")
+    if value["schema_version"] != _STATE_VERSION or not isinstance(value["seen_event_ids"], list):
+        raise SourceContractError("STREAM_STATE", "watermark_state", "version is invalid")
+    return StreamState(
+        source_id=value["source_id"],
+        watermark=value["watermark"],
+        seen_event_ids=tuple(value["seen_event_ids"]),
+        last_batch_content_sha256=value["last_batch_content_sha256"],
+        last_batch_event_ids_digest=value["last_batch_event_ids_digest"],
+        last_query_digest=value["last_query_digest"],
+        updated_at=value["updated_at"],
+    )
+
+
+def _atomic_state_replace(root_fd: int, name: str, state: StreamState) -> None:
+    payload = _serialize_state(state)
+    temporary = f".{name}.{os.getpid()}.tmp"
+    with contextlib.suppress(FileNotFoundError):
+        os.unlink(temporary, dir_fd=root_fd)
+    descriptor = os.open(
+        temporary,
+        os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
+        0o600,
+        dir_fd=root_fd,
+    )
+    try:
+        view = memoryview(payload)
+        written = 0
+        while written < len(view):
+            count = os.write(descriptor, view[written:])
+            if count <= 0:
+                raise SourceContractError(
+                    "STREAM_STATE_WRITE",
+                    "watermark_state",
+                    "write made no progress",
+                )
+            written += count
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+    try:
+        os.replace(temporary, name, src_dir_fd=root_fd, dst_dir_fd=root_fd)
+        os.fsync(root_fd)
+    finally:
+        with contextlib.suppress(FileNotFoundError):
+            os.unlink(temporary, dir_fd=root_fd)
+
+
+def _serialize_state(state: StreamState) -> bytes:
+    try:
+        payload = canonical_json_bytes(state.to_dict())
+    except CanonicalJSONError as error:
+        raise SourceContractError(
+            "STREAM_STATE_LIMIT",
+            "stream_state.seen_event_ids",
+            "state exceeds the canonical parser member budget",
+        ) from error
+    if len(payload) > MAX_STATE_BYTES:
+        raise SourceContractError(
+            "STREAM_STATE_LIMIT",
+            "watermark_state",
+            f"canonical state exceeds its {MAX_STATE_BYTES}-byte durable loader budget",
+        )
+    return payload
+
+
+def _read_bounded(descriptor: int, maximum: int) -> bytes:
+    chunks: list[bytes] = []
+    total = 0
+    while total <= maximum:
+        chunk = os.read(descriptor, min(64 * 1024, maximum + 1 - total))
+        if not chunk:
+            break
+        chunks.append(chunk)
+        total += len(chunk)
+    if total > maximum:
+        raise SourceContractError(
+            "STREAM_STATE_LIMIT",
+            "watermark_state",
+            "state exceeds its byte budget",
+        )
+    return b"".join(chunks)