mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/visual_run/query.py

@@ -0,0 +1,259 @@
+"""Bounded read-only query boundary over exact sealed Parquet bytes."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any
+
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from mostlyright.data_harness.plan_graph import HARD_MAX_OUTPUT_ROWS
+from mostlyright.data_harness.rowset import open_sealed_parquet, render_filtered_window
+from mostlyright.data_harness.visual_run.observations import (
+    safe_column_for_display,
+    safe_for_display,
+)
+
+MAX_QUERY_ROWS = 200
+MAX_QUERY_BYTES = 1024 * 1024
+MAX_PREDICATES = 8
+QUERY_OPERATORS = frozenset({"eq", "ne", "lt", "le", "gt", "ge", "is_null", "is_not_null"})
+QUERY_DISPLAY_DISPOSITIONS = frozenset({"full", "redacted", "aggregate_only", "schema_only"})
+PUBLIC_RIGHTS = frozenset({"public_domain", "permissive_license"})
+
+
+@dataclass(frozen=True)
+class QueryEvidence:
+    """Exact verified evidence that authorizes and explains a row-query result."""
+
+    candidate_digest: str
+    table_sha256: str
+    display: str
+    rights_statuses: tuple[str, ...]
+    classifications: tuple[str, ...]
+    profiles: Mapping[str, Mapping[str, Any]]
+    findings: tuple[Mapping[str, Any], ...]
+    lineage: Mapping[str, Mapping[str, Any]]
+    evidence_sha256: Mapping[str, str]
+
+    def validate(self, parquet_raw: bytes, available: Sequence[str]) -> None:
+        digests = (self.candidate_digest, self.table_sha256, *self.evidence_sha256.values())
+        if any(
+            not isinstance(digest, str)
+            or len(digest) != 64
+            or any(character not in "0123456789abcdef" for character in digest)
+            for digest in digests
+        ):
+            raise QueryError("QUERY_EVIDENCE", "verified evidence digests are invalid")
+        if hashlib.sha256(parquet_raw).hexdigest() != self.table_sha256:
+            raise QueryError("QUERY_EVIDENCE", "row bytes differ from verified evidence")
+        if self.display not in QUERY_DISPLAY_DISPOSITIONS:
+            raise QueryError("QUERY_EVIDENCE", "display disposition is invalid")
+        if (
+            self.display != "full"
+            or not self.rights_statuses
+            or any(status not in PUBLIC_RIGHTS for status in self.rights_statuses)
+            or not self.classifications
+            or any(classification != "public" for classification in self.classifications)
+        ):
+            raise QueryError(
+                "QUERY_DISPLAY_POLICY", "verified source governance forbids row-level display"
+            )
+        if set(self.profiles) != set(available) or set(self.lineage) != set(available):
+            raise QueryError(
+                "QUERY_EVIDENCE", "verified profile or lineage does not cover the sealed schema"
+            )
+        if any(not safe_column_for_display(column) for column in available):
+            raise QueryError("QUERY_SENSITIVE_COLUMN", "sealed schema contains a secret coordinate")
+
+
+class QueryError(ValueError):
+    def __init__(self, code: str, detail: str) -> None:
+        self.code = code
+        super().__init__(detail)
+
+
+def _request(
+    columns: Sequence[str], predicates: Sequence[Mapping[str, Any]], offset: int, limit: int
+) -> None:
+    if (
+        not columns
+        or len(columns) > 256
+        or len(columns) != len(set(columns))
+        or any(not isinstance(column, str) for column in columns)
+    ):
+        raise QueryError("QUERY_COLUMNS", "selected columns are invalid")
+    if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
+        raise QueryError("QUERY_WINDOW", "offset is invalid")
+    if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= MAX_QUERY_ROWS:
+        raise QueryError("QUERY_WINDOW", "limit is invalid")
+    if len(predicates) > MAX_PREDICATES:
+        raise QueryError("QUERY_FILTER_BOUNDS", "too many predicates")
+    for predicate in predicates:
+        if not isinstance(predicate, Mapping) or set(predicate) not in (
+            {"column", "op"},
+            {"column", "op", "value"},
+        ):
+            raise QueryError("QUERY_FILTER", "predicate fields are invalid")
+        if not isinstance(predicate["column"], str) or predicate["op"] not in QUERY_OPERATORS:
+            raise QueryError("QUERY_FILTER", "predicate coordinate is invalid")
+        nullary = predicate["op"] in {"is_null", "is_not_null"}
+        if nullary != ("value" not in predicate):
+            raise QueryError("QUERY_FILTER", "predicate value shape is invalid")
+        if "value" in predicate and type(predicate["value"]) not in {bool, int, float, str}:
+            raise QueryError("QUERY_FILTER", "predicate value is not a scalar")
+
+
+def query_sealed_parquet(
+    parquet_raw: bytes,
+    *,
+    columns: Sequence[str],
+    predicates: Sequence[Mapping[str, Any]] = (),
+    offset: int = 0,
+    limit: int = 50,
+    evidence: QueryEvidence,
+) -> dict[str, Any]:
+    """Return one deterministic physical-order window; never SQL or arbitrary expressions."""
+
+    _request(columns, predicates, offset, limit)
+    parquet, available, total = open_sealed_parquet(
+        parquet_raw,
+        max_columns=256,
+        max_rows=HARD_MAX_OUTPUT_ROWS,
+        max_uncompressed_bytes=256 * 1024 * 1024,
+    )
+    evidence.validate(parquet_raw, available)
+    unknown = sorted(({*columns} | {str(item["column"]) for item in predicates}) - set(available))
+    if unknown:
+        raise QueryError("QUERY_COLUMN_UNKNOWN", f"columns are absent: {unknown}")
+    if any(
+        not safe_column_for_display(column)
+        for column in (*columns, *(str(item["column"]) for item in predicates))
+    ):
+        raise QueryError("QUERY_SENSITIVE_COLUMN", "selected column is a secret coordinate")
+    if any(
+        "value" in predicate and not safe_for_display(predicate["value"])
+        for predicate in predicates
+    ):
+        raise QueryError("QUERY_SENSITIVE_VALUE", "filter contains credential-bearing text")
+
+    inspected = {*columns, *(str(item["column"]) for item in predicates)}
+    for batch in parquet.iter_batches(batch_size=1_024, columns=sorted(inspected)):
+        for column in inspected:
+            array = batch.column(batch.schema.get_field_index(column))
+            if any(
+                scalar.is_valid
+                and isinstance(scalar.as_py(), str)
+                and not safe_for_display(scalar.as_py())
+                for scalar in array
+            ):
+                raise QueryError(
+                    "QUERY_SENSITIVE_VALUE", "selected data contains credential-bearing text"
+                )
+
+    def select(batch: pa.RecordBatch) -> pa.RecordBatch | pa.Table:
+        mask: pa.Array | pa.ChunkedArray | None = None
+        for predicate in predicates:
+            array = batch.column(batch.schema.get_field_index(str(predicate["column"])))
+            op = str(predicate["op"])
+            if op == "is_null":
+                condition = pc.is_null(array)
+            elif op == "is_not_null":
+                condition = pc.invert(pc.is_null(array))
+            else:
+                functions = {
+                    "eq": pc.equal,
+                    "ne": pc.not_equal,
+                    "lt": pc.less,
+                    "le": pc.less_equal,
+                    "gt": pc.greater,
+                    "ge": pc.greater_equal,
+                }
+                try:
+                    condition = functions[op](array, predicate["value"])
+                except (ArrowInvalid, ArrowNotImplementedError) as exc:  # type: ignore[name-defined]
+                    raise QueryError(
+                        "QUERY_FILTER_TYPE", "filter value does not match column"
+                    ) from exc
+            mask = condition if mask is None else pc.and_kleene(mask, condition)
+        retained: pa.RecordBatch | pa.Table = batch if mask is None else batch.filter(mask)
+        return retained.select(columns)
+
+    try:
+        matched, rows = render_filtered_window(
+            parquet,
+            columns,
+            select=select,
+            offset=offset,
+            limit=limit,
+            exact_match_count=total if not predicates else None,
+        )
+    except QueryError:
+        raise
+    payload = {
+        "schema_version": "sealed-row-query.v1",
+        "columns": list(columns),
+        "total_row_count": total,
+        "matched_row_count": matched,
+        "offset": offset,
+        "limit": limit,
+        "rows": rows,
+        "truncated": offset + len(rows) < matched,
+        "evidence": {
+            "candidate_digest": evidence.candidate_digest,
+            "table_sha256": evidence.table_sha256,
+            "display": evidence.display,
+            "rights_statuses": list(evidence.rights_statuses),
+            "classifications": list(evidence.classifications),
+            "profiles": [dict(evidence.profiles[column]) for column in columns],
+            "findings": [
+                dict(finding)
+                for finding in evidence.findings
+                if finding.get("column") in {None, *columns}
+            ],
+            "lineage": [dict(evidence.lineage[column]) for column in columns],
+            "member_sha256": dict(evidence.evidence_sha256),
+        },
+    }
+    while (
+        len(json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode())
+        > MAX_QUERY_BYTES
+    ):
+        if not payload["rows"]:
+            raise QueryError("QUERY_RESPONSE_BOUNDS", "query metadata exceeds response budget")
+        payload["rows"] = payload["rows"][:-1]
+        payload["truncated"] = True
+    return payload
+
+
+def sealed_parquet_columns(parquet_raw: bytes, *, evidence: QueryEvidence) -> tuple[str, ...]:
+    """Return the bounded sealed schema for the viewer's initial explorer request."""
+
+    _parquet, available, _total = open_sealed_parquet(
+        parquet_raw,
+        max_columns=256,
+        max_rows=HARD_MAX_OUTPUT_ROWS,
+        max_uncompressed_bytes=256 * 1024 * 1024,
+    )
+    evidence.validate(parquet_raw, available)
+    return tuple(available)
+
+
+# Local aliases keep pyarrow exceptions out of the public contract.
+ArrowInvalid = pa.ArrowInvalid
+ArrowNotImplementedError = pa.ArrowNotImplementedError
+
+__all__ = [
+    "MAX_PREDICATES",
+    "MAX_QUERY_BYTES",
+    "MAX_QUERY_ROWS",
+    "QUERY_OPERATORS",
+    "QueryError",
+    "QueryEvidence",
+    "query_sealed_parquet",
+    "sealed_parquet_columns",
+]

mostlyright/data_harness/visual_run/reducer.py

@@ -0,0 +1,280 @@
+"""Pure deterministic reduction of visual-run events."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, replace
+from types import MappingProxyType
+
+from mostlyright.data_harness.visual_run.contracts import (
+    EVENT_REGISTRY,
+    ArtifactReference,
+    TerminalError,
+    VisualRunError,
+    VisualRunEvent,
+)
+
+
+@dataclass(frozen=True)
+class CellState:
+    cell_id: str
+    cell_type: str
+    version: int
+    phase: str
+    stage: str
+    status: str
+    source_length: int = 0
+    source: ArtifactReference | None = None
+    execution_count: int | None = None
+    outputs: tuple[ArtifactReference, ...] = ()
+    error: TerminalError | None = None
+
+
+@dataclass(frozen=True)
+class ObjectState:
+    """Complete current state for every non-cell registry object family."""
+
+    object_id: str
+    kind: str
+    version: int
+    phase: str
+    stage: str
+    status: str
+    payload: Mapping[str, object]
+    artifact: ArtifactReference | None = None
+    error: TerminalError | None = None
+
+
+@dataclass(frozen=True)
+class ReducedRun:
+    run_id: str | None = None
+    cursor: int = 0
+    lifecycle: str = "waiting"
+    active_object_id: str | None = None
+    cells: Mapping[str, CellState] = MappingProxyType({})
+    observations: tuple[ArtifactReference, ...] = ()
+    quality: tuple[int, int] | None = None
+    candidate_digest: str | None = None
+    table_ready: bool = False
+    terminal_error: TerminalError | None = None
+    terminal: bool = False
+    object_versions: Mapping[str, int] = MappingProxyType({})
+    object_kinds: Mapping[str, str] = MappingProxyType({})
+    objects: Mapping[str, ObjectState] = MappingProxyType({})
+
+
+def _fail(code: str, message: str) -> None:
+    raise VisualRunError(code, message)
+
+
+def apply_event(state: ReducedRun, event: VisualRunEvent) -> ReducedRun:
+    if event.sequence != state.cursor + 1:
+        _fail("VISUAL_RUN_SEQUENCE_GAP", "reducer requires the next durable sequence")
+    if state.run_id is not None and event.run_id != state.run_id:
+        _fail("VISUAL_RUN_RUN_MISMATCH", "event belongs to another run")
+    if state.terminal:
+        _fail("VISUAL_RUN_TERMINAL_APPEND", "events cannot follow a terminal run event")
+    cells = dict(state.cells)
+    observations = state.observations
+    lifecycle = state.lifecycle
+    active = state.active_object_id
+    quality = state.quality
+    candidate_digest = state.candidate_digest
+    table_ready = state.table_ready
+    terminal_error = state.terminal_error
+    terminal = state.terminal
+    object_versions = dict(state.object_versions)
+    object_kinds = dict(state.object_kinds)
+    objects = dict(state.objects)
+
+    event_type = event.event_type
+    spec = EVENT_REGISTRY[event_type]
+    event_family = event_type.split(".", 1)[0]
+    current_version = object_versions.get(event.object_id)
+    if spec.object_rule == "create":
+        if current_version is not None:
+            _fail("VISUAL_RUN_OBJECT_EXISTS", "object creation was already recorded")
+        if event.previous_object_version is not None or event.object_version != 1:
+            _fail("VISUAL_RUN_OBJECT_VERSION", "object creation must start at version one")
+    else:
+        if current_version is None:
+            _fail("VISUAL_RUN_OBJECT_MISSING", "object update has no creation event")
+        if (
+            event.previous_object_version != current_version
+            or event.object_version != current_version + 1
+        ):
+            _fail("VISUAL_RUN_OBJECT_VERSION", "object update does not follow current version")
+    if event_type != "run.started" and state.run_id is None:
+        _fail("VISUAL_RUN_TRANSITION", "run must start before other events")
+    established_kind = object_kinds.get(event.object_id)
+    compatible_kinds = {established_kind}
+    if established_kind == "transformation":
+        compatible_kinds.add("stage")
+    if current_version is not None and event_family not in compatible_kinds:
+        _fail("VISUAL_RUN_OBJECT_KIND", "object update changes registry family")
+    if event_type == "run.started":
+        lifecycle = "researching"
+    elif event_type in {"run.completed", "run.failed", "run.interrupted"}:
+        lifecycle = event.status or "failed"
+        terminal_error = event.error
+        terminal = True
+        active = None
+    elif event_type == "phase.started":
+        lifecycle = event.phase
+        terminal_error = None
+        active = event.object_id
+    elif event_type == "phase.completed":
+        active = None
+    elif event_type == "phase.failed":
+        lifecycle = "failed"
+        terminal_error = event.error
+        active = None
+        terminal_error = event.error
+    elif event_type == "cell.created":
+        cells[event.object_id] = CellState(
+            event.object_id,
+            str(event.payload["cell_type"]),
+            event.object_version,
+            event.phase,
+            event.stage,
+            "writing",
+        )
+        active = event.object_id
+    elif event_type.startswith("cell."):
+        moved = False
+        cell = cells.get(event.object_id)
+        if cell is None:
+            _fail("VISUAL_RUN_OBJECT_MISSING", "cell update has no creation event")
+        if cell.status in {"complete", "failed"} and event_type not in {
+            "cell.source_revised",
+            "cell.moved",
+        }:
+            _fail("VISUAL_RUN_OBJECT_TERMINAL", "terminal cell cannot be changed")
+        if event_type in {"cell.source_extended", "cell.source_revised"}:
+            length = int(event.payload["source_length"])
+            if event_type == "cell.source_extended" and length < cell.source_length:
+                _fail("VISUAL_RUN_TRANSITION", "source length cannot shrink")
+            if event.artifact is None:
+                _fail("VISUAL_RUN_ARTIFACT_REQUIRED", "source update requires artifact")
+            cell = replace(
+                cell,
+                source_length=length,
+                source=event.artifact,
+                status="writing",
+                execution_count=None,
+                outputs=(),
+                error=None,
+            )
+            active = event.object_id
+        elif event_type == "cell.moved":
+            after = str(event.payload["after_cell_id"])
+            if after == event.object_id or after not in cells:
+                _fail("VISUAL_RUN_TRANSITION", "cell move anchor is invalid")
+            moving = replace(cells.pop(event.object_id), version=event.object_version)
+            reordered: dict[str, CellState] = {}
+            for identifier, existing in cells.items():
+                reordered[identifier] = existing
+                if identifier == after:
+                    reordered[event.object_id] = moving
+            cells = reordered
+            cell = moving
+            moved = True
+        elif event_type == "cell.execution_started":
+            cell = replace(
+                cell, execution_count=int(event.payload["execution_count"]), status="executing"
+            )
+        elif event_type == "cell.output_appended":
+            if event.artifact is None:
+                _fail("VISUAL_RUN_ARTIFACT_REQUIRED", "output requires artifact")
+            index = int(event.payload["output_index"])
+            if index != len(cell.outputs):
+                _fail("VISUAL_RUN_TRANSITION", "appended output index must be next")
+            cell = replace(cell, outputs=(*cell.outputs, event.artifact), status="executing")
+        elif event_type == "cell.output_replaced":
+            if event.artifact is None:
+                _fail("VISUAL_RUN_ARTIFACT_REQUIRED", "output requires artifact")
+            index = int(event.payload["output_index"])
+            if not 0 <= index < len(cell.outputs):
+                _fail("VISUAL_RUN_TRANSITION", "replacement output index is absent")
+            outputs = list(cell.outputs)
+            outputs[index] = event.artifact
+            cell = replace(cell, outputs=tuple(outputs), status="executing")
+        elif event_type == "cell.completed":
+            cell = replace(cell, status="complete")
+            active = None
+        elif event_type == "cell.failed":
+            cell = replace(cell, status="failed", error=event.error)
+            active = None
+        if not moved:
+            cells[event.object_id] = replace(cell, version=event.object_version)
+    elif event_type in {
+        "source.schema_observed",
+        "source.observation_ready",
+        "stage.observation_ready",
+    }:
+        if event.artifact is not None:
+            observations = (*observations, event.artifact)
+    elif event_type == "quality.completed":
+        quality = (int(event.payload["checks_passed"]), int(event.payload["checks_total"]))
+    elif event_type == "build.sealed":
+        candidate_digest = str(event.payload["candidate_digest"])
+        lifecycle = "building"
+    elif event_type == "build.verified":
+        digest = str(event.payload["candidate_digest"])
+        if candidate_digest is None:
+            _fail("VISUAL_RUN_TRANSITION", "build must be sealed before verification")
+        if digest != candidate_digest:
+            _fail("VISUAL_RUN_BUILD_MISMATCH", "verified digest differs from sealed digest")
+        candidate_digest = digest
+        table_ready = True
+        lifecycle = "table-ready"
+
+    if event.status is not None:
+        status = event.status
+    elif event_type.endswith(".failed"):
+        status = "failed"
+    elif event_type == "source.rejected":
+        status = "rejected"
+    elif event_type.endswith((".completed", ".verified", ".written", ".materialized")) or (
+        event_type.endswith((".observation_ready", ".schema_observed"))
+    ):
+        status = "complete"
+    else:
+        status = "active"
+    objects[event.object_id] = ObjectState(
+        object_id=event.object_id,
+        kind=established_kind or event_family,
+        version=event.object_version,
+        phase=event.phase,
+        stage=event.stage,
+        status=status,
+        payload=event.payload,
+        artifact=event.artifact,
+        error=event.error,
+    )
+
+    object_versions[event.object_id] = event.object_version
+    object_kinds.setdefault(event.object_id, event_family)
+    return ReducedRun(
+        run_id=event.run_id,
+        cursor=event.sequence,
+        lifecycle=lifecycle,
+        active_object_id=active,
+        cells=MappingProxyType(cells),
+        observations=observations,
+        quality=quality,
+        candidate_digest=candidate_digest,
+        table_ready=table_ready,
+        terminal_error=terminal_error,
+        terminal=terminal,
+        object_versions=MappingProxyType(object_versions),
+        object_kinds=MappingProxyType(object_kinds),
+        objects=MappingProxyType(objects),
+    )
+
+
+def reduce_events(events: tuple[VisualRunEvent, ...] | list[VisualRunEvent]) -> ReducedRun:
+    state = ReducedRun()
+    for event in events:
+        state = apply_event(state, event)
+    return state