mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/preparation/engine.py

@@ -0,0 +1,221 @@
+"""Fixed-order deterministic preparation executor."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from mostlyright.data_harness.canonical import canonical_sha256
+from mostlyright.data_harness.preparation.contracts import (
+    AsOfJoinSpec,
+    ColumnLineage,
+    PreparationPlan,
+)
+from mostlyright.data_harness.preparation.errors import PreparationError
+from mostlyright.data_harness.preparation.gates import (
+    QualityReport,
+    TargetRationale,
+    TemporalReport,
+    construct_target,
+    validate_quality,
+    validate_temporal_safety,
+)
+from mostlyright.data_harness.preparation.joins import (
+    JoinAudit,
+    backward_asof_join,
+    equality_left_join,
+)
+from mostlyright.data_harness.preparation.profile import TableProfile
+from mostlyright.data_harness.preparation.table import TableState, validate_columns
+from mostlyright.data_harness.preparation.transforms import (
+    OperationAudit,
+    PreparedSource,
+    apply_operations,
+)
+
+
+@dataclass(frozen=True)
+class PreparationResult:
+    """Complete deterministic preparation evidence and immutable output state."""
+
+    plan_id: str
+    plan_digest: str
+    output: TableState
+    source_before_profiles: tuple[tuple[str, TableProfile], ...]
+    source_after_profiles: tuple[tuple[str, TableProfile], ...]
+    operation_audits: tuple[OperationAudit, ...]
+    join_audits: tuple[JoinAudit, ...]
+    temporal_report: TemporalReport
+    target_rationale: TargetRationale | None
+    output_profile: TableProfile
+    quality_report: QualityReport
+
+    @property
+    def analysis_ready(self) -> bool:
+        return self.quality_report.status == "passed"
+
+    @property
+    def evidence_digest(self) -> str:
+        return canonical_sha256(self.to_evidence_dict())
+
+    def to_evidence_dict(self) -> dict[str, Any]:
+        return {
+            "plan_id": self.plan_id,
+            "plan_digest": self.plan_digest,
+            "source_before_profiles": [
+                {"source_id": source_id, "profile": profile.to_dict()}
+                for source_id, profile in self.source_before_profiles
+            ],
+            "source_after_profiles": [
+                {"source_id": source_id, "profile": profile.to_dict()}
+                for source_id, profile in self.source_after_profiles
+            ],
+            "operation_audits": [item.to_dict() for item in self.operation_audits],
+            "join_audits": [item.to_dict() for item in self.join_audits],
+            "temporal_report": self.temporal_report.to_dict(),
+            "target_rationale": (
+                self.target_rationale.to_dict() if self.target_rationale is not None else None
+            ),
+            "output_profile": self.output_profile.to_dict(),
+            "quality_report": self.quality_report.to_dict(),
+            "column_lineage": [item.to_dict() for item in self.output.lineage],
+            "snapshot_digests": list(self.output.snapshot_digests),
+            "analysis_ready": self.analysis_ready,
+        }
+
+
+def execute_preparation(
+    plan: PreparationPlan,
+    inputs: Mapping[str, TableState],
+) -> PreparationResult:
+    """Execute the only supported source→operations→joins→temporal→target→quality order."""
+
+    if not isinstance(plan, PreparationPlan):
+        raise PreparationError("PLAN_TYPE", "plan", "must be a PreparationPlan")
+    if not isinstance(inputs, Mapping):
+        raise PreparationError("INPUT_TYPE", "inputs", "must be a source mapping")
+    states = dict(inputs)
+    if not states or any(not isinstance(key, str) for key in states):
+        raise PreparationError("INPUT_SET", "inputs", "must contain named immutable tables")
+    if any(not isinstance(state, TableState) for state in states.values()):
+        raise PreparationError("INPUT_TYPE", "inputs", "contains a non-TableState value")
+    if any(source_id != state.table_id for source_id, state in states.items()):
+        raise PreparationError("INPUT_SOURCE", "inputs", "mapping keys must equal source table IDs")
+    expected_sources = {item.source_id for item in plan.temporal_evidence}
+    if set(states) != expected_sources:
+        raise PreparationError(
+            "INPUT_SET",
+            "inputs",
+            "must exactly match declared temporal evidence sources",
+        )
+    if plan.primary_source_id not in states:
+        raise PreparationError("INPUT_PRIMARY", "plan.primary_source_id", "is not an input")
+    operations_by_source: dict[str, list[Any]] = {source_id: [] for source_id in states}
+    for operation in plan.operations:
+        if operation.source_id not in states:
+            raise PreparationError(
+                "OPERATION_SOURCE",
+                operation.operation_id,
+                "references an unknown input source",
+            )
+        operations_by_source[operation.source_id].append(operation)
+    prepared: dict[str, PreparedSource] = {}
+    for source_id in sorted(states):
+        prepared[source_id] = apply_operations(
+            states[source_id],
+            tuple(operations_by_source[source_id]),
+        )
+    current = prepared[plan.primary_source_id].state
+    join_audits: list[JoinAudit] = []
+    consumed_right_sources: set[str] = set()
+    for join in plan.joins:
+        if join.right_source_id == plan.primary_source_id:
+            raise PreparationError("JOIN_SOURCE", join.join_id, "cannot join primary onto itself")
+        if join.right_source_id in consumed_right_sources:
+            raise PreparationError(
+                "JOIN_SOURCE",
+                join.join_id,
+                "a right source can be consumed only once",
+            )
+        try:
+            right = prepared[join.right_source_id].state
+        except KeyError:
+            raise PreparationError(
+                "JOIN_SOURCE",
+                join.join_id,
+                "references an unknown right source",
+            ) from None
+        if isinstance(join, AsOfJoinSpec):
+            current, audit = backward_asof_join(current, right, join)
+        else:
+            current, audit = equality_left_join(current, right, join)
+        join_audits.append(audit)
+        consumed_right_sources.add(join.right_source_id)
+    used_sources = {item.source_id for lineage in current.lineage for item in lineage.inputs}
+    if used_sources != expected_sources:
+        raise PreparationError(
+            "JOIN_SOURCE",
+            "plan.joins",
+            "not every declared input contributes to the prepared table",
+        )
+    temporal = validate_temporal_safety(
+        current,
+        evidence=plan.temporal_evidence,
+        policy=plan.temporal_policy,
+    )
+    current, target_rationale = construct_target(
+        current,
+        plan.target,
+        target_policy=plan.target_policy,
+    )
+    current = _final_projection(current, plan.output_columns)
+    output_profile, quality_report = validate_quality(
+        current,
+        spec=plan.quality,
+        joins=tuple(join_audits),
+        temporal=temporal,
+    )
+    source_before = tuple(
+        (source_id, prepared[source_id].before_profile) for source_id in sorted(prepared)
+    )
+    source_after = tuple(
+        (source_id, prepared[source_id].after_profile) for source_id in sorted(prepared)
+    )
+    operation_audits = tuple(
+        audit for source_id in sorted(prepared) for audit in prepared[source_id].operation_audits
+    )
+    return PreparationResult(
+        plan_id=plan.plan_id,
+        plan_digest=plan.digest,
+        output=current,
+        source_before_profiles=source_before,
+        source_after_profiles=source_after,
+        operation_audits=operation_audits,
+        join_audits=tuple(join_audits),
+        temporal_report=temporal,
+        target_rationale=target_rationale,
+        output_profile=output_profile,
+        quality_report=quality_report,
+    )
+
+
+def _final_projection(state: TableState, columns: tuple[str, ...]) -> TableState:
+    indexes = validate_columns(state, columns, "plan.output_columns")
+    lineage_by_name = {item.output_column: item for item in state.lineage}
+    operation_id = "final_projection"
+    return state.replace(
+        fields=tuple(state.fields[index] for index in indexes),
+        rows=tuple(tuple(row[index] for index in indexes) for row in state.rows),
+        lineage=tuple(
+            ColumnLineage(
+                output_column=column,
+                inputs=lineage_by_name[column].inputs,
+                operation_ids=(*lineage_by_name[column].operation_ids, operation_id),
+            )
+            for column in columns
+        ),
+    )
+
+
+__all__ = ["PreparationResult", "execute_preparation"]

mostlyright/data_harness/preparation/errors.py

@@ -0,0 +1,14 @@
+"""Typed deterministic preparation failures."""
+
+from __future__ import annotations
+
+
+class PreparationError(ValueError):
+    """A stable preparation contract, execution, or quality failure."""
+
+    def __init__(self, code: str, path: str, detail: str, *, severity: str = "blocker") -> None:
+        self.code = code
+        self.path = path
+        self.detail = detail
+        self.severity = severity
+        super().__init__(f"{path}: {detail} [{code}]")