cfa-kernel 0.1.0__py3-none-any.whl

cfa/core/planner.py

@@ -0,0 +1,290 @@
+"""
+CFA Execution Planner
+=====================
+Generates a governed execution DAG from an approved State Signature.
+
+The Planner is NOT free — it fills templates, follows the plan approved
+by the Policy Engine, and respects all constraints declared in the Signature.
+
+Key properties:
+- Every plan is idempotent (merge with deterministic key, partition overwrite)
+- Supports Composite Intent decomposition
+- Consistency unit selection follows whitepaper enum (partition | dataset | dag_branch | time_window)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any
+
+from cfa.types import StateSignature, TargetLayer
+
+# ── Consistency Unit ─────────────────────────────────────────────────────────
+
+
+class ConsistencyUnit(StrEnum):
+    PARTITION = "partition"
+    DATASET = "dataset"
+    DAG_BRANCH = "dag_branch"
+    TIME_WINDOW = "time_window"
+
+
+# ── Execution Steps ──────────────────────────────────────────────────────────
+
+
+class StepType(StrEnum):
+    EXTRACT = "extract"
+    ANONYMIZE = "anonymize"
+    JOIN = "join"
+    TRANSFORM = "transform"
+    LOAD = "load"
+    FILTER = "filter"
+    AGGREGATE = "aggregate"
+
+
+@dataclass(frozen=True)
+class ExecutionStep:
+    """Single node in the execution DAG."""
+
+    id: str
+    step_type: StepType
+    source: str | None = None
+    target: str | None = None
+    config: dict[str, Any] = field(default_factory=dict)
+    depends_on: tuple[str, ...] = ()
+
+    @property
+    def description(self) -> str:
+        parts = [f"{self.step_type.value}"]
+        if self.source:
+            parts.append(f"source={self.source}")
+        if self.target:
+            parts.append(f"target={self.target}")
+        return " | ".join(parts)
+
+
+# ── Execution Plan ───────────────────────────────────────────────────────────
+
+
+class WriteMode(StrEnum):
+    MERGE = "merge"
+    OVERWRITE_PARTITION = "overwrite_partition"
+    APPEND = "append"  # only allowed in Bronze
+
+
+@dataclass
+class ExecutionPlan:
+    """
+    Governed execution DAG generated from an approved Signature.
+    Immutable once finalized — any change requires a new plan.
+    """
+
+    signature_hash: str
+    intent_id: str
+    steps: list[ExecutionStep]
+    consistency_unit: ConsistencyUnit
+    write_mode: WriteMode
+    idempotent: bool = True
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def step_ids(self) -> list[str]:
+        return [s.id for s in self.steps]
+
+    @property
+    def step_count(self) -> int:
+        return len(self.steps)
+
+    def get_step(self, step_id: str) -> ExecutionStep | None:
+        return next((s for s in self.steps if s.id == step_id), None)
+
+    def execution_order(self) -> list[ExecutionStep]:
+        """Topological sort of steps respecting depends_on."""
+        resolved: list[ExecutionStep] = []
+        resolved_ids: set[str] = set()
+        pending = list(self.steps)
+
+        max_iterations = len(pending) * len(pending)
+        iteration = 0
+        while pending:
+            iteration += 1
+            if iteration > max_iterations:
+                raise ValueError("Cyclic dependency detected in execution plan")
+
+            for step in list(pending):
+                if all(dep in resolved_ids for dep in step.depends_on):
+                    resolved.append(step)
+                    resolved_ids.add(step.id)
+                    pending.remove(step)
+
+        return resolved
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "signature_hash": self.signature_hash,
+            "intent_id": self.intent_id,
+            "consistency_unit": self.consistency_unit.value,
+            "write_mode": self.write_mode.value,
+            "idempotent": self.idempotent,
+            "steps": [
+                {
+                    "id": s.id,
+                    "type": s.step_type.value,
+                    "source": s.source,
+                    "target": s.target,
+                    "config": s.config,
+                    "depends_on": list(s.depends_on),
+                }
+                for s in self.steps
+            ],
+        }
+
+
+# ── Execution Planner ────────────────────────────────────────────────────────
+
+
+class ExecutionPlanner:
+    """
+    Generates an ExecutionPlan from an approved StateSignature.
+
+    The planner does NOT generate arbitrary code — it assembles governed steps
+    based on the Signature's intent, datasets, constraints and target layer.
+    """
+
+    def plan(self, signature: StateSignature) -> ExecutionPlan:
+        steps = self._build_steps(signature)
+        consistency_unit = self._select_consistency_unit(signature)
+        write_mode = self._select_write_mode(signature)
+
+        return ExecutionPlan(
+            signature_hash=signature.signature_hash,
+            intent_id=signature.intent_id,
+            steps=steps,
+            consistency_unit=consistency_unit,
+            write_mode=write_mode,
+            metadata={
+                "domain": signature.domain,
+                "target_layer": signature.target_layer.value,
+            },
+        )
+
+    def _build_steps(self, sig: StateSignature) -> list[ExecutionStep]:
+        steps: list[ExecutionStep] = []
+        extract_ids: list[str] = []
+        post_extract_ids: list[str] = []
+
+        # Step 1: Extract each dataset (with partition filter if required)
+        for _i, ds in enumerate(sig.datasets):
+            step_id = f"extract_{ds.name}"
+            config: dict[str, Any] = {}
+
+            if sig.constraints.partition_by:
+                config["filter"] = {
+                    "column": sig.constraints.partition_by[0],
+                    "predicate": ">=",
+                    "required_by": "FINOPS",
+                }
+
+            steps.append(ExecutionStep(
+                id=step_id,
+                step_type=StepType.EXTRACT,
+                source=ds.name,
+                config=config,
+            ))
+            extract_ids.append(step_id)
+
+        # Step 2: Anonymize datasets with PII
+        for ds in sig.datasets:
+            if ds.contains_pii and sig.constraints.no_pii_raw:
+                anon_id = f"anonymize_{ds.name}"
+                depends = (f"extract_{ds.name}",)
+                steps.append(ExecutionStep(
+                    id=anon_id,
+                    step_type=StepType.ANONYMIZE,
+                    source=ds.name,
+                    config={
+                        "pii_columns": list(ds.pii_columns),
+                        "strategy": "sha256",
+                    },
+                    depends_on=depends,
+                ))
+                post_extract_ids.append(anon_id)
+            else:
+                post_extract_ids.append(f"extract_{ds.name}")
+
+        # Step 3: Join if multiple datasets and intent is reconciliation
+        if len(sig.datasets) > 1 and "reconcil" in sig.intent:
+            join_id = "join_datasets"
+            merge_keys = list(sig.datasets[0].merge_keys) if sig.datasets[0].merge_keys else ["id"]
+            steps.append(ExecutionStep(
+                id=join_id,
+                step_type=StepType.JOIN,
+                config={
+                    "type": "broadcast" if self._needs_broadcast(sig) else "sort_merge",
+                    "datasets": [d.name for d in sig.datasets],
+                    "merge_keys": merge_keys,
+                },
+                depends_on=tuple(post_extract_ids),
+            ))
+            load_depends = (join_id,)
+        elif len(post_extract_ids) == 1:
+            load_depends = (post_extract_ids[0],)
+        else:
+            load_depends = tuple(post_extract_ids)
+
+        # Step 4: Aggregate if intent calls for it
+        if "aggregate" in sig.intent:
+            agg_id = "aggregate"
+            steps.append(ExecutionStep(
+                id=agg_id,
+                step_type=StepType.AGGREGATE,
+                config={"group_by": list(sig.constraints.partition_by)},
+                depends_on=load_depends,
+            ))
+            load_depends = (agg_id,)
+
+        # Step 5: Load to target
+        target_name = self._derive_target_name(sig)
+        load_config: dict[str, Any] = {
+            "write_mode": self._select_write_mode(sig).value,
+        }
+        if sig.constraints.merge_key_required:
+            load_config["merge_key"] = True
+            load_config["merge_keys"] = list(sig.datasets[0].merge_keys) if sig.datasets and sig.datasets[0].merge_keys else ["id"]
+        if sig.constraints.partition_by:
+            load_config["partition_by"] = list(sig.constraints.partition_by)
+
+        steps.append(ExecutionStep(
+            id="load_target",
+            step_type=StepType.LOAD,
+            target=target_name,
+            config=load_config,
+            depends_on=load_depends,
+        ))
+
+        return steps
+
+    def _needs_broadcast(self, sig: StateSignature) -> bool:
+        """Use broadcast join when one dataset is much smaller than the other."""
+        if len(sig.datasets) != 2:
+            return False
+        sizes = sorted(d.size_gb for d in sig.datasets)
+        return sizes[0] < 1.0 and sizes[1] > 100.0
+
+    def _select_consistency_unit(self, sig: StateSignature) -> ConsistencyUnit:
+        """Per whitepaper: selection based on execution context."""
+        if sig.constraints.partition_by:
+            return ConsistencyUnit.PARTITION
+        if len(sig.datasets) > 2:
+            return ConsistencyUnit.DAG_BRANCH
+        return ConsistencyUnit.DATASET
+
+    def _select_write_mode(self, sig: StateSignature) -> WriteMode:
+        """Per whitepaper: append only in Bronze, merge in Silver/Gold."""
+        if sig.target_layer == TargetLayer.BRONZE:
+            return WriteMode.OVERWRITE_PARTITION if sig.constraints.partition_by else WriteMode.APPEND
+        return WriteMode.MERGE
+
+    def _derive_target_name(self, sig: StateSignature) -> str:
+        return sig.target_dataset_name

cfa/execution/__init__.py

@@ -0,0 +1,12 @@
+"""CFA Execution — partial execution and state projection."""
+from cfa._lazy import LazyLoader
+
+__getattr__ = LazyLoader({
+    "PartialExecutionManager": ("cfa.execution.partial", "PartialExecutionManager"),
+    "PartialExecutionState": ("cfa.execution.partial", "PartialExecutionState"),
+    "PublishState": ("cfa.execution.partial", "PublishState"),
+    "FailurePolicy": ("cfa.execution.partial", "FailurePolicy"),
+    "RetryPolicy": ("cfa.execution.partial", "RetryPolicy"),
+    "StateProjectionProtocol": ("cfa.execution.state_projection", "StateProjectionProtocol"),
+    "ProjectionResult": ("cfa.execution.state_projection", "ProjectionResult"),
+})

cfa/execution/partial.py

@@ -0,0 +1,339 @@
+"""
+CFA Partial Execution State
+============================
+Manages partial failures, retry policies, and publish semantics.
+
+When a plan partially fails, CFA does NOT silently succeed or blindly fail.
+Instead, it applies a FailurePolicy to determine next action:
+- FULL_ROLLBACK: discard everything, mark as rolled_back
+- SELECTIVE_QUARANTINE: quarantine failed consistency units, commit the rest
+- PARTIAL_COMMIT_NO_PUBLISH: commit all succeeded, but do not publish
+- DEGRADED_PUBLISH: commit and publish with degradation flag
+
+Retry policy: max 3 attempts, failed consistency units only.
+Publish semantics: committed_not_published -> published | degraded.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+from typing import Any
+
+from cfa.core.codegen import GeneratedCode
+from cfa.core.planner import ExecutionPlan
+from cfa.sandbox import (
+    ExecutionMetrics,
+    SandboxOutcome,
+    SandboxResult,
+    StepOutcome,
+)
+from cfa.sandbox.executor import SandboxExecutor
+from cfa.types import (
+    Fault,
+    FaultFamily,
+    FaultSeverity,
+    PolicyAction,
+    StateSignature,
+)
+from cfa.validation.runtime import RuntimeValidationResult, RuntimeValidator
+
+# ── Enums ───────────────────────────────────────────────────────────────────
+
+
+class FailurePolicy(StrEnum):
+    FULL_ROLLBACK = "full_rollback"
+    SELECTIVE_QUARANTINE = "selective_quarantine"
+    PARTIAL_COMMIT_NO_PUBLISH = "partial_commit_no_publish"
+    DEGRADED_PUBLISH = "degraded_publish"
+
+
+class PublishState(StrEnum):
+    NOT_STARTED = "not_started"
+    COMMITTED_NOT_PUBLISHED = "committed_not_published"
+    PUBLISHED = "published"
+    DEGRADED = "degraded"
+    ROLLED_BACK = "rolled_back"
+    QUARANTINED = "quarantined"
+
+
+# ── Retry Policy ────────────────────────────────────────────────────────────
+
+
+@dataclass(frozen=True)
+class RetryPolicy:
+    """Controls retry behavior for failed steps."""
+
+    max_attempts: int = 3
+    retry_failed_only: bool = True  # only retry failed consistency units
+
+
+# ── Execution State ─────────────────────────────────────────────────────────
+
+
+@dataclass
+class PartialExecutionState:
+    """
+    Tracks the state of a partially executed plan.
+    Supports retry, quarantine, and publish semantics.
+    """
+
+    plan_signature_hash: str
+    publish_state: PublishState = PublishState.NOT_STARTED
+    sandbox_result: SandboxResult | None = None
+    runtime_validation: RuntimeValidationResult | None = None
+    retry_count: int = 0
+    quarantined_steps: list[str] = field(default_factory=list)
+    committed_steps: list[str] = field(default_factory=list)
+    faults: list[Fault] = field(default_factory=list)
+    failure_policy_applied: FailurePolicy | None = None
+
+    @property
+    def is_fully_committed(self) -> bool:
+        return (
+            self.sandbox_result is not None
+            and self.sandbox_result.all_succeeded
+            and self.publish_state in (PublishState.COMMITTED_NOT_PUBLISHED, PublishState.PUBLISHED)
+        )
+
+    @property
+    def has_quarantined(self) -> bool:
+        return len(self.quarantined_steps) > 0
+
+
+# ── Partial Execution Manager ───────────────────────────────────────────────
+
+
+class PartialExecutionManager:
+    """
+    Orchestrates sandbox execution with failure policy, retry, and publish semantics.
+
+    Flow:
+    1. Execute plan in sandbox
+    2. Validate runtime metrics
+    3. On partial failure, apply failure policy
+    4. Retry if policy allows
+    5. Determine publish state
+    """
+
+    def __init__(
+        self,
+        sandbox: SandboxExecutor,
+        runtime_validator: RuntimeValidator | None = None,
+        failure_policy: FailurePolicy = FailurePolicy.SELECTIVE_QUARANTINE,
+        retry_policy: RetryPolicy | None = None,
+    ) -> None:
+        self.sandbox = sandbox
+        self.runtime_validator = runtime_validator or RuntimeValidator()
+        self.failure_policy = failure_policy
+        self.retry_policy = retry_policy or RetryPolicy()
+
+    def execute(
+        self,
+        plan: ExecutionPlan,
+        code: GeneratedCode,
+        signature: StateSignature,
+        schema_contract: dict[str, Any] | None = None,
+    ) -> PartialExecutionState:
+        state = PartialExecutionState(plan_signature_hash=plan.signature_hash)
+
+        # ── Execute in sandbox ──────────────────────────────────────────
+        sandbox_result = self.sandbox.execute(plan, code, signature)
+        state.sandbox_result = sandbox_result
+        state.faults.extend(sandbox_result.faults)
+
+        # ── Handle panic (environmental fault) ──────────────────────────
+        if sandbox_result.outcome == SandboxOutcome.PANIC:
+            state.publish_state = PublishState.ROLLED_BACK
+            state.failure_policy_applied = FailurePolicy.FULL_ROLLBACK
+            return state
+
+        # ── Runtime validation ──────────────────────────────────────────
+        rv_result = self.runtime_validator.validate(sandbox_result, signature, schema_contract)
+        state.runtime_validation = rv_result
+        state.faults.extend(rv_result.faults)
+
+        # ── All succeeded + validation passed ───────────────────────────
+        if sandbox_result.outcome == SandboxOutcome.COMPLETED and rv_result.passed:
+            state.publish_state = PublishState.PUBLISHED
+            state.committed_steps = [r.step_id for r in sandbox_result.step_results]
+            return state
+
+        # ── Runtime validation failed on complete execution ─────────────
+        if sandbox_result.outcome == SandboxOutcome.COMPLETED and not rv_result.passed:
+            return self._apply_failure_policy_for_validation(state)
+
+        # ── Partial failure: some steps failed ──────────────────────────
+        if sandbox_result.outcome in (SandboxOutcome.PARTIAL, SandboxOutcome.FAILED):
+            retried = self._retry_failed_steps(state, plan, code, signature, schema_contract)
+            if retried is not None:
+                return retried
+            return self._apply_failure_policy(state)
+
+        return state
+
+    def _retry_failed_steps(
+        self,
+        state: PartialExecutionState,
+        plan: ExecutionPlan,
+        code: GeneratedCode,
+        signature: StateSignature,
+        schema_contract: dict[str, Any] | None,
+    ) -> PartialExecutionState | None:
+        """Retry failed consistency units before terminal policy application."""
+        assert state.sandbox_result is not None
+        failed_step_ids = [r.step_id for r in state.sandbox_result.failed_steps]
+        if not failed_step_ids or self.retry_policy.max_attempts <= 1:
+            return None
+
+        latest_result = state.sandbox_result
+        retryable_ids = list(failed_step_ids)
+
+        for attempt in range(1, self.retry_policy.max_attempts):
+            retry_result = self.sandbox.execute(plan, code, signature, step_ids=retryable_ids)
+            state.retry_count = attempt
+            state.faults.extend(retry_result.faults)
+
+            latest_result = self._merge_retry_result(latest_result, retry_result)
+            remaining_failed = [r.step_id for r in latest_result.failed_steps]
+
+            if not remaining_failed:
+                state.sandbox_result = latest_result
+                rv_result = self.runtime_validator.validate(latest_result, signature, schema_contract)
+                state.runtime_validation = rv_result
+                state.faults.extend(rv_result.faults)
+                if rv_result.passed:
+                    state.publish_state = PublishState.PUBLISHED
+                    state.committed_steps = [r.step_id for r in latest_result.step_results]
+                    state.quarantined_steps = []
+                    return state
+                return self._apply_failure_policy_for_validation(state)
+
+            if not self.retry_policy.retry_failed_only:
+                retryable_ids = [s.id for s in plan.execution_order()]
+            else:
+                retryable_ids = remaining_failed
+
+        state.sandbox_result = latest_result
+        return None
+
+    def _merge_retry_result(
+        self,
+        base: SandboxResult,
+        retry: SandboxResult,
+    ) -> SandboxResult:
+        """Merge retry outcomes over the original attempt, keeping latest result per retried step."""
+        replacement = {r.step_id: r for r in retry.step_results}
+        merged_steps = [replacement.get(step.step_id, step) for step in base.step_results]
+
+        aggregate = ExecutionMetrics()
+        all_faults: list[Fault] = []
+        for step in merged_steps:
+            if step.faults:
+                all_faults.extend(step.faults)
+            if step.outcome == StepOutcome.SUCCESS:
+                aggregate.rows_output = step.metrics.rows_output
+                aggregate.shuffle_bytes += step.metrics.shuffle_bytes
+                aggregate.duration_seconds += step.metrics.duration_seconds
+                aggregate.cost_dbu += step.metrics.cost_dbu
+                aggregate.output_schema = step.metrics.output_schema
+                for col, cnt in step.metrics.null_counts.items():
+                    aggregate.null_counts[col] = aggregate.null_counts.get(col, 0) + cnt
+
+        failed = [r for r in merged_steps if r.outcome == StepOutcome.FAILED]
+        if not failed:
+            outcome = SandboxOutcome.COMPLETED
+        elif len(failed) < len(merged_steps):
+            outcome = SandboxOutcome.PARTIAL
+        else:
+            outcome = SandboxOutcome.FAILED
+
+        return SandboxResult(
+            outcome=outcome,
+            step_results=merged_steps,
+            aggregate_metrics=aggregate,
+            faults=all_faults,
+            panic_reason=retry.panic_reason or base.panic_reason,
+        )
+
+    def _apply_failure_policy(self, state: PartialExecutionState) -> PartialExecutionState:
+        """Apply failure policy when sandbox has partial/full failure."""
+        assert state.sandbox_result is not None
+
+        state.failure_policy_applied = self.failure_policy
+        succeeded = state.sandbox_result.successful_steps
+        failed = state.sandbox_result.failed_steps
+
+        match self.failure_policy:
+            case FailurePolicy.FULL_ROLLBACK:
+                state.publish_state = PublishState.ROLLED_BACK
+                state.quarantined_steps = [r.step_id for r in state.sandbox_result.step_results]
+
+            case FailurePolicy.SELECTIVE_QUARANTINE:
+                state.committed_steps = [r.step_id for r in succeeded]
+                state.quarantined_steps = [r.step_id for r in failed]
+                if succeeded:
+                    state.publish_state = PublishState.QUARANTINED
+                else:
+                    state.publish_state = PublishState.ROLLED_BACK
+
+            case FailurePolicy.PARTIAL_COMMIT_NO_PUBLISH:
+                state.committed_steps = [r.step_id for r in succeeded]
+                state.quarantined_steps = [r.step_id for r in failed]
+                state.publish_state = PublishState.COMMITTED_NOT_PUBLISHED
+
+            case FailurePolicy.DEGRADED_PUBLISH:
+                state.committed_steps = [r.step_id for r in succeeded]
+                state.quarantined_steps = [r.step_id for r in failed]
+                if succeeded:
+                    state.publish_state = PublishState.DEGRADED
+                    state.faults.append(Fault(
+                        code="PARTIAL_DEGRADED_PUBLISH",
+                        family=FaultFamily.RUNTIME,
+                        severity=FaultSeverity.WARNING,
+                        stage="partial_execution",
+                        message=(
+                            f"Degraded publish: {len(failed)} of "
+                            f"{len(state.sandbox_result.step_results)} steps failed."
+                        ),
+                        mandatory_action=PolicyAction.APPROVE,
+                        detected_before_execution=False,
+                    ))
+                else:
+                    state.publish_state = PublishState.ROLLED_BACK
+
+        return state
+
+    def _apply_failure_policy_for_validation(
+        self, state: PartialExecutionState
+    ) -> PartialExecutionState:
+        """Apply failure policy when runtime validation fails on complete execution."""
+        state.failure_policy_applied = self.failure_policy
+
+        match self.failure_policy:
+            case FailurePolicy.FULL_ROLLBACK:
+                state.publish_state = PublishState.ROLLED_BACK
+
+            case FailurePolicy.SELECTIVE_QUARANTINE:
+                # All steps succeeded but validation failed — quarantine the whole batch
+                state.publish_state = PublishState.QUARANTINED
+                if state.sandbox_result:
+                    state.quarantined_steps = [
+                        r.step_id for r in state.sandbox_result.step_results
+                    ]
+
+            case FailurePolicy.PARTIAL_COMMIT_NO_PUBLISH:
+                state.publish_state = PublishState.COMMITTED_NOT_PUBLISHED
+                if state.sandbox_result:
+                    state.committed_steps = [
+                        r.step_id for r in state.sandbox_result.step_results
+                    ]
+
+            case FailurePolicy.DEGRADED_PUBLISH:
+                state.publish_state = PublishState.DEGRADED
+                if state.sandbox_result:
+                    state.committed_steps = [
+                        r.step_id for r in state.sandbox_result.step_results
+                    ]
+
+        return state