cfa-kernel 0.1.0__py3-none-any.whl

cfa/runtime/gate.py

@@ -0,0 +1,287 @@
+"""
+CFA Runtime Gate
+================
+Production-grade governance gate for wrapping pipeline execution.
+
+Two surfaces, one core:
+- cfa.testing → pytest-native for CI/CD
+- cfa.runtime → production decorator/context-manager for live pipelines
+
+Usage:
+    from cfa.runtime import RuntimeGate, GateConfig
+
+    gate = RuntimeGate(
+        config=GateConfig(policy_bundle="prod_v4.2"),
+        catalog=PROD_CATALOG,
+    )
+
+    # Pre-execution validation
+    gate.validate("agregar vendas mensais com PII anonimizado")
+
+    # Scoped execution with metrics
+    with gate.scope("monthly_aggregation"):
+        df = run_pipeline()
+        gate.record_metrics(rows=1000000, shuffle_mb=450, cost_dbu=12.0)
+
+    # Decorator for simple functions
+    @gate.guard("agregar vendas")
+    def my_pipeline():
+        ...
+"""
+
+from __future__ import annotations
+
+import uuid
+from collections.abc import Callable
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from typing import Any
+
+from cfa.audit.context import ContextRegistry
+from cfa.audit.trail import AuditTrail
+from cfa.core.kernel import KernelConfig, KernelOrchestrator
+from cfa.policy.engine import PolicyRule
+from cfa.types import DecisionState
+
+
+class GateViolation(str):
+    """Policy for handling governance violations at runtime."""
+
+    BLOCK = "block"          # Raise exception, stop execution
+    WARN = "warn"            # Log warning, continue execution
+    AUDIT_ONLY = "audit_only"  # Record in audit trail, always continue
+
+
+@dataclass
+class GateConfig:
+    """Production configuration for the Runtime Gate."""
+
+    policy_bundle: str = "prod_v1.0"
+    on_violation: str = GateViolation.BLOCK
+    backend: str = "pyspark"
+    sandbox: str = ""             # name in SandboxRegistry, empty = use kernel default
+    execute: bool = False         # run sandbox execution phase (Phase 4)
+    max_replan_attempts: int = 3
+    warnings_are_blocking: bool = True
+    enable_planning: bool = True
+    enable_codegen: bool = True
+    enable_static_validation: bool = True
+    enable_sandbox: bool = False  # disabled by default — gate is pre-execution
+    enable_promotion: bool = True
+
+    def to_kernel_config(self) -> KernelConfig:
+        return KernelConfig(
+            policy_bundle_version=self.policy_bundle,
+            backend=self.backend,
+            max_replan_attempts=self.max_replan_attempts,
+            warnings_are_blocking=self.warnings_are_blocking,
+            enable_planning=self.enable_planning,
+            enable_codegen=self.enable_codegen,
+            enable_static_validation=self.enable_static_validation,
+            enable_sandbox=self.enable_sandbox,
+            enable_promotion=self.enable_promotion,
+        )
+
+
+class GovernanceViolation(Exception):
+    """Raised when the runtime gate blocks an intent."""
+
+    def __init__(self, gate_id: str, intent: str, reason: str, faults: list[str]) -> None:
+        self.gate_id = gate_id
+        self.intent = intent
+        self.reason = reason
+        self.faults = faults
+        super().__init__(
+            f"[gate={gate_id}] Governance violation for '{intent[:80]}': {reason}"
+        )
+
+
+@dataclass
+class GateResult:
+    """Result of a runtime gate validation."""
+
+    gate_id: str
+    intent: str
+    passed: bool
+    state: DecisionState
+    blocked_reason: str = ""
+    faults: list[str] = field(default_factory=list)
+    signature_hash: str = ""
+    replay_count: int = 0
+    execution_id: str = ""
+
+
+class RuntimeGate:
+    """Production governance gate for live pipelines.
+
+    Wraps KernelOrchestrator with production defaults, metrics recording,
+    and configurable violation handling.
+
+    Usage modes:
+    - validate(intent) → GateResult for pre-execution checks
+    - scope(name) → context manager for scoped execution
+    - guard(intent) → decorator for function wrapping
+    """
+
+    def __init__(
+        self,
+        config: GateConfig | None = None,
+        catalog: dict[str, Any] | None = None,
+        policy_rules: list[PolicyRule] | None = None,
+        context_registry: ContextRegistry | None = None,
+        audit_trail: AuditTrail | None = None,
+    ) -> None:
+        self.config = config or GateConfig()
+        self._gate_id = str(uuid.uuid4())[:8]
+
+        sandbox_backend = None
+        if self.config.sandbox:
+            from cfa.sandbox import SandboxRegistry
+            registry = SandboxRegistry.singleton()
+            sandbox_backend = registry.get(self.config.sandbox)()
+
+        self._kernel = KernelOrchestrator(
+            catalog=catalog,
+            config=self.config.to_kernel_config(),
+            policy_rules=policy_rules,
+            context_registry=context_registry,
+            audit_trail=audit_trail,
+            sandbox_backend=sandbox_backend,
+        )
+        self._last_result: GateResult | None = None
+
+    @property
+    def gate_id(self) -> str:
+        return self._gate_id
+
+    # ── Pre-execution validation ──────────────────────────────────────────
+
+    def validate(self, intent: str) -> GateResult:
+        """Validate an intent before execution. Does NOT execute code.
+
+        Raises GovernanceViolation if on_violation='block' and intent fails.
+        Always returns GateResult.
+        """
+        kresult = self._kernel.process(intent)
+        passed = kresult.is_executable
+
+        faults: list[str] = []
+        if kresult.policy_result:
+            faults = [f.code for f in kresult.policy_result.faults]
+
+        result = GateResult(
+            gate_id=self._gate_id,
+            intent=intent,
+            passed=passed,
+            state=kresult.state,
+            blocked_reason=kresult.blocked_reason,
+            faults=faults,
+            signature_hash=kresult.signature.signature_hash if kresult.signature else "",
+            replay_count=len(kresult.replan_history),
+            execution_id=kresult.intent_id,
+        )
+        self._last_result = result
+
+        if not passed and self.config.on_violation == GateViolation.BLOCK:
+            raise GovernanceViolation(
+                gate_id=self._gate_id,
+                intent=intent,
+                reason=kresult.blocked_reason,
+                faults=faults,
+            )
+
+        return result
+
+    # ── Scoped execution ──────────────────────────────────────────────────
+
+    @contextmanager
+    def scope(self, name: str) -> Any:
+        """Context manager for governed execution scope.
+
+        Usage:
+            with gate.scope("monthly_aggregation"):
+                df = process(...)
+                gate.record_metrics(rows=1000, shuffle_mb=5)
+        """
+        execution_id = str(uuid.uuid4())
+        try:
+            yield execution_id
+        except Exception:
+            self._kernel.audit_trail.record(
+                intent_id=execution_id,
+                stage="runtime_gate",
+                event_type="scope_error",
+                outcome="error",
+                scope_name=name,
+                policy_bundle_version=self.config.policy_bundle,
+            )
+            raise
+
+    def record_metrics(
+        self,
+        rows: int = 0,
+        shuffle_mb: float = 0.0,
+        cost_dbu: float = 0.0,
+        duration_seconds: float = 0.0,
+        **extra: Any,
+    ) -> None:
+        """Record execution metrics for the current scope."""
+        if self._last_result:
+            self._kernel.audit_trail.record(
+                intent_id=self._last_result.execution_id,
+                stage="runtime_gate",
+                event_type="execution_metrics",
+                outcome="recorded",
+                rows=rows,
+                shuffle_mb=shuffle_mb,
+                cost_dbu=cost_dbu,
+                duration_seconds=duration_seconds,
+                policy_bundle_version=self.config.policy_bundle,
+                **extra,
+            )
+
+    # ── Decorator ─────────────────────────────────────────────────────────
+
+    def guard(self, intent: str) -> Callable[[Callable], Callable]:
+        """Decorator that guards a function with governance validation.
+
+        Usage:
+            @gate.guard("agregar vendas")
+            def my_pipeline():
+                ...
+        """
+
+        def decorator(fn: Callable) -> Callable:
+            def wrapper(*args: Any, **kwargs: Any) -> Any:
+                self.validate(intent)
+                return fn(*args, **kwargs)
+
+            return wrapper
+
+        return decorator
+
+
+def runtime_gate(
+    intent: str,
+    *,
+    policy_bundle: str = "prod_v1.0",
+    on_violation: str = GateViolation.BLOCK,
+    catalog: dict[str, Any] | None = None,
+    **gate_kwargs: Any,
+) -> Callable[[Callable], Callable]:
+    """Standalone decorator for quick runtime governance.
+
+    Usage:
+        @runtime_gate("agregar vendas", policy_bundle="prod_v2.0")
+        def my_pipeline():
+            ...
+
+    This creates a temporary RuntimeGate for the decorated function.
+    For multiple functions sharing the same gate, use RuntimeGate directly.
+    """
+    gate = RuntimeGate(
+        config=GateConfig(policy_bundle=policy_bundle, on_violation=on_violation),
+        catalog=catalog,
+        **gate_kwargs,
+    )
+    return gate.guard(intent)

cfa/sandbox/__init__.py

@@ -0,0 +1,189 @@
+"""
+CFA Sandbox Registry
+====================
+Pluggable execution backends for governed code.
+
+Each sandbox backend declares its capabilities (execution mode,
+rollback support, metrics accuracy) via ``SandboxCapabilities``.
+
+The registry follows the same pattern as ``BackendRegistry``.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from cfa.core.codegen import GeneratedCode  # noqa: F401
+from cfa.core.planner import ExecutionStep
+from cfa.types import Fault
+
+# ── Execution Metrics ────────────────────────────────────────────────────────
+
+
+@dataclass
+class ExecutionMetrics:
+    rows_output: int = 0
+    shuffle_bytes: int = 0
+    duration_seconds: float = 0.0
+    cost_dbu: float = 0.0
+    null_counts: dict[str, int] = field(default_factory=dict)
+    output_schema: list[str] = field(default_factory=list)
+    peak_memory_mb: float = 0.0
+
+    @property
+    def shuffle_mb(self) -> float:
+        return self.shuffle_bytes / (1024 * 1024)
+
+    def null_ratio(self, column: str, total_rows: int | None = None) -> float:
+        rows = total_rows or self.rows_output
+        if rows == 0:
+            return 0.0
+        return self.null_counts.get(column, 0) / rows
+
+
+# ── Step Result ──────────────────────────────────────────────────────────────
+
+
+class StepOutcome(str):
+    SUCCESS = "success"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+    INTERRUPTED = "interrupted"
+
+
+@dataclass
+class StepResult:
+    step_id: str
+    outcome: str  # StepOutcome value
+    metrics: ExecutionMetrics = field(default_factory=ExecutionMetrics)
+    error: str = ""
+    faults: list[Fault] = field(default_factory=list)
+    retry_count: int = 0
+
+
+# ── Sandbox Result ───────────────────────────────────────────────────────────
+
+
+class SandboxOutcome(str):
+    COMPLETED = "completed"
+    PARTIAL = "partial"
+    FAILED = "failed"
+    PANIC = "panic"
+
+
+@dataclass
+class SandboxResult:
+    outcome: str  # SandboxOutcome value
+    step_results: list[StepResult] = field(default_factory=list)
+    aggregate_metrics: ExecutionMetrics = field(default_factory=ExecutionMetrics)
+    faults: list[Fault] = field(default_factory=list)
+    panic_reason: str = ""
+
+    @property
+    def successful_steps(self) -> list[StepResult]:
+        return [r for r in self.step_results if r.outcome == StepOutcome.SUCCESS]
+
+    @property
+    def failed_steps(self) -> list[StepResult]:
+        return [r for r in self.step_results if r.outcome == StepOutcome.FAILED]
+
+    @property
+    def all_succeeded(self) -> bool:
+        return all(r.outcome == StepOutcome.SUCCESS for r in self.step_results)
+
+
+# ── Capabilities ─────────────────────────────────────────────────────────────
+
+
+@dataclass
+class SandboxCapabilities:
+    backend_name: str = ""
+    backend_version: str = ""
+
+    execution_mode: str = "simulation"  # "simulation" | "local" | "cluster"
+    supports_rollback: bool = False
+    supports_metrics: bool = True
+    supports_environment_check: bool = False
+    max_parallel_steps: int = 1
+
+    custom: dict[str, Any] = field(default_factory=dict)
+
+
+# ── Sandbox Backend ──────────────────────────────────────────────────────────
+
+
+class SandboxBackend(ABC):
+    """Extension point: pluggable execution backend."""
+
+    @abstractmethod
+    def execute_step(
+        self, step: ExecutionStep, code: str, context: dict[str, Any]
+    ) -> StepResult: ...
+
+    @abstractmethod
+    def check_environment(self) -> list[Fault]: ...
+
+    def get_capabilities(self) -> SandboxCapabilities:
+        return SandboxCapabilities()
+
+
+# ── Sandbox Registry ─────────────────────────────────────────────────────────
+
+
+SandboxFactory = Any  # Callable[[], SandboxBackend]
+
+
+class SandboxRegistry:
+    """Global registry of available sandbox backend factories."""
+
+    _instance: SandboxRegistry | None = None
+    _lock: Any = None
+
+    def __init__(self) -> None:
+        self._backends: dict[str, SandboxFactory] = {}
+
+    @classmethod
+    def singleton(cls) -> SandboxRegistry:
+        if cls._lock is None:
+            import threading
+            cls._lock = threading.Lock()
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = cls()
+                cls._instance._bootstrap()
+            return cls._instance
+
+    def _bootstrap(self) -> None:
+        if self._backends:
+            return
+        from .mock import MockSandboxBackend  # noqa: F811
+        from .panic import PanicSandboxBackend  # noqa: F811
+
+        self.register("mock", lambda: MockSandboxBackend())
+        self.register("panic", lambda: PanicSandboxBackend(panic_on_step="extract_nfe"))
+
+    def register(self, name: str, factory: SandboxFactory) -> None:
+        self._backends[name] = factory
+
+    def get(self, name: str) -> SandboxFactory:
+        if name not in self._backends:
+            available = ", ".join(sorted(self._backends))
+            raise KeyError(f"Unknown sandbox '{name}'. Registered: {available or '(none)'}")
+        return self._backends[name]
+
+    def list(self) -> list[str]:
+        return sorted(self._backends)
+
+    def __contains__(self, name: str) -> bool:
+        return name in self._backends
+
+
+# ── Backward-compatible re-exports ──────────────────────────────────────────
+
+__all__ = ["SandboxExecutor", "MockSandboxBackend", "PanicSandboxBackend"]
+
+from .executor import SandboxExecutor  # noqa: E402
+from .mock import MockSandboxBackend  # noqa: E402
+from .panic import PanicSandboxBackend  # noqa: E402

cfa/sandbox/executor.py

@@ -0,0 +1,92 @@
+"""Sandbox executor — orchestrates plan execution through a sandbox backend."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from cfa.core.codegen import GeneratedCode
+from cfa.core.planner import ExecutionPlan
+from cfa.types import StateSignature
+
+from . import (
+    ExecutionMetrics,
+    SandboxBackend,
+    SandboxOutcome,
+    SandboxResult,
+    StepOutcome,
+    StepResult,
+)
+
+
+class SandboxExecutor:
+    """Orchestrates isolated execution of a plan in a sandbox."""
+
+    def __init__(self, backend: SandboxBackend) -> None:
+        self.backend = backend
+
+    def execute(
+        self,
+        plan: ExecutionPlan,
+        code: GeneratedCode,
+        signature: StateSignature,
+        schema_contract: dict[str, Any] | None = None,
+        step_ids: list[str] | None = None,
+    ) -> SandboxResult:
+        step_results: list[StepResult] = []
+        faults: list = []
+        aggregate = None
+
+        context: dict[str, Any] = {"steps": [s.id for s in plan.steps]}
+        target_ids = set(step_ids) if step_ids else None
+
+        for step in plan.execution_order():
+            if target_ids is not None and step.id not in target_ids:
+                continue
+            env_faults = self.backend.check_environment()
+            if env_faults:
+                return SandboxResult(
+                    outcome=SandboxOutcome.PANIC,
+                    step_results=step_results,
+                    faults=env_faults,
+                    panic_reason="environment_check_failed",
+                )
+
+            step_code = code.step_code_map.get(step.id, code.code)
+            result = self.backend.execute_step(step, step_code, context)
+
+            if result.outcome == StepOutcome.INTERRUPTED:
+                return SandboxResult(
+                    outcome=SandboxOutcome.PANIC,
+                    step_results=step_results + [result],
+                    faults=result.faults,
+                    panic_reason=result.error,
+                )
+
+            step_results.append(result)
+            faults.extend(result.faults)
+
+            if result.outcome == StepOutcome.FAILED:
+                return SandboxResult(
+                    outcome=SandboxOutcome.PARTIAL,
+                    step_results=step_results,
+                    faults=faults,
+                )
+
+        total_rows = sum(r.metrics.rows_output for r in step_results)
+        total_shuffle = sum(r.metrics.shuffle_bytes for r in step_results)
+        total_duration = sum(r.metrics.duration_seconds for r in step_results)
+        total_cost = sum(r.metrics.cost_dbu for r in step_results)
+
+        aggregate = ExecutionMetrics(
+            rows_output=total_rows,
+            shuffle_bytes=total_shuffle,
+            duration_seconds=total_duration,
+            cost_dbu=total_cost,
+        )
+
+        return SandboxResult(
+            outcome=SandboxOutcome.COMPLETED,
+            step_results=step_results,
+            aggregate_metrics=aggregate,
+            faults=faults,
+        )

cfa/sandbox/mock.py

@@ -0,0 +1,89 @@
+"""Mock sandbox backend — deterministic simulation for testing."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from cfa.core.planner import ExecutionStep
+from cfa.types import Fault, FaultFamily, FaultSeverity, PolicyAction
+
+from . import (
+    ExecutionMetrics,
+    SandboxBackend,
+    SandboxCapabilities,
+    StepOutcome,
+    StepResult,
+)
+
+
+class MockSandboxBackend(SandboxBackend):
+    """Deterministic sandbox for testing and local simulation.
+
+    Configurable outcomes allow testing failure scenarios without
+    a real execution environment.
+    """
+
+    def __init__(
+        self,
+        default_rows: int = 100_000,
+        default_shuffle_mb: float = 50.0,
+        default_cost_dbu: float = 5.0,
+        fail_steps: set[str] | None = None,
+        null_ratio: float = 0.01,
+        output_schema: list[str] | None = None,
+    ) -> None:
+        self.default_rows = default_rows
+        self.default_shuffle_mb = default_shuffle_mb
+        self.default_cost_dbu = default_cost_dbu
+        self.fail_steps = fail_steps or set()
+        self.null_ratio = null_ratio
+        self.output_schema = output_schema or ["nfe_id", "cpf_hash", "processing_date"]
+
+    def get_capabilities(self) -> SandboxCapabilities:
+        return SandboxCapabilities(
+            backend_name="mock",
+            backend_version="sim-1.0",
+            execution_mode="simulation",
+            supports_rollback=False,
+            supports_metrics=True,
+            supports_environment_check=False,
+        )
+
+    def execute_step(
+        self, step: ExecutionStep, code: str, context: dict[str, Any]
+    ) -> StepResult:
+        if step.id in self.fail_steps:
+            return StepResult(
+                step_id=step.id,
+                outcome=StepOutcome.FAILED,
+                error=f"Simulated failure on step {step.id}",
+                faults=[
+                    Fault(
+                        code="RUNTIME_STEP_FAILED",
+                        family=FaultFamily.RUNTIME,
+                        severity=FaultSeverity.HIGH,
+                        stage="sandbox",
+                        message=f"Step {step.id} failed during execution.",
+                        mandatory_action=PolicyAction.BLOCK,
+                        detected_before_execution=False,
+                    )
+                ],
+            )
+
+        null_counts = {col: int(self.default_rows * self.null_ratio) for col in self.output_schema}
+
+        return StepResult(
+            step_id=step.id,
+            outcome=StepOutcome.SUCCESS,
+            metrics=ExecutionMetrics(
+                rows_output=self.default_rows,
+                shuffle_bytes=int(self.default_shuffle_mb * 1024 * 1024),
+                duration_seconds=2.5,
+                cost_dbu=self.default_cost_dbu / max(len(context.get("steps", [1])), 1),
+                null_counts=null_counts,
+                output_schema=list(self.output_schema),
+            ),
+        )
+
+    def check_environment(self) -> list[Fault]:
+        return []