cfa-kernel 0.1.0__py3-none-any.whl

cfa/core/codegen.py

@@ -0,0 +1,65 @@
+"""
+CFA Code Generator
+==================
+Generates deterministic, governed code from an ExecutionPlan.
+
+The code generator is NOT creative — it fills templates governed by the plan.
+
+Key properties:
+- Output is deterministic (same plan = same code)
+- All PII handling is explicit (sha256/drop)
+- Partition filters are always present when required
+- Write operations use merge (never raw append in Silver/Gold)
+
+Backend-specific implementations live in cfa.backends.*.
+This module provides the core ABC, the GeneratedCode artifact,
+and backward-compatible re-exports.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+from cfa.core.planner import ExecutionPlan
+
+# ── Generated Code ───────────────────────────────────────────────────────────
+
+
+@dataclass
+class GeneratedCode:
+    """Complete code artifact generated from an execution plan."""
+
+    plan_signature_hash: str
+    intent_id: str
+    language: str
+    code: str
+    step_code_map: dict[str, str] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def line_count(self) -> int:
+        return len(self.code.strip().splitlines())
+
+
+# ── Code Generator Backend ───────────────────────────────────────────────────
+
+
+class CodeGenBackend(ABC):
+    """Extension point: different code generation targets."""
+
+    @abstractmethod
+    def generate(self, plan: ExecutionPlan) -> GeneratedCode: ...
+
+
+# ── Backward-compatible re-exports ───────────────────────────────────────────
+# PySparkGenerator now lives in cfa.backends.pyspark as PySparkBackend.
+# Lazy import to avoid circular dependency with backends.__init__.
+
+
+def __getattr__(name):
+    if name == "PySparkGenerator":
+        from cfa.backends.pyspark import PySparkBackend
+        return PySparkBackend
+    raise AttributeError(f"module 'cfa.core.codegen' has no attribute {name!r}")

cfa/core/conditions.py

@@ -0,0 +1,129 @@
+"""
+CFA Condition Registry
+======================
+Central registry mapping condition strings to callable checks.
+
+Used by:
+- PolicyBundle YAML/JSON loader (maps "pii_in_protected_layer" → lambda)
+- BehaviorSpec Systematizer (maps ConditionType → lambda)
+- Programmatic PolicyRule creation
+
+Single source of truth for all built-in governance conditions.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from cfa.types import DatasetClassification, StateSignature
+
+ConditionFn = Callable[[StateSignature], bool]
+
+
+def _pii_in_protected_layer(meta: dict[str, Any]) -> ConditionFn:
+    target_layer = meta.get("target_layer", "")
+    def check(sig: StateSignature) -> bool:
+        protected = (target_layer in ("silver", "gold")) or sig.writes_to_protected_layer
+        return protected and sig.contains_pii and not sig.constraints.no_pii_raw
+    return check
+
+
+def _missing_merge_key(meta: dict[str, Any]) -> ConditionFn:
+    def check(sig: StateSignature) -> bool:
+        return sig.writes_to_protected_layer and not sig.constraints.merge_key_required
+    return check
+
+
+def _missing_partition(meta: dict[str, Any]) -> ConditionFn:
+    min_size_gb = meta.get("min_size_gb", 1.0)
+    def check(sig: StateSignature) -> bool:
+        has_large = any(
+            d.classification in (DatasetClassification.HIGH_VOLUME, DatasetClassification.SENSITIVE)
+            or d.size_gb >= min_size_gb
+            for d in sig.datasets
+        )
+        return has_large and len(sig.constraints.partition_by) == 0
+    return check
+
+
+def _enforce_types_disabled(meta: dict[str, Any]) -> ConditionFn:
+    def check(sig: StateSignature) -> bool:
+        return sig.writes_to_protected_layer and not sig.constraints.enforce_types
+    return check
+
+
+def _pii_without_policy(meta: dict[str, Any]) -> ConditionFn:
+    def check(sig: StateSignature) -> bool:
+        return sig.contains_pii and not sig.constraints.no_pii_raw
+    return check
+
+
+def _sensitive_without_partition(meta: dict[str, Any]) -> ConditionFn:
+    def check(sig: StateSignature) -> bool:
+        return (
+            any(d.classification == DatasetClassification.SENSITIVE for d in sig.datasets)
+            and len(sig.constraints.partition_by) == 0
+        )
+    return check
+
+
+def _cost_budget_exceeded(meta: dict[str, Any]) -> ConditionFn:
+    max_dbu = meta.get("max_dbu", 0.0)
+    def check(sig: StateSignature) -> bool:
+        if max_dbu > 0 and sig.constraints.max_cost_dbu is not None:
+            return sig.constraints.max_cost_dbu > max_dbu
+        return sig.constraints.max_cost_dbu is not None and sig.constraints.max_cost_dbu <= 0
+    return check
+
+
+CONDITION_REGISTRY: dict[str, Callable[[dict[str, Any]], ConditionFn]] = {
+    "pii_in_protected_layer": _pii_in_protected_layer,
+    "missing_merge_key": _missing_merge_key,
+    "missing_partition": _missing_partition,
+    "enforce_types_disabled": _enforce_types_disabled,
+    "pii_without_policy": _pii_without_policy,
+    "sensitive_without_partition": _sensitive_without_partition,
+    "cost_budget_exceeded": _cost_budget_exceeded,
+    "schema_mismatch": _missing_partition,   # alias: schema mismatch = check partition/schema
+    "shuffle_budget_exceeded": _missing_partition,  # alias
+    "unauthorized_gold_write": _pii_in_protected_layer,  # alias: gold = protected
+}
+
+
+def register_condition(name: str, builder: Callable[[dict[str, Any]], ConditionFn]) -> None:
+    """Register a custom condition builder for use in policy bundles.
+
+    Args:
+        name: Condition name used in YAML/JSON (e.g. "my_custom_check").
+        builder: Function that takes metadata dict and returns ConditionFn.
+    """
+    CONDITION_REGISTRY[name] = builder
+
+
+def build_condition(name: str, metadata: dict[str, Any] | None = None) -> ConditionFn:
+    """Build a condition function from a registered name and metadata.
+
+    Args:
+        name: Registered condition name (e.g. "pii_in_protected_layer").
+        metadata: Optional parameters for the condition builder.
+
+    Returns:
+        A callable that takes StateSignature and returns bool.
+
+    Raises:
+        KeyError: If the condition name is not registered.
+    """
+    meta = metadata or {}
+    builder = CONDITION_REGISTRY.get(name)
+    if builder is None:
+        raise KeyError(
+            f"Unknown condition '{name}'. "
+            f"Registered conditions: {', '.join(sorted(CONDITION_REGISTRY))}"
+        )
+    return builder(meta)
+
+
+def list_conditions() -> list[str]:
+    """Return all registered condition names."""
+    return sorted(CONDITION_REGISTRY)

cfa/core/kernel.py

@@ -0,0 +1,224 @@
+"""
+CFA Kernel Orchestrator
+=======================
+Single entry point for the CFA Kernel. Delegates the 5-phase pipeline
+to KernelPhases (core/phases/runner.py).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Any
+
+from cfa.audit.context import ContextRegistry
+from cfa.audit.trail import AuditTrail
+from cfa.backends import BackendRegistry
+from cfa.core.codegen import CodeGenBackend
+from cfa.core.planner import ExecutionPlanner
+from cfa.execution.partial import (
+    FailurePolicy,
+    PartialExecutionManager,
+    RetryPolicy,
+)
+from cfa.execution.state_projection import StateProjectionProtocol
+from cfa.normalizer.base import (
+    AutoApproveHandler,
+    ConfirmationHandler,
+    ConfirmationOrchestrator,
+    IntentNormalizer,
+    MockNormalizerBackend,
+    NormalizerBackend,
+    RuleBasedNormalizerBackend,
+)
+from cfa.observability.promotion import PromotionEngine
+from cfa.policy.engine import PolicyEngine, PolicyRule
+from cfa.sandbox import SandboxBackend
+from cfa.sandbox.executor import SandboxExecutor
+from cfa.sandbox.mock import MockSandboxBackend
+from cfa.types import KernelResult
+from cfa.validation.runtime import RuntimeValidator
+from cfa.validation.static import StaticValidator
+
+# ── Pipeline Phase ────────────────────────────────────────────────────────────
+
+
+class PipelinePhase(StrEnum):
+    FORMALIZE = "formalize"
+    GOVERN = "govern"
+    GENERATE = "generate"
+    EXECUTE = "execute"
+    VALIDATE = "validate"
+
+
+# ── Kernel Config ────────────────────────────────────────────────────────────
+
+
+@dataclass
+class KernelConfig:
+    policy_bundle_version: str = "v1.0"
+    catalog_snapshot_version: str = "catalog_default"
+    max_replan_attempts: int = 3
+    confirmation_timeout_seconds: int = 300
+    warnings_are_blocking: bool = False
+    backend: str = "pyspark"
+
+    phase_formalize: bool = True
+    phase_govern: bool = True
+    phase_generate: bool = True
+    phase_execute: bool = True
+    phase_validate: bool = True
+
+    enable_planning: bool = True
+    enable_codegen: bool = True
+    enable_static_validation: bool = True
+    enable_sandbox: bool = True
+    failure_policy: FailurePolicy = FailurePolicy.SELECTIVE_QUARANTINE
+    enable_promotion: bool = True
+    normalizer: str = "rule_based"
+    strict_normalization: bool = False
+    min_normalizer_confidence: float = 0.65
+
+
+# ── Kernel Orchestrator ──────────────────────────────────────────────────────
+
+
+class KernelOrchestrator:
+    """Single entry point for the CFA Kernel.
+
+    Creates a KernelPhases runner with all dependencies and delegates.
+    Pipeline phases (5): Formalize → Govern → Generate → Execute → Validate
+    """
+
+    def __init__(
+        self,
+        normalizer_backend: NormalizerBackend | None = None,
+        confirmation_handler: ConfirmationHandler | None = None,
+        policy_rules: list[PolicyRule] | None = None,
+        context_registry: ContextRegistry | None = None,
+        audit_trail: AuditTrail | None = None,
+        catalog: dict[str, Any] | None = None,
+        config: KernelConfig | None = None,
+        planner: ExecutionPlanner | None = None,
+        codegen_backend: CodeGenBackend | str | None = None,
+        static_validator: StaticValidator | None = None,
+        sandbox_backend: SandboxBackend | None = None,
+        runtime_validator: RuntimeValidator | None = None,
+        retry_policy: RetryPolicy | None = None,
+        promotion_engine: PromotionEngine | None = None,
+        schema_contract: dict[str, Any] | None = None,
+    ) -> None:
+        self.config = config or KernelConfig()
+        self._context_registry = context_registry or ContextRegistry()
+        self._audit_trail = audit_trail or AuditTrail()
+        self._catalog = catalog or {"datasets": {}}
+        self._schema_contract = schema_contract
+
+        self._normalizer = IntentNormalizer(
+            backend=normalizer_backend or self._resolve_normalizer_backend(),
+            policy_bundle_version=self.config.policy_bundle_version,
+            catalog_snapshot_version=self.config.catalog_snapshot_version,
+        )
+        self._confirmation = ConfirmationOrchestrator(
+            handler=confirmation_handler or AutoApproveHandler(),
+            timeout_seconds=self.config.confirmation_timeout_seconds,
+        )
+        self._policy = PolicyEngine(
+            rules=policy_rules, policy_bundle_version=self.config.policy_bundle_version,
+            max_replan_attempts=self.config.max_replan_attempts,
+        )
+        self._planner = planner or ExecutionPlanner()
+        self._codegen = self._resolve_codegen_backend(codegen_backend)
+        self._static_validator = static_validator or StaticValidator()
+        self._sandbox_executor = SandboxExecutor(backend=sandbox_backend or MockSandboxBackend())
+        self._runtime_validator = runtime_validator or RuntimeValidator()
+        self._retry_policy = retry_policy or RetryPolicy()
+        self._promotion_engine = promotion_engine or PromotionEngine(system_version="cfa_v0.1.0")
+
+    # Backward-compatible public aliases for internal components
+    @property
+    def audit_trail(self) -> AuditTrail:
+        return self._audit_trail
+
+    @property
+    def context_registry(self) -> ContextRegistry:
+        return self._context_registry
+
+    @property
+    def catalog(self) -> dict[str, Any]:
+        return self._catalog
+
+    @property
+    def promotion_engine(self) -> PromotionEngine:
+        return self._promotion_engine
+
+    def process(self, raw_intent: str) -> KernelResult:
+        from cfa.core.phases.runner import KernelPhases
+
+        phases = KernelPhases(
+            config=self.config,
+            context_registry=self._context_registry,
+            audit_trail=self._audit_trail,
+            catalog=self._catalog,
+            schema_contract=self._schema_contract,
+            normalizer=self._normalizer,
+            confirmation=self._confirmation,
+            policy=self._policy,
+            planner=self._planner,
+            codegen=self._codegen,
+            static_validator=self._static_validator,
+            sandbox_executor=self._sandbox_executor,
+            runtime_validator=self._runtime_validator,
+            partial_execution_manager=PartialExecutionManager(
+                sandbox=self._sandbox_executor,
+                runtime_validator=self._runtime_validator,
+                failure_policy=self.config.failure_policy,
+                retry_policy=self._retry_policy,
+            ),
+            state_projection=StateProjectionProtocol(self._context_registry),
+            promotion_engine=self._promotion_engine,
+        )
+        return phases.process(raw_intent)
+
+    def describe(self) -> dict[str, Any]:
+        return {
+            "config": {
+                "policy_bundle_version": self.config.policy_bundle_version,
+                "catalog_snapshot_version": self.config.catalog_snapshot_version,
+                "max_replan_attempts": self.config.max_replan_attempts,
+                "backend": self.config.backend,
+            },
+            "pipeline_phases": self.pipeline_config(),
+            "context_registry_version": self._context_registry.version_id,
+            "catalog_datasets": list(self._catalog.get("datasets", {}).keys()),
+            "policy_rules": len(self._policy.rules),
+            "audit_events": self._audit_trail.event_count,
+        }
+
+    def pipeline_config(self) -> dict[str, bool]:
+        return {
+            PipelinePhase.FORMALIZE: self.config.phase_formalize,
+            PipelinePhase.GOVERN: self.config.phase_govern,
+            PipelinePhase.GENERATE: self.config.phase_generate,
+            PipelinePhase.EXECUTE: self.config.phase_execute,
+            PipelinePhase.VALIDATE: self.config.phase_validate,
+        }
+
+    # ── Internal helpers ──────────────────────────────────────────────────
+
+    def _resolve_codegen_backend(
+        self, backend: CodeGenBackend | str | None
+    ) -> CodeGenBackend:
+        if isinstance(backend, CodeGenBackend):
+            return backend
+        name: str = backend if isinstance(backend, str) else self.config.backend
+        factory = BackendRegistry.singleton().get(name)
+        return factory()
+
+    def _resolve_normalizer_backend(self) -> NormalizerBackend:
+        if self.config.normalizer == "mock":
+            return MockNormalizerBackend()
+        return RuleBasedNormalizerBackend(
+            strict=self.config.strict_normalization,
+            min_confidence=self.config.min_normalizer_confidence,
+        )