cfa-kernel 0.1.0__py3-none-any.whl

cfa/audit/trail.py

@@ -0,0 +1,194 @@
+"""
+CFA Audit Trail
+================
+Append-only, causally ordered record of all decision events.
+
+Phase 1: in-memory list.
+Phase 4: persistent backend (JSON Lines file, extensible to S3/Kafka/OpenLineage).
+
+Properties (Invariant I5):
+- Immutable after write
+- Causal ordering per intent_id
+- Complete per intent (start and end recorded)
+- Cryptographic hash chain for tamper detection
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+from cfa.types import _utcnow
+
+
+@dataclass
+class AuditEvent:
+    """Single typed event in the audit trail."""
+
+    intent_id: str
+    stage: str
+    event_type: str
+    outcome: str
+    policy_bundle_version: str = ""
+    details: dict[str, Any] = field(default_factory=dict)
+    timestamp: str = field(default_factory=lambda: _utcnow().isoformat())
+    event_hash: str = ""
+    previous_hash: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+# ── Audit Storage Backend ───────────────────────────────────────────────────
+
+
+class AuditStorageBackend(ABC):
+    """Extension point: pluggable persistence for audit events."""
+
+    @abstractmethod
+    def append(self, event: AuditEvent) -> None: ...
+
+    @abstractmethod
+    def load_all(self) -> list[AuditEvent]: ...
+
+    @abstractmethod
+    def load_by_intent(self, intent_id: str) -> list[AuditEvent]: ...
+
+
+class InMemoryAuditStorage(AuditStorageBackend):
+    """In-memory storage for testing."""
+
+    def __init__(self) -> None:
+        self._events: list[AuditEvent] = []
+
+    def append(self, event: AuditEvent) -> None:
+        self._events.append(event)
+
+    def load_all(self) -> list[AuditEvent]:
+        return list(self._events)
+
+    def load_by_intent(self, intent_id: str) -> list[AuditEvent]:
+        return [e for e in self._events if e.intent_id == intent_id]
+
+
+class JsonLinesAuditStorage(AuditStorageBackend):
+    """
+    JSON Lines file-based persistent audit storage.
+    Each line is a JSON-serialized AuditEvent.
+    Append-only — never modifies existing lines (Invariant I5).
+    """
+
+    def __init__(self, file_path: str | Path) -> None:
+        self.file_path = Path(file_path)
+        self.file_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def append(self, event: AuditEvent) -> None:
+        with open(self.file_path, "a", encoding="utf-8") as f:
+            f.write(json.dumps(event.to_dict(), default=str) + "\n")
+
+    def load_all(self) -> list[AuditEvent]:
+        if not self.file_path.exists():
+            return []
+        events = []
+        for line in self.file_path.read_text(encoding="utf-8").strip().splitlines():
+            if line:
+                data = json.loads(line)
+                events.append(AuditEvent(**data))
+        return events
+
+    def load_by_intent(self, intent_id: str) -> list[AuditEvent]:
+        return [e for e in self.load_all() if e.intent_id == intent_id]
+
+
+# ── Audit Trail ─────────────────────────────────────────────────────────────
+
+
+class AuditTrail:
+    """
+    Append-only audit trail with hash chain.
+
+    Two consumption modes (per whitepaper):
+    - Operational (engineering): JSON, debugging, tuning
+    - Regulatory (audit): normalized + cryptographic hash chain
+
+    The hash chain ensures tamper detection: each event's hash
+    includes the previous event's hash, creating a causal chain.
+    """
+
+    def __init__(self, storage: AuditStorageBackend | None = None) -> None:
+        self._storage = storage or InMemoryAuditStorage()
+        self._last_hash = ""
+        # Restore hash chain from existing events
+        existing = self._storage.load_all()
+        if existing:
+            self._last_hash = existing[-1].event_hash
+
+    def record(
+        self,
+        intent_id: str,
+        stage: str,
+        event_type: str,
+        outcome: str,
+        policy_bundle_version: str = "",
+        **details: Any,
+    ) -> AuditEvent:
+        event = AuditEvent(
+            intent_id=intent_id,
+            stage=stage,
+            event_type=event_type,
+            outcome=outcome,
+            policy_bundle_version=policy_bundle_version,
+            details=details,
+            previous_hash=self._last_hash,
+        )
+        # Compute hash chain
+        event.event_hash = self._compute_hash(event)
+        self._last_hash = event.event_hash
+        self._storage.append(event)
+        return event
+
+    def get_events_for_intent(self, intent_id: str) -> list[AuditEvent]:
+        return self._storage.load_by_intent(intent_id)
+
+    def get_all_events(self) -> list[AuditEvent]:
+        return self._storage.load_all()
+
+    @property
+    def event_count(self) -> int:
+        return len(self._storage.load_all())
+
+    def verify_chain(self) -> bool:
+        """Verify the integrity of the hash chain. Returns True if valid."""
+        events = self._storage.load_all()
+        prev_hash = ""
+        for event in events:
+            if event.previous_hash != prev_hash:
+                return False
+            expected = self._compute_hash(event)
+            legacy_expected = self._compute_hash_legacy(event)
+            if event.event_hash not in (expected, legacy_expected):
+                return False
+            prev_hash = event.event_hash
+        return True
+
+    @staticmethod
+    def _compute_hash(event: AuditEvent) -> str:
+        payload = json.dumps({
+            "intent_id": event.intent_id,
+            "stage": event.stage,
+            "event_type": event.event_type,
+            "outcome": event.outcome,
+            "timestamp": event.timestamp,
+            "previous_hash": event.previous_hash,
+            "details": event.details,
+        }, sort_keys=True, default=str)
+        return hashlib.sha256(payload.encode()).hexdigest()
+
+    @staticmethod
+    def _compute_hash_legacy(event: AuditEvent) -> str:
+        """Return the pre-3.1 truncated hash for backward verification only."""
+        return AuditTrail._compute_hash(event)[:16]

cfa/backends/__init__.py

@@ -0,0 +1,132 @@
+"""
+CFA Backend Registry
+====================
+Pluggable backends for code generation and execution.
+
+A BackendAdapter extends CodeGenBackend with capability introspection,
+enabling the Policy Engine to validate whether a backend can satisfy
+a given intent's constraints before attempting code generation.
+
+Usage:
+    from cfa.backends import BackendRegistry, BackendAdapter, BackendCapabilities
+
+    registry = BackendRegistry()
+    registry.register("pyspark", PySparkBackend)
+    registry.register("duckdb", DuckDBBackend)
+
+    backend = registry.get("pyspark")()
+    caps = backend.get_capabilities()
+"""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from cfa.core.codegen import CodeGenBackend
+from cfa.validation.static import ForbiddenToken
+
+
+@dataclass
+class BackendCapabilities:
+    """Capabilities exposed by a backend for policy validation.
+
+    The PolicyEngine queries these before approving an intent —
+    if a backend cannot satisfy a required constraint (e.g., merge_key
+    on a Silver write), the intent is blocked before code generation.
+    """
+
+    backend_name: str = ""
+    backend_version: str = ""
+
+    supports_merge: bool = False
+    supports_partition_overwrite: bool = False
+    supports_anonymization: bool = False
+    supports_schema_enforcement: bool = False
+
+    pii_anonymization_methods: list[str] = field(default_factory=lambda: ["sha256", "drop"])
+    cost_model_available: bool = False
+    max_recommended_rows: int = 10_000_000
+    supported_languages: list[str] = field(default_factory=lambda: ["python"])
+
+    forbidden_tokens: list[ForbiddenToken] = field(default_factory=list)
+
+    custom: dict[str, Any] = field(default_factory=dict)
+
+
+class BackendAdapter(CodeGenBackend):
+    """Extension of CodeGenBackend with capability introspection.
+
+    Implement this to create a pluggable backend for any target:
+    PySpark, DuckDB, BigQuery, REST API, LLM chain, etc.
+    """
+
+    @abstractmethod
+    def get_capabilities(self) -> BackendCapabilities: ...
+
+
+BackendFactory = Callable[[], BackendAdapter]
+
+
+class BackendRegistry:
+    """Global registry of available backend factories.
+
+    Supports:
+    - register(name, factory) — add a backend
+    - get(name) -> factory — retrieve by name
+    - list() -> list of registered names
+    - remove(name) — unregister a backend
+    """
+
+    _instance: BackendRegistry | None = None
+    _lock: Any = None
+
+    def __init__(self) -> None:
+        self._backends: dict[str, BackendFactory] = {}
+
+    @classmethod
+    def singleton(cls) -> BackendRegistry:
+        if cls._lock is None:
+            import threading
+            cls._lock = threading.Lock()
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = cls()
+                cls._instance._bootstrap()
+            return cls._instance
+
+    def _bootstrap(self) -> None:
+        if self._backends:
+            return
+        from .dbt import DbtBackend  # noqa: F811
+        from .pyspark import PySparkBackend  # noqa: F811
+        from .sql import SqlBackend  # noqa: F811
+
+        self.register("dbt", lambda: DbtBackend())
+        self.register("pyspark", lambda: PySparkBackend())
+        self.register("sql", lambda: SqlBackend())
+
+    def register(self, name: str, factory: BackendFactory) -> None:
+        self._backends[name] = factory
+
+    def get(self, name: str) -> BackendFactory:
+        if name not in self._backends:
+            available = ", ".join(sorted(self._backends))
+            raise KeyError(
+                f"Unknown backend '{name}'. Registered backends: {available or '(none)'}"
+            )
+        return self._backends[name]
+
+    def list(self) -> list[str]:
+        return sorted(self._backends)
+
+    def remove(self, name: str) -> None:
+        self._backends.pop(name, None)
+
+    def clear(self) -> None:
+        self._backends.clear()
+
+    def __contains__(self, name: str) -> bool:
+        return name in self._backends

cfa/backends/dbt.py

@@ -0,0 +1,338 @@
+"""
+dbt Backend
+============
+Code generation backend targeting dbt (data build tool).
+
+Generates governed dbt models from an ExecutionPlan. Output includes:
+- ``model.sql`` with dbt Jinja templating and governed SQL
+- ``schema.yml`` with data quality tests derived from governance rules
+- Partition configs, merge keys as uniqueness tests, PII annotations
+
+Template-based — no LLM involved.
+"""
+
+from __future__ import annotations
+
+from cfa.core.codegen import GeneratedCode
+from cfa.core.planner import ExecutionPlan, ExecutionStep, StepType, WriteMode
+from cfa.types import FaultSeverity
+from cfa.validation.static import ForbiddenToken
+
+from . import BackendAdapter, BackendCapabilities
+
+_DBT_FORBIDDEN_TOKENS: list[ForbiddenToken] = [
+    ForbiddenToken(pattern=r"\bDROP\s+TABLE\b", fault_code="STATIC_DBT_DROP_TABLE",
+                   severity=FaultSeverity.CRITICAL,
+                   message="DROP TABLE in governed dbt model forbidden.", is_regex=True),
+    ForbiddenToken(pattern=r"\bDROP\s+DATABASE\b", fault_code="STATIC_DBT_DROP_DATABASE",
+                   severity=FaultSeverity.CRITICAL,
+                   message="DROP DATABASE in governed dbt model forbidden.", is_regex=True),
+    ForbiddenToken(pattern=r"\bTRUNCATE\b", fault_code="STATIC_DBT_TRUNCATE",
+                   severity=FaultSeverity.CRITICAL,
+                   message="TRUNCATE forbidden in dbt governed models.", is_regex=True),
+    ForbiddenToken(pattern=r"\bDELETE\s+FROM\b", fault_code="STATIC_DBT_DELETE",
+                   severity=FaultSeverity.CRITICAL,
+                   message="DELETE FROM forbidden in dbt governed models.", is_regex=True),
+    ForbiddenToken(pattern=r"\bALTER\s+TABLE\b", fault_code="STATIC_DBT_ALTER_TABLE",
+                   severity=FaultSeverity.HIGH,
+                   message="ALTER TABLE forbidden — use dbt model rebuild.", is_regex=True),
+]
+
+
+class DbtBackend(BackendAdapter):
+    """Generates governed dbt model + schema.yml from an ExecutionPlan."""
+
+    def get_capabilities(self) -> BackendCapabilities:
+        return BackendCapabilities(
+            backend_name="dbt",
+            backend_version="dbt-core-1.x",
+            supports_merge=True,
+            supports_partition_overwrite=True,
+            supports_anonymization=True,
+            supports_schema_enforcement=True,
+            pii_anonymization_methods=["sha256", "drop", "md5", "tokenize"],
+            cost_model_available=False,
+            max_recommended_rows=500_000_000,
+            supported_languages=["sql", "yaml"],
+            forbidden_tokens=_DBT_FORBIDDEN_TOKENS,
+        )
+
+    def generate(self, plan: ExecutionPlan) -> GeneratedCode:
+        sql_lines: list[str] = []
+        schema_columns: list[dict[str, object]] = []
+        model_name = self._derive_model_name(plan)
+        pii_columns_anonymized: set[str] = set()
+        merge_keys: list[str] = []
+        partition_cols: list[str] = []
+        step_code: dict[str, str] = {}
+        ordered = plan.execution_order()
+
+        for step in ordered:
+            code = self._generate_step(step, plan)
+            step_code[step.id] = code
+            sql_lines.append(f"-- Step: {step.id} ({step.step_type.value})")
+            sql_lines.append(code)
+            sql_lines.append("")
+
+            merge_keys = step.config.get("merge_keys", merge_keys)
+            partition_cols = step.config.get("partition_by", partition_cols)
+            if step.step_type == StepType.ANONYMIZE:
+                for col in step.config.get("pii_columns", []):
+                    pii_columns_anonymized.add(str(col))
+
+        schema = self._build_schema_yml(
+            model_name=model_name,
+            plan=plan,
+            merge_keys=merge_keys,
+            partition_cols=partition_cols,
+            pii_columns_anonymized=pii_columns_anonymized,
+            schema_columns=schema_columns,
+        )
+
+        config_block = self._build_config(
+            plan=plan,
+            partition_cols=partition_cols,
+            merge_keys=merge_keys,
+        )
+
+        full_body = "\n".join(sql_lines)
+        model_sql = f"{config_block}\n\n{full_body}"
+
+        combined = (
+            f"-- dbt model: {model_name}\n"
+            f"-- ============={'=' * len(model_name)}\n\n"
+            f"{model_sql}\n\n"
+            f"-- schema.yml (generated by CFA governance)\n"
+            f"-- ==========================================\n\n"
+            f"{schema}"
+        )
+
+        return GeneratedCode(
+            plan_signature_hash=plan.signature_hash,
+            intent_id=plan.intent_id,
+            language="dbt",
+            code=combined,
+            step_code_map=step_code,
+            metadata={
+                "model_name": model_name,
+                "write_mode": plan.write_mode.value,
+                "consistency_unit": plan.consistency_unit.value,
+                "step_count": plan.step_count,
+            },
+        )
+
+    # ── Step generators ───────────────────────────────────────────────────
+
+    def _generate_step(self, step: ExecutionStep, plan: ExecutionPlan) -> str:
+        match step.step_type:
+            case StepType.EXTRACT:
+                return self._gen_extract(step)
+            case StepType.ANONYMIZE:
+                return self._gen_anonymize(step)
+            case StepType.JOIN:
+                return self._gen_join(step, plan)
+            case StepType.AGGREGATE:
+                return self._gen_aggregate(step)
+            case StepType.LOAD:
+                return self._gen_load(step, plan)
+            case StepType.FILTER:
+                return self._gen_filter(step)
+            case StepType.TRANSFORM:
+                return self._gen_transform(step)
+            case _:
+                return f"-- TODO: unsupported step type {step.step_type.value}"
+
+    def _gen_extract(self, step: ExecutionStep) -> str:
+        source = step.source or "unknown_source"
+        model_ref = _ref_name(source)
+        lines = [f"-- EXTRACT: {source}"]
+        lines.append(f"SELECT * FROM {{{{ ref('{model_ref}') }}}}")
+
+        filt = step.config.get("filter")
+        if filt:
+            col = _quote_ident(filt["column"])
+            pred = filt["predicate"]
+            lines.append(f"WHERE {col} {pred} '{{{{ var(\"date_param\") }}}}'")
+
+        return "\n".join(lines)
+
+    def _gen_anonymize(self, step: ExecutionStep) -> str:
+        source = step.source or "source"
+        pii_cols = step.config.get("pii_columns", [])
+        strategy = step.config.get("strategy", "sha256")
+
+        if not pii_cols:
+            return f"-- No PII columns to anonymize for {source}"
+
+        col_list = ", ".join(_quote_ident(c) for c in pii_cols)
+        lines: list[str] = [
+            f"-- ANONYMIZE: {source} (strategy={strategy}, columns={col_list})",
+        ]
+        for col in pii_cols:
+            safe = _quote_ident(col)
+            if strategy == "sha256":
+                lines.append(f"--   {safe} replaced with SHA256({safe})")
+            elif strategy == "drop":
+                lines.append(f"--   {safe} dropped")
+            else:
+                lines.append(f"--   {safe} anonymized ({strategy})")
+        return "\n".join(lines)
+
+    def _gen_join(self, step: ExecutionStep, plan: ExecutionPlan) -> str:
+        datasets = step.config.get("datasets", [])
+        merge_keys = step.config.get("merge_keys", ["id"])
+
+        if len(datasets) < 2:
+            return "-- Join requires at least 2 datasets"
+
+        left_alias = _cte_name(datasets[0])
+        right_alias = _cte_name(datasets[1])
+        left_ref = _ref_name(datasets[0])
+        right_ref = _ref_name(datasets[1])
+        on_clause = " AND ".join(
+            f"{left_alias}.{_quote_ident(k)} = {right_alias}.{_quote_ident(k)}"
+            for k in merge_keys
+        )
+
+        lines: list[str] = [f"-- JOIN: {datasets[0]} + {datasets[1]}"]
+        lines.append(f"SELECT {left_alias}.*, {right_alias}.*")
+        lines.append(f"FROM {{{{ ref('{left_ref}') }}}} {left_alias}")
+        lines.append(f"INNER JOIN {{{{ ref('{right_ref}') }}}} {right_alias}")
+        lines.append(f"  ON {on_clause}")
+
+        return "\n".join(lines)
+
+    def _gen_aggregate(self, step: ExecutionStep) -> str:
+        group_by = step.config.get("group_by", [])
+        lines: list[str] = ["-- AGGREGATE"]
+        if not group_by:
+            lines.append("SELECT COUNT(*) AS cnt FROM joined_cte")
+        else:
+            cols = ", ".join(_quote_ident(c) for c in group_by)
+            lines.append(f"SELECT {cols}, COUNT(*) AS cnt")
+            lines.append("FROM joined_cte")
+            lines.append(f"GROUP BY {cols}")
+        return "\n".join(lines)
+
+    def _gen_load(self, step: ExecutionStep, plan: ExecutionPlan) -> str:
+        target = step.target or "target_model"
+        merge_keys = step.config.get("merge_keys", ["id"])
+        partition_by = step.config.get("partition_by", [])
+
+        lines: list[str] = [f"-- LOAD: {target}"]
+        lines.append(f"--   Model: {{{{ ref('{_ref_name(target)}') }}}}")
+        if merge_keys:
+            lines.append(f"--   Merge keys: {', '.join(merge_keys)}")
+        if partition_by:
+            lines.append(f"--   Partition by: {', '.join(partition_by)}")
+        lines.append("SELECT * FROM joined_cte")
+        return "\n".join(lines)
+
+    def _gen_filter(self, step: ExecutionStep) -> str:
+        condition = step.config.get("condition", "1=1")
+        return f"-- FILTER: WHERE {condition}"
+
+    def _gen_transform(self, step: ExecutionStep) -> str:
+        return f"-- TRANSFORM: {step.config}"
+
+    # ── Config & schema generation ────────────────────────────────────────
+
+    def _build_config(
+        self,
+        plan: ExecutionPlan,
+        partition_cols: list[str],
+        merge_keys: list[str],
+    ) -> str:
+        materialized = "table" if plan.write_mode == WriteMode.MERGE else "incremental"
+        lines = [
+            "{{ config(",
+            f"    materialized='{materialized}',",
+        ]
+        if partition_cols:
+            field_list = ", ".join(f"'{p}'" for p in partition_cols)
+            lines.append(f"    partition_by={{'field': [{field_list}], 'data_type': 'date'}},")
+        if merge_keys and plan.write_mode == WriteMode.MERGE:
+            key_list = ", ".join(f"'{k}'" for k in merge_keys)
+            lines.append(f"    unique_key=['{key_list}'],")
+        lines.append(") }}")
+        return "\n".join(lines)
+
+    def _build_schema_yml(
+        self,
+        model_name: str,
+        plan: ExecutionPlan,
+        merge_keys: list[str],
+        partition_cols: list[str],
+        pii_columns_anonymized: set[str],
+        schema_columns: list[dict[str, object]],
+    ) -> str:
+        lines = [
+            "version: 2",
+            "",
+            "models:",
+            f"  - name: {_ref_name(model_name)}",
+            f'    description: "Governed model generated by CFA. Intent hash: {plan.signature_hash}"',
+        ]
+
+        columns: list[str] = list({
+            str(k) for k in merge_keys
+        } | {
+            str(p) for p in partition_cols
+        })
+
+        if columns:
+            lines.append("    columns:")
+            for col in sorted(columns):
+                lines.append(f"      - name: {_quote_ident(col)}")
+                tests: list[str] = []
+                if col in merge_keys:
+                    tests.append("not_null")
+                    if len(merge_keys) == 1:
+                        tests.append("unique")
+                if col in pii_columns_anonymized:
+                    lines.append('        description: "PII column anonimizada via SHA256"')
+                for test in tests:
+                    lines.append("        tests:")
+                    lines.append(f"          - {test}")
+
+        if len(merge_keys) > 1:
+            lines.append("    tests:")
+            lines.append("      - dbt_utils.unique_combination_of_columns:")
+            lines.append("          combination_of_columns:")
+            for k in merge_keys:
+                lines.append(f"            - {_quote_ident(k)}")
+
+        return "\n".join(lines)
+
+    @staticmethod
+    def _derive_model_name(plan: ExecutionPlan) -> str:
+        target = plan.write_mode.value
+        domain = plan.metadata.get("domain", "governed")
+        layer = plan.metadata.get("target_layer", "silver")
+        return f"{layer}_{domain}_{target}"
+
+
+# ── dbt-specific helpers ────────────────────────────────────────────────────
+
+
+def _ref_name(source: str) -> str:
+    clean = source.replace("-", "_").replace(".", "_").replace(" ", "_").lower()
+    return clean
+
+
+def _cte_name(source: str) -> str:
+    clean = _ref_name(source)
+    if len(clean) <= 8:
+        return clean
+    parts = clean.split("_")
+    if len(parts) >= 2:
+        return parts[0][:4] + "_" + parts[-1][:4]
+    return clean[:8]
+
+
+def _quote_ident(name: str) -> str:
+    return str(name)
+
+
+def _var_name(source: str) -> str:
+    return f"cte_{_ref_name(source)}"