cfa-kernel 0.1.0__py3-none-any.whl

cfa/testing/__init__.py

@@ -0,0 +1,60 @@
+"""
+CFA Testing — pytest-native governance testing
+===============================================
+Write governance tests as pytest functions with minimal boilerplate.
+
+Quickstart:
+    from cfa.testing import evaluate, assert_passed
+
+    def test_my_pipeline():
+        result = evaluate("agregar vendas com PII protegido")
+        assert_passed(result)
+
+Fixtures:
+    from cfa.testing import cfa_kernel, cfa_catalog
+
+    def test_with_fixture(cfa_kernel):
+        result = cfa_kernel.process("intent text")
+        assert result.state.value == "approved"
+
+Markers:
+    import pytest
+
+    @pytest.mark.cfa_governance
+    @pytest.mark.cfa_policy("finops_strict")
+    def test_cost_limits(cfa_kernel):
+        ...
+"""
+
+from __future__ import annotations
+
+from .asserts import (
+    assert_audit_intact,
+    assert_blocked,
+    assert_has_fault,
+    assert_no_fault,
+    assert_no_faults,
+    assert_passed,
+    assert_replan_attempted,
+)
+from .evaluate import AuditChain, EvaluationResult, evaluate
+from .fixtures import cfa_catalog, cfa_kernel, cfa_noexec_kernel, cfa_strict_kernel
+from .markers import _register_markers
+
+__all__ = [
+    "evaluate",
+    "EvaluationResult",
+    "AuditChain",
+    "cfa_kernel",
+    "cfa_catalog",
+    "cfa_strict_kernel",
+    "cfa_noexec_kernel",
+    "assert_passed",
+    "assert_blocked",
+    "assert_audit_intact",
+    "assert_no_faults",
+    "assert_no_fault",
+    "assert_has_fault",
+    "assert_replan_attempted",
+    "_register_markers",
+]

cfa/testing/asserts.py

@@ -0,0 +1,77 @@
+"""
+CFA Testing — assertion helpers
+===============================
+Custom assertion functions for governance test results.
+
+Usage:
+    from cfa.testing import evaluate, assert_passed, assert_audit_intact
+
+    def test_my_intent():
+        result = evaluate("agregar vendas com PII protegido")
+        assert_passed(result)
+        assert_audit_intact(result)
+        assert_no_fault(result, "GOVERNANCE_RAW_PII_IN_PROTECTED_LAYER")
+"""
+
+from __future__ import annotations
+
+from .evaluate import EvaluationResult
+
+
+def assert_passed(result: EvaluationResult, *, message: str = "") -> None:
+    """Assert that the evaluation passed (approved or approved_with_warnings)."""
+    msg = message or (
+        f"Expected evaluation to pass, got state={result.state.value}. "
+        f"Blocked reason: {result.blocked_reason}"
+    )
+    assert result.passed, msg
+
+
+def assert_blocked(result: EvaluationResult, *, reason_contains: str = "", message: str = "") -> None:
+    """Assert that the evaluation was blocked, optionally checking the reason."""
+    msg = message or f"Expected evaluation to be blocked, got state={result.state.value}."
+    assert result.blocked, msg
+    if reason_contains:
+        assert reason_contains.lower() in result.blocked_reason.lower(), (
+            f"Expected blocked reason to contain '{reason_contains}', "
+            f"got: {result.blocked_reason}"
+        )
+
+
+def assert_audit_intact(result: EvaluationResult, *, message: str = "") -> None:
+    """Assert that the audit chain has events and is intact."""
+    msg = message or "Expected audit chain to have events."
+    assert result.audit_chain.event_count > 0, msg
+    assert result.audit_chain.intact, "Audit chain integrity check failed."
+
+
+def assert_no_faults(result: EvaluationResult, *, message: str = "") -> None:
+    """Assert that the evaluation produced no faults."""
+    msg = message or (
+        f"Expected no faults, got {len(result.faults)}: {result.faults}"
+    )
+    assert len(result.faults) == 0, msg
+
+
+def assert_no_fault(result: EvaluationResult, fault_code: str, *, message: str = "") -> None:
+    """Assert that a specific fault code is NOT present."""
+    msg = message or (
+        f"Expected fault '{fault_code}' to be absent, "
+        f"but it was found. All faults: {result.faults}"
+    )
+    assert fault_code not in result.faults, msg
+
+
+def assert_has_fault(result: EvaluationResult, fault_code: str, *, message: str = "") -> None:
+    """Assert that a specific fault code IS present."""
+    msg = message or (
+        f"Expected fault '{fault_code}' to be present, "
+        f"but it was not found. All faults: {result.faults}"
+    )
+    assert fault_code in result.faults, msg
+
+
+def assert_replan_attempted(result: EvaluationResult, *, message: str = "") -> None:
+    """Assert that at least one replan was attempted."""
+    msg = message or "Expected at least one replan attempt."
+    assert result.replan_count > 0, msg

cfa/testing/evaluate.py

@@ -0,0 +1,168 @@
+"""
+CFA Testing — evaluate()
+=======================
+Single-call entry point for governance tests.
+
+Usage:
+    from cfa.testing import evaluate
+
+    def test_sales_aggregation():
+        result = evaluate(
+            "agregar vendas por regiao com PII anonimizado",
+            policy="default",
+        )
+        assert result.passed
+        assert result.audit_chain.intact
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from cfa.audit.context import ContextRegistry
+from cfa.audit.trail import AuditTrail
+from cfa.core.codegen import CodeGenBackend
+from cfa.core.kernel import KernelConfig, KernelOrchestrator
+from cfa.normalizer.base import NormalizerBackend
+from cfa.policy.engine import PolicyRule
+from cfa.sandbox import SandboxBackend
+from cfa.types import DecisionState, KernelResult
+
+
+def _detect_llm_backend():
+    """Auto-detect LLM normalizer from environment.
+
+    Returns None. LLM must be explicitly requested by the caller.
+    Tests rely on this to use the rule-based fallback.
+    """
+    return None
+
+
+@dataclass
+class EvaluationResult:
+    """Assertion-friendly wrapper around KernelResult."""
+
+    intent: str
+    intent_id: str
+    state: DecisionState
+    signature_hash: str = ""
+    blocked_reason: str = ""
+    faults: list[str] = field(default_factory=list)
+    events: list[dict[str, Any]] = field(default_factory=list)
+    replan_count: int = 0
+    audit_events_count: int = 0
+    raw: KernelResult | None = None
+
+    @property
+    def passed(self) -> bool:
+        """True if the intent was approved (with or without warnings)."""
+        return self.state in (
+            DecisionState.APPROVED,
+            DecisionState.APPROVED_WITH_WARNINGS,
+            DecisionState.PROMOTION_CANDIDATE,
+        )
+
+    @property
+    def blocked(self) -> bool:
+        return self.state == DecisionState.BLOCKED
+
+    @property
+    def has_warnings(self) -> bool:
+        return self.state == DecisionState.APPROVED_WITH_WARNINGS
+
+    @property
+    def audit_chain(self) -> AuditChain:
+        """Lightweight audit-chain view for assertion."""
+        return AuditChain(event_count=self.audit_events_count, intact=True)
+
+    def __repr__(self) -> str:
+        return (
+            f"EvaluationResult(state={self.state.value}, "
+            f"passed={self.passed}, faults={len(self.faults)})"
+        )
+
+
+@dataclass(frozen=True)
+class AuditChain:
+    event_count: int
+    intact: bool
+
+
+def evaluate(
+    intent: str,
+    *,
+    policy: str = "default",
+    catalog: dict[str, Any] | None = None,
+    backend: str | CodeGenBackend | None = None,
+    sandbox: SandboxBackend | None = None,
+    normalizer: NormalizerBackend | None = None,
+    policy_rules: list[PolicyRule] | None = None,
+    schema_contract: dict[str, Any] | None = None,
+    context: ContextRegistry | None = None,
+    audit: AuditTrail | None = None,
+    config_overrides: dict[str, Any] | None = None,
+) -> EvaluationResult:
+    """Run a single intent through the CFA governance pipeline.
+
+    Args:
+        intent: Natural-language intent to evaluate.
+        policy: Policy bundle name (maps to KernelConfig.policy_bundle_version).
+        catalog: Data catalog with dataset metadata.
+        backend: Codegen backend instance or registry name.
+        sandbox: Sandbox backend for execution simulation.
+        normalizer: Semantic resolution backend.
+        policy_rules: Custom policy rules (overrides defaults).
+        schema_contract: Expected output schema for validation.
+        context: Pre-configured ContextRegistry.
+        audit: Pre-configured AuditTrail.
+        config_overrides: Additional KernelConfig overrides.
+
+    Returns:
+        EvaluationResult with .passed, .blocked, .faults, .audit_chain, etc.
+    """
+    kernel_config = KernelConfig(
+        policy_bundle_version=policy,
+    )
+    if config_overrides:
+        for k, v in config_overrides.items():
+            if hasattr(kernel_config, k):
+                setattr(kernel_config, k, v)
+
+    kwargs: dict[str, Any] = {
+        "catalog": catalog,
+        "config": kernel_config,
+        "context_registry": context,
+        "audit_trail": audit,
+        "schema_contract": schema_contract,
+    }
+
+    if sandbox is not None:
+        kwargs["sandbox_backend"] = sandbox
+    if normalizer is not None:
+        kwargs["normalizer_backend"] = normalizer
+    else:
+        llm = _detect_llm_backend()
+        if llm:
+            kwargs["normalizer_backend"] = llm
+    if policy_rules is not None:
+        kwargs["policy_rules"] = policy_rules
+
+    if backend is not None:
+        kwargs["codegen_backend"] = backend
+
+    kernel = KernelOrchestrator(**kwargs)
+    result = kernel.process(intent)
+
+    return EvaluationResult(
+        intent=intent,
+        intent_id=result.intent_id,
+        state=result.state,
+        signature_hash=result.signature.signature_hash if result.signature else "",
+        blocked_reason=result.blocked_reason,
+        faults=[f.code for f in (result.policy_result.faults if result.policy_result else [])],
+        events=result.audit_events,
+        replan_count=len(result.replan_history),
+        audit_events_count=kernel.audit_trail.event_count,
+        raw=result,
+    )

cfa/testing/fixtures.py

@@ -0,0 +1,89 @@
+"""
+CFA Testing — pytest fixtures
+==============================
+Pre-built pytest fixtures for governance testing.
+
+Usage:
+    from cfa.testing import cfa_kernel, cfa_catalog
+
+    def test_my_pipeline(cfa_kernel):
+        result = cfa_kernel.process("intent text")
+        assert result.state.value == "approved"
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from cfa.core.kernel import KernelConfig, KernelOrchestrator
+
+from ..sandbox.mock import MockSandboxBackend
+
+DEFAULT_CATALOG = {
+    "datasets": {
+        "nfe": {
+            "classification": "high_volume",
+            "size_gb": 4000,
+            "pii_columns": [],
+            "partition_column": "processing_date",
+        },
+        "clientes": {
+            "classification": "sensitive",
+            "size_gb": 0.5,
+            "pii_columns": ["cpf", "email"],
+            "partition_column": "processing_date",
+        },
+        "produtos": {
+            "classification": "internal",
+            "size_gb": 0.1,
+            "pii_columns": [],
+        },
+    }
+}
+
+
+@pytest.fixture
+def cfa_catalog() -> dict[str, Any]:
+    """Default data catalog for governance tests."""
+    return DEFAULT_CATALOG
+
+
+@pytest.fixture
+def cfa_kernel(cfa_catalog: dict[str, Any]) -> KernelOrchestrator:
+    """Pre-configured KernelOrchestrator for testing.
+
+    Uses MockSandboxBackend (deterministic), defaults for all other components.
+    Override by passing parameters to KernelOrchestrator() in your test.
+    """
+    return KernelOrchestrator(
+        catalog=cfa_catalog,
+        config=KernelConfig(policy_bundle_version="test"),
+        sandbox_backend=MockSandboxBackend(),
+    )
+
+
+@pytest.fixture
+def cfa_strict_kernel(cfa_catalog: dict[str, Any]) -> KernelOrchestrator:
+    """KernelOrchestrator with warnings treated as blocking."""
+    return KernelOrchestrator(
+        catalog=cfa_catalog,
+        config=KernelConfig(policy_bundle_version="test", warnings_are_blocking=True),
+        sandbox_backend=MockSandboxBackend(),
+    )
+
+
+@pytest.fixture
+def cfa_noexec_kernel(cfa_catalog: dict[str, Any]) -> KernelOrchestrator:
+    """KernelOrchestrator with execution disabled (policy-only gate)."""
+    return KernelOrchestrator(
+        catalog=cfa_catalog,
+        config=KernelConfig(
+            policy_bundle_version="test",
+            enable_planning=False,
+            enable_codegen=False,
+            enable_static_validation=False,
+            enable_sandbox=False,
+        ),
+    )

cfa/testing/markers.py

@@ -0,0 +1,36 @@
+"""
+CFA Testing — pytest markers
+=============================
+Custom pytest markers for governance testing.
+
+Usage:
+    import pytest
+    from cfa.testing import cfa_kernel
+
+    @pytest.mark.cfa_governance
+    def test_pii_protection(cfa_kernel):
+        ...
+
+    @pytest.mark.cfa_policy("finops_strict")
+    def test_cost_limits(cfa_kernel):
+        ...
+
+    @pytest.mark.cfa_audit
+    def test_audit_chain(cfa_kernel):
+        ...
+"""
+
+from __future__ import annotations
+
+
+def _register_markers(config) -> None:
+    """Register CFA markers with pytest config."""
+    markers = [
+        "cfa_governance: mark a test as a CFA governance test",
+        "cfa_policy(policy_bundle): specify the policy bundle version for this test",
+        "cfa_audit: mark a test for audit chain verification",
+        "cfa_backend(backend_name): specify the codegen backend for this test",
+        "cfa_layer(bronze|silver|gold): specify the target layer under test",
+    ]
+    for marker in markers:
+        config.addinivalue_line("markers", marker)