cfa-kernel 0.1.7__tar.gz → 0.1.9__tar.gz

{cfa_kernel-0.1.7 → cfa_kernel-0.1.9}/.gitignore

@@ -70,3 +70,8 @@ website/.docusaurus/
 cfa_test_results_*/
 test_battery.py
 uv.lock
+
+CFA_LLM_Demo.dbc
+CFA_LLM_Demo.html
+CFA_LLM_Demo.py
+CFA_LLM_Demo.ipynb.py

{cfa_kernel-0.1.7 → cfa_kernel-0.1.9}/CFA_LLM_Demo.ipynb

@@ -145,27 +145,32 @@
    "source": [
     "CATALOG = {\n",
     "    'nfe_bronze': {\n",
+    "            'classification': 'internal',\n",
     "        'type': 'delta', 'layer': 'bronze', 'size_gb': 50,\n",
     "        'partition_by': ['processing_date'], 'pii': False,\n",
     "        'description': 'Notas Fiscais Eletronicas brutas',\n",
     "    },\n",
     "    'clientes_bronze': {\n",
+    "            'classification': 'sensitive',\n",
     "        'type': 'delta', 'layer': 'bronze', 'size_gb': 10,\n",
     "        'partition_by': ['processing_date'], 'pii': True,\n",
     "        'pii_columns': ['cpf', 'nome', 'endereco'],\n",
     "        'description': 'Dados cadastrais com CPF e endereco',\n",
     "    },\n",
     "    'vendas_bronze': {\n",
+    "            'classification': 'high_volume',\n",
     "        'type': 'delta', 'layer': 'bronze', 'size_gb': 2000,\n",
     "        'pii': False,\n",
     "        'description': 'Registros de transacoes de venda',\n",
     "    },\n",
     "    'fornecedores_bronze': {\n",
+    "            'classification': 'internal',\n",
     "        'type': 'delta', 'layer': 'bronze', 'size_gb': 10,\n",
     "        'pii': False,\n",
     "        'description': 'Cadastro de fornecedores',\n",
     "    },\n",
     "    'vendas_gold_agregado': {\n",
+    "            'classification': 'high_volume',\n",
     "        'type': 'delta', 'layer': 'gold', 'size_gb': 500,\n",
     "        'pii': False,\n",
     "        'description': 'Agregados de vendas para BI',\n",

cfa_kernel-0.1.7/CFA_LLM_Demo.ipynb.py → cfa_kernel-0.1.9/CFA_LLM_Demo_fixed.py

@@ -24,28 +24,27 @@
 # COMMAND ----------
 
 # MAGIC %md
-# MAGIC ## Section 1 — Setup & Secrets
+# MAGIC ## Section 0 — Install
 # MAGIC
-# MAGIC API key read from Databricks Secret Scope `cfa-secrets/deepseek-key`.
-# MAGIC Falls back to `DEEPSEEK_API_KEY` env var for local testing.
+# MAGIC Run this cell first to install dependencies.
 # MAGIC
 
 # COMMAND ----------
 
 # MAGIC %md
-# MAGIC ## Section 0 — Install
+# MAGIC ## Section 1 — Setup & Secrets
 # MAGIC
-# MAGIC Run this cell first to install dependencies.
+# MAGIC API key read from Databricks Secret Scope `cfa-secrets/openai-key` or `cfa-secrets/deepseek-key`.
+# MAGIC Falls back to `OPENAI_API_KEY` / `DEEPSEEK_API_KEY` env vars for local testing.
 # MAGIC
 
 # COMMAND ----------
 
 # MAGIC %pip install -q cfa-kernel[all] openai
-# MAGIC
 
 # COMMAND ----------
 
-dbutils.library.restartPython()
+# MAGIC %restart_python
 
 # COMMAND ----------
 
@@ -83,7 +82,7 @@ DEEPSEEK_KEY = _try_secret("deepseek-key") or os.environ.get("DEEPSEEK_API_KEY")
 # Auto-select: prefer OpenAI (better reachability from Azure), fallback to DeepSeek
 if OPENAI_KEY:
     LLM_MODEL = "gpt-4o-mini"
-    LLM_BASE_URL = "api.openai.com"
+    LLM_BASE_URL = None  # default api.openai.com
     LLM_API_KEY = OPENAI_KEY
     print(f"Using OpenAI ({LLM_MODEL})")
 elif DEEPSEEK_KEY:
@@ -127,28 +126,37 @@ def make_provider():
 CATALOG = {
     'nfe_bronze': {
         'type': 'delta', 'layer': 'bronze', 'size_gb': 50,
+        'classification': 'high_volume',
         'partition_by': ['processing_date'], 'pii': False,
+        'pii_columns': [],
         'description': 'Notas Fiscais Eletronicas brutas',
     },
     'clientes_bronze': {
         'type': 'delta', 'layer': 'bronze', 'size_gb': 10,
+        'classification': 'sensitive',
         'partition_by': ['processing_date'], 'pii': True,
         'pii_columns': ['cpf', 'nome', 'endereco'],
         'description': 'Dados cadastrais com CPF e endereco',
     },
     'vendas_bronze': {
         'type': 'delta', 'layer': 'bronze', 'size_gb': 2000,
-        'pii': False,
+        'classification': 'high_volume',
+        'partition_by': ['data_venda'], 'pii': False,
+        'pii_columns': [],
         'description': 'Registros de transacoes de venda',
     },
     'fornecedores_bronze': {
         'type': 'delta', 'layer': 'bronze', 'size_gb': 10,
-        'pii': False,
+        'classification': 'internal',
+        'partition_by': ['updated_at'], 'pii': False,
+        'pii_columns': [],
         'description': 'Cadastro de fornecedores',
     },
     'vendas_gold_agregado': {
         'type': 'delta', 'layer': 'gold', 'size_gb': 500,
-        'pii': False,
+        'classification': 'internal',
+        'partition_by': ['data_venda'], 'pii': False,
+        'pii_columns': [],
         'description': 'Agregados de vendas para BI',
     },
 }
@@ -157,6 +165,9 @@ from cfa.policy.catalog import validate_catalog
 result = validate_catalog(CATALOG)
 print(f"Catalog valid: {result.valid}")
 print(f"  {len(CATALOG)} datasets registered")
+print("  Classifications: nfe_bronze=high_volume, clientes_bronze=sensitive,")
+print("                   vendas_bronze=high_volume, fornecedores_bronze=internal,")
+print("                   vendas_gold_agregado=internal")
 for msg in result.messages:
     print(f"  {msg}")
 
@@ -176,18 +187,14 @@ for msg in result.messages:
 # COMMAND ----------
 
 if not HAS_LLM:
-    print("SKIP: no API key configured. Set DEEPSEEK_API_KEY or Databricks secret.")
+    print("SKIP: no API key configured.")
 else:
-    provider = OpenAILMProvider(
-        model="deepseek-chat",
-        base_url="https://api.deepseek.com",
-        temperature=0.0,
-    )
+    provider = make_provider()
     llm_backend = LLMNormalizerBackend(provider=provider, strict=False)
     normalizer = IntentNormalizer(backend=llm_backend)
 
     print("=" * 55)
-    print("LLM NORMALIZER — Semantic Resolution")
+    print("LLM NORMALIZER -- Semantic Resolution")
     print("=" * 55)
 
     intents = [
@@ -197,13 +204,13 @@ else:
     ]
 
     for raw in intents:
-        print(f"\nIntent: \"{raw}\"")
+        print(f'\nIntent: "{raw}"')
         res = normalizer.normalize(raw, {}, CATALOG)
         sig = res.signature
         print(f"  Domain       : {sig.domain}")
         print(f"  Intent       : {sig.intent}")
         print(f"  Target layer : {sig.target_layer}")
-        ds_names = [d.get('name', '?') for d in sig.datasets]
+        ds_names = [d.name for d in sig.datasets]
         print(f"  Datasets     : {ds_names}")
         print(f"  Confidence   : {res.confidence_score:.2f}")
         print(f"  Ambiguity    : {res.ambiguity_level}")
@@ -235,19 +242,28 @@ else:
     print("=" * 55)
     print("STRICT MODE — Catalog-grounded LLM")
     print("=" * 55)
+    print("  strict=True: LLM output validated against catalog.")
+    print("  Hallucinated datasets or invalid classifications raise ValueError.")
+    print()
 
-    for raw in [
-        "Join NFe with Clientes anonymize CPF persist to Silver",
-        "Full scan vendas without partition to Gold",
-    ]:
-        print(f"\nIntent: \"{raw}\"")
+    test_cases = [
+        # (intent, expect_pass)
+        ("Join NFe with Clientes anonymize CPF persist to Silver", True),
+        ("Full scan vendas without partition to Gold", False),
+    ]
+
+    for raw, expect_pass in test_cases:
+        print(f"Intent: \"{raw}\"")
         try:
             res = sn.normalize(raw, {}, CATALOG)
-            print(f"  PASSED  | confidence={res.confidence_score:.2f}  layer={res.signature.target_layer}")
+            status = "PASSED " if expect_pass else "UNEXPECTED PASS"
+            print(f"  {status} | confidence={res.confidence_score:.2f}  layer={res.signature.target_layer}")
         except ValueError as e:
-            print(f"  REJECTED | {str(e)[:120]}")
+            status = "REJECTED (expected)" if not expect_pass else "REJECTED (unexpected)"
+            print(f"  {status} | {str(e)[:120]}")
+        print()
 
-    print("\nStrict mode demo complete")
+    print("Strict mode demo complete")
 
 
 # COMMAND ----------
@@ -265,8 +281,14 @@ else:
 if not HAS_LLM:
     print("SKIP: no API key available")
 else:
-    # The 'sb' backend from section 4 already recorded calls
-    records = sb.audit_records
+    # 'sb' is the strict backend created in Section 4.
+    # Guard against running this cell out of order.
+    try:
+        records = sb.audit_records
+    except NameError:
+        print("NOTE: 'sb' not found — run Section 4 first, or re-run from Section 1.")
+        records = []
+
     print("=" * 55)
     print("LLM AUDIT TRAIL — Tamper-evident LLM calls")
     print("=" * 55)
@@ -376,7 +398,9 @@ if not HAS_LLM:
     print("SKIP: no API key available")
 else:
     provider = make_provider()
-    backend = LLMNormalizerBackend(provider=provider, strict=True)
+    # strict=False: LLM resolves intent semantically; catalog used for grounding
+    # but validation errors do not block execution (strict mode is for S4 demo only)
+    backend = LLMNormalizerBackend(provider=provider, strict=False)
 
     kernel = KernelOrchestrator(
         catalog=CATALOG,
@@ -406,7 +430,12 @@ else:
                 print(f"  -> Replans   : {len(result.replan_history)}")
             if result.generated_code and result.generated_code.code:
                 lines = result.generated_code.code.splitlines()
-                print(f"  -> Code gen  : {len(lines)} lines {result.generated_code.language}")
+                print(f"  -> Code gen  : {len(lines)} lines ({result.generated_code.language})")
+                preview = [l for l in lines if l.strip()][:6]
+                for line in preview:
+                    print(f"     {line}")
+                if len(lines) > 6:
+                    print(f"     ... ({len(lines) - 6} more lines)")
         except Exception as e:
             print(f"  -> BLOCKED   : {type(e).__name__} | {str(e)[:100]}")
 
@@ -430,25 +459,43 @@ else:
 
 # COMMAND ----------
 
+import traceback
 print("=" * 55)
 print("RUNTIME GATE")
 print("=" * 55)
 
-gate = RuntimeGate(
-    catalog=CATALOG,
-    policy_rules=rules if rules is not None else None,
-)
+# Guard: 'rules' is set by Section 6 (LLM Systematizer).
+# If that section was skipped or failed, fall back to default rules gracefully.
+try:
+    _rules = rules
+except NameError:
+    _rules = None
+
+if _rules is None:
+    print("NOTE: LLM-generated rules not available (Section 6 skipped or no API key).")
+    print("      Using default CFA policy rules instead.")
+else:
+    print(f"  Using {len(_rules)} LLM-generated rules from Section 6")
 
-result = gate.validate("Join NFe with Clientes and persist to Silver")
-print(f"  validate() -> state={result.state.value}  passed={result.passed}")
-print(f"  gate_id={result.gate_id}  execution_id={result.execution_id[:8]}...")
+try:
+    gate = RuntimeGate(
+        catalog=CATALOG,
+        policy_rules=_rules,
+    )
+
+    result = gate.validate("Join NFe with Clientes and persist to Silver")
+    print(f"  validate() -> state={result.state.value}  passed={result.passed}")
+    print(f"  gate_id={result.gate_id}  execution_id={result.execution_id[:8]}...")
 
-@gate.guard("aggregate sales data with PII protected")
-def my_pipeline():
-    return "pipeline executed"
+    @gate.guard("aggregate sales data with PII protected")
+    def my_pipeline():
+        return "pipeline executed"
 
-print(f"  @gate.guard -> {my_pipeline()}")
-print("\nRuntime Gate demo complete")
+    print(f"  @gate.guard -> {my_pipeline()}")
+    print("\nRuntime Gate demo complete")
+except Exception as e:
+    print(f"  RUNTIME GATE FAILED: {type(e).__name__}")
+    traceback.print_exc()
 
 
 # COMMAND ----------
@@ -470,9 +517,10 @@ rule_norm = IntentNormalizer(backend=RuleBasedNormalizerBackend())
 llm_norm = None
 if HAS_LLM:
     provider = make_provider()
-    llm_norm = IntentNormalizer(backend=LLMNormalizerBackend(provider=provider, strict=True))
+    # strict=False for comparison — we want to see the LLM output, not block it
+    llm_norm = IntentNormalizer(backend=LLMNormalizerBackend(provider=provider, strict=False))
 
-header = f"{'Intent':<50s} | {'Method':<12s} | {'Domain':<10s} | {'Layer':<8s} | {'Confidence':>10s}"
+header = f"{'Intent':<50s} | {'Method':<12s} | {'Domain':<14s} | {'Layer':<8s} | {'Confidence':>10s}"
 print(header)
 print("-" * len(header))
 
@@ -484,22 +532,21 @@ for raw in [
     # Rule-based
     rb = rule_norm.normalize(raw, {}, CATALOG)
     sig_rb = rb.signature
-    print(f"{raw[:47]:<50s} | {'rule-based':<12s} | {sig_rb.domain:<10s} | {sig_rb.target_layer:<8s} | {rb.confidence_score:>9.2f}")
+    print(f"{raw[:47]:<50s} | {'rule-based':<12s} | {sig_rb.domain:<14s} | {str(sig_rb.target_layer):<8s} | {rb.confidence_score:>9.2f}")
     # LLM
     if llm_norm is not None:
         try:
             llm = llm_norm.normalize(raw, {}, CATALOG)
             sig_llm = llm.signature
-            print(f"{'':50s} | {'LLM':<12s} | {sig_llm.domain:<10s} | {sig_llm.target_layer:<8s} | {llm.confidence_score:>9.2f}")
+            print(f"{'':50s} | {'LLM':<12s} | {sig_llm.domain:<14s} | {str(sig_llm.target_layer):<8s} | {llm.confidence_score:>9.2f}")
         except Exception as e:
-            print(f"{'':50s} | {'LLM':<12s} | ERROR: {type(e).__name__}")
+            print(f"{'':50s} | {'LLM':<12s} | ERROR: {type(e).__name__}: {str(e)[:60]}")
     else:
         print(f"{'':50s} | {'LLM':<12s} | (no API key)")
     print()
 
 print("Normalizer comparison complete")
 
-
 # COMMAND ----------
 
 # MAGIC %md
@@ -518,13 +565,19 @@ print("=" * 55)
 print("POLICY ENGINE WITH LLM-GENERATED RULES")
 print("=" * 55)
 
-if rules is None:
-    print("SKIP: rules not available (LLM Systematizer requires API key)")
-    print("   Using default policy rules instead.")
+# Guard: 'rules' is set by Section 6 (LLM Systematizer).
+try:
+    _rules_s10 = rules
+except NameError:
+    _rules_s10 = None
+
+if _rules_s10 is None:
+    print("NOTE: LLM-generated rules not available (Section 6 skipped or no API key).")
+    print("      Using default CFA policy rules instead.")
     engine = PolicyEngine()  # default rules
 else:
-    print(f"  Using {len(rules)} LLM-generated rules from Section 6")
-    engine = PolicyEngine(rules=rules)
+    print(f"  Using {len(_rules_s10)} LLM-generated rules from Section 6")
+    engine = PolicyEngine(rules=_rules_s10)
 
 # --- Safe signature ---
 sig_safe = StateSignature(
@@ -588,14 +641,14 @@ print("\nLLM-rules PolicyEngine complete")
 # MAGIC
 # MAGIC | LLM Feature | Status | Notes |
 # MAGIC |---|---|---|
-# MAGIC | LLM Normalizer | \\u2713 | Semantic intent resolution via DeepSeek |
-# MAGIC | Strict Mode | \\u2713 | LLM output validated against catalog |
-# MAGIC | LLM Audit Trail | \\u2713 | Every call SHA-256 traceable |
-# MAGIC | LLM Systematizer | \\u2713 | NL → BehaviorSpec → PolicyRules |
-# MAGIC | Full Kernel + LLM | \\u2713 | End-to-end with LLM normalizer |
-# MAGIC | Runtime Gate | \\u2713 | Guard with LLM-generated policy rules |
-# MAGIC | Normalizer Comparison | \\u2713 | Rule-based vs LLM side-by-side |
-# MAGIC | Secret Management | \\u2713 | Databricks Secret Scope `cfa-secrets` |
+# MAGIC | LLM Normalizer | ✓ | Semantic intent resolution via DeepSeek/OpenAI |
+# MAGIC | Strict Mode | ✓ | LLM output validated against catalog |
+# MAGIC | LLM Audit Trail | ✓ | Every call SHA-256 traceable |
+# MAGIC | LLM Systematizer | ✓ | NL → BehaviorSpec → PolicyRules |
+# MAGIC | Full Kernel + LLM | ✓ | End-to-end with LLM normalizer |
+# MAGIC | Runtime Gate | ✓ | Guard with LLM-generated policy rules |
+# MAGIC | Normalizer Comparison | ✓ | Rule-based vs LLM side-by-side |
+# MAGIC | Secret Management | ✓ | Databricks Secret Scope `cfa-secrets` |
 # MAGIC
 # MAGIC **Key differences from rule-based:**
 # MAGIC

{cfa_kernel-0.1.7 → cfa_kernel-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cfa-kernel
-Version: 0.1.7
+Version: 0.1.9
 Summary: CFA — Governed execution for AI agents and data systems
 Project-URL: Homepage, https://marquesantero.github.io/cfa/
 Project-URL: Repository, https://github.com/marquesantero/cfa
@@ -38,7 +38,7 @@ Provides-Extra: yaml
 Requires-Dist: pyyaml>=6.0; extra == 'yaml'
 Description-Content-Type: text/markdown
 
-# CFA v0.1.7
+# CFA v0.1.9
 
 [![CI](https://github.com/marquesantero/cfa/actions/workflows/ci.yml/badge.svg)](https://github.com/marquesantero/cfa/actions/workflows/ci.yml)
 [![codecov](https://codecov.io/github/marquesantero/cfa/graph/badge.svg?token=P5NFQBZGYT)](https://codecov.io/github/marquesantero/cfa)