churnkit 1.2.2a2__tar.gz → 1.2.2a4__tar.gz

{churnkit-1.2.2a2 → churnkit-1.2.2a4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: churnkit
-Version: 1.2.2a2
+Version: 1.2.2a4
 Summary: Structured ML framework for customer churn prediction -- from exploration notebooks to production pipelines, locally or on Databricks.
 Project-URL: Homepage, https://github.com/aladjov/CR
 Project-URL: Documentation, https://github.com/aladjov/CR/wiki

{churnkit-1.2.2a2 → churnkit-1.2.2a4}/causal_notebooks/c02_archetype_derivation.ipynb

@@ -4,7 +4,20 @@
    "cell_type": "markdown",
    "id": "046ea43d",
    "metadata": {},
-   "source": "[//]: # (cr:doc name='chapter_c02_archetype_derivation' id=046ea43d)\n# Chapter c02: Archetype Derivation (Causal Track)\n\nHeart of the causal track:\n1. Loads the `@production` model + the gold features.\n2. Loads the frozen `ShapAttribution` artifact persisted with the training run (per-feature importance + background means — no background sample is re-computed here).\n3. Emits per-row linear SHAP via a single distributed `spark_df.select(...)` — `shap_i(row) = importance_i * (x_i - background_mean_i)`. No `TreeExplainer`, no `pandas_udf`.\n4. Pre-selects top features by mean |SHAP| (`KMEANS_FEATURE_CAP`).\n5. Runs a silhouette-swept Spark KMeans over the reduced SHAP space.\n6. Fits per-cluster surrogate trees → JSON predicates.\n7. Maps archetypes ↔ playbooks via feature-overlap baseline + optional LLM refinement.\n8. Writes `archetype_catalog` + `eligibility_policy` rows as `pending_review` for the c03 approval gate.\n"
+   "source": [
+    "[//]: # (cr:doc name='chapter_c02_archetype_derivation' id=046ea43d)\n",
+    "# Chapter c02: Archetype Derivation (Causal Track)\n",
+    "\n",
+    "Heart of the causal track:\n",
+    "1. Loads the `@production` model + the gold features.\n",
+    "2. Freezes a stratified SHAP background sample.\n",
+    "3. Computes per-row SHAP via partition-wise `pandas_udf` (TreeExplainer with the frozen background).\n",
+    "4. Pre-selects top features by mean |SHAP| (`KMEANS_FEATURE_CAP`).\n",
+    "5. Runs a silhouette-swept Spark KMeans over the reduced SHAP space.\n",
+    "6. Fits per-cluster surrogate trees → JSON predicates.\n",
+    "7. Maps archetypes ↔ playbooks via feature-overlap baseline + optional LLM refinement.\n",
+    "8. Writes `archetype_catalog` + `eligibility_policy` rows as `pending_review` for the c03 approval gate.\n"
+   ]
   },
   {
    "cell_type": "code",
@@ -236,22 +249,6 @@
     "    llm_namer = build_llm_namer(LLM_ENDPOINT_NAME)\n",
     "    print(f\"Resolved LLM matcher: {llm_namer.model_id}\")\n",
     "\n",
-    "    try:\n",
-    "        from customer_retention.analysis.auto_explorer.run_namespace import RunNamespace\n",
-    "        from customer_retention.stages.causal.interpretation import (\n",
-    "            enrich_archetype_from_namespace,\n",
-    "        )\n",
-    "        _enrich_ns = RunNamespace.from_env_or_latest(get_experiments_dir())\n",
-    "\n",
-    "        def _enrichment_builder(_arch, _ctx):\n",
-    "            return enrich_archetype_from_namespace(\n",
-    "                _ctx, _enrich_ns, composite_name=COMPOSITE_NAME or \"\",\n",
-    "            )\n",
-    "        print(f\"Enrichment builder active: namespace={_enrich_ns.run_dir}\")\n",
-    "    except Exception as _enrich_exc:\n",
-    "        print(f\"Enrichment disabled ({type(_enrich_exc).__name__}: {_enrich_exc}) — LLM prompt falls back to raw column names.\")\n",
-    "        _enrichment_builder = None\n",
-    "\n",
     "    cfg = DerivationConfig(\n",
     "        spark=spark,\n",
     "        training_df=training_df,\n",
@@ -271,8 +268,6 @@
     "        feature_cap=KMEANS_FEATURE_CAP,\n",
     "        llm_endpoint_name=LLM_ENDPOINT_NAME,\n",
     "        llm_namer=llm_namer,\n",
-    "        enrichment_builder=_enrichment_builder,\n",
-    "        namespace=_enrich_ns,\n",
     "        fit_thresholds=FitThresholds(\n",
     "            auto=float(FIT_AUTO_THRESHOLD),\n",
     "            review=float(FIT_REVIEW_THRESHOLD),\n",
@@ -288,12 +283,54 @@
    ]
   },
   {
-   "cell_type": "code",
-   "id": "a20abbe5",
-   "source": "# @cr:code name='backfill_eligibility_rules_prose' id=a20abbe5\n# Re-render `eligibility_rules_prose` for every existing active policy row whose\n# prose column is NULL — derivation populates the column at row-creation time\n# only, so rows written before column_descriptions / feature_meta /\n# feature_population_stats sidecars existed stay NULL until we rewrite them.\n# Cycle 013 P4 surfaced exactly this gap; the backfill closes it idempotently.\nif is_databricks() and spark is not None and spark.catalog.tableExists(ELIGIBILITY_POLICY_FQN):\n    from customer_retention.stages.causal.interpretation import (\n        backfill_eligibility_prose,\n    )\n\n    _backfill = backfill_eligibility_prose(\n        spark, ELIGIBILITY_POLICY_FQN, namespace=_enrich_ns,\n    )\n    print(_backfill.summary())\n    if _backfill.warnings:\n        print(\"(interpretation-layer warnings — see logs for details)\")\n        for _w in _backfill.warnings:\n            print(f\"  - {_w}\")\n",
+   "cell_type": "markdown",
+   "id": "2e42d59c",
    "metadata": {},
+   "source": [
+    "[//]: # (cr:doc name='c02_backfill_prose_section' id=2e42d59c)\n",
+    "## 2.2 Backfill `eligibility_rules_prose`\n"
+   ]
+  },
+  {
+   "cell_type": "code",
    "execution_count": null,
-   "outputs": []
+   "id": "04993f07",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# @cr:code name='backfill_eligibility_rules_prose' id=04993f07\n",
+    "# Re-render `eligibility_rules_prose` for every existing active policy row whose\n",
+    "# prose column is NULL — derivation populates the column at row-creation time\n",
+    "# only, so rows written before column_descriptions / feature_meta /\n",
+    "# feature_population_stats sidecars existed stay NULL until we rewrite them.\n",
+    "# Cycle 013 P4 surfaced exactly this gap; the backfill closes it idempotently.\n",
+    "if is_databricks() and spark is not None and spark.catalog.tableExists(ELIGIBILITY_POLICY_FQN):\n",
+    "    from customer_retention.analysis.auto_explorer.run_namespace import RunNamespace\n",
+    "    from customer_retention.stages.causal.interpretation import (\n",
+    "        backfill_eligibility_prose,\n",
+    "    )\n",
+    "\n",
+    "    # Resolve the namespace locally so this cell runs standalone — the\n",
+    "    # derivation cell only defines `_enrich_ns` on the freshly-derive path,\n",
+    "    # so a backfill-only rerun (active archetypes already exist) would\n",
+    "    # otherwise NameError on the reference below.\n",
+    "    _backfill_ns = globals().get(\"_enrich_ns\")\n",
+    "    if _backfill_ns is None:\n",
+    "        try:\n",
+    "            _backfill_ns = RunNamespace.from_env_or_latest(get_experiments_dir())\n",
+    "        except Exception as _ns_exc:\n",
+    "            print(f\"Backfill namespace lookup failed ({type(_ns_exc).__name__}: {_ns_exc}) — proceeding without sidecar context.\")\n",
+    "            _backfill_ns = None\n",
+    "\n",
+    "    _backfill = backfill_eligibility_prose(\n",
+    "        spark, ELIGIBILITY_POLICY_FQN, namespace=_backfill_ns,\n",
+    "    )\n",
+    "    print(_backfill.summary())\n",
+    "    if _backfill.warnings:\n",
+    "        print(\"(interpretation-layer warnings — see logs for details)\")\n",
+    "        for _w in _backfill.warnings:\n",
+    "            print(f\"  - {_w}\")\n"
+   ]
   },
   {
    "cell_type": "code",

{churnkit-1.2.2a2 → churnkit-1.2.2a4}/causal_notebooks/c05_snapshot_and_dashboard.ipynb

@@ -63,7 +63,10 @@
    "cell_type": "markdown",
    "id": "96b40ce2",
    "metadata": {},
-   "source": "[//]: # (cr:doc name='c05_configuration' id=96b40ce2)\n## Configuration\n\nThe cell below is the only place you should need to edit. Every value here is read by the snapshot writer, the per-slice SHAP writer, and the dashboard publisher — nothing is hardcoded inside the algorithmic cells.\n\n- **`SNAPSHOT_RISK_TIER_HIGH` / `SNAPSHOT_RISK_TIER_MEDIUM`** — risk-tier thresholds applied at snapshot time. Leave as `None` to fall back to the values stored on the active `decision_policy` row (the canonical source — set in `c01_publish_definitions`).\n- **`SNAPSHOT_CAPACITY_PARTITION_COLUMN`** — optional partition column for capacity caps (e.g. `\"csm_owner_id\"`). Leave as `\"\"` to apply caps globally per playbook.\n- **`SHAP_PER_SLICE_K`** — per `(playbook, archetype, risk_tier)` slice, this many top-ranked accounts (by `expected_loss = churn_probability × value_at_risk`, breaking ties on `churn_probability`) get per-row SHAP attributed and persisted to `top_shap_drivers`. This is the **only** cap controlling how many rows reach the L3 cohort dashboard view (`v_eligible_all_playbooks`) — slices without SHAP simply do not surface. Set to `0` to skip SHAP enrichment entirely.\n- **`SHAP_TOP_DRIVERS_PER_ROW`** — number of top drivers (by `|shap_contribution|`) emitted per scored account into the `top_drivers` array."
+   "source": [
+    "[//]: # (cr:doc name='c05_configuration' id=96b40ce2)\n",
+    "## Configuration\n\nThe cell below is the only place you should need to edit. Every value here is read by the snapshot writer and the dashboard publisher — nothing is hardcoded inside the algorithmic cells.\n\n- **`SNAPSHOT_RISK_TIER_HIGH` / `SNAPSHOT_RISK_TIER_MEDIUM`** — risk-tier thresholds applied at snapshot time. Leave as `None` to fall back to the values stored on the active `decision_policy` row (the canonical source — set in `c01_publish_definitions`).\n- **`SNAPSHOT_CAPACITY_PARTITION_COLUMN`** — optional partition column for capacity caps (e.g. `\"csm_owner_id\"`). Leave as `\"\"` to apply caps globally per playbook.\n"
+   ]
   },
   {
    "cell_type": "code",
@@ -75,10 +78,7 @@
     "# @cr:config name='configuration' id=2586e099\n",
     "SNAPSHOT_RISK_TIER_HIGH = None\n",
     "SNAPSHOT_RISK_TIER_MEDIUM = None\n",
-    "SNAPSHOT_CAPACITY_PARTITION_COLUMN = \"\"\n",
-    "\n",
-    "SHAP_PER_SLICE_K = 20\n",
-    "SHAP_TOP_DRIVERS_PER_ROW = 5\n"
+    "SNAPSHOT_CAPACITY_PARTITION_COLUMN = \"\"\n"
    ]
   },
   {
@@ -204,67 +204,21 @@
   },
   {
    "cell_type": "markdown",
-   "id": "48bcbe87",
+   "id": "9c2e5465",
    "metadata": {},
-   "source": "[//]: # (cr:doc name='c05_top_shap_section' id=48bcbe87)\n## 5.2 Compute Per-Slice Top SHAP Drivers\n\nPicks the top `SHAP_PER_SLICE_K` accounts from each `(playbook, archetype, risk_tier)` slice — ranked by `expected_loss DESC, churn_probability DESC` over the snapshot just written — joins their latest gold-feature row, runs a single distributed linear-SHAP `select` (importance × (x − background_mean) using the frozen `shap_attribution.json` artifact from training), and reduces each row to its `SHAP_TOP_DRIVERS_PER_ROW` strongest drivers by `|shap_contribution|`. The result is merged into `top_shap_drivers` keyed on `(model_name, model_version, entity_id)`.\n\nThe L3 cohort view (`v_eligible_all_playbooks`) shows only rows that have these drivers populated — the per-slice K is the single cap controlling its size, replacing the previous defense-in-depth `slice_rank <= 500` SQL filter. The L1 portfolio matrix and L2 archetype rollup remain populated from the full eligibility snapshot, so totals and capacity widgets stay truthful regardless of the SHAP cap.\n\nSkipped automatically when no model URI / attribution artifact is reachable (local runs without a trained MLflow run) or when `SHAP_PER_SLICE_K = 0`."
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6509814d",
-   "metadata": {},
-   "outputs": [],
    "source": [
-    "# @cr:code name='compute_top_shap_drivers' id=6509814d\n",
-    "\n",
-    "from customer_retention.stages.causal import (\n",
-    "    TopDriversConfig,\n",
-    "    compute_and_write_top_shap_drivers,\n",
-    ")\n",
-    "\n",
-    "top_drivers_result = None\n",
-    "if spark is None:\n",
-    "    print(\"SKIPPED: no Spark session (Databricks-only cell)\")\n",
-    "elif snapshot_result is None:\n",
-    "    print(\"SKIPPED: snapshot_result is None — 5.1 did not produce a run\")\n",
-    "elif int(SHAP_PER_SLICE_K) <= 0:\n",
-    "    print(\"SKIPPED: SHAP_PER_SLICE_K == 0 (per-slice SHAP enrichment disabled)\")\n",
-    "elif MODEL_URI is None:\n",
-    "    print(\"SKIPPED: MODEL_URI is None — local runs without an MLflow attribution artifact cannot replay SHAP\")\n",
-    "elif not spark.catalog.tableExists(GOLD_FEATURES_FQN):\n",
-    "    print(f\"SKIPPED: {GOLD_FEATURES_FQN} does not exist — gold features required for SHAP replay\")\n",
-    "else:\n",
-    "    top_drivers_cfg = TopDriversConfig(\n",
-    "        spark=spark,\n",
-    "        snapshot_table_fqn=ELIGIBILITY_SNAPSHOT_FQN,\n",
-    "        gold_features_fqn=GOLD_FEATURES_FQN,\n",
-    "        top_shap_drivers_fqn=TOP_SHAP_DRIVERS_FQN,\n",
-    "        model_name=MODEL_NAME,\n",
-    "        model_version=str(MODEL_VERSION),\n",
-    "        scoring_run_id=snapshot_result.scoring_run_id,\n",
-    "        as_of_date=snapshot_result.as_of_date,\n",
-    "        model_uri=MODEL_URI,\n",
-    "        per_slice_k=int(SHAP_PER_SLICE_K),\n",
-    "        top_drivers_per_row=int(SHAP_TOP_DRIVERS_PER_ROW),\n",
-    "    )\n",
-    "    top_drivers_result = compute_and_write_top_shap_drivers(top_drivers_cfg)\n",
-    "    print(top_drivers_result.summary())\n"
+    "[//]: # (cr:doc name='c05_write_run_context_section' id=9c2e5465)\n",
+    "## 5.2 Write Run Context (app masthead projection)\n"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "834012b4",
-   "metadata": {},
-   "source": "[//]: # (cr:doc name='c05_write_run_context_section' id=834012b4)\n## 5.3 Write Run Context (app masthead projection)\n\nUpserts a single row into `{catalog}.{schema}.run_context` keyed on the snapshot's `scoring_run_id`. Reads the optional `project_context.yaml` from the run namespace to populate horizon, primary objective, and temporal posture; when the YAML is unreachable the writer still emits a row so the dashboard view always returns model metadata. The companion `v_run_context` view (published in 5.4) selects the row with the most recent `as_of_date`, so re-running this cell is an idempotent MERGE."
-  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "6b63ca6a",
+   "id": "9a423d80",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# @cr:code name='write_run_context' id=6b63ca6a\n",
+    "# @cr:code name='write_run_context' id=9a423d80\n",
     "from customer_retention.stages.causal import (\n",
     "    from_project_context,\n",
     "    write_run_context,\n",
@@ -305,7 +259,6 @@
     "        from mlflow.models import Model as _MlflowModel\n",
     "        _info = _MlflowModel.load(MODEL_URI)\n",
     "        _flavors = list((_info.flavors or {}).keys())\n",
-    "        # Prefer the most specific flavor over ``python_function``.\n",
     "        for preferred in (\"xgboost\", \"lightgbm\", \"catboost\", \"pytorch\", \"tensorflow\", \"sklearn\"):\n",
     "            if preferred in _flavors:\n",
     "                return preferred\n",
@@ -346,7 +299,10 @@
    "cell_type": "markdown",
    "id": "a6cbe88f",
    "metadata": {},
-   "source": "[//]: # (cr:doc name='c05_publish_views_section' id=a6cbe88f)\n## 5.4 Publish Dashboard SQL Views"
+   "source": [
+    "[//]: # (cr:doc name='c05_publish_views_section' id=a6cbe88f)\n",
+    "## 5.3 Publish Dashboard SQL Views\n"
+   ]
   },
   {
    "cell_type": "code",
@@ -357,7 +313,6 @@
    "source": [
     "# @cr:code name='publish_dashboard_views' id=00cf348e\n",
     "from customer_retention.stages.causal.dashboard_views import (\n",
-    "    DASHBOARD_DEVIATION_VIEW_NAMES,\n",
     "    DASHBOARD_VIEW_NAMES,\n",
     "    publish_dashboard_views,\n",
     ")\n",
@@ -367,26 +322,20 @@
     "elif not spark.catalog.tableExists(ELIGIBILITY_SNAPSHOT_FQN):\n",
     "    print(f\"SKIPPED: {ELIGIBILITY_SNAPSHOT_FQN} not populated yet\")\n",
     "else:\n",
-    "    # Pass composite_name so the per-run deviation views\n",
-    "    # (v_account_feature_deviation / _topn) are also published -- they\n",
-    "    # reference gold_features_<composite_name> and stay skipped when\n",
-    "    # composite_name is None.\n",
-    "    statements = publish_dashboard_views(\n",
-    "        spark, CATALOG, SCHEMA, composite_name=COMPOSITE_NAME or None\n",
-    "    )\n",
+    "    statements = publish_dashboard_views(spark, CATALOG, SCHEMA)\n",
     "    print(f\"Published {len(statements)} dashboard views:\")\n",
     "    for view_name in DASHBOARD_VIEW_NAMES:\n",
-    "        print(f\"  - {CATALOG}.{SCHEMA}.{view_name}\")\n",
-    "    if COMPOSITE_NAME:\n",
-    "        for view_name in DASHBOARD_DEVIATION_VIEW_NAMES:\n",
-    "            print(f\"  - {CATALOG}.{SCHEMA}.{view_name}\")\n"
+    "        print(f\"  - {CATALOG}.{SCHEMA}.{view_name}\")\n"
    ]
   },
   {
    "cell_type": "markdown",
    "id": "fd17e5f0",
    "metadata": {},
-   "source": "[//]: # (cr:doc name='c05_summary_section' id=fd17e5f0)\n## 5.5 Print Run Summary"
+   "source": [
+    "[//]: # (cr:doc name='c05_summary_section' id=fd17e5f0)\n",
+    "## 5.4 Print Run Summary\n"
+   ]
   },
   {
    "cell_type": "code",
@@ -397,7 +346,7 @@
    "source": [
     "# @cr:code name='print_run_summary' id=b2232bcf\n",
     "if spark is None or not spark.catalog.tableExists(ARCHETYPE_CATALOG_FQN):\n",
-    "    print(\"(no archetype_catalog yet \\u2014 run c01..c03 first)\")\n",
+    "    print(\"(no archetype_catalog yet — run c01..c03 first)\")\n",
     "else:\n",
     "    counts = spark.sql(\n",
     "        f\"SELECT status, COUNT(*) AS n FROM {ARCHETYPE_CATALOG_FQN} GROUP BY status\"\n",
@@ -407,30 +356,7 @@
     "        print(f\"  {row['status']}: {row['n']}\")\n",
     "    print(f\"Model: {MODEL_NAME} v{MODEL_VERSION}\")\n",
     "    if snapshot_result is not None:\n",
-    "        print(snapshot_result.summary())\n",
-    "    if top_drivers_result is not None:\n",
-    "        print(top_drivers_result.summary())\n",
-    "    # SHAP-readiness check for the dashboard's right panel. The default\n",
-    "    # profile (and the ECH/SPS overrides) render the SHAP drivers panel\n",
-    "    # from ``v_account_explanation.account_top_shap_features`` -- sourced\n",
-    "    # from ``top_shap_drivers`` (per-row) with a fallback to\n",
-    "    # ``archetype_catalog.top_shap_features`` (per-archetype). If the row\n",
-    "    # count below is zero, the dashboard will show no SHAP bars; rerun\n",
-    "    # 5.2 ``compute_top_shap_drivers`` first.\n",
-    "    if spark.catalog.tableExists(TOP_SHAP_DRIVERS_FQN) and snapshot_result is not None:\n",
-    "        shap_counts = spark.sql(\n",
-    "            f\"SELECT COUNT(*) AS n, COUNT(DISTINCT entity_id) AS entities \"\n",
-    "            f\"FROM {TOP_SHAP_DRIVERS_FQN} \"\n",
-    "            f\"WHERE model_name = '{MODEL_NAME}' \"\n",
-    "            f\"  AND model_version = '{MODEL_VERSION}'\"\n",
-    "        ).collect()[0]\n",
-    "        print(\n",
-    "            f\"top_shap_drivers (model={MODEL_NAME} v{MODEL_VERSION}): \"\n",
-    "            f\"{shap_counts['n']:,} rows / {shap_counts['entities']:,} entities -- \"\n",
-    "            f\"{'ready' if shap_counts['entities'] > 0 else 'EMPTY (rerun 5.2)'}\"\n",
-    "        )\n",
-    "    else:\n",
-    "        print(\"top_shap_drivers: not populated -- rerun 5.2 to enable the dashboard SHAP panel\")\n"
+    "        print(snapshot_result.summary())\n"
    ]
   },
   {

{churnkit-1.2.2a2 → churnkit-1.2.2a4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "churnkit"
-version = "1.02.2a2"
+version = "1.02.2a4"
 description = "Structured ML framework for customer churn prediction -- from exploration notebooks to production pipelines, locally or on Databricks."
 readme = "README.md"
 license = {text = "Apache-2.0"}

{churnkit-1.2.2a2 → churnkit-1.2.2a4}/scripts/notebooks/build_causal_notebooks.py

@@ -438,12 +438,25 @@ _C02_BACKFILL_PROSE_BODY = '''# Re-render `eligibility_rules_prose` for every ex
 # feature_population_stats sidecars existed stay NULL until we rewrite them.
 # Cycle 013 P4 surfaced exactly this gap; the backfill closes it idempotently.
 if is_databricks() and spark is not None and spark.catalog.tableExists(ELIGIBILITY_POLICY_FQN):
+    from customer_retention.analysis.auto_explorer.run_namespace import RunNamespace
     from customer_retention.stages.causal.interpretation import (
         backfill_eligibility_prose,
     )
 
+    # Resolve the namespace locally so this cell runs standalone — the
+    # derivation cell only defines `_enrich_ns` on the freshly-derive path,
+    # so a backfill-only rerun (active archetypes already exist) would
+    # otherwise NameError on the reference below.
+    _backfill_ns = globals().get("_enrich_ns")
+    if _backfill_ns is None:
+        try:
+            _backfill_ns = RunNamespace.from_env_or_latest(get_experiments_dir())
+        except Exception as _ns_exc:
+            print(f"Backfill namespace lookup failed ({type(_ns_exc).__name__}: {_ns_exc}) — proceeding without sidecar context.")
+            _backfill_ns = None
+
     _backfill = backfill_eligibility_prose(
-        spark, ELIGIBILITY_POLICY_FQN, namespace=_enrich_ns,
+        spark, ELIGIBILITY_POLICY_FQN, namespace=_backfill_ns,
     )
     print(_backfill.summary())
     if _backfill.warnings:

{churnkit-1.2.2a2 → churnkit-1.2.2a4}/src/customer_retention/__init__.py

@@ -17,7 +17,7 @@ Main module categories:
                 llm_context, iteration)
 """
 
-__version__ = "1.02.2a2"
+__version__ = "1.02.2a4"
 
 # Environment utilities (always available)
 from .core.compat import (

{churnkit-1.2.2a2 → churnkit-1.2.2a4}/src/customer_retention/stages/causal/dashboard_views.py

@@ -31,6 +31,13 @@ _VIEW_FILE_NAME = "dashboard_views.sql"
 _DEVIATION_BLOCK_OPEN = "-- @cr:deviation-block:open"
 _DEVIATION_BLOCK_CLOSE = "-- @cr:deviation-block:close"
 
+# Sentinel-delimited block for v_feature_provenance — gated on the optional
+# feature_meta + column_descriptions tables existing in UC. Older causal-track
+# builds did not produce these tables; the view's CREATE OR REPLACE would fail
+# at validation time on those clusters and take the whole publish call down.
+_PROVENANCE_BLOCK_OPEN = "-- @cr:provenance-block:open"
+_PROVENANCE_BLOCK_CLOSE = "-- @cr:provenance-block:close"
+
 DASHBOARD_VIEW_NAMES: tuple[str, ...] = (
     "v_ranked_at_risk_customers",
     "v_archetype_overview",
@@ -44,12 +51,14 @@ DASHBOARD_VIEW_NAMES: tuple[str, ...] = (
     "v_eligible_all_playbooks",
     "v_account_explanation",
     "v_run_context",
-    "v_feature_provenance",
 )
 DASHBOARD_DEVIATION_VIEW_NAMES: tuple[str, ...] = (
     "v_account_feature_deviation",
     "v_account_feature_deviation_topn",
 )
+DASHBOARD_PROVENANCE_VIEW_NAMES: tuple[str, ...] = (
+    "v_feature_provenance",
+)
 
 
 def load_dashboard_view_sql() -> str:
@@ -82,25 +91,50 @@ def _strip_deviation_markers(sql_text: str) -> str:
     )
 
 
+def _strip_provenance_block(sql_text: str) -> str:
+    """Drop the ``@cr:provenance-block`` section so the rendered SQL parses
+    on clusters that don't yet have ``feature_meta`` / ``column_descriptions``.
+    """
+    pattern = re.compile(
+        re.escape(_PROVENANCE_BLOCK_OPEN) + r".*?" + re.escape(_PROVENANCE_BLOCK_CLOSE),
+        re.DOTALL,
+    )
+    return pattern.sub("", sql_text)
+
+
+def _strip_provenance_markers(sql_text: str) -> str:
+    return "\n".join(
+        line for line in sql_text.splitlines()
+        if line.strip() not in (_PROVENANCE_BLOCK_OPEN, _PROVENANCE_BLOCK_CLOSE)
+    )
+
+
 def render_dashboard_view_sql(
     catalog: str,
     schema: str,
     *,
     composite_name: Optional[str] = None,
+    include_provenance: bool = True,
 ) -> str:
     """Substitute ``{catalog}`` / ``{schema}`` (and optionally ``{composite_name}``)
     into the raw SQL template.
 
     When ``composite_name`` is omitted, the deviation views (which reference
     ``gold_features_{composite_name}``) are stripped from the output so the
-    remaining DDL stays parseable.  Existing callers that don't supply the
-    parameter keep their previous behaviour.
+    remaining DDL stays parseable. ``include_provenance=False`` strips the
+    ``v_feature_provenance`` block — used by the publisher when the optional
+    ``feature_meta`` / ``column_descriptions`` tables aren't materialized yet
+    so the view's CREATE OR REPLACE wouldn't validate.
     """
     text = load_dashboard_view_sql()
     if composite_name:
         text = _strip_deviation_markers(text).replace("{composite_name}", composite_name)
     else:
         text = _strip_deviation_block(text)
+    if include_provenance:
+        text = _strip_provenance_markers(text)
+    else:
+        text = _strip_provenance_block(text)
     return text.replace("{catalog}", catalog).replace("{schema}", schema)
 
 
@@ -230,6 +264,29 @@ def _try_materialize_population_stats_from_sidecar(
     return spark.catalog.tableExists(fqn)
 
 
+_PROVENANCE_PREREQ_TABLES: tuple[str, ...] = (
+    "feature_meta",
+    "column_descriptions",
+)
+
+
+def _provenance_prerequisites_present(
+    spark: "SparkSession", catalog: str, schema: str
+) -> tuple[bool, list[str]]:
+    """Return ``(ok, missing)`` for tables ``v_feature_provenance`` reads.
+
+    Older causal-track builds did not write ``feature_meta`` /
+    ``column_descriptions`` — on those clusters the provenance view is
+    skipped so the rest of the dashboard publishes successfully.
+    """
+    missing: list[str] = []
+    for tbl in _PROVENANCE_PREREQ_TABLES:
+        fqn = f"{catalog}.{schema}.{tbl}"
+        if not spark.catalog.tableExists(fqn):
+            missing.append(fqn)
+    return (not missing), missing
+
+
 def _deviation_prerequisites_present(
     spark: "SparkSession", catalog: str, schema: str, composite_name: str
 ) -> tuple[bool, list[str]]:
@@ -296,7 +353,21 @@ def publish_dashboard_views(
             )
             effective_composite = None
 
-    rendered = render_dashboard_view_sql(catalog, schema, composite_name=effective_composite)
+    include_provenance, prov_missing = _provenance_prerequisites_present(spark, catalog, schema)
+    if not include_provenance:
+        logger.warning(
+            "v_feature_provenance skipped: missing prerequisite table(s) %s; "
+            "publishing the rest of the dashboard. Re-run gold materialization "
+            "to populate feature_meta / column_descriptions and republish.",
+            ", ".join(prov_missing),
+        )
+
+    rendered = render_dashboard_view_sql(
+        catalog,
+        schema,
+        composite_name=effective_composite,
+        include_provenance=include_provenance,
+    )
     statements = split_view_statements(rendered)
     submitted: List[str] = []
     for stmt in statements:

{churnkit-1.2.2a2 → churnkit-1.2.2a4}/src/customer_retention/stages/causal/sql/dashboard_views.sql

@@ -811,6 +811,7 @@ FROM ranked
 WHERE deviation_rank <= 12;
 -- @cr:deviation-block:close
 
+-- @cr:provenance-block:open
 -- ============================================================================
 -- 14. v_feature_provenance (per-feature lineage + business definition)
 -- ----------------------------------------------------------------------------
@@ -884,3 +885,4 @@ SELECT
     cd.source_column_defs
 FROM fm
 LEFT JOIN col_defs cd ON cd.feature_name = fm.feature_name;
+-- @cr:provenance-block:close