churnkit 1.2.8a2__tar.gz → 1.2.8a3__tar.gz

{churnkit-1.2.8a2 → churnkit-1.2.8a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: churnkit
-Version: 1.2.8a2
+Version: 1.2.8a3
 Summary: Structured ML framework for customer churn prediction -- from exploration notebooks to production pipelines, locally or on Databricks.
 Project-URL: Homepage, https://github.com/aladjov/CR
 Project-URL: Documentation, https://github.com/aladjov/CR/wiki

{churnkit-1.2.8a2 → churnkit-1.2.8a3}/exploration_notebooks/10_spec_generation.ipynb

@@ -991,7 +991,7 @@
     "tags": []
    },
    "outputs": [],
-   "source": "# @cr:code name='run_pipeline' id=8b659505\nRUN_PIPELINE = True\n# Resume gate: skip any landing/bronze/silver/gold notebook whose output UC table\n# already exists. Operator-facing knob — flip to False for a forced full re-run\n# (e.g. when findings or generated code changed in a way that invalidates prior\n# tables). Training is never resume-skipped (no UC table to check; MLflow runs\n# are tracked separately).\nRESUME_FROM_EXISTING_TABLES = True\n\nif RUN_PIPELINE and GENERATION_TARGET == GenerationTarget.DATABRICKS:\n    import json as _json\n    import time as _time\n\n    def _resume_table_for(stage, nb_name, catalog, schema):\n        \"\"\"Resolve the UC table written by a generated stage notebook.\n\n        Conventions established by the Databricks renderer:\n          - landing/landing_<src>          -> {cat}.{sch}.landing_<src>\n          - bronze/bronze_entity_<src>     -> {cat}.{sch}.bronze_entity_<src>\n          - bronze/bronze_event_<src>      -> {cat}.{sch}.bronze_entity_<src>_events  (Shape A)\n          - bronze/bronze_entity_<src>_aggregated -> same name as notebook\n          - silver/silver_featureset_<CN>  -> same name as notebook\n          - gold/gold_features_<CN>        -> same name as notebook\n          - training/*                     -> None (no UC table)\n        \"\"\"\n        base = f\"{catalog}.{schema}\"\n        if stage == \"landing\":\n            return f\"{base}.{nb_name}\"\n        if stage == \"bronze\":\n            if nb_name.startswith(\"bronze_event_\"):\n                source = nb_name[len(\"bronze_event_\"):]\n                return f\"{base}.bronze_entity_{source}_events\"\n            return f\"{base}.{nb_name}\"\n        if stage in (\"silver\", \"gold\"):\n            return f\"{base}.{nb_name}\"\n        return None\n\n    print(f\"Databricks pipeline: {output_dir}\\n\")\n    if RESUME_FROM_EXISTING_TABLES:\n        print(\"[RESUME] table-existence resume enabled — stages with existing UC tables will be skipped\\n\")\n    spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {DATABRICKS_CATALOG}.{DATABRICKS_SCHEMA}\")\n    _ns_params = {\"experiments_dir\": str(_namespace.root), \"run_id\": _namespace.run_id} if _namespace else {}\n    _stages = [\"landing\", \"bronze\", \"silver\", \"gold\", \"training\"]\n    _total_start = _time.time()\n    _training_results = None\n    _silver_results = None\n    _skipped_count = 0\n    for _stage in _stages:\n        _stage_dir = output_dir / _stage\n        if not _stage_dir.exists():\n            continue\n        if _stage == \"bronze\":\n            # (retires NB10 patch §2.11): events must run before\n            # entities so the aggregator inputs exist when consumers try\n            # to read them. Default sorted() puts `bronze_entity_*`\n            # before `bronze_event_*` (i < v) and raises\n            # [TABLE_OR_VIEW_NOT_FOUND] on the first aggregated bronze.\n            _notebooks = sorted(\n                (f.stem for f in _stage_dir.iterdir() if f.suffix == \".py\"),\n                key=lambda n: (0 if n.startswith(\"bronze_event_\") else 1, n),\n            )\n            print(f\"[bronze order] {len(_notebooks)} notebooks; events first:\")\n            for _n in _notebooks[:6]:\n                print(f\"   {_n}\")\n            if len(_notebooks) > 6:\n                print(f\"   ... + {len(_notebooks) - 6} more\")\n        else:\n            _notebooks = sorted(f.stem for f in _stage_dir.iterdir() if f.suffix == \".py\")\n        for _nb in _notebooks:\n            _path = str(output_dir / _stage / _nb)\n\n            if RESUME_FROM_EXISTING_TABLES:\n                _expected = _resume_table_for(_stage, _nb, DATABRICKS_CATALOG, DATABRICKS_SCHEMA)\n                if _expected and spark.catalog.tableExists(_expected):\n                    print(f\"[SKIP] {_stage.upper()}/{_nb}: {_expected} already exists\")\n                    _skipped_count += 1\n                    continue\n\n            _start = _time.time()\n            _result = dbutils.notebook.run(_path, 86400, _ns_params)\n            _elapsed = _time.time() - _start\n            print(f\"[{_stage.upper()}] {_nb}: {_elapsed:.1f}s\")\n            # Generated stage notebooks structured-exit on top-level exceptions\n            # with {\"status\":\"FAILED\",...}; raise so the pipeline halts at the\n            # failed stage instead of cascading opaque downstream errors.\n            _parsed = None\n            if _result:\n                try:\n                    _parsed = _json.loads(_result)\n                except (ValueError, TypeError):\n                    if _stage in (\"training\", \"silver\"):\n                        print(f\"  result: {_result}\")\n            if isinstance(_parsed, dict) and _parsed.get(\"status\") == \"FAILED\":\n                raise RuntimeError(\n                    f\"[{_stage.upper()}] {_nb} failed: \"\n                    f\"{_parsed.get('error_type', '?')}: {_parsed.get('error_message', '')} \"\n                    \"(see spawned-notebook output for full traceback)\"\n                )\n            if isinstance(_parsed, dict) and _stage == \"training\":\n                _training_results = _parsed\n            elif isinstance(_parsed, dict) and _stage == \"silver\":\n                _silver_results = _parsed\n    print(f\"\\nTotal: {_time.time() - _total_start:.1f}s ({_skipped_count} stages skipped via resume)\")\n\n    _ns = _namespace\n    if _ns is not None:\n        _bronze_path = _ns.bronze_metadata_path\n        if _bronze_path.exists():\n            _bm = _json.loads(_bronze_path.read_text())\n            print(\"\\n\" + \"=\" * 60)\n            print(\"BRONZE SUMMARY\")\n            print(\"=\" * 60)\n            print(f\"Sources: {_bm.get('total_sources', 0)}\")\n            for _src, _info in _bm.get(\"sources\", {}).items():\n                if isinstance(_info, dict):\n                    print(f\"  {_src}: {_info.get('rows', '?'):,} rows, {_info.get('columns', '?')} columns\")\n                else:\n                    print(f\"  {_src}: {_info}\")\n\n        _gold_path = _ns.gold_metadata_path\n        if _gold_path.exists():\n            _gm = _json.loads(_gold_path.read_text())\n            print(\"\\n\" + \"=\" * 60)\n            print(\"GOLD SUMMARY\")\n            print(\"=\" * 60)\n            print(f\"Rows: {_gm.get('rows', '?'):,}, Columns: {_gm.get('columns', '?')}\")\n            if \"feature_count\" in _gm:\n                print(f\"Features: {_gm['feature_count']}\")\n            if \"feature_version\" in _gm:\n                print(f\"Version: {_gm['feature_version']}\")\n            if \"elapsed_seconds\" in _gm:\n                print(f\"Elapsed: {_gm['elapsed_seconds']}s\")\n\n    if _silver_results:\n        print(\"\\n\" + \"=\" * 60)\n        print(\"SILVER RESULTS\")\n        print(\"=\" * 60)\n        print(f\"Rows: {_silver_results.get('rows', '?'):,}, Columns: {_silver_results.get('columns', '?')}\")\n        _es = _silver_results.get(\"elapsed_seconds\", {})\n        if _es:\n            print(\"\\nStage timings:\")\n            for _name in (\"load_bronze\", \"merge_sources\", \"apply_derived\", \"holdout_mask\", \"delta_write\", \"optimize\", \"total\"):\n                if _name in _es:\n                    print(f\"  {_name:<18} {_es[_name]:>8.1f}s\")\n        _mb = _silver_results.get(\"merge_breakdown\")\n        if _mb:\n            print(\"\\nMerge breakdown:\")\n            print(f\"  Spine: {_mb.get('spine_rows', 0):,} rows = {_mb.get('spine_entities', 0):,} entities x {_mb.get('spine_dates', 0)} dates ({_mb.get('spine_stats_seconds', 0):.1f}s)\")\n            print(f\"  Checkpoints: {_mb.get('checkpoint_count', 0)} ({_mb.get('checkpoint_seconds', 0):.1f}s)\")\n            print(f\"  Validation: {_mb.get('validation_seconds', 0):.1f}s\")\n            print(f\"  Merge total: {_mb.get('merge_total_seconds', 0):.1f}s\")\n            _spd = _mb.get(\"seconds_per_dataset\", {})\n            _cpd = _mb.get(\"columns_per_dataset\", {})\n            print(\"\\nPer-source merge timing:\")\n            _ranked = sorted(_mb.get(\"datasets_merged\", []), key=lambda n: _spd.get(n, 0), reverse=True)\n            for _name in _ranked:\n                print(f\"  {_name:<40} {_spd.get(_name, 0):>8.1f}s (+{_cpd.get(_name, 0)} cols)\")\n\n    if _training_results:\n        print(\"\\n\" + \"=\" * 60)\n        print(\"TRAINING RESULTS\")\n        print(\"=\" * 60)\n        if \"gold_data\" in _training_results:\n            _gd = _training_results[\"gold_data\"]\n            print(f\"\\nGold data: {_gd.get('rows', '?'):,} rows, {_gd.get('columns', '?')} columns\")\n            print(f\"Column types: {_gd.get('column_types', {})}\")\n        if \"feature_count\" in _training_results:\n            print(f\"Features: {_training_results['feature_count']}\")\n        if \"split\" in _training_results:\n            _sp = _training_results[\"split\"]\n            print(f\"\\nSplit: train={_sp.get('train_count', '?'):,}, test={_sp.get('test_count', '?'):,}\")\n            print(f\"Cutoff date: {_sp.get('cutoff_date', 'N/A')}\")\n        if \"label_distribution\" in _training_results:\n            print(f\"Label distribution: {_training_results['label_distribution']}\")\n        _fp = _training_results.get(\"feature_profile\", {})\n        if _fp:\n            print(f\"\\nFeature profile: production={_fp.get('production_features', '?')}, exploration={_fp.get('exploration_features', '?')}\")\n            _disc = _fp.get(\"discrepancies\", [])\n            if _disc:\n                print(f\"WARNING: {len(_disc)} feature discrepancies vs exploration:\")\n                for _d in _disc:\n                    print(f\"  {_d}\")\n            elif \"exploration_features\" in _fp:\n                print(\"Feature profile matches exploration\")\n            _excl = _fp.get(\"excluded_details\", {})\n            if _excl:\n                print(f\"\\nExclusion details ({len(_excl)} columns):\")\n                for _col, _reason in sorted(_excl.items()):\n                    print(f\"  {_col}: {_reason}\")\n        if \"models\" in _training_results:\n            print(\"\\nModel Results:\")\n            print(f\"{'Model':<25} {'AUC':>8} {'PR-AUC':>8} {'F1':>8}\")\n            print(\"-\" * 53)\n            for _name, _metrics in _training_results[\"models\"].items():\n                print(f\"{_name:<25} {_metrics.get('roc_auc', 0):.4f}   {_metrics.get('pr_auc', 0):.4f}   {_metrics.get('f1', 0):.4f}\")\n        if \"best_model\" in _training_results:\n            print(f\"\\nBest: {_training_results['best_model']} (AUC={_training_results.get('best_roc_auc', 0):.4f})\")\n\nelif RUN_PIPELINE and GENERATION_TARGET == GenerationTarget.LOCAL_FEAST_MLFLOW:\n    import subprocess\n    import sys\n    runner_path = output_dir / \"pipeline_runner.py\"\n    if runner_path.exists():\n        print(f\"Running: python {runner_path.name}\")\n        print(\"Pipeline: Landing -> Bronze -> Silver -> Gold -> Training...\\n\")\n        proc = subprocess.Popen(\n            [sys.executable, \"-u\", \"pipeline_runner.py\"],\n            cwd=str(output_dir.resolve()),\n            stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,\n        )\n        for line in proc.stdout:\n            print(line, end=\"\", flush=True)\n        if proc.wait() != 0:\n            raise subprocess.CalledProcessError(proc.returncode, \"pipeline_runner.py\")\n    else:\n        print(\"pipeline_runner.py not found. Generate first by running cells above.\")\n\nelse:\n    print(\"To run the complete pipeline:\")\n    print(f\"\\n  cd {output_dir}\")\n    print(\"  python pipeline_runner.py\")"
+   "source": "# @cr:code name='run_pipeline' id=8b659505\nRUN_PIPELINE = True\n# Resume gate: skip any landing/bronze/silver/gold notebook whose output UC table\n# already exists. Operator-facing knob — flip to False for a forced full re-run\n# (e.g. when findings or generated code changed in a way that invalidates prior\n# tables). Training is never resume-skipped (no UC table to check; MLflow runs\n# are tracked separately).\nRESUME_FROM_EXISTING_TABLES = True\n\ndef _fmt_int(value, default=\"?\"):\n    # Stage metadata writers can emit `null` for row counts when the post-write\n    # readback is skipped (a wide-table perf knob); `:,` format on None/str\n    # raises TypeError. Wrap every `:,`-formatted summary read.\n    if isinstance(value, bool) or value is None:\n        return default\n    if isinstance(value, (int, float)):\n        return f\"{value:,}\"\n    return default\n\nif RUN_PIPELINE and GENERATION_TARGET == GenerationTarget.DATABRICKS:\n    import json as _json\n    import time as _time\n\n    def _resume_table_for(stage, nb_name, catalog, schema):\n        \"\"\"Resolve the UC table written by a generated stage notebook.\n\n        Conventions established by the Databricks renderer:\n          - landing/landing_<src>          -> {cat}.{sch}.landing_<src>\n          - bronze/bronze_entity_<src>     -> {cat}.{sch}.bronze_entity_<src>\n          - bronze/bronze_event_<src>      -> {cat}.{sch}.bronze_entity_<src>_events  (Shape A)\n          - bronze/bronze_entity_<src>_aggregated -> same name as notebook\n          - silver/silver_featureset_<CN>  -> same name as notebook\n          - gold/gold_features_<CN>        -> same name as notebook\n          - training/*                     -> None (no UC table)\n        \"\"\"\n        base = f\"{catalog}.{schema}\"\n        if stage == \"landing\":\n            return f\"{base}.{nb_name}\"\n        if stage == \"bronze\":\n            if nb_name.startswith(\"bronze_event_\"):\n                source = nb_name[len(\"bronze_event_\"):]\n                return f\"{base}.bronze_entity_{source}_events\"\n            return f\"{base}.{nb_name}\"\n        if stage in (\"silver\", \"gold\"):\n            return f\"{base}.{nb_name}\"\n        return None\n\n    print(f\"Databricks pipeline: {output_dir}\\n\")\n    if RESUME_FROM_EXISTING_TABLES:\n        print(\"[RESUME] table-existence resume enabled — stages with existing UC tables will be skipped\\n\")\n    spark.sql(f\"CREATE SCHEMA IF NOT EXISTS {DATABRICKS_CATALOG}.{DATABRICKS_SCHEMA}\")\n    _ns_params = {\"experiments_dir\": str(_namespace.root), \"run_id\": _namespace.run_id} if _namespace else {}\n    _stages = [\"landing\", \"bronze\", \"silver\", \"gold\", \"training\"]\n    _total_start = _time.time()\n    _training_results = None\n    _silver_results = None\n    _skipped_count = 0\n    for _stage in _stages:\n        _stage_dir = output_dir / _stage\n        if not _stage_dir.exists():\n            continue\n        if _stage == \"bronze\":\n            # (retires NB10 patch §2.11): events must run before\n            # entities so the aggregator inputs exist when consumers try\n            # to read them. Default sorted() puts `bronze_entity_*`\n            # before `bronze_event_*` (i < v) and raises\n            # [TABLE_OR_VIEW_NOT_FOUND] on the first aggregated bronze.\n            _notebooks = sorted(\n                (f.stem for f in _stage_dir.iterdir() if f.suffix == \".py\"),\n                key=lambda n: (0 if n.startswith(\"bronze_event_\") else 1, n),\n            )\n            print(f\"[bronze order] {len(_notebooks)} notebooks; events first:\")\n            for _n in _notebooks[:6]:\n                print(f\"   {_n}\")\n            if len(_notebooks) > 6:\n                print(f\"   ... + {len(_notebooks) - 6} more\")\n        else:\n            _notebooks = sorted(f.stem for f in _stage_dir.iterdir() if f.suffix == \".py\")\n        for _nb in _notebooks:\n            _path = str(output_dir / _stage / _nb)\n\n            if RESUME_FROM_EXISTING_TABLES:\n                _expected = _resume_table_for(_stage, _nb, DATABRICKS_CATALOG, DATABRICKS_SCHEMA)\n                if _expected and spark.catalog.tableExists(_expected):\n                    print(f\"[SKIP] {_stage.upper()}/{_nb}: {_expected} already exists\")\n                    _skipped_count += 1\n                    continue\n\n            _start = _time.time()\n            _result = dbutils.notebook.run(_path, 86400, _ns_params)\n            _elapsed = _time.time() - _start\n            print(f\"[{_stage.upper()}] {_nb}: {_elapsed:.1f}s\")\n            # Generated stage notebooks structured-exit on top-level exceptions\n            # with {\"status\":\"FAILED\",...}; raise so the pipeline halts at the\n            # failed stage instead of cascading opaque downstream errors.\n            _parsed = None\n            if _result:\n                try:\n                    _parsed = _json.loads(_result)\n                except (ValueError, TypeError):\n                    if _stage in (\"training\", \"silver\"):\n                        print(f\"  result: {_result}\")\n            if isinstance(_parsed, dict) and _parsed.get(\"status\") == \"FAILED\":\n                raise RuntimeError(\n                    f\"[{_stage.upper()}] {_nb} failed: \"\n                    f\"{_parsed.get('error_type', '?')}: {_parsed.get('error_message', '')} \"\n                    \"(see spawned-notebook output for full traceback)\"\n                )\n            if isinstance(_parsed, dict) and _stage == \"training\":\n                _training_results = _parsed\n            elif isinstance(_parsed, dict) and _stage == \"silver\":\n                _silver_results = _parsed\n    print(f\"\\nTotal: {_time.time() - _total_start:.1f}s ({_skipped_count} stages skipped via resume)\")\n\n    _ns = _namespace\n    if _ns is not None:\n        _bronze_path = _ns.bronze_metadata_path\n        if _bronze_path.exists():\n            _bm = _json.loads(_bronze_path.read_text())\n            print(\"\\n\" + \"=\" * 60)\n            print(\"BRONZE SUMMARY\")\n            print(\"=\" * 60)\n            print(f\"Sources: {_bm.get('total_sources', 0)}\")\n            for _src, _info in _bm.get(\"sources\", {}).items():\n                if isinstance(_info, dict):\n                    print(f\"  {_src}: {_fmt_int(_info.get('rows'))} rows, {_info.get('columns', '?')} columns\")\n                else:\n                    print(f\"  {_src}: {_info}\")\n\n        _gold_path = _ns.gold_metadata_path\n        if _gold_path.exists():\n            _gm = _json.loads(_gold_path.read_text())\n            print(\"\\n\" + \"=\" * 60)\n            print(\"GOLD SUMMARY\")\n            print(\"=\" * 60)\n            print(f\"Rows: {_fmt_int(_gm.get('rows'))}, Columns: {_gm.get('columns', '?')}\")\n            if \"feature_count\" in _gm:\n                print(f\"Features: {_gm['feature_count']}\")\n            if \"feature_version\" in _gm:\n                print(f\"Version: {_gm['feature_version']}\")\n            if \"elapsed_seconds\" in _gm:\n                print(f\"Elapsed: {_gm['elapsed_seconds']}s\")\n\n    if _silver_results:\n        print(\"\\n\" + \"=\" * 60)\n        print(\"SILVER RESULTS\")\n        print(\"=\" * 60)\n        print(f\"Rows: {_fmt_int(_silver_results.get('rows'))}, Columns: {_silver_results.get('columns', '?')}\")\n        _es = _silver_results.get(\"elapsed_seconds\", {})\n        if _es:\n            print(\"\\nStage timings:\")\n            for _name in (\"load_bronze\", \"merge_sources\", \"apply_derived\", \"holdout_mask\", \"delta_write\", \"optimize\", \"total\"):\n                if _name in _es:\n                    print(f\"  {_name:<18} {_es[_name]:>8.1f}s\")\n        _mb = _silver_results.get(\"merge_breakdown\")\n        if _mb:\n            print(\"\\nMerge breakdown:\")\n            print(f\"  Spine: {_fmt_int(_mb.get('spine_rows'), '0')} rows = {_fmt_int(_mb.get('spine_entities'), '0')} entities x {_mb.get('spine_dates', 0)} dates ({_mb.get('spine_stats_seconds', 0):.1f}s)\")\n            print(f\"  Checkpoints: {_mb.get('checkpoint_count', 0)} ({_mb.get('checkpoint_seconds', 0):.1f}s)\")\n            print(f\"  Validation: {_mb.get('validation_seconds', 0):.1f}s\")\n            print(f\"  Merge total: {_mb.get('merge_total_seconds', 0):.1f}s\")\n            _spd = _mb.get(\"seconds_per_dataset\", {})\n            _cpd = _mb.get(\"columns_per_dataset\", {})\n            print(\"\\nPer-source merge timing:\")\n            _ranked = sorted(_mb.get(\"datasets_merged\", []), key=lambda n: _spd.get(n, 0), reverse=True)\n            for _name in _ranked:\n                print(f\"  {_name:<40} {_spd.get(_name, 0):>8.1f}s (+{_cpd.get(_name, 0)} cols)\")\n\n    if _training_results:\n        print(\"\\n\" + \"=\" * 60)\n        print(\"TRAINING RESULTS\")\n        print(\"=\" * 60)\n        if \"gold_data\" in _training_results:\n            _gd = _training_results[\"gold_data\"]\n            print(f\"\\nGold data: {_fmt_int(_gd.get('rows'))} rows, {_gd.get('columns', '?')} columns\")\n            print(f\"Column types: {_gd.get('column_types', {})}\")\n        if \"feature_count\" in _training_results:\n            print(f\"Features: {_training_results['feature_count']}\")\n        if \"split\" in _training_results:\n            _sp = _training_results[\"split\"]\n            print(f\"\\nSplit: train={_fmt_int(_sp.get('train_count'))}, test={_fmt_int(_sp.get('test_count'))}\")\n            print(f\"Cutoff date: {_sp.get('cutoff_date', 'N/A')}\")\n        if \"label_distribution\" in _training_results:\n            print(f\"Label distribution: {_training_results['label_distribution']}\")\n        _fp = _training_results.get(\"feature_profile\", {})\n        if _fp:\n            print(f\"\\nFeature profile: production={_fp.get('production_features', '?')}, exploration={_fp.get('exploration_features', '?')}\")\n            _disc = _fp.get(\"discrepancies\", [])\n            if _disc:\n                print(f\"WARNING: {len(_disc)} feature discrepancies vs exploration:\")\n                for _d in _disc:\n                    print(f\"  {_d}\")\n            elif \"exploration_features\" in _fp:\n                print(\"Feature profile matches exploration\")\n            _excl = _fp.get(\"excluded_details\", {})\n            if _excl:\n                print(f\"\\nExclusion details ({len(_excl)} columns):\")\n                for _col, _reason in sorted(_excl.items()):\n                    print(f\"  {_col}: {_reason}\")\n        if \"models\" in _training_results:\n            print(\"\\nModel Results:\")\n            print(f\"{'Model':<25} {'AUC':>8} {'PR-AUC':>8} {'F1':>8}\")\n            print(\"-\" * 53)\n            for _name, _metrics in _training_results[\"models\"].items():\n                print(f\"{_name:<25} {_metrics.get('roc_auc', 0):.4f}   {_metrics.get('pr_auc', 0):.4f}   {_metrics.get('f1', 0):.4f}\")\n        if \"best_model\" in _training_results:\n            print(f\"\\nBest: {_training_results['best_model']} (AUC={_training_results.get('best_roc_auc', 0):.4f})\")\n\nelif RUN_PIPELINE and GENERATION_TARGET == GenerationTarget.LOCAL_FEAST_MLFLOW:\n    import subprocess\n    import sys\n    runner_path = output_dir / \"pipeline_runner.py\"\n    if runner_path.exists():\n        print(f\"Running: python {runner_path.name}\")\n        print(\"Pipeline: Landing -> Bronze -> Silver -> Gold -> Training...\\n\")\n        proc = subprocess.Popen(\n            [sys.executable, \"-u\", \"pipeline_runner.py\"],\n            cwd=str(output_dir.resolve()),\n            stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,\n        )\n        for line in proc.stdout:\n            print(line, end=\"\", flush=True)\n        if proc.wait() != 0:\n            raise subprocess.CalledProcessError(proc.returncode, \"pipeline_runner.py\")\n    else:\n        print(\"pipeline_runner.py not found. Generate first by running cells above.\")\n\nelse:\n    print(\"To run the complete pipeline:\")\n    print(f\"\\n  cd {output_dir}\")\n    print(\"  python pipeline_runner.py\")"
   },
   {
    "cell_type": "markdown",

{churnkit-1.2.8a2 → churnkit-1.2.8a3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "churnkit"
-version = "1.02.8a2"
+version = "1.02.8a3"
 description = "Structured ML framework for customer churn prediction -- from exploration notebooks to production pipelines, locally or on Databricks."
 readme = "README.md"
 license = {text = "Apache-2.0"}

{churnkit-1.2.8a2 → churnkit-1.2.8a3}/src/customer_retention/__init__.py

@@ -17,7 +17,7 @@ Main module categories:
                 llm_context, iteration)
 """
 
-__version__ = "1.02.8a2"
+__version__ = "1.02.8a3"
 
 # Environment utilities (always available)
 from .core.compat import (

{churnkit-1.2.8a2 → churnkit-1.2.8a3}/src/customer_retention/generators/pipeline_generator/findings_parser.py

@@ -522,7 +522,8 @@ class FindingsParser:
 
     def _index_raw_source_columns(self, discovered_events: Dict[str, ExplorationFindings]) -> None:
         for name, findings in discovered_events.items():
-            self._raw_source_columns[name] = set(findings.columns.keys())
+            cols = set(findings.columns.keys())
+            self._raw_source_columns.setdefault(name, set()).update(cols)
             per_col: Dict[str, List[str]] = {}
             for col_name, col_finding in findings.columns.items():
                 tm = getattr(col_finding, "type_metrics", None) or {}
@@ -537,7 +538,15 @@ class FindingsParser:
                         for c in top_cats
                     ]
             if per_col:
-                self._value_counts_by_source[name] = per_col
+                # Merge — don't replace. Two findings files (entity-level and
+                # pre-aggregated) can both register columns under the same
+                # source name; the second pass otherwise clobbers the first
+                # and loses raw event-stream value-counts (e.g. event_type)
+                # that only the entity-level file carries.
+                bucket = self._value_counts_by_source.setdefault(name, {})
+                for col, vals in per_col.items():
+                    if col not in bucket:
+                        bucket[col] = vals
         self._augment_value_counts_from_spec()
 
     _SPEC_TRAILING_WINDOW_RE = re.compile(r"^(?P<head>.+)_count_(?:\d+[dhw]|all_time)$")
@@ -1211,6 +1220,7 @@ class FindingsParser:
         source_findings: Optional[Dict[str, "ExplorationFindings"]] = None,
     ) -> None:
         self._apply_bronze_recommendations(config, registry)
+        self._apply_registry_bronze_aggregations(config, registry)
         self._apply_imbalance_recommendations(config, registry)
         self._apply_silver_recommendations(config, registry)
         zero_inflation_opt_in_prefixes = self._collect_zero_inflation_opt_in_prefixes(
@@ -1596,6 +1606,43 @@ class FindingsParser:
                 return name
         return None
 
+    def _apply_registry_bronze_aggregations(
+        self, config: PipelineConfig, registry: RecommendationRegistry
+    ) -> None:
+        """Promote ``registry.bronze_aggregations`` onto event_cfg for sources
+        the operator didn't restate in NB10's ``BRONZE_AGGREGATIONS`` dict.
+
+        Without this, a per-source declaration made via
+        ``add_bronze_value_counts`` / ``add_bronze_per_grid_date_mode`` during
+        exploration is silently ignored at codegen unless the operator
+        manually echoes it into NB10. Recs that depend on the per-value count
+        shape (e.g. ``subscription_terminate_to_start_ratio_*``) get dropped.
+        Operator overrides win on conflict; only un-overridden sources are
+        filled in here.
+        """
+        ba = getattr(registry, "bronze_aggregations", None) or {}
+        if not isinstance(ba, dict) or not ba:
+            return
+        if self._bronze_aggregation_overrides is None:
+            self._bronze_aggregation_overrides = {}
+        for src, entry in ba.items():
+            if src in self._bronze_aggregation_overrides:
+                continue
+            if src not in config.bronze_event:
+                continue
+            if not isinstance(entry, dict):
+                continue
+            recognized = {
+                k: v for k, v in entry.items()
+                if k in _RECOGNIZED_BRONZE_OVERRIDE_KEYS
+            }
+            if not recognized:
+                continue
+            self._bronze_aggregation_overrides[src] = recognized
+            self._apply_bronze_aggregation_overrides(
+                config.bronze_event[src], src,
+            )
+
     def _apply_bronze_recommendations(self, config: PipelineConfig, registry: RecommendationRegistry) -> None:
         sources_to_process = dict(registry.sources)
         if not sources_to_process and hasattr(registry, "bronze") and registry.bronze is not None:
@@ -2480,6 +2527,17 @@ class FindingsParser:
             for src in dd.source_columns:
                 for suffix in ("_delta_hours", "_hour", "_dow", "_is_weekend"):
                     columns.add(f"{src}{suffix}")
+            # The runtime aggregator computes datetime-derived features per row
+            # (`compute_temporal_features`) and then runs them through the same
+            # window aggregation as numeric `value_columns`. Mirror that fan-out
+            # so silver_derived recs referencing `<src>_<suffix>_<func>_<window>`
+            # (e.g. `PAYMENT_DATE_is_weekend_max_all_time`) survive parity.
+            if agg and agg.windows and agg.agg_funcs:
+                for src in dd.source_columns:
+                    for suffix in ("_delta_hours", "_hour", "_dow", "_is_weekend"):
+                        for window in agg.windows:
+                            for func in agg.agg_funcs:
+                                columns.add(f"{src}{suffix}_{func}_{window}")
         for text_cfg in event_cfg.text_features:
             if text_cfg.component_columns:
                 columns |= set(text_cfg.component_columns)