churnkit 1.3.2a2__tar.gz → 1.3.2a3__tar.gz

{churnkit-1.3.2a2 → churnkit-1.3.2a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: churnkit
-Version: 1.3.2a2
+Version: 1.3.2a3
 Summary: Structured ML framework for customer churn prediction -- from exploration notebooks to production pipelines, locally or on Databricks.
 Project-URL: Homepage, https://github.com/aladjov/CR
 Project-URL: Documentation, https://github.com/aladjov/CR/wiki

{churnkit-1.3.2a2 → churnkit-1.3.2a3}/apps/databricks_app/src/theme.css

@@ -82,7 +82,7 @@ footer,
 }
 
 .block-container {
-  max-width: 1120px !important;
+  max-width: 1320px !important;
   padding-top: 2rem !important;
   padding-bottom: 8rem !important;
   padding-left: 2rem !important;

{churnkit-1.3.2a2 → churnkit-1.3.2a3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "churnkit"
-version = "1.03.2a2"
+version = "1.03.2a3"
 description = "Structured ML framework for customer churn prediction -- from exploration notebooks to production pipelines, locally or on Databricks."
 readme = "README.md"
 license = {text = "Apache-2.0"}

{churnkit-1.3.2a2 → churnkit-1.3.2a3}/src/customer_retention/__init__.py

@@ -17,7 +17,7 @@ Main module categories:
                 llm_context, iteration)
 """
 
-__version__ = "1.03.2a2"
+__version__ = "1.03.2a3"
 
 # Environment utilities (always available)
 from .core.compat import (

{churnkit-1.3.2a2 → churnkit-1.3.2a3}/src/customer_retention/generators/notebook_generator/stages/s_c04_batch_inference.py

@@ -101,53 +101,67 @@ def _resolve_scope_filter():
 
 
 def _resolve_already_positive_exclusion():
-    """Return a Spark-SQL predicate that drops entities whose target label is
-    already 1 (e.g. ``churned = 0``), resolved from
-    ``ProjectContext.target_column``. Always-on at scoring time: the trained
-    model has nothing useful to say about a row that is already a positive
-    outcome, and surfacing such rows in the CSM-facing dashboard wastes
-    headcount on accounts the business can no longer recover. Distinct from
-    ``_resolve_scope_filter`` so the training parity contract stays clean —
-    training cohort keeps positives (the model needs them to learn the
-    boundary); scoring drops them because they aren't actionable.
-
-    Returns ``None`` when project_context is absent or carries no
-    ``target_column`` — the cell then falls back to the unmodified scope
-    filter and logs the omission so the operator can see it didn't apply."""
+    """Return ``(target_column, landing_fqn, raw_entity_key)`` so scoring can
+    drop entities whose target label has EVER been 1 in landing — applied
+    by the framework as a ``left_anti`` join against
+    ``landing_<target>.filter(<target_col> = 1)``.
+
+    Entity-level by construction: a row-level ``<target_col> = 0`` clause
+    cannot work against a snapshot-panel landing table, because an entity
+    with both ``<target_col>=0`` and ``<target_col>=1`` rows would still
+    survive the row-level filter (the 0-rows pass; ``distinct(entity_id)``
+    re-admits the entity). Anti-join on the "ever positive" entity set is
+    the only shape that matches the intent — drop the entity wholesale if
+    landing has ANY row at the positive outcome.
+
+    Always-on at scoring time: the trained model has nothing useful to say
+    about an already-positive row, and surfacing such rows in the
+    CSM-facing dashboard wastes headcount on accounts the business can no
+    longer recover. Distinct from ``_resolve_scope_filter`` so the training
+    parity contract stays clean — training cohort keeps positives (the
+    model needs them to learn the boundary); scoring drops them because
+    they aren't actionable.
+
+    Returns ``(None, None, None)`` when project_context is absent or
+    cannot resolve a target column / raw entity key — the cell then falls
+    back to the unmodified scope filter and logs the omission so the
+    operator can see it didn't apply."""
     try:
         from customer_retention.analysis.auto_explorer.project_context import ProjectContext
         from customer_retention.analysis.auto_explorer.run_namespace import RunNamespace
     except ImportError:
-        return None
+        return None, None, None
     _ns = RunNamespace.from_env_or_latest()
     if _ns is None:
-        return None
+        return None, None, None
     _path = _ns.project_context_path
     if not _path.exists():
-        return None
+        return None, None, None
     _ctx = ProjectContext.load(_path)
     _target_col = getattr(_ctx, "target_column", None)
     if not _target_col:
-        return None
-    return f"{_target_col} = 0"
-
-
-def _compose_scoring_filter(*parts):
-    """AND-combine non-empty Spark-SQL predicates, parenthesizing each so
-    later predicates can't accidentally rebind the earlier ones via
-    operator-precedence surprises. Returns ``None`` when every part is
-    falsy."""
-    _kept = [p for p in parts if p]
-    if not _kept:
-        return None
-    if len(_kept) == 1:
-        return _kept[0]
-    return " and ".join(f"({p})" for p in _kept)
+        return None, None, None
+    _target_name = (
+        getattr(_ctx, "target_dataset", None)
+        or next(
+            (n for n, d in _ctx.datasets.items() if getattr(d, "role", None) == "target"),
+            None,
+        )
+    )
+    if _target_name is None and len(_ctx.datasets) == 1:
+        _target_name = next(iter(_ctx.datasets))
+    if _target_name is None:
+        return None, None, None
+    _target_ds = _ctx.datasets.get(_target_name)
+    _raw_key = getattr(_target_ds, "entity_column", None) if _target_ds is not None else None
+    if not _raw_key:
+        return None, None, None
+    _landing_fqn = f"{CATALOG}.{SCHEMA}.landing_{_target_name}"
+    return _target_col, _landing_fqn, _raw_key
 
 
 _scope_filter = _resolve_scope_filter()
-_already_positive_exclusion = _resolve_already_positive_exclusion()
-_scoring_filter = _compose_scoring_filter(_scope_filter, _already_positive_exclusion)
+_excl_target_col, _excl_landing_fqn, _excl_entity_key = _resolve_already_positive_exclusion()
 
 batch_inference_result = None
 _predictions_status = "UNKNOWN"
@@ -190,12 +204,17 @@ else:
             print(f"Scope filter (from NB00 project_context): {_scope_filter}")
         else:
             print("Scope filter: (none — scoring full entity population)")
-        if _already_positive_exclusion:
-            print(f"Already-positive exclusion (target_column): {_already_positive_exclusion}")
+        if _excl_target_col and _excl_landing_fqn and _excl_entity_key:
+            print(
+                f"Already-positive exclusion (entity-level anti-join): "
+                f"DROP entity where any row of {_excl_landing_fqn} has "
+                f"{_excl_target_col} = 1 (key={_excl_entity_key})"
+            )
         else:
             print(
                 "Already-positive exclusion: (none — project_context.target_column "
-                "not set; rows already at the positive outcome will be scored)"
+                "or target dataset entity_column not resolvable; entities already "
+                "at the positive outcome may be scored)"
             )
         config = BatchInferenceConfig(
             catalog=CATALOG,
@@ -206,7 +225,10 @@ else:
             risk_tier_high=RISK_TIER_HIGH,
             risk_tier_medium=RISK_TIER_MEDIUM,
             inference_timestamp=datetime.now(timezone.utc),
-            filter_expression=_scoring_filter,
+            filter_expression=_scope_filter,
+            exclude_already_positive_target_column=_excl_target_col,
+            exclude_already_positive_via_table=_excl_landing_fqn,
+            exclude_already_positive_entity_key=_excl_entity_key,
         )
         batch_inference_result = run_batch_inference(config)
         _predictions_status = batch_inference_result.summary()

{churnkit-1.3.2a2 → churnkit-1.3.2a3}/src/customer_retention/stages/scoring/batch_inference.py

@@ -223,6 +223,31 @@ class BatchInferenceConfig:
     # ``project_context.datasets[<target>].entity_column`` automatically.
     filter_via_table_entity_key: Optional[str] = None
 
+    # Entity-level exclusion of rows already at the positive outcome.
+    # Distinct from ``filter_expression`` because ``churned = 0`` is a
+    # *row-level* predicate and landing is a per-snapshot panel: an entity
+    # that was churned=0 in earlier snapshots and churned=1 later still has
+    # rows passing the row-level filter, so its entity_id survives
+    # ``.select().distinct()`` and the scoring cohort silently leaks
+    # already-positive accounts onto the CSM-facing dashboard.
+    #
+    # Set all three together to drop any entity whose target has EVER been
+    # 1 across any row of ``via_table`` (the landing table for the target
+    # dataset). Applied as a one-sided ``left_anti`` join after the scope
+    # filter resolves entity_df — single Spark job, narrow shuffle, no
+    # collect-to-driver. The trained model has nothing useful to say about
+    # an already-positive row and the business can no longer recover that
+    # account; no opt-out by design.
+    #
+    # When any of the three fields is ``None`` the exclusion is silently
+    # skipped. The c04 stage generator populates them automatically from
+    # ``project_context.target_column`` + the target dataset's
+    # ``entity_column`` so old callers (that pre-date this change) keep
+    # working unchanged but get the looser row-level semantics.
+    exclude_already_positive_target_column: Optional[str] = None
+    exclude_already_positive_via_table: Optional[str] = None
+    exclude_already_positive_entity_key: Optional[str] = None
+
 
 @dataclass
 class BatchInferenceResult:
@@ -480,14 +505,13 @@ def _run_databricks(config: BatchInferenceConfig) -> BatchInferenceResult:
     t_prep = time.perf_counter()
     df_customers = spark.table(customer_table)
     if config.filter_expression:
-        # NB00's `sample_filter` predicate is stored in pandas/Python
-        # syntax (`column in ['a', 'b']`) because exploration filters via
-        # `df.query()`. Spark `df.filter()` needs SQL-tuple syntax
-        # (`column IN ('a', 'b')`). The same translator landing applies
-        # before its `df.filter(...)` call (see `findings_parser`
-        # invocation of `_spark_safe_query_expr`) — apply it here so the
-        # same predicate string drives both stages without operator
-        # intervention.
+        # The NB00 sample_filter predicate is stored in pandas-style
+        # syntax (Python list literals after IN) because exploration
+        # filters via the pandas query API. Spark df.filter needs
+        # SQL-tuple syntax instead. The same translator landing applies
+        # before its df.filter call (see findings_parser invocation of
+        # _spark_safe_query_expr) — apply it here so the same predicate
+        # string drives both stages without operator intervention.
         from customer_retention.core.compat import _spark_safe_query_expr
         _sql_filter = _spark_safe_query_expr(config.filter_expression)
         # Register sibling-temp-views for any bare-name table references
@@ -576,6 +600,42 @@ def _run_databricks(config: BatchInferenceConfig) -> BatchInferenceResult:
     entity_count = int(entity_df.count())
     prep_seconds = time.perf_counter() - t_prep
     _emit(f"{entity_count:,} entities after filter/dedup (prep {prep_seconds:.1f}s)")
+
+    # Entity-level already-positive exclusion. Runs in addition to the
+    # row-level scope filter above because ``churned = 0`` applied per-row
+    # to a snapshot-panel landing table can never produce the right
+    # answer: an entity with both churned=0 and churned=1 rows passes the
+    # row-level filter and slips into the scored cohort. Here we build
+    # the "ever-positive" entity set from landing and ``left_anti`` join
+    # it out — one narrow shuffle, no collect-to-driver, no temp views.
+    _excl_col = config.exclude_already_positive_target_column
+    _excl_table = config.exclude_already_positive_via_table
+    _excl_key = config.exclude_already_positive_entity_key
+    if _excl_col and _excl_table and _excl_key:
+        from pyspark.sql.functions import col as _col  # noqa: N812 — Spark API name
+
+        if not spark.catalog.tableExists(_excl_table):
+            _emit(
+                f"WARNING: exclude_already_positive_via_table {_excl_table} does not "
+                "exist; skipping entity-level already-positive exclusion."
+            )
+        else:
+            _ever_positive = (
+                spark.table(_excl_table)
+                .filter(_col(_excl_col) == lit(1))
+                .select(_col(_excl_key).alias("entity_id"))
+                .distinct()
+            )
+            entity_df = entity_df.join(_ever_positive, on="entity_id", how="left_anti")
+            _post_excl = int(entity_df.count())
+            _dropped = entity_count - _post_excl
+            _emit(
+                f"dropped {_dropped:,} entities with {_excl_col}=1 in any "
+                f"{_excl_table} row (entity-level exclusion); "
+                f"{_post_excl:,} entities remain"
+            )
+            entity_count = _post_excl
+
     if entity_count == 0:
         raise ValueError(
             f"Scope filter + distinct produced 0 entities in {customer_table}; "