SuperModelingFactory 0.3.15__tar.gz → 0.5.0__tar.gz

{supermodelingfactory-0.3.15 → supermodelingfactory-0.5.0}/Modeling_Tool/Core/ODPS_Tool.py

@@ -247,7 +247,7 @@ class ODPSRunner(object):
 
         return table_schema
 
-    def upload_df(self, df, table_name, table_schema=None, partition=None):
+    def upload_df(self, df, table_name, table_schema=None, partition=None, atomic=True):
         """上传数据集至mc中创建新表
 
         Parameters
@@ -260,20 +260,113 @@ class ODPSRunner(object):
             数据集
         partition: string
             保存分区
+        atomic: bool, default True
+            When ``True`` (recommended, default from 0.4.2), the target table is
+            replaced through a temp-table + rename swap so that a failure between
+            ``delete_table`` and the completed write never leaves the caller
+            with a dropped-and-empty table. When ``False``, the pre-0.4.2
+            behaviour is used: the existing table is dropped first and then
+            recreated, so any exception during write leaves the target table
+            missing. Only set ``atomic=False`` if you deliberately want the
+            legacy behaviour, e.g. for tables whose downstream readers expect
+            a specific object identity.
         """
         if table_schema is None:
             df.loc[:, "py_inserttime"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
             table_schema = self.cre_table_schema(df=df, partition_name=None)
 
-        self.o.delete_table(table_name, if_exists=True)
-        t = self.o.create_table(table_name, table_schema)
+        if not atomic:
+            # Legacy pre-0.4.2 non-atomic path — retained under an explicit
+            # opt-out for callers that need the old identity semantics.
+            self.o.delete_table(table_name, if_exists=True)
+            t = self.o.create_table(table_name, table_schema)
+            if bool(partition):
+                with t.open_writer(partition=partition, create_partition=True) as writer:
+                    writer.write(df.values.tolist())
+            else:
+                with t.open_writer() as writer:
+                    writer.write(df.values.tolist())
+            logger.info(f'<<<< 完成数据入表{table_name}: shape={df.shape} >>>>')
+            return
+
+        # Atomic swap path (default from 0.4.2). If any step before the final
+        # swap fails, the original table is untouched.
+        ts = datetime.now().strftime('%Y%m%d%H%M%S%f')
+        tmp_name = f"{table_name}__tmp_{ts}"
+        old_name = f"{table_name}__old_{ts}"
+
+        # Step 1: write data into the temp table. If this fails, drop the
+        # partial temp table and re-raise; the target table is untouched.
+        self.o.delete_table(tmp_name, if_exists=True)
+        tmp_table = self.o.create_table(tmp_name, table_schema)
+        try:
+            if bool(partition):
+                with tmp_table.open_writer(partition=partition, create_partition=True) as writer:
+                    writer.write(df.values.tolist())
+            else:
+                with tmp_table.open_writer() as writer:
+                    writer.write(df.values.tolist())
+        except Exception:
+            try:
+                self.o.delete_table(tmp_name, if_exists=True)
+            except Exception as cleanup_exc:  # noqa: BLE001
+                logger.warning(
+                    f"upload_df: failed to drop temp table {tmp_name} after write error: {cleanup_exc!r}"
+                )
+            raise
+
+        # Step 2: swap. If the original table exists, rename it aside first so
+        # that the rename of tmp -> target is guaranteed to succeed. Then drop
+        # the old copy (best-effort, warn on failure).
+        target_exists = self.o.exist_table(table_name)
+        if target_exists:
+            try:
+                self.o.run_sql(f"ALTER TABLE {table_name} RENAME TO {old_name};")
+            except Exception:
+                # Rename of the live target failed — do not touch it. Drop the
+                # tmp table so we don't leak it, then re-raise.
+                try:
+                    self.o.delete_table(tmp_name, if_exists=True)
+                except Exception as cleanup_exc:  # noqa: BLE001
+                    logger.warning(
+                        f"upload_df: failed to drop temp table {tmp_name} after rename-target failure: {cleanup_exc!r}"
+                    )
+                raise
+
+        try:
+            self.o.run_sql(f"ALTER TABLE {tmp_name} RENAME TO {table_name};")
+        except Exception:
+            # tmp -> target rename failed. If we already moved the original,
+            # try to restore it so the caller is not left with a missing table.
+            if target_exists:
+                try:
+                    self.o.run_sql(f"ALTER TABLE {old_name} RENAME TO {table_name};")
+                    logger.warning(
+                        f"upload_df: swap of {tmp_name} -> {table_name} failed; restored original from {old_name}"
+                    )
+                except Exception as restore_exc:  # noqa: BLE001
+                    logger.error(
+                        f"upload_df: swap of {tmp_name} -> {table_name} failed AND restore of "
+                        f"{old_name} -> {table_name} failed: {restore_exc!r}. "
+                        f"Manual recovery required — original data is in {old_name}."
+                    )
+            try:
+                self.o.delete_table(tmp_name, if_exists=True)
+            except Exception:  # noqa: BLE001
+                pass
+            raise
+
+        # Step 3: drop the moved-aside original — best-effort, do not fail the
+        # call if this cleanup errors, since the swap already succeeded.
+        if target_exists:
+            try:
+                self.o.delete_table(old_name, if_exists=True)
+            except Exception as cleanup_exc:  # noqa: BLE001
+                logger.warning(
+                    f"upload_df: swap succeeded but failed to drop old copy {old_name}: {cleanup_exc!r}. "
+                    f"Safe to delete manually."
+                )
 
-        if bool(partition):
-            with t.open_writer(partition=partition, create_partition=True) as writer:
-                writer.write(df.values.tolist())
-        else:
-            with t.open_writer() as writer:
-                writer.write(df.values.tolist())
         logger.info(f'<<<< 完成数据入表{table_name}: shape={df.shape} >>>>')
 
     def insert_df(self, df, table_name, overwrite=True, partition=None):

{supermodelingfactory-0.3.15 → supermodelingfactory-0.5.0}/Modeling_Tool/Feature/PSI_Tool.py

@@ -7,12 +7,20 @@ to measure the distribution drift between expected and actual datasets.
 Author: Matrix Agent
 """
 
+import warnings
+
 import numpy as np
 import pandas as pd
 from typing import Union, List, Dict, Optional, Tuple, Callable, Any
 from tqdm import tqdm
 from Modeling_Tool.Core.Binning_Tool import quick_binning
 
+# Missing bucket label used across PSI helpers when ``missing_policy="include"``
+# (the 0.4.2 default). Kept identical to the sentinel already used by
+# ``Feature.Weighted_Screen`` and ``WOE_Adapter`` so downstream aggregators can
+# recognise it without special-casing.
+_MISSING_BIN = "__MISSING__"
+
 # ============================================================================
 # Classes
 # ============================================================================
@@ -50,7 +58,8 @@ class PSICalculator:
         equal_freq: bool = True,
         min_bin_prop: float = 0.05,
         content: float = 1e-6,
-        precision: int = 5
+        precision: int = 5,
+        missing_policy: str = "include"
     ):
         """
         Initialize PSICalculator with configuration parameters.
@@ -67,12 +76,26 @@ class PSICalculator:
             Small value to prevent division by zero. Default is 1e-6.
         precision : int, optional
             Decimal precision for rounding. Default is 5.
+        missing_policy : {"drop", "include", "warn_and_drop"}, optional
+            How NaN rows are handled. Default in 0.5.0 is "include", which routes
+            NaN rows through a dedicated "__MISSING__" bin so missing-rate drift
+            contributes to the PSI. This is the recommended production behaviour
+            and became the default in 0.5.0 (previously "drop" in 0.4.2 for
+            numeric backward-compat). Pass "drop" to reproduce pre-0.5.0
+            numbers, or "warn_and_drop" for legacy numbers with a RuntimeWarning
+            naming the NaN counts.
         """
+        if missing_policy not in {"include", "drop", "warn_and_drop"}:
+            raise ValueError(
+                f"PSICalculator.__init__: missing_policy must be one of "
+                f"'include', 'drop', 'warn_and_drop'; got {missing_policy!r}."
+            )
         self.buckets = buckets
         self.equal_freq = equal_freq
         self.min_bin_prop = min_bin_prop
         self.content = content
         self.precision = precision
+        self.missing_policy = missing_policy
     
 #     def _calculate_single_psi(
 #         self,
@@ -410,7 +433,8 @@ class PSICalculator:
         varlist: List[str],
         group_by: Optional[str] = None,
         group_name: Optional[str] = None,
-        return_details = False
+        return_details = False,
+        missing_policy: Optional[str] = None
     ) -> pd.DataFrame:
         """
         Calculate grouped PSI comparing two datasets, using expected as benchmark.
@@ -427,12 +451,18 @@ class PSICalculator:
             Column to group by in both datasets.
         group_name : str, optional
             Specific group column name for multi-group calculation.
+        missing_policy : {"drop", "include", "warn_and_drop"}, optional
+            How NaN rows are handled. If None (default), falls back to the value
+            configured on ``self.missing_policy``. Pass an explicit value to
+            override the class-level default for this call only. See
+            ``PSICalculator.__init__`` for the semantics of each mode.
             
         Returns
         -------
         pandas.DataFrame
             Grouped PSI results.
         """
+        effective_policy = missing_policy if missing_policy is not None else self.missing_policy
         return calculate_multigroup_psi_two_sets(
             expected_df = expected_df,
             actual_df = current_data,
@@ -444,7 +474,8 @@ class PSICalculator:
             content = self.content,
             precision = self.precision,
             group_name = group_name,
-            return_details = return_details
+            return_details = return_details,
+            missing_policy = effective_policy
         )
 
 
@@ -460,14 +491,29 @@ def _calculate_single_psi(
     return_details: bool = False,
     min_bin_prop: float = 0.05,
     content: float = 1e-6,
-    precision: int = 5
+    precision: int = 5,
+    missing_policy: str = "include",
 ) -> Union[float, Tuple[float, pd.DataFrame]]:
     """
     Calculate Population Stability Index (PSI) for a single variable.
-    
+
     This function bins both expected and actual series using the same breakpoints,
     then calculates PSI based on the distribution difference between them.
-    
+
+    Missing handling (0.4.2, N29)
+    ------------------------------
+    Before 0.4.2, both series were unconditionally passed through ``.dropna()``
+    before binning. Any drift in the *missing rate itself* — the single most
+    common early signal of a data-pipeline break — was silently invisible in
+    the returned PSI. 0.4.2 adds the ``missing_policy`` parameter so callers
+    can opt in to the corrected behaviour. The default remains ``"drop"`` for
+    strict backward compatibility with the pre-0.4.2 numeric output; the next
+    minor release will flip the default to ``"include"``. Pass
+    ``missing_policy="include"`` to route NaN rows through a dedicated
+    ``"__MISSING__"`` bin on both sides so missing-rate drift contributes to
+    the PSI. Pass ``missing_policy="warn_and_drop"`` to keep the old numbers
+    but at least surface a ``RuntimeWarning`` naming the two NaN counts.
+
     Parameters
     ----------
     expected_series : pandas.Series
@@ -486,62 +532,112 @@ def _calculate_single_psi(
         Small value to avoid division by zero. Default is 1e-6.
     precision : int, optional
         Decimal precision. Default is 5.
-        
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        How NaN rows are handled. ``"drop"`` (default in 0.4.2) reproduces the
+        pre-0.4.2 behaviour and silently excludes NaN rows from both sides
+        before binning. ``"include"`` treats NaN as its own ``"__MISSING__"``
+        bin so missing-rate drift shows up in the PSI — recommended for any
+        production drift monitor, and will become the default in the next
+        minor release. ``"warn_and_drop"`` behaves like ``"drop"`` but emits
+        a ``RuntimeWarning`` naming the two NaN counts so the caller sees what
+        was dropped.
+
     Returns
     -------
     float or tuple
         If return_details is False: Returns total PSI value.
         If return_details is True: Returns tuple of (PSI value, details DataFrame).
-        
+
     Notes
     -----
     PSI Formula: Σ (Actual% - Expected%) * ln(Actual% / Expected%)
     A PSI < 0.1 indicates stable population, 0.1-0.25 suggests some change,
     and > 0.25 indicates significant drift.
     """
-    # Drop NA
+    if missing_policy not in {"include", "drop", "warn_and_drop"}:
+        raise ValueError(
+            f"_calculate_single_psi: missing_policy must be one of "
+            f"'include', 'drop', 'warn_and_drop'; got {missing_policy!r}."
+        )
+
+    # Record NaN counts before splitting so we can (a) re-attach the missing
+    # bin under "include" and (b) warn under "warn_and_drop".
+    n_expected = int(len(expected_series))
+    n_actual = int(len(actual_series))
+    n_expected_na = int(expected_series.isna().sum())
+    n_actual_na = int(actual_series.isna().sum())
+
+    if missing_policy == "warn_and_drop" and (n_expected_na or n_actual_na):
+        warnings.warn(
+            f"_calculate_single_psi: dropping NaN rows before binning. "
+            f"expected: {n_expected_na}/{n_expected} NaN, "
+            f"actual: {n_actual_na}/{n_actual} NaN. "
+            f"Pass missing_policy='include' to treat NaN as its own bin so "
+            f"missing-rate drift is captured.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
     expected_clean = expected_series.dropna()
     actual_clean = actual_series.dropna()
-    
+
     # Bin the expected data to get breakpoints
     expected_bins, breakpoints = quick_binning(
-        pd.DataFrame(expected_clean), 
-        expected_clean.name, 
-        labels=None, 
-        nbins=buckets, 
-        precision=precision, 
-        equal_freq=equal_freq, 
-        right=True, 
+        pd.DataFrame(expected_clean),
+        expected_clean.name,
+        labels=None,
+        nbins=buckets,
+        precision=precision,
+        equal_freq=equal_freq,
+        right=True,
         include_lowest=False,
-        min_bin_prop=min_bin_prop, 
-        tree_binning=False, 
-        target=None, 
+        min_bin_prop=min_bin_prop,
+        tree_binning=False,
+        target=None,
         random_state=42
     )
-    
+
     # Bin the actual data using the same breakpoints
     actual_bins, _ = quick_binning(
-        pd.DataFrame(actual_clean), 
-        actual_clean.name, 
-        labels=None, 
-        nbins=list(breakpoints), 
-        precision=precision, 
-        equal_freq=equal_freq, 
-        right=True, 
+        pd.DataFrame(actual_clean),
+        actual_clean.name,
+        labels=None,
+        nbins=list(breakpoints),
+        precision=precision,
+        equal_freq=equal_freq,
+        right=True,
         include_lowest=False,
-        min_bin_prop=min_bin_prop, 
-        tree_binning=False, 
-        target=None, 
+        min_bin_prop=min_bin_prop,
+        tree_binning=False,
+        target=None,
         random_state=42
     )
-    
-    # Get bin counts
+
+    # Get bin counts (finite-value rows only, from quick_binning)
     expected_count = expected_bins.value_counts(normalize=False, sort=False)
     actual_count = actual_bins.value_counts(normalize=False, sort=False)
-    
-    # Get bin proportions
-    expected_percents = expected_bins.value_counts(normalize=True, sort=False)
-    actual_percents = actual_bins.value_counts(normalize=True, sort=False)
+
+    if missing_policy == "include":
+        # Re-attach the __MISSING__ bin so that missing-rate drift contributes
+        # to the PSI on both sides. The denominators use the *original* row
+        # counts (before dropna) so the fractions are directly comparable.
+        if n_expected_na > 0:
+            expected_count = pd.concat(
+                [expected_count, pd.Series({_MISSING_BIN: n_expected_na})]
+            )
+        if n_actual_na > 0:
+            actual_count = pd.concat(
+                [actual_count, pd.Series({_MISSING_BIN: n_actual_na})]
+            )
+        expected_denom = float(n_expected) if n_expected > 0 else 1.0
+        actual_denom = float(n_actual) if n_actual > 0 else 1.0
+        expected_percents = expected_count / expected_denom
+        actual_percents = actual_count / actual_denom
+    else:
+        # "drop" / "warn_and_drop": legacy behaviour, denominators are the
+        # NaN-dropped row counts, so missing-rate drift is invisible.
+        expected_percents = expected_bins.value_counts(normalize=True, sort=False)
+        actual_percents = actual_bins.value_counts(normalize=True, sort=False)
     
     # Ensure both series have the same bin indices
     all_bins = expected_percents.index.union(actual_percents.index)
@@ -579,7 +675,8 @@ def calculate_psi(
     return_details: bool = False,
     min_bin_prop: float = 0.05,
     content: float = 1e-6,
-    precision: int = 5
+    precision: int = 5,
+    missing_policy: str = "include"
 ) -> Union[float, pd.DataFrame, Tuple[Dict, Dict]]:
     """
     Calculate Population Stability Index (PSI) for a variable, optionally by groups.
@@ -609,6 +706,14 @@ def calculate_psi(
         Small value to avoid division by zero. Default is 1e-6.
     precision : int, optional
         Decimal precision. Default is 5.
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        How NaN rows are handled. Default in 0.5.0 is "include", which routes
+        NaN rows through a dedicated "__MISSING__" bin so missing-rate drift
+        contributes to the PSI. This is the recommended production behaviour
+        and became the default in 0.5.0 (previously "drop" in 0.4.2 for
+        numeric backward-compat). Pass "drop" to reproduce pre-0.5.0
+        numbers, or "warn_and_drop" for legacy numbers with a RuntimeWarning
+        naming the NaN counts.
         
     Returns
     -------
@@ -661,7 +766,8 @@ def calculate_psi(
                     True,
                     min_bin_prop,
                     content,
-                    precision
+                    precision,
+                    missing_policy
                 )
                 results[group] = psi_value
                 details_dict[group] = detail
@@ -674,7 +780,8 @@ def calculate_psi(
                     False,
                     min_bin_prop,
                     content,
-                    precision
+                    precision,
+                    missing_policy
                 )
         
         if return_details:
@@ -692,7 +799,8 @@ def calculate_psi(
                 True, 
                 min_bin_prop, 
                 content, 
-                precision
+                precision,
+                missing_policy
             )
         else:
             return _calculate_single_psi(
@@ -703,7 +811,8 @@ def calculate_psi(
                 False, 
                 min_bin_prop, 
                 content, 
-                precision
+                precision,
+                missing_policy
             )
 
 
@@ -718,7 +827,8 @@ def calculate_within_psi(
     min_bin_prop: float = 0.05,
     content: float = 1e-6,
     precision: int = 5,
-    benchmark_display_name: Optional[str] = None
+    benchmark_display_name: Optional[str] = None,
+    missing_policy: str = "include"
 ) -> Union[pd.DataFrame, Dict]:
     """
     Calculate PSI values within a single dataset, comparing groups to a benchmark.
@@ -750,6 +860,14 @@ def calculate_within_psi(
         Decimal precision. Default is 5.
     benchmark_display_name : str, optional
         Custom name for benchmark in results.
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        How NaN rows are handled. Default in 0.5.0 is "include", which routes
+        NaN rows through a dedicated "__MISSING__" bin so missing-rate drift
+        contributes to the PSI. This is the recommended production behaviour
+        and became the default in 0.5.0 (previously "drop" in 0.4.2 for
+        numeric backward-compat). Pass "drop" to reproduce pre-0.5.0
+        numbers, or "warn_and_drop" for legacy numbers with a RuntimeWarning
+        naming the NaN counts.
         
     Returns
     -------
@@ -785,7 +903,8 @@ def calculate_within_psi(
                 return_details=True, 
                 min_bin_prop=min_bin_prop, 
                 content=content, 
-                precision=precision
+                precision=precision,
+                missing_policy=missing_policy
             )
             res_dict[obs_value] = psi
             detail_dict[obs_value] = details
@@ -799,7 +918,8 @@ def calculate_within_psi(
                 return_details=False, 
                 min_bin_prop=min_bin_prop, 
                 content=content, 
-                precision=precision
+                precision=precision,
+                missing_policy=missing_policy
             )
             res_dict[obs_value] = psi
     
@@ -822,7 +942,8 @@ def calculate_psi_within_dataset(
     buckets: int = 10,
     min_bin_prop: float = 0.05,
     content: float = 1e-6,
-    precision: int = 5
+    precision: int = 5,
+    missing_policy: str = "include"
 ) -> pd.DataFrame:
     """
     Calculate PSI for multiple variables within a dataset, comparing groups to a benchmark.
@@ -850,6 +971,14 @@ def calculate_psi_within_dataset(
         Small value to avoid division by zero. Default is 1e-6.
     precision : int, optional
         Decimal precision. Default is 5.
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        How NaN rows are handled. Default in 0.5.0 is "include", which routes
+        NaN rows through a dedicated "__MISSING__" bin so missing-rate drift
+        contributes to the PSI. This is the recommended production behaviour
+        and became the default in 0.5.0 (previously "drop" in 0.4.2 for
+        numeric backward-compat). Pass "drop" to reproduce pre-0.5.0
+        numbers, or "warn_and_drop" for legacy numbers with a RuntimeWarning
+        naming the NaN counts.
         
     Returns
     -------
@@ -873,7 +1002,8 @@ def calculate_psi_within_dataset(
             return_details=False, 
             min_bin_prop=min_bin_prop, 
             content=content, 
-            precision=precision
+            precision=precision,
+            missing_policy=missing_policy
         ).sort_values([grp_name]).reset_index(drop=True)
         
         single_psi['var'] = var
@@ -891,7 +1021,8 @@ def calculate_multivar_psi_two_sets(
     equal_freq: bool = True,
     min_bin_prop: float = 0.05,
     content: float = 1e-6,
-    precision: int = 5
+    precision: int = 5,
+    missing_policy: str = "include"
 ) -> pd.DataFrame:
     """
     Calculate PSI for multiple variables by comparing two different datasets.
@@ -919,6 +1050,14 @@ def calculate_multivar_psi_two_sets(
         Small value to avoid division by zero. Default is 1e-6.
     precision : int, optional
         Decimal precision. Default is 5.
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        How NaN rows are handled. Default in 0.5.0 is "include", which routes
+        NaN rows through a dedicated "__MISSING__" bin so missing-rate drift
+        contributes to the PSI. This is the recommended production behaviour
+        and became the default in 0.5.0 (previously "drop" in 0.4.2 for
+        numeric backward-compat). Pass "drop" to reproduce pre-0.5.0
+        numbers, or "warn_and_drop" for legacy numbers with a RuntimeWarning
+        naming the NaN counts.
         
     Returns
     -------
@@ -938,7 +1077,8 @@ def calculate_multivar_psi_two_sets(
             target_col=var, 
             group_by=group_by, 
             buckets=buckets,
-            return_details=False
+            return_details=False,
+            missing_policy=missing_policy
         )
         if group_by is None:
             single_psi = pd.DataFrame([single_psi], columns=['psi'])
@@ -1078,7 +1218,8 @@ def calculate_multigroup_psi_two_sets(
     content: float = 1e-6,
     precision: int = 5,
     group_name: Optional[str] = None,
-    return_details: bool = False
+    return_details: bool = False,
+    missing_policy: str = "include"
 ) -> Union[pd.DataFrame, Dict[str, pd.DataFrame]]:
     """
     Calculate grouped PSI using expected DataFrame as benchmark, applied to actual DataFrame groups.
@@ -1108,6 +1249,12 @@ def calculate_multigroup_psi_two_sets(
     return_details : bool, optional
         是否返回详细分箱信息。若为 True，返回字典 {'psi': psi_df, 'details': details_df}，
         details_df 包含列：['bin', 'expected_percent', 'actual_percent', 'psi_component', group_name, 'var']
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        NaN 行处理策略。0.5.0 起默认为 "include"，将 NaN 行汇入独立的
+        "__MISSING__" 桶，使缺失率漂移进入 PSI 计算；这是推荐的生产行为，
+        并在 0.5.0 成为默认值（0.4.2 为 "drop" 以保持数值向后兼容）。
+        传入 "drop" 可复现 0.5.0 之前的数值，或 "warn_and_drop" 保留旧数值
+        同时发出 RuntimeWarning 报告两侧 NaN 数量。
     """
     if group_name is not None:
         if actual_df[group_name].isna().sum() > 0:
@@ -1130,7 +1277,8 @@ def calculate_multigroup_psi_two_sets(
                         return_details=True,
                         min_bin_prop=min_bin_prop,
                         content=content,
-                        precision=precision
+                        precision=precision,
+                        missing_policy=missing_policy
                     )
                     
                     psi_records.append({group_name: group, 'var': var, 'psi': psi_val})
@@ -1177,7 +1325,8 @@ def calculate_multigroup_psi_two_sets(
                     return_details=True,
                     min_bin_prop=min_bin_prop,
                     content=content,
-                    precision=precision
+                    precision=precision,
+                    missing_policy=missing_policy
                 )
                 psi_records.append({'var': var, 'psi': psi_val})
                 
@@ -1212,7 +1361,8 @@ def calculate_multigroup_psi_two_sets(
                 equal_freq=equal_freq, 
                 min_bin_prop=min_bin_prop, 
                 content=content, 
-                precision=precision
+                precision=precision,
+                missing_policy=missing_policy
             )
             
             return group_psi

{supermodelingfactory-0.3.15 → supermodelingfactory-0.5.0}/Modeling_Tool/Feature/Weighted_Screen.py

@@ -113,11 +113,25 @@ def _weighted_bin_distribution(
     w: np.ndarray,
     content: float,
 ) -> pd.Series:
+    """Return the weighted bin distribution as an observed-only frequency
+    Series.
+
+    Fix (0.4.2, N23): the pre-0.4.2 version unioned ``_MISSING_BIN`` into the
+    returned index and clipped every entry to ``content`` (a small positive
+    floor such as ``1e-5``). When only one side of a PSI comparison had missing
+    values, that forced a large synthetic mass on the missing bin of the side
+    that actually observed zero missings — which then propagated through
+    ``_psi_from_distributions`` as a large spurious PSI component. The fix is
+    to return only the bins actually observed in ``bins``; the alignment and
+    zero-safety are done exactly once, in ``_psi_from_distributions``, so both
+    sides are treated symmetrically. The ``content`` parameter is retained for
+    signature stability but is now unused in this helper.
+    """
+    del content  # retained for signature stability; alignment is elsewhere.
     df = pd.DataFrame({"bin": bins, "w": w})
     dist = df.groupby("bin", dropna=False)["w"].sum()
     total = float(dist.sum()) or 1.0
-    dist = dist / total
-    return dist.reindex(dist.index.union([_MISSING_BIN]), fill_value=0.0).clip(lower=content)
+    return dist / total
 
 
 def _psi_from_distributions(expected: pd.Series, actual: pd.Series, content: float) -> float: