SuperModelingFactory 0.4.2__tar.gz → 0.5.1__tar.gz

{supermodelingfactory-0.4.2 → supermodelingfactory-0.5.1}/Modeling_Tool/Feature/Feature_Insights.py

@@ -4,9 +4,11 @@
 """
 
 import pandas as pd
+import warnings
 from tqdm import tqdm
 
 from .Distribution_Tool import proc_means_by_grp, proc_means_for_screening
+from Modeling_Tool._utils.robust import smf_logger
 import logging
 logger = logging.getLogger(__name__)
 
@@ -108,6 +110,7 @@ class VarExtractionInsights:
         self.seed = seed
         self.missing_rate_ref = missing_rate_ref
         self.spec_values = spec_values if spec_values is not None else []
+        self.failed_variables = []
 
     @staticmethod
     def remove_folder(file_path):
@@ -170,6 +173,7 @@ class VarExtractionInsights:
         """
         if dep is None:
             dep = self.dep
+        self.failed_variables = []
 
         from Modeling_Tool.Eval.Model_Eval_Tool import get_gains_table
 
@@ -200,9 +204,27 @@ class VarExtractionInsights:
                     attr_iv['var'] = var
                     iv_info_res.append(attr_iv)
 
-                except TypeError:
+                except (TypeError, ValueError, KeyError, ZeroDivisionError) as exc:
+                    row = smf_logger.record_and_continue(var, exc, stage="feature_insights")
+                    self.failed_variables.append((row["feature"], row["exception_type"]))
                     continue
 
+        if self.failed_variables:
+            failed = [name for name, _ in self.failed_variables]
+            warnings.warn(
+                f"{len(failed)}/{len(varlist)} variables failed insight computation: "
+                f"{failed[:10]}{'...' if len(failed) > 10 else ''} — see smf_logger for details",
+                UserWarning,
+                stacklevel=2,
+            )
+
+        output_cols = [
+            'var', 'n_all', 'n', 'ks_in_gains', 'lift_in_gains', 'iv',
+            'n_bump', 'missing_rate', 'min', 'mean', 'max', 'n_bins'
+        ]
+        if not iv_info_res:
+            return pd.DataFrame(columns=output_cols)
+
         iv_info_res = pd.concat(iv_info_res).sort_values("IV", ascending=False)
 
         high_iv_summary = iv_info_res.query(f"IV >= {iv_cut}").round(4)
@@ -222,10 +244,7 @@ class VarExtractionInsights:
             how='left'
         )
         fnl_summary.columns = [x.lower() for x in fnl_summary.columns]
-        fnl_summary = fnl_summary[[
-            'var', 'n_all', 'n', 'ks_in_gains', 'lift_in_gains', 'iv',
-            'n_bump', 'missing_rate', 'min', 'mean', 'max', 'n_bins'
-        ]]
+        fnl_summary = fnl_summary[output_cols]
 
         return fnl_summary
 

{supermodelingfactory-0.4.2 → supermodelingfactory-0.5.1}/Modeling_Tool/Feature/Feature_Screen.py

@@ -10,6 +10,7 @@ import numpy as np
 import pandas as pd
 
 from Modeling_Tool.Core.sample_weight_utils import resolve_sample_weight
+from Modeling_Tool._utils.sentinels import SMF_MISSING_BIN
 
 from .Weighted_Screen import (
     WeightedScreenResult,
@@ -163,7 +164,7 @@ def fit_screening_woe_engine(
 
     master_params = dict(woe_params or {})
     woe_suffix = master_params.pop("woe_suffix", "_woe")
-    missing_ref_value = master_params.pop("missing_ref_value", -999999)
+    missing_ref_value = master_params.pop("missing_ref_value", SMF_MISSING_BIN)
     master = WOE_Master(
         train_data=fit_ins,
         varlist=features,
@@ -224,7 +225,7 @@ def _woe_bins_unweighted_screen(
         missing_rate_ref=config.missing_rate_ref,
         on_empty_stage=config.on_empty_stage,
     )
-    binner = _resolve_screening_binner(splits, feature_cols, target_col, config, prefit_woe_engine)
+    binner = _resolve_screening_binner(splits, current, target_col, config, prefit_woe_engine)
 
     psi_table = pd.DataFrame(columns=["var", "psi_ins_oos", "psi_ins_oot", "psi_max"])
     if config.psi_enabled:
@@ -357,7 +358,7 @@ def _weighted_woe_bins_screen(
     if binner is None:
         binner = fit_screening_woe_engine(
             ins,
-            feature_cols,
+            current,
             target_col,
             woe_engine=config.woe_engine,
             woe_fit_query=config.woe_fit_query,
@@ -369,7 +370,7 @@ def _weighted_woe_bins_screen(
 
     psi_records: list[dict] = []
     if config.psi_enabled:
-        for var in feature_cols:
+        for var in current:
             if var not in ins.columns or ins[var].nunique(dropna=False) <= 1:
                 continue
             bins_ins = _bins_to_numpy(adapter.assign_bins(ins, var))
@@ -583,4 +584,4 @@ __all__ = [
     "feature_screen_from_dataframe",
     "fit_screening_woe_engine",
     "screen_config_from_mapping",
-]
+]

{supermodelingfactory-0.4.2 → supermodelingfactory-0.5.1}/Modeling_Tool/Feature/PSI_Tool.py

@@ -20,6 +20,58 @@ from Modeling_Tool.Core.Binning_Tool import quick_binning
 # ``Feature.Weighted_Screen`` and ``WOE_Adapter`` so downstream aggregators can
 # recognise it without special-casing.
 _MISSING_BIN = "__MISSING__"
+_PSI_BUCKET_POLICIES = {"floor_1e6", "smooth_laplace", "exclude"}
+
+
+def _validate_psi_bucket_policy(policy: str, caller: str) -> None:
+    if policy not in _PSI_BUCKET_POLICIES:
+        raise ValueError(
+            f"{caller}: psi_missing_bucket_policy must be one of "
+            f"{sorted(_PSI_BUCKET_POLICIES)}; got {policy!r}."
+        )
+
+
+def _psi_distributions_from_counts(
+    expected_count: pd.Series,
+    actual_count: pd.Series,
+    expected_total: float,
+    actual_total: float,
+    *,
+    content: float,
+    policy: str,
+) -> tuple[pd.Series, pd.Series, pd.Series, pd.Series]:
+    _validate_psi_bucket_policy(policy, "_psi_distributions_from_counts")
+    all_bins = expected_count.index.union(actual_count.index)
+    expected_aligned = expected_count.reindex(all_bins, fill_value=0).astype(float)
+    actual_aligned = actual_count.reindex(all_bins, fill_value=0).astype(float)
+    one_sided = (expected_aligned == 0) | (actual_aligned == 0)
+
+    expected_total = float(expected_total) if expected_total > 0 else 1.0
+    actual_total = float(actual_total) if actual_total > 0 else 1.0
+
+    if policy == "exclude":
+        keep = ~one_sided
+        expected_aligned = expected_aligned[keep]
+        actual_aligned = actual_aligned[keep]
+        if expected_aligned.empty:
+            empty = pd.Series(dtype=float)
+            return empty, empty, empty, empty
+        expected_pct = expected_aligned / expected_total
+        actual_pct = actual_aligned / actual_total
+    elif policy == "smooth_laplace" and bool(one_sided.any()):
+        n_buckets = max(len(all_bins), 1)
+        expected_pct = (expected_aligned + 1.0) / (expected_total + n_buckets)
+        actual_pct = (actual_aligned + 1.0) / (actual_total + n_buckets)
+    else:
+        expected_pct = (expected_aligned / expected_total).clip(lower=content)
+        actual_pct = (actual_aligned / actual_total).clip(lower=content)
+
+    psi_values = (actual_pct - expected_pct) * np.log(actual_pct / expected_pct)
+
+    floor_expected = (expected_aligned / expected_total).clip(lower=content)
+    floor_actual = (actual_aligned / actual_total).clip(lower=content)
+    floor_values = (floor_actual - floor_expected) * np.log(floor_actual / floor_expected)
+    return expected_pct, actual_pct, psi_values, floor_values
 
 # ============================================================================
 # Classes
@@ -58,7 +110,9 @@ class PSICalculator:
         equal_freq: bool = True,
         min_bin_prop: float = 0.05,
         content: float = 1e-6,
-        precision: int = 5
+        precision: int = 5,
+        missing_policy: str = "include",
+        psi_missing_bucket_policy: str = "smooth_laplace",
     ):
         """
         Initialize PSICalculator with configuration parameters.
@@ -75,12 +129,28 @@ class PSICalculator:
             Small value to prevent division by zero. Default is 1e-6.
         precision : int, optional
             Decimal precision for rounding. Default is 5.
+        missing_policy : {"drop", "include", "warn_and_drop"}, optional
+            How NaN rows are handled. Default in 0.5.0 is "include", which routes
+            NaN rows through a dedicated "__MISSING__" bin so missing-rate drift
+            contributes to the PSI. This is the recommended production behaviour
+            and became the default in 0.5.0 (previously "drop" in 0.4.2 for
+            numeric backward-compat). Pass "drop" to reproduce pre-0.5.0
+            numbers, or "warn_and_drop" for legacy numbers with a RuntimeWarning
+            naming the NaN counts.
         """
+        if missing_policy not in {"include", "drop", "warn_and_drop"}:
+            raise ValueError(
+                f"PSICalculator.__init__: missing_policy must be one of "
+                f"'include', 'drop', 'warn_and_drop'; got {missing_policy!r}."
+            )
+        _validate_psi_bucket_policy(psi_missing_bucket_policy, "PSICalculator.__init__")
         self.buckets = buckets
         self.equal_freq = equal_freq
         self.min_bin_prop = min_bin_prop
         self.content = content
         self.precision = precision
+        self.missing_policy = missing_policy
+        self.psi_missing_bucket_policy = psi_missing_bucket_policy
     
 #     def _calculate_single_psi(
 #         self,
@@ -418,7 +488,9 @@ class PSICalculator:
         varlist: List[str],
         group_by: Optional[str] = None,
         group_name: Optional[str] = None,
-        return_details = False
+        return_details = False,
+        missing_policy: Optional[str] = None,
+        psi_missing_bucket_policy: Optional[str] = None,
     ) -> pd.DataFrame:
         """
         Calculate grouped PSI comparing two datasets, using expected as benchmark.
@@ -435,12 +507,23 @@ class PSICalculator:
             Column to group by in both datasets.
         group_name : str, optional
             Specific group column name for multi-group calculation.
+        missing_policy : {"drop", "include", "warn_and_drop"}, optional
+            How NaN rows are handled. If None (default), falls back to the value
+            configured on ``self.missing_policy``. Pass an explicit value to
+            override the class-level default for this call only. See
+            ``PSICalculator.__init__`` for the semantics of each mode.
             
         Returns
         -------
         pandas.DataFrame
             Grouped PSI results.
         """
+        effective_policy = missing_policy if missing_policy is not None else self.missing_policy
+        effective_bucket_policy = (
+            psi_missing_bucket_policy
+            if psi_missing_bucket_policy is not None
+            else self.psi_missing_bucket_policy
+        )
         return calculate_multigroup_psi_two_sets(
             expected_df = expected_df,
             actual_df = current_data,
@@ -452,7 +535,9 @@ class PSICalculator:
             content = self.content,
             precision = self.precision,
             group_name = group_name,
-            return_details = return_details
+            return_details = return_details,
+            missing_policy = effective_policy,
+            psi_missing_bucket_policy = effective_bucket_policy,
         )
 
 
@@ -469,7 +554,8 @@ def _calculate_single_psi(
     min_bin_prop: float = 0.05,
     content: float = 1e-6,
     precision: int = 5,
-    missing_policy: str = "drop",
+    missing_policy: str = "include",
+    psi_missing_bucket_policy: str = "smooth_laplace",
 ) -> Union[float, Tuple[float, pd.DataFrame]]:
     """
     Calculate Population Stability Index (PSI) for a single variable.
@@ -536,6 +622,7 @@ def _calculate_single_psi(
             f"_calculate_single_psi: missing_policy must be one of "
             f"'include', 'drop', 'warn_and_drop'; got {missing_policy!r}."
         )
+    _validate_psi_bucket_policy(psi_missing_bucket_policy, "_calculate_single_psi")
 
     # Record NaN counts before splitting so we can (a) re-attach the missing
     # bin under "include" and (b) warn under "warn_and_drop".
@@ -608,25 +695,24 @@ def _calculate_single_psi(
             )
         expected_denom = float(n_expected) if n_expected > 0 else 1.0
         actual_denom = float(n_actual) if n_actual > 0 else 1.0
-        expected_percents = expected_count / expected_denom
-        actual_percents = actual_count / actual_denom
     else:
         # "drop" / "warn_and_drop": legacy behaviour, denominators are the
         # NaN-dropped row counts, so missing-rate drift is invisible.
-        expected_percents = expected_bins.value_counts(normalize=True, sort=False)
-        actual_percents = actual_bins.value_counts(normalize=True, sort=False)
-    
-    # Ensure both series have the same bin indices
-    all_bins = expected_percents.index.union(actual_percents.index)
-    expected_percents = expected_percents.reindex(all_bins, fill_value=content)
-    actual_percents = actual_percents.reindex(all_bins, fill_value=content)
-    
-    # Clip to avoid division by zero
-    expected_percents = expected_percents.clip(lower=content)
-    actual_percents = actual_percents.clip(lower=content)
-    
-    # Calculate PSI
-    psi_values = (actual_percents - expected_percents) * np.log(actual_percents / expected_percents)
+        expected_denom = float(len(expected_bins)) if len(expected_bins) > 0 else 1.0
+        actual_denom = float(len(actual_bins)) if len(actual_bins) > 0 else 1.0
+
+    all_bins = expected_count.index.union(actual_count.index)
+    expected_count = expected_count.reindex(all_bins, fill_value=0).astype(float)
+    actual_count = actual_count.reindex(all_bins, fill_value=0).astype(float)
+
+    expected_percents, actual_percents, psi_values, psi_floor_values = _psi_distributions_from_counts(
+        expected_count,
+        actual_count,
+        expected_denom,
+        actual_denom,
+        content=content,
+        policy=psi_missing_bucket_policy,
+    )
     psi_total = psi_values.sum()
     
     if return_details:
@@ -635,8 +721,13 @@ def _calculate_single_psi(
             'actual_count': actual_count,
             'expected_percent': expected_percents,
             'actual_percent': actual_percents,
-            'psi_component': psi_values
+            'psi_component': psi_values,
+            'psi_component_floor_1e6': psi_floor_values,
         })
+        details["bucket_status"] = "common"
+        details.loc[(details["expected_count"] == 0) & (details["actual_count"] > 0), "bucket_status"] = "actual_only"
+        details.loc[(details["expected_count"] > 0) & (details["actual_count"] == 0), "bucket_status"] = "expected_only"
+        details["psi_missing_bucket_policy"] = psi_missing_bucket_policy
         return psi_total, details
     else:
         return psi_total
@@ -652,7 +743,9 @@ def calculate_psi(
     return_details: bool = False,
     min_bin_prop: float = 0.05,
     content: float = 1e-6,
-    precision: int = 5
+    precision: int = 5,
+    missing_policy: str = "include",
+    psi_missing_bucket_policy: str = "smooth_laplace",
 ) -> Union[float, pd.DataFrame, Tuple[Dict, Dict]]:
     """
     Calculate Population Stability Index (PSI) for a variable, optionally by groups.
@@ -682,6 +775,14 @@ def calculate_psi(
         Small value to avoid division by zero. Default is 1e-6.
     precision : int, optional
         Decimal precision. Default is 5.
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        How NaN rows are handled. Default in 0.5.0 is "include", which routes
+        NaN rows through a dedicated "__MISSING__" bin so missing-rate drift
+        contributes to the PSI. This is the recommended production behaviour
+        and became the default in 0.5.0 (previously "drop" in 0.4.2 for
+        numeric backward-compat). Pass "drop" to reproduce pre-0.5.0
+        numbers, or "warn_and_drop" for legacy numbers with a RuntimeWarning
+        naming the NaN counts.
         
     Returns
     -------
@@ -734,7 +835,9 @@ def calculate_psi(
                     True,
                     min_bin_prop,
                     content,
-                    precision
+                    precision,
+                    missing_policy,
+                    psi_missing_bucket_policy,
                 )
                 results[group] = psi_value
                 details_dict[group] = detail
@@ -747,7 +850,9 @@ def calculate_psi(
                     False,
                     min_bin_prop,
                     content,
-                    precision
+                    precision,
+                    missing_policy,
+                    psi_missing_bucket_policy,
                 )
         
         if return_details:
@@ -765,7 +870,9 @@ def calculate_psi(
                 True, 
                 min_bin_prop, 
                 content, 
-                precision
+                precision,
+                missing_policy,
+                psi_missing_bucket_policy,
             )
         else:
             return _calculate_single_psi(
@@ -776,7 +883,9 @@ def calculate_psi(
                 False, 
                 min_bin_prop, 
                 content, 
-                precision
+                precision,
+                missing_policy,
+                psi_missing_bucket_policy,
             )
 
 
@@ -791,7 +900,9 @@ def calculate_within_psi(
     min_bin_prop: float = 0.05,
     content: float = 1e-6,
     precision: int = 5,
-    benchmark_display_name: Optional[str] = None
+    benchmark_display_name: Optional[str] = None,
+    missing_policy: str = "include",
+    psi_missing_bucket_policy: str = "smooth_laplace",
 ) -> Union[pd.DataFrame, Dict]:
     """
     Calculate PSI values within a single dataset, comparing groups to a benchmark.
@@ -823,6 +934,14 @@ def calculate_within_psi(
         Decimal precision. Default is 5.
     benchmark_display_name : str, optional
         Custom name for benchmark in results.
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        How NaN rows are handled. Default in 0.5.0 is "include", which routes
+        NaN rows through a dedicated "__MISSING__" bin so missing-rate drift
+        contributes to the PSI. This is the recommended production behaviour
+        and became the default in 0.5.0 (previously "drop" in 0.4.2 for
+        numeric backward-compat). Pass "drop" to reproduce pre-0.5.0
+        numbers, or "warn_and_drop" for legacy numbers with a RuntimeWarning
+        naming the NaN counts.
         
     Returns
     -------
@@ -858,7 +977,9 @@ def calculate_within_psi(
                 return_details=True, 
                 min_bin_prop=min_bin_prop, 
                 content=content, 
-                precision=precision
+                precision=precision,
+                missing_policy=missing_policy,
+                psi_missing_bucket_policy=psi_missing_bucket_policy,
             )
             res_dict[obs_value] = psi
             detail_dict[obs_value] = details
@@ -872,7 +993,9 @@ def calculate_within_psi(
                 return_details=False, 
                 min_bin_prop=min_bin_prop, 
                 content=content, 
-                precision=precision
+                precision=precision,
+                missing_policy=missing_policy,
+                psi_missing_bucket_policy=psi_missing_bucket_policy,
             )
             res_dict[obs_value] = psi
     
@@ -895,7 +1018,9 @@ def calculate_psi_within_dataset(
     buckets: int = 10,
     min_bin_prop: float = 0.05,
     content: float = 1e-6,
-    precision: int = 5
+    precision: int = 5,
+    missing_policy: str = "include",
+    psi_missing_bucket_policy: str = "smooth_laplace",
 ) -> pd.DataFrame:
     """
     Calculate PSI for multiple variables within a dataset, comparing groups to a benchmark.
@@ -923,6 +1048,14 @@ def calculate_psi_within_dataset(
         Small value to avoid division by zero. Default is 1e-6.
     precision : int, optional
         Decimal precision. Default is 5.
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        How NaN rows are handled. Default in 0.5.0 is "include", which routes
+        NaN rows through a dedicated "__MISSING__" bin so missing-rate drift
+        contributes to the PSI. This is the recommended production behaviour
+        and became the default in 0.5.0 (previously "drop" in 0.4.2 for
+        numeric backward-compat). Pass "drop" to reproduce pre-0.5.0
+        numbers, or "warn_and_drop" for legacy numbers with a RuntimeWarning
+        naming the NaN counts.
         
     Returns
     -------
@@ -946,7 +1079,9 @@ def calculate_psi_within_dataset(
             return_details=False, 
             min_bin_prop=min_bin_prop, 
             content=content, 
-            precision=precision
+            precision=precision,
+            missing_policy=missing_policy,
+            psi_missing_bucket_policy=psi_missing_bucket_policy,
         ).sort_values([grp_name]).reset_index(drop=True)
         
         single_psi['var'] = var
@@ -964,7 +1099,9 @@ def calculate_multivar_psi_two_sets(
     equal_freq: bool = True,
     min_bin_prop: float = 0.05,
     content: float = 1e-6,
-    precision: int = 5
+    precision: int = 5,
+    missing_policy: str = "include",
+    psi_missing_bucket_policy: str = "smooth_laplace",
 ) -> pd.DataFrame:
     """
     Calculate PSI for multiple variables by comparing two different datasets.
@@ -992,6 +1129,14 @@ def calculate_multivar_psi_two_sets(
         Small value to avoid division by zero. Default is 1e-6.
     precision : int, optional
         Decimal precision. Default is 5.
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        How NaN rows are handled. Default in 0.5.0 is "include", which routes
+        NaN rows through a dedicated "__MISSING__" bin so missing-rate drift
+        contributes to the PSI. This is the recommended production behaviour
+        and became the default in 0.5.0 (previously "drop" in 0.4.2 for
+        numeric backward-compat). Pass "drop" to reproduce pre-0.5.0
+        numbers, or "warn_and_drop" for legacy numbers with a RuntimeWarning
+        naming the NaN counts.
         
     Returns
     -------
@@ -1011,7 +1156,13 @@ def calculate_multivar_psi_two_sets(
             target_col=var, 
             group_by=group_by, 
             buckets=buckets,
-            return_details=False
+            equal_freq=equal_freq,
+            min_bin_prop=min_bin_prop,
+            content=content,
+            precision=precision,
+            return_details=False,
+            missing_policy=missing_policy,
+            psi_missing_bucket_policy=psi_missing_bucket_policy,
         )
         if group_by is None:
             single_psi = pd.DataFrame([single_psi], columns=['psi'])
@@ -1151,7 +1302,9 @@ def calculate_multigroup_psi_two_sets(
     content: float = 1e-6,
     precision: int = 5,
     group_name: Optional[str] = None,
-    return_details: bool = False
+    return_details: bool = False,
+    missing_policy: str = "include",
+    psi_missing_bucket_policy: str = "smooth_laplace",
 ) -> Union[pd.DataFrame, Dict[str, pd.DataFrame]]:
     """
     Calculate grouped PSI using expected DataFrame as benchmark, applied to actual DataFrame groups.
@@ -1181,6 +1334,12 @@ def calculate_multigroup_psi_two_sets(
     return_details : bool, optional
         是否返回详细分箱信息。若为 True，返回字典 {'psi': psi_df, 'details': details_df}，
         details_df 包含列：['bin', 'expected_percent', 'actual_percent', 'psi_component', group_name, 'var']
+    missing_policy : {"drop", "include", "warn_and_drop"}, optional
+        NaN 行处理策略。0.5.0 起默认为 "include"，将 NaN 行汇入独立的
+        "__MISSING__" 桶，使缺失率漂移进入 PSI 计算；这是推荐的生产行为，
+        并在 0.5.0 成为默认值（0.4.2 为 "drop" 以保持数值向后兼容）。
+        传入 "drop" 可复现 0.5.0 之前的数值，或 "warn_and_drop" 保留旧数值
+        同时发出 RuntimeWarning 报告两侧 NaN 数量。
     """
     if group_name is not None:
         if actual_df[group_name].isna().sum() > 0:
@@ -1203,7 +1362,9 @@ def calculate_multigroup_psi_two_sets(
                         return_details=True,
                         min_bin_prop=min_bin_prop,
                         content=content,
-                        precision=precision
+                        precision=precision,
+                        missing_policy=missing_policy,
+                        psi_missing_bucket_policy=psi_missing_bucket_policy,
                     )
                     
                     psi_records.append({group_name: group, 'var': var, 'psi': psi_val})
@@ -1221,7 +1382,7 @@ def calculate_multigroup_psi_two_sets(
                         detail_df[group_name] = group
                         detail_df['var'] = var
                         
-                        required_cols = ['bin', 'expected_count', 'actual_count', 'expected_percent', 'actual_percent', 'psi_component', group_name, 'var']
+                        required_cols = ['bin', 'expected_count', 'actual_count', 'expected_percent', 'actual_percent', 'psi_component', 'bucket_status', 'psi_missing_bucket_policy', group_name, 'var']
                         for col in required_cols:
                             if col not in detail_df.columns:
                                 detail_df[col] = np.nan
@@ -1250,7 +1411,9 @@ def calculate_multigroup_psi_two_sets(
                     return_details=True,
                     min_bin_prop=min_bin_prop,
                     content=content,
-                    precision=precision
+                    precision=precision,
+                    missing_policy=missing_policy,
+                    psi_missing_bucket_policy=psi_missing_bucket_policy,
                 )
                 psi_records.append({'var': var, 'psi': psi_val})
                 
@@ -1262,7 +1425,7 @@ def calculate_multigroup_psi_two_sets(
                         index_col = detail_df.columns[0]
                         detail_df = detail_df.rename(columns={index_col: 'bin'})
                     detail_df['var'] = var
-                    required_cols = ['bin', 'expected_count', 'actual_count', 'expected_percent', 'actual_percent', 'psi_component', 'var']
+                    required_cols = ['bin', 'expected_count', 'actual_count', 'expected_percent', 'actual_percent', 'psi_component', 'bucket_status', 'psi_missing_bucket_policy', 'var']
                     for col in required_cols:
                         if col not in detail_df.columns:
                             detail_df[col] = np.nan
@@ -1285,7 +1448,9 @@ def calculate_multigroup_psi_two_sets(
                 equal_freq=equal_freq, 
                 min_bin_prop=min_bin_prop, 
                 content=content, 
-                precision=precision
+                precision=precision,
+                missing_policy=missing_policy,
+                psi_missing_bucket_policy=psi_missing_bucket_policy,
             )
             
             return group_psi

{supermodelingfactory-0.4.2 → supermodelingfactory-0.5.1}/Modeling_Tool/Feature/PSI_Tool.pyi

@@ -20,10 +20,10 @@ from tqdm import tqdm
 from Modeling_Tool.Core.Binning_Tool import quick_binning
 
 class PSICalculator:
-    def __init__(self, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, binning_engine: Any = None): ...
-    def calculate(self, expected_df: pd.DataFrame, current_data: pd.DataFrame, varlist: List[str], group_by: Optional[str] = None, group_name: Optional[str] = None, return_details = False) -> pd.DataFrame: ...
-def calculate_psi(expected: Union[pd.DataFrame, pd.Series], actual: Union[pd.DataFrame, pd.Series], target_col: str, buckets: int = 10, equal_freq: bool = True, group_by: Optional[Union[str, List[str]]] = None, return_details: bool = False, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5) -> Union[float, pd.DataFrame, Tuple[Dict, Dict]]: ...
-def calculate_within_psi(data: pd.DataFrame, grp_name: str, target_col: str, benchmark: Optional[Any] = None, equal_freq: bool = True, buckets: int = 10, return_details: bool = False, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, benchmark_display_name: Optional[str] = None) -> Union[pd.DataFrame, Dict]: ...
-def calculate_psi_within_dataset(data: pd.DataFrame, grp_name: str, varlist: List[str], benchmark: Optional[Any] = None, equal_freq: bool = True, buckets: int = 10, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5) -> pd.DataFrame: ...
-def calculate_multivar_psi_two_sets(expected_df: pd.DataFrame, actual_df: pd.DataFrame, varlist: List[str], group_by: Optional[Union[str, List[str]]] = None, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5) -> pd.DataFrame: ...
-def calculate_multigroup_psi_two_sets(expected_df: pd.DataFrame, actual_df: pd.DataFrame, varlist: List[str], group_by: Optional[Union[str, List[str]]] = None, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, group_name: Optional[str] = None, return_details: bool = False) -> Union[pd.DataFrame, Dict[str, pd.DataFrame]]: ...
+    def __init__(self, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace", binning_engine: Any = None): ...
+    def calculate(self, expected_df: pd.DataFrame, current_data: pd.DataFrame, varlist: List[str], group_by: Optional[str] = None, group_name: Optional[str] = None, return_details = False, missing_policy: Optional[str] = None, psi_missing_bucket_policy: Optional[str] = None) -> pd.DataFrame: ...
+def calculate_psi(expected: Union[pd.DataFrame, pd.Series], actual: Union[pd.DataFrame, pd.Series], target_col: str, buckets: int = 10, equal_freq: bool = True, group_by: Optional[Union[str, List[str]]] = None, return_details: bool = False, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace") -> Union[float, pd.DataFrame, Tuple[Dict, Dict]]: ...
+def calculate_within_psi(data: pd.DataFrame, grp_name: str, target_col: str, benchmark: Optional[Any] = None, equal_freq: bool = True, buckets: int = 10, return_details: bool = False, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, benchmark_display_name: Optional[str] = None, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace") -> Union[pd.DataFrame, Dict]: ...
+def calculate_psi_within_dataset(data: pd.DataFrame, grp_name: str, varlist: List[str], benchmark: Optional[Any] = None, equal_freq: bool = True, buckets: int = 10, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace") -> pd.DataFrame: ...
+def calculate_multivar_psi_two_sets(expected_df: pd.DataFrame, actual_df: pd.DataFrame, varlist: List[str], group_by: Optional[Union[str, List[str]]] = None, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace") -> pd.DataFrame: ...
+def calculate_multigroup_psi_two_sets(expected_df: pd.DataFrame, actual_df: pd.DataFrame, varlist: List[str], group_by: Optional[Union[str, List[str]]] = None, buckets: int = 10, equal_freq: bool = True, min_bin_prop: float = 0.05, content: float = 1e-06, precision: int = 5, group_name: Optional[str] = None, return_details: bool = False, missing_policy: str = "include", psi_missing_bucket_policy: str = "smooth_laplace") -> Union[pd.DataFrame, Dict[str, pd.DataFrame]]: ...