SuperModelingFactory 0.3.14__tar.gz → 0.3.15__tar.gz

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/Distribution_Tool.py

@@ -186,6 +186,40 @@ def proc_means_by_grp(data, varlist, groupby=None, spec_missing_value=None, q=No
     return means_rpt
 
 
+_SCREENING_MEANS_COLS = ["attribute", "N_ALL", "N", "MISSING_RATE", "MIN", "MEAN", "MAX"]
+
+
+def proc_means_for_screening(data, varlist, spec_missing_value=None, q=None):
+    """Build a uniform means table for IV screening reports.
+
+    ``proc_means_by_grp`` returns object stats (unique/top/freq) when numeric and
+    categorical columns are melted together.  This helper splits by dtype so MIN,
+    MEAN, and MAX are always present (NaN for non-numeric features).
+    """
+    if not varlist:
+        return pd.DataFrame(columns=_SCREENING_MEANS_COLS)
+
+    numeric = [v for v in varlist if v in data.columns and pd.api.types.is_numeric_dtype(data[v])]
+    other = [v for v in varlist if v not in numeric and v in data.columns]
+    frames = []
+
+    if numeric:
+        frames.append(proc_means_by_grp(data, numeric, spec_missing_value=spec_missing_value, q=q))
+
+    for var in other:
+        part = proc_means_by_grp(data, [var], spec_missing_value=spec_missing_value, q=q)
+        for col in ("MIN", "MEAN", "MAX"):
+            if col not in part.columns:
+                part[col] = np.nan
+        frames.append(part)
+
+    if not frames:
+        return pd.DataFrame(columns=_SCREENING_MEANS_COLS)
+
+    out = pd.concat(frames, ignore_index=True)
+    return out[_SCREENING_MEANS_COLS]
+
+
 class DistributionShiftAnalyzer:
     """分布偏移分析器。
     

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/Distribution_Tool.pyi

@@ -25,6 +25,7 @@ class proc_means:
     def group_sum(self): ...
     def __call__(self, q = None): ...
 def proc_means_by_grp(data, varlist, groupby = None, spec_missing_value = None, q = None): ...
+def proc_means_for_screening(data, varlist, spec_missing_value = None, q = None): ...
 
 class DistributionShiftAnalyzer:
     def __init__(self, data, grp_name, benchmark_value): ...

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/Feature_Insights.py

@@ -6,7 +6,7 @@
 import pandas as pd
 from tqdm import tqdm
 
-from .Distribution_Tool import proc_means_by_grp
+from .Distribution_Tool import proc_means_by_grp, proc_means_for_screening
 import logging
 logger = logging.getLogger(__name__)
 
@@ -208,14 +208,15 @@ class VarExtractionInsights:
         high_iv_summary = iv_info_res.query(f"IV >= {iv_cut}").round(4)
         high_iv_varlist = high_iv_summary['var'].tolist()
 
-        means = proc_means_by_grp(data, high_iv_varlist, spec_missing_value=self.missing_rate_ref)
-
         if len(high_iv_varlist) == 0:
             logger.info(f"WARNING: No variable with IV >= {iv_cut}")
-            means = means.rename(columns={"index": "attribute"})
+
+        means = proc_means_for_screening(
+            data, high_iv_varlist, spec_missing_value=self.missing_rate_ref,
+        )
 
         fnl_summary = high_iv_summary.merge(
-            means[['attribute', 'N_ALL', 'N', 'MISSING_RATE', 'MIN', 'MEAN', 'MAX']],
+            means,
             left_on=['var'],
             right_on=['attribute'],
             how='left'

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/Feature_Screen.py

@@ -13,6 +13,7 @@ from Modeling_Tool.Core.sample_weight_utils import resolve_sample_weight
 
 from .Weighted_Screen import (
     WeightedScreenResult,
+    _apply_missing_rate_stage,
     _apply_stage_keep,
     _corr_dedup_weighted,
     _legacy_unweighted_screen,
@@ -58,6 +59,8 @@ class FeatureScreenConfig:
     corr_use_woe_bins: bool = False
     corr_nan_policy: Literal["pairwise", "median_fill", "raise"] = "pairwise"
     on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn"
+    missing_rate_threshold: float | None = None
+    missing_rate_ref: float | int = -999999
     woe_engine: str = "equal_freq"
     woe_fit_query: str | None = None
     woe_params: dict[str, Any] = field(
@@ -105,6 +108,8 @@ def screen_config_from_mapping(
         corr_use_woe_bins=bool(cfg.get("corr_use_woe_bins", False)),
         corr_nan_policy=str(cfg.get("corr_nan_policy", "pairwise")),  # type: ignore[arg-type]
         on_empty_stage=str(cfg.get("on_empty_stage", "keep_all_warn")),  # type: ignore[arg-type]
+        missing_rate_threshold=cfg.get("missing_rate_threshold"),
+        missing_rate_ref=cfg.get("missing_rate_ref", -999999),
         woe_engine=str(woe_engine or cfg.get("woe_engine", "equal_freq")),
         woe_fit_query=woe_fit_query if woe_fit_query is not None else cfg.get("woe_fit_query"),
         woe_params=dict(FeatureScreenConfig().woe_params | dict(woe_params or cfg.get("woe_params") or {})),
@@ -211,6 +216,14 @@ def _woe_bins_unweighted_screen(
     ins, oos, oot = splits["ins"], splits["oos"], splits["oot"]
     current = list(feature_cols)
     summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, None)]
+    current, missing_rate_table, missing_rate_dropped = _apply_missing_rate_stage(
+        ins,
+        current,
+        summary_rows,
+        missing_rate_threshold=config.missing_rate_threshold,
+        missing_rate_ref=config.missing_rate_ref,
+        on_empty_stage=config.on_empty_stage,
+    )
     binner = _resolve_screening_binner(splits, feature_cols, target_col, config, prefit_woe_engine)
 
     psi_table = pd.DataFrame(columns=["var", "psi_ins_oos", "psi_ins_oot", "psi_max"])
@@ -303,6 +316,8 @@ def _woe_bins_unweighted_screen(
         psi_table=psi_table,
         corr_dropped=corr_dropped,
         summary=pd.DataFrame(summary_rows),
+        missing_rate_table=missing_rate_table,
+        missing_rate_dropped=missing_rate_dropped,
     )
 
 
@@ -328,6 +343,15 @@ def _weighted_woe_bins_screen(
     y_ins = ins[target_col].astype(float).to_numpy()
     current = list(feature_cols)
     summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, weight_col)]
+    current, missing_rate_table, missing_rate_dropped = _apply_missing_rate_stage(
+        ins,
+        current,
+        summary_rows,
+        missing_rate_threshold=config.missing_rate_threshold,
+        missing_rate_ref=config.missing_rate_ref,
+        weight_col=weight_col,
+        on_empty_stage=config.on_empty_stage,
+    )
 
     binner = prefit_woe_engine
     if binner is None:
@@ -438,6 +462,8 @@ def _weighted_woe_bins_screen(
         psi_table=psi_table,
         corr_dropped=corr_dropped,
         summary=pd.DataFrame(summary_rows),
+        missing_rate_table=missing_rate_table,
+        missing_rate_dropped=missing_rate_dropped,
     )
 
 
@@ -485,6 +511,8 @@ def feature_screen(
             corr_nan_policy=cfg.corr_nan_policy,
             on_empty_stage=cfg.on_empty_stage,
             prefit_woe_engine=prefit_woe_engine,
+            missing_rate_threshold=cfg.missing_rate_threshold,
+            missing_rate_ref=cfg.missing_rate_ref,
         )
 
     if use_woe_bins or prefit_woe_engine is not None:
@@ -515,6 +543,8 @@ def feature_screen(
         plot_outputs=cfg.plot_outputs,
         iv_equal_freq=cfg.iv_equal_freq,
         on_empty_stage=cfg.on_empty_stage,
+        missing_rate_threshold=cfg.missing_rate_threshold,
+        missing_rate_ref=cfg.missing_rate_ref,
     )
 
 

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py

@@ -19,7 +19,7 @@ from .Feature_Insights import (
     VarExtractionInsights as _BaseVarExtractionInsights,
     var_corr_filter,
 )
-from .Distribution_Tool import proc_means_by_grp
+from .Distribution_Tool import proc_means_for_screening
 
 
 def _psi_from_bins(expected_bins: pd.Series, current_bins: pd.Series, content: float) -> float:
@@ -110,9 +110,9 @@ def _screening_summary_from_engine(
     high_iv_summary = pd.DataFrame(rows).query(f"iv >= {iv_cut}").round(4)
     high_iv_varlist = high_iv_summary["var"].tolist()
     if high_iv_varlist:
-        means = proc_means_by_grp(data, high_iv_varlist, spec_missing_value=missing_rate_ref)
+        means = proc_means_for_screening(data, high_iv_varlist, spec_missing_value=missing_rate_ref)
         summary = high_iv_summary.merge(
-            means[["attribute", "N_ALL", "N", "MISSING_RATE", "MIN", "MEAN", "MAX"]],
+            means,
             left_on="var",
             right_on="attribute",
             how="left",

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/Weighted_Screen.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import warnings
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Literal
 
@@ -16,6 +16,9 @@ from Modeling_Tool.Core.utils import calc_iv
 _MISSING_BIN = "__MISSING__"
 
 
+_MISSING_RATE_COLS = ["var", "missing_rate"]
+
+
 @dataclass
 class WeightedScreenResult:
     selected_features: list[str]
@@ -23,6 +26,12 @@ class WeightedScreenResult:
     psi_table: pd.DataFrame
     corr_dropped: pd.DataFrame
     summary: pd.DataFrame
+    missing_rate_table: pd.DataFrame = field(
+        default_factory=lambda: pd.DataFrame(columns=_MISSING_RATE_COLS),
+    )
+    missing_rate_dropped: pd.DataFrame = field(
+        default_factory=lambda: pd.DataFrame(columns=_MISSING_RATE_COLS),
+    )
 
 
 def _summary_row(stage: str, n_in: int, n_out: int, threshold: Any, weight_col: str | None) -> dict:
@@ -327,6 +336,83 @@ def _weighted_corr_for_screen(
     return _weighted_pearson_corr_matrix(X, w_ins, nan_policy=corr_nan_policy)
 
 
+def _missing_rate_for_series(
+    series: pd.Series,
+    *,
+    w: np.ndarray | None = None,
+    missing_rate_ref: Any = None,
+) -> float:
+    values = series.copy()
+    if missing_rate_ref is not None:
+        values = values.replace(missing_rate_ref, np.nan)
+    if w is None:
+        total = len(values)
+        if total == 0:
+            return 1.0
+        valid = int(values.notna().sum())
+        return 1.0 - float(valid) / float(total)
+    arr = values.to_numpy(dtype=float, copy=False)
+    finite = np.isfinite(arr) & values.notna().to_numpy()
+    total_w = float(np.sum(w))
+    if total_w <= 0:
+        return 1.0
+    return 1.0 - float(np.sum(w[finite])) / total_w
+
+
+def _apply_missing_rate_stage(
+    ins: pd.DataFrame,
+    current: list[str],
+    summary_rows: list[dict],
+    *,
+    missing_rate_threshold: float | None,
+    missing_rate_ref: Any = None,
+    weight_col: str | None = None,
+    on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
+) -> tuple[list[str], pd.DataFrame, pd.DataFrame]:
+    empty = pd.DataFrame(columns=_MISSING_RATE_COLS)
+    if missing_rate_threshold is None:
+        return current, empty.copy(), empty.copy()
+
+    w = (
+        resolve_sample_weight(data=ins, weight_col=weight_col, expected_len=len(ins))
+        if weight_col is not None
+        else None
+    )
+    records: list[dict] = []
+    for var in current:
+        if var not in ins.columns:
+            continue
+        records.append({
+            "var": var,
+            "missing_rate": _missing_rate_for_series(
+                ins[var], w=w, missing_rate_ref=missing_rate_ref,
+            ),
+        })
+    missing_rate_table = pd.DataFrame(records) if records else empty.copy()
+    keep = missing_rate_table.loc[
+        missing_rate_table["missing_rate"] <= missing_rate_threshold, "var"
+    ].tolist()
+    missing_rate_dropped = missing_rate_table.loc[
+        missing_rate_table["missing_rate"] > missing_rate_threshold, _MISSING_RATE_COLS
+    ].copy()
+
+    n_before = len(current)
+    current = _apply_stage_keep(
+        current,
+        keep,
+        "missing_rate",
+        summary_rows,
+        on_empty_stage=on_empty_stage,
+        weight_col=weight_col,
+        threshold=missing_rate_threshold,
+        intersect=True,
+    )
+    summary_rows.append(
+        _summary_row("missing_rate", n_before, len(current), missing_rate_threshold, weight_col),
+    )
+    return current, missing_rate_table, missing_rate_dropped
+
+
 def _apply_stage_keep(
     current: list[str],
     keep: list[str],
@@ -439,12 +525,22 @@ def _legacy_unweighted_screen(
     plot_outputs: bool,
     iv_equal_freq: bool = True,
     on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
+    missing_rate_threshold: float | None = None,
+    missing_rate_ref: Any = None,
 ) -> WeightedScreenResult:
     from Modeling_Tool import CorrelationFilter, PSICalculator, VarExtractionInsights
 
     ins, oos, oot = splits["ins"], splits["oos"], splits["oot"]
     current = list(feature_cols)
     summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, None)]
+    current, missing_rate_table, missing_rate_dropped = _apply_missing_rate_stage(
+        ins,
+        current,
+        summary_rows,
+        missing_rate_threshold=missing_rate_threshold,
+        missing_rate_ref=missing_rate_ref,
+        on_empty_stage=on_empty_stage,
+    )
 
     psi_table = pd.DataFrame(columns=["var", "psi_ins_oos", "psi_ins_oot", "psi_max"])
     if psi_enabled:
@@ -528,6 +624,8 @@ def _legacy_unweighted_screen(
         psi_table=psi_table,
         corr_dropped=corr_dropped,
         summary=pd.DataFrame(summary_rows),
+        missing_rate_table=missing_rate_table,
+        missing_rate_dropped=missing_rate_dropped,
     )
 
 
@@ -553,6 +651,8 @@ def _weighted_screen_impl(
     corr_nan_policy: Literal["pairwise", "median_fill", "raise"] = "pairwise",
     on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
     prefit_woe_engine: Any | None = None,
+    missing_rate_threshold: float | None = None,
+    missing_rate_ref: Any = None,
 ) -> WeightedScreenResult:
     ins = splits["ins"]
     oos = splits["oos"]
@@ -561,6 +661,15 @@ def _weighted_screen_impl(
 
     current = list(feature_cols)
     summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, weight_col)]
+    current, missing_rate_table, missing_rate_dropped = _apply_missing_rate_stage(
+        ins,
+        current,
+        summary_rows,
+        missing_rate_threshold=missing_rate_threshold,
+        missing_rate_ref=missing_rate_ref,
+        weight_col=weight_col,
+        on_empty_stage=on_empty_stage,
+    )
 
     # Precompute bin edges on ins for PSI / IV
     edges_cache: dict[str, list[float] | None] = {}
@@ -668,6 +777,8 @@ def _weighted_screen_impl(
         psi_table=psi_table,
         corr_dropped=corr_dropped,
         summary=pd.DataFrame(summary_rows),
+        missing_rate_table=missing_rate_table,
+        missing_rate_dropped=missing_rate_dropped,
     )
 
 

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/Modeling_Tool/Pipeline/feature_validation.py

@@ -97,6 +97,7 @@ class FeatureValidationPipelineConfig:
         }
     )
 
+    missing_rate_threshold: float | None = None
     selection_enabled: bool = False
     selection_params: dict[str, Any] = field(default_factory=dict)
     weight_col: str | None = None
@@ -1174,6 +1175,7 @@ class FeatureValidationPipeline:
         params = dict(cfg.selection_params or {})
         corr_params = dict(cfg.corr_params or {})
         psi_params = dict(cfg.psi_params or {})
+        missing_rate_threshold = params.get("missing_rate_threshold", cfg.missing_rate_threshold)
         mapping = {
             "psi_enabled": params.get("psi_enabled", cfg.psi_enabled),
             "psi_threshold": params.get("psi_threshold", 0.2),
@@ -1196,6 +1198,11 @@ class FeatureValidationPipeline:
                 corr_params.get("max_iterations", 10),
             ),
             "corr_use_woe_bins": params.get("corr_use_woe_bins", cfg.corr_use_woe_bins),
+            "missing_rate_threshold": missing_rate_threshold,
+            "missing_rate_ref": params.get(
+                "missing_rate_ref",
+                cfg.woe_params.get("missing_ref_value", -999999),
+            ),
         }
         return screen_config_from_mapping(
             mapping,
@@ -1238,6 +1245,7 @@ class FeatureValidationPipeline:
         config_snapshot = {
             "selection_enabled": True,
             "selection_params": dict(cfg.selection_params or {}),
+            "missing_rate_threshold": screen_cfg.missing_rate_threshold,
             "weight_col": cfg.weight_col,
             "psi_use_woe_bins": screen_cfg.psi_use_woe_bins,
             "iv_use_woe_bins": screen_cfg.iv_use_woe_bins,
@@ -1673,9 +1681,15 @@ class FeatureValidationPipeline:
         }
         if selected_features:
             tables["selected_features"] = pd.DataFrame({"feature": selected_features})
-        screen_summary = (selection_summary or {}).get("screen_summary")
-        if isinstance(screen_summary, pd.DataFrame) and not screen_summary.empty:
-            tables["selection_screen_summary"] = screen_summary
+        if selection_summary:
+            for key in ("missing_rate", "missing_rate_dropped", "screen_summary"):
+                value = selection_summary.get(key)
+                if isinstance(value, pd.DataFrame) and not value.empty:
+                    tables[f"selection_{key}"] = value
+            if "final_features" in selection_summary:
+                tables["selection_final_features"] = pd.DataFrame(
+                    {"feature": list(selection_summary["final_features"])},
+                )
         for name, df in distribution_summary.items():
             tables[f"distribution_{name}"] = df
         return tables

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/Modeling_Tool/Pipeline/screening_artifact.py

@@ -31,6 +31,10 @@ def screen_result_to_summary(
         summary["iv"] = iv
     if not result.corr_dropped.empty:
         summary["corr_dropped"] = result.corr_dropped
+    if not result.missing_rate_table.empty:
+        summary["missing_rate"] = result.missing_rate_table
+    if not result.missing_rate_dropped.empty:
+        summary["missing_rate_dropped"] = result.missing_rate_dropped
     summary["corr_features"] = list(result.selected_features)
     summary["screen_summary"] = result.summary
     summary["final_features"] = list(result.selected_features)

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/Modeling_Tool/__init__.py

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 __author__ = "Jingkai Sun"
-__version__ = "0.3.14"
+__version__ = "0.3.15"
 
 # Core module - base data structures
 from .Core import (

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SuperModelingFactory
-Version: 0.3.14
+Version: 0.3.15
 Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
 Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
 Author: Kyle Sun

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/SuperModelingFactory.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SuperModelingFactory
-Version: 0.3.14
+Version: 0.3.15
 Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
 Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
 Author: Kyle Sun

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/pyproject.toml

@@ -12,7 +12,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "SuperModelingFactory"
-version = "0.3.14"
+version = "0.3.15"
 description = "Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting."
 readme = "README.md"
 requires-python = ">=3.10"

{supermodelingfactory-0.3.14 → supermodelingfactory-0.3.15}/setup.py

@@ -15,7 +15,7 @@ def _read(path: str) -> str:
 
 setup(
     name="SuperModelingFactory",
-    version=os.environ.get("SMF_VERSION", "0.3.14"),
+    version=os.environ.get("SMF_VERSION", "0.3.15"),
     description="Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.",
     long_description=_read("README.md"),
     long_description_content_type="text/markdown",