SuperModelingFactory 0.3.13__tar.gz → 0.3.15__tar.gz

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/Distribution_Tool.py

@@ -186,6 +186,40 @@ def proc_means_by_grp(data, varlist, groupby=None, spec_missing_value=None, q=No
     return means_rpt
 
 
+_SCREENING_MEANS_COLS = ["attribute", "N_ALL", "N", "MISSING_RATE", "MIN", "MEAN", "MAX"]
+
+
+def proc_means_for_screening(data, varlist, spec_missing_value=None, q=None):
+    """Build a uniform means table for IV screening reports.
+
+    ``proc_means_by_grp`` returns object stats (unique/top/freq) when numeric and
+    categorical columns are melted together.  This helper splits by dtype so MIN,
+    MEAN, and MAX are always present (NaN for non-numeric features).
+    """
+    if not varlist:
+        return pd.DataFrame(columns=_SCREENING_MEANS_COLS)
+
+    numeric = [v for v in varlist if v in data.columns and pd.api.types.is_numeric_dtype(data[v])]
+    other = [v for v in varlist if v not in numeric and v in data.columns]
+    frames = []
+
+    if numeric:
+        frames.append(proc_means_by_grp(data, numeric, spec_missing_value=spec_missing_value, q=q))
+
+    for var in other:
+        part = proc_means_by_grp(data, [var], spec_missing_value=spec_missing_value, q=q)
+        for col in ("MIN", "MEAN", "MAX"):
+            if col not in part.columns:
+                part[col] = np.nan
+        frames.append(part)
+
+    if not frames:
+        return pd.DataFrame(columns=_SCREENING_MEANS_COLS)
+
+    out = pd.concat(frames, ignore_index=True)
+    return out[_SCREENING_MEANS_COLS]
+
+
 class DistributionShiftAnalyzer:
     """分布偏移分析器。
     

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/Distribution_Tool.pyi

@@ -25,6 +25,7 @@ class proc_means:
     def group_sum(self): ...
     def __call__(self, q = None): ...
 def proc_means_by_grp(data, varlist, groupby = None, spec_missing_value = None, q = None): ...
+def proc_means_for_screening(data, varlist, spec_missing_value = None, q = None): ...
 
 class DistributionShiftAnalyzer:
     def __init__(self, data, grp_name, benchmark_value): ...

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/Feature_Insights.py

@@ -6,7 +6,7 @@
 import pandas as pd
 from tqdm import tqdm
 
-from .Distribution_Tool import proc_means_by_grp
+from .Distribution_Tool import proc_means_by_grp, proc_means_for_screening
 import logging
 logger = logging.getLogger(__name__)
 
@@ -208,14 +208,15 @@ class VarExtractionInsights:
         high_iv_summary = iv_info_res.query(f"IV >= {iv_cut}").round(4)
         high_iv_varlist = high_iv_summary['var'].tolist()
 
-        means = proc_means_by_grp(data, high_iv_varlist, spec_missing_value=self.missing_rate_ref)
-
         if len(high_iv_varlist) == 0:
             logger.info(f"WARNING: No variable with IV >= {iv_cut}")
-            means = means.rename(columns={"index": "attribute"})
+
+        means = proc_means_for_screening(
+            data, high_iv_varlist, spec_missing_value=self.missing_rate_ref,
+        )
 
         fnl_summary = high_iv_summary.merge(
-            means[['attribute', 'N_ALL', 'N', 'MISSING_RATE', 'MIN', 'MEAN', 'MAX']],
+            means,
             left_on=['var'],
             right_on=['attribute'],
             how='left'

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/Feature_Screen.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, Mapping
+from typing import Any, Literal, Mapping
 
 import numpy as np
 import pandas as pd
@@ -13,14 +13,16 @@ from Modeling_Tool.Core.sample_weight_utils import resolve_sample_weight
 
 from .Weighted_Screen import (
     WeightedScreenResult,
+    _apply_missing_rate_stage,
+    _apply_stage_keep,
     _corr_dedup_weighted,
     _legacy_unweighted_screen,
     _psi_from_distributions,
     _resolve_splits,
     _summary_row,
     _weighted_bin_distribution,
+    _weighted_corr_for_screen,
     _weighted_iv_from_assigned_bins,
-    _weighted_pearson_corr_matrix,
     _weighted_screen_impl,
 )
 
@@ -55,6 +57,10 @@ class FeatureScreenConfig:
     corr_threshold: float = 0.75
     corr_max_iterations: int = 10
     corr_use_woe_bins: bool = False
+    corr_nan_policy: Literal["pairwise", "median_fill", "raise"] = "pairwise"
+    on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn"
+    missing_rate_threshold: float | None = None
+    missing_rate_ref: float | int = -999999
     woe_engine: str = "equal_freq"
     woe_fit_query: str | None = None
     woe_params: dict[str, Any] = field(
@@ -100,6 +106,10 @@ def screen_config_from_mapping(
         corr_threshold=float(cfg.get("corr_threshold", 0.75)),
         corr_max_iterations=int(cfg.get("corr_max_iterations", 10)),
         corr_use_woe_bins=bool(cfg.get("corr_use_woe_bins", False)),
+        corr_nan_policy=str(cfg.get("corr_nan_policy", "pairwise")),  # type: ignore[arg-type]
+        on_empty_stage=str(cfg.get("on_empty_stage", "keep_all_warn")),  # type: ignore[arg-type]
+        missing_rate_threshold=cfg.get("missing_rate_threshold"),
+        missing_rate_ref=cfg.get("missing_rate_ref", -999999),
         woe_engine=str(woe_engine or cfg.get("woe_engine", "equal_freq")),
         woe_fit_query=woe_fit_query if woe_fit_query is not None else cfg.get("woe_fit_query"),
         woe_params=dict(FeatureScreenConfig().woe_params | dict(woe_params or cfg.get("woe_params") or {})),
@@ -206,6 +216,14 @@ def _woe_bins_unweighted_screen(
     ins, oos, oot = splits["ins"], splits["oos"], splits["oot"]
     current = list(feature_cols)
     summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, None)]
+    current, missing_rate_table, missing_rate_dropped = _apply_missing_rate_stage(
+        ins,
+        current,
+        summary_rows,
+        missing_rate_threshold=config.missing_rate_threshold,
+        missing_rate_ref=config.missing_rate_ref,
+        on_empty_stage=config.on_empty_stage,
+    )
     binner = _resolve_screening_binner(splits, feature_cols, target_col, config, prefit_woe_engine)
 
     psi_table = pd.DataFrame(columns=["var", "psi_ins_oos", "psi_ins_oot", "psi_max"])
@@ -236,7 +254,10 @@ def _woe_bins_unweighted_screen(
             psi_table["psi_max"] = psi_table[compare_cols].max(axis=1, skipna=True)
             keep = psi_table.loc[psi_table["psi_max"] < config.psi_threshold, "var"].tolist()
             n_before = len(current)
-            current = keep or current
+            current = _apply_stage_keep(
+                current, keep, "psi", summary_rows,
+                on_empty_stage=config.on_empty_stage, threshold=config.psi_threshold,
+            )
             summary_rows.append(_summary_row("psi", n_before, len(current), config.psi_threshold, None))
 
     iv_table = pd.DataFrame(columns=["var", "iv_weighted", "n_bins", "missing_rate"])
@@ -268,7 +289,10 @@ def _woe_bins_unweighted_screen(
             iv_table = iv.rename(columns={"iv": "iv_weighted"})[["var", "iv_weighted", "n_bins", "missing_rate"]]
             keep = iv.loc[iv["iv"] >= config.iv_threshold, "var"].tolist()
             n_before = len(current)
-            current = keep or current
+            current = _apply_stage_keep(
+                current, keep, "iv", summary_rows,
+                on_empty_stage=config.on_empty_stage, threshold=config.iv_threshold,
+            )
             summary_rows.append(_summary_row("iv", n_before, len(current), config.iv_threshold, None))
 
     corr_dropped = pd.DataFrame(columns=["var_a", "var_b", "corr", "iv_a", "iv_b", "kept", "dropped"])
@@ -292,6 +316,8 @@ def _woe_bins_unweighted_screen(
         psi_table=psi_table,
         corr_dropped=corr_dropped,
         summary=pd.DataFrame(summary_rows),
+        missing_rate_table=missing_rate_table,
+        missing_rate_dropped=missing_rate_dropped,
     )
 
 
@@ -317,6 +343,15 @@ def _weighted_woe_bins_screen(
     y_ins = ins[target_col].astype(float).to_numpy()
     current = list(feature_cols)
     summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, weight_col)]
+    current, missing_rate_table, missing_rate_dropped = _apply_missing_rate_stage(
+        ins,
+        current,
+        summary_rows,
+        missing_rate_threshold=config.missing_rate_threshold,
+        missing_rate_ref=config.missing_rate_ref,
+        weight_col=weight_col,
+        on_empty_stage=config.on_empty_stage,
+    )
 
     binner = prefit_woe_engine
     if binner is None:
@@ -360,7 +395,11 @@ def _weighted_woe_bins_screen(
             psi_table["psi_max"] = psi_table[compare_cols].max(axis=1, skipna=True)
             keep = psi_table.loc[psi_table["psi_max"] < config.psi_threshold, "var"].tolist()
             n_before = len(current)
-            current = [v for v in current if v in keep] or current
+            current = _apply_stage_keep(
+                current, keep, "psi", summary_rows,
+                on_empty_stage=config.on_empty_stage, weight_col=weight_col,
+                threshold=config.psi_threshold, intersect=True,
+            )
             summary_rows.append(_summary_row("psi", n_before, len(current), config.psi_threshold, weight_col))
         else:
             psi_table["psi_max"] = pd.Series(dtype=float)
@@ -387,15 +426,24 @@ def _weighted_woe_bins_screen(
         if not iv_table.empty:
             keep = iv_table.loc[iv_table["iv_weighted"] >= config.iv_threshold, "var"].tolist()
             n_before = len(current)
-            current = [v for v in current if v in keep] or current
+            current = _apply_stage_keep(
+                current, keep, "iv", summary_rows,
+                on_empty_stage=config.on_empty_stage, weight_col=weight_col,
+                threshold=config.iv_threshold, intersect=True,
+            )
             summary_rows.append(_summary_row("iv", n_before, len(current), config.iv_threshold, weight_col))
     else:
         iv_table = pd.DataFrame(columns=["var", "iv_weighted", "n_bins", "missing_rate"])
 
     corr_dropped = pd.DataFrame(columns=["var_a", "var_b", "corr", "iv_a", "iv_b", "kept", "dropped"])
     if config.corr_enabled and len(current) > 1:
-        X = ins[current].astype(float).to_numpy()
-        corr = _weighted_pearson_corr_matrix(X, w_ins)
+        corr = _weighted_corr_for_screen(
+            ins, current, w_ins,
+            corr_use_woe_bins=config.corr_use_woe_bins,
+            corr_nan_policy=config.corr_nan_policy,
+            adapter=adapter,
+            binner=binner,
+        )
         iv_map = dict(zip(iv_table["var"], iv_table["iv_weighted"])) if not iv_table.empty else {}
         n_before = len(current)
         current, corr_dropped = _corr_dedup_weighted(
@@ -414,6 +462,8 @@ def _weighted_woe_bins_screen(
         psi_table=psi_table,
         corr_dropped=corr_dropped,
         summary=pd.DataFrame(summary_rows),
+        missing_rate_table=missing_rate_table,
+        missing_rate_dropped=missing_rate_dropped,
     )
 
 
@@ -457,6 +507,12 @@ def feature_screen(
             corr_max_iterations=cfg.corr_max_iterations,
             content=cfg.content,
             precision=cfg.precision,
+            corr_use_woe_bins=cfg.corr_use_woe_bins,
+            corr_nan_policy=cfg.corr_nan_policy,
+            on_empty_stage=cfg.on_empty_stage,
+            prefit_woe_engine=prefit_woe_engine,
+            missing_rate_threshold=cfg.missing_rate_threshold,
+            missing_rate_ref=cfg.missing_rate_ref,
         )
 
     if use_woe_bins or prefit_woe_engine is not None:
@@ -486,6 +542,9 @@ def feature_screen(
         plot_path=cfg.plot_path,
         plot_outputs=cfg.plot_outputs,
         iv_equal_freq=cfg.iv_equal_freq,
+        on_empty_stage=cfg.on_empty_stage,
+        missing_rate_threshold=cfg.missing_rate_threshold,
+        missing_rate_ref=cfg.missing_rate_ref,
     )
 
 

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py

@@ -19,7 +19,7 @@ from .Feature_Insights import (
     VarExtractionInsights as _BaseVarExtractionInsights,
     var_corr_filter,
 )
-from .Distribution_Tool import proc_means_by_grp
+from .Distribution_Tool import proc_means_for_screening
 
 
 def _psi_from_bins(expected_bins: pd.Series, current_bins: pd.Series, content: float) -> float:
@@ -110,9 +110,9 @@ def _screening_summary_from_engine(
     high_iv_summary = pd.DataFrame(rows).query(f"iv >= {iv_cut}").round(4)
     high_iv_varlist = high_iv_summary["var"].tolist()
     if high_iv_varlist:
-        means = proc_means_by_grp(data, high_iv_varlist, spec_missing_value=missing_rate_ref)
+        means = proc_means_for_screening(data, high_iv_varlist, spec_missing_value=missing_rate_ref)
         summary = high_iv_summary.merge(
-            means[["attribute", "N_ALL", "N", "MISSING_RATE", "MIN", "MEAN", "MAX"]],
+            means,
             left_on="var",
             right_on="attribute",
             how="left",

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/Modeling_Tool/Feature/Weighted_Screen.py

@@ -2,9 +2,10 @@
 
 from __future__ import annotations
 
-from dataclasses import dataclass
+import warnings
+from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any
+from typing import Any, Literal
 
 import numpy as np
 import pandas as pd
@@ -15,6 +16,9 @@ from Modeling_Tool.Core.utils import calc_iv
 _MISSING_BIN = "__MISSING__"
 
 
+_MISSING_RATE_COLS = ["var", "missing_rate"]
+
+
 @dataclass
 class WeightedScreenResult:
     selected_features: list[str]
@@ -22,6 +26,12 @@ class WeightedScreenResult:
     psi_table: pd.DataFrame
     corr_dropped: pd.DataFrame
     summary: pd.DataFrame
+    missing_rate_table: pd.DataFrame = field(
+        default_factory=lambda: pd.DataFrame(columns=_MISSING_RATE_COLS),
+    )
+    missing_rate_dropped: pd.DataFrame = field(
+        default_factory=lambda: pd.DataFrame(columns=_MISSING_RATE_COLS),
+    )
 
 
 def _summary_row(stage: str, n_in: int, n_out: int, threshold: Any, weight_col: str | None) -> dict:
@@ -185,21 +195,255 @@ def _weighted_iv_for_var(
     return iv, len(rows), missing_rate
 
 
-def _weighted_pearson_corr_matrix(X: np.ndarray, w: np.ndarray) -> np.ndarray:
+def _weighted_median(x: np.ndarray, w: np.ndarray) -> float:
+    mask = np.isfinite(x)
+    if not mask.any():
+        return np.nan
+    xv = x[mask].astype(float)
+    wv = w[mask].astype(float)
+    order = np.argsort(xv)
+    xv = xv[order]
+    wv = wv[order]
+    cumw = np.cumsum(wv)
+    total = float(cumw[-1])
+    if total <= 0:
+        return float(np.median(xv))
+    idx = min(int(np.searchsorted(cumw, total / 2.0, side="right")), len(xv) - 1)
+    return float(xv[idx])
+
+
+def _weighted_corr_pair(xi: np.ndarray, xj: np.ndarray, w: np.ndarray) -> float:
+    mask = np.isfinite(xi) & np.isfinite(xj)
+    if mask.sum() < 2:
+        return 0.0
+    wi = w[mask].astype(float)
+    xi = xi[mask].astype(float)
+    xj = xj[mask].astype(float)
+    w_sum = wi.sum()
+    if w_sum <= 0:
+        c = np.corrcoef(xi, xj)
+        return float(c[0, 1]) if np.isfinite(c[0, 1]) else 0.0
+    w_norm = wi / w_sum
+    mi = np.average(xi, weights=w_norm)
+    mj = np.average(xj, weights=w_norm)
+    di = xi - mi
+    dj = xj - mj
+    cov = np.average(di * dj, weights=w_norm)
+    si = np.sqrt(np.average(di * di, weights=w_norm))
+    sj = np.sqrt(np.average(dj * dj, weights=w_norm))
+    if si == 0 or sj == 0:
+        return 0.0
+    return float(cov / (si * sj))
+
+
+def _weighted_pearson_corr_matrix(
+    X: np.ndarray,
+    w: np.ndarray,
+    *,
+    nan_policy: Literal["pairwise", "median_fill", "raise"] = "pairwise",
+) -> np.ndarray:
     if X.shape[1] == 0:
         return np.empty((0, 0))
     w = np.asarray(w, dtype=float)
+    k = X.shape[1]
+
+    if nan_policy == "raise" and np.any(~np.isfinite(X)):
+        raise ValueError(
+            "weighted correlation input contains NaN; set corr_nan_policy to "
+            "'pairwise' or 'median_fill' to proceed"
+        )
+
+    nan_mask = ~np.isfinite(X)
+    nan_fraction = float(nan_mask.sum()) / float(X.size) if X.size else 0.0
+
+    if nan_policy == "pairwise":
+        corr = np.eye(k, dtype=float)
+        insufficient_pairs = 0
+        total_pairs = k * (k - 1) // 2
+        for i in range(k):
+            for j in range(i + 1, k):
+                c = _weighted_corr_pair(X[:, i], X[:, j], w)
+                corr[i, j] = c
+                corr[j, i] = c
+                overlap = int((np.isfinite(X[:, i]) & np.isfinite(X[:, j])).sum())
+                if overlap < 2:
+                    insufficient_pairs += 1
+        if total_pairs > 0 and insufficient_pairs / total_pairs > 0.01:
+            warnings.warn(
+                f"[feature_screen] weighted corr: >1% of feature pairs have insufficient "
+                f"overlapping observations ({insufficient_pairs}/{total_pairs})",
+                stacklevel=3,
+            )
+        return corr
+
+    X_use = X.astype(float).copy()
+    if nan_policy == "median_fill":
+        for j in range(k):
+            col = X_use[:, j]
+            bad = ~np.isfinite(col)
+            if bad.any():
+                fill = _weighted_median(col, w)
+                if np.isfinite(fill):
+                    col[bad] = fill
+                X_use[:, j] = col
+
     w_sum = w.sum()
     if w_sum <= 0:
-        return np.corrcoef(X, rowvar=False)
-    w_norm = w / w_sum
-    mean = np.average(X, axis=0, weights=w_norm)
-    Xc = X - mean
-    cov = np.cov(Xc, rowvar=False, aweights=w, ddof=0)
-    std = np.sqrt(np.diag(cov))
-    std[std == 0] = np.nan
-    corr = cov / np.outer(std, std)
-    return np.nan_to_num(corr, nan=0.0)
+        out = np.corrcoef(X_use, rowvar=False)
+        out = np.nan_to_num(out, nan=0.0)
+    else:
+        w_norm = w / w_sum
+        mean = np.average(X_use, axis=0, weights=w_norm)
+        Xc = X_use - mean
+        cov = np.cov(Xc, rowvar=False, aweights=w, ddof=0)
+        std = np.sqrt(np.diag(cov))
+        std[std == 0] = np.nan
+        out = cov / np.outer(std, std)
+        out = np.nan_to_num(out, nan=0.0)
+
+    if nan_fraction > 0.01:
+        warnings.warn(
+            f"[feature_screen] weighted corr: {nan_fraction:.1%} of matrix entries were NaN "
+            f"before computation (policy={nan_policy!r})",
+            stacklevel=3,
+        )
+    return out
+
+
+def _weighted_corr_for_screen(
+    ins: pd.DataFrame,
+    current: list[str],
+    w_ins: np.ndarray,
+    *,
+    corr_use_woe_bins: bool,
+    corr_nan_policy: Literal["pairwise", "median_fill", "raise"],
+    adapter: Any | None = None,
+    binner: Any | None = None,
+) -> np.ndarray:
+    """Build weighted correlation matrix for screening (WOE or raw-value path)."""
+    from Modeling_Tool.WOE.WOE_Adapter import as_woe_engine
+
+    if corr_use_woe_bins:
+        eng = adapter if adapter is not None else (as_woe_engine(binner) if binner is not None else None)
+        if eng is not None:
+            suffix = getattr(eng, "woe_suffix", "_woe")
+            woe_ins = eng.transform(ins, varlist=current)
+            cols = [f"{v}{suffix}" for v in current if f"{v}{suffix}" in woe_ins.columns]
+            if cols:
+                X = woe_ins[cols].to_numpy(dtype=float)
+                return _weighted_pearson_corr_matrix(X, w_ins, nan_policy="pairwise")
+    X = ins[current].astype(float).to_numpy()
+    return _weighted_pearson_corr_matrix(X, w_ins, nan_policy=corr_nan_policy)
+
+
+def _missing_rate_for_series(
+    series: pd.Series,
+    *,
+    w: np.ndarray | None = None,
+    missing_rate_ref: Any = None,
+) -> float:
+    values = series.copy()
+    if missing_rate_ref is not None:
+        values = values.replace(missing_rate_ref, np.nan)
+    if w is None:
+        total = len(values)
+        if total == 0:
+            return 1.0
+        valid = int(values.notna().sum())
+        return 1.0 - float(valid) / float(total)
+    arr = values.to_numpy(dtype=float, copy=False)
+    finite = np.isfinite(arr) & values.notna().to_numpy()
+    total_w = float(np.sum(w))
+    if total_w <= 0:
+        return 1.0
+    return 1.0 - float(np.sum(w[finite])) / total_w
+
+
+def _apply_missing_rate_stage(
+    ins: pd.DataFrame,
+    current: list[str],
+    summary_rows: list[dict],
+    *,
+    missing_rate_threshold: float | None,
+    missing_rate_ref: Any = None,
+    weight_col: str | None = None,
+    on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
+) -> tuple[list[str], pd.DataFrame, pd.DataFrame]:
+    empty = pd.DataFrame(columns=_MISSING_RATE_COLS)
+    if missing_rate_threshold is None:
+        return current, empty.copy(), empty.copy()
+
+    w = (
+        resolve_sample_weight(data=ins, weight_col=weight_col, expected_len=len(ins))
+        if weight_col is not None
+        else None
+    )
+    records: list[dict] = []
+    for var in current:
+        if var not in ins.columns:
+            continue
+        records.append({
+            "var": var,
+            "missing_rate": _missing_rate_for_series(
+                ins[var], w=w, missing_rate_ref=missing_rate_ref,
+            ),
+        })
+    missing_rate_table = pd.DataFrame(records) if records else empty.copy()
+    keep = missing_rate_table.loc[
+        missing_rate_table["missing_rate"] <= missing_rate_threshold, "var"
+    ].tolist()
+    missing_rate_dropped = missing_rate_table.loc[
+        missing_rate_table["missing_rate"] > missing_rate_threshold, _MISSING_RATE_COLS
+    ].copy()
+
+    n_before = len(current)
+    current = _apply_stage_keep(
+        current,
+        keep,
+        "missing_rate",
+        summary_rows,
+        on_empty_stage=on_empty_stage,
+        weight_col=weight_col,
+        threshold=missing_rate_threshold,
+        intersect=True,
+    )
+    summary_rows.append(
+        _summary_row("missing_rate", n_before, len(current), missing_rate_threshold, weight_col),
+    )
+    return current, missing_rate_table, missing_rate_dropped
+
+
+def _apply_stage_keep(
+    current: list[str],
+    keep: list[str],
+    stage: str,
+    summary_rows: list[dict],
+    *,
+    on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
+    weight_col: str | None = None,
+    threshold: Any = None,
+    intersect: bool = False,
+) -> list[str]:
+    if intersect:
+        keep_set = set(keep)
+        new_current = [v for v in current if v in keep_set]
+    else:
+        new_current = list(keep) if keep else []
+    if new_current:
+        return new_current
+    if on_empty_stage == "raise":
+        raise ValueError(
+            f"feature screen stage {stage!r} eliminated all {len(current)} variables; "
+            f"on_empty_stage='raise'"
+        )
+    warnings.warn(
+        f"[feature_screen] stage {stage!r} 全军覆没, 按 keep_all_warn 保留全部 {len(current)} 个",
+        stacklevel=3,
+    )
+    summary_rows.append(
+        _summary_row(f"{stage}_fallback", len(current), len(current), threshold, weight_col)
+    )
+    return current
 
 
 def _high_corr_pairs(varlist: list[str], corr: np.ndarray, threshold: float) -> pd.DataFrame:
@@ -280,12 +524,23 @@ def _legacy_unweighted_screen(
     plot_path: str | None,
     plot_outputs: bool,
     iv_equal_freq: bool = True,
+    on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
+    missing_rate_threshold: float | None = None,
+    missing_rate_ref: Any = None,
 ) -> WeightedScreenResult:
     from Modeling_Tool import CorrelationFilter, PSICalculator, VarExtractionInsights
 
     ins, oos, oot = splits["ins"], splits["oos"], splits["oot"]
     current = list(feature_cols)
     summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, None)]
+    current, missing_rate_table, missing_rate_dropped = _apply_missing_rate_stage(
+        ins,
+        current,
+        summary_rows,
+        missing_rate_threshold=missing_rate_threshold,
+        missing_rate_ref=missing_rate_ref,
+        on_empty_stage=on_empty_stage,
+    )
 
     psi_table = pd.DataFrame(columns=["var", "psi_ins_oos", "psi_ins_oot", "psi_max"])
     if psi_enabled:
@@ -314,7 +569,10 @@ def _legacy_unweighted_screen(
             psi_table["psi_max"] = psi_table[compare_cols].max(axis=1, skipna=True)
             keep = psi_table.loc[psi_table["psi_max"] < psi_threshold, "var"].tolist()
             n_before = len(current)
-            current = keep or current
+            current = _apply_stage_keep(
+                current, keep, "psi", summary_rows,
+                on_empty_stage=on_empty_stage, threshold=psi_threshold,
+            )
             summary_rows.append(_summary_row("psi", n_before, len(current), psi_threshold, None))
 
     iv_table = pd.DataFrame(columns=["var", "iv_weighted", "n_bins", "missing_rate"])
@@ -342,7 +600,10 @@ def _legacy_unweighted_screen(
         iv_table = iv.rename(columns={"iv": "iv_weighted"})[["var", "iv_weighted", "n_bins", "missing_rate"]]
         keep = iv.loc[iv["iv"] >= iv_threshold, "var"].tolist()
         n_before = len(current)
-        current = keep or current
+        current = _apply_stage_keep(
+            current, keep, "iv", summary_rows,
+            on_empty_stage=on_empty_stage, threshold=iv_threshold,
+        )
         summary_rows.append(_summary_row("iv", n_before, len(current), iv_threshold, None))
 
     corr_dropped = pd.DataFrame(columns=["var_a", "var_b", "corr", "iv_a", "iv_b", "kept", "dropped"])
@@ -363,6 +624,8 @@ def _legacy_unweighted_screen(
         psi_table=psi_table,
         corr_dropped=corr_dropped,
         summary=pd.DataFrame(summary_rows),
+        missing_rate_table=missing_rate_table,
+        missing_rate_dropped=missing_rate_dropped,
     )
 
 
@@ -384,6 +647,12 @@ def _weighted_screen_impl(
     corr_max_iterations: int,
     content: float,
     precision: int,
+    corr_use_woe_bins: bool = False,
+    corr_nan_policy: Literal["pairwise", "median_fill", "raise"] = "pairwise",
+    on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
+    prefit_woe_engine: Any | None = None,
+    missing_rate_threshold: float | None = None,
+    missing_rate_ref: Any = None,
 ) -> WeightedScreenResult:
     ins = splits["ins"]
     oos = splits["oos"]
@@ -392,6 +661,15 @@ def _weighted_screen_impl(
 
     current = list(feature_cols)
     summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, weight_col)]
+    current, missing_rate_table, missing_rate_dropped = _apply_missing_rate_stage(
+        ins,
+        current,
+        summary_rows,
+        missing_rate_threshold=missing_rate_threshold,
+        missing_rate_ref=missing_rate_ref,
+        weight_col=weight_col,
+        on_empty_stage=on_empty_stage,
+    )
 
     # Precompute bin edges on ins for PSI / IV
     edges_cache: dict[str, list[float] | None] = {}
@@ -432,7 +710,11 @@ def _weighted_screen_impl(
             psi_table["psi_max"] = psi_table[compare_cols].max(axis=1, skipna=True)
             keep = psi_table.loc[psi_table["psi_max"] < psi_threshold, "var"].tolist()
             n_before = len(current)
-            current = [v for v in current if v in keep] or current
+            current = _apply_stage_keep(
+                current, keep, "psi", summary_rows,
+                on_empty_stage=on_empty_stage, weight_col=weight_col,
+                threshold=psi_threshold, intersect=True,
+            )
             summary_rows.append(_summary_row("psi", n_before, len(current), psi_threshold, weight_col))
         else:
             psi_table["psi_max"] = pd.Series(dtype=float)
@@ -464,15 +746,23 @@ def _weighted_screen_impl(
         if not iv_table.empty:
             keep = iv_table.loc[iv_table["iv_weighted"] >= iv_threshold, "var"].tolist()
             n_before = len(current)
-            current = [v for v in current if v in keep] or current
+            current = _apply_stage_keep(
+                current, keep, "iv", summary_rows,
+                on_empty_stage=on_empty_stage, weight_col=weight_col,
+                threshold=iv_threshold, intersect=True,
+            )
             summary_rows.append(_summary_row("iv", n_before, len(current), iv_threshold, weight_col))
     else:
         iv_table = pd.DataFrame(columns=["var", "iv_weighted", "n_bins", "missing_rate"])
 
     corr_dropped = pd.DataFrame(columns=["var_a", "var_b", "corr", "iv_a", "iv_b", "kept", "dropped"])
     if corr_enabled and len(current) > 1:
-        X = ins[current].astype(float).to_numpy()
-        corr = _weighted_pearson_corr_matrix(X, w_ins)
+        corr = _weighted_corr_for_screen(
+            ins, current, w_ins,
+            corr_use_woe_bins=corr_use_woe_bins,
+            corr_nan_policy=corr_nan_policy,
+            binner=prefit_woe_engine,
+        )
         iv_map = dict(zip(iv_table["var"], iv_table["iv_weighted"])) if not iv_table.empty else {}
         n_before = len(current)
         current, corr_dropped = _corr_dedup_weighted(
@@ -487,6 +777,8 @@ def _weighted_screen_impl(
         psi_table=psi_table,
         corr_dropped=corr_dropped,
         summary=pd.DataFrame(summary_rows),
+        missing_rate_table=missing_rate_table,
+        missing_rate_dropped=missing_rate_dropped,
     )
 
 
@@ -521,6 +813,8 @@ def weighted_feature_screen(
     woe_params: dict[str, Any] | None = None,
     monotone_woe_params: dict[str, Any] | None = None,
     prefit_woe_engine: Any | None = None,
+    corr_nan_policy: Literal["pairwise", "median_fill", "raise"] = "pairwise",
+    on_empty_stage: Literal["keep_all_warn", "raise"] = "keep_all_warn",
 ) -> WeightedScreenResult:
     """Run PSI -> IV -> correlation screening with optional sample weights.
 
@@ -563,6 +857,8 @@ def weighted_feature_screen(
         content=content,
         precision=precision,
         min_bin_prop=min_bin_prop,
+        corr_nan_policy=corr_nan_policy,
+        on_empty_stage=on_empty_stage,
     )
     return feature_screen_from_dataframe(
         data,

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/Modeling_Tool/Pipeline/feature_validation.py

@@ -97,6 +97,7 @@ class FeatureValidationPipelineConfig:
         }
     )
 
+    missing_rate_threshold: float | None = None
     selection_enabled: bool = False
     selection_params: dict[str, Any] = field(default_factory=dict)
     weight_col: str | None = None
@@ -1174,6 +1175,7 @@ class FeatureValidationPipeline:
         params = dict(cfg.selection_params or {})
         corr_params = dict(cfg.corr_params or {})
         psi_params = dict(cfg.psi_params or {})
+        missing_rate_threshold = params.get("missing_rate_threshold", cfg.missing_rate_threshold)
         mapping = {
             "psi_enabled": params.get("psi_enabled", cfg.psi_enabled),
             "psi_threshold": params.get("psi_threshold", 0.2),
@@ -1196,6 +1198,11 @@ class FeatureValidationPipeline:
                 corr_params.get("max_iterations", 10),
             ),
             "corr_use_woe_bins": params.get("corr_use_woe_bins", cfg.corr_use_woe_bins),
+            "missing_rate_threshold": missing_rate_threshold,
+            "missing_rate_ref": params.get(
+                "missing_rate_ref",
+                cfg.woe_params.get("missing_ref_value", -999999),
+            ),
         }
         return screen_config_from_mapping(
             mapping,
@@ -1238,6 +1245,7 @@ class FeatureValidationPipeline:
         config_snapshot = {
             "selection_enabled": True,
             "selection_params": dict(cfg.selection_params or {}),
+            "missing_rate_threshold": screen_cfg.missing_rate_threshold,
             "weight_col": cfg.weight_col,
             "psi_use_woe_bins": screen_cfg.psi_use_woe_bins,
             "iv_use_woe_bins": screen_cfg.iv_use_woe_bins,
@@ -1673,9 +1681,15 @@ class FeatureValidationPipeline:
         }
         if selected_features:
             tables["selected_features"] = pd.DataFrame({"feature": selected_features})
-        screen_summary = (selection_summary or {}).get("screen_summary")
-        if isinstance(screen_summary, pd.DataFrame) and not screen_summary.empty:
-            tables["selection_screen_summary"] = screen_summary
+        if selection_summary:
+            for key in ("missing_rate", "missing_rate_dropped", "screen_summary"):
+                value = selection_summary.get(key)
+                if isinstance(value, pd.DataFrame) and not value.empty:
+                    tables[f"selection_{key}"] = value
+            if "final_features" in selection_summary:
+                tables["selection_final_features"] = pd.DataFrame(
+                    {"feature": list(selection_summary["final_features"])},
+                )
         for name, df in distribution_summary.items():
             tables[f"distribution_{name}"] = df
         return tables

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/Modeling_Tool/Pipeline/screening_artifact.py

@@ -31,6 +31,10 @@ def screen_result_to_summary(
         summary["iv"] = iv
     if not result.corr_dropped.empty:
         summary["corr_dropped"] = result.corr_dropped
+    if not result.missing_rate_table.empty:
+        summary["missing_rate"] = result.missing_rate_table
+    if not result.missing_rate_dropped.empty:
+        summary["missing_rate_dropped"] = result.missing_rate_dropped
     summary["corr_features"] = list(result.selected_features)
     summary["screen_summary"] = result.summary
     summary["final_features"] = list(result.selected_features)

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/Modeling_Tool/__init__.py

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 __author__ = "Jingkai Sun"
-__version__ = "0.3.13"
+__version__ = "0.3.15"
 
 # Core module - base data structures
 from .Core import (

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SuperModelingFactory
-Version: 0.3.13
+Version: 0.3.15
 Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
 Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
 Author: Kyle Sun

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/SuperModelingFactory.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SuperModelingFactory
-Version: 0.3.13
+Version: 0.3.15
 Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
 Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
 Author: Kyle Sun

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/pyproject.toml

@@ -12,7 +12,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "SuperModelingFactory"
-version = "0.3.13"
+version = "0.3.15"
 description = "Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting."
 readme = "README.md"
 requires-python = ">=3.10"

{supermodelingfactory-0.3.13 → supermodelingfactory-0.3.15}/setup.py

@@ -15,7 +15,7 @@ def _read(path: str) -> str:
 
 setup(
     name="SuperModelingFactory",
-    version=os.environ.get("SMF_VERSION", "0.3.13"),
+    version=os.environ.get("SMF_VERSION", "0.3.15"),
     description="Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.",
     long_description=_read("README.md"),
     long_description_content_type="text/markdown",