SuperModelingFactory 0.3.7__tar.gz → 0.3.8__tar.gz

supermodelingfactory-0.3.8/Modeling_Tool/Feature/Weighted_Screen.py

@@ -0,0 +1,548 @@
+"""Weighted feature screening: PSI -> IV -> correlation dedup."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from Modeling_Tool.Core.sample_weight_utils import resolve_sample_weight, weighted_rate
+from Modeling_Tool.Core.utils import calc_iv
+
+_MISSING_BIN = "__MISSING__"
+
+
+@dataclass
+class WeightedScreenResult:
+    selected_features: list[str]
+    iv_table: pd.DataFrame
+    psi_table: pd.DataFrame
+    corr_dropped: pd.DataFrame
+    summary: pd.DataFrame
+
+
+def _summary_row(stage: str, n_in: int, n_out: int, threshold: Any, weight_col: str | None) -> dict:
+    return {
+        "stage": stage,
+        "n_in": n_in,
+        "n_out": n_out,
+        "threshold": threshold,
+        "weight_col": weight_col,
+    }
+
+
+def _resolve_splits(data: pd.DataFrame, split_col: str) -> dict[str, pd.DataFrame]:
+    if split_col not in data.columns:
+        raise KeyError(f"Missing split_col {split_col!r}")
+    ins = data[data[split_col] == "ins"].copy()
+    oos = data[data[split_col] == "oos"].copy()
+    oot = data[data[split_col] == "oot"].copy()
+    if len(ins) == 0:
+        raise ValueError("No rows with split_col == 'ins'.")
+    return {"ins": ins, "oos": oos, "oot": oot}
+
+
+def _max_nbins(n_obs: int, nbins: int, min_bin_prop: float) -> int:
+    if min_bin_prop <= 0 or n_obs <= 0:
+        return max(1, nbins)
+    return max(1, min(nbins, int(1 / min_bin_prop)))
+
+
+def _weighted_equal_freq_edges(
+    x: np.ndarray,
+    w: np.ndarray,
+    n_bins: int,
+    min_bin_prop: float,
+    precision: int = 5,
+) -> list[float] | None:
+    mask = np.isfinite(x)
+    xv = x[mask].astype(float)
+    wv = w[mask].astype(float)
+    if xv.size == 0 or wv.sum() <= 0:
+        return None
+
+    n_bins = _max_nbins(xv.size, n_bins, min_bin_prop)
+    order = np.argsort(xv)
+    xv = xv[order]
+    wv = wv[order]
+    cumw = np.cumsum(wv)
+    total_w = float(cumw[-1])
+    targets = [total_w * i / n_bins for i in range(1, n_bins)]
+
+    edges: list[float] = []
+    for target in targets:
+        idx = int(np.searchsorted(cumw, target, side="right"))
+        idx = min(max(idx, 1), xv.size) - 1
+        edges.append(round(float(xv[idx]), precision))
+
+    uniq = sorted(set(edges))
+    if not uniq:
+        return [-np.inf, np.inf]
+    return [-np.inf, *uniq, np.inf]
+
+
+def _assign_bins(x: np.ndarray, edges: list[float]) -> np.ndarray:
+    out = np.empty(len(x), dtype=object)
+    missing = ~np.isfinite(x)
+    out[missing] = _MISSING_BIN
+    if edges is None:
+        out[~missing] = "all"
+        return out
+    valid = x[~missing].astype(float)
+    # pd.cut for consistent interval labels
+    cats = pd.cut(valid, bins=edges, include_lowest=True, duplicates="drop")
+    out[~missing] = np.asarray(cats.astype(str))
+    return out
+
+
+def _weighted_bin_distribution(
+    bins: np.ndarray,
+    w: np.ndarray,
+    content: float,
+) -> pd.Series:
+    df = pd.DataFrame({"bin": bins, "w": w})
+    dist = df.groupby("bin", dropna=False)["w"].sum()
+    total = float(dist.sum()) or 1.0
+    dist = dist / total
+    return dist.reindex(dist.index.union([_MISSING_BIN]), fill_value=0.0).clip(lower=content)
+
+
+def _psi_from_distributions(expected: pd.Series, actual: pd.Series, content: float) -> float:
+    all_bins = expected.index.union(actual.index)
+    e = expected.reindex(all_bins, fill_value=content).astype(float)
+    a = actual.reindex(all_bins, fill_value=content).astype(float)
+    e = e.clip(lower=content)
+    a = a.clip(lower=content)
+    return float(((a - e) * np.log(a / e)).sum())
+
+
+def _weighted_iv_for_var(
+    x: np.ndarray,
+    y: np.ndarray,
+    w: np.ndarray,
+    n_bins: int,
+    min_bin_prop: float,
+    precision: int,
+) -> tuple[float, int, float]:
+    edges = _weighted_equal_freq_edges(x, w, n_bins, min_bin_prop, precision=precision)
+    bins = _assign_bins(x, edges)
+    total_bad = float(np.sum(w * y))
+    total_good = float(np.sum(w * (1.0 - y)))
+    if total_bad <= 0 or total_good <= 0:
+        missing_rate = 1.0 - float(np.sum(w[np.isfinite(x)])) / float(np.sum(w) or 1.0)
+        return 0.0, 0, missing_rate
+
+    rows = []
+    for b in pd.unique(bins):
+        m = bins == b
+        bad_w = float(np.sum(w[m] * y[m]))
+        good_w = float(np.sum(w[m] * (1.0 - y[m])))
+        rows.append({
+            "bad_pct": bad_w / total_bad,
+            "good_pct": good_w / total_good,
+        })
+    if not rows:
+        missing_rate = 1.0 - float(np.sum(w[np.isfinite(x)])) / float(np.sum(w) or 1.0)
+        return 0.0, 0, missing_rate
+
+    stats = pd.DataFrame(rows)
+    iv = float(calc_iv(stats, "bad_pct", "good_pct").sum())
+    missing_rate = 1.0 - float(np.sum(w[np.isfinite(x)])) / float(np.sum(w) or 1.0)
+    return iv, len(rows), missing_rate
+
+
+def _weighted_pearson_corr_matrix(X: np.ndarray, w: np.ndarray) -> np.ndarray:
+    if X.shape[1] == 0:
+        return np.empty((0, 0))
+    w = np.asarray(w, dtype=float)
+    w_sum = w.sum()
+    if w_sum <= 0:
+        return np.corrcoef(X, rowvar=False)
+    w_norm = w / w_sum
+    mean = np.average(X, axis=0, weights=w_norm)
+    Xc = X - mean
+    cov = np.cov(Xc, rowvar=False, aweights=w, ddof=0)
+    std = np.sqrt(np.diag(cov))
+    std[std == 0] = np.nan
+    corr = cov / np.outer(std, std)
+    return np.nan_to_num(corr, nan=0.0)
+
+
+def _high_corr_pairs(varlist: list[str], corr: np.ndarray, threshold: float) -> pd.DataFrame:
+    rows = []
+    for i in range(len(varlist)):
+        for j in range(i + 1, len(varlist)):
+            c = corr[i, j]
+            if abs(c) > threshold:
+                rows.append({"var_a": varlist[i], "var_b": varlist[j], "corr": float(c)})
+    return pd.DataFrame(rows)
+
+
+def _corr_dedup_weighted(
+    varlist: list[str],
+    corr: np.ndarray,
+    iv_map: dict[str, float],
+    threshold: float,
+    max_iterations: int,
+) -> tuple[list[str], pd.DataFrame]:
+    dropped_rows: list[dict] = []
+    current = list(varlist)
+    idx = {v: i for i, v in enumerate(varlist)}
+
+    for _ in range(max_iterations):
+        if len(current) <= 1:
+            break
+        sub_idx = [idx[v] for v in current]
+        sub_corr = corr[np.ix_(sub_idx, sub_idx)]
+        pairs = _high_corr_pairs(current, sub_corr, threshold)
+        if pairs.empty:
+            break
+
+        remove: set[str] = set()
+        for _, row in pairs.iterrows():
+            v1, v2 = row["var_a"], row["var_b"]
+            if v1 in remove or v2 in remove:
+                continue
+            iv1 = iv_map.get(v1, 0.0)
+            iv2 = iv_map.get(v2, 0.0)
+            if iv1 >= iv2:
+                kept, dropped = v1, v2
+            else:
+                kept, dropped = v2, v1
+            remove.add(dropped)
+            dropped_rows.append({
+                "var_a": v1,
+                "var_b": v2,
+                "corr": row["corr"],
+                "iv_a": iv1,
+                "iv_b": iv2,
+                "kept": kept,
+                "dropped": dropped,
+            })
+
+        if not remove:
+            break
+        current = [v for v in current if v not in remove]
+
+    return current, pd.DataFrame(dropped_rows)
+
+
+def _legacy_unweighted_screen(
+    splits: dict[str, pd.DataFrame],
+    feature_cols: list[str],
+    target_col: str,
+    *,
+    psi_enabled: bool,
+    psi_threshold: float,
+    psi_compare_splits: list[str],
+    iv_enabled: bool,
+    iv_threshold: float,
+    iv_bins: int,
+    iv_min_bin_prop: float,
+    corr_enabled: bool,
+    corr_threshold: float,
+    corr_max_iterations: int,
+    psi_buckets: int,
+    plot_path: str | None,
+    plot_outputs: bool,
+    iv_equal_freq: bool = True,
+) -> WeightedScreenResult:
+    from Modeling_Tool import CorrelationFilter, PSICalculator, VarExtractionInsights
+
+    ins, oos, oot = splits["ins"], splits["oos"], splits["oot"]
+    current = list(feature_cols)
+    summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, None)]
+
+    psi_table = pd.DataFrame(columns=["var", "psi_ins_oos", "psi_ins_oot", "psi_max"])
+    if psi_enabled:
+        psi_frames = []
+        if "oos" in psi_compare_splits and len(oos) > 0:
+            psi_oos = PSICalculator(buckets=psi_buckets).calculate(
+                expected_df=ins, current_data=oos, varlist=current,
+            )
+            psi_oos = psi_oos.rename(columns={"psi": "psi_ins_oos"})[["var", "psi_ins_oos"]]
+            psi_frames.append(psi_oos)
+        if "oot" in psi_compare_splits and len(oot) > 0:
+            psi_oot = PSICalculator(buckets=psi_buckets).calculate(
+                expected_df=ins, current_data=oot, varlist=current,
+            )
+            psi_oot = psi_oot.rename(columns={"psi": "psi_ins_oot"})[["var", "psi_ins_oot"]]
+            psi_frames.append(psi_oot)
+
+        if psi_frames:
+            psi_table = psi_frames[0]
+            for frame in psi_frames[1:]:
+                psi_table = psi_table.merge(frame, on="var", how="outer")
+            for col in ("psi_ins_oos", "psi_ins_oot"):
+                if col not in psi_table.columns:
+                    psi_table[col] = np.nan
+            compare_cols = [c for c in ("psi_ins_oos", "psi_ins_oot") if c in psi_table.columns]
+            psi_table["psi_max"] = psi_table[compare_cols].max(axis=1, skipna=True)
+            keep = psi_table.loc[psi_table["psi_max"] < psi_threshold, "var"].tolist()
+            n_before = len(current)
+            current = keep or current
+            summary_rows.append(_summary_row("psi", n_before, len(current), psi_threshold, None))
+
+    iv_table = pd.DataFrame(columns=["var", "iv_weighted", "n_bins", "missing_rate"])
+    if iv_enabled:
+        vi = VarExtractionInsights(
+            data=ins,
+            dep=target_col,
+            plot_path=plot_path or "",
+            nbins=iv_bins,
+            equal_freq=iv_equal_freq,
+            min_bin_prop=iv_min_bin_prop,
+        )
+        iv = vi.get_var_analysis_report(data=ins, varlist=current, dep=target_col, iv_cut=0.0)
+        if plot_path and plot_outputs and current:
+            Path(plot_path, "overall").mkdir(parents=True, exist_ok=True)
+            plot_data = ins.copy()
+            plot_data["_smf_plot_group"] = "overall"
+            vi.plot_woe(
+                data=plot_data,
+                varlist=current,
+                plot_group="_smf_plot_group",
+                plot_dirname="overall",
+                plot_path=plot_path,
+            )
+        iv_table = iv.rename(columns={"iv": "iv_weighted"})[["var", "iv_weighted", "n_bins", "missing_rate"]]
+        keep = iv.loc[iv["iv"] >= iv_threshold, "var"].tolist()
+        n_before = len(current)
+        current = keep or current
+        summary_rows.append(_summary_row("iv", n_before, len(current), iv_threshold, None))
+
+    corr_dropped = pd.DataFrame(columns=["var_a", "var_b", "corr", "iv_a", "iv_b", "kept", "dropped"])
+    if corr_enabled and len(current) > 1:
+        cf = CorrelationFilter(
+            data=ins[current + [target_col]],
+            dep=target_col,
+            corr_cutpoint=corr_threshold,
+        )
+        n_before = len(current)
+        current = cf.remove_highly_correlated(current, max_iterations=corr_max_iterations)
+        summary_rows.append(_summary_row("corr", n_before, len(current), corr_threshold, None))
+
+    summary_rows.append(_summary_row("final", len(feature_cols), len(current), None, None))
+    return WeightedScreenResult(
+        selected_features=list(current),
+        iv_table=iv_table,
+        psi_table=psi_table,
+        corr_dropped=corr_dropped,
+        summary=pd.DataFrame(summary_rows),
+    )
+
+
+def _weighted_screen_impl(
+    splits: dict[str, pd.DataFrame],
+    feature_cols: list[str],
+    target_col: str,
+    weight_col: str,
+    *,
+    psi_enabled: bool,
+    psi_threshold: float,
+    psi_compare_splits: list[str],
+    iv_enabled: bool,
+    iv_threshold: float,
+    iv_bins: int,
+    min_bin_prop: float,
+    corr_enabled: bool,
+    corr_threshold: float,
+    corr_max_iterations: int,
+    content: float,
+    precision: int,
+) -> WeightedScreenResult:
+    ins = splits["ins"]
+    oos = splits["oos"]
+    oot = splits["oot"]
+    w_ins = resolve_sample_weight(data=ins, weight_col=weight_col, expected_len=len(ins))
+
+    current = list(feature_cols)
+    summary_rows = [_summary_row("initial", len(feature_cols), len(current), None, weight_col)]
+
+    # Precompute bin edges on ins for PSI / IV
+    edges_cache: dict[str, list[float] | None] = {}
+    y_ins = ins[target_col].astype(float).to_numpy()
+
+    psi_records: list[dict] = []
+    if psi_enabled:
+        for var in feature_cols:
+            if var not in ins.columns or ins[var].nunique(dropna=False) <= 1:
+                continue
+            x_ins = ins[var].to_numpy(dtype=float)
+            if var not in edges_cache:
+                edges_cache[var] = _weighted_equal_freq_edges(
+                    x_ins, w_ins, iv_bins, min_bin_prop, precision=precision,
+                )
+            edges = edges_cache[var]
+            bins_ins = _assign_bins(x_ins, edges) if edges else _assign_bins(x_ins, None)
+            exp_dist = _weighted_bin_distribution(bins_ins, w_ins, content)
+
+            row: dict[str, Any] = {"var": var, "psi_ins_oos": np.nan, "psi_ins_oot": np.nan}
+            if "oos" in psi_compare_splits and len(oos) > 0:
+                w_oos = resolve_sample_weight(data=oos, weight_col=weight_col, expected_len=len(oos))
+                bins_oos = _assign_bins(oos[var].to_numpy(dtype=float), edges)
+                act = _weighted_bin_distribution(bins_oos, w_oos, content)
+                row["psi_ins_oos"] = _psi_from_distributions(exp_dist, act, content)
+            if "oot" in psi_compare_splits and len(oot) > 0:
+                w_oot = resolve_sample_weight(data=oot, weight_col=weight_col, expected_len=len(oot))
+                bins_oot = _assign_bins(oot[var].to_numpy(dtype=float), edges)
+                act = _weighted_bin_distribution(bins_oot, w_oot, content)
+                row["psi_ins_oot"] = _psi_from_distributions(exp_dist, act, content)
+            psi_records.append(row)
+
+        psi_table = pd.DataFrame(psi_records) if psi_records else pd.DataFrame(
+            columns=["var", "psi_ins_oos", "psi_ins_oot"],
+        )
+        if not psi_table.empty:
+            compare_cols = [c for c in ("psi_ins_oos", "psi_ins_oot") if c in psi_table.columns]
+            psi_table["psi_max"] = psi_table[compare_cols].max(axis=1, skipna=True)
+            keep = psi_table.loc[psi_table["psi_max"] < psi_threshold, "var"].tolist()
+            n_before = len(current)
+            current = [v for v in current if v in keep] or current
+            summary_rows.append(_summary_row("psi", n_before, len(current), psi_threshold, weight_col))
+        else:
+            psi_table["psi_max"] = pd.Series(dtype=float)
+    else:
+        psi_table = pd.DataFrame(columns=["var", "psi_ins_oos", "psi_ins_oot", "psi_max"])
+
+    iv_records: list[dict] = []
+    if iv_enabled:
+        for var in current:
+            if var not in ins.columns or ins[var].nunique(dropna=False) <= 1:
+                continue
+            iv_val, n_b, miss = _weighted_iv_for_var(
+                ins[var].to_numpy(dtype=float),
+                y_ins,
+                w_ins,
+                iv_bins,
+                min_bin_prop,
+                precision,
+            )
+            iv_records.append({
+                "var": var,
+                "iv_weighted": iv_val,
+                "n_bins": n_b,
+                "missing_rate": miss,
+            })
+        iv_table = pd.DataFrame(iv_records) if iv_records else pd.DataFrame(
+            columns=["var", "iv_weighted", "n_bins", "missing_rate"],
+        )
+        if not iv_table.empty:
+            keep = iv_table.loc[iv_table["iv_weighted"] >= iv_threshold, "var"].tolist()
+            n_before = len(current)
+            current = [v for v in current if v in keep] or current
+            summary_rows.append(_summary_row("iv", n_before, len(current), iv_threshold, weight_col))
+    else:
+        iv_table = pd.DataFrame(columns=["var", "iv_weighted", "n_bins", "missing_rate"])
+
+    corr_dropped = pd.DataFrame(columns=["var_a", "var_b", "corr", "iv_a", "iv_b", "kept", "dropped"])
+    if corr_enabled and len(current) > 1:
+        X = ins[current].astype(float).to_numpy()
+        corr = _weighted_pearson_corr_matrix(X, w_ins)
+        iv_map = dict(zip(iv_table["var"], iv_table["iv_weighted"])) if not iv_table.empty else {}
+        n_before = len(current)
+        current, corr_dropped = _corr_dedup_weighted(
+            current, corr, iv_map, corr_threshold, corr_max_iterations,
+        )
+        summary_rows.append(_summary_row("corr", n_before, len(current), corr_threshold, weight_col))
+
+    summary_rows.append(_summary_row("final", len(feature_cols), len(current), None, weight_col))
+    return WeightedScreenResult(
+        selected_features=list(current),
+        iv_table=iv_table,
+        psi_table=psi_table,
+        corr_dropped=corr_dropped,
+        summary=pd.DataFrame(summary_rows),
+    )
+
+
+def weighted_feature_screen(
+    data: pd.DataFrame,
+    feature_cols: list[str],
+    target_col: str,
+    split_col: str,
+    weight_col: str | None = None,
+    psi_enabled: bool = True,
+    psi_threshold: float = 0.2,
+    psi_compare_splits: list[str] | None = None,
+    iv_enabled: bool = True,
+    iv_threshold: float = 0.02,
+    corr_enabled: bool = True,
+    corr_threshold: float = 0.75,
+    iv_bins: int = 10,
+    min_bin_prop: float = 0.05,
+    iv_min_bin_prop: float | None = None,
+    corr_max_iterations: int = 10,
+    content: float = 1e-6,
+    precision: int = 5,
+    psi_buckets: int | None = None,
+    plot_path: str | None = None,
+    plot_outputs: bool = False,
+    iv_equal_freq: bool = True,
+) -> WeightedScreenResult:
+    """Run PSI -> IV -> correlation screening with optional sample weights.
+
+    Bin edges are fit on ``ins`` only. When ``weight_col`` is None the legacy
+    unweighted tools are used for backward-compatible results with
+    ``CreditModelPipeline`` (IV uses tree binning via ``VarExtractionInsights``).
+    Weighted screening uses equal-frequency bins on weighted quantiles.
+    """
+    if iv_min_bin_prop is None:
+        iv_min_bin_prop = min_bin_prop
+    if psi_buckets is None:
+        psi_buckets = iv_bins
+    if psi_compare_splits is None:
+        psi_compare_splits = ["oos", "oot"]
+
+    exclude = {target_col, split_col}
+    if weight_col:
+        exclude.add(weight_col)
+    features = [c for c in feature_cols if c not in exclude and c in data.columns]
+
+    splits = _resolve_splits(data, split_col)
+
+    if weight_col is None:
+        return _legacy_unweighted_screen(
+            splits,
+            features,
+            target_col,
+            psi_enabled=psi_enabled,
+            psi_threshold=psi_threshold,
+            psi_compare_splits=list(psi_compare_splits),
+            iv_enabled=iv_enabled,
+            iv_threshold=iv_threshold,
+            iv_bins=iv_bins,
+            iv_min_bin_prop=iv_min_bin_prop,
+            corr_enabled=corr_enabled,
+            corr_threshold=corr_threshold,
+            corr_max_iterations=corr_max_iterations,
+            psi_buckets=psi_buckets,
+            plot_path=plot_path,
+            plot_outputs=plot_outputs,
+            iv_equal_freq=iv_equal_freq,
+        )
+
+    if weight_col not in data.columns:
+        raise KeyError(f"Missing weight_col {weight_col!r}")
+
+    return _weighted_screen_impl(
+        splits,
+        features,
+        target_col,
+        weight_col,
+        psi_enabled=psi_enabled,
+        psi_threshold=psi_threshold,
+        psi_compare_splits=list(psi_compare_splits),
+        iv_enabled=iv_enabled,
+        iv_threshold=iv_threshold,
+        iv_bins=iv_bins,
+        min_bin_prop=min_bin_prop,
+        corr_enabled=corr_enabled,
+        corr_threshold=corr_threshold,
+        corr_max_iterations=corr_max_iterations,
+        content=content,
+        precision=precision,
+    )

{supermodelingfactory-0.3.7 → supermodelingfactory-0.3.8}/Modeling_Tool/Feature/__init__.py

@@ -25,6 +25,11 @@ from .WOE_Engine_Feature_Patch import (
     CorrelationFilter,
 )
 
+from .Weighted_Screen import (
+    WeightedScreenResult,
+    weighted_feature_screen,
+)
+
 __all__ = [
     # Distribution_Tool
     'proc_means', 'proc_means_by_grp', 'get_distribution_shift',
@@ -37,4 +42,7 @@ __all__ = [
     # PSI_Tool
     'PSICalculator', 'calculate_psi', 'calculate_within_psi',
     'calculate_psi_within_dataset', 'calculate_multivar_psi_two_sets',
+
+    # Weighted_Screen
+    'WeightedScreenResult', 'weighted_feature_screen',
 ]

{supermodelingfactory-0.3.7 → supermodelingfactory-0.3.8}/Modeling_Tool/Pipeline/credit_model.py

@@ -45,10 +45,12 @@ class CreditModelPipelineConfig:
         default_factory=lambda: {
             "psi_enabled": True,
             "psi_threshold": 0.2,
+            "psi_compare_splits": ["oos"],
             "iv_enabled": True,
             "iv_threshold": 0.02,
             "corr_enabled": True,
             "corr_threshold": 0.75,
+            "corr_max_iterations": 10,
         }
     )
     woe_engine: str = "equal_freq"
@@ -427,79 +429,78 @@ class CreditModelPipeline:
         splits: dict[str, pd.DataFrame],
         feature_cols: list[str],
     ) -> tuple[dict[str, Any], list[str]]:
-        from Modeling_Tool import CorrelationFilter, PSICalculator, VarExtractionInsights
+        from Modeling_Tool.Feature.Weighted_Screen import weighted_feature_screen
 
         cfg = self.config
         fs_cfg = cfg.feature_selection
-        ins, oos = splits["ins"], splits["oos"]
-        current = list(feature_cols)
-        summary: dict[str, Any] = {"initial_features": list(feature_cols)}
-
-        if fs_cfg.get("psi_enabled", True):
-            try:
-                psi = PSICalculator(buckets=int(fs_cfg.get("psi_buckets", 10))).calculate(
-                    expected_df=ins,
-                    current_data=oos,
-                    varlist=current,
-                )
-                keep = psi.loc[psi["psi"] < float(fs_cfg.get("psi_threshold", 0.2)), "var"].tolist()
-                current = keep or current
-                summary["psi"] = psi
-            except Exception as exc:
-                summary["psi_error"] = repr(exc)
-
+        split_col = "_smf_split"
+        screen_frames = []
+        for name, df in splits.items():
+            part = df.copy()
+            part[split_col] = name
+            screen_frames.append(part)
+        screen_data = pd.concat(screen_frames, ignore_index=True)
+
+        plot_path = None
         if fs_cfg.get("iv_enabled", True):
-            try:
-                plot_path = str(Path(cfg.output_dir) / "figs" / "var_analysis")
-                make_dirs(plot_path)
-                vi = VarExtractionInsights(
-                    data=ins,
-                    dep=cfg.target_col,
-                    plot_path=plot_path,
-                    nbins=int(fs_cfg.get("iv_nbins", 10)),
-                    equal_freq=bool(fs_cfg.get("iv_equal_freq", True)),
-                    min_bin_prop=float(fs_cfg.get("iv_min_bin_prop", 0.05)),
-                )
-                iv = vi.get_var_analysis_report(
-                    data=ins,
-                    varlist=current,
-                    dep=cfg.target_col,
-                    iv_cut=0.0,
-                )
-                if cfg.write_outputs and cfg.plot_outputs and current:
-                    make_dirs(Path(plot_path) / "overall")
-                    plot_data = ins.copy()
-                    plot_data["_smf_plot_group"] = "overall"
-                    vi.plot_woe(
-                        data=plot_data,
-                        varlist=current,
-                        plot_group="_smf_plot_group",
-                        plot_dirname="overall",
-                        plot_path=plot_path,
-                    )
-                keep = iv.loc[iv["iv"] >= float(fs_cfg.get("iv_threshold", 0.02)), "var"].tolist()
-                current = keep or current
-                summary["iv"] = iv
-            except Exception as exc:
-                summary["iv_error"] = repr(exc)
+            plot_path = str(Path(cfg.output_dir) / "figs" / "var_analysis")
+            if cfg.write_outputs and cfg.plot_outputs:
+                make_dirs(plot_path, Path(plot_path) / "overall")
 
-        if fs_cfg.get("corr_enabled", True) and len(current) > 1:
-            try:
-                corr = CorrelationFilter(
-                    data=ins[current + [cfg.target_col]],
-                    dep=cfg.target_col,
-                    corr_cutpoint=float(fs_cfg.get("corr_threshold", 0.75)),
-                )
-                current = corr.remove_highly_correlated(
-                    current,
-                    max_iterations=int(fs_cfg.get("corr_max_iterations", 10)),
-                )
-                summary["corr_features"] = list(current)
-            except Exception as exc:
-                summary["corr_error"] = repr(exc)
+        summary: dict[str, Any] = {"initial_features": list(feature_cols)}
+        try:
+            result = weighted_feature_screen(
+                data=screen_data,
+                feature_cols=feature_cols,
+                target_col=cfg.target_col,
+                split_col=split_col,
+                weight_col=cfg.weight_col,
+                psi_enabled=bool(fs_cfg.get("psi_enabled", True)),
+                psi_threshold=float(fs_cfg.get("psi_threshold", 0.2)),
+                psi_compare_splits=list(fs_cfg.get("psi_compare_splits", ["oos"])),
+                iv_enabled=bool(fs_cfg.get("iv_enabled", True)),
+                iv_threshold=float(fs_cfg.get("iv_threshold", 0.02)),
+                corr_enabled=bool(fs_cfg.get("corr_enabled", True)),
+                corr_threshold=float(fs_cfg.get("corr_threshold", 0.75)),
+                iv_bins=int(fs_cfg.get("iv_nbins", 10)),
+                iv_min_bin_prop=float(fs_cfg.get("iv_min_bin_prop", 0.05)),
+                corr_max_iterations=int(fs_cfg.get("corr_max_iterations", 10)),
+                psi_buckets=int(fs_cfg.get("psi_buckets", fs_cfg.get("iv_nbins", 10))),
+                plot_path=plot_path,
+                plot_outputs=bool(cfg.write_outputs and cfg.plot_outputs),
+                iv_equal_freq=bool(fs_cfg.get("iv_equal_freq", True)),
+            )
+            summary = self._screen_result_to_summary(result, feature_cols)
+            return summary, list(result.selected_features)
+        except Exception as exc:
+            summary["error"] = repr(exc)
+            summary["final_features"] = list(feature_cols)
+            return summary, list(feature_cols)
 
-        summary["final_features"] = list(current)
-        return summary, list(current)
+    def _screen_result_to_summary(
+        self,
+        result: Any,
+        initial_features: list[str],
+    ) -> dict[str, Any]:
+        summary: dict[str, Any] = {"initial_features": list(initial_features)}
+        if not result.psi_table.empty:
+            psi = result.psi_table.copy()
+            if "psi_ins_oos" in psi.columns:
+                psi["psi"] = psi["psi_ins_oos"]
+            elif "psi_max" in psi.columns:
+                psi["psi"] = psi["psi_max"]
+            summary["psi"] = psi
+        if not result.iv_table.empty:
+            iv = result.iv_table.copy()
+            if "iv_weighted" in iv.columns:
+                iv = iv.rename(columns={"iv_weighted": "iv"})
+            summary["iv"] = iv
+        if not result.corr_dropped.empty:
+            summary["corr_dropped"] = result.corr_dropped
+        summary["corr_features"] = list(result.selected_features)
+        summary["screen_summary"] = result.summary
+        summary["final_features"] = list(result.selected_features)
+        return summary
 
     def _transform_extra_eval_datasets(
         self,

{supermodelingfactory-0.3.7 → supermodelingfactory-0.3.8}/Modeling_Tool/__init__.py

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 __author__ = "Jingkai Sun"
-__version__ = "0.3.7"
+__version__ = "0.3.8"
 
 # Core module - base data structures
 from .Core import (
@@ -87,6 +87,8 @@ from .Feature import (
     CorrelationFilter,
     PSICalculator,
     calculate_psi_within_dataset,
+    WeightedScreenResult,
+    weighted_feature_screen,
 )
 
 _MODEL_EXPORTS = frozenset({
@@ -228,4 +230,6 @@ __all__ = [
     'CorrelationFilter',
     'PSICalculator',
     'calculate_psi_within_dataset',
+    'WeightedScreenResult',
+    'weighted_feature_screen',
 ]

{supermodelingfactory-0.3.7 → supermodelingfactory-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SuperModelingFactory
-Version: 0.3.7
+Version: 0.3.8
 Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
 Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
 Author: Kyle Sun

{supermodelingfactory-0.3.7 → supermodelingfactory-0.3.8}/SuperModelingFactory.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: SuperModelingFactory
-Version: 0.3.7
+Version: 0.3.8
 Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
 Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
 Author: Kyle Sun

{supermodelingfactory-0.3.7 → supermodelingfactory-0.3.8}/SuperModelingFactory.egg-info/SOURCES.txt

@@ -46,6 +46,7 @@ Modeling_Tool/Feature/Feature_Insights.pyi
 Modeling_Tool/Feature/PSI_Tool.py
 Modeling_Tool/Feature/PSI_Tool.pyi
 Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py
+Modeling_Tool/Feature/Weighted_Screen.py
 Modeling_Tool/Feature/__init__.py
 Modeling_Tool/Model/Backward_Tool.py
 Modeling_Tool/Model/Backward_Tool.pyi

{supermodelingfactory-0.3.7 → supermodelingfactory-0.3.8}/pyproject.toml

@@ -12,7 +12,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "SuperModelingFactory"
-version = "0.3.7"
+version = "0.3.8"
 description = "Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting."
 readme = "README.md"
 requires-python = ">=3.10"

{supermodelingfactory-0.3.7 → supermodelingfactory-0.3.8}/setup.py

@@ -15,7 +15,7 @@ def _read(path: str) -> str:
 
 setup(
     name="SuperModelingFactory",
-    version=os.environ.get("SMF_VERSION", "0.3.7"),
+    version=os.environ.get("SMF_VERSION", "0.3.8"),
     description="Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.",
     long_description=_read("README.md"),
     long_description_content_type="text/markdown",