SuperModelingFactory 0.6.4__tar.gz → 0.6.6__tar.gz

{supermodelingfactory-0.6.4 → supermodelingfactory-0.6.6}/Modeling_Tool/Eval/Model_Eval_Tool.py

@@ -2169,7 +2169,7 @@ class PerformanceEvaluator:
                  dist_bins = 20, pct_bins = 10, precision = 5, min_bin_prop = 0.05,
                  include_missing = False, equal_freq = True, chi2_method = False,
                  init_equi_bins = 1000, chi2_p = 0.9, tree_binning = False, random_state = 42,
-                 weight_col = None):
+                 weight_col = None, spec_values = None, ascending = None):
         """
         初始化性能评估器。
         
@@ -2225,9 +2225,29 @@ class PerformanceEvaluator:
         self.tree_binning = tree_binning
         self.random_state = random_state
         self.weight_col = weight_col
+        # spec_values: sentinel scores (e.g. -1 for all-missing overrides)
+        # that get their own evaluation bin and are excluded from quantile
+        # edges and ranking metrics. ascending: None keeps every underlying
+        # function's own legacy default; an explicit bool is threaded through
+        # summary/gains/plot and the weighted path uniformly.
+        self.spec_values = list(spec_values) if spec_values else []
+        self.ascending = ascending
         self.datasets = {}
         self.dataset_weight_cols = {}
         self.evaluate_status = None
+
+    def _governed_eval_kwargs(self):
+        """Kwargs to thread spec_values/ascending into underlying eval calls.
+
+        Nothing is passed when unset, so every legacy default stays intact
+        (unweighted summary False, comparison paths True, weighted False).
+        """
+        kwargs = {}
+        if self.spec_values:
+            kwargs["spec_values"] = list(self.spec_values)
+        if self.ascending is not None:
+            kwargs["ascending"] = bool(self.ascending)
+        return kwargs
     
     def add_dataset(self, name, data, weight_col = None, overwrite = False):
         """
@@ -2351,6 +2371,8 @@ class PerformanceEvaluator:
                         scr_name,
                         weight_col=wc,
                         nbins=self.pct_bins,
+                        ascending=bool(self.ascending) if self.ascending is not None else False,
+                        spec_values=self.spec_values or None,
                     )
                 )
             fnl_df = pd.DataFrame(rows)
@@ -2368,15 +2390,23 @@ class PerformanceEvaluator:
                     if scr_name is None:
                         scr_name = "_mdl_scr"
                         work_data[scr_name] = self.model.predict_proba(work_data.loc[:, self.feature_cols])[:, 1]
-                    payload = {
-                        "y_true": work_data[self.tgt_name],
-                        "y_score": work_data[scr_name],
-                    }
                     resolved_weight = _weighted_eval.resolve_weights(
                         work_data,
                         weight_col=wc,
                         expected_len=len(work_data),
                     )
+                    y_true = work_data[self.tgt_name].to_numpy()
+                    y_score = work_data[scr_name].to_numpy()
+                    if self.spec_values:
+                        keep = ~pd.Series(y_score).isin(self.spec_values).to_numpy()
+                        y_true = y_true[keep]
+                        y_score = y_score[keep]
+                        if resolved_weight is not None:
+                            resolved_weight = np.asarray(resolved_weight)[keep]
+                    payload = {
+                        "y_true": y_true,
+                        "y_score": y_score,
+                    }
                     if resolved_weight is not None:
                         payload["sample_weight"] = resolved_weight
                     eval_datasets[name] = payload
@@ -2445,6 +2475,21 @@ class PerformanceEvaluator:
                 return data[self.scr_name]
             return self.model.predict_proba(data.loc[:, self.feature_cols])[:, 1]
 
+        def _rank_payload(data):
+            """(y_true, y_score) with special sentinel scores removed.
+
+            Sentinels like the all-missing override -1 carry no ordering
+            information: AUC/KS/distribution figures are computed on the
+            non-special rows; their bin-level accounting lives in the gains
+            table via spec_values.
+            """
+            y_true = data[self.tgt_name]
+            y_score = _get_score(data)
+            if not self.spec_values:
+                return y_true, y_score
+            keep = ~pd.Series(np.asarray(y_score)).isin(self.spec_values).to_numpy()
+            return np.asarray(y_true)[keep], np.asarray(y_score)[keep]
+
         def _get_benchmark_bin_edges():
             if benchmark_dataset is None:
                 return None
@@ -2492,9 +2537,10 @@ class PerformanceEvaluator:
             for name, data in dataset_dict.items():
                 if data is None:
                     continue
+                y_true, y_score = _rank_payload(data)
                 eval_datasets[name] = {
-                    "y_true": data[self.tgt_name],
-                    "y_score": _get_score(data)
+                    "y_true": y_true,
+                    "y_score": y_score
                 }
 
             if len(eval_datasets) == 0:
@@ -2553,7 +2599,8 @@ class PerformanceEvaluator:
                         chi2_p = self.chi2_p,
                         retSummary = True,
                         tree_binning = False if benchmark_bin_edges is not None else self.tree_binning,
-                        random_state = self.random_state
+                        random_state = self.random_state,
+                        **self._governed_eval_kwargs(),
                     )
                 gains_res['index'] = name
                 gains_summ_list.append(gains_res)

{supermodelingfactory-0.6.4 → supermodelingfactory-0.6.6}/Modeling_Tool/Eval/Model_Eval_Tool.pyi

@@ -32,6 +32,6 @@ class GainsTableCalculator:
     def calculate(self, grp_name = None, min_data_size = 100, grp_colname = None, sync_range = True, retSummary = False, withSummary = False, wholeGroup = False, add_func = None): ...
 
 class PerformanceEvaluator:
-    def __init__(self, tgt_name, scr_name = None, model = None, feature_cols = None, dist_bins = 20, pct_bins = 10, precision = 5, min_bin_prop = 0.05, include_missing = False, equal_freq = True, chi2_method = False, init_equi_bins = 1000, chi2_p = 0.9, tree_binning = False, random_state = 42, weight_col = None): ...
+    def __init__(self, tgt_name, scr_name = None, model = None, feature_cols = None, dist_bins = 20, pct_bins = 10, precision = 5, min_bin_prop = 0.05, include_missing = False, equal_freq = True, chi2_method = False, init_equi_bins = 1000, chi2_p = 0.9, tree_binning = False, random_state = 42, weight_col = None, spec_values = None, ascending = None): ...
     def add_dataset(self, name, data, weight_col = None, overwrite = False): ...
     def evaluate(self, oot_grp_name = None, min_data_size = 100, grp_colname = None, fig_save_path = None, rpt_save_path = None, to_show = False, display = True, gains_table = False, benchmark_dataset = None, weight_col = None): ...

{supermodelingfactory-0.6.4 → supermodelingfactory-0.6.6}/Modeling_Tool/Eval/weighted_eval_utils.py

@@ -95,10 +95,18 @@ def calc_pr(y_true, y_score, sample_weight=None):
     )
 
 
-def rank_bins(score, weight, nbins):
+def rank_bins(score, weight, nbins, ascending=False):
+    """Weighted equal-frequency rank bins.
+
+    ``ascending=False`` (default, legacy) sorts scores descending — bin 1 is
+    the highest-score bucket. ``ascending=True`` sorts scores ascending so
+    bin 1 is the lowest-score bucket, matching the unweighted
+    ``gains_ascending=True`` convention.
+    """
     score = np.asarray(score, dtype=float)
     weight = np.ones(len(score), dtype=float) if weight is None else np.asarray(weight, dtype=float)
-    order = np.lexsort((np.arange(len(score)), -score))
+    sort_key = score if ascending else -score
+    order = np.lexsort((np.arange(len(score)), sort_key))
     bins = np.empty(len(score), dtype=int)
     total = float(weight.sum()) or float(len(score)) or 1.0
     cum_weight = np.cumsum(weight[order])
@@ -108,18 +116,35 @@ def rank_bins(score, weight, nbins):
     return bins
 
 
-def get_gains_table(data, dep, score, nbins=10, weight_col=None, weighted_binning=None, **kwargs):
+def _split_special_scores(df, score, spec_values):
+    """Split a frame into (normal, special) rows by exact score membership.
+
+    Special score values (e.g. the all-missing override ``-1``) are business
+    sentinels, not model outputs: they must not participate in quantile
+    binning or ranking metrics.
+    """
+    if not spec_values:
+        return df, None
+    spec_mask = df[score].isin(list(spec_values))
+    if not bool(spec_mask.any()):
+        return df, None
+    return df[~spec_mask], df[spec_mask]
+
+
+def get_gains_table(data, dep, score, nbins=10, weight_col=None, weighted_binning=None,
+                    ascending=False, spec_values=None, **kwargs):
     cols = [dep, score]
     if weight_col is not None and weight_col in data.columns:
         cols.append(weight_col)
     df = data[cols].copy()
+    df, special_df = _split_special_scores(df, score, spec_values)
     weight = resolve_weights(df, weight_col=weight_col, expected_len=len(df))
     if weight is None:
         weight = np.ones(len(df), dtype=float)
 
     y = df[dep].astype(float).to_numpy()
     s = df[score].astype(float).to_numpy()
-    df["_bin_num"] = rank_bins(s, weight, nbins)
+    df["_bin_num"] = rank_bins(s, weight, nbins, ascending=ascending)
     df["_bin_range"] = df["_bin_num"]
     df["_w"] = weight
     df["_bad_w"] = weight * y
@@ -165,13 +190,56 @@ def get_gains_table(data, dep, score, nbins=10, weight_col=None, weighted_binnin
     out["KS_PER_BIN"] = (out["CUM_BAD_PCT"] - out["CUM_GOOD_PCT"]).abs()
     out["KS"] = out["KS_PER_BIN"]
     out["LIFT"] = out["AVG_BAD"] / overall_bad_rate if overall_bad_rate else np.nan
-    out["TRUE_BAD_SHIFT"] = out["AVG_BAD"].shift(1) / out["AVG_BAD"] - 1
+    out["TRUE_BAD_SHIFT"] = (
+        (out["AVG_BAD"].shift(1) / out["AVG_BAD"] - 1)
+        if not ascending
+        else (out["AVG_BAD"] / out["AVG_BAD"].shift(1) - 1)
+    )
     out["RANK_ORDER_BUMP"] = out["TRUE_BAD_SHIFT"].lt(0).astype(int)
     with np.errstate(divide="ignore", invalid="ignore"):
         out["WOE"] = np.log(out["BAD_PCT_IN_EACH_BIN"] / out["GOOD_PCT_IN_EACH_BIN"])
     out["WOE"] = out["WOE"].replace([np.inf, -np.inf], 0).fillna(0)
     out["IV"] = (out["BAD_PCT_IN_EACH_BIN"] - out["GOOD_PCT_IN_EACH_BIN"]) * out["WOE"]
     out["AUC"] = safe_auc(y, s, sample_weight=weight)
+
+    if special_df is not None and len(special_df):
+        # Special sentinel scores (e.g. -1 for all-missing rows) get their own
+        # descriptive rows: never part of quantile edges, cumulative columns,
+        # or ranking metrics (those stay NaN by construction).
+        spec_weight = resolve_weights(special_df, weight_col=weight_col, expected_len=len(special_df))
+        if spec_weight is None:
+            spec_weight = np.ones(len(special_df), dtype=float)
+        grand_total = total_weight + float(np.sum(spec_weight))
+        spec_rows = []
+        for value, part in special_df.groupby(score, sort=True):
+            part_w = resolve_weights(part, weight_col=weight_col, expected_len=len(part))
+            if part_w is None:
+                part_w = np.ones(len(part), dtype=float)
+            part_y = part[dep].astype(float).to_numpy()
+            n_w = float(np.sum(part_w))
+            n_bad = float(np.sum(part_w * part_y))
+            label = f"special:{value}"
+            spec_rows.append(
+                pd.DataFrame(
+                    {
+                        "MIN": [value],
+                        "MAX": [value],
+                        "N": [n_w],
+                        "N_RAW": [int(len(part))],
+                        "PERF_CNT": [n_w],
+                        "N_BAD": [n_bad],
+                        "N_GOOD": [n_w - n_bad],
+                        "AVG_SCORE": [float(value)],
+                        "UNIQUE_SCORE": [1],
+                        "PROP": [n_w / grand_total if grand_total else np.nan],
+                        "AVG_BAD": [n_bad / n_w if n_w else np.nan],
+                        "AVG_GOOD": [(n_w - n_bad) / n_w if n_w else np.nan],
+                    },
+                    index=pd.MultiIndex.from_tuples([(label, label)], names=out.index.names),
+                )
+            )
+        out = pd.concat([out] + spec_rows)
+        out["AUC"] = out["AUC"].iloc[0]
     return out
 
 
@@ -212,32 +280,53 @@ def calc_fixed_pct(y_true, y_score, sample_weight=None, **kwargs):
     return calc_equid_dist(y_true, y_score, sample_weight=sample_weight, **kwargs)
 
 
-def dataset_summary(name, data, tgt_name, scr_name, weight_col=None, nbins=10):
+def dataset_summary(name, data, tgt_name, scr_name, weight_col=None, nbins=10,
+                    ascending=False, spec_values=None):
     weight = resolve_weights(data, weight_col=weight_col, expected_len=len(data))
     y_true = data[tgt_name].to_numpy()
     y_score = data[scr_name].to_numpy()
-    roc_df = calc_roc(y_true, y_score, sample_weight=weight)
-    gains = get_gains_table(data, tgt_name, scr_name, nbins=nbins, weight_col=weight_col)
-    return {
+    # Ranking metrics (AUC/KS/avgScore) are computed on non-special rows only:
+    # sentinel scores like -1 carry no ordering information.
+    rank_data, special_part = _split_special_scores(data, scr_name, spec_values)
+    rank_weight = resolve_weights(rank_data, weight_col=weight_col, expected_len=len(rank_data))
+    rank_true = rank_data[tgt_name].to_numpy()
+    rank_score = rank_data[scr_name].to_numpy()
+    roc_df = calc_roc(rank_true, rank_score, sample_weight=rank_weight)
+    gains = get_gains_table(
+        data, tgt_name, scr_name, nbins=nbins, weight_col=weight_col,
+        ascending=ascending, spec_values=spec_values,
+    )
+    summary = {
         "index": name,
         "dataset": name,
         "DATASET": name,
-        "AUC": safe_auc(y_true, y_score, sample_weight=weight),
+        "AUC": safe_auc(rank_true, rank_score, sample_weight=rank_weight),
         "KS": float(roc_df["KS"].max()) if "KS" in roc_df else np.nan,
         "LIFT": float(gains["LIFT"].max()) if "LIFT" in gains else np.nan,
         "IV": float(gains["IV"].sum()) if "IV" in gains else np.nan,
         "N": float(np.sum(weight)) if weight is not None else float(len(data)),
         "N_RAW": int(len(data)),
         "avgTrue": safe_weighted_average(y_true, weight),
-        "avgScore": safe_weighted_average(y_score, weight),
+        "avgScore": safe_weighted_average(rank_score, rank_weight),
     }
+    if special_part is not None:
+        spec_weight = resolve_weights(special_part, weight_col=weight_col, expected_len=len(special_part))
+        summary["N_SPECIAL"] = (
+            float(np.sum(spec_weight)) if spec_weight is not None else float(len(special_part))
+        )
+        summary["N_SPECIAL_RAW"] = int(len(special_part))
+    return summary
 
 
-def get_perf_summary(train=None, validation=None, oot=None, tgt_name=None, scr_name=None, weight_col=None, nbins=10, **kwargs):
+def get_perf_summary(train=None, validation=None, oot=None, tgt_name=None, scr_name=None, weight_col=None, nbins=10,
+                     ascending=False, spec_values=None, **kwargs):
     rows = []
     for name, data in (("ins", train), ("oos", validation), ("oot", oot)):
         if data is not None:
-            rows.append(dataset_summary(name, data, tgt_name, scr_name, weight_col=weight_col, nbins=nbins))
+            rows.append(dataset_summary(
+                name, data, tgt_name, scr_name, weight_col=weight_col, nbins=nbins,
+                ascending=ascending, spec_values=spec_values,
+            ))
     return pd.DataFrame(rows)
 
 

{supermodelingfactory-0.6.4 → supermodelingfactory-0.6.6}/Modeling_Tool/Pipeline/_common.py

@@ -532,6 +532,69 @@ def copy_column_length_checked(
     dst[col] = src[col].to_numpy()
 
 
+def all_missing_mask(data: pd.DataFrame, feature_cols: list[str]) -> np.ndarray:
+    """Rows where every listed feature is missing.
+
+    Same membership test as ``Core.scoring``'s all-missing branch
+    (``pd.isnull(data[varlist]).sum(axis=1) == len(varlist)``), so the
+    pipeline's evaluation override and the production scoring API agree on
+    which rows count as un-scorable.
+    """
+    features = list(feature_cols or [])
+    if not features:
+        return np.zeros(len(data), dtype=bool)
+    absent = [col for col in features if col not in data.columns]
+    if absent:
+        raise KeyError(
+            f"all-missing score override needs raw feature columns "
+            f"{absent[:5]} which are absent from the evaluation frame."
+        )
+    return (pd.isnull(data[features]).sum(axis=1) == len(features)).to_numpy()
+
+
+def resolve_missing_oot(
+    oos: pd.DataFrame,
+    synthesize: bool,
+    pipeline: str,
+) -> pd.DataFrame | None:
+    """Decide what to do when a run has no real OOT rows.
+
+    ``synthesize=True`` preserves the historical behavior — the OOS frame is
+    copied in as a stand-in OOT — but now emits a loud ``UserWarning`` so the
+    placeholder can never be mistaken for a real out-of-time sample.
+    ``synthesize=False`` returns ``None``: the caller must then leave OOT out
+    of its splits (missing key or empty frame) so no downstream stage
+    evaluates, searches, or reports against a synthetic OOT.
+    """
+    if synthesize:
+        warnings.warn(
+            f"{pipeline}: no real OOT rows found; OOS ({len(oos)} rows) was "
+            f"copied in as a stand-in OOT. Metrics labeled 'oot' are OOS "
+            f"metrics, NOT out-of-time performance. Set "
+            f"synthesize_missing_oot=False (or candidate_mode=True) to drop "
+            f"the placeholder instead.",
+            UserWarning,
+            stacklevel=3,
+        )
+        return oos.copy()
+    return None
+
+
+def assert_split_allowed(
+    split: str,
+    forbidden: list[str] | set[str] | None,
+    component: str,
+) -> None:
+    """Hard gate for split governance: raise if ``component`` is about to
+    consume a split listed in ``forbidden_splits``."""
+    if forbidden and split in set(forbidden):
+        raise ValueError(
+            f"{component} requested split {split!r}, but it is listed in "
+            f"forbidden_splits {sorted(set(forbidden))}. Remove the split from "
+            f"the component's configuration or lift the restriction."
+        )
+
+
 def split_oot_by_flag(
     data: pd.DataFrame,
     oot_col: str | None,