SuperModelingFactory 0.5.10__tar.gz → 0.6.1__tar.gz

{supermodelingfactory-0.5.10 → supermodelingfactory-0.6.1}/Modeling_Tool/Core/Proc_Compare.py

@@ -34,6 +34,7 @@ class ProcCompareConfig:
     chunk_size: int = 200_000
     n_partitions: int = 16
     backend: CompareBackend = "sequential"
+    compare_block_size: int = 64
 
     numeric_tol: float = 1e-8
     numeric_rtol: float = 0.0
@@ -107,6 +108,8 @@ class ProcCompareEngine:
             raise ValueError("chunk_size must be a positive integer.")
         if cfg.n_partitions <= 0:
             raise ValueError("n_partitions must be a positive integer.")
+        if cfg.compare_block_size <= 0:
+            raise ValueError("compare_block_size must be a positive integer.")
         if cfg.numeric_tol < 0 or cfg.numeric_rtol < 0 or cfg.datetime_tol_seconds < 0:
             raise ValueError("tolerance values must be non-negative.")
         if cfg.top_n <= 0:
@@ -294,44 +297,54 @@ class ProcCompareEngine:
         mismatch_frames = []
         mismatch_cols_by_row: dict[int, list[str]] = {}
 
-        for col in compare_cols:
-            left_col = f"{col}_left"
-            right_col = f"{col}_right"
-            comp = self._compare_series(col, merged[left_col], merged[right_col], common_mask)
-            mismatch_mask = comp["mismatch"]
-            n_mismatch = int(mismatch_mask.sum())
-            if n_mismatch:
-                mismatch_indexes = merged.index[mismatch_mask]
-                for idx in mismatch_indexes:
+        common_values = common_mask.to_numpy(dtype=bool)
+        n_compared = int(common_values.sum())
+        block_size = int(self.config.compare_block_size)
+        for start in range(0, len(compare_cols), block_size):
+            block_cols = compare_cols[start : start + block_size]
+            comp = self._compare_column_block(merged, block_cols, common_values)
+            mismatch_matrix = comp["mismatch"]
+            one_null_matrix = comp["one_null"]
+            both_null_matrix = comp["both_null"]
+            diff_matrix = comp["diff"]
+            abs_diff_matrix = comp["abs_diff"]
+            left_values = comp["left_values"]
+            right_values = comp["right_values"]
+
+            for col_idx, col in enumerate(block_cols):
+                mismatch_positions = np.flatnonzero(mismatch_matrix[:, col_idx])
+                n_mismatch = int(len(mismatch_positions))
+                for idx in mismatch_positions:
                     mismatch_cols_by_row.setdefault(int(idx), []).append(col)
-                if self.config.detail_mode != "none":
-                    detail = merged.loc[mismatch_mask, key_cols].copy()
+                if n_mismatch and self.config.detail_mode != "none":
+                    detail = merged.iloc[mismatch_positions][key_cols].copy()
                     detail["column"] = col
-                    detail["left_value"] = merged.loc[mismatch_mask, left_col].to_numpy()
-                    detail["right_value"] = merged.loc[mismatch_mask, right_col].to_numpy()
-                    detail["diff"] = comp["diff"].loc[mismatch_mask].to_numpy()
-                    detail["abs_diff"] = comp["abs_diff"].loc[mismatch_mask].to_numpy()
+                    detail["left_value"] = left_values[mismatch_positions, col_idx]
+                    detail["right_value"] = right_values[mismatch_positions, col_idx]
+                    detail["diff"] = diff_matrix[mismatch_positions, col_idx]
+                    detail["abs_diff"] = abs_diff_matrix[mismatch_positions, col_idx]
                     mismatch_frames.append(detail)
 
-            n_compared = int(common_mask.sum())
-            n_one_null = int(comp["one_null"].sum())
-            n_both_null = int(comp["both_null"].sum())
-            n_equal = n_compared - n_mismatch
-            column_rows.append(
-                {
-                    "column": col,
-                    "n_compared": n_compared,
-                    "n_equal": n_equal,
-                    "n_mismatch": n_mismatch,
-                    "pct_mismatch": round(n_mismatch / n_compared * 100, 6) if n_compared else 0.0,
-                    "n_one_side_null": n_one_null,
-                    "n_both_null": n_both_null,
-                    "mean_diff": comp["diff"].mean(skipna=True),
-                    "max_abs_diff": comp["abs_diff"].max(skipna=True),
-                    "_diff_sum": comp["diff"].sum(skipna=True),
-                    "_diff_count": int(comp["diff"].notna().sum()),
-                }
-            )
+                diff_values = diff_matrix[:, col_idx]
+                abs_diff_values = abs_diff_matrix[:, col_idx]
+                diff_valid = np.isfinite(diff_values)
+                diff_sum = float(np.nansum(diff_values)) if diff_valid.any() else 0.0
+                diff_count = int(diff_valid.sum())
+                column_rows.append(
+                    {
+                        "column": col,
+                        "n_compared": n_compared,
+                        "n_equal": n_compared - n_mismatch,
+                        "n_mismatch": n_mismatch,
+                        "pct_mismatch": round(n_mismatch / n_compared * 100, 6) if n_compared else 0.0,
+                        "n_one_side_null": int(one_null_matrix[:, col_idx].sum()),
+                        "n_both_null": int(both_null_matrix[:, col_idx].sum()),
+                        "mean_diff": diff_sum / diff_count if diff_count else np.nan,
+                        "max_abs_diff": float(np.nanmax(abs_diff_values)) if np.isfinite(abs_diff_values).any() else np.nan,
+                        "_diff_sum": diff_sum,
+                        "_diff_count": diff_count,
+                    }
+                )
 
         for idx, cols in mismatch_cols_by_row.items():
             row_summary.loc[idx, "n_cell_mismatch"] = len(cols)
@@ -375,6 +388,146 @@ class ProcCompareEngine:
             duplicate_key_summary=duplicate_key_summary,
         )
 
+    def _compare_column_block(
+        self,
+        merged: pd.DataFrame,
+        columns: list[str],
+        common_mask: np.ndarray,
+    ) -> dict[str, np.ndarray]:
+        """Compare a bounded field block with vectorized type-specific kernels."""
+        n_rows = len(merged)
+        n_cols = len(columns)
+        left_series = [self._normalize_series_missing(merged[f"{col}_left"]) for col in columns]
+        right_series = [self._normalize_series_missing(merged[f"{col}_right"]) for col in columns]
+        left_values = np.column_stack([series.to_numpy(dtype=object) for series in left_series])
+        right_values = np.column_stack([series.to_numpy(dtype=object) for series in right_series])
+        mismatch = np.zeros((n_rows, n_cols), dtype=bool)
+        one_null = np.zeros((n_rows, n_cols), dtype=bool)
+        both_null = np.zeros((n_rows, n_cols), dtype=bool)
+        diff = np.full((n_rows, n_cols), np.nan, dtype=float)
+        abs_diff = np.full((n_rows, n_cols), np.nan, dtype=float)
+
+        datetime_idx = [
+            idx
+            for idx, col in enumerate(columns)
+            if self._should_compare_datetime(col, left_series[idx], right_series[idx])
+        ]
+        datetime_set = set(datetime_idx)
+        numeric_idx = [
+            idx
+            for idx in range(n_cols)
+            if idx not in datetime_set
+            and self._should_compare_numeric(left_series[idx], right_series[idx])
+        ]
+        numeric_set = set(numeric_idx)
+        string_idx = [
+            idx
+            for idx in range(n_cols)
+            if idx not in datetime_set and idx not in numeric_set
+        ]
+        common = common_mask[:, None]
+
+        if datetime_idx:
+            left_dt = np.column_stack(
+                [self._datetime_seconds(left_series[idx]) for idx in datetime_idx]
+            )
+            right_dt = np.column_stack(
+                [self._datetime_seconds(right_series[idx]) for idx in datetime_idx]
+            )
+            block_diff = left_dt - right_dt
+            block_abs = np.abs(block_diff)
+            left_null = ~np.isfinite(left_dt)
+            right_null = ~np.isfinite(right_dt)
+            tolerances = np.asarray(
+                [self._column_datetime_tol(columns[idx]) for idx in datetime_idx],
+                dtype=float,
+            )
+            block_one = (left_null ^ right_null) & common
+            block_both = left_null & right_null & common
+            block_mismatch = (block_abs > tolerances[None, :]) | block_one
+            if not self.config.both_null_equal:
+                block_mismatch |= block_both
+            block_mismatch &= common
+            diff[:, datetime_idx] = block_diff
+            abs_diff[:, datetime_idx] = block_abs
+            one_null[:, datetime_idx] = block_one
+            both_null[:, datetime_idx] = block_both
+            mismatch[:, datetime_idx] = block_mismatch
+
+        if numeric_idx:
+            left_num = np.column_stack(
+                [pd.to_numeric(left_series[idx], errors="coerce").to_numpy(dtype=float, na_value=np.nan) for idx in numeric_idx]
+            )
+            right_num = np.column_stack(
+                [pd.to_numeric(right_series[idx], errors="coerce").to_numpy(dtype=float, na_value=np.nan) for idx in numeric_idx]
+            )
+            block_diff = left_num - right_num
+            block_abs = np.abs(block_diff)
+            left_null = ~np.isfinite(left_num)
+            right_null = ~np.isfinite(right_num)
+            tolerances = np.asarray(
+                [self._column_numeric_tol(columns[idx]) for idx in numeric_idx],
+                dtype=float,
+            )
+            limit = tolerances[:, 0][None, :] + tolerances[:, 1][None, :] * np.abs(right_num)
+            block_one = (left_null ^ right_null) & common
+            block_both = left_null & right_null & common
+            block_mismatch = (block_abs > limit) | block_one
+            if not self.config.both_null_equal:
+                block_mismatch |= block_both
+            block_mismatch &= common
+            diff[:, numeric_idx] = block_diff
+            abs_diff[:, numeric_idx] = block_abs
+            one_null[:, numeric_idx] = block_one
+            both_null[:, numeric_idx] = block_both
+            mismatch[:, numeric_idx] = block_mismatch
+
+        if string_idx:
+            left_null = np.column_stack([left_series[idx].isna().to_numpy() for idx in string_idx])
+            right_null = np.column_stack([right_series[idx].isna().to_numpy() for idx in string_idx])
+            left_text = np.column_stack(
+                [left_series[idx].astype("string").fillna("").astype(str).to_numpy() for idx in string_idx]
+            )
+            right_text = np.column_stack(
+                [right_series[idx].astype("string").fillna("").astype(str).to_numpy() for idx in string_idx]
+            )
+            block_one = (left_null ^ right_null) & common
+            block_both = left_null & right_null & common
+            block_mismatch = (left_text != right_text) | block_one
+            if self.config.both_null_equal:
+                block_mismatch &= ~block_both
+            else:
+                block_mismatch |= block_both
+            block_mismatch &= common
+            one_null[:, string_idx] = block_one
+            both_null[:, string_idx] = block_both
+            mismatch[:, string_idx] = block_mismatch
+
+        return {
+            "mismatch": mismatch,
+            "one_null": one_null,
+            "both_null": both_null,
+            "diff": diff,
+            "abs_diff": abs_diff,
+            "left_values": left_values,
+            "right_values": right_values,
+        }
+
+    @staticmethod
+    def _datetime_seconds(series: pd.Series) -> np.ndarray:
+        values = ProcCompareEngine._to_datetime_series(series)
+        missing = values.isna().to_numpy()
+        result = values.astype("int64").to_numpy(dtype=float) / 1_000_000_000.0
+        result[missing] = np.nan
+        return result
+
+    @staticmethod
+    def _to_datetime_series(series: pd.Series) -> pd.Series:
+        try:
+            return pd.to_datetime(series, errors="coerce", format="mixed")
+        except (TypeError, ValueError):
+            return pd.to_datetime(series, errors="coerce")
+
     def _compare_series(
         self,
         col: str,
@@ -391,8 +544,8 @@ class ProcCompareEngine:
         abs_diff = pd.Series(np.nan, index=left.index, dtype="float64")
 
         if self._should_compare_datetime(col, left, right):
-            left_dt = pd.to_datetime(left, errors="coerce")
-            right_dt = pd.to_datetime(right, errors="coerce")
+            left_dt = self._to_datetime_series(left)
+            right_dt = self._to_datetime_series(right)
             diff = (left_dt - right_dt).dt.total_seconds()
             abs_diff = diff.abs()
             tol = self._column_datetime_tol(col)

{supermodelingfactory-0.5.10 → supermodelingfactory-0.6.1}/Modeling_Tool/Core/kDataFrame.py

@@ -57,13 +57,7 @@ class kSeries(Series):
         """
         Scale the model scores (for internt segment of MCI model)
         """
-        data = self.copy()
-        def app_func(x):
-            scores = x * 1.112
-            if (scores > 0.9999999):
-                return 0.9999999
-            return scores
-        return data.apply(app_func)
+        return (self * 1.112).clip(upper=0.9999999)
     
     def proc_freq(self) -> pd.DataFrame:
         """
@@ -128,8 +122,12 @@ class kDataFrame(DataFrame):
             data = self
         else:
             data = self.copy()
-        import re
-        data[vintage_colname] = data[by].apply(lambda x: re.search("\d{4}-\d{2}", x).group().replace('-', ''))
+        data[vintage_colname] = (
+            data[by]
+            .astype("string")
+            .str.extract(r"(\d{4}-\d{2})", expand=False)
+            .str.replace("-", "", regex=False)
+        )
         if inplace:
             self = data
         return data
@@ -165,13 +163,7 @@ class kDataFrame(DataFrame):
         """
         Scale the model scores (for internt segment of MCI model)
         """
-        data = self.copy()
-        def app_func(x):
-            scores = x * 1.112
-            if (scores > 0.9999999):
-                return 0.9999999
-            return scores
-        return data[pb_score].apply(app_func)
+        return (self[pb_score] * 1.112).clip(upper=0.9999999)
     
     def proc_freq(self, var: str):
         """

{supermodelingfactory-0.5.10 → supermodelingfactory-0.6.1}/Modeling_Tool/Core/utils.py

@@ -342,8 +342,12 @@ def convert_to_vintage(data, vintage_colname = 'VINTAGE', by = 'TRAN_TMS', retur
     >>> df = pd.DataFrame({'TRAN_TMS': ['2025-03-15 10:00:00', '2025-03-20 11:00:00']})
     >>> convert_to_vintage(df)
     """
-    import re
-    data[vintage_colname] = data[by].apply(lambda x: re.search("\\d{4}-\\d{2}", x).group().replace('-', ''))
+    data[vintage_colname] = (
+        data[by]
+        .astype("string")
+        .str.extract(r"(\d{4}-\d{2})", expand=False)
+        .str.replace("-", "", regex=False)
+    )
 
     if return_kDF:
         return kDataFrame(data)
@@ -1052,10 +1056,9 @@ def bool_to_str(data):
     >>> bool_to_str(df)
     """
     dfc = data.copy()
-    type_dict = data.dtypes.to_dict()
-    for k,v in type_dict.items():
-        if str(v).lower() == 'bool':
-            dfc = dfc.astype({k: str})
+    bool_cols = [col for col in data.columns if pd.api.types.is_bool_dtype(data[col])]
+    if bool_cols:
+        dfc[bool_cols] = dfc[bool_cols].astype(str)
     return dfc
 
 
@@ -2670,4 +2673,4 @@ def get_feature_names_batch(models, model_type=None):
     elif isinstance(models, list):
         return [get_feature_names(model, model_type=model_type) for model in models]
     else:
-        raise TypeError("models参数应为dict或list类型")
+        raise TypeError("models参数应为dict或list类型")

{supermodelingfactory-0.5.10 → supermodelingfactory-0.6.1}/Modeling_Tool/Eval/Model_Eval_Tool.py

@@ -88,51 +88,31 @@ def _get_gains_table_scr(data, score, dep, nbins = 10, precision = 5,
                                bin_colnames = ("_bin_num", "_bin_range"),
                                ascending = ascending)
     
-    def _compute_gains_tmp(group):
-        # 计算各项指标
-        min_val = group[score].min()
-        max_val = group[score].max()
-        # NOTE: pandas 2.3+ 起 groupby(...).apply(func) 传入 func 的 group
-        # DataFrame 不再包含分组列,不能再用 group['_bin_num']。
-        # len(group) 与原 .count() 数值等价(_bin_num 为分组主键,不为 NaN)。
-        n = len(group)
-        avg_score = group[score].mean()
-        unique_score = group[score].nunique()
-
-        # dep相关
-        dep_vals = group[dep]
-        
-        grand_total = res.shape[0]
-        grand_perf_cnt = res[dep].count()
-        grand_total_bad = res[dep].sum()
-        grand_total_good = (grand_perf_cnt - grand_total_bad)
-        
-        perf_cnt = dep_vals.count()
-        n_bad = (dep_vals == 1).sum()
-        n_good = (dep_vals == 0).sum()
-        avg_bad = n_bad / dep_vals.count() if dep_vals.count() > 0 else 0  # 避免除以0
-        avg_good = n_good / dep_vals.count() if dep_vals.count() > 0 else 0
-        
-        lift = avg_bad / res[dep].mean()
-        prop = n / grand_total
-
-        # 返回Series
-        return pd.Series({
-            'MIN': min_val,
-            'MAX': max_val,
-            'N': n,
-            'PROP': prop,
-            'PERF_CNT': perf_cnt,
-            'AVG_SCORE': avg_score,
-            'UNIQUE_SCORE': unique_score,
-            'AVG_BAD': avg_bad,
-            'AVG_GOOD': avg_good,
-            'N_BAD': n_bad,
-            'N_GOOD': n_good,
-            'LIFT': lift
-        })
-    
-    gains_table = res.groupby(["_bin_num", "_bin_range"], dropna=False).apply(_compute_gains_tmp)
+    res = res.copy()
+    res["_smf_bad_ind"] = res[dep].eq(1).astype(np.int64)
+    res["_smf_good_ind"] = res[dep].eq(0).astype(np.int64)
+    grouped = res.groupby(["_bin_num", "_bin_range"], dropna=False)
+    gains_table = grouped.agg(
+        MIN=(score, "min"),
+        MAX=(score, "max"),
+        N=(dep, "size"),
+        PERF_CNT=(dep, "count"),
+        AVG_SCORE=(score, "mean"),
+        UNIQUE_SCORE=(score, "nunique"),
+        N_BAD=("_smf_bad_ind", "sum"),
+        N_GOOD=("_smf_good_ind", "sum"),
+    )
+    perf_denom = gains_table["PERF_CNT"].replace(0, np.nan)
+    gains_table["PROP"] = gains_table["N"] / max(len(res), 1)
+    gains_table["AVG_BAD"] = (gains_table["N_BAD"] / perf_denom).fillna(0.0)
+    gains_table["AVG_GOOD"] = (gains_table["N_GOOD"] / perf_denom).fillna(0.0)
+    gains_table["LIFT"] = gains_table["AVG_BAD"] / res[dep].mean()
+    gains_table = gains_table[
+        [
+            "MIN", "MAX", "N", "PROP", "PERF_CNT", "AVG_SCORE",
+            "UNIQUE_SCORE", "AVG_BAD", "AVG_GOOD", "N_BAD", "N_GOOD", "LIFT",
+        ]
+    ]
 
     gains_table["BAD_PCT_IN_EACH_BIN"] = gains_table["N_BAD"] / gains_table["N_BAD"].sum()
     gains_table["GOOD_PCT_IN_EACH_BIN"] = gains_table["N_GOOD"] / gains_table["N_GOOD"].sum()
@@ -146,7 +126,7 @@ def _get_gains_table_scr(data, score, dep, nbins = 10, precision = 5,
 
 
     gains_table["TRUE_BAD_SHIFT"] = (gains_table['AVG_BAD'].shift(1) / gains_table['AVG_BAD'] - 1) if not ascending else (gains_table['AVG_BAD'] / gains_table['AVG_BAD'].shift(1) - 1) 
-    gains_table["RANK_ORDER_BUMP"] = gains_table["TRUE_BAD_SHIFT"].apply(lambda x: 1 if x < 0 else 0)
+    gains_table["RANK_ORDER_BUMP"] = gains_table["TRUE_BAD_SHIFT"].lt(0).astype(int)
     
     gains_table["WOE"] = calc_woe(data = gains_table, bad_pct = "BAD_PCT_IN_EACH_BIN", good_pct = "GOOD_PCT_IN_EACH_BIN")
     gains_table["IV"] = calc_iv(data = gains_table, bad_pct = "BAD_PCT_IN_EACH_BIN", good_pct = "GOOD_PCT_IN_EACH_BIN")

{supermodelingfactory-0.5.10 → supermodelingfactory-0.6.1}/Modeling_Tool/Eval/evaluate_model.py

@@ -648,25 +648,46 @@ def __agg(df):
         sum_score=pd.NamedAgg(column='y_score', aggfunc='sum'),
         ).reset_index()
     df_agg[['n', 'sum_true', 'sum_score']] = df_agg[['n', 'sum_true', 'sum_score']].fillna(0)
-    df_agg.loc[:, 'proportion'] = [x / N for x in df_agg['n']]
-    df_agg.loc[:, 'capture_rate'] = [x / N1 for x in df_agg['sum_true']]
+    df_agg.loc[:, 'proportion'] = np.divide(
+        df_agg['n'].to_numpy(dtype=float),
+        float(N),
+        out=np.full(len(df_agg), np.nan),
+        where=N > 0,
+    )
+    df_agg.loc[:, 'capture_rate'] = np.divide(
+        df_agg['sum_true'].to_numpy(dtype=float),
+        float(N1),
+        out=np.full(len(df_agg), np.nan),
+        where=N1 > 0,
+    )
 
     if 'y_group' in df_agg.columns:
-        gs = list(set(df_agg['y_group']))
-        for g in gs:
-            g_idx = (df_agg['y_group'] == g)
-            df_agg.loc[g_idx, 'cumsum_n'] = np.cumsum(df_agg.loc[g_idx, 'n'])
-            df_agg.loc[g_idx, 'cumsum_proportion'] = np.cumsum(df_agg.loc[g_idx, 'proportion'])
-            df_agg.loc[g_idx, 'cumsum_true'] = np.cumsum(df_agg.loc[g_idx, 'sum_true'])
-            df_agg.loc[g_idx, 'cumsum_score'] = np.cumsum(df_agg.loc[g_idx, 'sum_score'])
+        cumulative = df_agg.groupby('y_group', sort=False)[
+            ['n', 'proportion', 'sum_true', 'sum_score']
+        ].cumsum()
+        cumulative.columns = [
+            'cumsum_n', 'cumsum_proportion', 'cumsum_true', 'cumsum_score'
+        ]
+        df_agg.loc[:, cumulative.columns] = cumulative.to_numpy()
     else:
         df_agg.loc[:, 'cumsum_n'] = np.cumsum(df_agg['n'])
         df_agg.loc[:, 'cumsum_proportion'] = np.cumsum(df_agg['proportion'])
         df_agg.loc[:, 'cumsum_true'] = np.cumsum(df_agg['sum_true'])
         df_agg.loc[:, 'cumsum_score'] = np.cumsum(df_agg['sum_score'])
 
-    df_agg.loc[:, 'cumavg_true'] = [x / y if y>0 else np.nan for x,y in zip(df_agg['cumsum_true'], df_agg['cumsum_n'])]
-    df_agg.loc[:, 'cumavg_score'] = [x / y if y>0 else np.nan for x,y in zip(df_agg['cumsum_score'], df_agg['cumsum_n'])]
+    cumulative_n = df_agg['cumsum_n'].to_numpy(dtype=float)
+    df_agg.loc[:, 'cumavg_true'] = np.divide(
+        df_agg['cumsum_true'].to_numpy(dtype=float),
+        cumulative_n,
+        out=np.full(len(df_agg), np.nan),
+        where=cumulative_n > 0,
+    )
+    df_agg.loc[:, 'cumavg_score'] = np.divide(
+        df_agg['cumsum_score'].to_numpy(dtype=float),
+        cumulative_n,
+        out=np.full(len(df_agg), np.nan),
+        where=cumulative_n > 0,
+    )
     
     columns = group_cols + ['min_score', 'max_score', 'n', 'proportion', 'sum_true', 'sum_score', 'avg_true', 'avg_score', 'capture_rate',
                             'cumsum_n', 'cumsum_proportion', 'cumsum_true', 'cumsum_score', 'cumavg_true', 'cumavg_score', ]
@@ -794,20 +815,28 @@ def calc_equid_pct(y_true, y_score, y_group=None, bins=10, ascending=True, sampl
     binsize = int(size / bins) # 向下取整
     indices = np.argsort(y_score) if ascending else np.argsort(y_score)[::-1] 
     
-    thresholds = np.array([0]*size)
-    thresholds_percentile = [100 * (i+1) / bins for i in range(bins)]
-    for i in range(bins):
-        s = i*binsize
-        e = (i+1)*binsize if i < bins-1 else np.max([size, (i+1)*binsize]) # 严格排序
-        thresholds[indices[s:e]] = thresholds_percentile[i]
-    if bool(y_group):
+    # Keep the historical integer percentile labels while assigning them in
+    # one pass. For bin counts that do not divide 100, the legacy code
+    # truncated labels during assignment to its integer array.
+    thresholds = np.zeros(size, dtype=int)
+    if binsize > 0:
+        sorted_bin = np.minimum(np.arange(size) // binsize, bins - 1)
+    else:
+        sorted_bin = np.full(size, bins - 1, dtype=int)
+    thresholds[indices] = 100.0 * (sorted_bin + 1) / bins
+    if y_group is not None:
         df = pd.DataFrame({'y_true': y_true, 'y_score': y_score, 'y_group': y_group, 'thresholds': thresholds})
     else:
         df = pd.DataFrame({'y_true': y_true, 'y_score': y_score, 'thresholds': thresholds})
     pct_df = __agg(df)
 
     avg_true = np.mean(y_true)
-    pct_df['lift'] = [x / avg_true for x in pct_df['cumavg_true']]
+    pct_df['lift'] = np.divide(
+        pct_df['cumavg_true'].to_numpy(dtype=float),
+        float(avg_true),
+        out=np.full(len(pct_df), np.nan),
+        where=avg_true != 0,
+    )
     pct_df['gain'] = np.cumsum(pct_df['capture_rate'])
 
     return pct_df

{supermodelingfactory-0.5.10 → supermodelingfactory-0.6.1}/Modeling_Tool/Eval/weighted_eval_utils.py

@@ -124,6 +124,7 @@ def get_gains_table(data, dep, score, nbins=10, weight_col=None, weighted_binnin
     df["_w"] = weight
     df["_bad_w"] = weight * y
     df["_good_w"] = weight * (1.0 - y)
+    df["_score_w"] = weight * s
     grouped = df.groupby(["_bin_num", "_bin_range"], sort=True, dropna=False)
     out = grouped.agg(
         MIN=(score, "min"),
@@ -133,9 +134,19 @@ def get_gains_table(data, dep, score, nbins=10, weight_col=None, weighted_binnin
         PERF_CNT=("_w", "sum"),
         N_BAD=("_bad_w", "sum"),
         N_GOOD=("_good_w", "sum"),
-        AVG_SCORE=(score, lambda x: safe_weighted_average(x, df.loc[x.index, "_w"])),
+        SCORE_W=("_score_w", "sum"),
+        SCORE_COUNT=(score, "count"),
         UNIQUE_SCORE=(score, "nunique"),
     )
+    out["AVG_SCORE"] = out["SCORE_W"] / out["N"].replace(0, np.nan)
+    out.loc[out["SCORE_COUNT"].ne(out["N_RAW"]), "AVG_SCORE"] = np.nan
+    out = out.drop(columns=["SCORE_W", "SCORE_COUNT"])
+    out = out[
+        [
+            "MIN", "MAX", "N", "N_RAW", "PERF_CNT", "N_BAD",
+            "N_GOOD", "AVG_SCORE", "UNIQUE_SCORE",
+        ]
+    ]
 
     total_weight = float(out["N"].sum()) or 1.0
     total_bad = float(out["N_BAD"].sum()) or 1.0
@@ -155,7 +166,7 @@ def get_gains_table(data, dep, score, nbins=10, weight_col=None, weighted_binnin
     out["KS"] = out["KS_PER_BIN"]
     out["LIFT"] = out["AVG_BAD"] / overall_bad_rate if overall_bad_rate else np.nan
     out["TRUE_BAD_SHIFT"] = out["AVG_BAD"].shift(1) / out["AVG_BAD"] - 1
-    out["RANK_ORDER_BUMP"] = out["TRUE_BAD_SHIFT"].apply(lambda x: 1 if x < 0 else 0)
+    out["RANK_ORDER_BUMP"] = out["TRUE_BAD_SHIFT"].lt(0).astype(int)
     with np.errstate(divide="ignore", invalid="ignore"):
         out["WOE"] = np.log(out["BAD_PCT_IN_EACH_BIN"] / out["GOOD_PCT_IN_EACH_BIN"])
     out["WOE"] = out["WOE"].replace([np.inf, -np.inf], 0).fillna(0)

{supermodelingfactory-0.5.10 → supermodelingfactory-0.6.1}/Modeling_Tool/Explainability/Coalition_Structure.py

@@ -8,6 +8,7 @@ accepted by ``shap.maskers.Partition``.
 """
 from __future__ import annotations
 
+from collections import Counter
 from typing import Dict, Iterable, List, Mapping, Optional, Sequence
 
 import numpy as np
@@ -248,7 +249,7 @@ def validate_groups(groups: Mapping[str, Sequence[str]], features: Sequence[str]
     feature_set = set(features)
     missing = feature_set - set(all_assigned)
     unknown = set(all_assigned) - feature_set
-    duplicated = {feat for feat in all_assigned if all_assigned.count(feat) > 1}
+    duplicated = {feat for feat, count in Counter(all_assigned).items() if count > 1}
     ok = not missing and not duplicated and not unknown
     if not ok and raise_error:
         parts = []
@@ -316,10 +317,9 @@ def groups_to_shap_clustering(
 
     for feats in groups.values():
         idxs = [feat_idx[feat] for feat in feats if feat in feat_idx]
-        for i in idxs:
-            for j in idxs:
-                if i != j:
-                    dist_mat[i, j] = float(intra_dist)
+        if idxs:
+            dist_mat[np.ix_(idxs, idxs)] = float(intra_dist)
+            dist_mat[idxs, idxs] = 0.0
 
     condensed = squareform(dist_mat, checks=False)
     return linkage(condensed, method="complete")