SuperModelingFactory 0.5.9__tar.gz → 0.6.0__tar.gz

{supermodelingfactory-0.5.9 → supermodelingfactory-0.6.0}/Modeling_Tool/Core/Binning_Tool.py

@@ -706,6 +706,46 @@ def get_bin_range(edges, precision = 5, ascending = False, left_sign = '(', righ
     return res
 
 
+def _materialize_bin_columns(data, binned, bin_range_list, bin_num_col, bin_range_col):
+    """Attach categorical bin numbers and labels without Python row callbacks."""
+    codes = binned.cat.codes.to_numpy(dtype=np.intp, copy=False)
+    range_values = np.empty(len(codes), dtype=object)
+    range_values[:] = np.nan
+    valid = codes >= 0
+    if valid.any():
+        labels = np.asarray(bin_range_list, dtype=object)
+        if codes[valid].max() >= len(labels):
+            raise ValueError("Binning category codes do not match bin range labels")
+        range_values[valid] = np.take(labels, codes[valid])
+
+    data[bin_num_col] = binned.astype(object)
+    data[bin_range_col] = range_values
+    return data
+
+
+def _parse_bin_range_bounds(data, col="_bin_range"):
+    """Parse interval labels into numeric left and right bounds vectorially."""
+    if col not in data.columns:
+        raise KeyError(f"Column {col!r} is not present in the bin mapping table")
+
+    ranges = data[col].astype("string").str.strip()
+    bounds = ranges.str.extract(
+        r"^[\[\(]\s*([^,]+?)\s*,\s*([^\]\)]+?)\s*[\]\)]$",
+        expand=True,
+    )
+    invalid = bounds.isna().any(axis=1)
+    if invalid.any():
+        examples = ranges.loc[invalid].dropna().head(3).tolist()
+        raise ValueError(f"Unable to parse bin range labels in {col!r}: {examples}")
+
+    try:
+        left = bounds.iloc[:, 0].str.strip().astype(float).to_numpy()
+        right = bounds.iloc[:, 1].str.strip().astype(float).to_numpy()
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"Bin range column {col!r} contains non-numeric bounds") from exc
+    return left, right
+
+
 def get_bin_range_list(data, col = "_bin_range"):
     """
     将分箱区间字符串列转换为边界值列表。
@@ -730,21 +770,8 @@ def get_bin_range_list(data, col = "_bin_range"):
     >>> unique_range = get_bin_range_list(data, col="bin_range")
     """
     
-    data["bin_value_list"] = data[col].apply(lambda x: x.replace("[", "")\
-                                                                  .replace("]", "")\
-                                                                  .replace("(", "")\
-                                                                  .replace(")", "")\
-                                                      .split(","))
-    bin_range = [np.inf if v.strip() == 'inf' else -np.inf if v.strip() == '-inf' else float(v.strip()) for x in data["bin_value_list"].tolist() for v in x]
-
-    unique_range = []
-    for x in bin_range:
-        if x not in unique_range:
-            unique_range.append(x)
-
-    unique_range.sort()
-        
-    return unique_range
+    left, right = _parse_bin_range_bounds(data, col=col)
+    return np.unique(np.concatenate([left, right])).tolist()
 
 
 def chi2_auto_binning(df_pvt, max_bins, min_cnt_in_bin, p=0.95):
@@ -1165,9 +1192,8 @@ class Binning:
         bin_num_col = self.bin_colnames[0]
         bin_range_col = self.bin_colnames[1]
         
-        self.data[bin_num_col] = binned.astype(object)
-        self.data[bin_range_col] = self.data[bin_num_col].apply(
-            lambda x: bin_range_list[int(x)] if self.include_missing else bin_range_list[int(x - 1)]
+        self.data = _materialize_bin_columns(
+            self.data, binned, bin_range_list, bin_num_col, bin_range_col
         )
         
         self.result = self.data
@@ -1237,9 +1263,8 @@ class Binning:
         bin_num_col = self.bin_colnames[0]
         bin_range_col = self.bin_colnames[1]
         
-        self.data[bin_num_col] = binned.astype(object)
-        self.data[bin_range_col] = self.data[bin_num_col].apply(
-            lambda x: bin_range_list[int(x)] if self.include_missing else bin_range_list[int(x - 1)]
+        self.data = _materialize_bin_columns(
+            self.data, binned, bin_range_list, bin_num_col, bin_range_col
         )
         
         self.bin_edges = sorted([np.inf if str(x).lower() == 'inf' else -np.inf if str(x).lower() == '-inf' else x for x in bin_edges])
@@ -1385,10 +1410,9 @@ def chi2_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, t
     binned = binning_series
     rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not include_missing else [i for i in range(0, len(bin_range_list))]
     binned = binned.cat.rename_categories(rename_catlist)
-    data[bin_num_col] = binned.astype(object)
-#     print(bin_range_list)
-#     print(data[bin_num_col])
-    data[bin_range_col] = data[bin_num_col].apply(lambda x: bin_range_list[int(x)] if include_missing else bin_range_list[int(x - 1)])
+    data = _materialize_bin_columns(
+        data, binned, bin_range_list, bin_num_col, bin_range_col
+    )
     
     fnl_res = data
     
@@ -1399,7 +1423,7 @@ def chi2_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, t
 
 def run_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, include_missing = True, equal_freq = True, 
                 bin_colnames = ("bin_num", "bin_range"), ascending = False, right = True, include_lowest = False, 
-                tree_binning = False, target = None, random_state=42, spec_values = []):
+                tree_binning = False, target = None, random_state=42, spec_values = [], fillna = -999999):
     """
     通用分箱函数，支持等频或等距分箱。
     
@@ -1437,6 +1461,8 @@ def run_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, in
         随机种子
     spec_values : list, default []
         特殊值列表
+    fillna : scalar, default -999999
+        `include_missing=True` 时用于承载缺失值的分箱哨兵。
     
     Returns
     -------
@@ -1481,7 +1507,8 @@ def run_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, in
                                       ascending = ascending,
                                       include_missing = include_missing,
                                       random_state = random_state,
-                                      spec_values = spec_values)
+                                      spec_values = spec_values,
+                                      fillna = fillna)
     
     left_sign='[' if include_lowest else '('
     right_sign=']' if right else ')'
@@ -1489,8 +1516,9 @@ def run_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, in
     
     rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not include_missing else [i for i in range(0, len(bin_range_list))]
     binned = binned.cat.rename_categories(rename_catlist)
-    data[bin_num_col] = binned.astype(object)
-    data[bin_range_col] = data[bin_num_col].apply(lambda x: bin_range_list[int(x)] if include_missing else bin_range_list[int(x - 1)])
+    data = _materialize_bin_columns(
+        data, binned, bin_range_list, bin_num_col, bin_range_col
+    )
         
     return data, bin_edges
 

{supermodelingfactory-0.5.9 → supermodelingfactory-0.6.0}/Modeling_Tool/Core/Binning_Tool.pyi

@@ -44,5 +44,5 @@ class Binning:
     def run(self): ...
     def get_result(self, return_edges = True): ...
 def chi2_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, tgt_name = None, include_missing = True, equal_freq = True, bin_colnames = ('_bin_num', '_bin_range'), ascending = True, chi2_p = 0.95, init_equi_bins = 100, fillna = -999999, spec_values = [], init_points = None): ...
-def run_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, include_missing = True, equal_freq = True, bin_colnames = ('bin_num', 'bin_range'), ascending = False, right = True, include_lowest = False, tree_binning = False, target = None, random_state = 42, spec_values = []): ...
+def run_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, include_missing = True, equal_freq = True, bin_colnames = ('bin_num', 'bin_range'), ascending = False, right = True, include_lowest = False, tree_binning = False, target = None, random_state = 42, spec_values = [], fillna = -999999): ...
 def super_binning(data, score, dep, nbins = 10, precision = 5, min_bin_prop = 0.05, include_missing = True, equal_freq = True, chi2_method = False, chi2_p = 0.95, init_equi_bins = 2000, fillna = -999999, spec_values = [], tree_binning = False, random_state = 42, return_edges = False, ascending = True, bin_colnames = ('_bin_num', '_bin_range')): ...

{supermodelingfactory-0.5.9 → supermodelingfactory-0.6.0}/Modeling_Tool/Core/Proc_Compare.py

@@ -34,6 +34,7 @@ class ProcCompareConfig:
     chunk_size: int = 200_000
     n_partitions: int = 16
     backend: CompareBackend = "sequential"
+    compare_block_size: int = 64
 
     numeric_tol: float = 1e-8
     numeric_rtol: float = 0.0
@@ -107,6 +108,8 @@ class ProcCompareEngine:
             raise ValueError("chunk_size must be a positive integer.")
         if cfg.n_partitions <= 0:
             raise ValueError("n_partitions must be a positive integer.")
+        if cfg.compare_block_size <= 0:
+            raise ValueError("compare_block_size must be a positive integer.")
         if cfg.numeric_tol < 0 or cfg.numeric_rtol < 0 or cfg.datetime_tol_seconds < 0:
             raise ValueError("tolerance values must be non-negative.")
         if cfg.top_n <= 0:
@@ -294,44 +297,54 @@ class ProcCompareEngine:
         mismatch_frames = []
         mismatch_cols_by_row: dict[int, list[str]] = {}
 
-        for col in compare_cols:
-            left_col = f"{col}_left"
-            right_col = f"{col}_right"
-            comp = self._compare_series(col, merged[left_col], merged[right_col], common_mask)
-            mismatch_mask = comp["mismatch"]
-            n_mismatch = int(mismatch_mask.sum())
-            if n_mismatch:
-                mismatch_indexes = merged.index[mismatch_mask]
-                for idx in mismatch_indexes:
+        common_values = common_mask.to_numpy(dtype=bool)
+        n_compared = int(common_values.sum())
+        block_size = int(self.config.compare_block_size)
+        for start in range(0, len(compare_cols), block_size):
+            block_cols = compare_cols[start : start + block_size]
+            comp = self._compare_column_block(merged, block_cols, common_values)
+            mismatch_matrix = comp["mismatch"]
+            one_null_matrix = comp["one_null"]
+            both_null_matrix = comp["both_null"]
+            diff_matrix = comp["diff"]
+            abs_diff_matrix = comp["abs_diff"]
+            left_values = comp["left_values"]
+            right_values = comp["right_values"]
+
+            for col_idx, col in enumerate(block_cols):
+                mismatch_positions = np.flatnonzero(mismatch_matrix[:, col_idx])
+                n_mismatch = int(len(mismatch_positions))
+                for idx in mismatch_positions:
                     mismatch_cols_by_row.setdefault(int(idx), []).append(col)
-                if self.config.detail_mode != "none":
-                    detail = merged.loc[mismatch_mask, key_cols].copy()
+                if n_mismatch and self.config.detail_mode != "none":
+                    detail = merged.iloc[mismatch_positions][key_cols].copy()
                     detail["column"] = col
-                    detail["left_value"] = merged.loc[mismatch_mask, left_col].to_numpy()
-                    detail["right_value"] = merged.loc[mismatch_mask, right_col].to_numpy()
-                    detail["diff"] = comp["diff"].loc[mismatch_mask].to_numpy()
-                    detail["abs_diff"] = comp["abs_diff"].loc[mismatch_mask].to_numpy()
+                    detail["left_value"] = left_values[mismatch_positions, col_idx]
+                    detail["right_value"] = right_values[mismatch_positions, col_idx]
+                    detail["diff"] = diff_matrix[mismatch_positions, col_idx]
+                    detail["abs_diff"] = abs_diff_matrix[mismatch_positions, col_idx]
                     mismatch_frames.append(detail)
 
-            n_compared = int(common_mask.sum())
-            n_one_null = int(comp["one_null"].sum())
-            n_both_null = int(comp["both_null"].sum())
-            n_equal = n_compared - n_mismatch
-            column_rows.append(
-                {
-                    "column": col,
-                    "n_compared": n_compared,
-                    "n_equal": n_equal,
-                    "n_mismatch": n_mismatch,
-                    "pct_mismatch": round(n_mismatch / n_compared * 100, 6) if n_compared else 0.0,
-                    "n_one_side_null": n_one_null,
-                    "n_both_null": n_both_null,
-                    "mean_diff": comp["diff"].mean(skipna=True),
-                    "max_abs_diff": comp["abs_diff"].max(skipna=True),
-                    "_diff_sum": comp["diff"].sum(skipna=True),
-                    "_diff_count": int(comp["diff"].notna().sum()),
-                }
-            )
+                diff_values = diff_matrix[:, col_idx]
+                abs_diff_values = abs_diff_matrix[:, col_idx]
+                diff_valid = np.isfinite(diff_values)
+                diff_sum = float(np.nansum(diff_values)) if diff_valid.any() else 0.0
+                diff_count = int(diff_valid.sum())
+                column_rows.append(
+                    {
+                        "column": col,
+                        "n_compared": n_compared,
+                        "n_equal": n_compared - n_mismatch,
+                        "n_mismatch": n_mismatch,
+                        "pct_mismatch": round(n_mismatch / n_compared * 100, 6) if n_compared else 0.0,
+                        "n_one_side_null": int(one_null_matrix[:, col_idx].sum()),
+                        "n_both_null": int(both_null_matrix[:, col_idx].sum()),
+                        "mean_diff": diff_sum / diff_count if diff_count else np.nan,
+                        "max_abs_diff": float(np.nanmax(abs_diff_values)) if np.isfinite(abs_diff_values).any() else np.nan,
+                        "_diff_sum": diff_sum,
+                        "_diff_count": diff_count,
+                    }
+                )
 
         for idx, cols in mismatch_cols_by_row.items():
             row_summary.loc[idx, "n_cell_mismatch"] = len(cols)
@@ -375,6 +388,146 @@ class ProcCompareEngine:
             duplicate_key_summary=duplicate_key_summary,
         )
 
+    def _compare_column_block(
+        self,
+        merged: pd.DataFrame,
+        columns: list[str],
+        common_mask: np.ndarray,
+    ) -> dict[str, np.ndarray]:
+        """Compare a bounded field block with vectorized type-specific kernels."""
+        n_rows = len(merged)
+        n_cols = len(columns)
+        left_series = [self._normalize_series_missing(merged[f"{col}_left"]) for col in columns]
+        right_series = [self._normalize_series_missing(merged[f"{col}_right"]) for col in columns]
+        left_values = np.column_stack([series.to_numpy(dtype=object) for series in left_series])
+        right_values = np.column_stack([series.to_numpy(dtype=object) for series in right_series])
+        mismatch = np.zeros((n_rows, n_cols), dtype=bool)
+        one_null = np.zeros((n_rows, n_cols), dtype=bool)
+        both_null = np.zeros((n_rows, n_cols), dtype=bool)
+        diff = np.full((n_rows, n_cols), np.nan, dtype=float)
+        abs_diff = np.full((n_rows, n_cols), np.nan, dtype=float)
+
+        datetime_idx = [
+            idx
+            for idx, col in enumerate(columns)
+            if self._should_compare_datetime(col, left_series[idx], right_series[idx])
+        ]
+        datetime_set = set(datetime_idx)
+        numeric_idx = [
+            idx
+            for idx in range(n_cols)
+            if idx not in datetime_set
+            and self._should_compare_numeric(left_series[idx], right_series[idx])
+        ]
+        numeric_set = set(numeric_idx)
+        string_idx = [
+            idx
+            for idx in range(n_cols)
+            if idx not in datetime_set and idx not in numeric_set
+        ]
+        common = common_mask[:, None]
+
+        if datetime_idx:
+            left_dt = np.column_stack(
+                [self._datetime_seconds(left_series[idx]) for idx in datetime_idx]
+            )
+            right_dt = np.column_stack(
+                [self._datetime_seconds(right_series[idx]) for idx in datetime_idx]
+            )
+            block_diff = left_dt - right_dt
+            block_abs = np.abs(block_diff)
+            left_null = ~np.isfinite(left_dt)
+            right_null = ~np.isfinite(right_dt)
+            tolerances = np.asarray(
+                [self._column_datetime_tol(columns[idx]) for idx in datetime_idx],
+                dtype=float,
+            )
+            block_one = (left_null ^ right_null) & common
+            block_both = left_null & right_null & common
+            block_mismatch = (block_abs > tolerances[None, :]) | block_one
+            if not self.config.both_null_equal:
+                block_mismatch |= block_both
+            block_mismatch &= common
+            diff[:, datetime_idx] = block_diff
+            abs_diff[:, datetime_idx] = block_abs
+            one_null[:, datetime_idx] = block_one
+            both_null[:, datetime_idx] = block_both
+            mismatch[:, datetime_idx] = block_mismatch
+
+        if numeric_idx:
+            left_num = np.column_stack(
+                [pd.to_numeric(left_series[idx], errors="coerce").to_numpy(dtype=float, na_value=np.nan) for idx in numeric_idx]
+            )
+            right_num = np.column_stack(
+                [pd.to_numeric(right_series[idx], errors="coerce").to_numpy(dtype=float, na_value=np.nan) for idx in numeric_idx]
+            )
+            block_diff = left_num - right_num
+            block_abs = np.abs(block_diff)
+            left_null = ~np.isfinite(left_num)
+            right_null = ~np.isfinite(right_num)
+            tolerances = np.asarray(
+                [self._column_numeric_tol(columns[idx]) for idx in numeric_idx],
+                dtype=float,
+            )
+            limit = tolerances[:, 0][None, :] + tolerances[:, 1][None, :] * np.abs(right_num)
+            block_one = (left_null ^ right_null) & common
+            block_both = left_null & right_null & common
+            block_mismatch = (block_abs > limit) | block_one
+            if not self.config.both_null_equal:
+                block_mismatch |= block_both
+            block_mismatch &= common
+            diff[:, numeric_idx] = block_diff
+            abs_diff[:, numeric_idx] = block_abs
+            one_null[:, numeric_idx] = block_one
+            both_null[:, numeric_idx] = block_both
+            mismatch[:, numeric_idx] = block_mismatch
+
+        if string_idx:
+            left_null = np.column_stack([left_series[idx].isna().to_numpy() for idx in string_idx])
+            right_null = np.column_stack([right_series[idx].isna().to_numpy() for idx in string_idx])
+            left_text = np.column_stack(
+                [left_series[idx].astype("string").fillna("").astype(str).to_numpy() for idx in string_idx]
+            )
+            right_text = np.column_stack(
+                [right_series[idx].astype("string").fillna("").astype(str).to_numpy() for idx in string_idx]
+            )
+            block_one = (left_null ^ right_null) & common
+            block_both = left_null & right_null & common
+            block_mismatch = (left_text != right_text) | block_one
+            if self.config.both_null_equal:
+                block_mismatch &= ~block_both
+            else:
+                block_mismatch |= block_both
+            block_mismatch &= common
+            one_null[:, string_idx] = block_one
+            both_null[:, string_idx] = block_both
+            mismatch[:, string_idx] = block_mismatch
+
+        return {
+            "mismatch": mismatch,
+            "one_null": one_null,
+            "both_null": both_null,
+            "diff": diff,
+            "abs_diff": abs_diff,
+            "left_values": left_values,
+            "right_values": right_values,
+        }
+
+    @staticmethod
+    def _datetime_seconds(series: pd.Series) -> np.ndarray:
+        values = ProcCompareEngine._to_datetime_series(series)
+        missing = values.isna().to_numpy()
+        result = values.astype("int64").to_numpy(dtype=float) / 1_000_000_000.0
+        result[missing] = np.nan
+        return result
+
+    @staticmethod
+    def _to_datetime_series(series: pd.Series) -> pd.Series:
+        try:
+            return pd.to_datetime(series, errors="coerce", format="mixed")
+        except (TypeError, ValueError):
+            return pd.to_datetime(series, errors="coerce")
+
     def _compare_series(
         self,
         col: str,
@@ -391,8 +544,8 @@ class ProcCompareEngine:
         abs_diff = pd.Series(np.nan, index=left.index, dtype="float64")
 
         if self._should_compare_datetime(col, left, right):
-            left_dt = pd.to_datetime(left, errors="coerce")
-            right_dt = pd.to_datetime(right, errors="coerce")
+            left_dt = self._to_datetime_series(left)
+            right_dt = self._to_datetime_series(right)
             diff = (left_dt - right_dt).dt.total_seconds()
             abs_diff = diff.abs()
             tol = self._column_datetime_tol(col)

{supermodelingfactory-0.5.9 → supermodelingfactory-0.6.0}/Modeling_Tool/Core/kDataFrame.py

@@ -57,13 +57,7 @@ class kSeries(Series):
         """
         Scale the model scores (for internt segment of MCI model)
         """
-        data = self.copy()
-        def app_func(x):
-            scores = x * 1.112
-            if (scores > 0.9999999):
-                return 0.9999999
-            return scores
-        return data.apply(app_func)
+        return (self * 1.112).clip(upper=0.9999999)
     
     def proc_freq(self) -> pd.DataFrame:
         """
@@ -128,8 +122,12 @@ class kDataFrame(DataFrame):
             data = self
         else:
             data = self.copy()
-        import re
-        data[vintage_colname] = data[by].apply(lambda x: re.search("\d{4}-\d{2}", x).group().replace('-', ''))
+        data[vintage_colname] = (
+            data[by]
+            .astype("string")
+            .str.extract(r"(\d{4}-\d{2})", expand=False)
+            .str.replace("-", "", regex=False)
+        )
         if inplace:
             self = data
         return data
@@ -165,13 +163,7 @@ class kDataFrame(DataFrame):
         """
         Scale the model scores (for internt segment of MCI model)
         """
-        data = self.copy()
-        def app_func(x):
-            scores = x * 1.112
-            if (scores > 0.9999999):
-                return 0.9999999
-            return scores
-        return data[pb_score].apply(app_func)
+        return (self[pb_score] * 1.112).clip(upper=0.9999999)
     
     def proc_freq(self, var: str):
         """

{supermodelingfactory-0.5.9 → supermodelingfactory-0.6.0}/Modeling_Tool/Core/utils.py

@@ -342,8 +342,12 @@ def convert_to_vintage(data, vintage_colname = 'VINTAGE', by = 'TRAN_TMS', retur
     >>> df = pd.DataFrame({'TRAN_TMS': ['2025-03-15 10:00:00', '2025-03-20 11:00:00']})
     >>> convert_to_vintage(df)
     """
-    import re
-    data[vintage_colname] = data[by].apply(lambda x: re.search("\\d{4}-\\d{2}", x).group().replace('-', ''))
+    data[vintage_colname] = (
+        data[by]
+        .astype("string")
+        .str.extract(r"(\d{4}-\d{2})", expand=False)
+        .str.replace("-", "", regex=False)
+    )
 
     if return_kDF:
         return kDataFrame(data)
@@ -1052,10 +1056,9 @@ def bool_to_str(data):
     >>> bool_to_str(df)
     """
     dfc = data.copy()
-    type_dict = data.dtypes.to_dict()
-    for k,v in type_dict.items():
-        if str(v).lower() == 'bool':
-            dfc = dfc.astype({k: str})
+    bool_cols = [col for col in data.columns if pd.api.types.is_bool_dtype(data[col])]
+    if bool_cols:
+        dfc[bool_cols] = dfc[bool_cols].astype(str)
     return dfc
 
 
@@ -1688,6 +1691,19 @@ def parse_sql_file(sql_path:str=None,
 
 
 
+def _calc_woe_iv_values(data, bad_pct, good_pct, fillwoe=True, filliv=True):
+    """Calculate WOE and IV together with one vectorized logarithm."""
+    if len(data[bad_pct]) > 0 and len(data[good_pct]) > 0:
+        bad_values = data[bad_pct]
+        good_values = data[good_pct]
+        woe = np.log(bad_values / good_values)
+        iv = (bad_values - good_values) * woe
+    else:
+        woe = 0 if fillwoe else np.nan
+        iv = 0 if filliv else np.nan
+    return woe, iv
+
+
 def calc_woe(data, bad_pct, good_pct, fillwoe=True):
     """
     计算WOE（Weight of Evidence）值。
@@ -1716,15 +1732,7 @@ def calc_woe(data, bad_pct, good_pct, fillwoe=True):
     >>> calc_woe(df, 'bad_pct', 'good_pct')
     """
     
-    if len(data[bad_pct]) > 0 and len(data[good_pct]) > 0:
-        woe = np.log(data[bad_pct] / data[good_pct])
-    else:
-        if fillwoe:
-            woe = 0
-        else:
-            woe = np.nan
-
-    return woe
+    return _calc_woe_iv_values(data, bad_pct, good_pct, fillwoe=fillwoe)[0]
 
 
 def calc_iv(data, bad_pct, good_pct, filliv=True):
@@ -1755,15 +1763,7 @@ def calc_iv(data, bad_pct, good_pct, filliv=True):
     >>> calc_iv(df, 'bad_pct', 'good_pct')
     """
     
-    if len(data[bad_pct]) > 0 and len(data[good_pct]) > 0:
-        iv = (data[bad_pct] - data[good_pct]) * np.log(data[bad_pct] / data[good_pct])
-    else:
-        if filliv:
-            iv = 0
-        else:
-            iv = np.nan
-
-    return iv
+    return _calc_woe_iv_values(data, bad_pct, good_pct, filliv=filliv)[1]
 
 
 def save_model(model, filename):
@@ -2413,9 +2413,13 @@ class WOEIVCalculator:
         -------
         tuple
         """
-        woe = self.calc_woe(fillna)
-        iv = self.calc_iv(fillna)
-        return woe, iv
+        return _calc_woe_iv_values(
+            self.data,
+            self.bad_pct_col,
+            self.good_pct_col,
+            fillwoe=fillna,
+            filliv=fillna,
+        )
 
 
 def get_feature_names(model, model_type=None):
@@ -2669,4 +2673,4 @@ def get_feature_names_batch(models, model_type=None):
     elif isinstance(models, list):
         return [get_feature_names(model, model_type=model_type) for model in models]
     else:
-        raise TypeError("models参数应为dict或list类型")
+        raise TypeError("models参数应为dict或list类型")

{supermodelingfactory-0.5.9 → supermodelingfactory-0.6.0}/Modeling_Tool/Eval/Model_Eval_Tool.py

@@ -88,51 +88,31 @@ def _get_gains_table_scr(data, score, dep, nbins = 10, precision = 5,
                                bin_colnames = ("_bin_num", "_bin_range"),
                                ascending = ascending)
     
-    def _compute_gains_tmp(group):
-        # 计算各项指标
-        min_val = group[score].min()
-        max_val = group[score].max()
-        # NOTE: pandas 2.3+ 起 groupby(...).apply(func) 传入 func 的 group
-        # DataFrame 不再包含分组列,不能再用 group['_bin_num']。
-        # len(group) 与原 .count() 数值等价(_bin_num 为分组主键,不为 NaN)。
-        n = len(group)
-        avg_score = group[score].mean()
-        unique_score = group[score].nunique()
-
-        # dep相关
-        dep_vals = group[dep]
-        
-        grand_total = res.shape[0]
-        grand_perf_cnt = res[dep].count()
-        grand_total_bad = res[dep].sum()
-        grand_total_good = (grand_perf_cnt - grand_total_bad)
-        
-        perf_cnt = dep_vals.count()
-        n_bad = (dep_vals == 1).sum()
-        n_good = (dep_vals == 0).sum()
-        avg_bad = n_bad / dep_vals.count() if dep_vals.count() > 0 else 0  # 避免除以0
-        avg_good = n_good / dep_vals.count() if dep_vals.count() > 0 else 0
-        
-        lift = avg_bad / res[dep].mean()
-        prop = n / grand_total
-
-        # 返回Series
-        return pd.Series({
-            'MIN': min_val,
-            'MAX': max_val,
-            'N': n,
-            'PROP': prop,
-            'PERF_CNT': perf_cnt,
-            'AVG_SCORE': avg_score,
-            'UNIQUE_SCORE': unique_score,
-            'AVG_BAD': avg_bad,
-            'AVG_GOOD': avg_good,
-            'N_BAD': n_bad,
-            'N_GOOD': n_good,
-            'LIFT': lift
-        })
-    
-    gains_table = res.groupby(["_bin_num", "_bin_range"], dropna=False).apply(_compute_gains_tmp)
+    res = res.copy()
+    res["_smf_bad_ind"] = res[dep].eq(1).astype(np.int64)
+    res["_smf_good_ind"] = res[dep].eq(0).astype(np.int64)
+    grouped = res.groupby(["_bin_num", "_bin_range"], dropna=False)
+    gains_table = grouped.agg(
+        MIN=(score, "min"),
+        MAX=(score, "max"),
+        N=(dep, "size"),
+        PERF_CNT=(dep, "count"),
+        AVG_SCORE=(score, "mean"),
+        UNIQUE_SCORE=(score, "nunique"),
+        N_BAD=("_smf_bad_ind", "sum"),
+        N_GOOD=("_smf_good_ind", "sum"),
+    )
+    perf_denom = gains_table["PERF_CNT"].replace(0, np.nan)
+    gains_table["PROP"] = gains_table["N"] / max(len(res), 1)
+    gains_table["AVG_BAD"] = (gains_table["N_BAD"] / perf_denom).fillna(0.0)
+    gains_table["AVG_GOOD"] = (gains_table["N_GOOD"] / perf_denom).fillna(0.0)
+    gains_table["LIFT"] = gains_table["AVG_BAD"] / res[dep].mean()
+    gains_table = gains_table[
+        [
+            "MIN", "MAX", "N", "PROP", "PERF_CNT", "AVG_SCORE",
+            "UNIQUE_SCORE", "AVG_BAD", "AVG_GOOD", "N_BAD", "N_GOOD", "LIFT",
+        ]
+    ]
 
     gains_table["BAD_PCT_IN_EACH_BIN"] = gains_table["N_BAD"] / gains_table["N_BAD"].sum()
     gains_table["GOOD_PCT_IN_EACH_BIN"] = gains_table["N_GOOD"] / gains_table["N_GOOD"].sum()
@@ -146,7 +126,7 @@ def _get_gains_table_scr(data, score, dep, nbins = 10, precision = 5,
 
 
     gains_table["TRUE_BAD_SHIFT"] = (gains_table['AVG_BAD'].shift(1) / gains_table['AVG_BAD'] - 1) if not ascending else (gains_table['AVG_BAD'] / gains_table['AVG_BAD'].shift(1) - 1) 
-    gains_table["RANK_ORDER_BUMP"] = gains_table["TRUE_BAD_SHIFT"].apply(lambda x: 1 if x < 0 else 0)
+    gains_table["RANK_ORDER_BUMP"] = gains_table["TRUE_BAD_SHIFT"].lt(0).astype(int)
     
     gains_table["WOE"] = calc_woe(data = gains_table, bad_pct = "BAD_PCT_IN_EACH_BIN", good_pct = "GOOD_PCT_IN_EACH_BIN")
     gains_table["IV"] = calc_iv(data = gains_table, bad_pct = "BAD_PCT_IN_EACH_BIN", good_pct = "GOOD_PCT_IN_EACH_BIN")