SuperModelingFactory 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. ExcelMaster/ExcelFormatTool.py +487 -0
  2. ExcelMaster/ExcelMaster.py +917 -0
  3. ExcelMaster/Template.py +525 -0
  4. ExcelMaster/Utility.py +233 -0
  5. ExcelMaster/__init__.py +3 -0
  6. Modeling_Tool/Core/Binning_Tool.py +1608 -0
  7. Modeling_Tool/Core/Binning_Tool.pyi +48 -0
  8. Modeling_Tool/Core/Check_DuckDB_Compatibility.py +835 -0
  9. Modeling_Tool/Core/Json_Data_Converter.py +621 -0
  10. Modeling_Tool/Core/Model_Registry_Tool.py +199 -0
  11. Modeling_Tool/Core/ODPS_Tool.py +284 -0
  12. Modeling_Tool/Core/Slope_Tool.py +356 -0
  13. Modeling_Tool/Core/Slope_Tool.pyi +31 -0
  14. Modeling_Tool/Core/XOR_Encryptor.py +207 -0
  15. Modeling_Tool/Core/XOR_Encryptor.pyi +26 -0
  16. Modeling_Tool/Core/__init__.py +99 -0
  17. Modeling_Tool/Core/kDataFrame.py +228 -0
  18. Modeling_Tool/Core/kDataFrame.pyi +43 -0
  19. Modeling_Tool/Core/sample_weight_utils.py +77 -0
  20. Modeling_Tool/Core/utils.py +2672 -0
  21. Modeling_Tool/Eval/Evaluation_Tool.py +1452 -0
  22. Modeling_Tool/Eval/Evaluation_Tool.pyi +47 -0
  23. Modeling_Tool/Eval/Model_Eval_Tool.py +2548 -0
  24. Modeling_Tool/Eval/Model_Eval_Tool.pyi +37 -0
  25. Modeling_Tool/Eval/__init__.py +54 -0
  26. Modeling_Tool/Eval/evaluate_model.py +2008 -0
  27. Modeling_Tool/Eval/evaluate_model.pyi +50 -0
  28. Modeling_Tool/Eval/weighted_eval_utils.py +326 -0
  29. Modeling_Tool/Explainability/Coalition_Structure.py +305 -0
  30. Modeling_Tool/Explainability/Model_Explainer.py +743 -0
  31. Modeling_Tool/Explainability/__init__.py +24 -0
  32. Modeling_Tool/Feature/Distribution_Tool.py +509 -0
  33. Modeling_Tool/Feature/Distribution_Tool.pyi +42 -0
  34. Modeling_Tool/Feature/Feature_Insights.py +762 -0
  35. Modeling_Tool/Feature/Feature_Insights.pyi +33 -0
  36. Modeling_Tool/Feature/PSI_Tool.py +1195 -0
  37. Modeling_Tool/Feature/PSI_Tool.pyi +29 -0
  38. Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py +355 -0
  39. Modeling_Tool/Feature/__init__.py +40 -0
  40. Modeling_Tool/Model/Backward_Tool.py +778 -0
  41. Modeling_Tool/Model/Backward_Tool.pyi +45 -0
  42. Modeling_Tool/Model/GBM_Search_Tool.py +251 -0
  43. Modeling_Tool/Model/GBM_Tool.py +1610 -0
  44. Modeling_Tool/Model/GBM_Tool.pyi +90 -0
  45. Modeling_Tool/Model/LRM_Tool.py +1198 -0
  46. Modeling_Tool/Model/LRM_Tool.pyi +47 -0
  47. Modeling_Tool/Model/__init__.py +61 -0
  48. Modeling_Tool/Sample/Distribution_Adaptation.py +131 -0
  49. Modeling_Tool/Sample/Distribution_Adaptation.pyi +30 -0
  50. Modeling_Tool/Sample/Reject_Infer.py +413 -0
  51. Modeling_Tool/Sample/Reject_Infer.pyi +43 -0
  52. Modeling_Tool/Sample/Sample_Split.py +520 -0
  53. Modeling_Tool/Sample/Sample_Split.pyi +43 -0
  54. Modeling_Tool/Sample/__init__.py +31 -0
  55. Modeling_Tool/UAT/UAT_Consistency_Checker.py +1180 -0
  56. Modeling_Tool/UAT/__init__.py +19 -0
  57. Modeling_Tool/WOE/WOE_Adapter.py +204 -0
  58. Modeling_Tool/WOE/WOE_Adapter.pyi +21 -0
  59. Modeling_Tool/WOE/WOE_Master.py +491 -0
  60. Modeling_Tool/WOE/WOE_Master.pyi +40 -0
  61. Modeling_Tool/WOE/WOE_Monotone_Binner.py +3324 -0
  62. Modeling_Tool/WOE/WOE_Monotone_Binner.pyi +71 -0
  63. Modeling_Tool/WOE/WOE_Plot_Tool.py +919 -0
  64. Modeling_Tool/WOE/WOE_Plot_Tool.pyi +46 -0
  65. Modeling_Tool/WOE/WOE_Report_Builder.py +214 -0
  66. Modeling_Tool/WOE/WOE_Report_Builder.pyi +24 -0
  67. Modeling_Tool/WOE/WOE_Tool.py +1094 -0
  68. Modeling_Tool/WOE/WOE_Tool.pyi +41 -0
  69. Modeling_Tool/WOE/__init__.py +86 -0
  70. Modeling_Tool/WOE/plot_woe_tool.py +290 -0
  71. Modeling_Tool/WOE/plot_woe_tool.pyi +25 -0
  72. Modeling_Tool/__init__.py +176 -0
  73. Report/Report_Tool.py +380 -0
  74. Report/__init__.py +3 -0
  75. supermodelingfactory-0.2.0.dist-info/METADATA +268 -0
  76. supermodelingfactory-0.2.0.dist-info/RECORD +79 -0
  77. supermodelingfactory-0.2.0.dist-info/WHEEL +5 -0
  78. supermodelingfactory-0.2.0.dist-info/licenses/LICENSE +102 -0
  79. supermodelingfactory-0.2.0.dist-info/top_level.txt +3 -0
@@ -0,0 +1,3324 @@
1
+ """
2
+ WOE_Monotone_Binner.py
3
+ ======================
4
+ 贪心单调 WOE 分箱器 — 可复用独立类
5
+
6
+ 用法示例:
7
+ from WOE_Monotone_Binner import MonotoneWOEBinner
8
+
9
+ binner = MonotoneWOEBinner(
10
+ feature_cols=["age", "income", "score"],
11
+ target_col="is_bad",
12
+ n_init_bins=20,
13
+ min_bin_size=0.03,
14
+ special_values=[-1, -100], # 这些值会单独成一箱
15
+ cate_feats=["city_grade", "edu_level"], # 已离散化的类别特征,直接算 WOE/IV,不做区间切分
16
+ )
17
+ binner.fit(train_df) # 训练拟合(贪心单调)
18
+ # 或开启卡方后合并
19
+ binner.fit(train_df, chi2_binning=True, chi2_p=0.95, chi2_init_size=2000)
20
+ # 类别特征:按坏率聚类合并坏率相近的类别(只作用于 cate_feats)
21
+ binner.refine_cate(max_bins=5)
22
+
23
+ # --- 或直接加载已有分箱结果,跳过 fit ---
24
+ bins_dict = binner.get_final_bins() # 获取分箱区间+WOE
25
+ edges_dict = binner.get_bin_edges() # 获取分箱边界列表(含 ±inf)
26
+ binner2 = MonotoneWOEBinner(feature_cols=[...], target_col="is_bad")
27
+ binner2.load_woe_bins(bins_dict) # 直接加载
28
+
29
+ df_woe = binner.apply_woe(test_df) # WOE转换
30
+ binner.export_woe_report("woe_report.xlsx") # 输出Excel报告(含图片Sheet)
31
+ binner.plot_woe_graph("woe_charts/") # 输出每个特征的图
32
+ binner.plot_woe_graph("woe_charts/", group_name="month", _df_for_group=df)
33
+
34
+ 依赖:
35
+ pip install pandas numpy matplotlib xlsxwriter pillow
36
+ (export_woe_report 通过 SuperModelingFactory 的 ExcelMaster 写出,
37
+ 底层依赖 xlsxwriter + pillow)
38
+ """
39
+
40
+ from __future__ import annotations
41
+
42
+ import logging
43
+ import io
44
+ import os
45
+ import math
46
+ import warnings
47
+ import textwrap
48
+ import tempfile
49
+ from typing import List, Optional, Dict, Any, Union
50
+ import copy
51
+ from concurrent.futures import ProcessPoolExecutor, as_completed
52
+
53
+ logger = logging.getLogger(__name__)
54
+
55
+ import numpy as np
56
+ import pandas as pd
57
+ import matplotlib
58
+ import matplotlib.pyplot as plt
59
+ import matplotlib.ticker as mticker
60
+
61
+ matplotlib.use("Agg")
62
+ matplotlib.rcParams["font.family"] = "DejaVu Sans"
63
+ warnings.filterwarnings("ignore")
64
+
65
+
66
+ # ══════════════════════════════════════════════════════════════════════════════
67
+ # 特殊值标签辅助
68
+ # ══════════════════════════════════════════════════════════════════════════════
69
+
70
+ _SPECIAL_BIN_PREFIX = "__special__" # 内部用于标记特殊箱的前缀
71
+ _CATE_GROUP_SEP = " | " # refine_cate 合并多个类别后,bin_label 的成员分隔符
72
+
73
+ def _sv_label(sv) -> str:
74
+ """将特殊值转为分箱标签,nan → '[Missing]',其余 → '[sv=xxx]'"""
75
+ if sv is None or (isinstance(sv, float) and math.isnan(sv)):
76
+ return "[Missing]"
77
+ return f"[sv={sv}]"
78
+
79
+
80
+ # ══════════════════════════════════════════════════════════════════════════════
81
+ # 多进程辅助函数(必须在类外定义,保证 pickle 兼容)
82
+ # ══════════════════════════════════════════════════════════════════════════════
83
+
84
+ def _chunk_fit_worker(args):
85
+ """fit() 并行 worker:对一批特征执行贪心单调 WOE 分箱。"""
86
+ binner_lite, df, chunk_feats, chi2_binning, chi2_p, chi2_init_size = args
87
+ ok, err = {}, {}
88
+ for feat in chunk_feats:
89
+ try:
90
+ ok[feat] = binner_lite._greedy_fit_one(
91
+ df, feat, chi2_binning, chi2_p, chi2_init_size
92
+ )
93
+ except Exception as exc:
94
+ import traceback
95
+ err[feat] = (exc, traceback.format_exc())
96
+ return ok, err
97
+
98
+
99
+ def _chunk_chi2_worker(args):
100
+ """refine_chi2() 并行 worker:对一批特征执行卡方后合并。"""
101
+ binner_lite, df, chunk_feats, edges_map, sv_iv_map, chi2_p, chi2_init_size = args
102
+ ok, err = {}, {}
103
+ for feat in chunk_feats:
104
+ edges = edges_map.get(feat, [])
105
+ if not edges:
106
+ ok[feat] = None # 标记为跳过(仅 1 箱)
107
+ continue
108
+ try:
109
+ df_normal, _ = binner_lite._split_special(df, feat)
110
+ new_edges = binner_lite._chi2_merge_one(
111
+ df_normal, feat, edges, chi2_p, chi2_init_size
112
+ )
113
+ wt, iv = binner_lite._compute_woe_table(df_normal, feat, new_edges)
114
+ woes = wt.sort_values("bin")["woe"].values
115
+ sv_iv = sv_iv_map.get(feat, 0.0)
116
+ ok[feat] = dict(
117
+ edges = new_edges,
118
+ woe_table = wt,
119
+ iv = round(iv + sv_iv, 6),
120
+ is_monotonic = MonotoneWOEBinner._is_monotone(woes),
121
+ n_bins = len(wt),
122
+ )
123
+ except Exception as exc:
124
+ import traceback
125
+ err[feat] = (exc, traceback.format_exc())
126
+ return ok, err
127
+
128
+
129
+ # ══════════════════════════════════════════════════════════════════════════════
130
+ # 主类
131
+ # ══════════════════════════════════════════════════════════════════════════════
132
+
133
+ class MonotoneWOEBinner:
134
+ """
135
+ 贪心合并单调 WOE 分箱器(支持特殊值单独分箱 + 可选卡方后合并)。
136
+
137
+ Parameters
138
+ ----------
139
+ feature_cols : 需要分箱的数值特征列名列表
140
+ target_col : 二分类目标变量列名(0=好,1=坏)
141
+ n_init_bins : 初始等频分箱数,默认 20
142
+ min_bin_size : 每箱最小样本占比,默认 0.03(3%)
143
+ min_n_bins : 最终分箱数下限(不含特殊值箱),默认 2
144
+ eps : 防止 log(0) 的微小量,默认 1e-6
145
+ missing_woe : 缺失值(NaN)对应的 WOE 填充值,默认 0.0(中性)
146
+ 注意:若 nan 已在 special_values 中则会独立计算 WOE,
147
+ 此参数仅对未列入 special_values 的 NaN 生效。
148
+ special_values : 需要单独成箱的特殊值列表,如 [-1, -100, float('nan')]
149
+ 这些值会在 fit 时先被剔除,对剩余数据做单调分箱;
150
+ 最终在汇总表中单独追加为独立箱,WOE 独立计算。
151
+ 支持 nan / None / float('nan') 表示"缺失值单独分箱"。
152
+ 注意:仅作用于 feature_cols(数值特征),不影响 cate_feats。
153
+ cate_feats : 已离散化的类别(离散)特征列名列表,默认 None。
154
+ 这些特征**不做任何区间切分**——每个不同的取值直接作为一箱,
155
+ 直接计算其 WOE / IV,箱标签即类别取值本身。
156
+ 缺失值(NaN)若存在则单独归为 [Missing] 箱(独立计算 WOE)。
157
+ 与 feature_cols 互斥(同名时按 cate_feats 处理);卡方/决策树
158
+ 后合并(refine_chi2 / refine_dtree)对类别特征自动跳过。
159
+ 可用 refine_cate() 按坏率(bad rate)聚类合并坏率相近的类别。
160
+ bin_label_decimals : 分箱区间边界值的小数点保留位数,默认 None(使用 .8g
161
+ 格式,最多 8 位有效数字)。设为正整数 N 时,边界值固定
162
+ 显示 N 位小数(:.Nf),例如 N=2 时 1234.5678 → 1234.57。
163
+ 注意:较低的精度会使 load_woe_bins(get_final_bins()) 的
164
+ round-trip 边界稍有误差,但通常可忽略。
165
+
166
+ fit() 参数(传入 fit() 方法,不在 __init__ 中设置)
167
+ -------------------------------------------------------
168
+ chi2_binning : 是否在贪心单调分箱后再做卡方后合并,默认 False。
169
+ True 时:以贪心结果为起点,迭代合并卡方值最小的相邻箱对,
170
+ 直到所有相邻对的卡方检验 p 值均 < (1 - chi2_p),
171
+ 合并过程中严格保持 WOE 单调(不满足则跳过该对)。
172
+ chi2_p : 卡方检验置信度阈值,默认 0.99。相邻箱 p > (1-chi2_p)
173
+ 时认为两箱分布无显著差异,可以合并。
174
+ chi2_init_size : 卡方计算时的全局 stratified 采样上限,默认 1000。
175
+ 若普通行数 > chi2_init_size,则按 target 比例分层
176
+ 抽样后再计算卡方,避免大数据集下卡方值虚高。
177
+ """
178
+
179
+ def __init__(
180
+ self,
181
+ feature_cols: List[str],
182
+ target_col: str,
183
+ n_init_bins: int = 20,
184
+ min_bin_size: float = 0.03,
185
+ min_n_bins: int = 2,
186
+ eps: float = 1e-6,
187
+ missing_woe: float = 0.0,
188
+ special_values: Optional[List] = None,
189
+ cate_feats: Optional[List[str]] = None,
190
+ bin_label_decimals: Optional[int] = None,
191
+ ):
192
+ self.feature_cols = list(feature_cols)
193
+ self.target_col = target_col
194
+ self.n_init_bins = n_init_bins
195
+ self.min_bin_size = min_bin_size
196
+ self.min_n_bins = min_n_bins
197
+ self.eps = eps
198
+ self.missing_woe = missing_woe
199
+ self.special_values = list(special_values) if special_values else []
200
+ self.cate_feats = list(cate_feats) if cate_feats else []
201
+ self._cate_feats_set = set(self.cate_feats)
202
+ self.bin_label_decimals = bin_label_decimals
203
+
204
+ # 判断 special_values 中是否包含 nan(缺失值独立分箱)
205
+ self._sv_has_nan = any(
206
+ v is None or (isinstance(v, float) and math.isnan(v))
207
+ for v in self.special_values
208
+ )
209
+ # 非 nan 特殊值列表
210
+ self._sv_numeric = [
211
+ v for v in self.special_values
212
+ if not (v is None or (isinstance(v, float) and math.isnan(v)))
213
+ ]
214
+
215
+ # 拟合结果,fit() 后填充
216
+ # {feat: {
217
+ # "edges" : list of float (普通箱切割点),
218
+ # "woe_table" : pd.DataFrame (普通箱 WOE 明细,bin 列 0-based),
219
+ # "sv_table" : pd.DataFrame (特殊值箱 WOE 明细,每行一个特殊值),
220
+ # "iv" : float (含特殊值箱的总 IV),
221
+ # "is_monotonic": bool (仅对普通箱),
222
+ # "n_bins" : int (普通箱数),
223
+ # --- 类别特征(cate_feats)额外字段 ---
224
+ # "is_categorical": True,
225
+ # "categories" : list (类别取值,按自然顺序;woe_table 每行一个类别,
226
+ # 含 cat_value/bin_label 列),
227
+ # }}
228
+ self._results: Dict[str, Any] = {}
229
+ self._is_fitted = False
230
+
231
+ # ─────────────────────────────────────────────────────────────────
232
+ # 内部工具
233
+ # ─────────────────────────────────────────────────────────────────
234
+
235
+ def _split_special(self, df: pd.DataFrame, feat: str):
236
+ """
237
+ 将 df 拆分为:普通行(用于单调分箱)+ 各特殊值行。
238
+
239
+ Returns
240
+ -------
241
+ df_normal : 剔除了特殊值和(视情况)NaN 的普通行
242
+ sv_groups : {sv -> sub_df},每个特殊值对应的行子集
243
+ """
244
+ mask_normal = pd.Series(True, index=df.index)
245
+
246
+ sv_groups: Dict[Any, pd.DataFrame] = {}
247
+
248
+ # NaN 单独分箱
249
+ if self._sv_has_nan:
250
+ nan_mask = df[feat].isna()
251
+ sv_groups[float("nan")] = df[nan_mask]
252
+ mask_normal &= ~nan_mask
253
+ else:
254
+ # NaN 不单独分箱 → 普通分箱时直接 dropna(_compute_woe_table 内部处理)
255
+ mask_normal &= df[feat].notna()
256
+
257
+ # 数值特殊值
258
+ for sv in self._sv_numeric:
259
+ sv_mask = (df[feat] == sv)
260
+ sv_groups[sv] = df[sv_mask]
261
+ mask_normal &= ~sv_mask
262
+
263
+ df_normal = df[mask_normal]
264
+ return df_normal, sv_groups
265
+
266
+ # ── 分组绘图(by-group)用的分箱辅助:数值=edges,类别=取值映射 ──────────────
267
+
268
+ @staticmethod
269
+ def _cat_to_bin_map(vr: Dict) -> Dict:
270
+ """类别特征:构建 {类别取值 -> 普通箱索引} 映射
271
+ (含 refine_cate 合并后的成员展开)。"""
272
+ wt = vr["woe_table"]
273
+ has_members = "cat_members" in wt.columns
274
+ has_value = "cat_value" in wt.columns
275
+ m: Dict = {}
276
+ for _, r in wt.iterrows():
277
+ if has_members and isinstance(r["cat_members"], (list, tuple)):
278
+ members = r["cat_members"]
279
+ elif has_value and not pd.isna(r["cat_value"]):
280
+ members = [r["cat_value"]]
281
+ else:
282
+ members = []
283
+ for cv in members:
284
+ if not pd.isna(cv):
285
+ m[cv] = int(r["bin"])
286
+ return m
287
+
288
+ def _split_special_for_plot(self, df: pd.DataFrame, feat: str, vr: Dict):
289
+ """分组绘图用的特殊值拆分。
290
+ 数值特征:沿用 _split_special(按 special_values 拆分)。
291
+ 类别特征:仅把 NaN 拆为 [Missing](与 _categorical_fit_one 口径一致),
292
+ 数值不视为特殊值。
293
+ """
294
+ if vr.get("is_categorical"):
295
+ nan_mask = df[feat].isna()
296
+ sv_groups: Dict[Any, pd.DataFrame] = {}
297
+ if bool(nan_mask.any()):
298
+ sv_groups[float("nan")] = df[nan_mask]
299
+ return df[~nan_mask], sv_groups
300
+ return self._split_special(df, feat)
301
+
302
+ def _assign_normal_bins(self, sub: pd.DataFrame, feat: str, vr: Dict,
303
+ fitted_edges: list) -> pd.Series:
304
+ """把普通行映射到普通箱索引(NaN = 未命中,不计入任何箱)。
305
+ 数值特征:pd.cut on edges;类别特征:按取值查 cat_to_bin。
306
+ """
307
+ if len(sub) == 0:
308
+ return pd.Series([], dtype=float, index=sub.index)
309
+ if vr.get("is_categorical"):
310
+ return sub[feat].map(self._cat_to_bin_map(vr))
311
+ if len(fitted_edges) > 0:
312
+ return pd.cut(sub[feat], bins=[-np.inf] + list(fitted_edges) + [np.inf],
313
+ labels=False, right=True)
314
+ return pd.Series(0, index=sub.index)
315
+
316
+ def _compute_woe_single_bin(
317
+ self, sub: pd.DataFrame, total_bad: float, total_good: float
318
+ ) -> Dict[str, float]:
319
+ """计算某子集的 bad/good/woe/iv 等统计量。"""
320
+ eps = self.eps
321
+ n = len(sub)
322
+ bad = float(sub[self.target_col].sum())
323
+ good = float((sub[self.target_col] == 0).sum())
324
+ bad_rate = bad / (bad + good) if (bad + good) > 0 else 0.0
325
+ pct_bad = bad / (total_bad + eps)
326
+ pct_good = good / (total_good + eps)
327
+ woe = math.log((pct_bad + eps) / (pct_good + eps))
328
+ iv = (pct_bad - pct_good) * woe
329
+ return dict(n=n, bad=int(bad), good=int(good),
330
+ bad_rate=bad_rate, pct_bad=pct_bad,
331
+ pct_good=pct_good, woe=woe, iv=iv)
332
+
333
+ def _compute_woe_table(
334
+ self, df: pd.DataFrame, feat: str, edges: list
335
+ ) -> tuple:
336
+ """给定分割点 edges,计算普通箱的 WOE 明细表和 IV。"""
337
+ sub = df[[feat, self.target_col]].dropna(subset=[feat])
338
+ if len(sub) == 0 or len(edges) == 0:
339
+ bins = pd.Series([0] * len(sub), index=sub.index)
340
+ else:
341
+ bins = pd.cut(
342
+ sub[feat],
343
+ bins=[-np.inf] + list(edges) + [np.inf],
344
+ labels=False, right=True,
345
+ )
346
+ sub = sub.copy()
347
+ sub["_bin"] = bins
348
+
349
+ total_bad = float(sub[self.target_col].sum())
350
+ total_good = float((sub[self.target_col] == 0).sum())
351
+ eps = self.eps
352
+ records = []
353
+ for b in sorted(sub["_bin"].dropna().unique()):
354
+ grp = sub[sub["_bin"] == b]
355
+ n = len(grp)
356
+ bad = float(grp[self.target_col].sum())
357
+ good = float((grp[self.target_col] == 0).sum())
358
+ pct_bad = bad / (total_bad + eps)
359
+ pct_good = good / (total_good + eps)
360
+ woe = math.log((pct_bad + eps) / (pct_good + eps))
361
+ iv = (pct_bad - pct_good) * woe
362
+ records.append(dict(
363
+ bin=int(b), n=n, bad=int(bad), good=int(good),
364
+ bad_rate=bad / (bad + good) if (bad + good) > 0 else 0.0,
365
+ pct_bad=pct_bad, pct_good=pct_good, woe=woe, iv=iv,
366
+ ))
367
+ wt = pd.DataFrame(records)
368
+ if len(wt) == 0:
369
+ wt = pd.DataFrame(columns=["bin", "n", "bad", "good", "bad_rate",
370
+ "pct_bad", "pct_good", "woe", "iv"])
371
+ return wt, 0.0
372
+ return wt, float(wt["iv"].sum())
373
+
374
+ def _compute_sv_table(
375
+ self, sv_groups: Dict, total_bad: float, total_good: float
376
+ ) -> pd.DataFrame:
377
+ """
378
+ 计算所有特殊值的独立 WOE 明细,返回 DataFrame。
379
+ 每行对应一个特殊值,bin_label 为 '[sv=xxx]' 或 '[Missing]'。
380
+ """
381
+ records = []
382
+ for sv, sv_df in sv_groups.items():
383
+ if len(sv_df) == 0:
384
+ continue
385
+ stats = self._compute_woe_single_bin(sv_df, total_bad, total_good)
386
+ stats["bin_label"] = _sv_label(sv)
387
+ stats["sv"] = sv
388
+ records.append(stats)
389
+ return pd.DataFrame(records) if records else pd.DataFrame()
390
+
391
+ @staticmethod
392
+ def _is_monotone(woe_values: np.ndarray) -> bool:
393
+ if len(woe_values) <= 1:
394
+ return True
395
+ inc = all(woe_values[i] <= woe_values[i+1] for i in range(len(woe_values)-1))
396
+ dec = all(woe_values[i] >= woe_values[i+1] for i in range(len(woe_values)-1))
397
+ return inc or dec
398
+
399
+ def _chi2_merge_one(
400
+ self,
401
+ df_normal: pd.DataFrame,
402
+ feat: str,
403
+ edges: list,
404
+ chi2_p: float,
405
+ chi2_init_size: int,
406
+ ) -> list:
407
+ """
408
+ 在贪心单调分箱结果的基础上,做卡方后合并。
409
+
410
+ 算法
411
+ ----
412
+ 1. 若普通行数 > chi2_init_size,按 target 比例分层采样 chi2_init_size 行
413
+ 用于卡方统计(不改动 edges 本身的全量评估)
414
+ 2. 迭代:计算所有相邻箱对的卡方 p 值
415
+ a. 若所有相邻对 p < alpha (= 1 - chi2_p),停止
416
+ b. 否则选 p 值最大(最不显著)的相邻对尝试合并
417
+ c. 合并后检验 WOE 是否仍单调(基于全量普通行)
418
+ - 单调:接受合并,更新 edges
419
+ - 不单调:标记该对为「禁止合并」,跳过后继续
420
+ d. 若所有可合并对均被禁止,停止
421
+ 3. 若合并后 edges 剩余箱数 < min_n_bins,停止
422
+
423
+ Parameters
424
+ ----------
425
+ df_normal : 已剔除特殊值的普通行 DataFrame
426
+ feat : 特征列名
427
+ edges : 贪心分箱的切割点列表(in-place 不修改,返回新列表)
428
+ chi2_p : 置信度,如 0.99;alpha = 1 - chi2_p
429
+ chi2_init_size : 采样上限
430
+
431
+ Returns
432
+ -------
433
+ new_edges : 卡方合并后的切割点列表
434
+ """
435
+ from scipy.stats import chi2 as chi2_dist
436
+
437
+ alpha = 1.0 - chi2_p
438
+ edges = list(edges) # 不改动原始列表
439
+
440
+ # ── 采样(stratified by target)──
441
+ sub_full = df_normal[[feat, self.target_col]].dropna(subset=[feat])
442
+ n_full = len(sub_full)
443
+ if n_full > chi2_init_size:
444
+ # 按 target 分层采样(用 index 采样避免 groupby 把 target 列变成索引)
445
+ sampled_idx = []
446
+ for tval, grp in sub_full.groupby(self.target_col, sort=False):
447
+ n_take = max(1, int(round(chi2_init_size * len(grp) / n_full)))
448
+ n_take = min(n_take, len(grp))
449
+ sampled_idx.extend(
450
+ grp.sample(n=n_take, random_state=42).index.tolist()
451
+ )
452
+ sampled = sub_full.loc[sampled_idx]
453
+ # 若分层采样行数不足(极不平衡),补全到 chi2_init_size
454
+ if len(sampled) < chi2_init_size:
455
+ remain = sub_full.drop(sampled.index)
456
+ n_extra = min(chi2_init_size - len(sampled), len(remain))
457
+ if n_extra > 0:
458
+ extra = remain.sample(n_extra, random_state=42)
459
+ sampled = pd.concat([sampled, extra])
460
+ df_chi2 = sampled
461
+ else:
462
+ df_chi2 = sub_full
463
+
464
+ def _bin_series(df_slice, edge_list):
465
+ """将 df_slice[feat] 按 edge_list 分箱,返回 bin 编号 Series。"""
466
+ if not edge_list:
467
+ return pd.Series(0, index=df_slice.index)
468
+ return pd.cut(
469
+ df_slice[feat],
470
+ bins=[-np.inf] + edge_list + [np.inf],
471
+ labels=False, right=True,
472
+ )
473
+
474
+ def _chi2_pval_pair(df_slice, edge_list, bi):
475
+ """
476
+ 计算 bi 和 bi+1 箱合并前的卡方 p 值(2x2 列联表)。
477
+ 返回 p_value;若某箱样本为 0 则返回 1.0(默认可合并)。
478
+ """
479
+ bins = _bin_series(df_slice, edge_list)
480
+ df_c = df_slice.copy()
481
+ df_c["_bin"] = bins
482
+
483
+ grp_i = df_c[df_c["_bin"] == bi]
484
+ grp_j = df_c[df_c["_bin"] == bi + 1]
485
+
486
+ if len(grp_i) == 0 or len(grp_j) == 0:
487
+ return 1.0 # 空箱,可合并
488
+
489
+ bad_i = float((grp_i[self.target_col] == 1).sum())
490
+ good_i = float((grp_i[self.target_col] == 0).sum())
491
+ bad_j = float((grp_j[self.target_col] == 1).sum())
492
+ good_j = float((grp_j[self.target_col] == 0).sum())
493
+
494
+ # 2×2 列联表: [[bad_i, good_i], [bad_j, good_j]]
495
+ table = np.array([[bad_i, good_i], [bad_j, good_j]])
496
+
497
+ # 若任一格期望值为 0,退化:返回 p=0(不合并)
498
+ row_sum = table.sum(axis=1, keepdims=True)
499
+ col_sum = table.sum(axis=0, keepdims=True)
500
+ total = table.sum()
501
+ if total == 0:
502
+ return 1.0
503
+ expected = row_sum * col_sum / total
504
+ if np.any(expected == 0):
505
+ return 0.0
506
+
507
+ # 手动计算卡方(避免 scipy 依赖问题时的稳定性)
508
+ chi2_val = float(np.sum((table - expected) ** 2 / expected))
509
+ # chi2 分布自由度 = (行-1)*(列-1) = 1
510
+ p_val = 1.0 - chi2_dist.cdf(chi2_val, df=1)
511
+ return p_val
512
+
513
+ forbidden = set() # 被禁止合并的 (bi) 索引集合(相对当前 edges)
514
+
515
+ for _iter in range(200):
516
+ n_bins = len(edges) + 1
517
+ if n_bins <= self.min_n_bins:
518
+ break
519
+
520
+ # 计算所有相邻对的 p 值
521
+ pvals = []
522
+ for bi in range(n_bins - 1):
523
+ p = _chi2_pval_pair(df_chi2, edges, bi)
524
+ pvals.append((bi, p))
525
+
526
+ # 过滤已禁止 & p < alpha 的对
527
+ candidates = [(bi, p) for bi, p in pvals
528
+ if p >= alpha and bi not in forbidden]
529
+
530
+ if not candidates:
531
+ break # 所有相邻对都显著(或被禁),停止
532
+
533
+ # 选 p 值最大(最不显著)的对尝试合并
534
+ best_bi, best_p = max(candidates, key=lambda x: x[1])
535
+
536
+ # 试合并:移除 edges[best_bi]
537
+ trial_edges = [e for i, e in enumerate(edges) if i != best_bi]
538
+
539
+ # 检验合并后 WOE 是否仍单调(基于全量普通行)
540
+ wt_trial, _ = self._compute_woe_table(sub_full, feat, trial_edges)
541
+ woes_trial = wt_trial.sort_values("bin")["woe"].values
542
+
543
+ if self._is_monotone(woes_trial):
544
+ # 接受合并
545
+ edges = trial_edges
546
+ # forbidden 索引需要更新(被合并箱之后的索引都 -1)
547
+ forbidden = {bi - (1 if bi > best_bi else 0)
548
+ for bi in forbidden if bi != best_bi}
549
+ else:
550
+ # 拒绝:禁止该对后继续
551
+ forbidden.add(best_bi)
552
+
553
+ return edges
554
+
555
+ def _greedy_fit_one(
556
+ self,
557
+ df: pd.DataFrame,
558
+ feat: str,
559
+ chi2_binning: bool = False,
560
+ chi2_p: float = 0.99,
561
+ chi2_init_size: int = 1000,
562
+ ) -> Dict[str, Any]:
563
+ """
564
+ 对单个特征进行贪心单调 WOE 分箱(+ 可选卡方后合并),支持特殊值剔除。
565
+ 类别特征(cate_feats)走 _categorical_fit_one,不做区间切分。
566
+ """
567
+ # 0. 类别特征:每个取值直接成箱,直接算 WOE/IV,不做区间切分
568
+ if feat in self._cate_feats_set:
569
+ return self._categorical_fit_one(df, feat)
570
+
571
+ # 1. 剔除特殊值,获取普通行和特殊值子集
572
+ df_normal, sv_groups = self._split_special(df, feat)
573
+
574
+ # 全量总体(含特殊值)的 bad/good,用于计算 pct_bad/pct_good
575
+ total_bad = float(df[self.target_col].sum())
576
+ total_good = float((df[self.target_col] == 0).sum())
577
+
578
+ # 2. 对普通行进行贪心单调分箱
579
+ col = df_normal[feat].dropna()
580
+ n = len(col)
581
+
582
+ if n < 10:
583
+ # 普通行太少,退化为单箱
584
+ wt, iv = self._compute_woe_table(df_normal, feat, [])
585
+ sv_table = self._compute_sv_table(sv_groups, total_bad, total_good)
586
+ sv_iv = float(sv_table["iv"].sum()) if len(sv_table) > 0 else 0.0
587
+ return dict(edges=[], woe_table=wt, sv_table=sv_table,
588
+ iv=round(iv + sv_iv, 6),
589
+ is_monotonic=True, n_bins=max(len(wt), 1))
590
+
591
+ min_n = max(int(n * self.min_bin_size), 5)
592
+
593
+ # 等频初始分位数边界
594
+ quantiles = np.linspace(0, 100, self.n_init_bins + 1)
595
+ raw_edges = np.unique(np.nanpercentile(col.values, quantiles[1:-1]))
596
+
597
+ if len(raw_edges) == 0:
598
+ wt, iv = self._compute_woe_table(df_normal, feat, [])
599
+ sv_table = self._compute_sv_table(sv_groups, total_bad, total_good)
600
+ sv_iv = float(sv_table["iv"].sum()) if len(sv_table) > 0 else 0.0
601
+ return dict(edges=[], woe_table=wt, sv_table=sv_table,
602
+ iv=round(iv + sv_iv, 6),
603
+ is_monotonic=True, n_bins=len(wt))
604
+
605
+ edges = list(raw_edges)
606
+ wt, iv = self._compute_woe_table(df_normal, feat, edges)
607
+
608
+ # 贪心合并
609
+ for _ in range(100):
610
+ woes = wt.sort_values("bin")["woe"].values
611
+ if self._is_monotone(woes):
612
+ break
613
+ if len(edges) < self.min_n_bins - 1:
614
+ break
615
+
616
+ inc_vio = [(i, abs(woes[i+1] - woes[i]))
617
+ for i in range(len(woes)-1) if woes[i] >= woes[i+1]]
618
+ dec_vio = [(i, abs(woes[i+1] - woes[i]))
619
+ for i in range(len(woes)-1) if woes[i] <= woes[i+1]]
620
+ violations = inc_vio if len(inc_vio) <= len(dec_vio) else dec_vio
621
+ if not violations:
622
+ break
623
+
624
+ merge_idx = min(violations, key=lambda x: x[1])[0]
625
+ if merge_idx < len(edges):
626
+ edges.pop(merge_idx)
627
+ wt, iv = self._compute_woe_table(df_normal, feat, edges)
628
+
629
+ woes_final = wt.sort_values("bin")["woe"].values
630
+
631
+ # ── 卡方后合并(可选)──────────────────────────────────────────────
632
+ if chi2_binning and len(edges) >= 1:
633
+ edges = self._chi2_merge_one(
634
+ df_normal, feat, edges, chi2_p, chi2_init_size
635
+ )
636
+ wt, iv = self._compute_woe_table(df_normal, feat, edges)
637
+ woes_final = wt.sort_values("bin")["woe"].values
638
+
639
+ # 计算特殊值箱
640
+ sv_table = self._compute_sv_table(sv_groups, total_bad, total_good)
641
+ sv_iv = float(sv_table["iv"].sum()) if len(sv_table) > 0 else 0.0
642
+
643
+ return dict(
644
+ edges = edges,
645
+ woe_table = wt,
646
+ sv_table = sv_table,
647
+ iv = round(iv + sv_iv, 6),
648
+ is_monotonic = self._is_monotone(woes_final),
649
+ n_bins = len(wt),
650
+ )
651
+
652
+ @staticmethod
653
+ def _sort_categories(cats: list) -> list:
654
+ """对类别取值做稳健排序(同类型自然排序;混合类型回退按字符串排序)。"""
655
+ try:
656
+ return sorted(cats)
657
+ except TypeError:
658
+ return sorted(cats, key=lambda x: str(x))
659
+
660
+ def _categorical_fit_one(self, df: pd.DataFrame, feat: str) -> Dict[str, Any]:
661
+ """
662
+ 对单个**已离散化的类别特征**直接计算 WOE/IV,不做任何区间切分。
663
+
664
+ 每个不同的取值各自成一箱,箱标签即取值本身;缺失值(NaN)若存在则单独
665
+ 归为 [Missing] 箱(追加进 sv_table,独立计算 WOE)。
666
+
667
+ WOE/IV 口径与数值特征保持一致:
668
+ - 普通类别箱:pct_bad/pct_good 以**非缺失**样本的 bad/good 为分母
669
+ - [Missing] 箱:以**全量**样本的 bad/good 为分母(与 _compute_sv_table 一致)
670
+ - 总 IV = 各类别箱 IV 之和 + [Missing] 箱 IV
671
+ """
672
+ sub = df[[feat, self.target_col]]
673
+
674
+ nan_mask = sub[feat].isna()
675
+ normal = sub[~nan_mask]
676
+
677
+ # 普通类别箱口径:非缺失样本的总 bad/good
678
+ norm_total_bad = float(normal[self.target_col].sum())
679
+ norm_total_good = float((normal[self.target_col] == 0).sum())
680
+ # [Missing] 箱口径:全量样本的总 bad/good
681
+ full_total_bad = float(sub[self.target_col].sum())
682
+ full_total_good = float((sub[self.target_col] == 0).sum())
683
+
684
+ cats = self._sort_categories(list(normal[feat].dropna().unique()))
685
+
686
+ records = []
687
+ for i, cat in enumerate(cats):
688
+ grp = normal[normal[feat] == cat]
689
+ stats = self._compute_woe_single_bin(grp, norm_total_bad, norm_total_good)
690
+ stats.update(bin=i, cat_value=cat, bin_label=str(cat))
691
+ records.append(stats)
692
+
693
+ cols = ["bin", "cat_value", "bin_label", "n", "bad", "good",
694
+ "bad_rate", "pct_bad", "pct_good", "woe", "iv"]
695
+ woe_table = pd.DataFrame(records, columns=cols) if records \
696
+ else pd.DataFrame(columns=cols)
697
+
698
+ normal_iv = float(woe_table["iv"].sum()) if len(woe_table) > 0 else 0.0
699
+
700
+ # 缺失值 → [Missing] 箱(复用 sv_table 机制)
701
+ sv_groups = {float("nan"): sub[nan_mask]} if int(nan_mask.sum()) > 0 else {}
702
+ sv_table = self._compute_sv_table(sv_groups, full_total_bad, full_total_good)
703
+ sv_iv = float(sv_table["iv"].sum()) if len(sv_table) > 0 else 0.0
704
+
705
+ woes = woe_table.sort_values("bin")["woe"].values if len(woe_table) > 0 else np.array([])
706
+
707
+ return dict(
708
+ edges = [],
709
+ woe_table = woe_table,
710
+ sv_table = sv_table,
711
+ iv = round(normal_iv + sv_iv, 6),
712
+ is_monotonic = self._is_monotone(woes),
713
+ n_bins = len(woe_table),
714
+ is_categorical = True,
715
+ categories = list(cats),
716
+ )
717
+
718
+ # ─────────────────────────────────────────────────────────────────
719
+ # 公开 API
720
+ # ─────────────────────────────────────────────────────────────────
721
+
722
+ def fit(
723
+ self,
724
+ df: pd.DataFrame,
725
+ chi2_binning: bool = False,
726
+ chi2_p: float = 0.99,
727
+ chi2_init_size: int = 1000,
728
+ n_jobs: int = 1,
729
+ ) -> "MonotoneWOEBinner":
730
+ """
731
+ 在训练集上拟合所有特征的单调 WOE 分箱。
732
+
733
+ Parameters
734
+ ----------
735
+ df : 训练集 DataFrame,需包含 feature_cols 和 target_col
736
+ chi2_binning : 是否在贪心单调分箱后再进行卡方后合并,默认 False。
737
+ True 时:当相邻箱的卡方检验 p > (1 - chi2_p)时,
738
+ 尝试合并该对,并保持 WOE 单调。
739
+ chi2_p : 卡方检验置信度,默认 0.99。
740
+ 较大的值(如 0.99)表示保留更多箱;
741
+ 较小的值(如 0.90)表示更容易合并。
742
+ chi2_init_size : 卡方计算时的全局 stratified 采样上限,默认 1000。
743
+ 若普通行数 > chi2_init_size,按 target 比例分层采样,
744
+ 避免大数据集下卡方值虚高导致不该合并的箱被强行分开。
745
+ n_jobs : 并行线程数,默认 1(顺序执行,行为与旧版完全相同)。
746
+ 设为 > 1 时使用指定数量的线程;设为 -1 时使用所有可用
747
+ CPU 核心。特征数较多(如 3000+)时可显著提速。
748
+
749
+ Returns
750
+ -------
751
+ self (支持链式调用)
752
+ """
753
+ # 待拟合的全部特征 = 数值特征 + 类别特征(去重,保持顺序)
754
+ all_fit_feats = list(dict.fromkeys(list(self.feature_cols) + list(self.cate_feats)))
755
+
756
+ missing_feats = [f for f in all_fit_feats if f not in df.columns]
757
+ if missing_feats:
758
+ raise ValueError(f"以下特征列不在 DataFrame 中: {missing_feats}")
759
+ if self.target_col not in df.columns:
760
+ raise ValueError(f"目标列 '{self.target_col}' 不在 DataFrame 中")
761
+
762
+ # 检查 scipy 可用性
763
+ if chi2_binning:
764
+ try:
765
+ from scipy.stats import chi2 as _chi2_check # noqa
766
+ except ImportError:
767
+ raise ImportError(
768
+ "chi2_binning=True 需要 scipy,请先安装: pip install scipy"
769
+ )
770
+
771
+ self._train_n = len(df)
772
+ self._bad_rate = float(df[self.target_col].mean())
773
+ self._chi2_binning = chi2_binning
774
+ self._chi2_p = chi2_p
775
+ self._chi2_init_size = chi2_init_size
776
+
777
+ sv_hint = f", special_values={self.special_values}" if self.special_values else ""
778
+ cate_hint = f", cate_feats={len(self.cate_feats)}个" if self.cate_feats else ""
779
+ chi2_hint = (f", chi2_binning=True (p={chi2_p}, sample={chi2_init_size})"
780
+ if chi2_binning else "")
781
+ logger.info(f"[MonotoneWOEBinner] 开始拟合 {len(all_fit_feats)} 个特征"
782
+ f"{sv_hint}{cate_hint}{chi2_hint} ...")
783
+
784
+ if n_jobs == 0:
785
+ raise ValueError("n_jobs 不能为 0;请使用正整数或 -1(全部核心)")
786
+
787
+ def _fit_one(feat):
788
+ try:
789
+ res = self._greedy_fit_one(
790
+ df, feat,
791
+ chi2_binning = chi2_binning,
792
+ chi2_p = chi2_p,
793
+ chi2_init_size = chi2_init_size,
794
+ )
795
+ return feat, res, None
796
+ except Exception as exc:
797
+ import traceback as _tb
798
+ return feat, None, (exc, _tb.format_exc())
799
+
800
+ def _log_feat(feat, res):
801
+ mono = res["is_monotonic"]
802
+ iv = res["iv"]
803
+ nb = res["n_bins"]
804
+ nsv = len(res["sv_table"])
805
+ sv_str = f" | sv_bins={nsv}" if nsv > 0 else ""
806
+ cat_str = " | CATE" if res.get("is_categorical") else ""
807
+ logger.info(f" ✓ {feat:40s} | n_bins={nb}{sv_str}{cat_str} | IV={iv:.4f} | mono={mono}")
808
+
809
+ if n_jobs == 1:
810
+ for feat in all_fit_feats:
811
+ _, res, err = _fit_one(feat)
812
+ if err is not None:
813
+ logger.info(f" ✗ {feat}: 拟合失败 — {err[0]}")
814
+ print(err[1])
815
+ else:
816
+ self._results[feat] = res
817
+ _log_feat(feat, res)
818
+ else:
819
+ # ── 多进程并行(ProcessPoolExecutor)──────────────────────────
820
+ # 策略:将特征列表均分为 N 块,每块整体发往一个进程,
821
+ # df 每块只序列化一次(而非每特征一次),大幅降低 IPC 开销。
822
+ max_workers = n_jobs if n_jobs > 0 else None
823
+ n_workers = max_workers or os.cpu_count() or 1
824
+ chunk_size = max(1, math.ceil(len(all_fit_feats) / n_workers))
825
+ chunks = [
826
+ all_fit_feats[i : i + chunk_size]
827
+ for i in range(0, len(all_fit_feats), chunk_size)
828
+ ]
829
+ # 轻量副本:只含配置,不含已有拟合结果,减少序列化体积
830
+ binner_lite = copy.copy(self)
831
+ binner_lite._results = {}
832
+ binner_lite._is_fitted = False
833
+
834
+ feat_ok, feat_err = {}, {}
835
+ with ProcessPoolExecutor(max_workers=max_workers) as executor:
836
+ futures = [
837
+ executor.submit(
838
+ _chunk_fit_worker,
839
+ (binner_lite, df, chunk, chi2_binning, chi2_p, chi2_init_size),
840
+ )
841
+ for chunk in chunks
842
+ ]
843
+ for fut in as_completed(futures):
844
+ ok, err = fut.result()
845
+ feat_ok.update(ok)
846
+ feat_err.update(err)
847
+ # 按原始顺序写回并打印日志
848
+ for feat in all_fit_feats:
849
+ if feat in feat_ok:
850
+ self._results[feat] = feat_ok[feat]
851
+ _log_feat(feat, feat_ok[feat])
852
+ elif feat in feat_err:
853
+ exc, tb = feat_err[feat]
854
+ logger.info(f" ✗ {feat}: 拟合失败 — {exc}")
855
+ print(tb)
856
+
857
+ self._is_fitted = True
858
+ n_mono = sum(1 for v in self._results.values() if v["is_monotonic"])
859
+ method = "greedy+chi2" if chi2_binning else "greedy"
860
+ logger.info(f"[MonotoneWOEBinner] 拟合完成 ({method}): "
861
+ f"{n_mono}/{len(self._results)} 个特征单调")
862
+ return self
863
+
864
+ def _check_fitted(self):
865
+ if not self._is_fitted:
866
+ raise RuntimeError("请先调用 fit() 或 load_woe_bins() 进行初始化")
867
+
868
+ def refine_chi2(
869
+ self,
870
+ df: pd.DataFrame,
871
+ features: Optional[List[str]] = None,
872
+ chi2_p: float = 0.99,
873
+ chi2_init_size: int = 1000,
874
+ n_jobs: int = 1,
875
+ ) -> "MonotoneWOEBinner":
876
+ """
877
+ 在已有贪心分箱结果的基础上,追加卡方后合并(不重跑贪心分箱)。
878
+
879
+ 与 fit(chi2_binning=True) 的区别
880
+ ---------------------------------
881
+ - 跳过贪心分箱阶段,直接以 self._results 中已有的 edges 为起点
882
+ - 可在同一份 fit 结果上反复以不同 chi2_p 调参,无需重跑贪心,速度更快
883
+ - 支持只对指定特征子集执行卡方合并
884
+ - 特殊值箱 WOE 不受影响,沿用 fit() 时的计算结果
885
+
886
+ Parameters
887
+ ----------
888
+ df : 原始训练数据(与 fit() 时相同),用于计算卡方统计量
889
+ features : 需要做卡方合并的特征列表;默认 None 表示所有已拟合特征
890
+ chi2_p : 卡方检验置信度,默认 0.99;较小值(如 0.90)更容易合并箱
891
+ chi2_init_size : 卡方计算时 stratified 采样上限,默认 1000
892
+ n_jobs : 并行线程数,默认 1(顺序执行)。设为 > 1 时使用指定数量
893
+ 的线程;设为 -1 时使用所有可用 CPU 核心。
894
+
895
+ Returns
896
+ -------
897
+ self(支持链式调用)
898
+
899
+ Examples
900
+ --------
901
+ >>> binner = MonotoneWOEBinner(feature_cols=["score"], target_col="is_bad")
902
+ >>> binner.fit(train_df) # 贪心分箱
903
+ >>> binner.refine_chi2(train_df, chi2_p=0.95, n_jobs=8) # 并行卡方合并
904
+ >>> # 或只对部分特征做卡方合并
905
+ >>> binner.refine_chi2(train_df, features=["score", "income"], chi2_p=0.90)
906
+ """
907
+ self._check_fitted()
908
+ try:
909
+ from scipy.stats import chi2 as _chi2_check # noqa
910
+ except ImportError:
911
+ raise ImportError(
912
+ "refine_chi2() 需要 scipy,请先安装: pip install scipy"
913
+ )
914
+ if n_jobs == 0:
915
+ raise ValueError("n_jobs 不能为 0;请使用正整数或 -1(全部核心)")
916
+
917
+ target_feats = features if features is not None else list(self._results.keys())
918
+ # 类别特征不适用卡方后合并,自动剔除
919
+ _cate_in = [f for f in target_feats if self._results.get(f, {}).get("is_categorical")]
920
+ if _cate_in:
921
+ logger.info(f"[refine_chi2] 跳过 {len(_cate_in)} 个类别特征(不适用卡方合并)")
922
+ target_feats = [f for f in target_feats
923
+ if not self._results.get(f, {}).get("is_categorical")]
924
+ missing_feats = [f for f in target_feats if f not in self._results]
925
+ if missing_feats:
926
+ raise ValueError(f"以下特征尚未拟合,无法做 chi2 合并: {missing_feats}")
927
+ if self.target_col not in df.columns:
928
+ raise ValueError(f"目标列 '{self.target_col}' 不在 DataFrame 中")
929
+
930
+ logger.info(
931
+ f"[refine_chi2] 对 {len(target_feats)} 个特征做卡方后合并 "
932
+ f"(chi2_p={chi2_p}, sample={chi2_init_size}, n_jobs={n_jobs}) ..."
933
+ )
934
+
935
+ # 每个特征的计算完全独立,可安全并行
936
+ def _refine_one(feat):
937
+ vr = self._results[feat]
938
+ edges = list(vr["edges"])
939
+ if len(edges) < 1:
940
+ return feat, None, None # 标记为"跳过"
941
+ try:
942
+ df_normal, _ = self._split_special(df, feat)
943
+ new_edges = self._chi2_merge_one(
944
+ df_normal, feat, edges, chi2_p, chi2_init_size
945
+ )
946
+ wt, iv = self._compute_woe_table(df_normal, feat, new_edges)
947
+ woes = wt.sort_values("bin")["woe"].values
948
+ sv_table = vr.get("sv_table", pd.DataFrame())
949
+ sv_iv = float(sv_table["iv"].sum()) if len(sv_table) > 0 else 0.0
950
+ update = dict(
951
+ edges = new_edges,
952
+ woe_table = wt,
953
+ iv = round(iv + sv_iv, 6),
954
+ is_monotonic = self._is_monotone(woes),
955
+ n_bins = len(wt),
956
+ )
957
+ return feat, (vr["n_bins"], update), None
958
+ except Exception as exc:
959
+ import traceback as _tb
960
+ return feat, None, (exc, _tb.format_exc())
961
+
962
+ def _apply_and_log(feat, ok, err):
963
+ if ok is None and err is None:
964
+ logger.info(f" - {feat:40s} | 仅 1 箱,跳过卡方合并")
965
+ elif err is not None:
966
+ logger.info(f" ✗ {feat}: chi2 合并失败 — {err[0]}")
967
+ print(err[1])
968
+ else:
969
+ old_nb, update = ok
970
+ self._results[feat].update(update)
971
+ logger.info(
972
+ f" ✓ {feat:40s} | bins: {old_nb} → {update['n_bins']} "
973
+ f"| IV={update['iv']:.4f} | mono={update['is_monotonic']}"
974
+ )
975
+
976
+ if n_jobs == 1:
977
+ for feat in target_feats:
978
+ feat, ok, err = _refine_one(feat)
979
+ _apply_and_log(feat, ok, err)
980
+ else:
981
+ # ── 多进程并行(ProcessPoolExecutor)──────────────────────────
982
+ max_workers = n_jobs if n_jobs > 0 else None
983
+ n_workers = max_workers or os.cpu_count() or 1
984
+ chunk_size = max(1, math.ceil(len(target_feats) / n_workers))
985
+ chunks = [
986
+ target_feats[i : i + chunk_size]
987
+ for i in range(0, len(target_feats), chunk_size)
988
+ ]
989
+ binner_lite = copy.copy(self)
990
+ binner_lite._results = {}
991
+ binner_lite._is_fitted = False
992
+ # 只传各特征的 edges 和 sv_iv(轻量),不传完整 _results
993
+ edges_map = {f: list(self._results[f]["edges"]) for f in target_feats}
994
+ sv_iv_map = {
995
+ f: float(self._results[f].get("sv_table", pd.DataFrame())["iv"].sum())
996
+ if len(self._results[f].get("sv_table", pd.DataFrame())) > 0 else 0.0
997
+ for f in target_feats
998
+ }
999
+ old_nb_map = {f: self._results[f]["n_bins"] for f in target_feats}
1000
+
1001
+ feat_ok, feat_err, feat_skip = {}, {}, set()
1002
+ with ProcessPoolExecutor(max_workers=max_workers) as executor:
1003
+ futures = [
1004
+ executor.submit(
1005
+ _chunk_chi2_worker,
1006
+ (binner_lite, df, chunk, edges_map, sv_iv_map,
1007
+ chi2_p, chi2_init_size),
1008
+ )
1009
+ for chunk in chunks
1010
+ ]
1011
+ for fut in as_completed(futures):
1012
+ ok, err = fut.result()
1013
+ for feat, res in ok.items():
1014
+ if res is None:
1015
+ feat_skip.add(feat)
1016
+ else:
1017
+ feat_ok[feat] = res
1018
+ feat_err.update(err)
1019
+ # 按原始顺序写回并打印日志
1020
+ for feat in target_feats:
1021
+ if feat in feat_skip:
1022
+ logger.info(f" - {feat:40s} | 仅 1 箱,跳过卡方合并")
1023
+ elif feat in feat_err:
1024
+ exc, tb = feat_err[feat]
1025
+ logger.info(f" ✗ {feat}: chi2 合并失败 — {exc}")
1026
+ print(tb)
1027
+ elif feat in feat_ok:
1028
+ upd = feat_ok[feat]
1029
+ self._results[feat].update(upd)
1030
+ logger.info(
1031
+ f" ✓ {feat:40s} | bins: {old_nb_map[feat]} → {upd['n_bins']} "
1032
+ f"| IV={upd['iv']:.4f} | mono={upd['is_monotonic']}"
1033
+ )
1034
+
1035
+ self._chi2_binning = True
1036
+ self._chi2_p = chi2_p
1037
+ self._chi2_init_size = chi2_init_size
1038
+
1039
+ n_skipped = sum(
1040
+ 1 for f in target_feats if len(self._results[f]["edges"]) < 1
1041
+ )
1042
+ logger.info(
1043
+ f"[refine_chi2] 完成,{len(target_feats) - n_skipped}/{len(target_feats)} "
1044
+ f"个特征参与合并"
1045
+ )
1046
+ return self
1047
+
1048
+ # ── refine_dtree ─────────────────────────────────────────────────────────
1049
+
1050
+ @staticmethod
1051
+ def _dtree_edges(df_normal: pd.DataFrame, feat: str, target_col: str,
1052
+ max_bins: int, min_samples_leaf) -> list:
1053
+ """用决策树找分割点,返回排好序的内部边界列表。"""
1054
+ from sklearn.tree import DecisionTreeClassifier
1055
+ sub = df_normal[[feat, target_col]].dropna(subset=[feat])
1056
+ if len(sub) < 4:
1057
+ return []
1058
+ X = sub[[feat]].values
1059
+ y = sub[target_col].values
1060
+ clf = DecisionTreeClassifier(
1061
+ max_leaf_nodes = max_bins,
1062
+ min_samples_leaf = min_samples_leaf,
1063
+ random_state = 42,
1064
+ )
1065
+ clf.fit(X, y)
1066
+ tree = clf.tree_
1067
+ # threshold == -2 表示叶节点,排除后去重排序
1068
+ raw = tree.threshold[tree.feature != -2]
1069
+ return sorted(set(float(t) for t in raw))
1070
+
1071
+ @staticmethod
1072
+ def _monotone_merge_edges(df_normal: pd.DataFrame, feat: str,
1073
+ target_col: str, edges: list,
1074
+ eps: float) -> list:
1075
+ """
1076
+ 在给定 edges 分箱后,若 WOE 不单调,贪心合并 WOE 方向反转的相邻箱,
1077
+ 直到单调为止。不改变箱的单调方向(自动判断升/降)。
1078
+ """
1079
+ import math as _math
1080
+
1081
+ def _woe_seq(edge_list):
1082
+ sub = df_normal[[feat, target_col]].dropna(subset=[feat])
1083
+ if len(sub) == 0 or len(edge_list) == 0:
1084
+ return np.array([])
1085
+ bins = pd.cut(sub[feat], bins=[-np.inf] + edge_list + [np.inf],
1086
+ labels=False, right=True)
1087
+ sub = sub.copy(); sub["_b"] = bins
1088
+ tb = float(sub[target_col].sum()); tg = float((sub[target_col] == 0).sum())
1089
+ recs = []
1090
+ for b in sorted(sub["_b"].dropna().unique()):
1091
+ g = sub[sub["_b"] == b]
1092
+ bad = float(g[target_col].sum()); good = float((g[target_col] == 0).sum())
1093
+ pb = bad / (tb + eps); pg = good / (tg + eps)
1094
+ recs.append(_math.log((pb + eps) / (pg + eps)))
1095
+ return np.array(recs)
1096
+
1097
+ def _is_mono(arr):
1098
+ if len(arr) <= 1: return True
1099
+ return (all(arr[i] <= arr[i+1] for i in range(len(arr)-1)) or
1100
+ all(arr[i] >= arr[i+1] for i in range(len(arr)-1)))
1101
+
1102
+ cur = list(edges)
1103
+ for _ in range(200):
1104
+ woes = _woe_seq(cur)
1105
+ if len(woes) <= 1 or _is_mono(woes):
1106
+ break
1107
+ # 找第一个方向反转位置(按主方向:第一个差值的符号)
1108
+ main_sign = np.sign(woes[1] - woes[0]) if len(woes) > 1 else 0
1109
+ for i in range(len(woes) - 1):
1110
+ if main_sign == 0:
1111
+ main_sign = np.sign(woes[i+1] - woes[i])
1112
+ if main_sign != 0 and np.sign(woes[i+1] - woes[i]) not in (main_sign, 0):
1113
+ # 合并第 i 和 i+1 箱(移除 cur[i])
1114
+ cur = [e for j, e in enumerate(cur) if j != i]
1115
+ break
1116
+ return cur
1117
+
1118
+ def refine_dtree(
1119
+ self,
1120
+ df: pd.DataFrame,
1121
+ features: Optional[List[str]] = None,
1122
+ max_bins: int = 6,
1123
+ min_samples_leaf: float = 0.05,
1124
+ monotone: bool = True,
1125
+ n_jobs: int = 1,
1126
+ ) -> "MonotoneWOEBinner":
1127
+ """
1128
+ 在已有贪心分箱结果的基础上,用决策树重新划定分割点。
1129
+
1130
+ 与 refine_chi2 的区别
1131
+ ---------------------
1132
+ - refine_chi2 : 在现有 edges 上做后合并(只减少箱数)
1133
+ - refine_dtree : 用决策树从头找最优分割点(可改变箱的位置和数量),
1134
+ 适合需要基于信息增益而非 IV 单调性重新划分的场景
1135
+
1136
+ 算法
1137
+ ----
1138
+ 1. 对每个特征的普通行拟合 DecisionTreeClassifier
1139
+ (max_leaf_nodes=max_bins, min_samples_leaf=min_samples_leaf)
1140
+ 2. 提取树的内部阈值作为新 edges
1141
+ 3. 若 monotone=True,贪心合并 WOE 方向反转的相邻箱,直到 WOE 单调
1142
+ 4. 重新计算 woe_table / iv / n_bins,写回 self._results
1143
+
1144
+ Parameters
1145
+ ----------
1146
+ df : 训练集 DataFrame(与 fit() 时相同)
1147
+ features : 特征子集;默认 None 表示所有已拟合特征
1148
+ max_bins : 决策树最大叶节点数(即分箱上限),默认 6
1149
+ min_samples_leaf : 决策树每个叶节点的最小样本占比(0~1)或绝对数(≥1),
1150
+ 默认 0.05(5%),用于防止过细分箱
1151
+ monotone : 是否在决策树分箱后强制 WOE 单调,默认 True
1152
+ n_jobs : 并行进程数,默认 1;-1 使用所有 CPU 核心
1153
+
1154
+ Returns
1155
+ -------
1156
+ self(支持链式调用)
1157
+
1158
+ Examples
1159
+ --------
1160
+ >>> binner.fit(train_df)
1161
+ >>> binner.refine_dtree(train_df, max_bins=5, min_samples_leaf=0.05)
1162
+ >>> # 先贪心分箱,再决策树重新划分,再 chi2 后合并
1163
+ >>> binner.fit(train_df).refine_dtree(train_df).refine_chi2(train_df, chi2_p=0.95)
1164
+ """
1165
+ self._check_fitted()
1166
+ try:
1167
+ from sklearn.tree import DecisionTreeClassifier # noqa
1168
+ except ImportError:
1169
+ raise ImportError(
1170
+ "refine_dtree() 需要 scikit-learn,请先安装: pip install scikit-learn"
1171
+ )
1172
+ if n_jobs == 0:
1173
+ raise ValueError("n_jobs 不能为 0;请使用正整数或 -1(全部核心)")
1174
+
1175
+ target_feats = features if features is not None else list(self._results.keys())
1176
+ # 类别特征不适用决策树重分箱,自动剔除(避免把类别编码当数值切分)
1177
+ _cate_in = [f for f in target_feats if self._results.get(f, {}).get("is_categorical")]
1178
+ if _cate_in:
1179
+ logger.info(f"[refine_dtree] 跳过 {len(_cate_in)} 个类别特征(不适用决策树重分箱)")
1180
+ target_feats = [f for f in target_feats
1181
+ if not self._results.get(f, {}).get("is_categorical")]
1182
+ missing_feats = [f for f in target_feats if f not in self._results]
1183
+ if missing_feats:
1184
+ raise ValueError(f"以下特征尚未拟合,无法做 dtree 重分箱: {missing_feats}")
1185
+ if self.target_col not in df.columns:
1186
+ raise ValueError(f"目标列 '{self.target_col}' 不在 DataFrame 中")
1187
+
1188
+ logger.info(
1189
+ f"[refine_dtree] 对 {len(target_feats)} 个特征做决策树重分箱 "
1190
+ f"(max_bins={max_bins}, min_samples_leaf={min_samples_leaf}, "
1191
+ f"monotone={monotone}, n_jobs={n_jobs}) ..."
1192
+ )
1193
+
1194
+ eps = self.eps
1195
+
1196
+ def _refine_one(feat):
1197
+ vr = self._results[feat]
1198
+ try:
1199
+ df_normal, _ = self._split_special(df, feat)
1200
+ # Step 1: 决策树找分割点
1201
+ new_edges = self._dtree_edges(
1202
+ df_normal, feat, self.target_col, max_bins, min_samples_leaf
1203
+ )
1204
+ # Step 2: 可选单调合并
1205
+ if monotone and len(new_edges) >= 1:
1206
+ new_edges = self._monotone_merge_edges(
1207
+ df_normal, feat, self.target_col, new_edges, eps
1208
+ )
1209
+ # Step 3: 重算 WOE 表
1210
+ wt, iv = self._compute_woe_table(df_normal, feat, new_edges)
1211
+ woes = wt.sort_values("bin")["woe"].values if len(wt) > 0 else np.array([])
1212
+ sv_table = vr.get("sv_table", pd.DataFrame())
1213
+ sv_iv = float(sv_table["iv"].sum()) if len(sv_table) > 0 else 0.0
1214
+ update = dict(
1215
+ edges = new_edges,
1216
+ woe_table = wt,
1217
+ iv = round(iv + sv_iv, 6),
1218
+ is_monotonic = self._is_monotone(woes),
1219
+ n_bins = len(wt),
1220
+ )
1221
+ return feat, (vr["n_bins"], update), None
1222
+ except Exception as exc:
1223
+ import traceback as _tb
1224
+ return feat, None, (exc, _tb.format_exc())
1225
+
1226
+ def _apply_and_log(feat, ok, err):
1227
+ if err is not None:
1228
+ logger.info(f" ✗ {feat}: dtree 重分箱失败 — {err[0]}")
1229
+ print(err[1])
1230
+ else:
1231
+ old_nb, update = ok
1232
+ self._results[feat].update(update)
1233
+ logger.info(
1234
+ f" ✓ {feat:40s} | bins: {old_nb} → {update['n_bins']} "
1235
+ f"| IV={update['iv']:.4f} | mono={update['is_monotonic']}"
1236
+ )
1237
+
1238
+ if n_jobs == 1:
1239
+ for feat in target_feats:
1240
+ feat, ok, err = _refine_one(feat)
1241
+ _apply_and_log(feat, ok, err)
1242
+ else:
1243
+ max_workers = n_jobs if n_jobs > 0 else None
1244
+ n_workers = max_workers or os.cpu_count() or 1
1245
+ chunk_size = max(1, math.ceil(len(target_feats) / n_workers))
1246
+ chunks = [
1247
+ target_feats[i : i + chunk_size]
1248
+ for i in range(0, len(target_feats), chunk_size)
1249
+ ]
1250
+ old_nb_map = {f: self._results[f]["n_bins"] for f in target_feats}
1251
+ feat_ok, feat_err = {}, {}
1252
+ with ProcessPoolExecutor(max_workers=max_workers) as executor:
1253
+ futures = {
1254
+ executor.submit(_refine_one, feat): feat
1255
+ for feat in target_feats
1256
+ }
1257
+ from concurrent.futures import as_completed as _asc
1258
+ for fut in _asc(futures):
1259
+ feat_r, ok, err = fut.result()
1260
+ if err is not None:
1261
+ feat_err[feat_r] = err
1262
+ else:
1263
+ feat_ok[feat_r] = ok
1264
+ for feat in target_feats:
1265
+ if feat in feat_err:
1266
+ exc, tb = feat_err[feat]
1267
+ logger.info(f" ✗ {feat}: dtree 重分箱失败 — {exc}")
1268
+ print(tb)
1269
+ elif feat in feat_ok:
1270
+ old_nb, upd = feat_ok[feat]
1271
+ self._results[feat].update(upd)
1272
+ logger.info(
1273
+ f" ✓ {feat:40s} | bins: {old_nb_map[feat]} → {upd['n_bins']} "
1274
+ f"| IV={upd['iv']:.4f} | mono={upd['is_monotonic']}"
1275
+ )
1276
+
1277
+ logger.info(f"[refine_dtree] 完成,{len(target_feats)} 个特征处理完毕")
1278
+ return self
1279
+
1280
+ # ── refine_cate ──────────────────────────────────────────────────────────
1281
+
1282
+ def _cluster_cate_one(
1283
+ self,
1284
+ vr: Dict,
1285
+ max_bins: int,
1286
+ min_bin_size: float,
1287
+ badrate_tol: Optional[float],
1288
+ ) -> Optional[Dict[str, Any]]:
1289
+ """
1290
+ 对单个类别特征,按坏率(bad rate)做凝聚式(agglomerative)聚类:
1291
+ 把坏率相近的类别合并成同一箱,直到箱数 ≤ max_bins。
1292
+
1293
+ 合并规则(每轮择一执行,直到无可合并):
1294
+ 1. min_bin_size 优先(稳定性):存在样本占比 < min_bin_size 的箱时,
1295
+ 先把最小的违例箱并入坏率更接近的相邻箱(忽略 badrate_tol)。
1296
+ 2. 否则若箱数 > max_bins:合并坏率差最小的相邻箱对;
1297
+ 若设了 badrate_tol 且最小坏率差 > badrate_tol,则停止
1298
+ (剩余相邻箱坏率差异都过大,不再强行合并)。
1299
+
1300
+ 合并仅发生在按坏率排序后的相邻箱之间,因此最终各箱坏率严格有序,
1301
+ WOE 天然单调。仅用已拟合的每类别计数(woe_table),无需重读原始数据。
1302
+
1303
+ Returns
1304
+ -------
1305
+ update dict(woe_table / iv / is_monotonic / n_bins / categories),
1306
+ 若无需聚类则返回 None。
1307
+ """
1308
+ wt = vr["woe_table"]
1309
+ eps = self.eps
1310
+ if len(wt) <= 1:
1311
+ return None
1312
+
1313
+ total_bad = float(wt["bad"].sum())
1314
+ total_good = float(wt["good"].sum())
1315
+ total_n = float(wt["n"].sum())
1316
+
1317
+ # 初始:每个(已有)箱作为一个组,保留成员类别与标签
1318
+ groups = []
1319
+ for _, r in wt.iterrows():
1320
+ if "cat_members" in wt.columns and isinstance(r["cat_members"], (list, tuple)):
1321
+ members = list(r["cat_members"])
1322
+ elif "cat_value" in wt.columns and not pd.isna(r["cat_value"]):
1323
+ members = [r["cat_value"]]
1324
+ else:
1325
+ members = [r["bin_label"]]
1326
+ groups.append(dict(
1327
+ members = members,
1328
+ label = str(r["bin_label"]),
1329
+ n = float(r["n"]),
1330
+ bad = float(r["bad"]),
1331
+ good = float(r["good"]),
1332
+ ))
1333
+
1334
+ def _br(g):
1335
+ return g["bad"] / (g["bad"] + g["good"] + eps)
1336
+
1337
+ groups.sort(key=_br)
1338
+
1339
+ def _merge(i):
1340
+ """合并相邻 groups[i] 与 groups[i+1]。"""
1341
+ a, b = groups[i], groups[i + 1]
1342
+ groups[i:i + 2] = [dict(
1343
+ members = a["members"] + b["members"],
1344
+ label = a["label"] + _CATE_GROUP_SEP + b["label"],
1345
+ n = a["n"] + b["n"],
1346
+ bad = a["bad"] + b["bad"],
1347
+ good = a["good"] + b["good"],
1348
+ )]
1349
+
1350
+ changed = False
1351
+ while len(groups) > 1:
1352
+ too_small = (
1353
+ [i for i, g in enumerate(groups)
1354
+ if g["n"] / (total_n + eps) < min_bin_size]
1355
+ if min_bin_size > 0 else []
1356
+ )
1357
+ if too_small:
1358
+ # 把最小的违例箱并入坏率更接近的相邻箱
1359
+ i = min(too_small, key=lambda k: groups[k]["n"])
1360
+ if i == 0:
1361
+ j = 0
1362
+ elif i == len(groups) - 1:
1363
+ j = i - 1
1364
+ else:
1365
+ dl = abs(_br(groups[i]) - _br(groups[i - 1]))
1366
+ dr = abs(_br(groups[i]) - _br(groups[i + 1]))
1367
+ j = i - 1 if dl <= dr else i
1368
+ _merge(j); changed = True
1369
+ continue
1370
+ if len(groups) > max_bins:
1371
+ gap, mi = min(
1372
+ (abs(_br(groups[i + 1]) - _br(groups[i])), i)
1373
+ for i in range(len(groups) - 1)
1374
+ )
1375
+ if badrate_tol is not None and gap > badrate_tol:
1376
+ break # 剩余相邻箱坏率差异都过大,停止合并
1377
+ _merge(mi); changed = True
1378
+ continue
1379
+ break
1380
+
1381
+ if not changed:
1382
+ return None
1383
+
1384
+ # 重算各组 WOE/IV,按坏率升序编号(WOE 天然单调)
1385
+ groups.sort(key=_br)
1386
+ records = []
1387
+ for i, g in enumerate(groups):
1388
+ bad, good, n = g["bad"], g["good"], g["n"]
1389
+ pct_bad = bad / (total_bad + eps)
1390
+ pct_good = good / (total_good + eps)
1391
+ woe = math.log((pct_bad + eps) / (pct_good + eps))
1392
+ iv = (pct_bad - pct_good) * woe
1393
+ records.append(dict(
1394
+ bin=i,
1395
+ cat_value=(g["members"][0] if len(g["members"]) == 1 else np.nan),
1396
+ cat_members=list(g["members"]),
1397
+ bin_label=g["label"],
1398
+ n=int(n), bad=int(bad), good=int(good),
1399
+ bad_rate=bad / (bad + good) if (bad + good) > 0 else 0.0,
1400
+ pct_bad=pct_bad, pct_good=pct_good, woe=woe, iv=iv,
1401
+ ))
1402
+ cols = ["bin", "cat_value", "cat_members", "bin_label", "n", "bad", "good",
1403
+ "bad_rate", "pct_bad", "pct_good", "woe", "iv"]
1404
+ new_wt = pd.DataFrame(records, columns=cols)
1405
+
1406
+ normal_iv = float(new_wt["iv"].sum())
1407
+ sv_table = vr.get("sv_table", pd.DataFrame())
1408
+ sv_iv = float(sv_table["iv"].sum()) if len(sv_table) > 0 else 0.0
1409
+
1410
+ return dict(
1411
+ woe_table = new_wt,
1412
+ iv = round(normal_iv + sv_iv, 6),
1413
+ is_monotonic = self._is_monotone(new_wt["woe"].values),
1414
+ n_bins = len(new_wt),
1415
+ categories = [m for g in groups for m in g["members"]],
1416
+ )
1417
+
1418
+ def refine_cate(
1419
+ self,
1420
+ features: Optional[List[str]] = None,
1421
+ max_bins: int = 5,
1422
+ min_bin_size: float = 0.0,
1423
+ badrate_tol: Optional[float] = None,
1424
+ ) -> "MonotoneWOEBinner":
1425
+ """
1426
+ 对已拟合的**类别特征(cate_feats)**按坏率(bad rate)做凝聚式聚类,
1427
+ 把坏率相近的类别合并成同一箱,降低箱数、提升稳定性。
1428
+
1429
+ 与 refine_chi2 / refine_dtree 的关系
1430
+ ------------------------------------
1431
+ - refine_chi2 / refine_dtree : 只作用于**数值**特征(类别特征自动跳过)
1432
+ - refine_cate : 只作用于**类别**特征(数值特征自动跳过)
1433
+
1434
+ 说明
1435
+ ----
1436
+ - 仅使用 fit() 已算好的每类别计数(woe_table),**无需重新传入 df**,速度极快。
1437
+ - 合并按坏率排序后的相邻类别进行,因此结果各箱坏率有序、WOE 天然单调。
1438
+ - 合并只会降低或维持 IV(信息合并不会增加 IV),换取更少的箱与更好的泛化。
1439
+ - [Missing] 箱不参与聚类,沿用 fit() 的结果。
1440
+ - 可重复调用(在已聚类结果上继续合并)。
1441
+
1442
+ Parameters
1443
+ ----------
1444
+ features : 需要聚类的类别特征列表;默认 None = 所有已拟合的类别特征。
1445
+ 传入数值特征会被自动跳过。
1446
+ max_bins : 聚类后每个特征的最大箱数,默认 5。
1447
+ min_bin_size : 每箱最小样本占比(0~1),默认 0.0(关闭)。> 0 时,样本占比
1448
+ 低于该阈值的箱会被强制并入坏率最接近的相邻箱(优先于 max_bins,
1449
+ 且忽略 badrate_tol)。
1450
+ badrate_tol : 坏率差阈值,默认 None(不启用)。设为正数时,当所有相邻箱的
1451
+ 坏率差都 > badrate_tol 时停止按 max_bins 合并,避免把坏率差异
1452
+ 很大的类别为了凑箱数而强行合并。
1453
+
1454
+ Returns
1455
+ -------
1456
+ self(支持链式调用)
1457
+
1458
+ Examples
1459
+ --------
1460
+ >>> binner = MonotoneWOEBinner(feature_cols=["score"], target_col="is_bad",
1461
+ ... cate_feats=["city", "industry"])
1462
+ >>> binner.fit(df)
1463
+ >>> binner.refine_cate(max_bins=5) # 全部类别特征聚类
1464
+ >>> binner.refine_cate(features=["city"], max_bins=4, # 仅 city,带约束
1465
+ ... min_bin_size=0.02, badrate_tol=0.03)
1466
+ """
1467
+ self._check_fitted()
1468
+ if max_bins < 1:
1469
+ raise ValueError(f"max_bins 必须 ≥ 1,收到: {max_bins}")
1470
+
1471
+ all_cate = [f for f in self._results if self._results[f].get("is_categorical")]
1472
+ if features is None:
1473
+ target_feats = all_cate
1474
+ else:
1475
+ _num_in = [f for f in features
1476
+ if f in self._results and not self._results[f].get("is_categorical")]
1477
+ if _num_in:
1478
+ logger.info(f"[refine_cate] 跳过 {len(_num_in)} 个非类别特征(仅适用类别特征)")
1479
+ target_feats = [f for f in features
1480
+ if self._results.get(f, {}).get("is_categorical")]
1481
+ missing_feats = [f for f in features if f not in self._results]
1482
+ if missing_feats:
1483
+ raise ValueError(f"以下特征尚未拟合,无法做类别聚类: {missing_feats}")
1484
+
1485
+ if not target_feats:
1486
+ logger.info("[refine_cate] 无类别特征可聚类(需先 fit 含 cate_feats 的特征)")
1487
+ return self
1488
+
1489
+ logger.info(
1490
+ f"[refine_cate] 对 {len(target_feats)} 个类别特征按坏率聚类 "
1491
+ f"(max_bins={max_bins}, min_bin_size={min_bin_size}, badrate_tol={badrate_tol}) ..."
1492
+ )
1493
+
1494
+ for feat in target_feats:
1495
+ vr = self._results[feat]
1496
+ old_nb = vr["n_bins"]
1497
+ try:
1498
+ update = self._cluster_cate_one(vr, max_bins, min_bin_size, badrate_tol)
1499
+ except Exception as exc:
1500
+ import traceback as _tb
1501
+ logger.info(f" ✗ {feat}: 类别聚类失败 — {exc}")
1502
+ print(_tb.format_exc())
1503
+ continue
1504
+ if update is None:
1505
+ logger.info(f" - {feat:40s} | {old_nb} 箱无需聚类")
1506
+ continue
1507
+ vr.update(update)
1508
+ logger.info(
1509
+ f" ✓ {feat:40s} | bins: {old_nb} → {update['n_bins']} "
1510
+ f"| IV={update['iv']:.4f} | mono={update['is_monotonic']}"
1511
+ )
1512
+
1513
+ logger.info(f"[refine_cate] 完成,{len(target_feats)} 个类别特征处理完毕")
1514
+ return self
1515
+
1516
+ @staticmethod
1517
+ def _bin_label(edges: list, bin_idx: int, n_bins: int,
1518
+ decimals: Optional[int] = None) -> str:
1519
+ """生成普通分箱区间字符串,bin_idx 为 0-based。
1520
+
1521
+ decimals=None → 使用 .8g(8 位有效数字),确保 load_woe_bins() 反向
1522
+ 重建 edges 时精度足够。
1523
+ decimals=N → 使用 :.Nf(固定 N 位小数),便于人工阅读。
1524
+ """
1525
+ def _fmt(v: float) -> str:
1526
+ return f"{v:.{decimals}f}" if decimals is not None else f"{v:.8g}"
1527
+
1528
+ if not edges:
1529
+ return "(-∞, +∞)"
1530
+ if bin_idx == 0:
1531
+ return f"(-∞, {_fmt(float(edges[0]))}]"
1532
+ elif bin_idx == n_bins - 1:
1533
+ return f"({_fmt(float(edges[-1]))}, +∞)"
1534
+ else:
1535
+ return f"({_fmt(float(edges[bin_idx-1]))}, {_fmt(float(edges[bin_idx]))}]"
1536
+
1537
+ # ── 1. get_final_bins ────────────────────────────────────────────
1538
+
1539
+ def get_final_bins(self) -> Dict[str, pd.DataFrame]:
1540
+ """
1541
+ 返回每个特征的最终分箱区间 + WOE 明细(含特殊值箱)。
1542
+
1543
+ 特殊值箱追加在普通箱之后,bin_no 继续编号,bin_label 为 '[sv=xxx]'。
1544
+
1545
+ Returns
1546
+ -------
1547
+ dict: {feature_name -> pd.DataFrame}
1548
+ DataFrame 列: bin_no | bin_label | n | bad | good |
1549
+ bad_rate | pct_n | lift |
1550
+ pct_bad | pct_good | woe | iv | cumiv
1551
+ is_special (bool, True=特殊值箱)
1552
+
1553
+ 其中:
1554
+ pct_n = 该箱样本量 / 所有箱样本量之和(含特殊值箱)
1555
+ lift = 该箱 bad_rate / 全局平均 bad_rate
1556
+ 全局 bad_rate 取 self._bad_rate(fit 时记录);
1557
+ 若未 fit 则退化为所有箱 bad 之和 / n 之和
1558
+ """
1559
+ self._check_fitted()
1560
+ result = {}
1561
+ for feat, vr in self._results.items():
1562
+ wt = vr["woe_table"].copy().sort_values("bin").reset_index(drop=True)
1563
+ edges = vr["edges"]
1564
+ n_bins = vr["n_bins"]
1565
+
1566
+ wt["bin_no"] = wt["bin"] + 1
1567
+ if vr.get("is_categorical"):
1568
+ # 类别特征:箱标签即类别取值本身,已存于 woe_table,无需重建
1569
+ if "bin_label" not in wt.columns:
1570
+ wt["bin_label"] = wt["bin"].astype(str)
1571
+ else:
1572
+ wt["bin_label"] = [
1573
+ self._bin_label(edges, int(row["bin"]), n_bins,
1574
+ self.bin_label_decimals)
1575
+ for _, row in wt.iterrows()
1576
+ ]
1577
+ wt["cumiv"] = wt["iv"].cumsum()
1578
+ wt["is_special"] = False
1579
+
1580
+ # 追加特殊值箱
1581
+ sv_table = vr.get("sv_table", pd.DataFrame())
1582
+ if len(sv_table) > 0:
1583
+ sv_rows = []
1584
+ base_bin_no = len(wt) + 1
1585
+ running_cumiv = float(wt["cumiv"].iloc[-1]) if len(wt) > 0 else 0.0
1586
+ for i, (_, svrow) in enumerate(sv_table.iterrows()):
1587
+ running_cumiv += float(svrow["iv"])
1588
+ sv_rows.append({
1589
+ "bin_no" : base_bin_no + i,
1590
+ "bin_label" : svrow["bin_label"],
1591
+ "n" : int(svrow["n"]),
1592
+ "bad" : int(svrow["bad"]),
1593
+ "good" : int(svrow["good"]),
1594
+ "bad_rate" : float(svrow["bad_rate"]),
1595
+ "pct_bad" : float(svrow["pct_bad"]),
1596
+ "pct_good" : float(svrow["pct_good"]),
1597
+ "woe" : float(svrow["woe"]),
1598
+ "iv" : float(svrow["iv"]),
1599
+ "cumiv" : round(running_cumiv, 6),
1600
+ "is_special": True,
1601
+ })
1602
+ sv_df = pd.DataFrame(sv_rows)
1603
+ wt = pd.concat([wt, sv_df], ignore_index=True)
1604
+
1605
+ # ── 计算 pct_n 和 lift ──
1606
+ total_n = float(wt["n"].sum())
1607
+ # 全局 bad_rate:优先用 fit() 时记录的,否则从分箱数据反推
1608
+ avg_bad_rate = getattr(self, "_bad_rate", None)
1609
+ if avg_bad_rate is None or avg_bad_rate == 0:
1610
+ total_bad_all = float(wt["bad"].sum())
1611
+ total_good_all = float(wt["good"].sum()) if "good" in wt.columns else 0.0
1612
+ avg_bad_rate = total_bad_all / (total_bad_all + total_good_all) if (total_bad_all + total_good_all) > 0 else self.eps
1613
+
1614
+ wt["pct_n"] = wt["n"] / total_n if total_n > 0 else 0.0
1615
+ wt["lift"] = wt["bad_rate"].apply(
1616
+ lambda br: round(br / avg_bad_rate, 4) if avg_bad_rate > 0 else 0.0
1617
+ )
1618
+
1619
+ # ── 补全可能缺失或 NaN 的列(格式 B 加载时 woe_table 无这些列,
1620
+ # pd.concat 后普通箱行为 NaN)──
1621
+ _eps = self.eps
1622
+ if "good" not in wt.columns or wt["good"].isna().any():
1623
+ wt["good"] = wt["good"].fillna(0)
1624
+ if "bad_rate" not in wt.columns or wt["bad_rate"].isna().any():
1625
+ g = wt["good"].fillna(0) if "good" in wt.columns else 0
1626
+ wt["bad_rate"] = wt["bad"] / (wt["bad"] + g + _eps)
1627
+ # pct_bad / pct_good:只对普通箱(non-special)重算;sv 行保持 0.0
1628
+ _need_pct = (
1629
+ "pct_bad" not in wt.columns or wt["pct_bad"].isna().any() or
1630
+ "pct_good" not in wt.columns or wt["pct_good"].isna().any()
1631
+ )
1632
+ if _need_pct:
1633
+ _normal_mask = ~wt["is_special"].astype(bool) if "is_special" in wt.columns else pd.Series(True, index=wt.index)
1634
+ _total_bad = float(wt.loc[_normal_mask, "bad"].sum())
1635
+ _total_good = float(wt.loc[_normal_mask, "good"].sum())
1636
+ if "pct_bad" not in wt.columns:
1637
+ wt["pct_bad"] = 0.0
1638
+ if "pct_good" not in wt.columns:
1639
+ wt["pct_good"] = 0.0
1640
+ wt.loc[_normal_mask, "pct_bad"] = (
1641
+ wt.loc[_normal_mask, "bad"] / (_total_bad + _eps)
1642
+ )
1643
+ wt.loc[_normal_mask, "pct_good"] = (
1644
+ wt.loc[_normal_mask, "good"] / (_total_good + _eps)
1645
+ )
1646
+
1647
+ cols = ["bin_no", "bin_label", "n", "bad", "good",
1648
+ "bad_rate", "pct_n", "lift",
1649
+ "pct_bad", "pct_good", "woe", "iv", "cumiv", "is_special"]
1650
+ result[feat] = wt[[c for c in cols if c in wt.columns]]
1651
+ return result
1652
+
1653
+ # ── 1a2. get_bin_edges ────────────────────────────────────────────
1654
+
1655
+ def get_bin_edges(self) -> Dict[str, List[float]]:
1656
+ """
1657
+ 返回每个特征的完整分箱边界列表(含 ±inf 端点),可直接用于
1658
+ ``pd.cut``、``get_gains_table`` 等下游函数。
1659
+
1660
+ 返回的边界列表与 ``get_final_bins()`` 中的普通箱 bin_label
1661
+ 一一对应:若边界为 ``[-inf, 1.5, 3.0, inf]``,则对应的三个
1662
+ 普通箱分别为 ``(-∞, 1.5]``、``(1.5, 3.0]``、``(3.0, +∞)``。
1663
+
1664
+ **注意**:特殊值箱(如 ``[sv=-1]``、``[Missing]``)不包含在
1665
+ 边界列表中 — 它们独立于普通分箱,由 ``MonotoneWOEBinner``
1666
+ 在 ``apply_woe()`` 时自动处理。类别特征(``cate_feats``)同样
1667
+ 不包含在内(无数值边界),其 WOE 映射由 ``apply_woe()`` 直接按取值查表。
1668
+
1669
+ Returns
1670
+ -------
1671
+ dict: ``{feature_name: [-inf, cut1, cut2, ..., inf]}``
1672
+ 每个特征的完整分箱边界列表,首尾固定为 ``-np.inf``
1673
+ 和 ``np.inf``。
1674
+
1675
+ Example
1676
+ -------
1677
+ >>> binner = MonotoneWOEBinner(feature_cols=["score"], target_col="is_bad")
1678
+ >>> binner.fit(df)
1679
+ >>> binner.get_bin_edges()
1680
+ {'score': [-inf, 450.0, 520.0, 600.0, 680.0, inf]}
1681
+
1682
+ >>> # 可直接用于下游分箱
1683
+ >>> edges = binner.get_bin_edges()["score"]
1684
+ >>> df["score_bin"] = pd.cut(df["score"], bins=edges, labels=False)
1685
+ """
1686
+ self._check_fitted()
1687
+ result = {}
1688
+ for feat, vr in self._results.items():
1689
+ if vr.get("is_categorical"):
1690
+ # 类别特征无数值边界,不适用 pd.cut,跳过
1691
+ continue
1692
+ edges = [float(e) for e in vr["edges"]]
1693
+ result[feat] = [-np.inf] + edges + [np.inf]
1694
+ return result
1695
+
1696
+ # ── 1b. load_woe_bins ────────────────────────────────────────────
1697
+
1698
+ def load_woe_bins(self, bins_dict: dict) -> "MonotoneWOEBinner":
1699
+ """
1700
+ 直接加载已有的分箱结果,跳过 fit()。支持两种输入格式:
1701
+
1702
+ 格式 A — get_final_bins() 的输出:
1703
+ {feature_name -> DataFrame}
1704
+ DataFrame 必须包含列: bin_label | n | bad | woe | iv
1705
+ (可含 is_special 列;无则假设全为普通箱)
1706
+ 类别特征自动识别:若普通箱 bin_label 不是数值区间格式(如 "(-∞, 1.5]"),
1707
+ 则按类别特征加载,apply_woe 时按取值直接查表。
1708
+
1709
+ 格式 B — 训练流水线 woe_results 格式:
1710
+ {feature_name -> dict},dict 包含:
1711
+ edges : list,含 ±inf 端点,如 [-inf, 1.5, 3.0, inf]
1712
+ woe_map : {bin_index -> woe_value}
1713
+ missing_woe : float
1714
+ bin_df : DataFrame,含列 b | n | nb | br | woe | pct
1715
+ total_iv : float(可选;若无则从 bin_df 推算)
1716
+ n_bins : int(可选)
1717
+
1718
+ 两种格式可在同一个 bins_dict 中混合使用。
1719
+
1720
+ Returns
1721
+ -------
1722
+ self (支持链式调用)
1723
+ """
1724
+ self._results = {}
1725
+
1726
+ for feat, payload in bins_dict.items():
1727
+
1728
+ # ── 判断格式 ──────────────────────────────────────────────
1729
+ if isinstance(payload, pd.DataFrame):
1730
+ # 格式 A(直接是 DataFrame)
1731
+ fmt = "A"
1732
+ df_bin = payload
1733
+ elif isinstance(payload, dict) and "woe_map" in payload:
1734
+ # 格式 B(dict with edges / woe_map / bin_df)
1735
+ fmt = "B"
1736
+ elif isinstance(payload, dict):
1737
+ # 格式 A 包在 dict 里(不常见,兼容)
1738
+ fmt = "A"
1739
+ df_bin = payload.get("bin_df") or payload.get("df")
1740
+ if df_bin is None:
1741
+ raise ValueError(
1742
+ f"特征 '{feat}': dict 格式既无 'woe_map' 也无 'bin_df',无法识别格式"
1743
+ )
1744
+ else:
1745
+ raise ValueError(
1746
+ f"特征 '{feat}': 不支持的类型 {type(payload)},"
1747
+ "期望 DataFrame 或含 woe_map 的 dict"
1748
+ )
1749
+
1750
+ # 类别特征标记(格式 A 自动识别;格式 B 暂不支持类别特征)
1751
+ is_categorical = False
1752
+
1753
+ # ════════════════════════════════════════════════════════
1754
+ # 格式 A 处理路径
1755
+ # ════════════════════════════════════════════════════════
1756
+ if fmt == "A":
1757
+ required_cols = {"bin_label", "n", "bad", "woe", "iv"}
1758
+ missing = required_cols - set(df_bin.columns)
1759
+ if missing:
1760
+ raise ValueError(f"特征 '{feat}' 的分箱表缺少列: {missing}")
1761
+
1762
+ df_bin = df_bin.copy().reset_index(drop=True)
1763
+
1764
+ if "is_special" in df_bin.columns:
1765
+ sv_mask = df_bin["is_special"].astype(bool)
1766
+ df_normal = df_bin[~sv_mask].copy()
1767
+ df_sv = df_bin[sv_mask].copy()
1768
+ else:
1769
+ df_normal = df_bin.copy()
1770
+ df_sv = pd.DataFrame()
1771
+
1772
+ # 自动识别类别特征:普通箱标签不全是数值区间格式 → 类别特征
1773
+ _norm_labels = df_normal["bin_label"].astype(str).tolist()
1774
+ is_categorical = (
1775
+ len(_norm_labels) > 0
1776
+ and not all(self._looks_like_interval(l) for l in _norm_labels)
1777
+ )
1778
+
1779
+ woe_table = df_normal.copy()
1780
+ woe_table["bin"] = range(len(df_normal))
1781
+ if is_categorical:
1782
+ edges = [] # 类别特征无数值边界
1783
+ # 还原成员类别:支持 refine_cate 合并后的 "A | B | C" 标签
1784
+ # 每个成员 int → float → 原字符串,供 apply_woe 精确匹配
1785
+ _members = [
1786
+ [self._infer_cat_value(p) for p in lbl.split(_CATE_GROUP_SEP)]
1787
+ for lbl in _norm_labels
1788
+ ]
1789
+ woe_table["cat_members"] = _members
1790
+ woe_table["cat_value"] = [
1791
+ ms[0] if len(ms) == 1 else np.nan for ms in _members
1792
+ ]
1793
+ else:
1794
+ edges = self._reconstruct_edges(_norm_labels)
1795
+ sv_table = df_sv.copy() if len(df_sv) > 0 else pd.DataFrame()
1796
+ total_iv = float(df_bin["iv"].sum())
1797
+ n_bins = len(df_normal)
1798
+ woes = df_normal["woe"].values if len(df_normal) > 0 else np.array([])
1799
+ missing_woe = 0.0 # 格式 A 无此字段,用中性 WOE
1800
+
1801
+ # ════════════════════════════════════════════════════════
1802
+ # 格式 B 处理路径
1803
+ # ════════════════════════════════════════════════════════
1804
+ else: # fmt == "B"
1805
+ raw_edges = list(payload["edges"]) # 含首尾 ±inf
1806
+ woe_map = payload["woe_map"] # {int -> float}
1807
+ missing_woe = float(payload.get("missing_woe", 0.0))
1808
+ bin_df = payload.get("bin_df", pd.DataFrame())
1809
+ total_iv = float(payload.get("total_iv", 0.0))
1810
+
1811
+ # edges:去掉首尾 ±inf,只保留内部切割点
1812
+ import math as _math
1813
+ edges = [
1814
+ float(e) for e in raw_edges
1815
+ if not (_math.isinf(float(e)) or _math.isnan(float(e)))
1816
+ ]
1817
+
1818
+ n_bins = len(woe_map)
1819
+
1820
+ # 构建 woe_table(与格式 A 的 woe_table 列对齐)
1821
+ if len(bin_df) > 0:
1822
+ bdf = bin_df.copy().reset_index(drop=True)
1823
+ # 列名映射:bin_df 用 b/nb/br/pct,woe_table 用 bin/bad/bad_rate/pct_n
1824
+ rename_map = {}
1825
+ if "b" in bdf.columns and "bin" not in bdf.columns:
1826
+ rename_map["b"] = "bin"
1827
+ if "nb" in bdf.columns and "bad" not in bdf.columns:
1828
+ rename_map["nb"] = "bad"
1829
+ if "br" in bdf.columns and "bad_rate" not in bdf.columns:
1830
+ rename_map["br"] = "bad_rate"
1831
+ if "pct" in bdf.columns and "pct_n" not in bdf.columns:
1832
+ rename_map["pct"] = "pct_n"
1833
+ bdf = bdf.rename(columns=rename_map)
1834
+
1835
+ # 确保有 woe 列(用 woe_map 覆盖,保证精度一致)
1836
+ bdf["woe"] = bdf["bin"].map({int(k): float(v)
1837
+ for k, v in woe_map.items()})
1838
+
1839
+ # 补充 good 列(若缺)
1840
+ if "good" not in bdf.columns:
1841
+ bdf["good"] = 0
1842
+
1843
+ # 补充 iv 列(若缺)
1844
+ if "iv" not in bdf.columns:
1845
+ total_bad = bdf["bad"].sum()
1846
+ total_good = bdf["good"].sum() if "good" in bdf.columns else 0
1847
+ eps = self.eps
1848
+ def _iv_row(r):
1849
+ pb = r["bad"] / (total_bad + eps)
1850
+ pg = r["good"] / (total_good + eps) if total_good > 0 else eps
1851
+ return (pb - pg) * r["woe"]
1852
+ bdf["iv"] = bdf.apply(_iv_row, axis=1)
1853
+ if total_iv == 0.0:
1854
+ total_iv = float(bdf["iv"].sum())
1855
+
1856
+ # 确保有 bin_label 列(从 edges 生成)
1857
+ if "bin_label" not in bdf.columns:
1858
+ labels = self._make_bin_labels(edges, n_bins, self.bin_label_decimals)
1859
+ bdf["bin_label"] = labels[: len(bdf)]
1860
+
1861
+ woe_table = bdf.copy()
1862
+ else:
1863
+ # bin_df 缺失,从 woe_map + edges 最小化构建
1864
+ labels = self._make_bin_labels(edges, n_bins)
1865
+ woe_table = pd.DataFrame({
1866
+ "bin": list(range(n_bins)),
1867
+ "bin_label": labels,
1868
+ "woe": [float(woe_map[k]) for k in sorted(woe_map)],
1869
+ "n": [0] * n_bins,
1870
+ "bad": [0] * n_bins,
1871
+ "good": [0] * n_bins,
1872
+ "bad_rate": [0.0] * n_bins,
1873
+ "pct_n": [0.0] * n_bins,
1874
+ "iv": [0.0] * n_bins,
1875
+ })
1876
+
1877
+ woes = np.array([float(woe_map[k]) for k in sorted(woe_map)])
1878
+
1879
+ # ── 根据 self.special_values 自动构建 sv_table ──
1880
+ # 格式 B 没有 sv 的统计数据,但有 missing_woe;
1881
+ # 用 missing_woe 作为 WOE,n/bad/good 等统计量置为 0(占位)。
1882
+ sv_rows = []
1883
+ for sv_val in (self.special_values or []):
1884
+ import math as _math2
1885
+ is_nan_sv = (sv_val is None or
1886
+ (isinstance(sv_val, float) and _math2.isnan(sv_val)))
1887
+ lbl = "[Missing]" if is_nan_sv else f"[sv={sv_val}]"
1888
+ sv_rows.append({
1889
+ "bin_label": lbl,
1890
+ "sv": "__nan__" if is_nan_sv else sv_val,
1891
+ "n": 0,
1892
+ "bad": 0,
1893
+ "good": 0,
1894
+ "bad_rate": 0.0,
1895
+ "pct_bad": 0.0,
1896
+ "pct_good": 0.0,
1897
+ "woe": missing_woe,
1898
+ "iv": 0.0,
1899
+ })
1900
+ sv_table = pd.DataFrame(sv_rows) if sv_rows else pd.DataFrame()
1901
+
1902
+ # ── 写入 _results ─────────────────────────────────────────
1903
+ res = dict(
1904
+ edges = edges,
1905
+ woe_table = woe_table,
1906
+ sv_table = sv_table,
1907
+ iv = round(total_iv, 6),
1908
+ missing_woe = missing_woe,
1909
+ is_monotonic = self._is_monotone(woes) if len(woes) > 1 else True,
1910
+ n_bins = n_bins,
1911
+ )
1912
+ if is_categorical:
1913
+ res["is_categorical"] = True
1914
+ res["categories"] = (
1915
+ [m for ms in woe_table["cat_members"] for m in ms]
1916
+ if "cat_members" in woe_table.columns else []
1917
+ )
1918
+ self._results[feat] = res
1919
+
1920
+ # 同步 feature_cols
1921
+ existing = set(self.feature_cols)
1922
+ for feat in bins_dict:
1923
+ if feat not in existing:
1924
+ self.feature_cols.append(feat)
1925
+
1926
+ self._is_fitted = True
1927
+ logger.info(f"[load_woe_bins] 加载完成: {len(self._results)} 个特征")
1928
+ return self
1929
+
1930
+ @staticmethod
1931
+ def _make_bin_labels(edges: List[float], n_bins: int,
1932
+ decimals: Optional[int] = None) -> List[str]:
1933
+ """
1934
+ 从内部切割点 edges(不含 ±inf)生成 bin_label 字符串列表。
1935
+ 例如 edges=[1.5, 3.0], n_bins=3 →
1936
+ ["(-∞, 1.5]", "(1.5, 3.0]", "(3.0, +∞)"]
1937
+
1938
+ decimals=None → :.8g;decimals=N → :.Nf(固定 N 位小数)。
1939
+ """
1940
+ def _fmt(v: float) -> str:
1941
+ return f"{v:.{decimals}f}" if decimals is not None else f"{v:.8g}"
1942
+
1943
+ labels = []
1944
+ all_edges = [-float("inf")] + list(edges) + [float("inf")]
1945
+ for i in range(n_bins):
1946
+ lo = all_edges[i]
1947
+ hi = all_edges[i + 1]
1948
+ lo_s = "-∞" if lo == -float("inf") else _fmt(lo)
1949
+ hi_s = "+∞" if hi == float("inf") else _fmt(hi)
1950
+ if i == 0:
1951
+ labels.append(f"(-∞, {hi_s}]")
1952
+ elif i == n_bins - 1:
1953
+ labels.append(f"({lo_s}, +∞)")
1954
+ else:
1955
+ labels.append(f"({lo_s}, {hi_s}]")
1956
+ return labels
1957
+
1958
+ @staticmethod
1959
+ def _reconstruct_edges(bin_labels: List[str]) -> List[float]:
1960
+ """
1961
+ 从 bin_label 列表反向推断切割点 edges。
1962
+ 例如 ["(-∞, 1.5]", "(1.5, 3.0]", "(3.0, +∞)"] → [1.5, 3.0]
1963
+ 若解析失败则返回空列表。
1964
+ """
1965
+ import re
1966
+ edges = []
1967
+ for lbl in bin_labels[:-1]: # 最后一箱没有右边界可提取
1968
+ # 匹配 "(xxx, yyy]" 或 "(-∞, yyy]" 格式的右端点
1969
+ m = re.search(r",\s*([+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?)\s*\]", lbl)
1970
+ if m:
1971
+ try:
1972
+ edges.append(float(m.group(1)))
1973
+ except ValueError:
1974
+ pass
1975
+ return edges
1976
+
1977
+ @staticmethod
1978
+ def _looks_like_interval(label: str) -> bool:
1979
+ """判断 bin_label 是否为数值区间格式,如 (-∞, 1.5] / (1.5, 3] / (3, +∞)。
1980
+
1981
+ 用于 load_woe_bins 区分数值特征与类别特征:类别特征的 bin_label 是
1982
+ 类别取值本身(任意字符串),不符合该区间格式。
1983
+ """
1984
+ import re
1985
+ _num = r"(?:[+-]?∞|[+-]?inf|[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?)"
1986
+ return bool(re.match(rf"^\(\s*{_num}\s*,\s*{_num}\s*[\]\)]$", str(label).strip()))
1987
+
1988
+ @staticmethod
1989
+ def _infer_cat_value(label):
1990
+ """从 bin_label 还原类别原始取值:能转 int 则 int,能转 float 则 float,否则原字符串。"""
1991
+ s = str(label)
1992
+ try:
1993
+ f = float(s)
1994
+ except (ValueError, OverflowError):
1995
+ return s
1996
+ i = int(f)
1997
+ return i if i == f else f
1998
+
1999
+ # ── 2. apply_woe ─────────────────────────────────────────────────
2000
+
2001
+ def apply_woe(
2002
+ self,
2003
+ data: pd.DataFrame,
2004
+ suffix: str = "_woe",
2005
+ inplace: bool = False,
2006
+ ) -> pd.DataFrame:
2007
+ """
2008
+ 将 data 中的特征原始数值转换为 WOE 值,添加 *_woe 列。
2009
+
2010
+ 特殊值处理:
2011
+ - 若某值在 special_values 中,直接查 sv_table 获取对应 WOE
2012
+ - NaN:若 nan 在 special_values 中则查 sv_table;否则填 missing_woe
2013
+ - 普通值:按 edges 做 pd.cut,然后查 woe_table
2014
+
2015
+ 类别特征(cate_feats)处理:
2016
+ - 按取值直接查表取 WOE(不做区间切分)
2017
+ - NaN → [Missing] 箱 WOE(若 fit 时存在缺失),否则 missing_woe
2018
+ - 训练时未出现过的新类别 → missing_woe(中性)
2019
+
2020
+ Parameters
2021
+ ----------
2022
+ data : 含原始特征列的 DataFrame
2023
+ suffix : WOE 列后缀,默认 "_woe"
2024
+ inplace : 是否在原 DataFrame 上操作(False = 返回副本)
2025
+
2026
+ Returns
2027
+ -------
2028
+ DataFrame,新增 {feat}{suffix} 列
2029
+ """
2030
+ self._check_fitted()
2031
+ df = data if inplace else data.copy()
2032
+
2033
+ for feat, vr in self._results.items():
2034
+ if feat not in df.columns:
2035
+ logger.info(f" [WARN] '{feat}' 不在 data 中,跳过")
2036
+ continue
2037
+
2038
+ sv_table = vr.get("sv_table", pd.DataFrame())
2039
+ woe_col = feat + suffix
2040
+ # 优先使用 _results 中存储的 per-feature missing_woe(格式 B 加载时设置)
2041
+ feat_missing_woe = float(vr.get("missing_woe", self.missing_woe))
2042
+
2043
+ series = df[feat]
2044
+
2045
+ # 构建特殊值 → WOE 映射(数值/类别特征通用,主要用于 NaN/[Missing])
2046
+ sv_woe_map: Dict = {}
2047
+ if len(sv_table) > 0 and "bin_label" in sv_table.columns:
2048
+ for _, svrow in sv_table.iterrows():
2049
+ lbl = svrow["bin_label"]
2050
+ sv_woe_val = float(svrow["woe"])
2051
+ # 解析 bin_label 还原特殊值
2052
+ if lbl == "[Missing]":
2053
+ sv_woe_map["__nan__"] = sv_woe_val
2054
+ else:
2055
+ import re
2056
+ m = re.match(r"\[sv=(.*)\]$", lbl)
2057
+ if m:
2058
+ raw = m.group(1)
2059
+ try:
2060
+ sv_woe_map[float(raw)] = sv_woe_val
2061
+ sv_woe_map[int(float(raw))] = sv_woe_val
2062
+ except (ValueError, OverflowError):
2063
+ sv_woe_map[raw] = sv_woe_val
2064
+
2065
+ # ── 类别特征:按取值直接查 WOE,不做区间切分 ──
2066
+ if vr.get("is_categorical"):
2067
+ wt = vr["woe_table"]
2068
+ # 原始取值 → WOE(fit 路径,精确匹配;int/float 由 dict 等价性兼容)
2069
+ # refine_cate 聚类后,一个箱含多个类别(cat_members);逐成员展开建表
2070
+ cat_woe_map: Dict = {}
2071
+ cat_woe_map_str: Dict = {}
2072
+ for _, r in wt.iterrows():
2073
+ woe_v = float(r["woe"])
2074
+ if "cat_members" in wt.columns and isinstance(r["cat_members"], (list, tuple)):
2075
+ members = r["cat_members"]
2076
+ elif "cat_value" in wt.columns and not pd.isna(r["cat_value"]):
2077
+ members = [r["cat_value"]]
2078
+ else:
2079
+ members = []
2080
+ for cv in members:
2081
+ if pd.isna(cv):
2082
+ continue
2083
+ cat_woe_map[cv] = woe_v # 精确匹配
2084
+ cat_woe_map_str[str(cv)] = woe_v # 类型不一致时回退匹配
2085
+ # 整箱标签也登记一次(兜底;通常不会命中真实类别)
2086
+ if "bin_label" in wt.columns:
2087
+ cat_woe_map_str.setdefault(str(r["bin_label"]), woe_v)
2088
+ nan_woe = sv_woe_map.get("__nan__", feat_missing_woe)
2089
+
2090
+ def _get_cat_woe(val):
2091
+ if val is None or (isinstance(val, float) and math.isnan(val)):
2092
+ return nan_woe
2093
+ if val in cat_woe_map:
2094
+ return cat_woe_map[val]
2095
+ sval = str(val)
2096
+ if sval in cat_woe_map_str:
2097
+ return cat_woe_map_str[sval]
2098
+ # 训练时未见过的新类别 → 中性 missing_woe
2099
+ return feat_missing_woe
2100
+
2101
+ df[woe_col] = series.apply(_get_cat_woe).astype(float)
2102
+ continue
2103
+
2104
+ # ── 数值特征:按 edges 做 bin 查找 ──
2105
+ edges = [float(e) for e in vr["edges"]]
2106
+ wt_map = vr["woe_table"].set_index("bin")["woe"].to_dict()
2107
+
2108
+ # 逐元素赋 WOE
2109
+ def _get_woe(val):
2110
+ # NaN
2111
+ if val is None or (isinstance(val, float) and math.isnan(val)):
2112
+ if "__nan__" in sv_woe_map:
2113
+ return sv_woe_map["__nan__"]
2114
+ return feat_missing_woe
2115
+ # 数值特殊值
2116
+ if val in sv_woe_map:
2117
+ return sv_woe_map[val]
2118
+ # 普通值:做 bin 查找
2119
+ if not edges:
2120
+ return wt_map.get(0, feat_missing_woe)
2121
+ import bisect
2122
+ # pd.cut right=True → bin_idx = bisect_right(edges, val)
2123
+ bin_idx = bisect.bisect_right(edges, val)
2124
+ return wt_map.get(bin_idx, feat_missing_woe)
2125
+
2126
+ df[woe_col] = series.apply(_get_woe).astype(float)
2127
+
2128
+ return df
2129
+
2130
+ # ── 3. export_woe_report ─────────────────────────────────────────
2131
+
2132
+ def export_woe_report(self, report_path: str) -> None:
2133
+ """
2134
+ 将所有特征的分箱结果输出为 Excel 报告(使用 SuperModelingFactory
2135
+ 的 ExcelMaster 工具包写入)。
2136
+
2137
+ Sheet 列表
2138
+ ----------
2139
+ Sheet 1 「WOE分箱明细」: 汇总表 + 逐特征明细(含特殊值箱,紫色标注)
2140
+ Sheet 2 「WOE分箱图」 : 每个特征嵌入整体 WOE 图
2141
+
2142
+ Parameters
2143
+ ----------
2144
+ report_path : 输出路径,如 "woe_report.xlsx"
2145
+ """
2146
+ self._check_fitted()
2147
+ from ExcelMaster.ExcelMaster import ExcelMaster
2148
+
2149
+ bins_dict = self.get_final_bins()
2150
+ os.makedirs(os.path.dirname(os.path.abspath(report_path)), exist_ok=True)
2151
+
2152
+ em = ExcelMaster(report_path, verbose=False, gap_number=1)
2153
+ wb = em.workbook # 底层 xlsxwriter workbook,用于自定义数字 / 颜色格式
2154
+
2155
+ # 自定义单元格格式(set_cell_format 可直接接受 format 对象)
2156
+ _base = {"border": 1, "align": "center", "valign": "vcenter",
2157
+ "font_name": "Calibri", "font_size": 9}
2158
+ fmt_pct = wb.add_format({**_base, "num_format": "0.00%"})
2159
+ fmt_num4 = wb.add_format({**_base, "num_format": "0.0000"})
2160
+ fmt_num2 = wb.add_format({**_base, "num_format": "0.00"})
2161
+ # 特殊值行:仅叠加紫底紫字(与上面的数字格式叠加生效)
2162
+ fmt_sv = wb.add_format({"bg_color": "#E8D5F5",
2163
+ "font_color": "#5B2C6F", "bold": True})
2164
+
2165
+ # ═══════════════════════════════════════════════════════════════
2166
+ # Sheet 1: WOE分箱明细
2167
+ # ═══════════════════════════════════════════════════════════════
2168
+ ws = em.add_worksheet("WOE分箱明细")
2169
+
2170
+ # 顶部总标题
2171
+ em.merge_col(ws, ncols=13, text="各特征 WOE 分箱明细", cformat="BLUE_H1")
2172
+
2173
+ # ── 汇总表 ──
2174
+ summary_rows = []
2175
+ for i, (feat, vr) in enumerate(self._results.items(), 1):
2176
+ wt = vr["woe_table"]
2177
+ sv_t = vr.get("sv_table", pd.DataFrame())
2178
+ woes = wt.sort_values("bin")["woe"].values if len(wt) > 0 else np.array([])
2179
+ if vr.get("is_categorical"):
2180
+ direction = "类别"
2181
+ else:
2182
+ direction = "↑ 递增" if (len(woes) >= 2 and woes[-1] > woes[0]) else "↓ 递减"
2183
+ summary_rows.append({
2184
+ "序号": i, "特征名": feat, "普通箱数": vr["n_bins"],
2185
+ "特殊值箱数": len(sv_t) if len(sv_t) > 0 else 0,
2186
+ "总IV值": round(float(vr["iv"]), 4),
2187
+ "WOE方向": direction,
2188
+ "是否单调": "✓" if vr["is_monotonic"] else "✗",
2189
+ })
2190
+ summary_df = pd.DataFrame(summary_rows)
2191
+ em.write_dataframe(ws, summary_df, title="▌ 汇总:各特征 IV 一览",
2192
+ index=False, header=True)
2193
+
2194
+ # ── 逐特征明细 ──
2195
+ col_rename = {
2196
+ "bin_no": "分箱编号", "bin_label": "分箱区间", "n": "样本量",
2197
+ "bad": "坏样本数", "good": "好样本数", "bad_rate": "箱内坏样本率",
2198
+ "pct_n": "样本占比", "lift": "Lift", "pct_bad": "坏件分布占比",
2199
+ "pct_good": "好件分布占比", "woe": "WOE值", "iv": "IV贡献", "cumiv": "累计IV",
2200
+ }
2201
+ order = list(col_rename.keys())
2202
+ pct_cn = {"箱内坏样本率", "样本占比", "坏件分布占比", "好件分布占比"}
2203
+ num4_cn = {"WOE值", "IV贡献", "累计IV"}
2204
+ num2_cn = {"Lift"}
2205
+
2206
+ for seq, (feat, wt_df) in enumerate(bins_dict.items(), 1):
2207
+ vr = self._results[feat]
2208
+ cols = [c for c in order if c in wt_df.columns]
2209
+ ddf = wt_df[cols].rename(columns=col_rename)
2210
+
2211
+ sv_mask = (wt_df["is_special"].astype(bool).tolist()
2212
+ if "is_special" in wt_df.columns else [False] * len(wt_df))
2213
+ n_normal = vr["n_bins"]
2214
+ sv_hint = f" | 特殊值箱={sum(sv_mask)}" if any(sv_mask) else ""
2215
+ title = (f" [{seq:02d}] {feat} | IV={vr['iv']:.4f} "
2216
+ f"| 普通箱={vr['n_bins']}{sv_hint} "
2217
+ f"| 单调={'✓' if vr['is_monotonic'] else '✗'}")
2218
+
2219
+ loc = em.write_dataframe(ws, ddf, title=title, index=False,
2220
+ header=True, titleformat="BLUE_H4",
2221
+ retCellRange="value")
2222
+ r0, c0, r1, c1 = loc
2223
+ data_r0 = r0 + 2 # 跳过 title(1) + header(1)
2224
+ data_r1 = r1
2225
+ colpos = {name: c0 + idx for idx, name in enumerate(ddf.columns)}
2226
+
2227
+ # 列数字格式
2228
+ for cn, fmt in ([(c, fmt_pct) for c in pct_cn]
2229
+ + [(c, fmt_num4) for c in num4_cn]
2230
+ + [(c, fmt_num2) for c in num2_cn]):
2231
+ if cn in colpos:
2232
+ cc = colpos[cn]
2233
+ em.set_cell_format(ws, [data_r0, cc, data_r1, cc], fmt)
2234
+
2235
+ # WOE / Lift 列色阶(仅普通箱行,避免特殊值极端值干扰色阶)
2236
+ if n_normal > 0:
2237
+ nb_r1 = data_r0 + n_normal - 1
2238
+ if "WOE值" in colpos:
2239
+ cc = colpos["WOE值"]
2240
+ em.set_color_scale(ws, [data_r0, cc, nb_r1, cc],
2241
+ colors=("#F4B183", "#FFFFFF", "#A9D08E"))
2242
+ if "Lift" in colpos:
2243
+ cc = colpos["Lift"]
2244
+ em.set_color_scale(ws, [data_r0, cc, nb_r1, cc],
2245
+ colors=("#9DC3E6", "#FFFFFF", "#F4B183"))
2246
+
2247
+ # 特殊值行整行紫色(叠加在数字格式之上)
2248
+ for ri, is_sv in enumerate(sv_mask):
2249
+ if is_sv:
2250
+ rr = data_r0 + ri
2251
+ em.set_cell_format(ws, [rr, c0, rr, c1], fmt_sv)
2252
+
2253
+ # 特殊值图例注释
2254
+ if any(sv_mask):
2255
+ em.merge_col(ws, ncols=13,
2256
+ text=" ★ 紫色行为特殊值独立分箱,WOE 独立计算,不参与单调约束",
2257
+ cformat="TEXT_ITALIC")
2258
+
2259
+ # ═══════════════════════════════════════════════════════════════
2260
+ # Sheet 2: WOE分箱图(每个特征嵌入整体 WOE 图)
2261
+ # ═══════════════════════════════════════════════════════════════
2262
+ ws2 = em.add_worksheet("WOE分箱图")
2263
+ em.merge_col(ws2, ncols=11, text="各特征 WOE 分箱图(整体)", cformat="BLUE_H1")
2264
+
2265
+ # 注意: xlsxwriter 的 insert_image 延迟到 close 时才读取图片文件,
2266
+ # 因此 close_workbook() 必须在 tempdir 仍存活时调用。
2267
+ with tempfile.TemporaryDirectory() as tmpdir:
2268
+ for feat_idx, (feat, wt_df) in enumerate(bins_dict.items()):
2269
+ vr = self._results[feat]
2270
+ normal_df = (wt_df[~wt_df["is_special"].astype(bool)]
2271
+ if "is_special" in wt_df.columns else wt_df)
2272
+ sv_df = (wt_df[wt_df["is_special"].astype(bool)]
2273
+ if "is_special" in wt_df.columns else pd.DataFrame())
2274
+
2275
+ # 渲染整体 WOE 图并落盘(insert_image 需要文件路径)
2276
+ img_buf = self._render_woe_chart(
2277
+ feat, normal_df, sv_df, vr, dpi=120, figsize=(9, 4.5),
2278
+ )
2279
+ safe_feat = feat.replace("/", "_").replace("\\", "_")
2280
+ img_path = os.path.join(tmpdir, f"{safe_feat}.png")
2281
+ with open(img_path, "wb") as f:
2282
+ f.write(img_buf.getbuffer())
2283
+
2284
+ # 特征标题
2285
+ em.merge_col(
2286
+ ws2, ncols=11,
2287
+ text=(f" [{feat_idx+1:02d}] {feat} | IV={vr['iv']:.4f} "
2288
+ f"| 普通箱={vr['n_bins']} "
2289
+ f"| 单调={'✓' if vr['is_monotonic'] else '✗'}"
2290
+ + (f" | 特殊值箱={len(sv_df)}" if len(sv_df) > 0 else "")),
2291
+ cformat="BLUE_H4",
2292
+ )
2293
+ # 插入图片(缩放到合适大小)
2294
+ em.insert_image(ws2, figPath=img_path, figScale=(0.62, 0.62),
2295
+ skipby="row")
2296
+
2297
+ em.close_workbook()
2298
+
2299
+ logger.info(f"[export_woe_report] 报告已保存至: {report_path} "
2300
+ f"(ExcelMaster, 含图片Sheet)")
2301
+
2302
+ def _render_woe_chart(
2303
+ self,
2304
+ feat: str,
2305
+ normal_df: pd.DataFrame,
2306
+ sv_df: pd.DataFrame,
2307
+ vr: Dict,
2308
+ dpi: int = 120,
2309
+ figsize: tuple = (9, 4.5),
2310
+ ) -> io.BytesIO:
2311
+ """
2312
+ 渲染单个特征的 WOE 复合图,返回 BytesIO(PNG 格式)。
2313
+ 普通箱:stacked 柱图 + WOE 折线 + 标注框
2314
+ 特殊值箱:独立虚线柱(右侧追加)+ 标注框,使用不同颜色
2315
+ """
2316
+ GOOD_COLOR = "#5BBCD6"
2317
+ BAD_COLOR = "#F4856A"
2318
+ WOE_COLOR = "#2E75B6"
2319
+ SV_GOOD = "#A8D8A8" # 特殊值箱好样本(浅绿)
2320
+ SV_BAD = "#F7B7A3" # 特殊值箱坏样本(浅橙)
2321
+ SV_WOE = "#8E44AD" # 特殊值箱 WOE 线(紫)
2322
+
2323
+ n_normal = len(normal_df)
2324
+ n_sv = len(sv_df)
2325
+ n_total = n_normal + n_sv
2326
+
2327
+ if n_total == 0:
2328
+ fig, ax = plt.subplots(figsize=figsize)
2329
+ ax.text(0.5, 0.5, "No data", ha="center", va="center")
2330
+ buf = io.BytesIO()
2331
+ plt.savefig(buf, format="png", dpi=dpi, bbox_inches="tight")
2332
+ plt.close(fig)
2333
+ buf.seek(0)
2334
+ return buf
2335
+
2336
+ fig, ax_bar = plt.subplots(figsize=figsize)
2337
+ ax_woe = ax_bar.twinx()
2338
+
2339
+ x_normal = np.arange(n_normal)
2340
+ x_sv = np.arange(n_normal, n_total)
2341
+ x_all = np.arange(n_total)
2342
+
2343
+ # ── 普通箱柱图 ──
2344
+ pct_bad_n = normal_df["pct_bad"].values if n_normal > 0 else np.array([])
2345
+ pct_good_n = normal_df["pct_good"].values if n_normal > 0 else np.array([])
2346
+ if n_normal > 0:
2347
+ ax_bar.bar(x_normal, pct_good_n, color=GOOD_COLOR, alpha=0.85,
2348
+ label="0 (Good)", width=0.6, zorder=2)
2349
+ ax_bar.bar(x_normal, pct_bad_n, bottom=pct_good_n, color=BAD_COLOR,
2350
+ alpha=0.85, label="1 (Bad)", width=0.6, zorder=2)
2351
+
2352
+ # ── 特殊值箱柱图(虚边框区分)──
2353
+ pct_bad_sv = sv_df["pct_bad"].values if n_sv > 0 else np.array([])
2354
+ pct_good_sv = sv_df["pct_good"].values if n_sv > 0 else np.array([])
2355
+ if n_sv > 0:
2356
+ ax_bar.bar(x_sv, pct_good_sv, color=SV_GOOD, alpha=0.85,
2357
+ width=0.6, zorder=2, edgecolor="#888", linewidth=1.2,
2358
+ linestyle="--", label="0 (sv)")
2359
+ ax_bar.bar(x_sv, pct_bad_sv, bottom=pct_good_sv, color=SV_BAD,
2360
+ alpha=0.85, width=0.6, zorder=2, edgecolor="#888",
2361
+ linewidth=1.2, linestyle="--", label="1 (sv)")
2362
+
2363
+ # 特殊值箱分隔线(用数据坐标,避免 bbox_inches='tight' 拉伸)
2364
+ ax_bar.axvline(x=n_normal - 0.5, color="#888", linewidth=1.2,
2365
+ linestyle=":", zorder=3, label="_nolegend_")
2366
+ ax_bar.text(n_normal - 0.35, 0.93, "Special",
2367
+ fontsize=7.5, color="#8E44AD", va="top",
2368
+ transform=ax_bar.transData)
2369
+
2370
+ # ── 普通箱 WOE 线 ──
2371
+ woe_n = normal_df["woe"].values if n_normal > 0 else np.array([])
2372
+ br_n = normal_df["bad_rate"].values if n_normal > 0 else np.array([])
2373
+ if n_normal > 0:
2374
+ ax_woe.plot(x_normal, woe_n, color=WOE_COLOR, marker="o",
2375
+ linewidth=2, markersize=6, zorder=5, label="WOE (normal)")
2376
+
2377
+ # ── 特殊值箱 WOE 线(断开,虚线)──
2378
+ woe_sv = sv_df["woe"].values if n_sv > 0 else np.array([])
2379
+ br_sv = sv_df["bad_rate"].values if n_sv > 0 else np.array([])
2380
+ if n_sv > 0:
2381
+ ax_woe.plot(x_sv, woe_sv, color=SV_WOE, marker="D",
2382
+ linewidth=1.5, markersize=6, linestyle="--",
2383
+ zorder=5, label="WOE (special)")
2384
+
2385
+ # ── 标注框(普通箱)──
2386
+ bar_ylim_max = 1.0
2387
+ label_offset = 0.04
2388
+ label_margin = 0.02
2389
+
2390
+ def _annotate(ax_b, ax_w, xi, wv, br, pt_good, pt_bad,
2391
+ lift=None, pct_n=None, fc="white", ec="black"):
2392
+ lines = [f"WOE: {wv:.3f}", f"BR: {br:.2%}"]
2393
+ if lift is not None:
2394
+ lines.append(f"Lift: {lift:.2f}x | {pct_n:.1%}"
2395
+ if pct_n is not None else f"Lift: {lift:.2f}x")
2396
+ label_txt = "\n".join(lines)
2397
+ bar_top = pt_good + pt_bad
2398
+ y_above = bar_top + label_offset
2399
+ if y_above + label_margin <= bar_ylim_max:
2400
+ y_text = y_above
2401
+ va_text = "bottom"
2402
+ else:
2403
+ y_text = min(bar_ylim_max - label_margin, bar_top) - 0.02
2404
+ va_text = "top"
2405
+ wv_clamped = max(-1.0, min(1.0, wv))
2406
+ ax_b.annotate(
2407
+ label_txt,
2408
+ xy=(xi, wv_clamped), xycoords=("data", ax_w.transData),
2409
+ xytext=(xi, y_text),
2410
+ textcoords=("data", ax_b.transData),
2411
+ fontsize=6.5, va=va_text, ha="center",
2412
+ bbox=dict(boxstyle="round,pad=0.3", fc=fc, ec=ec, lw=0.7),
2413
+ arrowprops=dict(arrowstyle="-", color=ec, lw=0.7),
2414
+ )
2415
+
2416
+ lift_n = normal_df["lift"].values if "lift" in normal_df.columns and n_normal > 0 else [None]*n_normal
2417
+ pct_n_n = normal_df["pct_n"].values if "pct_n" in normal_df.columns and n_normal > 0 else [None]*n_normal
2418
+
2419
+ for xi, (wv, br, pg, pb, lv, pn) in enumerate(
2420
+ zip(woe_n, br_n, pct_good_n, pct_bad_n, lift_n, pct_n_n)):
2421
+ _annotate(ax_bar, ax_woe, xi, wv, br, pg, pb, lift=lv, pct_n=pn)
2422
+
2423
+ # 特殊值箱标注(紫色,复用 _annotate)
2424
+ lift_sv = sv_df["lift"].values if "lift" in sv_df.columns and n_sv > 0 else [None]*n_sv
2425
+ pct_n_sv = sv_df["pct_n"].values if "pct_n" in sv_df.columns and n_sv > 0 else [None]*n_sv
2426
+ for i, (xi, wv, br, pg, pb, lv, pn) in enumerate(
2427
+ zip(x_sv, woe_sv, br_sv, pct_good_sv, pct_bad_sv, lift_sv, pct_n_sv)):
2428
+ _annotate(ax_bar, ax_woe, xi, wv, br, pg, pb,
2429
+ lift=lv, pct_n=pn, fc="#F5EEF8", ec="#8E44AD")
2430
+
2431
+ # ── 轴格式 ──
2432
+ all_labels = (
2433
+ [str(b) for b in normal_df["bin_label"]]
2434
+ + ([str(b) for b in sv_df["bin_label"]] if n_sv > 0 else [])
2435
+ )
2436
+ ax_bar.set_xlim(-0.5, n_total - 0.5)
2437
+ ax_bar.set_ylim(0, 1.0)
2438
+ ax_woe.set_ylim(-1.0, 1.0)
2439
+ ax_woe.axhline(0, color="gray", linewidth=0.6, linestyle="--", zorder=1)
2440
+
2441
+ ax_bar.set_xticks(x_all)
2442
+ ax_bar.set_xticklabels(
2443
+ [textwrap.fill(lb, width=13) for lb in all_labels],
2444
+ fontsize=7.5, rotation=30, ha="right",
2445
+ )
2446
+ ax_bar.set_ylabel("Proportion", fontsize=9)
2447
+ ax_bar.yaxis.set_major_formatter(mticker.PercentFormatter(xmax=1, decimals=0))
2448
+ ax_bar.tick_params(axis="y", labelsize=8)
2449
+ ax_woe.set_ylabel("WOE", fontsize=9, color="#333")
2450
+ ax_woe.tick_params(axis="y", labelsize=8)
2451
+ ax_bar.grid(axis="y", alpha=0.3, zorder=0)
2452
+ ax_bar.set_axisbelow(True)
2453
+
2454
+ # 图例定位策略:
2455
+ # 有 sv 时:两个 legend 合并放在 special 区域内当前最右一个 sv 笔的正上方(y=1.0 上边界)
2456
+ # 无 sv 时:Target legend 放右上角
2457
+ handles, labels = ax_bar.get_legend_handles_labels()
2458
+ if n_sv > 0:
2459
+ # 合并 bar 和 woe 的全部句柄 → 一个 legend
2460
+ woe_handles = [l for l in ax_woe.get_lines()
2461
+ if not l.get_label().startswith("_")]
2462
+ all_handles = handles + woe_handles
2463
+ all_labels = labels + [l.get_label() for l in woe_handles]
2464
+ # 锚点在最右 sv bin 的中心 x, y=1.0(ax_bar 上边界)
2465
+ anchor_x = n_total - 1.0 # 最右一个 bin 的 x
2466
+ ax_bar.legend(all_handles, all_labels,
2467
+ loc="upper right",
2468
+ bbox_to_anchor=(anchor_x, 1.0),
2469
+ bbox_transform=ax_bar.transData,
2470
+ fontsize=7.0, framealpha=0.88,
2471
+ ncol=2, borderpad=0.35, handlelength=1.2)
2472
+ else:
2473
+ ax_bar.legend(handles, labels,
2474
+ loc="upper right",
2475
+ fontsize=7.0, framealpha=0.85,
2476
+ title="Target", title_fontsize=7.0,
2477
+ ncol=2, borderpad=0.35, handlelength=1.2)
2478
+
2479
+ iv_total = vr["iv"]
2480
+ plt.title(f"{feat}: IV={iv_total:.3f}", fontsize=11, fontweight="bold", pad=8)
2481
+ plt.tight_layout()
2482
+
2483
+ buf = io.BytesIO()
2484
+ plt.savefig(buf, format="png", dpi=dpi, bbox_inches="tight")
2485
+ plt.close(fig)
2486
+ buf.seek(0)
2487
+ return buf
2488
+
2489
+ # ── 4. plot_woe_graph ────────────────────────────────────────────
2490
+
2491
+ def plot_woe_graph(
2492
+ self,
2493
+ graph_path: str,
2494
+ group_name: Optional[str] = None,
2495
+ _df_for_group: Optional[pd.DataFrame] = None,
2496
+ dpi: int = 150,
2497
+ figsize: tuple = (9, 6),
2498
+ bar_mode: str = "clustered",
2499
+ ) -> None:
2500
+ """
2501
+ 为每个特征绘制复合图(线图 + Stack 柱图),保存到 graph_path 目录。
2502
+
2503
+ 整体图(group_name=None):
2504
+ - 普通箱 stacked 柱图 + WOE 折线 + 标注框
2505
+ - 若有特殊值箱,追加在右侧(虚边框 + 紫色标注)
2506
+ - 标题:"{feat}: IV={iv:.3f}"
2507
+
2508
+ 分组图(group_name 不为 None):每个 group 一条 WOE 折线(仅普通箱),
2509
+ 柱图样式由 bar_mode 控制:
2510
+ - "pooled" : 单套柱,全量好坏占比(占全量样本)
2511
+ - "clustered" : 每个箱位置并排各组柱,柱高=占【该组】总样本(默认)
2512
+ - "small_multiples" : 每个 group 一个子图 panel,各画该组组内占比柱
2513
+ + 该组 WOE 线(WOE y 轴跨 panel 统一,便于对比)
2514
+ - 标题:"{feat}: IV_range={min}−{max}"
2515
+
2516
+ Parameters
2517
+ ----------
2518
+ graph_path : 图片保存目录(自动创建)
2519
+ group_name : 分组列名(如 "month"),None = 画整体图
2520
+ _df_for_group : 含原始特征+target+group_name 的 DataFrame(group 模式必填)
2521
+ dpi : 图片分辨率,默认 150
2522
+ figsize : 图片尺寸,默认 (9, 6)。small_multiples 模式下为单个 panel
2523
+ 的基准尺寸,整图按子图网格自动放大
2524
+ bar_mode : 分组图柱样式,"pooled" | "clustered" | "small_multiples",
2525
+ 默认 "clustered"。group_name=None(整体图)时此参数忽略
2526
+ """
2527
+ self._check_fitted()
2528
+ _valid_bar_modes = {"pooled", "clustered", "small_multiples"}
2529
+ if bar_mode not in _valid_bar_modes:
2530
+ raise ValueError(
2531
+ f"bar_mode 必须是 {_valid_bar_modes} 之一,收到: {bar_mode!r}"
2532
+ )
2533
+ os.makedirs(graph_path, exist_ok=True)
2534
+ bins_dict = self.get_final_bins()
2535
+
2536
+ GOOD_COLOR = "#5BBCD6"
2537
+ BAD_COLOR = "#F4856A"
2538
+ WOE_COLOR_OVERALL = "#2E75B6"
2539
+ SV_GOOD = "#A8D8A8"
2540
+ SV_BAD = "#F7B7A3"
2541
+ SV_WOE = "#8E44AD"
2542
+
2543
+ for feat, wt_df in bins_dict.items():
2544
+ vr = self._results[feat]
2545
+ edges = [float(e) for e in vr["edges"]]
2546
+
2547
+ # 分离普通箱和特殊值箱
2548
+ if "is_special" in wt_df.columns:
2549
+ normal_df = wt_df[~wt_df["is_special"].astype(bool)].copy()
2550
+ sv_df = wt_df[wt_df["is_special"].astype(bool)].copy()
2551
+ else:
2552
+ normal_df = wt_df.copy()
2553
+ sv_df = pd.DataFrame()
2554
+
2555
+ n_normal = len(normal_df)
2556
+ n_sv = len(sv_df)
2557
+ n_total = n_normal + n_sv
2558
+ iv_overall = vr["iv"]
2559
+
2560
+ x_normal = np.arange(n_normal)
2561
+ x_sv = np.arange(n_normal, n_total)
2562
+ x_all = np.arange(n_total)
2563
+
2564
+ # ── small_multiples 独立路径:每组一个子图,自建多子图 figure ──
2565
+ if group_name is not None and bar_mode == "small_multiples":
2566
+ self._plot_feat_small_multiples(
2567
+ feat, normal_df, sv_df,
2568
+ n_normal, n_sv, n_total, x_normal, x_sv, x_all,
2569
+ group_name, _df_for_group, graph_path, dpi, figsize,
2570
+ GOOD_COLOR, BAD_COLOR, SV_GOOD, SV_BAD, SV_WOE,
2571
+ )
2572
+ continue
2573
+
2574
+ fig, ax_bar = plt.subplots(figsize=figsize)
2575
+ ax_woe = ax_bar.twinx()
2576
+
2577
+ pct_bad_n = normal_df["pct_bad"].values if n_normal > 0 else np.array([])
2578
+ pct_good_n = normal_df["pct_good"].values if n_normal > 0 else np.array([])
2579
+ pct_bad_sv = sv_df["pct_bad"].values if n_sv > 0 else np.array([])
2580
+ pct_good_sv = sv_df["pct_good"].values if n_sv > 0 else np.array([])
2581
+
2582
+ if group_name is None:
2583
+ # ── 整体图 ──
2584
+ if n_normal > 0:
2585
+ ax_bar.bar(x_normal, pct_good_n, color=GOOD_COLOR,
2586
+ alpha=0.85, label="0", width=0.6, zorder=2)
2587
+ ax_bar.bar(x_normal, pct_bad_n, bottom=pct_good_n,
2588
+ color=BAD_COLOR, alpha=0.85, label="1",
2589
+ width=0.6, zorder=2)
2590
+
2591
+ if n_sv > 0:
2592
+ ax_bar.bar(x_sv, pct_good_sv, color=SV_GOOD, alpha=0.85,
2593
+ width=0.6, zorder=2, edgecolor="#888",
2594
+ linewidth=1.2, linestyle="--")
2595
+ ax_bar.bar(x_sv, pct_bad_sv, bottom=pct_good_sv, color=SV_BAD,
2596
+ alpha=0.85, width=0.6, zorder=2, edgecolor="#888",
2597
+ linewidth=1.2, linestyle="--")
2598
+ ax_bar.axvline(x=n_normal - 0.5, color="#888",
2599
+ linewidth=1.2, linestyle=":", zorder=3)
2600
+ # 注意: 用数据坐标而非 get_xaxis_transform(),避免 bbox_inches='tight' 拉伸
2601
+ ax_bar.text(n_normal - 0.35, 0.93, "Special",
2602
+ fontsize=8, color=SV_WOE, va="top",
2603
+ transform=ax_bar.transData)
2604
+
2605
+ # WOE 线(普通箱)
2606
+ woe_n = normal_df["woe"].values if n_normal > 0 else np.array([])
2607
+ br_n = normal_df["bad_rate"].values if n_normal > 0 else np.array([])
2608
+ if n_normal > 0:
2609
+ ax_woe.plot(x_normal, woe_n, color=WOE_COLOR_OVERALL,
2610
+ marker="o", linewidth=2, markersize=6, zorder=5)
2611
+
2612
+ # WOE 线(特殊值箱)
2613
+ woe_sv = sv_df["woe"].values if n_sv > 0 else np.array([])
2614
+ br_sv = sv_df["bad_rate"].values if n_sv > 0 else np.array([])
2615
+ if n_sv > 0:
2616
+ ax_woe.plot(x_sv, woe_sv, color=SV_WOE, marker="D",
2617
+ linewidth=1.5, markersize=6, linestyle="--", zorder=5)
2618
+
2619
+ # 标注框(普通箱)
2620
+ bar_ylim_max = 1.0
2621
+ label_offset = 0.04
2622
+ label_margin = 0.02
2623
+
2624
+ lift_nn = normal_df["lift"].values if "lift" in normal_df.columns and n_normal > 0 else [None]*n_normal
2625
+ pct_n_nn = normal_df["pct_n"].values if "pct_n" in normal_df.columns and n_normal > 0 else [None]*n_normal
2626
+ for xi, (wv, br, lv, pn) in enumerate(zip(woe_n, br_n, lift_nn, pct_n_nn)):
2627
+ lines_txt = [f"WOE: {wv:.3f}", f"BR: {br:.2%}"]
2628
+ if lv is not None:
2629
+ lines_txt.append(f"Lift: {lv:.2f}x | {pn:.1%}"
2630
+ if pn is not None else f"Lift: {lv:.2f}x")
2631
+ label_txt = "\n".join(lines_txt)
2632
+ bar_top = pct_good_n[xi] + pct_bad_n[xi]
2633
+ y_above = bar_top + label_offset
2634
+ if y_above + label_margin <= bar_ylim_max:
2635
+ y_text, va_text = y_above, "bottom"
2636
+ else:
2637
+ y_text = min(bar_ylim_max - label_margin, bar_top) - 0.02
2638
+ va_text = "top"
2639
+ ax_bar.annotate(
2640
+ label_txt,
2641
+ xy=(xi, wv), xycoords=("data", ax_woe.transData),
2642
+ xytext=(xi, y_text),
2643
+ textcoords=("data", ax_bar.transData),
2644
+ fontsize=7.0, va=va_text, ha="center",
2645
+ zorder=10,
2646
+ bbox=dict(boxstyle="round,pad=0.3", fc="white",
2647
+ ec="black", lw=0.8, zorder=10),
2648
+ arrowprops=dict(arrowstyle="-", color="black", lw=0.8),
2649
+ )
2650
+
2651
+ # 标注框(特殊值箱,紫色)
2652
+ lift_sv2 = sv_df["lift"].values if "lift" in sv_df.columns and n_sv > 0 else [None]*n_sv
2653
+ pct_n_sv2 = sv_df["pct_n"].values if "pct_n" in sv_df.columns and n_sv > 0 else [None]*n_sv
2654
+ for xi, (wv, br, pg, pb, lv, pn) in enumerate(
2655
+ zip(woe_sv, br_sv, pct_good_sv, pct_bad_sv, lift_sv2, pct_n_sv2)):
2656
+ xi_abs = n_normal + xi
2657
+ lines_sv = [f"WOE: {wv:.3f}", f"BR: {br:.2%}"]
2658
+ if lv is not None:
2659
+ lines_sv.append(f"Lift: {lv:.2f}x | {pn:.1%}"
2660
+ if pn is not None else f"Lift: {lv:.2f}x")
2661
+ label_txt = "\n".join(lines_sv)
2662
+ bar_top = pg + pb
2663
+ y_above = bar_top + label_offset
2664
+ if y_above + label_margin <= bar_ylim_max:
2665
+ y_text, va_text = y_above, "bottom"
2666
+ else:
2667
+ y_text = min(bar_ylim_max - label_margin, bar_top) - 0.02
2668
+ va_text = "top"
2669
+ ax_bar.annotate(
2670
+ label_txt,
2671
+ xy=(xi_abs, wv), xycoords=("data", ax_woe.transData),
2672
+ xytext=(xi_abs, y_text),
2673
+ textcoords=("data", ax_bar.transData),
2674
+ fontsize=7.0, va=va_text, ha="center",
2675
+ zorder=10,
2676
+ bbox=dict(boxstyle="round,pad=0.3", fc="#F5EEF8",
2677
+ ec=SV_WOE, lw=0.9, zorder=10),
2678
+ arrowprops=dict(arrowstyle="-", color=SV_WOE, lw=0.8),
2679
+ )
2680
+
2681
+ # Legend: sv 区域右上角内部,有 sv 时合并两个 legend
2682
+ bar_handles, bar_labels = ax_bar.get_legend_handles_labels()
2683
+ if n_sv > 0:
2684
+ woe_handles2 = [l for l in ax_woe.get_lines()
2685
+ if not l.get_label().startswith("_")]
2686
+ all_h = bar_handles + woe_handles2
2687
+ all_l = bar_labels + [l.get_label() for l in woe_handles2]
2688
+ anchor_x2 = n_total - 1.0
2689
+ ax_bar.legend(all_h, all_l,
2690
+ loc="upper right",
2691
+ bbox_to_anchor=(anchor_x2, 1.0),
2692
+ bbox_transform=ax_bar.transData,
2693
+ fontsize=8.0, framealpha=0.88,
2694
+ ncol=2, borderpad=0.35, handlelength=1.2)
2695
+ else:
2696
+ ax_bar.legend(bar_handles, bar_labels,
2697
+ loc="upper right",
2698
+ fontsize=8.0, framealpha=0.88,
2699
+ title="Target", title_fontsize=8.0,
2700
+ ncol=2, borderpad=0.35)
2701
+ ax_woe.set_ylabel("WOE (TargetRate)", fontsize=9, color="#333")
2702
+ title = f"{feat}: IV={iv_overall:.3f}"
2703
+
2704
+ else:
2705
+ # ── By-group 图 ──
2706
+ if _df_for_group is None:
2707
+ logger.info(f" [WARN] group_name='{group_name}' 需要传入 _df_for_group,跳过 {feat}")
2708
+ plt.close(fig)
2709
+ continue
2710
+ if group_name not in _df_for_group.columns or feat not in _df_for_group.columns:
2711
+ logger.info(f" [WARN] group '{group_name}' 或 feature '{feat}' 不在 DataFrame 中")
2712
+ plt.close(fig)
2713
+ continue
2714
+
2715
+ # ── 用拟合好的 edges 对各 group 分别分箱 ──
2716
+ fitted_edges = list(vr["edges"])
2717
+ eps = self.eps
2718
+
2719
+ groups = sorted(_df_for_group[group_name].dropna().unique())
2720
+ n_groups = len(groups)
2721
+ # tab10 调色板,最多 10 种颜色循环
2722
+ cmap_colors = plt.cm.tab10(np.linspace(0, 0.9, min(max(n_groups,1), 10)))
2723
+ group_ivs = []
2724
+
2725
+ # WOE 基准:全量 total_bad / total_good(各组 WOE 相对全量,保证跨组可比)
2726
+ all_normal_df, all_sv_groups = self._split_special_for_plot(_df_for_group, feat, vr)
2727
+ all_normal_sub = all_normal_df[[feat, self.target_col]].dropna(subset=[feat]).copy()
2728
+ all_normal_sub["_bin"] = self._assign_normal_bins(
2729
+ all_normal_sub, feat, vr, fitted_edges)
2730
+ all_normal_sub = all_normal_sub[all_normal_sub["_bin"].notna()]
2731
+ all_total_bad = float(all_normal_sub[self.target_col].sum())
2732
+ all_total_good = float((all_normal_sub[self.target_col] == 0).sum())
2733
+
2734
+ # ── 柱图模式:pooled(全量单套柱)vs clustered(各组并排柱)──
2735
+ if bar_mode == "pooled":
2736
+ # 全量普通箱比例(分母 = 全量普通行数)
2737
+ all_n_full = len(all_normal_sub)
2738
+ pct_good_n_grp = np.zeros(n_normal)
2739
+ pct_bad_n_grp = np.zeros(n_normal)
2740
+ for b in range(n_normal):
2741
+ grp_b = all_normal_sub[all_normal_sub["_bin"] == b]
2742
+ bad_b = float(grp_b[self.target_col].sum())
2743
+ good_b = float((grp_b[self.target_col] == 0).sum())
2744
+ pct_good_n_grp[b] = good_b / (all_n_full + eps) if all_n_full > 0 else 0.0
2745
+ pct_bad_n_grp[b] = bad_b / (all_n_full + eps) if all_n_full > 0 else 0.0
2746
+ # 全量特殊值箱比例(分母 = 全量行数)
2747
+ all_sv_n = len(_df_for_group)
2748
+ pct_good_sv_grp = np.zeros(n_sv)
2749
+ pct_bad_sv_grp = np.zeros(n_sv)
2750
+ if n_sv > 0:
2751
+ for si, sv_row in enumerate(sv_df.itertuples()):
2752
+ matched_sv_df = None
2753
+ for sv_key, sv_sub in all_sv_groups.items():
2754
+ if _sv_label(sv_key) == sv_row.bin_label:
2755
+ matched_sv_df = sv_sub
2756
+ break
2757
+ if matched_sv_df is not None and len(matched_sv_df) > 0:
2758
+ pct_good_sv_grp[si] = float((matched_sv_df[self.target_col] == 0).sum()) / (all_sv_n + eps)
2759
+ pct_bad_sv_grp[si] = float(matched_sv_df[self.target_col].sum()) / (all_sv_n + eps)
2760
+ # 画全量单套柱
2761
+ if n_normal > 0:
2762
+ ax_bar.bar(x_normal, pct_good_n_grp, color=GOOD_COLOR,
2763
+ alpha=0.45, width=0.6, zorder=2)
2764
+ ax_bar.bar(x_normal, pct_bad_n_grp, bottom=pct_good_n_grp,
2765
+ color=BAD_COLOR, alpha=0.45, width=0.6, zorder=2)
2766
+ if n_sv > 0:
2767
+ ax_bar.bar(x_sv, pct_good_sv_grp, color=SV_GOOD, alpha=0.35,
2768
+ width=0.6, zorder=2, edgecolor="#888",
2769
+ linewidth=1.0, linestyle="--")
2770
+ ax_bar.bar(x_sv, pct_bad_sv_grp, bottom=pct_good_sv_grp,
2771
+ color=SV_BAD, alpha=0.35, width=0.6, zorder=2,
2772
+ edgecolor="#888", linewidth=1.0, linestyle="--")
2773
+ else: # clustered:每个箱位置并排 n_groups 根柱,柱宽均分簇宽
2774
+ cluster_w = 0.8 # 每个箱簇占据的总宽度
2775
+ bar_w = cluster_w / max(n_groups, 1)
2776
+
2777
+ # 特殊值分隔线 + 标注(pooled / clustered 公共)
2778
+ if n_sv > 0:
2779
+ ax_bar.axvline(x=n_normal - 0.5, color="#888",
2780
+ linewidth=1.0, linestyle=":", zorder=3)
2781
+ ax_bar.text(n_normal + 0.05, 0.93, "Special",
2782
+ fontsize=8, color=SV_WOE, va="top",
2783
+ transform=ax_bar.transData)
2784
+
2785
+ # 逐组:算 WOE 折线(clustered 时另画该组组内占比柱)
2786
+ for gi, grp_val in enumerate(groups):
2787
+ grp_df_full = _df_for_group[_df_for_group[group_name] == grp_val]
2788
+ n_grp = len(grp_df_full)
2789
+ tr = grp_df_full[self.target_col].mean() if n_grp > 0 else 0.0
2790
+ clr = cmap_colors[gi % len(cmap_colors)]
2791
+
2792
+ # 分离该组的特殊值与普通行,并按 edges/类别取值分箱
2793
+ grp_normal_df, grp_sv_groups = self._split_special_for_plot(grp_df_full, feat, vr)
2794
+ grp_sub = grp_normal_df[[feat, self.target_col]].dropna(subset=[feat]).copy()
2795
+ grp_sub["_bin"] = self._assign_normal_bins(grp_sub, feat, vr, fitted_edges)
2796
+ grp_sub = grp_sub[grp_sub["_bin"].notna()]
2797
+
2798
+ # 普通箱:组内占比(分母=该组总样本 n_grp)+ WOE(相对全量基准)
2799
+ pct_good_n_g = np.zeros(n_normal)
2800
+ pct_bad_n_g = np.zeros(n_normal)
2801
+ grp_woe = []
2802
+ grp_iv = 0.0
2803
+ for b in range(n_normal):
2804
+ bin_rows = grp_sub[grp_sub["_bin"] == b]
2805
+ bad_b = float(bin_rows[self.target_col].sum())
2806
+ good_b = float((bin_rows[self.target_col] == 0).sum())
2807
+ pct_good_n_g[b] = good_b / (n_grp + eps)
2808
+ pct_bad_n_g[b] = bad_b / (n_grp + eps)
2809
+ if len(bin_rows) == 0:
2810
+ grp_woe.append(np.nan)
2811
+ continue
2812
+ pct_bad_w = bad_b / (all_total_bad + eps)
2813
+ pct_good_w = good_b / (all_total_good + eps)
2814
+ woe_b = math.log((pct_bad_w + eps) / (pct_good_w + eps))
2815
+ grp_iv += (pct_bad_w - pct_good_w) * woe_b
2816
+ grp_woe.append(woe_b)
2817
+
2818
+ # ── clustered:画该组组内占比柱(边框用组色,与 WOE 折线对应)──
2819
+ if bar_mode == "clustered":
2820
+ x_off = -cluster_w / 2 + bar_w * (gi + 0.5)
2821
+ x_normal_g = x_normal + x_off
2822
+ x_sv_g = x_sv + x_off
2823
+
2824
+ pct_good_sv_g = np.zeros(n_sv)
2825
+ pct_bad_sv_g = np.zeros(n_sv)
2826
+ if n_sv > 0:
2827
+ for si, sv_row in enumerate(sv_df.itertuples()):
2828
+ matched_sv_df = None
2829
+ for sv_key, sv_sub in grp_sv_groups.items():
2830
+ if _sv_label(sv_key) == sv_row.bin_label:
2831
+ matched_sv_df = sv_sub
2832
+ break
2833
+ if matched_sv_df is not None and len(matched_sv_df) > 0:
2834
+ pct_good_sv_g[si] = float((matched_sv_df[self.target_col] == 0).sum()) / (n_grp + eps)
2835
+ pct_bad_sv_g[si] = float(matched_sv_df[self.target_col].sum()) / (n_grp + eps)
2836
+
2837
+ if n_normal > 0:
2838
+ ax_bar.bar(x_normal_g, pct_good_n_g, color=GOOD_COLOR,
2839
+ alpha=0.55, width=bar_w, zorder=2,
2840
+ edgecolor=clr, linewidth=0.8)
2841
+ ax_bar.bar(x_normal_g, pct_bad_n_g, bottom=pct_good_n_g,
2842
+ color=BAD_COLOR, alpha=0.55, width=bar_w, zorder=2,
2843
+ edgecolor=clr, linewidth=0.8)
2844
+ if n_sv > 0:
2845
+ ax_bar.bar(x_sv_g, pct_good_sv_g, color=SV_GOOD,
2846
+ alpha=0.5, width=bar_w, zorder=2,
2847
+ edgecolor=clr, linewidth=0.8, linestyle="--")
2848
+ ax_bar.bar(x_sv_g, pct_bad_sv_g, bottom=pct_good_sv_g,
2849
+ color=SV_BAD, alpha=0.5, width=bar_w, zorder=2,
2850
+ edgecolor=clr, linewidth=0.8, linestyle="--")
2851
+
2852
+ # ── 画该组 WOE 折线(画在箱中心 x_normal,便于跨组对齐对比)──
2853
+ if n_grp < 5 or grp_df_full[self.target_col].nunique() < 2:
2854
+ group_ivs.append(0.0)
2855
+ lbl = f"{grp_val} N={n_grp:,} TR={tr:.1%} IV=0.000"
2856
+ ax_woe.plot(x_normal, [np.nan] * n_normal,
2857
+ color=clr, linewidth=1.5, marker="o",
2858
+ markersize=4, zorder=5, label=lbl)
2859
+ else:
2860
+ group_ivs.append(round(grp_iv, 4))
2861
+ lbl = f"{grp_val} N={n_grp:,} TR={tr:.1%} IV={grp_iv:.3f}"
2862
+ ax_woe.plot(x_normal, grp_woe, color=clr,
2863
+ linewidth=1.5, marker="o", markersize=4,
2864
+ zorder=5, label=lbl)
2865
+
2866
+ # Legend: 所有条目合并,放坐标轴右侧外部(更靠右,避免遮挡 WOE y轴标题)
2867
+ _bar_desc = "pooled %" if bar_mode == "pooled" else "in-group %"
2868
+ dummy_good = plt.Rectangle((0,0),1,1, color=GOOD_COLOR, alpha=0.6)
2869
+ dummy_bad = plt.Rectangle((0,0),1,1, color=BAD_COLOR, alpha=0.6)
2870
+ woe_lines_h = [l for l in ax_woe.get_lines()
2871
+ if not l.get_label().startswith("_")]
2872
+ woe_labels_h = [l.get_label() for l in woe_lines_h]
2873
+ all_handles = [dummy_good, dummy_bad] + woe_lines_h
2874
+ all_labels_l = ["0 (Good)", "1 (Bad)"] + woe_labels_h
2875
+ ax_bar.legend(all_handles, all_labels_l,
2876
+ loc="center left",
2877
+ bbox_to_anchor=(1.18, 0.5),
2878
+ bbox_transform=ax_bar.transAxes,
2879
+ fontsize=7.5, framealpha=0.92,
2880
+ ncol=1, borderpad=0.5, handlelength=1.5,
2881
+ title=f"Group WOE (bar={_bar_desc})", title_fontsize=8.0)
2882
+
2883
+ iv_vals = [v for v in group_ivs if v > 0]
2884
+ iv_range = (f"{min(iv_vals):.3f}-{max(iv_vals):.3f}"
2885
+ if iv_vals else "0.000-0.000")
2886
+ title = f"{feat}: IV_range={iv_range}"
2887
+ ax_woe.set_ylabel("WOE", fontsize=9, color="#333")
2888
+
2889
+ # ── 通用轴格式 ──
2890
+ all_labels = (
2891
+ [str(b) for b in normal_df["bin_label"]]
2892
+ + ([str(b) for b in sv_df["bin_label"]] if n_sv > 0 else [])
2893
+ )
2894
+ ax_bar.set_xlim(-0.5, n_total - 0.5)
2895
+ ax_bar.set_ylim(0, 1.0)
2896
+
2897
+ # ── 动态计算 WOE y 轴范围(避免折线超出坐标轴)──
2898
+ # 收集所有已绘制折线上的有效 WOE 值
2899
+ _all_woe_pts = []
2900
+ for _line in ax_woe.get_lines():
2901
+ _yd = np.array(_line.get_ydata(), dtype=float)
2902
+ _all_woe_pts.extend(_yd[~np.isnan(_yd)].tolist())
2903
+ if _all_woe_pts:
2904
+ _woe_min = min(_all_woe_pts)
2905
+ _woe_max = max(_all_woe_pts)
2906
+ # padding: 15% of span, 最少 ±0.1 的绝对余量
2907
+ _span = max(_woe_max - _woe_min, 0.2)
2908
+ _pad = max(_span * 0.15, 0.1)
2909
+ _y_lo = _woe_min - _pad
2910
+ _y_hi = _woe_max + _pad
2911
+ # 至少覆盖 [-0.5, 0.5],且 0 刻度要在范围内
2912
+ _y_lo = min(_y_lo, -0.5)
2913
+ _y_hi = max(_y_hi, 0.5)
2914
+ else:
2915
+ _y_lo, _y_hi = -1.0, 1.0
2916
+ ax_woe.set_ylim(_y_lo, _y_hi)
2917
+ ax_woe.axhline(0, color="gray", linewidth=0.6, linestyle="--", zorder=1)
2918
+
2919
+ ax_bar.set_xticks(x_all)
2920
+ ax_bar.set_xticklabels(
2921
+ [textwrap.fill(lb, width=14) for lb in all_labels],
2922
+ fontsize=8, rotation=30, ha="right",
2923
+ )
2924
+ ax_bar.set_ylabel("Proportion", fontsize=9)
2925
+ ax_bar.yaxis.set_major_formatter(mticker.PercentFormatter(xmax=1, decimals=0))
2926
+ ax_bar.tick_params(axis="y", labelsize=8)
2927
+ ax_woe.tick_params(axis="y", labelsize=8)
2928
+ ax_bar.grid(axis="y", alpha=0.3, zorder=0)
2929
+ ax_bar.set_axisbelow(True)
2930
+
2931
+ plt.title(title, fontsize=11, fontweight="bold", pad=10)
2932
+ if group_name is not None:
2933
+ # by-group 模式:legend 在右侧外,预留右边空间(含 IV 文字,留更多右边距)
2934
+ plt.tight_layout(rect=[0, 0, 0.78, 1])
2935
+ else:
2936
+ plt.tight_layout()
2937
+
2938
+ safe_feat = feat.replace("/", "_").replace("\\", "_")
2939
+ suffix_str = f"_by_{group_name}" if group_name else ""
2940
+ out_file = os.path.join(graph_path, f"{safe_feat}{suffix_str}.png")
2941
+ plt.savefig(out_file, dpi=dpi, bbox_inches="tight")
2942
+ plt.close(fig)
2943
+ logger.info(f" [plot_woe_graph] {out_file}")
2944
+
2945
+ logger.info(f"[plot_woe_graph] 全部图表已保存至: {graph_path}")
2946
+
2947
+ def _plot_feat_small_multiples(
2948
+ self, feat, normal_df, sv_df,
2949
+ n_normal, n_sv, n_total, x_normal, x_sv, x_all,
2950
+ group_name, _df_for_group, graph_path, dpi, figsize,
2951
+ GOOD_COLOR, BAD_COLOR, SV_GOOD, SV_BAD, SV_WOE,
2952
+ ):
2953
+ """small_multiples 模式:每个 group 一个子图 panel。
2954
+
2955
+ - 柱高 = 该箱样本 / 该组总样本(组内占比),good/bad 堆叠,含特殊值箱
2956
+ - WOE 相对全量基准计算;WOE y 轴范围跨全部 panel 统一,便于横向对比
2957
+ - 文件名后缀 _by_{group_name},与 pooled / clustered 模式一致
2958
+ """
2959
+ # ── guard(与单图路径一致)──
2960
+ if _df_for_group is None:
2961
+ logger.info(f" [WARN] group_name='{group_name}' 需要传入 _df_for_group,跳过 {feat}")
2962
+ return
2963
+ if group_name not in _df_for_group.columns or feat not in _df_for_group.columns:
2964
+ logger.info(f" [WARN] group '{group_name}' 或 feature '{feat}' 不在 DataFrame 中")
2965
+ return
2966
+
2967
+ vr = self._results[feat]
2968
+ fitted_edges = list(vr["edges"])
2969
+ eps = self.eps
2970
+
2971
+ all_labels = (
2972
+ [str(b) for b in normal_df["bin_label"]]
2973
+ + ([str(b) for b in sv_df["bin_label"]] if n_sv > 0 else [])
2974
+ )
2975
+
2976
+ # WOE 基准:全量 total_bad / total_good
2977
+ all_normal_df, _ = self._split_special_for_plot(_df_for_group, feat, vr)
2978
+ all_normal_sub = all_normal_df[[feat, self.target_col]].dropna(subset=[feat]).copy()
2979
+ all_normal_sub["_bin"] = self._assign_normal_bins(
2980
+ all_normal_sub, feat, vr, fitted_edges)
2981
+ all_normal_sub = all_normal_sub[all_normal_sub["_bin"].notna()]
2982
+ all_total_bad = float(all_normal_sub[self.target_col].sum())
2983
+ all_total_good = float((all_normal_sub[self.target_col] == 0).sum())
2984
+
2985
+ groups = sorted(_df_for_group[group_name].dropna().unique())
2986
+ n_groups = len(groups)
2987
+ if n_groups == 0:
2988
+ logger.info(f" [WARN] group '{group_name}' 无有效取值,跳过 {feat}")
2989
+ return
2990
+
2991
+ # 子图网格:最多 3 列
2992
+ ncols = min(n_groups, 3)
2993
+ nrows = math.ceil(n_groups / ncols)
2994
+ fig, axes = plt.subplots(
2995
+ nrows, ncols,
2996
+ figsize=(figsize[0] * 0.62 * ncols, figsize[1] * 0.62 * nrows),
2997
+ squeeze=False,
2998
+ )
2999
+ axes_flat = axes.flatten()
3000
+
3001
+ group_ivs = []
3002
+ woe_axes = []
3003
+ all_woe_pts = []
3004
+
3005
+ for gi, grp_val in enumerate(groups):
3006
+ ax_bar = axes_flat[gi]
3007
+ ax_woe = ax_bar.twinx()
3008
+ woe_axes.append(ax_woe)
3009
+
3010
+ grp_df_full = _df_for_group[_df_for_group[group_name] == grp_val]
3011
+ n_grp = len(grp_df_full)
3012
+ tr = grp_df_full[self.target_col].mean() if n_grp > 0 else 0.0
3013
+
3014
+ grp_normal_df, grp_sv_groups = self._split_special_for_plot(grp_df_full, feat, vr)
3015
+ grp_sub = grp_normal_df[[feat, self.target_col]].dropna(subset=[feat]).copy()
3016
+ grp_sub["_bin"] = self._assign_normal_bins(grp_sub, feat, vr, fitted_edges)
3017
+ grp_sub = grp_sub[grp_sub["_bin"].notna()]
3018
+
3019
+ # 普通箱:组内占比 + 组内 bad_rate + WOE(相对全量基准)
3020
+ pct_good_n_g = np.zeros(n_normal)
3021
+ pct_bad_n_g = np.zeros(n_normal)
3022
+ grp_woe = []
3023
+ grp_br = [] # 各箱组内 bad_rate(用于数据标签)
3024
+ grp_iv = 0.0
3025
+ for b in range(n_normal):
3026
+ bin_rows = grp_sub[grp_sub["_bin"] == b]
3027
+ bad_b = float(bin_rows[self.target_col].sum())
3028
+ good_b = float((bin_rows[self.target_col] == 0).sum())
3029
+ pct_good_n_g[b] = good_b / (n_grp + eps)
3030
+ pct_bad_n_g[b] = bad_b / (n_grp + eps)
3031
+ if len(bin_rows) == 0:
3032
+ grp_woe.append(np.nan)
3033
+ grp_br.append(np.nan)
3034
+ continue
3035
+ grp_br.append(bad_b / (bad_b + good_b + eps))
3036
+ pct_bad_w = bad_b / (all_total_bad + eps)
3037
+ pct_good_w = good_b / (all_total_good + eps)
3038
+ woe_b = math.log((pct_bad_w + eps) / (pct_good_w + eps))
3039
+ grp_iv += (pct_bad_w - pct_good_w) * woe_b
3040
+ grp_woe.append(woe_b)
3041
+
3042
+ # 特殊值箱:组内占比 + WOE(相对全量基准)+ 组内 bad_rate
3043
+ pct_good_sv_g = np.zeros(n_sv)
3044
+ pct_bad_sv_g = np.zeros(n_sv)
3045
+ sv_woe = [np.nan] * n_sv
3046
+ sv_br = [np.nan] * n_sv
3047
+ if n_sv > 0:
3048
+ for si, sv_row in enumerate(sv_df.itertuples()):
3049
+ matched_sv_df = None
3050
+ for sv_key, sv_sub in grp_sv_groups.items():
3051
+ if _sv_label(sv_key) == sv_row.bin_label:
3052
+ matched_sv_df = sv_sub
3053
+ break
3054
+ if matched_sv_df is not None and len(matched_sv_df) > 0:
3055
+ sv_bad_i = float(matched_sv_df[self.target_col].sum())
3056
+ sv_good_i = float((matched_sv_df[self.target_col] == 0).sum())
3057
+ pct_good_sv_g[si] = sv_good_i / (n_grp + eps)
3058
+ pct_bad_sv_g[si] = sv_bad_i / (n_grp + eps)
3059
+ sv_br[si] = sv_bad_i / (sv_bad_i + sv_good_i + eps)
3060
+ _pb = sv_bad_i / (all_total_bad + eps)
3061
+ _pg = sv_good_i / (all_total_good + eps)
3062
+ sv_woe[si] = math.log((_pb + eps) / (_pg + eps))
3063
+
3064
+ # 柱(单套,good 下 bad 上)
3065
+ if n_normal > 0:
3066
+ ax_bar.bar(x_normal, pct_good_n_g, color=GOOD_COLOR,
3067
+ alpha=0.85, width=0.6, zorder=2, label="0")
3068
+ ax_bar.bar(x_normal, pct_bad_n_g, bottom=pct_good_n_g,
3069
+ color=BAD_COLOR, alpha=0.85, width=0.6, zorder=2, label="1")
3070
+ if n_sv > 0:
3071
+ ax_bar.bar(x_sv, pct_good_sv_g, color=SV_GOOD, alpha=0.85,
3072
+ width=0.6, zorder=2, edgecolor="#888",
3073
+ linewidth=1.0, linestyle="--")
3074
+ ax_bar.bar(x_sv, pct_bad_sv_g, bottom=pct_good_sv_g, color=SV_BAD,
3075
+ alpha=0.85, width=0.6, zorder=2, edgecolor="#888",
3076
+ linewidth=1.0, linestyle="--")
3077
+ ax_bar.axvline(x=n_normal - 0.5, color="#888",
3078
+ linewidth=1.0, linestyle=":", zorder=3)
3079
+ ax_bar.text(n_normal + 0.05, 0.93, "Special",
3080
+ fontsize=7, color=SV_WOE, va="top",
3081
+ transform=ax_bar.transData)
3082
+
3083
+ # WOE 折线(单色;范围统一在循环后设置)
3084
+ if n_grp < 5 or grp_df_full[self.target_col].nunique() < 2:
3085
+ group_ivs.append(0.0)
3086
+ iv_disp = 0.0
3087
+ ax_woe.plot(x_normal, [np.nan] * n_normal, color="#2E75B6",
3088
+ linewidth=1.8, marker="o", markersize=5, zorder=5)
3089
+ else:
3090
+ group_ivs.append(round(grp_iv, 4))
3091
+ iv_disp = grp_iv
3092
+ ax_woe.plot(x_normal, grp_woe, color="#2E75B6",
3093
+ linewidth=1.8, marker="o", markersize=5, zorder=5)
3094
+ all_woe_pts.extend([w for w in grp_woe if not np.isnan(w)])
3095
+
3096
+ # 数据标签框(WOE / BR / Lift | 组内占比),仅普通箱有效点
3097
+ # lift 基准 = 该组整体 bad_rate (tr),每张小图自洽
3098
+ _ylim_max = 1.0
3099
+ for xi in range(n_normal):
3100
+ wv = grp_woe[xi]
3101
+ if np.isnan(wv):
3102
+ continue
3103
+ br = grp_br[xi]
3104
+ pn = pct_good_n_g[xi] + pct_bad_n_g[xi] # 组内占比 = 柱高
3105
+ lv = br / (tr + eps) if tr > 0 else None
3106
+ lines_txt = [f"WOE: {wv:.3f}", f"BR: {br:.2%}"]
3107
+ lines_txt.append(f"Lift: {lv:.2f}x | {pn:.1%}"
3108
+ if lv is not None else f"{pn:.1%}")
3109
+ bar_top = pn
3110
+ y_above = bar_top + 0.04
3111
+ if y_above + 0.02 <= _ylim_max:
3112
+ y_text, va_text = y_above, "bottom"
3113
+ else:
3114
+ y_text = min(_ylim_max - 0.02, bar_top) - 0.02
3115
+ va_text = "top"
3116
+ ax_bar.annotate(
3117
+ "\n".join(lines_txt),
3118
+ xy=(xi, wv), xycoords=("data", ax_woe.transData),
3119
+ xytext=(xi, y_text), textcoords=("data", ax_bar.transData),
3120
+ fontsize=5.5, va=va_text, ha="center", zorder=10,
3121
+ bbox=dict(boxstyle="round,pad=0.2", fc="white",
3122
+ ec="#888", lw=0.6, zorder=10),
3123
+ arrowprops=dict(arrowstyle="-", color="#888", lw=0.6),
3124
+ )
3125
+
3126
+ # 特殊值箱:紫色数据标签框(贴柱顶,含 WOE/BR/Lift|占比)
3127
+ # 不画 WOE 点、不纳入统一 y 轴,避免极端 sv WOE 压平普通折线
3128
+ for si in range(n_sv):
3129
+ if np.isnan(sv_woe[si]):
3130
+ continue
3131
+ xi_abs = n_normal + si
3132
+ br = sv_br[si]
3133
+ pn = pct_good_sv_g[si] + pct_bad_sv_g[si] # 组内占比 = 柱高
3134
+ lv = br / (tr + eps) if tr > 0 else None
3135
+ lines_sv = [f"WOE: {sv_woe[si]:.3f}", f"BR: {br:.2%}"]
3136
+ lines_sv.append(f"Lift: {lv:.2f}x | {pn:.1%}"
3137
+ if lv is not None else f"{pn:.1%}")
3138
+ y_above = pn + 0.04
3139
+ if y_above + 0.02 <= _ylim_max:
3140
+ y_text, va_text = y_above, "bottom"
3141
+ else:
3142
+ y_text = min(_ylim_max - 0.02, pn) - 0.02
3143
+ va_text = "top"
3144
+ ax_bar.annotate(
3145
+ "\n".join(lines_sv),
3146
+ xy=(xi_abs, pn), xycoords="data",
3147
+ xytext=(xi_abs, y_text), textcoords="data",
3148
+ fontsize=5.5, va=va_text, ha="center", zorder=10,
3149
+ bbox=dict(boxstyle="round,pad=0.2", fc="#F5EEF8",
3150
+ ec=SV_WOE, lw=0.7, zorder=10),
3151
+ arrowprops=dict(arrowstyle="-", color=SV_WOE, lw=0.6),
3152
+ )
3153
+
3154
+ # panel 轴格式
3155
+ ax_bar.set_title(f"{grp_val} N={n_grp:,} TR={tr:.1%} IV={iv_disp:.3f}",
3156
+ fontsize=9, fontweight="bold")
3157
+ ax_bar.set_xlim(-0.5, n_total - 0.5)
3158
+ ax_bar.set_ylim(0, 1.0)
3159
+ ax_bar.set_xticks(x_all)
3160
+ ax_bar.set_xticklabels(
3161
+ [textwrap.fill(lb, width=12) for lb in all_labels],
3162
+ fontsize=6.5, rotation=30, ha="right",
3163
+ )
3164
+ ax_bar.set_ylabel("Proportion", fontsize=8)
3165
+ ax_bar.yaxis.set_major_formatter(mticker.PercentFormatter(xmax=1, decimals=0))
3166
+ ax_bar.tick_params(axis="y", labelsize=7)
3167
+ ax_woe.tick_params(axis="y", labelsize=7)
3168
+ ax_woe.set_ylabel("WOE", fontsize=8, color="#333")
3169
+ ax_bar.grid(axis="y", alpha=0.3, zorder=0)
3170
+ ax_bar.set_axisbelow(True)
3171
+
3172
+ # 统一 WOE y 轴范围(跨 panel 可比)
3173
+ if all_woe_pts:
3174
+ _woe_min, _woe_max = min(all_woe_pts), max(all_woe_pts)
3175
+ _span = max(_woe_max - _woe_min, 0.2)
3176
+ _pad = max(_span * 0.15, 0.1)
3177
+ _y_lo = min(_woe_min - _pad, -0.5)
3178
+ _y_hi = max(_woe_max + _pad, 0.5)
3179
+ else:
3180
+ _y_lo, _y_hi = -1.0, 1.0
3181
+ for axw in woe_axes:
3182
+ axw.set_ylim(_y_lo, _y_hi)
3183
+ axw.axhline(0, color="gray", linewidth=0.6, linestyle="--", zorder=1)
3184
+
3185
+ # 隐藏多余空 panel
3186
+ for j in range(n_groups, nrows * ncols):
3187
+ axes_flat[j].axis("off")
3188
+
3189
+ iv_vals = [v for v in group_ivs if v > 0]
3190
+ iv_range = (f"{min(iv_vals):.3f}-{max(iv_vals):.3f}"
3191
+ if iv_vals else "0.000-0.000")
3192
+ fig.suptitle(f"{feat}: IV_range={iv_range} (bar=in-group %)",
3193
+ fontsize=12, fontweight="bold")
3194
+
3195
+ fig.tight_layout(rect=[0, 0, 1, 0.97])
3196
+ safe_feat = feat.replace("/", "_").replace("\\", "_")
3197
+ out_file = os.path.join(graph_path, f"{safe_feat}_by_{group_name}.png")
3198
+ fig.savefig(out_file, dpi=dpi, bbox_inches="tight")
3199
+ plt.close(fig)
3200
+ logger.info(f" [plot_woe_graph] {out_file}")
3201
+
3202
+ # ─────────────────────────────────────────────────────────────────
3203
+ # 便捷属性
3204
+ # ─────────────────────────────────────────────────────────────────
3205
+
3206
+ @property
3207
+ def iv_summary(self) -> pd.DataFrame:
3208
+ """返回所有特征的 IV 汇总 DataFrame,按 IV 降序排列。"""
3209
+ self._check_fitted()
3210
+ rows = [
3211
+ {"feature": feat, "iv": vr["iv"],
3212
+ "n_bins": vr["n_bins"],
3213
+ "n_sv_bins": len(vr.get("sv_table", pd.DataFrame())),
3214
+ "is_monotonic": vr["is_monotonic"],
3215
+ "is_categorical": vr.get("is_categorical", False)}
3216
+ for feat, vr in self._results.items()
3217
+ ]
3218
+ return pd.DataFrame(rows).sort_values("iv", ascending=False).reset_index(drop=True)
3219
+
3220
+ def __repr__(self) -> str:
3221
+ fitted = "fitted" if self._is_fitted else "not fitted"
3222
+ sv_hint = f", special_values={self.special_values}" if self.special_values else ""
3223
+ cate_hint = f", cate_feats={len(self.cate_feats)}" if self.cate_feats else ""
3224
+ dec_hint = (f", bin_label_decimals={self.bin_label_decimals}"
3225
+ if self.bin_label_decimals is not None else "")
3226
+ return (f"MonotoneWOEBinner({fitted}, "
3227
+ f"features={len(self.feature_cols)}, "
3228
+ f"n_init_bins={self.n_init_bins}{sv_hint}{cate_hint}{dec_hint})")
3229
+
3230
+
3231
+ # ══════════════════════════════════════════════════════════════════════════════
3232
+ # 快速使用示例 / 自测
3233
+ # ══════════════════════════════════════════════════════════════════════════════
3234
+
3235
+ if __name__ == "__main__":
3236
+ import numpy as np
3237
+
3238
+ rng = np.random.default_rng(42)
3239
+ N = 3000
3240
+ df_demo = pd.DataFrame({
3241
+ "id": range(N),
3242
+ "score": rng.normal(600, 80, N),
3243
+ "income": rng.exponential(5000, N),
3244
+ "tenure": rng.integers(1, 120, N).astype(float),
3245
+ "age": rng.integers(20, 65, N).astype(float),
3246
+ "month": rng.choice(["2026-01", "2026-02", "2026-03"], N),
3247
+ "is_bad": (rng.random(N) < 0.25).astype(int),
3248
+ })
3249
+ df_demo["is_bad"] = ((df_demo["score"] < 580) & (rng.random(N) < 0.35)).astype(int)
3250
+
3251
+ # 类别特征:已离散化的「城市等级」,与坏率挂钩(D/C 坏率更高)
3252
+ grade_pool = np.array(["A", "B", "C", "D"])
3253
+ df_demo["city_grade"] = np.where(
3254
+ df_demo["is_bad"].values == 1,
3255
+ rng.choice(grade_pool, N, p=[0.15, 0.20, 0.30, 0.35]),
3256
+ rng.choice(grade_pool, N, p=[0.35, 0.30, 0.20, 0.15]),
3257
+ )
3258
+
3259
+ # 人工注入特殊值
3260
+ sv_idx = rng.choice(N, 280, replace=False)
3261
+ df_demo.loc[sv_idx[:100], "score"] = -1 # 特殊值 -1(无信用记录)
3262
+ df_demo.loc[sv_idx[100:150], "income"] = -100 # 特殊值 -100
3263
+ df_demo.loc[sv_idx[150:230], "tenure"] = np.nan # NaN 单独分箱
3264
+ df_demo.loc[sv_idx[230:], "city_grade"] = np.nan # 类别特征的缺失 → [Missing] 箱
3265
+
3266
+ feats = ["score", "income", "tenure", "age"]
3267
+ cate_feats = ["city_grade"]
3268
+
3269
+ # ── 1. 正常 fit(含特殊值参数 + 类别特征)
3270
+ binner = MonotoneWOEBinner(
3271
+ feature_cols=feats,
3272
+ target_col="is_bad",
3273
+ n_init_bins=20,
3274
+ min_bin_size=0.03,
3275
+ special_values=[-1, -100, float("nan")], # ← 仅作用于数值特征
3276
+ cate_feats=cate_feats, # ← 新增:类别特征,直接算 WOE/IV
3277
+ )
3278
+ binner.fit(df_demo)
3279
+
3280
+ logger.info("\n=== get_final_bins() — city_grade (类别特征,4 个类别 + [Missing]) ===")
3281
+ logger.info(binner.get_final_bins()["city_grade"].to_string(index=False))
3282
+
3283
+ # 1b. refine_cate:按坏率聚类合并类别(演示 max_bins=2,把 4 个类别并成 2 箱)
3284
+ binner.refine_cate(features=["city_grade"], max_bins=2)
3285
+ logger.info("\n=== refine_cate(max_bins=2) — city_grade 聚类后 ===")
3286
+ logger.info(binner.get_final_bins()["city_grade"].to_string(index=False))
3287
+ logger.info("\n=== iv_summary ===")
3288
+ logger.info(binner.iv_summary.to_string(index=False))
3289
+
3290
+ # 2. get_final_bins(含特殊值箱)
3291
+ bins = binner.get_final_bins()
3292
+ logger.info("\n=== get_final_bins() — score (含特殊值箱) ===")
3293
+ logger.info(bins["score"].to_string(index=False))
3294
+
3295
+ # 3. apply_woe
3296
+ df_woe = binner.apply_woe(df_demo)
3297
+ logger.info("\n=== apply_woe() — first 5 rows ===")
3298
+ woe_cols = [f + "_woe" for f in feats + cate_feats]
3299
+ logger.info(df_woe[woe_cols].head())
3300
+
3301
+ # 4. export_woe_report(含图片 Sheet)
3302
+ binner.export_woe_report("/tmp/demo_woe_report_v2.xlsx")
3303
+ logger.info("报告已生成: /tmp/demo_woe_report_v2.xlsx")
3304
+
3305
+ # 5. plot_woe_graph(整体图,含特殊值箱)
3306
+ binner.plot_woe_graph("/tmp/demo_woe_charts_v2/")
3307
+
3308
+ # 5b. plot_woe_graph(分组图,按 month;类别特征 city_grade 同样支持)
3309
+ binner.plot_woe_graph("/tmp/demo_woe_charts_v2_bymonth/", group_name="month",
3310
+ _df_for_group=df_demo, bar_mode="clustered")
3311
+
3312
+ # 6. load_woe_bins — 跳过 fit,直接加载
3313
+ binner2 = MonotoneWOEBinner(feature_cols=[], target_col="is_bad")
3314
+ binner2.load_woe_bins(bins)
3315
+ df_woe2 = binner2.apply_woe(df_demo)
3316
+ logger.info("\n=== load_woe_bins + apply_woe() — first 5 rows ===")
3317
+ logger.info(df_woe2[woe_cols].head())
3318
+
3319
+ # 验证两者 WOE 输出一致
3320
+ for col in woe_cols:
3321
+ diff = (df_woe[col] - df_woe2[col]).abs().max()
3322
+ logger.info(f" {col}: max_diff={diff:.6f} {'✓' if diff < 1e-4 else '✗ MISMATCH'}")
3323
+
3324
+ logger.info("\n✓ 全部测试完成")