SuperModelingFactory 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ExcelMaster/ExcelFormatTool.py +487 -0
- ExcelMaster/ExcelMaster.py +917 -0
- ExcelMaster/Template.py +525 -0
- ExcelMaster/Utility.py +233 -0
- ExcelMaster/__init__.py +3 -0
- Modeling_Tool/Core/Binning_Tool.py +1608 -0
- Modeling_Tool/Core/Binning_Tool.pyi +48 -0
- Modeling_Tool/Core/Check_DuckDB_Compatibility.py +835 -0
- Modeling_Tool/Core/Json_Data_Converter.py +621 -0
- Modeling_Tool/Core/Model_Registry_Tool.py +199 -0
- Modeling_Tool/Core/ODPS_Tool.py +284 -0
- Modeling_Tool/Core/Slope_Tool.py +356 -0
- Modeling_Tool/Core/Slope_Tool.pyi +31 -0
- Modeling_Tool/Core/XOR_Encryptor.py +207 -0
- Modeling_Tool/Core/XOR_Encryptor.pyi +26 -0
- Modeling_Tool/Core/__init__.py +99 -0
- Modeling_Tool/Core/kDataFrame.py +228 -0
- Modeling_Tool/Core/kDataFrame.pyi +43 -0
- Modeling_Tool/Core/sample_weight_utils.py +77 -0
- Modeling_Tool/Core/utils.py +2672 -0
- Modeling_Tool/Eval/Evaluation_Tool.py +1452 -0
- Modeling_Tool/Eval/Evaluation_Tool.pyi +47 -0
- Modeling_Tool/Eval/Model_Eval_Tool.py +2548 -0
- Modeling_Tool/Eval/Model_Eval_Tool.pyi +37 -0
- Modeling_Tool/Eval/__init__.py +54 -0
- Modeling_Tool/Eval/evaluate_model.py +2008 -0
- Modeling_Tool/Eval/evaluate_model.pyi +50 -0
- Modeling_Tool/Eval/weighted_eval_utils.py +326 -0
- Modeling_Tool/Explainability/Coalition_Structure.py +305 -0
- Modeling_Tool/Explainability/Model_Explainer.py +743 -0
- Modeling_Tool/Explainability/__init__.py +24 -0
- Modeling_Tool/Feature/Distribution_Tool.py +509 -0
- Modeling_Tool/Feature/Distribution_Tool.pyi +42 -0
- Modeling_Tool/Feature/Feature_Insights.py +762 -0
- Modeling_Tool/Feature/Feature_Insights.pyi +33 -0
- Modeling_Tool/Feature/PSI_Tool.py +1195 -0
- Modeling_Tool/Feature/PSI_Tool.pyi +29 -0
- Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py +355 -0
- Modeling_Tool/Feature/__init__.py +40 -0
- Modeling_Tool/Model/Backward_Tool.py +778 -0
- Modeling_Tool/Model/Backward_Tool.pyi +45 -0
- Modeling_Tool/Model/GBM_Search_Tool.py +251 -0
- Modeling_Tool/Model/GBM_Tool.py +1610 -0
- Modeling_Tool/Model/GBM_Tool.pyi +90 -0
- Modeling_Tool/Model/LRM_Tool.py +1198 -0
- Modeling_Tool/Model/LRM_Tool.pyi +47 -0
- Modeling_Tool/Model/__init__.py +61 -0
- Modeling_Tool/Sample/Distribution_Adaptation.py +131 -0
- Modeling_Tool/Sample/Distribution_Adaptation.pyi +30 -0
- Modeling_Tool/Sample/Reject_Infer.py +413 -0
- Modeling_Tool/Sample/Reject_Infer.pyi +43 -0
- Modeling_Tool/Sample/Sample_Split.py +520 -0
- Modeling_Tool/Sample/Sample_Split.pyi +43 -0
- Modeling_Tool/Sample/__init__.py +31 -0
- Modeling_Tool/UAT/UAT_Consistency_Checker.py +1180 -0
- Modeling_Tool/UAT/__init__.py +19 -0
- Modeling_Tool/WOE/WOE_Adapter.py +204 -0
- Modeling_Tool/WOE/WOE_Adapter.pyi +21 -0
- Modeling_Tool/WOE/WOE_Master.py +491 -0
- Modeling_Tool/WOE/WOE_Master.pyi +40 -0
- Modeling_Tool/WOE/WOE_Monotone_Binner.py +3324 -0
- Modeling_Tool/WOE/WOE_Monotone_Binner.pyi +71 -0
- Modeling_Tool/WOE/WOE_Plot_Tool.py +919 -0
- Modeling_Tool/WOE/WOE_Plot_Tool.pyi +46 -0
- Modeling_Tool/WOE/WOE_Report_Builder.py +214 -0
- Modeling_Tool/WOE/WOE_Report_Builder.pyi +24 -0
- Modeling_Tool/WOE/WOE_Tool.py +1094 -0
- Modeling_Tool/WOE/WOE_Tool.pyi +41 -0
- Modeling_Tool/WOE/__init__.py +86 -0
- Modeling_Tool/WOE/plot_woe_tool.py +290 -0
- Modeling_Tool/WOE/plot_woe_tool.pyi +25 -0
- Modeling_Tool/__init__.py +176 -0
- Report/Report_Tool.py +380 -0
- Report/__init__.py +3 -0
- supermodelingfactory-0.2.0.dist-info/METADATA +268 -0
- supermodelingfactory-0.2.0.dist-info/RECORD +79 -0
- supermodelingfactory-0.2.0.dist-info/WHEEL +5 -0
- supermodelingfactory-0.2.0.dist-info/licenses/LICENSE +102 -0
- supermodelingfactory-0.2.0.dist-info/top_level.txt +3 -0
|
@@ -0,0 +1,1608 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
logger = logging.getLogger(__name__)
|
|
3
|
+
|
|
4
|
+
import pandas as pd
|
|
5
|
+
import numpy as np
|
|
6
|
+
from scipy.stats import chi2_contingency, chi2
|
|
7
|
+
|
|
8
|
+
# Available only in newer pandas versions. Older Airflow images should skip it.
|
|
9
|
+
try:
|
|
10
|
+
pd.set_option('future.no_silent_downcasting', True)
|
|
11
|
+
except (KeyError, ValueError):
|
|
12
|
+
pass
|
|
13
|
+
pd.options.mode.chained_assignment = None # default='warn'
|
|
14
|
+
|
|
15
|
+
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
16
|
+
|
|
17
|
+
def get_max_nbins(data, nbins, min_bin_prop = 0.05):
|
|
18
|
+
"""
|
|
19
|
+
根据给定的最小分箱比例计算最大分箱数。
|
|
20
|
+
|
|
21
|
+
根据数据总量和最小分箱比例,计算能够满足最小样本数要求的最大分箱数。
|
|
22
|
+
|
|
23
|
+
Parameters
|
|
24
|
+
----------
|
|
25
|
+
data : pandas.DataFrame
|
|
26
|
+
输入数据表
|
|
27
|
+
nbins : int
|
|
28
|
+
期望的分箱数量
|
|
29
|
+
min_bin_prop : float, default 0.05
|
|
30
|
+
每个分箱最小样本占比
|
|
31
|
+
|
|
32
|
+
Returns
|
|
33
|
+
-------
|
|
34
|
+
int
|
|
35
|
+
可行的最大分箱数
|
|
36
|
+
|
|
37
|
+
Examples
|
|
38
|
+
--------
|
|
39
|
+
>>> get_max_nbins(data, nbins=10, min_bin_prop=0.05)
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
n = data.shape[0]
|
|
43
|
+
if n == 0:
|
|
44
|
+
return nbins # 空数据无法分箱,返回原值
|
|
45
|
+
|
|
46
|
+
min_bin_size = min_bin_prop * n
|
|
47
|
+
|
|
48
|
+
if min_bin_size == 0:
|
|
49
|
+
# 此时 min_bin_prop == 0,退化为最多 nbins 个箱
|
|
50
|
+
return min(nbins, n)
|
|
51
|
+
|
|
52
|
+
nbins = min(nbins, max(5, n // min_bin_size))
|
|
53
|
+
return nbins
|
|
54
|
+
|
|
55
|
+
def get_decision_tree_binning_edges(feature, target, max_leaf_nodes=5, min_samples_leaf=0.05, random_state=42, missing_ref_value = None, spec_values = []):
|
|
56
|
+
"""
|
|
57
|
+
使用决策树对连续变量进行最优分箱。
|
|
58
|
+
|
|
59
|
+
通过决策树分类算法寻找最优的分箱边界点,适用于连续型变量的自动化分箱。
|
|
60
|
+
算法会自动处理缺失值和特殊值,并返回合适的分箱边界。
|
|
61
|
+
|
|
62
|
+
Parameters
|
|
63
|
+
----------
|
|
64
|
+
feature : array-like
|
|
65
|
+
待分箱的连续特征,Pandas Series或一维数组
|
|
66
|
+
target : array-like
|
|
67
|
+
目标变量,二分类标签(Pandas Series或一维数组)
|
|
68
|
+
max_leaf_nodes : int, default 5
|
|
69
|
+
决策树最大叶节点数,即最大分箱数
|
|
70
|
+
min_samples_leaf : float, default 0.05
|
|
71
|
+
叶节点最小样本比例,默认为0.05(5%)
|
|
72
|
+
random_state : int, default 42
|
|
73
|
+
随机种子,确保结果可重现
|
|
74
|
+
missing_ref_value : any, optional
|
|
75
|
+
缺失值参考值,该值将被视为缺失
|
|
76
|
+
spec_values : list, optional
|
|
77
|
+
特殊数值列表,这些值将被独立分箱
|
|
78
|
+
|
|
79
|
+
Returns
|
|
80
|
+
-------
|
|
81
|
+
bin_edges : list
|
|
82
|
+
分箱边界列表
|
|
83
|
+
|
|
84
|
+
Examples
|
|
85
|
+
--------
|
|
86
|
+
>>> edges = get_decision_tree_binning_edges(feature, target, max_leaf_nodes=5)
|
|
87
|
+
"""
|
|
88
|
+
from sklearn.tree import DecisionTreeClassifier, export_text
|
|
89
|
+
from sklearn.utils import check_random_state
|
|
90
|
+
|
|
91
|
+
if missing_ref_value:
|
|
92
|
+
feature = np.where(feature == missing_ref_value, np.nan, feature)
|
|
93
|
+
|
|
94
|
+
# 数据预处理:移除缺失值
|
|
95
|
+
df = pd.DataFrame({
|
|
96
|
+
'feature': feature,
|
|
97
|
+
'target': target
|
|
98
|
+
}).dropna()
|
|
99
|
+
|
|
100
|
+
if len(spec_values) > 0:
|
|
101
|
+
df = df[~df["feature"].isin(spec_values)]
|
|
102
|
+
|
|
103
|
+
feature_clean = df['feature']
|
|
104
|
+
target_clean = df['target']
|
|
105
|
+
|
|
106
|
+
# 如果特征方差为0或几乎为0,无法分箱
|
|
107
|
+
if feature_clean.nunique() <= 1:
|
|
108
|
+
logger.info("警告: 特征方差为0,无法分箱")
|
|
109
|
+
return [feature_clean.min(), feature_clean.max()], pd.cut(feature, bins=[feature_clean.min(), feature_clean.max()])
|
|
110
|
+
|
|
111
|
+
# 重塑特征为2D数组以适配sklearn
|
|
112
|
+
X = feature_clean.values.reshape(-1, 1)
|
|
113
|
+
y = target_clean.values
|
|
114
|
+
|
|
115
|
+
# 创建决策树分类器
|
|
116
|
+
tree_model = DecisionTreeClassifier(
|
|
117
|
+
max_leaf_nodes=max_leaf_nodes,
|
|
118
|
+
min_samples_leaf=min_samples_leaf,
|
|
119
|
+
random_state=random_state
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
# 拟合决策树模型
|
|
123
|
+
tree_model.fit(X, y)
|
|
124
|
+
|
|
125
|
+
# 提取分箱边界
|
|
126
|
+
threshold = tree_model.tree_.threshold
|
|
127
|
+
feature_min = feature_clean.min()
|
|
128
|
+
feature_max = feature_clean.max()
|
|
129
|
+
|
|
130
|
+
# 获取非叶子节点的分割阈值并排序
|
|
131
|
+
bin_edges = sorted([th for th in threshold if th != -2])
|
|
132
|
+
|
|
133
|
+
# 添加最小值和最大值作为边界
|
|
134
|
+
# bin_edges = [feature_min] + bin_edges + [feature_max]
|
|
135
|
+
|
|
136
|
+
# 移除可能重复的边界
|
|
137
|
+
bin_edges = sorted(list(set(bin_edges)))
|
|
138
|
+
if missing_ref_value:
|
|
139
|
+
bin_edges = sorted(list(set(bin_edges + [missing_ref_value])))
|
|
140
|
+
|
|
141
|
+
# print(tree_model.get_params())
|
|
142
|
+
|
|
143
|
+
return bin_edges
|
|
144
|
+
|
|
145
|
+
class NumVarBinning:
|
|
146
|
+
"""
|
|
147
|
+
根据分组数和方法, 计算数值型数值变量切分点数值序列。
|
|
148
|
+
|
|
149
|
+
根据指定分组数计算切分点数值, 分组方法包括"等距分组"和"等分分组":
|
|
150
|
+
(1) 当分组数不小于变量唯一值个数时, 切分点数值序列即为变量唯一值序列
|
|
151
|
+
(2) 当分组数小于变量唯一值个数时, 可选择"等距分组"或"等分分组"
|
|
152
|
+
|
|
153
|
+
若有指定需要独立成组的数值, 则:
|
|
154
|
+
(1) 将数值加入切分点数值序列
|
|
155
|
+
(2) 添加数值上确界=数值+0.1**(精度+1), 加入切分点数值序列
|
|
156
|
+
|
|
157
|
+
切分点数值精度:
|
|
158
|
+
(1) 若变量取值为整数, 则精度为保留1位小数
|
|
159
|
+
(2) 若变量取值为浮点数, 则精度为保留输入的位数
|
|
160
|
+
|
|
161
|
+
为了泛化性, 将切分点数值序列中的最小值替换为-inf, 最大值增加+inf
|
|
162
|
+
|
|
163
|
+
Parameters
|
|
164
|
+
----------
|
|
165
|
+
var_name : str
|
|
166
|
+
变量名
|
|
167
|
+
spec_values : list, optional
|
|
168
|
+
指定需要单独成组的特殊数值
|
|
169
|
+
spec_digit : int, default 3
|
|
170
|
+
指定特殊的切分点上确界保留小数位数, 若变量为整数则固定为1, 若非整数则为spec_digit值
|
|
171
|
+
|
|
172
|
+
Examples
|
|
173
|
+
--------
|
|
174
|
+
>>> nvb = NumVarBinning(var_name='income', spec_values=[-9999])
|
|
175
|
+
>>> binning = nvb.equi_binning(df, bins=10)
|
|
176
|
+
"""
|
|
177
|
+
def __init__(self, var_name, spec_values=None, spec_digit=3):
|
|
178
|
+
"""
|
|
179
|
+
初始化数值变量分箱对象。
|
|
180
|
+
|
|
181
|
+
Parameters
|
|
182
|
+
----------
|
|
183
|
+
var_name : str
|
|
184
|
+
变量名
|
|
185
|
+
spec_values : list, optional
|
|
186
|
+
指定需要单独成组的特殊数值
|
|
187
|
+
spec_digit : int, default 3
|
|
188
|
+
特殊值精度位数
|
|
189
|
+
"""
|
|
190
|
+
self.var_name = var_name
|
|
191
|
+
self.spec_values = spec_values
|
|
192
|
+
self.spec_digit = spec_digit
|
|
193
|
+
self.bins = None
|
|
194
|
+
self.cut_points = None
|
|
195
|
+
self.bin_names = None
|
|
196
|
+
|
|
197
|
+
def calc_equi_cutpoints(self, df, bins=10, equi_method="equif"):
|
|
198
|
+
"""
|
|
199
|
+
根据分组数和等值分组方法, 计算数值型数值变量切分点数值序列。
|
|
200
|
+
|
|
201
|
+
1、根据指定分组数计算切分点数值, 分组方法包括"等距分组"和"等分分组"
|
|
202
|
+
(1) 当分组数不小于变量唯一值个数时, 切分点数值序列即为变量唯一值序列
|
|
203
|
+
(2) 当分组数小于变量唯一值个数时, 可选择"等距分组"或"等分分组"
|
|
204
|
+
|
|
205
|
+
2、若有指定需要独立成组的数值, 则:
|
|
206
|
+
(1) 将数值加入切分点数值序列;
|
|
207
|
+
(2) 添加数值上确界=数值+0.1**(精度+1), 加入切分点数值序列;
|
|
208
|
+
|
|
209
|
+
3、切分点数值精度:
|
|
210
|
+
(1) 若变量取值为整数, 则精度为保留1位小数
|
|
211
|
+
(2) 若变量取值为浮点数, 则精度为保留输入的位数
|
|
212
|
+
|
|
213
|
+
Parameters
|
|
214
|
+
----------
|
|
215
|
+
df : pandas.DataFrame
|
|
216
|
+
数据表
|
|
217
|
+
bins : int, default 10
|
|
218
|
+
切分组数
|
|
219
|
+
equi_method : string, default "equif"
|
|
220
|
+
分组方法, 候选值{"equid"等距分组, "equif"等分分组}
|
|
221
|
+
|
|
222
|
+
Returns
|
|
223
|
+
-------
|
|
224
|
+
cut_points : numpy.array
|
|
225
|
+
切分点数值序列
|
|
226
|
+
"""
|
|
227
|
+
var_series = df[self.var_name]
|
|
228
|
+
|
|
229
|
+
if any(pd.isnull(var_series)):
|
|
230
|
+
raise ValueError(f"{self.var_name}变量取值中出现NaN值")
|
|
231
|
+
|
|
232
|
+
if pd.api.types.is_integer_dtype(var_series):
|
|
233
|
+
spec_digit = 1
|
|
234
|
+
else:
|
|
235
|
+
spec_digit = self.spec_digit
|
|
236
|
+
|
|
237
|
+
# 根据指定分bin数计算切分点值
|
|
238
|
+
cut_points = var_series.unique()
|
|
239
|
+
if len(cut_points) <= bins:
|
|
240
|
+
cut_points.sort()
|
|
241
|
+
elif equi_method == "equif":
|
|
242
|
+
pct = np.arange(1, bins) / bins
|
|
243
|
+
cut_points = var_series.quantile(pct, interpolation="higher").values
|
|
244
|
+
elif equi_method == "equid":
|
|
245
|
+
min_value = min(var_series)
|
|
246
|
+
max_value = max(var_series)
|
|
247
|
+
step = (max_value - min_value) / bins
|
|
248
|
+
cut_points = np.arange(start=min_value, stop=max_value, step=step)[1:]
|
|
249
|
+
else:
|
|
250
|
+
raise ValueError("equi_method取值错误.")
|
|
251
|
+
|
|
252
|
+
# 添加需要单独成组的特殊数值
|
|
253
|
+
if bool(self.spec_values) and len(self.spec_values) > 0:
|
|
254
|
+
self.spec_values.sort()
|
|
255
|
+
spec_values_upper = [x + 0.1**spec_digit for x in self.spec_values]
|
|
256
|
+
cut_points = np.append(cut_points, self.spec_values + spec_values_upper)
|
|
257
|
+
cut_points.sort()
|
|
258
|
+
|
|
259
|
+
# print("NumVarBinning Cut Points: ", cut_points)
|
|
260
|
+
# 返回Info
|
|
261
|
+
self.equi_method = equi_method
|
|
262
|
+
|
|
263
|
+
return cut_points
|
|
264
|
+
|
|
265
|
+
def modify_cutpoints(self, df, points):
|
|
266
|
+
"""
|
|
267
|
+
修正切分点数值序列。
|
|
268
|
+
|
|
269
|
+
1、切分点数值精度:
|
|
270
|
+
(1) 若变量取值为整数, 则精度为保留1位小数
|
|
271
|
+
(2) 若变量取值为浮点数, 则精度为保留输入的位数
|
|
272
|
+
|
|
273
|
+
2、过滤不在变量值域内的切分点, 并去重
|
|
274
|
+
|
|
275
|
+
3、为了泛化性, 将切分点数值序列中的最小值替换为-inf, 最大值增加+inf。
|
|
276
|
+
|
|
277
|
+
Parameters
|
|
278
|
+
----------
|
|
279
|
+
df : pandas.DataFrame
|
|
280
|
+
数据表
|
|
281
|
+
points : array-like
|
|
282
|
+
切分点数值序列
|
|
283
|
+
|
|
284
|
+
Returns
|
|
285
|
+
-------
|
|
286
|
+
cut_points : numpy.array
|
|
287
|
+
修正后的切分点数值序列
|
|
288
|
+
"""
|
|
289
|
+
var_series = df[self.var_name]
|
|
290
|
+
|
|
291
|
+
if pd.api.types.is_integer_dtype(var_series):
|
|
292
|
+
spec_digit = 1
|
|
293
|
+
else:
|
|
294
|
+
spec_digit = self.spec_digit
|
|
295
|
+
|
|
296
|
+
# 过滤变量值域内切分点
|
|
297
|
+
## Added
|
|
298
|
+
min_value_wo_spec_values = min(var_series[~var_series.isin(self.spec_values)])
|
|
299
|
+
max_value_wo_spec_values = max(var_series[~var_series.isin(self.spec_values)])
|
|
300
|
+
|
|
301
|
+
min_value = min(var_series)
|
|
302
|
+
max_value = max(var_series)
|
|
303
|
+
# print("(MIN, MAX): ", min_value, max_value)
|
|
304
|
+
cut_points = list(filter(lambda x: min_value <= x <= max_value, points))
|
|
305
|
+
# print("Modify Init Cut Points: ", cut_points)
|
|
306
|
+
if bool(self.spec_values):
|
|
307
|
+
spec_cut_points = list(filter(lambda x: min_value <= x <= max_value, self.spec_values))
|
|
308
|
+
# print("Modify Spec Cut Points: ", spec_cut_points)
|
|
309
|
+
|
|
310
|
+
## Added
|
|
311
|
+
spec_values_upper = [x + 0.1**spec_digit for x in self.spec_values]
|
|
312
|
+
|
|
313
|
+
cut_points.extend(spec_cut_points)
|
|
314
|
+
|
|
315
|
+
## Added
|
|
316
|
+
cut_points.extend(spec_values_upper)
|
|
317
|
+
|
|
318
|
+
cut_points = np.unique(np.array(cut_points).astype("float").round(spec_digit))
|
|
319
|
+
cut_points = np.insert(cut_points, 0, -np.inf)
|
|
320
|
+
cut_points = np.append(cut_points, np.inf)
|
|
321
|
+
|
|
322
|
+
# print("Final Cut Points Before Binning: ", cut_points)
|
|
323
|
+
# 过滤有样本的切分点
|
|
324
|
+
binning_series = pd.cut(df[self.var_name], cut_points, right=False, labels=cut_points[:-1])
|
|
325
|
+
binning_cnts = binning_series.value_counts()
|
|
326
|
+
cut_points = np.array(binning_cnts.index[binning_cnts > 0])
|
|
327
|
+
cut_points.sort()
|
|
328
|
+
|
|
329
|
+
# 若第一位切分点非spec_values, 则令其为-inf
|
|
330
|
+
if bool(self.spec_values):
|
|
331
|
+
if cut_points[0] not in self.spec_values:
|
|
332
|
+
cut_points[0] = -np.inf
|
|
333
|
+
else:
|
|
334
|
+
cut_points[0] = -np.inf
|
|
335
|
+
|
|
336
|
+
# 将变量的最大值增加+inf
|
|
337
|
+
cut_points = np.append(cut_points, np.inf)
|
|
338
|
+
|
|
339
|
+
return cut_points
|
|
340
|
+
|
|
341
|
+
def apt_binning(self, df, points, modify=True):
|
|
342
|
+
"""
|
|
343
|
+
使用指定切分点数值序列对变量进行分组。
|
|
344
|
+
|
|
345
|
+
切分点数值精度:
|
|
346
|
+
(1) 若变量取值为整数, 则精度为保留1位小数
|
|
347
|
+
(2) 若变量取值为浮点数, 则精度为保留输入的位数
|
|
348
|
+
|
|
349
|
+
过滤不在变量值域内的切分点
|
|
350
|
+
|
|
351
|
+
为了泛化性, 将切分点数值序列中的最小值替换为-inf, 最大值增加+inf。
|
|
352
|
+
|
|
353
|
+
Parameters
|
|
354
|
+
----------
|
|
355
|
+
df : pandas.DataFrame
|
|
356
|
+
数据表
|
|
357
|
+
points : array-like
|
|
358
|
+
切分点数值序列
|
|
359
|
+
modify : bool, default True
|
|
360
|
+
是否修正切分点数值序列
|
|
361
|
+
|
|
362
|
+
Returns
|
|
363
|
+
-------
|
|
364
|
+
binning_series : pandas.Categorical
|
|
365
|
+
分组后序列
|
|
366
|
+
"""
|
|
367
|
+
if modify:
|
|
368
|
+
self.cut_points = self.modify_cutpoints(df, points)
|
|
369
|
+
else:
|
|
370
|
+
points.sort()
|
|
371
|
+
self.cut_points = points
|
|
372
|
+
self.bins = len(self.cut_points) - 1
|
|
373
|
+
|
|
374
|
+
binning_series = pd.cut(df[self.var_name], self.cut_points, right=False)
|
|
375
|
+
self.bin_names = binning_series.values.categories
|
|
376
|
+
# self.bin_names = {i: str(c) for i, c in zip(range(self.bins), binning_series.values.categories)}
|
|
377
|
+
# binning_series = binning_series.values.rename_categories(range(self.bins))
|
|
378
|
+
|
|
379
|
+
return binning_series
|
|
380
|
+
|
|
381
|
+
def equi_binning(self, df, bins=10, equi_method="equif"):
|
|
382
|
+
"""
|
|
383
|
+
根据分组数和等值分组方法, 计算数值型数值变量切分点数值序列并分箱。
|
|
384
|
+
|
|
385
|
+
Parameters
|
|
386
|
+
----------
|
|
387
|
+
df : pandas.DataFrame
|
|
388
|
+
数据表
|
|
389
|
+
bins : int, default 10
|
|
390
|
+
切分组数
|
|
391
|
+
equi_method : string, default "equif"
|
|
392
|
+
分组方法, 候选值{"equid"等距分组, "equif"等分分组}
|
|
393
|
+
|
|
394
|
+
Returns
|
|
395
|
+
-------
|
|
396
|
+
binning_series : pandas.Categorical
|
|
397
|
+
分组后序列
|
|
398
|
+
"""
|
|
399
|
+
logging.info(f"-------- [{self.var_name} : EquiX Binning] --------")
|
|
400
|
+
logging.info(f"PARAMS: bins={bins}, equi_method={equi_method}")
|
|
401
|
+
cut_points = self.calc_equi_cutpoints(df=df, bins=bins, equi_method=equi_method)
|
|
402
|
+
binning_series = self.apt_binning(df=df, points=cut_points, modify=True)
|
|
403
|
+
|
|
404
|
+
return binning_series
|
|
405
|
+
|
|
406
|
+
def apply_binning(self, df):
|
|
407
|
+
"""
|
|
408
|
+
使用已保存的切分点数值序列对变量进行分组。
|
|
409
|
+
|
|
410
|
+
使用之前通过equi_binning或auto_binning等方法计算的切分点对新数据进行分组。
|
|
411
|
+
|
|
412
|
+
Parameters
|
|
413
|
+
----------
|
|
414
|
+
df : pandas.DataFrame
|
|
415
|
+
数据表
|
|
416
|
+
|
|
417
|
+
Returns
|
|
418
|
+
-------
|
|
419
|
+
binning_series : pandas.Categorical
|
|
420
|
+
分组后序列
|
|
421
|
+
"""
|
|
422
|
+
if self.cut_points is None:
|
|
423
|
+
raise ValueError("类self.cut_points变量为None。")
|
|
424
|
+
binning_series = self.apt_binning(df, self.cut_points, modify=False)
|
|
425
|
+
|
|
426
|
+
return binning_series
|
|
427
|
+
|
|
428
|
+
def auto_binning(self, df, tgt_name, max_bins=10, min_prop_in_bin=0.05, equi_bins=200, equi_method="equif", init_points = None, binning_criteria="chi2", chi2_p=0.95):
|
|
429
|
+
"""
|
|
430
|
+
基于卡方检验的自动分组。
|
|
431
|
+
|
|
432
|
+
基于分布距离计算方法, 按照规定最大组数和最小组样本数, 自动求解分组序列。
|
|
433
|
+
步骤:
|
|
434
|
+
1. 利用初始分组方法, 完成初始分组
|
|
435
|
+
2. 基于分布距离计算方法, 计算初始分组中各组样本大小、组内分布距离, 以及合并相邻组后对原分组的距离差异增益
|
|
436
|
+
3. 不断合并样本量不符合最小组样本数要求, 合并相邻组后距离差异增益较大的组, 直至满足最大组数和最小组样本数要求
|
|
437
|
+
4. 根据最终分组结果, 生成分组后变量序列。
|
|
438
|
+
|
|
439
|
+
Parameters
|
|
440
|
+
----------
|
|
441
|
+
df : pandas.DataFrame
|
|
442
|
+
数据表
|
|
443
|
+
tgt_name : str
|
|
444
|
+
目标变量名
|
|
445
|
+
max_bins : int, default 10
|
|
446
|
+
最大分组数量
|
|
447
|
+
min_prop_in_bin : float, default 0.05
|
|
448
|
+
每组最小样本数占比
|
|
449
|
+
equi_bins : int, default 200
|
|
450
|
+
初始分组数量
|
|
451
|
+
equi_method : string, default "equif"
|
|
452
|
+
初始分组方法, 候选值{"equid"等距分组, "equif"等分分组}
|
|
453
|
+
init_points : array-like, default None
|
|
454
|
+
初始切分点数值序列, 若非None时则equi_bins、equi_method失效
|
|
455
|
+
binning_criteria : string, default "chi2"
|
|
456
|
+
分组准则, 候选值{"chi2": "卡方值"}
|
|
457
|
+
chi2_p : float, default 0.95
|
|
458
|
+
用于计算自由度为1的卡方分布分位数的概率值. 当相邻组卡方值小于该值时, 认为不独立, 进行合并. 故越大独立性判断越严格
|
|
459
|
+
|
|
460
|
+
Returns
|
|
461
|
+
-------
|
|
462
|
+
binning_series : pandas.Categorical
|
|
463
|
+
分组后序列
|
|
464
|
+
"""
|
|
465
|
+
logging.info(f"\n---------------- [{self.var_name} : Auto binning] ----------------")
|
|
466
|
+
min_cnt_in_bin = np.floor(df.shape[0] * min_prop_in_bin)
|
|
467
|
+
logging.info(f"PARAMS: max_bins={max_bins}, min_cnt_in_bin={min_cnt_in_bin}, criteria={binning_criteria}")
|
|
468
|
+
|
|
469
|
+
# 初始分组
|
|
470
|
+
if init_points is None:
|
|
471
|
+
binning_series = self.equi_binning(df, bins=equi_bins, equi_method=equi_method)
|
|
472
|
+
else:
|
|
473
|
+
binning_series = self.apt_binning(df=df, points=init_points, modify=True)
|
|
474
|
+
cut_points = self.cut_points
|
|
475
|
+
|
|
476
|
+
# 初始分组字段透视表
|
|
477
|
+
df_tmp = df[[self.var_name, tgt_name]].copy()
|
|
478
|
+
df_tmp[self.var_name] = binning_series
|
|
479
|
+
df_pvt = cre_pvt(df=df_tmp, var_name=self.var_name, tgt_name=tgt_name)
|
|
480
|
+
df_pvt.index = cut_points[:-1]
|
|
481
|
+
|
|
482
|
+
# 单独列出特殊值
|
|
483
|
+
if bool(self.spec_values):
|
|
484
|
+
df_pvt_spec = df_pvt.loc[[x in self.spec_values for x in df_pvt.index], ].copy()
|
|
485
|
+
df_pvt = df_pvt.loc[[x not in self.spec_values for x in df_pvt.index], ].copy()
|
|
486
|
+
max_bins = max_bins - df_pvt_spec.shape[0]
|
|
487
|
+
|
|
488
|
+
# 自动分组
|
|
489
|
+
if not df_pvt.empty:
|
|
490
|
+
logging.info(f"-------- [{self.var_name} : Auto binning] --------")
|
|
491
|
+
df_pvt = chi2_auto_binning(df_pvt=df_pvt, max_bins=max_bins, min_cnt_in_bin=min_cnt_in_bin, p=chi2_p)
|
|
492
|
+
|
|
493
|
+
# 重新计算分组
|
|
494
|
+
cut_points = df_pvt.index
|
|
495
|
+
|
|
496
|
+
# 添加需要单独成组的特殊数值
|
|
497
|
+
if bool(self.spec_values) and len(self.spec_values) > 0:
|
|
498
|
+
# spec_values_upper = [x + 0.1**spec_digit for x in self.spec_values]
|
|
499
|
+
cut_points = np.append(cut_points, self.spec_values)
|
|
500
|
+
# cut_points = np.append(cut_points, self.spec_values_upper)
|
|
501
|
+
# cut_points.sort()
|
|
502
|
+
|
|
503
|
+
# print("Cut Points After Auto Binning: ", cut_points)
|
|
504
|
+
|
|
505
|
+
binning_series = self.apt_binning(
|
|
506
|
+
df=df,
|
|
507
|
+
points=cut_points,
|
|
508
|
+
modify=True
|
|
509
|
+
)
|
|
510
|
+
|
|
511
|
+
return binning_series
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def cre_pvt(df, var_name, tgt_name):
|
|
515
|
+
"""
|
|
516
|
+
生成以变量为行,目标变量为列的数据透视表。
|
|
517
|
+
|
|
518
|
+
根据指定变量和目标变量生成分组统计透视表,包含每组的负样本数、正样本数、
|
|
519
|
+
总样本数以及目标转化率。
|
|
520
|
+
|
|
521
|
+
Parameters
|
|
522
|
+
----------
|
|
523
|
+
df : pandas.DataFrame
|
|
524
|
+
数据表
|
|
525
|
+
var_name : str
|
|
526
|
+
变量名(分组依据)
|
|
527
|
+
tgt_name : str
|
|
528
|
+
目标变量名(二分类标签,0和1)
|
|
529
|
+
|
|
530
|
+
Returns
|
|
531
|
+
-------
|
|
532
|
+
df_pvt : pandas.DataFrame
|
|
533
|
+
变量数据透视表,包含列:
|
|
534
|
+
- 0: 负样本数
|
|
535
|
+
- 1: 正样本数
|
|
536
|
+
- n: 总样本数
|
|
537
|
+
- tr: 目标转化率(正样本占比)
|
|
538
|
+
|
|
539
|
+
Examples
|
|
540
|
+
--------
|
|
541
|
+
>>> df_pvt = cre_pvt(df, var_name='income', tgt_name='default')
|
|
542
|
+
"""
|
|
543
|
+
|
|
544
|
+
df_pvt = df.groupby(var_name)[tgt_name].value_counts().unstack().fillna(0)
|
|
545
|
+
df_pvt["n"] = df_pvt[0] + df_pvt[1]
|
|
546
|
+
df_pvt["tr"] = df_pvt[1] / df_pvt["n"]
|
|
547
|
+
|
|
548
|
+
return df_pvt
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def merge_bins(df_pvt, ilocs):
|
|
552
|
+
"""
|
|
553
|
+
基于位置编号的变量组合并。
|
|
554
|
+
|
|
555
|
+
将指定位置的分组进行合并,计算合并后的统计指标(样本数、转化率等),
|
|
556
|
+
并更新卡方值和转化率差异。
|
|
557
|
+
|
|
558
|
+
特别注意: 不可重置df_pvt的index
|
|
559
|
+
|
|
560
|
+
Parameters
|
|
561
|
+
----------
|
|
562
|
+
df_pvt : pandas.DataFrame
|
|
563
|
+
变量数据透视表,通过cre_pvt函数生成
|
|
564
|
+
ilocs : list
|
|
565
|
+
待合并的位置编号列表(如[0,1]表示合并前两个分组)
|
|
566
|
+
|
|
567
|
+
Returns
|
|
568
|
+
-------
|
|
569
|
+
df_pvt_new : pandas.DataFrame
|
|
570
|
+
合并后的变量数据透视表
|
|
571
|
+
|
|
572
|
+
Examples
|
|
573
|
+
--------
|
|
574
|
+
>>> df_pvt = merge_bins(df_pvt, ilocs=[0, 1])
|
|
575
|
+
"""
|
|
576
|
+
ilocs.sort()
|
|
577
|
+
df = df_pvt.copy()
|
|
578
|
+
|
|
579
|
+
# 汇总字段值至最小位置序号
|
|
580
|
+
idxes = df.index
|
|
581
|
+
l_idx = idxes[ilocs[0]]
|
|
582
|
+
df.loc[l_idx, [0, 1, "n"]] = np.apply_over_axes(np.sum, df.loc[idxes[ilocs], [0, 1, "n"]], axes=0)[0]
|
|
583
|
+
df.loc[l_idx, "tr"] = df.loc[l_idx, 1] / df.loc[l_idx, "n"]
|
|
584
|
+
df.drop(index=idxes[ilocs[1:]], inplace=True)
|
|
585
|
+
|
|
586
|
+
# 重新计算 tr_diff 和 chisq
|
|
587
|
+
idxes = df.index
|
|
588
|
+
if "tr_diff" in df.columns:
|
|
589
|
+
if ilocs[0]+1 < df.shape[0]:
|
|
590
|
+
df.loc[idxes[ilocs[0]], "tr_diff"] = df.loc[idxes[ilocs[0]+1], "tr"] - df.loc[idxes[ilocs[0]], "tr"]
|
|
591
|
+
else:
|
|
592
|
+
df.loc[idxes[ilocs[0]], "tr_diff"] = 0
|
|
593
|
+
|
|
594
|
+
if "chisq" in df.columns:
|
|
595
|
+
if ilocs[0]+1 < df.shape[0]:
|
|
596
|
+
df.loc[idxes[ilocs[0]], "chisq"] = chi2_contingency(observed_laplace(df.loc[idxes[[ilocs[0], ilocs[0]+1]], [0, 1]]), correction=False)[0]
|
|
597
|
+
else:
|
|
598
|
+
df.loc[idxes[ilocs[0]], "chisq"] = np.inf
|
|
599
|
+
|
|
600
|
+
return df
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def observed_laplace(observed, digit=6):
|
|
604
|
+
"""
|
|
605
|
+
对列联表数值进行拉普拉斯修正。
|
|
606
|
+
|
|
607
|
+
在列联表的每个单元格数值上加上一个小量(0.1**digit),以避免
|
|
608
|
+
零计数导致的卡方检验问题。
|
|
609
|
+
|
|
610
|
+
Parameters
|
|
611
|
+
----------
|
|
612
|
+
observed : array_like
|
|
613
|
+
列联表(二维数组或类似结构)
|
|
614
|
+
digit : int, default 6
|
|
615
|
+
对观测值增加小数的位数, 若为-1则增加整数1
|
|
616
|
+
|
|
617
|
+
Returns
|
|
618
|
+
-------
|
|
619
|
+
obs_laplace : numpy.ndarray
|
|
620
|
+
拉普拉斯修正后的列联表
|
|
621
|
+
|
|
622
|
+
Examples
|
|
623
|
+
--------
|
|
624
|
+
>>> observed = [[10, 20], [30, 40]]
|
|
625
|
+
>>> obs_laplace = observed_laplace(observed, digit=6)
|
|
626
|
+
"""
|
|
627
|
+
obs_laplace = np.asarray(observed) + 0.1**digit
|
|
628
|
+
|
|
629
|
+
return obs_laplace
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def cat_2_list(bin_series):
|
|
633
|
+
"""
|
|
634
|
+
将分箱后的Categorical序列转换为边界值列表。
|
|
635
|
+
|
|
636
|
+
从Categorical类型的数据中提取所有区间边界,返回所有左边界和右边界
|
|
637
|
+
组成的去重列表。
|
|
638
|
+
|
|
639
|
+
Parameters
|
|
640
|
+
----------
|
|
641
|
+
bin_series : pandas.Categorical
|
|
642
|
+
分箱后的Categorical序列
|
|
643
|
+
|
|
644
|
+
Returns
|
|
645
|
+
-------
|
|
646
|
+
list
|
|
647
|
+
包含所有区间边界值的列表
|
|
648
|
+
|
|
649
|
+
Examples
|
|
650
|
+
--------
|
|
651
|
+
>>> edges = cat_2_list(binning_series)
|
|
652
|
+
"""
|
|
653
|
+
|
|
654
|
+
interval_list = bin_series.cat.categories.tolist()
|
|
655
|
+
edges_res = set()
|
|
656
|
+
for x in interval_list:
|
|
657
|
+
edges_res.add(x.left)
|
|
658
|
+
edges_res.add(x.right)
|
|
659
|
+
edges_res = list(edges_res)
|
|
660
|
+
|
|
661
|
+
return edges_res
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def get_bin_range(edges, precision = 5, ascending = False, left_sign = '(', right_sign = ']'):
|
|
665
|
+
"""
|
|
666
|
+
根据分箱边界生成区间字符串描述列表。
|
|
667
|
+
|
|
668
|
+
将分箱边界点列表转换为带区间符号的字符串描述列表,支持自定义
|
|
669
|
+
区间开闭符号和精度。
|
|
670
|
+
|
|
671
|
+
Parameters
|
|
672
|
+
----------
|
|
673
|
+
edges : array-like
|
|
674
|
+
分箱边界值列表
|
|
675
|
+
precision : int, default 5
|
|
676
|
+
边界值精度(小数位数)
|
|
677
|
+
ascending : bool, default False
|
|
678
|
+
是否升序排列
|
|
679
|
+
left_sign : str, default '('
|
|
680
|
+
左区间符号,'['表示包含,'('表示不包含
|
|
681
|
+
right_sign : str, default ']'
|
|
682
|
+
右区间符号,']'表示包含,')'表示不包含
|
|
683
|
+
|
|
684
|
+
Returns
|
|
685
|
+
-------
|
|
686
|
+
list
|
|
687
|
+
区间字符串描述列表
|
|
688
|
+
|
|
689
|
+
Examples
|
|
690
|
+
--------
|
|
691
|
+
>>> edges = [0, 10, 20, 30]
|
|
692
|
+
>>> ranges = get_bin_range(edges, precision=0, ascending=True)
|
|
693
|
+
['[0, 10]', '[10, 20]', '[20, 30]']
|
|
694
|
+
"""
|
|
695
|
+
|
|
696
|
+
i = 0
|
|
697
|
+
reverse = not ascending
|
|
698
|
+
edges = sorted([round(x, precision) for x in edges], reverse = reverse)
|
|
699
|
+
res = []
|
|
700
|
+
while i < len(edges) - 1:
|
|
701
|
+
left = edges[i]
|
|
702
|
+
right = edges[i+1]
|
|
703
|
+
res.append(f"{left_sign}{left}, {right}{right_sign}")
|
|
704
|
+
left = right
|
|
705
|
+
i += 1
|
|
706
|
+
|
|
707
|
+
return res
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def get_bin_range_list(data, col = "_bin_range"):
|
|
711
|
+
"""
|
|
712
|
+
将分箱区间字符串列转换为边界值列表。
|
|
713
|
+
|
|
714
|
+
解析DataFrame中的分箱区间字符串列(如"[0, 10)", "(10, 20]"等),
|
|
715
|
+
提取所有唯一的边界值并排序返回。
|
|
716
|
+
|
|
717
|
+
Parameters
|
|
718
|
+
----------
|
|
719
|
+
data : pandas.DataFrame
|
|
720
|
+
包含分箱区间列的DataFrame
|
|
721
|
+
col : str, default "_bin_range"
|
|
722
|
+
分箱区间列名
|
|
723
|
+
|
|
724
|
+
Returns
|
|
725
|
+
-------
|
|
726
|
+
list
|
|
727
|
+
去重并排序后的边界值列表
|
|
728
|
+
|
|
729
|
+
Examples
|
|
730
|
+
--------
|
|
731
|
+
>>> unique_range = get_bin_range_list(data, col="bin_range")
|
|
732
|
+
"""
|
|
733
|
+
|
|
734
|
+
data["bin_value_list"] = data[col].apply(lambda x: x.replace("[", "")\
|
|
735
|
+
.replace("]", "")\
|
|
736
|
+
.replace("(", "")\
|
|
737
|
+
.replace(")", "")\
|
|
738
|
+
.split(","))
|
|
739
|
+
bin_range = [np.inf if v.strip() == 'inf' else -np.inf if v.strip() == '-inf' else float(v.strip()) for x in data["bin_value_list"].tolist() for v in x]
|
|
740
|
+
|
|
741
|
+
unique_range = []
|
|
742
|
+
for x in bin_range:
|
|
743
|
+
if x not in unique_range:
|
|
744
|
+
unique_range.append(x)
|
|
745
|
+
|
|
746
|
+
unique_range.sort()
|
|
747
|
+
|
|
748
|
+
return unique_range
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def chi2_auto_binning(df_pvt, max_bins, min_cnt_in_bin, p=0.95):
|
|
752
|
+
"""
|
|
753
|
+
基于卡方检验的自动分箱。
|
|
754
|
+
|
|
755
|
+
通过卡方检验判断相邻分箱是否应该合并,迭代进行以下步骤:
|
|
756
|
+
1. 首先处理样本量不满足最小要求的头尾分箱
|
|
757
|
+
2. 然后处理样本量不足的中间分箱
|
|
758
|
+
3. 最后进行卡方检验合并不独立的相邻分箱
|
|
759
|
+
|
|
760
|
+
Parameters
|
|
761
|
+
----------
|
|
762
|
+
df_pvt : pandas.DataFrame
|
|
763
|
+
变量数据透视表,通过cre_pvt函数生成,包含列:0, 1, n, tr
|
|
764
|
+
max_bins : int
|
|
765
|
+
最大分组数量
|
|
766
|
+
min_cnt_in_bin : int
|
|
767
|
+
每组最小样本数
|
|
768
|
+
p : float, default 0.95
|
|
769
|
+
用于计算自由度为1的卡方分布分位数的概率值. 当相邻组卡方值小于该值时,
|
|
770
|
+
认为不独立, 进行合并. 值越大独立性判断越严格
|
|
771
|
+
|
|
772
|
+
Returns
|
|
773
|
+
-------
|
|
774
|
+
df_pvt : pandas.DataFrame
|
|
775
|
+
合并后的变量数据透视表
|
|
776
|
+
|
|
777
|
+
Examples
|
|
778
|
+
--------
|
|
779
|
+
>>> df_pvt = chi2_auto_binning(df_pvt, max_bins=10, min_cnt_in_bin=100, p=0.95)
|
|
780
|
+
"""
|
|
781
|
+
# 计算样本数是否满足最小样本数要求
|
|
782
|
+
if np.sum(df_pvt["n"]) <= min_cnt_in_bin:
|
|
783
|
+
df_pvt = merge_bins(df_pvt=df_pvt, ilocs=list(range(df_pvt.shape[0])))
|
|
784
|
+
logging.info("Merge ilocs: all")
|
|
785
|
+
else:
|
|
786
|
+
# 头尾分箱
|
|
787
|
+
# 计算头尾两侧Bin是否满足最小样本要求
|
|
788
|
+
csumn_asc = np.cumsum(df_pvt["n"])
|
|
789
|
+
ilocs_head = np.min(np.where(csumn_asc >= min_cnt_in_bin))
|
|
790
|
+
if ilocs_head > 0:
|
|
791
|
+
ori_bins = df_pvt.shape[0]
|
|
792
|
+
df_pvt = merge_bins(df_pvt=df_pvt, ilocs=list(range(ilocs_head+1)))
|
|
793
|
+
logging.info(f"HeadMerge: ilocs={[0, ilocs_head+1]}, bins={ori_bins} -> {df_pvt.shape[0]}")
|
|
794
|
+
|
|
795
|
+
csumn_desc = np.cumsum(df_pvt["n"][::-1])
|
|
796
|
+
ilocs_tail = np.min(np.where(csumn_desc >= min_cnt_in_bin))
|
|
797
|
+
if ilocs_tail > 0:
|
|
798
|
+
ori_bins = df_pvt.shape[0]
|
|
799
|
+
df_pvt = merge_bins(df_pvt=df_pvt, ilocs=list(range(ori_bins-(ilocs_tail+1), ori_bins)))
|
|
800
|
+
logging.info(f"TailMerge: ilocs={[ori_bins-(ilocs_tail+1), ori_bins]}, bins={ori_bins} -> {df_pvt.shape[0]}")
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
# 自动分箱
|
|
804
|
+
# 计算初始分组每组卡方值、与后一组合并后的分布散度及增益值
|
|
805
|
+
chisq = [chi2_contingency(observed_laplace(df_pvt.loc[df_pvt.index[[i, i+1]], [0, 1]]), correction=False)[0] for i in range(df_pvt.shape[0] - 1)]
|
|
806
|
+
chisq.append(np.inf)
|
|
807
|
+
tr_diff = np.diff(df_pvt["tr"])
|
|
808
|
+
tr_diff = np.append(tr_diff, 0)
|
|
809
|
+
df_pvt["chisq"] = chisq
|
|
810
|
+
df_pvt["tr_diff"] = tr_diff
|
|
811
|
+
|
|
812
|
+
r = 0
|
|
813
|
+
ori_bins = df_pvt.shape[0]
|
|
814
|
+
while df_pvt.shape[0] > max_bins or any(df_pvt["n"] < min_cnt_in_bin):
|
|
815
|
+
r += 1
|
|
816
|
+
# 优先处理样本量不满足最低要求的Bin, 向前或向后合并
|
|
817
|
+
if any(df_pvt["n"] < min_cnt_in_bin):
|
|
818
|
+
iloc = np.min(np.where(df_pvt["n"] < min_cnt_in_bin))
|
|
819
|
+
# 计算变量组合并方向:位置与单调性
|
|
820
|
+
if iloc == 0:
|
|
821
|
+
iloc_merge = iloc + 1
|
|
822
|
+
elif iloc == df_pvt.shape[0]-1:
|
|
823
|
+
iloc_merge = iloc - 1
|
|
824
|
+
else:
|
|
825
|
+
idxes = df_pvt.index
|
|
826
|
+
if np.abs(df_pvt.loc[idxes[iloc-1], "tr_diff"]) <= np.abs(df_pvt.loc[idxes[iloc], "tr_diff"]):
|
|
827
|
+
iloc_merge = iloc - 1
|
|
828
|
+
else:
|
|
829
|
+
iloc_merge = iloc + 1
|
|
830
|
+
# 其次处理卡方值低的Bin, 向后合并
|
|
831
|
+
else:
|
|
832
|
+
chisq_list = df_pvt["chisq"].to_list()
|
|
833
|
+
iloc = chisq_list.index(np.min(chisq_list))
|
|
834
|
+
iloc_merge = iloc + 1
|
|
835
|
+
|
|
836
|
+
# 变量组合并
|
|
837
|
+
df_pvt = merge_bins(df_pvt=df_pvt, ilocs=[iloc, iloc_merge])
|
|
838
|
+
logging.info(f"LoopMerge: round={r}, bins={ori_bins} -> {df_pvt.shape[0]}")
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
# 卡方检验分箱
|
|
842
|
+
r = 0
|
|
843
|
+
ori_bins = df_pvt.shape[0]
|
|
844
|
+
while df_pvt["chisq"].min() <= chi2.ppf(p, 1):
|
|
845
|
+
r += 1
|
|
846
|
+
chisq_list = df_pvt["chisq"].to_list()
|
|
847
|
+
iloc = chisq_list.index(np.min(chisq_list))
|
|
848
|
+
df_pvt = merge_bins(df_pvt=df_pvt, ilocs=[iloc, iloc + 1])
|
|
849
|
+
logging.info(f"Chi2Merge: round={r}, bins={ori_bins} -> {df_pvt.shape[0]}")
|
|
850
|
+
|
|
851
|
+
return df_pvt
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
def quick_binning(data, column, labels = None, nbins = 10, precision = 5, equal_freq = True, right = True, include_lowest = False,
|
|
855
|
+
min_bin_prop = 0.05, ascending = True, include_missing = False, tree_binning = False, target = None, random_state=42,
|
|
856
|
+
fillna = -999999, spec_values = []):
|
|
857
|
+
"""
|
|
858
|
+
快速分箱函数。
|
|
859
|
+
|
|
860
|
+
对数据进行快速等频或等距分箱,支持多种分箱策略:
|
|
861
|
+
- 等频分箱:按分位数切分,保证每箱样本数接近
|
|
862
|
+
- 等距分箱:按数值区间均匀切分
|
|
863
|
+
- 决策树分箱:使用决策树寻找最优切分点
|
|
864
|
+
|
|
865
|
+
Parameters
|
|
866
|
+
----------
|
|
867
|
+
data : pandas.DataFrame
|
|
868
|
+
输入数据表
|
|
869
|
+
column : str
|
|
870
|
+
需要分箱的列名
|
|
871
|
+
labels : array-like, optional
|
|
872
|
+
自定义分箱标签
|
|
873
|
+
nbins : int or list/tuple, default 10
|
|
874
|
+
分箱数量(整数)或指定分箱边界(列表或元组)
|
|
875
|
+
precision : int, default 5
|
|
876
|
+
边界值精度(小数位数)
|
|
877
|
+
equal_freq : bool, default True
|
|
878
|
+
True为等频分箱,False为等距分箱
|
|
879
|
+
right : bool, default True
|
|
880
|
+
区间是否右闭合
|
|
881
|
+
include_lowest : bool, default False
|
|
882
|
+
是否包含最小值
|
|
883
|
+
min_bin_prop : float, default 0.05
|
|
884
|
+
每箱最小样本占比
|
|
885
|
+
ascending : bool, default True
|
|
886
|
+
分箱顺序是否升序
|
|
887
|
+
include_missing : bool, default False
|
|
888
|
+
是否包含缺失值
|
|
889
|
+
tree_binning : bool, default False
|
|
890
|
+
是否使用决策树分箱
|
|
891
|
+
target : str, optional
|
|
892
|
+
目标变量名(决策树分箱时必需)
|
|
893
|
+
random_state : int, default 42
|
|
894
|
+
随机种子
|
|
895
|
+
fillna : any, default -999999
|
|
896
|
+
缺失值填充值
|
|
897
|
+
spec_values : list, default []
|
|
898
|
+
特殊值列表,将独立成箱
|
|
899
|
+
|
|
900
|
+
Returns
|
|
901
|
+
-------
|
|
902
|
+
binned : pandas.Categorical
|
|
903
|
+
分箱后的序列
|
|
904
|
+
bin_edges : numpy.ndarray
|
|
905
|
+
分箱边界数组
|
|
906
|
+
|
|
907
|
+
Examples
|
|
908
|
+
--------
|
|
909
|
+
>>> binned, edges = quick_binning(data, 'income', nbins=10, equal_freq=True)
|
|
910
|
+
"""
|
|
911
|
+
|
|
912
|
+
binning_series = data[column].round(precision)
|
|
913
|
+
|
|
914
|
+
if include_missing:
|
|
915
|
+
binning_series = binning_series.fillna(fillna)
|
|
916
|
+
else:
|
|
917
|
+
binning_series = binning_series.dropna()
|
|
918
|
+
|
|
919
|
+
# Determine Binning Intervals
|
|
920
|
+
if isinstance(nbins, int):
|
|
921
|
+
|
|
922
|
+
value_no_spec_value = binning_series[~binning_series.isin(spec_values)]
|
|
923
|
+
# print("Tree Binning No Spec Value: ", value_no_spec_value)
|
|
924
|
+
|
|
925
|
+
if len(value_no_spec_value) == 0:
|
|
926
|
+
# 返回一个默认的空分箱结果
|
|
927
|
+
return pd.Series(), []
|
|
928
|
+
|
|
929
|
+
if equal_freq:
|
|
930
|
+
nbins = int(get_max_nbins(data, nbins, min_bin_prop))
|
|
931
|
+
breakpoints = np.percentile(value_no_spec_value, [100 / nbins * i for i in range(1, nbins)])
|
|
932
|
+
breakpoints = list(breakpoints) + spec_values
|
|
933
|
+
else:
|
|
934
|
+
nbins = int(get_max_nbins(data, nbins, min_bin_prop))
|
|
935
|
+
min_value = binning_series.replace(fillna, np.nan).min() if include_missing else value_no_spec_value.min()
|
|
936
|
+
# print("MIN Value: ", min_value)
|
|
937
|
+
breakpoints = np.linspace(min_value, value_no_spec_value.max(), nbins + 1)[1:-1]
|
|
938
|
+
breakpoints = np.sort(np.unique(list(breakpoints) + [fillna])) if include_missing else breakpoints
|
|
939
|
+
breakpoints = list(breakpoints) + spec_values
|
|
940
|
+
|
|
941
|
+
if tree_binning:
|
|
942
|
+
breakpoints = get_decision_tree_binning_edges(
|
|
943
|
+
binning_series,
|
|
944
|
+
data[target],
|
|
945
|
+
max_leaf_nodes = nbins,
|
|
946
|
+
min_samples_leaf = min_bin_prop,
|
|
947
|
+
random_state = random_state,
|
|
948
|
+
missing_ref_value = fillna if include_missing else None,
|
|
949
|
+
spec_values = spec_values
|
|
950
|
+
)
|
|
951
|
+
breakpoints = list(breakpoints)
|
|
952
|
+
# print("Tree Output: ", breakpoints)
|
|
953
|
+
breakpoints = [round(x, precision) for x in breakpoints]
|
|
954
|
+
fnl_breakpoints = np.sort(np.unique([-np.inf, *breakpoints, np.inf]))
|
|
955
|
+
# print("Final Tree Output: ", fnl_breakpoints)
|
|
956
|
+
|
|
957
|
+
if isinstance(nbins, list) or isinstance(nbins, tuple):
|
|
958
|
+
fnl_breakpoints = nbins
|
|
959
|
+
|
|
960
|
+
if len(spec_values) > 0:
|
|
961
|
+
fnl_breakpoints = sorted(list(set(list(fnl_breakpoints) + spec_values)))
|
|
962
|
+
|
|
963
|
+
binned, bin_edges = pd.cut(
|
|
964
|
+
binning_series,
|
|
965
|
+
bins = fnl_breakpoints,
|
|
966
|
+
labels = labels,
|
|
967
|
+
right = right,
|
|
968
|
+
include_lowest = include_lowest,
|
|
969
|
+
retbins = True
|
|
970
|
+
)
|
|
971
|
+
|
|
972
|
+
orig_cat = [x for x in binned.cat.categories.tolist()]
|
|
973
|
+
if ascending:
|
|
974
|
+
binned = binned
|
|
975
|
+
else:
|
|
976
|
+
orig_cat.reverse()
|
|
977
|
+
binned = binned.cat.reorder_categories([*orig_cat], ordered=True)
|
|
978
|
+
|
|
979
|
+
return binned, bin_edges
|
|
980
|
+
|
|
981
|
+
|
|
982
|
+
class Binning:
|
|
983
|
+
"""
|
|
984
|
+
统一的分箱操作类。
|
|
985
|
+
|
|
986
|
+
整合了快速分箱、卡方分箱等多种分箱方法,提供统一的接口进行数据分箱操作。
|
|
987
|
+
支持等频/等距分箱、决策树分箱、卡方自动分箱等多种策略。
|
|
988
|
+
|
|
989
|
+
Parameters
|
|
990
|
+
----------
|
|
991
|
+
data : pandas.DataFrame
|
|
992
|
+
输入数据表(将被复制,不修改原数据)
|
|
993
|
+
column : str
|
|
994
|
+
需要分箱的列名
|
|
995
|
+
tgt_name : str, optional
|
|
996
|
+
目标变量名(卡方分箱时必需)
|
|
997
|
+
nbins : int, default 10
|
|
998
|
+
分箱数量
|
|
999
|
+
precision : int, default 5
|
|
1000
|
+
边界值精度(小数位数)
|
|
1001
|
+
min_bin_prop : float, default 0.05
|
|
1002
|
+
每箱最小样本占比
|
|
1003
|
+
include_missing : bool, default True
|
|
1004
|
+
是否包含缺失值
|
|
1005
|
+
equal_freq : bool, default True
|
|
1006
|
+
True为等频分箱,False为等距分箱
|
|
1007
|
+
bin_colnames : tuple, default ("_bin_num", "_bin_range")
|
|
1008
|
+
分箱结果列名元组
|
|
1009
|
+
ascending : bool, default True
|
|
1010
|
+
分箱顺序是否升序
|
|
1011
|
+
right : bool, default True
|
|
1012
|
+
区间是否右闭合
|
|
1013
|
+
include_lowest : bool, default False
|
|
1014
|
+
是否包含最小值
|
|
1015
|
+
tree_binning : bool, default False
|
|
1016
|
+
是否使用决策树分箱
|
|
1017
|
+
chi2_method : bool, default False
|
|
1018
|
+
是否使用卡方分箱
|
|
1019
|
+
chi2_p : float, default 0.95
|
|
1020
|
+
卡方检验显著性水平
|
|
1021
|
+
init_equi_bins : int, default 200
|
|
1022
|
+
初始等频分箱数量
|
|
1023
|
+
fillna : any, default -999999
|
|
1024
|
+
缺失值填充值
|
|
1025
|
+
spec_values : list, default []
|
|
1026
|
+
特殊值列表
|
|
1027
|
+
random_state : int, default 42
|
|
1028
|
+
随机种子
|
|
1029
|
+
|
|
1030
|
+
Attributes
|
|
1031
|
+
----------
|
|
1032
|
+
result : pandas.DataFrame
|
|
1033
|
+
分箱结果数据
|
|
1034
|
+
bin_edges : numpy.ndarray
|
|
1035
|
+
分箱边界数组
|
|
1036
|
+
|
|
1037
|
+
Examples
|
|
1038
|
+
--------
|
|
1039
|
+
>>> binner = Binning(data, column='income', tgt_name='default', nbins=10)
|
|
1040
|
+
>>> binner.run()
|
|
1041
|
+
>>> result, edges = binner.get_result()
|
|
1042
|
+
|
|
1043
|
+
>>> # 使用卡方分箱
|
|
1044
|
+
>>> binner = Binning(data, column='income', tgt_name='default',
|
|
1045
|
+
... nbins=10, chi2_method=True, chi2_p=0.95)
|
|
1046
|
+
>>> binner.run()
|
|
1047
|
+
"""
|
|
1048
|
+
|
|
1049
|
+
def __init__(self, data, column, tgt_name=None, nbins=10, precision=5, min_bin_prop=0.05,
|
|
1050
|
+
include_missing=True, equal_freq=True, bin_colnames=("_bin_num", "_bin_range"),
|
|
1051
|
+
ascending=True, right=True, include_lowest=False, tree_binning=False,
|
|
1052
|
+
chi2_method=False, chi2_p=0.95, init_equi_bins=200, fillna=-999999,
|
|
1053
|
+
spec_values=[], random_state=42):
|
|
1054
|
+
"""
|
|
1055
|
+
初始化分箱对象。
|
|
1056
|
+
|
|
1057
|
+
Parameters
|
|
1058
|
+
----------
|
|
1059
|
+
data : pandas.DataFrame
|
|
1060
|
+
输入数据表
|
|
1061
|
+
column : str
|
|
1062
|
+
需要分箱的列名
|
|
1063
|
+
tgt_name : str, optional
|
|
1064
|
+
目标变量名
|
|
1065
|
+
nbins : int, default 10
|
|
1066
|
+
分箱数量
|
|
1067
|
+
precision : int, default 5
|
|
1068
|
+
边界值精度
|
|
1069
|
+
min_bin_prop : float, default 0.05
|
|
1070
|
+
每箱最小样本占比
|
|
1071
|
+
include_missing : bool, default True
|
|
1072
|
+
是否包含缺失值
|
|
1073
|
+
equal_freq : bool, default True
|
|
1074
|
+
等频分箱还是等距分箱
|
|
1075
|
+
bin_colnames : tuple, default ("_bin_num", "_bin_range")
|
|
1076
|
+
分箱结果列名
|
|
1077
|
+
ascending : bool, default True
|
|
1078
|
+
分箱顺序是否升序
|
|
1079
|
+
right : bool, default True
|
|
1080
|
+
区间是否右闭合
|
|
1081
|
+
include_lowest : bool, default False
|
|
1082
|
+
是否包含最小值
|
|
1083
|
+
tree_binning : bool, default False
|
|
1084
|
+
是否使用决策树分箱
|
|
1085
|
+
chi2_method : bool, default False
|
|
1086
|
+
是否使用卡方分箱
|
|
1087
|
+
chi2_p : float, default 0.95
|
|
1088
|
+
卡方检验显著性水平
|
|
1089
|
+
init_equi_bins : int, default 200
|
|
1090
|
+
初始等频分箱数量
|
|
1091
|
+
fillna : any, default -999999
|
|
1092
|
+
缺失值填充值
|
|
1093
|
+
spec_values : list, default []
|
|
1094
|
+
特殊值列表
|
|
1095
|
+
random_state : int, default 42
|
|
1096
|
+
随机种子
|
|
1097
|
+
"""
|
|
1098
|
+
self.data = data.copy()
|
|
1099
|
+
self.original_data = data.copy()
|
|
1100
|
+
self.column = column
|
|
1101
|
+
self.tgt_name = tgt_name
|
|
1102
|
+
self.nbins = nbins
|
|
1103
|
+
self.precision = precision
|
|
1104
|
+
self.min_bin_prop = min_bin_prop
|
|
1105
|
+
self.include_missing = include_missing
|
|
1106
|
+
self.equal_freq = equal_freq
|
|
1107
|
+
self.bin_colnames = bin_colnames
|
|
1108
|
+
self.ascending = ascending
|
|
1109
|
+
self.right = right
|
|
1110
|
+
self.include_lowest = include_lowest
|
|
1111
|
+
self.tree_binning = tree_binning
|
|
1112
|
+
self.chi2_method = chi2_method
|
|
1113
|
+
self.chi2_p = chi2_p
|
|
1114
|
+
self.init_equi_bins = init_equi_bins
|
|
1115
|
+
self.fillna = fillna
|
|
1116
|
+
self.spec_values = spec_values
|
|
1117
|
+
self.random_state = random_state
|
|
1118
|
+
self.result = None
|
|
1119
|
+
self.bin_edges = None
|
|
1120
|
+
|
|
1121
|
+
def run_quick_binning(self):
|
|
1122
|
+
"""
|
|
1123
|
+
执行快速分箱。
|
|
1124
|
+
|
|
1125
|
+
根据配置参数执行等频或等距分箱,并将结果添加到数据中。
|
|
1126
|
+
|
|
1127
|
+
Returns
|
|
1128
|
+
-------
|
|
1129
|
+
self
|
|
1130
|
+
返回自身以便链式调用
|
|
1131
|
+
"""
|
|
1132
|
+
if isinstance(self.nbins, int):
|
|
1133
|
+
self.nbins = int(get_max_nbins(data=self.data, nbins=self.nbins,
|
|
1134
|
+
min_bin_prop=self.min_bin_prop))
|
|
1135
|
+
|
|
1136
|
+
if not self.include_missing:
|
|
1137
|
+
self.data = self.data.dropna(subset=[self.column])
|
|
1138
|
+
|
|
1139
|
+
labels = None
|
|
1140
|
+
binned, self.bin_edges = quick_binning(
|
|
1141
|
+
data=self.data,
|
|
1142
|
+
column=self.column,
|
|
1143
|
+
nbins=self.nbins,
|
|
1144
|
+
precision=self.precision,
|
|
1145
|
+
equal_freq=self.equal_freq,
|
|
1146
|
+
labels=labels,
|
|
1147
|
+
right=self.right,
|
|
1148
|
+
include_lowest=self.include_lowest,
|
|
1149
|
+
tree_binning=self.tree_binning,
|
|
1150
|
+
target=self.tgt_name,
|
|
1151
|
+
ascending=self.ascending,
|
|
1152
|
+
include_missing=self.include_missing,
|
|
1153
|
+
random_state=self.random_state,
|
|
1154
|
+
spec_values=self.spec_values
|
|
1155
|
+
)
|
|
1156
|
+
|
|
1157
|
+
left_sign = '[' if self.include_lowest else '('
|
|
1158
|
+
right_sign = ']' if self.right else ')'
|
|
1159
|
+
bin_range_list = get_bin_range(edges=self.bin_edges, precision=self.precision,
|
|
1160
|
+
ascending=self.ascending, left_sign=left_sign,
|
|
1161
|
+
right_sign=right_sign)
|
|
1162
|
+
|
|
1163
|
+
rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not self.include_missing else [i for i in range(0, len(bin_range_list))]
|
|
1164
|
+
binned = binned.cat.rename_categories(rename_catlist)
|
|
1165
|
+
|
|
1166
|
+
bin_num_col = self.bin_colnames[0]
|
|
1167
|
+
bin_range_col = self.bin_colnames[1]
|
|
1168
|
+
|
|
1169
|
+
self.data[bin_num_col] = binned.astype(object)
|
|
1170
|
+
self.data[bin_range_col] = self.data[bin_num_col].apply(
|
|
1171
|
+
lambda x: bin_range_list[int(x)] if self.include_missing else bin_range_list[int(x - 1)]
|
|
1172
|
+
)
|
|
1173
|
+
|
|
1174
|
+
self.result = self.data
|
|
1175
|
+
return self
|
|
1176
|
+
|
|
1177
|
+
def run_chi2_binning(self, init_points=None):
|
|
1178
|
+
"""
|
|
1179
|
+
执行卡方分箱。
|
|
1180
|
+
|
|
1181
|
+
在快速分箱的基础上,进一步使用卡方检验进行自动分箱优化。
|
|
1182
|
+
|
|
1183
|
+
Parameters
|
|
1184
|
+
----------
|
|
1185
|
+
init_points : array-like, optional
|
|
1186
|
+
初始分箱边界,将作为卡方分箱的起点
|
|
1187
|
+
|
|
1188
|
+
Returns
|
|
1189
|
+
-------
|
|
1190
|
+
self
|
|
1191
|
+
返回自身以便链式调用
|
|
1192
|
+
"""
|
|
1193
|
+
self.data = self.original_data.copy()
|
|
1194
|
+
self.data = self.data.reset_index(drop=True)
|
|
1195
|
+
|
|
1196
|
+
df = self.data[[self.column, self.tgt_name]].copy()
|
|
1197
|
+
|
|
1198
|
+
equi_method = "equid"
|
|
1199
|
+
if self.equal_freq:
|
|
1200
|
+
equi_method = "equif"
|
|
1201
|
+
|
|
1202
|
+
if self.include_missing:
|
|
1203
|
+
df[self.column] = df[self.column].fillna(self.fillna)
|
|
1204
|
+
spec_values = [self.fillna, *self.spec_values]
|
|
1205
|
+
else:
|
|
1206
|
+
df = df.dropna()
|
|
1207
|
+
self.data = self.data[~pd.isnull(self.data[self.column])]
|
|
1208
|
+
|
|
1209
|
+
nvb = NumVarBinning(var_name=self.column, spec_values=spec_values,
|
|
1210
|
+
spec_digit=self.precision)
|
|
1211
|
+
binning_series = nvb.auto_binning(
|
|
1212
|
+
df=df,
|
|
1213
|
+
tgt_name=self.tgt_name,
|
|
1214
|
+
max_bins=self.nbins,
|
|
1215
|
+
min_prop_in_bin=self.min_bin_prop,
|
|
1216
|
+
equi_method=equi_method,
|
|
1217
|
+
equi_bins=self.init_equi_bins,
|
|
1218
|
+
binning_criteria='chi2',
|
|
1219
|
+
chi2_p=self.chi2_p,
|
|
1220
|
+
init_points=init_points
|
|
1221
|
+
)
|
|
1222
|
+
|
|
1223
|
+
bin_edges = cat_2_list(binning_series)
|
|
1224
|
+
|
|
1225
|
+
left_sign = '['
|
|
1226
|
+
right_sign = ')'
|
|
1227
|
+
if not self.ascending:
|
|
1228
|
+
left_sign = '('
|
|
1229
|
+
right_sign = ']'
|
|
1230
|
+
bin_range_list = get_bin_range(edges=bin_edges, precision=self.precision,
|
|
1231
|
+
ascending=self.ascending, left_sign=left_sign,
|
|
1232
|
+
right_sign=right_sign)
|
|
1233
|
+
|
|
1234
|
+
binned = binning_series
|
|
1235
|
+
rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not self.include_missing else [i for i in range(0, len(bin_range_list))]
|
|
1236
|
+
binned = binned.cat.rename_categories(rename_catlist)
|
|
1237
|
+
|
|
1238
|
+
bin_num_col = self.bin_colnames[0]
|
|
1239
|
+
bin_range_col = self.bin_colnames[1]
|
|
1240
|
+
|
|
1241
|
+
self.data[bin_num_col] = binned.astype(object)
|
|
1242
|
+
self.data[bin_range_col] = self.data[bin_num_col].apply(
|
|
1243
|
+
lambda x: bin_range_list[int(x)] if self.include_missing else bin_range_list[int(x - 1)]
|
|
1244
|
+
)
|
|
1245
|
+
|
|
1246
|
+
self.bin_edges = sorted([np.inf if str(x).lower() == 'inf' else -np.inf if str(x).lower() == '-inf' else x for x in bin_edges])
|
|
1247
|
+
|
|
1248
|
+
self.result = self.data
|
|
1249
|
+
return self
|
|
1250
|
+
|
|
1251
|
+
def run(self):
|
|
1252
|
+
"""
|
|
1253
|
+
执行分箱操作。
|
|
1254
|
+
|
|
1255
|
+
根据chi2_method参数决定执行快速分箱还是卡方分箱。
|
|
1256
|
+
|
|
1257
|
+
Returns
|
|
1258
|
+
-------
|
|
1259
|
+
self
|
|
1260
|
+
返回自身以便链式调用
|
|
1261
|
+
"""
|
|
1262
|
+
if self.chi2_method:
|
|
1263
|
+
# 先执行快速分箱获取初始边界
|
|
1264
|
+
self.run_quick_binning()
|
|
1265
|
+
init_points = self.bin_edges.copy()
|
|
1266
|
+
|
|
1267
|
+
# 再执行卡方分箱
|
|
1268
|
+
self.run_chi2_binning(init_points=init_points)
|
|
1269
|
+
else:
|
|
1270
|
+
self.run_quick_binning()
|
|
1271
|
+
|
|
1272
|
+
return self
|
|
1273
|
+
|
|
1274
|
+
def get_result(self, return_edges=True):
|
|
1275
|
+
"""
|
|
1276
|
+
获取分箱结果。
|
|
1277
|
+
|
|
1278
|
+
Parameters
|
|
1279
|
+
----------
|
|
1280
|
+
return_edges : bool, default True
|
|
1281
|
+
是否返回分箱边界
|
|
1282
|
+
|
|
1283
|
+
Returns
|
|
1284
|
+
-------
|
|
1285
|
+
tuple or pandas.DataFrame
|
|
1286
|
+
如果return_edges为True,返回(result, bin_edges)元组
|
|
1287
|
+
否则只返回result
|
|
1288
|
+
"""
|
|
1289
|
+
if return_edges:
|
|
1290
|
+
return self.result, self.bin_edges
|
|
1291
|
+
return self.result
|
|
1292
|
+
|
|
1293
|
+
|
|
1294
|
+
def chi2_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, tgt_name = None,
|
|
1295
|
+
include_missing = True, equal_freq = True, bin_colnames = ("_bin_num", "_bin_range"), ascending = True,
|
|
1296
|
+
chi2_p = 0.95, init_equi_bins = 100, fillna = -999999, spec_values = [], init_points = None):
|
|
1297
|
+
"""
|
|
1298
|
+
基于卡方检验的分箱函数。
|
|
1299
|
+
|
|
1300
|
+
使用卡方检验自动寻找最优分箱边界,通过卡方检验判断相邻分箱是否应该合并,
|
|
1301
|
+
最终得到统计上显著的分箱结果。
|
|
1302
|
+
|
|
1303
|
+
Parameters
|
|
1304
|
+
----------
|
|
1305
|
+
data : pandas.DataFrame
|
|
1306
|
+
输入数据表
|
|
1307
|
+
column : str
|
|
1308
|
+
需要分箱的列名
|
|
1309
|
+
nbins : int, default 10
|
|
1310
|
+
最大分箱数量
|
|
1311
|
+
precision : int, default 5
|
|
1312
|
+
边界值精度(小数位数)
|
|
1313
|
+
min_bin_prop : float, default 0.05
|
|
1314
|
+
每箱最小样本占比
|
|
1315
|
+
tgt_name : str
|
|
1316
|
+
目标变量名(二分类标签,0和1)
|
|
1317
|
+
include_missing : bool, default True
|
|
1318
|
+
是否包含缺失值
|
|
1319
|
+
equal_freq : bool, default True
|
|
1320
|
+
True为等频分箱,False为等距分箱
|
|
1321
|
+
bin_colnames : tuple, default ("_bin_num", "_bin_range")
|
|
1322
|
+
分箱结果列名元组
|
|
1323
|
+
ascending : bool, default True
|
|
1324
|
+
分箱顺序是否升序
|
|
1325
|
+
chi2_p : float, default 0.95
|
|
1326
|
+
卡方检验显著性水平
|
|
1327
|
+
init_equi_bins : int, default 100
|
|
1328
|
+
初始等频分箱数量
|
|
1329
|
+
fillna : any, default -999999
|
|
1330
|
+
缺失值填充值
|
|
1331
|
+
spec_values : list, default []
|
|
1332
|
+
特殊值列表
|
|
1333
|
+
init_points : array-like, optional
|
|
1334
|
+
初始分箱边界,将作为卡方分箱的起点
|
|
1335
|
+
|
|
1336
|
+
Returns
|
|
1337
|
+
-------
|
|
1338
|
+
tuple
|
|
1339
|
+
(result, bin_edges) - 分箱结果数据框和分箱边界数组
|
|
1340
|
+
|
|
1341
|
+
Examples
|
|
1342
|
+
--------
|
|
1343
|
+
>>> result, edges = chi2_binning(data, column='income', tgt_name='default', nbins=10)
|
|
1344
|
+
"""
|
|
1345
|
+
|
|
1346
|
+
data = data.reset_index(drop = True)
|
|
1347
|
+
|
|
1348
|
+
df = data[[column, tgt_name]].copy()
|
|
1349
|
+
|
|
1350
|
+
equi_method = "equid"
|
|
1351
|
+
if equal_freq:
|
|
1352
|
+
equi_method = "equif"
|
|
1353
|
+
|
|
1354
|
+
if include_missing:
|
|
1355
|
+
df[column] = df[column].fillna(fillna)
|
|
1356
|
+
spec_values = [fillna, *spec_values]
|
|
1357
|
+
else:
|
|
1358
|
+
df = df.dropna()
|
|
1359
|
+
data = data[~pd.isnull(data[column])]
|
|
1360
|
+
|
|
1361
|
+
nvb = NumVarBinning(var_name=column, spec_values=spec_values, spec_digit=precision)
|
|
1362
|
+
# print(init_points)
|
|
1363
|
+
binning_series = nvb.auto_binning(
|
|
1364
|
+
df=df,
|
|
1365
|
+
tgt_name=tgt_name,
|
|
1366
|
+
max_bins=nbins,
|
|
1367
|
+
min_prop_in_bin=min_bin_prop,
|
|
1368
|
+
equi_method=equi_method,
|
|
1369
|
+
equi_bins=init_equi_bins,
|
|
1370
|
+
binning_criteria='chi2',
|
|
1371
|
+
chi2_p=chi2_p,
|
|
1372
|
+
init_points=init_points)
|
|
1373
|
+
|
|
1374
|
+
bin_num_col = bin_colnames[0]
|
|
1375
|
+
bin_range_col = bin_colnames[1]
|
|
1376
|
+
|
|
1377
|
+
bin_edges = cat_2_list(binning_series)
|
|
1378
|
+
|
|
1379
|
+
left_sign='['
|
|
1380
|
+
right_sign=')'
|
|
1381
|
+
if not ascending:
|
|
1382
|
+
left_sign='('
|
|
1383
|
+
right_sign=']'
|
|
1384
|
+
bin_range_list = get_bin_range(edges = bin_edges, precision = precision, ascending = ascending, left_sign=left_sign, right_sign=right_sign)
|
|
1385
|
+
|
|
1386
|
+
binned = binning_series
|
|
1387
|
+
rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not include_missing else [i for i in range(0, len(bin_range_list))]
|
|
1388
|
+
binned = binned.cat.rename_categories(rename_catlist)
|
|
1389
|
+
data[bin_num_col] = binned.astype(object)
|
|
1390
|
+
# print(bin_range_list)
|
|
1391
|
+
# print(data[bin_num_col])
|
|
1392
|
+
data[bin_range_col] = data[bin_num_col].apply(lambda x: bin_range_list[int(x)] if include_missing else bin_range_list[int(x - 1)])
|
|
1393
|
+
|
|
1394
|
+
fnl_res = data
|
|
1395
|
+
|
|
1396
|
+
bin_edges = sorted([np.inf if str(x).lower() == 'inf' else -np.inf if str(x).lower() == '-inf' else x for x in bin_edges])
|
|
1397
|
+
|
|
1398
|
+
return fnl_res, bin_edges
|
|
1399
|
+
|
|
1400
|
+
|
|
1401
|
+
def run_binning(data, column, nbins = 10, precision = 5, min_bin_prop = 0.05, include_missing = True, equal_freq = True,
|
|
1402
|
+
bin_colnames = ("bin_num", "bin_range"), ascending = False, right = True, include_lowest = False,
|
|
1403
|
+
tree_binning = False, target = None, random_state=42, spec_values = []):
|
|
1404
|
+
"""
|
|
1405
|
+
通用分箱函数,支持等频或等距分箱。
|
|
1406
|
+
|
|
1407
|
+
对数值型变量进行分箱处理,支持多种配置选项,返回分箱后的数据和边界值。
|
|
1408
|
+
|
|
1409
|
+
Parameters
|
|
1410
|
+
----------
|
|
1411
|
+
data : pandas.DataFrame
|
|
1412
|
+
包含数据的DataFrame
|
|
1413
|
+
column : str
|
|
1414
|
+
需要分箱的列名
|
|
1415
|
+
nbins : int, default 10
|
|
1416
|
+
分箱数量
|
|
1417
|
+
precision : int, default 5
|
|
1418
|
+
边界值精度(小数位数)
|
|
1419
|
+
min_bin_prop : float, default 0.05
|
|
1420
|
+
每箱最小样本占比
|
|
1421
|
+
include_missing : bool, default True
|
|
1422
|
+
是否包含缺失值
|
|
1423
|
+
equal_freq : bool, default True
|
|
1424
|
+
True为等频分箱,False为等距分箱
|
|
1425
|
+
bin_colnames : tuple, default ("bin_num", "bin_range")
|
|
1426
|
+
分箱结果列名元组
|
|
1427
|
+
ascending : bool, default False
|
|
1428
|
+
分箱顺序是否升序
|
|
1429
|
+
right : bool, default True
|
|
1430
|
+
区间是否右闭合
|
|
1431
|
+
include_lowest : bool, default False
|
|
1432
|
+
是否包含最小值
|
|
1433
|
+
tree_binning : bool, default False
|
|
1434
|
+
是否使用决策树分箱
|
|
1435
|
+
target : str, optional
|
|
1436
|
+
目标变量名(决策树分箱时必需)
|
|
1437
|
+
random_state : int, default 42
|
|
1438
|
+
随机种子
|
|
1439
|
+
spec_values : list, default []
|
|
1440
|
+
特殊值列表
|
|
1441
|
+
|
|
1442
|
+
Returns
|
|
1443
|
+
-------
|
|
1444
|
+
tuple
|
|
1445
|
+
(data, bin_edges) - 添加了分箱列的数据和分箱边界数组
|
|
1446
|
+
|
|
1447
|
+
Examples
|
|
1448
|
+
--------
|
|
1449
|
+
>>> data, edges = run_binning(data, column='income', nbins=10, equal_freq=True)
|
|
1450
|
+
|
|
1451
|
+
"""
|
|
1452
|
+
|
|
1453
|
+
# 新增保护:如果数据为空或指定列全为缺失,直接返回占位结果
|
|
1454
|
+
if data.empty or data[column].isnull().all():
|
|
1455
|
+
# 返回一个包含缺失分箱的默认结果
|
|
1456
|
+
bin_num_col, bin_range_col = bin_colnames
|
|
1457
|
+
data = data.copy()
|
|
1458
|
+
data[bin_num_col] = 0 if include_missing else 1
|
|
1459
|
+
data[bin_range_col] = "Missing" if include_missing else "All"
|
|
1460
|
+
return data, []
|
|
1461
|
+
|
|
1462
|
+
bin_num_col = bin_colnames[0]
|
|
1463
|
+
bin_range_col = bin_colnames[1]
|
|
1464
|
+
|
|
1465
|
+
if isinstance(nbins, int):
|
|
1466
|
+
nbins = int(get_max_nbins(data = data, nbins = nbins, min_bin_prop = min_bin_prop))
|
|
1467
|
+
|
|
1468
|
+
if not include_missing:
|
|
1469
|
+
data = data.dropna(subset = [column])
|
|
1470
|
+
|
|
1471
|
+
labels = None
|
|
1472
|
+
binned, bin_edges = quick_binning(data = data,
|
|
1473
|
+
column = column,
|
|
1474
|
+
nbins = nbins,
|
|
1475
|
+
precision = precision,
|
|
1476
|
+
equal_freq = equal_freq,
|
|
1477
|
+
labels = labels,
|
|
1478
|
+
right = right,
|
|
1479
|
+
include_lowest = include_lowest,
|
|
1480
|
+
tree_binning = tree_binning,
|
|
1481
|
+
target = target,
|
|
1482
|
+
ascending = ascending,
|
|
1483
|
+
include_missing = include_missing,
|
|
1484
|
+
random_state = random_state,
|
|
1485
|
+
spec_values = spec_values)
|
|
1486
|
+
|
|
1487
|
+
left_sign='[' if include_lowest else '('
|
|
1488
|
+
right_sign=']' if right else ')'
|
|
1489
|
+
bin_range_list = get_bin_range(edges = bin_edges, precision = precision, ascending = ascending, left_sign=left_sign, right_sign=right_sign)
|
|
1490
|
+
|
|
1491
|
+
rename_catlist = [i for i in range(1, len(bin_range_list) + 1)] if not include_missing else [i for i in range(0, len(bin_range_list))]
|
|
1492
|
+
binned = binned.cat.rename_categories(rename_catlist)
|
|
1493
|
+
data[bin_num_col] = binned.astype(object)
|
|
1494
|
+
data[bin_range_col] = data[bin_num_col].apply(lambda x: bin_range_list[int(x)] if include_missing else bin_range_list[int(x - 1)])
|
|
1495
|
+
|
|
1496
|
+
return data, bin_edges
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
def super_binning(data, score, dep, nbins = 10, precision = 5, min_bin_prop = 0.05, include_missing = True,
|
|
1500
|
+
equal_freq = True, chi2_method = False, chi2_p = 0.95, init_equi_bins = 2000, fillna = -999999,
|
|
1501
|
+
spec_values = [], tree_binning = False, random_state=42, return_edges = False, ascending = True,
|
|
1502
|
+
bin_colnames = ("_bin_num", "_bin_range")):
|
|
1503
|
+
"""
|
|
1504
|
+
超级分箱函数,整合多种分箱策略。
|
|
1505
|
+
|
|
1506
|
+
提供统一的分箱接口,支持基础分箱和卡方分箱两种模式,
|
|
1507
|
+
可以根据参数配置自动选择合适的分箱策略。
|
|
1508
|
+
|
|
1509
|
+
Parameters
|
|
1510
|
+
----------
|
|
1511
|
+
data : pandas.DataFrame
|
|
1512
|
+
输入数据表
|
|
1513
|
+
score : str
|
|
1514
|
+
需要分箱的分数/数值列名
|
|
1515
|
+
dep : str
|
|
1516
|
+
目标变量名(二分类标签,0和1)
|
|
1517
|
+
nbins : int, default 10
|
|
1518
|
+
最大分箱数量
|
|
1519
|
+
precision : int, default 5
|
|
1520
|
+
边界值精度(小数位数)
|
|
1521
|
+
min_bin_prop : float, default 0.05
|
|
1522
|
+
每箱最小样本占比
|
|
1523
|
+
include_missing : bool, default True
|
|
1524
|
+
是否包含缺失值
|
|
1525
|
+
equal_freq : bool, default True
|
|
1526
|
+
True为等频分箱,False为等距分箱
|
|
1527
|
+
chi2_method : bool, default False
|
|
1528
|
+
是否使用卡方分箱进行精细化
|
|
1529
|
+
chi2_p : float, default 0.95
|
|
1530
|
+
卡方检验显著性水平
|
|
1531
|
+
init_equi_bins : int, default 2000
|
|
1532
|
+
初始等频分箱数量(卡方分箱前)
|
|
1533
|
+
fillna : any, default -999999
|
|
1534
|
+
缺失值填充值
|
|
1535
|
+
spec_values : list, default []
|
|
1536
|
+
特殊值列表
|
|
1537
|
+
tree_binning : bool, default False
|
|
1538
|
+
是否使用决策树分箱
|
|
1539
|
+
random_state : int, default 42
|
|
1540
|
+
随机种子
|
|
1541
|
+
return_edges : bool, default False
|
|
1542
|
+
是否返回分箱边界
|
|
1543
|
+
ascending : bool, default True
|
|
1544
|
+
分箱顺序是否升序
|
|
1545
|
+
bin_colnames : tuple, default ("_bin_num", "_bin_range")
|
|
1546
|
+
分箱结果列名元组
|
|
1547
|
+
|
|
1548
|
+
Returns
|
|
1549
|
+
-------
|
|
1550
|
+
pandas.DataFrame or tuple
|
|
1551
|
+
如果return_edges为False,返回分箱结果数据
|
|
1552
|
+
如果return_edges为True,返回(result, edges)元组
|
|
1553
|
+
|
|
1554
|
+
Examples
|
|
1555
|
+
--------
|
|
1556
|
+
>>> # 基础分箱
|
|
1557
|
+
>>> result = super_binning(data, score='income', dep='default', nbins=10)
|
|
1558
|
+
|
|
1559
|
+
>>> # 卡方分箱
|
|
1560
|
+
>>> result, edges = super_binning(data, score='income', dep='default',
|
|
1561
|
+
... nbins=10, chi2_method=True, return_edges=True)
|
|
1562
|
+
"""
|
|
1563
|
+
|
|
1564
|
+
res, output_edges = run_binning(data = data,
|
|
1565
|
+
column = score,
|
|
1566
|
+
nbins = nbins,
|
|
1567
|
+
precision = precision,
|
|
1568
|
+
min_bin_prop = min_bin_prop,
|
|
1569
|
+
include_missing = include_missing,
|
|
1570
|
+
equal_freq = equal_freq,
|
|
1571
|
+
bin_colnames = bin_colnames,
|
|
1572
|
+
ascending = ascending,
|
|
1573
|
+
tree_binning = tree_binning,
|
|
1574
|
+
target = dep,
|
|
1575
|
+
random_state = random_state,
|
|
1576
|
+
spec_values = spec_values)
|
|
1577
|
+
|
|
1578
|
+
# print("First Layer Edges: ", output_edges)
|
|
1579
|
+
|
|
1580
|
+
if chi2_method:
|
|
1581
|
+
""" Chi2 Binning. """
|
|
1582
|
+
|
|
1583
|
+
# chi2_edges = [x for x in output_edges if x not in spec_values + [-np.inf, np.inf]]
|
|
1584
|
+
chi2_edges = [x for x in output_edges if x not in [-np.inf, np.inf]]
|
|
1585
|
+
|
|
1586
|
+
# print("Special Value: ", spec_values)
|
|
1587
|
+
# print("Second Layer Inputs: ", chi2_edges)
|
|
1588
|
+
res, output_edges = chi2_binning(data = data,
|
|
1589
|
+
column = score,
|
|
1590
|
+
tgt_name = dep,
|
|
1591
|
+
nbins = nbins,
|
|
1592
|
+
precision = precision,
|
|
1593
|
+
min_bin_prop = min_bin_prop,
|
|
1594
|
+
include_missing = include_missing,
|
|
1595
|
+
equal_freq = equal_freq,
|
|
1596
|
+
bin_colnames = bin_colnames,
|
|
1597
|
+
ascending = ascending,
|
|
1598
|
+
chi2_p = chi2_p,
|
|
1599
|
+
init_equi_bins = init_equi_bins,
|
|
1600
|
+
fillna = fillna,
|
|
1601
|
+
spec_values = spec_values,
|
|
1602
|
+
init_points = chi2_edges)
|
|
1603
|
+
|
|
1604
|
+
|
|
1605
|
+
if return_edges:
|
|
1606
|
+
return res, output_edges
|
|
1607
|
+
|
|
1608
|
+
return res
|