SuperModelingFactory 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. ExcelMaster/ExcelFormatTool.py +487 -0
  2. ExcelMaster/ExcelMaster.py +917 -0
  3. ExcelMaster/Template.py +525 -0
  4. ExcelMaster/Utility.py +233 -0
  5. ExcelMaster/__init__.py +3 -0
  6. Modeling_Tool/Core/Binning_Tool.py +1608 -0
  7. Modeling_Tool/Core/Binning_Tool.pyi +48 -0
  8. Modeling_Tool/Core/Check_DuckDB_Compatibility.py +835 -0
  9. Modeling_Tool/Core/Json_Data_Converter.py +621 -0
  10. Modeling_Tool/Core/Model_Registry_Tool.py +199 -0
  11. Modeling_Tool/Core/ODPS_Tool.py +284 -0
  12. Modeling_Tool/Core/Slope_Tool.py +356 -0
  13. Modeling_Tool/Core/Slope_Tool.pyi +31 -0
  14. Modeling_Tool/Core/XOR_Encryptor.py +207 -0
  15. Modeling_Tool/Core/XOR_Encryptor.pyi +26 -0
  16. Modeling_Tool/Core/__init__.py +99 -0
  17. Modeling_Tool/Core/kDataFrame.py +228 -0
  18. Modeling_Tool/Core/kDataFrame.pyi +43 -0
  19. Modeling_Tool/Core/sample_weight_utils.py +77 -0
  20. Modeling_Tool/Core/utils.py +2672 -0
  21. Modeling_Tool/Eval/Evaluation_Tool.py +1452 -0
  22. Modeling_Tool/Eval/Evaluation_Tool.pyi +47 -0
  23. Modeling_Tool/Eval/Model_Eval_Tool.py +2548 -0
  24. Modeling_Tool/Eval/Model_Eval_Tool.pyi +37 -0
  25. Modeling_Tool/Eval/__init__.py +54 -0
  26. Modeling_Tool/Eval/evaluate_model.py +2008 -0
  27. Modeling_Tool/Eval/evaluate_model.pyi +50 -0
  28. Modeling_Tool/Eval/weighted_eval_utils.py +326 -0
  29. Modeling_Tool/Explainability/Coalition_Structure.py +305 -0
  30. Modeling_Tool/Explainability/Model_Explainer.py +743 -0
  31. Modeling_Tool/Explainability/__init__.py +24 -0
  32. Modeling_Tool/Feature/Distribution_Tool.py +509 -0
  33. Modeling_Tool/Feature/Distribution_Tool.pyi +42 -0
  34. Modeling_Tool/Feature/Feature_Insights.py +762 -0
  35. Modeling_Tool/Feature/Feature_Insights.pyi +33 -0
  36. Modeling_Tool/Feature/PSI_Tool.py +1195 -0
  37. Modeling_Tool/Feature/PSI_Tool.pyi +29 -0
  38. Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py +355 -0
  39. Modeling_Tool/Feature/__init__.py +40 -0
  40. Modeling_Tool/Model/Backward_Tool.py +778 -0
  41. Modeling_Tool/Model/Backward_Tool.pyi +45 -0
  42. Modeling_Tool/Model/GBM_Search_Tool.py +251 -0
  43. Modeling_Tool/Model/GBM_Tool.py +1610 -0
  44. Modeling_Tool/Model/GBM_Tool.pyi +90 -0
  45. Modeling_Tool/Model/LRM_Tool.py +1198 -0
  46. Modeling_Tool/Model/LRM_Tool.pyi +47 -0
  47. Modeling_Tool/Model/__init__.py +61 -0
  48. Modeling_Tool/Sample/Distribution_Adaptation.py +131 -0
  49. Modeling_Tool/Sample/Distribution_Adaptation.pyi +30 -0
  50. Modeling_Tool/Sample/Reject_Infer.py +413 -0
  51. Modeling_Tool/Sample/Reject_Infer.pyi +43 -0
  52. Modeling_Tool/Sample/Sample_Split.py +520 -0
  53. Modeling_Tool/Sample/Sample_Split.pyi +43 -0
  54. Modeling_Tool/Sample/__init__.py +31 -0
  55. Modeling_Tool/UAT/UAT_Consistency_Checker.py +1180 -0
  56. Modeling_Tool/UAT/__init__.py +19 -0
  57. Modeling_Tool/WOE/WOE_Adapter.py +204 -0
  58. Modeling_Tool/WOE/WOE_Adapter.pyi +21 -0
  59. Modeling_Tool/WOE/WOE_Master.py +491 -0
  60. Modeling_Tool/WOE/WOE_Master.pyi +40 -0
  61. Modeling_Tool/WOE/WOE_Monotone_Binner.py +3324 -0
  62. Modeling_Tool/WOE/WOE_Monotone_Binner.pyi +71 -0
  63. Modeling_Tool/WOE/WOE_Plot_Tool.py +919 -0
  64. Modeling_Tool/WOE/WOE_Plot_Tool.pyi +46 -0
  65. Modeling_Tool/WOE/WOE_Report_Builder.py +214 -0
  66. Modeling_Tool/WOE/WOE_Report_Builder.pyi +24 -0
  67. Modeling_Tool/WOE/WOE_Tool.py +1094 -0
  68. Modeling_Tool/WOE/WOE_Tool.pyi +41 -0
  69. Modeling_Tool/WOE/__init__.py +86 -0
  70. Modeling_Tool/WOE/plot_woe_tool.py +290 -0
  71. Modeling_Tool/WOE/plot_woe_tool.pyi +25 -0
  72. Modeling_Tool/__init__.py +176 -0
  73. Report/Report_Tool.py +380 -0
  74. Report/__init__.py +3 -0
  75. supermodelingfactory-0.2.0.dist-info/METADATA +268 -0
  76. supermodelingfactory-0.2.0.dist-info/RECORD +79 -0
  77. supermodelingfactory-0.2.0.dist-info/WHEEL +5 -0
  78. supermodelingfactory-0.2.0.dist-info/licenses/LICENSE +102 -0
  79. supermodelingfactory-0.2.0.dist-info/top_level.txt +3 -0
Report/Report_Tool.py ADDED
@@ -0,0 +1,380 @@
1
+ from ExcelMaster.ExcelMaster import ExcelMaster
2
+ import logging
3
+
4
+ def single_model_perf(em, ws, fig_path, res_path, model_name, image_size, text = None):
5
+ """ Put Single Model Performance Summary. """
6
+
7
+ em.gap_number = 0
8
+ if text is not None:
9
+ em.write_text_content(ws, input_text = text)
10
+ em._resize_image(imgPath = fig_path, resize = image_size, outPath = fig_path)
11
+ img_loc = em.insert_image(ws, figPath = fig_path, retCellRange = "value")
12
+
13
+ em.gap_number = 1
14
+ perf_res = pd.read_csv(res_path)
15
+ perf_res["Top10%_Lift"] = perf_res["Top10%_TargetRate"]/perf_res["avgTrue"]
16
+ perf_res["AUC_Shift"] = perf_res["AUC"].shift(1)/perf_res["AUC"] - 1
17
+ df_loc = em.write_dataframe(ws, df = perf_res.round(3), title=f"Performance for {model_name}",
18
+ index=False, header=True, retCellRange="value")
19
+
20
+ return img_loc, df_loc
21
+
22
+ def write_var_info(em, ws, var, var_name, data_dict, var_info_title = "", skipby = "row"):
23
+ """ Write Var Information Table. """
24
+ description = data_dict.loc[data_dict[var_name] == f"{var}", "description"].values[0]
25
+
26
+ em.gap_number = 0
27
+ single_var_info = data_dict.loc[data_dict[var_name] == var, :]
28
+ info_loc = em.write_dataframe(ws, df = single_var_info, index = False,
29
+ title=var_info_title, skipby=skipby,
30
+ retCellRange="value")
31
+
32
+ return description, info_loc
33
+
34
+
35
+ def plot_woe(em, ws, var, woe_bins, x_col, spec_missing_value = -99999, chart_size = (20, 5), var_name = "var_name", description = "", skipby = "row"):
36
+ """ Plot WOE Table. """
37
+
38
+ pd.options.mode.chained_assignment = None
39
+
40
+ em.gap_number = 1
41
+ single_var_bin = woe_bins[woe_bins[var_name] == var]
42
+ if single_var_bin.shape[0] > 0:
43
+ single_var_bin.loc[:,"mean"] = single_var_bin.loc[:, "tr"].mean()
44
+ single_var_bin.loc[:,"mean_wo_nan"] = single_var_bin.loc[~single_var_bin["bin_value"].str.startswith(f"[{str(spec_missing_value)}"),"tr"].mean()
45
+ iv = round(single_var_bin["iv"].sum(), 4)
46
+
47
+ chart_loc = em.write_duo_chart(worksheet=ws,
48
+ df = single_var_bin,
49
+ y1_list=["n1", "n0"], y2_list=["tr"],
50
+ x = x_col,
51
+ c1_type="stacked_column", c2_type="line",
52
+ y1_axis_range = None, y2_axis_range=None,
53
+ title=f"{var} \n (iv: {iv})",
54
+ chart_size=chart_size,
55
+ xy_axes_name=(description, "N", "Bad Rate"),
56
+ major_gridlines=False,
57
+ retChart=True if single_var_bin.shape[0] > 0 else False,
58
+ y2_num_format = "0.00%",
59
+ y2_line_type=None,
60
+ skipby=skipby,
61
+ retCellRange='value')
62
+
63
+ if single_var_bin.shape[0] > 0:
64
+ column_chart = chart_loc[0]
65
+ line_chart = chart_loc[1]
66
+ fnl_line_chart = em.write_chart(worksheet = ws,
67
+ df = single_var_bin,
68
+ y_list = ['mean', 'mean_wo_nan'],
69
+ x = x_col,
70
+ chart_type = "line",
71
+ chart_size = chart_size,
72
+ y_num_format = "0.00%",
73
+ line_type = "long_dash",
74
+ line_marker = "triangle",
75
+ xy_axes_name = (description, "Bad Rate"),
76
+ major_gridlines=False,
77
+ retChart = True,
78
+ y2_axis = True,
79
+ append_to_chart = line_chart)
80
+
81
+ chart_loc = em.write_combined_chart(ws,
82
+ chart1 = column_chart,
83
+ chart2 = fnl_line_chart,
84
+ chart_size = chart_size,
85
+ skipby = skipby,
86
+ retCellRange="value")
87
+ return chart_loc
88
+
89
+
90
+ def get_woe_plot_report(em, ws, analysis_dir, varlist, means_rpt = None):
91
+ """ Generate Plots for WOE BINS in Excel."""
92
+
93
+ train_image_dir = f"{analysis_dir}/woe_plot/"
94
+
95
+ group_woe_bins = pd.read_csv(f"{analysis_dir}/numvars_woe_group.csv").rename(columns={"Unnamed: 0":"VAR"})
96
+ woe_bins = pd.read_csv(f"{analysis_dir}/numvars_woe.csv").rename(columns={"Unnamed: 0":"VAR"})
97
+
98
+ woe_bins.columns = [x.lower() for x in woe_bins]
99
+ group_woe_bins.columns = [x.lower() for x in group_woe_bins]
100
+
101
+ valid_varlist = []
102
+ for var in varlist:
103
+ train_fig_path = f"{train_image_dir}/{var}_woe_group.png"
104
+ if os.path.isfile(train_fig_path):
105
+ valid_varlist.append(var)
106
+
107
+ logging.info(f"Valid Number of Vars: {len(valid_varlist)}")
108
+
109
+ em.merge_col(ws, ncols=5, text = "Bivar Table")
110
+
111
+ varlist = valid_varlist
112
+
113
+ image_size = (30, 9)
114
+ em.reset_curr_loc(loc = (3, 1))
115
+ info_loc_list = {"train":[], "train_group":[]}
116
+ image_loc_list = {"train":[], "train_group":[]}
117
+ for var in varlist:
118
+
119
+ train_fig_path = f"{train_image_dir}/{var}_woe.png"
120
+ train_group_fig_path = f"{train_image_dir}/{var}_woe_group.png"
121
+
122
+ ### Column 1 ###
123
+ em._resize_image(imgPath=train_fig_path, resize = image_size, outPath = train_fig_path)
124
+ train_image_loc = em.insert_image(ws, figPath=train_fig_path, retCellRange="value")
125
+ image_loc_list["train"].append(train_image_loc)
126
+
127
+ ### Column 2 ###
128
+ start_row = train_image_loc[0]
129
+ start_col = train_image_loc[3] + 1
130
+ em.reset_curr_loc(loc = (start_row, start_col))
131
+
132
+ em._resize_image(imgPath=train_group_fig_path, resize = image_size, outPath = train_group_fig_path)
133
+ train_group_image_loc = em.insert_image(ws, figPath=train_group_fig_path, retCellRange="value")
134
+ image_loc_list["train_group"].append(train_group_image_loc)
135
+
136
+ ### Column 3 ###
137
+ start_row = train_group_image_loc[0]
138
+ start_col = train_group_image_loc[3] + 1
139
+ em.reset_curr_loc(loc = (start_row, start_col))
140
+
141
+ if means_rpt is not None:
142
+
143
+ means_rpt_var = means_rpt.loc[means_rpt['attribute'] == var, :].round(2).drop(columns = ['attribute']).T
144
+ means_rpt_loc = em.write_dataframe(ws, df = means_rpt_var, title=f"Means for {var}", index=True, header=False, retCellRange="value")
145
+
146
+ for i in range(means_rpt_loc[2], means_rpt_loc[2] + 1):
147
+ cell_range = [i, # Start Row
148
+ means_rpt_loc[1] + 1, # Start Col
149
+ i, # End Row
150
+ means_rpt_loc[3] # End Col
151
+ ]
152
+ em.set_color_scale(ws, cell_range=cell_range, colors = ("#FFFFFF", "#F8696B"))
153
+ em.set_cell_format(ws, cell_range=cell_range, cformat = "NUM%.2")
154
+
155
+ ### Next Row ###
156
+ start_row = train_image_loc[2] + 3
157
+ start_col = train_image_loc[1]
158
+ em.reset_curr_loc(loc = (start_row, start_col))
159
+
160
+ em.reset_curr_loc(loc = (3, 0))
161
+ for i, var in enumerate(varlist):
162
+
163
+ em.gap_number = 0
164
+ c_len = image_size[0]
165
+ repeat_num = c_len + 3
166
+ var_name_df = pd.DataFrame([var] * repeat_num)
167
+ var_df_loc = em.write_dataframe(ws, df = var_name_df, title=None, index=False, header=False, retCellRange="value")
168
+
169
+ return 0
170
+
171
+
172
+ def get_woe_plot_report_new(em, ws, woe_plot_dir, grp_name, varlist, means_rpt = None):
173
+ """ Generate Plots for WOE BINS in Excel."""
174
+
175
+ train_image_dir = woe_plot_dir
176
+
177
+ valid_varlist = []
178
+ for var in varlist:
179
+ train_fig_path = f"{train_image_dir}/{var}_{grp_name}.png"
180
+ if os.path.isfile(train_fig_path):
181
+ valid_varlist.append(var)
182
+
183
+ logging.info(f"Valid Number of Vars: {len(valid_varlist)}")
184
+
185
+ em.merge_col(ws, ncols=5, text = "Bivar Table")
186
+
187
+ varlist = valid_varlist
188
+
189
+ image_size = (40, 9)
190
+ em.reset_curr_loc(loc = (3, 1))
191
+ info_loc_list = {"train":[], f"train_{grp_name}":[]}
192
+ image_loc_list = {"train":[], f"train_{grp_name}":[]}
193
+ for var in varlist:
194
+
195
+ train_fig_path = f"{train_image_dir}/{var}.png"
196
+ train_group_fig_path = f"{train_image_dir}/{var}_{grp_name}.png"
197
+
198
+ ### Column 1 ###
199
+ em._resize_image(imgPath=train_fig_path, resize = image_size, outPath = train_fig_path)
200
+ train_image_loc = em.insert_image(ws, figPath=train_fig_path, retCellRange="value")
201
+ image_loc_list["train"].append(train_image_loc)
202
+
203
+ ### Column 2 ###
204
+ start_row = train_image_loc[0]
205
+ start_col = train_image_loc[3] + 1
206
+ em.reset_curr_loc(loc = (start_row, start_col))
207
+
208
+ em._resize_image(imgPath=train_group_fig_path, resize = image_size, outPath = train_group_fig_path)
209
+ train_group_image_loc = em.insert_image(ws, figPath=train_group_fig_path, retCellRange="value")
210
+ image_loc_list[f"train_{grp_name}"].append(train_group_image_loc)
211
+
212
+ ### Column 3 ###
213
+ start_row = train_group_image_loc[0]
214
+ start_col = train_group_image_loc[3] + 1
215
+ em.reset_curr_loc(loc = (start_row, start_col))
216
+
217
+ if means_rpt is not None:
218
+
219
+ means_rpt_var = means_rpt.loc[means_rpt['attribute'] == var, :].round(2).drop(columns = ['attribute']).T
220
+ means_rpt_loc = em.write_dataframe(ws, df = means_rpt_var, title=f"Means for {var}", index=True, header=False, retCellRange="value")
221
+
222
+ for i in range(means_rpt_loc[2], means_rpt_loc[2] + 1):
223
+ cell_range = [i, # Start Row
224
+ means_rpt_loc[1] + 1, # Start Col
225
+ i, # End Row
226
+ means_rpt_loc[3] # End Col
227
+ ]
228
+ em.set_color_scale(ws, cell_range=cell_range, colors = ("#FFFFFF", "#F8696B"))
229
+ em.set_cell_format(ws, cell_range=cell_range, cformat = "NUM%.2")
230
+
231
+ ### Next Row ###
232
+ start_row = train_image_loc[2] + 3
233
+ start_col = train_image_loc[1]
234
+ em.reset_curr_loc(loc = (start_row, start_col))
235
+
236
+ em.reset_curr_loc(loc = (3, 0))
237
+ for i, var in enumerate(varlist):
238
+
239
+ em.gap_number = 0
240
+ c_len = image_size[0]
241
+ repeat_num = c_len + 3
242
+ var_name_df = pd.DataFrame([var] * repeat_num)
243
+ var_df_loc = em.write_dataframe(ws, df = var_name_df, title=None, index=False, header=False, retCellRange="value")
244
+
245
+ return 0
246
+
247
+
248
+ def get_multi_model_perf_report(em, ws, eval_img_path, eval_res_path):
249
+ """ Create Multi-models Evaluation Report. """
250
+
251
+ image_size = (39, 13)
252
+ init_format = {
253
+ 'bold': True,
254
+ 'underline': False,
255
+ 'font_name': 'Arial',
256
+ 'font_size': 18,
257
+ 'font_color': '#000000',
258
+ 'align': 'left'
259
+ }
260
+
261
+ em.add_new_format(format_dict = init_format, format_name="CUS_#")
262
+
263
+ init_format.update({"font_size": 14, "bg_color": "#FFD966"})
264
+ em.add_new_format(format_dict = init_format, format_name="CUS_##")
265
+
266
+ em.write_text_content(worksheet=ws, input_text="{CUS_#} 多模型评估(未调参版) \n \n")
267
+ h2_loc = em.write_text_content(worksheet=ws, input_text="{CUS_##} 直接使用三方特征 \n", retCellRange="value")
268
+
269
+ em.set_cell_format(ws, cell_range=[h2_loc[0], h2_loc[1], h2_loc[2] - 2, h2_loc[3] + 5], cformat = "CUS_##")
270
+
271
+ img_loc1, df_loc1 = single_model_perf(em, ws,
272
+ fig_path = f"{eval_img_path}/xgb_original_perf.jpg",
273
+ res_path = f"{eval_res_path}/xgb_original_perf.csv",
274
+ model_name = "XGBoost",
275
+ image_size = image_size)
276
+
277
+ em.reset_curr_loc(loc=(img_loc1[0], df_loc1[3] + 3))
278
+ img_loc2, df_loc2 = single_model_perf(em, ws,
279
+ fig_path = f"{eval_img_path}/lgb_original_perf.jpg",
280
+ res_path = f"{eval_res_path}/lgb_original_perf.csv",
281
+ model_name = "LightGBM",
282
+ image_size = image_size)
283
+
284
+ ################################## Division Line ###############################
285
+ tmp_color = em.add_new_format({'bg_color': '#D6DCE4'}, "bg_tmp")
286
+ div_line_loc = [df_loc2[2] + 2, 0, df_loc2[2] + 2, 200]
287
+ em.set_cell_format(ws, cell_range=div_line_loc, cformat = "bg_tmp")
288
+ ################################## Division Line ###############################
289
+
290
+ em.reset_curr_loc(loc=(div_line_loc[0] + 2, df_loc1[1]))
291
+ h2_loc = em.write_text_content(worksheet=ws, input_text="{CUS_##} 特征WOE处理后建模 \n", retCellRange="value")
292
+ em.set_cell_format(ws, cell_range=[h2_loc[0], h2_loc[1], h2_loc[2] - 2, h2_loc[3] + 5], cformat = "CUS_##")
293
+
294
+ img_loc, df_loc = single_model_perf(em, ws,
295
+ fig_path = f"{eval_img_path}/lr_woe_perf.jpg",
296
+ res_path = f"{eval_res_path}/lr_woe_perf.csv",
297
+ model_name = "Logistic Regression",
298
+ image_size = image_size)
299
+
300
+ em.reset_curr_loc(loc=(img_loc[0], df_loc[3] + 3))
301
+ img_loc, df_loc = single_model_perf(em, ws,
302
+ fig_path = f"{eval_img_path}/xgb_woe_perf.jpg",
303
+ res_path = f"{eval_res_path}/xgb_woe_perf.csv",
304
+ model_name = "XGBoost",
305
+ image_size = image_size)
306
+
307
+ em.reset_curr_loc(loc=(img_loc[0], df_loc[3] + 3))
308
+ img_loc, df_loc = single_model_perf(em, ws,
309
+ fig_path = f"{eval_img_path}/lgb_woe_perf.jpg",
310
+ res_path = f"{eval_res_path}/lgb_woe_perf.csv",
311
+ model_name = "LightGBM",
312
+ image_size = image_size)
313
+
314
+ return None
315
+
316
+ def get_multi_model_varimp(em, ws, raw_varimp = None, woe_varimp = None):
317
+ """ Varimp for Multi Model Evaluation. """
318
+
319
+ ################################# Varimp Worksheet ##################################
320
+ em.write_text_content(worksheet=ws, input_text="{CUS_#} 特征重要性评估 \n \n")
321
+
322
+ em.gap_number = 1
323
+
324
+ if raw_varimp is not None:
325
+ raw_varimp = raw_varimp[['variable', 'xgb_rank', 'xgb_varimp', 'lgb_rank', 'lgb_varimp']]
326
+ df1_loc = em.write_dataframe(ws, df = raw_varimp.round(4), title=f"Variable Importance (Original Feature)",
327
+ index=False, header=True, retCellRange="value", skipby = 'col')
328
+
329
+ if woe_varimp is not None:
330
+ woe_varimp = woe_varimp[['variable', 'lr_rank', 'coefficient', 'xgb_rank', 'xgb_varimp', 'lgb_rank', 'lgb_varimp']]
331
+ df2_loc = em.write_dataframe(ws, df = woe_varimp.round(4), title=f"Variable Importance (WOE-ed Feature)",
332
+ index=False, header=True, retCellRange="value")
333
+
334
+ return 0
335
+
336
+ def get_fnl_model_report(em, ws, result_dir):
337
+ """ Create Final Model Performance Report. """
338
+
339
+ image_size = (39, 13)
340
+ init_format = {
341
+ 'bold': True,
342
+ 'underline': False,
343
+ 'font_name': 'Arial',
344
+ 'font_size': 18,
345
+ 'font_color': '#000000',
346
+ 'align': 'left'
347
+ }
348
+
349
+ em.add_new_format(format_dict = init_format, format_name="CUS_#")
350
+
351
+ init_format.update({"font_size": 14, "bg_color": "#FFD966"})
352
+ em.add_new_format(format_dict = init_format, format_name="CUS_##")
353
+
354
+ em.write_text_content(worksheet=ws, input_text="{CUS_#} 终版模型评估 \n \n")
355
+
356
+ h2_loc = em.write_text_content(worksheet=ws, input_text="{CUS_##} 原始特征建模 \n", retCellRange="value")
357
+ em.set_cell_format(ws, cell_range=[h2_loc[0], h2_loc[1], h2_loc[2] - 2, h2_loc[3] + 5], cformat = "CUS_##")
358
+
359
+ img_loc, df_loc = single_model_perf(em, ws,
360
+ fig_path = f"{result_dir}/xgb_fnl_model_perf.jpg",
361
+ res_path = f"{result_dir}/xgb_fnl_model_perf.csv",
362
+ model_name = "XGBoost (Without MC)",
363
+ text = "{###} XGBoost (Without Monotonic Constraints) \n",
364
+ image_size = image_size)
365
+
366
+ return 0
367
+
368
+
369
+ def get_model_varimp(em, ws, varimp):
370
+ """ Varimp for Multi Model Evaluation. """
371
+
372
+ ################################# Varimp Worksheet ##################################
373
+ em.write_text_content(worksheet=ws, input_text="{CUS_#} 特征重要性评估 \n \n")
374
+
375
+ em.gap_number = 1
376
+
377
+ df1_loc = em.write_dataframe(ws, df = varimp.round(4), title=f"Variable Importance",
378
+ index=False, header=True, retCellRange="value", skipby = 'col')
379
+
380
+ return 0
Report/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ # encoding: utf-8
2
+ """Report composition helpers bundled with SuperModelingFactory."""
3
+
@@ -0,0 +1,268 @@
1
+ Metadata-Version: 2.4
2
+ Name: SuperModelingFactory
3
+ Version: 0.2.0
4
+ Summary: Credit risk modeling factory: WOE binning, scorecards, LightGBM, Excel reporting.
5
+ Home-page: https://github.com/Kyle-J-Sun/SuperModelingFactory
6
+ Author: Kyle Sun
7
+ Author-email: Kyle Sun <jingkai.sun20@alumni.imperial.ac.uk>
8
+ License: BUSL-1.1
9
+ Project-URL: Homepage, https://github.com/Kyle-J-Sun/SuperModelingFactory
10
+ Project-URL: Documentation, https://kyle-j-sun.github.io/SuperModelingFactory_doc
11
+ Project-URL: Source, https://github.com/Kyle-J-Sun/SuperModelingFactory
12
+ Keywords: credit-risk,scorecard,woe,lightgbm,psi,iv,modeling,fintech
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Intended Audience :: Financial and Insurance Industry
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
21
+ Classifier: License :: Other/Proprietary License
22
+ Requires-Python: >=3.10
23
+ Description-Content-Type: text/markdown
24
+ License-File: LICENSE
25
+ Requires-Dist: pandas<3,>=1.5.0
26
+ Requires-Dist: numpy>=1.23.0
27
+ Requires-Dist: scipy>=1.9.0
28
+ Requires-Dist: scikit-learn>=1.1.0
29
+ Requires-Dist: joblib>=1.2.0
30
+ Requires-Dist: python-dateutil>=2.8.0
31
+ Requires-Dist: lightgbm>=3.3.0
32
+ Requires-Dist: xgboost>=1.7.0
33
+ Requires-Dist: catboost>=1.2.0
34
+ Requires-Dist: matplotlib>=3.5.0
35
+ Requires-Dist: seaborn>=0.12.0
36
+ Requires-Dist: xlsxwriter>=3.0.0
37
+ Requires-Dist: openpyxl>=3.0.0
38
+ Requires-Dist: Pillow>=9.0.0
39
+ Requires-Dist: tqdm>=4.64.0
40
+ Provides-Extra: optuna
41
+ Requires-Dist: optuna>=4.0.0; extra == "optuna"
42
+ Provides-Extra: stats
43
+ Requires-Dist: statsmodels>=0.13.0; extra == "stats"
44
+ Provides-Extra: imblearn
45
+ Requires-Dist: imbalanced-learn>=0.10.0; extra == "imblearn"
46
+ Provides-Extra: explain
47
+ Requires-Dist: shap>=0.44.0; extra == "explain"
48
+ Requires-Dist: lime>=0.2.0.1; extra == "explain"
49
+ Provides-Extra: odps
50
+ Requires-Dist: pyodps>=0.11.0; extra == "odps"
51
+ Dynamic: author
52
+ Dynamic: home-page
53
+ Dynamic: license-file
54
+ Dynamic: requires-python
55
+
56
+ # SuperModelingFactory
57
+
58
+ [![PyPI](https://img.shields.io/pypi/v/supermodelingfactory.svg)](https://pypi.org/project/supermodelingfactory/)
59
+ [![Python](https://img.shields.io/pypi/pyversions/supermodelingfactory.svg)](https://pypi.org/project/supermodelingfactory/)
60
+ [![License: BSL 1.1](https://img.shields.io/badge/license-BSL%201.1-blue.svg)](LICENSE)
61
+ [![Build wheels](https://github.com/Kyle-J-Sun/SuperModelingFactory/actions/workflows/build.yml/badge.svg)](https://github.com/Kyle-J-Sun/SuperModelingFactory/actions/workflows/build.yml)
62
+ [![Docs](https://img.shields.io/badge/docs-mkdocs-blue.svg)](https://kyle-j-sun.github.io/SuperModelingFactory_doc/)
63
+
64
+ 风控建模工厂 —— 一套面向信用评分卡开发与模型管理的完整 Python 工具链。
65
+
66
+ 📖 **[在线文档](https://kyle-j-sun.github.io/SuperModelingFactory_doc/)** · 安装、快速上手、API 参考、用户指南一应俱全。
67
+
68
+ ## 安装
69
+
70
+ ```bash
71
+ pip install supermodelingfactory
72
+ ```
73
+
74
+ macOS 用户额外需要安装 OpenMP 运行时(lightgbm 依赖):
75
+
76
+ ```bash
77
+ brew install libomp
78
+ ```
79
+
80
+ 支持的环境:Python 3.10 / 3.11 / 3.12 / 3.13,平台 macOS arm64 / Linux x86_64 / Windows x86_64。
81
+
82
+ 详见 [INSTALL.md](INSTALL.md)。
83
+
84
+ ## 许可证
85
+
86
+ 本项目采用 **Business Source License 1.1**,Change Date 为 **2030-06-24**。之前:
87
+
88
+ - ✅ 允许:个人学习、学术研究、内部评估、原型、教学
89
+ - ❌ 不允许:任何生产 / 商业 / 营收性使用
90
+
91
+ 2030-06-24 后自动转为 Apache 2.0。商业授权请联系作者。
92
+
93
+ 项目现在以源码形式打包和分发;wheel 与 sdist 均包含 Python 源码,不再通过 Cython 隐藏核心模块实现。
94
+
95
+ ## 项目概述
96
+
97
+ SuperModelingFactory 整合了信贷风控建模全流程所需的三大能力:
98
+
99
+ | 子项目 | 功能定位 | 核心能力 |
100
+ |--------|---------|---------|
101
+ | **[Modeling_Tool](Modeling_Tool/)** | 建模引擎 | 数据分箱、WOE 编码、特征分析、模型训练与评估、样本管理 |
102
+ | **[ExcelMaster](ExcelMaster/)** | 报告引擎 | 程序化 Excel 工作簿生成,支持图表、条件格式、光标流式写入 |
103
+ | **[Report](Report/)** | 报告模板 | 模型性能报告、WOE 图批量导出、多模型对比报告 |
104
+
105
+ ## 项目结构
106
+
107
+ ```
108
+ SuperModelingFactory/
109
+ ├── Modeling_Tool/ # 核心建模工具包
110
+ │ ├── Core/ # 基础设施:分箱、ODPS、工具函数、加密
111
+ │ ├── WOE/ # WOE 编码:分箱、变换、映射、可视化
112
+ │ ├── Feature/ # 特征分析:分布偏移、PSI、相关性过滤
113
+ │ ├── Model/ # 模型训练:LR、LightGBM、XGBoost、变量选择
114
+ │ ├── Eval/ # 模型评估:Gains 表、ROC/KS、性能汇总
115
+ │ └── Sample/ # 样本管理:切分、分层、拒绝推断、分布适配
116
+ ├── ExcelMaster/ # Excel 报告引擎
117
+ │ ├── ExcelFormatTool.py # 格式定义(50+ 预设单元格格式)
118
+ │ ├── ExcelMaster.py # 核心引擎(光标流式写入、图表、条件格式)
119
+ │ ├── Template.py # 分析报告模板(PVA、Bivar、GridSearch 等)
120
+ │ └── Utility.py # 工具函数(颜色、路径、PSI 报表处理等)
121
+ └── Report/ # 模型评估报告模板
122
+ └── Report_Tool.py # 性能报告、WOE 绘图、多模型对比
123
+ ```
124
+
125
+ ## 安装
126
+
127
+ ### 依赖
128
+
129
+ ```bash
130
+ # 核心依赖
131
+ pip install pandas numpy scipy scikit-learn
132
+
133
+ # 建模引擎
134
+ pip install lightgbm xgboost joblib
135
+
136
+ # Excel 报告
137
+ pip install xlsxwriter openpyxl Pillow matplotlib seaborn
138
+
139
+ # 可选
140
+ pip install pyodps # 阿里云 MaxCompute 连接
141
+ pip install imbalanced-learn # SMOTE 采样
142
+ pip install tqdm # 进度条
143
+ ```
144
+
145
+ ### 使用
146
+
147
+ ```bash
148
+ git clone <repo-url>
149
+ cd SuperModelingFactory
150
+ export PYTHONPATH="${PYTHONPATH}:$(pwd)"
151
+ ```
152
+
153
+ ## 快速开始
154
+
155
+ ### 典型风控建模流程
156
+
157
+ ```python
158
+ from Modeling_Tool import (
159
+ # 分箱
160
+ Binning, super_binning,
161
+ # WOE 编码
162
+ WOE_Master,
163
+ # 特征分析
164
+ VarExtractionInsights, CorrelationFilter, PSICalculator,
165
+ # 模型训练
166
+ GradientBoostingModel, LRMaster,
167
+ # 模型评估
168
+ GainsTableCalculator, PerformanceEvaluator,
169
+ # 样本管理
170
+ SampleSplitter, RejectInferrer
171
+ )
172
+
173
+ # 1. 样本切分
174
+ splitter = SampleSplitter(test_size=0.3, random_state=42, stratify=True)
175
+ train_df, test_df = splitter.split_df(data, target='is_bad')
176
+
177
+ # 2. WOE 分箱与编码
178
+ woe_master = WOE_Master(train_data=train_df, varlist=feature_cols, dep='is_bad')
179
+ woe_master.fit(nbins=10, equal_freq=True)
180
+ train_woe = woe_master.transform(train_df)
181
+ test_woe = woe_master.transform(test_df)
182
+
183
+ # 3. 特征筛选
184
+ psi_calc = PSICalculator(buckets=10)
185
+ psi_result = psi_calc.calculate(expected_df=train_df, current_data=test_df, varlist=feature_cols)
186
+
187
+ corr_filter = CorrelationFilter(data=train_woe, dep='is_bad')
188
+ keep_vars = corr_filter.remove_highly_correlated(feature_cols)
189
+
190
+ # 4. 模型训练
191
+ model = GradientBoostingModel('lgb', params={'n_estimators': 100, 'learning_rate': 0.1})
192
+ model.fit(train_woe[keep_vars], train_woe['is_bad'], test_woe[keep_vars], test_woe['is_bad'])
193
+
194
+ # 5. 模型评估
195
+ evaluator = PerformanceEvaluator(tgt_name='is_bad', model=model.model, feature_cols=keep_vars)
196
+ evaluator.add_dataset('train', train_woe).add_dataset('test', test_woe)
197
+ perf_result = evaluator.evaluate()
198
+ ```
199
+
200
+ ### 使用 ExcelMaster 生成报告
201
+
202
+ ```python
203
+ from ExcelMaster.ExcelMaster import ExcelMaster
204
+
205
+ em = ExcelMaster('model_report.xlsx')
206
+ ws = em.add_worksheet('Performance')
207
+
208
+ # 流式写入 DataFrame
209
+ em.write_dataframe(ws, perf_result, title='模型性能汇总', titleformat='BLUE_H2')
210
+ em.insert_image(ws, 'roc_curve.png', figScale=(600, 400))
211
+
212
+ em.close_workbook()
213
+ ```
214
+
215
+ ## 架构设计
216
+
217
+ ### 依赖方向
218
+
219
+ ```
220
+ ┌─────────┐
221
+ │ Core │ (基础设施,无跨包依赖)
222
+ └────┬────┘
223
+ ┌─────────┬───┼───────┬─────────┐
224
+ ▼ ▼ ▼ ▼ ▼
225
+ WOE Model Eval Feature Sample
226
+ │ │ │ │
227
+ └─────────┴──────────────┴────────
228
+ (均单向依赖 Core,模块间延迟导入)
229
+ ```
230
+
231
+ - **Core** 是所有子包的基础,不依赖任何其他子包
232
+ - 其他子包之间通过**延迟导入**(函数体内 import)避免循环依赖
233
+ - 顶层 `Modeling_Tool/__init__.py` 提供精选的统一 API
234
+
235
+ ### 命名规范
236
+
237
+ - 所有公开 API 通过 `__init__.py` 导出,使用方只需 `from Modeling_Tool import ...`
238
+ - 类名采用 PascalCase,函数名采用 snake_case
239
+ - 以 `_` 开头的函数/方法为内部实现,不对外暴露
240
+
241
+ ## 持续集成
242
+
243
+ 本仓库的 GitHub Actions(`.github/workflows/tests.yml`)会在 push 到 `main` 与 PR 上自动跑 pytest,矩阵为:
244
+
245
+ - **Python**:`3.11`、`3.12`
246
+ - **依赖矩阵**:
247
+ - `legacy` — `numpy<2` + `scipy<1.13` + `lightgbm<4`
248
+ - `modern` — `numpy>=2` + `scipy>=1.13` + `lightgbm>=4`
249
+ - **共同约束**:`pandas>=2.0,<2.3`(等 [issue #2](https://github.com/Kyle-J-Sun/SuperModelingFactory/issues/2) 修复后放宽)
250
+
251
+ 测试用例托管在独立仓库 [`SuperModelingFactory_pytest`](https://github.com/Kyle-J-Sun/SuperModelingFactory_pytest)(私有),workflow 通过 `secrets.PYTEST_REPO_TOKEN` 跨仓 clone。
252
+
253
+ ### 配置 PAT(只需做一次)
254
+
255
+ 1. 进入 [GitHub Settings · Tokens (classic)](https://github.com/settings/tokens) 生成新 token,scope 勾选 `repo`(只读访问私有仓库即可)
256
+ 2. 进入本仓库 **Settings → Secrets and variables → Actions → New repository secret**
257
+ 3. Name: `PYTEST_REPO_TOKEN`,Value: 粘贴 token
258
+
259
+ 如改用 Fine-grained PAT,需将其授权访问 `SuperModelingFactory_pytest` 仓库的 *Contents: Read* 权限。
260
+
261
+ ## 版本
262
+
263
+ - **Version**: 0.2.0
264
+ - **Author**: Jingkai Sun
265
+
266
+ ## 许可证
267
+
268
+ 内部项目,仅供团队使用。