SuperModelingFactory 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ExcelMaster/ExcelFormatTool.py +487 -0
- ExcelMaster/ExcelMaster.py +917 -0
- ExcelMaster/Template.py +525 -0
- ExcelMaster/Utility.py +233 -0
- ExcelMaster/__init__.py +3 -0
- Modeling_Tool/Core/Binning_Tool.py +1608 -0
- Modeling_Tool/Core/Binning_Tool.pyi +48 -0
- Modeling_Tool/Core/Check_DuckDB_Compatibility.py +835 -0
- Modeling_Tool/Core/Json_Data_Converter.py +621 -0
- Modeling_Tool/Core/Model_Registry_Tool.py +199 -0
- Modeling_Tool/Core/ODPS_Tool.py +284 -0
- Modeling_Tool/Core/Slope_Tool.py +356 -0
- Modeling_Tool/Core/Slope_Tool.pyi +31 -0
- Modeling_Tool/Core/XOR_Encryptor.py +207 -0
- Modeling_Tool/Core/XOR_Encryptor.pyi +26 -0
- Modeling_Tool/Core/__init__.py +99 -0
- Modeling_Tool/Core/kDataFrame.py +228 -0
- Modeling_Tool/Core/kDataFrame.pyi +43 -0
- Modeling_Tool/Core/sample_weight_utils.py +77 -0
- Modeling_Tool/Core/utils.py +2672 -0
- Modeling_Tool/Eval/Evaluation_Tool.py +1452 -0
- Modeling_Tool/Eval/Evaluation_Tool.pyi +47 -0
- Modeling_Tool/Eval/Model_Eval_Tool.py +2548 -0
- Modeling_Tool/Eval/Model_Eval_Tool.pyi +37 -0
- Modeling_Tool/Eval/__init__.py +54 -0
- Modeling_Tool/Eval/evaluate_model.py +2008 -0
- Modeling_Tool/Eval/evaluate_model.pyi +50 -0
- Modeling_Tool/Eval/weighted_eval_utils.py +326 -0
- Modeling_Tool/Explainability/Coalition_Structure.py +305 -0
- Modeling_Tool/Explainability/Model_Explainer.py +743 -0
- Modeling_Tool/Explainability/__init__.py +24 -0
- Modeling_Tool/Feature/Distribution_Tool.py +509 -0
- Modeling_Tool/Feature/Distribution_Tool.pyi +42 -0
- Modeling_Tool/Feature/Feature_Insights.py +762 -0
- Modeling_Tool/Feature/Feature_Insights.pyi +33 -0
- Modeling_Tool/Feature/PSI_Tool.py +1195 -0
- Modeling_Tool/Feature/PSI_Tool.pyi +29 -0
- Modeling_Tool/Feature/WOE_Engine_Feature_Patch.py +355 -0
- Modeling_Tool/Feature/__init__.py +40 -0
- Modeling_Tool/Model/Backward_Tool.py +778 -0
- Modeling_Tool/Model/Backward_Tool.pyi +45 -0
- Modeling_Tool/Model/GBM_Search_Tool.py +251 -0
- Modeling_Tool/Model/GBM_Tool.py +1610 -0
- Modeling_Tool/Model/GBM_Tool.pyi +90 -0
- Modeling_Tool/Model/LRM_Tool.py +1198 -0
- Modeling_Tool/Model/LRM_Tool.pyi +47 -0
- Modeling_Tool/Model/__init__.py +61 -0
- Modeling_Tool/Sample/Distribution_Adaptation.py +131 -0
- Modeling_Tool/Sample/Distribution_Adaptation.pyi +30 -0
- Modeling_Tool/Sample/Reject_Infer.py +413 -0
- Modeling_Tool/Sample/Reject_Infer.pyi +43 -0
- Modeling_Tool/Sample/Sample_Split.py +520 -0
- Modeling_Tool/Sample/Sample_Split.pyi +43 -0
- Modeling_Tool/Sample/__init__.py +31 -0
- Modeling_Tool/UAT/UAT_Consistency_Checker.py +1180 -0
- Modeling_Tool/UAT/__init__.py +19 -0
- Modeling_Tool/WOE/WOE_Adapter.py +204 -0
- Modeling_Tool/WOE/WOE_Adapter.pyi +21 -0
- Modeling_Tool/WOE/WOE_Master.py +491 -0
- Modeling_Tool/WOE/WOE_Master.pyi +40 -0
- Modeling_Tool/WOE/WOE_Monotone_Binner.py +3324 -0
- Modeling_Tool/WOE/WOE_Monotone_Binner.pyi +71 -0
- Modeling_Tool/WOE/WOE_Plot_Tool.py +919 -0
- Modeling_Tool/WOE/WOE_Plot_Tool.pyi +46 -0
- Modeling_Tool/WOE/WOE_Report_Builder.py +214 -0
- Modeling_Tool/WOE/WOE_Report_Builder.pyi +24 -0
- Modeling_Tool/WOE/WOE_Tool.py +1094 -0
- Modeling_Tool/WOE/WOE_Tool.pyi +41 -0
- Modeling_Tool/WOE/__init__.py +86 -0
- Modeling_Tool/WOE/plot_woe_tool.py +290 -0
- Modeling_Tool/WOE/plot_woe_tool.pyi +25 -0
- Modeling_Tool/__init__.py +176 -0
- Report/Report_Tool.py +380 -0
- Report/__init__.py +3 -0
- supermodelingfactory-0.2.0.dist-info/METADATA +268 -0
- supermodelingfactory-0.2.0.dist-info/RECORD +79 -0
- supermodelingfactory-0.2.0.dist-info/WHEEL +5 -0
- supermodelingfactory-0.2.0.dist-info/licenses/LICENSE +102 -0
- supermodelingfactory-0.2.0.dist-info/top_level.txt +3 -0
|
@@ -0,0 +1,2672 @@
|
|
|
1
|
+
import subprocess
|
|
2
|
+
import os, sys, logging
|
|
3
|
+
logger = logging.getLogger(__name__)
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import numpy as np
|
|
7
|
+
from pandas import *
|
|
8
|
+
|
|
9
|
+
from datetime import date as dt
|
|
10
|
+
from dateutil.relativedelta import relativedelta as rd
|
|
11
|
+
|
|
12
|
+
from .kDataFrame import kDataFrame
|
|
13
|
+
|
|
14
|
+
from .sample_weight_utils import (
|
|
15
|
+
resolve_sample_weight,
|
|
16
|
+
validate_sample_weight,
|
|
17
|
+
weighted_sum,
|
|
18
|
+
weighted_mean,
|
|
19
|
+
weighted_rate,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
def bucket_by_cond(df: pd.DataFrame, cond_dict: dict, colname: str,
|
|
23
|
+
drop_unmatched: bool = True, default=np.nan) -> pd.DataFrame:
|
|
24
|
+
"""
|
|
25
|
+
根据 query 条件字典对数据分组打标签。
|
|
26
|
+
|
|
27
|
+
Parameters
|
|
28
|
+
----------
|
|
29
|
+
df : 原始 DataFrame
|
|
30
|
+
cond_dict : {标签: query条件字符串}
|
|
31
|
+
colname : 新增的标签列名
|
|
32
|
+
drop_unmatched : True=丢弃未命中行(原逻辑), False=保留全量
|
|
33
|
+
default : 未命中行的默认值(仅 drop_unmatched=False 时生效)
|
|
34
|
+
"""
|
|
35
|
+
if drop_unmatched:
|
|
36
|
+
res_list = []
|
|
37
|
+
for label, cond in cond_dict.items():
|
|
38
|
+
sub = df.query(cond).copy()
|
|
39
|
+
sub[colname] = label
|
|
40
|
+
res_list.append(sub)
|
|
41
|
+
return pd.concat(res_list)
|
|
42
|
+
else:
|
|
43
|
+
result = df.copy()
|
|
44
|
+
result[colname] = default
|
|
45
|
+
for label, cond in cond_dict.items():
|
|
46
|
+
mask = result.eval(cond)
|
|
47
|
+
result.loc[mask, colname] = label
|
|
48
|
+
return result
|
|
49
|
+
|
|
50
|
+
def cut2pieces(varlist, n = 4):
|
|
51
|
+
"""
|
|
52
|
+
将列表切分为多个子列表。
|
|
53
|
+
|
|
54
|
+
根据指定的切分数量,将列表均匀地切分为多个子列表。
|
|
55
|
+
|
|
56
|
+
Parameters
|
|
57
|
+
----------
|
|
58
|
+
varlist : list
|
|
59
|
+
需要切分的列表
|
|
60
|
+
n : int, default 4
|
|
61
|
+
切分的子列表数量
|
|
62
|
+
|
|
63
|
+
Returns
|
|
64
|
+
-------
|
|
65
|
+
list
|
|
66
|
+
切分后的子列表
|
|
67
|
+
|
|
68
|
+
Examples
|
|
69
|
+
--------
|
|
70
|
+
>>> cut2pieces([1, 2, 3, 4, 5, 6, 7, 8], n=4)
|
|
71
|
+
[[1, 2], [3, 4], [5, 6], [7, 8]]
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
cut_point = np.floor(len(varlist) / n)
|
|
75
|
+
cut_range = range(0, len(varlist), int(cut_point))
|
|
76
|
+
cut_points = [x for x in cut_range]
|
|
77
|
+
cut_points = cut_points[1: len(cut_points) - 1]
|
|
78
|
+
|
|
79
|
+
cut_list = []
|
|
80
|
+
|
|
81
|
+
i = 0
|
|
82
|
+
while i <= len(cut_points):
|
|
83
|
+
if i == 0:
|
|
84
|
+
cut_list.append(varlist[:cut_points[i]])
|
|
85
|
+
elif i == len(cut_points):
|
|
86
|
+
cut_list.append(varlist[cut_points[i - 1]:])
|
|
87
|
+
else:
|
|
88
|
+
cut_list.append(varlist[cut_points[i - 1]:cut_points[i]])
|
|
89
|
+
i += 1
|
|
90
|
+
|
|
91
|
+
return cut_list
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def check_colname_exist(data, colname):
|
|
95
|
+
"""
|
|
96
|
+
检查列名是否存在于DataFrame中。
|
|
97
|
+
|
|
98
|
+
Parameters
|
|
99
|
+
----------
|
|
100
|
+
data : pandas.DataFrame
|
|
101
|
+
输入数据表
|
|
102
|
+
colname : str
|
|
103
|
+
需要检查的列名
|
|
104
|
+
|
|
105
|
+
Returns
|
|
106
|
+
-------
|
|
107
|
+
bool
|
|
108
|
+
如果列名存在返回True,否则返回False
|
|
109
|
+
|
|
110
|
+
Examples
|
|
111
|
+
--------
|
|
112
|
+
>>> df = pd.DataFrame({'a': [1, 2], 'b': [3, 4]})
|
|
113
|
+
>>> check_colname_exist(df, 'a')
|
|
114
|
+
True
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
return colname in data.columns
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def get_curr_abs_path(path):
|
|
121
|
+
"""
|
|
122
|
+
获取当前模块目录下指定路径的绝对路径。
|
|
123
|
+
|
|
124
|
+
Parameters
|
|
125
|
+
----------
|
|
126
|
+
path : str
|
|
127
|
+
相对路径
|
|
128
|
+
|
|
129
|
+
Returns
|
|
130
|
+
-------
|
|
131
|
+
str
|
|
132
|
+
绝对路径字符串
|
|
133
|
+
"""
|
|
134
|
+
return os.path.dirname(os.path.abspath(__file__)) + "/" + path
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def get_curr_datetime(sep=''):
|
|
138
|
+
"""
|
|
139
|
+
获取当前日期时间字符串。
|
|
140
|
+
|
|
141
|
+
Parameters
|
|
142
|
+
----------
|
|
143
|
+
sep : str, default ''
|
|
144
|
+
日期和时间之间的分隔符
|
|
145
|
+
|
|
146
|
+
Returns
|
|
147
|
+
-------
|
|
148
|
+
str
|
|
149
|
+
格式化后的日期时间字符串,格式为YYYYMMDD{sep}HHMMSS
|
|
150
|
+
|
|
151
|
+
Examples
|
|
152
|
+
--------
|
|
153
|
+
>>> get_curr_datetime() # 返回类似 '20250330143624'
|
|
154
|
+
>>> get_curr_datetime('-') # 返回类似 '20250330-143624'
|
|
155
|
+
"""
|
|
156
|
+
import datetime as dt
|
|
157
|
+
return dt.datetime.now().strftime(f"%Y%m%d{sep}%H%M%S")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def get_buffer_date(start_date):
|
|
161
|
+
"""
|
|
162
|
+
获取起始日期前4周(28天)的日期。
|
|
163
|
+
|
|
164
|
+
Parameters
|
|
165
|
+
----------
|
|
166
|
+
start_date : str
|
|
167
|
+
起始日期,格式为'YYYY-MM-DD'
|
|
168
|
+
|
|
169
|
+
Returns
|
|
170
|
+
-------
|
|
171
|
+
str
|
|
172
|
+
起始日期前4周的日期,格式为'YYYY-MM-DD'
|
|
173
|
+
|
|
174
|
+
Examples
|
|
175
|
+
--------
|
|
176
|
+
>>> get_buffer_date('2025-03-30')
|
|
177
|
+
'2025-03-02'
|
|
178
|
+
"""
|
|
179
|
+
import datetime
|
|
180
|
+
d = datetime.date(int(start_date[0:4]),int(start_date[5:7]),int(start_date[8:10]))
|
|
181
|
+
res = (d + datetime.timedelta(weeks=-4)).strftime("%Y-%m-%d")
|
|
182
|
+
return res
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def get_quarter(strDate):
|
|
186
|
+
"""
|
|
187
|
+
从日期字符串获取季度值。
|
|
188
|
+
|
|
189
|
+
Parameters
|
|
190
|
+
----------
|
|
191
|
+
strDate : str
|
|
192
|
+
日期字符串,格式为'YYYYMM'或'YYYYMMDD'
|
|
193
|
+
|
|
194
|
+
Returns
|
|
195
|
+
-------
|
|
196
|
+
int
|
|
197
|
+
季度值(1-4)
|
|
198
|
+
|
|
199
|
+
Examples
|
|
200
|
+
--------
|
|
201
|
+
>>> get_quarter('202501')
|
|
202
|
+
1
|
|
203
|
+
>>> get_quarter('202506')
|
|
204
|
+
2
|
|
205
|
+
"""
|
|
206
|
+
return ((int(strDate[4:6])-1)//3) + 1
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def get_last_vintage():
|
|
210
|
+
"""
|
|
211
|
+
获取上一个月的年月字符串。
|
|
212
|
+
|
|
213
|
+
Returns
|
|
214
|
+
-------
|
|
215
|
+
str
|
|
216
|
+
上一年月的字符串,格式为'YYYYMM'
|
|
217
|
+
|
|
218
|
+
Examples
|
|
219
|
+
--------
|
|
220
|
+
>>> get_last_vintage() # 如果当前是2025年3月,返回'202502'
|
|
221
|
+
"""
|
|
222
|
+
import datetime
|
|
223
|
+
todayDate = datetime.date.today()
|
|
224
|
+
lastM = todayDate.replace(day=1) - datetime.timedelta(days=1)
|
|
225
|
+
return lastM.strftime("%Y%m")
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def read_csv(path, *args, **kwargs):
|
|
229
|
+
"""
|
|
230
|
+
读取CSV文件并返回kDataFrame对象。
|
|
231
|
+
|
|
232
|
+
Parameters
|
|
233
|
+
----------
|
|
234
|
+
path : str
|
|
235
|
+
CSV文件路径
|
|
236
|
+
*args
|
|
237
|
+
pandas.read_csv的其他位置参数
|
|
238
|
+
**kwargs
|
|
239
|
+
pandas.read_csv的其他关键字参数
|
|
240
|
+
|
|
241
|
+
Returns
|
|
242
|
+
-------
|
|
243
|
+
kDataFrame
|
|
244
|
+
包含数据的kDataFrame对象
|
|
245
|
+
|
|
246
|
+
Examples
|
|
247
|
+
--------
|
|
248
|
+
>>> df = read_csv('data.csv')
|
|
249
|
+
"""
|
|
250
|
+
data = kDataFrame(pd.read_csv(path, *args, **kwargs))
|
|
251
|
+
return data
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def df_to_h2oframe(data):
|
|
255
|
+
"""
|
|
256
|
+
将DataFrame转换为H2OFrame。
|
|
257
|
+
|
|
258
|
+
Parameters
|
|
259
|
+
----------
|
|
260
|
+
data : pandas.DataFrame
|
|
261
|
+
输入数据表
|
|
262
|
+
|
|
263
|
+
Returns
|
|
264
|
+
-------
|
|
265
|
+
h2o.H2OFrame
|
|
266
|
+
H2OFrame对象
|
|
267
|
+
|
|
268
|
+
Examples
|
|
269
|
+
--------
|
|
270
|
+
>>> hf = df_to_h2oframe(df)
|
|
271
|
+
"""
|
|
272
|
+
import h2o
|
|
273
|
+
if isinstance(data, h2o.H2OFrame):
|
|
274
|
+
return data
|
|
275
|
+
else:
|
|
276
|
+
return h2o.H2OFrame(data)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def move_column(data, colname, idx, return_kDF = True, h2o_frame = False):
|
|
280
|
+
"""
|
|
281
|
+
将指定列移动到DataFrame的特定位置。
|
|
282
|
+
|
|
283
|
+
Parameters
|
|
284
|
+
----------
|
|
285
|
+
data : pandas.DataFrame
|
|
286
|
+
输入数据表
|
|
287
|
+
colname : str
|
|
288
|
+
需要移动的列名
|
|
289
|
+
idx : int
|
|
290
|
+
目标位置索引
|
|
291
|
+
return_kDF : bool, default True
|
|
292
|
+
是否返回kDataFrame对象
|
|
293
|
+
h2o_frame : bool, default False
|
|
294
|
+
输入是否为H2OFrame
|
|
295
|
+
|
|
296
|
+
Returns
|
|
297
|
+
-------
|
|
298
|
+
pandas.DataFrame or kDataFrame
|
|
299
|
+
列顺序调整后的数据表
|
|
300
|
+
|
|
301
|
+
Examples
|
|
302
|
+
--------
|
|
303
|
+
>>> df = pd.DataFrame({'a': [1, 2], 'b': [3, 4], 'c': [5, 6]})
|
|
304
|
+
>>> move_column(df, 'c', 0) # 将'c'列移到第一列
|
|
305
|
+
"""
|
|
306
|
+
import h2o
|
|
307
|
+
if h2o_frame:
|
|
308
|
+
return_kDF = False
|
|
309
|
+
colarray = data.columns
|
|
310
|
+
else:
|
|
311
|
+
colarray = data.columns.tolist()
|
|
312
|
+
colarray.remove(colname)
|
|
313
|
+
colarray.insert(idx, colname)
|
|
314
|
+
data = data[colarray]
|
|
315
|
+
if return_kDF:
|
|
316
|
+
return kDataFrame(data)
|
|
317
|
+
return data
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def convert_to_vintage(data, vintage_colname = 'VINTAGE', by = 'TRAN_TMS', return_kDF = True):
|
|
321
|
+
"""
|
|
322
|
+
根据时间列生成Vintage列。
|
|
323
|
+
|
|
324
|
+
Parameters
|
|
325
|
+
----------
|
|
326
|
+
data : pandas.DataFrame
|
|
327
|
+
输入数据表
|
|
328
|
+
vintage_colname : str, default 'VINTAGE'
|
|
329
|
+
生成的Vintage列名
|
|
330
|
+
by : str, default 'TRAN_TMS'
|
|
331
|
+
时间列名
|
|
332
|
+
return_kDF : bool, default True
|
|
333
|
+
是否返回kDataFrame对象
|
|
334
|
+
|
|
335
|
+
Returns
|
|
336
|
+
-------
|
|
337
|
+
pandas.DataFrame or kDataFrame
|
|
338
|
+
添加了Vintage列的数据表
|
|
339
|
+
|
|
340
|
+
Examples
|
|
341
|
+
--------
|
|
342
|
+
>>> df = pd.DataFrame({'TRAN_TMS': ['2025-03-15 10:00:00', '2025-03-20 11:00:00']})
|
|
343
|
+
>>> convert_to_vintage(df)
|
|
344
|
+
"""
|
|
345
|
+
import re
|
|
346
|
+
data[vintage_colname] = data[by].apply(lambda x: re.search("\\d{4}-\\d{2}", x).group().replace('-', ''))
|
|
347
|
+
|
|
348
|
+
if return_kDF:
|
|
349
|
+
return kDataFrame(data)
|
|
350
|
+
return data
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def col_filter_regex(data, regex = ".*?of_co_at_12m", case_sensitive = True, h2o_frame=False, return_kDF = True):
|
|
354
|
+
"""
|
|
355
|
+
使用正则表达式过滤DataFrame的列名。
|
|
356
|
+
|
|
357
|
+
Parameters
|
|
358
|
+
----------
|
|
359
|
+
data : pandas.DataFrame or h2o.H2OFrame
|
|
360
|
+
输入数据表
|
|
361
|
+
regex : str, default ".*?of_co_at_12m"
|
|
362
|
+
正则表达式模式
|
|
363
|
+
case_sensitive : bool, default True
|
|
364
|
+
是否区分大小写
|
|
365
|
+
h2o_frame : bool, default False
|
|
366
|
+
输入是否为H2OFrame
|
|
367
|
+
return_kDF : bool, default True
|
|
368
|
+
是否返回kDataFrame对象
|
|
369
|
+
|
|
370
|
+
Returns
|
|
371
|
+
-------
|
|
372
|
+
pandas.DataFrame or kDataFrame
|
|
373
|
+
过滤后的数据表(只包含匹配的列)
|
|
374
|
+
|
|
375
|
+
Examples
|
|
376
|
+
--------
|
|
377
|
+
>>> df = pd.DataFrame({'score_at_12m': [1, 2], 'other_col': [3, 4]})
|
|
378
|
+
>>> col_filter_regex(df, regex='score_at_12m')
|
|
379
|
+
"""
|
|
380
|
+
if h2o_frame:
|
|
381
|
+
return_kDF = False
|
|
382
|
+
import re
|
|
383
|
+
fltr = []
|
|
384
|
+
for col in data.columns:
|
|
385
|
+
if re.search(regex, col):
|
|
386
|
+
fltr.append(col)
|
|
387
|
+
else:
|
|
388
|
+
fltr = data.columns[data.columns.str.contains(regex, regex = True, case = case_sensitive)]
|
|
389
|
+
if return_kDF:
|
|
390
|
+
return kDataFrame(data[fltr])
|
|
391
|
+
return data[fltr]
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def row_filter_regex(data, col, regex, case_sensitive = True,
|
|
395
|
+
as_index = False, return_kDF = True):
|
|
396
|
+
"""
|
|
397
|
+
使用正则表达式过滤DataFrame的行。
|
|
398
|
+
|
|
399
|
+
Parameters
|
|
400
|
+
----------
|
|
401
|
+
data : pandas.DataFrame
|
|
402
|
+
输入数据表
|
|
403
|
+
col : str
|
|
404
|
+
用于过滤的列名
|
|
405
|
+
regex : str
|
|
406
|
+
正则表达式模式
|
|
407
|
+
case_sensitive : bool, default True
|
|
408
|
+
是否区分大小写
|
|
409
|
+
as_index : bool, default False
|
|
410
|
+
是否将过滤列作为索引
|
|
411
|
+
return_kDF : bool, default True
|
|
412
|
+
是否返回kDataFrame对象
|
|
413
|
+
|
|
414
|
+
Returns
|
|
415
|
+
-------
|
|
416
|
+
pandas.DataFrame or kDataFrame
|
|
417
|
+
过滤后的数据表
|
|
418
|
+
|
|
419
|
+
Examples
|
|
420
|
+
--------
|
|
421
|
+
>>> df = pd.DataFrame({'name': ['apple', 'banana', 'cherry'], 'value': [1, 2, 3]})
|
|
422
|
+
>>> row_filter_regex(df, 'name', 'a.*')
|
|
423
|
+
"""
|
|
424
|
+
fltr = data[col].astype('str').str.contains(pat = regex, regex = True, case = case_sensitive)
|
|
425
|
+
if return_kDF:
|
|
426
|
+
return kDataFrame(data[fltr])
|
|
427
|
+
if as_index:
|
|
428
|
+
return data[fltr].set_index(col)
|
|
429
|
+
return data[fltr]
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def convert_colnames(data, how = "lowercase", return_kDF = True):
|
|
433
|
+
"""
|
|
434
|
+
统一DataFrame列名的格式。
|
|
435
|
+
|
|
436
|
+
Parameters
|
|
437
|
+
----------
|
|
438
|
+
data : pandas.DataFrame
|
|
439
|
+
输入数据表
|
|
440
|
+
how : str, default "lowercase"
|
|
441
|
+
转换方式,可选值:'lower'/'lowercase', 'upper'/'uppercase', 'cap'/'capitalize'
|
|
442
|
+
return_kDF : bool, default True
|
|
443
|
+
是否返回kDataFrame对象
|
|
444
|
+
|
|
445
|
+
Returns
|
|
446
|
+
-------
|
|
447
|
+
pandas.DataFrame or kDataFrame
|
|
448
|
+
列名统一后的数据表
|
|
449
|
+
|
|
450
|
+
Examples
|
|
451
|
+
--------
|
|
452
|
+
>>> df = pd.DataFrame({'NAME': [1], 'Age': [2]})
|
|
453
|
+
>>> convert_colnames(df, 'lower')
|
|
454
|
+
"""
|
|
455
|
+
cols = data.columns
|
|
456
|
+
if how.lower() == "lower" or how.lower() == "lowercase":
|
|
457
|
+
res = [name.lower() for name in cols]
|
|
458
|
+
if how.lower() == "upper" or how.lower() == "uppercase":
|
|
459
|
+
res = [name.upper() for name in cols]
|
|
460
|
+
if how.lower() == "cap" or how.lower() == "capitalize":
|
|
461
|
+
res = [name.capitalize() for name in cols]
|
|
462
|
+
data.columns = res
|
|
463
|
+
if return_kDF:
|
|
464
|
+
return kDataFrame(data)
|
|
465
|
+
return data
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def proc_freq(data, var: str, return_kDF = True) -> pd.DataFrame:
|
|
469
|
+
"""
|
|
470
|
+
实现SAS的PROC FREQ功能,计算频数和百分比。
|
|
471
|
+
|
|
472
|
+
Parameters
|
|
473
|
+
----------
|
|
474
|
+
data : pandas.DataFrame
|
|
475
|
+
输入数据表
|
|
476
|
+
var : str
|
|
477
|
+
需要统计的列名
|
|
478
|
+
return_kDF : bool, default True
|
|
479
|
+
是否返回kDataFrame对象
|
|
480
|
+
|
|
481
|
+
Returns
|
|
482
|
+
-------
|
|
483
|
+
pandas.DataFrame
|
|
484
|
+
包含frequency, percent, cumFrequency, cumPercent的统计表
|
|
485
|
+
|
|
486
|
+
Examples
|
|
487
|
+
--------
|
|
488
|
+
>>> df = pd.DataFrame({'category': ['A', 'B', 'A', 'C', 'A']})
|
|
489
|
+
>>> proc_freq(df, 'category')
|
|
490
|
+
"""
|
|
491
|
+
|
|
492
|
+
f = data[var].value_counts(dropna = False)
|
|
493
|
+
p = data[var].value_counts(dropna = False, normalize = True)
|
|
494
|
+
df = pd.concat([f,p], axis = 1, keys = ['frequency', 'percent'])
|
|
495
|
+
df = df.sort_index()
|
|
496
|
+
df['cumFrequency'] = df['frequency'].cumsum()
|
|
497
|
+
df['cumPercent'] = df['percent'].cumsum()
|
|
498
|
+
if return_kDF:
|
|
499
|
+
return kDataFrame(df)
|
|
500
|
+
return df
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def proc_means(data, varlist = None, quantiles = [0.05, 0.15, 0.25, 0.5, 0.75, 0.95, 0.99]):
|
|
504
|
+
"""
|
|
505
|
+
实现SAS的PROC MEANS功能,计算描述性统计量。
|
|
506
|
+
|
|
507
|
+
Parameters
|
|
508
|
+
----------
|
|
509
|
+
data : pandas.DataFrame
|
|
510
|
+
输入数据表
|
|
511
|
+
varlist : list, optional
|
|
512
|
+
需要统计的列名列表,默认为所有列
|
|
513
|
+
quantiles : list, default [0.05, 0.15, 0.25, 0.5, 0.75, 0.95, 0.99]
|
|
514
|
+
分位数列表
|
|
515
|
+
|
|
516
|
+
Returns
|
|
517
|
+
-------
|
|
518
|
+
pandas.DataFrame
|
|
519
|
+
包含统计量的数据表,包括count, mean, std, min, max及指定分位数
|
|
520
|
+
|
|
521
|
+
Examples
|
|
522
|
+
--------
|
|
523
|
+
>>> df = pd.DataFrame({'a': [1, 2, 3, 4, 5], 'b': [10, 20, 30, 40, 50]})
|
|
524
|
+
>>> proc_means(df)
|
|
525
|
+
"""
|
|
526
|
+
|
|
527
|
+
if varlist is None:
|
|
528
|
+
varlist = data.columns
|
|
529
|
+
|
|
530
|
+
means = data[varlist].describe(percentiles = quantiles).T.rename(columns={"count":"n"})
|
|
531
|
+
|
|
532
|
+
# Rename colnames.
|
|
533
|
+
means.columns = [x.upper() for x in means.columns]
|
|
534
|
+
quantile_rename = {str(int(x * 100)) + "%": "Q" + str(int(x * 100)) for x in quantiles}
|
|
535
|
+
means = means.rename(columns = quantile_rename)
|
|
536
|
+
|
|
537
|
+
# Compute Missing Rate
|
|
538
|
+
means["MISSING_RATE"] = 1 - means["N"]/data.shape[0]
|
|
539
|
+
return means
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def capping_score(data, pb_score: str, multiplier = 1, df_type: str = 'DataFrame'):
|
|
543
|
+
"""
|
|
544
|
+
对模型分数进行缩放和上限处理。
|
|
545
|
+
|
|
546
|
+
Parameters
|
|
547
|
+
----------
|
|
548
|
+
data : pandas.DataFrame
|
|
549
|
+
输入数据表
|
|
550
|
+
pb_score : str
|
|
551
|
+
分数列名
|
|
552
|
+
multiplier : float, default 1
|
|
553
|
+
分数缩放倍数
|
|
554
|
+
df_type : str, default 'DataFrame'
|
|
555
|
+
数据类型,'DataFrame'或'h2o'
|
|
556
|
+
|
|
557
|
+
Returns
|
|
558
|
+
-------
|
|
559
|
+
pandas.Series or h2o.H2OFrame
|
|
560
|
+
处理后的分数
|
|
561
|
+
|
|
562
|
+
Examples
|
|
563
|
+
--------
|
|
564
|
+
>>> df = pd.DataFrame({'score': [0.1, 0.5, 0.99, 1.0]})
|
|
565
|
+
>>> capping_score(df, 'score', multiplier=600, df_type='DataFrame')
|
|
566
|
+
"""
|
|
567
|
+
scores = data[pb_score] * multiplier
|
|
568
|
+
cond = (scores > 0.9999999)
|
|
569
|
+
|
|
570
|
+
if df_type.lower() == 'h2o':
|
|
571
|
+
return cond.ifelse(0.9999999, scores)
|
|
572
|
+
|
|
573
|
+
return (0.9999999 if scores > 0.9999999 else scores)
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def get_filenames(path: str, regex: str) -> [str]:
|
|
577
|
+
"""
|
|
578
|
+
获取指定路径下匹配正则表达式的文件名列表。
|
|
579
|
+
|
|
580
|
+
Parameters
|
|
581
|
+
----------
|
|
582
|
+
path : str
|
|
583
|
+
文件夹路径
|
|
584
|
+
regex : str
|
|
585
|
+
正则表达式模式
|
|
586
|
+
|
|
587
|
+
Returns
|
|
588
|
+
-------
|
|
589
|
+
list
|
|
590
|
+
匹配的文件名列表
|
|
591
|
+
|
|
592
|
+
Examples
|
|
593
|
+
--------
|
|
594
|
+
>>> get_filenames('/path/to/files', '.*\\.csv')
|
|
595
|
+
['file1.csv', 'file2.csv']
|
|
596
|
+
"""
|
|
597
|
+
import re
|
|
598
|
+
outfiles = []
|
|
599
|
+
for (Dirs, subdirs, files) in os.walk(path):
|
|
600
|
+
for file in files:
|
|
601
|
+
if re.search(regex, file):
|
|
602
|
+
outfiles.append(file)
|
|
603
|
+
return outfiles
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def sas_to_csv_by_folder(folder_path: str):
|
|
607
|
+
"""
|
|
608
|
+
将指定文件夹中的所有SAS数据集转换为CSV文件。
|
|
609
|
+
|
|
610
|
+
Parameters
|
|
611
|
+
----------
|
|
612
|
+
folder_path : str
|
|
613
|
+
包含SAS文件的文件夹路径
|
|
614
|
+
|
|
615
|
+
Returns
|
|
616
|
+
-------
|
|
617
|
+
int
|
|
618
|
+
执行状态码(0表示成功)
|
|
619
|
+
|
|
620
|
+
Examples
|
|
621
|
+
--------
|
|
622
|
+
>>> sas_to_csv_by_folder('/path/to/sas/files')
|
|
623
|
+
"""
|
|
624
|
+
filenames = get_filenames(path = folder_path, regex = ".*?sas7bdat")
|
|
625
|
+
sasfilepaths = [folder_path + file for file in filenames]
|
|
626
|
+
from tqdm import tqdm
|
|
627
|
+
with tqdm(total = len(sasfilepaths), position = 0, leave = True, file = sys.stdout) as pbar:
|
|
628
|
+
for i in range(len(sasfilepaths)):
|
|
629
|
+
saspath = sasfilepaths[i]
|
|
630
|
+
logger.info(f"=> converting {filenames[i]}...")
|
|
631
|
+
csvpath = sasfilepaths[i].replace("sas7bdat", "csv")
|
|
632
|
+
sas_to_csv(saspath, csvpath)
|
|
633
|
+
pbar.update()
|
|
634
|
+
return 0
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def _last_modified_date(filename):
|
|
638
|
+
"""
|
|
639
|
+
获取文件的最后修改日期。
|
|
640
|
+
|
|
641
|
+
Parameters
|
|
642
|
+
----------
|
|
643
|
+
filename : str
|
|
644
|
+
文件名
|
|
645
|
+
|
|
646
|
+
Returns
|
|
647
|
+
-------
|
|
648
|
+
str
|
|
649
|
+
文件的最后修改日期字符串
|
|
650
|
+
"""
|
|
651
|
+
proc = subprocess.Popen(["date", "-r", filename, '"+%m-%d-%Y %H:%M:%S"'], stdout=subprocess.PIPE, shell=True)
|
|
652
|
+
(out, err) = proc.communicate()
|
|
653
|
+
return out.decode('ascii')
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def read_attr_list(path: str = "pe_attr_list.txt", lower = False):
|
|
657
|
+
"""
|
|
658
|
+
读取属性列表文件(每行一个属性)。
|
|
659
|
+
|
|
660
|
+
Parameters
|
|
661
|
+
----------
|
|
662
|
+
path : str, default "pe_attr_list.txt"
|
|
663
|
+
文件路径
|
|
664
|
+
lower : bool, default False
|
|
665
|
+
是否转换为小写
|
|
666
|
+
|
|
667
|
+
Returns
|
|
668
|
+
-------
|
|
669
|
+
list
|
|
670
|
+
属性列表
|
|
671
|
+
|
|
672
|
+
Examples
|
|
673
|
+
--------
|
|
674
|
+
>>> read_attr_list('vars.txt', lower=True)
|
|
675
|
+
"""
|
|
676
|
+
with open(path) as f:
|
|
677
|
+
lines = f.readlines()
|
|
678
|
+
|
|
679
|
+
ls = []
|
|
680
|
+
for line in lines:
|
|
681
|
+
ls.append(line.strip().upper())
|
|
682
|
+
f.close()
|
|
683
|
+
if lower:
|
|
684
|
+
return [x.lower() for x in ls]
|
|
685
|
+
return ls
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def write_attr_list(var_list: list, path: str = "_vls_results.txt", sep="\n", quote='double'):
|
|
689
|
+
"""
|
|
690
|
+
将变量列表写入文件。
|
|
691
|
+
|
|
692
|
+
Parameters
|
|
693
|
+
----------
|
|
694
|
+
var_list : list
|
|
695
|
+
要写入的变量列表
|
|
696
|
+
path : str, default "_vls_results.txt"
|
|
697
|
+
输出文件路径
|
|
698
|
+
sep : str, default "\n"
|
|
699
|
+
分隔符
|
|
700
|
+
quote : str, default 'double'
|
|
701
|
+
引号类型,'double', 'single', 或 'none'
|
|
702
|
+
|
|
703
|
+
Returns
|
|
704
|
+
-------
|
|
705
|
+
None
|
|
706
|
+
|
|
707
|
+
Examples
|
|
708
|
+
--------
|
|
709
|
+
>>> write_attr_list(['var1', 'var2'], 'output.txt', quote='single')
|
|
710
|
+
"""
|
|
711
|
+
with open(path, "w+") as f:
|
|
712
|
+
for v in var_list:
|
|
713
|
+
if quote == 'double':
|
|
714
|
+
f.writelines('"'+str(v)+'"'+sep)
|
|
715
|
+
elif quote == 'single':
|
|
716
|
+
f.writelines("'"+str(v)+"'"+sep)
|
|
717
|
+
else:
|
|
718
|
+
f.writelines(str(v)+sep)
|
|
719
|
+
f.close()
|
|
720
|
+
return None
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def list_filter_regex(ls, regex):
|
|
724
|
+
"""
|
|
725
|
+
使用正则表达式过滤列表元素。
|
|
726
|
+
|
|
727
|
+
Parameters
|
|
728
|
+
----------
|
|
729
|
+
ls : list
|
|
730
|
+
输入列表
|
|
731
|
+
regex : str
|
|
732
|
+
正则表达式模式
|
|
733
|
+
|
|
734
|
+
Returns
|
|
735
|
+
-------
|
|
736
|
+
list
|
|
737
|
+
匹配的元素列表
|
|
738
|
+
|
|
739
|
+
Examples
|
|
740
|
+
--------
|
|
741
|
+
>>> list_filter_regex(['abc', 'def', 'abf'], 'ab.*')
|
|
742
|
+
['abc', 'abf']
|
|
743
|
+
"""
|
|
744
|
+
import re
|
|
745
|
+
ret = []
|
|
746
|
+
for elem in ls:
|
|
747
|
+
if re.search(regex, elem):
|
|
748
|
+
ret.append(elem)
|
|
749
|
+
return ret
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def list_to_h2oFrame(val: str or float or int, length: int):
|
|
753
|
+
"""
|
|
754
|
+
将值转换为指定长度的H2O Frame。
|
|
755
|
+
|
|
756
|
+
Parameters
|
|
757
|
+
----------
|
|
758
|
+
val : str or float or int
|
|
759
|
+
值
|
|
760
|
+
length : int
|
|
761
|
+
长度
|
|
762
|
+
|
|
763
|
+
Returns
|
|
764
|
+
-------
|
|
765
|
+
h2o.H2OFrame
|
|
766
|
+
包含重复值的H2OFrame
|
|
767
|
+
|
|
768
|
+
Examples
|
|
769
|
+
--------
|
|
770
|
+
>>> list_to_h2oFrame(5, 10)
|
|
771
|
+
"""
|
|
772
|
+
""" convert list to H2O Frame """
|
|
773
|
+
import h2o
|
|
774
|
+
return h2o.H2OFrame([val] * length)
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def odds_score(pb_score, event_ratio = 15, margin_point = 20, score_point = 500):
|
|
778
|
+
"""
|
|
779
|
+
根据概率分数计算Odds分数。
|
|
780
|
+
|
|
781
|
+
用于将概率值转换为信用分数刻度。
|
|
782
|
+
|
|
783
|
+
Parameters
|
|
784
|
+
----------
|
|
785
|
+
pb_score : float
|
|
786
|
+
预测概率(0到1之间)
|
|
787
|
+
event_ratio : float, default 15
|
|
788
|
+
事件比例
|
|
789
|
+
margin_point : float, default 20
|
|
790
|
+
分数点差
|
|
791
|
+
score_point : float, default 500
|
|
792
|
+
基础分数点
|
|
793
|
+
|
|
794
|
+
Returns
|
|
795
|
+
-------
|
|
796
|
+
float
|
|
797
|
+
Odds分数
|
|
798
|
+
|
|
799
|
+
Examples
|
|
800
|
+
--------
|
|
801
|
+
>>> odds_score(0.03, event_ratio=15, margin_point=20, score_point=500)
|
|
802
|
+
619.3...
|
|
803
|
+
"""
|
|
804
|
+
a = (margin_point / np.log(2))
|
|
805
|
+
b = (np.log(event_ratio) + np.log(pb_score/(1 - pb_score)))
|
|
806
|
+
return (score_point - a * b)
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def last_Month_Vintage(year: int, month: int, day: int) -> str:
|
|
810
|
+
"""
|
|
811
|
+
获取上一个月的年月字符串。
|
|
812
|
+
|
|
813
|
+
Parameters
|
|
814
|
+
----------
|
|
815
|
+
year : int
|
|
816
|
+
年份
|
|
817
|
+
month : int
|
|
818
|
+
月份
|
|
819
|
+
day : int
|
|
820
|
+
日期
|
|
821
|
+
|
|
822
|
+
Returns
|
|
823
|
+
-------
|
|
824
|
+
str
|
|
825
|
+
上一个月的年月字符串,格式为'YYYYMM'
|
|
826
|
+
|
|
827
|
+
Examples
|
|
828
|
+
--------
|
|
829
|
+
>>> last_Month_Vintage(2025, 3, 15)
|
|
830
|
+
202502
|
|
831
|
+
"""
|
|
832
|
+
lastDate = dt(year, month, day) + rd(months=-1)
|
|
833
|
+
yr = str(lastDate.year)
|
|
834
|
+
mth = "0" + str(lastDate.month) if len([char for char in str(lastDate.month)]) == 1 else str(lastDate.month)
|
|
835
|
+
lastMthVtge = yr + mth
|
|
836
|
+
return int(lastMthVtge)
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
def read_sas_file(file_path_name=''):
|
|
840
|
+
"""
|
|
841
|
+
读取SAS数据集文件。
|
|
842
|
+
|
|
843
|
+
使用latin-1编码读取SAS文件,这是SAS Studio和SAS Grid的默认编码。
|
|
844
|
+
|
|
845
|
+
Parameters
|
|
846
|
+
----------
|
|
847
|
+
file_path_name : str
|
|
848
|
+
SAS文件路径
|
|
849
|
+
|
|
850
|
+
Returns
|
|
851
|
+
-------
|
|
852
|
+
kDataFrame
|
|
853
|
+
包含数据的kDataFrame对象
|
|
854
|
+
|
|
855
|
+
Examples
|
|
856
|
+
--------
|
|
857
|
+
>>> df = read_sas_file('data.sas7bdat')
|
|
858
|
+
"""
|
|
859
|
+
df = pd.read_sas(file_path_name,
|
|
860
|
+
format = 'sas7bdat', encoding="latin-1")
|
|
861
|
+
return kDataFrame(df)
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
def sas_to_csv(fileNameWithPath, outputFileNameWithPath, timecounter = True):
|
|
865
|
+
"""
|
|
866
|
+
将SAS数据集转换为CSV文件。
|
|
867
|
+
|
|
868
|
+
Parameters
|
|
869
|
+
----------
|
|
870
|
+
fileNameWithPath : str
|
|
871
|
+
输入SAS文件路径
|
|
872
|
+
outputFileNameWithPath : str
|
|
873
|
+
输出CSV文件路径
|
|
874
|
+
timecounter : bool, default True
|
|
875
|
+
是否打印执行时间
|
|
876
|
+
|
|
877
|
+
Returns
|
|
878
|
+
-------
|
|
879
|
+
int
|
|
880
|
+
执行状态码(0表示成功)
|
|
881
|
+
|
|
882
|
+
Examples
|
|
883
|
+
--------
|
|
884
|
+
>>> sas_to_csv('input.sas7bdat', 'output.csv')
|
|
885
|
+
Completed! It took 0.1234 minutes to run.
|
|
886
|
+
0
|
|
887
|
+
"""
|
|
888
|
+
import time as t
|
|
889
|
+
|
|
890
|
+
t0 = t.time()
|
|
891
|
+
df = read_sas_file(fileNameWithPath)
|
|
892
|
+
df.to_csv(outputFileNameWithPath, index = False)
|
|
893
|
+
t1 = t.time()
|
|
894
|
+
if timecounter:
|
|
895
|
+
logger.info(f"Completed! It took {round((t1 - t0)/60, 4)} minutes to run.")
|
|
896
|
+
return 0
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def merge_all_data(*args, on = "APPLICATION_ID", how = "left", return_kDF = True):
|
|
900
|
+
"""
|
|
901
|
+
合并多个数据集。
|
|
902
|
+
|
|
903
|
+
Parameters
|
|
904
|
+
----------
|
|
905
|
+
*args
|
|
906
|
+
要合并的DataFrame列表
|
|
907
|
+
on : str, default "APPLICATION_ID"
|
|
908
|
+
连接键列名
|
|
909
|
+
how : str, default "left"
|
|
910
|
+
连接方式,'left', 'right', 'inner', 'outer'
|
|
911
|
+
return_kDF : bool, default True
|
|
912
|
+
是否返回kDataFrame对象
|
|
913
|
+
|
|
914
|
+
Returns
|
|
915
|
+
-------
|
|
916
|
+
pandas.DataFrame or kDataFrame
|
|
917
|
+
合并后的数据表
|
|
918
|
+
|
|
919
|
+
Examples
|
|
920
|
+
--------
|
|
921
|
+
>>> df1 = pd.DataFrame({'id': [1, 2], 'a': [3, 4]})
|
|
922
|
+
>>> df2 = pd.DataFrame({'id': [1, 2], 'b': [5, 6]})
|
|
923
|
+
>>> merge_all_data(df1, df2, on='id')
|
|
924
|
+
"""
|
|
925
|
+
argList = list(args)
|
|
926
|
+
data = argList[0]
|
|
927
|
+
for i in range(len(argList)-1):
|
|
928
|
+
data = pd.merge(data, argList[i+1], on=on, how = how, suffixes=(f'_merge{i}', f'_merge{i+1}'))
|
|
929
|
+
if return_kDF:
|
|
930
|
+
return kDataFrame(data)
|
|
931
|
+
return data
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
def get_valid_vintages(sVintage, eVintage):
|
|
935
|
+
"""
|
|
936
|
+
获取指定范围内的有效Vintage列表。
|
|
937
|
+
|
|
938
|
+
Parameters
|
|
939
|
+
----------
|
|
940
|
+
sVintage : int
|
|
941
|
+
起始Vintage(格式YYYYMM)
|
|
942
|
+
eVintage : int
|
|
943
|
+
结束Vintage(格式YYYYMM)
|
|
944
|
+
|
|
945
|
+
Returns
|
|
946
|
+
-------
|
|
947
|
+
list
|
|
948
|
+
有效的Vintage列表
|
|
949
|
+
|
|
950
|
+
Examples
|
|
951
|
+
--------
|
|
952
|
+
>>> get_valid_vintages(202001, 202503)
|
|
953
|
+
[202001, 202002, ..., 202012, 202101, ..., 202503]
|
|
954
|
+
"""
|
|
955
|
+
vintages = []
|
|
956
|
+
for vintage in range(sVintage, eVintage + 1):
|
|
957
|
+
if (vintage % 100) > 0 and (vintage % 100) < 13:
|
|
958
|
+
vintages.append(vintage)
|
|
959
|
+
return vintages
|
|
960
|
+
|
|
961
|
+
|
|
962
|
+
def set_non_number_str(h2o_tbl_path):
|
|
963
|
+
"""
|
|
964
|
+
导入文件为H2OFrame并将所有非数值列设置为字符串类型。
|
|
965
|
+
|
|
966
|
+
Parameters
|
|
967
|
+
----------
|
|
968
|
+
h2o_tbl_path : str
|
|
969
|
+
H2O表路径
|
|
970
|
+
|
|
971
|
+
Returns
|
|
972
|
+
-------
|
|
973
|
+
h2o.H2OFrame
|
|
974
|
+
处理后的H2OFrame
|
|
975
|
+
|
|
976
|
+
Examples
|
|
977
|
+
--------
|
|
978
|
+
>>> hf = set_non_number_str('/path/to/file.csv')
|
|
979
|
+
"""
|
|
980
|
+
""" Import file as H2O Frame and set all non-numeric columns to string type. """
|
|
981
|
+
import h2o
|
|
982
|
+
h2o.init(min_mem_size='100G')
|
|
983
|
+
df = h2o.import_file(h2o_tbl_path)
|
|
984
|
+
orig_types = list(df.types.values())
|
|
985
|
+
fnl_types = ['string' if ((tp !='int') and (tp != 'enum') and (tp != 'string') and (tp != 'real')) else str(tp) for tp in orig_types]
|
|
986
|
+
fnl_df = h2o.import_file(h2o_tbl_path, col_types = fnl_types)
|
|
987
|
+
return fnl_df
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
def list_to_SQL(ls, excl=[], prefix = '', wquote=False):
|
|
991
|
+
"""
|
|
992
|
+
将列表转换为SQL格式的字符串。
|
|
993
|
+
|
|
994
|
+
Parameters
|
|
995
|
+
----------
|
|
996
|
+
ls : list
|
|
997
|
+
输入列表
|
|
998
|
+
excl : list, default []
|
|
999
|
+
要排除的元素列表
|
|
1000
|
+
prefix : str, default ''
|
|
1001
|
+
列名前缀(如表别名)
|
|
1002
|
+
wquote : bool, default False
|
|
1003
|
+
是否用引号包裹元素
|
|
1004
|
+
|
|
1005
|
+
Returns
|
|
1006
|
+
-------
|
|
1007
|
+
str
|
|
1008
|
+
SQL格式的字符串
|
|
1009
|
+
|
|
1010
|
+
Examples
|
|
1011
|
+
--------
|
|
1012
|
+
>>> list_to_SQL(['col1', 'col2', 'col3'], prefix='t')
|
|
1013
|
+
't.col1,t.col2,t.col3'
|
|
1014
|
+
"""
|
|
1015
|
+
sqlFmt = ""
|
|
1016
|
+
|
|
1017
|
+
for i, var in enumerate(ls):
|
|
1018
|
+
if wquote:
|
|
1019
|
+
var = f"'{var}'"
|
|
1020
|
+
if var in excl:
|
|
1021
|
+
continue
|
|
1022
|
+
if i != len(ls) - 1:
|
|
1023
|
+
if prefix == '' or prefix is None:
|
|
1024
|
+
sqlFmt += var + ","
|
|
1025
|
+
else:
|
|
1026
|
+
sqlFmt += prefix + "." + var + ","
|
|
1027
|
+
else:
|
|
1028
|
+
if prefix == '' or prefix is None:
|
|
1029
|
+
sqlFmt += var
|
|
1030
|
+
else:
|
|
1031
|
+
sqlFmt += prefix + "." + var
|
|
1032
|
+
return sqlFmt
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
def bool_to_str(data):
|
|
1036
|
+
"""
|
|
1037
|
+
将DataFrame中的布尔类型列转换为字符串类型。
|
|
1038
|
+
|
|
1039
|
+
Parameters
|
|
1040
|
+
----------
|
|
1041
|
+
data : pandas.DataFrame
|
|
1042
|
+
输入数据表
|
|
1043
|
+
|
|
1044
|
+
Returns
|
|
1045
|
+
-------
|
|
1046
|
+
pandas.DataFrame
|
|
1047
|
+
转换后的数据表
|
|
1048
|
+
|
|
1049
|
+
Examples
|
|
1050
|
+
--------
|
|
1051
|
+
>>> df = pd.DataFrame({'a': [True, False], 'b': [1, 2]})
|
|
1052
|
+
>>> bool_to_str(df)
|
|
1053
|
+
"""
|
|
1054
|
+
dfc = data.copy()
|
|
1055
|
+
type_dict = data.dtypes.to_dict()
|
|
1056
|
+
for k,v in type_dict.items():
|
|
1057
|
+
if str(v).lower() == 'bool':
|
|
1058
|
+
dfc = dfc.astype({k: str})
|
|
1059
|
+
return dfc
|
|
1060
|
+
|
|
1061
|
+
|
|
1062
|
+
def get_dtypes_file(data, outputFile = None, ck_format=False):
|
|
1063
|
+
"""
|
|
1064
|
+
获取DataFrame各列的数据类型。
|
|
1065
|
+
|
|
1066
|
+
Parameters
|
|
1067
|
+
----------
|
|
1068
|
+
data : pandas.DataFrame
|
|
1069
|
+
输入数据表
|
|
1070
|
+
outputFile : str, optional
|
|
1071
|
+
输出文件路径
|
|
1072
|
+
ck_format : bool, default False
|
|
1073
|
+
是否使用自定义格式
|
|
1074
|
+
|
|
1075
|
+
Returns
|
|
1076
|
+
-------
|
|
1077
|
+
pandas.DataFrame
|
|
1078
|
+
包含列名和数据类型的DataFrame
|
|
1079
|
+
|
|
1080
|
+
Examples
|
|
1081
|
+
--------
|
|
1082
|
+
>>> df = pd.DataFrame({'a': [1], 'b': ['x'], 'c': [1.5]})
|
|
1083
|
+
>>> get_dtypes_file(df)
|
|
1084
|
+
"""
|
|
1085
|
+
df = bool_to_str(data)
|
|
1086
|
+
res = pd.DataFrame(df.dtypes)
|
|
1087
|
+
res = res.reset_index()
|
|
1088
|
+
res.columns = ["colname", "dtype"]
|
|
1089
|
+
if ck_format:
|
|
1090
|
+
res['dtype'] = res['dtype'].astype(str).str.strip().map(ck_dtype)
|
|
1091
|
+
res['dtype'] = res['dtype'].astype(str).str.strip()
|
|
1092
|
+
if outputFile is not None:
|
|
1093
|
+
res.to_csv(outputFile, index = False, header=False)
|
|
1094
|
+
return res
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
def add_path_suffix(file, suffix = "_cut"):
|
|
1098
|
+
"""
|
|
1099
|
+
为文件路径添加后缀。
|
|
1100
|
+
|
|
1101
|
+
Parameters
|
|
1102
|
+
----------
|
|
1103
|
+
file : str
|
|
1104
|
+
文件路径
|
|
1105
|
+
suffix : str, default "_cut"
|
|
1106
|
+
要添加的后缀
|
|
1107
|
+
|
|
1108
|
+
Returns
|
|
1109
|
+
-------
|
|
1110
|
+
str
|
|
1111
|
+
添加后缀后的文件路径
|
|
1112
|
+
|
|
1113
|
+
Examples
|
|
1114
|
+
--------
|
|
1115
|
+
>>> add_path_suffix('/path/to/file.csv', '_processed')
|
|
1116
|
+
'/path/to/file_processed.csv'
|
|
1117
|
+
"""
|
|
1118
|
+
whole_path = file.split("/")
|
|
1119
|
+
path = [item for item in whole_path if item != whole_path[-1]]
|
|
1120
|
+
file = whole_path[-1].split(".")
|
|
1121
|
+
filename = file[0]
|
|
1122
|
+
ext = file[1]
|
|
1123
|
+
res = "/".join(path)+"/"+filename+suffix+"."+ext
|
|
1124
|
+
return res
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
def h2o_apply_regex(data, colname, func):
|
|
1128
|
+
"""
|
|
1129
|
+
对H2O Frame的列应用正则表达式转换函数。
|
|
1130
|
+
|
|
1131
|
+
Parameters
|
|
1132
|
+
----------
|
|
1133
|
+
data : h2o.H2OFrame
|
|
1134
|
+
输入数据
|
|
1135
|
+
colname : str
|
|
1136
|
+
列名
|
|
1137
|
+
func : callable
|
|
1138
|
+
应用函数
|
|
1139
|
+
|
|
1140
|
+
Returns
|
|
1141
|
+
-------
|
|
1142
|
+
h2o.H2OFrame
|
|
1143
|
+
转换后的H2OFrame
|
|
1144
|
+
|
|
1145
|
+
Examples
|
|
1146
|
+
--------
|
|
1147
|
+
>>> h2o_apply_regex(hf, 'name', lambda x: x.upper())
|
|
1148
|
+
"""
|
|
1149
|
+
""" Apply lambda function to h2o frame. """
|
|
1150
|
+
if isinstance(colname, str):
|
|
1151
|
+
fnl_res = h2o.H2OFrame(data[colname].as_data_frame()[colname].apply(func).tolist())
|
|
1152
|
+
fnl_res = fnl_res.rename({'C1':colname})
|
|
1153
|
+
return fnl_res
|
|
1154
|
+
|
|
1155
|
+
|
|
1156
|
+
def get_summary_rpt(means_rpt, iv_psi_rpt, corr_rpt):
|
|
1157
|
+
"""
|
|
1158
|
+
合并生成特征汇总报告。
|
|
1159
|
+
|
|
1160
|
+
将Means报告、IV/PSI报告和相关性报告合并为一个综合报告。
|
|
1161
|
+
|
|
1162
|
+
Parameters
|
|
1163
|
+
----------
|
|
1164
|
+
means_rpt : pandas.DataFrame
|
|
1165
|
+
Means统计报告
|
|
1166
|
+
iv_psi_rpt : pandas.DataFrame
|
|
1167
|
+
IV/PSI报告
|
|
1168
|
+
corr_rpt : pandas.DataFrame
|
|
1169
|
+
相关性报告
|
|
1170
|
+
|
|
1171
|
+
Returns
|
|
1172
|
+
-------
|
|
1173
|
+
pandas.DataFrame
|
|
1174
|
+
合并后的汇总报告
|
|
1175
|
+
|
|
1176
|
+
Examples
|
|
1177
|
+
--------
|
|
1178
|
+
>>> summary = get_summary_rpt(means, iv_psi, corr)
|
|
1179
|
+
"""
|
|
1180
|
+
iv_psi_rpt = iv_psi_rpt.set_index("Var_Name")
|
|
1181
|
+
corr_rpt = corr_rpt.set_index("Var_Name")
|
|
1182
|
+
|
|
1183
|
+
fnl_rpt = means_rpt\
|
|
1184
|
+
.merge(iv_psi_rpt, left_index = True, right_index = True)\
|
|
1185
|
+
.merge(corr_rpt, left_index = True, right_index = True)
|
|
1186
|
+
|
|
1187
|
+
fnl_rpt.columns = [x.upper() for x in fnl_rpt.columns]
|
|
1188
|
+
return fnl_rpt
|
|
1189
|
+
|
|
1190
|
+
|
|
1191
|
+
def flatten_json_attr(data, jsonColname= "data"):
|
|
1192
|
+
"""
|
|
1193
|
+
展开JSON格式的模型属性列。
|
|
1194
|
+
|
|
1195
|
+
Parameters
|
|
1196
|
+
----------
|
|
1197
|
+
data : pandas.DataFrame
|
|
1198
|
+
包含JSON列的数据表
|
|
1199
|
+
jsonColname : str, default "data"
|
|
1200
|
+
JSON列名
|
|
1201
|
+
|
|
1202
|
+
Returns
|
|
1203
|
+
-------
|
|
1204
|
+
pandas.DataFrame
|
|
1205
|
+
展开后的数据表
|
|
1206
|
+
|
|
1207
|
+
Examples
|
|
1208
|
+
--------
|
|
1209
|
+
>>> df = pd.DataFrame({'id': [1], 'data': ['{"key1":"val1"}']})
|
|
1210
|
+
>>> flatten_json_attr(df)
|
|
1211
|
+
"""
|
|
1212
|
+
""" Flatten Json-format model attributes. """
|
|
1213
|
+
import ast
|
|
1214
|
+
json_data = data[jsonColname].tolist()
|
|
1215
|
+
json_data = [ast.literal_eval(x) for x in json_data]
|
|
1216
|
+
info_list = [x for x in data.columns if x != 'data']
|
|
1217
|
+
drv = data[info_list]
|
|
1218
|
+
drv_w_attr = pd.concat([drv, pd.json_normalize(data=json_data)], axis = 1)
|
|
1219
|
+
logging.info(f"Flattened Data Shape: {drv_w_attr.shape}")
|
|
1220
|
+
return drv_w_attr
|
|
1221
|
+
|
|
1222
|
+
|
|
1223
|
+
def parse_odps_schema(schema_list):
|
|
1224
|
+
"""
|
|
1225
|
+
解析ODPS Schema。
|
|
1226
|
+
|
|
1227
|
+
Parameters
|
|
1228
|
+
----------
|
|
1229
|
+
schema_list : list
|
|
1230
|
+
ODPS Schema列表
|
|
1231
|
+
|
|
1232
|
+
Returns
|
|
1233
|
+
-------
|
|
1234
|
+
dict
|
|
1235
|
+
字段名到数据类型的字典
|
|
1236
|
+
|
|
1237
|
+
Examples
|
|
1238
|
+
--------
|
|
1239
|
+
>>> schema = parse_odps_schema(['column col1 type=string, column col2 type=bigint'])
|
|
1240
|
+
{'col1': 'string', 'col2': 'bigint'}
|
|
1241
|
+
"""
|
|
1242
|
+
import re
|
|
1243
|
+
|
|
1244
|
+
fnl_dict = {}
|
|
1245
|
+
for x in schema_list:
|
|
1246
|
+
res = re.sub(r'[<>]', '', str(x).replace(" type ", "").replace("column ", "")).replace(" ", "").split(',')
|
|
1247
|
+
fnl_dict[res[0]] = res[1]
|
|
1248
|
+
return fnl_dict
|
|
1249
|
+
|
|
1250
|
+
|
|
1251
|
+
def npnan2none(df):
|
|
1252
|
+
"""
|
|
1253
|
+
将DataFrame中的np.nan和np.nat转换为None值。
|
|
1254
|
+
|
|
1255
|
+
Parameters
|
|
1256
|
+
----------
|
|
1257
|
+
df : pandas.DataFrame
|
|
1258
|
+
输入数据表
|
|
1259
|
+
|
|
1260
|
+
Returns
|
|
1261
|
+
-------
|
|
1262
|
+
pandas.DataFrame
|
|
1263
|
+
转换后的数据表
|
|
1264
|
+
|
|
1265
|
+
Examples
|
|
1266
|
+
--------
|
|
1267
|
+
>>> df = pd.DataFrame({'a': [1, np.nan], 'b': [np.nan, 2]})
|
|
1268
|
+
>>> npnan2none(df)
|
|
1269
|
+
"""
|
|
1270
|
+
""" Convert np.nan, np.nat to None value. """
|
|
1271
|
+
|
|
1272
|
+
obj_colist = [k for k, v in df.dtypes.items() if v == 'O']
|
|
1273
|
+
for x in obj_colist:
|
|
1274
|
+
try:
|
|
1275
|
+
df[x] = df[x].astype(float).where(df[x].notnull(), None)
|
|
1276
|
+
except:
|
|
1277
|
+
df[x] = df[x].astype(object).where(df[x].notnull(), None)
|
|
1278
|
+
|
|
1279
|
+
df = df.replace({np.nan: None})
|
|
1280
|
+
return df
|
|
1281
|
+
|
|
1282
|
+
|
|
1283
|
+
def drop_tmp_cols(df, drop_list = ['py_inserttime']):
|
|
1284
|
+
"""
|
|
1285
|
+
删除DataFrame中的临时列。
|
|
1286
|
+
|
|
1287
|
+
Parameters
|
|
1288
|
+
----------
|
|
1289
|
+
df : pandas.DataFrame
|
|
1290
|
+
输入数据表
|
|
1291
|
+
drop_list : list, default ['py_inserttime']
|
|
1292
|
+
要删除的临时列列表
|
|
1293
|
+
|
|
1294
|
+
Returns
|
|
1295
|
+
-------
|
|
1296
|
+
pandas.DataFrame
|
|
1297
|
+
删除临时列后的数据表
|
|
1298
|
+
|
|
1299
|
+
Examples
|
|
1300
|
+
--------
|
|
1301
|
+
>>> df = pd.DataFrame({'a': [1, 2], 'py_inserttime': [0, 0]})
|
|
1302
|
+
>>> drop_tmp_cols(df)
|
|
1303
|
+
"""
|
|
1304
|
+
""" Drop Temporary Columns. """
|
|
1305
|
+
|
|
1306
|
+
col_exist_dict = {}
|
|
1307
|
+
for x in drop_list:
|
|
1308
|
+
if x in df.columns:
|
|
1309
|
+
col_exist_dict[x] = 1
|
|
1310
|
+
else:
|
|
1311
|
+
col_exist_dict[x] = 0
|
|
1312
|
+
|
|
1313
|
+
df = df.drop(columns = [k for k, v in col_exist_dict.items() if v == 1])
|
|
1314
|
+
|
|
1315
|
+
return df
|
|
1316
|
+
|
|
1317
|
+
|
|
1318
|
+
def mkdir_if_not_exist(folder_path, replace = False):
|
|
1319
|
+
"""
|
|
1320
|
+
如果目录不存在则创建目录。
|
|
1321
|
+
|
|
1322
|
+
Parameters
|
|
1323
|
+
----------
|
|
1324
|
+
folder_path : str
|
|
1325
|
+
文件夹路径
|
|
1326
|
+
replace : bool, default False
|
|
1327
|
+
如果目录已存在是否替换
|
|
1328
|
+
|
|
1329
|
+
Returns
|
|
1330
|
+
-------
|
|
1331
|
+
int
|
|
1332
|
+
状态码:0表示成功,1表示目录已存在
|
|
1333
|
+
|
|
1334
|
+
Examples
|
|
1335
|
+
--------
|
|
1336
|
+
>>> mkdir_if_not_exist('/path/to/new/folder')
|
|
1337
|
+
"""
|
|
1338
|
+
""" Make new directory if the given path does not exist. """
|
|
1339
|
+
|
|
1340
|
+
if folder_path is None:
|
|
1341
|
+
return None
|
|
1342
|
+
|
|
1343
|
+
if os.path.isdir(folder_path):
|
|
1344
|
+
|
|
1345
|
+
if replace:
|
|
1346
|
+
logging.info(f"Folder {folder_path} has been replaced!")
|
|
1347
|
+
os.makedirs(folder_path, exist_ok=replace)
|
|
1348
|
+
return 0
|
|
1349
|
+
|
|
1350
|
+
# logging.info(f"Folder {folder_path} has already existed!")
|
|
1351
|
+
return 1
|
|
1352
|
+
|
|
1353
|
+
else:
|
|
1354
|
+
os.makedirs(folder_path, exist_ok=False)
|
|
1355
|
+
logging.info(f"Folder {folder_path} created!")
|
|
1356
|
+
|
|
1357
|
+
return 0
|
|
1358
|
+
|
|
1359
|
+
|
|
1360
|
+
def _remove_comments(sql):
|
|
1361
|
+
"""
|
|
1362
|
+
移除SQL查询中的所有注释。
|
|
1363
|
+
|
|
1364
|
+
Parameters
|
|
1365
|
+
----------
|
|
1366
|
+
sql : str
|
|
1367
|
+
SQL查询字符串
|
|
1368
|
+
|
|
1369
|
+
Returns
|
|
1370
|
+
-------
|
|
1371
|
+
str
|
|
1372
|
+
移除注释后的SQL字符串
|
|
1373
|
+
"""
|
|
1374
|
+
""" Remove all comments from the SQL query. """
|
|
1375
|
+
import re
|
|
1376
|
+
# =========================================================================
|
|
1377
|
+
# 正则分组 (按优先级排列,左起优先匹配):
|
|
1378
|
+
# group 1: 单引号字符串 '...' (支持 SQL 标准 '' 转义) — 保留
|
|
1379
|
+
# group 2: 双引号字符串/标识符 "..." — 保留
|
|
1380
|
+
# group 3: /*+ ... */ optimizer hint — 保留
|
|
1381
|
+
# group 4: /* ... */ 普通多行注释 — 删除
|
|
1382
|
+
# group 5: -- ... 单行注释 — 删除
|
|
1383
|
+
# =========================================================================
|
|
1384
|
+
# 修复记录 (2026-06-11):
|
|
1385
|
+
# 1. re.sub + 回调 替代 re.findall + str.replace,
|
|
1386
|
+
# 避免全局替换破坏字符串字面量内相同文本
|
|
1387
|
+
# 2. 新增双引号保护分组, 防止 "..." 内的注释标记被误删
|
|
1388
|
+
# 3. 改进单引号正则: '([^']|'')*' — 支持 SQL 标准转义
|
|
1389
|
+
# =========================================================================
|
|
1390
|
+
pattern = r"""(?ms)('[^']*(?:''[^']*)*')|("[^"]*")|(\/\*\+.*?\*\/)|(\/\*.*?\*\/)|(\-\-.*?)$"""
|
|
1391
|
+
|
|
1392
|
+
def _replacer(m):
|
|
1393
|
+
# Group 1 (单引号), Group 2 (双引号), Group 3 (hint): 保留原文
|
|
1394
|
+
if m.group(1) or m.group(2) or m.group(3):
|
|
1395
|
+
return m.group(0)
|
|
1396
|
+
# Group 4 (/* */) 和 Group 5 (--): 删除
|
|
1397
|
+
return ''
|
|
1398
|
+
|
|
1399
|
+
sql = re.sub(pattern, _replacer, sql)
|
|
1400
|
+
|
|
1401
|
+
# Clean up lines left empty by removed comments, but preserve SQL formatting
|
|
1402
|
+
sql = re.sub(r'[ \t]+\n', '\n', sql) # remove trailing whitespace on lines
|
|
1403
|
+
sql = re.sub(r'\n{3,}', '\n\n', sql) # collapse 3+ blank lines to at most 2
|
|
1404
|
+
sql = re.sub(r'\n\s*\n', '\n', sql) # collapse consecutive blank lines into one
|
|
1405
|
+
return sql.strip()
|
|
1406
|
+
|
|
1407
|
+
|
|
1408
|
+
def _split_select_fields(select_clause):
|
|
1409
|
+
"""
|
|
1410
|
+
拆分SELECT字段列表,按逗号分割但尊重括号嵌套深度。
|
|
1411
|
+
|
|
1412
|
+
该函数确保 EXCEPT(...)、COALESCE(...)、子查询等括号内的逗号不会被误当作字段分隔符。
|
|
1413
|
+
|
|
1414
|
+
Parameters
|
|
1415
|
+
----------
|
|
1416
|
+
select_clause : str
|
|
1417
|
+
SELECT 和下一个关键字(FROM/WHERE等)之间的字段列表字符串
|
|
1418
|
+
|
|
1419
|
+
Returns
|
|
1420
|
+
-------
|
|
1421
|
+
list
|
|
1422
|
+
字段名称列表,每个字段去除前后空白
|
|
1423
|
+
|
|
1424
|
+
Examples
|
|
1425
|
+
--------
|
|
1426
|
+
>>> _split_select_fields("a, b, COALESCE(x, y), c.* EXCEPT (d, e)")
|
|
1427
|
+
['a', 'b', 'COALESCE(x, y)', 'c.* EXCEPT (d, e)']
|
|
1428
|
+
"""
|
|
1429
|
+
fields = []
|
|
1430
|
+
current = ''
|
|
1431
|
+
depth = 0
|
|
1432
|
+
for char in select_clause:
|
|
1433
|
+
if char == '(':
|
|
1434
|
+
depth += 1
|
|
1435
|
+
current += char
|
|
1436
|
+
elif char == ')':
|
|
1437
|
+
depth -= 1
|
|
1438
|
+
current += char
|
|
1439
|
+
elif char == ',' and depth == 0:
|
|
1440
|
+
stripped = current.strip()
|
|
1441
|
+
if stripped:
|
|
1442
|
+
fields.append(stripped)
|
|
1443
|
+
current = ''
|
|
1444
|
+
else:
|
|
1445
|
+
current += char
|
|
1446
|
+
stripped = current.strip()
|
|
1447
|
+
if stripped:
|
|
1448
|
+
fields.append(stripped)
|
|
1449
|
+
return fields
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
def _format_sql_select(sql):
|
|
1453
|
+
"""
|
|
1454
|
+
格式化SQL的SELECT子句:SELECT独占一行,每个字段独占一行,使用前置逗号风格。
|
|
1455
|
+
|
|
1456
|
+
递归格式化所有嵌套子查询中的SELECT,包括 FROM (SELECT ...)、JOIN (SELECT ...)、
|
|
1457
|
+
WITH ... AS (SELECT ...) 等场景。
|
|
1458
|
+
|
|
1459
|
+
Parameters
|
|
1460
|
+
----------
|
|
1461
|
+
sql : str
|
|
1462
|
+
SQL查询字符串
|
|
1463
|
+
|
|
1464
|
+
Returns
|
|
1465
|
+
-------
|
|
1466
|
+
str
|
|
1467
|
+
格式化后的SQL字符串;如果找不到SELECT关键字则返回原字符串
|
|
1468
|
+
|
|
1469
|
+
Examples
|
|
1470
|
+
--------
|
|
1471
|
+
>>> _format_sql_select("SELECT a, b, c FROM t")
|
|
1472
|
+
'SELECT\\n a\\n , b\\n , c\\nFROM t'
|
|
1473
|
+
"""
|
|
1474
|
+
import re
|
|
1475
|
+
|
|
1476
|
+
# =========================================================================
|
|
1477
|
+
# Phase 1: 递归处理括号内的子查询(内向外格式化)
|
|
1478
|
+
# =========================================================================
|
|
1479
|
+
result_parts = []
|
|
1480
|
+
i = 0
|
|
1481
|
+
while i < len(sql):
|
|
1482
|
+
if sql[i] == '(':
|
|
1483
|
+
# 找到匹配的右括号
|
|
1484
|
+
depth = 1
|
|
1485
|
+
j = i + 1
|
|
1486
|
+
while j < len(sql) and depth > 0:
|
|
1487
|
+
if sql[j] == '(':
|
|
1488
|
+
depth += 1
|
|
1489
|
+
elif sql[j] == ')':
|
|
1490
|
+
depth -= 1
|
|
1491
|
+
j += 1
|
|
1492
|
+
# 递归格式化括号内的内容
|
|
1493
|
+
inner_content = sql[i + 1:j - 1]
|
|
1494
|
+
formatted_inner = _format_sql_select(inner_content)
|
|
1495
|
+
result_parts.append('(' + formatted_inner + ')')
|
|
1496
|
+
i = j
|
|
1497
|
+
else:
|
|
1498
|
+
result_parts.append(sql[i])
|
|
1499
|
+
i += 1
|
|
1500
|
+
|
|
1501
|
+
sql = ''.join(result_parts)
|
|
1502
|
+
|
|
1503
|
+
# =========================================================================
|
|
1504
|
+
# Phase 2: 格式化当前层级(depth=0)的 SELECT 子句
|
|
1505
|
+
# =========================================================================
|
|
1506
|
+
# 查找最外层SELECT关键字(不在括号内的SELECT)
|
|
1507
|
+
select_match = None
|
|
1508
|
+
depth = 0
|
|
1509
|
+
for m in re.finditer(r'\bSELECT\b', sql, re.IGNORECASE):
|
|
1510
|
+
prefix = sql[:m.start()]
|
|
1511
|
+
depth = prefix.count('(') - prefix.count(')')
|
|
1512
|
+
if depth == 0:
|
|
1513
|
+
select_match = m
|
|
1514
|
+
break
|
|
1515
|
+
|
|
1516
|
+
if not select_match:
|
|
1517
|
+
return sql
|
|
1518
|
+
|
|
1519
|
+
# 保留 SELECT 关键字之前的文本(如 WITH 子句)
|
|
1520
|
+
pre_select = sql[:select_match.start()]
|
|
1521
|
+
select_start = select_match.end()
|
|
1522
|
+
remainder = sql[select_start:]
|
|
1523
|
+
|
|
1524
|
+
# 查找SELECT子句结束位置(下一个SQL关键字,不在括号内)
|
|
1525
|
+
end_keywords = [
|
|
1526
|
+
r'\bFROM\b', r'\bWHERE\b', r'\bGROUP\s+BY\b', r'\bORDER\s+BY\b',
|
|
1527
|
+
r'\bHAVING\b', r'\bLIMIT\b', r'\bOFFSET\b',
|
|
1528
|
+
r'\bINNER\s+JOIN\b', r'\bLEFT\s+(?:OUTER\s+)?JOIN\b',
|
|
1529
|
+
r'\bRIGHT\s+(?:OUTER\s+)?JOIN\b', r'\bFULL\s+(?:OUTER\s+)?JOIN\b',
|
|
1530
|
+
r'\bCROSS\s+JOIN\b', r'\bJOIN\b',
|
|
1531
|
+
r'\bUNION\s+ALL\b', r'\bUNION\b', r'\bINTERSECT\b', r'\bMINUS\b',
|
|
1532
|
+
r'\bWINDOW\b', r'\bQUALIFY\b', r'\bDISTRIBUTE\s+BY\b',
|
|
1533
|
+
r'\bSORT\s+BY\b', r'\bCLUSTER\s+BY\b',
|
|
1534
|
+
r';', r'\)\s*$'
|
|
1535
|
+
]
|
|
1536
|
+
|
|
1537
|
+
# 组合成正则,在括号深度为0的位置匹配
|
|
1538
|
+
pattern = '|'.join(f'(?:{kw})' for kw in end_keywords)
|
|
1539
|
+
|
|
1540
|
+
# 使用字符遍历方式定位第一个不在括号内的结束关键字
|
|
1541
|
+
best_pos = len(remainder)
|
|
1542
|
+
for m in re.finditer(pattern, remainder, re.IGNORECASE):
|
|
1543
|
+
prefix = remainder[:m.start()]
|
|
1544
|
+
p_depth = prefix.count('(') - prefix.count(')')
|
|
1545
|
+
if p_depth == 0:
|
|
1546
|
+
best_pos = m.start()
|
|
1547
|
+
break
|
|
1548
|
+
|
|
1549
|
+
# 非SELECT语句(如只包含FROM子查询的语句)
|
|
1550
|
+
if best_pos == 0:
|
|
1551
|
+
return sql
|
|
1552
|
+
|
|
1553
|
+
select_clause = remainder[:best_pos]
|
|
1554
|
+
rest_of_sql = remainder[best_pos:]
|
|
1555
|
+
|
|
1556
|
+
# 如果SELECT子句为空(例如边缘情况),返回原SQL
|
|
1557
|
+
if not select_clause.strip():
|
|
1558
|
+
return sql
|
|
1559
|
+
|
|
1560
|
+
# 拆分字段
|
|
1561
|
+
fields = _split_select_fields(select_clause)
|
|
1562
|
+
|
|
1563
|
+
if not fields:
|
|
1564
|
+
return sql
|
|
1565
|
+
|
|
1566
|
+
# 标准化为前置逗号风格并组装
|
|
1567
|
+
formatted_fields = []
|
|
1568
|
+
for i, field in enumerate(fields):
|
|
1569
|
+
if i == 0:
|
|
1570
|
+
formatted_fields.append(f" {field}")
|
|
1571
|
+
else:
|
|
1572
|
+
# 移除字段已有的前置逗号,统一添加
|
|
1573
|
+
field_stripped = field.strip()
|
|
1574
|
+
if field_stripped.startswith(','):
|
|
1575
|
+
field_stripped = field_stripped[1:].strip()
|
|
1576
|
+
formatted_fields.append(f" , {field_stripped}")
|
|
1577
|
+
|
|
1578
|
+
formatted_select = pre_select + "SELECT\n" + "\n".join(formatted_fields) + "\n" + rest_of_sql
|
|
1579
|
+
return formatted_select
|
|
1580
|
+
|
|
1581
|
+
|
|
1582
|
+
def _split_sql_queries(query, split_mark = "$single_query_end$"):
|
|
1583
|
+
"""
|
|
1584
|
+
分割SQL查询字符串。
|
|
1585
|
+
|
|
1586
|
+
Parameters
|
|
1587
|
+
----------
|
|
1588
|
+
query : str
|
|
1589
|
+
SQL查询字符串
|
|
1590
|
+
split_mark : str, default "$single_query_end$"
|
|
1591
|
+
分割标记
|
|
1592
|
+
|
|
1593
|
+
Returns
|
|
1594
|
+
-------
|
|
1595
|
+
list
|
|
1596
|
+
分割后的SQL查询列表
|
|
1597
|
+
"""
|
|
1598
|
+
import re
|
|
1599
|
+
query = _remove_comments(query)
|
|
1600
|
+
# 保护字符串和标识符内的分号不被当作查询分隔符
|
|
1601
|
+
# group 1: 单引号字符串 (支持 SQL 标准 '' 转义)
|
|
1602
|
+
# group 2: 双引号字符串/标识符
|
|
1603
|
+
# group 3: 分号 (真正的查询分隔符)
|
|
1604
|
+
skip_semi_in_quote = r"""(?s)('[^']*(?:''[^']*)*')|("[^"]*")|(;)"""
|
|
1605
|
+
def _protect_semicolon(m):
|
|
1606
|
+
if m.group(1) or m.group(2):
|
|
1607
|
+
return m.group(0) # 保留字符串原文
|
|
1608
|
+
return split_mark # 将分号替换为分割标记
|
|
1609
|
+
query = re.sub(skip_semi_in_quote, _protect_semicolon, query)
|
|
1610
|
+
return query.split(split_mark)
|
|
1611
|
+
|
|
1612
|
+
|
|
1613
|
+
def parse_sql_file(sql_path:str=None,
|
|
1614
|
+
sql_query:str=None,
|
|
1615
|
+
split:bool=False,
|
|
1616
|
+
format_select:bool=False,
|
|
1617
|
+
**kwargs):
|
|
1618
|
+
"""
|
|
1619
|
+
解析SQL文件并替换变量。
|
|
1620
|
+
|
|
1621
|
+
Parameters
|
|
1622
|
+
----------
|
|
1623
|
+
sql_path : str, optional
|
|
1624
|
+
SQL文件路径
|
|
1625
|
+
sql_query : str, optional
|
|
1626
|
+
SQL查询字符串
|
|
1627
|
+
split : bool, default False
|
|
1628
|
+
是否分割多个查询
|
|
1629
|
+
format_select : bool, default False
|
|
1630
|
+
是否自动格式化SELECT字段(每个字段一行,前置逗号风格)
|
|
1631
|
+
**kwargs
|
|
1632
|
+
SQL中要替换的变量
|
|
1633
|
+
|
|
1634
|
+
Returns
|
|
1635
|
+
-------
|
|
1636
|
+
str or list
|
|
1637
|
+
解析后的SQL字符串或字符串列表
|
|
1638
|
+
|
|
1639
|
+
Examples
|
|
1640
|
+
--------
|
|
1641
|
+
>>> parse_sql_file(sql_path='query.sql', table_name='my_table', date='2025-01-01')
|
|
1642
|
+
>>> parse_sql_file(sql_path='query.sql', format_select=True, table_name='my_table')
|
|
1643
|
+
"""
|
|
1644
|
+
import re
|
|
1645
|
+
import warnings
|
|
1646
|
+
|
|
1647
|
+
if (sql_path is None) and (sql_query is None):
|
|
1648
|
+
raise AttributeError("please give either sql_path or sql_query.")
|
|
1649
|
+
|
|
1650
|
+
if (sql_path is not None) and (sql_query is not None):
|
|
1651
|
+
raise AttributeError("sql_path and sql_query can not be BOTH given.")
|
|
1652
|
+
|
|
1653
|
+
sql = sql_query
|
|
1654
|
+
if sql_path is not None:
|
|
1655
|
+
# Read sql file.
|
|
1656
|
+
with open(sql_path, 'r') as file:
|
|
1657
|
+
sql = file.read().strip()
|
|
1658
|
+
|
|
1659
|
+
# Remove all comments in sql file.
|
|
1660
|
+
sql = _remove_comments(sql)
|
|
1661
|
+
|
|
1662
|
+
# Find and parse all argments in sql file.
|
|
1663
|
+
all_args = list(set(re.findall(r"{(.*?)}", sql)))
|
|
1664
|
+
#### Parse Arguments
|
|
1665
|
+
for k, v in kwargs.items():
|
|
1666
|
+
if k in all_args:
|
|
1667
|
+
sql = sql.replace("{%s}" % k, v)
|
|
1668
|
+
args_left = list(set(re.findall(r"{(.*?)}", sql)))
|
|
1669
|
+
|
|
1670
|
+
# Identify if mutliple queries in one .sql file
|
|
1671
|
+
## if Yes: identify if split sql file.
|
|
1672
|
+
query_list = _split_sql_queries(sql)
|
|
1673
|
+
query_list = [query.strip() for query in query_list if query != '']
|
|
1674
|
+
|
|
1675
|
+
# Optionally format SELECT clause for readability
|
|
1676
|
+
if format_select:
|
|
1677
|
+
query_list = [_format_sql_select(q) for q in query_list]
|
|
1678
|
+
|
|
1679
|
+
if len(args_left) != 0:
|
|
1680
|
+
# Raise a warning if not all arguments are given through the function.
|
|
1681
|
+
warnings.warn(f"Missing argument(s) {', '.join(args_left)} in the given SQL file.")
|
|
1682
|
+
|
|
1683
|
+
# Ensure all the arguments have been claimed.
|
|
1684
|
+
if split:
|
|
1685
|
+
return query_list if len(query_list) > 1 else query_list[0]
|
|
1686
|
+
else:
|
|
1687
|
+
return '; '.join(query_list)+";"
|
|
1688
|
+
|
|
1689
|
+
|
|
1690
|
+
|
|
1691
|
+
def calc_woe(data, bad_pct, good_pct, fillwoe=True):
|
|
1692
|
+
"""
|
|
1693
|
+
计算WOE(Weight of Evidence)值。
|
|
1694
|
+
|
|
1695
|
+
WOE = ln(组正样本占比 / 组负样本占比)
|
|
1696
|
+
|
|
1697
|
+
Parameters
|
|
1698
|
+
----------
|
|
1699
|
+
data : pandas.DataFrame
|
|
1700
|
+
包含比例的数据表
|
|
1701
|
+
bad_pct : str
|
|
1702
|
+
坏样本占比列名
|
|
1703
|
+
good_pct : str
|
|
1704
|
+
好样本占比列名
|
|
1705
|
+
fillwoe : bool, default True
|
|
1706
|
+
当比例为0时是否将woe置为0
|
|
1707
|
+
|
|
1708
|
+
Returns
|
|
1709
|
+
-------
|
|
1710
|
+
float or pandas.Series
|
|
1711
|
+
WOE值
|
|
1712
|
+
|
|
1713
|
+
Examples
|
|
1714
|
+
--------
|
|
1715
|
+
>>> df = pd.DataFrame({'bad_pct': [0.3, 0.5], 'good_pct': [0.7, 0.5]})
|
|
1716
|
+
>>> calc_woe(df, 'bad_pct', 'good_pct')
|
|
1717
|
+
"""
|
|
1718
|
+
|
|
1719
|
+
if len(data[bad_pct]) > 0 and len(data[good_pct]) > 0:
|
|
1720
|
+
woe = np.log(data[bad_pct] / data[good_pct])
|
|
1721
|
+
else:
|
|
1722
|
+
if fillwoe:
|
|
1723
|
+
woe = 0
|
|
1724
|
+
else:
|
|
1725
|
+
woe = np.nan
|
|
1726
|
+
|
|
1727
|
+
return woe
|
|
1728
|
+
|
|
1729
|
+
|
|
1730
|
+
def calc_iv(data, bad_pct, good_pct, filliv=True):
|
|
1731
|
+
"""
|
|
1732
|
+
计算IV(Information Value)值。
|
|
1733
|
+
|
|
1734
|
+
IV = (组正样本占比 - 组负样本占比) * WOE
|
|
1735
|
+
|
|
1736
|
+
Parameters
|
|
1737
|
+
----------
|
|
1738
|
+
data : pandas.DataFrame
|
|
1739
|
+
包含比例的数据表
|
|
1740
|
+
bad_pct : str
|
|
1741
|
+
坏样本占比列名
|
|
1742
|
+
good_pct : str
|
|
1743
|
+
好样本占比列名
|
|
1744
|
+
filliv : bool, default True
|
|
1745
|
+
当比例为0时是否将iv置为0
|
|
1746
|
+
|
|
1747
|
+
Returns
|
|
1748
|
+
-------
|
|
1749
|
+
float or pandas.Series
|
|
1750
|
+
IV值
|
|
1751
|
+
|
|
1752
|
+
Examples
|
|
1753
|
+
--------
|
|
1754
|
+
>>> df = pd.DataFrame({'bad_pct': [0.3, 0.5], 'good_pct': [0.7, 0.5]})
|
|
1755
|
+
>>> calc_iv(df, 'bad_pct', 'good_pct')
|
|
1756
|
+
"""
|
|
1757
|
+
|
|
1758
|
+
if len(data[bad_pct]) > 0 and len(data[good_pct]) > 0:
|
|
1759
|
+
iv = (data[bad_pct] - data[good_pct]) * np.log(data[bad_pct] / data[good_pct])
|
|
1760
|
+
else:
|
|
1761
|
+
if filliv:
|
|
1762
|
+
iv = 0
|
|
1763
|
+
else:
|
|
1764
|
+
iv = np.nan
|
|
1765
|
+
|
|
1766
|
+
return iv
|
|
1767
|
+
|
|
1768
|
+
|
|
1769
|
+
def save_model(model, filename):
|
|
1770
|
+
"""
|
|
1771
|
+
使用pickle保存lightGBM模型。
|
|
1772
|
+
|
|
1773
|
+
Parameters
|
|
1774
|
+
----------
|
|
1775
|
+
model : object
|
|
1776
|
+
要保存的模型对象
|
|
1777
|
+
filename : str
|
|
1778
|
+
保存路径
|
|
1779
|
+
|
|
1780
|
+
Returns
|
|
1781
|
+
-------
|
|
1782
|
+
int
|
|
1783
|
+
执行状态码(0表示成功)
|
|
1784
|
+
|
|
1785
|
+
Examples
|
|
1786
|
+
--------
|
|
1787
|
+
>>> save_model(model, 'model.pkl')
|
|
1788
|
+
"""
|
|
1789
|
+
""" Save lightGBM model using pickle. """
|
|
1790
|
+
import joblib
|
|
1791
|
+
|
|
1792
|
+
joblib.dump(model, filename)
|
|
1793
|
+
return 0
|
|
1794
|
+
|
|
1795
|
+
|
|
1796
|
+
def load_model(model_path):
|
|
1797
|
+
"""
|
|
1798
|
+
加载pickle模型。
|
|
1799
|
+
|
|
1800
|
+
Parameters
|
|
1801
|
+
----------
|
|
1802
|
+
model_path : str
|
|
1803
|
+
模型文件路径
|
|
1804
|
+
|
|
1805
|
+
Returns
|
|
1806
|
+
-------
|
|
1807
|
+
object
|
|
1808
|
+
加载的模型对象
|
|
1809
|
+
|
|
1810
|
+
Examples
|
|
1811
|
+
--------
|
|
1812
|
+
>>> model = load_model('model.pkl')
|
|
1813
|
+
"""
|
|
1814
|
+
""" Load Pickle Model. """
|
|
1815
|
+
import joblib
|
|
1816
|
+
|
|
1817
|
+
model = joblib.load(model_path)
|
|
1818
|
+
return model
|
|
1819
|
+
|
|
1820
|
+
|
|
1821
|
+
def scoring(data, model, varlist, scr_name, keeplist = None, all_missing_spec_value = None):
|
|
1822
|
+
"""
|
|
1823
|
+
使用模型对数据进行评分。
|
|
1824
|
+
|
|
1825
|
+
Parameters
|
|
1826
|
+
----------
|
|
1827
|
+
data : pandas.DataFrame
|
|
1828
|
+
输入数据表
|
|
1829
|
+
model : sklearn-like model
|
|
1830
|
+
机器学习模型
|
|
1831
|
+
varlist : list
|
|
1832
|
+
特征变量列表
|
|
1833
|
+
scr_name : str
|
|
1834
|
+
分数列名
|
|
1835
|
+
keeplist : list, optional
|
|
1836
|
+
要保留的列列表
|
|
1837
|
+
all_missing_spec_value : float, optional
|
|
1838
|
+
全缺失样本的指定分数值
|
|
1839
|
+
|
|
1840
|
+
Returns
|
|
1841
|
+
-------
|
|
1842
|
+
pandas.DataFrame
|
|
1843
|
+
包含分数的数据表
|
|
1844
|
+
|
|
1845
|
+
Examples
|
|
1846
|
+
--------
|
|
1847
|
+
>>> df = scoring(data, model, ['feat1', 'feat2'], 'score')
|
|
1848
|
+
"""
|
|
1849
|
+
""" Model Soring. """
|
|
1850
|
+
|
|
1851
|
+
fnl_data = data.copy()
|
|
1852
|
+
fnl_data[scr_name] = model.predict_proba(fnl_data.loc[:, varlist])[:, 1]
|
|
1853
|
+
|
|
1854
|
+
nohit_condition = (pd.isnull(fnl_data[varlist]).sum(axis = 1) == len(varlist))
|
|
1855
|
+
if fnl_data[nohit_condition].shape[0] > 0:
|
|
1856
|
+
|
|
1857
|
+
all_missing_data = fnl_data[nohit_condition]
|
|
1858
|
+
other_data = fnl_data[~nohit_condition]
|
|
1859
|
+
|
|
1860
|
+
|
|
1861
|
+
all_missing_data[scr_name] = model.predict_proba(fnl_data[nohit_condition].loc[:, varlist])[:, 1]
|
|
1862
|
+
logger.info("Score for All-Missing Cases: ", all_missing_data[scr_name].unique())
|
|
1863
|
+
|
|
1864
|
+
if all_missing_spec_value:
|
|
1865
|
+
all_missing_data[scr_name] = all_missing_spec_value
|
|
1866
|
+
logger.info(f"Score for All-Missing Cases Has Been Reset to {all_missing_spec_value}")
|
|
1867
|
+
|
|
1868
|
+
fnl_data = pd.concat([other_data, all_missing_data])
|
|
1869
|
+
|
|
1870
|
+
assert fnl_data.shape[0] == data.shape[0]
|
|
1871
|
+
|
|
1872
|
+
if keeplist is None:
|
|
1873
|
+
keeplist = fnl_data.columns.tolist()
|
|
1874
|
+
else:
|
|
1875
|
+
keeplist = keeplist + [scr_name]
|
|
1876
|
+
|
|
1877
|
+
return fnl_data[keeplist]
|
|
1878
|
+
|
|
1879
|
+
def get_missing_indicator(data, subset = None):
|
|
1880
|
+
""" Add Missing Indicator. """
|
|
1881
|
+
|
|
1882
|
+
all_missing_logic = lambda data: (pd.isnull(data[subset]).sum(axis = 1) == len(subset))
|
|
1883
|
+
return all_missing_logic(data).astype(int)
|
|
1884
|
+
|
|
1885
|
+
def upload_score(data, model, varlist, scr_name, table_name, keeplist = None, retPandas = False, all_missing_spec_value = None):
|
|
1886
|
+
"""
|
|
1887
|
+
将模型分数上传到Maxcompute。
|
|
1888
|
+
|
|
1889
|
+
Parameters
|
|
1890
|
+
----------
|
|
1891
|
+
data : pandas.DataFrame
|
|
1892
|
+
输入数据表
|
|
1893
|
+
model : sklearn-like model
|
|
1894
|
+
机器学习模型
|
|
1895
|
+
varlist : list
|
|
1896
|
+
特征变量列表
|
|
1897
|
+
scr_name : str
|
|
1898
|
+
分数列名
|
|
1899
|
+
table_name : str
|
|
1900
|
+
目标表名
|
|
1901
|
+
keeplist : list, optional
|
|
1902
|
+
要保留的列列表
|
|
1903
|
+
retPandas : bool, default False
|
|
1904
|
+
是否返回pandas DataFrame
|
|
1905
|
+
all_missing_spec_value : float, optional
|
|
1906
|
+
全缺失样本的指定分数值
|
|
1907
|
+
|
|
1908
|
+
Returns
|
|
1909
|
+
-------
|
|
1910
|
+
int or pandas.DataFrame
|
|
1911
|
+
状态码或数据表
|
|
1912
|
+
|
|
1913
|
+
Examples
|
|
1914
|
+
--------
|
|
1915
|
+
>>> upload_score(data, model, ['feat1', 'feat2'], 'score', 'output_table')
|
|
1916
|
+
"""
|
|
1917
|
+
""" Upload Score to Maxcompute. """
|
|
1918
|
+
|
|
1919
|
+
data = scoring(data = data, model = model, varlist = varlist, scr_name = scr_name, keeplist = keeplist, all_missing_spec_value = all_missing_spec_value)
|
|
1920
|
+
|
|
1921
|
+
|
|
1922
|
+
sqlrunner = ODPSRunner()
|
|
1923
|
+
|
|
1924
|
+
fnl_scr_upload = data.copy()
|
|
1925
|
+
fnl_scr_upload = uf.npnan2none(fnl_scr_upload)
|
|
1926
|
+
fnl_scr_upload = uf.drop_tmp_cols(fnl_scr_upload)
|
|
1927
|
+
|
|
1928
|
+
if keeplist is None:
|
|
1929
|
+
keeplist = fnl_scr_upload.columns.tolist()
|
|
1930
|
+
else:
|
|
1931
|
+
keeplist = keeplist + [scr_name]
|
|
1932
|
+
|
|
1933
|
+
sqlrunner.upload_df(fnl_scr_upload[keeplist], table_name)
|
|
1934
|
+
|
|
1935
|
+
if retPandas:
|
|
1936
|
+
return fnl_scr_upload[keeplist]
|
|
1937
|
+
|
|
1938
|
+
return 0
|
|
1939
|
+
|
|
1940
|
+
|
|
1941
|
+
def pull_attributes_in_batch(table_name, varlist, batch_num = 6, unikey = 'flow_id', main_info_select = ['*'], add_query = ''):
|
|
1942
|
+
""" Pull Data from DataWorks in Vertical Batch. """
|
|
1943
|
+
|
|
1944
|
+
from .ODPS_Tool import ODPSRunner
|
|
1945
|
+
import multiprocessing
|
|
1946
|
+
|
|
1947
|
+
n_process = multiprocessing.cpu_count() - 1
|
|
1948
|
+
logger.info(n_process)
|
|
1949
|
+
sqlrunner = ODPSRunner()
|
|
1950
|
+
|
|
1951
|
+
n = 6
|
|
1952
|
+
|
|
1953
|
+
batch_varlist = cut2pieces(varlist, n)
|
|
1954
|
+
|
|
1955
|
+
assert (len(varlist) == len([x for varlist in batch_varlist for x in varlist]))
|
|
1956
|
+
|
|
1957
|
+
res = {}
|
|
1958
|
+
i = 0
|
|
1959
|
+
while i < len(batch_varlist):
|
|
1960
|
+
|
|
1961
|
+
logger.info(i)
|
|
1962
|
+
sql_query = f"""
|
|
1963
|
+
SELECT {unikey}, {", ".join(batch_varlist[i])}
|
|
1964
|
+
FROM {table_name}
|
|
1965
|
+
{add_query};
|
|
1966
|
+
"""
|
|
1967
|
+
|
|
1968
|
+
res[f'batch{i}'] = sqlrunner.run_sql(sql_query, n_process = n_process)
|
|
1969
|
+
|
|
1970
|
+
i += 1
|
|
1971
|
+
|
|
1972
|
+
sql_query = f"""
|
|
1973
|
+
SELECT {','.join(main_info_select)} EXCEPT ({", ".join(varlist)})
|
|
1974
|
+
FROM {table_name}
|
|
1975
|
+
{add_query};
|
|
1976
|
+
"""
|
|
1977
|
+
|
|
1978
|
+
main_info = sqlrunner.run_sql(sql_query, n_process = n_process)
|
|
1979
|
+
master_df = main_info.copy()
|
|
1980
|
+
|
|
1981
|
+
for k, data in res.items():
|
|
1982
|
+
master_df = master_df.merge(data, on = [unikey])
|
|
1983
|
+
|
|
1984
|
+
return master_df
|
|
1985
|
+
|
|
1986
|
+
|
|
1987
|
+
class DataFrameProcessor:
|
|
1988
|
+
"""
|
|
1989
|
+
DataFrame处理工具类。
|
|
1990
|
+
|
|
1991
|
+
提供DataFrame操作的统一接口,包括列操作、行过滤、类型转换等功能。
|
|
1992
|
+
|
|
1993
|
+
Parameters
|
|
1994
|
+
----------
|
|
1995
|
+
data : pandas.DataFrame
|
|
1996
|
+
要处理的DataFrame
|
|
1997
|
+
|
|
1998
|
+
Examples
|
|
1999
|
+
--------
|
|
2000
|
+
>>> processor = DataFrameProcessor(df)
|
|
2001
|
+
>>> processor.move_column('col_a', 0)
|
|
2002
|
+
>>> processor.convert_colnames('lower')
|
|
2003
|
+
"""
|
|
2004
|
+
|
|
2005
|
+
def __init__(self, data):
|
|
2006
|
+
"""
|
|
2007
|
+
初始化DataFrame处理器。
|
|
2008
|
+
|
|
2009
|
+
Parameters
|
|
2010
|
+
----------
|
|
2011
|
+
data : pandas.DataFrame
|
|
2012
|
+
要处理的DataFrame
|
|
2013
|
+
"""
|
|
2014
|
+
self.data = data
|
|
2015
|
+
|
|
2016
|
+
def move_column(self, colname, idx, return_kDF=True, h2o_frame=False):
|
|
2017
|
+
"""
|
|
2018
|
+
移动列到指定位置。
|
|
2019
|
+
|
|
2020
|
+
Parameters
|
|
2021
|
+
----------
|
|
2022
|
+
colname : str
|
|
2023
|
+
要移动的列名
|
|
2024
|
+
idx : int
|
|
2025
|
+
目标位置索引
|
|
2026
|
+
return_kDF : bool, default True
|
|
2027
|
+
是否返回kDataFrame
|
|
2028
|
+
h2o_frame : bool, default False
|
|
2029
|
+
输入是否为H2OFrame
|
|
2030
|
+
|
|
2031
|
+
Returns
|
|
2032
|
+
-------
|
|
2033
|
+
DataFrame or kDataFrame
|
|
2034
|
+
"""
|
|
2035
|
+
return move_column(self.data, colname, idx, return_kDF, h2o_frame)
|
|
2036
|
+
|
|
2037
|
+
def convert_colnames(self, how="lowercase", return_kDF=True):
|
|
2038
|
+
"""
|
|
2039
|
+
转换列名格式。
|
|
2040
|
+
|
|
2041
|
+
Parameters
|
|
2042
|
+
----------
|
|
2043
|
+
how : str, default "lowercase"
|
|
2044
|
+
转换方式
|
|
2045
|
+
return_kDF : bool, default True
|
|
2046
|
+
是否返回kDataFrame
|
|
2047
|
+
|
|
2048
|
+
Returns
|
|
2049
|
+
-------
|
|
2050
|
+
DataFrame or kDataFrame
|
|
2051
|
+
"""
|
|
2052
|
+
return convert_colnames(self.data, how, return_kDF)
|
|
2053
|
+
|
|
2054
|
+
def col_filter_regex(self, regex, case_sensitive=True, h2o_frame=False, return_kDF=True):
|
|
2055
|
+
"""
|
|
2056
|
+
使用正则表达式过滤列。
|
|
2057
|
+
|
|
2058
|
+
Parameters
|
|
2059
|
+
----------
|
|
2060
|
+
regex : str
|
|
2061
|
+
正则表达式
|
|
2062
|
+
case_sensitive : bool, default True
|
|
2063
|
+
是否区分大小写
|
|
2064
|
+
h2o_frame : bool, default False
|
|
2065
|
+
输入是否为H2OFrame
|
|
2066
|
+
return_kDF : bool, default True
|
|
2067
|
+
是否返回kDataFrame
|
|
2068
|
+
|
|
2069
|
+
Returns
|
|
2070
|
+
-------
|
|
2071
|
+
DataFrame or kDataFrame
|
|
2072
|
+
"""
|
|
2073
|
+
return col_filter_regex(self.data, regex, case_sensitive, h2o_frame, return_kDF)
|
|
2074
|
+
|
|
2075
|
+
def row_filter_regex(self, col, regex, case_sensitive=True, as_index=False, return_kDF=True):
|
|
2076
|
+
"""
|
|
2077
|
+
使用正则表达式过滤行。
|
|
2078
|
+
|
|
2079
|
+
Parameters
|
|
2080
|
+
----------
|
|
2081
|
+
col : str
|
|
2082
|
+
用于过滤的列名
|
|
2083
|
+
regex : str
|
|
2084
|
+
正则表达式
|
|
2085
|
+
case_sensitive : bool, default True
|
|
2086
|
+
是否区分大小写
|
|
2087
|
+
as_index : bool, default False
|
|
2088
|
+
是否将过滤列作为索引
|
|
2089
|
+
return_kDF : bool, default True
|
|
2090
|
+
是否返回kDataFrame
|
|
2091
|
+
|
|
2092
|
+
Returns
|
|
2093
|
+
-------
|
|
2094
|
+
DataFrame or kDataFrame
|
|
2095
|
+
"""
|
|
2096
|
+
return row_filter_regex(self.data, col, regex, case_sensitive, as_index, return_kDF)
|
|
2097
|
+
|
|
2098
|
+
def get_dtypes(self, outputFile=None, ck_format=False):
|
|
2099
|
+
"""
|
|
2100
|
+
获取数据类型。
|
|
2101
|
+
|
|
2102
|
+
Parameters
|
|
2103
|
+
----------
|
|
2104
|
+
outputFile : str, optional
|
|
2105
|
+
输出文件路径
|
|
2106
|
+
ck_format : bool, default False
|
|
2107
|
+
是否使用自定义格式
|
|
2108
|
+
|
|
2109
|
+
Returns
|
|
2110
|
+
-------
|
|
2111
|
+
pandas.DataFrame
|
|
2112
|
+
"""
|
|
2113
|
+
return get_dtypes_file(self.data, outputFile, ck_format)
|
|
2114
|
+
|
|
2115
|
+
def drop_tmp_cols(self, drop_list=['py_inserttime']):
|
|
2116
|
+
"""
|
|
2117
|
+
删除临时列。
|
|
2118
|
+
|
|
2119
|
+
Parameters
|
|
2120
|
+
----------
|
|
2121
|
+
drop_list : list, default ['py_inserttime']
|
|
2122
|
+
要删除的列列表
|
|
2123
|
+
|
|
2124
|
+
Returns
|
|
2125
|
+
-------
|
|
2126
|
+
DataFrame
|
|
2127
|
+
"""
|
|
2128
|
+
return drop_tmp_cols(self.data, drop_list)
|
|
2129
|
+
|
|
2130
|
+
def to_bool_str(self):
|
|
2131
|
+
"""
|
|
2132
|
+
将布尔列转换为字符串。
|
|
2133
|
+
|
|
2134
|
+
Returns
|
|
2135
|
+
-------
|
|
2136
|
+
DataFrame
|
|
2137
|
+
"""
|
|
2138
|
+
return bool_to_str(self.data)
|
|
2139
|
+
|
|
2140
|
+
|
|
2141
|
+
class FilePathManager:
|
|
2142
|
+
"""
|
|
2143
|
+
文件路径管理工具类。
|
|
2144
|
+
|
|
2145
|
+
提供路径操作、文件列表获取、目录创建等功能。
|
|
2146
|
+
|
|
2147
|
+
Examples
|
|
2148
|
+
--------
|
|
2149
|
+
>>> fpm = FilePathManager('/base/path')
|
|
2150
|
+
>>> fpm.get_filenames('.*\\.csv')
|
|
2151
|
+
>>> fpm.mkdir_if_not_exist('output')
|
|
2152
|
+
"""
|
|
2153
|
+
|
|
2154
|
+
def __init__(self, base_path=None):
|
|
2155
|
+
"""
|
|
2156
|
+
初始化路径管理器。
|
|
2157
|
+
|
|
2158
|
+
Parameters
|
|
2159
|
+
----------
|
|
2160
|
+
base_path : str, optional
|
|
2161
|
+
基础路径
|
|
2162
|
+
"""
|
|
2163
|
+
self.base_path = base_path or os.getcwd()
|
|
2164
|
+
|
|
2165
|
+
def get_filenames(self, path, regex):
|
|
2166
|
+
"""
|
|
2167
|
+
获取匹配的文件名列表。
|
|
2168
|
+
|
|
2169
|
+
Parameters
|
|
2170
|
+
----------
|
|
2171
|
+
path : str
|
|
2172
|
+
目录路径
|
|
2173
|
+
regex : str
|
|
2174
|
+
正则表达式
|
|
2175
|
+
|
|
2176
|
+
Returns
|
|
2177
|
+
-------
|
|
2178
|
+
list
|
|
2179
|
+
"""
|
|
2180
|
+
return get_filenames(path, regex)
|
|
2181
|
+
|
|
2182
|
+
def add_suffix(self, file, suffix="_cut"):
|
|
2183
|
+
"""
|
|
2184
|
+
添加文件后缀。
|
|
2185
|
+
|
|
2186
|
+
Parameters
|
|
2187
|
+
----------
|
|
2188
|
+
file : str
|
|
2189
|
+
文件路径
|
|
2190
|
+
suffix : str, default "_cut"
|
|
2191
|
+
后缀
|
|
2192
|
+
|
|
2193
|
+
Returns
|
|
2194
|
+
-------
|
|
2195
|
+
str
|
|
2196
|
+
"""
|
|
2197
|
+
return add_path_suffix(file, suffix)
|
|
2198
|
+
|
|
2199
|
+
def mkdir(self, folder_path, replace=False):
|
|
2200
|
+
"""
|
|
2201
|
+
创建目录。
|
|
2202
|
+
|
|
2203
|
+
Parameters
|
|
2204
|
+
----------
|
|
2205
|
+
folder_path : str
|
|
2206
|
+
目录路径
|
|
2207
|
+
replace : bool, default False
|
|
2208
|
+
是否替换已存在目录
|
|
2209
|
+
|
|
2210
|
+
Returns
|
|
2211
|
+
-------
|
|
2212
|
+
int
|
|
2213
|
+
"""
|
|
2214
|
+
return mkdir_if_not_exist(folder_path, replace)
|
|
2215
|
+
|
|
2216
|
+
def get_curr_abs_path(self, path):
|
|
2217
|
+
"""
|
|
2218
|
+
获取绝对路径。
|
|
2219
|
+
|
|
2220
|
+
Parameters
|
|
2221
|
+
----------
|
|
2222
|
+
path : str
|
|
2223
|
+
相对路径
|
|
2224
|
+
|
|
2225
|
+
Returns
|
|
2226
|
+
-------
|
|
2227
|
+
str
|
|
2228
|
+
"""
|
|
2229
|
+
return get_curr_abs_path(path)
|
|
2230
|
+
|
|
2231
|
+
|
|
2232
|
+
class DateTimeUtils:
|
|
2233
|
+
"""
|
|
2234
|
+
日期时间工具类。
|
|
2235
|
+
|
|
2236
|
+
提供日期时间相关的便捷方法。
|
|
2237
|
+
|
|
2238
|
+
Examples
|
|
2239
|
+
--------
|
|
2240
|
+
>>> dt_utils = DateTimeUtils()
|
|
2241
|
+
>>> dt_utils.get_curr_datetime('-')
|
|
2242
|
+
'2025-03-30-143624'
|
|
2243
|
+
>>> dt_utils.get_last_vintage()
|
|
2244
|
+
'202502'
|
|
2245
|
+
"""
|
|
2246
|
+
|
|
2247
|
+
def get_curr_datetime(self, sep=''):
|
|
2248
|
+
"""
|
|
2249
|
+
获取当前日期时间。
|
|
2250
|
+
|
|
2251
|
+
Parameters
|
|
2252
|
+
----------
|
|
2253
|
+
sep : str, default ''
|
|
2254
|
+
分隔符
|
|
2255
|
+
|
|
2256
|
+
Returns
|
|
2257
|
+
-------
|
|
2258
|
+
str
|
|
2259
|
+
"""
|
|
2260
|
+
return get_curr_datetime(sep)
|
|
2261
|
+
|
|
2262
|
+
def get_buffer_date(self, start_date):
|
|
2263
|
+
"""
|
|
2264
|
+
获取缓冲日期。
|
|
2265
|
+
|
|
2266
|
+
Parameters
|
|
2267
|
+
----------
|
|
2268
|
+
start_date : str
|
|
2269
|
+
起始日期
|
|
2270
|
+
|
|
2271
|
+
Returns
|
|
2272
|
+
-------
|
|
2273
|
+
str
|
|
2274
|
+
"""
|
|
2275
|
+
return get_buffer_date(start_date)
|
|
2276
|
+
|
|
2277
|
+
def get_quarter(self, strDate):
|
|
2278
|
+
"""
|
|
2279
|
+
获取季度。
|
|
2280
|
+
|
|
2281
|
+
Parameters
|
|
2282
|
+
----------
|
|
2283
|
+
strDate : str
|
|
2284
|
+
日期字符串
|
|
2285
|
+
|
|
2286
|
+
Returns
|
|
2287
|
+
-------
|
|
2288
|
+
int
|
|
2289
|
+
"""
|
|
2290
|
+
return get_quarter(strDate)
|
|
2291
|
+
|
|
2292
|
+
def get_last_vintage(self):
|
|
2293
|
+
"""
|
|
2294
|
+
获取上一个月的vintage。
|
|
2295
|
+
|
|
2296
|
+
Returns
|
|
2297
|
+
-------
|
|
2298
|
+
str
|
|
2299
|
+
"""
|
|
2300
|
+
return get_last_vintage()
|
|
2301
|
+
|
|
2302
|
+
def last_month_vintage(self, year, month, day):
|
|
2303
|
+
"""
|
|
2304
|
+
获取上个月vintage。
|
|
2305
|
+
|
|
2306
|
+
Parameters
|
|
2307
|
+
----------
|
|
2308
|
+
year : int
|
|
2309
|
+
month : int
|
|
2310
|
+
day : int
|
|
2311
|
+
|
|
2312
|
+
Returns
|
|
2313
|
+
-------
|
|
2314
|
+
int
|
|
2315
|
+
"""
|
|
2316
|
+
return last_Month_Vintage(year, month, day)
|
|
2317
|
+
|
|
2318
|
+
def get_valid_vintages(self, sVintage, eVintage):
|
|
2319
|
+
"""
|
|
2320
|
+
获取有效vintage列表。
|
|
2321
|
+
|
|
2322
|
+
Parameters
|
|
2323
|
+
----------
|
|
2324
|
+
sVintage : int
|
|
2325
|
+
eVintage : int
|
|
2326
|
+
|
|
2327
|
+
Returns
|
|
2328
|
+
-------
|
|
2329
|
+
list
|
|
2330
|
+
"""
|
|
2331
|
+
return get_valid_vintages(sVintage, eVintage)
|
|
2332
|
+
|
|
2333
|
+
|
|
2334
|
+
class WOEIVCalculator:
|
|
2335
|
+
"""
|
|
2336
|
+
WOE和IV计算工具类。
|
|
2337
|
+
|
|
2338
|
+
提供信用评分中常用的WOE和IV指标计算功能。
|
|
2339
|
+
|
|
2340
|
+
Parameters
|
|
2341
|
+
----------
|
|
2342
|
+
data : pandas.DataFrame
|
|
2343
|
+
包含比例的数据表
|
|
2344
|
+
bad_pct_col : str
|
|
2345
|
+
坏样本占比列名
|
|
2346
|
+
good_pct_col : str
|
|
2347
|
+
好样本占比列名
|
|
2348
|
+
|
|
2349
|
+
Examples
|
|
2350
|
+
--------
|
|
2351
|
+
>>> calc = WOEIVCalculator(df, 'bad_pct', 'good_pct')
|
|
2352
|
+
>>> calc.woe()
|
|
2353
|
+
>>> calc.iv()
|
|
2354
|
+
"""
|
|
2355
|
+
|
|
2356
|
+
def __init__(self, data, bad_pct_col, good_pct_col):
|
|
2357
|
+
"""
|
|
2358
|
+
初始化WOE/IV计算器。
|
|
2359
|
+
|
|
2360
|
+
Parameters
|
|
2361
|
+
----------
|
|
2362
|
+
data : pandas.DataFrame
|
|
2363
|
+
数据表
|
|
2364
|
+
bad_pct_col : str
|
|
2365
|
+
坏样本占比列名
|
|
2366
|
+
good_pct_col : str
|
|
2367
|
+
好样本占比列名
|
|
2368
|
+
"""
|
|
2369
|
+
self.data = data
|
|
2370
|
+
self.bad_pct_col = bad_pct_col
|
|
2371
|
+
self.good_pct_col = good_pct_col
|
|
2372
|
+
|
|
2373
|
+
def calc_woe(self, fillna=True):
|
|
2374
|
+
"""
|
|
2375
|
+
计算WOE值。
|
|
2376
|
+
|
|
2377
|
+
Parameters
|
|
2378
|
+
----------
|
|
2379
|
+
fillna : bool, default True
|
|
2380
|
+
是否填充NA为0
|
|
2381
|
+
|
|
2382
|
+
Returns
|
|
2383
|
+
-------
|
|
2384
|
+
float or Series
|
|
2385
|
+
"""
|
|
2386
|
+
return calc_woe(self.data, self.bad_pct_col, self.good_pct_col, fillna)
|
|
2387
|
+
|
|
2388
|
+
def calc_iv(self, fillna=True):
|
|
2389
|
+
"""
|
|
2390
|
+
计算IV值。
|
|
2391
|
+
|
|
2392
|
+
Parameters
|
|
2393
|
+
----------
|
|
2394
|
+
fillna : bool, default True
|
|
2395
|
+
是否填充NA为0
|
|
2396
|
+
|
|
2397
|
+
Returns
|
|
2398
|
+
-------
|
|
2399
|
+
float or Series
|
|
2400
|
+
"""
|
|
2401
|
+
return calc_iv(self.data, self.bad_pct_col, self.good_pct_col, fillna)
|
|
2402
|
+
|
|
2403
|
+
def calc_both(self, fillna=True):
|
|
2404
|
+
"""
|
|
2405
|
+
同时计算WOE和IV。
|
|
2406
|
+
|
|
2407
|
+
Parameters
|
|
2408
|
+
----------
|
|
2409
|
+
fillna : bool, default True
|
|
2410
|
+
是否填充NA为0
|
|
2411
|
+
|
|
2412
|
+
Returns
|
|
2413
|
+
-------
|
|
2414
|
+
tuple
|
|
2415
|
+
"""
|
|
2416
|
+
woe = self.calc_woe(fillna)
|
|
2417
|
+
iv = self.calc_iv(fillna)
|
|
2418
|
+
return woe, iv
|
|
2419
|
+
|
|
2420
|
+
|
|
2421
|
+
def get_feature_names(model, model_type=None):
|
|
2422
|
+
"""获取模型的特征名称列表。
|
|
2423
|
+
|
|
2424
|
+
自动检测模型类型并返回其特征名称。
|
|
2425
|
+
支持LightGBM、XGBoost、sklearn等多种模型。
|
|
2426
|
+
|
|
2427
|
+
Parameters
|
|
2428
|
+
----------
|
|
2429
|
+
model : object
|
|
2430
|
+
训练好的机器学习模型对象
|
|
2431
|
+
model_type : str, optional
|
|
2432
|
+
模型类型提示,可选值:
|
|
2433
|
+
- 'lgb' 或 'lightgbm': LightGBM模型
|
|
2434
|
+
- 'xgb' 或 'xgboost': XGBoost模型
|
|
2435
|
+
- 'sklearn': sklearn模型
|
|
2436
|
+
- None: 自动检测(默认)
|
|
2437
|
+
|
|
2438
|
+
Returns
|
|
2439
|
+
-------
|
|
2440
|
+
list
|
|
2441
|
+
特征名称列表
|
|
2442
|
+
|
|
2443
|
+
Raises
|
|
2444
|
+
------
|
|
2445
|
+
ValueError
|
|
2446
|
+
当无法获取特征名称时抛出
|
|
2447
|
+
|
|
2448
|
+
Examples
|
|
2449
|
+
--------
|
|
2450
|
+
>>> # 通用方式
|
|
2451
|
+
>>> feature_names = get_feature_names(model)
|
|
2452
|
+
|
|
2453
|
+
>>> # 指定类型
|
|
2454
|
+
>>> feature_names = get_feature_names(lgb_model, model_type='lgb')
|
|
2455
|
+
|
|
2456
|
+
>>> # 处理XGBoost
|
|
2457
|
+
>>> feature_names = get_feature_names(xgb_model, model_type='xgb')
|
|
2458
|
+
"""
|
|
2459
|
+
# 如果指定了模型类型,优先使用专用函数
|
|
2460
|
+
if model_type is not None:
|
|
2461
|
+
model_type_lower = model_type.lower()
|
|
2462
|
+
if model_type_lower in ['lgb', 'lightgbm']:
|
|
2463
|
+
return get_feature_names_lgb(model)
|
|
2464
|
+
elif model_type_lower in ['xgb', 'xgboost']:
|
|
2465
|
+
return get_feature_names_xgb(model)
|
|
2466
|
+
|
|
2467
|
+
# 自动检测模型类型并获取特征名
|
|
2468
|
+
model_class_name = model.__class__.__name__.lower()
|
|
2469
|
+
|
|
2470
|
+
# SMF GradientBoostingModel wraps the fitted estimator in _model.model and
|
|
2471
|
+
# stores DataFrame column names on _model.feature_names_ after fit.
|
|
2472
|
+
wrapped_model_type = getattr(model, 'model_type', None)
|
|
2473
|
+
wrapped_backend = getattr(model, '_model', None)
|
|
2474
|
+
if wrapped_model_type is not None and wrapped_backend is not None:
|
|
2475
|
+
wrapped_feature_names = getattr(wrapped_backend, 'feature_names_', None)
|
|
2476
|
+
if wrapped_feature_names is not None:
|
|
2477
|
+
return list(wrapped_feature_names)
|
|
2478
|
+
|
|
2479
|
+
wrapped_estimator = getattr(wrapped_backend, 'model', None)
|
|
2480
|
+
if wrapped_estimator is not None:
|
|
2481
|
+
wrapped_model_type_lower = str(wrapped_model_type).lower()
|
|
2482
|
+
if wrapped_model_type_lower in ['lgb', 'lightgbm']:
|
|
2483
|
+
return get_feature_names_lgb(wrapped_estimator)
|
|
2484
|
+
if wrapped_model_type_lower in ['xgb', 'xgboost']:
|
|
2485
|
+
return get_feature_names_xgb(wrapped_estimator)
|
|
2486
|
+
return get_feature_names(wrapped_estimator)
|
|
2487
|
+
|
|
2488
|
+
# LightGBM 检测
|
|
2489
|
+
if 'lgb' in model_class_name or 'lightgbm' in model_class_name:
|
|
2490
|
+
return get_feature_names_lgb(model)
|
|
2491
|
+
|
|
2492
|
+
# XGBoost 检测
|
|
2493
|
+
if 'xgb' in model_class_name or 'xgboost' in model_class_name:
|
|
2494
|
+
return get_feature_names_xgb(model)
|
|
2495
|
+
|
|
2496
|
+
if 'logisticregression' in model_class_name:
|
|
2497
|
+
return list(model.feature_names_in_)
|
|
2498
|
+
|
|
2499
|
+
# 尝试通用sklearn方式
|
|
2500
|
+
# 方法1: feature_names_in 属性 (sklearn >= 1.0)
|
|
2501
|
+
if hasattr(model, 'feature_names_in_'):
|
|
2502
|
+
return list(model.feature_names_in_)
|
|
2503
|
+
|
|
2504
|
+
# 方法2: feature_names 属性
|
|
2505
|
+
if hasattr(model, 'feature_names'):
|
|
2506
|
+
feature_names = model.feature_names
|
|
2507
|
+
if callable(feature_names):
|
|
2508
|
+
return list(feature_names())
|
|
2509
|
+
return list(feature_names)
|
|
2510
|
+
|
|
2511
|
+
# 方法3: booster方式 (LightGBM特有)
|
|
2512
|
+
if hasattr(model, 'booster_'):
|
|
2513
|
+
try:
|
|
2514
|
+
return model.booster_.feature_name()
|
|
2515
|
+
except (AttributeError, TypeError):
|
|
2516
|
+
pass
|
|
2517
|
+
|
|
2518
|
+
# 方法4: 尝试从模型参数中获取
|
|
2519
|
+
if hasattr(model, 'feature_name'):
|
|
2520
|
+
try:
|
|
2521
|
+
feature_names = model.feature_name
|
|
2522
|
+
if callable(feature_names):
|
|
2523
|
+
return list(feature_names())
|
|
2524
|
+
return list(feature_names)
|
|
2525
|
+
except (AttributeError, TypeError):
|
|
2526
|
+
pass
|
|
2527
|
+
|
|
2528
|
+
# 无法获取特征名
|
|
2529
|
+
raise ValueError(
|
|
2530
|
+
f"无法获取模型 '{model_class_name}' 的特征名称。\n"
|
|
2531
|
+
f"请尝试:\n"
|
|
2532
|
+
f"1. 显式指定 model_type 参数\n"
|
|
2533
|
+
f"2. 使用专用函数:get_feature_names_lgb() 或 get_feature_names_xgb()"
|
|
2534
|
+
)
|
|
2535
|
+
|
|
2536
|
+
def get_feature_names_lgb(model):
|
|
2537
|
+
"""获取LightGBM模型的特征名称。
|
|
2538
|
+
|
|
2539
|
+
Parameters
|
|
2540
|
+
----------
|
|
2541
|
+
model : lgb.LGBMClassifier or lgb.LGBMRegressor
|
|
2542
|
+
训练好的LightGBM模型
|
|
2543
|
+
|
|
2544
|
+
Returns
|
|
2545
|
+
-------
|
|
2546
|
+
list
|
|
2547
|
+
特征名称列表
|
|
2548
|
+
|
|
2549
|
+
Raises
|
|
2550
|
+
------
|
|
2551
|
+
ValueError
|
|
2552
|
+
当无法获取特征名称时抛出
|
|
2553
|
+
|
|
2554
|
+
Examples
|
|
2555
|
+
--------
|
|
2556
|
+
>>> import lightgbm as lgb
|
|
2557
|
+
>>> model = lgb.LGBMClassifier().fit(X_train, y_train)
|
|
2558
|
+
>>> feature_names = get_feature_names_lgb(model)
|
|
2559
|
+
>>> print(feature_names)
|
|
2560
|
+
['feature_1', 'feature_2', 'feature_3']
|
|
2561
|
+
"""
|
|
2562
|
+
# 方法1: booster_.feature_name() (最可靠)
|
|
2563
|
+
if hasattr(model, 'booster_') and model.booster_ is not None:
|
|
2564
|
+
try:
|
|
2565
|
+
return model.booster_.feature_name()
|
|
2566
|
+
except (AttributeError, TypeError):
|
|
2567
|
+
pass
|
|
2568
|
+
|
|
2569
|
+
# 方法2: feature_name_ 属性
|
|
2570
|
+
if hasattr(model, 'feature_name_'):
|
|
2571
|
+
return list(model.feature_name_)
|
|
2572
|
+
|
|
2573
|
+
# 方法3: feature_name 属性/方法
|
|
2574
|
+
if hasattr(model, 'feature_name'):
|
|
2575
|
+
feature_names = model.feature_name
|
|
2576
|
+
if callable(feature_names):
|
|
2577
|
+
return list(feature_names())
|
|
2578
|
+
return list(feature_names)
|
|
2579
|
+
|
|
2580
|
+
raise ValueError(
|
|
2581
|
+
"无法获取LightGBM模型的特征名称。\n"
|
|
2582
|
+
"确保模型已正确训练。"
|
|
2583
|
+
)
|
|
2584
|
+
|
|
2585
|
+
|
|
2586
|
+
def get_feature_names_xgb(model):
|
|
2587
|
+
"""获取XGBoost模型的特征名称。
|
|
2588
|
+
|
|
2589
|
+
Parameters
|
|
2590
|
+
----------
|
|
2591
|
+
model : xgb.XGBClassifier or xgb.XGBRegressor
|
|
2592
|
+
训练好的XGBoost模型
|
|
2593
|
+
|
|
2594
|
+
Returns
|
|
2595
|
+
-------
|
|
2596
|
+
list
|
|
2597
|
+
特征名称列表
|
|
2598
|
+
|
|
2599
|
+
Raises
|
|
2600
|
+
------
|
|
2601
|
+
ValueError
|
|
2602
|
+
当无法获取特征名称时抛出
|
|
2603
|
+
|
|
2604
|
+
Examples
|
|
2605
|
+
--------
|
|
2606
|
+
>>> import xgboost as xgb
|
|
2607
|
+
>>> model = xgb.XGBClassifier().fit(X_train, y_train)
|
|
2608
|
+
>>> feature_names = get_feature_names_xgb(model)
|
|
2609
|
+
>>> print(feature_names)
|
|
2610
|
+
['feature_1', 'feature_2', 'feature_3']
|
|
2611
|
+
"""
|
|
2612
|
+
# 方法1: feature_names_in 属性 (sklearn风格)
|
|
2613
|
+
if hasattr(model, 'feature_names_in_'):
|
|
2614
|
+
return list(model.feature_names_in_)
|
|
2615
|
+
|
|
2616
|
+
# 方法2: booster.get_feature_names() (原生XGBoost)
|
|
2617
|
+
if hasattr(model, 'get_booster'):
|
|
2618
|
+
try:
|
|
2619
|
+
booster = model.get_booster()
|
|
2620
|
+
feature_names = booster.get_feature_names()
|
|
2621
|
+
return list(feature_names) if feature_names else []
|
|
2622
|
+
except (AttributeError, TypeError):
|
|
2623
|
+
pass
|
|
2624
|
+
|
|
2625
|
+
# 方法3: feature_names 属性
|
|
2626
|
+
if hasattr(model, 'feature_names'):
|
|
2627
|
+
feature_names = model.feature_names
|
|
2628
|
+
if callable(feature_names):
|
|
2629
|
+
return list(feature_names())
|
|
2630
|
+
return list(feature_names)
|
|
2631
|
+
|
|
2632
|
+
raise ValueError(
|
|
2633
|
+
"无法获取XGBoost模型的特征名称。\n"
|
|
2634
|
+
"确保模型已正确训练。"
|
|
2635
|
+
)
|
|
2636
|
+
|
|
2637
|
+
|
|
2638
|
+
# ============================================================================
|
|
2639
|
+
# 便捷函数:批量获取特征名
|
|
2640
|
+
# ============================================================================
|
|
2641
|
+
|
|
2642
|
+
def get_feature_names_batch(models, model_type=None):
|
|
2643
|
+
"""批量获取多个模型的特征名称。
|
|
2644
|
+
|
|
2645
|
+
Parameters
|
|
2646
|
+
----------
|
|
2647
|
+
models : dict or list
|
|
2648
|
+
模型字典 {name: model} 或模型列表
|
|
2649
|
+
model_type : str, optional
|
|
2650
|
+
模型类型提示
|
|
2651
|
+
|
|
2652
|
+
Returns
|
|
2653
|
+
-------
|
|
2654
|
+
dict or list
|
|
2655
|
+
特征名称字典或列表,与输入结构对应
|
|
2656
|
+
|
|
2657
|
+
Examples
|
|
2658
|
+
--------
|
|
2659
|
+
>>> models = {'lgb': lgb_model, 'xgb': xgb_model}
|
|
2660
|
+
>>> feature_names_dict = get_feature_names_batch(models)
|
|
2661
|
+
>>> print(feature_names_dict)
|
|
2662
|
+
{'lgb': ['f1', 'f2'], 'xgb': ['f1', 'f2']}
|
|
2663
|
+
"""
|
|
2664
|
+
if isinstance(models, dict):
|
|
2665
|
+
return {
|
|
2666
|
+
name: get_feature_names(model, model_type=model_type)
|
|
2667
|
+
for name, model in models.items()
|
|
2668
|
+
}
|
|
2669
|
+
elif isinstance(models, list):
|
|
2670
|
+
return [get_feature_names(model, model_type=model_type) for model in models]
|
|
2671
|
+
else:
|
|
2672
|
+
raise TypeError("models参数应为dict或list类型")
|