kivi 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kivi-0.2.4/PKG-INFO +19 -0
- kivi-0.2.4/README.md +6 -0
- kivi-0.2.4/kivi/BucketsAnalysis/BucketsUtils.py +83 -0
- kivi-0.2.4/kivi/BucketsAnalysis/CSI.py +105 -0
- kivi-0.2.4/kivi/BucketsAnalysis/KS.py +374 -0
- kivi-0.2.4/kivi/BucketsAnalysis/PSI.py +124 -0
- kivi-0.2.4/kivi/BucketsAnalysis/__init__.py +3 -0
- kivi-0.2.4/kivi/Card/Calibration.py +138 -0
- kivi-0.2.4/kivi/Card/MasterScale.py +120 -0
- kivi-0.2.4/kivi/Card/ScoreCard.py +99 -0
- kivi-0.2.4/kivi/Card/__init__.py +6 -0
- kivi-0.2.4/kivi/Card/card_eval.py +110 -0
- kivi-0.2.4/kivi/Card/card_rant.py +15 -0
- kivi-0.2.4/kivi/Card/card_score.py +87 -0
- kivi-0.2.4/kivi/Card/utils.py +26 -0
- kivi-0.2.4/kivi/Dataset/Dataset.py +53 -0
- kivi-0.2.4/kivi/Dataset/__init__.py +1 -0
- kivi-0.2.4/kivi/Dataset/bank.xlsx +0 -0
- kivi-0.2.4/kivi/Dataset/crime_data.csv +52 -0
- kivi-0.2.4/kivi/DesStatistic/CategoryData.py +71 -0
- kivi-0.2.4/kivi/DesStatistic/DefaultAnalysis.py +290 -0
- kivi-0.2.4/kivi/DesStatistic/__init__.py +6 -0
- kivi-0.2.4/kivi/FeatureAnalysis/Corr.py +35 -0
- kivi-0.2.4/kivi/FeatureAnalysis/FeatureSelect.py +27 -0
- kivi-0.2.4/kivi/FeatureAnalysis/Univariate.py +122 -0
- kivi-0.2.4/kivi/FeatureAnalysis/VIF.py +140 -0
- kivi-0.2.4/kivi/FeatureAnalysis/__init__.py +3 -0
- kivi-0.2.4/kivi/FeatureTools/FeatureTools.py +352 -0
- kivi-0.2.4/kivi/FeatureTools/FeatureToolsV2.py +255 -0
- kivi-0.2.4/kivi/FeatureTools/Scale.py +21 -0
- kivi-0.2.4/kivi/FeatureTools/__init__.py +2 -0
- kivi-0.2.4/kivi/Model/Apriori.py +135 -0
- kivi-0.2.4/kivi/Model/Lasso.py +97 -0
- kivi-0.2.4/kivi/Model/Linear.py +359 -0
- kivi-0.2.4/kivi/Model/LogisticRegression.py +163 -0
- kivi-0.2.4/kivi/Model/Ridge.py +61 -0
- kivi-0.2.4/kivi/Model/Tree.py +77 -0
- kivi-0.2.4/kivi/Model/__init__.py +9 -0
- kivi-0.2.4/kivi/Model/simtext.py +102 -0
- kivi-0.2.4/kivi/ModelEval/ModelEval.py +248 -0
- kivi-0.2.4/kivi/ModelEval/__init__.py +1 -0
- kivi-0.2.4/kivi/ModelTools/Calibration.py +244 -0
- kivi-0.2.4/kivi/ModelTools/Metrics.py +170 -0
- kivi-0.2.4/kivi/ModelTools/Sampling.py +53 -0
- kivi-0.2.4/kivi/ModelTools/__init__.py +4 -0
- kivi-0.2.4/kivi/Plot/Heatmap.py +52 -0
- kivi-0.2.4/kivi/Plot/__init__.py +0 -0
- kivi-0.2.4/kivi/PortfolioAnalysis/PortfolioAnalysis.py +269 -0
- kivi-0.2.4/kivi/PortfolioAnalysis/__init__.py +1 -0
- kivi-0.2.4/kivi/SQLengine/SQLengine.py +152 -0
- kivi-0.2.4/kivi/SQLengine/__init__.py +6 -0
- kivi-0.2.4/kivi/SparkSupport/SparkSupport.py +530 -0
- kivi-0.2.4/kivi/SparkSupport/__init__.py +4 -0
- kivi-0.2.4/kivi/Stepwise/Stepwise.py +123 -0
- kivi-0.2.4/kivi/Stepwise/__init__.py +1 -0
- kivi-0.2.4/kivi/Utils/BeautyExcel.py +193 -0
- kivi-0.2.4/kivi/Utils/Bins.py +28 -0
- kivi-0.2.4/kivi/Utils/Operator.py +187 -0
- kivi-0.2.4/kivi/Utils/Typing.py +8 -0
- kivi-0.2.4/kivi/Utils/__init__.py +8 -0
- kivi-0.2.4/kivi/Utils/encrypt.py +61 -0
- kivi-0.2.4/kivi/Utils/markdown.py +56 -0
- kivi-0.2.4/kivi/Utils/utils.py +226 -0
- kivi-0.2.4/kivi/Utils/utils_date.py +75 -0
- kivi-0.2.4/kivi/WOE/FixWOE.py +232 -0
- kivi-0.2.4/kivi/WOE/TransWOEScore.py +161 -0
- kivi-0.2.4/kivi/WOE/WOE.py +360 -0
- kivi-0.2.4/kivi/WOE/WOEKmeans.py +60 -0
- kivi-0.2.4/kivi/WOE/WoeBatch.py +225 -0
- kivi-0.2.4/kivi/WOE/WoeCategory.py +38 -0
- kivi-0.2.4/kivi/WOE/WoeChi2.py +251 -0
- kivi-0.2.4/kivi/WOE/WoeCutOffPoint.py +43 -0
- kivi-0.2.4/kivi/WOE/WoeDistance.py +39 -0
- kivi-0.2.4/kivi/WOE/WoeFrequency.py +37 -0
- kivi-0.2.4/kivi/WOE/WoeKSMerge.py +90 -0
- kivi-0.2.4/kivi/WOE/WoePearson.py +153 -0
- kivi-0.2.4/kivi/WOE/WoeTree.py +76 -0
- kivi-0.2.4/kivi/WOE/__init__.py +13 -0
- kivi-0.2.4/kivi/__init__.py +5 -0
- kivi-0.2.4/kivi/fake_data/__init__.py +2 -0
- kivi-0.2.4/kivi/fake_data/fake_data.py +86 -0
- kivi-0.2.4/kivi/fake_data/fake_data_config.py +20 -0
- kivi-0.2.4/kivi.egg-info/PKG-INFO +19 -0
- kivi-0.2.4/kivi.egg-info/SOURCES.txt +87 -0
- kivi-0.2.4/kivi.egg-info/dependency_links.txt +1 -0
- kivi-0.2.4/kivi.egg-info/requires.txt +8 -0
- kivi-0.2.4/kivi.egg-info/top_level.txt +1 -0
- kivi-0.2.4/setup.cfg +4 -0
- kivi-0.2.4/setup.py +45 -0
kivi-0.2.4/PKG-INFO
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: kivi
|
|
3
|
+
Version: 0.2.4
|
|
4
|
+
Summary: A Bank Risk Manager
|
|
5
|
+
Home-page: https://github.com/Risk-KIVI/kivi
|
|
6
|
+
Author: chensy.cao
|
|
7
|
+
Author-email: chensy.cao@foxmail.com
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.8
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# kivi
|
|
15
|
+
|
|
16
|
+
**Author: Chensy**
|
|
17
|
+
**Email: chensy.cao@foxmail.com**
|
|
18
|
+
|
|
19
|
+
----
|
kivi-0.2.4/README.md
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
from pyspark.sql import functions as F
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"quantile",
|
|
7
|
+
"getQuantiles",
|
|
8
|
+
"psi",
|
|
9
|
+
"csi",
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def quantile(col: str, percent=0.5):
|
|
15
|
+
"""
|
|
16
|
+
描述:分位数计算
|
|
17
|
+
|
|
18
|
+
参数:
|
|
19
|
+
:param col: 字段名称
|
|
20
|
+
:param percent: 分位点
|
|
21
|
+
:return : pyspark.sql.functions
|
|
22
|
+
"""
|
|
23
|
+
return F.expr(f'percentile_approx({col}, {percent})')
|
|
24
|
+
|
|
25
|
+
def getQuantiles(df, column_name, ranges=np.linspace(0, 1, 11))->list:
|
|
26
|
+
"""
|
|
27
|
+
描述:获取字段的分位数点。
|
|
28
|
+
|
|
29
|
+
参数:
|
|
30
|
+
:param df:
|
|
31
|
+
:param column_name:
|
|
32
|
+
:param ranges:
|
|
33
|
+
:return : 分位数值 List
|
|
34
|
+
|
|
35
|
+
示例:获取多个字段的多个分位点
|
|
36
|
+
>>> quant_val_list = [getQuantiles(df, col) for col in cols]
|
|
37
|
+
>>> pd.DataFrame(dict(zip(cols, quant_val_list)), index=np.linspace(0, 1, 11)).T
|
|
38
|
+
"""
|
|
39
|
+
quantiles = [quantile(column_name, i).alias(f'i') for i in ranges]
|
|
40
|
+
return df.agg(*quantiles).toPandas().values.flatten().tolist()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def psi(df_buckets, ex_name='Expected', ac_name='Actual', fill_bin=True):
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
:param df_buckets:
|
|
47
|
+
:param ex_name:
|
|
48
|
+
:param ac_name:
|
|
49
|
+
:return:
|
|
50
|
+
"""
|
|
51
|
+
if fill_bin:
|
|
52
|
+
if 0 in df_buckets[ex_name].tolist():
|
|
53
|
+
df_buckets[ex_name] += 0.5
|
|
54
|
+
if 0 in df_buckets[ac_name].tolist():
|
|
55
|
+
df_buckets[ac_name] += 0.5
|
|
56
|
+
|
|
57
|
+
df_buckets['Expected_percentage'] = df_buckets[ex_name] / df_buckets[ex_name].sum()
|
|
58
|
+
df_buckets['Actual_percentage'] = df_buckets[ac_name] / df_buckets[ac_name].sum()
|
|
59
|
+
df_buckets['A-E'] = df_buckets.Actual_percentage - df_buckets.Expected_percentage
|
|
60
|
+
df_buckets['A/E'] = df_buckets.Actual_percentage / df_buckets.Expected_percentage
|
|
61
|
+
df_buckets['ln(A/E)'] = np.log(df_buckets['A/E'])
|
|
62
|
+
df_buckets['psi_val'] = df_buckets['A-E'] * df_buckets['ln(A/E)']
|
|
63
|
+
df_buckets['psi'] = df_buckets['psi_val'].sum()
|
|
64
|
+
return df_buckets
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def csi(df_buckets, ex_name='Expected', ac_name='Actual', fill_bin=True):
|
|
68
|
+
"""
|
|
69
|
+
Des: 计算CSI的核心函数
|
|
70
|
+
:param df_buckets: 明细表
|
|
71
|
+
:return: 明细表以及CSI值 df_buckets, csi
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
df_buckets['Expected_percentage'] = df_buckets.Expected / df_buckets.Expected.sum()
|
|
75
|
+
df_buckets['Actual_percentage'] = df_buckets.Actual / df_buckets.Actual.sum()
|
|
76
|
+
df_buckets['A-E'] = df_buckets.Actual_percentage - df_buckets.Expected_percentage
|
|
77
|
+
if len(df_buckets) != len(self.score):
|
|
78
|
+
raise Exception(TypeError, 'Score len error!')
|
|
79
|
+
df_buckets['score'] = self.score
|
|
80
|
+
df_buckets['index'] = df_buckets['A-E'] * self.score
|
|
81
|
+
df_buckets['csi'] = df_buckets['index'].sum()
|
|
82
|
+
return df_buckets
|
|
83
|
+
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
from ..Utils.utils import pd
|
|
2
|
+
from ..Utils.Bins import Bins
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class CSI(Bins):
|
|
6
|
+
def __init__(self, score=None):
|
|
7
|
+
"""
|
|
8
|
+
Des: 计算CSI
|
|
9
|
+
:param score: 分箱分数
|
|
10
|
+
:param margin: cutoff point 左右填充预留边界
|
|
11
|
+
"""
|
|
12
|
+
super(CSI, self).__init__()
|
|
13
|
+
self.score = score
|
|
14
|
+
|
|
15
|
+
def _csi(self, res):
|
|
16
|
+
"""
|
|
17
|
+
Des: 计算CSI的核心函数
|
|
18
|
+
:param res: 明细表
|
|
19
|
+
:return: 明细表以及CSI值 res, csi
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
res['Expected_percentage'] = res.Expected / res.Expected.sum()
|
|
23
|
+
res['Actual_percentage'] = res.Actual / res.Actual.sum()
|
|
24
|
+
res['A-E'] = res.Actual_percentage - res.Expected_percentage
|
|
25
|
+
if len(res) != len(self.score):
|
|
26
|
+
raise Exception(TypeError, 'Score len error!')
|
|
27
|
+
res['score'] = self.score
|
|
28
|
+
res['index'] = res['A-E'] * self.score
|
|
29
|
+
csi = res['index'].sum()
|
|
30
|
+
return res, csi
|
|
31
|
+
|
|
32
|
+
def Continue(self, expected, actual, bins=10, binType='distance', cutPoint=None):
|
|
33
|
+
"""
|
|
34
|
+
Des: 连续型变量的CSI指标
|
|
35
|
+
:param expected: 预期分布
|
|
36
|
+
:param actual: 实际分布
|
|
37
|
+
:param bins: 分箱数目
|
|
38
|
+
:param binType: 分箱类型 'distance'等距 or 'frequency'等频 or 'customize'自定义
|
|
39
|
+
:param csvName: 保存CSV Name, 默认不保存
|
|
40
|
+
:return: 明细表以及CSI值 res, CSI
|
|
41
|
+
"""
|
|
42
|
+
expected.dropna(inplace=True)
|
|
43
|
+
actual.dropna(inplace=True)
|
|
44
|
+
|
|
45
|
+
expectedMax = expected.max()
|
|
46
|
+
expectedMin = expected.min()
|
|
47
|
+
actualMax = actual.max()
|
|
48
|
+
actualMin = actual.min()
|
|
49
|
+
|
|
50
|
+
if not pd.api.types.is_numeric_dtype(expected):
|
|
51
|
+
raise Exception(TypeError, 'expected must be numeric!')
|
|
52
|
+
|
|
53
|
+
if not pd.api.types.is_numeric_dtype(actual):
|
|
54
|
+
raise Exception(TypeError, 'actual must be numeric!')
|
|
55
|
+
|
|
56
|
+
if (expectedMax == expectedMin) or (actualMax == actualMin):
|
|
57
|
+
raise Exception()
|
|
58
|
+
|
|
59
|
+
if (expectedMin > actualMin) or (expectedMax < actualMax):
|
|
60
|
+
raise Exception()
|
|
61
|
+
|
|
62
|
+
if binType == 'distance':
|
|
63
|
+
cutoffPoint = self._cutoffPoint(
|
|
64
|
+
bins=bins, min=expectedMin, max=expectedMax)
|
|
65
|
+
elif binType == 'frequency':
|
|
66
|
+
cutoffPoint = self._cutoffPoint(
|
|
67
|
+
bins=bins, percent=True, expected=expected)
|
|
68
|
+
elif binType == 'customize' and cutPoint:
|
|
69
|
+
cutoffPoint = cutPoint
|
|
70
|
+
else:
|
|
71
|
+
raise Exception(TypeError, '分箱错误!')
|
|
72
|
+
|
|
73
|
+
res = pd.DataFrame({
|
|
74
|
+
'Expected': pd.cut(expected, cutoffPoint).value_counts(sort=False),
|
|
75
|
+
'Actual': pd.cut(actual, cutoffPoint).value_counts(sort=False),
|
|
76
|
+
},)
|
|
77
|
+
|
|
78
|
+
return self._csi(res)
|
|
79
|
+
|
|
80
|
+
def Category(self, expected, actual):
|
|
81
|
+
"""
|
|
82
|
+
Des: 类别型变量的CSI
|
|
83
|
+
:param expected:
|
|
84
|
+
:param actual:
|
|
85
|
+
:return: 明细表以及CSI值 res, CSI
|
|
86
|
+
"""
|
|
87
|
+
if (not pd.api.types.is_string_dtype(expected)) or (not pd.api.types.is_object_dtype(expected)):
|
|
88
|
+
raise Exception(TypeError, 'expected must be string or object!')
|
|
89
|
+
if (not pd.api.types.is_string_dtype(actual)) or (not pd.api.types.is_object_dtype(actual)):
|
|
90
|
+
raise Exception(TypeError, 'actual must be string or object!')
|
|
91
|
+
|
|
92
|
+
expected.dropna(inplace=True)
|
|
93
|
+
actual.dropna(inplace=True)
|
|
94
|
+
|
|
95
|
+
expectedGroup = expected.value_counts(sort=False)
|
|
96
|
+
actualGroup = actual.value_counts(sort=False)
|
|
97
|
+
|
|
98
|
+
res = pd.merge(
|
|
99
|
+
pd.DataFrame({'category': expectedGroup.index, 'Expected': expectedGroup.values}),
|
|
100
|
+
pd.DataFrame({'category': actualGroup.index, 'Actual': actualGroup.values}),
|
|
101
|
+
on='category',
|
|
102
|
+
how='outer',)
|
|
103
|
+
|
|
104
|
+
self.res, self.csi = self._csi(res)
|
|
105
|
+
return self.res, self.csi
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from ..Utils.utils import pd, saveCsv
|
|
3
|
+
from ..Utils.Bins import Bins
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class KS(Bins):
|
|
7
|
+
|
|
8
|
+
def __init__(self, csvName=None, imageName=None, jupyterNotebook=True):
|
|
9
|
+
"""
|
|
10
|
+
Des: KS 初始化
|
|
11
|
+
:param csvName: 保存csv计算过程文件
|
|
12
|
+
:param imageName: 保存ks图像
|
|
13
|
+
:param jupyterNotebook 是否显示在JupyterNotebook中
|
|
14
|
+
"""
|
|
15
|
+
super(KS, self).__init__()
|
|
16
|
+
self.csvName = csvName
|
|
17
|
+
self.imageName = imageName
|
|
18
|
+
self.jupyterNotebook = jupyterNotebook
|
|
19
|
+
|
|
20
|
+
def _ks(self, variables, target, **kwargs):
|
|
21
|
+
"""
|
|
22
|
+
Des: KS 方法中的内部核心计算函数
|
|
23
|
+
:param variables:
|
|
24
|
+
:param target:
|
|
25
|
+
:param kwargs:
|
|
26
|
+
:return:
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
kwargs.setdefault('category', False)
|
|
30
|
+
|
|
31
|
+
from numpy import abs
|
|
32
|
+
total = len(target)
|
|
33
|
+
total_bad = target.sum()
|
|
34
|
+
|
|
35
|
+
if kwargs['category']:
|
|
36
|
+
bucket = DataFrame({
|
|
37
|
+
'variables': variables,
|
|
38
|
+
'target': target,
|
|
39
|
+
}).groupby('variables', as_index=True)
|
|
40
|
+
else:
|
|
41
|
+
bucket = DataFrame({
|
|
42
|
+
'variables': variables,
|
|
43
|
+
'target': target,
|
|
44
|
+
'Bucket': cut(variables, self.cutoffPoint, include_lowest=True)
|
|
45
|
+
}).groupby('Bucket', as_index=True)
|
|
46
|
+
|
|
47
|
+
res = DataFrame({
|
|
48
|
+
'min_bin': bucket.variables.min(),
|
|
49
|
+
'max_bin': bucket.variables.max(),
|
|
50
|
+
'bad': bucket.target.sum(),
|
|
51
|
+
'total': bucket.target.count(),
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
res['good'] = res['total'] - res['bad']
|
|
55
|
+
res['bad_rate'] = res['bad'] / res['total'] # 每个箱体中好样本所占总样本数的比例
|
|
56
|
+
res['total_rate'] = res['total'] / total
|
|
57
|
+
res['cum_bad_rate'] = res.bad.cumsum() / total_bad
|
|
58
|
+
res['cum_good_rate'] = res.good.cumsum() / (total - total_bad)
|
|
59
|
+
res['ks'] = abs(res.cum_bad_rate - res.cum_good_rate)
|
|
60
|
+
res = res[[
|
|
61
|
+
'min_bin', 'max_bin', 'total',
|
|
62
|
+
'total_rate', 'good', 'bad',
|
|
63
|
+
'bad_rate', 'cum_bad_rate',
|
|
64
|
+
'cum_good_rate', 'ks']]
|
|
65
|
+
return res, res.ks.max()
|
|
66
|
+
|
|
67
|
+
def Continue(
|
|
68
|
+
self,
|
|
69
|
+
variables: Series,
|
|
70
|
+
target: Series,
|
|
71
|
+
bins=10,
|
|
72
|
+
binType='distance'):
|
|
73
|
+
"""
|
|
74
|
+
Des: 连续变量的KS值计算
|
|
75
|
+
:param variables: 连续变量
|
|
76
|
+
:param target: target
|
|
77
|
+
:param bins: 分箱数目
|
|
78
|
+
:param binType: 分箱方式: 等距'distance', 等频'frequency'
|
|
79
|
+
:return: res, ks
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
variables.dropna(inplace=True)
|
|
83
|
+
target.dropna(inplace=True)
|
|
84
|
+
|
|
85
|
+
variablesMin = variables.min()
|
|
86
|
+
variablesMax = variables.max()
|
|
87
|
+
|
|
88
|
+
if not pd.api.types.is_numeric_dtype(variables):
|
|
89
|
+
raise Exception(TypeError, 'variables must be numeric!')
|
|
90
|
+
|
|
91
|
+
if not pd.api.types.is_numeric_dtype(target):
|
|
92
|
+
raise Exception(TypeError, 'target must be numeric!')
|
|
93
|
+
|
|
94
|
+
if variablesMax == variablesMin:
|
|
95
|
+
raise Exception()
|
|
96
|
+
|
|
97
|
+
if binType == 'distance':
|
|
98
|
+
self.cutoffPoint = self._cutoffPoint(
|
|
99
|
+
bins=bins, min=variablesMin, max=variablesMax)
|
|
100
|
+
if binType == 'frequency':
|
|
101
|
+
self.cutoffPoint = self._cutoffPoint(
|
|
102
|
+
bins=bins, percent=True, expected=variables)
|
|
103
|
+
|
|
104
|
+
self.res, self.ks = self._ks(variables, target)
|
|
105
|
+
|
|
106
|
+
if self.csvName:
|
|
107
|
+
saveCsv(self.res, self.csvName)
|
|
108
|
+
return self.ks
|
|
109
|
+
else:
|
|
110
|
+
return self.res, self.ks
|
|
111
|
+
|
|
112
|
+
def Category(self, variables: Series, target: Series):
|
|
113
|
+
"""
|
|
114
|
+
Des: 对于类别型变量的KS值计算
|
|
115
|
+
:param variables: 类别变量
|
|
116
|
+
:param target: target
|
|
117
|
+
:return: res, ks
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
variables = variables.dropna()
|
|
121
|
+
target = target.dropna()
|
|
122
|
+
|
|
123
|
+
if (not is_string_dtype(variables)) or (not is_object_dtype(variables)):
|
|
124
|
+
raise Exception(TypeError, 'variables must be string or object!')
|
|
125
|
+
if not is_numeric_dtype(target):
|
|
126
|
+
raise Exception(TypeError, 'target must be numeric!')
|
|
127
|
+
|
|
128
|
+
self.res, self.ks = self._ks(variables, target, category=True)
|
|
129
|
+
|
|
130
|
+
if self.csvName:
|
|
131
|
+
saveCsv(self.res, self.csvName)
|
|
132
|
+
return self.ks
|
|
133
|
+
else:
|
|
134
|
+
return self.res, self.ks
|
|
135
|
+
|
|
136
|
+
def KSPlot(self, **kwargs):
|
|
137
|
+
"""
|
|
138
|
+
Des: 绘制 KS 曲线
|
|
139
|
+
:param kwargs:
|
|
140
|
+
# 图像样式
|
|
141
|
+
style[str] = 'seaborn',
|
|
142
|
+
# 宽,高
|
|
143
|
+
figsize[tuple] = (9, 5),
|
|
144
|
+
# 分辨率
|
|
145
|
+
dpi[int] = 150,
|
|
146
|
+
# x 轴名称
|
|
147
|
+
x_label[str] = 'Bucket',
|
|
148
|
+
# label 字体大小
|
|
149
|
+
labelfontsize[int] = 10,
|
|
150
|
+
# tick 字体大小
|
|
151
|
+
tickfontsize[int] = 8,
|
|
152
|
+
# title 字体大小
|
|
153
|
+
titlefontsize[int] = 12,
|
|
154
|
+
:return: plt.show() or None
|
|
155
|
+
"""
|
|
156
|
+
from numpy import argmax
|
|
157
|
+
|
|
158
|
+
kwargs.setdefault('style', 'seaborn')
|
|
159
|
+
kwargs.setdefault('figsize', (9, 5))
|
|
160
|
+
kwargs.setdefault('dpi', 150)
|
|
161
|
+
kwargs.setdefault('labelfontsize', 10)
|
|
162
|
+
kwargs.setdefault('tickfontsize', 8)
|
|
163
|
+
kwargs.setdefault('titlefontsize', 12)
|
|
164
|
+
kwargs.setdefault('x_label', 'Bucket')
|
|
165
|
+
|
|
166
|
+
markLineData = self.res[self.res.ks == self.ks]
|
|
167
|
+
|
|
168
|
+
cumGood, cumBad = float(
|
|
169
|
+
markLineData['cum_good_rate']), float(
|
|
170
|
+
markLineData['cum_bad_rate'])
|
|
171
|
+
|
|
172
|
+
if isinstance(self.res.min_bin.to_list()[0], int):
|
|
173
|
+
group = self.res.min_bin.astype(
|
|
174
|
+
str) + '-' + self.res.max_bin.astype(str)
|
|
175
|
+
ksBinMin, ksBinMax = int(
|
|
176
|
+
markLineData['min_bin']), int(
|
|
177
|
+
markLineData['max_bin'])
|
|
178
|
+
|
|
179
|
+
if isinstance(self.res.min_bin.to_list()[0], float):
|
|
180
|
+
group = self.res.min_bin.round(2).astype(
|
|
181
|
+
str) + '-' + self.res.max_bin.round(2).astype(str)
|
|
182
|
+
ksBinMin, ksBinMax = float(
|
|
183
|
+
markLineData['min_bin'].round(2)), float(
|
|
184
|
+
markLineData['max_bin'].round(2))
|
|
185
|
+
|
|
186
|
+
bucket = '[' + str(ksBinMin) + ', ' + str(ksBinMax) + ']'
|
|
187
|
+
|
|
188
|
+
import matplotlib.pyplot as plt
|
|
189
|
+
plt.style.use(kwargs.get('style'))
|
|
190
|
+
fig, axs = plt.subplots(
|
|
191
|
+
figsize=kwargs.get('figsize'),
|
|
192
|
+
dpi=kwargs.get('dpi')
|
|
193
|
+
)
|
|
194
|
+
axs.plot(
|
|
195
|
+
group,
|
|
196
|
+
self.res.ks * 100,
|
|
197
|
+
marker='o',
|
|
198
|
+
linewidth=3,
|
|
199
|
+
c='tab:blue')
|
|
200
|
+
axs.plot(
|
|
201
|
+
group,
|
|
202
|
+
self.res.cum_good_rate *
|
|
203
|
+
100,
|
|
204
|
+
marker='o',
|
|
205
|
+
linewidth=3,
|
|
206
|
+
c='tab:green')
|
|
207
|
+
axs.plot(
|
|
208
|
+
group,
|
|
209
|
+
self.res.cum_bad_rate *
|
|
210
|
+
100,
|
|
211
|
+
marker='o',
|
|
212
|
+
linewidth=3,
|
|
213
|
+
c='tab:red')
|
|
214
|
+
|
|
215
|
+
# 参考线
|
|
216
|
+
axs.axvline(x=argmax(self.res.ks.to_list()), linestyle='--', c='gray')
|
|
217
|
+
axs.axhline(y=self.ks * 100, linestyle='--', c='tab:blue')
|
|
218
|
+
axs.axhline(y=cumGood * 100, linestyle='--', c='tab:green')
|
|
219
|
+
axs.axhline(y=cumBad * 100, linestyle='--', c='tab:red')
|
|
220
|
+
|
|
221
|
+
axs.set_title(
|
|
222
|
+
f'KS = {self.ks * 100: .2f}% at bucket {bucket}',
|
|
223
|
+
fontsize=kwargs.get('titlefontsize'))
|
|
224
|
+
axs.legend(fontsize=kwargs.get('tickfontsize'))
|
|
225
|
+
|
|
226
|
+
axs.set_xlabel(
|
|
227
|
+
kwargs.get('x_label'),
|
|
228
|
+
fontsize=kwargs.get('labelfontsize'),
|
|
229
|
+
)
|
|
230
|
+
axs.set_ylabel(
|
|
231
|
+
r'$Rate(\%)$',
|
|
232
|
+
fontsize=kwargs.get('labelfontsize'),)
|
|
233
|
+
axs.tick_params(axis="x", labelsize=kwargs.get('tickfontsize'))
|
|
234
|
+
axs.tick_params(axis="y", labelsize=kwargs.get('tickfontsize'))
|
|
235
|
+
axs.grid(True)
|
|
236
|
+
return plt.show()
|
|
237
|
+
|
|
238
|
+
def KSValue(self, variables, target):
|
|
239
|
+
"""
|
|
240
|
+
Des: 计算KS值,计算KS-test的P-value
|
|
241
|
+
:param variables:
|
|
242
|
+
:param target:
|
|
243
|
+
:return: ks_value, p_value
|
|
244
|
+
"""
|
|
245
|
+
from scipy.stats import ks_2samp
|
|
246
|
+
get_ks = lambda variables, target: ks_2samp(variables[target == 1], variables[target == 0])
|
|
247
|
+
ks_value, p_value = get_ks(variables, target)
|
|
248
|
+
return ks_value, p_value
|
|
249
|
+
|
|
250
|
+
def ROCPlot(self, **kwargs):
|
|
251
|
+
"""
|
|
252
|
+
Des: 根据KS计算ROC曲线
|
|
253
|
+
:param kwargs: 图像参数['style', 'figsize', 'dpi', 'labelfontsize', title, 'x_label', 'y_label'……]
|
|
254
|
+
:return: None
|
|
255
|
+
"""
|
|
256
|
+
|
|
257
|
+
kwargs.setdefault('style', 'seaborn')
|
|
258
|
+
kwargs.setdefault('figsize', (9, 5))
|
|
259
|
+
kwargs.setdefault('dpi', 150)
|
|
260
|
+
kwargs.setdefault('labelfontsize', 10)
|
|
261
|
+
kwargs.setdefault('tickfontsize', 8)
|
|
262
|
+
kwargs.setdefault('titlefontsize', 12)
|
|
263
|
+
kwargs.setdefault('x_label', 'False Positive Rate')
|
|
264
|
+
kwargs.setdefault('y_label', 'True Positive Rate')
|
|
265
|
+
kwargs.setdefault('title', 'Receiver operating characteristic')
|
|
266
|
+
|
|
267
|
+
import matplotlib.pyplot as plt
|
|
268
|
+
from numpy import abs, argmax
|
|
269
|
+
plt.style.use(kwargs.get('style'))
|
|
270
|
+
fig, axs = plt.subplots(
|
|
271
|
+
figsize=kwargs.get('figsize'),
|
|
272
|
+
dpi=kwargs.get('dpi')
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
FPR = self.res.cum_good_rate.tolist()
|
|
276
|
+
TPR = self.res.cum_bad_rate.tolist()
|
|
277
|
+
FPR.insert(0, 0)
|
|
278
|
+
TPR.insert(0, 0)
|
|
279
|
+
axs.plot(FPR, TPR, marker='o',
|
|
280
|
+
linewidth=3,
|
|
281
|
+
c='tab:green',
|
|
282
|
+
label='$ROC\ curve$',
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
axs.plot([0, 1], [0, 1], linestyle='--', c='gray')
|
|
286
|
+
|
|
287
|
+
maxIndex = argmax(abs(self.res.cum_bad_rate -
|
|
288
|
+
self.res.cum_good_rate).tolist())
|
|
289
|
+
axs.scatter(
|
|
290
|
+
self.res.cum_good_rate[maxIndex],
|
|
291
|
+
self.res.cum_bad_rate[maxIndex],
|
|
292
|
+
alpha=0.8,
|
|
293
|
+
marker='*',
|
|
294
|
+
s=100,
|
|
295
|
+
color='tab:red',
|
|
296
|
+
label='$KS\ max\|cum\_bad\_rate-cum\_good\_rate\|$'
|
|
297
|
+
).set_zorder(10)
|
|
298
|
+
|
|
299
|
+
axs.set_title(
|
|
300
|
+
kwargs.get('title'),
|
|
301
|
+
fontsize=kwargs.get('titlefontsize'))
|
|
302
|
+
axs.legend(fontsize=kwargs.get('tickfontsize'))
|
|
303
|
+
|
|
304
|
+
axs.set_xlabel(
|
|
305
|
+
kwargs.get('x_label'),
|
|
306
|
+
fontsize=kwargs.get('labelfontsize'),
|
|
307
|
+
)
|
|
308
|
+
axs.set_ylabel(
|
|
309
|
+
kwargs.get('y_label'),
|
|
310
|
+
fontsize=kwargs.get('labelfontsize'),
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
if self.imageName:
|
|
314
|
+
if not os.path.exists('img'):
|
|
315
|
+
os.mkdir('img')
|
|
316
|
+
plt.savefig(f'./img/{self.imageName}_roc.png', )
|
|
317
|
+
return None
|
|
318
|
+
else:
|
|
319
|
+
return plt.show()
|
|
320
|
+
|
|
321
|
+
def Distribution(self, data: dict, **kwargs):
|
|
322
|
+
"""
|
|
323
|
+
Des: 绘制多个样本的分布情况
|
|
324
|
+
:param data: 数据
|
|
325
|
+
:param kwargs: 图像参数['style', 'figsize', 'dpi', 'labelfontsize', title, 'x_label', 'y_label'……]
|
|
326
|
+
:return: plt.plot
|
|
327
|
+
"""
|
|
328
|
+
|
|
329
|
+
kwargs.setdefault('style', 'seaborn')
|
|
330
|
+
kwargs.setdefault('figsize', (9, 5))
|
|
331
|
+
kwargs.setdefault('dpi', 150)
|
|
332
|
+
kwargs.setdefault('labelfontsize', 10)
|
|
333
|
+
kwargs.setdefault('tickfontsize', 8)
|
|
334
|
+
kwargs.setdefault('titlefontsize', 12)
|
|
335
|
+
kwargs.setdefault('x_label', 'Variable')
|
|
336
|
+
kwargs.setdefault('y_label', 'Frequency')
|
|
337
|
+
kwargs.setdefault('title', 'Distribution')
|
|
338
|
+
|
|
339
|
+
import seaborn as sns
|
|
340
|
+
import matplotlib.pyplot as plt
|
|
341
|
+
|
|
342
|
+
plt.style.use(kwargs.get('style'))
|
|
343
|
+
|
|
344
|
+
fig, axs = plt.subplots(
|
|
345
|
+
figsize=kwargs.get('figsize'),
|
|
346
|
+
dpi=kwargs.get('dpi')
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
with sns.color_palette("muted", n_colors=len(data)):
|
|
350
|
+
for name, item in data.items():
|
|
351
|
+
sns.distplot(item, label=name)
|
|
352
|
+
|
|
353
|
+
axs.legend()
|
|
354
|
+
|
|
355
|
+
axs.set_title(
|
|
356
|
+
kwargs.get('title'),
|
|
357
|
+
fontsize=kwargs.get('titlefontsize'))
|
|
358
|
+
|
|
359
|
+
axs.set_xlabel(
|
|
360
|
+
kwargs.get('x_label'),
|
|
361
|
+
fontsize=kwargs.get('labelfontsize'),
|
|
362
|
+
)
|
|
363
|
+
axs.set_ylabel(
|
|
364
|
+
kwargs.get('y_label'),
|
|
365
|
+
fontsize=kwargs.get('labelfontsize'),
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
if self.imageName:
|
|
369
|
+
if not os.path.exists('img'):
|
|
370
|
+
os.mkdir('img')
|
|
371
|
+
plt.savefig(f'./img/{self.imageName}_distribution.png', )
|
|
372
|
+
return None
|
|
373
|
+
else:
|
|
374
|
+
return plt.show()
|