riskvalidation 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- riskvalidation/__init__.py +30 -0
- riskvalidation/aggregation/__init__.py +11 -0
- riskvalidation/aggregation/band.py +151 -0
- riskvalidation/aggregation/bounds.py +191 -0
- riskvalidation/aggregation/copulas.py +241 -0
- riskvalidation/aggregation/model_risk.py +133 -0
- riskvalidation/aggregation/simple.py +147 -0
- riskvalidation/ccr/__init__.py +13 -0
- riskvalidation/ccr/collateral.py +175 -0
- riskvalidation/ccr/cva.py +193 -0
- riskvalidation/ccr/exposure.py +192 -0
- riskvalidation/ccr/hw1f.py +426 -0
- riskvalidation/ccr/swaps.py +172 -0
- riskvalidation/core/__init__.py +1 -0
- riskvalidation/core/build.py +57 -0
- riskvalidation/core/hashing.py +48 -0
- riskvalidation/core/policy.py +130 -0
- riskvalidation/core/registry.py +65 -0
- riskvalidation/core/result.py +117 -0
- riskvalidation/core/suite.py +78 -0
- riskvalidation/engines/__init__.py +2 -0
- riskvalidation/engines/low_default.py +170 -0
- riskvalidation/engines/pd_curve.py +529 -0
- riskvalidation/engines/vasicek.py +72 -0
- riskvalidation/fairness/__init__.py +10 -0
- riskvalidation/fairness/_checks.py +115 -0
- riskvalidation/fairness/conditional.py +201 -0
- riskvalidation/fairness/inference.py +64 -0
- riskvalidation/fairness/metrics.py +261 -0
- riskvalidation/fairness/plants.py +240 -0
- riskvalidation/fairness/policy.py +97 -0
- riskvalidation/fairness/replication.py +39 -0
- riskvalidation/fairness/situation.py +307 -0
- riskvalidation/generators/__init__.py +68 -0
- riskvalidation/generators/_base.py +145 -0
- riskvalidation/generators/counts.py +232 -0
- riskvalidation/generators/drift.py +107 -0
- riskvalidation/generators/framingham.py +124 -0
- riskvalidation/generators/paths.py +349 -0
- riskvalidation/generators/ratings.py +186 -0
- riskvalidation/generators/scores.py +347 -0
- riskvalidation/harness/__init__.py +46 -0
- riskvalidation/harness/exact.py +137 -0
- riskvalidation/harness/performance.py +74 -0
- riskvalidation/harness/rates.py +103 -0
- riskvalidation/harness/run.py +197 -0
- riskvalidation/irrbb/__init__.py +9 -0
- riskvalidation/irrbb/behavioural.py +206 -0
- riskvalidation/irrbb/curves.py +286 -0
- riskvalidation/irrbb/eve_nii.py +245 -0
- riskvalidation/irrbb/shocks.py +171 -0
- riskvalidation/irrbb/simulation.py +203 -0
- riskvalidation/irrbb/slotting.py +184 -0
- riskvalidation/lgd_ead/__init__.py +8 -0
- riskvalidation/lgd_ead/_checks.py +50 -0
- riskvalidation/lgd_ead/ccf.py +228 -0
- riskvalidation/lgd_ead/downturn.py +64 -0
- riskvalidation/lgd_ead/floors.py +114 -0
- riskvalidation/lgd_ead/lgd_models.py +297 -0
- riskvalidation/lgd_ead/workout.py +162 -0
- riskvalidation/liquidity/__init__.py +9 -0
- riskvalidation/liquidity/_checks.py +78 -0
- riskvalidation/liquidity/cbc.py +158 -0
- riskvalidation/liquidity/deposits.py +151 -0
- riskvalidation/liquidity/ladder.py +119 -0
- riskvalidation/liquidity/reverse.py +141 -0
- riskvalidation/liquidity/runoff.py +220 -0
- riskvalidation/liquidity/scenarios.py +272 -0
- riskvalidation/market/__init__.py +8 -0
- riskvalidation/market/caviar.py +239 -0
- riskvalidation/market/conventions.py +83 -0
- riskvalidation/market/evt.py +111 -0
- riskvalidation/market/frtb.py +175 -0
- riskvalidation/market/garch.py +204 -0
- riskvalidation/market/gas_fz.py +219 -0
- riskvalidation/market/historical.py +96 -0
- riskvalidation/market/parametric.py +78 -0
- riskvalidation/market/scoring.py +145 -0
- riskvalidation/ml/__init__.py +11 -0
- riskvalidation/ml/_checks.py +67 -0
- riskvalidation/ml/_logistic.py +82 -0
- riskvalidation/ml/cbpe.py +91 -0
- riskvalidation/ml/challenger.py +73 -0
- riskvalidation/ml/conformal.py +116 -0
- riskvalidation/ml/dataquality.py +130 -0
- riskvalidation/ml/effects.py +67 -0
- riskvalidation/ml/explanations.py +125 -0
- riskvalidation/ml/generalisation.py +79 -0
- riskvalidation/ml/leakage.py +99 -0
- riskvalidation/ml/plants.py +425 -0
- riskvalidation/ml/policy.py +182 -0
- riskvalidation/ml/robustness.py +286 -0
- riskvalidation/ml/seeds.py +106 -0
- riskvalidation/oprisk/__init__.py +18 -0
- riskvalidation/oprisk/_checks.py +60 -0
- riskvalidation/oprisk/aggregate.py +375 -0
- riskvalidation/oprisk/distributions.py +758 -0
- riskvalidation/oprisk/scenarios.py +342 -0
- riskvalidation/oprisk/sla.py +80 -0
- riskvalidation/oprisk/sma.py +169 -0
- riskvalidation/policies/__init__.py +4 -0
- riskvalidation/policies/default.json +273 -0
- riskvalidation/portfolio/__init__.py +16 -0
- riskvalidation/portfolio/asrf_fit.py +283 -0
- riskvalidation/portfolio/copula.py +129 -0
- riskvalidation/portfolio/creditrisk_plus.py +232 -0
- riskvalidation/portfolio/euler.py +126 -0
- riskvalidation/portfolio/granularity.py +135 -0
- riskvalidation/portfolio/importance.py +277 -0
- riskvalidation/portfolio/portfolio.py +309 -0
- riskvalidation/provisions/__init__.py +8 -0
- riskvalidation/provisions/_checks.py +100 -0
- riskvalidation/provisions/ecl.py +233 -0
- riskvalidation/provisions/local_cl.py +360 -0
- riskvalidation/provisions/local_es.py +157 -0
- riskvalidation/provisions/overlays.py +102 -0
- riskvalidation/provisions/reconciliation.py +157 -0
- riskvalidation/provisions/staging.py +211 -0
- riskvalidation/provisions/term_structure.py +199 -0
- riskvalidation/regulatory/__init__.py +14 -0
- riskvalidation/regulatory/ba_cva.py +138 -0
- riskvalidation/regulatory/frtb.py +345 -0
- riskvalidation/regulatory/irb.py +314 -0
- riskvalidation/regulatory/lcr_nsfr.py +369 -0
- riskvalidation/regulatory/output_floor.py +45 -0
- riskvalidation/regulatory/sa_ccr.py +486 -0
- riskvalidation/regulatory/standardised.py +348 -0
- riskvalidation/stress/__init__.py +9 -0
- riskvalidation/stress/capital.py +476 -0
- riskvalidation/stress/paths.py +108 -0
- riskvalidation/stress/reverse.py +179 -0
- riskvalidation/stress/satellites.py +268 -0
- riskvalidation/stress/scenario_models.py +274 -0
- riskvalidation/transitions/__init__.py +0 -0
- riskvalidation/transitions/em.py +120 -0
- riskvalidation/transitions/embedding.py +221 -0
- riskvalidation/transitions/estimators.py +213 -0
- riskvalidation/transitions/intervals.py +306 -0
- riskvalidation/transitions/mobility.py +57 -0
- riskvalidation/transitions/ttc.py +171 -0
- riskvalidation/validation/__init__.py +103 -0
- riskvalidation/validation/_common.py +57 -0
- riskvalidation/validation/aggregation.py +425 -0
- riskvalidation/validation/calibration.py +757 -0
- riskvalidation/validation/ccr.py +530 -0
- riskvalidation/validation/discrimination.py +314 -0
- riskvalidation/validation/fairness.py +632 -0
- riskvalidation/validation/ifrs9.py +693 -0
- riskvalidation/validation/irrbb.py +389 -0
- riskvalidation/validation/lgd_ead.py +469 -0
- riskvalidation/validation/liquidity.py +647 -0
- riskvalidation/validation/market.py +807 -0
- riskvalidation/validation/ml.py +958 -0
- riskvalidation/validation/oprisk.py +357 -0
- riskvalidation/validation/portfolio.py +353 -0
- riskvalidation/validation/rating_system.py +245 -0
- riskvalidation/validation/stability.py +256 -0
- riskvalidation/validation/stress.py +552 -0
- riskvalidation/validation/transitions.py +386 -0
- riskvalidation-0.8.0.dist-info/METADATA +136 -0
- riskvalidation-0.8.0.dist-info/RECORD +164 -0
- riskvalidation-0.8.0.dist-info/WHEEL +5 -0
- riskvalidation-0.8.0.dist-info/licenses/LICENSE +21 -0
- riskvalidation-0.8.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
"""Discriminatory power: does the model rank defaulters as riskier than non-defaulters?
|
|
2
|
+
|
|
3
|
+
Inputs are risk-oriented: a higher value means a riskier obligor (a PD, a score oriented to risk, or a rating
|
|
4
|
+
grade number where higher is worse). Ties get half credit, as in the Mann-Whitney statistic.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
from collections.abc import Callable, Sequence
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
from scipy import stats
|
|
14
|
+
|
|
15
|
+
from riskvalidation.core.build import make_result
|
|
16
|
+
from riskvalidation.core.policy import Policy
|
|
17
|
+
from riskvalidation.core.registry import register
|
|
18
|
+
from riskvalidation.core.result import TestResult
|
|
19
|
+
from riskvalidation.validation._common import as_binary, as_float_array
|
|
20
|
+
|
|
21
|
+
ECB_2019 = (
|
|
22
|
+
"ECB (2019), Instructions for reporting the validation results of internal models: IRB Pillar I models for "
|
|
23
|
+
"credit risk"
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _split(risk: Sequence[float], defaults: Sequence[int]) -> tuple[np.ndarray, np.ndarray]:
|
|
28
|
+
r = as_float_array(risk, "risk")
|
|
29
|
+
y = as_binary(defaults)
|
|
30
|
+
if r.shape != y.shape:
|
|
31
|
+
raise ValueError("risk and defaults must have the same length")
|
|
32
|
+
a, b = r[y == 1], r[y == 0]
|
|
33
|
+
if a.size == 0 or b.size == 0:
|
|
34
|
+
raise ValueError("discrimination needs at least one defaulter and one non-defaulter")
|
|
35
|
+
return a, b
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def structural_components(defaulters: np.ndarray, survivors: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
|
39
|
+
"""DeLong placements: V10[a] = share of non-defaulters ranked safer than defaulter a (ties count 1/2);
|
|
40
|
+
V01[b] = share of defaulters ranked riskier than non-defaulter b. Both average to the AUC."""
|
|
41
|
+
sb = np.sort(survivors)
|
|
42
|
+
sa = np.sort(defaulters)
|
|
43
|
+
below = np.searchsorted(sb, defaulters, side="left")
|
|
44
|
+
ties_a = np.searchsorted(sb, defaulters, side="right") - below
|
|
45
|
+
v10 = (below + 0.5 * ties_a) / sb.size
|
|
46
|
+
above = sa.size - np.searchsorted(sa, survivors, side="right")
|
|
47
|
+
ties_b = np.searchsorted(sa, survivors, side="right") - np.searchsorted(sa, survivors, side="left")
|
|
48
|
+
v01 = (above + 0.5 * ties_b) / sa.size
|
|
49
|
+
return v10, v01
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def auc_with_variance(risk: Sequence[float], defaults: Sequence[int]) -> tuple[float, float, int, int]:
|
|
53
|
+
"""AUC and the ECB variance s^2 = var(V10)/|A| + var(V01)/|B| (unbiased sample variances)."""
|
|
54
|
+
a, b = _split(risk, defaults)
|
|
55
|
+
v10, v01 = structural_components(a, b)
|
|
56
|
+
auc = float(v10.mean())
|
|
57
|
+
va = float(np.var(v10, ddof=1)) if a.size > 1 else 0.0
|
|
58
|
+
vb = float(np.var(v01, ddof=1)) if b.size > 1 else 0.0
|
|
59
|
+
return auc, va / a.size + vb / b.size, int(a.size), int(b.size)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def cap_accuracy_ratio(risk: Sequence[float], defaults: Sequence[int]) -> float:
|
|
63
|
+
"""Accuracy ratio from the cumulative accuracy profile, tied groups interpolated linearly.
|
|
64
|
+
|
|
65
|
+
AR = (area under the model CAP - 1/2) / (area under the perfect CAP - 1/2); equals 2 AUC - 1 with the same
|
|
66
|
+
tie convention (BCBS WP14).
|
|
67
|
+
"""
|
|
68
|
+
r = as_float_array(risk, "risk")
|
|
69
|
+
y = as_binary(defaults)
|
|
70
|
+
n, nd = r.size, int(y.sum())
|
|
71
|
+
if nd == 0 or nd == n:
|
|
72
|
+
raise ValueError("the CAP needs defaulters and non-defaulters")
|
|
73
|
+
vals, inv = np.unique(-r, return_inverse=True) # riskiest first
|
|
74
|
+
cnt = np.bincount(inv)
|
|
75
|
+
dft = np.bincount(inv, weights=y)
|
|
76
|
+
x = np.concatenate([[0.0], np.cumsum(cnt) / n])
|
|
77
|
+
h = np.concatenate([[0.0], np.cumsum(dft) / nd])
|
|
78
|
+
area = float(np.sum((x[1:] - x[:-1]) * (h[1:] + h[:-1]) / 2.0))
|
|
79
|
+
pi = nd / n
|
|
80
|
+
perfect = 1.0 - pi / 2.0
|
|
81
|
+
return (area - 0.5) / (perfect - 0.5)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@register(
|
|
85
|
+
"disc.auc",
|
|
86
|
+
family="discrimination",
|
|
87
|
+
name="Area under the ROC curve",
|
|
88
|
+
h0="descriptive: probability that a defaulter is ranked riskier than a non-defaulter (ties count one half)",
|
|
89
|
+
reference=f"{ECB_2019}, Annex 3.1; Hanley and McNeil (1982), doi:10.1148/radiology.143.1.7063747",
|
|
90
|
+
decision="metric",
|
|
91
|
+
)
|
|
92
|
+
def disc_auc(
|
|
93
|
+
risk: Sequence[float],
|
|
94
|
+
defaults: Sequence[int],
|
|
95
|
+
*,
|
|
96
|
+
policy: Policy | None = None,
|
|
97
|
+
model_id: str | None = None,
|
|
98
|
+
segment: str | None = None,
|
|
99
|
+
) -> TestResult:
|
|
100
|
+
"""AUC with the ECB standard error, the Gini/accuracy ratio 2 AUC - 1 and a normal 95% interval."""
|
|
101
|
+
auc, s2, na, nb = auc_with_variance(risk, defaults)
|
|
102
|
+
se = math.sqrt(s2)
|
|
103
|
+
return make_result(
|
|
104
|
+
"disc.auc",
|
|
105
|
+
metric=auc,
|
|
106
|
+
policy=policy,
|
|
107
|
+
n=na + nb,
|
|
108
|
+
n_events=na,
|
|
109
|
+
model_id=model_id,
|
|
110
|
+
segment=segment,
|
|
111
|
+
inputs=(np.asarray(risk, float), np.asarray(defaults)),
|
|
112
|
+
extras={
|
|
113
|
+
"gini": 2.0 * auc - 1.0,
|
|
114
|
+
"s2": s2,
|
|
115
|
+
"se": se,
|
|
116
|
+
"ci95_low": max(0.0, auc - 1.959963984540054 * se),
|
|
117
|
+
"ci95_high": min(1.0, auc + 1.959963984540054 * se),
|
|
118
|
+
},
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@register(
|
|
123
|
+
"disc.auc_vs_initial",
|
|
124
|
+
family="discrimination",
|
|
125
|
+
name="AUC against the AUC at initial validation",
|
|
126
|
+
h0="the AUC at development is smaller than the AUC in the observation period",
|
|
127
|
+
reference=f"{ECB_2019}, section 2.5.4.1 and Annex 3.1",
|
|
128
|
+
decision="p_value",
|
|
129
|
+
)
|
|
130
|
+
def disc_auc_vs_initial(
|
|
131
|
+
auc_initial: float,
|
|
132
|
+
risk: Sequence[float],
|
|
133
|
+
defaults: Sequence[int],
|
|
134
|
+
*,
|
|
135
|
+
auc_initial_variance: float = 0.0,
|
|
136
|
+
policy: Policy | None = None,
|
|
137
|
+
model_id: str | None = None,
|
|
138
|
+
segment: str | None = None,
|
|
139
|
+
) -> TestResult:
|
|
140
|
+
"""S = (AUC_init - AUC_curr) / s, p = 1 - Phi(S); the ECB treats the development AUC as a known number.
|
|
141
|
+
|
|
142
|
+
When the development AUC was itself estimated on a sample independent of the current one, pass its variance (the
|
|
143
|
+
``s2`` of :func:`auc_with_variance` on the development sample) as ``auc_initial_variance``: the statistic then uses
|
|
144
|
+
sqrt(s^2 + s_init^2), the variance of the difference of two independent estimates. This extends the ECB formula
|
|
145
|
+
and the result says so. Measured with known-truth generators (riskvalidation 0.03.000), the ECB form rejects about
|
|
146
|
+
11% of the time at the 5% level when the development sample is as large as the current one and the AUC has not
|
|
147
|
+
changed; the extended form holds the size."""
|
|
148
|
+
if not 0.0 <= auc_initial <= 1.0:
|
|
149
|
+
raise ValueError("the initial AUC must lie in [0, 1]")
|
|
150
|
+
if not (math.isfinite(auc_initial_variance) and auc_initial_variance >= 0.0):
|
|
151
|
+
raise ValueError("the variance of the initial AUC must be a non-negative number")
|
|
152
|
+
auc, s2_cur, na, nb = auc_with_variance(risk, defaults)
|
|
153
|
+
s2 = s2_cur + auc_initial_variance
|
|
154
|
+
if s2 <= 0.0:
|
|
155
|
+
return make_result(
|
|
156
|
+
"disc.auc_vs_initial",
|
|
157
|
+
statistic=float("nan"),
|
|
158
|
+
policy=policy,
|
|
159
|
+
n=na + nb,
|
|
160
|
+
n_events=na,
|
|
161
|
+
model_id=model_id,
|
|
162
|
+
segment=segment,
|
|
163
|
+
notes="the variance of the current AUC is zero; the statistic is undefined",
|
|
164
|
+
extras={"auc_initial": auc_initial, "auc_current": auc},
|
|
165
|
+
)
|
|
166
|
+
s = (auc_initial - auc) / math.sqrt(s2)
|
|
167
|
+
extended = auc_initial_variance > 0.0
|
|
168
|
+
return make_result(
|
|
169
|
+
"disc.auc_vs_initial",
|
|
170
|
+
statistic=s,
|
|
171
|
+
p_value=float(stats.norm.sf(s)),
|
|
172
|
+
policy=policy,
|
|
173
|
+
alternative="discrimination deteriorated since development",
|
|
174
|
+
n=na + nb,
|
|
175
|
+
n_events=na,
|
|
176
|
+
model_id=model_id,
|
|
177
|
+
segment=segment,
|
|
178
|
+
inputs=(auc_initial, np.asarray(risk, float), np.asarray(defaults), auc_initial_variance),
|
|
179
|
+
notes="the development AUC's variance is added to the ECB statistic (an extension of the ECB 2019 formula): "
|
|
180
|
+
"both AUCs are estimates on independent samples" if extended else "",
|
|
181
|
+
extras={"auc_initial": auc_initial, "auc_current": auc, "s2": s2_cur,
|
|
182
|
+
"auc_initial_variance": auc_initial_variance, "s2_total": s2},
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@register(
|
|
187
|
+
"disc.delong",
|
|
188
|
+
family="discrimination",
|
|
189
|
+
name="DeLong test of two correlated AUCs",
|
|
190
|
+
h0="the two models have the same AUC on the same obligors",
|
|
191
|
+
reference="DeLong, DeLong and Clarke-Pearson (1988), Biometrics 44(3), doi:10.2307/2531595",
|
|
192
|
+
decision="p_value",
|
|
193
|
+
)
|
|
194
|
+
def disc_delong(
|
|
195
|
+
risk_1: Sequence[float],
|
|
196
|
+
risk_2: Sequence[float],
|
|
197
|
+
defaults: Sequence[int],
|
|
198
|
+
*,
|
|
199
|
+
policy: Policy | None = None,
|
|
200
|
+
model_id: str | None = None,
|
|
201
|
+
segment: str | None = None,
|
|
202
|
+
) -> TestResult:
|
|
203
|
+
"""Paired test for a champion and a challenger scored on the same sample; two-sided."""
|
|
204
|
+
y = as_binary(defaults)
|
|
205
|
+
r1 = as_float_array(risk_1, "risk_1")
|
|
206
|
+
r2 = as_float_array(risk_2, "risk_2")
|
|
207
|
+
if not (r1.shape == r2.shape == y.shape):
|
|
208
|
+
raise ValueError("both risk vectors and defaults must have the same length")
|
|
209
|
+
comps = [structural_components(r[y == 1], r[y == 0]) for r in (r1, r2)]
|
|
210
|
+
na, nb = int(y.sum()), int(y.size - y.sum())
|
|
211
|
+
if na < 2 or nb < 2:
|
|
212
|
+
raise ValueError("the DeLong test needs at least two defaulters and two non-defaulters")
|
|
213
|
+
v10 = np.vstack([c[0] for c in comps])
|
|
214
|
+
v01 = np.vstack([c[1] for c in comps])
|
|
215
|
+
s = np.cov(v10, ddof=1) / na + np.cov(v01, ddof=1) / nb
|
|
216
|
+
auc1, auc2 = float(v10[0].mean()), float(v10[1].mean())
|
|
217
|
+
var = float(s[0, 0] + s[1, 1] - 2.0 * s[0, 1])
|
|
218
|
+
extras = {"auc_1": auc1, "auc_2": auc2, "difference": auc1 - auc2, "variance": var}
|
|
219
|
+
if var <= 0.0:
|
|
220
|
+
return make_result(
|
|
221
|
+
"disc.delong",
|
|
222
|
+
statistic=float("nan"),
|
|
223
|
+
policy=policy,
|
|
224
|
+
n=int(y.size),
|
|
225
|
+
n_events=na,
|
|
226
|
+
model_id=model_id,
|
|
227
|
+
segment=segment,
|
|
228
|
+
notes="the variance of the AUC difference is not positive (identical rankings); undefined",
|
|
229
|
+
extras=extras,
|
|
230
|
+
)
|
|
231
|
+
z = (auc1 - auc2) / math.sqrt(var)
|
|
232
|
+
return make_result(
|
|
233
|
+
"disc.delong",
|
|
234
|
+
statistic=z,
|
|
235
|
+
p_value=float(2.0 * stats.norm.sf(abs(z))),
|
|
236
|
+
policy=policy,
|
|
237
|
+
alternative="the AUCs differ",
|
|
238
|
+
n=int(y.size),
|
|
239
|
+
n_events=na,
|
|
240
|
+
model_id=model_id,
|
|
241
|
+
segment=segment,
|
|
242
|
+
inputs=(r1, r2, y),
|
|
243
|
+
extras=extras,
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
@register(
|
|
248
|
+
"disc.ks",
|
|
249
|
+
family="discrimination",
|
|
250
|
+
name="Kolmogorov-Smirnov separation",
|
|
251
|
+
h0="descriptive: maximum distance between the risk distributions of defaulters and non-defaulters",
|
|
252
|
+
reference="BCBS (2005), Working Paper 14; two-sample Kolmogorov-Smirnov statistic (SciPy ks_2samp)",
|
|
253
|
+
decision="metric",
|
|
254
|
+
)
|
|
255
|
+
def disc_ks(
|
|
256
|
+
risk: Sequence[float],
|
|
257
|
+
defaults: Sequence[int],
|
|
258
|
+
*,
|
|
259
|
+
policy: Policy | None = None,
|
|
260
|
+
model_id: str | None = None,
|
|
261
|
+
segment: str | None = None,
|
|
262
|
+
) -> TestResult:
|
|
263
|
+
"""KS separation; its p-value tests identical distributions, which is rarely the question in validation."""
|
|
264
|
+
a, b = _split(risk, defaults)
|
|
265
|
+
res = stats.ks_2samp(a, b)
|
|
266
|
+
return make_result(
|
|
267
|
+
"disc.ks",
|
|
268
|
+
metric=float(res.statistic),
|
|
269
|
+
p_value=float(res.pvalue),
|
|
270
|
+
policy=policy,
|
|
271
|
+
n=int(a.size + b.size),
|
|
272
|
+
n_events=int(a.size),
|
|
273
|
+
model_id=model_id,
|
|
274
|
+
segment=segment,
|
|
275
|
+
inputs=(a, b),
|
|
276
|
+
notes="the p-value tests H0 of identical distributions (no discrimination at all)",
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def bootstrap_ci(
|
|
281
|
+
statistic: Callable[[np.ndarray, np.ndarray], float],
|
|
282
|
+
risk: Sequence[float],
|
|
283
|
+
defaults: Sequence[int],
|
|
284
|
+
*,
|
|
285
|
+
n_boot: int = 2000,
|
|
286
|
+
level: float = 0.95,
|
|
287
|
+
seed: int = 0,
|
|
288
|
+
) -> tuple[float, float]:
|
|
289
|
+
"""Percentile interval from a bootstrap stratified by default status (defaulters and non-defaulters are
|
|
290
|
+
resampled separately, so every replicate keeps the observed default count)."""
|
|
291
|
+
r = as_float_array(risk, "risk")
|
|
292
|
+
y = as_binary(defaults)
|
|
293
|
+
ia, ib = np.flatnonzero(y == 1), np.flatnonzero(y == 0)
|
|
294
|
+
if ia.size == 0 or ib.size == 0:
|
|
295
|
+
raise ValueError("the stratified bootstrap needs both classes")
|
|
296
|
+
rng = np.random.default_rng(seed)
|
|
297
|
+
vals = np.empty(n_boot)
|
|
298
|
+
for k in range(n_boot):
|
|
299
|
+
idx = np.concatenate([rng.choice(ia, ia.size), rng.choice(ib, ib.size)])
|
|
300
|
+
vals[k] = statistic(r[idx], y[idx])
|
|
301
|
+
lo, hi = np.quantile(vals, [(1.0 - level) / 2.0, (1.0 + level) / 2.0])
|
|
302
|
+
return float(lo), float(hi)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
__all__ = [
|
|
306
|
+
"auc_with_variance",
|
|
307
|
+
"bootstrap_ci",
|
|
308
|
+
"cap_accuracy_ratio",
|
|
309
|
+
"disc_auc",
|
|
310
|
+
"disc_auc_vs_initial",
|
|
311
|
+
"disc_delong",
|
|
312
|
+
"disc_ks",
|
|
313
|
+
"structural_components",
|
|
314
|
+
]
|