riskvalidation 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- riskvalidation/__init__.py +30 -0
- riskvalidation/aggregation/__init__.py +11 -0
- riskvalidation/aggregation/band.py +151 -0
- riskvalidation/aggregation/bounds.py +191 -0
- riskvalidation/aggregation/copulas.py +241 -0
- riskvalidation/aggregation/model_risk.py +133 -0
- riskvalidation/aggregation/simple.py +147 -0
- riskvalidation/ccr/__init__.py +13 -0
- riskvalidation/ccr/collateral.py +175 -0
- riskvalidation/ccr/cva.py +193 -0
- riskvalidation/ccr/exposure.py +192 -0
- riskvalidation/ccr/hw1f.py +426 -0
- riskvalidation/ccr/swaps.py +172 -0
- riskvalidation/core/__init__.py +1 -0
- riskvalidation/core/build.py +57 -0
- riskvalidation/core/hashing.py +48 -0
- riskvalidation/core/policy.py +130 -0
- riskvalidation/core/registry.py +65 -0
- riskvalidation/core/result.py +117 -0
- riskvalidation/core/suite.py +78 -0
- riskvalidation/engines/__init__.py +2 -0
- riskvalidation/engines/low_default.py +170 -0
- riskvalidation/engines/pd_curve.py +529 -0
- riskvalidation/engines/vasicek.py +72 -0
- riskvalidation/fairness/__init__.py +10 -0
- riskvalidation/fairness/_checks.py +115 -0
- riskvalidation/fairness/conditional.py +201 -0
- riskvalidation/fairness/inference.py +64 -0
- riskvalidation/fairness/metrics.py +261 -0
- riskvalidation/fairness/plants.py +240 -0
- riskvalidation/fairness/policy.py +97 -0
- riskvalidation/fairness/replication.py +39 -0
- riskvalidation/fairness/situation.py +307 -0
- riskvalidation/generators/__init__.py +68 -0
- riskvalidation/generators/_base.py +145 -0
- riskvalidation/generators/counts.py +232 -0
- riskvalidation/generators/drift.py +107 -0
- riskvalidation/generators/framingham.py +124 -0
- riskvalidation/generators/paths.py +349 -0
- riskvalidation/generators/ratings.py +186 -0
- riskvalidation/generators/scores.py +347 -0
- riskvalidation/harness/__init__.py +46 -0
- riskvalidation/harness/exact.py +137 -0
- riskvalidation/harness/performance.py +74 -0
- riskvalidation/harness/rates.py +103 -0
- riskvalidation/harness/run.py +197 -0
- riskvalidation/irrbb/__init__.py +9 -0
- riskvalidation/irrbb/behavioural.py +206 -0
- riskvalidation/irrbb/curves.py +286 -0
- riskvalidation/irrbb/eve_nii.py +245 -0
- riskvalidation/irrbb/shocks.py +171 -0
- riskvalidation/irrbb/simulation.py +203 -0
- riskvalidation/irrbb/slotting.py +184 -0
- riskvalidation/lgd_ead/__init__.py +8 -0
- riskvalidation/lgd_ead/_checks.py +50 -0
- riskvalidation/lgd_ead/ccf.py +228 -0
- riskvalidation/lgd_ead/downturn.py +64 -0
- riskvalidation/lgd_ead/floors.py +114 -0
- riskvalidation/lgd_ead/lgd_models.py +297 -0
- riskvalidation/lgd_ead/workout.py +162 -0
- riskvalidation/liquidity/__init__.py +9 -0
- riskvalidation/liquidity/_checks.py +78 -0
- riskvalidation/liquidity/cbc.py +158 -0
- riskvalidation/liquidity/deposits.py +151 -0
- riskvalidation/liquidity/ladder.py +119 -0
- riskvalidation/liquidity/reverse.py +141 -0
- riskvalidation/liquidity/runoff.py +220 -0
- riskvalidation/liquidity/scenarios.py +272 -0
- riskvalidation/market/__init__.py +8 -0
- riskvalidation/market/caviar.py +239 -0
- riskvalidation/market/conventions.py +83 -0
- riskvalidation/market/evt.py +111 -0
- riskvalidation/market/frtb.py +175 -0
- riskvalidation/market/garch.py +204 -0
- riskvalidation/market/gas_fz.py +219 -0
- riskvalidation/market/historical.py +96 -0
- riskvalidation/market/parametric.py +78 -0
- riskvalidation/market/scoring.py +145 -0
- riskvalidation/ml/__init__.py +11 -0
- riskvalidation/ml/_checks.py +67 -0
- riskvalidation/ml/_logistic.py +82 -0
- riskvalidation/ml/cbpe.py +91 -0
- riskvalidation/ml/challenger.py +73 -0
- riskvalidation/ml/conformal.py +116 -0
- riskvalidation/ml/dataquality.py +130 -0
- riskvalidation/ml/effects.py +67 -0
- riskvalidation/ml/explanations.py +125 -0
- riskvalidation/ml/generalisation.py +79 -0
- riskvalidation/ml/leakage.py +99 -0
- riskvalidation/ml/plants.py +425 -0
- riskvalidation/ml/policy.py +182 -0
- riskvalidation/ml/robustness.py +286 -0
- riskvalidation/ml/seeds.py +106 -0
- riskvalidation/oprisk/__init__.py +18 -0
- riskvalidation/oprisk/_checks.py +60 -0
- riskvalidation/oprisk/aggregate.py +375 -0
- riskvalidation/oprisk/distributions.py +758 -0
- riskvalidation/oprisk/scenarios.py +342 -0
- riskvalidation/oprisk/sla.py +80 -0
- riskvalidation/oprisk/sma.py +169 -0
- riskvalidation/policies/__init__.py +4 -0
- riskvalidation/policies/default.json +273 -0
- riskvalidation/portfolio/__init__.py +16 -0
- riskvalidation/portfolio/asrf_fit.py +283 -0
- riskvalidation/portfolio/copula.py +129 -0
- riskvalidation/portfolio/creditrisk_plus.py +232 -0
- riskvalidation/portfolio/euler.py +126 -0
- riskvalidation/portfolio/granularity.py +135 -0
- riskvalidation/portfolio/importance.py +277 -0
- riskvalidation/portfolio/portfolio.py +309 -0
- riskvalidation/provisions/__init__.py +8 -0
- riskvalidation/provisions/_checks.py +100 -0
- riskvalidation/provisions/ecl.py +233 -0
- riskvalidation/provisions/local_cl.py +360 -0
- riskvalidation/provisions/local_es.py +157 -0
- riskvalidation/provisions/overlays.py +102 -0
- riskvalidation/provisions/reconciliation.py +157 -0
- riskvalidation/provisions/staging.py +211 -0
- riskvalidation/provisions/term_structure.py +199 -0
- riskvalidation/regulatory/__init__.py +14 -0
- riskvalidation/regulatory/ba_cva.py +138 -0
- riskvalidation/regulatory/frtb.py +345 -0
- riskvalidation/regulatory/irb.py +314 -0
- riskvalidation/regulatory/lcr_nsfr.py +369 -0
- riskvalidation/regulatory/output_floor.py +45 -0
- riskvalidation/regulatory/sa_ccr.py +486 -0
- riskvalidation/regulatory/standardised.py +348 -0
- riskvalidation/stress/__init__.py +9 -0
- riskvalidation/stress/capital.py +476 -0
- riskvalidation/stress/paths.py +108 -0
- riskvalidation/stress/reverse.py +179 -0
- riskvalidation/stress/satellites.py +268 -0
- riskvalidation/stress/scenario_models.py +274 -0
- riskvalidation/transitions/__init__.py +0 -0
- riskvalidation/transitions/em.py +120 -0
- riskvalidation/transitions/embedding.py +221 -0
- riskvalidation/transitions/estimators.py +213 -0
- riskvalidation/transitions/intervals.py +306 -0
- riskvalidation/transitions/mobility.py +57 -0
- riskvalidation/transitions/ttc.py +171 -0
- riskvalidation/validation/__init__.py +103 -0
- riskvalidation/validation/_common.py +57 -0
- riskvalidation/validation/aggregation.py +425 -0
- riskvalidation/validation/calibration.py +757 -0
- riskvalidation/validation/ccr.py +530 -0
- riskvalidation/validation/discrimination.py +314 -0
- riskvalidation/validation/fairness.py +632 -0
- riskvalidation/validation/ifrs9.py +693 -0
- riskvalidation/validation/irrbb.py +389 -0
- riskvalidation/validation/lgd_ead.py +469 -0
- riskvalidation/validation/liquidity.py +647 -0
- riskvalidation/validation/market.py +807 -0
- riskvalidation/validation/ml.py +958 -0
- riskvalidation/validation/oprisk.py +357 -0
- riskvalidation/validation/portfolio.py +353 -0
- riskvalidation/validation/rating_system.py +245 -0
- riskvalidation/validation/stability.py +256 -0
- riskvalidation/validation/stress.py +552 -0
- riskvalidation/validation/transitions.py +386 -0
- riskvalidation-0.8.0.dist-info/METADATA +136 -0
- riskvalidation-0.8.0.dist-info/RECORD +164 -0
- riskvalidation-0.8.0.dist-info/WHEEL +5 -0
- riskvalidation-0.8.0.dist-info/licenses/LICENSE +21 -0
- riskvalidation-0.8.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,757 @@
|
|
|
1
|
+
"""PD calibration tests: is the default rate the model predicted the one that occurred?
|
|
2
|
+
|
|
3
|
+
BCBS Working Paper 14 (2005): "at present no really powerful tests of adequate calibration are currently
|
|
4
|
+
available. Due to the correlation effects that have to be respected there even seems to be no way to develop
|
|
5
|
+
such tests." The tests below therefore come in families with different assumptions (one grade or all grades,
|
|
6
|
+
one period or several, independent or correlated defaults); read them together, never one alone.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import functools
|
|
12
|
+
import math
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
from scipy import special, stats
|
|
17
|
+
|
|
18
|
+
from riskvalidation.core.build import make_result
|
|
19
|
+
from riskvalidation.core.policy import Policy
|
|
20
|
+
from riskvalidation.core.registry import register
|
|
21
|
+
from riskvalidation.core.result import TestResult
|
|
22
|
+
from riskvalidation.validation._common import as_binary, as_float_array, check_counts, check_pd, labels
|
|
23
|
+
|
|
24
|
+
ECB_2019 = (
|
|
25
|
+
"ECB (2019), Instructions for reporting the validation results of internal models: IRB Pillar I models for "
|
|
26
|
+
"credit risk"
|
|
27
|
+
)
|
|
28
|
+
WP14 = "BCBS (2005), Studies on the validation of internal rating systems, Working Paper 14 (revised)"
|
|
29
|
+
|
|
30
|
+
_GH_NODES, _GH_WEIGHTS = np.polynomial.hermite_e.hermegauss(256)
|
|
31
|
+
_GH_WEIGHTS = _GH_WEIGHTS / math.sqrt(2.0 * math.pi)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _conditional_pd(pd: float, rho: float, x: np.ndarray) -> np.ndarray:
|
|
35
|
+
"""PD conditional on the systematic factor x in the one-factor model (Vasicek)."""
|
|
36
|
+
return stats.norm.cdf((stats.norm.ppf(pd) - math.sqrt(rho) * x) / math.sqrt(1.0 - rho))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def default_count_sf(k: int, n: int, pd: float, rho: float = 0.0) -> float:
|
|
40
|
+
"""P(D >= k) for n obligors with PD ``pd`` and asset correlation ``rho`` (binomial when rho = 0)."""
|
|
41
|
+
if k <= 0:
|
|
42
|
+
return 1.0
|
|
43
|
+
if rho == 0.0:
|
|
44
|
+
return float(stats.binom.sf(k - 1, n, pd))
|
|
45
|
+
if not 0.0 < rho < 1.0:
|
|
46
|
+
raise ValueError("asset correlation must lie in [0, 1)")
|
|
47
|
+
cond = _conditional_pd(pd, rho, _GH_NODES)
|
|
48
|
+
return float(np.clip(np.sum(_GH_WEIGHTS * stats.binom.sf(k - 1, n, cond)), 0.0, 1.0))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def critical_default_count(n: int, pd: float, q: float = 0.99, rho: float = 0.0) -> int:
|
|
52
|
+
"""Smallest k with P(D >= k) <= 1 - q; the test rejects when D >= k.
|
|
53
|
+
|
|
54
|
+
This definition reproduces the critical values tabulated in BCBS WP14 for PD 1%, N 1,000, q 99%
|
|
55
|
+
(19, 35, 49, 63, 77 for rho = 0%, 5%, 10%, 15%, 20%).
|
|
56
|
+
"""
|
|
57
|
+
lo, hi = 0, n + 1
|
|
58
|
+
while lo < hi: # P(D >= k) decreases in k
|
|
59
|
+
mid = (lo + hi) // 2
|
|
60
|
+
if default_count_sf(mid, n, pd, rho) <= 1.0 - q:
|
|
61
|
+
hi = mid
|
|
62
|
+
else:
|
|
63
|
+
lo = mid + 1
|
|
64
|
+
return lo
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@register(
|
|
68
|
+
"pd.jeffreys",
|
|
69
|
+
family="pd-calibration",
|
|
70
|
+
name="Jeffreys test",
|
|
71
|
+
h0="the PD applied at the start of the period is greater than or equal to the true PD",
|
|
72
|
+
reference=f"{ECB_2019}, section 2.5.3.1; Brown, Cai and DasGupta (2001), doi:10.1214/ss/1009213286",
|
|
73
|
+
decision="p_value",
|
|
74
|
+
)
|
|
75
|
+
def pd_jeffreys(
|
|
76
|
+
n: int,
|
|
77
|
+
d: int,
|
|
78
|
+
pd: float,
|
|
79
|
+
*,
|
|
80
|
+
policy: Policy | None = None,
|
|
81
|
+
model_id: str | None = None,
|
|
82
|
+
segment: str | None = None,
|
|
83
|
+
) -> TestResult:
|
|
84
|
+
"""One-sided Jeffreys test: p = F_Beta(D+1/2, N-D+1/2)(PD), small when the PD is underestimated."""
|
|
85
|
+
n_a, d_a = check_counts(n, d)
|
|
86
|
+
pd_v = float(check_pd(pd)[0])
|
|
87
|
+
nn, dd = int(n_a[0]), int(d_a[0])
|
|
88
|
+
if nn == 0:
|
|
89
|
+
raise ValueError("the Jeffreys test needs at least one obligor")
|
|
90
|
+
a, b = dd + 0.5, nn - dd + 0.5
|
|
91
|
+
p = float(stats.beta.cdf(pd_v, a, b))
|
|
92
|
+
return make_result(
|
|
93
|
+
"pd.jeffreys",
|
|
94
|
+
statistic=dd / nn,
|
|
95
|
+
p_value=p,
|
|
96
|
+
policy=policy,
|
|
97
|
+
alternative="the true PD exceeds the PD applied (underestimation)",
|
|
98
|
+
n=nn,
|
|
99
|
+
n_events=dd,
|
|
100
|
+
model_id=model_id,
|
|
101
|
+
segment=segment,
|
|
102
|
+
inputs=(nn, dd, pd_v),
|
|
103
|
+
extras={
|
|
104
|
+
"pd_applied": pd_v,
|
|
105
|
+
"default_rate": dd / nn,
|
|
106
|
+
"posterior_q025": float(stats.beta.ppf(0.025, a, b)),
|
|
107
|
+
"posterior_q975": float(stats.beta.ppf(0.975, a, b)),
|
|
108
|
+
},
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def pd_jeffreys_grades(
|
|
113
|
+
n: Sequence[int],
|
|
114
|
+
d: Sequence[int],
|
|
115
|
+
pd: Sequence[float],
|
|
116
|
+
*,
|
|
117
|
+
grades: Sequence[str] | None = None,
|
|
118
|
+
policy: Policy | None = None,
|
|
119
|
+
model_id: str | None = None,
|
|
120
|
+
) -> list[TestResult]:
|
|
121
|
+
"""K + 1 Jeffreys tests: one per grade and one for the portfolio (ECB 2019, section 2.5.3.1).
|
|
122
|
+
|
|
123
|
+
The portfolio PD is the obligor-weighted average of the grade PDs, the average PD across customers.
|
|
124
|
+
"""
|
|
125
|
+
n_a, d_a = check_counts(n, d)
|
|
126
|
+
pd_a = check_pd(pd)
|
|
127
|
+
if pd_a.shape != n_a.shape:
|
|
128
|
+
raise ValueError("pd must have one value per grade")
|
|
129
|
+
names = labels(len(n_a), grades)
|
|
130
|
+
out = [
|
|
131
|
+
pd_jeffreys(int(ni), int(di), float(pi), policy=policy, model_id=model_id, segment=g)
|
|
132
|
+
for ni, di, pi, g in zip(n_a, d_a, pd_a, names, strict=True)
|
|
133
|
+
if ni > 0
|
|
134
|
+
]
|
|
135
|
+
total = int(n_a.sum())
|
|
136
|
+
pd_port = float(np.sum(n_a * pd_a) / total)
|
|
137
|
+
out.append(pd_jeffreys(total, int(d_a.sum()), pd_port, policy=policy, model_id=model_id, segment="portfolio"))
|
|
138
|
+
return out
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@register(
|
|
142
|
+
"pd.binomial",
|
|
143
|
+
family="pd-calibration",
|
|
144
|
+
name="Binomial test",
|
|
145
|
+
h0="the true PD is less than or equal to the PD applied, defaults independent",
|
|
146
|
+
reference=f"{WP14}, section on calibration (binomial test; an early warning when defaults are correlated)",
|
|
147
|
+
decision="p_value",
|
|
148
|
+
)
|
|
149
|
+
def pd_binomial(
|
|
150
|
+
n: int,
|
|
151
|
+
d: int,
|
|
152
|
+
pd: float,
|
|
153
|
+
*,
|
|
154
|
+
q: float = 0.99,
|
|
155
|
+
policy: Policy | None = None,
|
|
156
|
+
model_id: str | None = None,
|
|
157
|
+
segment: str | None = None,
|
|
158
|
+
) -> TestResult:
|
|
159
|
+
"""Exact one-sided binomial test, p = P(Bin(N, PD) >= D)."""
|
|
160
|
+
n_a, d_a = check_counts(n, d)
|
|
161
|
+
pd_v = float(check_pd(pd)[0])
|
|
162
|
+
nn, dd = int(n_a[0]), int(d_a[0])
|
|
163
|
+
p = default_count_sf(dd, nn, pd_v, 0.0)
|
|
164
|
+
k = critical_default_count(nn, pd_v, q, 0.0)
|
|
165
|
+
return make_result(
|
|
166
|
+
"pd.binomial",
|
|
167
|
+
statistic=float(dd),
|
|
168
|
+
p_value=p,
|
|
169
|
+
policy=policy,
|
|
170
|
+
alternative="the true PD exceeds the PD applied",
|
|
171
|
+
n=nn,
|
|
172
|
+
n_events=dd,
|
|
173
|
+
model_id=model_id,
|
|
174
|
+
segment=segment,
|
|
175
|
+
inputs=(nn, dd, pd_v, q),
|
|
176
|
+
notes="assumes independent defaults; WP14: the true type I error can be much larger if defaults are "
|
|
177
|
+
"correlated",
|
|
178
|
+
extras={"critical_count": k, "confidence": q, "expected_defaults": nn * pd_v},
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@register(
|
|
183
|
+
"pd.binomial_vasicek",
|
|
184
|
+
family="pd-calibration",
|
|
185
|
+
name="Vasicek-corrected binomial test",
|
|
186
|
+
h0="the true PD is less than or equal to the PD applied, defaults correlated through one systematic factor",
|
|
187
|
+
reference=f"{WP14}, appendix (one-factor model with asset correlation)",
|
|
188
|
+
decision="p_value",
|
|
189
|
+
)
|
|
190
|
+
def pd_binomial_vasicek(
|
|
191
|
+
n: int,
|
|
192
|
+
d: int,
|
|
193
|
+
pd: float,
|
|
194
|
+
rho: float,
|
|
195
|
+
*,
|
|
196
|
+
q: float = 0.99,
|
|
197
|
+
policy: Policy | None = None,
|
|
198
|
+
model_id: str | None = None,
|
|
199
|
+
segment: str | None = None,
|
|
200
|
+
) -> TestResult:
|
|
201
|
+
"""Binomial test under the Vasicek one-factor mixture; the asset correlation is an assumption to document."""
|
|
202
|
+
n_a, d_a = check_counts(n, d)
|
|
203
|
+
pd_v = float(check_pd(pd)[0])
|
|
204
|
+
if not 0.0 <= rho < 1.0:
|
|
205
|
+
raise ValueError("asset correlation must lie in [0, 1)")
|
|
206
|
+
nn, dd = int(n_a[0]), int(d_a[0])
|
|
207
|
+
p = default_count_sf(dd, nn, pd_v, rho)
|
|
208
|
+
k = critical_default_count(nn, pd_v, q, rho)
|
|
209
|
+
approx = nn * float(
|
|
210
|
+
stats.norm.cdf((stats.norm.ppf(pd_v) + math.sqrt(rho) * stats.norm.ppf(q)) / math.sqrt(1.0 - rho))
|
|
211
|
+
)
|
|
212
|
+
return make_result(
|
|
213
|
+
"pd.binomial_vasicek",
|
|
214
|
+
statistic=float(dd),
|
|
215
|
+
p_value=p,
|
|
216
|
+
policy=policy,
|
|
217
|
+
alternative="the true PD exceeds the PD applied",
|
|
218
|
+
n=nn,
|
|
219
|
+
n_events=dd,
|
|
220
|
+
model_id=model_id,
|
|
221
|
+
segment=segment,
|
|
222
|
+
inputs=(nn, dd, pd_v, rho, q),
|
|
223
|
+
notes=f"asset correlation {rho:g} is a modelling assumption",
|
|
224
|
+
extras={"critical_count": k, "confidence": q, "rho": rho, "large_n_critical_count": approx},
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@register(
|
|
229
|
+
"pd.chi2_grades",
|
|
230
|
+
family="pd-calibration",
|
|
231
|
+
name="Chi-square test over grades",
|
|
232
|
+
h0="every grade PD equals the true PD of its grade, defaults independent within and between grades",
|
|
233
|
+
reference=f"{WP14}, calibration (Hosmer-Lemeshow type statistic over rating grades)",
|
|
234
|
+
decision="p_value",
|
|
235
|
+
)
|
|
236
|
+
def pd_chi2_grades(
|
|
237
|
+
n: Sequence[int],
|
|
238
|
+
d: Sequence[int],
|
|
239
|
+
pd: Sequence[float],
|
|
240
|
+
*,
|
|
241
|
+
policy: Policy | None = None,
|
|
242
|
+
model_id: str | None = None,
|
|
243
|
+
segment: str | None = None,
|
|
244
|
+
) -> TestResult:
|
|
245
|
+
"""T = sum (n_i p_i - d_i)^2 / (n_i p_i (1 - p_i)), chi-square with as many degrees of freedom as grades."""
|
|
246
|
+
n_a, d_a = check_counts(n, d)
|
|
247
|
+
pd_a = check_pd(pd)
|
|
248
|
+
keep = n_a > 0
|
|
249
|
+
n_a, d_a, pd_a = n_a[keep], d_a[keep], pd_a[keep]
|
|
250
|
+
t = float(np.sum((n_a * pd_a - d_a) ** 2 / (n_a * pd_a * (1.0 - pd_a))))
|
|
251
|
+
dof = int(n_a.size)
|
|
252
|
+
return make_result(
|
|
253
|
+
"pd.chi2_grades",
|
|
254
|
+
statistic=t,
|
|
255
|
+
p_value=float(stats.chi2.sf(t, dof)),
|
|
256
|
+
policy=policy,
|
|
257
|
+
alternative="at least one grade PD differs from the true PD",
|
|
258
|
+
n=int(n_a.sum()),
|
|
259
|
+
n_events=int(d_a.sum()),
|
|
260
|
+
model_id=model_id,
|
|
261
|
+
segment=segment,
|
|
262
|
+
inputs=(n_a, d_a, pd_a),
|
|
263
|
+
notes="WP14: likely to understate the true type I error under default dependence; slow convergence for "
|
|
264
|
+
"small PDs",
|
|
265
|
+
extras={"dof": dof},
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
@register(
|
|
270
|
+
"pd.default_profile",
|
|
271
|
+
family="pd-calibration",
|
|
272
|
+
name="Chi-square test of the default profile",
|
|
273
|
+
h0="given their number, the defaults fall on the grades as the PD curve implies (the shape of the curve)",
|
|
274
|
+
reference="Tasche (2013), The art of probability-of-default curve calibration, Journal of Credit Risk 9(4), "
|
|
275
|
+
"doi:10.21314/jcr.2013.169, sections 3.3 and 4.2.5; Pearson (1900), doi:10.1080/14786440009463897",
|
|
276
|
+
decision="p_value",
|
|
277
|
+
)
|
|
278
|
+
def pd_default_profile(
|
|
279
|
+
n: Sequence[int],
|
|
280
|
+
d: Sequence[int],
|
|
281
|
+
pd: Sequence[float],
|
|
282
|
+
*,
|
|
283
|
+
n_sim: int = 100_000,
|
|
284
|
+
seed: int = 0,
|
|
285
|
+
policy: Policy | None = None,
|
|
286
|
+
model_id: str | None = None,
|
|
287
|
+
segment: str | None = None,
|
|
288
|
+
) -> TestResult:
|
|
289
|
+
"""T = sum_x (d_x - D q_x)^2 / (D q_x), with q_x = n_x PD_x / sum_t n_t PD_t the implied default profile (3.5b) and
|
|
290
|
+
D the observed number of defaults. Under H0 the default numbers are multinomial(D, q) given D, so the test sees the
|
|
291
|
+
shape of the curve and not its level (read it with a level test such as ``pd.binomial``). Expected counts are small
|
|
292
|
+
in good grades, so the p-value is a Monte Carlo one, (1 + #{T* >= T}) / (n_sim + 1) over ``n_sim`` multinomial
|
|
293
|
+
samples drawn with ``seed``; the asymptotic chi-square p-value with k - 1 degrees of freedom is kept for reference.
|
|
294
|
+
"""
|
|
295
|
+
n_a, d_a = check_counts(n, d)
|
|
296
|
+
pd_a = check_pd(pd)
|
|
297
|
+
if pd_a.shape != n_a.shape:
|
|
298
|
+
raise ValueError("pd needs one value per grade")
|
|
299
|
+
if n_sim < 1000:
|
|
300
|
+
raise ValueError("n_sim must be at least 1000")
|
|
301
|
+
keep = n_a > 0
|
|
302
|
+
n_a, d_a, pd_a = n_a[keep], d_a[keep], pd_a[keep]
|
|
303
|
+
total = int(d_a.sum())
|
|
304
|
+
if total == 0 or n_a.size < 2:
|
|
305
|
+
raise ValueError("the default profile needs at least one default and two populated grades")
|
|
306
|
+
q = n_a * pd_a / float(np.sum(n_a * pd_a))
|
|
307
|
+
expected = total * q
|
|
308
|
+
t = float(np.sum((d_a - expected) ** 2 / expected))
|
|
309
|
+
rng = np.random.default_rng(seed)
|
|
310
|
+
at_least, done = 0, 0
|
|
311
|
+
while done < n_sim:
|
|
312
|
+
size = min(50_000, n_sim - done)
|
|
313
|
+
draws = rng.multinomial(total, q, size=size)
|
|
314
|
+
at_least += int(np.sum(np.sum((draws - expected) ** 2 / expected, axis=1) >= t * (1.0 - 1e-12)))
|
|
315
|
+
done += size
|
|
316
|
+
p_mc = (1 + at_least) / (n_sim + 1)
|
|
317
|
+
dof = int(n_a.size - 1)
|
|
318
|
+
return make_result(
|
|
319
|
+
"pd.default_profile",
|
|
320
|
+
statistic=t,
|
|
321
|
+
p_value=float(p_mc),
|
|
322
|
+
policy=policy,
|
|
323
|
+
alternative="the defaults fall on the grades differently from the implied default profile",
|
|
324
|
+
n=int(n_a.sum()),
|
|
325
|
+
n_events=total,
|
|
326
|
+
model_id=model_id,
|
|
327
|
+
segment=segment,
|
|
328
|
+
inputs=(n_a, d_a, pd_a, n_sim, seed),
|
|
329
|
+
notes="Monte Carlo p-value; its standard error is in extras",
|
|
330
|
+
extras={
|
|
331
|
+
"n_sim": int(n_sim),
|
|
332
|
+
"seed": int(seed),
|
|
333
|
+
"mc_standard_error": float(math.sqrt(p_mc * (1.0 - p_mc) / n_sim)),
|
|
334
|
+
"implied_default_profile": [float(v) for v in q],
|
|
335
|
+
"dof": dof,
|
|
336
|
+
"p_value_asymptotic": float(stats.chi2.sf(t, dof)),
|
|
337
|
+
},
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
@register(
|
|
342
|
+
"pd.hosmer_lemeshow",
|
|
343
|
+
family="pd-calibration",
|
|
344
|
+
name="Hosmer-Lemeshow test",
|
|
345
|
+
h0="the predicted probabilities equal the true default probabilities in every risk group",
|
|
346
|
+
reference="Hosmer and Lemeshow (1980), Communications in Statistics, Theory and Methods 9(10), "
|
|
347
|
+
"doi:10.1080/03610928008827941",
|
|
348
|
+
decision="p_value",
|
|
349
|
+
)
|
|
350
|
+
def pd_hosmer_lemeshow(
|
|
351
|
+
defaults: Sequence[int],
|
|
352
|
+
pd: Sequence[float],
|
|
353
|
+
*,
|
|
354
|
+
groups: int = 10,
|
|
355
|
+
fitted: bool = False,
|
|
356
|
+
dof: int | None = None,
|
|
357
|
+
policy: Policy | None = None,
|
|
358
|
+
model_id: str | None = None,
|
|
359
|
+
segment: str | None = None,
|
|
360
|
+
) -> TestResult:
|
|
361
|
+
"""Groups by quantiles of predicted PD; chi-square with ``groups`` degrees of freedom for PDs given before the
|
|
362
|
+
outcomes (validation), ``groups - 2`` when ``fitted`` (the PDs were estimated on these same observations, the
|
|
363
|
+
original in-sample statistic). ``dof`` overrides both; the choice is recorded.
|
|
364
|
+
|
|
365
|
+
For given PDs each group's standardised difference is asymptotically an independent chi-square with one degree of
|
|
366
|
+
freedom, as BCBS WP14 states for the test over grades. On independent samples with the true PDs, G - 2 rejects
|
|
367
|
+
far more often than the nominal level (the size harness of riskvalidation 0.03.000; see the test's page).
|
|
368
|
+
"""
|
|
369
|
+
y = as_binary(defaults)
|
|
370
|
+
p = check_pd(pd)
|
|
371
|
+
if p.shape != y.shape:
|
|
372
|
+
raise ValueError("defaults and pd must have the same length")
|
|
373
|
+
if groups < 3 or groups > y.size:
|
|
374
|
+
raise ValueError("groups must be at least 3 and at most the sample size")
|
|
375
|
+
order = np.argsort(p, kind="mergesort")
|
|
376
|
+
hl = 0.0
|
|
377
|
+
for idx in np.array_split(order, groups):
|
|
378
|
+
o, e, m = float(y[idx].sum()), float(p[idx].sum()), idx.size
|
|
379
|
+
hl += (o - e) ** 2 / (e * (1.0 - e / m))
|
|
380
|
+
k = int(dof) if dof is not None else (groups - 2 if fitted else groups)
|
|
381
|
+
return make_result(
|
|
382
|
+
"pd.hosmer_lemeshow",
|
|
383
|
+
statistic=hl,
|
|
384
|
+
p_value=float(stats.chi2.sf(hl, k)),
|
|
385
|
+
policy=policy,
|
|
386
|
+
alternative="the predicted probabilities are miscalibrated in at least one group",
|
|
387
|
+
n=int(y.size),
|
|
388
|
+
n_events=int(y.sum()),
|
|
389
|
+
model_id=model_id,
|
|
390
|
+
segment=segment,
|
|
391
|
+
inputs=(y, p, groups, k),
|
|
392
|
+
extras={"groups": groups, "dof": k, "fitted": bool(fitted)},
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
@register(
|
|
397
|
+
"pd.spiegelhalter",
|
|
398
|
+
family="pd-calibration",
|
|
399
|
+
name="Spiegelhalter test",
|
|
400
|
+
h0="every obligor's predicted PD equals its true default probability",
|
|
401
|
+
reference="Spiegelhalter (1986), Statistics in Medicine 5(5), doi:10.1002/sim.4780050506; formula as in the "
|
|
402
|
+
"Stata brier manual",
|
|
403
|
+
decision="p_value",
|
|
404
|
+
)
|
|
405
|
+
def pd_spiegelhalter(
|
|
406
|
+
defaults: Sequence[int],
|
|
407
|
+
pd: Sequence[float],
|
|
408
|
+
*,
|
|
409
|
+
policy: Policy | None = None,
|
|
410
|
+
model_id: str | None = None,
|
|
411
|
+
segment: str | None = None,
|
|
412
|
+
) -> TestResult:
|
|
413
|
+
"""Z = (observed Brier - expected Brier) / sd under H0; one-sided upper tail. Obligor level, no binning."""
|
|
414
|
+
y = as_binary(defaults)
|
|
415
|
+
f = check_pd(pd)
|
|
416
|
+
if f.shape != y.shape:
|
|
417
|
+
raise ValueError("defaults and pd must have the same length")
|
|
418
|
+
nn = y.size
|
|
419
|
+
observed = float(np.mean((f - y) ** 2))
|
|
420
|
+
expected = float(np.mean(f * (1.0 - f)))
|
|
421
|
+
var = float(np.sum(f * (1.0 - f) * (1.0 - 2.0 * f) ** 2) / nn**2)
|
|
422
|
+
z = (observed - expected) / math.sqrt(var)
|
|
423
|
+
return make_result(
|
|
424
|
+
"pd.spiegelhalter",
|
|
425
|
+
statistic=z,
|
|
426
|
+
p_value=float(stats.norm.sf(z)),
|
|
427
|
+
policy=policy,
|
|
428
|
+
alternative="the forecasts are miscalibrated",
|
|
429
|
+
n=int(nn),
|
|
430
|
+
n_events=int(y.sum()),
|
|
431
|
+
model_id=model_id,
|
|
432
|
+
segment=segment,
|
|
433
|
+
inputs=(y, f),
|
|
434
|
+
extras={"brier_observed": observed, "brier_expected": expected, "brier_variance": var},
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
@register(
|
|
439
|
+
"pd.normal_multiperiod",
|
|
440
|
+
family="pd-calibration",
|
|
441
|
+
name="Normal test over several periods",
|
|
442
|
+
h0="none of the true PDs in years 1 to T is greater than its forecast",
|
|
443
|
+
reference=f"{WP14}, calibration (normal test; cross-sectional dependence admissible, independence over time)",
|
|
444
|
+
decision="p_value",
|
|
445
|
+
)
|
|
446
|
+
def pd_normal_multiperiod(
|
|
447
|
+
default_rates: Sequence[float],
|
|
448
|
+
pd: Sequence[float],
|
|
449
|
+
*,
|
|
450
|
+
policy: Policy | None = None,
|
|
451
|
+
model_id: str | None = None,
|
|
452
|
+
segment: str | None = None,
|
|
453
|
+
) -> TestResult:
|
|
454
|
+
"""sum(d_t - PD_t) / (sqrt(T) sigma_hat), compared with the standard normal upper tail."""
|
|
455
|
+
dr = as_float_array(default_rates, "default_rates")
|
|
456
|
+
pdt = check_pd(pd)
|
|
457
|
+
if dr.shape != pdt.shape:
|
|
458
|
+
raise ValueError("default rates and PDs must have one value per year")
|
|
459
|
+
t = dr.size
|
|
460
|
+
if t < 2:
|
|
461
|
+
raise ValueError("the multi-period normal test needs at least two years")
|
|
462
|
+
e = dr - pdt
|
|
463
|
+
var = (np.sum(e**2) - np.sum(e) ** 2 / t) / (t - 1)
|
|
464
|
+
if var <= 0:
|
|
465
|
+
return make_result(
|
|
466
|
+
"pd.normal_multiperiod",
|
|
467
|
+
statistic=float("nan"),
|
|
468
|
+
policy=policy,
|
|
469
|
+
n=t,
|
|
470
|
+
model_id=model_id,
|
|
471
|
+
segment=segment,
|
|
472
|
+
inputs=(dr, pdt),
|
|
473
|
+
notes="the estimated variance of the yearly differences is zero; the statistic is undefined",
|
|
474
|
+
)
|
|
475
|
+
stat = float(np.sum(e) / (math.sqrt(t) * math.sqrt(var)))
|
|
476
|
+
return make_result(
|
|
477
|
+
"pd.normal_multiperiod",
|
|
478
|
+
statistic=stat,
|
|
479
|
+
p_value=float(special.ndtr(-stat)),
|
|
480
|
+
policy=policy,
|
|
481
|
+
alternative="at least one yearly PD is underestimated",
|
|
482
|
+
n=t,
|
|
483
|
+
model_id=model_id,
|
|
484
|
+
segment=segment,
|
|
485
|
+
inputs=(dr, pdt),
|
|
486
|
+
notes="above the nominal level over short series: measured size 8.5% at the 5% level for five years of 1,000 "
|
|
487
|
+
"obligors at PD 3% (WP14 Table 7: 8.1%)",
|
|
488
|
+
extras={"years": t, "sigma_hat": math.sqrt(var)},
|
|
489
|
+
)
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
WP14_TRAFFIC_PROBS = (0.5, 0.3, 0.15, 0.05)
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
@functools.lru_cache(maxsize=64)
|
|
496
|
+
def _traffic_profile_cdf(t: int, probs: tuple[float, ...]) -> dict[tuple[int, int, int, int], float]:
|
|
497
|
+
"""P(profile <= p) for every colour profile p of ``t`` years, the profiles in lexicographic order of
|
|
498
|
+
(A_g, A_y, A_o, A_r); the same additions in the same order as a full enumeration, computed once per (t, probs)."""
|
|
499
|
+
out: dict[tuple[int, int, int, int], float] = {}
|
|
500
|
+
acc = 0.0
|
|
501
|
+
for a_g in range(t + 1):
|
|
502
|
+
for a_y in range(t - a_g + 1):
|
|
503
|
+
for a_o in range(t - a_g - a_y + 1):
|
|
504
|
+
prof = (a_g, a_y, a_o, t - a_g - a_y - a_o)
|
|
505
|
+
acc += float(stats.multinomial.pmf(prof, t, probs))
|
|
506
|
+
out[prof] = min(1.0, acc)
|
|
507
|
+
return out
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
@functools.lru_cache(maxsize=64)
|
|
511
|
+
def _traffic_cuts(q: tuple[float, ...]) -> np.ndarray:
|
|
512
|
+
"""Phi^-1 of the cumulative colour probabilities (norm.ppf is ndtri inside SciPy)."""
|
|
513
|
+
return special.ndtri(np.cumsum(q)[:3])
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def _traffic_colours(n_a: np.ndarray, d_a: np.ndarray, pd_a: np.ndarray, q: np.ndarray,
|
|
517
|
+
ties: str) -> tuple[np.ndarray, np.ndarray]:
|
|
518
|
+
if q.shape != (4,) or np.any(q <= 0) or not math.isclose(float(q.sum()), 1.0, abs_tol=1e-12):
|
|
519
|
+
raise ValueError("probs must be four positive numbers summing to 1")
|
|
520
|
+
if ties not in ("lower", "upper"):
|
|
521
|
+
raise ValueError("ties is 'lower' (the printed WP14 mapping) or 'upper'")
|
|
522
|
+
r = (d_a - n_a * pd_a) / np.sqrt(n_a * pd_a * (1.0 - pd_a))
|
|
523
|
+
cuts = _traffic_cuts(tuple(float(v) for v in q))
|
|
524
|
+
tol = 1e-12 # a rate within rounding of a boundary is on it
|
|
525
|
+
if ties == "lower":
|
|
526
|
+
colour = np.sum(cuts[None, :] < r[:, None] - tol, axis=1) # 0 green, 1 yellow, 2 orange, 3 red
|
|
527
|
+
else:
|
|
528
|
+
colour = np.sum(cuts[None, :] <= r[:, None] + tol, axis=1)
|
|
529
|
+
return r, colour
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def traffic_light_colours(
|
|
533
|
+
n: Sequence[int], d: Sequence[int], pd: Sequence[float], *, probs: Sequence[float] = WP14_TRAFFIC_PROBS,
|
|
534
|
+
ties: str = "lower",
|
|
535
|
+
) -> np.ndarray:
|
|
536
|
+
"""The colour of each year in the traffic-lights test, 0 green to 3 red: the standardised default rate
|
|
537
|
+
(D - N p) / sqrt(N p (1 - p)) against Phi^-1 of the cumulative colour probabilities (WP14), with ``ties`` as in
|
|
538
|
+
:func:`pd_traffic_lights`."""
|
|
539
|
+
n_a, d_a = check_counts(n, d)
|
|
540
|
+
pd_a = check_pd(pd)
|
|
541
|
+
if pd_a.shape != n_a.shape:
|
|
542
|
+
raise ValueError("one PD per year is required")
|
|
543
|
+
return _traffic_colours(n_a, d_a, pd_a, np.asarray(probs, dtype=float), ties)[1]
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def traffic_profile_cdf(t: int, probs: Sequence[float] = WP14_TRAFFIC_PROBS) -> dict[tuple[int, int, int, int], float]:
|
|
547
|
+
"""P(profile <= p) under the multinomial null for every colour profile of ``t`` years (lexicographic order)."""
|
|
548
|
+
return dict(_traffic_profile_cdf(int(t), tuple(float(v) for v in probs)))
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
@register(
|
|
552
|
+
"pd.traffic_lights",
|
|
553
|
+
family="pd-calibration",
|
|
554
|
+
name="Traffic-lights test (Blochwitz, Hohl and Wehn)",
|
|
555
|
+
h0="the yearly PDs are correct; the colour counts are multinomial with the stated probabilities",
|
|
556
|
+
reference=f"{WP14}, traffic lights approach (Blochwitz, Hohl and Wehn); probabilities 0.5, 0.3, 0.15, 0.05",
|
|
557
|
+
decision="p_value",
|
|
558
|
+
)
|
|
559
|
+
def pd_traffic_lights(
|
|
560
|
+
n: Sequence[int],
|
|
561
|
+
d: Sequence[int],
|
|
562
|
+
pd: Sequence[float],
|
|
563
|
+
*,
|
|
564
|
+
probs: Sequence[float] = WP14_TRAFFIC_PROBS,
|
|
565
|
+
ties: str = "lower",
|
|
566
|
+
policy: Policy | None = None,
|
|
567
|
+
model_id: str | None = None,
|
|
568
|
+
segment: str | None = None,
|
|
569
|
+
) -> TestResult:
|
|
570
|
+
"""Colour each year by its standardised default rate; p = P(colour profile no better than observed).
|
|
571
|
+
|
|
572
|
+
Profiles are ordered as V = 1000 A_g + 100 A_y + 10 A_o + A_r (WP14), which for any T is the lexicographic
|
|
573
|
+
order of (A_g, A_y, A_o, A_r); P(V <= v_obs) is a lookup in the exact multinomial table.
|
|
574
|
+
|
|
575
|
+
``ties`` decides the colour of a year whose standardised rate sits on a colour boundary (within 1e-12), which
|
|
576
|
+
happens with positive probability when N p is an integer (a year with exactly the expected number of defaults):
|
|
577
|
+
``"lower"`` gives it the better colour, as WP14's printed mapping does (green if x <= Phi^-1(q_g)); ``"upper"``
|
|
578
|
+
gives it the worse one, which is what WP14's own simulation study must have used: its Tables 7 and 8 are
|
|
579
|
+
reproduced exactly with ``"upper"`` and not with the printed rule (riskvalidation 0.03.000, the size harness).
|
|
580
|
+
"""
|
|
581
|
+
n_a, d_a = check_counts(n, d)
|
|
582
|
+
pd_a = check_pd(pd)
|
|
583
|
+
if pd_a.shape != n_a.shape:
|
|
584
|
+
raise ValueError("one PD per year is required")
|
|
585
|
+
q = np.asarray(probs, dtype=float)
|
|
586
|
+
r, colour = _traffic_colours(n_a, d_a, pd_a, q, ties)
|
|
587
|
+
counts = tuple(int(np.sum(colour == c)) for c in range(4))
|
|
588
|
+
t = int(n_a.size)
|
|
589
|
+
p_le = _traffic_profile_cdf(t, tuple(float(v) for v in q))[counts]
|
|
590
|
+
v = 1000 * counts[0] + 100 * counts[1] + 10 * counts[2] + counts[3]
|
|
591
|
+
return make_result(
|
|
592
|
+
"pd.traffic_lights",
|
|
593
|
+
statistic=float(v),
|
|
594
|
+
p_value=min(1.0, p_le),
|
|
595
|
+
policy=policy,
|
|
596
|
+
alternative="the PDs are underestimated in too many years",
|
|
597
|
+
n=t,
|
|
598
|
+
n_events=int(d_a.sum()),
|
|
599
|
+
model_id=model_id,
|
|
600
|
+
segment=segment,
|
|
601
|
+
inputs=(n_a, d_a, pd_a, q, ties),
|
|
602
|
+
extras={
|
|
603
|
+
"ties": ties,
|
|
604
|
+
"green": counts[0],
|
|
605
|
+
"yellow": counts[1],
|
|
606
|
+
"orange": counts[2],
|
|
607
|
+
"red": counts[3],
|
|
608
|
+
"standardised_rates": r.tolist(),
|
|
609
|
+
"colours": [("green", "yellow", "orange", "red")[c] for c in colour],
|
|
610
|
+
},
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
@register(
|
|
615
|
+
"pd.brier",
|
|
616
|
+
family="pd-calibration",
|
|
617
|
+
name="Brier score with Murphy decomposition",
|
|
618
|
+
h0="descriptive: mean squared error of the PD forecasts",
|
|
619
|
+
reference="Brier (1950), doi:10.1175/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2; Murphy (1973), "
|
|
620
|
+
"doi:10.1175/1520-0450(1973)012<0595:ANVPOT>2.0.CO;2",
|
|
621
|
+
decision="metric",
|
|
622
|
+
)
|
|
623
|
+
def pd_brier(
|
|
624
|
+
defaults: Sequence[int],
|
|
625
|
+
pd: Sequence[float],
|
|
626
|
+
*,
|
|
627
|
+
groups: Sequence[str] | None = None,
|
|
628
|
+
bins: int = 10,
|
|
629
|
+
policy: Policy | None = None,
|
|
630
|
+
model_id: str | None = None,
|
|
631
|
+
segment: str | None = None,
|
|
632
|
+
) -> TestResult:
|
|
633
|
+
"""BS = reliability - resolution + uncertainty; exact when forecasts are constant within each group.
|
|
634
|
+
|
|
635
|
+
Groups are rating grades when given, otherwise ``bins`` quantile bins of the forecast. The Brier skill
|
|
636
|
+
against the trivial forecast (the sample default rate) is reported, since low default rates make any Brier
|
|
637
|
+
score look small (WP14).
|
|
638
|
+
"""
|
|
639
|
+
y = as_binary(defaults)
|
|
640
|
+
f = check_pd(pd)
|
|
641
|
+
if f.shape != y.shape:
|
|
642
|
+
raise ValueError("defaults and pd must have the same length")
|
|
643
|
+
nn = y.size
|
|
644
|
+
if groups is not None:
|
|
645
|
+
g = np.asarray(groups)
|
|
646
|
+
if g.shape != y.shape:
|
|
647
|
+
raise ValueError("groups must have one label per obligor")
|
|
648
|
+
_, gid = np.unique(g, return_inverse=True)
|
|
649
|
+
else:
|
|
650
|
+
order = np.argsort(f, kind="mergesort")
|
|
651
|
+
gid = np.empty(nn, dtype=int)
|
|
652
|
+
for k, idx in enumerate(np.array_split(order, min(bins, nn))):
|
|
653
|
+
gid[idx] = k
|
|
654
|
+
bs = float(np.mean((f - y) ** 2))
|
|
655
|
+
dbar = float(y.mean())
|
|
656
|
+
rel = res = 0.0
|
|
657
|
+
exact = True
|
|
658
|
+
for k in np.unique(gid):
|
|
659
|
+
m = gid == k
|
|
660
|
+
nk = int(m.sum())
|
|
661
|
+
fk, dk = float(f[m].mean()), float(y[m].mean())
|
|
662
|
+
rel += nk * (fk - dk) ** 2
|
|
663
|
+
res += nk * (dk - dbar) ** 2
|
|
664
|
+
exact = exact and bool(np.ptp(f[m]) == 0.0)
|
|
665
|
+
rel, res = rel / nn, res / nn
|
|
666
|
+
unc = dbar * (1.0 - dbar)
|
|
667
|
+
return make_result(
|
|
668
|
+
"pd.brier",
|
|
669
|
+
metric=bs,
|
|
670
|
+
policy=policy,
|
|
671
|
+
n=int(nn),
|
|
672
|
+
n_events=int(y.sum()),
|
|
673
|
+
model_id=model_id,
|
|
674
|
+
segment=segment,
|
|
675
|
+
inputs=(y, f, gid),
|
|
676
|
+
extras={
|
|
677
|
+
"reliability": rel,
|
|
678
|
+
"resolution": res,
|
|
679
|
+
"uncertainty": unc,
|
|
680
|
+
"brier_skill": (1.0 - bs / unc) if unc > 0 else None,
|
|
681
|
+
"decomposition_exact": exact,
|
|
682
|
+
},
|
|
683
|
+
)
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
@register(
|
|
687
|
+
"pd.ece",
|
|
688
|
+
family="pd-calibration",
|
|
689
|
+
name="Expected calibration error",
|
|
690
|
+
h0="descriptive: weighted mean absolute gap between forecast and observed default rate per bin",
|
|
691
|
+
reference="Naeini, Cooper and Hauskrecht (2015), AAAI, doi:10.1609/aaai.v29i1.9602; Guo et al. (2017), "
|
|
692
|
+
"arXiv:1706.04599",
|
|
693
|
+
decision="metric",
|
|
694
|
+
)
|
|
695
|
+
def pd_ece(
|
|
696
|
+
defaults: Sequence[int],
|
|
697
|
+
pd: Sequence[float],
|
|
698
|
+
*,
|
|
699
|
+
bins: int = 10,
|
|
700
|
+
binning: str = "quantile",
|
|
701
|
+
policy: Policy | None = None,
|
|
702
|
+
model_id: str | None = None,
|
|
703
|
+
segment: str | None = None,
|
|
704
|
+
) -> TestResult:
|
|
705
|
+
"""ECE = sum_m |B_m|/n |mean(y) - mean(p)|; quantile bins by default, because PDs cluster near zero."""
|
|
706
|
+
y = as_binary(defaults)
|
|
707
|
+
f = check_pd(pd)
|
|
708
|
+
if f.shape != y.shape:
|
|
709
|
+
raise ValueError("defaults and pd must have the same length")
|
|
710
|
+
if binning == "quantile":
|
|
711
|
+
order = np.argsort(f, kind="mergesort")
|
|
712
|
+
parts = np.array_split(order, min(bins, y.size))
|
|
713
|
+
elif binning == "uniform":
|
|
714
|
+
edges = np.linspace(0.0, 1.0, bins + 1)
|
|
715
|
+
idx = np.clip(np.searchsorted(edges, f, side="right") - 1, 0, bins - 1)
|
|
716
|
+
parts = [np.flatnonzero(idx == b) for b in range(bins)]
|
|
717
|
+
else:
|
|
718
|
+
raise ValueError("binning must be 'quantile' or 'uniform'")
|
|
719
|
+
ece, table = 0.0, []
|
|
720
|
+
for part in parts:
|
|
721
|
+
if part.size == 0:
|
|
722
|
+
continue
|
|
723
|
+
mp, my = float(f[part].mean()), float(y[part].mean())
|
|
724
|
+
ece += part.size / y.size * abs(my - mp)
|
|
725
|
+
table.append({"n": int(part.size), "mean_pd": mp, "default_rate": my})
|
|
726
|
+
return make_result(
|
|
727
|
+
"pd.ece",
|
|
728
|
+
metric=ece,
|
|
729
|
+
policy=policy,
|
|
730
|
+
n=int(y.size),
|
|
731
|
+
n_events=int(y.sum()),
|
|
732
|
+
model_id=model_id,
|
|
733
|
+
segment=segment,
|
|
734
|
+
inputs=(y, f, bins, binning),
|
|
735
|
+
extras={"bins": table, "binning": binning},
|
|
736
|
+
)
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
__all__ = [
|
|
740
|
+
"critical_default_count",
|
|
741
|
+
"default_count_sf",
|
|
742
|
+
"pd_binomial",
|
|
743
|
+
"pd_binomial_vasicek",
|
|
744
|
+
"pd_brier",
|
|
745
|
+
"pd_chi2_grades",
|
|
746
|
+
"pd_default_profile",
|
|
747
|
+
"pd_ece",
|
|
748
|
+
"pd_hosmer_lemeshow",
|
|
749
|
+
"pd_jeffreys",
|
|
750
|
+
"pd_jeffreys_grades",
|
|
751
|
+
"pd_normal_multiperiod",
|
|
752
|
+
"pd_spiegelhalter",
|
|
753
|
+
"pd_traffic_lights",
|
|
754
|
+
"traffic_light_colours",
|
|
755
|
+
"traffic_profile_cdf",
|
|
756
|
+
]
|
|
757
|
+
|