riskvalidation 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. riskvalidation/__init__.py +30 -0
  2. riskvalidation/aggregation/__init__.py +11 -0
  3. riskvalidation/aggregation/band.py +151 -0
  4. riskvalidation/aggregation/bounds.py +191 -0
  5. riskvalidation/aggregation/copulas.py +241 -0
  6. riskvalidation/aggregation/model_risk.py +133 -0
  7. riskvalidation/aggregation/simple.py +147 -0
  8. riskvalidation/ccr/__init__.py +13 -0
  9. riskvalidation/ccr/collateral.py +175 -0
  10. riskvalidation/ccr/cva.py +193 -0
  11. riskvalidation/ccr/exposure.py +192 -0
  12. riskvalidation/ccr/hw1f.py +426 -0
  13. riskvalidation/ccr/swaps.py +172 -0
  14. riskvalidation/core/__init__.py +1 -0
  15. riskvalidation/core/build.py +57 -0
  16. riskvalidation/core/hashing.py +48 -0
  17. riskvalidation/core/policy.py +130 -0
  18. riskvalidation/core/registry.py +65 -0
  19. riskvalidation/core/result.py +117 -0
  20. riskvalidation/core/suite.py +78 -0
  21. riskvalidation/engines/__init__.py +2 -0
  22. riskvalidation/engines/low_default.py +170 -0
  23. riskvalidation/engines/pd_curve.py +529 -0
  24. riskvalidation/engines/vasicek.py +72 -0
  25. riskvalidation/fairness/__init__.py +10 -0
  26. riskvalidation/fairness/_checks.py +115 -0
  27. riskvalidation/fairness/conditional.py +201 -0
  28. riskvalidation/fairness/inference.py +64 -0
  29. riskvalidation/fairness/metrics.py +261 -0
  30. riskvalidation/fairness/plants.py +240 -0
  31. riskvalidation/fairness/policy.py +97 -0
  32. riskvalidation/fairness/replication.py +39 -0
  33. riskvalidation/fairness/situation.py +307 -0
  34. riskvalidation/generators/__init__.py +68 -0
  35. riskvalidation/generators/_base.py +145 -0
  36. riskvalidation/generators/counts.py +232 -0
  37. riskvalidation/generators/drift.py +107 -0
  38. riskvalidation/generators/framingham.py +124 -0
  39. riskvalidation/generators/paths.py +349 -0
  40. riskvalidation/generators/ratings.py +186 -0
  41. riskvalidation/generators/scores.py +347 -0
  42. riskvalidation/harness/__init__.py +46 -0
  43. riskvalidation/harness/exact.py +137 -0
  44. riskvalidation/harness/performance.py +74 -0
  45. riskvalidation/harness/rates.py +103 -0
  46. riskvalidation/harness/run.py +197 -0
  47. riskvalidation/irrbb/__init__.py +9 -0
  48. riskvalidation/irrbb/behavioural.py +206 -0
  49. riskvalidation/irrbb/curves.py +286 -0
  50. riskvalidation/irrbb/eve_nii.py +245 -0
  51. riskvalidation/irrbb/shocks.py +171 -0
  52. riskvalidation/irrbb/simulation.py +203 -0
  53. riskvalidation/irrbb/slotting.py +184 -0
  54. riskvalidation/lgd_ead/__init__.py +8 -0
  55. riskvalidation/lgd_ead/_checks.py +50 -0
  56. riskvalidation/lgd_ead/ccf.py +228 -0
  57. riskvalidation/lgd_ead/downturn.py +64 -0
  58. riskvalidation/lgd_ead/floors.py +114 -0
  59. riskvalidation/lgd_ead/lgd_models.py +297 -0
  60. riskvalidation/lgd_ead/workout.py +162 -0
  61. riskvalidation/liquidity/__init__.py +9 -0
  62. riskvalidation/liquidity/_checks.py +78 -0
  63. riskvalidation/liquidity/cbc.py +158 -0
  64. riskvalidation/liquidity/deposits.py +151 -0
  65. riskvalidation/liquidity/ladder.py +119 -0
  66. riskvalidation/liquidity/reverse.py +141 -0
  67. riskvalidation/liquidity/runoff.py +220 -0
  68. riskvalidation/liquidity/scenarios.py +272 -0
  69. riskvalidation/market/__init__.py +8 -0
  70. riskvalidation/market/caviar.py +239 -0
  71. riskvalidation/market/conventions.py +83 -0
  72. riskvalidation/market/evt.py +111 -0
  73. riskvalidation/market/frtb.py +175 -0
  74. riskvalidation/market/garch.py +204 -0
  75. riskvalidation/market/gas_fz.py +219 -0
  76. riskvalidation/market/historical.py +96 -0
  77. riskvalidation/market/parametric.py +78 -0
  78. riskvalidation/market/scoring.py +145 -0
  79. riskvalidation/ml/__init__.py +11 -0
  80. riskvalidation/ml/_checks.py +67 -0
  81. riskvalidation/ml/_logistic.py +82 -0
  82. riskvalidation/ml/cbpe.py +91 -0
  83. riskvalidation/ml/challenger.py +73 -0
  84. riskvalidation/ml/conformal.py +116 -0
  85. riskvalidation/ml/dataquality.py +130 -0
  86. riskvalidation/ml/effects.py +67 -0
  87. riskvalidation/ml/explanations.py +125 -0
  88. riskvalidation/ml/generalisation.py +79 -0
  89. riskvalidation/ml/leakage.py +99 -0
  90. riskvalidation/ml/plants.py +425 -0
  91. riskvalidation/ml/policy.py +182 -0
  92. riskvalidation/ml/robustness.py +286 -0
  93. riskvalidation/ml/seeds.py +106 -0
  94. riskvalidation/oprisk/__init__.py +18 -0
  95. riskvalidation/oprisk/_checks.py +60 -0
  96. riskvalidation/oprisk/aggregate.py +375 -0
  97. riskvalidation/oprisk/distributions.py +758 -0
  98. riskvalidation/oprisk/scenarios.py +342 -0
  99. riskvalidation/oprisk/sla.py +80 -0
  100. riskvalidation/oprisk/sma.py +169 -0
  101. riskvalidation/policies/__init__.py +4 -0
  102. riskvalidation/policies/default.json +273 -0
  103. riskvalidation/portfolio/__init__.py +16 -0
  104. riskvalidation/portfolio/asrf_fit.py +283 -0
  105. riskvalidation/portfolio/copula.py +129 -0
  106. riskvalidation/portfolio/creditrisk_plus.py +232 -0
  107. riskvalidation/portfolio/euler.py +126 -0
  108. riskvalidation/portfolio/granularity.py +135 -0
  109. riskvalidation/portfolio/importance.py +277 -0
  110. riskvalidation/portfolio/portfolio.py +309 -0
  111. riskvalidation/provisions/__init__.py +8 -0
  112. riskvalidation/provisions/_checks.py +100 -0
  113. riskvalidation/provisions/ecl.py +233 -0
  114. riskvalidation/provisions/local_cl.py +360 -0
  115. riskvalidation/provisions/local_es.py +157 -0
  116. riskvalidation/provisions/overlays.py +102 -0
  117. riskvalidation/provisions/reconciliation.py +157 -0
  118. riskvalidation/provisions/staging.py +211 -0
  119. riskvalidation/provisions/term_structure.py +199 -0
  120. riskvalidation/regulatory/__init__.py +14 -0
  121. riskvalidation/regulatory/ba_cva.py +138 -0
  122. riskvalidation/regulatory/frtb.py +345 -0
  123. riskvalidation/regulatory/irb.py +314 -0
  124. riskvalidation/regulatory/lcr_nsfr.py +369 -0
  125. riskvalidation/regulatory/output_floor.py +45 -0
  126. riskvalidation/regulatory/sa_ccr.py +486 -0
  127. riskvalidation/regulatory/standardised.py +348 -0
  128. riskvalidation/stress/__init__.py +9 -0
  129. riskvalidation/stress/capital.py +476 -0
  130. riskvalidation/stress/paths.py +108 -0
  131. riskvalidation/stress/reverse.py +179 -0
  132. riskvalidation/stress/satellites.py +268 -0
  133. riskvalidation/stress/scenario_models.py +274 -0
  134. riskvalidation/transitions/__init__.py +0 -0
  135. riskvalidation/transitions/em.py +120 -0
  136. riskvalidation/transitions/embedding.py +221 -0
  137. riskvalidation/transitions/estimators.py +213 -0
  138. riskvalidation/transitions/intervals.py +306 -0
  139. riskvalidation/transitions/mobility.py +57 -0
  140. riskvalidation/transitions/ttc.py +171 -0
  141. riskvalidation/validation/__init__.py +103 -0
  142. riskvalidation/validation/_common.py +57 -0
  143. riskvalidation/validation/aggregation.py +425 -0
  144. riskvalidation/validation/calibration.py +757 -0
  145. riskvalidation/validation/ccr.py +530 -0
  146. riskvalidation/validation/discrimination.py +314 -0
  147. riskvalidation/validation/fairness.py +632 -0
  148. riskvalidation/validation/ifrs9.py +693 -0
  149. riskvalidation/validation/irrbb.py +389 -0
  150. riskvalidation/validation/lgd_ead.py +469 -0
  151. riskvalidation/validation/liquidity.py +647 -0
  152. riskvalidation/validation/market.py +807 -0
  153. riskvalidation/validation/ml.py +958 -0
  154. riskvalidation/validation/oprisk.py +357 -0
  155. riskvalidation/validation/portfolio.py +353 -0
  156. riskvalidation/validation/rating_system.py +245 -0
  157. riskvalidation/validation/stability.py +256 -0
  158. riskvalidation/validation/stress.py +552 -0
  159. riskvalidation/validation/transitions.py +386 -0
  160. riskvalidation-0.8.0.dist-info/METADATA +136 -0
  161. riskvalidation-0.8.0.dist-info/RECORD +164 -0
  162. riskvalidation-0.8.0.dist-info/WHEEL +5 -0
  163. riskvalidation-0.8.0.dist-info/licenses/LICENSE +21 -0
  164. riskvalidation-0.8.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,757 @@
1
+ """PD calibration tests: is the default rate the model predicted the one that occurred?
2
+
3
+ BCBS Working Paper 14 (2005): "at present no really powerful tests of adequate calibration are currently
4
+ available. Due to the correlation effects that have to be respected there even seems to be no way to develop
5
+ such tests." The tests below therefore come in families with different assumptions (one grade or all grades,
6
+ one period or several, independent or correlated defaults); read them together, never one alone.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import functools
12
+ import math
13
+ from collections.abc import Sequence
14
+
15
+ import numpy as np
16
+ from scipy import special, stats
17
+
18
+ from riskvalidation.core.build import make_result
19
+ from riskvalidation.core.policy import Policy
20
+ from riskvalidation.core.registry import register
21
+ from riskvalidation.core.result import TestResult
22
+ from riskvalidation.validation._common import as_binary, as_float_array, check_counts, check_pd, labels
23
+
24
+ ECB_2019 = (
25
+ "ECB (2019), Instructions for reporting the validation results of internal models: IRB Pillar I models for "
26
+ "credit risk"
27
+ )
28
+ WP14 = "BCBS (2005), Studies on the validation of internal rating systems, Working Paper 14 (revised)"
29
+
30
+ _GH_NODES, _GH_WEIGHTS = np.polynomial.hermite_e.hermegauss(256)
31
+ _GH_WEIGHTS = _GH_WEIGHTS / math.sqrt(2.0 * math.pi)
32
+
33
+
34
+ def _conditional_pd(pd: float, rho: float, x: np.ndarray) -> np.ndarray:
35
+ """PD conditional on the systematic factor x in the one-factor model (Vasicek)."""
36
+ return stats.norm.cdf((stats.norm.ppf(pd) - math.sqrt(rho) * x) / math.sqrt(1.0 - rho))
37
+
38
+
39
+ def default_count_sf(k: int, n: int, pd: float, rho: float = 0.0) -> float:
40
+ """P(D >= k) for n obligors with PD ``pd`` and asset correlation ``rho`` (binomial when rho = 0)."""
41
+ if k <= 0:
42
+ return 1.0
43
+ if rho == 0.0:
44
+ return float(stats.binom.sf(k - 1, n, pd))
45
+ if not 0.0 < rho < 1.0:
46
+ raise ValueError("asset correlation must lie in [0, 1)")
47
+ cond = _conditional_pd(pd, rho, _GH_NODES)
48
+ return float(np.clip(np.sum(_GH_WEIGHTS * stats.binom.sf(k - 1, n, cond)), 0.0, 1.0))
49
+
50
+
51
+ def critical_default_count(n: int, pd: float, q: float = 0.99, rho: float = 0.0) -> int:
52
+ """Smallest k with P(D >= k) <= 1 - q; the test rejects when D >= k.
53
+
54
+ This definition reproduces the critical values tabulated in BCBS WP14 for PD 1%, N 1,000, q 99%
55
+ (19, 35, 49, 63, 77 for rho = 0%, 5%, 10%, 15%, 20%).
56
+ """
57
+ lo, hi = 0, n + 1
58
+ while lo < hi: # P(D >= k) decreases in k
59
+ mid = (lo + hi) // 2
60
+ if default_count_sf(mid, n, pd, rho) <= 1.0 - q:
61
+ hi = mid
62
+ else:
63
+ lo = mid + 1
64
+ return lo
65
+
66
+
67
+ @register(
68
+ "pd.jeffreys",
69
+ family="pd-calibration",
70
+ name="Jeffreys test",
71
+ h0="the PD applied at the start of the period is greater than or equal to the true PD",
72
+ reference=f"{ECB_2019}, section 2.5.3.1; Brown, Cai and DasGupta (2001), doi:10.1214/ss/1009213286",
73
+ decision="p_value",
74
+ )
75
+ def pd_jeffreys(
76
+ n: int,
77
+ d: int,
78
+ pd: float,
79
+ *,
80
+ policy: Policy | None = None,
81
+ model_id: str | None = None,
82
+ segment: str | None = None,
83
+ ) -> TestResult:
84
+ """One-sided Jeffreys test: p = F_Beta(D+1/2, N-D+1/2)(PD), small when the PD is underestimated."""
85
+ n_a, d_a = check_counts(n, d)
86
+ pd_v = float(check_pd(pd)[0])
87
+ nn, dd = int(n_a[0]), int(d_a[0])
88
+ if nn == 0:
89
+ raise ValueError("the Jeffreys test needs at least one obligor")
90
+ a, b = dd + 0.5, nn - dd + 0.5
91
+ p = float(stats.beta.cdf(pd_v, a, b))
92
+ return make_result(
93
+ "pd.jeffreys",
94
+ statistic=dd / nn,
95
+ p_value=p,
96
+ policy=policy,
97
+ alternative="the true PD exceeds the PD applied (underestimation)",
98
+ n=nn,
99
+ n_events=dd,
100
+ model_id=model_id,
101
+ segment=segment,
102
+ inputs=(nn, dd, pd_v),
103
+ extras={
104
+ "pd_applied": pd_v,
105
+ "default_rate": dd / nn,
106
+ "posterior_q025": float(stats.beta.ppf(0.025, a, b)),
107
+ "posterior_q975": float(stats.beta.ppf(0.975, a, b)),
108
+ },
109
+ )
110
+
111
+
112
+ def pd_jeffreys_grades(
113
+ n: Sequence[int],
114
+ d: Sequence[int],
115
+ pd: Sequence[float],
116
+ *,
117
+ grades: Sequence[str] | None = None,
118
+ policy: Policy | None = None,
119
+ model_id: str | None = None,
120
+ ) -> list[TestResult]:
121
+ """K + 1 Jeffreys tests: one per grade and one for the portfolio (ECB 2019, section 2.5.3.1).
122
+
123
+ The portfolio PD is the obligor-weighted average of the grade PDs, the average PD across customers.
124
+ """
125
+ n_a, d_a = check_counts(n, d)
126
+ pd_a = check_pd(pd)
127
+ if pd_a.shape != n_a.shape:
128
+ raise ValueError("pd must have one value per grade")
129
+ names = labels(len(n_a), grades)
130
+ out = [
131
+ pd_jeffreys(int(ni), int(di), float(pi), policy=policy, model_id=model_id, segment=g)
132
+ for ni, di, pi, g in zip(n_a, d_a, pd_a, names, strict=True)
133
+ if ni > 0
134
+ ]
135
+ total = int(n_a.sum())
136
+ pd_port = float(np.sum(n_a * pd_a) / total)
137
+ out.append(pd_jeffreys(total, int(d_a.sum()), pd_port, policy=policy, model_id=model_id, segment="portfolio"))
138
+ return out
139
+
140
+
141
+ @register(
142
+ "pd.binomial",
143
+ family="pd-calibration",
144
+ name="Binomial test",
145
+ h0="the true PD is less than or equal to the PD applied, defaults independent",
146
+ reference=f"{WP14}, section on calibration (binomial test; an early warning when defaults are correlated)",
147
+ decision="p_value",
148
+ )
149
+ def pd_binomial(
150
+ n: int,
151
+ d: int,
152
+ pd: float,
153
+ *,
154
+ q: float = 0.99,
155
+ policy: Policy | None = None,
156
+ model_id: str | None = None,
157
+ segment: str | None = None,
158
+ ) -> TestResult:
159
+ """Exact one-sided binomial test, p = P(Bin(N, PD) >= D)."""
160
+ n_a, d_a = check_counts(n, d)
161
+ pd_v = float(check_pd(pd)[0])
162
+ nn, dd = int(n_a[0]), int(d_a[0])
163
+ p = default_count_sf(dd, nn, pd_v, 0.0)
164
+ k = critical_default_count(nn, pd_v, q, 0.0)
165
+ return make_result(
166
+ "pd.binomial",
167
+ statistic=float(dd),
168
+ p_value=p,
169
+ policy=policy,
170
+ alternative="the true PD exceeds the PD applied",
171
+ n=nn,
172
+ n_events=dd,
173
+ model_id=model_id,
174
+ segment=segment,
175
+ inputs=(nn, dd, pd_v, q),
176
+ notes="assumes independent defaults; WP14: the true type I error can be much larger if defaults are "
177
+ "correlated",
178
+ extras={"critical_count": k, "confidence": q, "expected_defaults": nn * pd_v},
179
+ )
180
+
181
+
182
+ @register(
183
+ "pd.binomial_vasicek",
184
+ family="pd-calibration",
185
+ name="Vasicek-corrected binomial test",
186
+ h0="the true PD is less than or equal to the PD applied, defaults correlated through one systematic factor",
187
+ reference=f"{WP14}, appendix (one-factor model with asset correlation)",
188
+ decision="p_value",
189
+ )
190
+ def pd_binomial_vasicek(
191
+ n: int,
192
+ d: int,
193
+ pd: float,
194
+ rho: float,
195
+ *,
196
+ q: float = 0.99,
197
+ policy: Policy | None = None,
198
+ model_id: str | None = None,
199
+ segment: str | None = None,
200
+ ) -> TestResult:
201
+ """Binomial test under the Vasicek one-factor mixture; the asset correlation is an assumption to document."""
202
+ n_a, d_a = check_counts(n, d)
203
+ pd_v = float(check_pd(pd)[0])
204
+ if not 0.0 <= rho < 1.0:
205
+ raise ValueError("asset correlation must lie in [0, 1)")
206
+ nn, dd = int(n_a[0]), int(d_a[0])
207
+ p = default_count_sf(dd, nn, pd_v, rho)
208
+ k = critical_default_count(nn, pd_v, q, rho)
209
+ approx = nn * float(
210
+ stats.norm.cdf((stats.norm.ppf(pd_v) + math.sqrt(rho) * stats.norm.ppf(q)) / math.sqrt(1.0 - rho))
211
+ )
212
+ return make_result(
213
+ "pd.binomial_vasicek",
214
+ statistic=float(dd),
215
+ p_value=p,
216
+ policy=policy,
217
+ alternative="the true PD exceeds the PD applied",
218
+ n=nn,
219
+ n_events=dd,
220
+ model_id=model_id,
221
+ segment=segment,
222
+ inputs=(nn, dd, pd_v, rho, q),
223
+ notes=f"asset correlation {rho:g} is a modelling assumption",
224
+ extras={"critical_count": k, "confidence": q, "rho": rho, "large_n_critical_count": approx},
225
+ )
226
+
227
+
228
+ @register(
229
+ "pd.chi2_grades",
230
+ family="pd-calibration",
231
+ name="Chi-square test over grades",
232
+ h0="every grade PD equals the true PD of its grade, defaults independent within and between grades",
233
+ reference=f"{WP14}, calibration (Hosmer-Lemeshow type statistic over rating grades)",
234
+ decision="p_value",
235
+ )
236
+ def pd_chi2_grades(
237
+ n: Sequence[int],
238
+ d: Sequence[int],
239
+ pd: Sequence[float],
240
+ *,
241
+ policy: Policy | None = None,
242
+ model_id: str | None = None,
243
+ segment: str | None = None,
244
+ ) -> TestResult:
245
+ """T = sum (n_i p_i - d_i)^2 / (n_i p_i (1 - p_i)), chi-square with as many degrees of freedom as grades."""
246
+ n_a, d_a = check_counts(n, d)
247
+ pd_a = check_pd(pd)
248
+ keep = n_a > 0
249
+ n_a, d_a, pd_a = n_a[keep], d_a[keep], pd_a[keep]
250
+ t = float(np.sum((n_a * pd_a - d_a) ** 2 / (n_a * pd_a * (1.0 - pd_a))))
251
+ dof = int(n_a.size)
252
+ return make_result(
253
+ "pd.chi2_grades",
254
+ statistic=t,
255
+ p_value=float(stats.chi2.sf(t, dof)),
256
+ policy=policy,
257
+ alternative="at least one grade PD differs from the true PD",
258
+ n=int(n_a.sum()),
259
+ n_events=int(d_a.sum()),
260
+ model_id=model_id,
261
+ segment=segment,
262
+ inputs=(n_a, d_a, pd_a),
263
+ notes="WP14: likely to understate the true type I error under default dependence; slow convergence for "
264
+ "small PDs",
265
+ extras={"dof": dof},
266
+ )
267
+
268
+
269
+ @register(
270
+ "pd.default_profile",
271
+ family="pd-calibration",
272
+ name="Chi-square test of the default profile",
273
+ h0="given their number, the defaults fall on the grades as the PD curve implies (the shape of the curve)",
274
+ reference="Tasche (2013), The art of probability-of-default curve calibration, Journal of Credit Risk 9(4), "
275
+ "doi:10.21314/jcr.2013.169, sections 3.3 and 4.2.5; Pearson (1900), doi:10.1080/14786440009463897",
276
+ decision="p_value",
277
+ )
278
+ def pd_default_profile(
279
+ n: Sequence[int],
280
+ d: Sequence[int],
281
+ pd: Sequence[float],
282
+ *,
283
+ n_sim: int = 100_000,
284
+ seed: int = 0,
285
+ policy: Policy | None = None,
286
+ model_id: str | None = None,
287
+ segment: str | None = None,
288
+ ) -> TestResult:
289
+ """T = sum_x (d_x - D q_x)^2 / (D q_x), with q_x = n_x PD_x / sum_t n_t PD_t the implied default profile (3.5b) and
290
+ D the observed number of defaults. Under H0 the default numbers are multinomial(D, q) given D, so the test sees the
291
+ shape of the curve and not its level (read it with a level test such as ``pd.binomial``). Expected counts are small
292
+ in good grades, so the p-value is a Monte Carlo one, (1 + #{T* >= T}) / (n_sim + 1) over ``n_sim`` multinomial
293
+ samples drawn with ``seed``; the asymptotic chi-square p-value with k - 1 degrees of freedom is kept for reference.
294
+ """
295
+ n_a, d_a = check_counts(n, d)
296
+ pd_a = check_pd(pd)
297
+ if pd_a.shape != n_a.shape:
298
+ raise ValueError("pd needs one value per grade")
299
+ if n_sim < 1000:
300
+ raise ValueError("n_sim must be at least 1000")
301
+ keep = n_a > 0
302
+ n_a, d_a, pd_a = n_a[keep], d_a[keep], pd_a[keep]
303
+ total = int(d_a.sum())
304
+ if total == 0 or n_a.size < 2:
305
+ raise ValueError("the default profile needs at least one default and two populated grades")
306
+ q = n_a * pd_a / float(np.sum(n_a * pd_a))
307
+ expected = total * q
308
+ t = float(np.sum((d_a - expected) ** 2 / expected))
309
+ rng = np.random.default_rng(seed)
310
+ at_least, done = 0, 0
311
+ while done < n_sim:
312
+ size = min(50_000, n_sim - done)
313
+ draws = rng.multinomial(total, q, size=size)
314
+ at_least += int(np.sum(np.sum((draws - expected) ** 2 / expected, axis=1) >= t * (1.0 - 1e-12)))
315
+ done += size
316
+ p_mc = (1 + at_least) / (n_sim + 1)
317
+ dof = int(n_a.size - 1)
318
+ return make_result(
319
+ "pd.default_profile",
320
+ statistic=t,
321
+ p_value=float(p_mc),
322
+ policy=policy,
323
+ alternative="the defaults fall on the grades differently from the implied default profile",
324
+ n=int(n_a.sum()),
325
+ n_events=total,
326
+ model_id=model_id,
327
+ segment=segment,
328
+ inputs=(n_a, d_a, pd_a, n_sim, seed),
329
+ notes="Monte Carlo p-value; its standard error is in extras",
330
+ extras={
331
+ "n_sim": int(n_sim),
332
+ "seed": int(seed),
333
+ "mc_standard_error": float(math.sqrt(p_mc * (1.0 - p_mc) / n_sim)),
334
+ "implied_default_profile": [float(v) for v in q],
335
+ "dof": dof,
336
+ "p_value_asymptotic": float(stats.chi2.sf(t, dof)),
337
+ },
338
+ )
339
+
340
+
341
+ @register(
342
+ "pd.hosmer_lemeshow",
343
+ family="pd-calibration",
344
+ name="Hosmer-Lemeshow test",
345
+ h0="the predicted probabilities equal the true default probabilities in every risk group",
346
+ reference="Hosmer and Lemeshow (1980), Communications in Statistics, Theory and Methods 9(10), "
347
+ "doi:10.1080/03610928008827941",
348
+ decision="p_value",
349
+ )
350
+ def pd_hosmer_lemeshow(
351
+ defaults: Sequence[int],
352
+ pd: Sequence[float],
353
+ *,
354
+ groups: int = 10,
355
+ fitted: bool = False,
356
+ dof: int | None = None,
357
+ policy: Policy | None = None,
358
+ model_id: str | None = None,
359
+ segment: str | None = None,
360
+ ) -> TestResult:
361
+ """Groups by quantiles of predicted PD; chi-square with ``groups`` degrees of freedom for PDs given before the
362
+ outcomes (validation), ``groups - 2`` when ``fitted`` (the PDs were estimated on these same observations, the
363
+ original in-sample statistic). ``dof`` overrides both; the choice is recorded.
364
+
365
+ For given PDs each group's standardised difference is asymptotically an independent chi-square with one degree of
366
+ freedom, as BCBS WP14 states for the test over grades. On independent samples with the true PDs, G - 2 rejects
367
+ far more often than the nominal level (the size harness of riskvalidation 0.03.000; see the test's page).
368
+ """
369
+ y = as_binary(defaults)
370
+ p = check_pd(pd)
371
+ if p.shape != y.shape:
372
+ raise ValueError("defaults and pd must have the same length")
373
+ if groups < 3 or groups > y.size:
374
+ raise ValueError("groups must be at least 3 and at most the sample size")
375
+ order = np.argsort(p, kind="mergesort")
376
+ hl = 0.0
377
+ for idx in np.array_split(order, groups):
378
+ o, e, m = float(y[idx].sum()), float(p[idx].sum()), idx.size
379
+ hl += (o - e) ** 2 / (e * (1.0 - e / m))
380
+ k = int(dof) if dof is not None else (groups - 2 if fitted else groups)
381
+ return make_result(
382
+ "pd.hosmer_lemeshow",
383
+ statistic=hl,
384
+ p_value=float(stats.chi2.sf(hl, k)),
385
+ policy=policy,
386
+ alternative="the predicted probabilities are miscalibrated in at least one group",
387
+ n=int(y.size),
388
+ n_events=int(y.sum()),
389
+ model_id=model_id,
390
+ segment=segment,
391
+ inputs=(y, p, groups, k),
392
+ extras={"groups": groups, "dof": k, "fitted": bool(fitted)},
393
+ )
394
+
395
+
396
+ @register(
397
+ "pd.spiegelhalter",
398
+ family="pd-calibration",
399
+ name="Spiegelhalter test",
400
+ h0="every obligor's predicted PD equals its true default probability",
401
+ reference="Spiegelhalter (1986), Statistics in Medicine 5(5), doi:10.1002/sim.4780050506; formula as in the "
402
+ "Stata brier manual",
403
+ decision="p_value",
404
+ )
405
+ def pd_spiegelhalter(
406
+ defaults: Sequence[int],
407
+ pd: Sequence[float],
408
+ *,
409
+ policy: Policy | None = None,
410
+ model_id: str | None = None,
411
+ segment: str | None = None,
412
+ ) -> TestResult:
413
+ """Z = (observed Brier - expected Brier) / sd under H0; one-sided upper tail. Obligor level, no binning."""
414
+ y = as_binary(defaults)
415
+ f = check_pd(pd)
416
+ if f.shape != y.shape:
417
+ raise ValueError("defaults and pd must have the same length")
418
+ nn = y.size
419
+ observed = float(np.mean((f - y) ** 2))
420
+ expected = float(np.mean(f * (1.0 - f)))
421
+ var = float(np.sum(f * (1.0 - f) * (1.0 - 2.0 * f) ** 2) / nn**2)
422
+ z = (observed - expected) / math.sqrt(var)
423
+ return make_result(
424
+ "pd.spiegelhalter",
425
+ statistic=z,
426
+ p_value=float(stats.norm.sf(z)),
427
+ policy=policy,
428
+ alternative="the forecasts are miscalibrated",
429
+ n=int(nn),
430
+ n_events=int(y.sum()),
431
+ model_id=model_id,
432
+ segment=segment,
433
+ inputs=(y, f),
434
+ extras={"brier_observed": observed, "brier_expected": expected, "brier_variance": var},
435
+ )
436
+
437
+
438
+ @register(
439
+ "pd.normal_multiperiod",
440
+ family="pd-calibration",
441
+ name="Normal test over several periods",
442
+ h0="none of the true PDs in years 1 to T is greater than its forecast",
443
+ reference=f"{WP14}, calibration (normal test; cross-sectional dependence admissible, independence over time)",
444
+ decision="p_value",
445
+ )
446
+ def pd_normal_multiperiod(
447
+ default_rates: Sequence[float],
448
+ pd: Sequence[float],
449
+ *,
450
+ policy: Policy | None = None,
451
+ model_id: str | None = None,
452
+ segment: str | None = None,
453
+ ) -> TestResult:
454
+ """sum(d_t - PD_t) / (sqrt(T) sigma_hat), compared with the standard normal upper tail."""
455
+ dr = as_float_array(default_rates, "default_rates")
456
+ pdt = check_pd(pd)
457
+ if dr.shape != pdt.shape:
458
+ raise ValueError("default rates and PDs must have one value per year")
459
+ t = dr.size
460
+ if t < 2:
461
+ raise ValueError("the multi-period normal test needs at least two years")
462
+ e = dr - pdt
463
+ var = (np.sum(e**2) - np.sum(e) ** 2 / t) / (t - 1)
464
+ if var <= 0:
465
+ return make_result(
466
+ "pd.normal_multiperiod",
467
+ statistic=float("nan"),
468
+ policy=policy,
469
+ n=t,
470
+ model_id=model_id,
471
+ segment=segment,
472
+ inputs=(dr, pdt),
473
+ notes="the estimated variance of the yearly differences is zero; the statistic is undefined",
474
+ )
475
+ stat = float(np.sum(e) / (math.sqrt(t) * math.sqrt(var)))
476
+ return make_result(
477
+ "pd.normal_multiperiod",
478
+ statistic=stat,
479
+ p_value=float(special.ndtr(-stat)),
480
+ policy=policy,
481
+ alternative="at least one yearly PD is underestimated",
482
+ n=t,
483
+ model_id=model_id,
484
+ segment=segment,
485
+ inputs=(dr, pdt),
486
+ notes="above the nominal level over short series: measured size 8.5% at the 5% level for five years of 1,000 "
487
+ "obligors at PD 3% (WP14 Table 7: 8.1%)",
488
+ extras={"years": t, "sigma_hat": math.sqrt(var)},
489
+ )
490
+
491
+
492
+ WP14_TRAFFIC_PROBS = (0.5, 0.3, 0.15, 0.05)
493
+
494
+
495
+ @functools.lru_cache(maxsize=64)
496
+ def _traffic_profile_cdf(t: int, probs: tuple[float, ...]) -> dict[tuple[int, int, int, int], float]:
497
+ """P(profile <= p) for every colour profile p of ``t`` years, the profiles in lexicographic order of
498
+ (A_g, A_y, A_o, A_r); the same additions in the same order as a full enumeration, computed once per (t, probs)."""
499
+ out: dict[tuple[int, int, int, int], float] = {}
500
+ acc = 0.0
501
+ for a_g in range(t + 1):
502
+ for a_y in range(t - a_g + 1):
503
+ for a_o in range(t - a_g - a_y + 1):
504
+ prof = (a_g, a_y, a_o, t - a_g - a_y - a_o)
505
+ acc += float(stats.multinomial.pmf(prof, t, probs))
506
+ out[prof] = min(1.0, acc)
507
+ return out
508
+
509
+
510
+ @functools.lru_cache(maxsize=64)
511
+ def _traffic_cuts(q: tuple[float, ...]) -> np.ndarray:
512
+ """Phi^-1 of the cumulative colour probabilities (norm.ppf is ndtri inside SciPy)."""
513
+ return special.ndtri(np.cumsum(q)[:3])
514
+
515
+
516
+ def _traffic_colours(n_a: np.ndarray, d_a: np.ndarray, pd_a: np.ndarray, q: np.ndarray,
517
+ ties: str) -> tuple[np.ndarray, np.ndarray]:
518
+ if q.shape != (4,) or np.any(q <= 0) or not math.isclose(float(q.sum()), 1.0, abs_tol=1e-12):
519
+ raise ValueError("probs must be four positive numbers summing to 1")
520
+ if ties not in ("lower", "upper"):
521
+ raise ValueError("ties is 'lower' (the printed WP14 mapping) or 'upper'")
522
+ r = (d_a - n_a * pd_a) / np.sqrt(n_a * pd_a * (1.0 - pd_a))
523
+ cuts = _traffic_cuts(tuple(float(v) for v in q))
524
+ tol = 1e-12 # a rate within rounding of a boundary is on it
525
+ if ties == "lower":
526
+ colour = np.sum(cuts[None, :] < r[:, None] - tol, axis=1) # 0 green, 1 yellow, 2 orange, 3 red
527
+ else:
528
+ colour = np.sum(cuts[None, :] <= r[:, None] + tol, axis=1)
529
+ return r, colour
530
+
531
+
532
+ def traffic_light_colours(
533
+ n: Sequence[int], d: Sequence[int], pd: Sequence[float], *, probs: Sequence[float] = WP14_TRAFFIC_PROBS,
534
+ ties: str = "lower",
535
+ ) -> np.ndarray:
536
+ """The colour of each year in the traffic-lights test, 0 green to 3 red: the standardised default rate
537
+ (D - N p) / sqrt(N p (1 - p)) against Phi^-1 of the cumulative colour probabilities (WP14), with ``ties`` as in
538
+ :func:`pd_traffic_lights`."""
539
+ n_a, d_a = check_counts(n, d)
540
+ pd_a = check_pd(pd)
541
+ if pd_a.shape != n_a.shape:
542
+ raise ValueError("one PD per year is required")
543
+ return _traffic_colours(n_a, d_a, pd_a, np.asarray(probs, dtype=float), ties)[1]
544
+
545
+
546
+ def traffic_profile_cdf(t: int, probs: Sequence[float] = WP14_TRAFFIC_PROBS) -> dict[tuple[int, int, int, int], float]:
547
+ """P(profile <= p) under the multinomial null for every colour profile of ``t`` years (lexicographic order)."""
548
+ return dict(_traffic_profile_cdf(int(t), tuple(float(v) for v in probs)))
549
+
550
+
551
+ @register(
552
+ "pd.traffic_lights",
553
+ family="pd-calibration",
554
+ name="Traffic-lights test (Blochwitz, Hohl and Wehn)",
555
+ h0="the yearly PDs are correct; the colour counts are multinomial with the stated probabilities",
556
+ reference=f"{WP14}, traffic lights approach (Blochwitz, Hohl and Wehn); probabilities 0.5, 0.3, 0.15, 0.05",
557
+ decision="p_value",
558
+ )
559
+ def pd_traffic_lights(
560
+ n: Sequence[int],
561
+ d: Sequence[int],
562
+ pd: Sequence[float],
563
+ *,
564
+ probs: Sequence[float] = WP14_TRAFFIC_PROBS,
565
+ ties: str = "lower",
566
+ policy: Policy | None = None,
567
+ model_id: str | None = None,
568
+ segment: str | None = None,
569
+ ) -> TestResult:
570
+ """Colour each year by its standardised default rate; p = P(colour profile no better than observed).
571
+
572
+ Profiles are ordered as V = 1000 A_g + 100 A_y + 10 A_o + A_r (WP14), which for any T is the lexicographic
573
+ order of (A_g, A_y, A_o, A_r); P(V <= v_obs) is a lookup in the exact multinomial table.
574
+
575
+ ``ties`` decides the colour of a year whose standardised rate sits on a colour boundary (within 1e-12), which
576
+ happens with positive probability when N p is an integer (a year with exactly the expected number of defaults):
577
+ ``"lower"`` gives it the better colour, as WP14's printed mapping does (green if x <= Phi^-1(q_g)); ``"upper"``
578
+ gives it the worse one, which is what WP14's own simulation study must have used: its Tables 7 and 8 are
579
+ reproduced exactly with ``"upper"`` and not with the printed rule (riskvalidation 0.03.000, the size harness).
580
+ """
581
+ n_a, d_a = check_counts(n, d)
582
+ pd_a = check_pd(pd)
583
+ if pd_a.shape != n_a.shape:
584
+ raise ValueError("one PD per year is required")
585
+ q = np.asarray(probs, dtype=float)
586
+ r, colour = _traffic_colours(n_a, d_a, pd_a, q, ties)
587
+ counts = tuple(int(np.sum(colour == c)) for c in range(4))
588
+ t = int(n_a.size)
589
+ p_le = _traffic_profile_cdf(t, tuple(float(v) for v in q))[counts]
590
+ v = 1000 * counts[0] + 100 * counts[1] + 10 * counts[2] + counts[3]
591
+ return make_result(
592
+ "pd.traffic_lights",
593
+ statistic=float(v),
594
+ p_value=min(1.0, p_le),
595
+ policy=policy,
596
+ alternative="the PDs are underestimated in too many years",
597
+ n=t,
598
+ n_events=int(d_a.sum()),
599
+ model_id=model_id,
600
+ segment=segment,
601
+ inputs=(n_a, d_a, pd_a, q, ties),
602
+ extras={
603
+ "ties": ties,
604
+ "green": counts[0],
605
+ "yellow": counts[1],
606
+ "orange": counts[2],
607
+ "red": counts[3],
608
+ "standardised_rates": r.tolist(),
609
+ "colours": [("green", "yellow", "orange", "red")[c] for c in colour],
610
+ },
611
+ )
612
+
613
+
614
+ @register(
615
+ "pd.brier",
616
+ family="pd-calibration",
617
+ name="Brier score with Murphy decomposition",
618
+ h0="descriptive: mean squared error of the PD forecasts",
619
+ reference="Brier (1950), doi:10.1175/1520-0493(1950)078<0001:VOFEIT>2.0.CO;2; Murphy (1973), "
620
+ "doi:10.1175/1520-0450(1973)012<0595:ANVPOT>2.0.CO;2",
621
+ decision="metric",
622
+ )
623
+ def pd_brier(
624
+ defaults: Sequence[int],
625
+ pd: Sequence[float],
626
+ *,
627
+ groups: Sequence[str] | None = None,
628
+ bins: int = 10,
629
+ policy: Policy | None = None,
630
+ model_id: str | None = None,
631
+ segment: str | None = None,
632
+ ) -> TestResult:
633
+ """BS = reliability - resolution + uncertainty; exact when forecasts are constant within each group.
634
+
635
+ Groups are rating grades when given, otherwise ``bins`` quantile bins of the forecast. The Brier skill
636
+ against the trivial forecast (the sample default rate) is reported, since low default rates make any Brier
637
+ score look small (WP14).
638
+ """
639
+ y = as_binary(defaults)
640
+ f = check_pd(pd)
641
+ if f.shape != y.shape:
642
+ raise ValueError("defaults and pd must have the same length")
643
+ nn = y.size
644
+ if groups is not None:
645
+ g = np.asarray(groups)
646
+ if g.shape != y.shape:
647
+ raise ValueError("groups must have one label per obligor")
648
+ _, gid = np.unique(g, return_inverse=True)
649
+ else:
650
+ order = np.argsort(f, kind="mergesort")
651
+ gid = np.empty(nn, dtype=int)
652
+ for k, idx in enumerate(np.array_split(order, min(bins, nn))):
653
+ gid[idx] = k
654
+ bs = float(np.mean((f - y) ** 2))
655
+ dbar = float(y.mean())
656
+ rel = res = 0.0
657
+ exact = True
658
+ for k in np.unique(gid):
659
+ m = gid == k
660
+ nk = int(m.sum())
661
+ fk, dk = float(f[m].mean()), float(y[m].mean())
662
+ rel += nk * (fk - dk) ** 2
663
+ res += nk * (dk - dbar) ** 2
664
+ exact = exact and bool(np.ptp(f[m]) == 0.0)
665
+ rel, res = rel / nn, res / nn
666
+ unc = dbar * (1.0 - dbar)
667
+ return make_result(
668
+ "pd.brier",
669
+ metric=bs,
670
+ policy=policy,
671
+ n=int(nn),
672
+ n_events=int(y.sum()),
673
+ model_id=model_id,
674
+ segment=segment,
675
+ inputs=(y, f, gid),
676
+ extras={
677
+ "reliability": rel,
678
+ "resolution": res,
679
+ "uncertainty": unc,
680
+ "brier_skill": (1.0 - bs / unc) if unc > 0 else None,
681
+ "decomposition_exact": exact,
682
+ },
683
+ )
684
+
685
+
686
+ @register(
687
+ "pd.ece",
688
+ family="pd-calibration",
689
+ name="Expected calibration error",
690
+ h0="descriptive: weighted mean absolute gap between forecast and observed default rate per bin",
691
+ reference="Naeini, Cooper and Hauskrecht (2015), AAAI, doi:10.1609/aaai.v29i1.9602; Guo et al. (2017), "
692
+ "arXiv:1706.04599",
693
+ decision="metric",
694
+ )
695
+ def pd_ece(
696
+ defaults: Sequence[int],
697
+ pd: Sequence[float],
698
+ *,
699
+ bins: int = 10,
700
+ binning: str = "quantile",
701
+ policy: Policy | None = None,
702
+ model_id: str | None = None,
703
+ segment: str | None = None,
704
+ ) -> TestResult:
705
+ """ECE = sum_m |B_m|/n |mean(y) - mean(p)|; quantile bins by default, because PDs cluster near zero."""
706
+ y = as_binary(defaults)
707
+ f = check_pd(pd)
708
+ if f.shape != y.shape:
709
+ raise ValueError("defaults and pd must have the same length")
710
+ if binning == "quantile":
711
+ order = np.argsort(f, kind="mergesort")
712
+ parts = np.array_split(order, min(bins, y.size))
713
+ elif binning == "uniform":
714
+ edges = np.linspace(0.0, 1.0, bins + 1)
715
+ idx = np.clip(np.searchsorted(edges, f, side="right") - 1, 0, bins - 1)
716
+ parts = [np.flatnonzero(idx == b) for b in range(bins)]
717
+ else:
718
+ raise ValueError("binning must be 'quantile' or 'uniform'")
719
+ ece, table = 0.0, []
720
+ for part in parts:
721
+ if part.size == 0:
722
+ continue
723
+ mp, my = float(f[part].mean()), float(y[part].mean())
724
+ ece += part.size / y.size * abs(my - mp)
725
+ table.append({"n": int(part.size), "mean_pd": mp, "default_rate": my})
726
+ return make_result(
727
+ "pd.ece",
728
+ metric=ece,
729
+ policy=policy,
730
+ n=int(y.size),
731
+ n_events=int(y.sum()),
732
+ model_id=model_id,
733
+ segment=segment,
734
+ inputs=(y, f, bins, binning),
735
+ extras={"bins": table, "binning": binning},
736
+ )
737
+
738
+
739
+ __all__ = [
740
+ "critical_default_count",
741
+ "default_count_sf",
742
+ "pd_binomial",
743
+ "pd_binomial_vasicek",
744
+ "pd_brier",
745
+ "pd_chi2_grades",
746
+ "pd_default_profile",
747
+ "pd_ece",
748
+ "pd_hosmer_lemeshow",
749
+ "pd_jeffreys",
750
+ "pd_jeffreys_grades",
751
+ "pd_normal_multiperiod",
752
+ "pd_spiegelhalter",
753
+ "pd_traffic_lights",
754
+ "traffic_light_colours",
755
+ "traffic_profile_cdf",
756
+ ]
757
+