riskvalidation 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. riskvalidation/__init__.py +30 -0
  2. riskvalidation/aggregation/__init__.py +11 -0
  3. riskvalidation/aggregation/band.py +151 -0
  4. riskvalidation/aggregation/bounds.py +191 -0
  5. riskvalidation/aggregation/copulas.py +241 -0
  6. riskvalidation/aggregation/model_risk.py +133 -0
  7. riskvalidation/aggregation/simple.py +147 -0
  8. riskvalidation/ccr/__init__.py +13 -0
  9. riskvalidation/ccr/collateral.py +175 -0
  10. riskvalidation/ccr/cva.py +193 -0
  11. riskvalidation/ccr/exposure.py +192 -0
  12. riskvalidation/ccr/hw1f.py +426 -0
  13. riskvalidation/ccr/swaps.py +172 -0
  14. riskvalidation/core/__init__.py +1 -0
  15. riskvalidation/core/build.py +57 -0
  16. riskvalidation/core/hashing.py +48 -0
  17. riskvalidation/core/policy.py +130 -0
  18. riskvalidation/core/registry.py +65 -0
  19. riskvalidation/core/result.py +117 -0
  20. riskvalidation/core/suite.py +78 -0
  21. riskvalidation/engines/__init__.py +2 -0
  22. riskvalidation/engines/low_default.py +170 -0
  23. riskvalidation/engines/pd_curve.py +529 -0
  24. riskvalidation/engines/vasicek.py +72 -0
  25. riskvalidation/fairness/__init__.py +10 -0
  26. riskvalidation/fairness/_checks.py +115 -0
  27. riskvalidation/fairness/conditional.py +201 -0
  28. riskvalidation/fairness/inference.py +64 -0
  29. riskvalidation/fairness/metrics.py +261 -0
  30. riskvalidation/fairness/plants.py +240 -0
  31. riskvalidation/fairness/policy.py +97 -0
  32. riskvalidation/fairness/replication.py +39 -0
  33. riskvalidation/fairness/situation.py +307 -0
  34. riskvalidation/generators/__init__.py +68 -0
  35. riskvalidation/generators/_base.py +145 -0
  36. riskvalidation/generators/counts.py +232 -0
  37. riskvalidation/generators/drift.py +107 -0
  38. riskvalidation/generators/framingham.py +124 -0
  39. riskvalidation/generators/paths.py +349 -0
  40. riskvalidation/generators/ratings.py +186 -0
  41. riskvalidation/generators/scores.py +347 -0
  42. riskvalidation/harness/__init__.py +46 -0
  43. riskvalidation/harness/exact.py +137 -0
  44. riskvalidation/harness/performance.py +74 -0
  45. riskvalidation/harness/rates.py +103 -0
  46. riskvalidation/harness/run.py +197 -0
  47. riskvalidation/irrbb/__init__.py +9 -0
  48. riskvalidation/irrbb/behavioural.py +206 -0
  49. riskvalidation/irrbb/curves.py +286 -0
  50. riskvalidation/irrbb/eve_nii.py +245 -0
  51. riskvalidation/irrbb/shocks.py +171 -0
  52. riskvalidation/irrbb/simulation.py +203 -0
  53. riskvalidation/irrbb/slotting.py +184 -0
  54. riskvalidation/lgd_ead/__init__.py +8 -0
  55. riskvalidation/lgd_ead/_checks.py +50 -0
  56. riskvalidation/lgd_ead/ccf.py +228 -0
  57. riskvalidation/lgd_ead/downturn.py +64 -0
  58. riskvalidation/lgd_ead/floors.py +114 -0
  59. riskvalidation/lgd_ead/lgd_models.py +297 -0
  60. riskvalidation/lgd_ead/workout.py +162 -0
  61. riskvalidation/liquidity/__init__.py +9 -0
  62. riskvalidation/liquidity/_checks.py +78 -0
  63. riskvalidation/liquidity/cbc.py +158 -0
  64. riskvalidation/liquidity/deposits.py +151 -0
  65. riskvalidation/liquidity/ladder.py +119 -0
  66. riskvalidation/liquidity/reverse.py +141 -0
  67. riskvalidation/liquidity/runoff.py +220 -0
  68. riskvalidation/liquidity/scenarios.py +272 -0
  69. riskvalidation/market/__init__.py +8 -0
  70. riskvalidation/market/caviar.py +239 -0
  71. riskvalidation/market/conventions.py +83 -0
  72. riskvalidation/market/evt.py +111 -0
  73. riskvalidation/market/frtb.py +175 -0
  74. riskvalidation/market/garch.py +204 -0
  75. riskvalidation/market/gas_fz.py +219 -0
  76. riskvalidation/market/historical.py +96 -0
  77. riskvalidation/market/parametric.py +78 -0
  78. riskvalidation/market/scoring.py +145 -0
  79. riskvalidation/ml/__init__.py +11 -0
  80. riskvalidation/ml/_checks.py +67 -0
  81. riskvalidation/ml/_logistic.py +82 -0
  82. riskvalidation/ml/cbpe.py +91 -0
  83. riskvalidation/ml/challenger.py +73 -0
  84. riskvalidation/ml/conformal.py +116 -0
  85. riskvalidation/ml/dataquality.py +130 -0
  86. riskvalidation/ml/effects.py +67 -0
  87. riskvalidation/ml/explanations.py +125 -0
  88. riskvalidation/ml/generalisation.py +79 -0
  89. riskvalidation/ml/leakage.py +99 -0
  90. riskvalidation/ml/plants.py +425 -0
  91. riskvalidation/ml/policy.py +182 -0
  92. riskvalidation/ml/robustness.py +286 -0
  93. riskvalidation/ml/seeds.py +106 -0
  94. riskvalidation/oprisk/__init__.py +18 -0
  95. riskvalidation/oprisk/_checks.py +60 -0
  96. riskvalidation/oprisk/aggregate.py +375 -0
  97. riskvalidation/oprisk/distributions.py +758 -0
  98. riskvalidation/oprisk/scenarios.py +342 -0
  99. riskvalidation/oprisk/sla.py +80 -0
  100. riskvalidation/oprisk/sma.py +169 -0
  101. riskvalidation/policies/__init__.py +4 -0
  102. riskvalidation/policies/default.json +273 -0
  103. riskvalidation/portfolio/__init__.py +16 -0
  104. riskvalidation/portfolio/asrf_fit.py +283 -0
  105. riskvalidation/portfolio/copula.py +129 -0
  106. riskvalidation/portfolio/creditrisk_plus.py +232 -0
  107. riskvalidation/portfolio/euler.py +126 -0
  108. riskvalidation/portfolio/granularity.py +135 -0
  109. riskvalidation/portfolio/importance.py +277 -0
  110. riskvalidation/portfolio/portfolio.py +309 -0
  111. riskvalidation/provisions/__init__.py +8 -0
  112. riskvalidation/provisions/_checks.py +100 -0
  113. riskvalidation/provisions/ecl.py +233 -0
  114. riskvalidation/provisions/local_cl.py +360 -0
  115. riskvalidation/provisions/local_es.py +157 -0
  116. riskvalidation/provisions/overlays.py +102 -0
  117. riskvalidation/provisions/reconciliation.py +157 -0
  118. riskvalidation/provisions/staging.py +211 -0
  119. riskvalidation/provisions/term_structure.py +199 -0
  120. riskvalidation/regulatory/__init__.py +14 -0
  121. riskvalidation/regulatory/ba_cva.py +138 -0
  122. riskvalidation/regulatory/frtb.py +345 -0
  123. riskvalidation/regulatory/irb.py +314 -0
  124. riskvalidation/regulatory/lcr_nsfr.py +369 -0
  125. riskvalidation/regulatory/output_floor.py +45 -0
  126. riskvalidation/regulatory/sa_ccr.py +486 -0
  127. riskvalidation/regulatory/standardised.py +348 -0
  128. riskvalidation/stress/__init__.py +9 -0
  129. riskvalidation/stress/capital.py +476 -0
  130. riskvalidation/stress/paths.py +108 -0
  131. riskvalidation/stress/reverse.py +179 -0
  132. riskvalidation/stress/satellites.py +268 -0
  133. riskvalidation/stress/scenario_models.py +274 -0
  134. riskvalidation/transitions/__init__.py +0 -0
  135. riskvalidation/transitions/em.py +120 -0
  136. riskvalidation/transitions/embedding.py +221 -0
  137. riskvalidation/transitions/estimators.py +213 -0
  138. riskvalidation/transitions/intervals.py +306 -0
  139. riskvalidation/transitions/mobility.py +57 -0
  140. riskvalidation/transitions/ttc.py +171 -0
  141. riskvalidation/validation/__init__.py +103 -0
  142. riskvalidation/validation/_common.py +57 -0
  143. riskvalidation/validation/aggregation.py +425 -0
  144. riskvalidation/validation/calibration.py +757 -0
  145. riskvalidation/validation/ccr.py +530 -0
  146. riskvalidation/validation/discrimination.py +314 -0
  147. riskvalidation/validation/fairness.py +632 -0
  148. riskvalidation/validation/ifrs9.py +693 -0
  149. riskvalidation/validation/irrbb.py +389 -0
  150. riskvalidation/validation/lgd_ead.py +469 -0
  151. riskvalidation/validation/liquidity.py +647 -0
  152. riskvalidation/validation/market.py +807 -0
  153. riskvalidation/validation/ml.py +958 -0
  154. riskvalidation/validation/oprisk.py +357 -0
  155. riskvalidation/validation/portfolio.py +353 -0
  156. riskvalidation/validation/rating_system.py +245 -0
  157. riskvalidation/validation/stability.py +256 -0
  158. riskvalidation/validation/stress.py +552 -0
  159. riskvalidation/validation/transitions.py +386 -0
  160. riskvalidation-0.8.0.dist-info/METADATA +136 -0
  161. riskvalidation-0.8.0.dist-info/RECORD +164 -0
  162. riskvalidation-0.8.0.dist-info/WHEEL +5 -0
  163. riskvalidation-0.8.0.dist-info/licenses/LICENSE +21 -0
  164. riskvalidation-0.8.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,145 @@
1
+ """The FZ0 and tick losses and the comparison tests (dossier 05 A.1 and B.6).
2
+
3
+ All forecasts are in the return convention (VaR and ES negative decimals, `e <= v`). The FZ0 loss is a score: it can be
4
+ negative and only differences between models are meaningful.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import math
10
+ from dataclasses import dataclass
11
+
12
+ import numpy as np
13
+ from scipy import stats
14
+
15
+ from riskvalidation.market.conventions import check_alpha, vec
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class Scores:
20
+ n: int # days scored for the tick loss
21
+ n_fz0: int # days with both VaR and ES defined
22
+ fz0_mean: float | None # None when n_fz0 = 0
23
+ tick_mean: float
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class DmResult:
28
+ mean_diff: float # mean of loss_a - loss_b
29
+ statistic: float
30
+ p_value: float # two-sided, normal
31
+ n: int
32
+ lags: int
33
+
34
+
35
+ @dataclass(frozen=True)
36
+ class McsResult:
37
+ members: list[int] # indices of the models in the set
38
+ p_values: np.ndarray # (M,) MCS p-values
39
+ confidence: float
40
+ statistic: str # "Tmax" or "TR"
41
+
42
+
43
+ def _same_shape(**arrays: np.ndarray) -> None:
44
+ shapes = {k: a.shape for k, a in arrays.items()}
45
+ if len(set(shapes.values())) != 1:
46
+ raise ValueError(f"the inputs must have the same shape, got {shapes}")
47
+
48
+
49
+ def fz0_loss(y: np.ndarray, v: np.ndarray, e: np.ndarray, alpha: float) -> np.ndarray:
50
+ """`L = -(1/(alpha e)) 1{y <= v}(v - y) + v/e + log(-e) - 1` (dossier 05 A.1, arXiv 1707.05108 eq. 6), per day.
51
+
52
+ The Fissler-Ziegel loss of eq. 4 with `G1 = 0`, `G2(x) = -1/x` and `calG2(x) = -log(-x)`. ValueError unless `v < 0`,
53
+ `e < 0` and `e <= v` elementwise.
54
+ """
55
+ alpha = check_alpha(alpha)
56
+ y, v, e = (np.asarray(a, dtype=float) for a in (y, v, e))
57
+ _same_shape(y=y, v=v, e=e)
58
+ if not (np.all(np.isfinite(y)) and np.all(np.isfinite(v)) and np.all(np.isfinite(e))):
59
+ raise ValueError("y, v and e must be finite")
60
+ if np.any(v >= 0.0) or np.any(e >= 0.0) or np.any(e > v):
61
+ raise ValueError("the FZ0 loss needs v < 0, e < 0 and e <= v elementwise")
62
+ hit = (y <= v).astype(float)
63
+ return -(1.0 / (alpha * e)) * hit * (v - y) + v / e + np.log(-e) - 1.0
64
+
65
+
66
+ def tick_loss(y: np.ndarray, v: np.ndarray, alpha: float) -> np.ndarray:
67
+ """`(alpha - 1{y < v})(y - v)` per day (dossier 05 A.2 row 11): nonnegative, zero at `y = v`."""
68
+ alpha = check_alpha(alpha)
69
+ y, v = np.asarray(y, dtype=float), np.asarray(v, dtype=float)
70
+ _same_shape(y=y, v=v)
71
+ return (alpha - (y < v)) * (y - v)
72
+
73
+
74
+ def mean_scores(y: np.ndarray, v: np.ndarray, e: np.ndarray, alpha: float) -> Scores:
75
+ """Mean tick loss over all `n` days and mean FZ0 over the days with a finite `e` (NaN where the ES is undefined)."""
76
+ alpha = check_alpha(alpha)
77
+ y, v = vec(y, "y"), vec(v, "v")
78
+ e = np.array(e, dtype=float)
79
+ _same_shape(y=y, v=v, e=e)
80
+ ok = np.isfinite(e)
81
+ n_fz0 = int(ok.sum())
82
+ fz0 = float(np.mean(fz0_loss(y[ok], v[ok], e[ok], alpha))) if n_fz0 else None
83
+ return Scores(n=int(y.size), n_fz0=n_fz0, fz0_mean=fz0, tick_mean=float(np.mean(tick_loss(y, v, alpha))))
84
+
85
+
86
+ def diebold_mariano(loss_a: np.ndarray, loss_b: np.ndarray, lags: int) -> DmResult:
87
+ """Diebold-Mariano (1995): `d_t = loss_a_t - loss_b_t`, `DM = mean(d) / sqrt(LRV / T)` with the Newey-West
88
+ (Bartlett) long-run variance `LRV = g0 + 2 sum_{j=1}^{lags} (1 - j/(lags+1)) g_j`, `g_j = (1/T) sum_t (d_t -
89
+ mean)(d_{t-j} - mean)` (decision D-2) and `p_value = 2 (1 - Phi(|DM|))`. A negative `mean_diff` means model A has
90
+ the lower loss. One-step forecasts use `lags = 0`. ValueError if `d` has no spread or `T <= lags + 1`."""
91
+ a, b = vec(loss_a, "loss_a"), vec(loss_b, "loss_b")
92
+ _same_shape(loss_a=a, loss_b=b)
93
+ if int(lags) != lags or lags < 0:
94
+ raise ValueError(f"lags must be a non-negative integer, got {lags}")
95
+ d = a - b
96
+ t = d.size
97
+ if t <= lags + 1:
98
+ raise ValueError(f"need more than lags + 1 = {lags + 1} observations, got {t}")
99
+ mean = float(d.mean())
100
+ c = d - mean
101
+ if float(np.max(np.abs(c))) <= 1e-14 * max(1.0, abs(mean)):
102
+ raise ValueError("the loss differential has no spread: the statistic is undefined")
103
+ lrv = float(c @ c) / t
104
+ for j in range(1, lags + 1):
105
+ lrv += 2.0 * (1.0 - j / (lags + 1.0)) * float(c[j:] @ c[:-j]) / t
106
+ if lrv <= 0.0:
107
+ raise ValueError("the long-run variance is not positive: the statistic is undefined")
108
+ z = mean / math.sqrt(lrv / t)
109
+ return DmResult(mean_diff=mean, statistic=float(z), p_value=float(2.0 * stats.norm.sf(abs(z))), n=int(t),
110
+ lags=int(lags))
111
+
112
+
113
+ def model_confidence_set(losses: np.ndarray, confidence: float = 0.90, statistic: str = "TR", n_bootstrap: int = 1000,
114
+ block_length: int = 10, seed: int = 0) -> McsResult:
115
+ """The Model Confidence Set of Hansen, Lunde and Nason (2011) through `arch.bootstrap.MCS` (stationary bootstrap).
116
+
117
+ `losses` is `(T, M)`; a model is in the set iff its MCS p-value is at least `1 - confidence`; the set is never empty
118
+ (the last survivor has p-value 1); `M = 1` gives that model with p-value 1. Same inputs and seed, same result."""
119
+ if not (0.0 < confidence < 1.0):
120
+ raise ValueError(f"confidence must lie in (0, 1), got {confidence}")
121
+ if statistic not in ("Tmax", "TR"):
122
+ raise ValueError(f"statistic must be 'Tmax' or 'TR', got {statistic!r}")
123
+ x = np.array(losses, dtype=float)
124
+ if x.ndim != 2 or x.shape[0] < 2:
125
+ raise ValueError(f"losses must be a (T, M) array with T >= 2, got shape {x.shape}")
126
+ if not np.all(np.isfinite(x)):
127
+ raise ValueError("losses contain non-finite values")
128
+ m = x.shape[1]
129
+ if m == 1:
130
+ return McsResult(members=[0], p_values=np.array([1.0]), confidence=float(confidence), statistic=statistic)
131
+ try:
132
+ import pandas as pd # noqa: PLC0415
133
+ from arch.bootstrap import MCS # noqa: PLC0415
134
+ except ImportError as exc: # pragma: no cover
135
+ raise ImportError(
136
+ "model_confidence_set needs the optional package `arch`: pip install 'riskvalidation[market]'"
137
+ ) from exc
138
+ mcs = MCS(pd.DataFrame(x), size=1.0 - confidence, reps=int(n_bootstrap), block_size=int(block_length),
139
+ method="R" if statistic == "TR" else "max", seed=int(seed))
140
+ mcs.compute()
141
+ pv = np.empty(m)
142
+ for name, row in mcs.pvalues.iterrows():
143
+ pv[int(name)] = float(row["Pvalue"])
144
+ members = [i for i in range(m) if pv[i] >= 1.0 - confidence]
145
+ return McsResult(members=members, p_values=pv, confidence=float(confidence), statistic=statistic)
@@ -0,0 +1,11 @@
1
+ """ML model validation: perturbation robustness and shift resilience, seed stability, explanation checks, effect curves,
2
+ generalisation, split conformal prediction, the leakage screen, data quality, the label-free performance estimate, the
3
+ champion and challenger comparison and the planted-defect generators.
4
+
5
+ Modules are imported by path (for example ``from riskvalidation.ml import robustness``); nothing is re-exported here.
6
+ The engine imports no learning library: a model enters as a predict callable on arrays, and attributions as arrays
7
+ computed in the pipeline. Every function is pure and deterministic (randomness takes a ``seed``), never mutates its
8
+ inputs and raises ``ValueError`` naming the field when data is missing or invalid. Tolerances, grids, counts and cut
9
+ points are conventions, never regulatory thresholds. The engine computes and states no compliance claim. The
10
+ ``TestResult`` functions are in ``riskvalidation.validation.ml``. Contract: ``engine-ml-api`` 1.1.0.
11
+ """
@@ -0,0 +1,67 @@
1
+ """Input checks of the ML modules, each naming the offending field (the shared array checks are the LGD ones)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Callable
6
+ from fractions import Fraction
7
+ from typing import Any
8
+
9
+ import numpy as np
10
+
11
+ from riskvalidation.lgd_ead._checks import closed_unit, mat, scalar, vec
12
+
13
+ __all__ = ["binary", "closed_unit", "decimal", "index", "mat", "nonneg", "predictions", "same_rows", "scalar", "vec"]
14
+
15
+
16
+ def binary(y: Any, name: str = "y", *, both: bool = True) -> np.ndarray:
17
+ """A new int64 vector of 0 and 1; with ``both`` each class must be present."""
18
+ a = np.asarray(y)
19
+ if a.ndim != 1 or a.size == 0:
20
+ raise ValueError(f"{name} must be a non-empty one-dimensional array")
21
+ if not np.all(np.isin(a, (0, 1))):
22
+ raise ValueError(f"{name} must contain only 0 and 1")
23
+ out = a.astype(np.int64)
24
+ if both and (out.min() == out.max()):
25
+ raise ValueError(f"{name} must contain both classes")
26
+ return out
27
+
28
+
29
+ def predictions(
30
+ predict: Callable[[np.ndarray], Any], x: np.ndarray, name: str = "predict", *, unit: bool = True
31
+ ) -> np.ndarray:
32
+ """The predict callable's output on ``x``: shape (n,), finite and, for a probability, in [0, 1]."""
33
+ out = np.array(predict(x), dtype=float)
34
+ if out.shape != (x.shape[0],):
35
+ raise ValueError(f"{name} must return an array of shape ({x.shape[0]},), got {out.shape}")
36
+ if not np.all(np.isfinite(out)):
37
+ raise ValueError(f"{name} returned non-finite values")
38
+ if unit and (out.min() < 0.0 or out.max() > 1.0):
39
+ raise ValueError(f"{name} must return probabilities in [0, 1]")
40
+ return out
41
+
42
+
43
+ def same_rows(**arrays: np.ndarray) -> None:
44
+ sizes = {k: v.shape[0] for k, v in arrays.items()}
45
+ if len(set(sizes.values())) > 1:
46
+ raise ValueError(f"{', '.join(sizes)} must have the same number of rows, got {sizes}")
47
+
48
+
49
+ def nonneg(x: Any, name: str) -> float:
50
+ v = scalar(x, name)
51
+ if v < 0.0:
52
+ raise ValueError(f"{name} must be nonnegative, got {v}")
53
+ return v
54
+
55
+
56
+ def index(x: Any, name: str, hi: int, *, lo: int = 0) -> int:
57
+ """An integer in [lo, hi)."""
58
+ if isinstance(x, bool) or not isinstance(x, (int, np.integer)):
59
+ raise ValueError(f"{name} must be an integer, got {x!r}")
60
+ if not lo <= int(x) < hi:
61
+ raise ValueError(f"{name} must lie in [{lo}, {hi}), got {x}")
62
+ return int(x)
63
+
64
+
65
+ def decimal(x: float, name: str) -> Fraction:
66
+ """The exact decimal a float was written as (``Fraction(str(x))``), so that 101 * 0.9 cannot round the wrong way."""
67
+ return Fraction(str(float(scalar(x, name))))
@@ -0,0 +1,82 @@
1
+ """The engine core's default logistic fit (NumPy and SciPy only) and the cross-fitted resemblance score built on it.
2
+
3
+ The fit is ridge-penalised Newton on standardised features with an unpenalised intercept and balanced class weights, so
4
+ that two samples that agree score 0.5 on average whatever their sizes. Two conventions are fixed here and recorded in
5
+ ``docs/ml/01_robustness.md``: the ridge strength ``RIDGE`` (1, the default of scikit-learn's ``C = 1``) and the number
6
+ of cross-fitting folds ``FOLDS`` (5). Neither is a regulatory value.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import numpy as np
12
+ from scipy import special
13
+
14
+ RIDGE = 1.0
15
+ FOLDS = 5
16
+ MAX_ITER = 50
17
+ GRAD_TOL = 1e-8
18
+
19
+
20
+ def fit_logistic(x: np.ndarray, y: np.ndarray, weight: np.ndarray, ridge: float = RIDGE) -> np.ndarray:
21
+ """Coefficients (intercept first) minimising ``sum w (log(1 + e^z) - y z) + ridge / 2 |beta|^2``; the penalised
22
+ objective is strictly convex, so damped Newton converges."""
23
+ n, p = x.shape
24
+ a = np.column_stack([np.ones(n), x])
25
+ pen = np.r_[0.0, np.full(p, ridge)]
26
+ beta = np.zeros(p + 1)
27
+
28
+ def objective(b: np.ndarray) -> float:
29
+ z = a @ b
30
+ return float(np.sum(weight * (np.logaddexp(0.0, z) - y * z)) + 0.5 * np.sum(pen * b * b))
31
+
32
+ f = objective(beta)
33
+ for _ in range(MAX_ITER):
34
+ mu = special.expit(a @ beta)
35
+ grad = a.T @ (weight * (mu - y)) + pen * beta
36
+ if np.max(np.abs(grad)) <= GRAD_TOL:
37
+ break
38
+ hess = (a * (weight * mu * (1.0 - mu))[:, None]).T @ a + np.diag(pen)
39
+ step = np.linalg.solve(hess, grad)
40
+ t = 1.0
41
+ while t > 1e-12:
42
+ cand = beta - t * step
43
+ fc = objective(cand)
44
+ if fc <= f:
45
+ break
46
+ t *= 0.5
47
+ else:
48
+ break
49
+ beta, f = cand, fc
50
+ return beta
51
+
52
+
53
+ def predict_proba(beta: np.ndarray, x: np.ndarray) -> np.ndarray:
54
+ return special.expit(beta[0] + x @ beta[1:])
55
+
56
+
57
+ def crossfit_resemblance(x_a: np.ndarray, x_b: np.ndarray, seed: int, folds: int = FOLDS) -> np.ndarray:
58
+ """Out-of-fold probability of membership in sample ``b`` for every row of ``a`` stacked on ``b`` (length n_a + n_b).
59
+
60
+ Rows with an identical feature vector form one group and a group is never split across folds: a row whose exact twin
61
+ sits in the other sample would otherwise be scored by a model that saw the twin with the opposite label, which
62
+ pushes the AUC of two identical samples below 0.5. The groups are dealt to the folds by a seeded permutation; the
63
+ features are standardised on the training folds and the classes carry the weight n / (2 n_class)."""
64
+ x = np.vstack([x_a, x_b])
65
+ y = np.r_[np.zeros(len(x_a)), np.ones(len(x_b))]
66
+ n = len(y)
67
+ rng = np.random.default_rng(seed)
68
+ _, group = np.unique(x + 0.0, axis=0, return_inverse=True)
69
+ group = group.reshape(-1)
70
+ fold = (rng.permutation(group.max() + 1) % folds)[group]
71
+ out = np.empty(n)
72
+ for k in range(folds):
73
+ tr, te = fold != k, fold == k
74
+ mean = x[tr].mean(axis=0)
75
+ sd = x[tr].std(axis=0)
76
+ sd[sd == 0.0] = 1.0
77
+ if y[tr].min() == y[tr].max():
78
+ raise ValueError("a training fold holds a single class: the samples have too few distinct rows")
79
+ w = np.where(y[tr] == 1.0, n / (2.0 * y[tr].sum()), n / (2.0 * (len(y[tr]) - y[tr].sum())))
80
+ beta = fit_logistic((x[tr] - mean) / sd, y[tr], w)
81
+ out[te] = predict_proba(beta, (x[te] - mean) / sd)
82
+ return out
@@ -0,0 +1,91 @@
1
+ """Label-free performance estimation, confidence-based (NannyML CBPE; dossier 06 C.7 l.1052-1058; I.1 l.1199).
2
+
3
+ For a calibrated probability ``p_j`` and the predicted label ``yhat_j``, ``P(yhat = y)_j = 1 - |yhat_j - p_j|``; the
4
+ expected true positives and true negatives are summed over observations, and the ROC AUC is estimated by repeating the
5
+ confusion over thresholds (https://nannyml.readthedocs.io/en/stable/how_it_works/performance_estimation.html).
6
+ Assumptions: well-calibrated probabilities, no concept drift, no extrapolation into unseen regions, adequate sample
7
+ size. An early-warning estimate of the AUC while defaults mature: it complements, never replaces, back-testing, and it
8
+ is always flagged as an estimate. Contract: ``engine-ml-api`` 1.1.0, module ``riskvalidation/ml/cbpe.py``.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from dataclasses import dataclass
14
+
15
+ import numpy as np
16
+
17
+ from riskvalidation.ml._checks import closed_unit, index, scalar, vec
18
+
19
+ ASSUMPTIONS = ("calibrated-probabilities", "no-concept-drift", "no-extrapolation", "adequate-sample")
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class CbpeConfusion:
24
+ """The expected true positives, false positives, true negatives and false negatives (they sum to n)."""
25
+
26
+ tp: float
27
+ fp: float
28
+ tn: float
29
+ fn: float
30
+
31
+
32
+ def _probabilities(p: np.ndarray) -> np.ndarray:
33
+ a = vec(p, "p")
34
+ closed_unit(a, "p")
35
+ return a
36
+
37
+
38
+ def _confusion(p: np.ndarray, threshold: float) -> tuple[float, float, float, float]:
39
+ yhat = p >= threshold
40
+ # P(yhat = y) = 1 - |yhat - p|: p when the label is predicted 1, 1 - p when it is predicted 0
41
+ return (
42
+ float(np.sum(p[yhat])),
43
+ float(np.sum(1.0 - p[yhat])),
44
+ float(np.sum(1.0 - p[~yhat])),
45
+ float(np.sum(p[~yhat])),
46
+ )
47
+
48
+
49
+ def cbpe_confusion(p: np.ndarray, threshold: float) -> CbpeConfusion:
50
+ """The expected confusion counts of calibrated PDs ``p`` at ``threshold`` (``yhat = 1`` when ``p >= threshold``):
51
+ ``tp = sum p`` and ``fp = sum (1 - p)`` over the predicted positives, ``tn = sum (1 - p)`` and ``fn = sum p`` over
52
+ the predicted negatives."""
53
+ a = _probabilities(p)
54
+ t = scalar(threshold, "threshold")
55
+ if not 0.0 <= t <= 1.0:
56
+ raise ValueError(f"threshold must lie in [0, 1], got {t}")
57
+ return CbpeConfusion(*_confusion(a, t))
58
+
59
+
60
+ @dataclass(frozen=True)
61
+ class CbpeAuc:
62
+ """The estimated AUC, always labelled ``estimate``, with its four assumptions."""
63
+
64
+ estimated_auc: float
65
+ label: str
66
+ assumptions: tuple[str, ...]
67
+
68
+ def __post_init__(self) -> None:
69
+ if self.label != "estimate":
70
+ raise ValueError("label must be 'estimate': a label-free AUC is never a measurement")
71
+ if len(self.assumptions) != 4:
72
+ raise ValueError("assumptions must name the four assumptions of the estimate")
73
+
74
+
75
+ def cbpe_auc(p: np.ndarray, n_thresholds: int = 101) -> CbpeAuc:
76
+ """The ROC AUC estimated without labels: the expected confusion at ``n_thresholds`` thresholds evenly spaced on [0,
77
+ 1] gives the expected rates ``tpr = TP / (TP + FN)`` and ``fpr = FP / (FP + TN)``, and the AUC is the trapezoidal
78
+ area under them (the curve runs from (1, 1) at threshold 0 to (0, 0) above the largest probability). No label is
79
+ read."""
80
+ a = _probabilities(p)
81
+ m = index(n_thresholds, "n_thresholds", 10**9, lo=2)
82
+ if a.sum() <= 0.0 or (1.0 - a).sum() <= 0.0:
83
+ raise ValueError("p holds no expected positive or no expected negative: the ROC is undefined")
84
+ tpr, fpr = np.empty(m), np.empty(m)
85
+ for i, t in enumerate(np.linspace(0.0, 1.0, m)):
86
+ tp, fp, tn, fn = _confusion(a, float(t))
87
+ tpr[i], fpr[i] = tp / (tp + fn), fp / (fp + tn)
88
+ order = np.argsort(fpr, kind="stable")
89
+ x, y = fpr[order], tpr[order]
90
+ auc = float(np.sum((x[1:] - x[:-1]) * (y[1:] + y[:-1]) / 2.0))
91
+ return CbpeAuc(estimated_auc=min(1.0, max(0.0, auc)), label="estimate", assumptions=ASSUMPTIONS)
@@ -0,0 +1,73 @@
1
+ """Champion and challenger: the paired bootstrap of the Brier difference and the share of obligors whose grade differs
2
+ by two or more notches, beside the existing ``disc_delong`` (dossier 06 C.7 l.1041-1044; I.1 l.1196; SS1/23 principle
3
+ 3.3(b)(iii); ECB guide paragraph 60). The existing ``bootstrap_ci`` resamples one score and cannot resample two
4
+ forecasts jointly, so the paired form is new. Contract: ``engine-ml-api`` 1.1.0, module
5
+ ``riskvalidation/ml/challenger.py``.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from dataclasses import dataclass
11
+
12
+ import numpy as np
13
+
14
+ from riskvalidation.ml._checks import binary, closed_unit, index, same_rows, vec
15
+
16
+ LEVEL = 0.95 # the interval level of the paired bootstrap (a convention, 06 C.7 l.1041-1044 names no value)
17
+ CHUNK = 2_000_000 # resampled indices drawn per block, so that the memory of a long bootstrap stays bounded
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class PairedBrier:
22
+ """The Brier difference (challenger minus champion) with its 95 percent interval over ``n_boot`` resamples."""
23
+
24
+ diff: float
25
+ lo: float
26
+ hi: float
27
+ n_boot: int
28
+
29
+ def __post_init__(self) -> None:
30
+ if self.lo > self.hi:
31
+ raise ValueError("lo must not exceed hi")
32
+
33
+
34
+ def paired_brier_bootstrap(
35
+ p_champion: np.ndarray, p_challenger: np.ndarray, y: np.ndarray, n_boot: int, seed: int
36
+ ) -> PairedBrier:
37
+ """The Brier difference (challenger minus champion) with a 95 percent percentile interval from ``n_boot`` resamples
38
+ of the obligors drawn jointly (the same obligors score both forecasts). Deterministic for a ``seed``; identical
39
+ forecasts give a zero difference and a zero-width interval. The Brier score is ``mean((p - y)^2)``."""
40
+ champ, chall = vec(p_champion, "p_champion"), vec(p_challenger, "p_challenger")
41
+ yy = binary(y, "y", both=False)
42
+ same_rows(p_champion=champ, p_challenger=chall, y=yy)
43
+ closed_unit(champ, "p_champion")
44
+ closed_unit(chall, "p_challenger")
45
+ b = index(n_boot, "n_boot", 10**9, lo=2)
46
+ d = (chall - yy) ** 2 - (champ - yy) ** 2
47
+ n = len(d)
48
+ rng = np.random.default_rng(seed)
49
+ per_block = max(1, CHUNK // n)
50
+ means = np.empty(b)
51
+ done = 0
52
+ while done < b:
53
+ m = min(per_block, b - done)
54
+ means[done : done + m] = d[rng.integers(0, n, size=(m, n))].mean(axis=1)
55
+ done += m
56
+ alpha = (1.0 - LEVEL) / 2.0
57
+ lo, hi = np.percentile(means, [100.0 * alpha, 100.0 * (1.0 - alpha)])
58
+ return PairedBrier(diff=float(d.mean()), lo=float(lo), hi=float(hi), n_boot=b)
59
+
60
+
61
+ def grade_disagreement(pd_a: np.ndarray, pd_b: np.ndarray, edges: np.ndarray, notches: int = 2) -> float:
62
+ """The share of obligors whose grades under the two PD vectors differ by at least ``notches`` (the grade of a PD is
63
+ the number of ``edges`` at or below it). Symmetric in the two vectors."""
64
+ a, b = vec(pd_a, "pd_a"), vec(pd_b, "pd_b")
65
+ same_rows(pd_a=a, pd_b=b)
66
+ closed_unit(a, "pd_a")
67
+ closed_unit(b, "pd_b")
68
+ e = vec(edges, "edges")
69
+ if np.any(np.diff(e) <= 0.0):
70
+ raise ValueError("edges must be strictly increasing")
71
+ k = index(notches, "notches", 10**9, lo=1)
72
+ ga, gb = np.searchsorted(e, a, side="right"), np.searchsorted(e, b, side="right")
73
+ return float(np.mean(np.abs(ga - gb) >= k))
@@ -0,0 +1,116 @@
1
+ """Split conformal prediction (dossier 06 C.7 l.1019-1028; I.1 l.1194).
2
+
3
+ Scores ``s_i = s(x_i, y_i)`` on n calibration points (``1 - p_hat_{y_i}(x_i)`` for classes); ``q_hat`` is the k-th
4
+ smallest score with ``k = ceil((n + 1)(1 - alpha))``; the set ``C(x) = {y : s(x, y) <= q_hat}``. Under exchangeability
5
+ ``P(Y in C(X)) >= 1 - alpha`` (Angelopoulos and Bates, arXiv 2107.07511; Vovk, Gammerman and Shafer 2022,
6
+ doi:10.1007/978-3-031-06649-8). The validation test is the empirical coverage on out-of-time or shifted data against
7
+ ``1 - alpha``; given a calibration of ``n_cal`` points the coverage of the set is Beta(k, n_cal + 1 - k) distributed
8
+ under exchangeability, so the covered count of ``m`` test points is Beta-Binomial(m, k, n_cal + 1 - k) and the test
9
+ reads the observed count against that law (the binomial p-value of the dossier stays beside it and never drives the
10
+ light); a coverage loss is read as an exchangeability violation, never as proof. Contract: ``engine-ml-api``
11
+ 1.1.0, module ``riskvalidation/ml/conformal.py``.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import math
17
+ from dataclasses import dataclass
18
+ from fractions import Fraction
19
+
20
+ import numpy as np
21
+ from scipy import stats
22
+
23
+ from riskvalidation.ml._checks import binary, closed_unit, decimal, vec
24
+
25
+ REL_ERR = 1.0 + 1e-7 # the tolerance of scipy.stats.binomtest on "no more likely than the observed count"
26
+
27
+
28
+ def _alpha(alpha: float) -> Fraction:
29
+ a = decimal(alpha, "alpha")
30
+ if not 0 < a < 1:
31
+ raise ValueError(f"alpha must lie in (0, 1), got {alpha}")
32
+ return a
33
+
34
+
35
+ def _rank(n: int, a: Fraction) -> int:
36
+ return math.ceil((n + 1) * (1 - a))
37
+
38
+
39
+ def split_conformal_quantile(scores: np.ndarray, alpha: float) -> float:
40
+ """The k-th smallest score, ``k = ceil((n + 1)(1 - alpha))``, with k computed on the exact decimal ``alpha``
41
+ (``fractions.Fraction(str(alpha))``) so that ``101 * 0.9`` cannot round the wrong way.
42
+
43
+ Equal to ``numpy.sort(scores)[k - 1]`` and to ``numpy.quantile(scores, k / n, method="inverted_cdf")``; the
44
+ ``higher`` method of ``numpy.quantile`` is not the reference (it returns the (k + 1)-th order statistic). ``k > n``
45
+ returns ``+inf`` (the set is the whole label space); the quantile is monotone in ``alpha``."""
46
+ s = vec(scores, "scores")
47
+ a = _alpha(alpha)
48
+ n = s.size
49
+ k = _rank(n, a)
50
+ if k > n:
51
+ return math.inf
52
+ return float(np.sort(s)[k - 1])
53
+
54
+
55
+ def conformal_sets(p1: np.ndarray, qhat: float) -> np.ndarray:
56
+ """The prediction sets as a boolean array (n, 2): column y is true when the score of label y, ``1 - p(y)``, is at
57
+ most ``qhat``. A set may be empty or hold both labels; ``qhat = +inf`` gives both labels everywhere."""
58
+ p = vec(p1, "p1")
59
+ closed_unit(p, "p1")
60
+ q = float(qhat)
61
+ if math.isnan(q):
62
+ raise ValueError("qhat is NaN")
63
+ return np.column_stack([p <= q, 1.0 - p <= q]) # the score of label 0 is 1 - (1 - p1) = p1
64
+
65
+
66
+ @dataclass(frozen=True)
67
+ class CoverageResult:
68
+ """The empirical coverage of ``n`` labels and the target ``1 - alpha``, the size ``n_cal`` of the calibration window
69
+ and its rank ``k = ceil((n_cal + 1)(1 - alpha))``, the two-sided Beta-Binomial p-value (``p_value``, the one the
70
+ light reads; NaN when ``k > n_cal``: the set is every label and the coverage is 1 by construction) and the two-sided
71
+ binomial p-value of the dossier (``p_value_binomial``), reported beside it and never driving the light."""
72
+
73
+ n: int
74
+ coverage: float
75
+ target: float
76
+ p_value: float
77
+ n_cal: int
78
+ k: int
79
+ p_value_binomial: float
80
+
81
+
82
+ def conformal_coverage(sets: np.ndarray, y: np.ndarray, alpha: float, *, n_cal: int) -> CoverageResult:
83
+ """The empirical coverage (the share of rows whose set holds the true label), the target ``1 - alpha`` and two
84
+ two-sided p-values of the covered count ``c`` of the ``m`` rows.
85
+
86
+ ``n_cal`` is the number of calibration scores the quantile came from. Under exchangeability the coverage of the set
87
+ given the calibration is Beta(k, n_cal + 1 - k) with ``k = ceil((n_cal + 1)(1 - alpha))`` (the exact decimal
88
+ ``alpha``), so ``c`` is Beta-Binomial(m, k, n_cal + 1 - k): ``p_value`` is the sum of the probabilities of every
89
+ count no more likely than ``c`` (the minimum-likelihood rule of ``scipy.stats.binomtest``,
90
+ ``scipy.stats.betabinom`` for the law). ``p_value_binomial`` is the two-sided ``scipy.stats.binomtest`` of the
91
+ dossier (06 C.7 l.1019-1028), which ignores the randomness of the calibration quantile and is oversized when
92
+ ``m`` is not small against ``n_cal``. When ``k > n_cal`` the set is every label, the coverage is 1 by construction
93
+ and ``p_value`` is NaN."""
94
+ st = np.asarray(sets)
95
+ if st.ndim != 2 or st.shape[1] != 2 or st.dtype != bool or st.shape[0] == 0:
96
+ raise ValueError("sets must be a non-empty boolean array of shape (n, 2)")
97
+ yy = binary(y, "y", both=False)
98
+ if yy.shape[0] != st.shape[0]:
99
+ raise ValueError("y must have one label per row of sets")
100
+ if isinstance(n_cal, bool) or not isinstance(n_cal, (int, np.integer)) or n_cal < 1:
101
+ raise ValueError(f"n_cal must be an integer of at least 1, got {n_cal!r}")
102
+ nc = int(n_cal)
103
+ a = _alpha(alpha)
104
+ hit = st[np.arange(len(yy)), yy]
105
+ c, m = int(hit.sum()), int(len(yy))
106
+ target = float(1 - a)
107
+ k = _rank(nc, a)
108
+ binom = float(stats.binomtest(c, m, target, alternative="two-sided").pvalue)
109
+ p = math.nan
110
+ if k <= nc:
111
+ pmf = stats.betabinom.pmf(np.arange(m + 1), m, k, nc + 1 - k)
112
+ p = float(pmf[pmf <= pmf[c] * REL_ERR].sum())
113
+ p = min(1.0, max(0.0, p))
114
+ return CoverageResult(
115
+ n=m, coverage=c / m, target=target, p_value=p, n_cal=nc, k=k, p_value_binomial=min(1.0, max(0.0, binom))
116
+ )