riskvalidation 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- riskvalidation/__init__.py +30 -0
- riskvalidation/aggregation/__init__.py +11 -0
- riskvalidation/aggregation/band.py +151 -0
- riskvalidation/aggregation/bounds.py +191 -0
- riskvalidation/aggregation/copulas.py +241 -0
- riskvalidation/aggregation/model_risk.py +133 -0
- riskvalidation/aggregation/simple.py +147 -0
- riskvalidation/ccr/__init__.py +13 -0
- riskvalidation/ccr/collateral.py +175 -0
- riskvalidation/ccr/cva.py +193 -0
- riskvalidation/ccr/exposure.py +192 -0
- riskvalidation/ccr/hw1f.py +426 -0
- riskvalidation/ccr/swaps.py +172 -0
- riskvalidation/core/__init__.py +1 -0
- riskvalidation/core/build.py +57 -0
- riskvalidation/core/hashing.py +48 -0
- riskvalidation/core/policy.py +130 -0
- riskvalidation/core/registry.py +65 -0
- riskvalidation/core/result.py +117 -0
- riskvalidation/core/suite.py +78 -0
- riskvalidation/engines/__init__.py +2 -0
- riskvalidation/engines/low_default.py +170 -0
- riskvalidation/engines/pd_curve.py +529 -0
- riskvalidation/engines/vasicek.py +72 -0
- riskvalidation/fairness/__init__.py +10 -0
- riskvalidation/fairness/_checks.py +115 -0
- riskvalidation/fairness/conditional.py +201 -0
- riskvalidation/fairness/inference.py +64 -0
- riskvalidation/fairness/metrics.py +261 -0
- riskvalidation/fairness/plants.py +240 -0
- riskvalidation/fairness/policy.py +97 -0
- riskvalidation/fairness/replication.py +39 -0
- riskvalidation/fairness/situation.py +307 -0
- riskvalidation/generators/__init__.py +68 -0
- riskvalidation/generators/_base.py +145 -0
- riskvalidation/generators/counts.py +232 -0
- riskvalidation/generators/drift.py +107 -0
- riskvalidation/generators/framingham.py +124 -0
- riskvalidation/generators/paths.py +349 -0
- riskvalidation/generators/ratings.py +186 -0
- riskvalidation/generators/scores.py +347 -0
- riskvalidation/harness/__init__.py +46 -0
- riskvalidation/harness/exact.py +137 -0
- riskvalidation/harness/performance.py +74 -0
- riskvalidation/harness/rates.py +103 -0
- riskvalidation/harness/run.py +197 -0
- riskvalidation/irrbb/__init__.py +9 -0
- riskvalidation/irrbb/behavioural.py +206 -0
- riskvalidation/irrbb/curves.py +286 -0
- riskvalidation/irrbb/eve_nii.py +245 -0
- riskvalidation/irrbb/shocks.py +171 -0
- riskvalidation/irrbb/simulation.py +203 -0
- riskvalidation/irrbb/slotting.py +184 -0
- riskvalidation/lgd_ead/__init__.py +8 -0
- riskvalidation/lgd_ead/_checks.py +50 -0
- riskvalidation/lgd_ead/ccf.py +228 -0
- riskvalidation/lgd_ead/downturn.py +64 -0
- riskvalidation/lgd_ead/floors.py +114 -0
- riskvalidation/lgd_ead/lgd_models.py +297 -0
- riskvalidation/lgd_ead/workout.py +162 -0
- riskvalidation/liquidity/__init__.py +9 -0
- riskvalidation/liquidity/_checks.py +78 -0
- riskvalidation/liquidity/cbc.py +158 -0
- riskvalidation/liquidity/deposits.py +151 -0
- riskvalidation/liquidity/ladder.py +119 -0
- riskvalidation/liquidity/reverse.py +141 -0
- riskvalidation/liquidity/runoff.py +220 -0
- riskvalidation/liquidity/scenarios.py +272 -0
- riskvalidation/market/__init__.py +8 -0
- riskvalidation/market/caviar.py +239 -0
- riskvalidation/market/conventions.py +83 -0
- riskvalidation/market/evt.py +111 -0
- riskvalidation/market/frtb.py +175 -0
- riskvalidation/market/garch.py +204 -0
- riskvalidation/market/gas_fz.py +219 -0
- riskvalidation/market/historical.py +96 -0
- riskvalidation/market/parametric.py +78 -0
- riskvalidation/market/scoring.py +145 -0
- riskvalidation/ml/__init__.py +11 -0
- riskvalidation/ml/_checks.py +67 -0
- riskvalidation/ml/_logistic.py +82 -0
- riskvalidation/ml/cbpe.py +91 -0
- riskvalidation/ml/challenger.py +73 -0
- riskvalidation/ml/conformal.py +116 -0
- riskvalidation/ml/dataquality.py +130 -0
- riskvalidation/ml/effects.py +67 -0
- riskvalidation/ml/explanations.py +125 -0
- riskvalidation/ml/generalisation.py +79 -0
- riskvalidation/ml/leakage.py +99 -0
- riskvalidation/ml/plants.py +425 -0
- riskvalidation/ml/policy.py +182 -0
- riskvalidation/ml/robustness.py +286 -0
- riskvalidation/ml/seeds.py +106 -0
- riskvalidation/oprisk/__init__.py +18 -0
- riskvalidation/oprisk/_checks.py +60 -0
- riskvalidation/oprisk/aggregate.py +375 -0
- riskvalidation/oprisk/distributions.py +758 -0
- riskvalidation/oprisk/scenarios.py +342 -0
- riskvalidation/oprisk/sla.py +80 -0
- riskvalidation/oprisk/sma.py +169 -0
- riskvalidation/policies/__init__.py +4 -0
- riskvalidation/policies/default.json +273 -0
- riskvalidation/portfolio/__init__.py +16 -0
- riskvalidation/portfolio/asrf_fit.py +283 -0
- riskvalidation/portfolio/copula.py +129 -0
- riskvalidation/portfolio/creditrisk_plus.py +232 -0
- riskvalidation/portfolio/euler.py +126 -0
- riskvalidation/portfolio/granularity.py +135 -0
- riskvalidation/portfolio/importance.py +277 -0
- riskvalidation/portfolio/portfolio.py +309 -0
- riskvalidation/provisions/__init__.py +8 -0
- riskvalidation/provisions/_checks.py +100 -0
- riskvalidation/provisions/ecl.py +233 -0
- riskvalidation/provisions/local_cl.py +360 -0
- riskvalidation/provisions/local_es.py +157 -0
- riskvalidation/provisions/overlays.py +102 -0
- riskvalidation/provisions/reconciliation.py +157 -0
- riskvalidation/provisions/staging.py +211 -0
- riskvalidation/provisions/term_structure.py +199 -0
- riskvalidation/regulatory/__init__.py +14 -0
- riskvalidation/regulatory/ba_cva.py +138 -0
- riskvalidation/regulatory/frtb.py +345 -0
- riskvalidation/regulatory/irb.py +314 -0
- riskvalidation/regulatory/lcr_nsfr.py +369 -0
- riskvalidation/regulatory/output_floor.py +45 -0
- riskvalidation/regulatory/sa_ccr.py +486 -0
- riskvalidation/regulatory/standardised.py +348 -0
- riskvalidation/stress/__init__.py +9 -0
- riskvalidation/stress/capital.py +476 -0
- riskvalidation/stress/paths.py +108 -0
- riskvalidation/stress/reverse.py +179 -0
- riskvalidation/stress/satellites.py +268 -0
- riskvalidation/stress/scenario_models.py +274 -0
- riskvalidation/transitions/__init__.py +0 -0
- riskvalidation/transitions/em.py +120 -0
- riskvalidation/transitions/embedding.py +221 -0
- riskvalidation/transitions/estimators.py +213 -0
- riskvalidation/transitions/intervals.py +306 -0
- riskvalidation/transitions/mobility.py +57 -0
- riskvalidation/transitions/ttc.py +171 -0
- riskvalidation/validation/__init__.py +103 -0
- riskvalidation/validation/_common.py +57 -0
- riskvalidation/validation/aggregation.py +425 -0
- riskvalidation/validation/calibration.py +757 -0
- riskvalidation/validation/ccr.py +530 -0
- riskvalidation/validation/discrimination.py +314 -0
- riskvalidation/validation/fairness.py +632 -0
- riskvalidation/validation/ifrs9.py +693 -0
- riskvalidation/validation/irrbb.py +389 -0
- riskvalidation/validation/lgd_ead.py +469 -0
- riskvalidation/validation/liquidity.py +647 -0
- riskvalidation/validation/market.py +807 -0
- riskvalidation/validation/ml.py +958 -0
- riskvalidation/validation/oprisk.py +357 -0
- riskvalidation/validation/portfolio.py +353 -0
- riskvalidation/validation/rating_system.py +245 -0
- riskvalidation/validation/stability.py +256 -0
- riskvalidation/validation/stress.py +552 -0
- riskvalidation/validation/transitions.py +386 -0
- riskvalidation-0.8.0.dist-info/METADATA +136 -0
- riskvalidation-0.8.0.dist-info/RECORD +164 -0
- riskvalidation-0.8.0.dist-info/WHEEL +5 -0
- riskvalidation-0.8.0.dist-info/licenses/LICENSE +21 -0
- riskvalidation-0.8.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""The FZ0 and tick losses and the comparison tests (dossier 05 A.1 and B.6).
|
|
2
|
+
|
|
3
|
+
All forecasts are in the return convention (VaR and ES negative decimals, `e <= v`). The FZ0 loss is a score: it can be
|
|
4
|
+
negative and only differences between models are meaningful.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
from scipy import stats
|
|
14
|
+
|
|
15
|
+
from riskvalidation.market.conventions import check_alpha, vec
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class Scores:
|
|
20
|
+
n: int # days scored for the tick loss
|
|
21
|
+
n_fz0: int # days with both VaR and ES defined
|
|
22
|
+
fz0_mean: float | None # None when n_fz0 = 0
|
|
23
|
+
tick_mean: float
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class DmResult:
|
|
28
|
+
mean_diff: float # mean of loss_a - loss_b
|
|
29
|
+
statistic: float
|
|
30
|
+
p_value: float # two-sided, normal
|
|
31
|
+
n: int
|
|
32
|
+
lags: int
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class McsResult:
|
|
37
|
+
members: list[int] # indices of the models in the set
|
|
38
|
+
p_values: np.ndarray # (M,) MCS p-values
|
|
39
|
+
confidence: float
|
|
40
|
+
statistic: str # "Tmax" or "TR"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _same_shape(**arrays: np.ndarray) -> None:
|
|
44
|
+
shapes = {k: a.shape for k, a in arrays.items()}
|
|
45
|
+
if len(set(shapes.values())) != 1:
|
|
46
|
+
raise ValueError(f"the inputs must have the same shape, got {shapes}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def fz0_loss(y: np.ndarray, v: np.ndarray, e: np.ndarray, alpha: float) -> np.ndarray:
|
|
50
|
+
"""`L = -(1/(alpha e)) 1{y <= v}(v - y) + v/e + log(-e) - 1` (dossier 05 A.1, arXiv 1707.05108 eq. 6), per day.
|
|
51
|
+
|
|
52
|
+
The Fissler-Ziegel loss of eq. 4 with `G1 = 0`, `G2(x) = -1/x` and `calG2(x) = -log(-x)`. ValueError unless `v < 0`,
|
|
53
|
+
`e < 0` and `e <= v` elementwise.
|
|
54
|
+
"""
|
|
55
|
+
alpha = check_alpha(alpha)
|
|
56
|
+
y, v, e = (np.asarray(a, dtype=float) for a in (y, v, e))
|
|
57
|
+
_same_shape(y=y, v=v, e=e)
|
|
58
|
+
if not (np.all(np.isfinite(y)) and np.all(np.isfinite(v)) and np.all(np.isfinite(e))):
|
|
59
|
+
raise ValueError("y, v and e must be finite")
|
|
60
|
+
if np.any(v >= 0.0) or np.any(e >= 0.0) or np.any(e > v):
|
|
61
|
+
raise ValueError("the FZ0 loss needs v < 0, e < 0 and e <= v elementwise")
|
|
62
|
+
hit = (y <= v).astype(float)
|
|
63
|
+
return -(1.0 / (alpha * e)) * hit * (v - y) + v / e + np.log(-e) - 1.0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def tick_loss(y: np.ndarray, v: np.ndarray, alpha: float) -> np.ndarray:
|
|
67
|
+
"""`(alpha - 1{y < v})(y - v)` per day (dossier 05 A.2 row 11): nonnegative, zero at `y = v`."""
|
|
68
|
+
alpha = check_alpha(alpha)
|
|
69
|
+
y, v = np.asarray(y, dtype=float), np.asarray(v, dtype=float)
|
|
70
|
+
_same_shape(y=y, v=v)
|
|
71
|
+
return (alpha - (y < v)) * (y - v)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def mean_scores(y: np.ndarray, v: np.ndarray, e: np.ndarray, alpha: float) -> Scores:
|
|
75
|
+
"""Mean tick loss over all `n` days and mean FZ0 over the days with a finite `e` (NaN where the ES is undefined)."""
|
|
76
|
+
alpha = check_alpha(alpha)
|
|
77
|
+
y, v = vec(y, "y"), vec(v, "v")
|
|
78
|
+
e = np.array(e, dtype=float)
|
|
79
|
+
_same_shape(y=y, v=v, e=e)
|
|
80
|
+
ok = np.isfinite(e)
|
|
81
|
+
n_fz0 = int(ok.sum())
|
|
82
|
+
fz0 = float(np.mean(fz0_loss(y[ok], v[ok], e[ok], alpha))) if n_fz0 else None
|
|
83
|
+
return Scores(n=int(y.size), n_fz0=n_fz0, fz0_mean=fz0, tick_mean=float(np.mean(tick_loss(y, v, alpha))))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def diebold_mariano(loss_a: np.ndarray, loss_b: np.ndarray, lags: int) -> DmResult:
|
|
87
|
+
"""Diebold-Mariano (1995): `d_t = loss_a_t - loss_b_t`, `DM = mean(d) / sqrt(LRV / T)` with the Newey-West
|
|
88
|
+
(Bartlett) long-run variance `LRV = g0 + 2 sum_{j=1}^{lags} (1 - j/(lags+1)) g_j`, `g_j = (1/T) sum_t (d_t -
|
|
89
|
+
mean)(d_{t-j} - mean)` (decision D-2) and `p_value = 2 (1 - Phi(|DM|))`. A negative `mean_diff` means model A has
|
|
90
|
+
the lower loss. One-step forecasts use `lags = 0`. ValueError if `d` has no spread or `T <= lags + 1`."""
|
|
91
|
+
a, b = vec(loss_a, "loss_a"), vec(loss_b, "loss_b")
|
|
92
|
+
_same_shape(loss_a=a, loss_b=b)
|
|
93
|
+
if int(lags) != lags or lags < 0:
|
|
94
|
+
raise ValueError(f"lags must be a non-negative integer, got {lags}")
|
|
95
|
+
d = a - b
|
|
96
|
+
t = d.size
|
|
97
|
+
if t <= lags + 1:
|
|
98
|
+
raise ValueError(f"need more than lags + 1 = {lags + 1} observations, got {t}")
|
|
99
|
+
mean = float(d.mean())
|
|
100
|
+
c = d - mean
|
|
101
|
+
if float(np.max(np.abs(c))) <= 1e-14 * max(1.0, abs(mean)):
|
|
102
|
+
raise ValueError("the loss differential has no spread: the statistic is undefined")
|
|
103
|
+
lrv = float(c @ c) / t
|
|
104
|
+
for j in range(1, lags + 1):
|
|
105
|
+
lrv += 2.0 * (1.0 - j / (lags + 1.0)) * float(c[j:] @ c[:-j]) / t
|
|
106
|
+
if lrv <= 0.0:
|
|
107
|
+
raise ValueError("the long-run variance is not positive: the statistic is undefined")
|
|
108
|
+
z = mean / math.sqrt(lrv / t)
|
|
109
|
+
return DmResult(mean_diff=mean, statistic=float(z), p_value=float(2.0 * stats.norm.sf(abs(z))), n=int(t),
|
|
110
|
+
lags=int(lags))
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def model_confidence_set(losses: np.ndarray, confidence: float = 0.90, statistic: str = "TR", n_bootstrap: int = 1000,
|
|
114
|
+
block_length: int = 10, seed: int = 0) -> McsResult:
|
|
115
|
+
"""The Model Confidence Set of Hansen, Lunde and Nason (2011) through `arch.bootstrap.MCS` (stationary bootstrap).
|
|
116
|
+
|
|
117
|
+
`losses` is `(T, M)`; a model is in the set iff its MCS p-value is at least `1 - confidence`; the set is never empty
|
|
118
|
+
(the last survivor has p-value 1); `M = 1` gives that model with p-value 1. Same inputs and seed, same result."""
|
|
119
|
+
if not (0.0 < confidence < 1.0):
|
|
120
|
+
raise ValueError(f"confidence must lie in (0, 1), got {confidence}")
|
|
121
|
+
if statistic not in ("Tmax", "TR"):
|
|
122
|
+
raise ValueError(f"statistic must be 'Tmax' or 'TR', got {statistic!r}")
|
|
123
|
+
x = np.array(losses, dtype=float)
|
|
124
|
+
if x.ndim != 2 or x.shape[0] < 2:
|
|
125
|
+
raise ValueError(f"losses must be a (T, M) array with T >= 2, got shape {x.shape}")
|
|
126
|
+
if not np.all(np.isfinite(x)):
|
|
127
|
+
raise ValueError("losses contain non-finite values")
|
|
128
|
+
m = x.shape[1]
|
|
129
|
+
if m == 1:
|
|
130
|
+
return McsResult(members=[0], p_values=np.array([1.0]), confidence=float(confidence), statistic=statistic)
|
|
131
|
+
try:
|
|
132
|
+
import pandas as pd # noqa: PLC0415
|
|
133
|
+
from arch.bootstrap import MCS # noqa: PLC0415
|
|
134
|
+
except ImportError as exc: # pragma: no cover
|
|
135
|
+
raise ImportError(
|
|
136
|
+
"model_confidence_set needs the optional package `arch`: pip install 'riskvalidation[market]'"
|
|
137
|
+
) from exc
|
|
138
|
+
mcs = MCS(pd.DataFrame(x), size=1.0 - confidence, reps=int(n_bootstrap), block_size=int(block_length),
|
|
139
|
+
method="R" if statistic == "TR" else "max", seed=int(seed))
|
|
140
|
+
mcs.compute()
|
|
141
|
+
pv = np.empty(m)
|
|
142
|
+
for name, row in mcs.pvalues.iterrows():
|
|
143
|
+
pv[int(name)] = float(row["Pvalue"])
|
|
144
|
+
members = [i for i in range(m) if pv[i] >= 1.0 - confidence]
|
|
145
|
+
return McsResult(members=members, p_values=pv, confidence=float(confidence), statistic=statistic)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""ML model validation: perturbation robustness and shift resilience, seed stability, explanation checks, effect curves,
|
|
2
|
+
generalisation, split conformal prediction, the leakage screen, data quality, the label-free performance estimate, the
|
|
3
|
+
champion and challenger comparison and the planted-defect generators.
|
|
4
|
+
|
|
5
|
+
Modules are imported by path (for example ``from riskvalidation.ml import robustness``); nothing is re-exported here.
|
|
6
|
+
The engine imports no learning library: a model enters as a predict callable on arrays, and attributions as arrays
|
|
7
|
+
computed in the pipeline. Every function is pure and deterministic (randomness takes a ``seed``), never mutates its
|
|
8
|
+
inputs and raises ``ValueError`` naming the field when data is missing or invalid. Tolerances, grids, counts and cut
|
|
9
|
+
points are conventions, never regulatory thresholds. The engine computes and states no compliance claim. The
|
|
10
|
+
``TestResult`` functions are in ``riskvalidation.validation.ml``. Contract: ``engine-ml-api`` 1.1.0.
|
|
11
|
+
"""
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Input checks of the ML modules, each naming the offending field (the shared array checks are the LGD ones)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from fractions import Fraction
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
from riskvalidation.lgd_ead._checks import closed_unit, mat, scalar, vec
|
|
12
|
+
|
|
13
|
+
__all__ = ["binary", "closed_unit", "decimal", "index", "mat", "nonneg", "predictions", "same_rows", "scalar", "vec"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def binary(y: Any, name: str = "y", *, both: bool = True) -> np.ndarray:
|
|
17
|
+
"""A new int64 vector of 0 and 1; with ``both`` each class must be present."""
|
|
18
|
+
a = np.asarray(y)
|
|
19
|
+
if a.ndim != 1 or a.size == 0:
|
|
20
|
+
raise ValueError(f"{name} must be a non-empty one-dimensional array")
|
|
21
|
+
if not np.all(np.isin(a, (0, 1))):
|
|
22
|
+
raise ValueError(f"{name} must contain only 0 and 1")
|
|
23
|
+
out = a.astype(np.int64)
|
|
24
|
+
if both and (out.min() == out.max()):
|
|
25
|
+
raise ValueError(f"{name} must contain both classes")
|
|
26
|
+
return out
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def predictions(
|
|
30
|
+
predict: Callable[[np.ndarray], Any], x: np.ndarray, name: str = "predict", *, unit: bool = True
|
|
31
|
+
) -> np.ndarray:
|
|
32
|
+
"""The predict callable's output on ``x``: shape (n,), finite and, for a probability, in [0, 1]."""
|
|
33
|
+
out = np.array(predict(x), dtype=float)
|
|
34
|
+
if out.shape != (x.shape[0],):
|
|
35
|
+
raise ValueError(f"{name} must return an array of shape ({x.shape[0]},), got {out.shape}")
|
|
36
|
+
if not np.all(np.isfinite(out)):
|
|
37
|
+
raise ValueError(f"{name} returned non-finite values")
|
|
38
|
+
if unit and (out.min() < 0.0 or out.max() > 1.0):
|
|
39
|
+
raise ValueError(f"{name} must return probabilities in [0, 1]")
|
|
40
|
+
return out
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def same_rows(**arrays: np.ndarray) -> None:
|
|
44
|
+
sizes = {k: v.shape[0] for k, v in arrays.items()}
|
|
45
|
+
if len(set(sizes.values())) > 1:
|
|
46
|
+
raise ValueError(f"{', '.join(sizes)} must have the same number of rows, got {sizes}")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def nonneg(x: Any, name: str) -> float:
|
|
50
|
+
v = scalar(x, name)
|
|
51
|
+
if v < 0.0:
|
|
52
|
+
raise ValueError(f"{name} must be nonnegative, got {v}")
|
|
53
|
+
return v
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def index(x: Any, name: str, hi: int, *, lo: int = 0) -> int:
|
|
57
|
+
"""An integer in [lo, hi)."""
|
|
58
|
+
if isinstance(x, bool) or not isinstance(x, (int, np.integer)):
|
|
59
|
+
raise ValueError(f"{name} must be an integer, got {x!r}")
|
|
60
|
+
if not lo <= int(x) < hi:
|
|
61
|
+
raise ValueError(f"{name} must lie in [{lo}, {hi}), got {x}")
|
|
62
|
+
return int(x)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def decimal(x: float, name: str) -> Fraction:
|
|
66
|
+
"""The exact decimal a float was written as (``Fraction(str(x))``), so that 101 * 0.9 cannot round the wrong way."""
|
|
67
|
+
return Fraction(str(float(scalar(x, name))))
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""The engine core's default logistic fit (NumPy and SciPy only) and the cross-fitted resemblance score built on it.
|
|
2
|
+
|
|
3
|
+
The fit is ridge-penalised Newton on standardised features with an unpenalised intercept and balanced class weights, so
|
|
4
|
+
that two samples that agree score 0.5 on average whatever their sizes. Two conventions are fixed here and recorded in
|
|
5
|
+
``docs/ml/01_robustness.md``: the ridge strength ``RIDGE`` (1, the default of scikit-learn's ``C = 1``) and the number
|
|
6
|
+
of cross-fitting folds ``FOLDS`` (5). Neither is a regulatory value.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
from scipy import special
|
|
13
|
+
|
|
14
|
+
RIDGE = 1.0
|
|
15
|
+
FOLDS = 5
|
|
16
|
+
MAX_ITER = 50
|
|
17
|
+
GRAD_TOL = 1e-8
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def fit_logistic(x: np.ndarray, y: np.ndarray, weight: np.ndarray, ridge: float = RIDGE) -> np.ndarray:
|
|
21
|
+
"""Coefficients (intercept first) minimising ``sum w (log(1 + e^z) - y z) + ridge / 2 |beta|^2``; the penalised
|
|
22
|
+
objective is strictly convex, so damped Newton converges."""
|
|
23
|
+
n, p = x.shape
|
|
24
|
+
a = np.column_stack([np.ones(n), x])
|
|
25
|
+
pen = np.r_[0.0, np.full(p, ridge)]
|
|
26
|
+
beta = np.zeros(p + 1)
|
|
27
|
+
|
|
28
|
+
def objective(b: np.ndarray) -> float:
|
|
29
|
+
z = a @ b
|
|
30
|
+
return float(np.sum(weight * (np.logaddexp(0.0, z) - y * z)) + 0.5 * np.sum(pen * b * b))
|
|
31
|
+
|
|
32
|
+
f = objective(beta)
|
|
33
|
+
for _ in range(MAX_ITER):
|
|
34
|
+
mu = special.expit(a @ beta)
|
|
35
|
+
grad = a.T @ (weight * (mu - y)) + pen * beta
|
|
36
|
+
if np.max(np.abs(grad)) <= GRAD_TOL:
|
|
37
|
+
break
|
|
38
|
+
hess = (a * (weight * mu * (1.0 - mu))[:, None]).T @ a + np.diag(pen)
|
|
39
|
+
step = np.linalg.solve(hess, grad)
|
|
40
|
+
t = 1.0
|
|
41
|
+
while t > 1e-12:
|
|
42
|
+
cand = beta - t * step
|
|
43
|
+
fc = objective(cand)
|
|
44
|
+
if fc <= f:
|
|
45
|
+
break
|
|
46
|
+
t *= 0.5
|
|
47
|
+
else:
|
|
48
|
+
break
|
|
49
|
+
beta, f = cand, fc
|
|
50
|
+
return beta
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def predict_proba(beta: np.ndarray, x: np.ndarray) -> np.ndarray:
|
|
54
|
+
return special.expit(beta[0] + x @ beta[1:])
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def crossfit_resemblance(x_a: np.ndarray, x_b: np.ndarray, seed: int, folds: int = FOLDS) -> np.ndarray:
|
|
58
|
+
"""Out-of-fold probability of membership in sample ``b`` for every row of ``a`` stacked on ``b`` (length n_a + n_b).
|
|
59
|
+
|
|
60
|
+
Rows with an identical feature vector form one group and a group is never split across folds: a row whose exact twin
|
|
61
|
+
sits in the other sample would otherwise be scored by a model that saw the twin with the opposite label, which
|
|
62
|
+
pushes the AUC of two identical samples below 0.5. The groups are dealt to the folds by a seeded permutation; the
|
|
63
|
+
features are standardised on the training folds and the classes carry the weight n / (2 n_class)."""
|
|
64
|
+
x = np.vstack([x_a, x_b])
|
|
65
|
+
y = np.r_[np.zeros(len(x_a)), np.ones(len(x_b))]
|
|
66
|
+
n = len(y)
|
|
67
|
+
rng = np.random.default_rng(seed)
|
|
68
|
+
_, group = np.unique(x + 0.0, axis=0, return_inverse=True)
|
|
69
|
+
group = group.reshape(-1)
|
|
70
|
+
fold = (rng.permutation(group.max() + 1) % folds)[group]
|
|
71
|
+
out = np.empty(n)
|
|
72
|
+
for k in range(folds):
|
|
73
|
+
tr, te = fold != k, fold == k
|
|
74
|
+
mean = x[tr].mean(axis=0)
|
|
75
|
+
sd = x[tr].std(axis=0)
|
|
76
|
+
sd[sd == 0.0] = 1.0
|
|
77
|
+
if y[tr].min() == y[tr].max():
|
|
78
|
+
raise ValueError("a training fold holds a single class: the samples have too few distinct rows")
|
|
79
|
+
w = np.where(y[tr] == 1.0, n / (2.0 * y[tr].sum()), n / (2.0 * (len(y[tr]) - y[tr].sum())))
|
|
80
|
+
beta = fit_logistic((x[tr] - mean) / sd, y[tr], w)
|
|
81
|
+
out[te] = predict_proba(beta, (x[te] - mean) / sd)
|
|
82
|
+
return out
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Label-free performance estimation, confidence-based (NannyML CBPE; dossier 06 C.7 l.1052-1058; I.1 l.1199).
|
|
2
|
+
|
|
3
|
+
For a calibrated probability ``p_j`` and the predicted label ``yhat_j``, ``P(yhat = y)_j = 1 - |yhat_j - p_j|``; the
|
|
4
|
+
expected true positives and true negatives are summed over observations, and the ROC AUC is estimated by repeating the
|
|
5
|
+
confusion over thresholds (https://nannyml.readthedocs.io/en/stable/how_it_works/performance_estimation.html).
|
|
6
|
+
Assumptions: well-calibrated probabilities, no concept drift, no extrapolation into unseen regions, adequate sample
|
|
7
|
+
size. An early-warning estimate of the AUC while defaults mature: it complements, never replaces, back-testing, and it
|
|
8
|
+
is always flagged as an estimate. Contract: ``engine-ml-api`` 1.1.0, module ``riskvalidation/ml/cbpe.py``.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
from riskvalidation.ml._checks import closed_unit, index, scalar, vec
|
|
18
|
+
|
|
19
|
+
ASSUMPTIONS = ("calibrated-probabilities", "no-concept-drift", "no-extrapolation", "adequate-sample")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class CbpeConfusion:
|
|
24
|
+
"""The expected true positives, false positives, true negatives and false negatives (they sum to n)."""
|
|
25
|
+
|
|
26
|
+
tp: float
|
|
27
|
+
fp: float
|
|
28
|
+
tn: float
|
|
29
|
+
fn: float
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _probabilities(p: np.ndarray) -> np.ndarray:
|
|
33
|
+
a = vec(p, "p")
|
|
34
|
+
closed_unit(a, "p")
|
|
35
|
+
return a
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _confusion(p: np.ndarray, threshold: float) -> tuple[float, float, float, float]:
|
|
39
|
+
yhat = p >= threshold
|
|
40
|
+
# P(yhat = y) = 1 - |yhat - p|: p when the label is predicted 1, 1 - p when it is predicted 0
|
|
41
|
+
return (
|
|
42
|
+
float(np.sum(p[yhat])),
|
|
43
|
+
float(np.sum(1.0 - p[yhat])),
|
|
44
|
+
float(np.sum(1.0 - p[~yhat])),
|
|
45
|
+
float(np.sum(p[~yhat])),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def cbpe_confusion(p: np.ndarray, threshold: float) -> CbpeConfusion:
|
|
50
|
+
"""The expected confusion counts of calibrated PDs ``p`` at ``threshold`` (``yhat = 1`` when ``p >= threshold``):
|
|
51
|
+
``tp = sum p`` and ``fp = sum (1 - p)`` over the predicted positives, ``tn = sum (1 - p)`` and ``fn = sum p`` over
|
|
52
|
+
the predicted negatives."""
|
|
53
|
+
a = _probabilities(p)
|
|
54
|
+
t = scalar(threshold, "threshold")
|
|
55
|
+
if not 0.0 <= t <= 1.0:
|
|
56
|
+
raise ValueError(f"threshold must lie in [0, 1], got {t}")
|
|
57
|
+
return CbpeConfusion(*_confusion(a, t))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class CbpeAuc:
|
|
62
|
+
"""The estimated AUC, always labelled ``estimate``, with its four assumptions."""
|
|
63
|
+
|
|
64
|
+
estimated_auc: float
|
|
65
|
+
label: str
|
|
66
|
+
assumptions: tuple[str, ...]
|
|
67
|
+
|
|
68
|
+
def __post_init__(self) -> None:
|
|
69
|
+
if self.label != "estimate":
|
|
70
|
+
raise ValueError("label must be 'estimate': a label-free AUC is never a measurement")
|
|
71
|
+
if len(self.assumptions) != 4:
|
|
72
|
+
raise ValueError("assumptions must name the four assumptions of the estimate")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def cbpe_auc(p: np.ndarray, n_thresholds: int = 101) -> CbpeAuc:
|
|
76
|
+
"""The ROC AUC estimated without labels: the expected confusion at ``n_thresholds`` thresholds evenly spaced on [0,
|
|
77
|
+
1] gives the expected rates ``tpr = TP / (TP + FN)`` and ``fpr = FP / (FP + TN)``, and the AUC is the trapezoidal
|
|
78
|
+
area under them (the curve runs from (1, 1) at threshold 0 to (0, 0) above the largest probability). No label is
|
|
79
|
+
read."""
|
|
80
|
+
a = _probabilities(p)
|
|
81
|
+
m = index(n_thresholds, "n_thresholds", 10**9, lo=2)
|
|
82
|
+
if a.sum() <= 0.0 or (1.0 - a).sum() <= 0.0:
|
|
83
|
+
raise ValueError("p holds no expected positive or no expected negative: the ROC is undefined")
|
|
84
|
+
tpr, fpr = np.empty(m), np.empty(m)
|
|
85
|
+
for i, t in enumerate(np.linspace(0.0, 1.0, m)):
|
|
86
|
+
tp, fp, tn, fn = _confusion(a, float(t))
|
|
87
|
+
tpr[i], fpr[i] = tp / (tp + fn), fp / (fp + tn)
|
|
88
|
+
order = np.argsort(fpr, kind="stable")
|
|
89
|
+
x, y = fpr[order], tpr[order]
|
|
90
|
+
auc = float(np.sum((x[1:] - x[:-1]) * (y[1:] + y[:-1]) / 2.0))
|
|
91
|
+
return CbpeAuc(estimated_auc=min(1.0, max(0.0, auc)), label="estimate", assumptions=ASSUMPTIONS)
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Champion and challenger: the paired bootstrap of the Brier difference and the share of obligors whose grade differs
|
|
2
|
+
by two or more notches, beside the existing ``disc_delong`` (dossier 06 C.7 l.1041-1044; I.1 l.1196; SS1/23 principle
|
|
3
|
+
3.3(b)(iii); ECB guide paragraph 60). The existing ``bootstrap_ci`` resamples one score and cannot resample two
|
|
4
|
+
forecasts jointly, so the paired form is new. Contract: ``engine-ml-api`` 1.1.0, module
|
|
5
|
+
``riskvalidation/ml/challenger.py``.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
|
|
14
|
+
from riskvalidation.ml._checks import binary, closed_unit, index, same_rows, vec
|
|
15
|
+
|
|
16
|
+
LEVEL = 0.95 # the interval level of the paired bootstrap (a convention, 06 C.7 l.1041-1044 names no value)
|
|
17
|
+
CHUNK = 2_000_000 # resampled indices drawn per block, so that the memory of a long bootstrap stays bounded
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class PairedBrier:
|
|
22
|
+
"""The Brier difference (challenger minus champion) with its 95 percent interval over ``n_boot`` resamples."""
|
|
23
|
+
|
|
24
|
+
diff: float
|
|
25
|
+
lo: float
|
|
26
|
+
hi: float
|
|
27
|
+
n_boot: int
|
|
28
|
+
|
|
29
|
+
def __post_init__(self) -> None:
|
|
30
|
+
if self.lo > self.hi:
|
|
31
|
+
raise ValueError("lo must not exceed hi")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def paired_brier_bootstrap(
|
|
35
|
+
p_champion: np.ndarray, p_challenger: np.ndarray, y: np.ndarray, n_boot: int, seed: int
|
|
36
|
+
) -> PairedBrier:
|
|
37
|
+
"""The Brier difference (challenger minus champion) with a 95 percent percentile interval from ``n_boot`` resamples
|
|
38
|
+
of the obligors drawn jointly (the same obligors score both forecasts). Deterministic for a ``seed``; identical
|
|
39
|
+
forecasts give a zero difference and a zero-width interval. The Brier score is ``mean((p - y)^2)``."""
|
|
40
|
+
champ, chall = vec(p_champion, "p_champion"), vec(p_challenger, "p_challenger")
|
|
41
|
+
yy = binary(y, "y", both=False)
|
|
42
|
+
same_rows(p_champion=champ, p_challenger=chall, y=yy)
|
|
43
|
+
closed_unit(champ, "p_champion")
|
|
44
|
+
closed_unit(chall, "p_challenger")
|
|
45
|
+
b = index(n_boot, "n_boot", 10**9, lo=2)
|
|
46
|
+
d = (chall - yy) ** 2 - (champ - yy) ** 2
|
|
47
|
+
n = len(d)
|
|
48
|
+
rng = np.random.default_rng(seed)
|
|
49
|
+
per_block = max(1, CHUNK // n)
|
|
50
|
+
means = np.empty(b)
|
|
51
|
+
done = 0
|
|
52
|
+
while done < b:
|
|
53
|
+
m = min(per_block, b - done)
|
|
54
|
+
means[done : done + m] = d[rng.integers(0, n, size=(m, n))].mean(axis=1)
|
|
55
|
+
done += m
|
|
56
|
+
alpha = (1.0 - LEVEL) / 2.0
|
|
57
|
+
lo, hi = np.percentile(means, [100.0 * alpha, 100.0 * (1.0 - alpha)])
|
|
58
|
+
return PairedBrier(diff=float(d.mean()), lo=float(lo), hi=float(hi), n_boot=b)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def grade_disagreement(pd_a: np.ndarray, pd_b: np.ndarray, edges: np.ndarray, notches: int = 2) -> float:
|
|
62
|
+
"""The share of obligors whose grades under the two PD vectors differ by at least ``notches`` (the grade of a PD is
|
|
63
|
+
the number of ``edges`` at or below it). Symmetric in the two vectors."""
|
|
64
|
+
a, b = vec(pd_a, "pd_a"), vec(pd_b, "pd_b")
|
|
65
|
+
same_rows(pd_a=a, pd_b=b)
|
|
66
|
+
closed_unit(a, "pd_a")
|
|
67
|
+
closed_unit(b, "pd_b")
|
|
68
|
+
e = vec(edges, "edges")
|
|
69
|
+
if np.any(np.diff(e) <= 0.0):
|
|
70
|
+
raise ValueError("edges must be strictly increasing")
|
|
71
|
+
k = index(notches, "notches", 10**9, lo=1)
|
|
72
|
+
ga, gb = np.searchsorted(e, a, side="right"), np.searchsorted(e, b, side="right")
|
|
73
|
+
return float(np.mean(np.abs(ga - gb) >= k))
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Split conformal prediction (dossier 06 C.7 l.1019-1028; I.1 l.1194).
|
|
2
|
+
|
|
3
|
+
Scores ``s_i = s(x_i, y_i)`` on n calibration points (``1 - p_hat_{y_i}(x_i)`` for classes); ``q_hat`` is the k-th
|
|
4
|
+
smallest score with ``k = ceil((n + 1)(1 - alpha))``; the set ``C(x) = {y : s(x, y) <= q_hat}``. Under exchangeability
|
|
5
|
+
``P(Y in C(X)) >= 1 - alpha`` (Angelopoulos and Bates, arXiv 2107.07511; Vovk, Gammerman and Shafer 2022,
|
|
6
|
+
doi:10.1007/978-3-031-06649-8). The validation test is the empirical coverage on out-of-time or shifted data against
|
|
7
|
+
``1 - alpha``; given a calibration of ``n_cal`` points the coverage of the set is Beta(k, n_cal + 1 - k) distributed
|
|
8
|
+
under exchangeability, so the covered count of ``m`` test points is Beta-Binomial(m, k, n_cal + 1 - k) and the test
|
|
9
|
+
reads the observed count against that law (the binomial p-value of the dossier stays beside it and never drives the
|
|
10
|
+
light); a coverage loss is read as an exchangeability violation, never as proof. Contract: ``engine-ml-api``
|
|
11
|
+
1.1.0, module ``riskvalidation/ml/conformal.py``.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import math
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from fractions import Fraction
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
from scipy import stats
|
|
22
|
+
|
|
23
|
+
from riskvalidation.ml._checks import binary, closed_unit, decimal, vec
|
|
24
|
+
|
|
25
|
+
REL_ERR = 1.0 + 1e-7 # the tolerance of scipy.stats.binomtest on "no more likely than the observed count"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _alpha(alpha: float) -> Fraction:
|
|
29
|
+
a = decimal(alpha, "alpha")
|
|
30
|
+
if not 0 < a < 1:
|
|
31
|
+
raise ValueError(f"alpha must lie in (0, 1), got {alpha}")
|
|
32
|
+
return a
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _rank(n: int, a: Fraction) -> int:
|
|
36
|
+
return math.ceil((n + 1) * (1 - a))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def split_conformal_quantile(scores: np.ndarray, alpha: float) -> float:
|
|
40
|
+
"""The k-th smallest score, ``k = ceil((n + 1)(1 - alpha))``, with k computed on the exact decimal ``alpha``
|
|
41
|
+
(``fractions.Fraction(str(alpha))``) so that ``101 * 0.9`` cannot round the wrong way.
|
|
42
|
+
|
|
43
|
+
Equal to ``numpy.sort(scores)[k - 1]`` and to ``numpy.quantile(scores, k / n, method="inverted_cdf")``; the
|
|
44
|
+
``higher`` method of ``numpy.quantile`` is not the reference (it returns the (k + 1)-th order statistic). ``k > n``
|
|
45
|
+
returns ``+inf`` (the set is the whole label space); the quantile is monotone in ``alpha``."""
|
|
46
|
+
s = vec(scores, "scores")
|
|
47
|
+
a = _alpha(alpha)
|
|
48
|
+
n = s.size
|
|
49
|
+
k = _rank(n, a)
|
|
50
|
+
if k > n:
|
|
51
|
+
return math.inf
|
|
52
|
+
return float(np.sort(s)[k - 1])
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def conformal_sets(p1: np.ndarray, qhat: float) -> np.ndarray:
|
|
56
|
+
"""The prediction sets as a boolean array (n, 2): column y is true when the score of label y, ``1 - p(y)``, is at
|
|
57
|
+
most ``qhat``. A set may be empty or hold both labels; ``qhat = +inf`` gives both labels everywhere."""
|
|
58
|
+
p = vec(p1, "p1")
|
|
59
|
+
closed_unit(p, "p1")
|
|
60
|
+
q = float(qhat)
|
|
61
|
+
if math.isnan(q):
|
|
62
|
+
raise ValueError("qhat is NaN")
|
|
63
|
+
return np.column_stack([p <= q, 1.0 - p <= q]) # the score of label 0 is 1 - (1 - p1) = p1
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class CoverageResult:
|
|
68
|
+
"""The empirical coverage of ``n`` labels and the target ``1 - alpha``, the size ``n_cal`` of the calibration window
|
|
69
|
+
and its rank ``k = ceil((n_cal + 1)(1 - alpha))``, the two-sided Beta-Binomial p-value (``p_value``, the one the
|
|
70
|
+
light reads; NaN when ``k > n_cal``: the set is every label and the coverage is 1 by construction) and the two-sided
|
|
71
|
+
binomial p-value of the dossier (``p_value_binomial``), reported beside it and never driving the light."""
|
|
72
|
+
|
|
73
|
+
n: int
|
|
74
|
+
coverage: float
|
|
75
|
+
target: float
|
|
76
|
+
p_value: float
|
|
77
|
+
n_cal: int
|
|
78
|
+
k: int
|
|
79
|
+
p_value_binomial: float
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def conformal_coverage(sets: np.ndarray, y: np.ndarray, alpha: float, *, n_cal: int) -> CoverageResult:
|
|
83
|
+
"""The empirical coverage (the share of rows whose set holds the true label), the target ``1 - alpha`` and two
|
|
84
|
+
two-sided p-values of the covered count ``c`` of the ``m`` rows.
|
|
85
|
+
|
|
86
|
+
``n_cal`` is the number of calibration scores the quantile came from. Under exchangeability the coverage of the set
|
|
87
|
+
given the calibration is Beta(k, n_cal + 1 - k) with ``k = ceil((n_cal + 1)(1 - alpha))`` (the exact decimal
|
|
88
|
+
``alpha``), so ``c`` is Beta-Binomial(m, k, n_cal + 1 - k): ``p_value`` is the sum of the probabilities of every
|
|
89
|
+
count no more likely than ``c`` (the minimum-likelihood rule of ``scipy.stats.binomtest``,
|
|
90
|
+
``scipy.stats.betabinom`` for the law). ``p_value_binomial`` is the two-sided ``scipy.stats.binomtest`` of the
|
|
91
|
+
dossier (06 C.7 l.1019-1028), which ignores the randomness of the calibration quantile and is oversized when
|
|
92
|
+
``m`` is not small against ``n_cal``. When ``k > n_cal`` the set is every label, the coverage is 1 by construction
|
|
93
|
+
and ``p_value`` is NaN."""
|
|
94
|
+
st = np.asarray(sets)
|
|
95
|
+
if st.ndim != 2 or st.shape[1] != 2 or st.dtype != bool or st.shape[0] == 0:
|
|
96
|
+
raise ValueError("sets must be a non-empty boolean array of shape (n, 2)")
|
|
97
|
+
yy = binary(y, "y", both=False)
|
|
98
|
+
if yy.shape[0] != st.shape[0]:
|
|
99
|
+
raise ValueError("y must have one label per row of sets")
|
|
100
|
+
if isinstance(n_cal, bool) or not isinstance(n_cal, (int, np.integer)) or n_cal < 1:
|
|
101
|
+
raise ValueError(f"n_cal must be an integer of at least 1, got {n_cal!r}")
|
|
102
|
+
nc = int(n_cal)
|
|
103
|
+
a = _alpha(alpha)
|
|
104
|
+
hit = st[np.arange(len(yy)), yy]
|
|
105
|
+
c, m = int(hit.sum()), int(len(yy))
|
|
106
|
+
target = float(1 - a)
|
|
107
|
+
k = _rank(nc, a)
|
|
108
|
+
binom = float(stats.binomtest(c, m, target, alternative="two-sided").pvalue)
|
|
109
|
+
p = math.nan
|
|
110
|
+
if k <= nc:
|
|
111
|
+
pmf = stats.betabinom.pmf(np.arange(m + 1), m, k, nc + 1 - k)
|
|
112
|
+
p = float(pmf[pmf <= pmf[c] * REL_ERR].sum())
|
|
113
|
+
p = min(1.0, max(0.0, p))
|
|
114
|
+
return CoverageResult(
|
|
115
|
+
n=m, coverage=c / m, target=target, p_value=p, n_cal=nc, k=k, p_value_binomial=min(1.0, max(0.0, binom))
|
|
116
|
+
)
|