riskvalidation 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- riskvalidation/__init__.py +30 -0
- riskvalidation/aggregation/__init__.py +11 -0
- riskvalidation/aggregation/band.py +151 -0
- riskvalidation/aggregation/bounds.py +191 -0
- riskvalidation/aggregation/copulas.py +241 -0
- riskvalidation/aggregation/model_risk.py +133 -0
- riskvalidation/aggregation/simple.py +147 -0
- riskvalidation/ccr/__init__.py +13 -0
- riskvalidation/ccr/collateral.py +175 -0
- riskvalidation/ccr/cva.py +193 -0
- riskvalidation/ccr/exposure.py +192 -0
- riskvalidation/ccr/hw1f.py +426 -0
- riskvalidation/ccr/swaps.py +172 -0
- riskvalidation/core/__init__.py +1 -0
- riskvalidation/core/build.py +57 -0
- riskvalidation/core/hashing.py +48 -0
- riskvalidation/core/policy.py +130 -0
- riskvalidation/core/registry.py +65 -0
- riskvalidation/core/result.py +117 -0
- riskvalidation/core/suite.py +78 -0
- riskvalidation/engines/__init__.py +2 -0
- riskvalidation/engines/low_default.py +170 -0
- riskvalidation/engines/pd_curve.py +529 -0
- riskvalidation/engines/vasicek.py +72 -0
- riskvalidation/fairness/__init__.py +10 -0
- riskvalidation/fairness/_checks.py +115 -0
- riskvalidation/fairness/conditional.py +201 -0
- riskvalidation/fairness/inference.py +64 -0
- riskvalidation/fairness/metrics.py +261 -0
- riskvalidation/fairness/plants.py +240 -0
- riskvalidation/fairness/policy.py +97 -0
- riskvalidation/fairness/replication.py +39 -0
- riskvalidation/fairness/situation.py +307 -0
- riskvalidation/generators/__init__.py +68 -0
- riskvalidation/generators/_base.py +145 -0
- riskvalidation/generators/counts.py +232 -0
- riskvalidation/generators/drift.py +107 -0
- riskvalidation/generators/framingham.py +124 -0
- riskvalidation/generators/paths.py +349 -0
- riskvalidation/generators/ratings.py +186 -0
- riskvalidation/generators/scores.py +347 -0
- riskvalidation/harness/__init__.py +46 -0
- riskvalidation/harness/exact.py +137 -0
- riskvalidation/harness/performance.py +74 -0
- riskvalidation/harness/rates.py +103 -0
- riskvalidation/harness/run.py +197 -0
- riskvalidation/irrbb/__init__.py +9 -0
- riskvalidation/irrbb/behavioural.py +206 -0
- riskvalidation/irrbb/curves.py +286 -0
- riskvalidation/irrbb/eve_nii.py +245 -0
- riskvalidation/irrbb/shocks.py +171 -0
- riskvalidation/irrbb/simulation.py +203 -0
- riskvalidation/irrbb/slotting.py +184 -0
- riskvalidation/lgd_ead/__init__.py +8 -0
- riskvalidation/lgd_ead/_checks.py +50 -0
- riskvalidation/lgd_ead/ccf.py +228 -0
- riskvalidation/lgd_ead/downturn.py +64 -0
- riskvalidation/lgd_ead/floors.py +114 -0
- riskvalidation/lgd_ead/lgd_models.py +297 -0
- riskvalidation/lgd_ead/workout.py +162 -0
- riskvalidation/liquidity/__init__.py +9 -0
- riskvalidation/liquidity/_checks.py +78 -0
- riskvalidation/liquidity/cbc.py +158 -0
- riskvalidation/liquidity/deposits.py +151 -0
- riskvalidation/liquidity/ladder.py +119 -0
- riskvalidation/liquidity/reverse.py +141 -0
- riskvalidation/liquidity/runoff.py +220 -0
- riskvalidation/liquidity/scenarios.py +272 -0
- riskvalidation/market/__init__.py +8 -0
- riskvalidation/market/caviar.py +239 -0
- riskvalidation/market/conventions.py +83 -0
- riskvalidation/market/evt.py +111 -0
- riskvalidation/market/frtb.py +175 -0
- riskvalidation/market/garch.py +204 -0
- riskvalidation/market/gas_fz.py +219 -0
- riskvalidation/market/historical.py +96 -0
- riskvalidation/market/parametric.py +78 -0
- riskvalidation/market/scoring.py +145 -0
- riskvalidation/ml/__init__.py +11 -0
- riskvalidation/ml/_checks.py +67 -0
- riskvalidation/ml/_logistic.py +82 -0
- riskvalidation/ml/cbpe.py +91 -0
- riskvalidation/ml/challenger.py +73 -0
- riskvalidation/ml/conformal.py +116 -0
- riskvalidation/ml/dataquality.py +130 -0
- riskvalidation/ml/effects.py +67 -0
- riskvalidation/ml/explanations.py +125 -0
- riskvalidation/ml/generalisation.py +79 -0
- riskvalidation/ml/leakage.py +99 -0
- riskvalidation/ml/plants.py +425 -0
- riskvalidation/ml/policy.py +182 -0
- riskvalidation/ml/robustness.py +286 -0
- riskvalidation/ml/seeds.py +106 -0
- riskvalidation/oprisk/__init__.py +18 -0
- riskvalidation/oprisk/_checks.py +60 -0
- riskvalidation/oprisk/aggregate.py +375 -0
- riskvalidation/oprisk/distributions.py +758 -0
- riskvalidation/oprisk/scenarios.py +342 -0
- riskvalidation/oprisk/sla.py +80 -0
- riskvalidation/oprisk/sma.py +169 -0
- riskvalidation/policies/__init__.py +4 -0
- riskvalidation/policies/default.json +273 -0
- riskvalidation/portfolio/__init__.py +16 -0
- riskvalidation/portfolio/asrf_fit.py +283 -0
- riskvalidation/portfolio/copula.py +129 -0
- riskvalidation/portfolio/creditrisk_plus.py +232 -0
- riskvalidation/portfolio/euler.py +126 -0
- riskvalidation/portfolio/granularity.py +135 -0
- riskvalidation/portfolio/importance.py +277 -0
- riskvalidation/portfolio/portfolio.py +309 -0
- riskvalidation/provisions/__init__.py +8 -0
- riskvalidation/provisions/_checks.py +100 -0
- riskvalidation/provisions/ecl.py +233 -0
- riskvalidation/provisions/local_cl.py +360 -0
- riskvalidation/provisions/local_es.py +157 -0
- riskvalidation/provisions/overlays.py +102 -0
- riskvalidation/provisions/reconciliation.py +157 -0
- riskvalidation/provisions/staging.py +211 -0
- riskvalidation/provisions/term_structure.py +199 -0
- riskvalidation/regulatory/__init__.py +14 -0
- riskvalidation/regulatory/ba_cva.py +138 -0
- riskvalidation/regulatory/frtb.py +345 -0
- riskvalidation/regulatory/irb.py +314 -0
- riskvalidation/regulatory/lcr_nsfr.py +369 -0
- riskvalidation/regulatory/output_floor.py +45 -0
- riskvalidation/regulatory/sa_ccr.py +486 -0
- riskvalidation/regulatory/standardised.py +348 -0
- riskvalidation/stress/__init__.py +9 -0
- riskvalidation/stress/capital.py +476 -0
- riskvalidation/stress/paths.py +108 -0
- riskvalidation/stress/reverse.py +179 -0
- riskvalidation/stress/satellites.py +268 -0
- riskvalidation/stress/scenario_models.py +274 -0
- riskvalidation/transitions/__init__.py +0 -0
- riskvalidation/transitions/em.py +120 -0
- riskvalidation/transitions/embedding.py +221 -0
- riskvalidation/transitions/estimators.py +213 -0
- riskvalidation/transitions/intervals.py +306 -0
- riskvalidation/transitions/mobility.py +57 -0
- riskvalidation/transitions/ttc.py +171 -0
- riskvalidation/validation/__init__.py +103 -0
- riskvalidation/validation/_common.py +57 -0
- riskvalidation/validation/aggregation.py +425 -0
- riskvalidation/validation/calibration.py +757 -0
- riskvalidation/validation/ccr.py +530 -0
- riskvalidation/validation/discrimination.py +314 -0
- riskvalidation/validation/fairness.py +632 -0
- riskvalidation/validation/ifrs9.py +693 -0
- riskvalidation/validation/irrbb.py +389 -0
- riskvalidation/validation/lgd_ead.py +469 -0
- riskvalidation/validation/liquidity.py +647 -0
- riskvalidation/validation/market.py +807 -0
- riskvalidation/validation/ml.py +958 -0
- riskvalidation/validation/oprisk.py +357 -0
- riskvalidation/validation/portfolio.py +353 -0
- riskvalidation/validation/rating_system.py +245 -0
- riskvalidation/validation/stability.py +256 -0
- riskvalidation/validation/stress.py +552 -0
- riskvalidation/validation/transitions.py +386 -0
- riskvalidation-0.8.0.dist-info/METADATA +136 -0
- riskvalidation-0.8.0.dist-info/RECORD +164 -0
- riskvalidation-0.8.0.dist-info/WHEEL +5 -0
- riskvalidation-0.8.0.dist-info/licenses/LICENSE +21 -0
- riskvalidation-0.8.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""Fixed-float interest rate swaps priced along Hull-White paths on the single curve of the case (no spread): cash flows
|
|
2
|
+
by date, so a close-out window sees the flows it contains (the sticky-date approach of addendum R3).
|
|
3
|
+
|
|
4
|
+
The fixed leg pays ``N * fixed_rate * accrual`` at each fixed time. The floating leg resets on the
|
|
5
|
+
fixed schedule: at a reset date ``T_j`` the coupon ``N tau L_j`` with
|
|
6
|
+
``L_j = (1 / P(T_j, T_{j+1}) - 1) / tau`` is fixed and paid at ``T_{j+1}``.
|
|
7
|
+
Contract: ``engine-ccr-api`` 1.0.0, module swaps.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import math
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
from riskvalidation.ccr.hw1f import bond_price
|
|
19
|
+
from riskvalidation.lgd_ead._checks import mat, scalar, vec
|
|
20
|
+
|
|
21
|
+
TIME_TOL = 1e-9
|
|
22
|
+
"""Tolerance, in years, to decide that a time is a reset or a payment date (grids built by arithmetic differ
|
|
23
|
+
by 1e-16)."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class Swap:
|
|
28
|
+
trade_id: str
|
|
29
|
+
notional: float # money
|
|
30
|
+
fixed_rate: float # decimal, annual-equivalent coupon rate paid per fixed period
|
|
31
|
+
is_payer: bool # the bank pays the fixed rate
|
|
32
|
+
start: float # years
|
|
33
|
+
maturity: float # years
|
|
34
|
+
fixed_frequency: int # fixed payments per year
|
|
35
|
+
|
|
36
|
+
def __post_init__(self) -> None:
|
|
37
|
+
if not isinstance(self.trade_id, str) or not self.trade_id:
|
|
38
|
+
raise ValueError("trade_id must be a non-empty string")
|
|
39
|
+
if not (math.isfinite(self.notional) and self.notional > 0.0):
|
|
40
|
+
raise ValueError(f"notional must be positive, got {self.notional}")
|
|
41
|
+
if not math.isfinite(self.fixed_rate):
|
|
42
|
+
raise ValueError("fixed_rate is non-finite")
|
|
43
|
+
if not (math.isfinite(self.start) and self.start >= 0.0):
|
|
44
|
+
raise ValueError(f"start must be nonnegative, got {self.start}")
|
|
45
|
+
if not (math.isfinite(self.maturity) and self.maturity > self.start):
|
|
46
|
+
raise ValueError(f"maturity must exceed start, got maturity={self.maturity} and start={self.start}")
|
|
47
|
+
f = self.fixed_frequency
|
|
48
|
+
if isinstance(f, bool) or int(f) != f or f < 1:
|
|
49
|
+
raise ValueError(f"fixed_frequency must be a positive integer, got {f}")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def cash_flow_times(swap: Swap) -> np.ndarray:
|
|
53
|
+
"""(n,) the fixed payment times ``start + k / frequency`` up to the maturity, the maturity last
|
|
54
|
+
(a last period shorter than ``1 / frequency`` ends at the maturity)."""
|
|
55
|
+
f = int(swap.fixed_frequency)
|
|
56
|
+
n = max(int(math.ceil((swap.maturity - swap.start) * f - 1e-9)), 1)
|
|
57
|
+
times = swap.start + np.arange(1, n + 1) / f
|
|
58
|
+
times[-1] = swap.maturity
|
|
59
|
+
return times
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def reset_times(swap: Swap) -> np.ndarray:
|
|
63
|
+
"""(n,) the reset dates of the floating leg: the start and every fixed payment time but the last."""
|
|
64
|
+
times = cash_flow_times(swap)
|
|
65
|
+
return np.concatenate([[swap.start], times[:-1]])
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _accruals(swap: Swap) -> np.ndarray:
|
|
69
|
+
return cash_flow_times(swap) - reset_times(swap)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def swap_value(swap: Swap, t: float, r: np.ndarray, a: float, sigma: float, p0: Callable, forward: Callable,
|
|
73
|
+
last_fixing: np.ndarray | None = None) -> np.ndarray:
|
|
74
|
+
"""(P,) the value to the bank at ``t`` given the short rates ``r`` at ``t``; positive is an asset, a payer value is
|
|
75
|
+
minus the receiver value of the same terms. Only the flows after ``t`` are valued (sticky date).
|
|
76
|
+
|
|
77
|
+
Fixed leg: ``sum c_i P(t, T_i)``. Floating leg: with ``t`` on a reset date or before the first one,
|
|
78
|
+
``N (P(t, T_0') - P(t, T_n))`` with ``T_0'`` the next reset at or after ``t``; strictly between two resets
|
|
79
|
+
``T_j < t < T_{j+1}`` the coupon fixed at ``T_j`` is known and the leg is
|
|
80
|
+
``N tau L_j P(t, T_{j+1}) + N (P(t, T_{j+1}) - P(t, T_n))``, which needs ``last_fixing`` (the simple rate ``L_j``
|
|
81
|
+
of each path); its absence there raises ``ValueError`` (never silently approximated)."""
|
|
82
|
+
tt = scalar(t, "t")
|
|
83
|
+
if tt < 0.0:
|
|
84
|
+
raise ValueError("t must be nonnegative")
|
|
85
|
+
rr = vec(r, "r")
|
|
86
|
+
times, resets, accr = cash_flow_times(swap), reset_times(swap), _accruals(swap)
|
|
87
|
+
after = times > tt + TIME_TOL
|
|
88
|
+
if not np.any(after):
|
|
89
|
+
return np.zeros_like(rr)
|
|
90
|
+
n = swap.notional
|
|
91
|
+
|
|
92
|
+
def p(big_t: float) -> np.ndarray:
|
|
93
|
+
return bond_price(rr, tt, float(big_t), a, sigma, p0, forward)
|
|
94
|
+
|
|
95
|
+
fixed = np.zeros_like(rr)
|
|
96
|
+
for big_t, tau in zip(times[after], accr[after], strict=True):
|
|
97
|
+
fixed += n * swap.fixed_rate * tau * p(big_t)
|
|
98
|
+
j = int(np.searchsorted(resets, tt + TIME_TOL, side="right")) - 1 # the last reset on or before t
|
|
99
|
+
on_reset = j >= 0 and abs(resets[j] - tt) <= TIME_TOL
|
|
100
|
+
p_end = p(times[-1])
|
|
101
|
+
if j < 0 or on_reset:
|
|
102
|
+
nxt = resets[0] if j < 0 else resets[j]
|
|
103
|
+
head = np.ones_like(rr) if abs(nxt - tt) <= TIME_TOL else p(nxt)
|
|
104
|
+
floating = n * (head - p_end)
|
|
105
|
+
else:
|
|
106
|
+
if last_fixing is None:
|
|
107
|
+
raise ValueError(f"last_fixing is required strictly between two resets (t={tt}); the valuation is never "
|
|
108
|
+
"approximated without the path's fixing")
|
|
109
|
+
lf = np.array(last_fixing, dtype=float)
|
|
110
|
+
if lf.shape != rr.shape or not np.all(np.isfinite(lf)):
|
|
111
|
+
raise ValueError(f"last_fixing must be a finite array of the shape of r {rr.shape}, got {lf.shape}")
|
|
112
|
+
p_next = p(times[j])
|
|
113
|
+
floating = n * accr[j] * lf * p_next + n * (p_next - p_end)
|
|
114
|
+
return floating - fixed if swap.is_payer else fixed - floating
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def par_fixed_rate(maturity: float, frequency: int, p0: Callable, start: float = 0.0) -> float:
|
|
118
|
+
"""``(P(0, start) - P(0, maturity)) / sum (1 / frequency) P(0, T_i)``:
|
|
119
|
+
the fixed rate at which the value of a swap is 0."""
|
|
120
|
+
s = Swap("par", 1.0, 0.0, False, scalar(start, "start"), scalar(maturity, "maturity"), frequency)
|
|
121
|
+
times = cash_flow_times(s)
|
|
122
|
+
df = np.asarray(p0(np.concatenate([[s.start], times])), dtype=float)
|
|
123
|
+
return float((df[0] - df[-1]) / np.sum(_accruals(s) * df[1:]))
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def simulation_grid(grid: np.ndarray, swaps: list[Swap]) -> np.ndarray:
|
|
127
|
+
"""The union of ``grid`` and every reset date of every swap that is not after the last grid point: the grid to
|
|
128
|
+
simulate on, so that the fixing at the last reset is read from the same path (``swap_values_on_grid``)."""
|
|
129
|
+
g = vec(grid, "grid")
|
|
130
|
+
if np.any(np.diff(g) <= 0.0) or g[0] < 0.0:
|
|
131
|
+
raise ValueError("grid must be nonnegative and strictly increasing")
|
|
132
|
+
parts = [g] + [reset_times(s)[reset_times(s) <= g[-1] + TIME_TOL] for s in swaps]
|
|
133
|
+
merged = np.sort(np.concatenate(parts))
|
|
134
|
+
keep = np.concatenate([[True], np.diff(merged) > TIME_TOL])
|
|
135
|
+
return merged[keep]
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def swap_values_on_grid(swaps: list[Swap], grid: np.ndarray, short_rate: np.ndarray, a: float, sigma: float,
|
|
139
|
+
p0: Callable, forward: Callable) -> np.ndarray:
|
|
140
|
+
"""(S, P, K) the value of each swap along each path at each grid point; the sum over swaps is the netting-set value
|
|
141
|
+
(netting is on the sum, not on the sum of positive parts).
|
|
142
|
+
|
|
143
|
+
``short_rate`` (P, K) comes from ``simulate_short_rate`` on ``grid``, which must therefore hold every reset date the
|
|
144
|
+
valuation reads (use ``simulation_grid``); between two resets the fixing is the simple rate of the path at the last
|
|
145
|
+
reset, ``L_j = (1 / P(T_j, T_{j+1}; r(T_j)) - 1) / tau``, and a
|
|
146
|
+
reset date missing from ``grid`` raises ``ValueError``."""
|
|
147
|
+
if len(swaps) == 0:
|
|
148
|
+
raise ValueError("swaps must not be empty")
|
|
149
|
+
g = vec(grid, "grid")
|
|
150
|
+
if np.any(np.diff(g) <= 0.0) or g[0] < 0.0:
|
|
151
|
+
raise ValueError("grid must be nonnegative and strictly increasing")
|
|
152
|
+
sr = mat(short_rate, "short_rate")
|
|
153
|
+
if sr.shape[1] != g.size:
|
|
154
|
+
raise ValueError(f"short_rate has {sr.shape[1]} columns for a grid of {g.size} points")
|
|
155
|
+
out = np.zeros((len(swaps), sr.shape[0], g.size))
|
|
156
|
+
for s_i, sw in enumerate(swaps):
|
|
157
|
+
times, resets = cash_flow_times(sw), reset_times(sw)
|
|
158
|
+
for k, t in enumerate(g):
|
|
159
|
+
if t >= sw.maturity - TIME_TOL:
|
|
160
|
+
continue
|
|
161
|
+
j = int(np.searchsorted(resets, t + TIME_TOL, side="right")) - 1
|
|
162
|
+
fixing = None
|
|
163
|
+
if j >= 0 and abs(resets[j] - t) > TIME_TOL: # strictly between two resets
|
|
164
|
+
idx = int(np.argmin(np.abs(g - resets[j])))
|
|
165
|
+
if abs(g[idx] - resets[j]) > TIME_TOL:
|
|
166
|
+
raise ValueError(f"grid must contain the reset date {resets[j]} of swap {sw.trade_id!r} "
|
|
167
|
+
"(simulate on simulation_grid)")
|
|
168
|
+
tau = float(times[j] - resets[j])
|
|
169
|
+
p_fix = bond_price(sr[:, idx], float(resets[j]), float(times[j]), a, sigma, p0, forward)
|
|
170
|
+
fixing = (1.0 / p_fix - 1.0) / tau
|
|
171
|
+
out[s_i, :, k] = swap_value(sw, float(t), sr[:, k], a, sigma, p0, forward, fixing)
|
|
172
|
+
return out
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Core contracts: the result record, the policy, the registry, input hashing and the suite runner."""
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Assemble a :class:`TestResult` from a registered test's statistics and a policy."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from riskvalidation.core.hashing import inputs_hash
|
|
9
|
+
from riskvalidation.core.policy import Policy, default_policy
|
|
10
|
+
from riskvalidation.core.registry import spec as get_spec
|
|
11
|
+
from riskvalidation.core.result import Light, TestResult
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def make_result(
|
|
15
|
+
test_id: str,
|
|
16
|
+
*,
|
|
17
|
+
statistic: float | None = None,
|
|
18
|
+
p_value: float | None = None,
|
|
19
|
+
metric: float | None = None,
|
|
20
|
+
policy: Policy | None = None,
|
|
21
|
+
alternative: str = "",
|
|
22
|
+
n: int | None = None,
|
|
23
|
+
n_events: int | None = None,
|
|
24
|
+
model_id: str | None = None,
|
|
25
|
+
segment: str | None = None,
|
|
26
|
+
inputs: Sequence[Any] = (),
|
|
27
|
+
notes: str = "",
|
|
28
|
+
extras: Mapping[str, Any] | None = None,
|
|
29
|
+
) -> TestResult:
|
|
30
|
+
"""Build the result of a registered test, classifying it under ``policy`` (the default when ``None``)."""
|
|
31
|
+
s = get_spec(test_id)
|
|
32
|
+
pol = policy if policy is not None else default_policy()
|
|
33
|
+
light, th = pol.classify(test_id, p_value=p_value, metric=metric)
|
|
34
|
+
if th is not None and light is Light.NOT_EVALUATED and not notes:
|
|
35
|
+
notes = "statistic undefined for this sample; no light assigned"
|
|
36
|
+
elif th is None and not notes:
|
|
37
|
+
notes = f"policy {pol.label} holds no thresholds for this test; no light assigned"
|
|
38
|
+
return TestResult(
|
|
39
|
+
test_id=test_id,
|
|
40
|
+
h0=s.h0,
|
|
41
|
+
reference=s.reference,
|
|
42
|
+
policy_version=pol.label,
|
|
43
|
+
light=light,
|
|
44
|
+
statistic=statistic,
|
|
45
|
+
p_value=p_value,
|
|
46
|
+
metric=metric,
|
|
47
|
+
alternative=alternative,
|
|
48
|
+
alpha_amber=None if th is None else th.amber,
|
|
49
|
+
alpha_red=None if th is None else th.red,
|
|
50
|
+
n=n,
|
|
51
|
+
n_events=n_events,
|
|
52
|
+
model_id=model_id,
|
|
53
|
+
segment=segment,
|
|
54
|
+
inputs_hash=inputs_hash(*inputs) if inputs else "",
|
|
55
|
+
notes=notes,
|
|
56
|
+
extras=dict(extras or {}),
|
|
57
|
+
)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Deterministic fingerprints of test inputs.
|
|
2
|
+
|
|
3
|
+
A validation result is valid only for the exact inputs it tested. ``inputs_hash`` makes that checkable: equal
|
|
4
|
+
inputs give the same SHA-256 on every platform, and changing any value, shape or dtype changes it.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
from collections.abc import Mapping, Sequence
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _feed(h: Any, obj: Any) -> None:
|
|
18
|
+
if isinstance(obj, np.ndarray):
|
|
19
|
+
arr = np.ascontiguousarray(obj)
|
|
20
|
+
if arr.dtype.byteorder == ">": # canonical little-endian bytes
|
|
21
|
+
arr = arr.astype(arr.dtype.newbyteorder("<"))
|
|
22
|
+
h.update(b"ndarray")
|
|
23
|
+
h.update(arr.dtype.str.encode())
|
|
24
|
+
h.update(repr(arr.shape).encode())
|
|
25
|
+
h.update(arr.tobytes())
|
|
26
|
+
elif isinstance(obj, Mapping):
|
|
27
|
+
h.update(b"mapping")
|
|
28
|
+
for key in sorted(obj, key=str):
|
|
29
|
+
h.update(str(key).encode())
|
|
30
|
+
_feed(h, obj[key])
|
|
31
|
+
elif isinstance(obj, (list, tuple)) or (isinstance(obj, Sequence) and not isinstance(obj, (str, bytes))):
|
|
32
|
+
h.update(b"sequence")
|
|
33
|
+
h.update(str(len(obj)).encode())
|
|
34
|
+
for item in obj:
|
|
35
|
+
_feed(h, item)
|
|
36
|
+
elif isinstance(obj, np.generic):
|
|
37
|
+
_feed(h, np.asarray(obj))
|
|
38
|
+
else:
|
|
39
|
+
h.update(b"scalar")
|
|
40
|
+
h.update(json.dumps(obj, sort_keys=True, default=repr).encode())
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def inputs_hash(*objs: Any) -> str:
|
|
44
|
+
"""SHA-256 hex digest over the given inputs, in order."""
|
|
45
|
+
h = hashlib.sha256()
|
|
46
|
+
for obj in objs:
|
|
47
|
+
_feed(h, obj)
|
|
48
|
+
return h.hexdigest()
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Versioned validation policy: the thresholds that turn a statistic into a traffic light.
|
|
2
|
+
|
|
3
|
+
Thresholds are an institution's policy choice, not regulation. The ECB's 2019 instructions for reporting IRB
|
|
4
|
+
validation results define the tests and statistics but set no pass or fail thresholds; the PSI bands of 0.10
|
|
5
|
+
and 0.25 are an industry convention with no control of the type I error (Yurdakul and Naranjo 2020); BCBS
|
|
6
|
+
Working Paper 14 (2005) notes that no really powerful PD calibration test exists once defaults are correlated.
|
|
7
|
+
A policy therefore lives in a versioned data file, is shown to users as "policy", and travels with every
|
|
8
|
+
result.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
from collections.abc import Mapping
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from importlib import resources
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from riskvalidation.core.result import Light
|
|
22
|
+
|
|
23
|
+
KINDS = ("p_value", "upper_bad", "lower_bad")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class Threshold:
|
|
28
|
+
"""Two cut points for one test.
|
|
29
|
+
|
|
30
|
+
- ``p_value``: red if ``p < red``, amber if ``red <= p < amber``, green otherwise (``red <= amber``).
|
|
31
|
+
- ``upper_bad`` (larger is worse, for example PSI): red if ``m >= red``, amber if ``amber <= m < red``.
|
|
32
|
+
- ``lower_bad`` (smaller is worse, for example AUC): red if ``m <= red``, amber if ``red < m <= amber``.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
kind: str
|
|
36
|
+
amber: float
|
|
37
|
+
red: float
|
|
38
|
+
source: str = ""
|
|
39
|
+
|
|
40
|
+
def __post_init__(self) -> None:
|
|
41
|
+
if self.kind not in KINDS:
|
|
42
|
+
raise ValueError(f"threshold kind must be one of {KINDS}, got {self.kind!r}")
|
|
43
|
+
if not (math.isfinite(self.amber) and math.isfinite(self.red)):
|
|
44
|
+
raise ValueError("thresholds must be finite")
|
|
45
|
+
if self.kind in ("p_value", "lower_bad") and self.red > self.amber:
|
|
46
|
+
raise ValueError(f"{self.kind}: red ({self.red}) must not exceed amber ({self.amber})")
|
|
47
|
+
if self.kind == "upper_bad" and self.red < self.amber:
|
|
48
|
+
raise ValueError(f"upper_bad: red ({self.red}) must not be below amber ({self.amber})")
|
|
49
|
+
|
|
50
|
+
def classify(self, value: float | None) -> Light:
|
|
51
|
+
if value is None or not math.isfinite(float(value)):
|
|
52
|
+
return Light.NOT_EVALUATED
|
|
53
|
+
v = float(value)
|
|
54
|
+
if self.kind == "p_value":
|
|
55
|
+
if v < self.red:
|
|
56
|
+
return Light.RED
|
|
57
|
+
return Light.AMBER if v < self.amber else Light.GREEN
|
|
58
|
+
if self.kind == "upper_bad":
|
|
59
|
+
if v >= self.red:
|
|
60
|
+
return Light.RED
|
|
61
|
+
return Light.AMBER if v >= self.amber else Light.GREEN
|
|
62
|
+
if v <= self.red: # lower_bad
|
|
63
|
+
return Light.RED
|
|
64
|
+
return Light.AMBER if v <= self.amber else Light.GREEN
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class Policy:
|
|
69
|
+
"""A named, versioned set of thresholds keyed by test id."""
|
|
70
|
+
|
|
71
|
+
policy_id: str
|
|
72
|
+
version: str
|
|
73
|
+
description: str
|
|
74
|
+
thresholds: Mapping[str, Threshold] = field(default_factory=dict)
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def label(self) -> str:
|
|
78
|
+
return f"{self.policy_id} {self.version}"
|
|
79
|
+
|
|
80
|
+
def threshold(self, test_id: str) -> Threshold | None:
|
|
81
|
+
return self.thresholds.get(test_id)
|
|
82
|
+
|
|
83
|
+
def classify(
|
|
84
|
+
self, test_id: str, *, p_value: float | None = None, metric: float | None = None
|
|
85
|
+
) -> tuple[Light, Threshold | None]:
|
|
86
|
+
"""Light for a result. ``p_value`` thresholds read the p-value; the others read the metric."""
|
|
87
|
+
th = self.threshold(test_id)
|
|
88
|
+
if th is None:
|
|
89
|
+
return Light.NOT_EVALUATED, None
|
|
90
|
+
value = p_value if th.kind == "p_value" else metric
|
|
91
|
+
return th.classify(value), th
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def policy_from_mapping(data: Mapping[str, Any]) -> Policy:
|
|
95
|
+
try:
|
|
96
|
+
thresholds = {
|
|
97
|
+
str(tid): Threshold(
|
|
98
|
+
kind=str(spec["kind"]),
|
|
99
|
+
amber=float(spec["amber"]),
|
|
100
|
+
red=float(spec["red"]),
|
|
101
|
+
source=str(spec.get("source", "")),
|
|
102
|
+
)
|
|
103
|
+
for tid, spec in dict(data.get("thresholds", {})).items()
|
|
104
|
+
}
|
|
105
|
+
return Policy(
|
|
106
|
+
policy_id=str(data["policy_id"]),
|
|
107
|
+
version=str(data["version"]),
|
|
108
|
+
description=str(data.get("description", "")),
|
|
109
|
+
thresholds=thresholds,
|
|
110
|
+
)
|
|
111
|
+
except KeyError as exc:
|
|
112
|
+
raise ValueError(f"policy file is missing the field {exc.args[0]!r}") from exc
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def load_policy(path: str | Path) -> Policy:
|
|
116
|
+
"""Load a policy from a JSON file."""
|
|
117
|
+
with open(path, encoding="utf-8") as fh:
|
|
118
|
+
return policy_from_mapping(json.load(fh))
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
_DEFAULT: Policy | None = None
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def default_policy() -> Policy:
|
|
125
|
+
"""The packaged default policy (conventional thresholds, documented per test)."""
|
|
126
|
+
global _DEFAULT
|
|
127
|
+
if _DEFAULT is None:
|
|
128
|
+
text = resources.files("riskvalidation.policies").joinpath("default.json").read_text("utf-8")
|
|
129
|
+
_DEFAULT = policy_from_mapping(json.loads(text))
|
|
130
|
+
return _DEFAULT
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""The test catalogue: every validation test registers its identity, hypothesis and reference once.
|
|
2
|
+
|
|
3
|
+
The registry is what lets a suite, a report or a web page list the tests that exist, with the primary source
|
|
4
|
+
that defines each one, without importing anything but this package.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Callable
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
DECISIONS = ("p_value", "metric")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class TestSpec:
|
|
18
|
+
"""Identity of one registered test."""
|
|
19
|
+
|
|
20
|
+
__test__ = False # not a pytest test class
|
|
21
|
+
|
|
22
|
+
test_id: str
|
|
23
|
+
family: str
|
|
24
|
+
name: str
|
|
25
|
+
h0: str
|
|
26
|
+
reference: str
|
|
27
|
+
decision: str
|
|
28
|
+
func: Callable[..., Any]
|
|
29
|
+
|
|
30
|
+
def __post_init__(self) -> None:
|
|
31
|
+
if self.decision not in DECISIONS:
|
|
32
|
+
raise ValueError(f"decision must be one of {DECISIONS}, got {self.decision!r}")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
_CATALOGUE: dict[str, TestSpec] = {}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def register(
|
|
39
|
+
test_id: str, *, family: str, name: str, h0: str, reference: str, decision: str
|
|
40
|
+
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
|
|
41
|
+
"""Decorator that registers a test function under ``test_id``."""
|
|
42
|
+
|
|
43
|
+
def wrap(func: Callable[..., Any]) -> Callable[..., Any]:
|
|
44
|
+
existing = _CATALOGUE.get(test_id)
|
|
45
|
+
if existing is not None and existing.func.__qualname__ != func.__qualname__:
|
|
46
|
+
raise ValueError(f"test id {test_id!r} is already registered by {existing.func.__qualname__}")
|
|
47
|
+
spec = TestSpec(test_id, family, name, h0, reference, decision, func)
|
|
48
|
+
_CATALOGUE[test_id] = spec
|
|
49
|
+
func.__rv_spec__ = spec # type: ignore[attr-defined]
|
|
50
|
+
return func
|
|
51
|
+
|
|
52
|
+
return wrap
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def spec(test_id: str) -> TestSpec:
|
|
56
|
+
try:
|
|
57
|
+
return _CATALOGUE[test_id]
|
|
58
|
+
except KeyError as exc:
|
|
59
|
+
raise KeyError(f"no registered test {test_id!r}") from exc
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def catalogue(family: str | None = None) -> list[TestSpec]:
|
|
63
|
+
"""Registered tests, optionally of one family, sorted by id."""
|
|
64
|
+
items = (s for s in _CATALOGUE.values() if family is None or s.family == family)
|
|
65
|
+
return sorted(items, key=lambda s: s.test_id)
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""The single record every validation test returns.
|
|
2
|
+
|
|
3
|
+
A validation result is evidence, not a verdict on compliance: it carries the statistic, the null hypothesis it
|
|
4
|
+
was tested against, the reference that defines it, and the traffic light assigned by a versioned *policy*.
|
|
5
|
+
Thresholds never live in the test code (see :mod:`riskvalidation.core.policy`).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import enum
|
|
11
|
+
import math
|
|
12
|
+
from collections.abc import Mapping
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Light(str, enum.Enum):
|
|
18
|
+
"""Traffic light of one test result under a policy.
|
|
19
|
+
|
|
20
|
+
``NOT_EVALUATED`` means the policy holds no thresholds for the test, or the statistic is undefined (for
|
|
21
|
+
example a calibration test on a grade with no obligors); no colour is shown for it. ``ERROR`` means the
|
|
22
|
+
test raised; the exception is recorded in ``notes``.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
GREEN = "green"
|
|
26
|
+
AMBER = "amber"
|
|
27
|
+
RED = "red"
|
|
28
|
+
NOT_EVALUATED = "not_evaluated"
|
|
29
|
+
ERROR = "error"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _finite_or_none(x: float | None) -> float | None:
|
|
33
|
+
if x is None:
|
|
34
|
+
return None
|
|
35
|
+
x = float(x)
|
|
36
|
+
return x if math.isfinite(x) else None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class TestResult:
|
|
41
|
+
"""Outcome of one validation test on one model, segment and sample.
|
|
42
|
+
|
|
43
|
+
Required: ``test_id``, ``h0`` (the null hypothesis or, for descriptive metrics, what is being measured),
|
|
44
|
+
``reference`` (the primary source that defines the test), ``policy_version`` and ``light``. A result that
|
|
45
|
+
is not an error must carry at least one of ``statistic``, ``p_value`` or ``metric``.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
__test__ = False # not a pytest test class
|
|
49
|
+
|
|
50
|
+
test_id: str
|
|
51
|
+
h0: str
|
|
52
|
+
reference: str
|
|
53
|
+
policy_version: str
|
|
54
|
+
light: Light
|
|
55
|
+
statistic: float | None = None
|
|
56
|
+
p_value: float | None = None
|
|
57
|
+
metric: float | None = None
|
|
58
|
+
alternative: str = ""
|
|
59
|
+
alpha_amber: float | None = None
|
|
60
|
+
alpha_red: float | None = None
|
|
61
|
+
n: int | None = None
|
|
62
|
+
n_events: int | None = None
|
|
63
|
+
model_id: str | None = None
|
|
64
|
+
segment: str | None = None
|
|
65
|
+
inputs_hash: str = ""
|
|
66
|
+
notes: str = ""
|
|
67
|
+
extras: Mapping[str, Any] = field(default_factory=dict)
|
|
68
|
+
|
|
69
|
+
def __post_init__(self) -> None:
|
|
70
|
+
for name in ("test_id", "h0", "reference", "policy_version"):
|
|
71
|
+
value = getattr(self, name)
|
|
72
|
+
if not isinstance(value, str) or not value.strip():
|
|
73
|
+
raise ValueError(f"TestResult.{name} must be a non-empty string")
|
|
74
|
+
if not isinstance(self.light, Light):
|
|
75
|
+
raise TypeError("TestResult.light must be a Light")
|
|
76
|
+
if self.p_value is not None and not (0.0 <= self.p_value <= 1.0):
|
|
77
|
+
raise ValueError(f"TestResult.p_value must lie in [0, 1], got {self.p_value}")
|
|
78
|
+
if self.light is not Light.ERROR and all(
|
|
79
|
+
v is None for v in (self.statistic, self.p_value, self.metric)
|
|
80
|
+
):
|
|
81
|
+
raise ValueError("a non-error TestResult needs a statistic, a p-value or a metric")
|
|
82
|
+
|
|
83
|
+
def to_dict(self) -> dict[str, Any]:
|
|
84
|
+
"""JSON-safe dictionary; non-finite numbers become ``None``."""
|
|
85
|
+
out: dict[str, Any] = {
|
|
86
|
+
"test_id": self.test_id,
|
|
87
|
+
"model_id": self.model_id,
|
|
88
|
+
"segment": self.segment,
|
|
89
|
+
"statistic": _finite_or_none(self.statistic),
|
|
90
|
+
"p_value": _finite_or_none(self.p_value),
|
|
91
|
+
"metric": _finite_or_none(self.metric),
|
|
92
|
+
"h0": self.h0,
|
|
93
|
+
"alternative": self.alternative,
|
|
94
|
+
"policy_version": self.policy_version,
|
|
95
|
+
"alpha_amber": _finite_or_none(self.alpha_amber),
|
|
96
|
+
"alpha_red": _finite_or_none(self.alpha_red),
|
|
97
|
+
"light": self.light.value,
|
|
98
|
+
"n": self.n,
|
|
99
|
+
"n_events": self.n_events,
|
|
100
|
+
"inputs_hash": self.inputs_hash,
|
|
101
|
+
"reference": self.reference,
|
|
102
|
+
"notes": self.notes,
|
|
103
|
+
"extras": {k: _jsonable(v) for k, v in self.extras.items()},
|
|
104
|
+
}
|
|
105
|
+
return out
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _jsonable(v: Any) -> Any:
|
|
109
|
+
if isinstance(v, float):
|
|
110
|
+
return _finite_or_none(v)
|
|
111
|
+
if isinstance(v, (list, tuple)):
|
|
112
|
+
return [_jsonable(x) for x in v]
|
|
113
|
+
if isinstance(v, Mapping):
|
|
114
|
+
return {str(k): _jsonable(x) for k, x in v.items()}
|
|
115
|
+
if hasattr(v, "tolist"): # numpy scalars and arrays
|
|
116
|
+
return _jsonable(v.tolist())
|
|
117
|
+
return v
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Run a battery of validation tests under one policy, never stopping at the first failure.
|
|
2
|
+
|
|
3
|
+
A validator needs the whole picture: a test that raises (bad input, an empty segment) is recorded as an
|
|
4
|
+
``ERROR`` result with the exception text, and the remaining tests still run.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections import Counter
|
|
10
|
+
from collections.abc import Callable, Iterable, Mapping
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from riskvalidation.core.policy import Policy, default_policy
|
|
15
|
+
from riskvalidation.core.result import Light, TestResult
|
|
16
|
+
|
|
17
|
+
Task = tuple[Callable[..., TestResult], Mapping[str, Any]]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class SuiteRun:
|
|
22
|
+
"""Results of one suite run, in task order."""
|
|
23
|
+
|
|
24
|
+
policy_version: str
|
|
25
|
+
results: tuple[TestResult, ...]
|
|
26
|
+
|
|
27
|
+
def counts(self) -> dict[str, int]:
|
|
28
|
+
c = Counter(r.light.value for r in self.results)
|
|
29
|
+
return {light.value: c.get(light.value, 0) for light in Light}
|
|
30
|
+
|
|
31
|
+
def table(self) -> list[dict[str, Any]]:
|
|
32
|
+
return [r.to_dict() for r in self.results]
|
|
33
|
+
|
|
34
|
+
def worst(self) -> Light:
|
|
35
|
+
"""Most severe evaluated light (ERROR outranks RED); NOT_EVALUATED only if nothing was evaluated."""
|
|
36
|
+
order = [Light.ERROR, Light.RED, Light.AMBER, Light.GREEN]
|
|
37
|
+
present = {r.light for r in self.results}
|
|
38
|
+
for light in order:
|
|
39
|
+
if light in present:
|
|
40
|
+
return light
|
|
41
|
+
return Light.NOT_EVALUATED
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def run_suite(
|
|
45
|
+
tasks: Iterable[Task],
|
|
46
|
+
*,
|
|
47
|
+
policy: Policy | None = None,
|
|
48
|
+
model_id: str | None = None,
|
|
49
|
+
segment: str | None = None,
|
|
50
|
+
) -> SuiteRun:
|
|
51
|
+
"""Run each ``(test_function, kwargs)`` task with the shared policy, model id and segment."""
|
|
52
|
+
pol = policy if policy is not None else default_policy()
|
|
53
|
+
results: list[TestResult] = []
|
|
54
|
+
for func, kwargs in tasks:
|
|
55
|
+
call = dict(kwargs)
|
|
56
|
+
call.setdefault("policy", pol)
|
|
57
|
+
if model_id is not None:
|
|
58
|
+
call.setdefault("model_id", model_id)
|
|
59
|
+
if segment is not None:
|
|
60
|
+
call.setdefault("segment", segment)
|
|
61
|
+
try:
|
|
62
|
+
result = func(**call)
|
|
63
|
+
if not isinstance(result, TestResult):
|
|
64
|
+
raise TypeError(f"{func.__qualname__} returned {type(result).__name__}, not TestResult")
|
|
65
|
+
except Exception as exc: # recorded, never raised: the suite must finish
|
|
66
|
+
s = getattr(func, "__rv_spec__", None)
|
|
67
|
+
result = TestResult(
|
|
68
|
+
test_id=s.test_id if s else func.__qualname__,
|
|
69
|
+
h0=s.h0 if s else "not computed",
|
|
70
|
+
reference=s.reference if s else "not computed",
|
|
71
|
+
policy_version=pol.label,
|
|
72
|
+
light=Light.ERROR,
|
|
73
|
+
model_id=call.get("model_id"),
|
|
74
|
+
segment=call.get("segment"),
|
|
75
|
+
notes=f"{type(exc).__name__}: {exc}",
|
|
76
|
+
)
|
|
77
|
+
results.append(result)
|
|
78
|
+
return SuiteRun(policy_version=pol.label, results=tuple(results))
|