riskvalidation 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- riskvalidation/__init__.py +30 -0
- riskvalidation/aggregation/__init__.py +11 -0
- riskvalidation/aggregation/band.py +151 -0
- riskvalidation/aggregation/bounds.py +191 -0
- riskvalidation/aggregation/copulas.py +241 -0
- riskvalidation/aggregation/model_risk.py +133 -0
- riskvalidation/aggregation/simple.py +147 -0
- riskvalidation/ccr/__init__.py +13 -0
- riskvalidation/ccr/collateral.py +175 -0
- riskvalidation/ccr/cva.py +193 -0
- riskvalidation/ccr/exposure.py +192 -0
- riskvalidation/ccr/hw1f.py +426 -0
- riskvalidation/ccr/swaps.py +172 -0
- riskvalidation/core/__init__.py +1 -0
- riskvalidation/core/build.py +57 -0
- riskvalidation/core/hashing.py +48 -0
- riskvalidation/core/policy.py +130 -0
- riskvalidation/core/registry.py +65 -0
- riskvalidation/core/result.py +117 -0
- riskvalidation/core/suite.py +78 -0
- riskvalidation/engines/__init__.py +2 -0
- riskvalidation/engines/low_default.py +170 -0
- riskvalidation/engines/pd_curve.py +529 -0
- riskvalidation/engines/vasicek.py +72 -0
- riskvalidation/fairness/__init__.py +10 -0
- riskvalidation/fairness/_checks.py +115 -0
- riskvalidation/fairness/conditional.py +201 -0
- riskvalidation/fairness/inference.py +64 -0
- riskvalidation/fairness/metrics.py +261 -0
- riskvalidation/fairness/plants.py +240 -0
- riskvalidation/fairness/policy.py +97 -0
- riskvalidation/fairness/replication.py +39 -0
- riskvalidation/fairness/situation.py +307 -0
- riskvalidation/generators/__init__.py +68 -0
- riskvalidation/generators/_base.py +145 -0
- riskvalidation/generators/counts.py +232 -0
- riskvalidation/generators/drift.py +107 -0
- riskvalidation/generators/framingham.py +124 -0
- riskvalidation/generators/paths.py +349 -0
- riskvalidation/generators/ratings.py +186 -0
- riskvalidation/generators/scores.py +347 -0
- riskvalidation/harness/__init__.py +46 -0
- riskvalidation/harness/exact.py +137 -0
- riskvalidation/harness/performance.py +74 -0
- riskvalidation/harness/rates.py +103 -0
- riskvalidation/harness/run.py +197 -0
- riskvalidation/irrbb/__init__.py +9 -0
- riskvalidation/irrbb/behavioural.py +206 -0
- riskvalidation/irrbb/curves.py +286 -0
- riskvalidation/irrbb/eve_nii.py +245 -0
- riskvalidation/irrbb/shocks.py +171 -0
- riskvalidation/irrbb/simulation.py +203 -0
- riskvalidation/irrbb/slotting.py +184 -0
- riskvalidation/lgd_ead/__init__.py +8 -0
- riskvalidation/lgd_ead/_checks.py +50 -0
- riskvalidation/lgd_ead/ccf.py +228 -0
- riskvalidation/lgd_ead/downturn.py +64 -0
- riskvalidation/lgd_ead/floors.py +114 -0
- riskvalidation/lgd_ead/lgd_models.py +297 -0
- riskvalidation/lgd_ead/workout.py +162 -0
- riskvalidation/liquidity/__init__.py +9 -0
- riskvalidation/liquidity/_checks.py +78 -0
- riskvalidation/liquidity/cbc.py +158 -0
- riskvalidation/liquidity/deposits.py +151 -0
- riskvalidation/liquidity/ladder.py +119 -0
- riskvalidation/liquidity/reverse.py +141 -0
- riskvalidation/liquidity/runoff.py +220 -0
- riskvalidation/liquidity/scenarios.py +272 -0
- riskvalidation/market/__init__.py +8 -0
- riskvalidation/market/caviar.py +239 -0
- riskvalidation/market/conventions.py +83 -0
- riskvalidation/market/evt.py +111 -0
- riskvalidation/market/frtb.py +175 -0
- riskvalidation/market/garch.py +204 -0
- riskvalidation/market/gas_fz.py +219 -0
- riskvalidation/market/historical.py +96 -0
- riskvalidation/market/parametric.py +78 -0
- riskvalidation/market/scoring.py +145 -0
- riskvalidation/ml/__init__.py +11 -0
- riskvalidation/ml/_checks.py +67 -0
- riskvalidation/ml/_logistic.py +82 -0
- riskvalidation/ml/cbpe.py +91 -0
- riskvalidation/ml/challenger.py +73 -0
- riskvalidation/ml/conformal.py +116 -0
- riskvalidation/ml/dataquality.py +130 -0
- riskvalidation/ml/effects.py +67 -0
- riskvalidation/ml/explanations.py +125 -0
- riskvalidation/ml/generalisation.py +79 -0
- riskvalidation/ml/leakage.py +99 -0
- riskvalidation/ml/plants.py +425 -0
- riskvalidation/ml/policy.py +182 -0
- riskvalidation/ml/robustness.py +286 -0
- riskvalidation/ml/seeds.py +106 -0
- riskvalidation/oprisk/__init__.py +18 -0
- riskvalidation/oprisk/_checks.py +60 -0
- riskvalidation/oprisk/aggregate.py +375 -0
- riskvalidation/oprisk/distributions.py +758 -0
- riskvalidation/oprisk/scenarios.py +342 -0
- riskvalidation/oprisk/sla.py +80 -0
- riskvalidation/oprisk/sma.py +169 -0
- riskvalidation/policies/__init__.py +4 -0
- riskvalidation/policies/default.json +273 -0
- riskvalidation/portfolio/__init__.py +16 -0
- riskvalidation/portfolio/asrf_fit.py +283 -0
- riskvalidation/portfolio/copula.py +129 -0
- riskvalidation/portfolio/creditrisk_plus.py +232 -0
- riskvalidation/portfolio/euler.py +126 -0
- riskvalidation/portfolio/granularity.py +135 -0
- riskvalidation/portfolio/importance.py +277 -0
- riskvalidation/portfolio/portfolio.py +309 -0
- riskvalidation/provisions/__init__.py +8 -0
- riskvalidation/provisions/_checks.py +100 -0
- riskvalidation/provisions/ecl.py +233 -0
- riskvalidation/provisions/local_cl.py +360 -0
- riskvalidation/provisions/local_es.py +157 -0
- riskvalidation/provisions/overlays.py +102 -0
- riskvalidation/provisions/reconciliation.py +157 -0
- riskvalidation/provisions/staging.py +211 -0
- riskvalidation/provisions/term_structure.py +199 -0
- riskvalidation/regulatory/__init__.py +14 -0
- riskvalidation/regulatory/ba_cva.py +138 -0
- riskvalidation/regulatory/frtb.py +345 -0
- riskvalidation/regulatory/irb.py +314 -0
- riskvalidation/regulatory/lcr_nsfr.py +369 -0
- riskvalidation/regulatory/output_floor.py +45 -0
- riskvalidation/regulatory/sa_ccr.py +486 -0
- riskvalidation/regulatory/standardised.py +348 -0
- riskvalidation/stress/__init__.py +9 -0
- riskvalidation/stress/capital.py +476 -0
- riskvalidation/stress/paths.py +108 -0
- riskvalidation/stress/reverse.py +179 -0
- riskvalidation/stress/satellites.py +268 -0
- riskvalidation/stress/scenario_models.py +274 -0
- riskvalidation/transitions/__init__.py +0 -0
- riskvalidation/transitions/em.py +120 -0
- riskvalidation/transitions/embedding.py +221 -0
- riskvalidation/transitions/estimators.py +213 -0
- riskvalidation/transitions/intervals.py +306 -0
- riskvalidation/transitions/mobility.py +57 -0
- riskvalidation/transitions/ttc.py +171 -0
- riskvalidation/validation/__init__.py +103 -0
- riskvalidation/validation/_common.py +57 -0
- riskvalidation/validation/aggregation.py +425 -0
- riskvalidation/validation/calibration.py +757 -0
- riskvalidation/validation/ccr.py +530 -0
- riskvalidation/validation/discrimination.py +314 -0
- riskvalidation/validation/fairness.py +632 -0
- riskvalidation/validation/ifrs9.py +693 -0
- riskvalidation/validation/irrbb.py +389 -0
- riskvalidation/validation/lgd_ead.py +469 -0
- riskvalidation/validation/liquidity.py +647 -0
- riskvalidation/validation/market.py +807 -0
- riskvalidation/validation/ml.py +958 -0
- riskvalidation/validation/oprisk.py +357 -0
- riskvalidation/validation/portfolio.py +353 -0
- riskvalidation/validation/rating_system.py +245 -0
- riskvalidation/validation/stability.py +256 -0
- riskvalidation/validation/stress.py +552 -0
- riskvalidation/validation/transitions.py +386 -0
- riskvalidation-0.8.0.dist-info/METADATA +136 -0
- riskvalidation-0.8.0.dist-info/RECORD +164 -0
- riskvalidation-0.8.0.dist-info/WHEEL +5 -0
- riskvalidation-0.8.0.dist-info/licenses/LICENSE +21 -0
- riskvalidation-0.8.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
"""Situation testing with k-nearest-neighbour comparators, ceteris paribus against mutatis mutandis (dossier 07 A.3
|
|
2
|
+
l.158-177), transcribed from the companion paper, Alvarez and Ruggieri, "Counterfactual Situation Testing: From Single
|
|
3
|
+
to Multidimensional Discrimination", arXiv:2502.01267v2 = JAIR 82 (doi:10.1613/jair.1.17935), Section 4 and equations
|
|
4
|
+
3 to 10, and the main paper, arXiv:2405.13693v4 (doi:10.1111/coin.70238), for the comparators (contract
|
|
5
|
+
`engine-fairness-api` 1.0.0, `categorical` keyword of `situation_test` 1.1.0; research addendum 7.2 to 7.6).
|
|
6
|
+
|
|
7
|
+
Setting: the group indicator is 1 for the protected group (the control search space `D_c`) and 0 for the others (the
|
|
8
|
+
test search space `D_t`); the decision 0 is the negative one. For every complainant `c` of the protected group the
|
|
9
|
+
control group is the k nearest protected rows to `x_c` (the complainant excluded) and the test group is the k nearest
|
|
10
|
+
rows of the other group to `x_c` (ceteris paribus) or to the counterfactual `x_c^CF` (mutatis mutandis); neither the
|
|
11
|
+
group nor the decision enters the distance. The statistic is delta p = p_c - p_t, the share of negative decisions in
|
|
12
|
+
the control group minus that in the test group, with the denominator k (the "CST without centers" version). The claim
|
|
13
|
+
is statistically significant at level alpha when the lower bound of the one-sided Wald interval exceeds the accepted
|
|
14
|
+
deviation tau.
|
|
15
|
+
|
|
16
|
+
The distance normalization is a named choice. `range` is the paper's text (equation 5: the continuous term is divided by
|
|
17
|
+
max - min over the search space) and is the default. `standard` is the reference code's default (a scikit-learn
|
|
18
|
+
StandardScaler, the absolute z-score difference, not bounded by 1). The published counts of the loan-application
|
|
19
|
+
experiment were produced under `standard`, so they are claimed only there. Nothing here reads the reference code's
|
|
20
|
+
report columns.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import math
|
|
26
|
+
from dataclasses import dataclass
|
|
27
|
+
|
|
28
|
+
import numpy as np
|
|
29
|
+
from scipy import stats
|
|
30
|
+
|
|
31
|
+
from riskvalidation.fairness._checks import binary, finite_matrix, same_rows, unit_interval_open
|
|
32
|
+
|
|
33
|
+
NORMALIZATIONS = ("range", "standard")
|
|
34
|
+
COMPARATORS = ("ceteris-paribus", "mutatis-mutandis")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class SituationResult:
|
|
39
|
+
"""The counts of one situation test over the complainants of the protected group."""
|
|
40
|
+
|
|
41
|
+
n_complainants: int
|
|
42
|
+
n_cases: int
|
|
43
|
+
n_significant: int
|
|
44
|
+
k: int
|
|
45
|
+
tau: float
|
|
46
|
+
alpha: float
|
|
47
|
+
|
|
48
|
+
def __post_init__(self) -> None:
|
|
49
|
+
if not 0 <= self.n_significant <= self.n_cases <= self.n_complainants:
|
|
50
|
+
raise ValueError("n_significant must not exceed n_cases, and n_cases must not exceed n_complainants")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _normalization(normalization: str) -> str:
|
|
54
|
+
if normalization not in NORMALIZATIONS:
|
|
55
|
+
raise ValueError(f"normalization must be 'range' or 'standard', got {normalization!r}")
|
|
56
|
+
return normalization
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _attributes(x: object, name: str, p: int | None = None) -> np.ndarray:
|
|
60
|
+
a = np.array(x, dtype=float)
|
|
61
|
+
if a.ndim != 1 or a.size == 0:
|
|
62
|
+
raise ValueError(f"{name} must be a non-empty one-dimensional array of attributes")
|
|
63
|
+
if not np.all(np.isfinite(a)):
|
|
64
|
+
raise ValueError(f"{name} contains non-finite values")
|
|
65
|
+
if p is not None and a.shape[0] != p:
|
|
66
|
+
raise ValueError(f"{name} has {a.shape[0]} attributes but {p} were expected")
|
|
67
|
+
return a
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _scale_and_mask(categorical: object, scale: object, p: int) -> tuple[np.ndarray, np.ndarray]:
|
|
71
|
+
cat = np.asarray(categorical)
|
|
72
|
+
if cat.shape != (p,):
|
|
73
|
+
raise ValueError(f"categorical must have one flag per attribute ({p}), got shape {cat.shape}")
|
|
74
|
+
cat = cat.astype(bool)
|
|
75
|
+
sc = np.array(scale, dtype=float)
|
|
76
|
+
if sc.shape != (p,):
|
|
77
|
+
raise ValueError(f"scale must have one value per attribute ({p}), got shape {sc.shape}")
|
|
78
|
+
for j in np.flatnonzero(~cat):
|
|
79
|
+
if not (math.isfinite(sc[j]) and sc[j] > 0.0):
|
|
80
|
+
raise ValueError(f"scale of continuous attribute {j} must be positive and finite, got {sc[j]!r}")
|
|
81
|
+
return cat, np.where(cat, 1.0, sc)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _distances(center: np.ndarray, pool: np.ndarray, cat: np.ndarray, scale: np.ndarray) -> np.ndarray:
|
|
85
|
+
"""Gower distance (equations 3 to 5) of `center` to every row of `pool`: the mean over the attributes of the
|
|
86
|
+
overlap distance (categorical) or |difference| / scale (continuous)."""
|
|
87
|
+
per = np.where(cat, (pool != center).astype(float), np.abs(pool - center) / scale)
|
|
88
|
+
return per.sum(axis=1) / pool.shape[1]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def gower_distance(
|
|
92
|
+
x1: np.ndarray, x2: np.ndarray, categorical: np.ndarray, scale: np.ndarray, normalization: str = "range"
|
|
93
|
+
) -> float:
|
|
94
|
+
"""Mean over the non-protected attributes of the overlap distance (categorical) or the Manhattan distance over
|
|
95
|
+
the attribute scale (continuous). Under `range`, `scale` holds max - min of each continuous attribute over the
|
|
96
|
+
search space, and the distance lies in [0, 1] for attributes inside that range. Under `standard`, `scale` holds the
|
|
97
|
+
population standard deviation (ddof 0) of the attribute: the continuous term is the absolute z-score difference and
|
|
98
|
+
is not bounded by 1. The arithmetic is the same; the normalization states what `scale` holds and any other value
|
|
99
|
+
raises `ValueError`. Protected attributes never enter (the caller leaves them out); a non-positive scale of a
|
|
100
|
+
continuous attribute raises `ValueError` (the scale of a categorical attribute is not read)."""
|
|
101
|
+
_normalization(normalization)
|
|
102
|
+
a = _attributes(x1, "x1")
|
|
103
|
+
b = _attributes(x2, "x2", a.shape[0])
|
|
104
|
+
cat, sc = _scale_and_mask(categorical, scale, a.shape[0])
|
|
105
|
+
return float(_distances(a, b[None, :], cat, sc)[0])
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _select(d: np.ndarray, k: int) -> np.ndarray:
|
|
109
|
+
"""Indices of the k smallest distances in ascending order, ties broken by row order."""
|
|
110
|
+
kth = np.partition(d, k - 1)[k - 1]
|
|
111
|
+
below = np.flatnonzero(d < kth)
|
|
112
|
+
ties = np.flatnonzero(d == kth)[: k - below.size]
|
|
113
|
+
chosen = np.concatenate([below, ties])
|
|
114
|
+
return chosen[np.lexsort((chosen, d[chosen]))]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def k_nearest(
|
|
118
|
+
center: np.ndarray,
|
|
119
|
+
pool: np.ndarray,
|
|
120
|
+
k: int,
|
|
121
|
+
categorical: np.ndarray,
|
|
122
|
+
scale: np.ndarray,
|
|
123
|
+
normalization: str = "range",
|
|
124
|
+
) -> np.ndarray:
|
|
125
|
+
"""Indices of the k nearest rows of `pool` to `center` by ascending distance (the normalization of
|
|
126
|
+
`gower_distance`), ties broken by row order; neither the protected attribute nor the decision is used (equations 6
|
|
127
|
+
and 7). The center itself is not removed: pass a pool without it."""
|
|
128
|
+
_normalization(normalization)
|
|
129
|
+
c = _attributes(center, "center")
|
|
130
|
+
m = finite_matrix(pool, "pool", rows_of="pool")
|
|
131
|
+
if m.shape[1] != c.shape[0]:
|
|
132
|
+
raise ValueError(f"center has {c.shape[0]} attributes but pool has {m.shape[1]} columns")
|
|
133
|
+
if not isinstance(k, (int, np.integer)) or not 1 <= k <= m.shape[0]:
|
|
134
|
+
raise ValueError(f"k must be an integer in [1, {m.shape[0]}] (the rows of pool), got {k!r}")
|
|
135
|
+
cat, sc = _scale_and_mask(categorical, scale, c.shape[0])
|
|
136
|
+
return _select(_distances(c, m, cat, sc), int(k)).astype(np.int64)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def situation_statistic(
|
|
140
|
+
decision_control: np.ndarray, decision_test: np.ndarray, negative: int = 0
|
|
141
|
+
) -> tuple[float, float, float]:
|
|
142
|
+
"""(delta_p, p_c, p_t): the share of `negative` decisions in the control group and in the test group (the
|
|
143
|
+
denominator is the group size k, the centers excluded) and their difference p_c - p_t, in [-1, 1]
|
|
144
|
+
(equations 8 and 9)."""
|
|
145
|
+
if negative not in (0, 1):
|
|
146
|
+
raise ValueError(f"negative must be 0 or 1, got {negative!r}")
|
|
147
|
+
dc = binary(decision_control, "decision_control")
|
|
148
|
+
dt = binary(decision_test, "decision_test")
|
|
149
|
+
if dc.shape != dt.shape:
|
|
150
|
+
raise ValueError(
|
|
151
|
+
f"decision_test has {dt.shape[0]} rows but decision_control has {dc.shape[0]}: both groups have k rows"
|
|
152
|
+
)
|
|
153
|
+
p_c = float(np.mean(dc == negative))
|
|
154
|
+
p_t = float(np.mean(dt == negative))
|
|
155
|
+
return p_c - p_t, p_c, p_t
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _z(alpha: float) -> float:
|
|
159
|
+
return float(stats.norm.ppf(1.0 - alpha))
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _wald_lower(delta: float, p_c: float, p_t: float, k: int, z: float) -> float:
|
|
163
|
+
return delta - z * math.sqrt((p_c * (1.0 - p_c) + p_t * (1.0 - p_t)) / k)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def wald_interval_one_sided(delta: float, p_c: float, p_t: float, k: int, alpha: float = 0.05) -> float:
|
|
167
|
+
"""The lower bound delta - z_alpha sqrt((p_c (1 - p_c) + p_t (1 - p_t)) / k) of the one-sided interval
|
|
168
|
+
[lower, +inf) (equation 10), with z_alpha = Phi^-1(1 - alpha) the exact normal quantile and the plus sign of the
|
|
169
|
+
numerator (footnote 7: the EAAMO paper's minus sign is a typo). The claim is significant when the lower bound
|
|
170
|
+
exceeds tau."""
|
|
171
|
+
for name, v in (("delta", delta), ("p_c", p_c), ("p_t", p_t)):
|
|
172
|
+
if not math.isfinite(v):
|
|
173
|
+
raise ValueError(f"{name} must be finite, got {v!r}")
|
|
174
|
+
for name, v in (("p_c", p_c), ("p_t", p_t)):
|
|
175
|
+
if not 0.0 <= v <= 1.0:
|
|
176
|
+
raise ValueError(f"{name} must lie in [0, 1], got {v!r}")
|
|
177
|
+
if not isinstance(k, (int, np.integer)) or k < 1:
|
|
178
|
+
raise ValueError(f"k must be a positive integer, got {k!r}")
|
|
179
|
+
a = unit_interval_open(alpha, "alpha")
|
|
180
|
+
return float(_wald_lower(delta, p_c, p_t, int(k), _z(a)))
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def bonferroni_alpha(alpha: float, q: int) -> float:
|
|
184
|
+
"""The level alpha / q for q protected attributes (multiple discrimination, Definition 4.6)."""
|
|
185
|
+
a = unit_interval_open(alpha, "alpha")
|
|
186
|
+
if not isinstance(q, (int, np.integer)) or q < 1:
|
|
187
|
+
raise ValueError(f"q must be an integer of at least 1, got {q!r}")
|
|
188
|
+
return a / int(q)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _scales(x: np.ndarray, protected: np.ndarray, others: np.ndarray, normalization: str, cat: np.ndarray):
|
|
192
|
+
"""The scale of each continuous attribute; a categorical attribute's scale is not read (overlap distance), so it
|
|
193
|
+
is set to 1 and never raises for being constant."""
|
|
194
|
+
if normalization == "range":
|
|
195
|
+
ctrl = np.ptp(x[protected], axis=0)
|
|
196
|
+
test = np.ptp(x[others], axis=0)
|
|
197
|
+
else:
|
|
198
|
+
ctrl = test = x.std(axis=0)
|
|
199
|
+
ctrl, test = np.where(cat, 1.0, ctrl), np.where(cat, 1.0, test)
|
|
200
|
+
for name, sc in (("control", ctrl), ("test", test)):
|
|
201
|
+
bad = np.flatnonzero(~(sc > 0.0))
|
|
202
|
+
if bad.size:
|
|
203
|
+
raise ValueError(
|
|
204
|
+
f"attribute {int(bad[0])} has a zero scale over the {name} search space (a constant column): "
|
|
205
|
+
"remove it or choose another normalization"
|
|
206
|
+
)
|
|
207
|
+
return ctrl, test
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def situation_test(
|
|
211
|
+
X: np.ndarray,
|
|
212
|
+
group: np.ndarray,
|
|
213
|
+
decision: np.ndarray,
|
|
214
|
+
k: int,
|
|
215
|
+
tau: float = 0.0,
|
|
216
|
+
alpha: float = 0.05,
|
|
217
|
+
comparator: str = "ceteris-paribus",
|
|
218
|
+
counterfactual: np.ndarray | None = None,
|
|
219
|
+
normalization: str = "range",
|
|
220
|
+
categorical: np.ndarray | None = None,
|
|
221
|
+
) -> SituationResult:
|
|
222
|
+
"""Runs the test for every complainant of the protected group (group 1), regardless of the complainant's own
|
|
223
|
+
decision. `X` holds the non-protected attributes, `group` is 1 for the protected group and 0 for the others,
|
|
224
|
+
`decision` is 1 favourable and 0 negative. `categorical` (1.1.0) is a boolean mask over the columns of `X`: a
|
|
225
|
+
categorical column contributes the overlap measure (1 if the values differ, 0 otherwise), every other column the
|
|
226
|
+
normalised Manhattan distance, as in equations 3 to 5 of the paper, and the distance is their mean over the columns;
|
|
227
|
+
under `normalization="standard"` the categorical columns still use overlap and the standardisation applies to the
|
|
228
|
+
other columns only. `None` (the 1.0.0 behaviour) treats every column as continuous.
|
|
229
|
+
|
|
230
|
+
comparator 'ceteris-paribus': the test group is built around the complainant. 'mutatis-mutandis': around the row of
|
|
231
|
+
`counterfactual` aligned with the complainant (the algorithm supplies none; it must have the shape of `X`). Under
|
|
232
|
+
`range` the scale of each attribute is max - min over the search space the group is drawn from (control: the
|
|
233
|
+
protected rows; test: the other rows); under `standard` it is the population standard deviation over all of `X`,
|
|
234
|
+
shared by the counterfactual (the paper's one space), not the reference code's separate scaler for it.
|
|
235
|
+
|
|
236
|
+
n_cases counts the complainants with delta p > tau; n_significant those whose lower bound of equation 10 exceeds
|
|
237
|
+
tau. Raises `ValueError` naming the field for unknown comparators and normalizations, 'mutatis-mutandis' without a
|
|
238
|
+
counterfactual, malformed inputs (a `categorical` mask that is not one flag per column of `X` included), k beyond
|
|
239
|
+
the search spaces and a zero scale of a continuous column."""
|
|
240
|
+
if comparator not in COMPARATORS:
|
|
241
|
+
raise ValueError(f"comparator must be one of {COMPARATORS}, got {comparator!r}")
|
|
242
|
+
_normalization(normalization)
|
|
243
|
+
x = finite_matrix(X, "X", rows_of="the decision") # rows are checked against group below
|
|
244
|
+
g = binary(group, "group")
|
|
245
|
+
d = binary(decision, "decision")
|
|
246
|
+
same_rows(g, "group", x, "X")
|
|
247
|
+
same_rows(g, "group", d, "decision")
|
|
248
|
+
if g.min() == g.max():
|
|
249
|
+
raise ValueError("group must contain both the protected group (1) and the others (0)")
|
|
250
|
+
if not isinstance(k, (int, np.integer)) or k < 1:
|
|
251
|
+
raise ValueError(f"k must be a positive integer, got {k!r}")
|
|
252
|
+
k = int(k)
|
|
253
|
+
a = unit_interval_open(alpha, "alpha")
|
|
254
|
+
if not (math.isfinite(tau) and -1.0 <= tau <= 1.0):
|
|
255
|
+
raise ValueError(f"tau must lie in [-1, 1], got {tau!r}")
|
|
256
|
+
if comparator == "mutatis-mutandis":
|
|
257
|
+
if counterfactual is None:
|
|
258
|
+
raise ValueError("counterfactual is required for the 'mutatis-mutandis' comparator")
|
|
259
|
+
cf = np.array(counterfactual, dtype=float)
|
|
260
|
+
if cf.shape != x.shape or not np.all(np.isfinite(cf)):
|
|
261
|
+
raise ValueError(
|
|
262
|
+
f"counterfactual must be a finite array of the shape of X {x.shape}, aligned row by row, got {cf.shape}"
|
|
263
|
+
)
|
|
264
|
+
else:
|
|
265
|
+
cf = None
|
|
266
|
+
|
|
267
|
+
prot_idx, other_idx = np.flatnonzero(g == 1), np.flatnonzero(g == 0)
|
|
268
|
+
if k > prot_idx.size - 1:
|
|
269
|
+
raise ValueError(f"k = {k} exceeds the {prot_idx.size - 1} other complainants of the control search space")
|
|
270
|
+
if k > other_idx.size:
|
|
271
|
+
raise ValueError(f"k = {k} exceeds the {other_idx.size} rows of the test search space")
|
|
272
|
+
if categorical is None:
|
|
273
|
+
cat = np.zeros(x.shape[1], dtype=bool)
|
|
274
|
+
else:
|
|
275
|
+
cat = np.asarray(categorical)
|
|
276
|
+
if cat.shape != (x.shape[1],):
|
|
277
|
+
raise ValueError(f"categorical must have one flag per column of X ({x.shape[1]}), got shape {cat.shape}")
|
|
278
|
+
cat = cat.astype(bool)
|
|
279
|
+
ctrl_scale, test_scale = _scales(x, prot_idx, other_idx, normalization, cat)
|
|
280
|
+
xp, xo = x[prot_idx], x[other_idx]
|
|
281
|
+
neg_p, neg_o = d[prot_idx] == 0, d[other_idx] == 0
|
|
282
|
+
z = _z(a)
|
|
283
|
+
cases = significant = 0
|
|
284
|
+
for i in range(prot_idx.size):
|
|
285
|
+
d_ctrl = _distances(xp[i], xp, cat, ctrl_scale)
|
|
286
|
+
d_ctrl[i] = np.inf # the complainant is the search center: excluded from its own control group
|
|
287
|
+
centre = xp[i] if cf is None else cf[prot_idx[i]]
|
|
288
|
+
d_test = _distances(centre, xo, cat, test_scale)
|
|
289
|
+
p_c = float(neg_p[_select(d_ctrl, k)].sum()) / k
|
|
290
|
+
p_t = float(neg_o[_select(d_test, k)].sum()) / k
|
|
291
|
+
delta = p_c - p_t
|
|
292
|
+
if delta > tau:
|
|
293
|
+
cases += 1
|
|
294
|
+
if _wald_lower(delta, p_c, p_t, k, z) > tau:
|
|
295
|
+
significant += 1
|
|
296
|
+
return SituationResult(int(prot_idx.size), cases, significant, k, float(tau), float(a))
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
__all__ = [
|
|
300
|
+
"SituationResult",
|
|
301
|
+
"bonferroni_alpha",
|
|
302
|
+
"gower_distance",
|
|
303
|
+
"k_nearest",
|
|
304
|
+
"situation_statistic",
|
|
305
|
+
"situation_test",
|
|
306
|
+
"wald_interval_one_sided",
|
|
307
|
+
]
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Known-truth generators: data-generating mechanisms whose parameters are known, so the size and power of a test
|
|
2
|
+
and the error of an estimator can be measured (see :mod:`riskvalidation.harness`).
|
|
3
|
+
|
|
4
|
+
Every generator is a frozen configuration with ``config()`` (the configuration written next to the data),
|
|
5
|
+
``draw(rng)`` (one dataset, keyed by the names of the test arguments it feeds), ``truth()`` (exact) and ``verify()``
|
|
6
|
+
(a large draw on the generator's own stream reproduces the truth within Monte Carlo error).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from riskvalidation.generators._base import Check, GeneratorBase, bvn_equal, jsonable
|
|
10
|
+
from riskvalidation.generators.counts import (
|
|
11
|
+
WP14,
|
|
12
|
+
DefaultCounts,
|
|
13
|
+
default_correlation,
|
|
14
|
+
feed_cell,
|
|
15
|
+
feed_grades,
|
|
16
|
+
feed_portfolio,
|
|
17
|
+
feed_series,
|
|
18
|
+
joint_default_probability,
|
|
19
|
+
)
|
|
20
|
+
from riskvalidation.generators.drift import YURDAKUL_NARANJO, TwoSample, population_psi
|
|
21
|
+
from riskvalidation.generators.framingham import DEMLER, NestedLda, lda_auc
|
|
22
|
+
from riskvalidation.generators.ratings import (
|
|
23
|
+
GradeFrequencies,
|
|
24
|
+
Migrations,
|
|
25
|
+
break_monotonicity,
|
|
26
|
+
concentrate,
|
|
27
|
+
decaying_matrix,
|
|
28
|
+
monotone_cells,
|
|
29
|
+
)
|
|
30
|
+
from riskvalidation.generators.scores import (
|
|
31
|
+
HANLEY_MCNEIL,
|
|
32
|
+
BinormalScores,
|
|
33
|
+
LogisticPortfolio,
|
|
34
|
+
binormal_auc,
|
|
35
|
+
binormal_auc_variance,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
"DEMLER",
|
|
40
|
+
"HANLEY_MCNEIL",
|
|
41
|
+
"WP14",
|
|
42
|
+
"YURDAKUL_NARANJO",
|
|
43
|
+
"BinormalScores",
|
|
44
|
+
"Check",
|
|
45
|
+
"DefaultCounts",
|
|
46
|
+
"GeneratorBase",
|
|
47
|
+
"GradeFrequencies",
|
|
48
|
+
"LogisticPortfolio",
|
|
49
|
+
"Migrations",
|
|
50
|
+
"NestedLda",
|
|
51
|
+
"TwoSample",
|
|
52
|
+
"binormal_auc",
|
|
53
|
+
"binormal_auc_variance",
|
|
54
|
+
"break_monotonicity",
|
|
55
|
+
"bvn_equal",
|
|
56
|
+
"concentrate",
|
|
57
|
+
"decaying_matrix",
|
|
58
|
+
"default_correlation",
|
|
59
|
+
"feed_cell",
|
|
60
|
+
"feed_grades",
|
|
61
|
+
"feed_portfolio",
|
|
62
|
+
"feed_series",
|
|
63
|
+
"joint_default_probability",
|
|
64
|
+
"jsonable",
|
|
65
|
+
"lda_auc",
|
|
66
|
+
"monotone_cells",
|
|
67
|
+
"population_psi",
|
|
68
|
+
]
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""The generator contract: a frozen configuration, one dataset per draw, an exact truth, a verifier.
|
|
2
|
+
|
|
3
|
+
A generator is the data-generating mechanism of a simulation study (Morris, White and Crowther 2019,
|
|
4
|
+
doi:10.1002/sim.8086): its parameters are known, so what a test or an estimator should find is known too. Each
|
|
5
|
+
generator here serialises its configuration (the configuration written next to the data), draws one dataset from a
|
|
6
|
+
NumPy ``Generator`` (so one seed gives one dataset), returns its truth computed exactly, and verifies itself: a large
|
|
7
|
+
sample drawn on its own stream reproduces the truth within Monte Carlo error.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import math
|
|
13
|
+
from dataclasses import dataclass, fields
|
|
14
|
+
from typing import Any, ClassVar
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
# Gauss-Hermite rule for expectations over a standard normal factor: E[f(Z)] = sum w_i f(x_i)
|
|
19
|
+
GH_NODES, GH_WEIGHTS = np.polynomial.hermite_e.hermegauss(256)
|
|
20
|
+
GH_WEIGHTS = GH_WEIGHTS / math.sqrt(2.0 * math.pi)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def jsonable(value: Any) -> Any:
|
|
24
|
+
"""NumPy arrays, tuples and scalars as plain JSON values (non-finite floats become ``None``)."""
|
|
25
|
+
if isinstance(value, np.ndarray):
|
|
26
|
+
return jsonable(value.tolist())
|
|
27
|
+
if isinstance(value, (list, tuple)):
|
|
28
|
+
return [jsonable(v) for v in value]
|
|
29
|
+
if isinstance(value, dict):
|
|
30
|
+
return {str(k): jsonable(v) for k, v in value.items()}
|
|
31
|
+
if isinstance(value, (bool, np.bool_)):
|
|
32
|
+
return bool(value)
|
|
33
|
+
if isinstance(value, (int, np.integer)):
|
|
34
|
+
return int(value)
|
|
35
|
+
if isinstance(value, (float, np.floating)):
|
|
36
|
+
v = float(value)
|
|
37
|
+
return v if math.isfinite(v) else None
|
|
38
|
+
return value
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def frozen_array(value: Any, *, dtype: type = float, ndim: int | None = None) -> tuple:
|
|
42
|
+
"""A nested tuple (hashable, so the dataclass stays frozen) from an array-like."""
|
|
43
|
+
arr = np.asarray(value, dtype=dtype)
|
|
44
|
+
if ndim is not None and arr.ndim != ndim:
|
|
45
|
+
raise ValueError(f"expected {ndim} dimensions, got shape {arr.shape}")
|
|
46
|
+
return tuple(_to_tuple(arr.tolist()))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _to_tuple(x: Any) -> Any:
|
|
50
|
+
return tuple(_to_tuple(v) for v in x) if isinstance(x, list) else x
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class Check:
|
|
55
|
+
"""One verification: the truth, what a large draw observed, and the Monte Carlo standard error."""
|
|
56
|
+
|
|
57
|
+
name: str
|
|
58
|
+
expected: float
|
|
59
|
+
observed: float
|
|
60
|
+
se: float
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def z(self) -> float:
|
|
64
|
+
if self.se > 0:
|
|
65
|
+
return (self.observed - self.expected) / self.se
|
|
66
|
+
return 0.0 if math.isclose(self.observed, self.expected, rel_tol=1e-12, abs_tol=1e-15) else math.inf
|
|
67
|
+
|
|
68
|
+
def ok(self, z_max: float = 4.0) -> bool:
|
|
69
|
+
return abs(self.z) <= z_max
|
|
70
|
+
|
|
71
|
+
def to_dict(self) -> dict[str, Any]:
|
|
72
|
+
return jsonable({"name": self.name, "expected": self.expected, "observed": self.observed, "se": self.se,
|
|
73
|
+
"z": self.z, "ok": self.ok()})
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class GeneratorBase:
|
|
77
|
+
"""Shared behaviour of the generators: the serialised configuration and the verification report."""
|
|
78
|
+
|
|
79
|
+
name: ClassVar[str] = "generator"
|
|
80
|
+
|
|
81
|
+
def config(self) -> dict[str, Any]:
|
|
82
|
+
"""The generator's name, the package version and every parameter, JSON-safe."""
|
|
83
|
+
from riskvalidation import __version__
|
|
84
|
+
|
|
85
|
+
params = {f.name: jsonable(getattr(self, f.name)) for f in fields(self)} # type: ignore[arg-type]
|
|
86
|
+
return {"generator": self.name, "riskvalidation": __version__, "parameters": params}
|
|
87
|
+
|
|
88
|
+
def draw(self, rng: np.random.Generator) -> dict[str, Any]: # pragma: no cover - interface
|
|
89
|
+
raise NotImplementedError
|
|
90
|
+
|
|
91
|
+
def truth(self) -> dict[str, Any]: # pragma: no cover - interface
|
|
92
|
+
raise NotImplementedError
|
|
93
|
+
|
|
94
|
+
def verify(self, n_draws: int = 20_000, seed: int = 20261005) -> list[Check]: # pragma: no cover - interface
|
|
95
|
+
raise NotImplementedError
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def bvn_equal_excess(c: float, rho: float) -> float:
|
|
99
|
+
r"""$\Phi_2(c, c; \rho) - \Phi(c)^2 = \int_0^\rho e^{-c^2/(1+r)}/(2\pi\sqrt{1-r^2})\,dr$, the bivariate standard
|
|
100
|
+
normal distribution function at (c, c) above its value under independence (Plackett's identity: the derivative of
|
|
101
|
+
the distribution function in the correlation is the density)."""
|
|
102
|
+
from scipy import integrate
|
|
103
|
+
|
|
104
|
+
if not -1.0 < rho < 1.0:
|
|
105
|
+
raise ValueError("correlation must lie in (-1, 1)")
|
|
106
|
+
if rho == 0.0:
|
|
107
|
+
return 0.0
|
|
108
|
+
val, _ = integrate.quad(lambda r: math.exp(-c * c / (1.0 + r)) / (2.0 * math.pi * math.sqrt(1.0 - r * r)),
|
|
109
|
+
0.0, rho, epsabs=1e-16, epsrel=1e-12, limit=200)
|
|
110
|
+
return float(val)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def bvn_equal(c: float, rho: float) -> float:
|
|
114
|
+
r"""The bivariate standard normal distribution function $\Phi_2(c, c; \rho)$."""
|
|
115
|
+
from scipy import stats
|
|
116
|
+
|
|
117
|
+
return float(stats.norm.cdf(c)) ** 2 + bvn_equal_excess(c, rho)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def mean_check(name: str, expected: float, samples: np.ndarray) -> Check:
|
|
121
|
+
"""The sample mean of ``samples`` against ``expected``, with the standard error of a mean."""
|
|
122
|
+
s = np.asarray(samples, dtype=float)
|
|
123
|
+
return Check(name, float(expected), float(s.mean()), float(s.std(ddof=1) / math.sqrt(s.size)))
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def variance_check(name: str, expected: float, samples: np.ndarray) -> Check:
|
|
127
|
+
"""The sample variance against ``expected``; its standard error from the fourth central moment."""
|
|
128
|
+
s = np.asarray(samples, dtype=float)
|
|
129
|
+
r = s.size
|
|
130
|
+
c = s - s.mean()
|
|
131
|
+
var = float(np.sum(c**2) / (r - 1))
|
|
132
|
+
m4 = float(np.mean(c**4))
|
|
133
|
+
se = math.sqrt(max(m4 - var**2 * (r - 3) / (r - 1), 0.0) / r)
|
|
134
|
+
return Check(name, float(expected), var, se)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def proportion_check(name: str, expected: float, hits: np.ndarray) -> Check:
|
|
138
|
+
"""A share of hits against ``expected``, with the binomial standard error at the expected value."""
|
|
139
|
+
h = np.asarray(hits, dtype=bool)
|
|
140
|
+
p = float(expected)
|
|
141
|
+
return Check(name, p, float(h.mean()), math.sqrt(max(p * (1.0 - p), 1e-300) / h.size))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
__all__ = ["Check", "GeneratorBase", "GH_NODES", "GH_WEIGHTS", "bvn_equal", "bvn_equal_excess", "frozen_array",
|
|
145
|
+
"jsonable", "mean_check", "proportion_check", "variance_check"]
|