riskvalidation 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- riskvalidation/__init__.py +30 -0
- riskvalidation/aggregation/__init__.py +11 -0
- riskvalidation/aggregation/band.py +151 -0
- riskvalidation/aggregation/bounds.py +191 -0
- riskvalidation/aggregation/copulas.py +241 -0
- riskvalidation/aggregation/model_risk.py +133 -0
- riskvalidation/aggregation/simple.py +147 -0
- riskvalidation/ccr/__init__.py +13 -0
- riskvalidation/ccr/collateral.py +175 -0
- riskvalidation/ccr/cva.py +193 -0
- riskvalidation/ccr/exposure.py +192 -0
- riskvalidation/ccr/hw1f.py +426 -0
- riskvalidation/ccr/swaps.py +172 -0
- riskvalidation/core/__init__.py +1 -0
- riskvalidation/core/build.py +57 -0
- riskvalidation/core/hashing.py +48 -0
- riskvalidation/core/policy.py +130 -0
- riskvalidation/core/registry.py +65 -0
- riskvalidation/core/result.py +117 -0
- riskvalidation/core/suite.py +78 -0
- riskvalidation/engines/__init__.py +2 -0
- riskvalidation/engines/low_default.py +170 -0
- riskvalidation/engines/pd_curve.py +529 -0
- riskvalidation/engines/vasicek.py +72 -0
- riskvalidation/fairness/__init__.py +10 -0
- riskvalidation/fairness/_checks.py +115 -0
- riskvalidation/fairness/conditional.py +201 -0
- riskvalidation/fairness/inference.py +64 -0
- riskvalidation/fairness/metrics.py +261 -0
- riskvalidation/fairness/plants.py +240 -0
- riskvalidation/fairness/policy.py +97 -0
- riskvalidation/fairness/replication.py +39 -0
- riskvalidation/fairness/situation.py +307 -0
- riskvalidation/generators/__init__.py +68 -0
- riskvalidation/generators/_base.py +145 -0
- riskvalidation/generators/counts.py +232 -0
- riskvalidation/generators/drift.py +107 -0
- riskvalidation/generators/framingham.py +124 -0
- riskvalidation/generators/paths.py +349 -0
- riskvalidation/generators/ratings.py +186 -0
- riskvalidation/generators/scores.py +347 -0
- riskvalidation/harness/__init__.py +46 -0
- riskvalidation/harness/exact.py +137 -0
- riskvalidation/harness/performance.py +74 -0
- riskvalidation/harness/rates.py +103 -0
- riskvalidation/harness/run.py +197 -0
- riskvalidation/irrbb/__init__.py +9 -0
- riskvalidation/irrbb/behavioural.py +206 -0
- riskvalidation/irrbb/curves.py +286 -0
- riskvalidation/irrbb/eve_nii.py +245 -0
- riskvalidation/irrbb/shocks.py +171 -0
- riskvalidation/irrbb/simulation.py +203 -0
- riskvalidation/irrbb/slotting.py +184 -0
- riskvalidation/lgd_ead/__init__.py +8 -0
- riskvalidation/lgd_ead/_checks.py +50 -0
- riskvalidation/lgd_ead/ccf.py +228 -0
- riskvalidation/lgd_ead/downturn.py +64 -0
- riskvalidation/lgd_ead/floors.py +114 -0
- riskvalidation/lgd_ead/lgd_models.py +297 -0
- riskvalidation/lgd_ead/workout.py +162 -0
- riskvalidation/liquidity/__init__.py +9 -0
- riskvalidation/liquidity/_checks.py +78 -0
- riskvalidation/liquidity/cbc.py +158 -0
- riskvalidation/liquidity/deposits.py +151 -0
- riskvalidation/liquidity/ladder.py +119 -0
- riskvalidation/liquidity/reverse.py +141 -0
- riskvalidation/liquidity/runoff.py +220 -0
- riskvalidation/liquidity/scenarios.py +272 -0
- riskvalidation/market/__init__.py +8 -0
- riskvalidation/market/caviar.py +239 -0
- riskvalidation/market/conventions.py +83 -0
- riskvalidation/market/evt.py +111 -0
- riskvalidation/market/frtb.py +175 -0
- riskvalidation/market/garch.py +204 -0
- riskvalidation/market/gas_fz.py +219 -0
- riskvalidation/market/historical.py +96 -0
- riskvalidation/market/parametric.py +78 -0
- riskvalidation/market/scoring.py +145 -0
- riskvalidation/ml/__init__.py +11 -0
- riskvalidation/ml/_checks.py +67 -0
- riskvalidation/ml/_logistic.py +82 -0
- riskvalidation/ml/cbpe.py +91 -0
- riskvalidation/ml/challenger.py +73 -0
- riskvalidation/ml/conformal.py +116 -0
- riskvalidation/ml/dataquality.py +130 -0
- riskvalidation/ml/effects.py +67 -0
- riskvalidation/ml/explanations.py +125 -0
- riskvalidation/ml/generalisation.py +79 -0
- riskvalidation/ml/leakage.py +99 -0
- riskvalidation/ml/plants.py +425 -0
- riskvalidation/ml/policy.py +182 -0
- riskvalidation/ml/robustness.py +286 -0
- riskvalidation/ml/seeds.py +106 -0
- riskvalidation/oprisk/__init__.py +18 -0
- riskvalidation/oprisk/_checks.py +60 -0
- riskvalidation/oprisk/aggregate.py +375 -0
- riskvalidation/oprisk/distributions.py +758 -0
- riskvalidation/oprisk/scenarios.py +342 -0
- riskvalidation/oprisk/sla.py +80 -0
- riskvalidation/oprisk/sma.py +169 -0
- riskvalidation/policies/__init__.py +4 -0
- riskvalidation/policies/default.json +273 -0
- riskvalidation/portfolio/__init__.py +16 -0
- riskvalidation/portfolio/asrf_fit.py +283 -0
- riskvalidation/portfolio/copula.py +129 -0
- riskvalidation/portfolio/creditrisk_plus.py +232 -0
- riskvalidation/portfolio/euler.py +126 -0
- riskvalidation/portfolio/granularity.py +135 -0
- riskvalidation/portfolio/importance.py +277 -0
- riskvalidation/portfolio/portfolio.py +309 -0
- riskvalidation/provisions/__init__.py +8 -0
- riskvalidation/provisions/_checks.py +100 -0
- riskvalidation/provisions/ecl.py +233 -0
- riskvalidation/provisions/local_cl.py +360 -0
- riskvalidation/provisions/local_es.py +157 -0
- riskvalidation/provisions/overlays.py +102 -0
- riskvalidation/provisions/reconciliation.py +157 -0
- riskvalidation/provisions/staging.py +211 -0
- riskvalidation/provisions/term_structure.py +199 -0
- riskvalidation/regulatory/__init__.py +14 -0
- riskvalidation/regulatory/ba_cva.py +138 -0
- riskvalidation/regulatory/frtb.py +345 -0
- riskvalidation/regulatory/irb.py +314 -0
- riskvalidation/regulatory/lcr_nsfr.py +369 -0
- riskvalidation/regulatory/output_floor.py +45 -0
- riskvalidation/regulatory/sa_ccr.py +486 -0
- riskvalidation/regulatory/standardised.py +348 -0
- riskvalidation/stress/__init__.py +9 -0
- riskvalidation/stress/capital.py +476 -0
- riskvalidation/stress/paths.py +108 -0
- riskvalidation/stress/reverse.py +179 -0
- riskvalidation/stress/satellites.py +268 -0
- riskvalidation/stress/scenario_models.py +274 -0
- riskvalidation/transitions/__init__.py +0 -0
- riskvalidation/transitions/em.py +120 -0
- riskvalidation/transitions/embedding.py +221 -0
- riskvalidation/transitions/estimators.py +213 -0
- riskvalidation/transitions/intervals.py +306 -0
- riskvalidation/transitions/mobility.py +57 -0
- riskvalidation/transitions/ttc.py +171 -0
- riskvalidation/validation/__init__.py +103 -0
- riskvalidation/validation/_common.py +57 -0
- riskvalidation/validation/aggregation.py +425 -0
- riskvalidation/validation/calibration.py +757 -0
- riskvalidation/validation/ccr.py +530 -0
- riskvalidation/validation/discrimination.py +314 -0
- riskvalidation/validation/fairness.py +632 -0
- riskvalidation/validation/ifrs9.py +693 -0
- riskvalidation/validation/irrbb.py +389 -0
- riskvalidation/validation/lgd_ead.py +469 -0
- riskvalidation/validation/liquidity.py +647 -0
- riskvalidation/validation/market.py +807 -0
- riskvalidation/validation/ml.py +958 -0
- riskvalidation/validation/oprisk.py +357 -0
- riskvalidation/validation/portfolio.py +353 -0
- riskvalidation/validation/rating_system.py +245 -0
- riskvalidation/validation/stability.py +256 -0
- riskvalidation/validation/stress.py +552 -0
- riskvalidation/validation/transitions.py +386 -0
- riskvalidation-0.8.0.dist-info/METADATA +136 -0
- riskvalidation-0.8.0.dist-info/RECORD +164 -0
- riskvalidation-0.8.0.dist-info/WHEEL +5 -0
- riskvalidation-0.8.0.dist-info/licenses/LICENSE +21 -0
- riskvalidation-0.8.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
"""Planted-bias generators for the size and power of the fairness tests (SDD l.335; dossier 07 D pattern 6): a
|
|
2
|
+
group-dependent decision threshold, group-dependent label noise and a proxy feature, each built on
|
|
3
|
+
`generators.LogisticPortfolio`, each with `truth()` (exact) and `verify()` (a large draw reproduces the truth within
|
|
4
|
+
Monte Carlo error) as the generator contract asks.
|
|
5
|
+
|
|
6
|
+
Conventions of a draw: `group` is 1 for the protected group and 0 for the reference group; `decision` is 1 for the
|
|
7
|
+
favourable decision (the model approves the obligors of low linear predictor); `defaults` is the default indicator (1
|
|
8
|
+
is a default) and `y` the outcome aligned with the decision (1 repaid, `y = 1 - defaults`), both as observed (the
|
|
9
|
+
label-noise plant flips them for the protected group; `y_true` and `defaults_true` are the unflipped ones); `pd` is
|
|
10
|
+
the model's PD (calibrated) and `risk` its linear predictor. The seed of a plant is the stream of `draw_default()` and
|
|
11
|
+
of the default `verify()`; the harness draws from its own streams.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import math
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from functools import cached_property
|
|
19
|
+
from typing import Any, ClassVar
|
|
20
|
+
|
|
21
|
+
import numpy as np
|
|
22
|
+
from scipy import special, stats
|
|
23
|
+
|
|
24
|
+
from riskvalidation.generators._base import Check, GeneratorBase, mean_check
|
|
25
|
+
from riskvalidation.generators.scores import LogisticPortfolio
|
|
26
|
+
|
|
27
|
+
DEFAULT_COEFFICIENTS = (0.8, 0.5, 0.3)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _check_common(n: int, base_rate: float, share: float) -> None:
|
|
31
|
+
if not isinstance(n, (int, np.integer)) or n < 2:
|
|
32
|
+
raise ValueError(f"n must be an integer of at least 2, got {n!r}")
|
|
33
|
+
if not 0.0 < base_rate < 1.0:
|
|
34
|
+
raise ValueError(f"base_rate must lie strictly between 0 and 1, got {base_rate!r}")
|
|
35
|
+
if not 0.0 < share < 1.0:
|
|
36
|
+
raise ValueError(f"protected_share must lie strictly between 0 and 1, got {share!r}")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class _PortfolioPlant(GeneratorBase):
|
|
40
|
+
"""What the three plants share: the portfolio, the stream of a draw and the decision cutoff."""
|
|
41
|
+
|
|
42
|
+
n: int
|
|
43
|
+
seed: int
|
|
44
|
+
base_rate: float
|
|
45
|
+
protected_share: float
|
|
46
|
+
intercept: float
|
|
47
|
+
coefficients: Any
|
|
48
|
+
correlation: float
|
|
49
|
+
|
|
50
|
+
@cached_property
|
|
51
|
+
def _portfolio(self) -> LogisticPortfolio:
|
|
52
|
+
return LogisticPortfolio(n=self.n, intercept=self.intercept, coefficients=self.coefficients,
|
|
53
|
+
correlation=self.correlation)
|
|
54
|
+
|
|
55
|
+
@cached_property
|
|
56
|
+
def _moments(self) -> tuple[float, float]:
|
|
57
|
+
"""Mean and standard deviation of the linear predictor, which is normal."""
|
|
58
|
+
return self._portfolio._eta_moments()
|
|
59
|
+
|
|
60
|
+
def _cutoff(self) -> float:
|
|
61
|
+
"""The linear-predictor cutoff that approves `base_rate` of the obligors (eta is normal, exactly)."""
|
|
62
|
+
m, s = self._moments
|
|
63
|
+
return m + s * float(stats.norm.ppf(self.base_rate))
|
|
64
|
+
|
|
65
|
+
def _base_draw(self, rng: np.random.Generator) -> tuple[dict[str, Any], np.ndarray]:
|
|
66
|
+
d = self._portfolio.draw(rng)
|
|
67
|
+
group = (rng.random(self.n) < self.protected_share).astype(np.int64)
|
|
68
|
+
return d, group
|
|
69
|
+
|
|
70
|
+
def draw_default(self) -> dict[str, Any]:
|
|
71
|
+
"""One dataset drawn from the plant's own seed."""
|
|
72
|
+
return self.draw(np.random.default_rng(self.seed)) # type: ignore[attr-defined]
|
|
73
|
+
|
|
74
|
+
def _stream(self, seed: int | None, salt: int) -> np.random.Generator:
|
|
75
|
+
return np.random.default_rng(np.random.SeedSequence([self.seed if seed is None else seed, salt]))
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class ThresholdBiasPlant(_PortfolioPlant):
|
|
80
|
+
"""The model's linear predictor is the same for both groups but the protected group is approved below a cutoff
|
|
81
|
+
lower by `gap` (in log-odds): a group-dependent decision threshold. `base_rate` is the reference group's selection
|
|
82
|
+
rate; `gap` 0 is the null. `truth()` gives the exact selection rates, since the linear predictor is normal."""
|
|
83
|
+
|
|
84
|
+
name: ClassVar[str] = "threshold_bias_plant"
|
|
85
|
+
n: int
|
|
86
|
+
gap: float
|
|
87
|
+
seed: int
|
|
88
|
+
base_rate: float = 0.7
|
|
89
|
+
protected_share: float = 0.5
|
|
90
|
+
intercept: float = -1.0
|
|
91
|
+
coefficients: Any = DEFAULT_COEFFICIENTS
|
|
92
|
+
correlation: float = 0.3
|
|
93
|
+
|
|
94
|
+
def __post_init__(self) -> None:
|
|
95
|
+
_check_common(self.n, self.base_rate, self.protected_share)
|
|
96
|
+
if not (math.isfinite(self.gap) and self.gap >= 0.0):
|
|
97
|
+
raise ValueError(f"gap must be a non-negative number, got {self.gap!r}")
|
|
98
|
+
object.__setattr__(self, "n", int(self.n))
|
|
99
|
+
|
|
100
|
+
def draw(self, rng: np.random.Generator) -> dict[str, Any]:
|
|
101
|
+
d, group = self._base_draw(rng)
|
|
102
|
+
eta = d["risk"]
|
|
103
|
+
decision = (eta <= self._cutoff() - self.gap * group).astype(np.int64)
|
|
104
|
+
defaults = d["defaults"].astype(np.int64)
|
|
105
|
+
return {"group": group, "decision": decision, "y": 1 - defaults, "defaults": defaults, "pd": d["pd"],
|
|
106
|
+
"risk": eta, "x": d["x"]}
|
|
107
|
+
|
|
108
|
+
def truth(self) -> dict[str, Any]:
|
|
109
|
+
s = self._moments[1]
|
|
110
|
+
ref = float(self.base_rate)
|
|
111
|
+
prot = ref if self.gap == 0.0 else float(stats.norm.cdf(stats.norm.ppf(self.base_rate) - self.gap / s))
|
|
112
|
+
return {"selection_rate_reference": ref, "selection_rate_protected": prot,
|
|
113
|
+
"selection_rate_gap": ref - prot if self.gap > 0.0 else 0.0, "gap": float(self.gap),
|
|
114
|
+
"protected_share": float(self.protected_share), "null": self.gap == 0.0}
|
|
115
|
+
|
|
116
|
+
def verify(self, n_draws: int = 200, seed: int | None = None) -> list[Check]:
|
|
117
|
+
rng = self._stream(seed, 1469)
|
|
118
|
+
tr = self.truth()
|
|
119
|
+
ref, prot, share = [], [], []
|
|
120
|
+
for _ in range(n_draws):
|
|
121
|
+
d = self.draw(rng)
|
|
122
|
+
ref.append(d["decision"][d["group"] == 0].mean())
|
|
123
|
+
prot.append(d["decision"][d["group"] == 1].mean())
|
|
124
|
+
share.append(d["group"].mean())
|
|
125
|
+
return [mean_check("selection rate, reference group", tr["selection_rate_reference"], np.array(ref)),
|
|
126
|
+
mean_check("selection rate, protected group", tr["selection_rate_protected"], np.array(prot)),
|
|
127
|
+
mean_check("protected share", tr["protected_share"], np.array(share))]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass(frozen=True)
|
|
131
|
+
class LabelNoiseBiasPlant(_PortfolioPlant):
|
|
132
|
+
"""The model decides identically for both groups, but the observed outcome of the protected group is wrong at the
|
|
133
|
+
rate `noise`: each of its labels is flipped with that probability, so the error rates measured against the observed
|
|
134
|
+
outcome differ between groups although the model treats them alike. `noise` is in [0, 0.5]; 0 is the null."""
|
|
135
|
+
|
|
136
|
+
name: ClassVar[str] = "label_noise_bias_plant"
|
|
137
|
+
n: int
|
|
138
|
+
noise: float
|
|
139
|
+
seed: int
|
|
140
|
+
base_rate: float = 0.7
|
|
141
|
+
protected_share: float = 0.5
|
|
142
|
+
intercept: float = -1.0
|
|
143
|
+
coefficients: Any = DEFAULT_COEFFICIENTS
|
|
144
|
+
correlation: float = 0.3
|
|
145
|
+
|
|
146
|
+
def __post_init__(self) -> None:
|
|
147
|
+
_check_common(self.n, self.base_rate, self.protected_share)
|
|
148
|
+
if not (math.isfinite(self.noise) and 0.0 <= self.noise <= 0.5):
|
|
149
|
+
raise ValueError(f"noise must lie in [0, 0.5], got {self.noise!r}")
|
|
150
|
+
object.__setattr__(self, "n", int(self.n))
|
|
151
|
+
|
|
152
|
+
def draw(self, rng: np.random.Generator) -> dict[str, Any]:
|
|
153
|
+
d, group = self._base_draw(rng)
|
|
154
|
+
eta = d["risk"]
|
|
155
|
+
decision = (eta <= self._cutoff()).astype(np.int64)
|
|
156
|
+
true_defaults = d["defaults"].astype(np.int64)
|
|
157
|
+
flips = ((rng.random(self.n) < self.noise) & (group == 1)).astype(np.int64)
|
|
158
|
+
defaults = true_defaults ^ flips
|
|
159
|
+
return {"group": group, "decision": decision, "y": 1 - defaults, "y_true": 1 - true_defaults,
|
|
160
|
+
"defaults": defaults, "defaults_true": true_defaults, "pd": d["pd"], "risk": eta, "x": d["x"]}
|
|
161
|
+
|
|
162
|
+
def truth(self) -> dict[str, Any]:
|
|
163
|
+
return {"flip_rate": float(self.noise), "flipped_group": 1, "selection_rate": float(self.base_rate),
|
|
164
|
+
"protected_share": float(self.protected_share)}
|
|
165
|
+
|
|
166
|
+
def verify(self, n_draws: int = 200, seed: int | None = None) -> list[Check]:
|
|
167
|
+
rng = self._stream(seed, 1470)
|
|
168
|
+
tr = self.truth()
|
|
169
|
+
prot, ref, rate = [], [], []
|
|
170
|
+
for _ in range(n_draws):
|
|
171
|
+
d = self.draw(rng)
|
|
172
|
+
flipped = d["y"] != d["y_true"]
|
|
173
|
+
prot.append(flipped[d["group"] == 1].mean())
|
|
174
|
+
ref.append(flipped[d["group"] == 0].mean())
|
|
175
|
+
rate.append(d["decision"].mean())
|
|
176
|
+
return [mean_check("flip rate, protected group", tr["flip_rate"], np.array(prot)),
|
|
177
|
+
mean_check("flip rate, reference group", 0.0, np.array(ref)),
|
|
178
|
+
mean_check("selection rate", tr["selection_rate"], np.array(rate))]
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@dataclass(frozen=True)
|
|
182
|
+
class ProxyFeaturePlant(_PortfolioPlant):
|
|
183
|
+
"""A feature, the proxy, has correlation `proxy_strength` with the protected indicator, and the decision depends on
|
|
184
|
+
the covariates and the proxy only (never on the protected indicator): the disparity is transmitted by the proxy
|
|
185
|
+
alone. A decision model that controls for the covariates but not the proxy finds a protected effect; once the
|
|
186
|
+
proxy is a control that effect is 0 exactly (`truth()['protected_effect_given_proxy']`). `proxy_strength` 0 is
|
|
187
|
+
the null."""
|
|
188
|
+
|
|
189
|
+
name: ClassVar[str] = "proxy_feature_plant"
|
|
190
|
+
n: int
|
|
191
|
+
proxy_strength: float
|
|
192
|
+
seed: int
|
|
193
|
+
proxy_effect: float = 1.0
|
|
194
|
+
base_rate: float = 0.7
|
|
195
|
+
protected_share: float = 0.5
|
|
196
|
+
intercept: float = 0.2
|
|
197
|
+
coefficients: Any = DEFAULT_COEFFICIENTS
|
|
198
|
+
correlation: float = 0.3
|
|
199
|
+
|
|
200
|
+
def __post_init__(self) -> None:
|
|
201
|
+
_check_common(self.n, self.base_rate, self.protected_share)
|
|
202
|
+
if not (math.isfinite(self.proxy_strength) and 0.0 <= self.proxy_strength < 1.0):
|
|
203
|
+
raise ValueError(f"proxy_strength must lie in [0, 1), got {self.proxy_strength!r}")
|
|
204
|
+
if not math.isfinite(self.proxy_effect):
|
|
205
|
+
raise ValueError("proxy_effect must be finite")
|
|
206
|
+
object.__setattr__(self, "n", int(self.n))
|
|
207
|
+
|
|
208
|
+
def draw(self, rng: np.random.Generator) -> dict[str, Any]:
|
|
209
|
+
d, group = self._base_draw(rng)
|
|
210
|
+
share = self.protected_share
|
|
211
|
+
standardised = (group - share) / math.sqrt(share * (1.0 - share)) # mean 0, variance 1
|
|
212
|
+
proxy = self.proxy_strength * standardised + math.sqrt(1.0 - self.proxy_strength**2) * rng.standard_normal(
|
|
213
|
+
self.n)
|
|
214
|
+
x = d["x"]
|
|
215
|
+
logit = self.intercept + x @ np.asarray(self.coefficients, dtype=float) + self.proxy_effect * proxy
|
|
216
|
+
decision = (rng.random(self.n) < special.expit(logit)).astype(np.int64)
|
|
217
|
+
return {"group": group, "decision": decision, "protected": group.astype(float)[:, None], "controls": x,
|
|
218
|
+
"proxy": proxy, "controls_with_proxy": np.column_stack([x, proxy])}
|
|
219
|
+
|
|
220
|
+
def truth(self) -> dict[str, Any]:
|
|
221
|
+
return {"proxy": "proxy", "proxy_index": len(self.coefficients), "proxy_strength": float(self.proxy_strength),
|
|
222
|
+
"proxy_effect": float(self.proxy_effect), "protected_effect_given_proxy": 0.0,
|
|
223
|
+
"protected_share": float(self.protected_share)}
|
|
224
|
+
|
|
225
|
+
def verify(self, n_draws: int = 200, seed: int | None = None) -> list[Check]:
|
|
226
|
+
rng = self._stream(seed, 1471)
|
|
227
|
+
tr = self.truth()
|
|
228
|
+
corr, share, mean_proxy = [], [], []
|
|
229
|
+
for _ in range(n_draws):
|
|
230
|
+
d = self.draw(rng)
|
|
231
|
+
corr.append(np.corrcoef(d["proxy"], d["group"])[0, 1])
|
|
232
|
+
share.append(d["group"].mean())
|
|
233
|
+
mean_proxy.append(d["proxy"].mean())
|
|
234
|
+
return [mean_check("correlation of the proxy with the protected indicator", tr["proxy_strength"],
|
|
235
|
+
np.array(corr)),
|
|
236
|
+
mean_check("protected share", tr["protected_share"], np.array(share)),
|
|
237
|
+
mean_check("mean of the proxy", 0.0, np.array(mean_proxy))]
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
__all__ = ["LabelNoiseBiasPlant", "ProxyFeaturePlant", "ThresholdBiasPlant"]
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""The `fair.` rows of the validation policy: the cut points of the fairness lights (contract `engine-fairness-api`
|
|
2
|
+
1.0.0, section Policy thresholds).
|
|
3
|
+
|
|
4
|
+
The values are conventions, never regulatory thresholds (dossier 06 I.6 l.1268-1270): each is a named parameter for the
|
|
5
|
+
main session's approval. A row marked strict states a tolerance that 'exceeds' triggers: the cut point is stored one
|
|
6
|
+
float above the tolerance (`math.nextafter`), so a metric equal to the tolerance is not a finding and is green
|
|
7
|
+
(`Threshold.classify` is inclusive at its cut points). A red cut of 2.0 is a sentinel above the range of a metric that
|
|
8
|
+
is at most 1: that light is never red.
|
|
9
|
+
|
|
10
|
+
`fairness_default_policy()` is the packaged default policy with these rows added where it holds none; once the rows
|
|
11
|
+
sit in `policies/default.json` the two are the same object. A policy a caller supplies is used as given: a policy that
|
|
12
|
+
lacks a key gives the `not_evaluated` light, never a colour.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import math
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from functools import lru_cache
|
|
20
|
+
|
|
21
|
+
from riskvalidation.core.policy import Policy, Threshold, default_policy
|
|
22
|
+
|
|
23
|
+
POLICY_SUFFIX = "fair-1.0.0"
|
|
24
|
+
NEVER_RED = 2.0
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class FairRow:
|
|
29
|
+
"""One row of the contract's Policy thresholds table."""
|
|
30
|
+
|
|
31
|
+
kind: str
|
|
32
|
+
amber: float
|
|
33
|
+
red: float
|
|
34
|
+
strict: bool
|
|
35
|
+
source: str
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
FAIR_ROWS: dict[str, FairRow] = {
|
|
39
|
+
"fair.demographic_parity": FairRow(
|
|
40
|
+
"upper_bad", 0.05, NEVER_RED, True,
|
|
41
|
+
"convention: the amber tolerance 0.05 on the largest absolute selection-rate gap is proposed, no dossier "
|
|
42
|
+
"value; "
|
|
43
|
+
"strict (a gap equal to the tolerance is green); red 2.0 is a sentinel above the metric's range: never red"),
|
|
44
|
+
"fair.two_proportion_z": FairRow(
|
|
45
|
+
"p_value", 0.05, 0.01, False,
|
|
46
|
+
"convention: the engine's default p-value cut points (5% and 1%); dossier 06 C.7 l.1036-1039"),
|
|
47
|
+
"fair.fisher_exact": FairRow(
|
|
48
|
+
"p_value", 0.05, 0.01, False,
|
|
49
|
+
"convention: the engine's default p-value cut points (5% and 1%); dossier 06 C.7 l.1036-1039"),
|
|
50
|
+
"fair.equalized_odds": FairRow(
|
|
51
|
+
"upper_bad", 0.05, 0.10, False,
|
|
52
|
+
"convention: proposed for the Taiwan battery, no dossier value (06 C.7 l.1029-1033)"),
|
|
53
|
+
"fair.equal_opportunity": FairRow(
|
|
54
|
+
"upper_bad", 0.05, 0.10, False,
|
|
55
|
+
"convention: proposed for the Taiwan battery, no dossier value (06 C.7 l.1029-1033)"),
|
|
56
|
+
"fair.calibration_within_groups": FairRow(
|
|
57
|
+
"upper_bad", 0.05, 0.10, False,
|
|
58
|
+
"convention: proposed for the Taiwan battery, no dossier value (04 J.3 l.904-905)"),
|
|
59
|
+
"fair.auc_by_group": FairRow(
|
|
60
|
+
"upper_bad", 0.05, 0.10, False,
|
|
61
|
+
"convention: proposed for the Taiwan battery, no dossier value (04 J.3 l.904-905)"),
|
|
62
|
+
"fair.situation_testing": FairRow(
|
|
63
|
+
"upper_bad", 0.20, NEVER_RED, False,
|
|
64
|
+
"convention: the amber share 0.20 of significant complainants, set on the measured size 0.039 and power 1.000 "
|
|
65
|
+
"at n 400 and k 30 (docs/fairness/06), no source value; red 2.0 is a sentinel above the metric's range: "
|
|
66
|
+
"never red"),
|
|
67
|
+
}
|
|
68
|
+
FAIR_KEYS = tuple(FAIR_ROWS)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def fair_threshold(key: str) -> Threshold:
|
|
72
|
+
"""The `Threshold` of a `fair.` row, the strict amber cut stored one float above the tolerance."""
|
|
73
|
+
row = FAIR_ROWS[key]
|
|
74
|
+
amber = math.nextafter(row.amber, math.inf) if row.strict else row.amber
|
|
75
|
+
return Threshold(kind=row.kind, amber=amber, red=row.red, source=row.source)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@lru_cache(maxsize=1)
|
|
79
|
+
def fairness_default_policy() -> Policy:
|
|
80
|
+
"""The packaged default policy with the `fair.` rows added where it holds none; the default policy itself,
|
|
81
|
+
label unchanged, when it already holds every `fair.` key."""
|
|
82
|
+
base = default_policy()
|
|
83
|
+
if all(key in base.thresholds for key in FAIR_KEYS):
|
|
84
|
+
return base
|
|
85
|
+
thresholds = dict(base.thresholds)
|
|
86
|
+
for key in FAIR_KEYS:
|
|
87
|
+
thresholds.setdefault(key, fair_threshold(key))
|
|
88
|
+
return Policy(
|
|
89
|
+
policy_id=base.policy_id,
|
|
90
|
+
version=f"{base.version}+{POLICY_SUFFIX}",
|
|
91
|
+
description=f"{base.description} Fairness rows ({POLICY_SUFFIX}): conventions for the main session's approval, "
|
|
92
|
+
"never regulatory thresholds.",
|
|
93
|
+
thresholds=thresholds,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
__all__ = ["FAIR_KEYS", "FAIR_ROWS", "FairRow", "NEVER_RED", "fair_threshold", "fairness_default_policy"]
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Year-to-year replication of an estimate (decision Q4; dossier 08 l.839 asks whether the finding is stable year to
|
|
2
|
+
year): the estimate fitted on year t and the same estimate refitted on year t+1, compared by sign and by size. No
|
|
3
|
+
verdict beyond sign and size is returned (contract `engine-fairness-api` 1.0.0)."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import math
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
|
|
10
|
+
from riskvalidation.fairness.conditional import Effect
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class Replication:
|
|
15
|
+
"""`same_sign` and the size ratio confirm over fit (signed; None when the fit coefficient is 0)."""
|
|
16
|
+
|
|
17
|
+
same_sign: bool
|
|
18
|
+
size_ratio: float | None
|
|
19
|
+
fit_coefficient: float
|
|
20
|
+
confirm_coefficient: float
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _sign(v: float) -> int:
|
|
24
|
+
return (v > 0.0) - (v < 0.0)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def replicate_effect(fit: Effect, confirm: Effect) -> Replication:
|
|
28
|
+
"""same_sign is the equality of the signs of the two coefficients (0 has its own sign); size_ratio is
|
|
29
|
+
confirm.coefficient / fit.coefficient, so a flip gives a negative ratio, and None when the fit coefficient is 0.
|
|
30
|
+
A fit coefficient of exactly 0 therefore gives the light `not_evaluated` in `validation.fairness.replication`."""
|
|
31
|
+
for name, eff in (("fit", fit), ("confirm", confirm)):
|
|
32
|
+
if not isinstance(eff, Effect) or not math.isfinite(eff.coefficient):
|
|
33
|
+
raise ValueError(f"{name} must be an Effect with a finite coefficient")
|
|
34
|
+
ratio = None if fit.coefficient == 0.0 else confirm.coefficient / fit.coefficient
|
|
35
|
+
return Replication(_sign(fit.coefficient) == _sign(confirm.coefficient), ratio, fit.coefficient,
|
|
36
|
+
confirm.coefficient)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
__all__ = ["Replication", "replicate_effect"]
|