riskvalidation 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- riskvalidation/__init__.py +30 -0
- riskvalidation/aggregation/__init__.py +11 -0
- riskvalidation/aggregation/band.py +151 -0
- riskvalidation/aggregation/bounds.py +191 -0
- riskvalidation/aggregation/copulas.py +241 -0
- riskvalidation/aggregation/model_risk.py +133 -0
- riskvalidation/aggregation/simple.py +147 -0
- riskvalidation/ccr/__init__.py +13 -0
- riskvalidation/ccr/collateral.py +175 -0
- riskvalidation/ccr/cva.py +193 -0
- riskvalidation/ccr/exposure.py +192 -0
- riskvalidation/ccr/hw1f.py +426 -0
- riskvalidation/ccr/swaps.py +172 -0
- riskvalidation/core/__init__.py +1 -0
- riskvalidation/core/build.py +57 -0
- riskvalidation/core/hashing.py +48 -0
- riskvalidation/core/policy.py +130 -0
- riskvalidation/core/registry.py +65 -0
- riskvalidation/core/result.py +117 -0
- riskvalidation/core/suite.py +78 -0
- riskvalidation/engines/__init__.py +2 -0
- riskvalidation/engines/low_default.py +170 -0
- riskvalidation/engines/pd_curve.py +529 -0
- riskvalidation/engines/vasicek.py +72 -0
- riskvalidation/fairness/__init__.py +10 -0
- riskvalidation/fairness/_checks.py +115 -0
- riskvalidation/fairness/conditional.py +201 -0
- riskvalidation/fairness/inference.py +64 -0
- riskvalidation/fairness/metrics.py +261 -0
- riskvalidation/fairness/plants.py +240 -0
- riskvalidation/fairness/policy.py +97 -0
- riskvalidation/fairness/replication.py +39 -0
- riskvalidation/fairness/situation.py +307 -0
- riskvalidation/generators/__init__.py +68 -0
- riskvalidation/generators/_base.py +145 -0
- riskvalidation/generators/counts.py +232 -0
- riskvalidation/generators/drift.py +107 -0
- riskvalidation/generators/framingham.py +124 -0
- riskvalidation/generators/paths.py +349 -0
- riskvalidation/generators/ratings.py +186 -0
- riskvalidation/generators/scores.py +347 -0
- riskvalidation/harness/__init__.py +46 -0
- riskvalidation/harness/exact.py +137 -0
- riskvalidation/harness/performance.py +74 -0
- riskvalidation/harness/rates.py +103 -0
- riskvalidation/harness/run.py +197 -0
- riskvalidation/irrbb/__init__.py +9 -0
- riskvalidation/irrbb/behavioural.py +206 -0
- riskvalidation/irrbb/curves.py +286 -0
- riskvalidation/irrbb/eve_nii.py +245 -0
- riskvalidation/irrbb/shocks.py +171 -0
- riskvalidation/irrbb/simulation.py +203 -0
- riskvalidation/irrbb/slotting.py +184 -0
- riskvalidation/lgd_ead/__init__.py +8 -0
- riskvalidation/lgd_ead/_checks.py +50 -0
- riskvalidation/lgd_ead/ccf.py +228 -0
- riskvalidation/lgd_ead/downturn.py +64 -0
- riskvalidation/lgd_ead/floors.py +114 -0
- riskvalidation/lgd_ead/lgd_models.py +297 -0
- riskvalidation/lgd_ead/workout.py +162 -0
- riskvalidation/liquidity/__init__.py +9 -0
- riskvalidation/liquidity/_checks.py +78 -0
- riskvalidation/liquidity/cbc.py +158 -0
- riskvalidation/liquidity/deposits.py +151 -0
- riskvalidation/liquidity/ladder.py +119 -0
- riskvalidation/liquidity/reverse.py +141 -0
- riskvalidation/liquidity/runoff.py +220 -0
- riskvalidation/liquidity/scenarios.py +272 -0
- riskvalidation/market/__init__.py +8 -0
- riskvalidation/market/caviar.py +239 -0
- riskvalidation/market/conventions.py +83 -0
- riskvalidation/market/evt.py +111 -0
- riskvalidation/market/frtb.py +175 -0
- riskvalidation/market/garch.py +204 -0
- riskvalidation/market/gas_fz.py +219 -0
- riskvalidation/market/historical.py +96 -0
- riskvalidation/market/parametric.py +78 -0
- riskvalidation/market/scoring.py +145 -0
- riskvalidation/ml/__init__.py +11 -0
- riskvalidation/ml/_checks.py +67 -0
- riskvalidation/ml/_logistic.py +82 -0
- riskvalidation/ml/cbpe.py +91 -0
- riskvalidation/ml/challenger.py +73 -0
- riskvalidation/ml/conformal.py +116 -0
- riskvalidation/ml/dataquality.py +130 -0
- riskvalidation/ml/effects.py +67 -0
- riskvalidation/ml/explanations.py +125 -0
- riskvalidation/ml/generalisation.py +79 -0
- riskvalidation/ml/leakage.py +99 -0
- riskvalidation/ml/plants.py +425 -0
- riskvalidation/ml/policy.py +182 -0
- riskvalidation/ml/robustness.py +286 -0
- riskvalidation/ml/seeds.py +106 -0
- riskvalidation/oprisk/__init__.py +18 -0
- riskvalidation/oprisk/_checks.py +60 -0
- riskvalidation/oprisk/aggregate.py +375 -0
- riskvalidation/oprisk/distributions.py +758 -0
- riskvalidation/oprisk/scenarios.py +342 -0
- riskvalidation/oprisk/sla.py +80 -0
- riskvalidation/oprisk/sma.py +169 -0
- riskvalidation/policies/__init__.py +4 -0
- riskvalidation/policies/default.json +273 -0
- riskvalidation/portfolio/__init__.py +16 -0
- riskvalidation/portfolio/asrf_fit.py +283 -0
- riskvalidation/portfolio/copula.py +129 -0
- riskvalidation/portfolio/creditrisk_plus.py +232 -0
- riskvalidation/portfolio/euler.py +126 -0
- riskvalidation/portfolio/granularity.py +135 -0
- riskvalidation/portfolio/importance.py +277 -0
- riskvalidation/portfolio/portfolio.py +309 -0
- riskvalidation/provisions/__init__.py +8 -0
- riskvalidation/provisions/_checks.py +100 -0
- riskvalidation/provisions/ecl.py +233 -0
- riskvalidation/provisions/local_cl.py +360 -0
- riskvalidation/provisions/local_es.py +157 -0
- riskvalidation/provisions/overlays.py +102 -0
- riskvalidation/provisions/reconciliation.py +157 -0
- riskvalidation/provisions/staging.py +211 -0
- riskvalidation/provisions/term_structure.py +199 -0
- riskvalidation/regulatory/__init__.py +14 -0
- riskvalidation/regulatory/ba_cva.py +138 -0
- riskvalidation/regulatory/frtb.py +345 -0
- riskvalidation/regulatory/irb.py +314 -0
- riskvalidation/regulatory/lcr_nsfr.py +369 -0
- riskvalidation/regulatory/output_floor.py +45 -0
- riskvalidation/regulatory/sa_ccr.py +486 -0
- riskvalidation/regulatory/standardised.py +348 -0
- riskvalidation/stress/__init__.py +9 -0
- riskvalidation/stress/capital.py +476 -0
- riskvalidation/stress/paths.py +108 -0
- riskvalidation/stress/reverse.py +179 -0
- riskvalidation/stress/satellites.py +268 -0
- riskvalidation/stress/scenario_models.py +274 -0
- riskvalidation/transitions/__init__.py +0 -0
- riskvalidation/transitions/em.py +120 -0
- riskvalidation/transitions/embedding.py +221 -0
- riskvalidation/transitions/estimators.py +213 -0
- riskvalidation/transitions/intervals.py +306 -0
- riskvalidation/transitions/mobility.py +57 -0
- riskvalidation/transitions/ttc.py +171 -0
- riskvalidation/validation/__init__.py +103 -0
- riskvalidation/validation/_common.py +57 -0
- riskvalidation/validation/aggregation.py +425 -0
- riskvalidation/validation/calibration.py +757 -0
- riskvalidation/validation/ccr.py +530 -0
- riskvalidation/validation/discrimination.py +314 -0
- riskvalidation/validation/fairness.py +632 -0
- riskvalidation/validation/ifrs9.py +693 -0
- riskvalidation/validation/irrbb.py +389 -0
- riskvalidation/validation/lgd_ead.py +469 -0
- riskvalidation/validation/liquidity.py +647 -0
- riskvalidation/validation/market.py +807 -0
- riskvalidation/validation/ml.py +958 -0
- riskvalidation/validation/oprisk.py +357 -0
- riskvalidation/validation/portfolio.py +353 -0
- riskvalidation/validation/rating_system.py +245 -0
- riskvalidation/validation/stability.py +256 -0
- riskvalidation/validation/stress.py +552 -0
- riskvalidation/validation/transitions.py +386 -0
- riskvalidation-0.8.0.dist-info/METADATA +136 -0
- riskvalidation-0.8.0.dist-info/RECORD +164 -0
- riskvalidation-0.8.0.dist-info/WHEEL +5 -0
- riskvalidation-0.8.0.dist-info/licenses/LICENSE +21 -0
- riskvalidation-0.8.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Input checks shared by the fairness modules, each naming the offending field.
|
|
2
|
+
|
|
3
|
+
Two kinds of refusal. A value that is malformed (a decision that is not 0 or 1, rows of unequal length, a non-finite
|
|
4
|
+
entry) raises `ValueError` naming the field. A value that is well formed but too thin to compute from (a single group,
|
|
5
|
+
a group without both outcome classes, a group of no rows) raises `InsufficientData`, a `ValueError` subclass, so the
|
|
6
|
+
`TestResult` functions can return the `not_evaluated` light with the reason instead of failing.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class InsufficientData(ValueError):
|
|
17
|
+
"""Valid data too thin for the quantity asked: a validation test returns `not_evaluated` with this reason."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def binary(x: Any, name: str) -> np.ndarray:
|
|
21
|
+
"""A new one-dimensional int64 array holding only 0 and 1."""
|
|
22
|
+
a = np.asarray(x)
|
|
23
|
+
if a.ndim != 1 or a.size == 0:
|
|
24
|
+
raise ValueError(f"{name} must be a non-empty one-dimensional array of 0 and 1")
|
|
25
|
+
if a.dtype.kind == "f" and not np.all(np.isfinite(a)):
|
|
26
|
+
raise ValueError(f"{name} contains non-finite values; it must hold only 0 and 1")
|
|
27
|
+
if a.dtype.kind not in "biuf" or not np.all(np.isin(a, (0, 1))):
|
|
28
|
+
raise ValueError(f"{name} must contain only 0 and 1")
|
|
29
|
+
return a.astype(np.int64)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def outcome(y: Any, name: str = "y") -> np.ndarray:
|
|
33
|
+
"""The outcome of the outcome-conditioned tests; a missing outcome is refused with the reason."""
|
|
34
|
+
if y is None:
|
|
35
|
+
raise ValueError(
|
|
36
|
+
f"{name} is missing: the dataset declares no outcome (outcome: none), and the outcome-conditioned tests "
|
|
37
|
+
"(equalized odds, equal opportunity, calibration within groups, AUC by group) never run on it"
|
|
38
|
+
)
|
|
39
|
+
return binary(y, name)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def group_labels(group: Any, n: int, name: str = "group") -> np.ndarray:
|
|
43
|
+
"""The group of each row, length `n`: numeric and boolean labels are kept as given (and sort numerically), other
|
|
44
|
+
labels become strings. A group is named by `str(label)` in every result."""
|
|
45
|
+
g = np.asarray(group)
|
|
46
|
+
if g.ndim != 1 or g.size == 0:
|
|
47
|
+
raise ValueError(f"{name} must be a non-empty one-dimensional array of labels")
|
|
48
|
+
if g.shape[0] != n:
|
|
49
|
+
raise ValueError(f"{name} has {g.shape[0]} rows but {n} were expected")
|
|
50
|
+
if g.dtype.kind == "f" and not np.all(np.isfinite(g)):
|
|
51
|
+
raise ValueError(f"{name} contains a missing label")
|
|
52
|
+
if g.dtype.kind in "biuf":
|
|
53
|
+
return g
|
|
54
|
+
if g.dtype.kind == "O":
|
|
55
|
+
if any(v is None or (isinstance(v, float) and v != v) for v in g):
|
|
56
|
+
raise ValueError(f"{name} contains a missing label")
|
|
57
|
+
return np.array([str(v) for v in g])
|
|
58
|
+
return g.astype(str)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def group_names(g: np.ndarray) -> list[str]:
|
|
62
|
+
"""The names of the groups present, in label order."""
|
|
63
|
+
return [str(v) for v in np.unique(g)]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def same_rows(a: np.ndarray, name_a: str, b: np.ndarray, name_b: str) -> None:
|
|
67
|
+
if a.shape[0] != b.shape[0]:
|
|
68
|
+
raise ValueError(f"{name_b} has {b.shape[0]} rows but {name_a} has {a.shape[0]}")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def count(v: Any, name: str) -> int:
|
|
72
|
+
"""An integer count (a Python or NumPy integer, or a float with an integral value)."""
|
|
73
|
+
if isinstance(v, (bool, np.bool_)):
|
|
74
|
+
raise ValueError(f"{name} must be an integer count, got a boolean")
|
|
75
|
+
if isinstance(v, (int, np.integer)):
|
|
76
|
+
return int(v)
|
|
77
|
+
if isinstance(v, (float, np.floating)) and np.isfinite(v) and float(v).is_integer():
|
|
78
|
+
return int(v)
|
|
79
|
+
raise ValueError(f"{name} must be an integer count, got {v!r}")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def count_pair(x: Any, n: Any, x_name: str, n_name: str) -> tuple[int, int]:
|
|
83
|
+
"""A favourable count and a group size with 0 <= x <= n; a group of no rows is `InsufficientData`."""
|
|
84
|
+
xi, ni = count(x, x_name), count(n, n_name)
|
|
85
|
+
if ni < 1:
|
|
86
|
+
raise InsufficientData(f"{n_name} must be at least 1: the group has no rows")
|
|
87
|
+
if not 0 <= xi <= ni:
|
|
88
|
+
raise ValueError(f"{x_name} must lie in [0, {n_name}] = [0, {ni}], got {xi}")
|
|
89
|
+
return xi, ni
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def finite_matrix(
|
|
93
|
+
x: Any, name: str, *, rows: int | None = None, rows_of: str = "the outcome", allow_no_columns: bool = False
|
|
94
|
+
) -> np.ndarray:
|
|
95
|
+
"""A new two-dimensional finite float64 array; a one-dimensional input is read as one column."""
|
|
96
|
+
a = np.array(x, dtype=float)
|
|
97
|
+
if a.ndim == 1:
|
|
98
|
+
a = a[:, None]
|
|
99
|
+
if a.ndim != 2:
|
|
100
|
+
raise ValueError(f"{name} must be one- or two-dimensional, got shape {a.shape}")
|
|
101
|
+
if a.shape[1] == 0 and not allow_no_columns:
|
|
102
|
+
raise ValueError(f"{name} has no columns")
|
|
103
|
+
if not np.all(np.isfinite(a)):
|
|
104
|
+
raise ValueError(f"{name} contains non-finite values")
|
|
105
|
+
if rows is not None and a.shape[0] != rows:
|
|
106
|
+
raise ValueError(f"{name} has {a.shape[0]} rows but {rows_of} has {rows}")
|
|
107
|
+
return a
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def unit_interval_open(v: float, name: str) -> float:
|
|
111
|
+
"""A float strictly inside (0, 1)."""
|
|
112
|
+
f = float(v)
|
|
113
|
+
if not (np.isfinite(f) and 0.0 < f < 1.0):
|
|
114
|
+
raise ValueError(f"{name} must lie strictly between 0 and 1, got {v!r}")
|
|
115
|
+
return f
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""The disparity that persists after the permitted controls (the C21 question, dossier 08 l.839; decision Q4): a
|
|
2
|
+
logistic decision model of the approval on the protected indicators and the controls, and a linear model of the rate
|
|
3
|
+
spread on the same terms, each with the Wald interval of one protected effect (contract `engine-fairness-api` 1.0.0).
|
|
4
|
+
|
|
5
|
+
The design of both models is, in this order, the intercept, the `q` protected indicator columns and the `c` control
|
|
6
|
+
columns; `coefficients[0]` is the intercept, `coefficients[1 + j]` the effect of protected indicator `j`, and
|
|
7
|
+
`protected_effect(model, index)` reads the coefficient at `index`. Coefficients of the decision model are log-odds,
|
|
8
|
+
those of the rate-spread model are in the units of the rate spread (percentage points). The columns are standardised
|
|
9
|
+
internally for conditioning and the results are returned in the original units. The model is an approximation of
|
|
10
|
+
comparative evidence: the controls the caller could not supply (public HMDA has no credit score) are the caller's
|
|
11
|
+
statement, never imputed here.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import math
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
from scipy import stats
|
|
21
|
+
from scipy.special import expit
|
|
22
|
+
|
|
23
|
+
from riskvalidation.fairness._checks import binary, finite_matrix, unit_interval_open
|
|
24
|
+
from riskvalidation.lgd_ead._checks import vec
|
|
25
|
+
|
|
26
|
+
MAX_ITER = 100
|
|
27
|
+
STEP_TOL = 1e-11
|
|
28
|
+
DIVERGED = 50.0 # a standardised coefficient beyond this is complete or quasi-complete separation
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class DecisionModel:
|
|
33
|
+
"""Coefficients and standard errors (equal lengths; intercept first), the rows used and the convergence flag."""
|
|
34
|
+
|
|
35
|
+
coefficients: tuple[float, ...]
|
|
36
|
+
standard_errors: tuple[float, ...]
|
|
37
|
+
n: int
|
|
38
|
+
converged: bool
|
|
39
|
+
|
|
40
|
+
def __post_init__(self) -> None:
|
|
41
|
+
if len(self.coefficients) != len(self.standard_errors):
|
|
42
|
+
raise ValueError("coefficients and standard_errors must have the same length")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class Effect:
|
|
47
|
+
"""One coefficient with its odds ratio exp(coefficient) and its Wald interval at `level`."""
|
|
48
|
+
|
|
49
|
+
coefficient: float
|
|
50
|
+
odds_ratio: float
|
|
51
|
+
lo: float
|
|
52
|
+
hi: float
|
|
53
|
+
level: float
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _design(protected: np.ndarray, controls: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
57
|
+
"""The standardised non-intercept columns, their means and standard deviations; a constant column is refused."""
|
|
58
|
+
cols = np.hstack([protected, controls])
|
|
59
|
+
names = [f"protected[{j}]" for j in range(protected.shape[1])]
|
|
60
|
+
names += [f"controls[{j}]" for j in range(controls.shape[1])]
|
|
61
|
+
mu, sd = cols.mean(axis=0), cols.std(axis=0)
|
|
62
|
+
for name, s in zip(names, sd, strict=True):
|
|
63
|
+
if not s > 0.0:
|
|
64
|
+
raise ValueError(f"{name} is constant: the design matrix is rank deficient (collinear with the intercept)")
|
|
65
|
+
z = (cols - mu) / sd
|
|
66
|
+
x = np.hstack([np.ones((cols.shape[0], 1)), z])
|
|
67
|
+
if np.linalg.matrix_rank(x) < x.shape[1]:
|
|
68
|
+
raise ValueError("the design matrix of protected and controls is rank deficient: a column is collinear")
|
|
69
|
+
return x, mu, sd
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _to_original(beta: np.ndarray, cov: np.ndarray, mu: np.ndarray, sd: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
|
73
|
+
"""Coefficients and standard errors of the standardised design in the units of the original columns."""
|
|
74
|
+
k = beta.size
|
|
75
|
+
a = np.zeros((k, k))
|
|
76
|
+
a[0, 0] = 1.0
|
|
77
|
+
a[0, 1:] = -mu / sd
|
|
78
|
+
a[1:, 1:] = np.diag(1.0 / sd)
|
|
79
|
+
return a @ beta, np.sqrt(np.diag(a @ cov @ a.T))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _failed(k: int, n: int) -> DecisionModel:
|
|
83
|
+
nan = (math.nan,) * k
|
|
84
|
+
return DecisionModel(nan, nan, n, False)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _check_inputs(
|
|
88
|
+
protected: np.ndarray, controls: np.ndarray, n: int, rows_of: str
|
|
89
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
90
|
+
p = finite_matrix(protected, "protected", rows=n, rows_of=rows_of)
|
|
91
|
+
c = finite_matrix(controls, "controls", rows=n, rows_of=rows_of, allow_no_columns=True)
|
|
92
|
+
return p, c
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def fit_decision_model(decision: np.ndarray, protected: np.ndarray, controls: np.ndarray) -> DecisionModel:
|
|
96
|
+
"""Logistic model of the decision (1 favourable: approval by the C21 convention) on the intercept, the protected
|
|
97
|
+
indicators (n, q) and the controls (n, c), by Newton's method (iteratively reweighted least squares) with step
|
|
98
|
+
halving from the zero start. Converged False, with NaN coefficients and no exception, when the iteration does not
|
|
99
|
+
settle in 100 steps or the classes are completely separated. Raises `ValueError` naming the field for a decision
|
|
100
|
+
that is not 0 and 1 with both classes present, non-finite entries, rows that do not match and a rank-deficient
|
|
101
|
+
design."""
|
|
102
|
+
y = binary(decision, "decision")
|
|
103
|
+
if y.min() == y.max():
|
|
104
|
+
raise ValueError("decision must contain both 0 and 1")
|
|
105
|
+
n = y.shape[0]
|
|
106
|
+
prot, ctrl = _check_inputs(protected, controls, n, "decision")
|
|
107
|
+
x, mu, sd = _design(prot, ctrl)
|
|
108
|
+
k = x.shape[1]
|
|
109
|
+
yf = y.astype(float)
|
|
110
|
+
|
|
111
|
+
def loglik(b: np.ndarray) -> float:
|
|
112
|
+
z = x @ b
|
|
113
|
+
return float(np.sum(yf * z - np.logaddexp(0.0, z)))
|
|
114
|
+
|
|
115
|
+
beta = np.zeros(k)
|
|
116
|
+
ll = loglik(beta)
|
|
117
|
+
converged = False
|
|
118
|
+
for _ in range(MAX_ITER):
|
|
119
|
+
mu_hat = expit(x @ beta)
|
|
120
|
+
grad = x.T @ (yf - mu_hat)
|
|
121
|
+
info = x.T @ (x * (mu_hat * (1.0 - mu_hat))[:, None])
|
|
122
|
+
try:
|
|
123
|
+
step = np.linalg.solve(info, grad)
|
|
124
|
+
except np.linalg.LinAlgError:
|
|
125
|
+
return _failed(k, n)
|
|
126
|
+
scale, cand, new = 1.0, beta, ll
|
|
127
|
+
for _ in range(40):
|
|
128
|
+
cand = beta + scale * step
|
|
129
|
+
new = loglik(cand)
|
|
130
|
+
if new >= ll - 1e-14:
|
|
131
|
+
break
|
|
132
|
+
scale *= 0.5
|
|
133
|
+
else:
|
|
134
|
+
return _failed(k, n)
|
|
135
|
+
beta, ll = cand, new
|
|
136
|
+
if not np.all(np.isfinite(beta)) or np.max(np.abs(beta[1:])) > DIVERGED:
|
|
137
|
+
return _failed(k, n)
|
|
138
|
+
if np.max(np.abs(scale * step)) < STEP_TOL:
|
|
139
|
+
converged = True
|
|
140
|
+
break
|
|
141
|
+
if not converged:
|
|
142
|
+
return _failed(k, n)
|
|
143
|
+
mu_hat = expit(x @ beta)
|
|
144
|
+
info = x.T @ (x * (mu_hat * (1.0 - mu_hat))[:, None])
|
|
145
|
+
try:
|
|
146
|
+
cov = np.linalg.inv(info)
|
|
147
|
+
except np.linalg.LinAlgError:
|
|
148
|
+
return _failed(k, n)
|
|
149
|
+
coef, se = _to_original(beta, cov, mu, sd)
|
|
150
|
+
if not (np.all(np.isfinite(coef)) and np.all(np.isfinite(se))):
|
|
151
|
+
return _failed(k, n)
|
|
152
|
+
return DecisionModel(tuple(float(v) for v in coef), tuple(float(v) for v in se), n, True)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def fit_rate_spread_model(rate_spread: np.ndarray, protected: np.ndarray, controls: np.ndarray) -> DecisionModel:
|
|
156
|
+
"""Linear least squares of the rate spread (percentage points, for the loans that carry it) on the same terms.
|
|
157
|
+
Rows with a missing (NaN) rate spread are dropped and never imputed; the number of rows dropped is the rows given
|
|
158
|
+
minus `n` of the result. Infinite values, a rank-deficient design, non-finite controls and fewer rows than
|
|
159
|
+
coefficients plus one raise `ValueError` naming the field. Standard errors use the residual variance
|
|
160
|
+
RSS / (n - k)."""
|
|
161
|
+
rs = vec(rate_spread, "rate_spread", allow_nan=True)
|
|
162
|
+
prot, ctrl = _check_inputs(protected, controls, rs.shape[0], "rate_spread")
|
|
163
|
+
keep = ~np.isnan(rs)
|
|
164
|
+
if not keep.any():
|
|
165
|
+
raise ValueError("rate_spread has no non-missing rows")
|
|
166
|
+
rs, prot, ctrl = rs[keep], prot[keep], ctrl[keep]
|
|
167
|
+
n = rs.shape[0]
|
|
168
|
+
x, mu, sd = _design(prot, ctrl)
|
|
169
|
+
k = x.shape[1]
|
|
170
|
+
if n <= k:
|
|
171
|
+
raise ValueError(f"rate_spread has {n} usable rows for {k} coefficients: at least {k + 1} are needed")
|
|
172
|
+
beta, *_ = np.linalg.lstsq(x, rs, rcond=None)
|
|
173
|
+
resid = rs - x @ beta
|
|
174
|
+
sigma2 = float(resid @ resid) / (n - k)
|
|
175
|
+
cov = sigma2 * np.linalg.inv(x.T @ x)
|
|
176
|
+
coef, se = _to_original(beta, cov, mu, sd)
|
|
177
|
+
return DecisionModel(tuple(float(v) for v in coef), tuple(float(v) for v in se), n, True)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def protected_effect(model: DecisionModel, index: int, level: float = 0.95) -> Effect:
|
|
181
|
+
"""The coefficient at `index` of the model, where `index` is the position in `model.coefficients`: index 0 is the
|
|
182
|
+
intercept and is refused, protected indicator j is index 1 + j. It returns the coefficient, its odds ratio
|
|
183
|
+
exp(coefficient) and the Wald interval coefficient plus or minus z se at `level` (z the normal quantile of
|
|
184
|
+
1 - (1 - level) / 2). The odds ratio is meaningful for the decision model only; for the linear rate-spread model
|
|
185
|
+
read the coefficient."""
|
|
186
|
+
k = len(model.coefficients)
|
|
187
|
+
if isinstance(index, (int, np.integer)) and index == 0:
|
|
188
|
+
raise ValueError("index 0 is the intercept, not a protected indicator; indicator j is index 1 + j")
|
|
189
|
+
if not isinstance(index, (int, np.integer)) or not 1 <= index < k:
|
|
190
|
+
raise ValueError(f"index must lie in [1, {k - 1}] (indicator j is index 1 + j), got {index!r}")
|
|
191
|
+
lv = unit_interval_open(level, "level")
|
|
192
|
+
coef, se = model.coefficients[index], model.standard_errors[index]
|
|
193
|
+
if not (math.isfinite(coef) and math.isfinite(se) and se >= 0.0):
|
|
194
|
+
raise ValueError(f"the standard error of coefficient {index} is not finite: the model did not converge")
|
|
195
|
+
z = float(stats.norm.ppf(1.0 - (1.0 - lv) / 2.0))
|
|
196
|
+
with np.errstate(over="ignore"):
|
|
197
|
+
odds = float(np.exp(coef))
|
|
198
|
+
return Effect(float(coef), odds, float(coef - z * se), float(coef + z * se), lv)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
__all__ = ["DecisionModel", "Effect", "fit_decision_model", "fit_rate_spread_model", "protected_effect"]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Inference on the selection rates of two groups: the pooled two-proportion z-test and Fisher's exact test (dossier 06
|
|
2
|
+
C.7 l.1036-1039: smaller differences can matter when statistically significant; contract `engine-fairness-api` 1.0.0).
|
|
3
|
+
|
|
4
|
+
Group 1 minus group 2 throughout. The interval of a controlled effect comes from the decision model of
|
|
5
|
+
`fairness.conditional`, not from here.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
from scipy import stats
|
|
14
|
+
|
|
15
|
+
from riskvalidation.fairness._checks import count_pair
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class ZResult:
|
|
20
|
+
"""Pooled z statistic (group 1 minus group 2), two-sided p-value, the group sizes and a note (None unless the
|
|
21
|
+
pooled rate is 0 or 1)."""
|
|
22
|
+
|
|
23
|
+
statistic: float
|
|
24
|
+
p_value: float
|
|
25
|
+
n1: int
|
|
26
|
+
n2: int
|
|
27
|
+
note: str | None
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class FisherResult:
|
|
32
|
+
"""The sample odds ratio (None when undefined) and the two-sided exact p-value."""
|
|
33
|
+
|
|
34
|
+
odds_ratio: float | None
|
|
35
|
+
p_value: float
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def two_proportion_z_test(x1: int, n1: int, x2: int, n2: int) -> ZResult:
|
|
39
|
+
"""z = (x1/n1 - x2/n2) / sqrt(p (1 - p) (1/n1 + 1/n2)) with the pooled rate p = (x1 + x2) / (n1 + n2); the p-value
|
|
40
|
+
is 2 (1 - Phi(|z|)). A pooled rate of 0 or 1 has no variance: statistic 0, p-value 1 and a note."""
|
|
41
|
+
a, na = count_pair(x1, n1, "x1", "n1")
|
|
42
|
+
b, nb = count_pair(x2, n2, "x2", "n2")
|
|
43
|
+
pooled_num, pooled_den = a + b, na + nb
|
|
44
|
+
if pooled_num in (0, pooled_den):
|
|
45
|
+
return ZResult(0.0, 1.0, na, nb, "the pooled rate is 0 or 1: the statistic is 0 and the p-value 1")
|
|
46
|
+
p = pooled_num / pooled_den
|
|
47
|
+
se = math.sqrt(p * (1.0 - p) * (1.0 / na + 1.0 / nb))
|
|
48
|
+
z = (a / na - b / nb) / se
|
|
49
|
+
return ZResult(float(z), float(2.0 * stats.norm.sf(abs(z))), na, nb, None)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def fisher_exact_test(x1: int, n1: int, x2: int, n2: int) -> FisherResult:
|
|
53
|
+
"""Fisher's exact test on [[x1, n1 - x1], [x2, n2 - x2]]: the two-sided p-value of scipy.stats.fisher_exact and the
|
|
54
|
+
sample odds ratio x1 (n2 - x2) / ((n1 - x1) x2) (not the conditional maximum likelihood estimate); None when the
|
|
55
|
+
denominator is 0, where the ratio is infinite or 0/0."""
|
|
56
|
+
a, na = count_pair(x1, n1, "x1", "n1")
|
|
57
|
+
b, nb = count_pair(x2, n2, "x2", "n2")
|
|
58
|
+
res = stats.fisher_exact([[a, na - a], [b, nb - b]], alternative="two-sided")
|
|
59
|
+
denominator = (na - a) * b
|
|
60
|
+
odds = None if denominator == 0 else float(a * (nb - b) / denominator)
|
|
61
|
+
return FisherResult(odds, float(min(1.0, max(0.0, res.pvalue))))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
__all__ = ["FisherResult", "ZResult", "fisher_exact_test", "two_proportion_z_test"]
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"""Group fairness metrics: selection rates, demographic parity, equalized odds, the four-fifths flag, calibration and
|
|
2
|
+
AUC within groups (dossier 06 C.7 l.1029-1040; dossier 04 J.3 l.897-905; contract `engine-fairness-api` 1.0.0).
|
|
3
|
+
|
|
4
|
+
Every comparison of two rates is made on the counts, in exact integer arithmetic, and rounded once when it becomes a
|
|
5
|
+
float: a rate difference or ratio that sits exactly on a convention (a tolerance, the four-fifths line) is then decided
|
|
6
|
+
on the exact value and not on a float that missed it by one unit in the last place (60/100 over 75/100 is
|
|
7
|
+
0.7999999999999999 in IEEE doubles and exactly 4/5). The four-fifths line is an employment benchmark (29 CFR 1607.4(D))
|
|
8
|
+
and never a credit-law threshold. The outcome-conditioned metrics need an outcome and never run on a dataset that has
|
|
9
|
+
none (HMDA has no outcome). Groups are named by `str(label)` and ordered by their label (numeric labels numerically).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from collections.abc import Mapping
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from fractions import Fraction
|
|
17
|
+
|
|
18
|
+
import numpy as np
|
|
19
|
+
|
|
20
|
+
from riskvalidation.fairness._checks import (
|
|
21
|
+
InsufficientData,
|
|
22
|
+
binary,
|
|
23
|
+
group_labels,
|
|
24
|
+
outcome,
|
|
25
|
+
same_rows,
|
|
26
|
+
)
|
|
27
|
+
from riskvalidation.validation.calibration import pd_ece
|
|
28
|
+
from riskvalidation.validation.discrimination import disc_auc
|
|
29
|
+
|
|
30
|
+
FOUR_FIFTHS_THRESHOLD = 0.8
|
|
31
|
+
EMPLOYMENT_BENCHMARK = "employment benchmark"
|
|
32
|
+
HIGHEST_RATE_GROUP = "highest-rate-group"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class SelectionRate:
|
|
37
|
+
"""One group's size, favourable count and selection rate (rate == favourable / n)."""
|
|
38
|
+
|
|
39
|
+
group: str
|
|
40
|
+
n: int
|
|
41
|
+
favourable: int
|
|
42
|
+
rate: float
|
|
43
|
+
|
|
44
|
+
def __post_init__(self) -> None:
|
|
45
|
+
if self.n < 1:
|
|
46
|
+
raise ValueError("n must be at least 1")
|
|
47
|
+
if not 0 <= self.favourable <= self.n:
|
|
48
|
+
raise ValueError(f"favourable must lie in [0, n], got {self.favourable} of {self.n}")
|
|
49
|
+
if self.rate != self.favourable / self.n:
|
|
50
|
+
raise ValueError(f"rate must equal favourable / n = {self.favourable / self.n}, got {self.rate}")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def group_selection_rates(decision: np.ndarray, group: np.ndarray) -> dict[str, SelectionRate]:
|
|
54
|
+
"""The selection rate of each group present (a group with no rows is absent), ordered by label."""
|
|
55
|
+
d = binary(decision, "decision")
|
|
56
|
+
g = group_labels(group, d.shape[0], "group")
|
|
57
|
+
labels, inverse = np.unique(g, return_inverse=True)
|
|
58
|
+
n = np.bincount(inverse, minlength=labels.size)
|
|
59
|
+
fav = np.bincount(inverse, weights=d, minlength=labels.size).astype(np.int64)
|
|
60
|
+
return {
|
|
61
|
+
str(lab): SelectionRate(str(lab), int(n[i]), int(fav[i]), int(fav[i]) / int(n[i]))
|
|
62
|
+
for i, lab in enumerate(labels)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _need(rates: Mapping[str, SelectionRate], reference: str | None = None) -> None:
|
|
67
|
+
if not isinstance(rates, Mapping) or not rates:
|
|
68
|
+
raise ValueError("rates must be a non-empty mapping of group to SelectionRate")
|
|
69
|
+
for lab, s in rates.items():
|
|
70
|
+
if not isinstance(s, SelectionRate):
|
|
71
|
+
raise ValueError(f"rates[{lab!r}] must be a SelectionRate")
|
|
72
|
+
if reference is not None and reference not in rates:
|
|
73
|
+
raise ValueError(f"reference {reference!r} is not one of the groups {list(rates)}")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def demographic_parity_difference(rates: Mapping[str, SelectionRate], reference: str) -> dict[str, float]:
|
|
77
|
+
"""Rate of each group minus the reference rate, computed on the counts (exact) and rounded once."""
|
|
78
|
+
_need(rates, reference)
|
|
79
|
+
r = rates[reference]
|
|
80
|
+
return {g: (s.favourable * r.n - r.favourable * s.n) / (s.n * r.n) for g, s in rates.items()}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def demographic_parity_ratio(rates: Mapping[str, SelectionRate], reference: str) -> dict[str, float | None]:
|
|
84
|
+
"""Rate of each group over the reference rate; None for every group when the reference rate is 0."""
|
|
85
|
+
_need(rates, reference)
|
|
86
|
+
r = rates[reference]
|
|
87
|
+
if r.favourable == 0:
|
|
88
|
+
return {g: None for g in rates}
|
|
89
|
+
return {g: (s.favourable * r.n) / (s.n * r.favourable) for g, s in rates.items()}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def adverse_impact_ratio(rates: Mapping[str, SelectionRate], reference: str) -> dict[str, float | None]:
|
|
93
|
+
"""The protected group's selection rate over the chosen reference group's: the parity ratio. It is not the
|
|
94
|
+
four-fifths ratio, which is taken against the highest-rate group (`four_fifths_flag`)."""
|
|
95
|
+
return demographic_parity_ratio(rates, reference)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass(frozen=True)
|
|
99
|
+
class FourFifths:
|
|
100
|
+
"""The four-fifths flag per group against the highest-rate group, labelled an employment benchmark."""
|
|
101
|
+
|
|
102
|
+
highest_rate_group: str
|
|
103
|
+
ratio_to_highest: dict[str, float | None]
|
|
104
|
+
flag: dict[str, bool]
|
|
105
|
+
threshold: float = FOUR_FIFTHS_THRESHOLD
|
|
106
|
+
label: str = EMPLOYMENT_BENCHMARK
|
|
107
|
+
is_credit_law_threshold: bool = False
|
|
108
|
+
compared_with: str = HIGHEST_RATE_GROUP
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def four_fifths_flag(rates: Mapping[str, SelectionRate]) -> FourFifths:
|
|
112
|
+
"""A group is flagged when 5 x_g n_h < 4 x_h n_g, with x the favourable count and n the size, h the highest-rate
|
|
113
|
+
group (29 CFR 1607.4(D); dossier 06 C.7 l.1034-1035). The highest-rate group is the first of the groups of the
|
|
114
|
+
highest rate, in the order of `rates`. The ratio is for display; the flag never reads it."""
|
|
115
|
+
_need(rates)
|
|
116
|
+
items = list(rates.items())
|
|
117
|
+
top_name, top = items[0]
|
|
118
|
+
for name, s in items[1:]:
|
|
119
|
+
if s.favourable * top.n > top.favourable * s.n: # strictly higher rate, by cross-multiplication
|
|
120
|
+
top_name, top = name, s
|
|
121
|
+
ratio: dict[str, float | None] = {}
|
|
122
|
+
flag: dict[str, bool] = {}
|
|
123
|
+
for name, s in items:
|
|
124
|
+
ratio[name] = None if top.favourable == 0 else (s.favourable * top.n) / (s.n * top.favourable)
|
|
125
|
+
flag[name] = 5 * s.favourable * top.n < 4 * top.favourable * s.n
|
|
126
|
+
return FourFifths(highest_rate_group=top_name, ratio_to_highest=ratio, flag=flag)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# ---------------------------------------------------------------------------------------------------------------
|
|
130
|
+
# Outcome-conditioned metrics: they need an outcome
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass(frozen=True)
|
|
134
|
+
class _Confusion:
|
|
135
|
+
positives: int
|
|
136
|
+
negatives: int
|
|
137
|
+
true_positives: int
|
|
138
|
+
false_positives: int
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _confusion(y: np.ndarray, d: np.ndarray, g: np.ndarray) -> dict[str, _Confusion]:
|
|
142
|
+
labels, codes = np.unique(g, return_inverse=True)
|
|
143
|
+
k = labels.size
|
|
144
|
+
|
|
145
|
+
def count(mask: np.ndarray) -> np.ndarray:
|
|
146
|
+
return np.bincount(codes, weights=mask, minlength=k).astype(np.int64)
|
|
147
|
+
|
|
148
|
+
pos, neg = count(y == 1), count(y == 0)
|
|
149
|
+
tp, fp = count((y == 1) & (d == 1)), count((y == 0) & (d == 1))
|
|
150
|
+
return {str(lab): _Confusion(int(pos[i]), int(neg[i]), int(tp[i]), int(fp[i])) for i, lab in enumerate(labels)}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def outcome_rates(y: np.ndarray, decision: np.ndarray, group: np.ndarray) -> dict[str, dict[str, float | None]]:
|
|
154
|
+
"""The true positive rate and false positive rate of each group (None where the class is absent)."""
|
|
155
|
+
yy = outcome(y, "y")
|
|
156
|
+
d = binary(decision, "decision")
|
|
157
|
+
same_rows(yy, "y", d, "decision")
|
|
158
|
+
g = group_labels(group, yy.shape[0], "group")
|
|
159
|
+
return {
|
|
160
|
+
lab: {
|
|
161
|
+
"tpr": None if c.positives == 0 else c.true_positives / c.positives,
|
|
162
|
+
"fpr": None if c.negatives == 0 else c.false_positives / c.negatives,
|
|
163
|
+
}
|
|
164
|
+
for lab, c in _confusion(yy, d, g).items()
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _gaps(
|
|
169
|
+
y: np.ndarray, decision: np.ndarray, group: np.ndarray, *, need_negatives: bool
|
|
170
|
+
) -> tuple[float, float | None]:
|
|
171
|
+
yy = outcome(y, "y")
|
|
172
|
+
d = binary(decision, "decision")
|
|
173
|
+
same_rows(yy, "y", d, "decision")
|
|
174
|
+
g = group_labels(group, yy.shape[0], "group")
|
|
175
|
+
conf = _confusion(yy, d, g)
|
|
176
|
+
if len(conf) < 2:
|
|
177
|
+
raise InsufficientData("at least two groups are needed to compare their error rates")
|
|
178
|
+
for lab, c in conf.items():
|
|
179
|
+
if c.positives == 0:
|
|
180
|
+
raise InsufficientData(f"group {lab!r} has no positives (y = 1): the true positive rate is undefined")
|
|
181
|
+
if need_negatives and c.negatives == 0:
|
|
182
|
+
raise InsufficientData(f"group {lab!r} lacks one of both classes: needs both classes of y")
|
|
183
|
+
tprs = [Fraction(c.true_positives, c.positives) for c in conf.values()]
|
|
184
|
+
tpr_gap = float(max(tprs) - min(tprs))
|
|
185
|
+
if not need_negatives:
|
|
186
|
+
return tpr_gap, None
|
|
187
|
+
fprs = [Fraction(c.false_positives, c.negatives) for c in conf.values()]
|
|
188
|
+
return tpr_gap, float(max(fprs) - min(fprs))
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def equalized_odds_difference(y: np.ndarray, decision: np.ndarray, group: np.ndarray) -> float:
|
|
192
|
+
"""The larger of the true positive rate gap and the false positive rate gap across groups (Hardt, Price and
|
|
193
|
+
Srebro 2016). Needs both classes of y in every group (`InsufficientData`, a `ValueError`, otherwise) and an outcome:
|
|
194
|
+
never run on HMDA."""
|
|
195
|
+
tpr_gap, fpr_gap = _gaps(y, decision, group, need_negatives=True)
|
|
196
|
+
return max(tpr_gap, fpr_gap) # type: ignore[type-var]
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def equal_opportunity_difference(y: np.ndarray, decision: np.ndarray, group: np.ndarray) -> float:
|
|
200
|
+
"""The true positive rate gap across groups. Needs positives in every group and an outcome: never run on HMDA."""
|
|
201
|
+
return _gaps(y, decision, group, need_negatives=False)[0]
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def calibration_within_groups(p: np.ndarray, y: np.ndarray, group: np.ndarray, n_bins: int = 10) -> dict[str, float]:
|
|
205
|
+
"""The expected calibration error of PD `p` against the default indicator `y` within each group, by the engine's
|
|
206
|
+
`pd_ece` (quantile bins). A group with fewer rows than `n_bins` is not evaluated and is absent from the result; the
|
|
207
|
+
caller counts it as the groups of `group` minus the keys returned."""
|
|
208
|
+
yy = outcome(y, "y")
|
|
209
|
+
pv = np.array(p, dtype=float)
|
|
210
|
+
if pv.ndim != 1 or pv.shape[0] != yy.shape[0]:
|
|
211
|
+
raise ValueError(f"p must be one-dimensional with the {yy.shape[0]} rows of y, got shape {pv.shape}")
|
|
212
|
+
if not np.all(np.isfinite(pv)) or np.any(pv <= 0.0) or np.any(pv >= 1.0):
|
|
213
|
+
raise ValueError("p must lie strictly between 0 and 1")
|
|
214
|
+
if not isinstance(n_bins, (int, np.integer)) or n_bins < 1:
|
|
215
|
+
raise ValueError(f"n_bins must be a positive integer, got {n_bins!r}")
|
|
216
|
+
g = group_labels(group, yy.shape[0], "group")
|
|
217
|
+
out: dict[str, float] = {}
|
|
218
|
+
for lab in np.unique(g):
|
|
219
|
+
m = g == lab
|
|
220
|
+
if int(m.sum()) < n_bins:
|
|
221
|
+
continue
|
|
222
|
+
out[str(lab)] = float(pd_ece(yy[m], pv[m], bins=int(n_bins)).metric)
|
|
223
|
+
return out
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def auc_by_group(score: np.ndarray, y: np.ndarray, group: np.ndarray) -> dict[str, float]:
|
|
227
|
+
"""The AUC of `score` within each group, by the engine's `disc_auc` (y = 1 is the class the score ranks higher). A
|
|
228
|
+
group with a single class is omitted; the caller counts it as the groups of `group` minus the keys returned."""
|
|
229
|
+
yy = outcome(y, "y")
|
|
230
|
+
s = np.array(score, dtype=float)
|
|
231
|
+
if s.ndim != 1 or s.shape[0] != yy.shape[0]:
|
|
232
|
+
raise ValueError(f"score must be one-dimensional with the {yy.shape[0]} rows of y, got shape {s.shape}")
|
|
233
|
+
if not np.all(np.isfinite(s)):
|
|
234
|
+
raise ValueError("score contains non-finite values")
|
|
235
|
+
g = group_labels(group, yy.shape[0], "group")
|
|
236
|
+
out: dict[str, float] = {}
|
|
237
|
+
for lab in np.unique(g):
|
|
238
|
+
m = g == lab
|
|
239
|
+
ym = yy[m]
|
|
240
|
+
if ym.min() == ym.max():
|
|
241
|
+
continue
|
|
242
|
+
out[str(lab)] = float(disc_auc(s[m], ym).metric)
|
|
243
|
+
return out
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
__all__ = [
|
|
247
|
+
"EMPLOYMENT_BENCHMARK",
|
|
248
|
+
"FOUR_FIFTHS_THRESHOLD",
|
|
249
|
+
"FourFifths",
|
|
250
|
+
"SelectionRate",
|
|
251
|
+
"adverse_impact_ratio",
|
|
252
|
+
"auc_by_group",
|
|
253
|
+
"calibration_within_groups",
|
|
254
|
+
"demographic_parity_difference",
|
|
255
|
+
"demographic_parity_ratio",
|
|
256
|
+
"equal_opportunity_difference",
|
|
257
|
+
"equalized_odds_difference",
|
|
258
|
+
"four_fifths_flag",
|
|
259
|
+
"group_selection_rates",
|
|
260
|
+
"outcome_rates",
|
|
261
|
+
]
|