riskaudit 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- riskaudit/__init__.py +4 -0
- riskaudit/_config.py +12 -0
- riskaudit/audit/__init__.py +24 -0
- riskaudit/audit/_common.py +107 -0
- riskaudit/audit/ablation.py +99 -0
- riskaudit/audit/capture.py +83 -0
- riskaudit/audit/curves.py +101 -0
- riskaudit/audit/lift.py +98 -0
- riskaudit/audit/reclassification.py +54 -0
- riskaudit/audit/report.py +85 -0
- riskaudit/audit/rtm.py +108 -0
- riskaudit/etl/__init__.py +0 -0
- riskaudit/etl/dictionary.yml +99 -0
- riskaudit/etl/download.py +71 -0
- riskaudit/etl/meps.py +107 -0
- riskaudit/features.py +65 -0
- riskaudit/models.py +108 -0
- riskaudit-0.1.1.dist-info/METADATA +247 -0
- riskaudit-0.1.1.dist-info/RECORD +21 -0
- riskaudit-0.1.1.dist-info/WHEEL +4 -0
- riskaudit-0.1.1.dist-info/licenses/LICENSE +21 -0
riskaudit/__init__.py
ADDED
riskaudit/_config.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
SEED = 2026
|
|
5
|
+
|
|
6
|
+
PROJECT_ROOT = Path(os.environ.get("RISKAUDIT_ROOT", Path.cwd()))
|
|
7
|
+
DATA_DIR = PROJECT_ROOT / "data"
|
|
8
|
+
ARTIFACTS_DIR = PROJECT_ROOT / "artifacts"
|
|
9
|
+
|
|
10
|
+
RAW_DIR = DATA_DIR / "raw"
|
|
11
|
+
PROCESSED_DIR = DATA_DIR / "processed"
|
|
12
|
+
SYNTHETIC_DIR = DATA_DIR / "synthetic"
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from riskaudit.audit._common import SurveyDesign
|
|
2
|
+
from riskaudit.audit.ablation import AblationResult, ablation
|
|
3
|
+
from riskaudit.audit.capture import CaptureResult, top_k_capture
|
|
4
|
+
from riskaudit.audit.curves import CurveResult, label_choice_curve
|
|
5
|
+
from riskaudit.audit.lift import LiftResult, incremental_lift
|
|
6
|
+
from riskaudit.audit.reclassification import reclassification
|
|
7
|
+
from riskaudit.audit.report import audit_report
|
|
8
|
+
from riskaudit.audit.rtm import RTMResult, regression_to_mean
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"AblationResult",
|
|
12
|
+
"CaptureResult",
|
|
13
|
+
"CurveResult",
|
|
14
|
+
"LiftResult",
|
|
15
|
+
"RTMResult",
|
|
16
|
+
"SurveyDesign",
|
|
17
|
+
"ablation",
|
|
18
|
+
"audit_report",
|
|
19
|
+
"incremental_lift",
|
|
20
|
+
"label_choice_curve",
|
|
21
|
+
"reclassification",
|
|
22
|
+
"regression_to_mean",
|
|
23
|
+
"top_k_capture",
|
|
24
|
+
]
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
from numpy.typing import ArrayLike
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class SurveyDesign:
|
|
10
|
+
"""Stratum and PSU labels for design-based (cluster) bootstrap resampling.
|
|
11
|
+
|
|
12
|
+
Pass one to the CI-bearing audit functions to resample PSUs within strata
|
|
13
|
+
(VARSTR/VARPSU) instead of individual rows, so the interval reflects the
|
|
14
|
+
survey design. Strata with a single PSU are held fixed (their variance
|
|
15
|
+
contribution is zero), and a warning is emitted when that happens.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
strata: np.ndarray
|
|
19
|
+
psu: np.ndarray
|
|
20
|
+
|
|
21
|
+
def __post_init__(self):
|
|
22
|
+
self.strata = np.asarray(self.strata)
|
|
23
|
+
self.psu = np.asarray(self.psu)
|
|
24
|
+
singles = sum(
|
|
25
|
+
np.unique(self.psu[self.strata == s]).size < 2 for s in np.unique(self.strata)
|
|
26
|
+
)
|
|
27
|
+
if singles:
|
|
28
|
+
warnings.warn(
|
|
29
|
+
f"{singles} stratum(s) have a single PSU and are held fixed; their variance "
|
|
30
|
+
"contribution is zero, so the CI is biased low. Consider collapsing strata.",
|
|
31
|
+
stacklevel=2,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def check_inputs(**arrays: ArrayLike) -> None:
|
|
36
|
+
"""Validate that public inputs are finite and equally long (fail at the boundary)."""
|
|
37
|
+
n = None
|
|
38
|
+
for name, a in arrays.items():
|
|
39
|
+
arr = np.asarray(a, dtype=float)
|
|
40
|
+
if not np.isfinite(arr).all():
|
|
41
|
+
raise ValueError(f"{name} contains NaN or inf")
|
|
42
|
+
if n is None:
|
|
43
|
+
n = arr.shape[0]
|
|
44
|
+
elif arr.shape[0] != n:
|
|
45
|
+
raise ValueError(f"length mismatch: {name} has {arr.shape[0]}, expected {n}")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def to_float(x: ArrayLike) -> np.ndarray:
|
|
49
|
+
return np.asarray(x, dtype=float)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def weights_or_ones(weights: ArrayLike | None, n: int) -> np.ndarray:
|
|
53
|
+
return np.ones(n) if weights is None else np.asarray(weights, dtype=float)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def topk_mask(scores: np.ndarray, weights: np.ndarray, k: float) -> np.ndarray:
|
|
57
|
+
"""Boolean mask of the highest-scoring units whose weight first reaches ``k``.
|
|
58
|
+
|
|
59
|
+
Ties are broken by row order (stable sort), so with discretized scores the
|
|
60
|
+
exact top-``k`` membership can depend on the input order.
|
|
61
|
+
"""
|
|
62
|
+
order = np.argsort(-scores, kind="stable")
|
|
63
|
+
cumw = np.cumsum(weights[order])
|
|
64
|
+
j = int(np.searchsorted(cumw, k * cumw[-1]))
|
|
65
|
+
mask = np.zeros(scores.shape[0], dtype=bool)
|
|
66
|
+
mask[order[: j + 1]] = True
|
|
67
|
+
return mask
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def wmean(x: np.ndarray, w: np.ndarray) -> float:
|
|
71
|
+
return float(np.sum(w * x) / np.sum(w))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def weighted_capture(scores: np.ndarray, need: np.ndarray, weights: np.ndarray, k: float) -> float:
|
|
75
|
+
"""Weighted share of total need falling in the top-``k`` (by weight) of ``scores``."""
|
|
76
|
+
mask = topk_mask(scores, weights, k)
|
|
77
|
+
wn = weights * need
|
|
78
|
+
return float(wn[mask].sum() / wn.sum())
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _cluster_indices(rng, design: SurveyDesign) -> np.ndarray:
|
|
82
|
+
parts = []
|
|
83
|
+
for st in np.unique(design.strata):
|
|
84
|
+
rows = np.where(design.strata == st)[0]
|
|
85
|
+
psus = np.unique(design.psu[rows])
|
|
86
|
+
if psus.size < 2:
|
|
87
|
+
parts.append(rows)
|
|
88
|
+
continue
|
|
89
|
+
for c in rng.choice(psus, size=psus.size, replace=True):
|
|
90
|
+
parts.append(rows[design.psu[rows] == c])
|
|
91
|
+
return np.concatenate(parts)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def boot_ci(stat, n: int, n_boot: int, seed: int, design: SurveyDesign | None = None):
|
|
95
|
+
"""Percentile CI from a seeded bootstrap of ``stat``.
|
|
96
|
+
|
|
97
|
+
Resamples rows uniformly, or PSUs within strata when a ``design`` is given.
|
|
98
|
+
The seed is fixed, so CIs of different metrics in the same report share the
|
|
99
|
+
same resamples (deterministic, but their sampling errors are correlated).
|
|
100
|
+
"""
|
|
101
|
+
rng = np.random.default_rng(seed)
|
|
102
|
+
|
|
103
|
+
def draw():
|
|
104
|
+
return rng.integers(0, n, n) if design is None else _cluster_indices(rng, design)
|
|
105
|
+
|
|
106
|
+
vals = np.array([stat(draw()) for _ in range(n_boot)])
|
|
107
|
+
return float(np.nanpercentile(vals, 2.5)), float(np.nanpercentile(vals, 97.5))
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from numpy.typing import ArrayLike
|
|
7
|
+
from sklearn.metrics import r2_score, roc_auc_score
|
|
8
|
+
from sklearn.model_selection import KFold
|
|
9
|
+
|
|
10
|
+
from riskaudit._config import SEED
|
|
11
|
+
from riskaudit.audit._common import to_float, weighted_capture, weights_or_ones
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class AblationResult:
|
|
16
|
+
global_delta: pd.DataFrame
|
|
17
|
+
capture_delta: pd.DataFrame
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _perf(y: np.ndarray, pred: np.ndarray, w: np.ndarray) -> float:
|
|
21
|
+
if np.isin(np.unique(y), [0, 1]).all():
|
|
22
|
+
return float(roc_auc_score(y, pred, sample_weight=w))
|
|
23
|
+
return float(r2_score(y, pred, sample_weight=w))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def ablation(
|
|
27
|
+
fit_fn: Callable[[pd.DataFrame, ArrayLike], object],
|
|
28
|
+
X: pd.DataFrame,
|
|
29
|
+
y: ArrayLike,
|
|
30
|
+
feature_groups: Mapping[str, Sequence[str]],
|
|
31
|
+
need: ArrayLike | None = None,
|
|
32
|
+
k: float = 0.10,
|
|
33
|
+
weights: ArrayLike | None = None,
|
|
34
|
+
cv: int = 5,
|
|
35
|
+
seed: int = SEED,
|
|
36
|
+
) -> AblationResult:
|
|
37
|
+
r"""Refit without each feature group; contrast global loss with capture loss.
|
|
38
|
+
|
|
39
|
+
Predictions are **cross-fitted** (``cv``-fold): every row is scored by a model
|
|
40
|
+
that did not train on it, so the deltas reflect generalization, not in-sample
|
|
41
|
+
memorization. For each group :math:`g` the model is refit on :math:`X` with the
|
|
42
|
+
columns of :math:`g` removed and compared to the full-feature model on:
|
|
43
|
+
|
|
44
|
+
- :math:`\Delta\text{global}` — change in cross-fitted performance (R\ :sup:`2`
|
|
45
|
+
or AUC) predicting the target :math:`y`;
|
|
46
|
+
- :math:`\Delta\text{capture}` — change in weighted top-``k`` capture of
|
|
47
|
+
``need`` (the independent need measure; defaults to the target ``y``).
|
|
48
|
+
|
|
49
|
+
The expected finding for a spend target with ``need`` set to measured distress:
|
|
50
|
+
dropping the mental-health group barely moves :math:`\Delta\text{global}` yet
|
|
51
|
+
collapses :math:`\Delta\text{capture}`.
|
|
52
|
+
|
|
53
|
+
Parameters
|
|
54
|
+
----------
|
|
55
|
+
fit_fn : callable ``(X, y) -> fitted model with .predict``
|
|
56
|
+
Trains one model; called once per fold per ablation with the same settings.
|
|
57
|
+
X : pandas.DataFrame of shape (n, p)
|
|
58
|
+
Feature matrix.
|
|
59
|
+
y : array-like of shape (n,)
|
|
60
|
+
Target the model is trained on.
|
|
61
|
+
feature_groups : mapping of str to sequence of str
|
|
62
|
+
Group name to its column names in ``X``.
|
|
63
|
+
need : array-like of shape (n,), optional
|
|
64
|
+
Independent need measure for the capture delta; defaults to ``y``.
|
|
65
|
+
k : float, default 0.10
|
|
66
|
+
Top fraction for the capture comparison.
|
|
67
|
+
weights : array-like of shape (n,), optional
|
|
68
|
+
Survey weights; all ones when omitted.
|
|
69
|
+
cv : int, default 5
|
|
70
|
+
Number of cross-fitting folds.
|
|
71
|
+
seed : int, default SEED
|
|
72
|
+
Fold shuffle seed.
|
|
73
|
+
|
|
74
|
+
Returns
|
|
75
|
+
-------
|
|
76
|
+
AblationResult
|
|
77
|
+
Per-group global and capture deltas.
|
|
78
|
+
"""
|
|
79
|
+
w = weights_or_ones(weights, len(X))
|
|
80
|
+
yv = np.asarray(y)
|
|
81
|
+
nd = to_float(y if need is None else need)
|
|
82
|
+
folds = list(KFold(max(2, min(cv, len(X))), shuffle=True, random_state=seed).split(X))
|
|
83
|
+
|
|
84
|
+
def cross_fit(xd: pd.DataFrame) -> np.ndarray:
|
|
85
|
+
pred = np.empty(len(xd))
|
|
86
|
+
for tr, te in folds:
|
|
87
|
+
model = fit_fn(xd.iloc[tr], yv[tr])
|
|
88
|
+
pred[te] = np.asarray(model.predict(xd.iloc[te]), dtype=float)
|
|
89
|
+
return pred
|
|
90
|
+
|
|
91
|
+
p0 = cross_fit(X)
|
|
92
|
+
g0, c0 = _perf(yv, p0, w), weighted_capture(p0, nd, w, k)
|
|
93
|
+
|
|
94
|
+
gd, cd = {}, {}
|
|
95
|
+
for name, cols in feature_groups.items():
|
|
96
|
+
pg = cross_fit(X.drop(columns=list(cols)))
|
|
97
|
+
gd[name] = g0 - _perf(yv, pg, w)
|
|
98
|
+
cd[name] = c0 - weighted_capture(pg, nd, w, k)
|
|
99
|
+
return AblationResult(pd.DataFrame({"delta": gd}), pd.DataFrame({"delta": cd}))
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
from numpy.typing import ArrayLike
|
|
5
|
+
|
|
6
|
+
from riskaudit._config import SEED
|
|
7
|
+
from riskaudit.audit._common import (
|
|
8
|
+
SurveyDesign,
|
|
9
|
+
boot_ci,
|
|
10
|
+
check_inputs,
|
|
11
|
+
to_float,
|
|
12
|
+
weighted_capture,
|
|
13
|
+
weights_or_ones,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class CaptureResult:
|
|
19
|
+
value: float
|
|
20
|
+
ci: tuple[float, float]
|
|
21
|
+
k: float
|
|
22
|
+
baseline: float # capture expected from a random score (= k)
|
|
23
|
+
oracle: float # capture from ranking by need itself (the achievable ceiling)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def top_k_capture(
|
|
27
|
+
scores: ArrayLike,
|
|
28
|
+
need: ArrayLike,
|
|
29
|
+
k: float = 0.10,
|
|
30
|
+
weights: ArrayLike | None = None,
|
|
31
|
+
n_boot: int = 1000,
|
|
32
|
+
design: SurveyDesign | None = None,
|
|
33
|
+
) -> CaptureResult:
|
|
34
|
+
r"""Weighted share of total need captured by the top-``k`` of ``scores``.
|
|
35
|
+
|
|
36
|
+
Units are ranked by ``scores`` descending and the top ``k`` fraction *by
|
|
37
|
+
weight* is selected. The statistic is the weighted share of total need that
|
|
38
|
+
falls in that group,
|
|
39
|
+
|
|
40
|
+
.. math::
|
|
41
|
+
C_k \;=\; \frac{\sum_{i \in T_k} w_i\, n_i}{\sum_i w_i\, n_i},
|
|
42
|
+
|
|
43
|
+
where :math:`T_k` is the highest-scoring set whose cumulative weight first
|
|
44
|
+
reaches a fraction :math:`k` of the total weight, :math:`n_i` is need and
|
|
45
|
+
:math:`w_i` the survey weight. A model that ranks by true need attains a
|
|
46
|
+
high :math:`C_k`; the gap between a spend-ranked and a need-ranked model is
|
|
47
|
+
the label-choice cost this toolkit measures.
|
|
48
|
+
|
|
49
|
+
Parameters
|
|
50
|
+
----------
|
|
51
|
+
scores : array-like of shape (n,)
|
|
52
|
+
Model score per unit; higher means higher predicted priority.
|
|
53
|
+
need : array-like of shape (n,)
|
|
54
|
+
Independent measure of realized need on the same units.
|
|
55
|
+
k : float, default 0.10
|
|
56
|
+
Top fraction, by weight, to select.
|
|
57
|
+
weights : array-like of shape (n,), optional
|
|
58
|
+
Survey weights; treated as all ones when omitted.
|
|
59
|
+
n_boot : int, default 1000
|
|
60
|
+
Weighted bootstrap resamples for the confidence interval.
|
|
61
|
+
design : SurveyDesign, optional
|
|
62
|
+
When given, the CI resamples PSUs within strata (VARSTR/VARPSU) instead
|
|
63
|
+
of rows, for a design-based interval.
|
|
64
|
+
|
|
65
|
+
Returns
|
|
66
|
+
-------
|
|
67
|
+
CaptureResult
|
|
68
|
+
``value``, its 95% ``ci``, the random-score ``baseline`` (= ``k``), and
|
|
69
|
+
the ``oracle`` (ranking by ``need`` itself). Read ``value`` against both:
|
|
70
|
+
a raw capture is uninterpretable without its floor and ceiling.
|
|
71
|
+
"""
|
|
72
|
+
s = to_float(scores)
|
|
73
|
+
nd = to_float(need)
|
|
74
|
+
w = weights_or_ones(weights, s.shape[0])
|
|
75
|
+
check_inputs(scores=s, need=nd, weights=w)
|
|
76
|
+
|
|
77
|
+
def stat(idx: np.ndarray) -> float:
|
|
78
|
+
return weighted_capture(s[idx], nd[idx], w[idx], k)
|
|
79
|
+
|
|
80
|
+
value = stat(np.arange(s.shape[0]))
|
|
81
|
+
ci = boot_ci(stat, s.shape[0], n_boot, SEED, design)
|
|
82
|
+
oracle = weighted_capture(nd, nd, w, k)
|
|
83
|
+
return CaptureResult(value, ci, k, k, oracle)
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
from numpy.typing import ArrayLike
|
|
5
|
+
|
|
6
|
+
from riskaudit._config import SEED
|
|
7
|
+
from riskaudit.audit._common import (
|
|
8
|
+
SurveyDesign,
|
|
9
|
+
_cluster_indices,
|
|
10
|
+
check_inputs,
|
|
11
|
+
to_float,
|
|
12
|
+
weights_or_ones,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class CurveResult:
|
|
18
|
+
percentile: np.ndarray
|
|
19
|
+
need_mean: np.ndarray
|
|
20
|
+
need_lo: np.ndarray
|
|
21
|
+
need_hi: np.ndarray
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def label_choice_curve(
|
|
25
|
+
scores: ArrayLike,
|
|
26
|
+
need: ArrayLike,
|
|
27
|
+
weights: ArrayLike | None = None,
|
|
28
|
+
bins: int = 20,
|
|
29
|
+
design: SurveyDesign | None = None,
|
|
30
|
+
n_boot: int = 1000,
|
|
31
|
+
) -> CurveResult:
|
|
32
|
+
r"""Mean observed need across score percentiles, Obermeyer-style.
|
|
33
|
+
|
|
34
|
+
Units are grouped into ``bins`` equal-weight strata of the score, and each
|
|
35
|
+
stratum's weighted mean need is reported,
|
|
36
|
+
|
|
37
|
+
.. math::
|
|
38
|
+
\bar{n}_b \;=\; \frac{\sum_{i \in b} w_i\, n_i}{\sum_{i \in b} w_i}.
|
|
39
|
+
|
|
40
|
+
A model that ranks by true need yields a clean monotone rise; label-choice
|
|
41
|
+
bias shows as units with high need sitting at low score percentiles. The band
|
|
42
|
+
is a percentile bootstrap — a design-based cluster bootstrap when ``design``
|
|
43
|
+
is given, otherwise a row bootstrap.
|
|
44
|
+
|
|
45
|
+
Parameters
|
|
46
|
+
----------
|
|
47
|
+
scores : array-like of shape (n,)
|
|
48
|
+
Model score per unit.
|
|
49
|
+
need : array-like of shape (n,)
|
|
50
|
+
Observed need on the same units.
|
|
51
|
+
weights : array-like of shape (n,), optional
|
|
52
|
+
Survey weights; all ones when omitted.
|
|
53
|
+
bins : int, default 20
|
|
54
|
+
Number of equal-weight score strata.
|
|
55
|
+
design : SurveyDesign, optional
|
|
56
|
+
When given, the band resamples PSUs within strata (VARSTR/VARPSU).
|
|
57
|
+
n_boot : int, default 1000
|
|
58
|
+
Bootstrap resamples for the band.
|
|
59
|
+
|
|
60
|
+
Returns
|
|
61
|
+
-------
|
|
62
|
+
CurveResult
|
|
63
|
+
Per-bin score percentile, mean need, and its 95% band.
|
|
64
|
+
"""
|
|
65
|
+
s = to_float(scores)
|
|
66
|
+
nd = to_float(need)
|
|
67
|
+
w = weights_or_ones(weights, s.shape[0])
|
|
68
|
+
check_inputs(scores=s, need=nd, weights=w)
|
|
69
|
+
|
|
70
|
+
order = np.argsort(s, kind="stable")
|
|
71
|
+
edges = np.linspace(0, w[order].sum(), bins + 1)
|
|
72
|
+
label = np.empty(s.shape[0], dtype=int)
|
|
73
|
+
label[order] = np.clip(
|
|
74
|
+
np.searchsorted(edges, np.cumsum(w[order]), side="left") - 1, 0, bins - 1
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
def bin_means(idx: np.ndarray) -> np.ndarray:
|
|
78
|
+
out = np.full(bins, np.nan)
|
|
79
|
+
lab = label[idx]
|
|
80
|
+
for b in range(bins):
|
|
81
|
+
sel = idx[lab == b]
|
|
82
|
+
if sel.size:
|
|
83
|
+
out[b] = np.sum(w[sel] * nd[sel]) / np.sum(w[sel])
|
|
84
|
+
return out
|
|
85
|
+
|
|
86
|
+
mean = bin_means(np.arange(s.shape[0]))
|
|
87
|
+
pct = (np.arange(bins) + 0.5) / bins * 100
|
|
88
|
+
|
|
89
|
+
rng = np.random.default_rng(SEED)
|
|
90
|
+
|
|
91
|
+
def draw():
|
|
92
|
+
return (
|
|
93
|
+
rng.integers(0, s.shape[0], s.shape[0])
|
|
94
|
+
if design is None
|
|
95
|
+
else _cluster_indices(rng, design)
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
boot = np.array([bin_means(draw()) for _ in range(n_boot)])
|
|
99
|
+
lo = np.nanpercentile(boot, 2.5, axis=0)
|
|
100
|
+
hi = np.nanpercentile(boot, 97.5, axis=0)
|
|
101
|
+
return CurveResult(pct, mean, lo, hi)
|
riskaudit/audit/lift.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from numpy.typing import ArrayLike
|
|
4
|
+
|
|
5
|
+
from riskaudit._config import SEED
|
|
6
|
+
from riskaudit.audit._common import (
|
|
7
|
+
SurveyDesign,
|
|
8
|
+
boot_ci,
|
|
9
|
+
check_inputs,
|
|
10
|
+
to_float,
|
|
11
|
+
topk_mask,
|
|
12
|
+
weights_or_ones,
|
|
13
|
+
wmean,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class LiftResult:
|
|
19
|
+
value: float
|
|
20
|
+
ci: tuple[float, float]
|
|
21
|
+
residual_distressed: float
|
|
22
|
+
residual_other: float
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def incremental_lift(
|
|
26
|
+
y_t1: ArrayLike,
|
|
27
|
+
y_pred: ArrayLike,
|
|
28
|
+
distress: ArrayLike,
|
|
29
|
+
scores: ArrayLike,
|
|
30
|
+
k: float = 0.10,
|
|
31
|
+
weights: ArrayLike | None = None,
|
|
32
|
+
n_boot: int = 1000,
|
|
33
|
+
design: SurveyDesign | None = None,
|
|
34
|
+
) -> LiftResult:
|
|
35
|
+
r"""Incremental need lift in the low-score tail (the contribution metric).
|
|
36
|
+
|
|
37
|
+
Among the units a model *deprioritizes* — those outside the top ``k`` of
|
|
38
|
+
``scores`` — this contrasts how much more future need distressed people
|
|
39
|
+
generated than the model predicted, versus everyone else in that tail:
|
|
40
|
+
|
|
41
|
+
.. math::
|
|
42
|
+
\text{Lift}_k \;=\;
|
|
43
|
+
\frac{\sum_{i \in L} w_i d_i r_i}{\sum_{i \in L} w_i d_i}
|
|
44
|
+
\;-\;
|
|
45
|
+
\frac{\sum_{i \in L} w_i (1-d_i) r_i}{\sum_{i \in L} w_i (1-d_i)},
|
|
46
|
+
|
|
47
|
+
where :math:`L` is the deprioritized tail, :math:`d_i` marks measured
|
|
48
|
+
distress, and :math:`r_i = y_{i,t+1} - \hat y_i` is the realized residual of
|
|
49
|
+
the outcome the model targets. A positive lift means the model was blind to
|
|
50
|
+
real downstream need concentrated in the distressed — the non-circular core
|
|
51
|
+
of the argument, since :math:`y_{t+1}` is the model's own currency, not the
|
|
52
|
+
distress measure (``docs/methods.md`` §2).
|
|
53
|
+
|
|
54
|
+
Parameters
|
|
55
|
+
----------
|
|
56
|
+
y_t1 : array-like of shape (n,)
|
|
57
|
+
Realized outcome at ``t+1`` (the spending/utilization the model targets).
|
|
58
|
+
y_pred : array-like of shape (n,)
|
|
59
|
+
The model's prediction of that outcome.
|
|
60
|
+
distress : array-like of shape (n,)
|
|
61
|
+
1 where measured need is present, 0 otherwise.
|
|
62
|
+
scores : array-like of shape (n,)
|
|
63
|
+
Model risk score defining the priority set.
|
|
64
|
+
k : float, default 0.10
|
|
65
|
+
Top fraction, by weight; its complement is the deprioritized tail.
|
|
66
|
+
weights : array-like of shape (n,), optional
|
|
67
|
+
Survey weights; all ones when omitted.
|
|
68
|
+
n_boot : int, default 1000
|
|
69
|
+
Bootstrap resamples for the confidence interval.
|
|
70
|
+
design : SurveyDesign, optional
|
|
71
|
+
When given, the CI resamples PSUs within strata (VARSTR/VARPSU) instead
|
|
72
|
+
of rows, for a design-based interval.
|
|
73
|
+
|
|
74
|
+
Returns
|
|
75
|
+
-------
|
|
76
|
+
LiftResult
|
|
77
|
+
The lift, its 95% ``ci``, and the two tail residual means.
|
|
78
|
+
"""
|
|
79
|
+
y = to_float(y_t1)
|
|
80
|
+
p = to_float(y_pred)
|
|
81
|
+
d = to_float(distress)
|
|
82
|
+
s = to_float(scores)
|
|
83
|
+
w = weights_or_ones(weights, y.shape[0])
|
|
84
|
+
check_inputs(y_t1=y, y_pred=p, distress=d, scores=s, weights=w)
|
|
85
|
+
r = y - p
|
|
86
|
+
|
|
87
|
+
def stat(idx) -> float:
|
|
88
|
+
tail = ~topk_mask(s[idx], w[idx], k)
|
|
89
|
+
di = d[idx]
|
|
90
|
+
one, zero = tail & (di == 1), tail & (di == 0)
|
|
91
|
+
return wmean(r[idx][one], w[idx][one]) - wmean(r[idx][zero], w[idx][zero])
|
|
92
|
+
|
|
93
|
+
tail = ~topk_mask(s, w, k)
|
|
94
|
+
one, zero = tail & (d == 1), tail & (d == 0)
|
|
95
|
+
rd = wmean(r[one], w[one])
|
|
96
|
+
ro = wmean(r[zero], w[zero])
|
|
97
|
+
ci = boot_ci(stat, y.shape[0], n_boot, SEED, design)
|
|
98
|
+
return LiftResult(float(rd - ro), ci, float(rd), float(ro))
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
from numpy.typing import ArrayLike
|
|
3
|
+
|
|
4
|
+
from riskaudit.audit._common import check_inputs, to_float, topk_mask, weights_or_ones
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def reclassification(
|
|
8
|
+
scores_a: ArrayLike,
|
|
9
|
+
scores_b: ArrayLike,
|
|
10
|
+
k: float = 0.10,
|
|
11
|
+
weights: ArrayLike | None = None,
|
|
12
|
+
) -> pd.DataFrame:
|
|
13
|
+
r"""Who enters and leaves the top-``k`` when the label switches from A to B.
|
|
14
|
+
|
|
15
|
+
Let :math:`T_k^A` and :math:`T_k^B` be the top-``k`` sets (by weight) under
|
|
16
|
+
scores A and B. The result is the weighted 2x2 contingency of membership,
|
|
17
|
+
|
|
18
|
+
================== ================ ==================
|
|
19
|
+
. in :math:`T_k^B` not in :math:`T_k^B`
|
|
20
|
+
================== ================ ==================
|
|
21
|
+
in :math:`T_k^A` stay dropped
|
|
22
|
+
not in :math:`T_k^A` added out
|
|
23
|
+
================== ================ ==================
|
|
24
|
+
|
|
25
|
+
``added`` are the units a need-based label would prioritize that a
|
|
26
|
+
spend-based one misses — the population cost of the label choice.
|
|
27
|
+
|
|
28
|
+
Parameters
|
|
29
|
+
----------
|
|
30
|
+
scores_a, scores_b : array-like of shape (n,)
|
|
31
|
+
Scores under label A and label B for the same units.
|
|
32
|
+
k : float, default 0.10
|
|
33
|
+
Top fraction, by weight.
|
|
34
|
+
weights : array-like of shape (n,), optional
|
|
35
|
+
Survey weights; all ones when omitted.
|
|
36
|
+
|
|
37
|
+
Returns
|
|
38
|
+
-------
|
|
39
|
+
pandas.DataFrame
|
|
40
|
+
Weighted counts and shares for stay / dropped / added / out.
|
|
41
|
+
"""
|
|
42
|
+
a = to_float(scores_a)
|
|
43
|
+
b = to_float(scores_b)
|
|
44
|
+
w = weights_or_ones(weights, a.shape[0])
|
|
45
|
+
check_inputs(scores_a=a, scores_b=b, weights=w)
|
|
46
|
+
ta = topk_mask(a, w, k)
|
|
47
|
+
tb = topk_mask(b, w, k)
|
|
48
|
+
cells = {"stay": ta & tb, "dropped": ta & ~tb, "added": ~ta & tb, "out": ~ta & ~tb}
|
|
49
|
+
total = w.sum()
|
|
50
|
+
rows = {
|
|
51
|
+
name: {"weight": float(w[m].sum()), "share": float(w[m].sum() / total)}
|
|
52
|
+
for name, m in cells.items()
|
|
53
|
+
}
|
|
54
|
+
return pd.DataFrame(rows).T[["weight", "share"]]
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import io
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import matplotlib
|
|
7
|
+
|
|
8
|
+
matplotlib.use("Agg")
|
|
9
|
+
import matplotlib.pyplot as plt # noqa: E402
|
|
10
|
+
import pandas as pd # noqa: E402
|
|
11
|
+
|
|
12
|
+
from riskaudit.audit.ablation import AblationResult # noqa: E402
|
|
13
|
+
from riskaudit.audit.capture import CaptureResult # noqa: E402
|
|
14
|
+
from riskaudit.audit.curves import CurveResult # noqa: E402
|
|
15
|
+
from riskaudit.audit.lift import LiftResult # noqa: E402
|
|
16
|
+
from riskaudit.audit.rtm import RTMResult # noqa: E402
|
|
17
|
+
|
|
18
|
+
_STYLE = (
|
|
19
|
+
"body{font-family:system-ui,sans-serif;margin:2rem;max-width:820px;line-height:1.5}"
|
|
20
|
+
"table{border-collapse:collapse;margin:.5rem 0}td,th{border:1px solid #ccc;padding:4px 10px}"
|
|
21
|
+
"img{max-width:100%}"
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _fig_uri(fig) -> str:
|
|
26
|
+
buf = io.BytesIO()
|
|
27
|
+
fig.savefig(buf, format="png", bbox_inches="tight", dpi=90)
|
|
28
|
+
plt.close(fig)
|
|
29
|
+
return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _render(name: str, r: Any) -> str:
|
|
33
|
+
if isinstance(r, CaptureResult):
|
|
34
|
+
return (
|
|
35
|
+
f"<p><b>{name}</b>: top-{r.k:.0%} capture = {r.value:.1%} "
|
|
36
|
+
f"(95% CI {r.ci[0]:.1%}–{r.ci[1]:.1%}; "
|
|
37
|
+
f"floor {r.baseline:.1%}, oracle {r.oracle:.1%})</p>"
|
|
38
|
+
)
|
|
39
|
+
if isinstance(r, RTMResult):
|
|
40
|
+
return (
|
|
41
|
+
f"<p><b>{name}</b>: observed drop {r.observed_drop:.3g}, RTM-expected "
|
|
42
|
+
f"{r.rtm_expected_drop:.3g}, RTM share {r.rtm_share:.1%} "
|
|
43
|
+
f"(95% CI {r.ci[0]:.1%}–{r.ci[1]:.1%})</p>"
|
|
44
|
+
)
|
|
45
|
+
if isinstance(r, LiftResult):
|
|
46
|
+
return (
|
|
47
|
+
f"<p><b>{name}</b>: incremental lift = {r.value:.3g} "
|
|
48
|
+
f"(distressed residual {r.residual_distressed:.3g} vs "
|
|
49
|
+
f"{r.residual_other:.3g}; 95% CI {r.ci[0]:.3g}–{r.ci[1]:.3g})</p>"
|
|
50
|
+
)
|
|
51
|
+
if isinstance(r, CurveResult):
|
|
52
|
+
fig, ax = plt.subplots(figsize=(5, 3))
|
|
53
|
+
ax.plot(r.percentile, r.need_mean, marker="o")
|
|
54
|
+
ax.fill_between(r.percentile, r.need_lo, r.need_hi, alpha=0.2)
|
|
55
|
+
ax.set_xlabel("score percentile")
|
|
56
|
+
ax.set_ylabel("mean need")
|
|
57
|
+
ax.set_title(name)
|
|
58
|
+
return f'<img src="{_fig_uri(fig)}" alt="{name}">'
|
|
59
|
+
if isinstance(r, AblationResult):
|
|
60
|
+
return (
|
|
61
|
+
f"<h3>{name}</h3><h4>global Δ</h4>{r.global_delta.to_html()}"
|
|
62
|
+
f"<h4>capture Δ</h4>{r.capture_delta.to_html()}"
|
|
63
|
+
)
|
|
64
|
+
if isinstance(r, pd.DataFrame):
|
|
65
|
+
return f"<h3>{name}</h3>{r.to_html()}"
|
|
66
|
+
return f"<p><b>{name}</b>: {r}</p>"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def audit_report(results: dict[str, Any], out_html: str | Path) -> Path:
|
|
70
|
+
"""Render audit results into one self-contained HTML file.
|
|
71
|
+
|
|
72
|
+
Takes the objects returned by the audit functions (keyed by name) and
|
|
73
|
+
writes a single HTML document with every figure inlined as a data URI, so
|
|
74
|
+
it opens with no external assets. Returns the written path.
|
|
75
|
+
"""
|
|
76
|
+
body = "\n".join(_render(k, v) for k, v in results.items())
|
|
77
|
+
html = (
|
|
78
|
+
f"<!doctype html><html><head><meta charset='utf-8'>"
|
|
79
|
+
f"<title>riskaudit report</title><style>{_STYLE}</style></head>"
|
|
80
|
+
f"<body><h1>riskaudit report</h1>{body}</body></html>"
|
|
81
|
+
)
|
|
82
|
+
out = Path(out_html)
|
|
83
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
84
|
+
out.write_text(html, encoding="utf-8")
|
|
85
|
+
return out
|