drcurves 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- drcurves/__init__.py +291 -0
- drcurves/balance.py +153 -0
- drcurves/bootstrap.py +126 -0
- drcurves/check.py +183 -0
- drcurves/did.py +56 -0
- drcurves/models.py +96 -0
- drcurves/nuisance.py +481 -0
- drcurves/pooled.py +630 -0
- drcurves/riesz.py +261 -0
- drcurves/simulate.py +150 -0
- drcurves-0.1.0.dist-info/METADATA +272 -0
- drcurves-0.1.0.dist-info/RECORD +15 -0
- drcurves-0.1.0.dist-info/WHEEL +5 -0
- drcurves-0.1.0.dist-info/licenses/LICENSE +21 -0
- drcurves-0.1.0.dist-info/top_level.txt +1 -0
drcurves/__init__.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
"""Doubly robust, nonlinear, hierarchically pooled dose-response curves."""
|
|
2
|
+
import warnings
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from .models import default_model
|
|
8
|
+
from .nuisance import GPS, DiscreteGPS, Nuisances, cross_fit_nuisances, pseudo_outcomes
|
|
9
|
+
from .pooled import HierarchicalDoseResponse
|
|
10
|
+
from .bootstrap import bootstrap_curves
|
|
11
|
+
from .balance import BalancingWeights
|
|
12
|
+
from .check import coverage_check
|
|
13
|
+
from .did import placebo_check
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
16
|
+
|
|
17
|
+
# Supported public API. Everything else (drcurves.riesz, drcurves.simulate, prepare_X) is
|
|
18
|
+
# experimental or internal and may change without notice before 1.0.
|
|
19
|
+
__all__ = [
|
|
20
|
+
# one call
|
|
21
|
+
"fit_dose_response", "HierarchicalDoseResponse",
|
|
22
|
+
# models and weights
|
|
23
|
+
"default_model", "GPS", "DiscreteGPS", "BalancingWeights",
|
|
24
|
+
# checks
|
|
25
|
+
"coverage_check", "bootstrap_curves", "placebo_check",
|
|
26
|
+
# building blocks for custom pipelines
|
|
27
|
+
"cross_fit_nuisances", "pseudo_outcomes", "Nuisances",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def prepare_X(X, groups=None, add_group_codes=True):
|
|
32
|
+
"""Numeric feature matrix for the nuisance models.
|
|
33
|
+
|
|
34
|
+
DataFrame columns of dtype object/category/bool are integer-coded. With
|
|
35
|
+
add_group_codes, one integer code per hierarchy level is appended so the
|
|
36
|
+
nuisance models can learn group effects (tree models split on codes fine;
|
|
37
|
+
pass categorical_features to HistGradientBoosting for native handling,
|
|
38
|
+
which is limited to 255 levels).
|
|
39
|
+
"""
|
|
40
|
+
if isinstance(X, pd.DataFrame):
|
|
41
|
+
cols = []
|
|
42
|
+
for c in X.columns:
|
|
43
|
+
col = X[c]
|
|
44
|
+
if not pd.api.types.is_numeric_dtype(col) or pd.api.types.is_bool_dtype(col):
|
|
45
|
+
cols.append(pd.factorize(col)[0].astype(float))
|
|
46
|
+
else:
|
|
47
|
+
cols.append(col.to_numpy(float))
|
|
48
|
+
Xn = np.column_stack(cols) if cols else np.empty((len(X), 0))
|
|
49
|
+
else:
|
|
50
|
+
Xn = np.asarray(X, float)
|
|
51
|
+
if Xn.ndim == 1:
|
|
52
|
+
Xn = Xn[:, None]
|
|
53
|
+
if groups is not None and add_group_codes:
|
|
54
|
+
g = groups.to_frame() if isinstance(groups, pd.Series) else pd.DataFrame(groups)
|
|
55
|
+
codes = [pd.factorize(g.iloc[:, : l + 1].astype(str).agg("|".join, axis=1))[0]
|
|
56
|
+
for l in range(g.shape[1])]
|
|
57
|
+
Xn = np.column_stack([np.column_stack(codes).astype(float), Xn])
|
|
58
|
+
if np.isinf(Xn).any():
|
|
59
|
+
raise ValueError("X contains inf.")
|
|
60
|
+
return Xn
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _check_dose(A, max_levels):
|
|
64
|
+
A = np.asarray(A, float)
|
|
65
|
+
if not np.all(np.isfinite(A)):
|
|
66
|
+
raise ValueError("A (the dose) must be finite.")
|
|
67
|
+
vals, counts = np.unique(A, return_counts=True)
|
|
68
|
+
discrete = len(vals) <= max_levels
|
|
69
|
+
top = counts.max() / len(A)
|
|
70
|
+
if not discrete and top > 0.05:
|
|
71
|
+
v = vals[np.argmax(counts)]
|
|
72
|
+
warnings.warn(f"{top:.0%} of doses equal {v:g}: a point mass the continuous GPS cannot "
|
|
73
|
+
"represent (e.g. zero spend). Weights near that value are unreliable; "
|
|
74
|
+
"consider modelling 'any dose' separately or analysing A > 0 only.",
|
|
75
|
+
stacklevel=3)
|
|
76
|
+
return discrete
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _code_columns(Xn, groups, add_group_codes):
|
|
80
|
+
"""Indices of the group-code columns prepare_X puts first (treated as categorical)."""
|
|
81
|
+
n_codes = groups.shape[1] if (groups is not None and add_group_codes) else 0
|
|
82
|
+
return list(range(n_codes)) or None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def fit_dose_response(X, A, Y, outcome_model=None, gps=None, groups=None, pooling="partial",
|
|
86
|
+
n_folds=5, n_basis=10, cluster=None, fold_groups=None,
|
|
87
|
+
cov_type="bayes", max_weight_quantile=0.99, add_group_codes=True,
|
|
88
|
+
max_levels=30, y_pre=None, design=None, nuisance_var=True, n_jobs=None,
|
|
89
|
+
random_state=0):
|
|
90
|
+
"""Fit doubly robust dose-response curves in one call.
|
|
91
|
+
|
|
92
|
+
Cross-fits the outcome model and the dose model, builds the doubly robust
|
|
93
|
+
pseudo-outcomes of Kennedy et al. (2017) and smooths them against the dose
|
|
94
|
+
with penalised splines, pooled across ``groups`` if given.
|
|
95
|
+
|
|
96
|
+
Parameters
|
|
97
|
+
----------
|
|
98
|
+
X : array-like or DataFrame
|
|
99
|
+
Confounders. Non-numeric DataFrame columns are coded; missing values
|
|
100
|
+
are allowed.
|
|
101
|
+
A : array-like
|
|
102
|
+
The dose (treatment), one value per row.
|
|
103
|
+
Y : array-like
|
|
104
|
+
The outcome.
|
|
105
|
+
outcome_model : sklearn regressor, optional
|
|
106
|
+
Model for E[Y | X, A]. None uses :func:`default_model`, a spline-ridge
|
|
107
|
+
and boosting stack.
|
|
108
|
+
gps : GPS, DiscreteGPS or BalancingWeights, optional
|
|
109
|
+
The weight model. None picks :class:`DiscreteGPS` when A has at most
|
|
110
|
+
``max_levels`` distinct values, otherwise :class:`GPS` (density-based
|
|
111
|
+
weights, Kennedy et al. 2017). :class:`BalancingWeights` is an
|
|
112
|
+
alternative for a population curve (``groups=None``).
|
|
113
|
+
groups : DataFrame, optional
|
|
114
|
+
None for one population curve; a DataFrame with one column for flat
|
|
115
|
+
groups, each shrunk toward the global curve; or several columns, top to
|
|
116
|
+
bottom (e.g. ``df[["region", "store"]]``). Levels must be nested.
|
|
117
|
+
pooling : {"partial", "none", "complete"}, default "partial"
|
|
118
|
+
See :class:`HierarchicalDoseResponse`.
|
|
119
|
+
n_folds : int, default 5
|
|
120
|
+
Cross-fitting folds.
|
|
121
|
+
n_basis : int, default 10
|
|
122
|
+
Number of B-spline basis functions for the curve.
|
|
123
|
+
cluster : array-like, optional
|
|
124
|
+
Cluster ids for the robust covariance types.
|
|
125
|
+
fold_groups : array-like, optional
|
|
126
|
+
Ids kept together in cross-fitting (e.g. customer id).
|
|
127
|
+
cov_type : {"bayes", "robust_bayes", "sandwich"}, default "bayes"
|
|
128
|
+
Default covariance for intervals.
|
|
129
|
+
max_weight_quantile : float, default 0.99
|
|
130
|
+
Weights are capped at this quantile, trading a little bias for a lot
|
|
131
|
+
of variance.
|
|
132
|
+
add_group_codes : bool, default True
|
|
133
|
+
Add the group labels to X so the nuisance models can use them.
|
|
134
|
+
max_levels : int, default 30
|
|
135
|
+
A dose with at most this many distinct values is treated as discrete.
|
|
136
|
+
y_pre : array-like, optional
|
|
137
|
+
The outcome before the dose, in the same units. With it, choose
|
|
138
|
+
``design``.
|
|
139
|
+
design : {"did", "lagged"}, optional
|
|
140
|
+
Only used with ``y_pre`` (default "did").
|
|
141
|
+
|
|
142
|
+
- ``"did"``: fit on the change ``Y - y_pre``. Removes stable unobserved
|
|
143
|
+
traits, but comparing doses needs strong parallel trends: units on
|
|
144
|
+
different doses would have followed the same path had their doses
|
|
145
|
+
been equal.
|
|
146
|
+
- ``"lagged"``: keep Y and adjust for ``y_pre`` as a confounder. Right
|
|
147
|
+
when the dose responds to past outcomes rather than to a stable trait.
|
|
148
|
+
|
|
149
|
+
The two bracket the truth when you don't know which applies; check
|
|
150
|
+
``"did"`` with :func:`placebo_check` on two pre-periods.
|
|
151
|
+
nuisance_var : bool, default True
|
|
152
|
+
Add the variance from estimating m(a), the outcome model averaged over
|
|
153
|
+
X (Kennedy et al. 2017), to the "bayes" and "robust_bayes" bands. Costs
|
|
154
|
+
one extra prediction per row and dose-grid point.
|
|
155
|
+
n_jobs : int, optional
|
|
156
|
+
Parallel jobs for the default models (None = 1, -1 = all cores).
|
|
157
|
+
random_state : int, default 0
|
|
158
|
+
Seed for the folds and the default models.
|
|
159
|
+
|
|
160
|
+
Returns
|
|
161
|
+
-------
|
|
162
|
+
HierarchicalDoseResponse
|
|
163
|
+
The fitted curves, with ``.nuisances_``, ``.pseudo_outcomes_`` and
|
|
164
|
+
``.diagnostics()`` (overlap and fit checks worth reading first).
|
|
165
|
+
|
|
166
|
+
See Also
|
|
167
|
+
--------
|
|
168
|
+
placebo_check : Checks the parallel-trends assumption behind ``design="did"``.
|
|
169
|
+
coverage_check : Checks the bands on data shaped like yours.
|
|
170
|
+
|
|
171
|
+
Examples
|
|
172
|
+
--------
|
|
173
|
+
>>> import numpy as np
|
|
174
|
+
>>> from drcurves import fit_dose_response
|
|
175
|
+
>>> from drcurves.simulate import sim_nonlinear
|
|
176
|
+
>>> X, A, Y, truth = sim_nonlinear(n=3000, seed=1)
|
|
177
|
+
>>> model = fit_dose_response(X, A, Y)
|
|
178
|
+
>>> grid = np.linspace(*np.quantile(A, [0.05, 0.95]), 50)
|
|
179
|
+
>>> est, lo, hi, _ = model.band(grid)
|
|
180
|
+
"""
|
|
181
|
+
Y = np.asarray(Y, float)
|
|
182
|
+
A = np.asarray(A, float)
|
|
183
|
+
if design is not None and y_pre is None:
|
|
184
|
+
raise ValueError("design needs y_pre (the outcome before the dose).")
|
|
185
|
+
if y_pre is not None:
|
|
186
|
+
y_pre = np.asarray(y_pre, float)
|
|
187
|
+
design = design or "did"
|
|
188
|
+
if design not in ("did", "lagged"):
|
|
189
|
+
raise ValueError('design must be "did" or "lagged".')
|
|
190
|
+
if y_pre.shape != Y.shape or not np.all(np.isfinite(y_pre)):
|
|
191
|
+
raise ValueError("y_pre must be finite and the same length as Y.")
|
|
192
|
+
if design == "did":
|
|
193
|
+
Y = Y - y_pre
|
|
194
|
+
else:
|
|
195
|
+
X = pd.concat([X.reset_index(drop=True), pd.Series(y_pre, name="_y_pre")], axis=1) \
|
|
196
|
+
if isinstance(X, pd.DataFrame) else np.column_stack([np.asarray(X, float), y_pre])
|
|
197
|
+
_check_inputs(X, A, Y, groups)
|
|
198
|
+
if groups is not None:
|
|
199
|
+
groups = groups.to_frame() if isinstance(groups, pd.Series) else pd.DataFrame(groups)
|
|
200
|
+
groups = groups.reset_index(drop=True)
|
|
201
|
+
discrete = _check_dose(A, max_levels)
|
|
202
|
+
Xn = prepare_X(X, groups, add_group_codes)
|
|
203
|
+
cat = _code_columns(Xn, groups, add_group_codes)
|
|
204
|
+
if outcome_model is None:
|
|
205
|
+
# the outcome model sees [X, A]; A is appended last, so the code columns keep their indices
|
|
206
|
+
outcome_model = default_model(cat, n_jobs=n_jobs, random_state=random_state)
|
|
207
|
+
if gps is None:
|
|
208
|
+
gps = (DiscreteGPS(random_state=random_state) if discrete
|
|
209
|
+
else GPS(categorical_features=cat, n_jobs=n_jobs, random_state=random_state))
|
|
210
|
+
elif discrete and type(gps) is GPS:
|
|
211
|
+
warnings.warn(f"A has only {len(np.unique(A))} distinct values; a continuous GPS is not "
|
|
212
|
+
"a valid density for it. Use DiscreteGPS (the default when gps=None).",
|
|
213
|
+
stacklevel=2)
|
|
214
|
+
leaf = None if groups is None else groups.astype(str).agg("\x1f".join, axis=1).to_numpy()
|
|
215
|
+
nuis = cross_fit_nuisances(Xn, A, Y, outcome_model, gps, groups=leaf, n_folds=n_folds,
|
|
216
|
+
max_levels=max_levels, fold_groups=fold_groups,
|
|
217
|
+
n_jobs=n_jobs, nuisance_var=nuisance_var, random_state=random_state)
|
|
218
|
+
xi = pseudo_outcomes(nuis, Y, kind="dr", max_weight_quantile=max_weight_quantile)
|
|
219
|
+
model = HierarchicalDoseResponse(n_basis=n_basis, pooling=pooling, cov_type=cov_type).fit(
|
|
220
|
+
A, xi, groups, cluster=cluster, nuisance_term=(nuis.a_grid, nuis.q))
|
|
221
|
+
model.nuisances_ = nuis
|
|
222
|
+
model.pseudo_outcomes_ = xi
|
|
223
|
+
model._diag_data = (A, leaf, max_weight_quantile)
|
|
224
|
+
model.diagnostics = lambda: _diagnostics(model)
|
|
225
|
+
model.design_ = design or "levels"
|
|
226
|
+
ess = model.diagnostics()["weights"]["ess_share"]
|
|
227
|
+
if np.isfinite(ess) and ess < 0.1:
|
|
228
|
+
warnings.warn(f"Poor overlap: the weights' effective sample size is {ess:.0%} of n, so a few "
|
|
229
|
+
"units dominate the curve. Check model.diagnostics(), consider trimming the dose "
|
|
230
|
+
"range, or try gps=BalancingWeights().", stacklevel=2)
|
|
231
|
+
return model
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
MIN_GROUP_ROWS = 10
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _check_inputs(X, A, Y, groups):
|
|
238
|
+
"""Fail early, with a clear message, on inputs the pipeline cannot handle."""
|
|
239
|
+
n = len(Y)
|
|
240
|
+
if Y.ndim != 1 or A.ndim != 1:
|
|
241
|
+
raise ValueError("A and Y must be one-dimensional.")
|
|
242
|
+
if len(A) != n or len(X) != n:
|
|
243
|
+
raise ValueError(f"X, A and Y must have the same length (got {len(X)}, {len(A)}, {n}).")
|
|
244
|
+
if groups is not None and len(groups) != n:
|
|
245
|
+
raise ValueError(f"groups must have one row per observation (got {len(groups)}, expected {n}).")
|
|
246
|
+
if not np.all(np.isfinite(Y)):
|
|
247
|
+
raise ValueError("Y must be finite (found NaN or inf).")
|
|
248
|
+
if not np.all(np.isfinite(A)):
|
|
249
|
+
raise ValueError("A must be finite (found NaN or inf).")
|
|
250
|
+
if np.ptp(A) == 0:
|
|
251
|
+
raise ValueError("A is constant: there is no variation in the dose to estimate a curve from.")
|
|
252
|
+
if n < 200:
|
|
253
|
+
warnings.warn(f"Only {n} rows: cross-fitted nuisance models and the curve will be very noisy.",
|
|
254
|
+
stacklevel=3)
|
|
255
|
+
if groups is not None:
|
|
256
|
+
groups = groups.to_frame() if isinstance(groups, pd.Series) else pd.DataFrame(groups)
|
|
257
|
+
if groups.isna().any().any():
|
|
258
|
+
raise ValueError("groups contains missing values.")
|
|
259
|
+
leaf_n = groups.astype(str).agg("\x1f".join, axis=1).value_counts()
|
|
260
|
+
small = int((leaf_n < MIN_GROUP_ROWS).sum())
|
|
261
|
+
if small:
|
|
262
|
+
warnings.warn(f"{small} of {len(leaf_n)} groups have fewer than {MIN_GROUP_ROWS} rows; their curves "
|
|
263
|
+
"will lean almost entirely on their parent group.", stacklevel=3)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _diagnostics(model):
|
|
267
|
+
"""Overlap and fit checks worth reading before trusting the curves."""
|
|
268
|
+
A, leaf, q = model._diag_data
|
|
269
|
+
nuis = model.nuisances_
|
|
270
|
+
w = nuis.varpi / nuis.pi
|
|
271
|
+
cap = np.quantile(w, q) if q is not None else np.inf
|
|
272
|
+
wc = np.minimum(w, cap) # the weights actually used
|
|
273
|
+
out = {
|
|
274
|
+
"weights": {"median": float(np.median(w)), "p99": float(np.quantile(w, 0.99)),
|
|
275
|
+
"max_raw": float(w.max()), "cap": float(cap),
|
|
276
|
+
"share_capped": float(np.mean(w > cap)),
|
|
277
|
+
# effective sample size of the capped weights, as a share of n
|
|
278
|
+
"ess_share": float(wc.sum() ** 2 / (wc ** 2).sum() / len(wc))},
|
|
279
|
+
"dose_range_fitted": (float(model.lo_), float(model.hi_)),
|
|
280
|
+
"penalties": {"smooth": float(model.lam_smooth_),
|
|
281
|
+
"levels": [float(v) for v in model.lam_levels_],
|
|
282
|
+
"grid_edges": (float(model.lam_grid.min()), float(model.lam_grid.max()))},
|
|
283
|
+
"edf": float(model.edf_),
|
|
284
|
+
}
|
|
285
|
+
if leaf is not None:
|
|
286
|
+
df = pd.DataFrame({"g": leaf, "A": A})
|
|
287
|
+
stats = df.groupby("g").A.agg(n="size", iqr=lambda v: np.subtract(*np.quantile(v, [.75, .25])))
|
|
288
|
+
out["groups"] = {"n_groups": int(len(stats)), "min_rows": int(stats.n.min()),
|
|
289
|
+
"groups_with_constant_dose": [k.replace("\x1f", "/")
|
|
290
|
+
for k in stats.index[stats.iqr == 0]][:20]}
|
|
291
|
+
return out
|
drcurves/balance.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""Balancing weights for a continuous dose (entropy balancing).
|
|
2
|
+
|
|
3
|
+
Under the target distribution p(x) p(a) the dose is independent of the
|
|
4
|
+
covariates. Entropy balancing finds the weights closest to uniform (in KL
|
|
5
|
+
divergence) such that, in the weighted sample,
|
|
6
|
+
|
|
7
|
+
* the covariate basis h(X) keeps its sample mean,
|
|
8
|
+
* the dose features g(A) keep their sample mean,
|
|
9
|
+
* every product g(A) * h(X) has mean E[g(A)] E[h(X)] (no association),
|
|
10
|
+
|
|
11
|
+
i.e. the dose is decorrelated from each covariate feature
|
|
12
|
+
(Tübbicke 2022; Vegetabile et al. 2021; Fong, Hazlett & Imai 2018 for the
|
|
13
|
+
related CBGPS). These weights estimate varpi(a) / pi(a | x) on the features
|
|
14
|
+
used, without a density model, and are stable when there are many covariates.
|
|
15
|
+
They are computed in-sample (not as a fitted function), so they are not
|
|
16
|
+
cross-fitted. Solved through the convex dual with a small ridge penalty, which
|
|
17
|
+
allows approximate balance when exact balance is impossible.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import numpy as np
|
|
22
|
+
from scipy.optimize import minimize
|
|
23
|
+
from scipy.special import logsumexp
|
|
24
|
+
from sklearn.base import clone
|
|
25
|
+
from sklearn.model_selection import KFold, cross_val_predict
|
|
26
|
+
from sklearn.preprocessing import SplineTransformer
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class BalancingWeights:
|
|
30
|
+
"""Entropy-balancing weights for a continuous dose (Tübbicke 2022).
|
|
31
|
+
|
|
32
|
+
Chooses weights close to uniform such that, after weighting, the dose is
|
|
33
|
+
uncorrelated with functions of the confounders. An alternative to
|
|
34
|
+
:class:`GPS` for a population curve (``groups=None``); it helped with many
|
|
35
|
+
confounders in our simulations.
|
|
36
|
+
|
|
37
|
+
Parameters
|
|
38
|
+
----------
|
|
39
|
+
basis : {"quadratic", "linear", "spline"}, default "quadratic"
|
|
40
|
+
Balance terms h(X): standardised X, plus X^2 (quadratic) or splines.
|
|
41
|
+
dose_powers : tuple of int, default (1, 2)
|
|
42
|
+
Powers of the standardised dose used in g(A).
|
|
43
|
+
ridge : float, default 0.01
|
|
44
|
+
Penalty on the dual variables; larger means smoother weights and
|
|
45
|
+
looser balance.
|
|
46
|
+
max_weight : float, default 50.0
|
|
47
|
+
Cap on any single weight.
|
|
48
|
+
scores : bool, default False
|
|
49
|
+
Also balance cross-fitted predictions of the dose (and of the outcome,
|
|
50
|
+
if given) and their squares: the directions along which confounding
|
|
51
|
+
acts.
|
|
52
|
+
score_model : sklearn regressor, optional
|
|
53
|
+
Model for those predictions.
|
|
54
|
+
n_knots : int, default 4
|
|
55
|
+
Knots for ``basis="spline"``.
|
|
56
|
+
"""
|
|
57
|
+
is_ratio = True
|
|
58
|
+
in_sample = True
|
|
59
|
+
|
|
60
|
+
def __init__(self, basis="quadratic", dose_powers=(1, 2), ridge=1e-2, max_weight=50.0,
|
|
61
|
+
scores=False, score_model=None, n_knots=4):
|
|
62
|
+
self.basis = basis
|
|
63
|
+
self.dose_powers = dose_powers
|
|
64
|
+
self.ridge = ridge
|
|
65
|
+
self.max_weight = max_weight
|
|
66
|
+
self.scores = scores
|
|
67
|
+
self.score_model = score_model
|
|
68
|
+
self.n_knots = n_knots
|
|
69
|
+
|
|
70
|
+
def _score_features(self, X, A, Y):
|
|
71
|
+
"""Cross-fitted predicted dose E[A|X] and predicted outcome E[Y|X]: the
|
|
72
|
+
directions along which confounding acts. Balancing them (and their
|
|
73
|
+
squares) targets the bias directly."""
|
|
74
|
+
from .riesz import default_dose_model
|
|
75
|
+
mdl = self.score_model if self.score_model is not None else default_dose_model()
|
|
76
|
+
cv = KFold(5, shuffle=True, random_state=0)
|
|
77
|
+
cols = [cross_val_predict(clone(mdl), X, A, cv=cv)]
|
|
78
|
+
if Y is not None:
|
|
79
|
+
cols.append(cross_val_predict(clone(mdl), X, Y, cv=cv))
|
|
80
|
+
S = np.column_stack(cols)
|
|
81
|
+
return np.hstack([S, S ** 2])
|
|
82
|
+
|
|
83
|
+
def _constraints(self, X, A, Y=None):
|
|
84
|
+
X = np.asarray(X, float)
|
|
85
|
+
sd = X.std(0)
|
|
86
|
+
keep = sd > 0
|
|
87
|
+
Z = (X[:, keep] - X[:, keep].mean(0)) / sd[keep]
|
|
88
|
+
H = [Z]
|
|
89
|
+
if self.basis == "spline":
|
|
90
|
+
many = np.array([len(np.unique(Z[:, j])) > 10 for j in range(Z.shape[1])])
|
|
91
|
+
if many.any():
|
|
92
|
+
St = SplineTransformer(n_knots=self.n_knots, degree=3, include_bias=False)
|
|
93
|
+
H.append(St.fit_transform(Z[:, many]))
|
|
94
|
+
if self.scores:
|
|
95
|
+
H.append(self._score_features(X, A, Y))
|
|
96
|
+
if self.basis == "quadratic":
|
|
97
|
+
Z2 = Z ** 2
|
|
98
|
+
s2 = Z2.std(0)
|
|
99
|
+
ok = s2 > 1e-8 # skip binary columns, whose square is collinear
|
|
100
|
+
H.append((Z2[:, ok] - Z2[:, ok].mean(0)) / s2[ok])
|
|
101
|
+
H = np.hstack(H)
|
|
102
|
+
H = (H - H.mean(0)) / np.maximum(H.std(0), 1e-12)
|
|
103
|
+
a = (A - A.mean()) / A.std()
|
|
104
|
+
G = []
|
|
105
|
+
for k in self.dose_powers:
|
|
106
|
+
gk = a ** k
|
|
107
|
+
G.append((gk - gk.mean()) / gk.std())
|
|
108
|
+
G = np.column_stack(G)
|
|
109
|
+
GH = (G[:, :, None] * H[:, None, :]).reshape(len(A), -1)
|
|
110
|
+
# g and h are centred, so independence means E_w[g h] = 0 (no recentring!)
|
|
111
|
+
C = np.hstack([H, G, GH])
|
|
112
|
+
return C / np.maximum(C.std(0), 1e-12)
|
|
113
|
+
|
|
114
|
+
def fit_weights(self, X, A, groups=None, Y=None):
|
|
115
|
+
"""Weights that make the weighted dose uncorrelated with the balance terms.
|
|
116
|
+
|
|
117
|
+
Parameters
|
|
118
|
+
----------
|
|
119
|
+
X : array-like
|
|
120
|
+
Confounders (numeric).
|
|
121
|
+
A : array-like
|
|
122
|
+
Dose.
|
|
123
|
+
groups : None
|
|
124
|
+
Must be None: balancing weights support one population only.
|
|
125
|
+
Y : array-like, optional
|
|
126
|
+
Outcome, used only with ``scores=True``.
|
|
127
|
+
|
|
128
|
+
Returns
|
|
129
|
+
-------
|
|
130
|
+
ndarray
|
|
131
|
+
One weight per row.
|
|
132
|
+
"""
|
|
133
|
+
if groups is not None:
|
|
134
|
+
raise NotImplementedError("BalancingWeights supports a single population (groups=None); "
|
|
135
|
+
"use GPS() for group curves.")
|
|
136
|
+
A = np.asarray(A, float)
|
|
137
|
+
C = self._constraints(X, A, Y if self.scores else None)
|
|
138
|
+
n, k = C.shape
|
|
139
|
+
|
|
140
|
+
def dual(lam):
|
|
141
|
+
s = C @ lam
|
|
142
|
+
lse = logsumexp(s)
|
|
143
|
+
w = np.exp(s - lse)
|
|
144
|
+
return lse + 0.5 * self.ridge * lam @ lam, C.T @ w + self.ridge * lam
|
|
145
|
+
|
|
146
|
+
res = minimize(dual, np.zeros(k), jac=True, method="L-BFGS-B",
|
|
147
|
+
options={"maxiter": 2000, "gtol": 1e-8})
|
|
148
|
+
s = C @ res.x
|
|
149
|
+
w = np.exp(s - logsumexp(s)) * n # mean weight 1
|
|
150
|
+
self.dual_result_ = res
|
|
151
|
+
self.max_imbalance_ = float(np.max(np.abs(C.T @ (w / n))))
|
|
152
|
+
self.ess_share_ = float(w.sum() ** 2 / (w ** 2).sum() / n)
|
|
153
|
+
return np.minimum(w, self.max_weight)
|
drcurves/bootstrap.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Bootstrap for the full pipeline (nuisances -> pseudo-outcomes -> pooled splines).
|
|
2
|
+
|
|
3
|
+
The analytic bands from HierarchicalDoseResponse treat the nuisance models and
|
|
4
|
+
the tuned penalties as fixed. The bootstrap refits everything, so it also
|
|
5
|
+
captures their variability. It is slow (every replicate refits the boosted
|
|
6
|
+
models), so use it to check the analytic bands once, not routinely.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
from .nuisance import GPS, cross_fit_nuisances, pseudo_outcomes
|
|
14
|
+
from .pooled import HierarchicalDoseResponse
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _resample(n, groups, stratify, rng):
|
|
18
|
+
if groups is None or not stratify:
|
|
19
|
+
return rng.integers(0, n, n)
|
|
20
|
+
leaf = groups.astype(str).agg("\x1f".join, axis=1).to_numpy()
|
|
21
|
+
idx = []
|
|
22
|
+
for g in pd.unique(leaf):
|
|
23
|
+
rows = np.flatnonzero(leaf == g)
|
|
24
|
+
idx.append(rng.choice(rows, len(rows), replace=True))
|
|
25
|
+
return np.concatenate(idx)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def bootstrap_curves(X, A, Y, outcome_model, gps, a_grid, paths=(None,), groups=None,
|
|
29
|
+
pooling="partial", n_boot=100, refit_nuisances=True, retune=True,
|
|
30
|
+
n_folds=5, n_basis=10, level=0.95, centre=False, random_state=0,
|
|
31
|
+
verbose=False):
|
|
32
|
+
"""Percentile bootstrap bands for one or more curves.
|
|
33
|
+
|
|
34
|
+
Rows are resampled within each leaf group, so every group appears in every
|
|
35
|
+
replicate and its curve stays defined. The band is pointwise; use the
|
|
36
|
+
stored draws for any other summary.
|
|
37
|
+
|
|
38
|
+
Parameters
|
|
39
|
+
----------
|
|
40
|
+
X : array-like
|
|
41
|
+
Confounders, already numeric (see ``drcurves.prepare_X``).
|
|
42
|
+
A, Y : array-like
|
|
43
|
+
Dose and outcome.
|
|
44
|
+
outcome_model : sklearn regressor or None
|
|
45
|
+
None uses :func:`default_model`.
|
|
46
|
+
gps : GPS, DiscreteGPS or None
|
|
47
|
+
None uses :class:`GPS`.
|
|
48
|
+
a_grid : array-like
|
|
49
|
+
Doses at which to evaluate the curves.
|
|
50
|
+
paths : sequence of tuples, default (None,)
|
|
51
|
+
Group paths to return, e.g. ``[None, ("north",), ("north", "s17")]``.
|
|
52
|
+
groups : DataFrame, optional
|
|
53
|
+
Hierarchy, top to bottom.
|
|
54
|
+
pooling : {"partial", "none", "complete"}, default "partial"
|
|
55
|
+
See :class:`HierarchicalDoseResponse`.
|
|
56
|
+
n_boot : int, default 100
|
|
57
|
+
Bootstrap replicates.
|
|
58
|
+
refit_nuisances : bool, default True
|
|
59
|
+
Refit the outcome model and GPS in every replicate (slow, complete), or
|
|
60
|
+
keep the original pseudo-outcomes (fast, misses nuisance noise).
|
|
61
|
+
retune : bool, default True
|
|
62
|
+
Re-select the penalties by GCV in every replicate.
|
|
63
|
+
n_folds : int, default 5
|
|
64
|
+
Cross-fitting folds.
|
|
65
|
+
n_basis : int, default 10
|
|
66
|
+
B-spline basis functions.
|
|
67
|
+
level : float, default 0.95
|
|
68
|
+
Band level.
|
|
69
|
+
centre : bool, default False
|
|
70
|
+
Bands for the shapes.
|
|
71
|
+
random_state : int, default 0
|
|
72
|
+
Seed.
|
|
73
|
+
verbose : bool, default False
|
|
74
|
+
Print progress.
|
|
75
|
+
|
|
76
|
+
Returns
|
|
77
|
+
-------
|
|
78
|
+
dict
|
|
79
|
+
``{path: {"est", "lo", "hi", "draws"}}``.
|
|
80
|
+
"""
|
|
81
|
+
X, A, Y = np.asarray(X, float), np.asarray(A, float), np.asarray(Y, float)
|
|
82
|
+
a_grid = np.asarray(a_grid, float)
|
|
83
|
+
rng = np.random.default_rng(random_state)
|
|
84
|
+
if groups is not None:
|
|
85
|
+
groups = pd.DataFrame(groups).reset_index(drop=True)
|
|
86
|
+
leaf = None if groups is None else groups.astype(str).agg("\x1f".join, axis=1).to_numpy()
|
|
87
|
+
if outcome_model is None:
|
|
88
|
+
from .models import default_model
|
|
89
|
+
outcome_model = default_model(random_state=random_state)
|
|
90
|
+
if gps is None:
|
|
91
|
+
gps = GPS(random_state=random_state)
|
|
92
|
+
|
|
93
|
+
nuis = cross_fit_nuisances(X, A, Y, outcome_model, gps, groups=leaf, n_folds=n_folds,
|
|
94
|
+
random_state=random_state)
|
|
95
|
+
xi = pseudo_outcomes(nuis, Y)
|
|
96
|
+
base = HierarchicalDoseResponse(n_basis=n_basis, pooling=pooling).fit(A, xi, groups)
|
|
97
|
+
pen = (base.lam_smooth_, base.lam_levels_)
|
|
98
|
+
est = {p: base.curve(a_grid, p, centre)[0] for p in paths}
|
|
99
|
+
|
|
100
|
+
draws = {p: [] for p in paths}
|
|
101
|
+
for b in range(n_boot):
|
|
102
|
+
idx = _resample(len(Y), groups, True, rng)
|
|
103
|
+
g_b = None if groups is None else groups.iloc[idx].reset_index(drop=True)
|
|
104
|
+
if refit_nuisances:
|
|
105
|
+
leaf_b = None if leaf is None else leaf[idx]
|
|
106
|
+
# duplicates of one original row stay in the same fold (no leakage)
|
|
107
|
+
nb = cross_fit_nuisances(X[idx], A[idx], Y[idx], outcome_model, gps, groups=leaf_b,
|
|
108
|
+
n_folds=n_folds, fold_groups=idx,
|
|
109
|
+
random_state=random_state + b + 1)
|
|
110
|
+
xi_b = pseudo_outcomes(nb, Y[idx])
|
|
111
|
+
else:
|
|
112
|
+
xi_b = xi[idx]
|
|
113
|
+
m = HierarchicalDoseResponse(n_basis=n_basis, pooling=pooling).fit(
|
|
114
|
+
A[idx], xi_b, g_b, penalties=None if retune else pen)
|
|
115
|
+
for p in paths:
|
|
116
|
+
draws[p].append(m.curve(a_grid, p, centre)[0])
|
|
117
|
+
if verbose:
|
|
118
|
+
print(f"bootstrap {b + 1}/{n_boot}", flush=True)
|
|
119
|
+
|
|
120
|
+
out = {}
|
|
121
|
+
q = [(1 - level) / 2, (1 + level) / 2]
|
|
122
|
+
for p in paths:
|
|
123
|
+
d = np.array(draws[p])
|
|
124
|
+
lo, hi = np.quantile(d, q, axis=0)
|
|
125
|
+
out[p] = {"est": est[p], "lo": lo, "hi": hi, "draws": d}
|
|
126
|
+
return out
|