pyrake-survey 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyrake/USAGE.md +639 -0
- pyrake/__init__.py +185 -0
- pyrake/_typing.py +17 -0
- pyrake/calibration/__init__.py +47 -0
- pyrake/calibration/distance_metrics.py +465 -0
- pyrake/calibration/frontier.py +2840 -0
- pyrake/calibration/joint_calibrator.py +1929 -0
- pyrake/calibration/rake.py +556 -0
- pyrake/calibration/visualizations.py +582 -0
- pyrake/estimation/__init__.py +57 -0
- pyrake/estimation/base_classes.py +1288 -0
- pyrake/estimation/linear_combination.py +351 -0
- pyrake/estimation/population.py +1733 -0
- pyrake/estimation/sensitivity_solvers.py +416 -0
- pyrake/estimation/treatment_effects.py +1493 -0
- pyrake/estimation/visualizations.py +182 -0
- pyrake_survey-0.2.0.dist-info/METADATA +551 -0
- pyrake_survey-0.2.0.dist-info/RECORD +20 -0
- pyrake_survey-0.2.0.dist-info/WHEEL +4 -0
- pyrake_survey-0.2.0.dist-info/licenses/LICENSE.txt +202 -0
pyrake/USAGE.md
ADDED
|
@@ -0,0 +1,639 @@
|
|
|
1
|
+
# PyRake User Guide
|
|
2
|
+
|
|
3
|
+
This guide teaches an AI agent how to use PyRake to analyze surveys.
|
|
4
|
+
PyRake is a survey calibration and estimation library for causal
|
|
5
|
+
inference and observational studies. It solves constrained convex
|
|
6
|
+
optimization problems to find balancing weights, then uses those
|
|
7
|
+
weights for estimation and inference.
|
|
8
|
+
|
|
9
|
+
**For AI agents:** This guide provides recipes and conceptual
|
|
10
|
+
orientation, but every class and method in this library has a detailed
|
|
11
|
+
NumPy-style docstring documenting its parameters, return values, and
|
|
12
|
+
usage notes. Before calling any function, read its docstring via
|
|
13
|
+
`help()`:
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
from pyrake import (
|
|
17
|
+
JointCalibrator,
|
|
18
|
+
SIPWEstimator,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
help(JointCalibrator)
|
|
22
|
+
help(JointCalibrator.solve_with_fallback)
|
|
23
|
+
help(SIPWEstimator)
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The docstrings are the authoritative reference for parameter names,
|
|
27
|
+
types, defaults, and edge cases.
|
|
28
|
+
|
|
29
|
+
The two main workflows are:
|
|
30
|
+
|
|
31
|
+
1. **Calibration**: Find weights that balance covariates between a
|
|
32
|
+
sample and a target population (or between treatment and control
|
|
33
|
+
groups).
|
|
34
|
+
2. **Estimation**: Use the calibrated weights (expressed as propensity
|
|
35
|
+
scores) to estimate population means, treatment effects, ratios,
|
|
36
|
+
and linear combinations, with proper variance estimation and
|
|
37
|
+
sensitivity analysis.
|
|
38
|
+
|
|
39
|
+
All classes are accessible from the top-level `pyrake` namespace:
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
import pyrake
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Calibration
|
|
46
|
+
|
|
47
|
+
### Single-Group Calibration with `Rake`
|
|
48
|
+
|
|
49
|
+
Use `Rake` when calibrating a single set of weights (e.g., one survey
|
|
50
|
+
sample to a known population).
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
rake = pyrake.Rake(
|
|
54
|
+
distance=pyrake.KLDivergence(),
|
|
55
|
+
X=X, # shape (M, p): covariates to balance exactly
|
|
56
|
+
mu=mu, # shape (p,): population means of X
|
|
57
|
+
Z=Z, # shape (M, q): covariates to balance approximately (optional)
|
|
58
|
+
nu=nu, # shape (q,): population means of Z (required if Z given)
|
|
59
|
+
psi=psi, # float or array: max imbalance for Z (optional; default 1.5/sqrt(M))
|
|
60
|
+
phi=phi, # float: upper bound on mean squared weight (optional)
|
|
61
|
+
min_weight=M / N, # lower bound on weights (scalar or array)
|
|
62
|
+
)
|
|
63
|
+
result = rake.solve()
|
|
64
|
+
weights = result.solution # shape (M,)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
**Key arguments:**
|
|
68
|
+
|
|
69
|
+
- `distance`: The objective function to minimize. Use
|
|
70
|
+
`pyrake.KLDivergence()` (recommended for survey calibration),
|
|
71
|
+
`pyrake.SquaredL2()`, or `pyrake.Huber()`. The distance is measured
|
|
72
|
+
from baseline weights (default: all ones). Pass
|
|
73
|
+
`pyrake.KLDivergence(v=baseline_weights)` to use custom baselines.
|
|
74
|
+
- `X`, `mu`: Covariates balanced *exactly*. Use sparingly — exact
|
|
75
|
+
balance for many covariates inflates variance.
|
|
76
|
+
- `Z`, `nu`: Covariates balanced *approximately*, via an l-infinity
|
|
77
|
+
constraint. This is the workhorse for most covariate balancing.
|
|
78
|
+
- `psi`: Controls the tightness of approximate balance. The default
|
|
79
|
+
`1.5 / sqrt(M)` follows Cochran and Chambers (1965, section 3.1).
|
|
80
|
+
If `Z` is whitened (divided by population standard deviations),
|
|
81
|
+
then `psi = 1.96 / sqrt(M)` would constrain imbalances to be
|
|
82
|
+
non-significant at alpha = 0.05.
|
|
83
|
+
- `phi`: Upper bound on the mean squared weight, `(1/M) * ||w||_2^2`.
|
|
84
|
+
Controls variance inflation. When set deliberately infeasible
|
|
85
|
+
(e.g., `phi=0.99`), the solver minimizes the norm, yielding a
|
|
86
|
+
minimum-variance solution.
|
|
87
|
+
- `min_weight`: Lower bound on each weight. Setting this to
|
|
88
|
+
`M / N` (sample size divided by population size) prevents
|
|
89
|
+
implied propensity scores greater than 1.
|
|
90
|
+
- `constrain_mean_weight_to`: Constrains the average weight.
|
|
91
|
+
Defaults to 1.0. Set to `None` to remove this constraint.
|
|
92
|
+
|
|
93
|
+
**Standardizing covariates:**
|
|
94
|
+
|
|
95
|
+
Always standardize `Z` and `nu` by dividing by population standard
|
|
96
|
+
deviations before passing them to `Rake` or `JointCalibrator`. This
|
|
97
|
+
puts all covariates on a common scale, making `psi` interpretable as
|
|
98
|
+
a z-statistic bound.
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
MIN_SIGMA = 1e-6
|
|
102
|
+
sigma = np.clip(population_std, MIN_SIGMA, None)
|
|
103
|
+
Z_standardized = Z / sigma
|
|
104
|
+
nu_standardized = nu / sigma
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
### Joint Calibration with `JointCalibrator`
|
|
108
|
+
|
|
109
|
+
When analyzing treatment effects, jointly calibrate treatment and
|
|
110
|
+
control group weights using `JointCalibrator`. This achieves both
|
|
111
|
+
*internal validity* (balance between groups) and *external validity*
|
|
112
|
+
(balance with the target population).
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
rake = pyrake.JointCalibrator(
|
|
116
|
+
distance=pyrake.KLDivergence(),
|
|
117
|
+
# External validity: each group vs population (approximate balance)
|
|
118
|
+
Z1=Z_control_standardized, # shape (M1, q): control covariates
|
|
119
|
+
Z2=Z_treated_standardized, # shape (M2, q): treated covariates
|
|
120
|
+
nu1=nu_control / sigma, # population means for control covariates
|
|
121
|
+
nu2=nu_treated / sigma, # population means for treated covariates
|
|
122
|
+
# Internal validity: control vs treated (approximate balance)
|
|
123
|
+
Z3=Z_control_standardized, # same covariates, for between-group comparison
|
|
124
|
+
Z4=Z_treated_standardized,
|
|
125
|
+
# Optional: treated-only exact balance covariates
|
|
126
|
+
X2=X_treated_only, # shape (M2, p2)
|
|
127
|
+
mu2=mu_treated_only, # shape (p2,)
|
|
128
|
+
# Bounds
|
|
129
|
+
min_weight1=M1 / N, # control lower bound
|
|
130
|
+
min_weight2=M2 / N, # treated lower bound
|
|
131
|
+
# Variance control
|
|
132
|
+
phi=0.99, # deliberately infeasible → minimize norm
|
|
133
|
+
settings=cvxium.OptimizationSettings(
|
|
134
|
+
outer=cvxium.OuterSettings(tolerance_soft=0.1),
|
|
135
|
+
),
|
|
136
|
+
)
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
**Constraint types:**
|
|
140
|
+
|
|
141
|
+
| Constraints | Purpose | Arguments |
|
|
142
|
+
|---------------|---------------------------------------|-------------------|
|
|
143
|
+
| X1, mu1 | Exact balance, control vs population | External validity |
|
|
144
|
+
| X2, mu2 | Exact balance, treated vs population | External validity |
|
|
145
|
+
| X3, X4 | Exact balance, control vs treated | Internal validity |
|
|
146
|
+
| Z1, nu1, psi1 | Approx balance, control vs population | External validity |
|
|
147
|
+
| Z2, nu2, psi2 | Approx balance, treated vs population | External validity |
|
|
148
|
+
| Z3, Z4, psi3 | Approx balance, control vs treated | Internal validity |
|
|
149
|
+
|
|
150
|
+
All constraint groups are optional. Specify only the ones needed. At
|
|
151
|
+
minimum, the mean-weight constraint (default: average weight = 1 for
|
|
152
|
+
each group) provides the required equality constraints.
|
|
153
|
+
|
|
154
|
+
**Default `psi` values** are set automatically:
|
|
155
|
+
|
|
156
|
+
- External validity: `psi1 = 1.5 / sqrt(M1)`, `psi2 = 1.5 / sqrt(M2)`
|
|
157
|
+
(one-sample z-test scale).
|
|
158
|
+
- Internal validity: `psi3 = 1.5 * sqrt(1/M1 + 1/M2)` (two-sample
|
|
159
|
+
z-test scale).
|
|
160
|
+
|
|
161
|
+
#### Solving with Fallback
|
|
162
|
+
|
|
163
|
+
Always use `solve_with_fallback()` instead of `solve()` for
|
|
164
|
+
`JointCalibrator`. It implements a graceful degradation cascade when
|
|
165
|
+
the full problem is infeasible:
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
result = rake.solve_with_fallback()
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
The result is a `FallbackOptimizationResult` with:
|
|
172
|
+
|
|
173
|
+
- `result.solution`: The weight vector, shape `(M1 + M2,)`. The
|
|
174
|
+
first `M1` entries are control weights; the remaining `M2` are
|
|
175
|
+
treated weights.
|
|
176
|
+
- `result.fallback_status`: Integer code indicating what was achieved.
|
|
177
|
+
- `result.fallback_message`: Human-readable description.
|
|
178
|
+
|
|
179
|
+
**Fallback levels** (in order of preference):
|
|
180
|
+
|
|
181
|
+
| Status | Description |
|
|
182
|
+
|----------|----------------------------------------------------------------------------------------------------|
|
|
183
|
+
| 0 | Full solution: all constraints satisfied |
|
|
184
|
+
| 1 | Phi relaxed: covariate balance achieved, but design effect (phi) minimized rather than constrained |
|
|
185
|
+
| 2 | Ambivalent: internal validity achieved, external validity maximized best-effort |
|
|
186
|
+
| 3 | Sad: internal validity was infeasible; internal imbalance minimized best-effort |
|
|
187
|
+
| (raises) | Fatal: basic bounds + equality constraints are infeasible |
|
|
188
|
+
|
|
189
|
+
Internal validity should *always* be prioritized over external validity.
|
|
190
|
+
|
|
191
|
+
#### Extracting Propensity Scores from Weights
|
|
192
|
+
|
|
193
|
+
After solving, convert weights to propensity scores for use with
|
|
194
|
+
estimators:
|
|
195
|
+
|
|
196
|
+
```python
|
|
197
|
+
w = result.solution
|
|
198
|
+
w_control = w[:M1] # control weights
|
|
199
|
+
w_treated = w[M1:] # treated weights
|
|
200
|
+
|
|
201
|
+
# Propensity scores: pi = (M / N) / w
|
|
202
|
+
control_propensities = (M1 / N) / w_control
|
|
203
|
+
treated_propensities = (M2 / N) / w_treated
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
### Assessing Covariate Balance
|
|
207
|
+
|
|
208
|
+
Use `plot_balance` for external validity (one-sample: group vs
|
|
209
|
+
population) and `plot_balance_2_sample` for internal validity
|
|
210
|
+
(two-sample: control vs treated).
|
|
211
|
+
|
|
212
|
+
```python
|
|
213
|
+
# External validity: control group vs population
|
|
214
|
+
pyrake.plot_balance(
|
|
215
|
+
X=df_control[covariates].values.astype(float),
|
|
216
|
+
mu=population_means.values,
|
|
217
|
+
covariates=covariate_names,
|
|
218
|
+
sigma=population_stds.values,
|
|
219
|
+
weights={
|
|
220
|
+
"Unadjusted": np.ones(M1),
|
|
221
|
+
"Calibrated": w_control,
|
|
222
|
+
},
|
|
223
|
+
title="External Validity: Control",
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
# Internal validity: control vs treated
|
|
227
|
+
pyrake.plot_balance_2_sample(
|
|
228
|
+
X1=df_control[covariates].values.astype(float),
|
|
229
|
+
X2=df_treated[covariates].values.astype(float),
|
|
230
|
+
covariates=covariate_names,
|
|
231
|
+
sigma1=population_stds.values,
|
|
232
|
+
sigma2=population_stds.values,
|
|
233
|
+
weights1={
|
|
234
|
+
"Unadjusted": np.ones(M1),
|
|
235
|
+
"Calibrated": w_control,
|
|
236
|
+
},
|
|
237
|
+
weights2={
|
|
238
|
+
"Unadjusted": np.ones(M2),
|
|
239
|
+
"Calibrated": w_treated,
|
|
240
|
+
},
|
|
241
|
+
title="Internal Validity: Control vs Treated",
|
|
242
|
+
)
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
The `weights` argument is a dictionary mapping strategy names to
|
|
246
|
+
weight arrays, allowing visual comparison of different weighting
|
|
247
|
+
strategies.
|
|
248
|
+
|
|
249
|
+
## Estimation
|
|
250
|
+
|
|
251
|
+
### Common Estimator Interface
|
|
252
|
+
|
|
253
|
+
All estimators share these methods:
|
|
254
|
+
|
|
255
|
+
```python
|
|
256
|
+
estimator.point_estimate() # float
|
|
257
|
+
estimator.variance() # float
|
|
258
|
+
estimator.confidence_interval(alpha=0.10) # (lb, ub)
|
|
259
|
+
estimator.pvalue(null_value=0.0) # float
|
|
260
|
+
estimator.sensitivity_analysis(gamma=6.0) # (lb, ub)
|
|
261
|
+
estimator.expanded_confidence_interval(alpha=0.10, gamma=6.0) # (lb, ub)
|
|
262
|
+
estimator.plot_sensitivity(gamma_lower=1.0, gamma_upper=6.0)
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
### Estimating Population Means with `SIPWEstimator`
|
|
266
|
+
|
|
267
|
+
`SIPWEstimator` (Stabilized Inverse Propensity Weighted) estimates
|
|
268
|
+
the weighted mean `sum(w_i * y_i) / sum(w_i)`. It does not require
|
|
269
|
+
knowing the population size.
|
|
270
|
+
|
|
271
|
+
```python
|
|
272
|
+
estimator = pyrake.SIPWEstimator(
|
|
273
|
+
propensity_scores=propensities, # shape (M,)
|
|
274
|
+
outcomes=Y, # shape (M,)
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
pe = estimator.point_estimate()
|
|
278
|
+
ci = estimator.confidence_interval(alpha=0.05)
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
By default, `SIPWEstimator` uses the `PopulationMean` estimand,
|
|
282
|
+
which sets weights to `1 / propensity_scores`. This is the standard
|
|
283
|
+
choice for survey calibration: we want to estimate the population
|
|
284
|
+
mean, and the propensity scores describe how likely each person is
|
|
285
|
+
to appear in the sample.
|
|
286
|
+
|
|
287
|
+
For binary outcomes (e.g., "yes/no" survey responses), the default
|
|
288
|
+
`binary_outcomes=True` enables a faster sensitivity analysis
|
|
289
|
+
algorithm. Set `binary_outcomes=False` for continuous outcomes.
|
|
290
|
+
|
|
291
|
+
### Estimating Means within Sub-Populations with `RatioEstimator`
|
|
292
|
+
|
|
293
|
+
When analyzing a metric within a sub-population that is only
|
|
294
|
+
observable in the sample (e.g., people who like chocolate), use
|
|
295
|
+
`RatioEstimator`. We cannot calibrate weights specifically for
|
|
296
|
+
chocolate-lovers because we do not know who in the population
|
|
297
|
+
loves chocolate. Instead, we calibrate weights for the full sample
|
|
298
|
+
and estimate the sub-population mean as a ratio:
|
|
299
|
+
|
|
300
|
+
```
|
|
301
|
+
E[C * Y] / E[C]
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
where `C` is a binary indicator for the sub-population (1 if the
|
|
305
|
+
person likes chocolate, 0 otherwise), and `Y` is the outcome.
|
|
306
|
+
|
|
307
|
+
```python
|
|
308
|
+
ratio_estimator = pyrake.RatioEstimator(
|
|
309
|
+
propensity_scores=propensities,
|
|
310
|
+
numerator_outcomes=C * Y, # C_i * Y_i for each respondent
|
|
311
|
+
denominator_outcomes=C, # C_i for each respondent
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
pe = ratio_estimator.point_estimate() # estimates E[C*Y] / E[C]
|
|
315
|
+
ci = ratio_estimator.confidence_interval()
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
Variance is computed via the delta method, properly accounting for
|
|
319
|
+
correlation between numerator and denominator.
|
|
320
|
+
|
|
321
|
+
### Treatment Effects in a Properly-Randomized A/B Test
|
|
322
|
+
|
|
323
|
+
In a properly-randomized A/B test with survey-based measurement,
|
|
324
|
+
the causal effect is the difference between two population means
|
|
325
|
+
corresponding to the two potential outcomes, Y(0) and Y(1).
|
|
326
|
+
By randomization, the population mean of `Y(1)` equals the mean
|
|
327
|
+
in the treated group, and the population mean of `Y(0)` equals the
|
|
328
|
+
mean in the control group. So we estimate each group's population
|
|
329
|
+
mean via `SIPWEstimator` (adjusting for survey non-response), and
|
|
330
|
+
the treatment effect is their difference.
|
|
331
|
+
|
|
332
|
+
Use `SimpleDifferenceEstimator`:
|
|
333
|
+
|
|
334
|
+
```python
|
|
335
|
+
uplift = pyrake.SimpleDifferenceEstimator(
|
|
336
|
+
control_outcomes=Y_control,
|
|
337
|
+
treated_outcomes=Y_treated,
|
|
338
|
+
control_sampling_propensity_scores=control_propensities,
|
|
339
|
+
treated_sampling_propensity_scores=treated_propensities,
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
pe = uplift.point_estimate() # treated mean - control mean
|
|
343
|
+
ci = uplift.confidence_interval()
|
|
344
|
+
pv = uplift.pvalue(null_value=0.0)
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
The `sampling_propensity_scores` adjust for survey non-response
|
|
348
|
+
(generalizing from respondents to the population). When omitted,
|
|
349
|
+
the estimator reduces to a simple difference in sample means.
|
|
350
|
+
|
|
351
|
+
**Important:** `SimpleDifferenceEstimator` is appropriate *only*
|
|
352
|
+
when treatment assignment is random (e.g., a proper A/B test). It
|
|
353
|
+
does not adjust for treatment selection bias.
|
|
354
|
+
|
|
355
|
+
### Treatment Effects Outside a Randomized Experiment
|
|
356
|
+
|
|
357
|
+
Outside a randomized experiment, we need *treatment propensity
|
|
358
|
+
scores* (how likely someone is to select treatment) in addition
|
|
359
|
+
to *response propensity scores* (how likely someone is to respond
|
|
360
|
+
to the survey). Use `ATEEstimator`, `ATTEstimator`, or
|
|
361
|
+
`ATCEstimator`:
|
|
362
|
+
|
|
363
|
+
```python
|
|
364
|
+
# Average Treatment Effect (ATE)
|
|
365
|
+
ate = pyrake.ATEEstimator(
|
|
366
|
+
control_propensity_scores=treatment_propensities_control,
|
|
367
|
+
control_outcomes=Y_control,
|
|
368
|
+
treated_propensity_scores=treatment_propensities_treated,
|
|
369
|
+
treated_outcomes=Y_treated,
|
|
370
|
+
# Optional: also adjust for survey non-response
|
|
371
|
+
control_sampling_propensity_scores=response_propensities_control,
|
|
372
|
+
treated_sampling_propensity_scores=response_propensities_treated,
|
|
373
|
+
)
|
|
374
|
+
```
|
|
375
|
+
|
|
376
|
+
When both treatment and response propensity scores are present, the
|
|
377
|
+
estimator uses `DoubleSamplingEstimand` internally, which multiplies
|
|
378
|
+
the two sets of weights together.
|
|
379
|
+
|
|
380
|
+
**Choosing the right estimand:**
|
|
381
|
+
|
|
382
|
+
- **ATE** (`ATEEstimator`): Average effect across the whole
|
|
383
|
+
population. Uses `PopulationMean` estimand for both groups. Control
|
|
384
|
+
weights are `1/(1-pi)`, treated weights are `1/pi`.
|
|
385
|
+
- **ATT** (`ATTEstimator`): Average effect on the treated. Uses
|
|
386
|
+
`NonRespondentMean` for control (weights = `(1-pi)/pi`) and
|
|
387
|
+
`SampleMean` for treated (weights = 1). Only requires
|
|
388
|
+
`control_propensity_scores`.
|
|
389
|
+
- **ATC** (`ATCEstimator`): Average effect on the control. Mirror of
|
|
390
|
+
ATT. Only requires `treated_propensity_scores`.
|
|
391
|
+
|
|
392
|
+
All three support optional `control_denominator_outcomes` and
|
|
393
|
+
`treated_denominator_outcomes` for ratio-based treatment effects
|
|
394
|
+
(see below).
|
|
395
|
+
|
|
396
|
+
### Treatment Effects within Sub-Populations
|
|
397
|
+
|
|
398
|
+
To estimate treatment effects within a sub-population (e.g., among
|
|
399
|
+
chocolate lovers), take the *difference of two `RatioEstimator`s*.
|
|
400
|
+
Use the denominator outcomes feature of the treatment effect
|
|
401
|
+
estimators:
|
|
402
|
+
|
|
403
|
+
```python
|
|
404
|
+
# Uplift among chocolate lovers in a randomized A/B test
|
|
405
|
+
uplift_choc = pyrake.SimpleDifferenceEstimator(
|
|
406
|
+
control_outcomes=C_control * Y_control,
|
|
407
|
+
treated_outcomes=C_treated * Y_treated,
|
|
408
|
+
control_denominator_outcomes=C_control,
|
|
409
|
+
treated_denominator_outcomes=C_treated,
|
|
410
|
+
control_sampling_propensity_scores=control_propensities,
|
|
411
|
+
treated_sampling_propensity_scores=treated_propensities,
|
|
412
|
+
)
|
|
413
|
+
```
|
|
414
|
+
|
|
415
|
+
This internally creates `RatioEstimator`s for each group and takes
|
|
416
|
+
their difference.
|
|
417
|
+
|
|
418
|
+
### Cost Efficiency: Sentiment Lift per Cost
|
|
419
|
+
|
|
420
|
+
When analyzing marketing impact, we often want the cost per unit of
|
|
421
|
+
incremental sentiment. It is better to estimate **sentiment lift per
|
|
422
|
+
cost** via `TreatmentEffectRatioEstimator` and then invert, because
|
|
423
|
+
the ratio estimator provides proper variance estimates.
|
|
424
|
+
|
|
425
|
+
```python
|
|
426
|
+
lift_per_cost = pyrake.TreatmentEffectRatioEstimator(
|
|
427
|
+
treatment_effect_estimator_class="SimpleDifference",
|
|
428
|
+
control_propensity_scores=[0.5], # irrelevant for SimpleDifference
|
|
429
|
+
treated_propensity_scores=[0.5], # irrelevant for SimpleDifference
|
|
430
|
+
numerator_control_outcomes=Y_control,
|
|
431
|
+
numerator_treated_outcomes=Y_treated,
|
|
432
|
+
denominator_control_outcomes=spend_control,
|
|
433
|
+
denominator_treated_outcomes=spend_treated,
|
|
434
|
+
control_sampling_propensity_scores=control_propensities,
|
|
435
|
+
treated_sampling_propensity_scores=treated_propensities,
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
pe = lift_per_cost.point_estimate()
|
|
439
|
+
ci = lift_per_cost.confidence_interval()
|
|
440
|
+
|
|
441
|
+
# Invert to get cost per unit of sentiment lift
|
|
442
|
+
cost_per_lift = 1.0 / pe
|
|
443
|
+
cost_per_lift_ci = (1.0 / ci[1], 1.0 / ci[0]) # flip bounds
|
|
444
|
+
```
|
|
445
|
+
|
|
446
|
+
Note: when using `treatment_effect_estimator_class="SimpleDifference"`,
|
|
447
|
+
the `control_propensity_scores` and `treated_propensity_scores`
|
|
448
|
+
arguments are ignored (they are only used for ATE/ATT/ATC), but must
|
|
449
|
+
still be provided. Pass `[0.5]` as a placeholder.
|
|
450
|
+
|
|
451
|
+
### Combining Estimates with `LinearCombinationEstimator`
|
|
452
|
+
|
|
453
|
+
`LinearCombinationEstimator` estimates `theta = offset + sum(a_i * theta_i)`
|
|
454
|
+
for independent sub-estimators. This is useful for:
|
|
455
|
+
|
|
456
|
+
1. **Meta-analysis**: Combining estimates across sub-groups, weighted
|
|
457
|
+
by their population fractions.
|
|
458
|
+
2. **Comparing sub-populations**: Testing whether estimates differ
|
|
459
|
+
between groups (e.g., men vs women).
|
|
460
|
+
|
|
461
|
+
#### Combining across cells/sub-groups
|
|
462
|
+
|
|
463
|
+
```python
|
|
464
|
+
# Per-cell estimators, one for each study cell
|
|
465
|
+
cell_estimators = [estimator_cell1, estimator_cell2, estimator_cell3]
|
|
466
|
+
cell_fractions = [0.4, 0.35, 0.25] # sum to 1
|
|
467
|
+
|
|
468
|
+
total = pyrake.LinearCombinationEstimator(
|
|
469
|
+
estimators=cell_estimators,
|
|
470
|
+
multipliers=cell_fractions,
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
pe = total.point_estimate() # weighted average
|
|
474
|
+
ci = total.confidence_interval()
|
|
475
|
+
```
|
|
476
|
+
|
|
477
|
+
#### Comparing sub-populations
|
|
478
|
+
|
|
479
|
+
```python
|
|
480
|
+
# Is the treatment effect different for men vs women?
|
|
481
|
+
men_vs_women = pyrake.LinearCombinationEstimator(
|
|
482
|
+
estimators=[men_estimator, women_estimator],
|
|
483
|
+
multipliers=[1.0, -1.0], # men minus women
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
pe = men_vs_women.point_estimate()
|
|
487
|
+
ci = men_vs_women.confidence_interval()
|
|
488
|
+
pv = men_vs_women.pvalue(null_value=0.0) # test H0: no difference
|
|
489
|
+
```
|
|
490
|
+
|
|
491
|
+
### Sensitivity Analysis
|
|
492
|
+
|
|
493
|
+
All estimators support sensitivity analysis following Rosenbaum's
|
|
494
|
+
framework, which quantifies robustness to hidden biases from
|
|
495
|
+
unobserved confounders. The `gamma` parameter controls how "wrong"
|
|
496
|
+
the propensity scores might be:
|
|
497
|
+
|
|
498
|
+
```python
|
|
499
|
+
# Point estimate bounds under hidden bias
|
|
500
|
+
lb, ub = estimator.sensitivity_analysis(gamma=6.0)
|
|
501
|
+
|
|
502
|
+
# Confidence interval that accounts for both sampling
|
|
503
|
+
# uncertainty and propensity score error
|
|
504
|
+
ci = estimator.expanded_confidence_interval(
|
|
505
|
+
alpha=0.10, gamma=6.0, B=1_000
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
# Visual summary across a range of gamma values
|
|
509
|
+
df, ax = estimator.plot_sensitivity(
|
|
510
|
+
gamma_lower=1.0,
|
|
511
|
+
gamma_upper=6.0,
|
|
512
|
+
title="Sensitivity to Hidden Bias",
|
|
513
|
+
)
|
|
514
|
+
```
|
|
515
|
+
|
|
516
|
+
A `gamma` of 6.0 is a reasonable default for reporting, following
|
|
517
|
+
Rosenbaum's convention based on Hammond's smoking-cancer
|
|
518
|
+
meta-analysis.
|
|
519
|
+
|
|
520
|
+
### `DoubleSamplingEstimand`
|
|
521
|
+
|
|
522
|
+
When there are two sources of selection — treatment assignment and
|
|
523
|
+
survey response — compose two `SimpleEstimand` objects:
|
|
524
|
+
|
|
525
|
+
```python
|
|
526
|
+
# Treatment propensity: how likely to be treated
|
|
527
|
+
treatment_estimand = pyrake.PopulationMean(treatment_propensities)
|
|
528
|
+
|
|
529
|
+
# Response propensity: how likely to respond to the survey
|
|
530
|
+
response_estimand = pyrake.PopulationMean(response_propensities)
|
|
531
|
+
|
|
532
|
+
# Combined: weights are the product
|
|
533
|
+
double_estimand = pyrake.DoubleSamplingEstimand(
|
|
534
|
+
estimand1=treatment_estimand,
|
|
535
|
+
estimand2=response_estimand,
|
|
536
|
+
)
|
|
537
|
+
|
|
538
|
+
# Use with any mean estimator
|
|
539
|
+
estimator = pyrake.SIPWEstimator(
|
|
540
|
+
propensity_scores=np.ones(M), # placeholder; weights come from estimand
|
|
541
|
+
outcomes=Y,
|
|
542
|
+
estimand=double_estimand,
|
|
543
|
+
)
|
|
544
|
+
```
|
|
545
|
+
|
|
546
|
+
In practice, the treatment effect estimators (`ATEEstimator`, etc.)
|
|
547
|
+
construct `DoubleSamplingEstimand` automatically when you pass
|
|
548
|
+
both treatment propensity scores and `*_sampling_propensity_scores`.
|
|
549
|
+
|
|
550
|
+
## Complete Example: Survey Analysis in an A/B Test
|
|
551
|
+
|
|
552
|
+
```python
|
|
553
|
+
import numpy as np
|
|
554
|
+
import pyrake
|
|
555
|
+
import cvxium
|
|
556
|
+
|
|
557
|
+
# --- Step 1: Prepare data ---
|
|
558
|
+
# df_control, df_treated: DataFrames of survey respondents
|
|
559
|
+
# covariates: list of column names to balance
|
|
560
|
+
# pop_means, pop_stds: population mean/std of covariates
|
|
561
|
+
# N: population size
|
|
562
|
+
|
|
563
|
+
M1 = len(df_control)
|
|
564
|
+
M2 = len(df_treated)
|
|
565
|
+
|
|
566
|
+
MIN_SIGMA = 1e-6
|
|
567
|
+
sigma = np.clip(pop_stds.values, MIN_SIGMA, None)
|
|
568
|
+
|
|
569
|
+
Z_control = df_control[covariates].values.astype(float) / sigma
|
|
570
|
+
Z_treated = df_treated[covariates].values.astype(float) / sigma
|
|
571
|
+
nu_control = pop_means.values / sigma
|
|
572
|
+
nu_treated = pop_means.values / sigma
|
|
573
|
+
|
|
574
|
+
# --- Step 2: Calibrate weights jointly ---
|
|
575
|
+
rake = pyrake.JointCalibrator(
|
|
576
|
+
distance=pyrake.KLDivergence(),
|
|
577
|
+
Z1=Z_control,
|
|
578
|
+
Z2=Z_treated,
|
|
579
|
+
Z3=Z_control,
|
|
580
|
+
Z4=Z_treated,
|
|
581
|
+
nu1=nu_control,
|
|
582
|
+
nu2=nu_treated,
|
|
583
|
+
min_weight1=M1 / N,
|
|
584
|
+
min_weight2=M2 / N,
|
|
585
|
+
phi=0.99,
|
|
586
|
+
settings=cvxium.OptimizationSettings(
|
|
587
|
+
outer=cvxium.OuterSettings(tolerance_soft=0.1),
|
|
588
|
+
),
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
result = rake.solve_with_fallback()
|
|
592
|
+
print(f"Fallback status: {result.fallback_status}")
|
|
593
|
+
print(result.fallback_message)
|
|
594
|
+
|
|
595
|
+
# --- Step 3: Extract propensity scores ---
|
|
596
|
+
w = result.solution
|
|
597
|
+
control_propensities = (M1 / N) / w[:M1]
|
|
598
|
+
treated_propensities = (M2 / N) / w[M1:]
|
|
599
|
+
|
|
600
|
+
# --- Step 4: Estimate treatment effect ---
|
|
601
|
+
Y_control = df_control["outcome"].values.astype(float)
|
|
602
|
+
Y_treated = df_treated["outcome"].values.astype(float)
|
|
603
|
+
|
|
604
|
+
uplift = pyrake.SimpleDifferenceEstimator(
|
|
605
|
+
control_outcomes=Y_control,
|
|
606
|
+
treated_outcomes=Y_treated,
|
|
607
|
+
control_sampling_propensity_scores=control_propensities,
|
|
608
|
+
treated_sampling_propensity_scores=treated_propensities,
|
|
609
|
+
)
|
|
610
|
+
|
|
611
|
+
print(f"Uplift: {uplift.point_estimate():.4f}")
|
|
612
|
+
print(f"95% CI: {uplift.confidence_interval(alpha=0.05)}")
|
|
613
|
+
print(f"p-value: {uplift.pvalue(null_value=0.0):.4f}")
|
|
614
|
+
|
|
615
|
+
# --- Step 5: Assess covariate balance ---
|
|
616
|
+
pyrake.plot_balance_2_sample(
|
|
617
|
+
X1=df_control[covariates].values.astype(float),
|
|
618
|
+
X2=df_treated[covariates].values.astype(float),
|
|
619
|
+
covariates=covariates,
|
|
620
|
+
sigma1=pop_stds.values,
|
|
621
|
+
sigma2=pop_stds.values,
|
|
622
|
+
weights1={"Calibrated": w[:M1]},
|
|
623
|
+
weights2={"Calibrated": w[M1:]},
|
|
624
|
+
title="Internal Validity",
|
|
625
|
+
)
|
|
626
|
+
```
|
|
627
|
+
|
|
628
|
+
## Summary of When to Use Each Estimator
|
|
629
|
+
|
|
630
|
+
| Scenario | Estimator |
|
|
631
|
+
|------------------------------------------------------------|----------------------------------------------------------|
|
|
632
|
+
| Population mean from a single survey | `SIPWEstimator` |
|
|
633
|
+
| Mean within an observable sub-population | `RatioEstimator` |
|
|
634
|
+
| Treatment effect in a randomized A/B test | `SimpleDifferenceEstimator` |
|
|
635
|
+
| Treatment effect, observational study | `ATEEstimator` / `ATTEstimator` / `ATCEstimator` |
|
|
636
|
+
| Treatment effect within a sub-population | Treatment effect estimator with `denominator_outcomes` |
|
|
637
|
+
| Ratio of two treatment effects (e.g., sentiment lift per cost) | `TreatmentEffectRatioEstimator` |
|
|
638
|
+
| Combining estimates across sub-groups | `LinearCombinationEstimator` |
|
|
639
|
+
| Comparing sub-populations (e.g., men vs women) | `LinearCombinationEstimator` with `[+1, -1]` multipliers |
|