diff-stream 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diff_stream/__init__.py +52 -0
- diff_stream/_bootstrap_worker.py +48 -0
- diff_stream/_core/__init__.py +5 -0
- diff_stream/_core/contrasts.py +199 -0
- diff_stream/_core/covariate_kernels.py +420 -0
- diff_stream/_core/cs_kernels.py +536 -0
- diff_stream/_core/cs_row_kernels.py +495 -0
- diff_stream/_core/design.py +105 -0
- diff_stream/_core/etwfe_kernels.py +288 -0
- diff_stream/_core/functionals.py +30 -0
- diff_stream/_core/multiplier.py +90 -0
- diff_stream/_core/nuisance.py +255 -0
- diff_stream/_core/partition.py +158 -0
- diff_stream/_core/process.py +61 -0
- diff_stream/_core/resample.py +171 -0
- diff_stream/_core/staging_kernels.py +90 -0
- diff_stream/_core/twostage_kernels.py +342 -0
- diff_stream/_core/weights.py +26 -0
- diff_stream/_core/within.py +81 -0
- diff_stream/_docs.py +100 -0
- diff_stream/_options.py +62 -0
- diff_stream/_present.py +202 -0
- diff_stream/absorbed.py +621 -0
- diff_stream/aggregation.py +225 -0
- diff_stream/api.py +1112 -0
- diff_stream/bacon.py +460 -0
- diff_stream/bacon_controls.py +188 -0
- diff_stream/balance.py +149 -0
- diff_stream/bjs.py +101 -0
- diff_stream/bjs_placebo.py +167 -0
- diff_stream/bootstrap.py +704 -0
- diff_stream/cohort_scores.py +68 -0
- diff_stream/cs.py +197 -0
- diff_stream/cs_unbalanced.py +336 -0
- diff_stream/effective_sample.py +73 -0
- diff_stream/estimators.py +95 -0
- diff_stream/etwfe.py +589 -0
- diff_stream/gardner.py +500 -0
- diff_stream/linear.py +339 -0
- diff_stream/periods.py +72 -0
- diff_stream/pretrend.py +97 -0
- diff_stream/py.typed +0 -0
- diff_stream/reference.py +125 -0
- diff_stream/results.py +521 -0
- diff_stream/simulate.py +80 -0
- diff_stream/stacked.py +405 -0
- diff_stream/store.py +706 -0
- diff_stream/twfe_store.py +181 -0
- diff_stream/twfe_weights.py +552 -0
- diff_stream/twostage_absorbed.py +190 -0
- diff_stream/unbalanced.py +297 -0
- diff_stream-0.1.0.dev0.dist-info/METADATA +192 -0
- diff_stream-0.1.0.dev0.dist-info/RECORD +56 -0
- diff_stream-0.1.0.dev0.dist-info/WHEEL +5 -0
- diff_stream-0.1.0.dev0.dist-info/licenses/LICENSE.md +21 -0
- diff_stream-0.1.0.dev0.dist-info/top_level.txt +1 -0
diff_stream/bacon.py
ADDED
|
@@ -0,0 +1,460 @@
|
|
|
1
|
+
"""Goodman–Bacon decomposition from balanced cohort-time sufficient statistics.
|
|
2
|
+
|
|
3
|
+
Independent implementation of the published two-group comparison identity.
|
|
4
|
+
Treatment timing and decomposition rules are DiD-specific; bounded source
|
|
5
|
+
validation/partitioning is shared with the estimation entry points.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from itertools import combinations
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from tempfile import TemporaryDirectory
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
import polars as pl
|
|
16
|
+
from hdfe_stream.core.profile import Component, Profile
|
|
17
|
+
from stream_limits import MemoryBudget, current_rss_gb, detect_cpus, thread_scope
|
|
18
|
+
|
|
19
|
+
from . import _docs, _options
|
|
20
|
+
from ._core.partition import preprocess_budget_mb, staging_bytes
|
|
21
|
+
from ._present import label_frame, public_diagnostics, public_profile
|
|
22
|
+
from .store import PanelStore, inspect
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(kw_only=True)
|
|
26
|
+
class BaconResult:
|
|
27
|
+
"""Descriptive TWFE decomposition; no separate causal ATT or standard errors."""
|
|
28
|
+
|
|
29
|
+
fml: str
|
|
30
|
+
depvar: str
|
|
31
|
+
twfe_estimate: float
|
|
32
|
+
decomposed_estimate: float
|
|
33
|
+
n_units: int
|
|
34
|
+
n_obs: int
|
|
35
|
+
diagnostics: dict
|
|
36
|
+
_table: pl.DataFrame = field(repr=False)
|
|
37
|
+
within_weight: float = 0.0
|
|
38
|
+
within_estimate: float | None = None
|
|
39
|
+
within_contribution: float = 0.0
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def reconstruction_error(self):
|
|
43
|
+
return self.decomposed_estimate - self.twfe_estimate
|
|
44
|
+
|
|
45
|
+
def comparisons(self):
|
|
46
|
+
"""Two-block comparisons, or full-period dyads when covariate-adjusted."""
|
|
47
|
+
return self._table.clone()
|
|
48
|
+
|
|
49
|
+
def tidy(self):
|
|
50
|
+
"""Alias for comparisons()."""
|
|
51
|
+
return self.comparisons()
|
|
52
|
+
|
|
53
|
+
def within_timing(self):
|
|
54
|
+
"""Within-timing component of a covariate-adjusted decomposition.
|
|
55
|
+
|
|
56
|
+
The estimate is undefined/null when its weight is zero. Contributions
|
|
57
|
+
plus the contributions in comparisons() reconstruct the target TWFE.
|
|
58
|
+
"""
|
|
59
|
+
if not self.diagnostics.get("covariate_adjusted", False):
|
|
60
|
+
raise ValueError("within_timing is available for covariate-adjusted decompositions")
|
|
61
|
+
return pl.DataFrame(
|
|
62
|
+
{
|
|
63
|
+
"comparison_type": ["within_timing"],
|
|
64
|
+
"weight": [self.within_weight],
|
|
65
|
+
"estimate": [self.within_estimate],
|
|
66
|
+
"contribution": [self.within_contribution],
|
|
67
|
+
},
|
|
68
|
+
schema={
|
|
69
|
+
"comparison_type": pl.String,
|
|
70
|
+
"weight": pl.Float64,
|
|
71
|
+
"estimate": pl.Float64,
|
|
72
|
+
"contribution": pl.Float64,
|
|
73
|
+
},
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
def summary(self):
|
|
77
|
+
"""Weights, within-type weighted estimates and contributions by type."""
|
|
78
|
+
table = self._table.group_by("comparison_type", maintain_order=True).agg(
|
|
79
|
+
pl.len().cast(pl.UInt32).alias("n_comparisons"),
|
|
80
|
+
pl.col("weight").sum(),
|
|
81
|
+
pl.col("contribution").sum(),
|
|
82
|
+
)
|
|
83
|
+
if self.diagnostics.get("covariate_adjusted", False):
|
|
84
|
+
within = self.within_timing().with_columns(
|
|
85
|
+
pl.lit(1, dtype=pl.UInt32).alias("n_comparisons")
|
|
86
|
+
)
|
|
87
|
+
table = pl.concat([table, within.select(table.columns)])
|
|
88
|
+
return table.with_columns(
|
|
89
|
+
pl.when(pl.col("weight") > 0)
|
|
90
|
+
.then(pl.col("contribution") / pl.col("weight"))
|
|
91
|
+
.otherwise(None)
|
|
92
|
+
.alias("estimate")
|
|
93
|
+
).select("comparison_type", "n_comparisons", "weight", "estimate", "contribution")
|
|
94
|
+
|
|
95
|
+
def summary_dict(self, *, comparisons=False):
|
|
96
|
+
out = dict(
|
|
97
|
+
fml=self.fml,
|
|
98
|
+
depvar=self.depvar,
|
|
99
|
+
twfe_estimate=self.twfe_estimate,
|
|
100
|
+
decomposed_estimate=self.decomposed_estimate,
|
|
101
|
+
reconstruction_error=self.reconstruction_error,
|
|
102
|
+
n_units=self.n_units,
|
|
103
|
+
n_obs=self.n_obs,
|
|
104
|
+
diagnostics=self.diagnostics,
|
|
105
|
+
by_type=self.summary().to_dicts(),
|
|
106
|
+
)
|
|
107
|
+
if self.diagnostics.get("covariate_adjusted", False):
|
|
108
|
+
out["within_timing"] = self.within_timing().to_dicts()[0]
|
|
109
|
+
if comparisons:
|
|
110
|
+
out["comparisons"] = self.comparisons().to_dicts()
|
|
111
|
+
return out
|
|
112
|
+
|
|
113
|
+
def summary_json(self, path=None, *, comparisons=False):
|
|
114
|
+
"""Return JSON and optionally save it to a file."""
|
|
115
|
+
content = json.dumps(self.summary_dict(comparisons=comparisons), indent=2, allow_nan=False)
|
|
116
|
+
if path is not None:
|
|
117
|
+
Path(path).write_text(content + "\n")
|
|
118
|
+
return content
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _decompose(store):
|
|
122
|
+
groups = sorted(store.gc)
|
|
123
|
+
index = {g: j for j, g in enumerate(groups)}
|
|
124
|
+
sizes = np.array([store.gc[g] for g in groups], dtype=float)
|
|
125
|
+
masses = np.array([store.gw[g] for g in groups], dtype=float)
|
|
126
|
+
proportions = masses / masses.sum()
|
|
127
|
+
raw_masses = masses * (store.weight_scale / store.weight_normalizer)
|
|
128
|
+
times, T = store.times, store.periods
|
|
129
|
+
first, last = int(times[0]), int(times[-1])
|
|
130
|
+
# Cohort constants cancel from every contrast. Removing them before prefix
|
|
131
|
+
# sums reduces cancellation from large unit/cohort intercepts.
|
|
132
|
+
means = store.means - store.means[:, :1]
|
|
133
|
+
prefix = np.column_stack([np.zeros(len(groups)), np.cumsum(means, axis=1)])
|
|
134
|
+
rows = []
|
|
135
|
+
|
|
136
|
+
def add(treated, control, pre_start, pre_end, post_start, post_end, kind):
|
|
137
|
+
i, j = index[treated], index[control]
|
|
138
|
+
pre, post = pre_end - pre_start + 1, post_end - post_start + 1
|
|
139
|
+
|
|
140
|
+
def change(k):
|
|
141
|
+
return (prefix[k, post_end - first + 1] - prefix[k, post_start - first]) / post - (
|
|
142
|
+
prefix[k, pre_end - first + 1] - prefix[k, pre_start - first]
|
|
143
|
+
) / pre
|
|
144
|
+
|
|
145
|
+
rows.append(
|
|
146
|
+
dict(
|
|
147
|
+
treated_cohort=int(treated),
|
|
148
|
+
control_cohort=int(control),
|
|
149
|
+
comparison_type=kind,
|
|
150
|
+
control_already_treated=kind in ("later_vs_earlier", "later_vs_always"),
|
|
151
|
+
pre_start=int(pre_start),
|
|
152
|
+
pre_end=int(pre_end),
|
|
153
|
+
post_start=int(post_start),
|
|
154
|
+
post_end=int(post_end),
|
|
155
|
+
pre_periods=int(pre),
|
|
156
|
+
post_periods=int(post),
|
|
157
|
+
n_treated_units=int(sizes[i]),
|
|
158
|
+
n_control_units=int(sizes[j]),
|
|
159
|
+
treated_weight_total=float(raw_masses[i]),
|
|
160
|
+
control_weight_total=float(raw_masses[j]),
|
|
161
|
+
estimate=float(change(i) - change(j)),
|
|
162
|
+
weight=float(proportions[i] * proportions[j] * (pre / T) * (post / T)),
|
|
163
|
+
)
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
treated = [g for g in groups if g > first]
|
|
167
|
+
if 0 in index:
|
|
168
|
+
for g in treated:
|
|
169
|
+
add(g, 0, first, g - 1, g, last, "treated_vs_never")
|
|
170
|
+
if first in index:
|
|
171
|
+
for g in treated:
|
|
172
|
+
add(g, first, first, g - 1, g, last, "later_vs_always")
|
|
173
|
+
for early, late in combinations(treated, 2):
|
|
174
|
+
add(early, late, first, early - 1, early, late - 1, "earlier_vs_later")
|
|
175
|
+
add(late, early, early, late - 1, late, last, "later_vs_earlier")
|
|
176
|
+
weight_sum = sum(row["weight"] for row in rows)
|
|
177
|
+
D = np.array([(times >= g) & (g != 0) for g in groups], dtype=float)
|
|
178
|
+
d = D - D.mean(axis=1, keepdims=True)
|
|
179
|
+
d -= proportions @ d
|
|
180
|
+
y = means - means.mean(axis=1, keepdims=True)
|
|
181
|
+
y -= proportions @ y
|
|
182
|
+
variance = float(np.sum(proportions[:, None] * d**2) / T)
|
|
183
|
+
if not weight_sum > 0 or not variance > 0:
|
|
184
|
+
raise ValueError(
|
|
185
|
+
"treatment is collinear with unit/time effects; no identified TWFE decomposition"
|
|
186
|
+
)
|
|
187
|
+
if not np.isclose(weight_sum, variance, rtol=1e-10, atol=0):
|
|
188
|
+
raise FloatingPointError(
|
|
189
|
+
"decomposition weights do not reconstruct the residualized treatment variance"
|
|
190
|
+
)
|
|
191
|
+
twfe = float(np.sum(proportions[:, None] * d * y) / T / variance)
|
|
192
|
+
for row in rows:
|
|
193
|
+
row["weight"] /= weight_sum
|
|
194
|
+
row["contribution"] = row["weight"] * row["estimate"]
|
|
195
|
+
reconstructed = sum(row["contribution"] for row in rows)
|
|
196
|
+
tolerance = 1e-10 * max(1.0, abs(twfe), max(abs(row["estimate"]) for row in rows))
|
|
197
|
+
if (
|
|
198
|
+
not np.isfinite(twfe)
|
|
199
|
+
or not np.isfinite(reconstructed)
|
|
200
|
+
or abs(twfe - reconstructed) > tolerance
|
|
201
|
+
):
|
|
202
|
+
raise FloatingPointError(
|
|
203
|
+
"Goodman–Bacon reconstruction lost numerical precision; rescale the outcome"
|
|
204
|
+
)
|
|
205
|
+
return pl.DataFrame(rows), twfe, float(reconstructed), variance
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _bacon_decomp(
|
|
209
|
+
fml,
|
|
210
|
+
data,
|
|
211
|
+
*,
|
|
212
|
+
panel,
|
|
213
|
+
cohort=None,
|
|
214
|
+
treatment=None,
|
|
215
|
+
panel_mode="balanced",
|
|
216
|
+
weights=None,
|
|
217
|
+
threads=None,
|
|
218
|
+
memory_gb=None,
|
|
219
|
+
memory_reserve=None,
|
|
220
|
+
workdir=None,
|
|
221
|
+
profile=False,
|
|
222
|
+
preprocess_mb=None,
|
|
223
|
+
batch_units=1024,
|
|
224
|
+
):
|
|
225
|
+
"""Goodman-Bacon decomposition from internal option names; see bacon_decomp.
|
|
226
|
+
|
|
227
|
+
Positive cohorts on or before the first date form an always-treated
|
|
228
|
+
group; future cohorts are never treated within the sample. With
|
|
229
|
+
covariates, the within-timing component and the full-period timing-group
|
|
230
|
+
dyads jointly recover the TWFE coefficient.
|
|
231
|
+
"""
|
|
232
|
+
if panel_mode not in ("balanced", "complete"):
|
|
233
|
+
raise ValueError(
|
|
234
|
+
"Goodman–Bacon requires panel_mode='balanced' or 'complete'; native unbalanced panels are unsupported"
|
|
235
|
+
)
|
|
236
|
+
if isinstance(batch_units, bool) or not isinstance(batch_units, int) or batch_units < 1:
|
|
237
|
+
raise ValueError("batch_units must be a positive integer")
|
|
238
|
+
if threads is not None and (
|
|
239
|
+
isinstance(threads, bool) or not isinstance(threads, int) or threads < 1
|
|
240
|
+
):
|
|
241
|
+
raise ValueError("n_threads must be a positive integer")
|
|
242
|
+
budget = MemoryBudget(memory_gb, memory_reserve)
|
|
243
|
+
preprocess_mb = preprocess_budget_mb(preprocess_mb, budget)
|
|
244
|
+
baseline = current_rss_gb() or 0
|
|
245
|
+
with thread_scope(threads):
|
|
246
|
+
lf, yname, design = inspect(
|
|
247
|
+
fml, data, panel, cohort, treatment=treatment, weights=weights, allow_covariates=True
|
|
248
|
+
)
|
|
249
|
+
if not budget.fits((preprocess_mb * 1e6 + 96e6) / 1e9):
|
|
250
|
+
raise MemoryError(
|
|
251
|
+
"preprocessing buffers exceed job headroom before staging; reduce preprocess_mb or request more memory"
|
|
252
|
+
)
|
|
253
|
+
if workdir is not None:
|
|
254
|
+
Path(workdir).mkdir(parents=True, exist_ok=True)
|
|
255
|
+
with TemporaryDirectory(prefix="diff-stream-bacon-", dir=workdir) as run:
|
|
256
|
+
store = PanelStore(
|
|
257
|
+
lf,
|
|
258
|
+
run,
|
|
259
|
+
batch_units,
|
|
260
|
+
budget,
|
|
261
|
+
preprocess_mb,
|
|
262
|
+
design=design,
|
|
263
|
+
panel_mode=panel_mode,
|
|
264
|
+
require_never_treated=False,
|
|
265
|
+
allow_always_treated=True,
|
|
266
|
+
)
|
|
267
|
+
G, T = len(store.gc), store.periods
|
|
268
|
+
always = int(store.times[0]) in store.gc
|
|
269
|
+
K = G - int(0 in store.gc) - int(always)
|
|
270
|
+
covariates = (
|
|
271
|
+
[j for j, name in enumerate(design.names) if name != "Intercept"]
|
|
272
|
+
if design is not None
|
|
273
|
+
else []
|
|
274
|
+
)
|
|
275
|
+
adjusted = bool(covariates)
|
|
276
|
+
comparisons = (
|
|
277
|
+
G * (G - 1) // 2
|
|
278
|
+
if adjusted
|
|
279
|
+
else K * (K - 1) + K * (int(0 in store.gc) + int(always))
|
|
280
|
+
)
|
|
281
|
+
numeric = 8 * 24 * G * T + 4096 * comparisons + 16e6
|
|
282
|
+
if adjusted:
|
|
283
|
+
P = T - 1 + len(covariates)
|
|
284
|
+
B = min(batch_units, store.n_units)
|
|
285
|
+
numeric += 8 * (10 * P * P + 12 * B * T * (P + 3) + 8 * G * T * len(covariates))
|
|
286
|
+
diagnostics = dict(
|
|
287
|
+
covariate_adjusted=adjusted,
|
|
288
|
+
covariate_names=[design.names[j] for j in covariates],
|
|
289
|
+
always_treated_units=store.gc.get(int(store.times[0]), 0),
|
|
290
|
+
always_treated_cohort=int(store.times[0]) if always else None,
|
|
291
|
+
batch_units=batch_units,
|
|
292
|
+
sampling_weights=weights,
|
|
293
|
+
weight_convention="positive time-invariant unit sampling weights"
|
|
294
|
+
if store.weighted
|
|
295
|
+
else "equal unit weights",
|
|
296
|
+
weight_scale=store.weight_scale / store.weight_normalizer,
|
|
297
|
+
total_unit_weight=float(
|
|
298
|
+
store.n_units * (store.weight_scale / store.weight_normalizer)
|
|
299
|
+
),
|
|
300
|
+
cohort_weight_totals={
|
|
301
|
+
str(g): float(m * (store.weight_scale / store.weight_normalizer))
|
|
302
|
+
for g, m in store.gw.items()
|
|
303
|
+
},
|
|
304
|
+
panel_mode=panel_mode,
|
|
305
|
+
input_units=store.n_units_input,
|
|
306
|
+
input_rows=store.n_rows,
|
|
307
|
+
dropped_units=store.n_units_input - store.n_units,
|
|
308
|
+
dropped_rows=store.n_rows - store.n_rows_used,
|
|
309
|
+
time_min=int(store.times[0]),
|
|
310
|
+
time_max=int(store.times[-1]),
|
|
311
|
+
periods=T,
|
|
312
|
+
cohort_sizes={str(g): int(n) for g, n in store.gc.items()},
|
|
313
|
+
n_cohorts=G,
|
|
314
|
+
n_comparisons=comparisons,
|
|
315
|
+
partitions=store.partition_count,
|
|
316
|
+
preprocess_mb=preprocess_mb,
|
|
317
|
+
max_decoded_leaf_bytes=store.max_leaf_bytes,
|
|
318
|
+
treatment="1[cohort > 0 and time >= cohort]",
|
|
319
|
+
scope="balanced, covariate-adjusted unit/time fixed effects"
|
|
320
|
+
if adjusted
|
|
321
|
+
else "balanced, no covariates; weighted unit/time fixed effects"
|
|
322
|
+
if store.weighted
|
|
323
|
+
else "balanced, no covariates, equal weights; OLS unit/time fixed effects",
|
|
324
|
+
)
|
|
325
|
+
if profile:
|
|
326
|
+
return Profile(
|
|
327
|
+
model="diff-stream Goodman–Bacon",
|
|
328
|
+
threads=threads or detect_cpus(),
|
|
329
|
+
memory_limit_gb=budget.limit_gb,
|
|
330
|
+
memory_reserve=budget.reserve,
|
|
331
|
+
polars_threads=pl.thread_pool_size(),
|
|
332
|
+
baseline_gb=baseline,
|
|
333
|
+
components=[
|
|
334
|
+
Component(
|
|
335
|
+
"cohort-time reductions, projection and comparison table"
|
|
336
|
+
if adjusted
|
|
337
|
+
else "cohort-time reductions and comparison table",
|
|
338
|
+
numeric,
|
|
339
|
+
numeric * 1.5,
|
|
340
|
+
),
|
|
341
|
+
Component(
|
|
342
|
+
"bounded partition preprocessing",
|
|
343
|
+
staging_bytes(preprocess_mb, store) + 96e6,
|
|
344
|
+
preprocess_mb * 1e6 + 256e6,
|
|
345
|
+
),
|
|
346
|
+
Component(
|
|
347
|
+
"snapshot, hash partitions and sorted leaves",
|
|
348
|
+
64 * store.n_rows,
|
|
349
|
+
128 * store.n_rows,
|
|
350
|
+
where="disk",
|
|
351
|
+
),
|
|
352
|
+
],
|
|
353
|
+
notes=diagnostics,
|
|
354
|
+
warnings=[
|
|
355
|
+
"Heuristic estimate; source readers and allocators have separate overhead. Comparison output grows quadratically with cohorts."
|
|
356
|
+
],
|
|
357
|
+
)
|
|
358
|
+
if not budget.fits(numeric / 1e9):
|
|
359
|
+
raise MemoryError(
|
|
360
|
+
"Goodman–Bacon comparison table exceeds job headroom; reduce the number of cohorts or request more memory"
|
|
361
|
+
)
|
|
362
|
+
within_weight, within_estimate, within_contribution = 0.0, None, 0.0
|
|
363
|
+
if adjusted:
|
|
364
|
+
from .bacon_controls import decompose_controls
|
|
365
|
+
|
|
366
|
+
(
|
|
367
|
+
table,
|
|
368
|
+
twfe,
|
|
369
|
+
decomposed,
|
|
370
|
+
variance,
|
|
371
|
+
within_weight,
|
|
372
|
+
within_estimate,
|
|
373
|
+
within_contribution,
|
|
374
|
+
metadata,
|
|
375
|
+
) = decompose_controls(store, covariates)
|
|
376
|
+
diagnostics.update(metadata)
|
|
377
|
+
else:
|
|
378
|
+
table, twfe, decomposed, variance = _decompose(store)
|
|
379
|
+
diagnostics.update(
|
|
380
|
+
residualized_treatment_variance=variance,
|
|
381
|
+
weight_sum=float(table["weight"].sum()) + within_weight,
|
|
382
|
+
between_weight_sum=float(table["weight"].sum()),
|
|
383
|
+
already_treated_control_weight=(
|
|
384
|
+
None
|
|
385
|
+
if adjusted
|
|
386
|
+
else float(table.filter(pl.col("control_already_treated"))["weight"].sum())
|
|
387
|
+
),
|
|
388
|
+
)
|
|
389
|
+
return BaconResult(
|
|
390
|
+
fml=fml,
|
|
391
|
+
depvar=yname,
|
|
392
|
+
twfe_estimate=twfe,
|
|
393
|
+
decomposed_estimate=decomposed,
|
|
394
|
+
n_units=store.n_units,
|
|
395
|
+
n_obs=store.n_rows_used,
|
|
396
|
+
diagnostics=public_diagnostics(diagnostics, store.period_labels),
|
|
397
|
+
_table=label_frame(table, store.period_labels),
|
|
398
|
+
within_weight=within_weight,
|
|
399
|
+
within_estimate=within_estimate,
|
|
400
|
+
within_contribution=within_contribution,
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
@_docs.document
|
|
405
|
+
def bacon_decomp(
|
|
406
|
+
fml,
|
|
407
|
+
data,
|
|
408
|
+
*,
|
|
409
|
+
panel,
|
|
410
|
+
cohort=None,
|
|
411
|
+
treatment=None,
|
|
412
|
+
weights=None,
|
|
413
|
+
incomplete_units="error",
|
|
414
|
+
n_threads=None,
|
|
415
|
+
max_memory_gb=None,
|
|
416
|
+
memory_reserve=None,
|
|
417
|
+
workdir=None,
|
|
418
|
+
profile=False,
|
|
419
|
+
batch_units=1024,
|
|
420
|
+
preprocess_mb=None,
|
|
421
|
+
):
|
|
422
|
+
"""Goodman-Bacon's decomposition of the two-way fixed effects estimate.
|
|
423
|
+
|
|
424
|
+
Writes the coefficient on treatment in a regression with unit and time
|
|
425
|
+
fixed effects as a weighted average of every two-group, two-period
|
|
426
|
+
comparison in the data, including the "forbidden" ones that use
|
|
427
|
+
already-treated units as controls. It is a diagnostic of what the
|
|
428
|
+
conventional estimate averages, not an estimator of a causal effect.
|
|
429
|
+
|
|
430
|
+
Goodman-Bacon, A. (2021). Difference-in-differences with variation in
|
|
431
|
+
treatment timing. *Journal of Econometrics* 225(2), 254-277.
|
|
432
|
+
|
|
433
|
+
Parameters
|
|
434
|
+
----------
|
|
435
|
+
fml : str
|
|
436
|
+
``"y ~ 1"``, or ``"y ~ x + z"`` for the covariate-adjusted
|
|
437
|
+
decomposition. Treatment comes from ``cohort``.
|
|
438
|
+
{DATA}
|
|
439
|
+
Units treated before the first period (always treated) are allowed;
|
|
440
|
+
a never-treated group is not required.
|
|
441
|
+
weights : str, optional
|
|
442
|
+
A column of sampling weights, positive and constant within each unit.
|
|
443
|
+
incomplete_units : {"error", "drop"}, default "error"
|
|
444
|
+
Refuse a panel with missing unit-periods, or drop the incomplete
|
|
445
|
+
units. The decomposition needs a balanced panel.
|
|
446
|
+
{RESOURCES}
|
|
447
|
+
|
|
448
|
+
Returns
|
|
449
|
+
-------
|
|
450
|
+
BaconResult or Profile
|
|
451
|
+
``comparisons()`` lists each comparison's estimate, weight and type;
|
|
452
|
+
``summary()`` totals them by type.
|
|
453
|
+
"""
|
|
454
|
+
options = dict(locals())
|
|
455
|
+
fml, data = options.pop("fml"), options.pop("data")
|
|
456
|
+
if incomplete_units == "keep":
|
|
457
|
+
raise ValueError(
|
|
458
|
+
"bacon_decomp needs a balanced panel: incomplete_units must be 'error' or 'drop'"
|
|
459
|
+
)
|
|
460
|
+
return public_profile(_bacon_decomp(fml, data, **_options.internal(options)))
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""Covariate-adjusted Bacon identity via weighted orthogonal projections.
|
|
2
|
+
|
|
3
|
+
Unit/time/covariate residuals split into within-timing-group variation and
|
|
4
|
+
cohort-time means. Pairwise mean contrasts recover the between component;
|
|
5
|
+
no unit-by-pair table or pair-specific source regression is required.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from itertools import combinations
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import polars as pl
|
|
12
|
+
|
|
13
|
+
from ._core.nuisance import solve_checked
|
|
14
|
+
from ._core.within import center_groups, independent_columns
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def decompose_controls(store, columns):
|
|
18
|
+
T = store.periods
|
|
19
|
+
P = T - 1 + len(columns)
|
|
20
|
+
groups = sorted(store.gc)
|
|
21
|
+
index = {g: j for j, g in enumerate(groups)}
|
|
22
|
+
masses = np.array([store.gw[g] for g in groups])
|
|
23
|
+
total = masses.sum()
|
|
24
|
+
physical = np.array([store.gc[g] for g in groups])
|
|
25
|
+
rawmass = masses * store.weight_scale / store.weight_normalizer
|
|
26
|
+
time = np.eye(T)[:, 1:]
|
|
27
|
+
|
|
28
|
+
def batches():
|
|
29
|
+
for g, y, x, w in store.batches(include_x=True, include_weights=True, row_weights=True):
|
|
30
|
+
X = np.concatenate([np.broadcast_to(time, (*y.shape, T - 1)), x[:, :, columns]], axis=2)
|
|
31
|
+
D = ((g[:, None] != 0) & (store.times >= g[:, None])).astype(float)
|
|
32
|
+
xc, yc, _, _ = center_groups(X, y, w)
|
|
33
|
+
dc = D - D.mean(axis=1, keepdims=True)
|
|
34
|
+
yield g, xc, yc, dc, w, X
|
|
35
|
+
|
|
36
|
+
Q, q, norm = np.zeros((P, P)), np.zeros(P), np.zeros(P)
|
|
37
|
+
for _, X, _, D, w, raw in batches():
|
|
38
|
+
z = X.reshape(-1, P)
|
|
39
|
+
sw = w.ravel()
|
|
40
|
+
Q += z.T @ (sw[:, None] * z)
|
|
41
|
+
q += z.T @ (sw * D.ravel())
|
|
42
|
+
norm += np.einsum("btp,bt,btp->p", raw, w, raw, optimize=True)
|
|
43
|
+
keep = independent_columns(Q, original_norm=norm)
|
|
44
|
+
coef = np.zeros(P)
|
|
45
|
+
if len(keep):
|
|
46
|
+
coef[keep] = solve_checked(Q[np.ix_(keep, keep)], q[keep], "Bacon treatment projection")
|
|
47
|
+
rmean = np.zeros((len(groups), T))
|
|
48
|
+
ymean = np.zeros_like(rmean)
|
|
49
|
+
rr, ry = 0.0, 0.0
|
|
50
|
+
for g, X, y, D, w, _ in batches():
|
|
51
|
+
r = D - X @ coef
|
|
52
|
+
rr += float(np.sum(w * r * r))
|
|
53
|
+
ry += float(np.sum(w * r * y))
|
|
54
|
+
for cg in np.unique(g):
|
|
55
|
+
mask = g == cg
|
|
56
|
+
rmean[index[cg]] += np.sum(w[mask] * r[mask], axis=0) / masses[index[cg]]
|
|
57
|
+
ymean[index[cg]] += np.sum(w[mask] * y[mask], axis=0) / masses[index[cg]]
|
|
58
|
+
if not np.isfinite(rr) or rr <= 1e-20 * max(total * T, 1):
|
|
59
|
+
raise ValueError(
|
|
60
|
+
"treatment is collinear with unit/time effects and covariates; no identified TWFE decomposition"
|
|
61
|
+
)
|
|
62
|
+
within_rr, within_ry = 0.0, 0.0
|
|
63
|
+
for g, X, y, D, w, _ in batches():
|
|
64
|
+
r = D - X @ coef
|
|
65
|
+
rw = r - np.stack([rmean[index[cg]] for cg in g])
|
|
66
|
+
within_rr += float(np.sum(w * rw * rw))
|
|
67
|
+
within_ry += float(np.sum(w * rw * y))
|
|
68
|
+
# Exact zero components can pick up rounding noise after group reduction.
|
|
69
|
+
if within_rr <= rr * 1e-24 and abs(within_ry) <= 1e-12 * max(abs(ry), 1):
|
|
70
|
+
within_rr, within_ry = 0.0, 0.0
|
|
71
|
+
omega = within_rr / rr
|
|
72
|
+
beta_within = within_ry / within_rr if within_rr > 0 else None
|
|
73
|
+
within_contribution = within_ry / rr
|
|
74
|
+
if omega > 1 + 1e-10:
|
|
75
|
+
raise FloatingPointError(
|
|
76
|
+
"Bacon within/between treatment projection lost numerical precision"
|
|
77
|
+
)
|
|
78
|
+
rows = []
|
|
79
|
+
first = int(store.times[0])
|
|
80
|
+
for a, b in combinations(groups, 2):
|
|
81
|
+
i, j = index[a], index[b]
|
|
82
|
+
dr = rmean[i] - rmean[j]
|
|
83
|
+
dy = ymean[i] - ymean[j]
|
|
84
|
+
pair_rr = float(dr @ dr)
|
|
85
|
+
pair_ry = float(dr @ dy)
|
|
86
|
+
raw = masses[i] * masses[j] / total * pair_rr
|
|
87
|
+
weight = raw / rr
|
|
88
|
+
if pair_rr <= 1e-24 * rr / total:
|
|
89
|
+
estimate = None
|
|
90
|
+
weight = 0.0
|
|
91
|
+
contribution = 0.0
|
|
92
|
+
else:
|
|
93
|
+
estimate = pair_ry / pair_rr
|
|
94
|
+
contribution = weight * estimate
|
|
95
|
+
if a == 0 or b == 0:
|
|
96
|
+
treated = b if a == 0 else a
|
|
97
|
+
control = 0
|
|
98
|
+
kind = "always_vs_never" if treated == first else "treated_vs_never"
|
|
99
|
+
elif a == first or b == first:
|
|
100
|
+
treated = b if a == first else a
|
|
101
|
+
control = first
|
|
102
|
+
kind = "later_vs_always"
|
|
103
|
+
else:
|
|
104
|
+
treated, control, kind = a, b, "both_treated"
|
|
105
|
+
ti, ci = index[treated], index[control]
|
|
106
|
+
rows.append(
|
|
107
|
+
dict(
|
|
108
|
+
treated_cohort=int(treated),
|
|
109
|
+
control_cohort=int(control),
|
|
110
|
+
comparison_type=kind,
|
|
111
|
+
control_already_treated=True
|
|
112
|
+
if kind == "later_vs_always"
|
|
113
|
+
else None
|
|
114
|
+
if kind == "both_treated"
|
|
115
|
+
else False,
|
|
116
|
+
pre_start=None,
|
|
117
|
+
pre_end=None,
|
|
118
|
+
post_start=None,
|
|
119
|
+
post_end=None,
|
|
120
|
+
pre_periods=None,
|
|
121
|
+
post_periods=None,
|
|
122
|
+
n_treated_units=int(physical[ti]),
|
|
123
|
+
n_control_units=int(physical[ci]),
|
|
124
|
+
treated_weight_total=float(rawmass[ti]),
|
|
125
|
+
control_weight_total=float(rawmass[ci]),
|
|
126
|
+
estimate=float(estimate) if estimate is not None else None,
|
|
127
|
+
weight=float(weight),
|
|
128
|
+
contribution=float(contribution),
|
|
129
|
+
)
|
|
130
|
+
)
|
|
131
|
+
between = sum(row["weight"] for row in rows)
|
|
132
|
+
for row in rows:
|
|
133
|
+
row["between_weight"] = row["weight"] / between if between > 0 else None
|
|
134
|
+
twfe = ry / rr
|
|
135
|
+
reconstructed = within_contribution + sum(row["contribution"] for row in rows)
|
|
136
|
+
scale = max(
|
|
137
|
+
1.0,
|
|
138
|
+
abs(twfe),
|
|
139
|
+
max((abs(row["estimate"]) for row in rows if row["estimate"] is not None), default=0.0),
|
|
140
|
+
)
|
|
141
|
+
if not np.isfinite(twfe) or not np.isfinite(reconstructed):
|
|
142
|
+
raise FloatingPointError(
|
|
143
|
+
"covariate-adjusted Bacon calculation overflowed; rescale the outcome"
|
|
144
|
+
)
|
|
145
|
+
if not np.isclose(between + omega, 1, atol=1e-10, rtol=0) or not np.isclose(
|
|
146
|
+
reconstructed, twfe, atol=1e-10 * scale, rtol=0
|
|
147
|
+
):
|
|
148
|
+
raise FloatingPointError(
|
|
149
|
+
"covariate-adjusted Bacon reconstruction lost numerical precision; rescale covariates/outcome"
|
|
150
|
+
)
|
|
151
|
+
names = [f"time::{t}" for t in store.times[1:]] + [store.design.names[j] for j in columns]
|
|
152
|
+
metadata = dict(
|
|
153
|
+
scope="balanced covariate-adjusted unit/time fixed effects; within-timing plus full-period timing-group dyads",
|
|
154
|
+
within_timing_weight=float(omega),
|
|
155
|
+
between_timing_weight=float(between),
|
|
156
|
+
comparison_weights="total contribution weights; between_weight additionally normalizes dyads alone",
|
|
157
|
+
residualized_treatment_variance=rr / (total * T),
|
|
158
|
+
nuisance_columns=[names[j] for j in keep],
|
|
159
|
+
absorbed_or_redundant_columns=[name for j, name in enumerate(names) if j not in keep],
|
|
160
|
+
comparison_interpretation="full-period residualized timing-group contrasts; both_treated combines clean/forbidden comparisons",
|
|
161
|
+
)
|
|
162
|
+
return (
|
|
163
|
+
pl.DataFrame(
|
|
164
|
+
rows,
|
|
165
|
+
schema_overrides={
|
|
166
|
+
"control_already_treated": pl.Boolean,
|
|
167
|
+
"estimate": pl.Float64,
|
|
168
|
+
**{
|
|
169
|
+
name: pl.Int64
|
|
170
|
+
for name in [
|
|
171
|
+
"pre_start",
|
|
172
|
+
"pre_end",
|
|
173
|
+
"post_start",
|
|
174
|
+
"post_end",
|
|
175
|
+
"pre_periods",
|
|
176
|
+
"post_periods",
|
|
177
|
+
]
|
|
178
|
+
},
|
|
179
|
+
},
|
|
180
|
+
),
|
|
181
|
+
float(twfe),
|
|
182
|
+
float(reconstructed),
|
|
183
|
+
rr / (total * T),
|
|
184
|
+
float(omega),
|
|
185
|
+
float(beta_within) if beta_within is not None else None,
|
|
186
|
+
float(within_contribution),
|
|
187
|
+
metadata,
|
|
188
|
+
)
|