hdfe-stream 0.2.2__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/CHANGELOG.md +71 -5
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/CITATION.cff +2 -2
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/PKG-INFO +54 -4
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/README.md +52 -3
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/kss.md +9 -5
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/kss_methodological_differences.md +10 -3
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/__init__.py +10 -6
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/_types.py +1 -2
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/api.py +4 -11
- hdfe_stream-0.3.0/hdfe_stream/connected.py +654 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/estimator.py +67 -13
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/glm.py +8 -20
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/inference.py +3 -2
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/inverse.py +3 -3
- hdfe_stream-0.3.0/hdfe_stream/kernels_graph.py +352 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/kernels_inverse.py +0 -17
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/leaveout.py +30 -176
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/leaveout_match.py +0 -1
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/passes.py +113 -14
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/results.py +25 -4
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/utils.py +6 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream.egg-info/PKG-INFO +54 -4
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream.egg-info/SOURCES.txt +2 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/pyproject.toml +2 -1
- hdfe_stream-0.3.0/tests/test_connected.py +449 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_inverse.py +27 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_leaveout.py +161 -63
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_sample.py +4 -3
- hdfe_stream-0.2.2/hdfe_stream/kernels_graph.py +0 -135
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/LICENSE.md +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/MANIFEST.in +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_disk.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_disk.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_memory.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_memory.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_time.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_time.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_disk.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_disk.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_memory.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_memory.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_time.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_time.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_disk.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_disk.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_memory.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_memory.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_time.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_time.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_disk.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_disk.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_memory.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_memory.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_time.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_time.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_disk.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_disk.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_memory.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_memory.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_time.dark.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_time.light.svg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/glm.md +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/_columns.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/am_interval.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/families.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/feterms.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/formula.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/kernels_base.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/kernels_glm.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/kernels_se.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/kernels_slopes.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/leaveout_se.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/leaveout_weakid.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/py.typed +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/report.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/reporting.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/simulate.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/solve.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/workspace.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream.egg-info/dependency_links.txt +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream.egg-info/requires.txt +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream.egg-info/top_level.txt +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/setup.cfg +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/conftest.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_am_interval.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_cleanup.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_column_names.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_crv3.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_feis.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_fewer_fe.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_formula.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_glm.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_iv_weights.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_logging.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_lowlevel.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_ols.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_reporting.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_simulate.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_singletons.py +0 -0
- {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_vcov.py +0 -0
|
@@ -1,5 +1,76 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.0
|
|
4
|
+
|
|
5
|
+
New features.
|
|
6
|
+
|
|
7
|
+
- **`connected_components`** finds the connected components of the graph of
|
|
8
|
+
two fixed-effect dimensions (either may be an interaction, `firm_id^year`)
|
|
9
|
+
by union-find over the rows as they stream, holding level-sized arrays
|
|
10
|
+
only. It returns a table per dimension giving each level its component,
|
|
11
|
+
numbered by size so that component 0 is the largest (ties go to the
|
|
12
|
+
component with the smallest level of the first dimension), and a table of
|
|
13
|
+
component sizes. `largest()` filters the data to the largest connected
|
|
14
|
+
set and `label()` adds each row's component, both lazily (on the data the
|
|
15
|
+
components were found in, or on other data passed to them). `min_degree`
|
|
16
|
+
removes levels with fewer distinct partners in the other dimension,
|
|
17
|
+
repeating until none is left, before the components are found: `(2, 1)`
|
|
18
|
+
keeps only movers, `2` the 2-core. `leave_one_out=True` prunes to the
|
|
19
|
+
leave-one-out connected set of KSS estimation instead. `rank_by` says
|
|
20
|
+
whether "largest" counts rows (the default) or the levels of one dimension.
|
|
21
|
+
- **`connected=`, `min_degree=` and `connected_fe=`** in `feols_stream`,
|
|
22
|
+
`fepois_stream`, `feglm_stream` and `StreamingHDFE` restrict the fit to a
|
|
23
|
+
pair of fixed effects (`connected_fe`, by default the first two listed),
|
|
24
|
+
among the rows with no missing values and before singletons are dropped:
|
|
25
|
+
`min_degree` prunes levels with too few distinct partners, then
|
|
26
|
+
`connected="largest"` keeps the largest connected set of what is left, or
|
|
27
|
+
`connected="leave_one_out"` the leave-one-out connected set, as
|
|
28
|
+
`connected_components` does. `sample()` reports the rows they drop as
|
|
29
|
+
`"min_degree"`, `"disconnected"` and `"leave_one_out"`, and
|
|
30
|
+
`diagnostics["connected"]` and the summary count them.
|
|
31
|
+
- **Python 3.15** is supported and tested in CI. pyfixest, which the
|
|
32
|
+
`formula` extra needs, has no 3.15 wheels yet (0.60.0), and building it
|
|
33
|
+
from source needs Rust and `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1`; the
|
|
34
|
+
rest of the package installs from wheels.
|
|
35
|
+
|
|
36
|
+
Changes that can affect existing code.
|
|
37
|
+
|
|
38
|
+
- **`leave_one_out_connected` and `LeaveOutSet` are removed.** Use
|
|
39
|
+
`connected_components(data, "worker_id + firm_id", leave_one_out=True)`,
|
|
40
|
+
whose `.largest()` is the pruned panel, or `connected="leave_one_out"`
|
|
41
|
+
in a fit. The set is the same: on simulated panels with many components,
|
|
42
|
+
workers observed once and several rounds of cut workers, the workers and
|
|
43
|
+
firms kept are identical. It is found 5-6 times faster (38s against 218s on
|
|
44
|
+
150 million rows), and a null worker or firm id no longer makes it fail.
|
|
45
|
+
- **`leave_out_kss` prunes in the fit**, with `connected="leave_one_out"`, so
|
|
46
|
+
the leave-one-out connected set is that of the rows the model can use:
|
|
47
|
+
before, rows with a missing outcome or covariate were part of the network
|
|
48
|
+
being pruned. `lo.pruning` is now a dict of the pruning's counts (the
|
|
49
|
+
fit's `diagnostics["connected"]["leave_one_out"]`), and the rows it dropped
|
|
50
|
+
are those `lo.fit.sample()` labels `"leave_one_out"`.
|
|
51
|
+
- **The `component` column of `fixef()` is numbered by size**, so component
|
|
52
|
+
0 is the largest (by rows in the estimation sample), as in
|
|
53
|
+
`connected_components`. Ties go to the component with the smallest level of
|
|
54
|
+
that dimension. It was numbered by each component's smallest level.
|
|
55
|
+
- **Removed `hdfe_stream.iter_group_chunks` and `StreamingHDFE.from_formula`.**
|
|
56
|
+
The first read only the fit's internal row files, sorted by its internal
|
|
57
|
+
group code, and stays internal; the second duplicated `feols_stream`.
|
|
58
|
+
|
|
59
|
+
Fixes.
|
|
60
|
+
|
|
61
|
+
- **The leave-out operator's null space is right with several connected
|
|
62
|
+
components.** `apply_inverse` built one null vector for every number up to
|
|
63
|
+
the largest component label, which with several components left gaps, and
|
|
64
|
+
each gap added a spurious vector. Leave-out estimation runs on a connected
|
|
65
|
+
set and was not affected.
|
|
66
|
+
|
|
67
|
+
Documentation.
|
|
68
|
+
|
|
69
|
+
- **New example, `connected_sets.py`**, and additions to the others:
|
|
70
|
+
`sample()` and joining the residuals back on `row_id`, `fe_dof`, and CRV3
|
|
71
|
+
(`formulas.py`); `summary_json` (`quickstart.py`); `keep=` and a
|
|
72
|
+
decomposition by group (`akm_variance.py`); and `StreamingGLM` (`glm.py`).
|
|
73
|
+
|
|
3
74
|
## 0.2.2
|
|
4
75
|
|
|
5
76
|
- **Polars 2.0 support.** `fit.sample()` returns the source's rows in the source's
|
|
@@ -38,11 +109,6 @@
|
|
|
38
109
|
unchanged up to rounding.
|
|
39
110
|
- **Removed `triple_budget` and `dense_max_levels`** from `StreamingHDFE`. They
|
|
40
111
|
tuned the old way of building S and have no counterpart in the new one.
|
|
41
|
-
- **Per-iteration CG messages are logged at DEBUG.** With `verbose=True`, the
|
|
42
|
-
"CG iter N: max rel. residual" lines go to the logger at DEBUG, whatever
|
|
43
|
-
`log_level` is, so they appear only when the logger is enabled for DEBUG.
|
|
44
|
-
Without a logger they are no longer printed. Other progress messages are
|
|
45
|
-
unchanged.
|
|
46
112
|
- **Importing hdfe_stream costs a little less memory.** p-values, critical
|
|
47
113
|
values and the weak-identification interval use `scipy.special`'s
|
|
48
114
|
distribution functions instead of `scipy.stats`, which is no longer
|
|
@@ -7,8 +7,8 @@ authors:
|
|
|
7
7
|
- family-names: Tucker
|
|
8
8
|
given-names: Lee
|
|
9
9
|
email: lee.c.tucker@census.gov
|
|
10
|
-
version: 0.
|
|
11
|
-
date-released: 2026-10-
|
|
10
|
+
version: 0.3.0
|
|
11
|
+
date-released: 2026-10-14
|
|
12
12
|
license: MIT
|
|
13
13
|
repository-code: "https://github.com/leetucker/hdfe-stream"
|
|
14
14
|
url: "https://github.com/leetucker/hdfe-stream"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hdfe-stream
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Out-of-core regression with several high-dimensional fixed effects: OLS, Poisson, logit and probit, with AKM/KSS variance decompositions
|
|
5
5
|
Author: Lee Tucker
|
|
6
6
|
License-Expression: MIT
|
|
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
22
23
|
Classifier: Topic :: Scientific/Engineering
|
|
23
24
|
Classifier: Typing :: Typed
|
|
24
25
|
Requires-Python: >=3.10
|
|
@@ -139,6 +140,11 @@ pip install hdfe-stream[formula] # formula syntax needs pyfixest
|
|
|
139
140
|
| `all` | all of the above |
|
|
140
141
|
| `test` | the test suite |
|
|
141
142
|
|
|
143
|
+
On Python 3.15, pyfixest 0.60 (for the `formula` extra) has no wheels yet. It
|
|
144
|
+
builds from source with a Rust toolchain and
|
|
145
|
+
`PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` set; everything else installs from
|
|
146
|
+
wheels.
|
|
147
|
+
|
|
142
148
|
Without `formula` you still get `StreamingHDFE`, the lower-level interface that
|
|
143
149
|
takes column names or Polars expressions instead of a formula string.
|
|
144
150
|
|
|
@@ -163,6 +169,47 @@ left (`fixef_rm="singleton"`, the default; `fixef_rm="none"` keeps them).
|
|
|
163
169
|
Finding them costs a few extra passes over the fixed-effect columns and memory
|
|
164
170
|
for the singleton levels, not for the rows.
|
|
165
171
|
|
|
172
|
+
**Connected sets.** Worker and firm effects are only comparable within a
|
|
173
|
+
connected set: firms linked, directly or through other firms, by workers who
|
|
174
|
+
moved between them. `connected_components(data, "worker_id + firm_id")` finds
|
|
175
|
+
the connected components of the graph of two fixed effects (either may be an
|
|
176
|
+
interaction, `firm_id^year`) by union-find over the rows as they stream, and
|
|
177
|
+
returns small tables, one per dimension, giving each level its component.
|
|
178
|
+
Components are numbered by size, so `component == 0` is the largest.
|
|
179
|
+
`.largest()` is the largest connected set and `.label()` every row with
|
|
180
|
+
its component, both lazily.
|
|
181
|
+
|
|
182
|
+
`min_degree` first removes levels with too few distinct partners in the other
|
|
183
|
+
dimension (distinct firms for a worker, distinct workers for a firm),
|
|
184
|
+
repeating until none is left, then finds the components of what remains.
|
|
185
|
+
`min_degree=(2, 1)` keeps movers only: workers seen at two or more firms, and
|
|
186
|
+
the firms they were seen at. `min_degree=2` also removes firms with a single
|
|
187
|
+
worker, and repeats, since removing a firm can leave a worker with one firm and
|
|
188
|
+
removing a worker can leave a firm with one worker, until every worker left
|
|
189
|
+
was seen at two or more firms and every firm employed two or more of them (the
|
|
190
|
+
network's "2-core").
|
|
191
|
+
|
|
192
|
+
`leave_one_out=True` instead prunes to the **leave-one-out connected set** that
|
|
193
|
+
KSS estimation needs (and `leave_out_kss` applies), so it can be found ahead of
|
|
194
|
+
time: the largest connected set, without workers observed only once, and
|
|
195
|
+
without any worker whose departure would split the network into pieces,
|
|
196
|
+
removed repeatedly until none is left. It differs from `min_degree=2`. It keeps
|
|
197
|
+
stayers with at least two observations, which `min_degree=2` removes, and it
|
|
198
|
+
removes movers that `min_degree=2` keeps. Every mover and firm it keeps,
|
|
199
|
+
`min_degree=2` keeps too, and on the movers and firms the two coincide only
|
|
200
|
+
when no single mover holds the network together.
|
|
201
|
+
|
|
202
|
+
In a fit, `min_degree=` prunes first, then `connected="largest"` keeps the
|
|
203
|
+
largest connected set or `connected="leave_one_out"` the leave-one-out
|
|
204
|
+
connected set, all on the first two fixed effects listed unless
|
|
205
|
+
`connected_fe=` names another pair, and all before singletons are dropped, so
|
|
206
|
+
the fit equals one on data restricted beforehand. `sample()` labels the rows
|
|
207
|
+
they drop `"min_degree"`, `"disconnected"` and `"leave_one_out"`. The
|
|
208
|
+
`component` column of `fixef()` for the first non-streamed dimension is
|
|
209
|
+
numbered the same way, by rows in the estimation sample. With three or more
|
|
210
|
+
fixed effects, connectedness of one pair is necessary for identification but
|
|
211
|
+
not sufficient.
|
|
212
|
+
|
|
166
213
|
**Covariates.** Transformations (`I(age**2)`, `log(x)`), categoricals (`C(x)`,
|
|
167
214
|
string columns), interactions (`:`, `*`), event-study terms
|
|
168
215
|
(`i(year, treat, ref=2009)`). Terms spanned by the fixed effects are dropped
|
|
@@ -212,7 +259,8 @@ order, not the source's, so every row carries `row_id`, its position in the
|
|
|
212
259
|
source before any row was dropped (`row_id=` renames the column).
|
|
213
260
|
`fit.resid()` joins back onto the source on it, and `fit.sample()` returns the
|
|
214
261
|
source, lazily, with `row_id`, `in_sample` and `dropped_because` (null for rows
|
|
215
|
-
used; otherwise `"missing"`, `"
|
|
262
|
+
used; otherwise `"missing"`, `"min_degree"`, `"disconnected"`,
|
|
263
|
+
`"leave_one_out"`, `"singleton"` or `"separation"`), for looking at
|
|
216
264
|
the data net of what the fit dropped:
|
|
217
265
|
|
|
218
266
|
```python
|
|
@@ -479,6 +527,7 @@ Runnable, on simulated data, no setup — see [examples/](https://github.com/lee
|
|
|
479
527
|
| [`varying_slopes.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/varying_slopes.py) | worker-specific trends, and the low-level interface |
|
|
480
528
|
| [`out_of_core.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/out_of_core.py) | memory, disk, solvers, logging |
|
|
481
529
|
| [`reporting.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/reporting.py) | tables and plots via pyfixest |
|
|
530
|
+
| [`connected_sets.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/connected_sets.py) | connected components, `min_degree` pruning and the leave-one-out connected set, compared |
|
|
482
531
|
| [`leave_out_kss.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/leave_out_kss.py) | the KSS bias correction, checked against the effects the data was built from |
|
|
483
532
|
|
|
484
533
|
## Simulated data
|
|
@@ -546,8 +595,9 @@ print(lo.summary())
|
|
|
546
595
|
|
|
547
596
|
It follows Saggio's reference implementation,
|
|
548
597
|
[LeaveOutTwoWay](https://github.com/rsaggio87/LeaveOutTwoWay). It prunes to the
|
|
549
|
-
leave-one-out connected set
|
|
550
|
-
|
|
598
|
+
leave-one-out connected set (which `connected_components(...,
|
|
599
|
+
leave_one_out=True)` finds on its own, ahead of time), leaves out a
|
|
600
|
+
worker–firm match by default, and approximates leverages by random projection. It supports weights, standard
|
|
551
601
|
errors, and KSS's weak-identification diagnostic and q = 1 interval.
|
|
552
602
|
|
|
553
603
|
**[docs/kss.md](https://github.com/leetucker/hdfe-stream/blob/main/docs/kss.md)** covers the options, the standard errors and their
|
|
@@ -90,6 +90,11 @@ pip install hdfe-stream[formula] # formula syntax needs pyfixest
|
|
|
90
90
|
| `all` | all of the above |
|
|
91
91
|
| `test` | the test suite |
|
|
92
92
|
|
|
93
|
+
On Python 3.15, pyfixest 0.60 (for the `formula` extra) has no wheels yet. It
|
|
94
|
+
builds from source with a Rust toolchain and
|
|
95
|
+
`PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` set; everything else installs from
|
|
96
|
+
wheels.
|
|
97
|
+
|
|
93
98
|
Without `formula` you still get `StreamingHDFE`, the lower-level interface that
|
|
94
99
|
takes column names or Polars expressions instead of a formula string.
|
|
95
100
|
|
|
@@ -114,6 +119,47 @@ left (`fixef_rm="singleton"`, the default; `fixef_rm="none"` keeps them).
|
|
|
114
119
|
Finding them costs a few extra passes over the fixed-effect columns and memory
|
|
115
120
|
for the singleton levels, not for the rows.
|
|
116
121
|
|
|
122
|
+
**Connected sets.** Worker and firm effects are only comparable within a
|
|
123
|
+
connected set: firms linked, directly or through other firms, by workers who
|
|
124
|
+
moved between them. `connected_components(data, "worker_id + firm_id")` finds
|
|
125
|
+
the connected components of the graph of two fixed effects (either may be an
|
|
126
|
+
interaction, `firm_id^year`) by union-find over the rows as they stream, and
|
|
127
|
+
returns small tables, one per dimension, giving each level its component.
|
|
128
|
+
Components are numbered by size, so `component == 0` is the largest.
|
|
129
|
+
`.largest()` is the largest connected set and `.label()` every row with
|
|
130
|
+
its component, both lazily.
|
|
131
|
+
|
|
132
|
+
`min_degree` first removes levels with too few distinct partners in the other
|
|
133
|
+
dimension (distinct firms for a worker, distinct workers for a firm),
|
|
134
|
+
repeating until none is left, then finds the components of what remains.
|
|
135
|
+
`min_degree=(2, 1)` keeps movers only: workers seen at two or more firms, and
|
|
136
|
+
the firms they were seen at. `min_degree=2` also removes firms with a single
|
|
137
|
+
worker, and repeats, since removing a firm can leave a worker with one firm and
|
|
138
|
+
removing a worker can leave a firm with one worker, until every worker left
|
|
139
|
+
was seen at two or more firms and every firm employed two or more of them (the
|
|
140
|
+
network's "2-core").
|
|
141
|
+
|
|
142
|
+
`leave_one_out=True` instead prunes to the **leave-one-out connected set** that
|
|
143
|
+
KSS estimation needs (and `leave_out_kss` applies), so it can be found ahead of
|
|
144
|
+
time: the largest connected set, without workers observed only once, and
|
|
145
|
+
without any worker whose departure would split the network into pieces,
|
|
146
|
+
removed repeatedly until none is left. It differs from `min_degree=2`. It keeps
|
|
147
|
+
stayers with at least two observations, which `min_degree=2` removes, and it
|
|
148
|
+
removes movers that `min_degree=2` keeps. Every mover and firm it keeps,
|
|
149
|
+
`min_degree=2` keeps too, and on the movers and firms the two coincide only
|
|
150
|
+
when no single mover holds the network together.
|
|
151
|
+
|
|
152
|
+
In a fit, `min_degree=` prunes first, then `connected="largest"` keeps the
|
|
153
|
+
largest connected set or `connected="leave_one_out"` the leave-one-out
|
|
154
|
+
connected set, all on the first two fixed effects listed unless
|
|
155
|
+
`connected_fe=` names another pair, and all before singletons are dropped, so
|
|
156
|
+
the fit equals one on data restricted beforehand. `sample()` labels the rows
|
|
157
|
+
they drop `"min_degree"`, `"disconnected"` and `"leave_one_out"`. The
|
|
158
|
+
`component` column of `fixef()` for the first non-streamed dimension is
|
|
159
|
+
numbered the same way, by rows in the estimation sample. With three or more
|
|
160
|
+
fixed effects, connectedness of one pair is necessary for identification but
|
|
161
|
+
not sufficient.
|
|
162
|
+
|
|
117
163
|
**Covariates.** Transformations (`I(age**2)`, `log(x)`), categoricals (`C(x)`,
|
|
118
164
|
string columns), interactions (`:`, `*`), event-study terms
|
|
119
165
|
(`i(year, treat, ref=2009)`). Terms spanned by the fixed effects are dropped
|
|
@@ -163,7 +209,8 @@ order, not the source's, so every row carries `row_id`, its position in the
|
|
|
163
209
|
source before any row was dropped (`row_id=` renames the column).
|
|
164
210
|
`fit.resid()` joins back onto the source on it, and `fit.sample()` returns the
|
|
165
211
|
source, lazily, with `row_id`, `in_sample` and `dropped_because` (null for rows
|
|
166
|
-
used; otherwise `"missing"`, `"
|
|
212
|
+
used; otherwise `"missing"`, `"min_degree"`, `"disconnected"`,
|
|
213
|
+
`"leave_one_out"`, `"singleton"` or `"separation"`), for looking at
|
|
167
214
|
the data net of what the fit dropped:
|
|
168
215
|
|
|
169
216
|
```python
|
|
@@ -430,6 +477,7 @@ Runnable, on simulated data, no setup — see [examples/](https://github.com/lee
|
|
|
430
477
|
| [`varying_slopes.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/varying_slopes.py) | worker-specific trends, and the low-level interface |
|
|
431
478
|
| [`out_of_core.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/out_of_core.py) | memory, disk, solvers, logging |
|
|
432
479
|
| [`reporting.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/reporting.py) | tables and plots via pyfixest |
|
|
480
|
+
| [`connected_sets.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/connected_sets.py) | connected components, `min_degree` pruning and the leave-one-out connected set, compared |
|
|
433
481
|
| [`leave_out_kss.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/leave_out_kss.py) | the KSS bias correction, checked against the effects the data was built from |
|
|
434
482
|
|
|
435
483
|
## Simulated data
|
|
@@ -497,8 +545,9 @@ print(lo.summary())
|
|
|
497
545
|
|
|
498
546
|
It follows Saggio's reference implementation,
|
|
499
547
|
[LeaveOutTwoWay](https://github.com/rsaggio87/LeaveOutTwoWay). It prunes to the
|
|
500
|
-
leave-one-out connected set
|
|
501
|
-
|
|
548
|
+
leave-one-out connected set (which `connected_components(...,
|
|
549
|
+
leave_one_out=True)` finds on its own, ahead of time), leaves out a
|
|
550
|
+
worker–firm match by default, and approximates leverages by random projection. It supports weights, standard
|
|
502
551
|
errors, and KSS's weak-identification diagnostic and q = 1 interval.
|
|
503
552
|
|
|
504
553
|
**[docs/kss.md](https://github.com/leetucker/hdfe-stream/blob/main/docs/kss.md)** covers the options, the standard errors and their
|
|
@@ -51,8 +51,12 @@ be describing a different model, so there is no "fit then decompose it" path
|
|
|
51
51
|
that skips this. `prune=False` checks instead of pruning, and fails if the panel
|
|
52
52
|
is not already connected — for callers who pruned upstream.
|
|
53
53
|
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
The pruning is the fit option `connected="leave_one_out"`, so it runs on the
|
|
55
|
+
rows the model can use, and the fit's `sample()` labels the rows it drops
|
|
56
|
+
`"leave_one_out"` (`lo.fit.sample()`); `lo.pruning` has the counts. To find the
|
|
57
|
+
set ahead of time, or to fit other things on it,
|
|
58
|
+
`connected_components(data, "worker_id + firm_id", leave_one_out=True)` returns
|
|
59
|
+
it as a table of the levels kept, and its `.largest()` is the pruned panel.
|
|
56
60
|
|
|
57
61
|
## What is left out: a match, or an observation
|
|
58
62
|
|
|
@@ -116,9 +120,9 @@ out an observation reads the sorted rows and cell tables a fit otherwise deletes
|
|
|
116
120
|
when it finishes, so that needs `keep_intermediates=True`:
|
|
117
121
|
|
|
118
122
|
```python
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
lo
|
|
123
|
+
fit = feols_stream(fml, "data/*.parquet", workdir="scratch",
|
|
124
|
+
connected="leave_one_out")
|
|
125
|
+
lo = fit.leave_out_kss(n_draws=250)
|
|
122
126
|
```
|
|
123
127
|
|
|
124
128
|
The two give identical numbers.
|
|
@@ -112,7 +112,14 @@ Were W2 kept, `leave_out_components` would find leverage 1 and raise "not
|
|
|
112
112
|
leave-one-out connected".
|
|
113
113
|
|
|
114
114
|
The pruned sample matches xhdfe's row for row on three panels, with the same
|
|
115
|
-
number of rounds (3, 1, 2). Diagnostics report
|
|
115
|
+
number of rounds (3, 1, 2). Diagnostics report the rounds
|
|
116
|
+
(`lo.pruning["rounds"]`).
|
|
117
|
+
|
|
118
|
+
The pruning runs on the rows the model can use: it is the fit's
|
|
119
|
+
`connected="leave_one_out"`, which comes after rows with a missing outcome,
|
|
120
|
+
covariate or fixed effect are dropped. A row the regression cannot use connects
|
|
121
|
+
nothing in it, so it cannot keep a worker or firm in the set. On complete data
|
|
122
|
+
this changes nothing.
|
|
116
123
|
|
|
117
124
|
### 1.3 Workers with a single observation · *agrees*
|
|
118
125
|
|
|
@@ -125,8 +132,8 @@ number of rounds (3, 1, 2). Diagnostics report `pruning_rounds`.
|
|
|
125
132
|
A worker seen once has leverage exactly 1: the worker effect is determined by that
|
|
126
133
|
observation. A pendant worker is never an articulation point, so pruning alone
|
|
127
134
|
never removes it, and the leverage check would raise. Tested on a minimal
|
|
128
|
-
panel. The resulting sample matches xhdfe's. Diagnostics report
|
|
129
|
-
`
|
|
135
|
+
panel. The resulting sample matches xhdfe's. Diagnostics report them
|
|
136
|
+
(`lo.pruning["single_observation"]`).
|
|
130
137
|
|
|
131
138
|
### 1.4 The graph searched for articulation points · *Numerical*
|
|
132
139
|
|
|
@@ -56,7 +56,9 @@ group_by a slice of whole fe[0] groups (about batch_rows rows) at a time.
|
|
|
56
56
|
Pipeline
|
|
57
57
|
--------
|
|
58
58
|
Pass 0 (Polars, streaming) evaluate the design as Polars expressions; drop
|
|
59
|
-
rows with missing / non-finite values
|
|
59
|
+
rows with missing / non-finite values (and, as
|
|
60
|
+
asked, outside the largest connected set, and
|
|
61
|
+
singletons); factorize
|
|
60
62
|
the non-streamed FE and cluster ids;
|
|
61
63
|
hash-partition rows into fe[0] buckets; sort each
|
|
62
64
|
bucket and assign dense fe[0] codes.
|
|
@@ -94,6 +96,8 @@ kernels_glm.py numba kernels, GLM cell sums and group effects
|
|
|
94
96
|
formula.py pyfixest formula -> Polars expressions (input side)
|
|
95
97
|
reporting.py pyfixest etable/coefplot views (output side)
|
|
96
98
|
feterms.py 'worker_id[t]' / 'firm_id^year' parsing
|
|
99
|
+
connected.py `connected_components`: components of a pair of FE
|
|
100
|
+
dimensions, the largest connected set (also `connected=`)
|
|
97
101
|
results.py `HDFEResult`, `HDFEMulti`
|
|
98
102
|
workspace.py run directories, `cleanup()`
|
|
99
103
|
report.py logging / printing
|
|
@@ -101,14 +105,14 @@ utils.py small shared helpers
|
|
|
101
105
|
"""
|
|
102
106
|
|
|
103
107
|
from .api import feglm_stream, fepois_stream, feols_stream
|
|
108
|
+
from .connected import ConnectedComponents, connected_components
|
|
104
109
|
from .estimator import StreamingHDFE
|
|
105
110
|
from .glm import StreamingGLM
|
|
106
111
|
from .leaveout_se import ComponentSE
|
|
107
112
|
from .leaveout_weakid import WeakIdDiagnostics
|
|
108
|
-
from .leaveout import
|
|
113
|
+
from .leaveout import leave_out_kss
|
|
109
114
|
from .reporting import etable
|
|
110
115
|
from .results import HDFEMulti, HDFEResult
|
|
111
|
-
from .utils import iter_group_chunks
|
|
112
116
|
from .workspace import cleanup
|
|
113
117
|
|
|
114
118
|
__all__ = [
|
|
@@ -123,9 +127,9 @@ __all__ = [
|
|
|
123
127
|
"leave_out_kss",
|
|
124
128
|
"ComponentSE",
|
|
125
129
|
"WeakIdDiagnostics",
|
|
126
|
-
"
|
|
130
|
+
"connected_components",
|
|
131
|
+
"ConnectedComponents",
|
|
127
132
|
"cleanup",
|
|
128
|
-
"iter_group_chunks",
|
|
129
133
|
]
|
|
130
134
|
|
|
131
135
|
def _detect_version():
|
|
@@ -133,7 +137,7 @@ def _detect_version():
|
|
|
133
137
|
from importlib.metadata import version
|
|
134
138
|
return version("hdfe-stream")
|
|
135
139
|
except Exception: # source tree without an installed dist
|
|
136
|
-
return "0.
|
|
140
|
+
return "0.3.0"
|
|
137
141
|
|
|
138
142
|
|
|
139
143
|
__version__ = _detect_version()
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import
|
|
6
|
+
from typing import Mapping, Sequence, Union
|
|
7
7
|
|
|
8
8
|
import polars as pl
|
|
9
9
|
|
|
@@ -15,4 +15,3 @@ Vcov = Union[str, Mapping[str, str]]
|
|
|
15
15
|
# column name, (name, expression), or a mapping / sequence of them
|
|
16
16
|
Variables = Union[
|
|
17
17
|
str, tuple, Sequence[Union[str, tuple]], Mapping[str, pl.Expr]]
|
|
18
|
-
Options = Any
|
|
@@ -17,16 +17,6 @@ from .results import HDFEMulti, HDFEResult, _vcov_key
|
|
|
17
17
|
from .utils import _as_lazy
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
class _FormulaEstimator:
|
|
21
|
-
def __init__(self, fml: str, workdir: PathLike | None, options: dict[str, Any]) -> None:
|
|
22
|
-
self.fml, self.workdir, self.options = fml, workdir, options
|
|
23
|
-
|
|
24
|
-
def fit(self, data: Source, vcov: Vcov = "iid", cluster: Sequence[str] | str = (),
|
|
25
|
-
fe_dof: str = "exact") -> HDFEResult | HDFEMulti:
|
|
26
|
-
return feols_stream(self.fml, data, self.workdir, vcov=vcov, cluster=cluster,
|
|
27
|
-
fe_dof=fe_dof, **self.options)
|
|
28
|
-
|
|
29
|
-
|
|
30
20
|
def feols_stream(fml: str, data: Source, workdir: PathLike | None = None,
|
|
31
21
|
vcov: Vcov = "iid", cluster: Sequence[str] | str = (),
|
|
32
22
|
fe_dof: str = "exact", **options: Any) -> HDFEResult | HDFEMulti:
|
|
@@ -49,7 +39,10 @@ def feols_stream(fml: str, data: Source, workdir: PathLike | None = None,
|
|
|
49
39
|
**options : passed to StreamingHDFE (fixef_rm, stream, weights,
|
|
50
40
|
weights_type, solver, precond, keep, verbose, logger, log_level,
|
|
51
41
|
...). fixef_rm='singleton' (the default, as in pyfixest) drops
|
|
52
|
-
singleton observations; 'none' keeps them.
|
|
42
|
+
singleton observations; 'none' keeps them. connected='largest'
|
|
43
|
+
keeps only the largest connected set of the first two fixed
|
|
44
|
+
effects listed, and min_degree= prunes their levels with too
|
|
45
|
+
few distinct partners first (see `connected_components`).
|
|
53
46
|
|
|
54
47
|
Varying slopes on the streamed dimension: "y ~ x | worker_id[t] + firm_id".
|
|
55
48
|
The worker FE file then has fe_worker_id (intercept) and fe_worker_id[t] (slope)
|