hdfe-stream 0.2.2__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/CHANGELOG.md +71 -5
  2. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/CITATION.cff +2 -2
  3. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/PKG-INFO +54 -4
  4. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/README.md +52 -3
  5. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/kss.md +9 -5
  6. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/kss_methodological_differences.md +10 -3
  7. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/__init__.py +10 -6
  8. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/_types.py +1 -2
  9. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/api.py +4 -11
  10. hdfe_stream-0.3.0/hdfe_stream/connected.py +654 -0
  11. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/estimator.py +67 -13
  12. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/glm.py +8 -20
  13. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/inference.py +3 -2
  14. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/inverse.py +3 -3
  15. hdfe_stream-0.3.0/hdfe_stream/kernels_graph.py +352 -0
  16. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/kernels_inverse.py +0 -17
  17. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/leaveout.py +30 -176
  18. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/leaveout_match.py +0 -1
  19. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/passes.py +113 -14
  20. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/results.py +25 -4
  21. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/utils.py +6 -0
  22. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream.egg-info/PKG-INFO +54 -4
  23. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream.egg-info/SOURCES.txt +2 -0
  24. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/pyproject.toml +2 -1
  25. hdfe_stream-0.3.0/tests/test_connected.py +449 -0
  26. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_inverse.py +27 -0
  27. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_leaveout.py +161 -63
  28. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_sample.py +4 -3
  29. hdfe_stream-0.2.2/hdfe_stream/kernels_graph.py +0 -135
  30. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/LICENSE.md +0 -0
  31. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/MANIFEST.in +0 -0
  32. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_disk.dark.svg +0 -0
  33. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_disk.light.svg +0 -0
  34. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_memory.dark.svg +0 -0
  35. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_memory.light.svg +0 -0
  36. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_time.dark.svg +0 -0
  37. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/akm_time.light.svg +0 -0
  38. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_disk.dark.svg +0 -0
  39. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_disk.light.svg +0 -0
  40. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_memory.dark.svg +0 -0
  41. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_memory.light.svg +0 -0
  42. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_time.dark.svg +0 -0
  43. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/covariates_time.light.svg +0 -0
  44. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_disk.dark.svg +0 -0
  45. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_disk.light.svg +0 -0
  46. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_memory.dark.svg +0 -0
  47. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_memory.light.svg +0 -0
  48. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_time.dark.svg +0 -0
  49. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_covariates_time.light.svg +0 -0
  50. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_disk.dark.svg +0 -0
  51. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_disk.light.svg +0 -0
  52. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_memory.dark.svg +0 -0
  53. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_memory.light.svg +0 -0
  54. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_time.dark.svg +0 -0
  55. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/glm_time.light.svg +0 -0
  56. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_disk.dark.svg +0 -0
  57. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_disk.light.svg +0 -0
  58. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_memory.dark.svg +0 -0
  59. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_memory.light.svg +0 -0
  60. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_time.dark.svg +0 -0
  61. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/figures/kss_time.light.svg +0 -0
  62. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/docs/glm.md +0 -0
  63. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/_columns.py +0 -0
  64. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/am_interval.py +0 -0
  65. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/families.py +0 -0
  66. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/feterms.py +0 -0
  67. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/formula.py +0 -0
  68. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/kernels_base.py +0 -0
  69. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/kernels_glm.py +0 -0
  70. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/kernels_se.py +0 -0
  71. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/kernels_slopes.py +0 -0
  72. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/leaveout_se.py +0 -0
  73. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/leaveout_weakid.py +0 -0
  74. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/py.typed +0 -0
  75. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/report.py +0 -0
  76. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/reporting.py +0 -0
  77. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/simulate.py +0 -0
  78. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/solve.py +0 -0
  79. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream/workspace.py +0 -0
  80. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream.egg-info/dependency_links.txt +0 -0
  81. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream.egg-info/requires.txt +0 -0
  82. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/hdfe_stream.egg-info/top_level.txt +0 -0
  83. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/setup.cfg +0 -0
  84. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/conftest.py +0 -0
  85. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_am_interval.py +0 -0
  86. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_cleanup.py +0 -0
  87. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_column_names.py +0 -0
  88. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_crv3.py +0 -0
  89. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_feis.py +0 -0
  90. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_fewer_fe.py +0 -0
  91. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_formula.py +0 -0
  92. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_glm.py +0 -0
  93. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_iv_weights.py +0 -0
  94. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_logging.py +0 -0
  95. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_lowlevel.py +0 -0
  96. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_ols.py +0 -0
  97. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_reporting.py +0 -0
  98. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_simulate.py +0 -0
  99. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_singletons.py +0 -0
  100. {hdfe_stream-0.2.2 → hdfe_stream-0.3.0}/tests/test_vcov.py +0 -0
@@ -1,5 +1,76 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.3.0
4
+
5
+ New features.
6
+
7
+ - **`connected_components`** finds the connected components of the graph of
8
+ two fixed-effect dimensions (either may be an interaction, `firm_id^year`)
9
+ by union-find over the rows as they stream, holding level-sized arrays
10
+ only. It returns a table per dimension giving each level its component,
11
+ numbered by size so that component 0 is the largest (ties go to the
12
+ component with the smallest level of the first dimension), and a table of
13
+ component sizes. `largest()` filters the data to the largest connected
14
+ set and `label()` adds each row's component, both lazily (on the data the
15
+ components were found in, or on other data passed to them). `min_degree`
16
+ removes levels with fewer distinct partners in the other dimension,
17
+ repeating until none is left, before the components are found: `(2, 1)`
18
+ keeps only movers, `2` the 2-core. `leave_one_out=True` prunes to the
19
+ leave-one-out connected set of KSS estimation instead. `rank_by` says
20
+ whether "largest" counts rows (the default) or the levels of one dimension.
21
+ - **`connected=`, `min_degree=` and `connected_fe=`** in `feols_stream`,
22
+ `fepois_stream`, `feglm_stream` and `StreamingHDFE` restrict the fit to a
23
+ pair of fixed effects (`connected_fe`, by default the first two listed),
24
+ among the rows with no missing values and before singletons are dropped:
25
+ `min_degree` prunes levels with too few distinct partners, then
26
+ `connected="largest"` keeps the largest connected set of what is left, or
27
+ `connected="leave_one_out"` the leave-one-out connected set, as
28
+ `connected_components` does. `sample()` reports the rows they drop as
29
+ `"min_degree"`, `"disconnected"` and `"leave_one_out"`, and
30
+ `diagnostics["connected"]` and the summary count them.
31
+ - **Python 3.15** is supported and tested in CI. pyfixest, which the
32
+ `formula` extra needs, has no 3.15 wheels yet (0.60.0), and building it
33
+ from source needs Rust and `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1`; the
34
+ rest of the package installs from wheels.
35
+
36
+ Changes that can affect existing code.
37
+
38
+ - **`leave_one_out_connected` and `LeaveOutSet` are removed.** Use
39
+ `connected_components(data, "worker_id + firm_id", leave_one_out=True)`,
40
+ whose `.largest()` is the pruned panel, or `connected="leave_one_out"`
41
+ in a fit. The set is the same: on simulated panels with many components,
42
+ workers observed once and several rounds of cut workers, the workers and
43
+ firms kept are identical. It is found 5-6 times faster (38s against 218s on
44
+ 150 million rows), and a null worker or firm id no longer makes it fail.
45
+ - **`leave_out_kss` prunes in the fit**, with `connected="leave_one_out"`, so
46
+ the leave-one-out connected set is that of the rows the model can use:
47
+ before, rows with a missing outcome or covariate were part of the network
48
+ being pruned. `lo.pruning` is now a dict of the pruning's counts (the
49
+ fit's `diagnostics["connected"]["leave_one_out"]`), and the rows it dropped
50
+ are those `lo.fit.sample()` labels `"leave_one_out"`.
51
+ - **The `component` column of `fixef()` is numbered by size**, so component
52
+ 0 is the largest (by rows in the estimation sample), as in
53
+ `connected_components`. Ties go to the component with the smallest level of
54
+ that dimension. It was numbered by each component's smallest level.
55
+ - **Removed `hdfe_stream.iter_group_chunks` and `StreamingHDFE.from_formula`.**
56
+ The first read only the fit's internal row files, sorted by its internal
57
+ group code, and stays internal; the second duplicated `feols_stream`.
58
+
59
+ Fixes.
60
+
61
+ - **The leave-out operator's null space is right with several connected
62
+ components.** `apply_inverse` built one null vector for every number up to
63
+ the largest component label, which with several components left gaps, and
64
+ each gap added a spurious vector. Leave-out estimation runs on a connected
65
+ set and was not affected.
66
+
67
+ Documentation.
68
+
69
+ - **New example, `connected_sets.py`**, and additions to the others:
70
+ `sample()` and joining the residuals back on `row_id`, `fe_dof`, and CRV3
71
+ (`formulas.py`); `summary_json` (`quickstart.py`); `keep=` and a
72
+ decomposition by group (`akm_variance.py`); and `StreamingGLM` (`glm.py`).
73
+
3
74
  ## 0.2.2
4
75
 
5
76
  - **Polars 2.0 support.** `fit.sample()` returns the source's rows in the source's
@@ -38,11 +109,6 @@
38
109
  unchanged up to rounding.
39
110
  - **Removed `triple_budget` and `dense_max_levels`** from `StreamingHDFE`. They
40
111
  tuned the old way of building S and have no counterpart in the new one.
41
- - **Per-iteration CG messages are logged at DEBUG.** With `verbose=True`, the
42
- "CG iter N: max rel. residual" lines go to the logger at DEBUG, whatever
43
- `log_level` is, so they appear only when the logger is enabled for DEBUG.
44
- Without a logger they are no longer printed. Other progress messages are
45
- unchanged.
46
112
  - **Importing hdfe_stream costs a little less memory.** p-values, critical
47
113
  values and the weak-identification interval use `scipy.special`'s
48
114
  distribution functions instead of `scipy.stats`, which is no longer
@@ -7,8 +7,8 @@ authors:
7
7
  - family-names: Tucker
8
8
  given-names: Lee
9
9
  email: lee.c.tucker@census.gov
10
- version: 0.2.2
11
- date-released: 2026-10-08
10
+ version: 0.3.0
11
+ date-released: 2026-10-14
12
12
  license: MIT
13
13
  repository-code: "https://github.com/leetucker/hdfe-stream"
14
14
  url: "https://github.com/leetucker/hdfe-stream"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hdfe-stream
3
- Version: 0.2.2
3
+ Version: 0.3.0
4
4
  Summary: Out-of-core regression with several high-dimensional fixed effects: OLS, Poisson, logit and probit, with AKM/KSS variance decompositions
5
5
  Author: Lee Tucker
6
6
  License-Expression: MIT
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.11
19
19
  Classifier: Programming Language :: Python :: 3.12
20
20
  Classifier: Programming Language :: Python :: 3.13
21
21
  Classifier: Programming Language :: Python :: 3.14
22
+ Classifier: Programming Language :: Python :: 3.15
22
23
  Classifier: Topic :: Scientific/Engineering
23
24
  Classifier: Typing :: Typed
24
25
  Requires-Python: >=3.10
@@ -139,6 +140,11 @@ pip install hdfe-stream[formula] # formula syntax needs pyfixest
139
140
  | `all` | all of the above |
140
141
  | `test` | the test suite |
141
142
 
143
+ On Python 3.15, pyfixest 0.60 (for the `formula` extra) has no wheels yet. It
144
+ builds from source with a Rust toolchain and
145
+ `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` set; everything else installs from
146
+ wheels.
147
+
142
148
  Without `formula` you still get `StreamingHDFE`, the lower-level interface that
143
149
  takes column names or Polars expressions instead of a formula string.
144
150
 
@@ -163,6 +169,47 @@ left (`fixef_rm="singleton"`, the default; `fixef_rm="none"` keeps them).
163
169
  Finding them costs a few extra passes over the fixed-effect columns and memory
164
170
  for the singleton levels, not for the rows.
165
171
 
172
+ **Connected sets.** Worker and firm effects are only comparable within a
173
+ connected set: firms linked, directly or through other firms, by workers who
174
+ moved between them. `connected_components(data, "worker_id + firm_id")` finds
175
+ the connected components of the graph of two fixed effects (either may be an
176
+ interaction, `firm_id^year`) by union-find over the rows as they stream, and
177
+ returns small tables, one per dimension, giving each level its component.
178
+ Components are numbered by size, so `component == 0` is the largest.
179
+ `.largest()` is the largest connected set and `.label()` every row with
180
+ its component, both lazily.
181
+
182
+ `min_degree` first removes levels with too few distinct partners in the other
183
+ dimension (distinct firms for a worker, distinct workers for a firm),
184
+ repeating until none is left, then finds the components of what remains.
185
+ `min_degree=(2, 1)` keeps movers only: workers seen at two or more firms, and
186
+ the firms they were seen at. `min_degree=2` also removes firms with a single
187
+ worker, and repeats, since removing a firm can leave a worker with one firm and
188
+ removing a worker can leave a firm with one worker, until every worker left
189
+ was seen at two or more firms and every firm employed two or more of them (the
190
+ network's "2-core").
191
+
192
+ `leave_one_out=True` instead prunes to the **leave-one-out connected set** that
193
+ KSS estimation needs (and `leave_out_kss` applies), so it can be found ahead of
194
+ time: the largest connected set, without workers observed only once, and
195
+ without any worker whose departure would split the network into pieces,
196
+ removed repeatedly until none is left. It differs from `min_degree=2`. It keeps
197
+ stayers with at least two observations, which `min_degree=2` removes, and it
198
+ removes movers that `min_degree=2` keeps. Every mover and firm it keeps,
199
+ `min_degree=2` keeps too, and on the movers and firms the two coincide only
200
+ when no single mover holds the network together.
201
+
202
+ In a fit, `min_degree=` prunes first, then `connected="largest"` keeps the
203
+ largest connected set or `connected="leave_one_out"` the leave-one-out
204
+ connected set, all on the first two fixed effects listed unless
205
+ `connected_fe=` names another pair, and all before singletons are dropped, so
206
+ the fit equals one on data restricted beforehand. `sample()` labels the rows
207
+ they drop `"min_degree"`, `"disconnected"` and `"leave_one_out"`. The
208
+ `component` column of `fixef()` for the first non-streamed dimension is
209
+ numbered the same way, by rows in the estimation sample. With three or more
210
+ fixed effects, connectedness of one pair is necessary for identification but
211
+ not sufficient.
212
+
166
213
  **Covariates.** Transformations (`I(age**2)`, `log(x)`), categoricals (`C(x)`,
167
214
  string columns), interactions (`:`, `*`), event-study terms
168
215
  (`i(year, treat, ref=2009)`). Terms spanned by the fixed effects are dropped
@@ -212,7 +259,8 @@ order, not the source's, so every row carries `row_id`, its position in the
212
259
  source before any row was dropped (`row_id=` renames the column).
213
260
  `fit.resid()` joins back onto the source on it, and `fit.sample()` returns the
214
261
  source, lazily, with `row_id`, `in_sample` and `dropped_because` (null for rows
215
- used; otherwise `"missing"`, `"singleton"` or `"separation"`), for looking at
262
+ used; otherwise `"missing"`, `"min_degree"`, `"disconnected"`,
263
+ `"leave_one_out"`, `"singleton"` or `"separation"`), for looking at
216
264
  the data net of what the fit dropped:
217
265
 
218
266
  ```python
@@ -479,6 +527,7 @@ Runnable, on simulated data, no setup — see [examples/](https://github.com/lee
479
527
  | [`varying_slopes.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/varying_slopes.py) | worker-specific trends, and the low-level interface |
480
528
  | [`out_of_core.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/out_of_core.py) | memory, disk, solvers, logging |
481
529
  | [`reporting.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/reporting.py) | tables and plots via pyfixest |
530
+ | [`connected_sets.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/connected_sets.py) | connected components, `min_degree` pruning and the leave-one-out connected set, compared |
482
531
  | [`leave_out_kss.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/leave_out_kss.py) | the KSS bias correction, checked against the effects the data was built from |
483
532
 
484
533
  ## Simulated data
@@ -546,8 +595,9 @@ print(lo.summary())
546
595
 
547
596
  It follows Saggio's reference implementation,
548
597
  [LeaveOutTwoWay](https://github.com/rsaggio87/LeaveOutTwoWay). It prunes to the
549
- leave-one-out connected set, leaves out a worker–firm match by default, and
550
- approximates leverages by random projection. It supports weights, standard
598
+ leave-one-out connected set (which `connected_components(...,
599
+ leave_one_out=True)` finds on its own, ahead of time), leaves out a
600
+ worker–firm match by default, and approximates leverages by random projection. It supports weights, standard
551
601
  errors, and KSS's weak-identification diagnostic and q = 1 interval.
552
602
 
553
603
  **[docs/kss.md](https://github.com/leetucker/hdfe-stream/blob/main/docs/kss.md)** covers the options, the standard errors and their
@@ -90,6 +90,11 @@ pip install hdfe-stream[formula] # formula syntax needs pyfixest
90
90
  | `all` | all of the above |
91
91
  | `test` | the test suite |
92
92
 
93
+ On Python 3.15, pyfixest 0.60 (for the `formula` extra) has no wheels yet. It
94
+ builds from source with a Rust toolchain and
95
+ `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` set; everything else installs from
96
+ wheels.
97
+
93
98
  Without `formula` you still get `StreamingHDFE`, the lower-level interface that
94
99
  takes column names or Polars expressions instead of a formula string.
95
100
 
@@ -114,6 +119,47 @@ left (`fixef_rm="singleton"`, the default; `fixef_rm="none"` keeps them).
114
119
  Finding them costs a few extra passes over the fixed-effect columns and memory
115
120
  for the singleton levels, not for the rows.
116
121
 
122
+ **Connected sets.** Worker and firm effects are only comparable within a
123
+ connected set: firms linked, directly or through other firms, by workers who
124
+ moved between them. `connected_components(data, "worker_id + firm_id")` finds
125
+ the connected components of the graph of two fixed effects (either may be an
126
+ interaction, `firm_id^year`) by union-find over the rows as they stream, and
127
+ returns small tables, one per dimension, giving each level its component.
128
+ Components are numbered by size, so `component == 0` is the largest.
129
+ `.largest()` is the largest connected set and `.label()` every row with
130
+ its component, both lazily.
131
+
132
+ `min_degree` first removes levels with too few distinct partners in the other
133
+ dimension (distinct firms for a worker, distinct workers for a firm),
134
+ repeating until none is left, then finds the components of what remains.
135
+ `min_degree=(2, 1)` keeps movers only: workers seen at two or more firms, and
136
+ the firms they were seen at. `min_degree=2` also removes firms with a single
137
+ worker, and repeats, since removing a firm can leave a worker with one firm and
138
+ removing a worker can leave a firm with one worker, until every worker left
139
+ was seen at two or more firms and every firm employed two or more of them (the
140
+ network's "2-core").
141
+
142
+ `leave_one_out=True` instead prunes to the **leave-one-out connected set** that
143
+ KSS estimation needs (and `leave_out_kss` applies), so it can be found ahead of
144
+ time: the largest connected set, without workers observed only once, and
145
+ without any worker whose departure would split the network into pieces,
146
+ removed repeatedly until none is left. It differs from `min_degree=2`. It keeps
147
+ stayers with at least two observations, which `min_degree=2` removes, and it
148
+ removes movers that `min_degree=2` keeps. Every mover and firm it keeps,
149
+ `min_degree=2` keeps too, and on the movers and firms the two coincide only
150
+ when no single mover holds the network together.
151
+
152
+ In a fit, `min_degree=` prunes first, then `connected="largest"` keeps the
153
+ largest connected set or `connected="leave_one_out"` the leave-one-out
154
+ connected set, all on the first two fixed effects listed unless
155
+ `connected_fe=` names another pair, and all before singletons are dropped, so
156
+ the fit equals one on data restricted beforehand. `sample()` labels the rows
157
+ they drop `"min_degree"`, `"disconnected"` and `"leave_one_out"`. The
158
+ `component` column of `fixef()` for the first non-streamed dimension is
159
+ numbered the same way, by rows in the estimation sample. With three or more
160
+ fixed effects, connectedness of one pair is necessary for identification but
161
+ not sufficient.
162
+
117
163
  **Covariates.** Transformations (`I(age**2)`, `log(x)`), categoricals (`C(x)`,
118
164
  string columns), interactions (`:`, `*`), event-study terms
119
165
  (`i(year, treat, ref=2009)`). Terms spanned by the fixed effects are dropped
@@ -163,7 +209,8 @@ order, not the source's, so every row carries `row_id`, its position in the
163
209
  source before any row was dropped (`row_id=` renames the column).
164
210
  `fit.resid()` joins back onto the source on it, and `fit.sample()` returns the
165
211
  source, lazily, with `row_id`, `in_sample` and `dropped_because` (null for rows
166
- used; otherwise `"missing"`, `"singleton"` or `"separation"`), for looking at
212
+ used; otherwise `"missing"`, `"min_degree"`, `"disconnected"`,
213
+ `"leave_one_out"`, `"singleton"` or `"separation"`), for looking at
167
214
  the data net of what the fit dropped:
168
215
 
169
216
  ```python
@@ -430,6 +477,7 @@ Runnable, on simulated data, no setup — see [examples/](https://github.com/lee
430
477
  | [`varying_slopes.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/varying_slopes.py) | worker-specific trends, and the low-level interface |
431
478
  | [`out_of_core.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/out_of_core.py) | memory, disk, solvers, logging |
432
479
  | [`reporting.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/reporting.py) | tables and plots via pyfixest |
480
+ | [`connected_sets.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/connected_sets.py) | connected components, `min_degree` pruning and the leave-one-out connected set, compared |
433
481
  | [`leave_out_kss.py`](https://github.com/leetucker/hdfe-stream/blob/main/examples/leave_out_kss.py) | the KSS bias correction, checked against the effects the data was built from |
434
482
 
435
483
  ## Simulated data
@@ -497,8 +545,9 @@ print(lo.summary())
497
545
 
498
546
  It follows Saggio's reference implementation,
499
547
  [LeaveOutTwoWay](https://github.com/rsaggio87/LeaveOutTwoWay). It prunes to the
500
- leave-one-out connected set, leaves out a worker–firm match by default, and
501
- approximates leverages by random projection. It supports weights, standard
548
+ leave-one-out connected set (which `connected_components(...,
549
+ leave_one_out=True)` finds on its own, ahead of time), leaves out a
550
+ worker–firm match by default, and approximates leverages by random projection. It supports weights, standard
502
551
  errors, and KSS's weak-identification diagnostic and q = 1 interval.
503
552
 
504
553
  **[docs/kss.md](https://github.com/leetucker/hdfe-stream/blob/main/docs/kss.md)** covers the options, the standard errors and their
@@ -51,8 +51,12 @@ be describing a different model, so there is no "fit then decompose it" path
51
51
  that skips this. `prune=False` checks instead of pruning, and fails if the panel
52
52
  is not already connected — for callers who pruned upstream.
53
53
 
54
- `leave_one_out_connected(data)` is available on its own if you want the pruned
55
- panel for something else.
54
+ The pruning is the fit option `connected="leave_one_out"`, so it runs on the
55
+ rows the model can use, and the fit's `sample()` labels the rows it drops
56
+ `"leave_one_out"` (`lo.fit.sample()`); `lo.pruning` has the counts. To find the
57
+ set ahead of time, or to fit other things on it,
58
+ `connected_components(data, "worker_id + firm_id", leave_one_out=True)` returns
59
+ it as a table of the levels kept, and its `.largest()` is the pruned panel.
56
60
 
57
61
  ## What is left out: a match, or an observation
58
62
 
@@ -116,9 +120,9 @@ out an observation reads the sorted rows and cell tables a fit otherwise deletes
116
120
  when it finishes, so that needs `keep_intermediates=True`:
117
121
 
118
122
  ```python
119
- panel = leave_one_out_connected("data/*.parquet")
120
- fit = feols_stream(fml, panel.data, workdir="scratch")
121
- lo = fit.leave_out_kss(n_draws=250)
123
+ fit = feols_stream(fml, "data/*.parquet", workdir="scratch",
124
+ connected="leave_one_out")
125
+ lo = fit.leave_out_kss(n_draws=250)
122
126
  ```
123
127
 
124
128
  The two give identical numbers.
@@ -112,7 +112,14 @@ Were W2 kept, `leave_out_components` would find leverage 1 and raise "not
112
112
  leave-one-out connected".
113
113
 
114
114
  The pruned sample matches xhdfe's row for row on three panels, with the same
115
- number of rounds (3, 1, 2). Diagnostics report `pruning_rounds`.
115
+ number of rounds (3, 1, 2). Diagnostics report the rounds
116
+ (`lo.pruning["rounds"]`).
117
+
118
+ The pruning runs on the rows the model can use: it is the fit's
119
+ `connected="leave_one_out"`, which comes after rows with a missing outcome,
120
+ covariate or fixed effect are dropped. A row the regression cannot use connects
121
+ nothing in it, so it cannot keep a worker or firm in the set. On complete data
122
+ this changes nothing.
116
123
 
117
124
  ### 1.3 Workers with a single observation · *agrees*
118
125
 
@@ -125,8 +132,8 @@ number of rounds (3, 1, 2). Diagnostics report `pruning_rounds`.
125
132
  A worker seen once has leverage exactly 1: the worker effect is determined by that
126
133
  observation. A pendant worker is never an articulation point, so pruning alone
127
134
  never removes it, and the leverage check would raise. Tested on a minimal
128
- panel. The resulting sample matches xhdfe's. Diagnostics report
129
- `single_observation_workers`.
135
+ panel. The resulting sample matches xhdfe's. Diagnostics report them
136
+ (`lo.pruning["single_observation"]`).
130
137
 
131
138
  ### 1.4 The graph searched for articulation points · *Numerical*
132
139
 
@@ -56,7 +56,9 @@ group_by a slice of whole fe[0] groups (about batch_rows rows) at a time.
56
56
  Pipeline
57
57
  --------
58
58
  Pass 0 (Polars, streaming) evaluate the design as Polars expressions; drop
59
- rows with missing / non-finite values; factorize
59
+ rows with missing / non-finite values (and, as
60
+ asked, outside the largest connected set, and
61
+ singletons); factorize
60
62
  the non-streamed FE and cluster ids;
61
63
  hash-partition rows into fe[0] buckets; sort each
62
64
  bucket and assign dense fe[0] codes.
@@ -94,6 +96,8 @@ kernels_glm.py numba kernels, GLM cell sums and group effects
94
96
  formula.py pyfixest formula -> Polars expressions (input side)
95
97
  reporting.py pyfixest etable/coefplot views (output side)
96
98
  feterms.py 'worker_id[t]' / 'firm_id^year' parsing
99
+ connected.py `connected_components`: components of a pair of FE
100
+ dimensions, the largest connected set (also `connected=`)
97
101
  results.py `HDFEResult`, `HDFEMulti`
98
102
  workspace.py run directories, `cleanup()`
99
103
  report.py logging / printing
@@ -101,14 +105,14 @@ utils.py small shared helpers
101
105
  """
102
106
 
103
107
  from .api import feglm_stream, fepois_stream, feols_stream
108
+ from .connected import ConnectedComponents, connected_components
104
109
  from .estimator import StreamingHDFE
105
110
  from .glm import StreamingGLM
106
111
  from .leaveout_se import ComponentSE
107
112
  from .leaveout_weakid import WeakIdDiagnostics
108
- from .leaveout import leave_one_out_connected, leave_out_kss
113
+ from .leaveout import leave_out_kss
109
114
  from .reporting import etable
110
115
  from .results import HDFEMulti, HDFEResult
111
- from .utils import iter_group_chunks
112
116
  from .workspace import cleanup
113
117
 
114
118
  __all__ = [
@@ -123,9 +127,9 @@ __all__ = [
123
127
  "leave_out_kss",
124
128
  "ComponentSE",
125
129
  "WeakIdDiagnostics",
126
- "leave_one_out_connected",
130
+ "connected_components",
131
+ "ConnectedComponents",
127
132
  "cleanup",
128
- "iter_group_chunks",
129
133
  ]
130
134
 
131
135
  def _detect_version():
@@ -133,7 +137,7 @@ def _detect_version():
133
137
  from importlib.metadata import version
134
138
  return version("hdfe-stream")
135
139
  except Exception: # source tree without an installed dist
136
- return "0.2.2"
140
+ return "0.3.0"
137
141
 
138
142
 
139
143
  __version__ = _detect_version()
@@ -3,7 +3,7 @@
3
3
  from __future__ import annotations
4
4
 
5
5
  from pathlib import Path
6
- from typing import Any, Mapping, Sequence, Union
6
+ from typing import Mapping, Sequence, Union
7
7
 
8
8
  import polars as pl
9
9
 
@@ -15,4 +15,3 @@ Vcov = Union[str, Mapping[str, str]]
15
15
  # column name, (name, expression), or a mapping / sequence of them
16
16
  Variables = Union[
17
17
  str, tuple, Sequence[Union[str, tuple]], Mapping[str, pl.Expr]]
18
- Options = Any
@@ -17,16 +17,6 @@ from .results import HDFEMulti, HDFEResult, _vcov_key
17
17
  from .utils import _as_lazy
18
18
 
19
19
 
20
- class _FormulaEstimator:
21
- def __init__(self, fml: str, workdir: PathLike | None, options: dict[str, Any]) -> None:
22
- self.fml, self.workdir, self.options = fml, workdir, options
23
-
24
- def fit(self, data: Source, vcov: Vcov = "iid", cluster: Sequence[str] | str = (),
25
- fe_dof: str = "exact") -> HDFEResult | HDFEMulti:
26
- return feols_stream(self.fml, data, self.workdir, vcov=vcov, cluster=cluster,
27
- fe_dof=fe_dof, **self.options)
28
-
29
-
30
20
  def feols_stream(fml: str, data: Source, workdir: PathLike | None = None,
31
21
  vcov: Vcov = "iid", cluster: Sequence[str] | str = (),
32
22
  fe_dof: str = "exact", **options: Any) -> HDFEResult | HDFEMulti:
@@ -49,7 +39,10 @@ def feols_stream(fml: str, data: Source, workdir: PathLike | None = None,
49
39
  **options : passed to StreamingHDFE (fixef_rm, stream, weights,
50
40
  weights_type, solver, precond, keep, verbose, logger, log_level,
51
41
  ...). fixef_rm='singleton' (the default, as in pyfixest) drops
52
- singleton observations; 'none' keeps them.
42
+ singleton observations; 'none' keeps them. connected='largest'
43
+ keeps only the largest connected set of the first two fixed
44
+ effects listed, and min_degree= prunes their levels with too
45
+ few distinct partners first (see `connected_components`).
53
46
 
54
47
  Varying slopes on the streamed dimension: "y ~ x | worker_id[t] + firm_id".
55
48
  The worker FE file then has fe_worker_id (intercept) and fe_worker_id[t] (slope)