logit-stream 0.3.1__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. {logit_stream-0.3.1 → logit_stream-0.4.0}/CHANGELOG.md +44 -0
  2. {logit_stream-0.3.1 → logit_stream-0.4.0}/CITATION.cff +2 -2
  3. {logit_stream-0.3.1 → logit_stream-0.4.0}/PKG-INFO +2 -2
  4. {logit_stream-0.3.1 → logit_stream-0.4.0}/README.md +1 -1
  5. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/__init__.py +5 -2
  6. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/api.py +85 -12
  7. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/estimator.py +217 -14
  8. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/fe.py +197 -11
  9. logit_stream-0.4.0/logit_stream/fe_kernels.py +726 -0
  10. logit_stream-0.4.0/logit_stream/meat_kernels.py +629 -0
  11. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/models.py +718 -6
  12. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/newton.py +20 -4
  13. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/results.py +167 -1
  14. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/store.py +70 -5
  15. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream.egg-info/PKG-INFO +2 -2
  16. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream.egg-info/SOURCES.txt +6 -0
  17. {logit_stream-0.3.1 → logit_stream-0.4.0}/pyproject.toml +2 -2
  18. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_api.py +5 -2
  19. logit_stream-0.4.0/tests/test_binary_logit.py +197 -0
  20. logit_stream-0.4.0/tests/test_fe_kernels.py +290 -0
  21. logit_stream-0.4.0/tests/test_kernels.py +295 -0
  22. logit_stream-0.4.0/tests/test_meat_kernels.py +189 -0
  23. logit_stream-0.4.0/tests/test_own_covariates.py +190 -0
  24. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_profile.py +4 -2
  25. logit_stream-0.3.1/tests/test_kernels.py +0 -105
  26. {logit_stream-0.3.1 → logit_stream-0.4.0}/LICENSE.md +0 -0
  27. {logit_stream-0.3.1 → logit_stream-0.4.0}/MANIFEST.in +0 -0
  28. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/_choice.py +0 -0
  29. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/_frontend.py +0 -0
  30. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/_limits.py +0 -0
  31. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/__init__.py +0 -0
  32. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/_common.py +0 -0
  33. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/fe.py +0 -0
  34. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/meat.py +0 -0
  35. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/mixed.py +0 -0
  36. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/models.py +0 -0
  37. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/nested.py +0 -0
  38. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/random_effects.py +0 -0
  39. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/store.py +0 -0
  40. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/mixed.py +0 -0
  41. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/multi_fe.py +0 -0
  42. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/nested.py +0 -0
  43. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/optimize.py +0 -0
  44. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/profile.py +0 -0
  45. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/py.typed +0 -0
  46. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/random_effects/__init__.py +0 -0
  47. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/random_effects/fit.py +0 -0
  48. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/random_effects/linalg.py +0 -0
  49. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/random_effects/model.py +0 -0
  50. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/random_effects/sigma.py +0 -0
  51. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/simulate.py +0 -0
  52. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream.egg-info/dependency_links.txt +0 -0
  53. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream.egg-info/requires.txt +0 -0
  54. {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream.egg-info/top_level.txt +0 -0
  55. {logit_stream-0.3.1 → logit_stream-0.4.0}/setup.cfg +0 -0
  56. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_cfelogit.py +0 -0
  57. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_clustering.py +0 -0
  58. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_conditional.py +0 -0
  59. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_device.py +0 -0
  60. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_firth_and_fit.py +0 -0
  61. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_fixed_effects.py +0 -0
  62. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_headroom.py +0 -0
  63. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_index_dtype.py +0 -0
  64. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_mixed.py +0 -0
  65. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_multi_fe.py +0 -0
  66. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_multinomial.py +0 -0
  67. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_nested.py +0 -0
  68. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_offset_start.py +0 -0
  69. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_ordered.py +0 -0
  70. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_probit.py +0 -0
  71. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_random.py +0 -0
  72. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_resources.py +0 -0
  73. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_simulate.py +0 -0
  74. {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_weights_scale.py +0 -0
@@ -1,5 +1,49 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.4.0
4
+
5
+ New
6
+ - **`mlogit_stream(own_covariates=...)`: covariates that enter only their own level's equation.**
7
+ For example, each group's baseline share can predict that group alone, leaving out the other
8
+ groups' shares. This drops the off-diagonal coefficients, which are often poorly determined
9
+ where the groups are segregated, and fits fewer parameters.
10
+ - **`logit_stream` and `probit_stream` have their own engine** for no fixed effect or one, about
11
+ 2 times faster than hdfe-stream's. They fit the two-level ordered logit or probit with this
12
+ package's Newton iterations and return a `BinaryLogitResult` or `BinaryProbitResult` (new
13
+ classes; `predict(type="class")` is new). Estimates are at the maximum to machine precision
14
+ (Newton's decrement judges convergence; `iwls_tol` and `iwls_maxiter` still work). Coefficients,
15
+ variances, degrees of freedom, fit statistics, `fixef`, `predict`, `resid` and `sample` agree
16
+ with `feglm_stream`'s; the probit's variances use the expected information, as its iteratively
17
+ reweighted fit does. Several fixed effects, `bias_correction=True`, `bootcluster=`, CRV3,
18
+ `separation_check=False` and `profile=True` still run `feglm_stream` and return its
19
+ `HDFEResult`.
20
+
21
+ Faster (estimates unchanged)
22
+ - **Multinomial logit** without fixed effects: about 5 times on dense designs, 1.3-1.8 times on
23
+ designs of indicators. **Ordered logit and probit** without fixed effects: about 4 times. Each
24
+ pass over the data is now one fused parallel kernel.
25
+ - **Multinomial and ordered logit with an absorbed fixed effect:** 2-4 times. The first pass
26
+ writes the rows grouped by the fixed effect, so each thread owns whole groups. Fixed effects
27
+ with many small groups are a further 1.5-2 times faster, and the ordered logit and probit no
28
+ longer slow down with many groups (100,000 groups: 47 s, now 2 s).
29
+ - **Robust and clustered variances:** 3-18 times faster for the multinomial and ordered logit, and
30
+ 2.5-7 times for the conditional logit and `cfelogit_stream`, from a fused kernel that forms each
31
+ score in a thread.
32
+ - `fit.diagnostics["information"]` says how the information matrix was formed.
33
+
34
+ Fixed
35
+ - Cluster counts `G` and degrees of freedom count only clusters with rows used, and analytic
36
+ weights are rescaled over the rows used after a fixed effect's separation drops some, in the
37
+ binary, multinomial and ordered logit, as hdfe-stream does. Before, the multinomial and ordered
38
+ logit counted clusters whose rows were all dropped (variances slightly small) and scaled the
39
+ weights over all rows read (iid variance and log-likelihoods).
40
+ - Sorting rows by a fixed effect respects `max_memory_gb`: rows not already grouped are
41
+ partitioned in memory to sort them, and when that would not fit they stay in their order and the
42
+ passes are slower.
43
+ - `resid()` of a result fitted with an offset works.
44
+ - The within pseudo R-squared is nan for an ordered or multinomial fit with an offset and an
45
+ absorbed fixed effect (the fixed-effect-only log-likelihood it came from ignored the offset).
46
+
3
47
  ## 0.3.1
4
48
 
5
49
  - **Works with pyfixest 0.60 and 0.70.** Formulas are read through hdfe-stream's own formula
@@ -7,8 +7,8 @@ authors:
7
7
  - family-names: Tucker
8
8
  given-names: Lee
9
9
  email: lee.c.tucker@census.gov
10
- version: 0.3.1
11
- date-released: 2026-11-09
10
+ version: 0.4.0
11
+ date-released: 2026-11-11
12
12
  license: MIT
13
13
  repository-code: "https://github.com/leetucker/logit-stream"
14
14
  url: "https://github.com/leetucker/logit-stream"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: logit-stream
3
- Version: 0.3.1
3
+ Version: 0.4.0
4
4
  Summary: Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects
5
5
  Author: Lee Tucker
6
6
  License-Expression: MIT
@@ -77,7 +77,7 @@ fit.predict("people/*.parquet").sink_parquet("imputed.parquet") # lazily
77
77
 
78
78
  | | |
79
79
  |---|---|
80
- | `logit_stream`, `probit_stream` | binary logit and probit with any number of fixed effects ([hdfe-stream](https://github.com/leetucker/hdfe-stream)'s `feglm_stream`) |
80
+ | `logit_stream`, `probit_stream` | binary logit and probit with any number of fixed effects (with at most one through this package's own engine, otherwise [hdfe-stream](https://github.com/leetucker/hdfe-stream)'s `feglm_stream`) |
81
81
  | `mlogit_stream` | multinomial logit (any number of fixed effects; several through a Poisson regression) |
82
82
  | `ologit_stream`, `oprobit_stream` | ordered logit and probit |
83
83
  | `clogit_stream` | conditional (McFadden) logit on long-format choice data |
@@ -20,7 +20,7 @@ fit.predict("people/*.parquet").sink_parquet("imputed.parquet") # lazily
20
20
 
21
21
  | | |
22
22
  |---|---|
23
- | `logit_stream`, `probit_stream` | binary logit and probit with any number of fixed effects ([hdfe-stream](https://github.com/leetucker/hdfe-stream)'s `feglm_stream`) |
23
+ | `logit_stream`, `probit_stream` | binary logit and probit with any number of fixed effects (with at most one through this package's own engine, otherwise [hdfe-stream](https://github.com/leetucker/hdfe-stream)'s `feglm_stream`) |
24
24
  | `mlogit_stream` | multinomial logit (any number of fixed effects; several through a Poisson regression) |
25
25
  | `ologit_stream`, `oprobit_stream` | ordered logit and probit |
26
26
  | `clogit_stream` | conditional (McFadden) logit on long-format choice data |
@@ -45,6 +45,8 @@ from .api import (
45
45
  probit_stream,
46
46
  )
47
47
  from .results import (
48
+ BinaryLogitResult,
49
+ BinaryProbitResult,
48
50
  ConditionalFELogitResult,
49
51
  ConditionalLogitResult,
50
52
  LogitMulti,
@@ -62,7 +64,7 @@ def _version():
62
64
  from importlib.metadata import version
63
65
  return version("logit-stream")
64
66
  except Exception: # noqa: BLE001 -- not installed
65
- return "0.3.1"
67
+ return "0.4.0"
66
68
 
67
69
 
68
70
  __version__ = _version()
@@ -70,7 +72,8 @@ __version__ = _version()
70
72
  __all__ = [
71
73
  "logit_stream", "probit_stream", "mlogit_stream", "ologit_stream", "oprobit_stream",
72
74
  "clogit_stream", "cfelogit_stream", "mixlogit_stream", "nlogit_stream",
73
- "LogitResult", "MultinomialLogitResult", "OrderedLogitResult", "OrderedProbitResult",
75
+ "LogitResult", "BinaryLogitResult", "BinaryProbitResult", "MultinomialLogitResult",
76
+ "OrderedLogitResult", "OrderedProbitResult",
74
77
  "ConditionalLogitResult", "ConditionalFELogitResult",
75
78
  "MixedLogitResult", "NestedLogitResult",
76
79
  "LogitMulti", "etable", "bonferroni", "holm", "__version__",
@@ -184,18 +184,80 @@ def logit_stream(fml: str, data: Source, workdir: PathLike | None = None,
184
184
 
185
185
  Notes
186
186
  -----
187
- Two conventions are hdfe-stream's (and pyfixest's) rather than this
188
- package's: convergence is judged on the relative change in deviance
189
- (``iwls_tol``), and the iid vcov has the factor (N - 1) / (N - K).
187
+ With no fixed effect or one, and none of hdfe-stream's options that have
188
+ no counterpart here (``bias_correction``, ``bootcluster``, a CRV3
189
+ variance, the separation check switched off, ...), the fit is this
190
+ package's: the two-level ordered logit's Newton iterations over the
191
+ stored blocks (see `ologit_stream`), reported as a `BinaryLogitResult`
192
+ with hdfe-stream's conventions for the variance (the iid vcov has the
193
+ factor (N - 1) / (N - K)) and for the coefficients (``Intercept`` first).
194
+ It converges on the Newton decrement (``iwls_tol``, if given, is that
195
+ tolerance; default 1e-8), so the estimates are at the maximum to machine
196
+ precision. Otherwise the fit is hdfe-stream's, `feglm_stream`, whose
197
+ convergence is judged on the relative change in deviance.
190
198
  A two-level `mlogit_stream` fits the same model, with this package's
191
- conventions, but absorbs at most one fixed effect; it is also the way
192
- to Firth's penalized likelihood for a binary outcome (``firth=True``).
199
+ conventions for the variance; it is also the way to Firth's penalized
200
+ likelihood for a binary outcome (``firth=True``).
193
201
  """
202
+ return _binary_fit("logit", fml, data, workdir, vcov, cluster, options)
203
+
204
+
205
+ def _binary_fit(family, fml, data, workdir, vcov, cluster, options):
206
+ """A binary logit or probit: this package's fit where it can, else
207
+ hdfe-stream's `feglm_stream`."""
208
+ route = _binary_route(vcov, cluster, options)
209
+ if route is not None:
210
+ from .estimator import _fit, _NeedsHdfe
211
+ try:
212
+ return _fit("ologit", fml, data, workdir=workdir, vcov=vcov, cluster=cluster,
213
+ _binary=True, link=family, **route)
214
+ except _NeedsHdfe:
215
+ pass
194
216
  from hdfe_stream import feglm_stream
195
- return feglm_stream(fml, data, "logit", workdir=workdir, vcov=vcov, cluster=cluster,
217
+ return feglm_stream(fml, data, family, workdir=workdir, vcov=vcov, cluster=cluster,
196
218
  **options)
197
219
 
198
220
 
221
+ # options of hdfe-stream's `feglm_stream` that the fit here ignores (it has no
222
+ # row files, it counts a single fixed effect's parameters without `fe_dof`, and
223
+ # it reports the within pseudo R-squared whenever there is a fixed effect)
224
+ _BINARY_IGNORED = {"save_resid", "within_pseudo_r2", "fe_dof"}
225
+
226
+
227
+ def _binary_route(vcov, cluster, options):
228
+ """The options for this package's fit of a binary logit, or None when only
229
+ hdfe-stream's does it: an option it has no counterpart of
230
+ (``bias_correction``, ``bootcluster``, ...), the separation check
231
+ switched off, or CRV3."""
232
+ import inspect
233
+
234
+ from .estimator import _fit
235
+ known = set(inspect.signature(_fit).parameters) - {
236
+ "kind", "fml", "data", "workdir", "vcov", "cluster", "levels", "case", "l2", "random",
237
+ "random_cov", "random_tol", "random_method", "firth", "device", "link", "altwise",
238
+ "poisson_options", "profile", "own_covariates", "_binary"}
239
+ opts = dict(options)
240
+ for k in _BINARY_IGNORED:
241
+ opts.pop(k, None)
242
+ if opts.pop("separation_check", True) is not True:
243
+ return None
244
+ for off, default in (("bias_correction", False), ("bootcluster", None)):
245
+ if off in opts and opts[off] == default: # switched off: as not given
246
+ del opts[off]
247
+ if "iwls_tol" in opts:
248
+ opts["tol"] = opts.pop("iwls_tol")
249
+ if "iwls_maxiter" in opts:
250
+ opts["maxiter"] = opts.pop("iwls_maxiter")
251
+ if any(k not in known for k in opts):
252
+ return None
253
+ vs = [vcov] + ([cluster] if isinstance(cluster, str) else list(cluster or []))
254
+ if any("CRV3" in (v if isinstance(v, str) else " ".join(map(str, v))) for v in vs):
255
+ return None
256
+ if options.get("fe_dof", "exact") not in ("exact", "pyfixest"):
257
+ return None
258
+ return opts
259
+
260
+
199
261
  def mlogit_stream(fml: str, data: Source, workdir: PathLike | None = None,
200
262
  vcov: Vcov = "iid", cluster: Sequence[str] | str = (),
201
263
  levels: Sequence | None = None, **options: Any):
@@ -221,12 +283,12 @@ def probit_stream(fml: str, data: Source, workdir: PathLike | None = None,
221
283
  vcov: Vcov = "iid", cluster: Sequence[str] | str = (),
222
284
  **options: Any):
223
285
  """Out-of-core binary probit, with any number of high-dimensional fixed
224
- effects: hdfe-stream's `hdfe_stream.feglm_stream` with
225
- ``family="probit"``, as `logit_stream` is with ``family="logit"`` (see
226
- there: the options, the result, the bias correction)."""
227
- from hdfe_stream import feglm_stream
228
- return feglm_stream(fml, data, "probit", workdir=workdir, vcov=vcov, cluster=cluster,
229
- **options)
286
+ effects: as `logit_stream` (see there: the options, the routing, the
287
+ bias correction), with the standard normal in place of the logistic. With
288
+ no fixed effect or one the fit is this package's, the two-level ordered
289
+ probit's Newton iterations, reported as a `BinaryProbitResult`; its
290
+ variances use the expected information, as `feglm_stream`'s do."""
291
+ return _binary_fit("probit", fml, data, workdir, vcov, cluster, options)
230
292
 
231
293
 
232
294
  def cfelogit_stream(fml: str, data: Source, workdir: PathLike | None = None,
@@ -340,6 +402,17 @@ mlogit_stream.__doc__ = """Out-of-core multinomial logit by streamed Newton.
340
402
  levels : sequence, optional
341
403
  The outcome's levels in the order wanted, base first. Default: the
342
404
  distinct values, sorted.
405
+ own_covariates : dict, optional
406
+ Covariates that enter only one level's equation: ``{level: column}``
407
+ (or a list of columns), such as each group's baseline share
408
+ predicting that group alone, ``{"Black": "share_black", "Asian":
409
+ "share_asian"}``. A column named here is not a coefficient in the
410
+ other levels' equations, so ``k - 1`` shares add ``k - 1``
411
+ parameters rather than ``(k - 1)^2``. Name the non-base levels; the
412
+ columns must not also be in the formula. The coefficients are named
413
+ ``"<level>:<column>"``, and `predict` takes the columns from the new
414
+ data. Not available with fixed or random effects, ``firth=``,
415
+ ``profile=`` or ``device="cuda"``.
343
416
  poisson_options : dict, optional
344
417
  With several fixed effects: further options of hdfe-stream's
345
418
  `fepois_stream` (``solver``, ``precond``, ``max_memory_gb``,
@@ -33,6 +33,8 @@ from .models import _CFELogit, _Conditional, _Multinomial, _Ordered
33
33
  from .newton import _Driver
34
34
  from .random_effects import _fit_random, _RandomEffects, _refine_laplace, _sigma_inference
35
35
  from .results import (
36
+ BinaryLogitResult,
37
+ BinaryProbitResult,
36
38
  ConditionalFELogitResult,
37
39
  ConditionalLogitResult,
38
40
  LogitMulti,
@@ -41,7 +43,7 @@ from .results import (
41
43
  OrderedProbitResult,
42
44
  _ic,
43
45
  )
44
- from .store import _build
46
+ from .store import SORT_FE, _build
45
47
 
46
48
  WORKDIR_ENV = "LOGIT_STREAM_WORKDIR"
47
49
 
@@ -49,6 +51,61 @@ _KINDS = {"mlogit": "multinomial logit", "ologit": "ordered logit",
49
51
  "clogit": "conditional logit", "cfelogit": "conditional fixed-effects logit"}
50
52
 
51
53
 
54
+ class _NeedsHdfe(Exception):
55
+ """A binary fit that only hdfe-stream's engine does (several absorbed
56
+ fixed effects): `logit_stream` catches it and runs `feglm_stream`."""
57
+
58
+
59
+ def _ratio(a, b):
60
+ return a / b if b > 0 else np.nan
61
+
62
+
63
+ def _nested_in_cluster(p0, fe, col):
64
+ """Whether every level of the absorbed fixed effect (among the rows
65
+ used) lies in one cluster of the store's cluster column `col`: then the
66
+ cluster-robust variance does not count that effect's parameters."""
67
+ G = int(max(1, getattr(fe, "G", 1)))
68
+ lo = np.full(G, np.iinfo(np.int64).max)
69
+ hi = np.full(G, -1)
70
+ keep_g = getattr(fe, "keep_g", None)
71
+ for blk in p0.store:
72
+ f = np.asarray(blk["fe"], dtype=np.int64)
73
+ c = np.asarray(blk["cl"][:, col], dtype=np.int64)
74
+ if keep_g is not None:
75
+ m = keep_g[f]
76
+ f, c = f[m], c[m]
77
+ np.minimum.at(lo, f, c)
78
+ np.maximum.at(hi, f, c)
79
+ seen = hi >= 0
80
+ return bool(np.all(lo[seen] == hi[seen]))
81
+
82
+
83
+ def _binary_report(names, beta, all_vcovs, N, P, k_fe, nested_by_req):
84
+ """The binary logit's report from the two-level ordered fit's: the
85
+ variances scaled as hdfe-stream's and pyfixest's are, K = P + k_fe
86
+ (iid (N - 1) / (N - K), hetero N / (N - K), clustered times
87
+ (N - 1) / (N - Kc), Kc counting the fixed effect only if it is not nested
88
+ in the request's clusters), and the cutpoint as the intercept (minus it,
89
+ first)."""
90
+ K = P + k_fe
91
+ fac = {"iid": _ratio(N - 1, N - K), "hetero": _ratio(N - 1, N - K)}
92
+ out = {}
93
+ for key, (V, df) in all_vcovs.items():
94
+ if key in fac:
95
+ out[key] = (V * fac[key], int(N - K))
96
+ else:
97
+ Kc = P + (1 if nested_by_req.get(key) else k_fe)
98
+ fac[key] = _ratio(N - 1, N - Kc)
99
+ out[key] = (V * fac[key], df)
100
+ if names and names[-1] == "cut1":
101
+ perm = np.r_[P - 1, np.arange(P - 1)]
102
+ sgn = np.r_[-1.0, np.ones(P - 1)]
103
+ out = {k: ((V[np.ix_(perm, perm)] * np.outer(sgn, sgn)), df) for k, (V, df) in out.items()}
104
+ beta = np.asarray(beta)[perm] * sgn
105
+ names = ["Intercept"] + list(names[:-1])
106
+ return names, beta, out, fac
107
+
108
+
52
109
  def _log(verbose, msg, logger=None, level=None):
53
110
  """Progress: to `logger` if given, else printed with a timestamp."""
54
111
  if not verbose:
@@ -110,7 +167,11 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
110
167
  offset=None, start=None, device: str = "cpu", link: str = "logit",
111
168
  altwise: bool = False, poisson_options: dict | None = None,
112
169
  profile: bool = False, n_threads: int | None = None,
113
- max_memory_gb: float | None = None, memory_reserve: float | None = None):
170
+ max_memory_gb: float | None = None, memory_reserve: float | None = None,
171
+ own_covariates: dict | None = None, _binary: bool = False):
172
+ if own_covariates is not None:
173
+ own_covariates = _check_own(kind, own_covariates, random=random, firth=firth,
174
+ profile=profile, device=device, poisson=poisson_options)
114
175
  if poisson_options is not None and kind != "mlogit":
115
176
  raise ValueError("poisson_options= is for mlogit_stream with several fixed effects")
116
177
  if weights_type not in ("aweights", "fweights"):
@@ -189,6 +250,11 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
189
250
  collect=lambda q, what: engine.collect(q, what))
190
251
  results = []
191
252
  for spec in specs:
253
+ if own_covariates is not None:
254
+ if spec.fe:
255
+ raise ValueError(f"{spec.fml}: own_covariates= is not available with an "
256
+ "absorbed fixed effect")
257
+ spec = _with_own(spec, own_covariates, lf)
192
258
  if kind == "cfelogit":
193
259
  # the formula's one fixed effect is the unit, conditioned out
194
260
  if len(spec.fe) != 1:
@@ -211,6 +277,8 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
211
277
  # the groupings' codes are the store's fe, fe2, ... in this order
212
278
  spec = replace(spec, fe=[*random, *spec.fe])
213
279
  elif spec.fe:
280
+ if _binary and len(spec.fe) > 1:
281
+ raise _NeedsHdfe
214
282
  if kind == "clogit":
215
283
  raise ValueError(f"{spec.fml}: fixed effects are not available for the "
216
284
  "conditional logit (a characteristic of the choice set "
@@ -267,7 +335,8 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
267
335
  logger=logger, random=random, random_cov=random_cov,
268
336
  random_tol=random_tol, random_method=random_method,
269
337
  firth=firth, offset=offset, start=start, device=device,
270
- link=link, altwise=altwise, budget=budget))
338
+ link=link, altwise=altwise, budget=budget,
339
+ own_covariates=own_covariates, binary=_binary))
271
340
  if profile:
272
341
  from .profile import combine_profiles
273
342
  return combine_profiles(results)
@@ -278,7 +347,8 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
278
347
  levels, case, tol, maxiter, l2, storage, dtype, block_mb, chunk_mb, workdir,
279
348
  keep_intermediates, log, logger, random=None, random_cov="unstructured",
280
349
  random_tol=1e-6, random_method="laplace", firth=False, offset=None, start=None,
281
- device=None, link="logit", altwise=False, budget=None):
350
+ device=None, link="logit", altwise=False, budget=None, own_covariates=None,
351
+ binary=False):
282
352
  t0 = time.time()
283
353
  log(f"{_KINDS[kind]}: {spec.fml}")
284
354
  run = Run(_base_dir(workdir), auto_cleanup=not keep_intermediates)
@@ -287,15 +357,26 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
287
357
  p0 = _build(lf, spec, kind, run.path, engine, weights=weights,
288
358
  clusters=clusters, case=case, levels=levels,
289
359
  in_memory=(storage == "memory"),
360
+ sort_fe=(SORT_FE and kind in ("mlogit", "ologit") and random is None
361
+ and len(spec.fe) == 1 and device is None),
290
362
  dtype=np.float32 if dtype == "float32" else np.float64,
291
363
  block_mb=block_mb, log=log, offset=offset, weights_type=weights_type,
292
- altwise=altwise)
364
+ altwise=altwise, budget=budget)
293
365
  disk_peak = p0.store.bytes if storage == "disk" else 0
366
+ if binary:
367
+ try:
368
+ ok = [float(v) for v in p0.levels] == [0.0, 1.0]
369
+ except (TypeError, ValueError):
370
+ ok = False
371
+ if not ok:
372
+ raise ValueError(f"{spec.fml}: {link} needs a dependent variable of 0s and 1s; "
373
+ f"its values are {list(p0.levels)[:6]}")
294
374
  xnames = [nm for nm, _ in spec.x]
295
375
  K = len(xnames)
296
376
  if kind == "mlogit":
297
377
  ic = xnames.index("Intercept") if "Intercept" in xnames else None
298
- model = _Multinomial(xnames, K, p0.levels, ic)
378
+ model = _Multinomial(xnames, K, p0.levels, ic,
379
+ own=_resolve_own(own_covariates, p0.levels, xnames, spec.fml))
299
380
  elif kind == "ologit":
300
381
  model = _Ordered(xnames, K, p0.levels, link=link)
301
382
  elif kind == "cfelogit":
@@ -314,8 +395,8 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
314
395
  else None
315
396
  if random is not None:
316
397
  keys = {nm: ("fe" if i == 0 else f"fe{i + 1}") for i, nm in enumerate(spec.fe)}
317
- fe = _RandomEffects(model, p0, random, outer, keys, log=log, logger=logger, budget=budget,
318
- cov=random_cov)
398
+ fe = _RandomEffects(model, p0, random, outer, keys, log=log, logger=logger,
399
+ budget=budget, cov=random_cov)
319
400
  if clusters:
320
401
  cl = {k: (i, 0) for i, k in enumerate(clusters)}
321
402
  bad = [k for k, ok in fe.nested(cl).items() if not ok]
@@ -358,6 +439,14 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
358
439
  "standard errors are not meaningful at the boundary.", logger)
359
440
  else:
360
441
  fit = driver.run(theta0, spec.fml)
442
+ if binary and link == "probit":
443
+ # the variances use the expected information, as an IRLS fit's do
444
+ # (for the logit it is the observed one)
445
+ model.expected = True
446
+ ll_e, _, H_e = driver.objective(fit.theta, hessian=True)
447
+ if H_e is None:
448
+ raise RuntimeError(f"{spec.fml}: the information at the estimates is not finite")
449
+ fit.H = H_e
361
450
  names = model.names()
362
451
  P = len(names)
363
452
  N = p0.wsum if weights_type == "fweights" else (
@@ -373,13 +462,26 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
373
462
  # (the coefficients' block of the inverse of the whole information)
374
463
  sg = re_info["sigma"]
375
464
  Vi = Vi + sg["J"] @ sg["vcov"] @ sg["J"].T
376
- all_vcovs = {"iid": (Vi, int(N - P))}
465
+ # analytic weights are scaled to a mean of one over the rows read, before
466
+ # a fixed effect's separation drops some: the binary logit rescales them
467
+ # over the rows used, as hdfe-stream does (it matters to the iid vcov and
468
+ # the log-likelihood, not to the robust variances); so do the multinomial
469
+ # and ordered logit
470
+ wfix = (p0.n_obs / p0.wsum if (binary or kind in ("mlogit", "ologit"))
471
+ and random is None and weights and weights_type != "fweights"
472
+ and p0.wsum > 0 else 1.0)
473
+ all_vcovs = {"iid": (Vi / wfix, int(N - P))}
377
474
  n_clusters = {}
378
475
  if hetero or clusters:
379
476
  log("final pass: scores for the robust variance")
380
477
  cl_index = {k: (i, int(p0.cluster_tables[k].height))
381
478
  for i, k in enumerate(clusters)}
382
479
  M, U = driver.meat(fit.theta, fit.H, weights_type, cl_index, hetero)
480
+ if kind in ("mlogit", "ologit"):
481
+ # a cluster whose rows were all dropped (a fixed effect's separation)
482
+ # is not one: G counts the clusters of the rows used
483
+ cl_index = {k: (i, int(np.count_nonzero(np.any(U[k] != 0, axis=1))))
484
+ for k, (i, _) in cl_index.items()}
383
485
  Hinv = driver.inverse(fit.H) if fe is None else None
384
486
  if hetero:
385
487
  all_vcovs["hetero"] = (_sandwich(fe, fit, Hinv, M) * N / max(N - 1, 1),
@@ -412,9 +514,12 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
412
514
  c = p0.counts
413
515
  ll0 = float(np.sum(c * np.log(c / c.sum())))
414
516
  k_fe = fe.n_params if fe is not None else 0
517
+ ll, ll0 = ll * wfix, ll0 * wfix
415
518
  aic, bic = _ic(ll, P + k_fe, N)
416
519
  # the null with only the fixed effect: free, from its outcome counts
417
- ll_fe0 = fe.loglik_fe_only() if fe is not None and random is None else np.nan
520
+ # (with an offset the groups' shares are not the fixed-effect-only fit: none is reported)
521
+ ll_fe0 = (fe.loglik_fe_only() * wfix
522
+ if fe is not None and random is None and offset is None else np.nan)
418
523
  diagnostics = {
419
524
  "newton": {"iterations": fit.iterations, "converged": fit.converged,
420
525
  "decrement": fit.decrement, "step_halvings": fit.halvings,
@@ -428,6 +533,12 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
428
533
  "disk_peak_gb": round(disk_peak / 1e9, 4),
429
534
  "seconds_total": round(time.time() - t0, 2),
430
535
  }
536
+ if kind in ("mlogit", "ologit") and device is None and (
537
+ fe is None or hasattr(fe, "information_form")):
538
+ diagnostics["information"] = {
539
+ "form": model.information_form(p0.n_obs) if fe is None else fe.information_form(),
540
+ "density": None if getattr(model, "density", None) is None
541
+ else round(float(model.density), 4)}
431
542
  if engine.fallbacks:
432
543
  diagnostics["polars_gpu_fallbacks"] = list(engine.fallbacks)
433
544
  if device is not None:
@@ -445,11 +556,28 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
445
556
  diagnostics["separation"] = fe.separation
446
557
  _separation_warning(spec, model, beta, np.sqrt(np.diag(all_vcovs["iid"][0])), fit, p0,
447
558
  logger)
559
+ if binary:
560
+ nested_by_req, nested_terms = {}, {}
561
+ if fe is not None and requests:
562
+ for req, parts in requests.items():
563
+ for one, _, _ in parts:
564
+ if one not in nested_terms:
565
+ nested_terms[one] = _nested_in_cluster(p0, fe, cl_index[one][0])
566
+ nested_by_req[req] = any(nested_terms[one] for one, _, _ in parts)
567
+ names, beta, all_vcovs, vfac = _binary_report(names, beta, all_vcovs, N, P, k_fe,
568
+ nested_by_req)
569
+ V, df_t = all_vcovs[vkey]
570
+ diagnostics["irls"] = {"iterations": fit.iterations, "converged": fit.converged,
571
+ "decrement": fit.decrement, "tol": tol,
572
+ "step_halvings": fit.halvings}
573
+ diagnostics["vcov_factors"] = {k: float(v) for k, v in vfac.items()}
574
+ diagnostics["nested_in_cluster"] = {k: bool(v) for k, v in nested_by_req.items()}
448
575
  common = dict(fml=spec.fml, depvar=spec.depvar, coefnames=names, beta=beta, vcov=V,
449
576
  vcov_type=vkey, df_t=df_t, n_obs=int(N) if float(N).is_integer() else N,
450
577
  all_vcovs=all_vcovs, n_clusters=n_clusters, collin_vars=fit.dropped,
451
578
  diagnostics=diagnostics, weights=weights, weights_type=weights_type,
452
- model=("ordered probit" if link == "probit" else _KINDS[kind]),
579
+ model=(f"binary {link}" if binary else
580
+ "ordered probit" if link == "probit" else _KINDS[kind]),
453
581
  loglik=ll, loglik_null=ll0,
454
582
  pseudo_r2=float(1 - ll / ll0) if ll0 else np.nan, aic=aic, bic=bic,
455
583
  loglik_fe_null=ll_fe0,
@@ -478,12 +606,20 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
478
606
  "incomplete": p0.incomplete}
479
607
  if fe is not None:
480
608
  rspec.update(fe.result_spec(fit.theta))
609
+ if kind == "mlogit" and model.own:
610
+ rspec["free"] = model.free().copy()
481
611
  if kind == "mlogit":
482
612
  res = MultinomialLogitResult(**common, levels=list(p0.levels), base=p0.levels[0],
483
613
  _spec=rspec)
484
614
  elif kind == "ologit":
485
- cls = OrderedProbitResult if link == "probit" else OrderedLogitResult
486
- res = cls(**common, levels=list(p0.levels), _spec=rspec)
615
+ if binary:
616
+ common["n_levels"] = ({spec.fe[0]: int(fe.n_params)} if fe is not None else {})
617
+ cls = BinaryProbitResult if link == "probit" else BinaryLogitResult
618
+ res = cls(**common, levels=list(p0.levels), family=link,
619
+ deviance=float(-2 * ll), k_fe=int(k_fe), _spec=rspec)
620
+ else:
621
+ cls = OrderedProbitResult if link == "probit" else OrderedLogitResult
622
+ res = cls(**common, levels=list(p0.levels), _spec=rspec)
487
623
  elif kind == "cfelogit":
488
624
  res = ConditionalFELogitResult(**common, unit=list(case),
489
625
  n_units=p0.units["units"],
@@ -499,6 +635,72 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
499
635
  run.cleanup()
500
636
 
501
637
 
638
+ def _own_columns(own_covariates):
639
+ """{level: [columns]} from `own_covariates`: a column or a list of them
640
+ for each level."""
641
+ return {lv: ([c] if isinstance(c, str) else list(c)) for lv, c in own_covariates.items()}
642
+
643
+
644
+ def _check_own(kind, own_covariates, *, random, firth, profile, device, poisson):
645
+ if kind != "mlogit":
646
+ raise ValueError("own_covariates= is for mlogit_stream")
647
+ if not isinstance(own_covariates, dict) or not own_covariates:
648
+ raise ValueError("own_covariates= is a dict {outcome level: column, or list of "
649
+ "columns}, e.g. {'Black': 'share_black', 'Asian': 'share_asian'}")
650
+ own = _own_columns(own_covariates)
651
+ flat = [c for cs in own.values() for c in cs]
652
+ if not all(isinstance(c, str) for c in flat) or not flat:
653
+ raise ValueError("own_covariates= names columns by their names")
654
+ if len(set(flat)) < len(flat):
655
+ raise ValueError("own_covariates= gives a column to more than one level (each level's "
656
+ "own covariates are its own columns)")
657
+ for name, bad in (("random=", random is not None), ("firth=", firth),
658
+ ("profile=", profile), ("device='cuda'", device not in (None, "cpu")),
659
+ ("poisson_options=", poisson is not None)):
660
+ if bad:
661
+ raise ValueError(f"own_covariates= and {name} cannot be combined")
662
+ return own
663
+
664
+
665
+ def _with_own(spec, own, lf):
666
+ """`spec` with the own-covariate columns added to its design, after the
667
+ formula's (so the formula's terms are a prefix)."""
668
+ have = {nm for nm, _ in spec.x}
669
+ schema = lf.collect_schema()
670
+ extra = []
671
+ for cs in own.values():
672
+ for c in cs:
673
+ if c not in schema:
674
+ raise ValueError(f"{spec.fml}: own_covariates= names {c!r}, which is not a "
675
+ "column of the data")
676
+ if c in have:
677
+ raise ValueError(f"{spec.fml}: {c!r} is both in the formula (where it enters "
678
+ "every level's equation) and in own_covariates=")
679
+ extra.append((c, pl.col(c).cast(pl.Float64)))
680
+ return replace(spec, x=list(spec.x) + extra)
681
+
682
+
683
+ def _resolve_own(own, levels, xnames, fml):
684
+ """{design column: equation index (0-based among the non-base levels)}
685
+ from `own_covariates` ({level: [columns]}), matching the keys to the
686
+ outcome's levels (by value, else by string)."""
687
+ if own is None:
688
+ return {}
689
+ out = {}
690
+ for lv, cs in own.items():
691
+ hit = [i for i, v in enumerate(levels) if v == lv] or \
692
+ [i for i, v in enumerate(levels) if str(v) == str(lv)]
693
+ if not hit:
694
+ raise ValueError(f"{fml}: own_covariates= names the level {lv!r}, and the outcome's "
695
+ f"levels are {list(levels)}")
696
+ if hit[0] == 0:
697
+ raise ValueError(f"{fml}: own_covariates= names the base level {levels[0]!r}, which "
698
+ "has no equation (its coefficients are zero); name the others")
699
+ for c in cs:
700
+ out[xnames.index(c)] = hit[0] - 1
701
+ return out
702
+
703
+
502
704
  def _warm_start(theta0, start, model, fml):
503
705
  """`start=` over the starting coefficients: a mapping {name: value} for
504
706
  some of them (the rest keep the null model's), a sequence of all of them
@@ -614,9 +816,10 @@ def _separation_warning(spec, model, beta, se, fit, p0, logger):
614
816
  cols = list(model.cols)
615
817
  K = len(cols)
616
818
  flagged = []
819
+ pcols = model.param_cols() if model.kind == "mlogit" else None
617
820
  for i, nm in enumerate(names):
618
821
  if model.kind == "mlogit":
619
- c = cols[i % K]
822
+ c = pcols[i]
620
823
  elif i < K:
621
824
  c = cols[i]
622
825
  else: