logit-stream 0.3.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {logit_stream-0.3.1 → logit_stream-0.4.0}/CHANGELOG.md +44 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/CITATION.cff +2 -2
- {logit_stream-0.3.1 → logit_stream-0.4.0}/PKG-INFO +2 -2
- {logit_stream-0.3.1 → logit_stream-0.4.0}/README.md +1 -1
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/__init__.py +5 -2
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/api.py +85 -12
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/estimator.py +217 -14
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/fe.py +197 -11
- logit_stream-0.4.0/logit_stream/fe_kernels.py +726 -0
- logit_stream-0.4.0/logit_stream/meat_kernels.py +629 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/models.py +718 -6
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/newton.py +20 -4
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/results.py +167 -1
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/store.py +70 -5
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream.egg-info/PKG-INFO +2 -2
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream.egg-info/SOURCES.txt +6 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/pyproject.toml +2 -2
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_api.py +5 -2
- logit_stream-0.4.0/tests/test_binary_logit.py +197 -0
- logit_stream-0.4.0/tests/test_fe_kernels.py +290 -0
- logit_stream-0.4.0/tests/test_kernels.py +295 -0
- logit_stream-0.4.0/tests/test_meat_kernels.py +189 -0
- logit_stream-0.4.0/tests/test_own_covariates.py +190 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_profile.py +4 -2
- logit_stream-0.3.1/tests/test_kernels.py +0 -105
- {logit_stream-0.3.1 → logit_stream-0.4.0}/LICENSE.md +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/MANIFEST.in +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/_choice.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/_frontend.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/_limits.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/__init__.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/_common.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/fe.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/meat.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/mixed.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/models.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/nested.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/random_effects.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/device/store.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/mixed.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/multi_fe.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/nested.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/optimize.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/profile.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/py.typed +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/random_effects/__init__.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/random_effects/fit.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/random_effects/linalg.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/random_effects/model.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/random_effects/sigma.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream/simulate.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream.egg-info/dependency_links.txt +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream.egg-info/requires.txt +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/logit_stream.egg-info/top_level.txt +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/setup.cfg +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_cfelogit.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_clustering.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_conditional.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_device.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_firth_and_fit.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_fixed_effects.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_headroom.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_index_dtype.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_mixed.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_multi_fe.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_multinomial.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_nested.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_offset_start.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_ordered.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_probit.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_random.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_resources.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_simulate.py +0 -0
- {logit_stream-0.3.1 → logit_stream-0.4.0}/tests/test_weights_scale.py +0 -0
|
@@ -1,5 +1,49 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.4.0
|
|
4
|
+
|
|
5
|
+
New
|
|
6
|
+
- **`mlogit_stream(own_covariates=...)`: covariates that enter only their own level's equation.**
|
|
7
|
+
For example, each group's baseline share can predict that group alone, leaving out the other
|
|
8
|
+
groups' shares. This drops the off-diagonal coefficients, which are often poorly determined
|
|
9
|
+
where the groups are segregated, and fits fewer parameters.
|
|
10
|
+
- **`logit_stream` and `probit_stream` have their own engine** for no fixed effect or one, about
|
|
11
|
+
2 times faster than hdfe-stream's. They fit the two-level ordered logit or probit with this
|
|
12
|
+
package's Newton iterations and return a `BinaryLogitResult` or `BinaryProbitResult` (new
|
|
13
|
+
classes; `predict(type="class")` is new). Estimates are at the maximum to machine precision
|
|
14
|
+
(Newton's decrement judges convergence; `iwls_tol` and `iwls_maxiter` still work). Coefficients,
|
|
15
|
+
variances, degrees of freedom, fit statistics, `fixef`, `predict`, `resid` and `sample` agree
|
|
16
|
+
with `feglm_stream`'s; the probit's variances use the expected information, as its iteratively
|
|
17
|
+
reweighted fit does. Several fixed effects, `bias_correction=True`, `bootcluster=`, CRV3,
|
|
18
|
+
`separation_check=False` and `profile=True` still run `feglm_stream` and return its
|
|
19
|
+
`HDFEResult`.
|
|
20
|
+
|
|
21
|
+
Faster (estimates unchanged)
|
|
22
|
+
- **Multinomial logit** without fixed effects: about 5 times on dense designs, 1.3-1.8 times on
|
|
23
|
+
designs of indicators. **Ordered logit and probit** without fixed effects: about 4 times. Each
|
|
24
|
+
pass over the data is now one fused parallel kernel.
|
|
25
|
+
- **Multinomial and ordered logit with an absorbed fixed effect:** 2-4 times. The first pass
|
|
26
|
+
writes the rows grouped by the fixed effect, so each thread owns whole groups. Fixed effects
|
|
27
|
+
with many small groups are a further 1.5-2 times faster, and the ordered logit and probit no
|
|
28
|
+
longer slow down with many groups (100,000 groups: 47 s, now 2 s).
|
|
29
|
+
- **Robust and clustered variances:** 3-18 times faster for the multinomial and ordered logit, and
|
|
30
|
+
2.5-7 times for the conditional logit and `cfelogit_stream`, from a fused kernel that forms each
|
|
31
|
+
score in a thread.
|
|
32
|
+
- `fit.diagnostics["information"]` says how the information matrix was formed.
|
|
33
|
+
|
|
34
|
+
Fixed
|
|
35
|
+
- Cluster counts `G` and degrees of freedom count only clusters with rows used, and analytic
|
|
36
|
+
weights are rescaled over the rows used after a fixed effect's separation drops some, in the
|
|
37
|
+
binary, multinomial and ordered logit, as hdfe-stream does. Before, the multinomial and ordered
|
|
38
|
+
logit counted clusters whose rows were all dropped (variances slightly small) and scaled the
|
|
39
|
+
weights over all rows read (iid variance and log-likelihoods).
|
|
40
|
+
- Sorting rows by a fixed effect respects `max_memory_gb`: rows not already grouped are
|
|
41
|
+
partitioned in memory to sort them, and when that would not fit they stay in their order and the
|
|
42
|
+
passes are slower.
|
|
43
|
+
- `resid()` of a result fitted with an offset works.
|
|
44
|
+
- The within pseudo R-squared is nan for an ordered or multinomial fit with an offset and an
|
|
45
|
+
absorbed fixed effect (the fixed-effect-only log-likelihood it came from ignored the offset).
|
|
46
|
+
|
|
3
47
|
## 0.3.1
|
|
4
48
|
|
|
5
49
|
- **Works with pyfixest 0.60 and 0.70.** Formulas are read through hdfe-stream's own formula
|
|
@@ -7,8 +7,8 @@ authors:
|
|
|
7
7
|
- family-names: Tucker
|
|
8
8
|
given-names: Lee
|
|
9
9
|
email: lee.c.tucker@census.gov
|
|
10
|
-
version: 0.
|
|
11
|
-
date-released: 2026-11-
|
|
10
|
+
version: 0.4.0
|
|
11
|
+
date-released: 2026-11-11
|
|
12
12
|
license: MIT
|
|
13
13
|
repository-code: "https://github.com/leetucker/logit-stream"
|
|
14
14
|
url: "https://github.com/leetucker/logit-stream"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: logit-stream
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects
|
|
5
5
|
Author: Lee Tucker
|
|
6
6
|
License-Expression: MIT
|
|
@@ -77,7 +77,7 @@ fit.predict("people/*.parquet").sink_parquet("imputed.parquet") # lazily
|
|
|
77
77
|
|
|
78
78
|
| | |
|
|
79
79
|
|---|---|
|
|
80
|
-
| `logit_stream`, `probit_stream` | binary logit and probit with any number of fixed effects ([hdfe-stream](https://github.com/leetucker/hdfe-stream)'s `feglm_stream`) |
|
|
80
|
+
| `logit_stream`, `probit_stream` | binary logit and probit with any number of fixed effects (with at most one through this package's own engine, otherwise [hdfe-stream](https://github.com/leetucker/hdfe-stream)'s `feglm_stream`) |
|
|
81
81
|
| `mlogit_stream` | multinomial logit (any number of fixed effects; several through a Poisson regression) |
|
|
82
82
|
| `ologit_stream`, `oprobit_stream` | ordered logit and probit |
|
|
83
83
|
| `clogit_stream` | conditional (McFadden) logit on long-format choice data |
|
|
@@ -20,7 +20,7 @@ fit.predict("people/*.parquet").sink_parquet("imputed.parquet") # lazily
|
|
|
20
20
|
|
|
21
21
|
| | |
|
|
22
22
|
|---|---|
|
|
23
|
-
| `logit_stream`, `probit_stream` | binary logit and probit with any number of fixed effects ([hdfe-stream](https://github.com/leetucker/hdfe-stream)'s `feglm_stream`) |
|
|
23
|
+
| `logit_stream`, `probit_stream` | binary logit and probit with any number of fixed effects (with at most one through this package's own engine, otherwise [hdfe-stream](https://github.com/leetucker/hdfe-stream)'s `feglm_stream`) |
|
|
24
24
|
| `mlogit_stream` | multinomial logit (any number of fixed effects; several through a Poisson regression) |
|
|
25
25
|
| `ologit_stream`, `oprobit_stream` | ordered logit and probit |
|
|
26
26
|
| `clogit_stream` | conditional (McFadden) logit on long-format choice data |
|
|
@@ -45,6 +45,8 @@ from .api import (
|
|
|
45
45
|
probit_stream,
|
|
46
46
|
)
|
|
47
47
|
from .results import (
|
|
48
|
+
BinaryLogitResult,
|
|
49
|
+
BinaryProbitResult,
|
|
48
50
|
ConditionalFELogitResult,
|
|
49
51
|
ConditionalLogitResult,
|
|
50
52
|
LogitMulti,
|
|
@@ -62,7 +64,7 @@ def _version():
|
|
|
62
64
|
from importlib.metadata import version
|
|
63
65
|
return version("logit-stream")
|
|
64
66
|
except Exception: # noqa: BLE001 -- not installed
|
|
65
|
-
return "0.
|
|
67
|
+
return "0.4.0"
|
|
66
68
|
|
|
67
69
|
|
|
68
70
|
__version__ = _version()
|
|
@@ -70,7 +72,8 @@ __version__ = _version()
|
|
|
70
72
|
__all__ = [
|
|
71
73
|
"logit_stream", "probit_stream", "mlogit_stream", "ologit_stream", "oprobit_stream",
|
|
72
74
|
"clogit_stream", "cfelogit_stream", "mixlogit_stream", "nlogit_stream",
|
|
73
|
-
"LogitResult", "
|
|
75
|
+
"LogitResult", "BinaryLogitResult", "BinaryProbitResult", "MultinomialLogitResult",
|
|
76
|
+
"OrderedLogitResult", "OrderedProbitResult",
|
|
74
77
|
"ConditionalLogitResult", "ConditionalFELogitResult",
|
|
75
78
|
"MixedLogitResult", "NestedLogitResult",
|
|
76
79
|
"LogitMulti", "etable", "bonferroni", "holm", "__version__",
|
|
@@ -184,18 +184,80 @@ def logit_stream(fml: str, data: Source, workdir: PathLike | None = None,
|
|
|
184
184
|
|
|
185
185
|
Notes
|
|
186
186
|
-----
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
187
|
+
With no fixed effect or one, and none of hdfe-stream's options that have
|
|
188
|
+
no counterpart here (``bias_correction``, ``bootcluster``, a CRV3
|
|
189
|
+
variance, the separation check switched off, ...), the fit is this
|
|
190
|
+
package's: the two-level ordered logit's Newton iterations over the
|
|
191
|
+
stored blocks (see `ologit_stream`), reported as a `BinaryLogitResult`
|
|
192
|
+
with hdfe-stream's conventions for the variance (the iid vcov has the
|
|
193
|
+
factor (N - 1) / (N - K)) and for the coefficients (``Intercept`` first).
|
|
194
|
+
It converges on the Newton decrement (``iwls_tol``, if given, is that
|
|
195
|
+
tolerance; default 1e-8), so the estimates are at the maximum to machine
|
|
196
|
+
precision. Otherwise the fit is hdfe-stream's, `feglm_stream`, whose
|
|
197
|
+
convergence is judged on the relative change in deviance.
|
|
190
198
|
A two-level `mlogit_stream` fits the same model, with this package's
|
|
191
|
-
conventions
|
|
192
|
-
|
|
199
|
+
conventions for the variance; it is also the way to Firth's penalized
|
|
200
|
+
likelihood for a binary outcome (``firth=True``).
|
|
193
201
|
"""
|
|
202
|
+
return _binary_fit("logit", fml, data, workdir, vcov, cluster, options)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _binary_fit(family, fml, data, workdir, vcov, cluster, options):
|
|
206
|
+
"""A binary logit or probit: this package's fit where it can, else
|
|
207
|
+
hdfe-stream's `feglm_stream`."""
|
|
208
|
+
route = _binary_route(vcov, cluster, options)
|
|
209
|
+
if route is not None:
|
|
210
|
+
from .estimator import _fit, _NeedsHdfe
|
|
211
|
+
try:
|
|
212
|
+
return _fit("ologit", fml, data, workdir=workdir, vcov=vcov, cluster=cluster,
|
|
213
|
+
_binary=True, link=family, **route)
|
|
214
|
+
except _NeedsHdfe:
|
|
215
|
+
pass
|
|
194
216
|
from hdfe_stream import feglm_stream
|
|
195
|
-
return feglm_stream(fml, data,
|
|
217
|
+
return feglm_stream(fml, data, family, workdir=workdir, vcov=vcov, cluster=cluster,
|
|
196
218
|
**options)
|
|
197
219
|
|
|
198
220
|
|
|
221
|
+
# options of hdfe-stream's `feglm_stream` that the fit here ignores (it has no
|
|
222
|
+
# row files, it counts a single fixed effect's parameters without `fe_dof`, and
|
|
223
|
+
# it reports the within pseudo R-squared whenever there is a fixed effect)
|
|
224
|
+
_BINARY_IGNORED = {"save_resid", "within_pseudo_r2", "fe_dof"}
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _binary_route(vcov, cluster, options):
|
|
228
|
+
"""The options for this package's fit of a binary logit, or None when only
|
|
229
|
+
hdfe-stream's does it: an option it has no counterpart of
|
|
230
|
+
(``bias_correction``, ``bootcluster``, ...), the separation check
|
|
231
|
+
switched off, or CRV3."""
|
|
232
|
+
import inspect
|
|
233
|
+
|
|
234
|
+
from .estimator import _fit
|
|
235
|
+
known = set(inspect.signature(_fit).parameters) - {
|
|
236
|
+
"kind", "fml", "data", "workdir", "vcov", "cluster", "levels", "case", "l2", "random",
|
|
237
|
+
"random_cov", "random_tol", "random_method", "firth", "device", "link", "altwise",
|
|
238
|
+
"poisson_options", "profile", "own_covariates", "_binary"}
|
|
239
|
+
opts = dict(options)
|
|
240
|
+
for k in _BINARY_IGNORED:
|
|
241
|
+
opts.pop(k, None)
|
|
242
|
+
if opts.pop("separation_check", True) is not True:
|
|
243
|
+
return None
|
|
244
|
+
for off, default in (("bias_correction", False), ("bootcluster", None)):
|
|
245
|
+
if off in opts and opts[off] == default: # switched off: as not given
|
|
246
|
+
del opts[off]
|
|
247
|
+
if "iwls_tol" in opts:
|
|
248
|
+
opts["tol"] = opts.pop("iwls_tol")
|
|
249
|
+
if "iwls_maxiter" in opts:
|
|
250
|
+
opts["maxiter"] = opts.pop("iwls_maxiter")
|
|
251
|
+
if any(k not in known for k in opts):
|
|
252
|
+
return None
|
|
253
|
+
vs = [vcov] + ([cluster] if isinstance(cluster, str) else list(cluster or []))
|
|
254
|
+
if any("CRV3" in (v if isinstance(v, str) else " ".join(map(str, v))) for v in vs):
|
|
255
|
+
return None
|
|
256
|
+
if options.get("fe_dof", "exact") not in ("exact", "pyfixest"):
|
|
257
|
+
return None
|
|
258
|
+
return opts
|
|
259
|
+
|
|
260
|
+
|
|
199
261
|
def mlogit_stream(fml: str, data: Source, workdir: PathLike | None = None,
|
|
200
262
|
vcov: Vcov = "iid", cluster: Sequence[str] | str = (),
|
|
201
263
|
levels: Sequence | None = None, **options: Any):
|
|
@@ -221,12 +283,12 @@ def probit_stream(fml: str, data: Source, workdir: PathLike | None = None,
|
|
|
221
283
|
vcov: Vcov = "iid", cluster: Sequence[str] | str = (),
|
|
222
284
|
**options: Any):
|
|
223
285
|
"""Out-of-core binary probit, with any number of high-dimensional fixed
|
|
224
|
-
effects:
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
286
|
+
effects: as `logit_stream` (see there: the options, the routing, the
|
|
287
|
+
bias correction), with the standard normal in place of the logistic. With
|
|
288
|
+
no fixed effect or one the fit is this package's, the two-level ordered
|
|
289
|
+
probit's Newton iterations, reported as a `BinaryProbitResult`; its
|
|
290
|
+
variances use the expected information, as `feglm_stream`'s do."""
|
|
291
|
+
return _binary_fit("probit", fml, data, workdir, vcov, cluster, options)
|
|
230
292
|
|
|
231
293
|
|
|
232
294
|
def cfelogit_stream(fml: str, data: Source, workdir: PathLike | None = None,
|
|
@@ -340,6 +402,17 @@ mlogit_stream.__doc__ = """Out-of-core multinomial logit by streamed Newton.
|
|
|
340
402
|
levels : sequence, optional
|
|
341
403
|
The outcome's levels in the order wanted, base first. Default: the
|
|
342
404
|
distinct values, sorted.
|
|
405
|
+
own_covariates : dict, optional
|
|
406
|
+
Covariates that enter only one level's equation: ``{level: column}``
|
|
407
|
+
(or a list of columns), such as each group's baseline share
|
|
408
|
+
predicting that group alone, ``{"Black": "share_black", "Asian":
|
|
409
|
+
"share_asian"}``. A column named here is not a coefficient in the
|
|
410
|
+
other levels' equations, so ``k - 1`` shares add ``k - 1``
|
|
411
|
+
parameters rather than ``(k - 1)^2``. Name the non-base levels; the
|
|
412
|
+
columns must not also be in the formula. The coefficients are named
|
|
413
|
+
``"<level>:<column>"``, and `predict` takes the columns from the new
|
|
414
|
+
data. Not available with fixed or random effects, ``firth=``,
|
|
415
|
+
``profile=`` or ``device="cuda"``.
|
|
343
416
|
poisson_options : dict, optional
|
|
344
417
|
With several fixed effects: further options of hdfe-stream's
|
|
345
418
|
`fepois_stream` (``solver``, ``precond``, ``max_memory_gb``,
|
|
@@ -33,6 +33,8 @@ from .models import _CFELogit, _Conditional, _Multinomial, _Ordered
|
|
|
33
33
|
from .newton import _Driver
|
|
34
34
|
from .random_effects import _fit_random, _RandomEffects, _refine_laplace, _sigma_inference
|
|
35
35
|
from .results import (
|
|
36
|
+
BinaryLogitResult,
|
|
37
|
+
BinaryProbitResult,
|
|
36
38
|
ConditionalFELogitResult,
|
|
37
39
|
ConditionalLogitResult,
|
|
38
40
|
LogitMulti,
|
|
@@ -41,7 +43,7 @@ from .results import (
|
|
|
41
43
|
OrderedProbitResult,
|
|
42
44
|
_ic,
|
|
43
45
|
)
|
|
44
|
-
from .store import _build
|
|
46
|
+
from .store import SORT_FE, _build
|
|
45
47
|
|
|
46
48
|
WORKDIR_ENV = "LOGIT_STREAM_WORKDIR"
|
|
47
49
|
|
|
@@ -49,6 +51,61 @@ _KINDS = {"mlogit": "multinomial logit", "ologit": "ordered logit",
|
|
|
49
51
|
"clogit": "conditional logit", "cfelogit": "conditional fixed-effects logit"}
|
|
50
52
|
|
|
51
53
|
|
|
54
|
+
class _NeedsHdfe(Exception):
|
|
55
|
+
"""A binary fit that only hdfe-stream's engine does (several absorbed
|
|
56
|
+
fixed effects): `logit_stream` catches it and runs `feglm_stream`."""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _ratio(a, b):
|
|
60
|
+
return a / b if b > 0 else np.nan
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _nested_in_cluster(p0, fe, col):
|
|
64
|
+
"""Whether every level of the absorbed fixed effect (among the rows
|
|
65
|
+
used) lies in one cluster of the store's cluster column `col`: then the
|
|
66
|
+
cluster-robust variance does not count that effect's parameters."""
|
|
67
|
+
G = int(max(1, getattr(fe, "G", 1)))
|
|
68
|
+
lo = np.full(G, np.iinfo(np.int64).max)
|
|
69
|
+
hi = np.full(G, -1)
|
|
70
|
+
keep_g = getattr(fe, "keep_g", None)
|
|
71
|
+
for blk in p0.store:
|
|
72
|
+
f = np.asarray(blk["fe"], dtype=np.int64)
|
|
73
|
+
c = np.asarray(blk["cl"][:, col], dtype=np.int64)
|
|
74
|
+
if keep_g is not None:
|
|
75
|
+
m = keep_g[f]
|
|
76
|
+
f, c = f[m], c[m]
|
|
77
|
+
np.minimum.at(lo, f, c)
|
|
78
|
+
np.maximum.at(hi, f, c)
|
|
79
|
+
seen = hi >= 0
|
|
80
|
+
return bool(np.all(lo[seen] == hi[seen]))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _binary_report(names, beta, all_vcovs, N, P, k_fe, nested_by_req):
|
|
84
|
+
"""The binary logit's report from the two-level ordered fit's: the
|
|
85
|
+
variances scaled as hdfe-stream's and pyfixest's are, K = P + k_fe
|
|
86
|
+
(iid (N - 1) / (N - K), hetero N / (N - K), clustered times
|
|
87
|
+
(N - 1) / (N - Kc), Kc counting the fixed effect only if it is not nested
|
|
88
|
+
in the request's clusters), and the cutpoint as the intercept (minus it,
|
|
89
|
+
first)."""
|
|
90
|
+
K = P + k_fe
|
|
91
|
+
fac = {"iid": _ratio(N - 1, N - K), "hetero": _ratio(N - 1, N - K)}
|
|
92
|
+
out = {}
|
|
93
|
+
for key, (V, df) in all_vcovs.items():
|
|
94
|
+
if key in fac:
|
|
95
|
+
out[key] = (V * fac[key], int(N - K))
|
|
96
|
+
else:
|
|
97
|
+
Kc = P + (1 if nested_by_req.get(key) else k_fe)
|
|
98
|
+
fac[key] = _ratio(N - 1, N - Kc)
|
|
99
|
+
out[key] = (V * fac[key], df)
|
|
100
|
+
if names and names[-1] == "cut1":
|
|
101
|
+
perm = np.r_[P - 1, np.arange(P - 1)]
|
|
102
|
+
sgn = np.r_[-1.0, np.ones(P - 1)]
|
|
103
|
+
out = {k: ((V[np.ix_(perm, perm)] * np.outer(sgn, sgn)), df) for k, (V, df) in out.items()}
|
|
104
|
+
beta = np.asarray(beta)[perm] * sgn
|
|
105
|
+
names = ["Intercept"] + list(names[:-1])
|
|
106
|
+
return names, beta, out, fac
|
|
107
|
+
|
|
108
|
+
|
|
52
109
|
def _log(verbose, msg, logger=None, level=None):
|
|
53
110
|
"""Progress: to `logger` if given, else printed with a timestamp."""
|
|
54
111
|
if not verbose:
|
|
@@ -110,7 +167,11 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
|
|
|
110
167
|
offset=None, start=None, device: str = "cpu", link: str = "logit",
|
|
111
168
|
altwise: bool = False, poisson_options: dict | None = None,
|
|
112
169
|
profile: bool = False, n_threads: int | None = None,
|
|
113
|
-
max_memory_gb: float | None = None, memory_reserve: float | None = None
|
|
170
|
+
max_memory_gb: float | None = None, memory_reserve: float | None = None,
|
|
171
|
+
own_covariates: dict | None = None, _binary: bool = False):
|
|
172
|
+
if own_covariates is not None:
|
|
173
|
+
own_covariates = _check_own(kind, own_covariates, random=random, firth=firth,
|
|
174
|
+
profile=profile, device=device, poisson=poisson_options)
|
|
114
175
|
if poisson_options is not None and kind != "mlogit":
|
|
115
176
|
raise ValueError("poisson_options= is for mlogit_stream with several fixed effects")
|
|
116
177
|
if weights_type not in ("aweights", "fweights"):
|
|
@@ -189,6 +250,11 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
|
|
|
189
250
|
collect=lambda q, what: engine.collect(q, what))
|
|
190
251
|
results = []
|
|
191
252
|
for spec in specs:
|
|
253
|
+
if own_covariates is not None:
|
|
254
|
+
if spec.fe:
|
|
255
|
+
raise ValueError(f"{spec.fml}: own_covariates= is not available with an "
|
|
256
|
+
"absorbed fixed effect")
|
|
257
|
+
spec = _with_own(spec, own_covariates, lf)
|
|
192
258
|
if kind == "cfelogit":
|
|
193
259
|
# the formula's one fixed effect is the unit, conditioned out
|
|
194
260
|
if len(spec.fe) != 1:
|
|
@@ -211,6 +277,8 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
|
|
|
211
277
|
# the groupings' codes are the store's fe, fe2, ... in this order
|
|
212
278
|
spec = replace(spec, fe=[*random, *spec.fe])
|
|
213
279
|
elif spec.fe:
|
|
280
|
+
if _binary and len(spec.fe) > 1:
|
|
281
|
+
raise _NeedsHdfe
|
|
214
282
|
if kind == "clogit":
|
|
215
283
|
raise ValueError(f"{spec.fml}: fixed effects are not available for the "
|
|
216
284
|
"conditional logit (a characteristic of the choice set "
|
|
@@ -267,7 +335,8 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
|
|
|
267
335
|
logger=logger, random=random, random_cov=random_cov,
|
|
268
336
|
random_tol=random_tol, random_method=random_method,
|
|
269
337
|
firth=firth, offset=offset, start=start, device=device,
|
|
270
|
-
link=link, altwise=altwise, budget=budget
|
|
338
|
+
link=link, altwise=altwise, budget=budget,
|
|
339
|
+
own_covariates=own_covariates, binary=_binary))
|
|
271
340
|
if profile:
|
|
272
341
|
from .profile import combine_profiles
|
|
273
342
|
return combine_profiles(results)
|
|
@@ -278,7 +347,8 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
278
347
|
levels, case, tol, maxiter, l2, storage, dtype, block_mb, chunk_mb, workdir,
|
|
279
348
|
keep_intermediates, log, logger, random=None, random_cov="unstructured",
|
|
280
349
|
random_tol=1e-6, random_method="laplace", firth=False, offset=None, start=None,
|
|
281
|
-
device=None, link="logit", altwise=False, budget=None
|
|
350
|
+
device=None, link="logit", altwise=False, budget=None, own_covariates=None,
|
|
351
|
+
binary=False):
|
|
282
352
|
t0 = time.time()
|
|
283
353
|
log(f"{_KINDS[kind]}: {spec.fml}")
|
|
284
354
|
run = Run(_base_dir(workdir), auto_cleanup=not keep_intermediates)
|
|
@@ -287,15 +357,26 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
287
357
|
p0 = _build(lf, spec, kind, run.path, engine, weights=weights,
|
|
288
358
|
clusters=clusters, case=case, levels=levels,
|
|
289
359
|
in_memory=(storage == "memory"),
|
|
360
|
+
sort_fe=(SORT_FE and kind in ("mlogit", "ologit") and random is None
|
|
361
|
+
and len(spec.fe) == 1 and device is None),
|
|
290
362
|
dtype=np.float32 if dtype == "float32" else np.float64,
|
|
291
363
|
block_mb=block_mb, log=log, offset=offset, weights_type=weights_type,
|
|
292
|
-
altwise=altwise)
|
|
364
|
+
altwise=altwise, budget=budget)
|
|
293
365
|
disk_peak = p0.store.bytes if storage == "disk" else 0
|
|
366
|
+
if binary:
|
|
367
|
+
try:
|
|
368
|
+
ok = [float(v) for v in p0.levels] == [0.0, 1.0]
|
|
369
|
+
except (TypeError, ValueError):
|
|
370
|
+
ok = False
|
|
371
|
+
if not ok:
|
|
372
|
+
raise ValueError(f"{spec.fml}: {link} needs a dependent variable of 0s and 1s; "
|
|
373
|
+
f"its values are {list(p0.levels)[:6]}")
|
|
294
374
|
xnames = [nm for nm, _ in spec.x]
|
|
295
375
|
K = len(xnames)
|
|
296
376
|
if kind == "mlogit":
|
|
297
377
|
ic = xnames.index("Intercept") if "Intercept" in xnames else None
|
|
298
|
-
model = _Multinomial(xnames, K, p0.levels, ic
|
|
378
|
+
model = _Multinomial(xnames, K, p0.levels, ic,
|
|
379
|
+
own=_resolve_own(own_covariates, p0.levels, xnames, spec.fml))
|
|
299
380
|
elif kind == "ologit":
|
|
300
381
|
model = _Ordered(xnames, K, p0.levels, link=link)
|
|
301
382
|
elif kind == "cfelogit":
|
|
@@ -314,8 +395,8 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
314
395
|
else None
|
|
315
396
|
if random is not None:
|
|
316
397
|
keys = {nm: ("fe" if i == 0 else f"fe{i + 1}") for i, nm in enumerate(spec.fe)}
|
|
317
|
-
fe = _RandomEffects(model, p0, random, outer, keys, log=log, logger=logger,
|
|
318
|
-
cov=random_cov)
|
|
398
|
+
fe = _RandomEffects(model, p0, random, outer, keys, log=log, logger=logger,
|
|
399
|
+
budget=budget, cov=random_cov)
|
|
319
400
|
if clusters:
|
|
320
401
|
cl = {k: (i, 0) for i, k in enumerate(clusters)}
|
|
321
402
|
bad = [k for k, ok in fe.nested(cl).items() if not ok]
|
|
@@ -358,6 +439,14 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
358
439
|
"standard errors are not meaningful at the boundary.", logger)
|
|
359
440
|
else:
|
|
360
441
|
fit = driver.run(theta0, spec.fml)
|
|
442
|
+
if binary and link == "probit":
|
|
443
|
+
# the variances use the expected information, as an IRLS fit's do
|
|
444
|
+
# (for the logit it is the observed one)
|
|
445
|
+
model.expected = True
|
|
446
|
+
ll_e, _, H_e = driver.objective(fit.theta, hessian=True)
|
|
447
|
+
if H_e is None:
|
|
448
|
+
raise RuntimeError(f"{spec.fml}: the information at the estimates is not finite")
|
|
449
|
+
fit.H = H_e
|
|
361
450
|
names = model.names()
|
|
362
451
|
P = len(names)
|
|
363
452
|
N = p0.wsum if weights_type == "fweights" else (
|
|
@@ -373,13 +462,26 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
373
462
|
# (the coefficients' block of the inverse of the whole information)
|
|
374
463
|
sg = re_info["sigma"]
|
|
375
464
|
Vi = Vi + sg["J"] @ sg["vcov"] @ sg["J"].T
|
|
376
|
-
|
|
465
|
+
# analytic weights are scaled to a mean of one over the rows read, before
|
|
466
|
+
# a fixed effect's separation drops some: the binary logit rescales them
|
|
467
|
+
# over the rows used, as hdfe-stream does (it matters to the iid vcov and
|
|
468
|
+
# the log-likelihood, not to the robust variances); so do the multinomial
|
|
469
|
+
# and ordered logit
|
|
470
|
+
wfix = (p0.n_obs / p0.wsum if (binary or kind in ("mlogit", "ologit"))
|
|
471
|
+
and random is None and weights and weights_type != "fweights"
|
|
472
|
+
and p0.wsum > 0 else 1.0)
|
|
473
|
+
all_vcovs = {"iid": (Vi / wfix, int(N - P))}
|
|
377
474
|
n_clusters = {}
|
|
378
475
|
if hetero or clusters:
|
|
379
476
|
log("final pass: scores for the robust variance")
|
|
380
477
|
cl_index = {k: (i, int(p0.cluster_tables[k].height))
|
|
381
478
|
for i, k in enumerate(clusters)}
|
|
382
479
|
M, U = driver.meat(fit.theta, fit.H, weights_type, cl_index, hetero)
|
|
480
|
+
if kind in ("mlogit", "ologit"):
|
|
481
|
+
# a cluster whose rows were all dropped (a fixed effect's separation)
|
|
482
|
+
# is not one: G counts the clusters of the rows used
|
|
483
|
+
cl_index = {k: (i, int(np.count_nonzero(np.any(U[k] != 0, axis=1))))
|
|
484
|
+
for k, (i, _) in cl_index.items()}
|
|
383
485
|
Hinv = driver.inverse(fit.H) if fe is None else None
|
|
384
486
|
if hetero:
|
|
385
487
|
all_vcovs["hetero"] = (_sandwich(fe, fit, Hinv, M) * N / max(N - 1, 1),
|
|
@@ -412,9 +514,12 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
412
514
|
c = p0.counts
|
|
413
515
|
ll0 = float(np.sum(c * np.log(c / c.sum())))
|
|
414
516
|
k_fe = fe.n_params if fe is not None else 0
|
|
517
|
+
ll, ll0 = ll * wfix, ll0 * wfix
|
|
415
518
|
aic, bic = _ic(ll, P + k_fe, N)
|
|
416
519
|
# the null with only the fixed effect: free, from its outcome counts
|
|
417
|
-
|
|
520
|
+
# (with an offset the groups' shares are not the fixed-effect-only fit: none is reported)
|
|
521
|
+
ll_fe0 = (fe.loglik_fe_only() * wfix
|
|
522
|
+
if fe is not None and random is None and offset is None else np.nan)
|
|
418
523
|
diagnostics = {
|
|
419
524
|
"newton": {"iterations": fit.iterations, "converged": fit.converged,
|
|
420
525
|
"decrement": fit.decrement, "step_halvings": fit.halvings,
|
|
@@ -428,6 +533,12 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
428
533
|
"disk_peak_gb": round(disk_peak / 1e9, 4),
|
|
429
534
|
"seconds_total": round(time.time() - t0, 2),
|
|
430
535
|
}
|
|
536
|
+
if kind in ("mlogit", "ologit") and device is None and (
|
|
537
|
+
fe is None or hasattr(fe, "information_form")):
|
|
538
|
+
diagnostics["information"] = {
|
|
539
|
+
"form": model.information_form(p0.n_obs) if fe is None else fe.information_form(),
|
|
540
|
+
"density": None if getattr(model, "density", None) is None
|
|
541
|
+
else round(float(model.density), 4)}
|
|
431
542
|
if engine.fallbacks:
|
|
432
543
|
diagnostics["polars_gpu_fallbacks"] = list(engine.fallbacks)
|
|
433
544
|
if device is not None:
|
|
@@ -445,11 +556,28 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
445
556
|
diagnostics["separation"] = fe.separation
|
|
446
557
|
_separation_warning(spec, model, beta, np.sqrt(np.diag(all_vcovs["iid"][0])), fit, p0,
|
|
447
558
|
logger)
|
|
559
|
+
if binary:
|
|
560
|
+
nested_by_req, nested_terms = {}, {}
|
|
561
|
+
if fe is not None and requests:
|
|
562
|
+
for req, parts in requests.items():
|
|
563
|
+
for one, _, _ in parts:
|
|
564
|
+
if one not in nested_terms:
|
|
565
|
+
nested_terms[one] = _nested_in_cluster(p0, fe, cl_index[one][0])
|
|
566
|
+
nested_by_req[req] = any(nested_terms[one] for one, _, _ in parts)
|
|
567
|
+
names, beta, all_vcovs, vfac = _binary_report(names, beta, all_vcovs, N, P, k_fe,
|
|
568
|
+
nested_by_req)
|
|
569
|
+
V, df_t = all_vcovs[vkey]
|
|
570
|
+
diagnostics["irls"] = {"iterations": fit.iterations, "converged": fit.converged,
|
|
571
|
+
"decrement": fit.decrement, "tol": tol,
|
|
572
|
+
"step_halvings": fit.halvings}
|
|
573
|
+
diagnostics["vcov_factors"] = {k: float(v) for k, v in vfac.items()}
|
|
574
|
+
diagnostics["nested_in_cluster"] = {k: bool(v) for k, v in nested_by_req.items()}
|
|
448
575
|
common = dict(fml=spec.fml, depvar=spec.depvar, coefnames=names, beta=beta, vcov=V,
|
|
449
576
|
vcov_type=vkey, df_t=df_t, n_obs=int(N) if float(N).is_integer() else N,
|
|
450
577
|
all_vcovs=all_vcovs, n_clusters=n_clusters, collin_vars=fit.dropped,
|
|
451
578
|
diagnostics=diagnostics, weights=weights, weights_type=weights_type,
|
|
452
|
-
model=("
|
|
579
|
+
model=(f"binary {link}" if binary else
|
|
580
|
+
"ordered probit" if link == "probit" else _KINDS[kind]),
|
|
453
581
|
loglik=ll, loglik_null=ll0,
|
|
454
582
|
pseudo_r2=float(1 - ll / ll0) if ll0 else np.nan, aic=aic, bic=bic,
|
|
455
583
|
loglik_fe_null=ll_fe0,
|
|
@@ -478,12 +606,20 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
478
606
|
"incomplete": p0.incomplete}
|
|
479
607
|
if fe is not None:
|
|
480
608
|
rspec.update(fe.result_spec(fit.theta))
|
|
609
|
+
if kind == "mlogit" and model.own:
|
|
610
|
+
rspec["free"] = model.free().copy()
|
|
481
611
|
if kind == "mlogit":
|
|
482
612
|
res = MultinomialLogitResult(**common, levels=list(p0.levels), base=p0.levels[0],
|
|
483
613
|
_spec=rspec)
|
|
484
614
|
elif kind == "ologit":
|
|
485
|
-
|
|
486
|
-
|
|
615
|
+
if binary:
|
|
616
|
+
common["n_levels"] = ({spec.fe[0]: int(fe.n_params)} if fe is not None else {})
|
|
617
|
+
cls = BinaryProbitResult if link == "probit" else BinaryLogitResult
|
|
618
|
+
res = cls(**common, levels=list(p0.levels), family=link,
|
|
619
|
+
deviance=float(-2 * ll), k_fe=int(k_fe), _spec=rspec)
|
|
620
|
+
else:
|
|
621
|
+
cls = OrderedProbitResult if link == "probit" else OrderedLogitResult
|
|
622
|
+
res = cls(**common, levels=list(p0.levels), _spec=rspec)
|
|
487
623
|
elif kind == "cfelogit":
|
|
488
624
|
res = ConditionalFELogitResult(**common, unit=list(case),
|
|
489
625
|
n_units=p0.units["units"],
|
|
@@ -499,6 +635,72 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
499
635
|
run.cleanup()
|
|
500
636
|
|
|
501
637
|
|
|
638
|
+
def _own_columns(own_covariates):
|
|
639
|
+
"""{level: [columns]} from `own_covariates`: a column or a list of them
|
|
640
|
+
for each level."""
|
|
641
|
+
return {lv: ([c] if isinstance(c, str) else list(c)) for lv, c in own_covariates.items()}
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def _check_own(kind, own_covariates, *, random, firth, profile, device, poisson):
|
|
645
|
+
if kind != "mlogit":
|
|
646
|
+
raise ValueError("own_covariates= is for mlogit_stream")
|
|
647
|
+
if not isinstance(own_covariates, dict) or not own_covariates:
|
|
648
|
+
raise ValueError("own_covariates= is a dict {outcome level: column, or list of "
|
|
649
|
+
"columns}, e.g. {'Black': 'share_black', 'Asian': 'share_asian'}")
|
|
650
|
+
own = _own_columns(own_covariates)
|
|
651
|
+
flat = [c for cs in own.values() for c in cs]
|
|
652
|
+
if not all(isinstance(c, str) for c in flat) or not flat:
|
|
653
|
+
raise ValueError("own_covariates= names columns by their names")
|
|
654
|
+
if len(set(flat)) < len(flat):
|
|
655
|
+
raise ValueError("own_covariates= gives a column to more than one level (each level's "
|
|
656
|
+
"own covariates are its own columns)")
|
|
657
|
+
for name, bad in (("random=", random is not None), ("firth=", firth),
|
|
658
|
+
("profile=", profile), ("device='cuda'", device not in (None, "cpu")),
|
|
659
|
+
("poisson_options=", poisson is not None)):
|
|
660
|
+
if bad:
|
|
661
|
+
raise ValueError(f"own_covariates= and {name} cannot be combined")
|
|
662
|
+
return own
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
def _with_own(spec, own, lf):
|
|
666
|
+
"""`spec` with the own-covariate columns added to its design, after the
|
|
667
|
+
formula's (so the formula's terms are a prefix)."""
|
|
668
|
+
have = {nm for nm, _ in spec.x}
|
|
669
|
+
schema = lf.collect_schema()
|
|
670
|
+
extra = []
|
|
671
|
+
for cs in own.values():
|
|
672
|
+
for c in cs:
|
|
673
|
+
if c not in schema:
|
|
674
|
+
raise ValueError(f"{spec.fml}: own_covariates= names {c!r}, which is not a "
|
|
675
|
+
"column of the data")
|
|
676
|
+
if c in have:
|
|
677
|
+
raise ValueError(f"{spec.fml}: {c!r} is both in the formula (where it enters "
|
|
678
|
+
"every level's equation) and in own_covariates=")
|
|
679
|
+
extra.append((c, pl.col(c).cast(pl.Float64)))
|
|
680
|
+
return replace(spec, x=list(spec.x) + extra)
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def _resolve_own(own, levels, xnames, fml):
|
|
684
|
+
"""{design column: equation index (0-based among the non-base levels)}
|
|
685
|
+
from `own_covariates` ({level: [columns]}), matching the keys to the
|
|
686
|
+
outcome's levels (by value, else by string)."""
|
|
687
|
+
if own is None:
|
|
688
|
+
return {}
|
|
689
|
+
out = {}
|
|
690
|
+
for lv, cs in own.items():
|
|
691
|
+
hit = [i for i, v in enumerate(levels) if v == lv] or \
|
|
692
|
+
[i for i, v in enumerate(levels) if str(v) == str(lv)]
|
|
693
|
+
if not hit:
|
|
694
|
+
raise ValueError(f"{fml}: own_covariates= names the level {lv!r}, and the outcome's "
|
|
695
|
+
f"levels are {list(levels)}")
|
|
696
|
+
if hit[0] == 0:
|
|
697
|
+
raise ValueError(f"{fml}: own_covariates= names the base level {levels[0]!r}, which "
|
|
698
|
+
"has no equation (its coefficients are zero); name the others")
|
|
699
|
+
for c in cs:
|
|
700
|
+
out[xnames.index(c)] = hit[0] - 1
|
|
701
|
+
return out
|
|
702
|
+
|
|
703
|
+
|
|
502
704
|
def _warm_start(theta0, start, model, fml):
|
|
503
705
|
"""`start=` over the starting coefficients: a mapping {name: value} for
|
|
504
706
|
some of them (the rest keep the null model's), a sequence of all of them
|
|
@@ -614,9 +816,10 @@ def _separation_warning(spec, model, beta, se, fit, p0, logger):
|
|
|
614
816
|
cols = list(model.cols)
|
|
615
817
|
K = len(cols)
|
|
616
818
|
flagged = []
|
|
819
|
+
pcols = model.param_cols() if model.kind == "mlogit" else None
|
|
617
820
|
for i, nm in enumerate(names):
|
|
618
821
|
if model.kind == "mlogit":
|
|
619
|
-
c =
|
|
822
|
+
c = pcols[i]
|
|
620
823
|
elif i < K:
|
|
621
824
|
c = cols[i]
|
|
622
825
|
else:
|