logit-stream 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {logit_stream-0.1.0 → logit_stream-0.3.0}/CHANGELOG.md +51 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/CITATION.cff +2 -2
- {logit_stream-0.1.0 → logit_stream-0.3.0}/PKG-INFO +4 -2
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/__init__.py +1 -1
- logit_stream-0.3.0/logit_stream/_limits.py +31 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/api.py +1 -1
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/estimator.py +37 -33
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/fe.py +7 -6
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/mixed.py +12 -1
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/models.py +225 -63
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/nested.py +12 -1
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/newton.py +12 -0
- logit_stream-0.3.0/logit_stream/profile.py +270 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/random_effects/model.py +9 -5
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/store.py +150 -58
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream.egg-info/PKG-INFO +4 -2
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream.egg-info/SOURCES.txt +6 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream.egg-info/requires.txt +3 -1
- {logit_stream-0.1.0 → logit_stream-0.3.0}/pyproject.toml +6 -2
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_cfelogit.py +18 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_conditional.py +22 -0
- logit_stream-0.3.0/tests/test_headroom.py +61 -0
- logit_stream-0.3.0/tests/test_index_dtype.py +51 -0
- logit_stream-0.3.0/tests/test_profile.py +161 -0
- logit_stream-0.3.0/tests/test_resources.py +135 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/LICENSE.md +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/MANIFEST.in +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/README.md +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/_choice.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/_frontend.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/__init__.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/_common.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/fe.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/meat.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/mixed.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/models.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/nested.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/random_effects.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/store.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/multi_fe.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/optimize.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/py.typed +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/random_effects/__init__.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/random_effects/fit.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/random_effects/linalg.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/random_effects/sigma.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/results.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/simulate.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream.egg-info/dependency_links.txt +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream.egg-info/top_level.txt +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/setup.cfg +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_api.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_clustering.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_device.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_firth_and_fit.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_fixed_effects.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_kernels.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_mixed.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_multi_fe.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_multinomial.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_nested.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_offset_start.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_ordered.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_probit.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_random.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_simulate.py +0 -0
- {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_weights_scale.py +0 -0
|
@@ -1,5 +1,56 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.0
|
|
4
|
+
|
|
5
|
+
- **The conditional logit is 2.3-2.8 times faster.** Its log-likelihood, gradient and information
|
|
6
|
+
are computed in one parallel numba pass over each block's choice sets, giving the same
|
|
7
|
+
estimates (4 million sets: 13 s, now 5 s; 8 million: 26 s, now 9-12 s). Designs of more than
|
|
8
|
+
256 parameters keep the previous BLAS-based pass, which is faster there.
|
|
9
|
+
- **The fixed-effects logit (`cfelogit_stream`) balances work across threads better.** Panels
|
|
10
|
+
are dealt out in small pieces to more parts than threads, so a run of long panels no longer
|
|
11
|
+
leaves threads idle: 0.55 of the time on panels sorted by length, 0.8-1.0 on shuffled
|
|
12
|
+
ones, with the same results.
|
|
13
|
+
- **Lower memory for conditional, mixed and fixed-effects logit.** Choice sets are now checked
|
|
14
|
+
and counted a block at a time instead of from a table of every set, so memory no longer grows
|
|
15
|
+
with the number of sets (8 million sets: peak 1.71 GB down to 1.33 GB). Same errors, same
|
|
16
|
+
results.
|
|
17
|
+
- **Memory tests.** `pytest -m resource` fits the multinomial, ordered and conditional logit
|
|
18
|
+
on data larger than a fixed memory limit; CI runs them on Linux, macOS and Windows. The
|
|
19
|
+
default `pytest` run leaves them out.
|
|
20
|
+
- Requires hdfe-stream 0.9.0 or later, which reads Parquet in less memory.
|
|
21
|
+
|
|
22
|
+
## 0.2.0
|
|
23
|
+
|
|
24
|
+
- **Fit resources (CPUs and memory) are constrained to your job's limits automatically.** On a
|
|
25
|
+
cluster (PBS, Slurm, or a container with cgroup limits), importing `logit_stream` caps the threads of
|
|
26
|
+
Polars, numba, BLAS and pyarrow at the job's CPU allocation, and with Polars 2.x gives Polars a
|
|
27
|
+
memory budget so its streaming operations spill to disk instead of growing. Caveats: the Polars
|
|
28
|
+
budget is a target for its buffers, not a hard cap on the process, and Polars 1.x has no such
|
|
29
|
+
option, so there only the threads are capped. The caps must be in place before Polars is
|
|
30
|
+
imported: import `logit_stream` first, or call `stream_limits.set_limits()` at the top of the script.
|
|
31
|
+
With no limit found nothing changes. See "Resource limits and profiling" in the documentation.
|
|
32
|
+
- **`profile=True`: estimate a fit's memory and disk before running it.** On
|
|
33
|
+
every estimator, profiling returns a `hdfe_stream.Profile` -- the estimated
|
|
34
|
+
memory and disk, from one scan of the data -- instead of fitting. P comes from
|
|
35
|
+
the model and the outcome's levels; the information matrix, its per-thread
|
|
36
|
+
copies, the fixed effect's cross blocks and the clusters' score sums are sized
|
|
37
|
+
from it. The multinomial logit with several fixed effects (fitted by
|
|
38
|
+
`fepois_stream` on data expanded J-fold) is profiled as that regression, from
|
|
39
|
+
counts worked out from one scan of the original data. Not modeled: random
|
|
40
|
+
intercepts, and the mixed and nested logit's workspaces beyond the per-thread
|
|
41
|
+
accumulators.
|
|
42
|
+
- **`max_memory_gb` and `memory_reserve`**: the fixed effect's cross-block guard and
|
|
43
|
+
the random intercepts' guard (both a fixed 8 GB) and CRV3's (25% of physical
|
|
44
|
+
memory) now ask whether their exact allocation fits in the memory left (the limit
|
|
45
|
+
less a reserve, less what the process holds). The limit defaults to the job's
|
|
46
|
+
allocation (cgroup, `STREAM_LIMITS_MEMORY_GB`, or on PBS `qstat`).
|
|
47
|
+
- **`n_threads=`** on every estimator: numba and BLAS threads for one fit,
|
|
48
|
+
restored afterwards (Polars' pool is fixed at import; see hdfe-stream's
|
|
49
|
+
"Resource limits and profiling").
|
|
50
|
+
- **`diagnostics["peak_rss_gb"]`** in every fit.
|
|
51
|
+
- Depends on `stream-limits`, and needs hdfe-stream 0.8.0 or later (the `<0.7` upper bound on
|
|
52
|
+
hdfe-stream is gone).
|
|
53
|
+
|
|
3
54
|
## 0.1.0
|
|
4
55
|
|
|
5
56
|
The first release: logit models by streamed Newton's method, for data that
|
|
@@ -7,8 +7,8 @@ authors:
|
|
|
7
7
|
- family-names: Tucker
|
|
8
8
|
given-names: Lee
|
|
9
9
|
email: lee.c.tucker@census.gov
|
|
10
|
-
version: 0.
|
|
11
|
-
date-released: 2026-
|
|
10
|
+
version: 0.3.0
|
|
11
|
+
date-released: 2026-11-05
|
|
12
12
|
license: MIT
|
|
13
13
|
repository-code: "https://github.com/leetucker/logit-stream"
|
|
14
14
|
url: "https://github.com/leetucker/logit-stream"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: logit-stream
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects
|
|
5
5
|
Author: Lee Tucker
|
|
6
6
|
License-Expression: MIT
|
|
@@ -25,7 +25,8 @@ Classifier: Typing :: Typed
|
|
|
25
25
|
Requires-Python: >=3.10
|
|
26
26
|
Description-Content-Type: text/markdown
|
|
27
27
|
License-File: LICENSE.md
|
|
28
|
-
Requires-Dist: hdfe-stream[formula]
|
|
28
|
+
Requires-Dist: hdfe-stream[formula]>=0.9.0
|
|
29
|
+
Requires-Dist: stream-limits>=0.1.1
|
|
29
30
|
Requires-Dist: polars!=1.41.*,>=1.37
|
|
30
31
|
Requires-Dist: numpy>=1.24
|
|
31
32
|
Requires-Dist: numba>=0.59
|
|
@@ -48,6 +49,7 @@ Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
|
|
|
48
49
|
Requires-Dist: sphinx-design>=0.6; extra == "docs"
|
|
49
50
|
Provides-Extra: test
|
|
50
51
|
Requires-Dist: pytest>=7.0; extra == "test"
|
|
52
|
+
Requires-Dist: stream-limits[testing]>=0.2.1; extra == "test"
|
|
51
53
|
Requires-Dist: scikit-learn>=1.5; extra == "test"
|
|
52
54
|
Requires-Dist: statsmodels>=0.14; extra == "test"
|
|
53
55
|
Requires-Dist: pandas>=2.0; extra == "test"
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Resource limits for a fit: the `n_threads=` option, and the peak memory the
|
|
2
|
+
fit used. The code is hdfe-stream's (`hdfe_stream.core.resources`); see
|
|
3
|
+
"Resource limits and profiling" in its documentation."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import functools
|
|
8
|
+
|
|
9
|
+
from hdfe_stream.core.profile import Profile
|
|
10
|
+
from hdfe_stream.core.resources import peak_rss_gb, reset_peak_rss, thread_scope
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def limited(fit):
|
|
14
|
+
"""Give an estimator's fit function an ``n_threads`` keyword that holds
|
|
15
|
+
numba and BLAS to that many threads for the fit (restored afterwards; Polars
|
|
16
|
+
cannot be limited per fit), and record ``diagnostics["peak_rss_gb"]``."""
|
|
17
|
+
|
|
18
|
+
@functools.wraps(fit)
|
|
19
|
+
def wrapper(*args, n_threads=None, **kwargs):
|
|
20
|
+
reset_peak_rss()
|
|
21
|
+
with thread_scope(n_threads):
|
|
22
|
+
out = fit(*args, **kwargs)
|
|
23
|
+
rss = peak_rss_gb()
|
|
24
|
+
rss = None if rss is None else round(rss, 3)
|
|
25
|
+
if isinstance(out, Profile):
|
|
26
|
+
return out
|
|
27
|
+
for r in ([out] if hasattr(out, "diagnostics") else out):
|
|
28
|
+
r.diagnostics["peak_rss_gb"] = rss
|
|
29
|
+
return out
|
|
30
|
+
|
|
31
|
+
return wrapper
|
|
@@ -342,7 +342,7 @@ mlogit_stream.__doc__ = """Out-of-core multinomial logit by streamed Newton.
|
|
|
342
342
|
distinct values, sorted.
|
|
343
343
|
poisson_options : dict, optional
|
|
344
344
|
With several fixed effects: further options of hdfe-stream's
|
|
345
|
-
`fepois_stream` (``solver``, ``precond``, ``
|
|
345
|
+
`fepois_stream` (``solver``, ``precond``, ``max_memory_gb``,
|
|
346
346
|
``fixef_rm``, ...), except those the multinomial logit sets itself
|
|
347
347
|
(vcov, cluster, weights, keep, workdir).""" + _RANDOM + """
|
|
348
348
|
|
|
@@ -12,7 +12,6 @@ from __future__ import annotations
|
|
|
12
12
|
|
|
13
13
|
import logging
|
|
14
14
|
import os
|
|
15
|
-
import sys
|
|
16
15
|
import tempfile
|
|
17
16
|
import time
|
|
18
17
|
from collections.abc import Sequence
|
|
@@ -23,11 +22,13 @@ from typing import Any
|
|
|
23
22
|
import numpy as np
|
|
24
23
|
import polars as pl
|
|
25
24
|
from hdfe_stream.core.report import warn
|
|
25
|
+
from hdfe_stream.core.resources import MemoryBudget
|
|
26
26
|
from hdfe_stream.core.results_base import vcov_key
|
|
27
27
|
from hdfe_stream.core.store import PolarsEngine
|
|
28
28
|
from hdfe_stream.core.workspace import Run
|
|
29
29
|
|
|
30
30
|
from ._frontend import _plan
|
|
31
|
+
from ._limits import limited
|
|
31
32
|
from .models import _CFELogit, _Conditional, _Multinomial, _Ordered
|
|
32
33
|
from .newton import _Driver
|
|
33
34
|
from .random_effects import _fit_random, _RandomEffects, _refine_laplace, _sigma_inference
|
|
@@ -96,6 +97,7 @@ def _clusterings(vcov, cluster):
|
|
|
96
97
|
return key, clusters, hetero, requests
|
|
97
98
|
|
|
98
99
|
|
|
100
|
+
@limited
|
|
99
101
|
def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
|
|
100
102
|
cluster: Sequence[str] | str = (), weights: str | None = None,
|
|
101
103
|
weights_type: str = "aweights", levels: Sequence | None = None,
|
|
@@ -106,7 +108,9 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
|
|
|
106
108
|
log_level=None, random: str | None = None, random_cov: str = "unstructured",
|
|
107
109
|
random_tol: float = 1e-6, random_method: str = "laplace", firth: bool = False,
|
|
108
110
|
offset=None, start=None, device: str = "cpu", link: str = "logit",
|
|
109
|
-
altwise: bool = False, poisson_options: dict | None = None
|
|
111
|
+
altwise: bool = False, poisson_options: dict | None = None,
|
|
112
|
+
profile: bool = False, n_threads: int | None = None,
|
|
113
|
+
max_memory_gb: float | None = None, memory_reserve: float | None = None):
|
|
110
114
|
if poisson_options is not None and kind != "mlogit":
|
|
111
115
|
raise ValueError("poisson_options= is for mlogit_stream with several fixed effects")
|
|
112
116
|
if weights_type not in ("aweights", "fweights"):
|
|
@@ -172,6 +176,7 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
|
|
|
172
176
|
if l2:
|
|
173
177
|
raise ValueError("l2= and random= cannot be combined")
|
|
174
178
|
vkey, clusters, hetero, requests = _clusterings(vcov, cluster)
|
|
179
|
+
budget = MemoryBudget(max_memory_gb, memory_reserve)
|
|
175
180
|
|
|
176
181
|
def log(msg):
|
|
177
182
|
_log(verbose, msg, logger, log_level)
|
|
@@ -214,6 +219,14 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
|
|
|
214
219
|
raise ValueError(f"{spec.fml}: firth= is not available with an absorbed "
|
|
215
220
|
"fixed effect")
|
|
216
221
|
if len(spec.fe) > 1 and kind == "mlogit":
|
|
222
|
+
if profile:
|
|
223
|
+
from .profile import profile_multi_fe
|
|
224
|
+
results.append(profile_multi_fe(
|
|
225
|
+
spec, lf, data, weights=weights, weights_type=weights_type,
|
|
226
|
+
clusters=clusters, requests=requests, hetero=hetero, vkey=vkey,
|
|
227
|
+
levels=levels, options=poisson_options, budget=budget,
|
|
228
|
+
n_threads=n_threads, tol=tol, maxiter=maxiter))
|
|
229
|
+
continue
|
|
217
230
|
# several: as a Poisson regression on the data expanded by level
|
|
218
231
|
bad = [nm for nm, v in (("l2=", l2), ("start=", start),
|
|
219
232
|
("storage='memory'", storage == "memory"))
|
|
@@ -236,6 +249,14 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
|
|
|
236
249
|
raise ValueError(f"{spec.fml}: one absorbed fixed effect is available for "
|
|
237
250
|
f"{_KINDS[kind]} (got {list(spec.fe)}); put small "
|
|
238
251
|
"dimensions in as C(var)")
|
|
252
|
+
if profile:
|
|
253
|
+
from .profile import profile_model
|
|
254
|
+
results.append(profile_model(
|
|
255
|
+
kind, spec, lf, data, weights=weights, clusters=clusters, requests=requests,
|
|
256
|
+
case=case, levels=levels, storage=storage, dtype=dtype, block_mb=block_mb,
|
|
257
|
+
chunk_mb=chunk_mb, offset=offset, random=random, firth=firth,
|
|
258
|
+
n_threads=n_threads, budget=budget))
|
|
259
|
+
continue
|
|
239
260
|
results.append(_fit_one(kind, spec, lf, engine, vkey=vkey, clusters=clusters,
|
|
240
261
|
requests=requests,
|
|
241
262
|
hetero=hetero, weights=weights, weights_type=weights_type,
|
|
@@ -246,7 +267,10 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
|
|
|
246
267
|
logger=logger, random=random, random_cov=random_cov,
|
|
247
268
|
random_tol=random_tol, random_method=random_method,
|
|
248
269
|
firth=firth, offset=offset, start=start, device=device,
|
|
249
|
-
link=link, altwise=altwise))
|
|
270
|
+
link=link, altwise=altwise, budget=budget))
|
|
271
|
+
if profile:
|
|
272
|
+
from .profile import combine_profiles
|
|
273
|
+
return combine_profiles(results)
|
|
250
274
|
return results[0] if len(results) == 1 else LogitMulti(results)
|
|
251
275
|
|
|
252
276
|
|
|
@@ -254,7 +278,7 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
254
278
|
levels, case, tol, maxiter, l2, storage, dtype, block_mb, chunk_mb, workdir,
|
|
255
279
|
keep_intermediates, log, logger, random=None, random_cov="unstructured",
|
|
256
280
|
random_tol=1e-6, random_method="laplace", firth=False, offset=None, start=None,
|
|
257
|
-
device=None, link="logit", altwise=False):
|
|
281
|
+
device=None, link="logit", altwise=False, budget=None):
|
|
258
282
|
t0 = time.time()
|
|
259
283
|
log(f"{_KINDS[kind]}: {spec.fml}")
|
|
260
284
|
run = Run(_base_dir(workdir), auto_cleanup=not keep_intermediates)
|
|
@@ -290,7 +314,7 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
290
314
|
else None
|
|
291
315
|
if random is not None:
|
|
292
316
|
keys = {nm: ("fe" if i == 0 else f"fe{i + 1}") for i, nm in enumerate(spec.fe)}
|
|
293
|
-
fe = _RandomEffects(model, p0, random, outer, keys, log=log, logger=logger,
|
|
317
|
+
fe = _RandomEffects(model, p0, random, outer, keys, log=log, logger=logger, budget=budget,
|
|
294
318
|
cov=random_cov)
|
|
295
319
|
if clusters:
|
|
296
320
|
cl = {k: (i, 0) for i, k in enumerate(clusters)}
|
|
@@ -302,7 +326,7 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
302
326
|
f"crossed groupings allow none; {bad} do not")
|
|
303
327
|
elif spec.fe:
|
|
304
328
|
from .fe import _AbsorbedFE
|
|
305
|
-
fe = _AbsorbedFE(model, p0, spec.fe[0], log=log, logger=logger)
|
|
329
|
+
fe = _AbsorbedFE(model, p0, spec.fe[0], log=log, logger=logger, budget=budget)
|
|
306
330
|
driver = _Driver(p0.store, model, tol=tol, maxiter=maxiter, l2=l2, firth=firth,
|
|
307
331
|
chunk_mb=chunk_mb, device=device,
|
|
308
332
|
log=log, logger=logger, fe=fe)
|
|
@@ -366,7 +390,8 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
|
|
|
366
390
|
raw = {k: _sandwich(fe, fit, Hinv, U[k].T @ U[k]) for k in cl_index}
|
|
367
391
|
for req, parts in requests.items():
|
|
368
392
|
if req.startswith("CRV3:"):
|
|
369
|
-
all_vcovs[req] = _crv3(driver, fit, U, cl_index, parts[0][0], spec, logger
|
|
393
|
+
all_vcovs[req] = _crv3(driver, fit, U, cl_index, parts[0][0], spec, logger,
|
|
394
|
+
budget)
|
|
370
395
|
n_clusters[req.split(":", 1)[1]] = cl_index[parts[0][0]][1]
|
|
371
396
|
continue
|
|
372
397
|
Vc, df, Gs = _clustered(raw, parts, cl_index, req, spec.fml, logger)
|
|
@@ -502,29 +527,7 @@ def _warm_start(theta0, start, model, fml):
|
|
|
502
527
|
return th
|
|
503
528
|
|
|
504
529
|
|
|
505
|
-
def
|
|
506
|
-
"""The machine's physical memory in GB: sysconf where there is one, the
|
|
507
|
-
Windows API otherwise; 16 if neither answers."""
|
|
508
|
-
try:
|
|
509
|
-
return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / 1e9
|
|
510
|
-
except (AttributeError, ValueError, OSError):
|
|
511
|
-
pass
|
|
512
|
-
if sys.platform == "win32":
|
|
513
|
-
import ctypes
|
|
514
|
-
|
|
515
|
-
class _Status(ctypes.Structure):
|
|
516
|
-
_fields_ = [("length", ctypes.c_ulong), ("load", ctypes.c_ulong),
|
|
517
|
-
("total", ctypes.c_ulonglong), ("avail", ctypes.c_ulonglong),
|
|
518
|
-
("page_total", ctypes.c_ulonglong), ("page_avail", ctypes.c_ulonglong),
|
|
519
|
-
("virt_total", ctypes.c_ulonglong), ("virt_avail", ctypes.c_ulonglong),
|
|
520
|
-
("ext", ctypes.c_ulonglong)]
|
|
521
|
-
status = _Status(length=ctypes.sizeof(_Status))
|
|
522
|
-
if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
|
|
523
|
-
return status.total / 1e9
|
|
524
|
-
return 16.0
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
def _crv3(driver, fit, U, cl_index, key, spec, logger):
|
|
530
|
+
def _crv3(driver, fit, U, cl_index, key, spec, logger, budget=None):
|
|
528
531
|
"""The cluster jackknife (CV3; MacKinnon, Nielsen and Webb 2023):
|
|
529
532
|
(G - 1) / G sum_g d_g d_g', with d_g the change in the estimates from
|
|
530
533
|
dropping cluster g, by one Newton step from them -- the information
|
|
@@ -535,11 +538,12 @@ def _crv3(driver, fit, U, cl_index, key, spec, logger):
|
|
|
535
538
|
col, G = cl_index[key]
|
|
536
539
|
P = len(fit.theta)
|
|
537
540
|
need = G * P * P * 8 / 1e9
|
|
538
|
-
|
|
539
|
-
if
|
|
541
|
+
budget = budget if budget is not None else MemoryBudget()
|
|
542
|
+
if not budget.fits(need):
|
|
540
543
|
raise MemoryError(f"{spec.fml}: CRV3 by {key.split(':', 1)[1]!r} needs {need:.1f} GB "
|
|
541
544
|
f"for each cluster's information ({G:,} clusters x {P} parameters "
|
|
542
|
-
"squared); use
|
|
545
|
+
f"squared) and {budget.headroom_gb():.1f} GB is free ({budget}); use "
|
|
546
|
+
"CRV1, or raise max_memory_gb")
|
|
543
547
|
Hg = driver.cluster_information(fit.theta, col, G)
|
|
544
548
|
H, S = fit.H, U[key]
|
|
545
549
|
D = np.empty((G, P))
|
|
@@ -44,6 +44,7 @@ import numba as nb
|
|
|
44
44
|
import numpy as np
|
|
45
45
|
import polars as pl
|
|
46
46
|
from hdfe_stream.core.report import warn
|
|
47
|
+
from hdfe_stream.core.resources import MemoryBudget
|
|
47
48
|
from scipy.optimize import minimize
|
|
48
49
|
from scipy.special import xlogy
|
|
49
50
|
|
|
@@ -51,8 +52,6 @@ from .models import _gram, _mnl_info_sparse
|
|
|
51
52
|
from .newton import _chunks, _design, _Driver
|
|
52
53
|
from .store import CODE
|
|
53
54
|
|
|
54
|
-
_MAX_GROUP_GB = 8.0
|
|
55
|
-
|
|
56
55
|
|
|
57
56
|
@nb.njit(parallel=True, cache=True)
|
|
58
57
|
def _mnl_group_info(order, seg, segg, X, P, w, Haa, B):
|
|
@@ -147,8 +146,9 @@ class _AbsorbedFE:
|
|
|
147
146
|
"""The fixed effect's bookkeeping, passes and linear algebra for the
|
|
148
147
|
Newton driver (see the module docstring)."""
|
|
149
148
|
|
|
150
|
-
def __init__(self, model, p0, name, log, logger=None):
|
|
149
|
+
def __init__(self, model, p0, name, log, logger=None, budget=None):
|
|
151
150
|
self.model, self.p0, self.name = model, p0, name
|
|
151
|
+
self.budget = budget if budget is not None else MemoryBudget()
|
|
152
152
|
self.store = p0.store
|
|
153
153
|
self.logger = logger
|
|
154
154
|
G = p0.fe_tables[name].height
|
|
@@ -200,11 +200,12 @@ class _AbsorbedFE:
|
|
|
200
200
|
"outcome never varies")
|
|
201
201
|
self.n_params = int(self.free.sum())
|
|
202
202
|
gb = G * self.m * max(model.P, 1) * 8 * 2 / 1e9
|
|
203
|
-
if gb
|
|
203
|
+
if not self.budget.fits(gb):
|
|
204
204
|
raise MemoryError(
|
|
205
205
|
f"absorbing {name} needs about {gb:.1f} GB for its per-level blocks "
|
|
206
|
-
f"({G:,} levels x {self.m} x {model.P} parameters, twice)
|
|
207
|
-
"
|
|
206
|
+
f"({G:,} levels x {self.m} x {model.P} parameters, twice) and "
|
|
207
|
+
f"{self.budget.headroom_gb():.1f} GB is free ({self.budget}); fit with fewer "
|
|
208
|
+
"covariates, a coarser fixed effect, or a larger max_memory_gb")
|
|
208
209
|
|
|
209
210
|
def loglik_fe_only(self):
|
|
210
211
|
"""The log-likelihood of the model with the fixed effect and no
|
|
@@ -38,6 +38,7 @@ import time
|
|
|
38
38
|
import numba as nb
|
|
39
39
|
import numpy as np
|
|
40
40
|
|
|
41
|
+
from ._limits import limited
|
|
41
42
|
from .newton import _Driver
|
|
42
43
|
from .optimize import _GradientDriver
|
|
43
44
|
|
|
@@ -388,13 +389,16 @@ class _MixedDriver(_GradientDriver):
|
|
|
388
389
|
# fitting
|
|
389
390
|
# --------------------------------------------------------------------------
|
|
390
391
|
|
|
392
|
+
@limited
|
|
391
393
|
def _fit_mixed(fml, data, *, case, panel=None, random, draws=500, halton=True, seed=0,
|
|
392
394
|
drop=100, vcov="iid", workdir=None, tol=1e-8, maxiter=200, switch=np.inf,
|
|
393
395
|
draws_cache_mb=512.0,
|
|
394
396
|
storage="disk", dtype="float64", block_mb=64.0, polars_engine="cpu",
|
|
395
397
|
keep_intermediates=False, verbose=True, logger=None, log_level=None,
|
|
396
|
-
device="cpu"
|
|
398
|
+
device="cpu", profile=False, chunk_mb=64.0, max_memory_gb=None,
|
|
399
|
+
memory_reserve=None):
|
|
397
400
|
from hdfe_stream.core.report import warn
|
|
401
|
+
from hdfe_stream.core.resources import MemoryBudget
|
|
398
402
|
from hdfe_stream.core.results_base import vcov_key
|
|
399
403
|
from hdfe_stream.core.store import PolarsEngine
|
|
400
404
|
from hdfe_stream.core.utils import as_lazy
|
|
@@ -446,6 +450,13 @@ def _fit_mixed(fml, data, *, case, panel=None, random, draws=500, halton=True, s
|
|
|
446
450
|
kind[k], ridx[k] = DISTRIBUTIONS[dist], len(rnames)
|
|
447
451
|
rnames.append(nm)
|
|
448
452
|
|
|
453
|
+
if profile:
|
|
454
|
+
from .profile import profile_model
|
|
455
|
+
return profile_model("mixlogit", spec, lf, data, case=case, panel=panel if own_panel else None,
|
|
456
|
+
storage=storage, dtype=dtype, block_mb=block_mb, chunk_mb=chunk_mb,
|
|
457
|
+
n_random=len(rnames), draws=int(draws),
|
|
458
|
+
draws_cache_mb=draws_cache_mb,
|
|
459
|
+
budget=MemoryBudget(max_memory_gb, memory_reserve))
|
|
449
460
|
t0 = time.time()
|
|
450
461
|
log(f"mixed logit: {spec.fml}")
|
|
451
462
|
run = Run(_base_dir(workdir), auto_cleanup=not keep_intermediates)
|