logit-stream 0.1.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {logit_stream-0.1.0 → logit_stream-0.3.0}/CHANGELOG.md +51 -0
  2. {logit_stream-0.1.0 → logit_stream-0.3.0}/CITATION.cff +2 -2
  3. {logit_stream-0.1.0 → logit_stream-0.3.0}/PKG-INFO +4 -2
  4. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/__init__.py +1 -1
  5. logit_stream-0.3.0/logit_stream/_limits.py +31 -0
  6. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/api.py +1 -1
  7. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/estimator.py +37 -33
  8. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/fe.py +7 -6
  9. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/mixed.py +12 -1
  10. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/models.py +225 -63
  11. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/nested.py +12 -1
  12. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/newton.py +12 -0
  13. logit_stream-0.3.0/logit_stream/profile.py +270 -0
  14. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/random_effects/model.py +9 -5
  15. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/store.py +150 -58
  16. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream.egg-info/PKG-INFO +4 -2
  17. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream.egg-info/SOURCES.txt +6 -0
  18. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream.egg-info/requires.txt +3 -1
  19. {logit_stream-0.1.0 → logit_stream-0.3.0}/pyproject.toml +6 -2
  20. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_cfelogit.py +18 -0
  21. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_conditional.py +22 -0
  22. logit_stream-0.3.0/tests/test_headroom.py +61 -0
  23. logit_stream-0.3.0/tests/test_index_dtype.py +51 -0
  24. logit_stream-0.3.0/tests/test_profile.py +161 -0
  25. logit_stream-0.3.0/tests/test_resources.py +135 -0
  26. {logit_stream-0.1.0 → logit_stream-0.3.0}/LICENSE.md +0 -0
  27. {logit_stream-0.1.0 → logit_stream-0.3.0}/MANIFEST.in +0 -0
  28. {logit_stream-0.1.0 → logit_stream-0.3.0}/README.md +0 -0
  29. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/_choice.py +0 -0
  30. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/_frontend.py +0 -0
  31. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/__init__.py +0 -0
  32. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/_common.py +0 -0
  33. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/fe.py +0 -0
  34. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/meat.py +0 -0
  35. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/mixed.py +0 -0
  36. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/models.py +0 -0
  37. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/nested.py +0 -0
  38. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/random_effects.py +0 -0
  39. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/device/store.py +0 -0
  40. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/multi_fe.py +0 -0
  41. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/optimize.py +0 -0
  42. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/py.typed +0 -0
  43. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/random_effects/__init__.py +0 -0
  44. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/random_effects/fit.py +0 -0
  45. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/random_effects/linalg.py +0 -0
  46. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/random_effects/sigma.py +0 -0
  47. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/results.py +0 -0
  48. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream/simulate.py +0 -0
  49. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream.egg-info/dependency_links.txt +0 -0
  50. {logit_stream-0.1.0 → logit_stream-0.3.0}/logit_stream.egg-info/top_level.txt +0 -0
  51. {logit_stream-0.1.0 → logit_stream-0.3.0}/setup.cfg +0 -0
  52. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_api.py +0 -0
  53. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_clustering.py +0 -0
  54. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_device.py +0 -0
  55. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_firth_and_fit.py +0 -0
  56. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_fixed_effects.py +0 -0
  57. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_kernels.py +0 -0
  58. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_mixed.py +0 -0
  59. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_multi_fe.py +0 -0
  60. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_multinomial.py +0 -0
  61. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_nested.py +0 -0
  62. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_offset_start.py +0 -0
  63. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_ordered.py +0 -0
  64. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_probit.py +0 -0
  65. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_random.py +0 -0
  66. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_simulate.py +0 -0
  67. {logit_stream-0.1.0 → logit_stream-0.3.0}/tests/test_weights_scale.py +0 -0
@@ -1,5 +1,56 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.3.0
4
+
5
+ - **The conditional logit is 2.3-2.8 times faster.** Its log-likelihood, gradient and information
6
+ are computed in one parallel numba pass over each block's choice sets, giving the same
7
+ estimates (4 million sets: 13 s, now 5 s; 8 million: 26 s, now 9-12 s). Designs of more than
8
+ 256 parameters keep the previous BLAS-based pass, which is faster there.
9
+ - **The fixed-effects logit (`cfelogit_stream`) balances work across threads better.** Panels
10
+ are dealt out in small pieces to more parts than threads, so a run of long panels no longer
11
+ leaves threads idle: 0.55 of the time on panels sorted by length, 0.8-1.0 on shuffled
12
+ ones, with the same results.
13
+ - **Lower memory for conditional, mixed and fixed-effects logit.** Choice sets are now checked
14
+ and counted a block at a time instead of from a table of every set, so memory no longer grows
15
+ with the number of sets (8 million sets: peak 1.71 GB down to 1.33 GB). Same errors, same
16
+ results.
17
+ - **Memory tests.** `pytest -m resource` fits the multinomial, ordered and conditional logit
18
+ on data larger than a fixed memory limit; CI runs them on Linux, macOS and Windows. The
19
+ default `pytest` run leaves them out.
20
+ - Requires hdfe-stream 0.9.0 or later, which reads Parquet in less memory.
21
+
22
+ ## 0.2.0
23
+
24
+ - **Fit resources (CPUs and memory) are constrained to your job's limits automatically.** On a
25
+ cluster (PBS, Slurm, or a container with cgroup limits), importing `logit_stream` caps the threads of
26
+ Polars, numba, BLAS and pyarrow at the job's CPU allocation, and with Polars 2.x gives Polars a
27
+ memory budget so its streaming operations spill to disk instead of growing. Caveats: the Polars
28
+ budget is a target for its buffers, not a hard cap on the process, and Polars 1.x has no such
29
+ option, so there only the threads are capped. The caps must be in place before Polars is
30
+ imported: import `logit_stream` first, or call `stream_limits.set_limits()` at the top of the script.
31
+ With no limit found nothing changes. See "Resource limits and profiling" in the documentation.
32
+ - **`profile=True`: estimate a fit's memory and disk before running it.** On
33
+ every estimator, profiling returns a `hdfe_stream.Profile` -- the estimated
34
+ memory and disk, from one scan of the data -- instead of fitting. P comes from
35
+ the model and the outcome's levels; the information matrix, its per-thread
36
+ copies, the fixed effect's cross blocks and the clusters' score sums are sized
37
+ from it. The multinomial logit with several fixed effects (fitted by
38
+ `fepois_stream` on data expanded J-fold) is profiled as that regression, from
39
+ counts worked out from one scan of the original data. Not modeled: random
40
+ intercepts, and the mixed and nested logit's workspaces beyond the per-thread
41
+ accumulators.
42
+ - **`max_memory_gb` and `memory_reserve`**: the fixed effect's cross-block guard and
43
+ the random intercepts' guard (both a fixed 8 GB) and CRV3's (25% of physical
44
+ memory) now ask whether their exact allocation fits in the memory left (the limit
45
+ less a reserve, less what the process holds). The limit defaults to the job's
46
+ allocation (cgroup, `STREAM_LIMITS_MEMORY_GB`, or on PBS `qstat`).
47
+ - **`n_threads=`** on every estimator: numba and BLAS threads for one fit,
48
+ restored afterwards (Polars' pool is fixed at import; see hdfe-stream's
49
+ "Resource limits and profiling").
50
+ - **`diagnostics["peak_rss_gb"]`** in every fit.
51
+ - Depends on `stream-limits`, and needs hdfe-stream 0.8.0 or later (the `<0.7` upper bound on
52
+ hdfe-stream is gone).
53
+
3
54
  ## 0.1.0
4
55
 
5
56
  The first release: logit models by streamed Newton's method, for data that
@@ -7,8 +7,8 @@ authors:
7
7
  - family-names: Tucker
8
8
  given-names: Lee
9
9
  email: lee.c.tucker@census.gov
10
- version: 0.1.0
11
- date-released: 2026-10-29
10
+ version: 0.3.0
11
+ date-released: 2026-11-05
12
12
  license: MIT
13
13
  repository-code: "https://github.com/leetucker/logit-stream"
14
14
  url: "https://github.com/leetucker/logit-stream"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: logit-stream
3
- Version: 0.1.0
3
+ Version: 0.3.0
4
4
  Summary: Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects
5
5
  Author: Lee Tucker
6
6
  License-Expression: MIT
@@ -25,7 +25,8 @@ Classifier: Typing :: Typed
25
25
  Requires-Python: >=3.10
26
26
  Description-Content-Type: text/markdown
27
27
  License-File: LICENSE.md
28
- Requires-Dist: hdfe-stream[formula]<0.7,>=0.6
28
+ Requires-Dist: hdfe-stream[formula]>=0.9.0
29
+ Requires-Dist: stream-limits>=0.1.1
29
30
  Requires-Dist: polars!=1.41.*,>=1.37
30
31
  Requires-Dist: numpy>=1.24
31
32
  Requires-Dist: numba>=0.59
@@ -48,6 +49,7 @@ Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
48
49
  Requires-Dist: sphinx-design>=0.6; extra == "docs"
49
50
  Provides-Extra: test
50
51
  Requires-Dist: pytest>=7.0; extra == "test"
52
+ Requires-Dist: stream-limits[testing]>=0.2.1; extra == "test"
51
53
  Requires-Dist: scikit-learn>=1.5; extra == "test"
52
54
  Requires-Dist: statsmodels>=0.14; extra == "test"
53
55
  Requires-Dist: pandas>=2.0; extra == "test"
@@ -62,7 +62,7 @@ def _version():
62
62
  from importlib.metadata import version
63
63
  return version("logit-stream")
64
64
  except Exception: # noqa: BLE001 -- not installed
65
- return "0.1.0"
65
+ return "0.3.0"
66
66
 
67
67
 
68
68
  __version__ = _version()
@@ -0,0 +1,31 @@
1
+ """Resource limits for a fit: the `n_threads=` option, and the peak memory the
2
+ fit used. The code is hdfe-stream's (`hdfe_stream.core.resources`); see
3
+ "Resource limits and profiling" in its documentation."""
4
+
5
+ from __future__ import annotations
6
+
7
+ import functools
8
+
9
+ from hdfe_stream.core.profile import Profile
10
+ from hdfe_stream.core.resources import peak_rss_gb, reset_peak_rss, thread_scope
11
+
12
+
13
+ def limited(fit):
14
+ """Give an estimator's fit function an ``n_threads`` keyword that holds
15
+ numba and BLAS to that many threads for the fit (restored afterwards; Polars
16
+ cannot be limited per fit), and record ``diagnostics["peak_rss_gb"]``."""
17
+
18
+ @functools.wraps(fit)
19
+ def wrapper(*args, n_threads=None, **kwargs):
20
+ reset_peak_rss()
21
+ with thread_scope(n_threads):
22
+ out = fit(*args, **kwargs)
23
+ rss = peak_rss_gb()
24
+ rss = None if rss is None else round(rss, 3)
25
+ if isinstance(out, Profile):
26
+ return out
27
+ for r in ([out] if hasattr(out, "diagnostics") else out):
28
+ r.diagnostics["peak_rss_gb"] = rss
29
+ return out
30
+
31
+ return wrapper
@@ -342,7 +342,7 @@ mlogit_stream.__doc__ = """Out-of-core multinomial logit by streamed Newton.
342
342
  distinct values, sorted.
343
343
  poisson_options : dict, optional
344
344
  With several fixed effects: further options of hdfe-stream's
345
- `fepois_stream` (``solver``, ``precond``, ``max_s_gb``,
345
+ `fepois_stream` (``solver``, ``precond``, ``max_memory_gb``,
346
346
  ``fixef_rm``, ...), except those the multinomial logit sets itself
347
347
  (vcov, cluster, weights, keep, workdir).""" + _RANDOM + """
348
348
 
@@ -12,7 +12,6 @@ from __future__ import annotations
12
12
 
13
13
  import logging
14
14
  import os
15
- import sys
16
15
  import tempfile
17
16
  import time
18
17
  from collections.abc import Sequence
@@ -23,11 +22,13 @@ from typing import Any
23
22
  import numpy as np
24
23
  import polars as pl
25
24
  from hdfe_stream.core.report import warn
25
+ from hdfe_stream.core.resources import MemoryBudget
26
26
  from hdfe_stream.core.results_base import vcov_key
27
27
  from hdfe_stream.core.store import PolarsEngine
28
28
  from hdfe_stream.core.workspace import Run
29
29
 
30
30
  from ._frontend import _plan
31
+ from ._limits import limited
31
32
  from .models import _CFELogit, _Conditional, _Multinomial, _Ordered
32
33
  from .newton import _Driver
33
34
  from .random_effects import _fit_random, _RandomEffects, _refine_laplace, _sigma_inference
@@ -96,6 +97,7 @@ def _clusterings(vcov, cluster):
96
97
  return key, clusters, hetero, requests
97
98
 
98
99
 
100
+ @limited
99
101
  def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
100
102
  cluster: Sequence[str] | str = (), weights: str | None = None,
101
103
  weights_type: str = "aweights", levels: Sequence | None = None,
@@ -106,7 +108,9 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
106
108
  log_level=None, random: str | None = None, random_cov: str = "unstructured",
107
109
  random_tol: float = 1e-6, random_method: str = "laplace", firth: bool = False,
108
110
  offset=None, start=None, device: str = "cpu", link: str = "logit",
109
- altwise: bool = False, poisson_options: dict | None = None):
111
+ altwise: bool = False, poisson_options: dict | None = None,
112
+ profile: bool = False, n_threads: int | None = None,
113
+ max_memory_gb: float | None = None, memory_reserve: float | None = None):
110
114
  if poisson_options is not None and kind != "mlogit":
111
115
  raise ValueError("poisson_options= is for mlogit_stream with several fixed effects")
112
116
  if weights_type not in ("aweights", "fweights"):
@@ -172,6 +176,7 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
172
176
  if l2:
173
177
  raise ValueError("l2= and random= cannot be combined")
174
178
  vkey, clusters, hetero, requests = _clusterings(vcov, cluster)
179
+ budget = MemoryBudget(max_memory_gb, memory_reserve)
175
180
 
176
181
  def log(msg):
177
182
  _log(verbose, msg, logger, log_level)
@@ -214,6 +219,14 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
214
219
  raise ValueError(f"{spec.fml}: firth= is not available with an absorbed "
215
220
  "fixed effect")
216
221
  if len(spec.fe) > 1 and kind == "mlogit":
222
+ if profile:
223
+ from .profile import profile_multi_fe
224
+ results.append(profile_multi_fe(
225
+ spec, lf, data, weights=weights, weights_type=weights_type,
226
+ clusters=clusters, requests=requests, hetero=hetero, vkey=vkey,
227
+ levels=levels, options=poisson_options, budget=budget,
228
+ n_threads=n_threads, tol=tol, maxiter=maxiter))
229
+ continue
217
230
  # several: as a Poisson regression on the data expanded by level
218
231
  bad = [nm for nm, v in (("l2=", l2), ("start=", start),
219
232
  ("storage='memory'", storage == "memory"))
@@ -236,6 +249,14 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
236
249
  raise ValueError(f"{spec.fml}: one absorbed fixed effect is available for "
237
250
  f"{_KINDS[kind]} (got {list(spec.fe)}); put small "
238
251
  "dimensions in as C(var)")
252
+ if profile:
253
+ from .profile import profile_model
254
+ results.append(profile_model(
255
+ kind, spec, lf, data, weights=weights, clusters=clusters, requests=requests,
256
+ case=case, levels=levels, storage=storage, dtype=dtype, block_mb=block_mb,
257
+ chunk_mb=chunk_mb, offset=offset, random=random, firth=firth,
258
+ n_threads=n_threads, budget=budget))
259
+ continue
239
260
  results.append(_fit_one(kind, spec, lf, engine, vkey=vkey, clusters=clusters,
240
261
  requests=requests,
241
262
  hetero=hetero, weights=weights, weights_type=weights_type,
@@ -246,7 +267,10 @@ def _fit(kind: str, fml: str, data: Any, *, workdir=None, vcov="iid",
246
267
  logger=logger, random=random, random_cov=random_cov,
247
268
  random_tol=random_tol, random_method=random_method,
248
269
  firth=firth, offset=offset, start=start, device=device,
249
- link=link, altwise=altwise))
270
+ link=link, altwise=altwise, budget=budget))
271
+ if profile:
272
+ from .profile import combine_profiles
273
+ return combine_profiles(results)
250
274
  return results[0] if len(results) == 1 else LogitMulti(results)
251
275
 
252
276
 
@@ -254,7 +278,7 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
254
278
  levels, case, tol, maxiter, l2, storage, dtype, block_mb, chunk_mb, workdir,
255
279
  keep_intermediates, log, logger, random=None, random_cov="unstructured",
256
280
  random_tol=1e-6, random_method="laplace", firth=False, offset=None, start=None,
257
- device=None, link="logit", altwise=False):
281
+ device=None, link="logit", altwise=False, budget=None):
258
282
  t0 = time.time()
259
283
  log(f"{_KINDS[kind]}: {spec.fml}")
260
284
  run = Run(_base_dir(workdir), auto_cleanup=not keep_intermediates)
@@ -290,7 +314,7 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
290
314
  else None
291
315
  if random is not None:
292
316
  keys = {nm: ("fe" if i == 0 else f"fe{i + 1}") for i, nm in enumerate(spec.fe)}
293
- fe = _RandomEffects(model, p0, random, outer, keys, log=log, logger=logger,
317
+ fe = _RandomEffects(model, p0, random, outer, keys, log=log, logger=logger, budget=budget,
294
318
  cov=random_cov)
295
319
  if clusters:
296
320
  cl = {k: (i, 0) for i, k in enumerate(clusters)}
@@ -302,7 +326,7 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
302
326
  f"crossed groupings allow none; {bad} do not")
303
327
  elif spec.fe:
304
328
  from .fe import _AbsorbedFE
305
- fe = _AbsorbedFE(model, p0, spec.fe[0], log=log, logger=logger)
329
+ fe = _AbsorbedFE(model, p0, spec.fe[0], log=log, logger=logger, budget=budget)
306
330
  driver = _Driver(p0.store, model, tol=tol, maxiter=maxiter, l2=l2, firth=firth,
307
331
  chunk_mb=chunk_mb, device=device,
308
332
  log=log, logger=logger, fe=fe)
@@ -366,7 +390,8 @@ def _fit_one(kind, spec, lf, engine, *, vkey, clusters, requests, hetero, weight
366
390
  raw = {k: _sandwich(fe, fit, Hinv, U[k].T @ U[k]) for k in cl_index}
367
391
  for req, parts in requests.items():
368
392
  if req.startswith("CRV3:"):
369
- all_vcovs[req] = _crv3(driver, fit, U, cl_index, parts[0][0], spec, logger)
393
+ all_vcovs[req] = _crv3(driver, fit, U, cl_index, parts[0][0], spec, logger,
394
+ budget)
370
395
  n_clusters[req.split(":", 1)[1]] = cl_index[parts[0][0]][1]
371
396
  continue
372
397
  Vc, df, Gs = _clustered(raw, parts, cl_index, req, spec.fml, logger)
@@ -502,29 +527,7 @@ def _warm_start(theta0, start, model, fml):
502
527
  return th
503
528
 
504
529
 
505
- def _physical_memory_gb():
506
- """The machine's physical memory in GB: sysconf where there is one, the
507
- Windows API otherwise; 16 if neither answers."""
508
- try:
509
- return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / 1e9
510
- except (AttributeError, ValueError, OSError):
511
- pass
512
- if sys.platform == "win32":
513
- import ctypes
514
-
515
- class _Status(ctypes.Structure):
516
- _fields_ = [("length", ctypes.c_ulong), ("load", ctypes.c_ulong),
517
- ("total", ctypes.c_ulonglong), ("avail", ctypes.c_ulonglong),
518
- ("page_total", ctypes.c_ulonglong), ("page_avail", ctypes.c_ulonglong),
519
- ("virt_total", ctypes.c_ulonglong), ("virt_avail", ctypes.c_ulonglong),
520
- ("ext", ctypes.c_ulonglong)]
521
- status = _Status(length=ctypes.sizeof(_Status))
522
- if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
523
- return status.total / 1e9
524
- return 16.0
525
-
526
-
527
- def _crv3(driver, fit, U, cl_index, key, spec, logger):
530
+ def _crv3(driver, fit, U, cl_index, key, spec, logger, budget=None):
528
531
  """The cluster jackknife (CV3; MacKinnon, Nielsen and Webb 2023):
529
532
  (G - 1) / G sum_g d_g d_g', with d_g the change in the estimates from
530
533
  dropping cluster g, by one Newton step from them -- the information
@@ -535,11 +538,12 @@ def _crv3(driver, fit, U, cl_index, key, spec, logger):
535
538
  col, G = cl_index[key]
536
539
  P = len(fit.theta)
537
540
  need = G * P * P * 8 / 1e9
538
- mem = _physical_memory_gb()
539
- if need > 0.25 * mem:
541
+ budget = budget if budget is not None else MemoryBudget()
542
+ if not budget.fits(need):
540
543
  raise MemoryError(f"{spec.fml}: CRV3 by {key.split(':', 1)[1]!r} needs {need:.1f} GB "
541
544
  f"for each cluster's information ({G:,} clusters x {P} parameters "
542
- "squared); use CRV1")
545
+ f"squared) and {budget.headroom_gb():.1f} GB is free ({budget}); use "
546
+ "CRV1, or raise max_memory_gb")
543
547
  Hg = driver.cluster_information(fit.theta, col, G)
544
548
  H, S = fit.H, U[key]
545
549
  D = np.empty((G, P))
@@ -44,6 +44,7 @@ import numba as nb
44
44
  import numpy as np
45
45
  import polars as pl
46
46
  from hdfe_stream.core.report import warn
47
+ from hdfe_stream.core.resources import MemoryBudget
47
48
  from scipy.optimize import minimize
48
49
  from scipy.special import xlogy
49
50
 
@@ -51,8 +52,6 @@ from .models import _gram, _mnl_info_sparse
51
52
  from .newton import _chunks, _design, _Driver
52
53
  from .store import CODE
53
54
 
54
- _MAX_GROUP_GB = 8.0
55
-
56
55
 
57
56
  @nb.njit(parallel=True, cache=True)
58
57
  def _mnl_group_info(order, seg, segg, X, P, w, Haa, B):
@@ -147,8 +146,9 @@ class _AbsorbedFE:
147
146
  """The fixed effect's bookkeeping, passes and linear algebra for the
148
147
  Newton driver (see the module docstring)."""
149
148
 
150
- def __init__(self, model, p0, name, log, logger=None):
149
+ def __init__(self, model, p0, name, log, logger=None, budget=None):
151
150
  self.model, self.p0, self.name = model, p0, name
151
+ self.budget = budget if budget is not None else MemoryBudget()
152
152
  self.store = p0.store
153
153
  self.logger = logger
154
154
  G = p0.fe_tables[name].height
@@ -200,11 +200,12 @@ class _AbsorbedFE:
200
200
  "outcome never varies")
201
201
  self.n_params = int(self.free.sum())
202
202
  gb = G * self.m * max(model.P, 1) * 8 * 2 / 1e9
203
- if gb > _MAX_GROUP_GB:
203
+ if not self.budget.fits(gb):
204
204
  raise MemoryError(
205
205
  f"absorbing {name} needs about {gb:.1f} GB for its per-level blocks "
206
- f"({G:,} levels x {self.m} x {model.P} parameters, twice); fit with fewer "
207
- "covariates, or a coarser fixed effect")
206
+ f"({G:,} levels x {self.m} x {model.P} parameters, twice) and "
207
+ f"{self.budget.headroom_gb():.1f} GB is free ({self.budget}); fit with fewer "
208
+ "covariates, a coarser fixed effect, or a larger max_memory_gb")
208
209
 
209
210
  def loglik_fe_only(self):
210
211
  """The log-likelihood of the model with the fixed effect and no
@@ -38,6 +38,7 @@ import time
38
38
  import numba as nb
39
39
  import numpy as np
40
40
 
41
+ from ._limits import limited
41
42
  from .newton import _Driver
42
43
  from .optimize import _GradientDriver
43
44
 
@@ -388,13 +389,16 @@ class _MixedDriver(_GradientDriver):
388
389
  # fitting
389
390
  # --------------------------------------------------------------------------
390
391
 
392
+ @limited
391
393
  def _fit_mixed(fml, data, *, case, panel=None, random, draws=500, halton=True, seed=0,
392
394
  drop=100, vcov="iid", workdir=None, tol=1e-8, maxiter=200, switch=np.inf,
393
395
  draws_cache_mb=512.0,
394
396
  storage="disk", dtype="float64", block_mb=64.0, polars_engine="cpu",
395
397
  keep_intermediates=False, verbose=True, logger=None, log_level=None,
396
- device="cpu"):
398
+ device="cpu", profile=False, chunk_mb=64.0, max_memory_gb=None,
399
+ memory_reserve=None):
397
400
  from hdfe_stream.core.report import warn
401
+ from hdfe_stream.core.resources import MemoryBudget
398
402
  from hdfe_stream.core.results_base import vcov_key
399
403
  from hdfe_stream.core.store import PolarsEngine
400
404
  from hdfe_stream.core.utils import as_lazy
@@ -446,6 +450,13 @@ def _fit_mixed(fml, data, *, case, panel=None, random, draws=500, halton=True, s
446
450
  kind[k], ridx[k] = DISTRIBUTIONS[dist], len(rnames)
447
451
  rnames.append(nm)
448
452
 
453
+ if profile:
454
+ from .profile import profile_model
455
+ return profile_model("mixlogit", spec, lf, data, case=case, panel=panel if own_panel else None,
456
+ storage=storage, dtype=dtype, block_mb=block_mb, chunk_mb=chunk_mb,
457
+ n_random=len(rnames), draws=int(draws),
458
+ draws_cache_mb=draws_cache_mb,
459
+ budget=MemoryBudget(max_memory_gb, memory_reserve))
449
460
  t0 = time.time()
450
461
  log(f"mixed logit: {spec.fml}")
451
462
  run = Run(_base_dir(workdir), auto_cleanup=not keep_intermediates)