logit-stream 0.2.0__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {logit_stream-0.2.0 → logit_stream-0.3.1}/CHANGELOG.md +25 -0
  2. {logit_stream-0.2.0 → logit_stream-0.3.1}/CITATION.cff +2 -2
  3. {logit_stream-0.2.0 → logit_stream-0.3.1}/PKG-INFO +4 -6
  4. {logit_stream-0.2.0 → logit_stream-0.3.1}/README.md +1 -4
  5. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/__init__.py +1 -1
  6. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/_frontend.py +9 -10
  7. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/models.py +225 -63
  8. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/newton.py +12 -0
  9. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/store.py +150 -58
  10. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream.egg-info/PKG-INFO +4 -6
  11. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream.egg-info/SOURCES.txt +1 -0
  12. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream.egg-info/requires.txt +2 -1
  13. {logit_stream-0.2.0 → logit_stream-0.3.1}/pyproject.toml +5 -2
  14. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_cfelogit.py +18 -0
  15. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_conditional.py +22 -0
  16. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_multinomial.py +4 -2
  17. logit_stream-0.3.1/tests/test_resources.py +135 -0
  18. {logit_stream-0.2.0 → logit_stream-0.3.1}/LICENSE.md +0 -0
  19. {logit_stream-0.2.0 → logit_stream-0.3.1}/MANIFEST.in +0 -0
  20. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/_choice.py +0 -0
  21. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/_limits.py +0 -0
  22. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/api.py +0 -0
  23. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/__init__.py +0 -0
  24. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/_common.py +0 -0
  25. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/fe.py +0 -0
  26. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/meat.py +0 -0
  27. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/mixed.py +0 -0
  28. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/models.py +0 -0
  29. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/nested.py +0 -0
  30. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/random_effects.py +0 -0
  31. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/store.py +0 -0
  32. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/estimator.py +0 -0
  33. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/fe.py +0 -0
  34. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/mixed.py +0 -0
  35. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/multi_fe.py +0 -0
  36. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/nested.py +0 -0
  37. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/optimize.py +0 -0
  38. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/profile.py +0 -0
  39. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/py.typed +0 -0
  40. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/random_effects/__init__.py +0 -0
  41. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/random_effects/fit.py +0 -0
  42. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/random_effects/linalg.py +0 -0
  43. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/random_effects/model.py +0 -0
  44. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/random_effects/sigma.py +0 -0
  45. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/results.py +0 -0
  46. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/simulate.py +0 -0
  47. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream.egg-info/dependency_links.txt +0 -0
  48. {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream.egg-info/top_level.txt +0 -0
  49. {logit_stream-0.2.0 → logit_stream-0.3.1}/setup.cfg +0 -0
  50. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_api.py +0 -0
  51. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_clustering.py +0 -0
  52. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_device.py +0 -0
  53. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_firth_and_fit.py +0 -0
  54. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_fixed_effects.py +0 -0
  55. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_headroom.py +0 -0
  56. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_index_dtype.py +0 -0
  57. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_kernels.py +0 -0
  58. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_mixed.py +0 -0
  59. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_multi_fe.py +0 -0
  60. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_nested.py +0 -0
  61. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_offset_start.py +0 -0
  62. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_ordered.py +0 -0
  63. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_probit.py +0 -0
  64. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_profile.py +0 -0
  65. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_random.py +0 -0
  66. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_simulate.py +0 -0
  67. {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_weights_scale.py +0 -0
@@ -1,5 +1,30 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.3.1
4
+
5
+ - **Works with pyfixest 0.60 and 0.70.** Formulas are read through hdfe-stream's own formula
6
+ parser, so either pyfixest version can be installed.
7
+ - Requires hdfe-stream 0.10.0 or later.
8
+
9
+ ## 0.3.0
10
+
11
+ - **The conditional logit is 2.3-2.8 times faster.** Its log-likelihood, gradient and information
12
+ are computed in one parallel numba pass over each block's choice sets, giving the same
13
+ estimates (4 million sets: 13 s, now 5 s; 8 million: 26 s, now 9-12 s). Designs of more than
14
+ 256 parameters keep the previous BLAS-based pass, which is faster there.
15
+ - **The fixed-effects logit (`cfelogit_stream`) balances work across threads better.** Panels
16
+ are dealt out in small pieces to more parts than threads, so a run of long panels no longer
17
+ leaves threads idle: 0.55 of the time on panels sorted by length, 0.8-1.0 on shuffled
18
+ ones, with the same results.
19
+ - **Lower memory for conditional, mixed and fixed-effects logit.** Choice sets are now checked
20
+ and counted a block at a time instead of from a table of every set, so memory no longer grows
21
+ with the number of sets (8 million sets: peak 1.71 GB down to 1.33 GB). Same errors, same
22
+ results.
23
+ - **Memory tests.** `pytest -m resource` fits the multinomial, ordered and conditional logit
24
+ on data larger than a fixed memory limit; CI runs them on Linux, macOS and Windows. The
25
+ default `pytest` run leaves them out.
26
+ - Requires hdfe-stream 0.9.0 or later, which reads Parquet in less memory.
27
+
3
28
  ## 0.2.0
4
29
 
5
30
  - **Fit resources (CPUs and memory) are constrained to your job's limits automatically.** On a
@@ -7,8 +7,8 @@ authors:
7
7
  - family-names: Tucker
8
8
  given-names: Lee
9
9
  email: lee.c.tucker@census.gov
10
- version: 0.2.0
11
- date-released: 2026-10-31
10
+ version: 0.3.1
11
+ date-released: 2026-11-09
12
12
  license: MIT
13
13
  repository-code: "https://github.com/leetucker/logit-stream"
14
14
  url: "https://github.com/leetucker/logit-stream"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: logit-stream
3
- Version: 0.2.0
3
+ Version: 0.3.1
4
4
  Summary: Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects
5
5
  Author: Lee Tucker
6
6
  License-Expression: MIT
@@ -25,7 +25,7 @@ Classifier: Typing :: Typed
25
25
  Requires-Python: >=3.10
26
26
  Description-Content-Type: text/markdown
27
27
  License-File: LICENSE.md
28
- Requires-Dist: hdfe-stream[formula]>=0.8.0
28
+ Requires-Dist: hdfe-stream[formula]>=0.10.0
29
29
  Requires-Dist: stream-limits>=0.1.1
30
30
  Requires-Dist: polars!=1.41.*,>=1.37
31
31
  Requires-Dist: numpy>=1.24
@@ -49,6 +49,7 @@ Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
49
49
  Requires-Dist: sphinx-design>=0.6; extra == "docs"
50
50
  Provides-Extra: test
51
51
  Requires-Dist: pytest>=7.0; extra == "test"
52
+ Requires-Dist: stream-limits[testing]>=0.2.1; extra == "test"
52
53
  Requires-Dist: scikit-learn>=1.5; extra == "test"
53
54
  Requires-Dist: statsmodels>=0.14; extra == "test"
54
55
  Requires-Dist: pandas>=2.0; extra == "test"
@@ -108,10 +109,7 @@ data.
108
109
  pip install logit-stream
109
110
  ```
110
111
 
111
- Python 3.10 or later. On Python 3.15, pyfixest 0.60 (which hdfe-stream's
112
- formula extra brings in) has no wheels yet: it builds from source with a Rust
113
- toolchain and `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` set; everything else
114
- installs from wheels.
112
+ Python 3.10 or later.
115
113
 
116
114
  ## Benchmarks
117
115
 
@@ -52,10 +52,7 @@ data.
52
52
  pip install logit-stream
53
53
  ```
54
54
 
55
- Python 3.10 or later. On Python 3.15, pyfixest 0.60 (which hdfe-stream's
56
- formula extra brings in) has no wheels yet: it builds from source with a Rust
57
- toolchain and `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` set; everything else
58
- installs from wheels.
55
+ Python 3.10 or later.
59
56
 
60
57
  ## Benchmarks
61
58
 
@@ -62,7 +62,7 @@ def _version():
62
62
  from importlib.metadata import version
63
63
  return version("logit-stream")
64
64
  except Exception: # noqa: BLE001 -- not installed
65
- return "0.2.0"
65
+ return "0.3.1"
66
66
 
67
67
 
68
68
  __version__ = _version()
@@ -19,7 +19,7 @@ from hdfe_stream.core.formula import (
19
19
  categorical_vars,
20
20
  compile_design,
21
21
  numeric_expr,
22
- pyfixest_formula_api,
22
+ parse_formula,
23
23
  )
24
24
 
25
25
 
@@ -52,27 +52,26 @@ def _plan(fml: str, lf: pl.LazyFrame, intercept: bool, collect) -> list[_Spec]:
52
52
  categorical coding is still reduced rank, as with it.
53
53
  `collect(lf, what)` runs a query (so that the GPU engine applies).
54
54
  """
55
- Formula, _ = pyfixest_formula_api()
56
55
  schema = lf.collect_schema()
57
- specs = Formula.parse(fml)
56
+ specs = parse_formula(fml)
58
57
  cats = set()
59
58
  for s in specs:
60
- if s.first_stage is not None:
61
- raise ValueError(f"{s.formula}: IV formulas are not available for logit models")
62
- cats |= categorical_vars(s.second_stage.split("~", 1)[1], schema)
59
+ if s.iv is not None:
60
+ raise ValueError(f"{s.key}: IV formulas are not available for logit models")
61
+ cats |= categorical_vars(s.rhs, schema)
63
62
  levels = {}
64
63
  for v in sorted(cats):
65
64
  u = collect(lf.select(pl.col(v).drop_nulls().unique().sort()), f"levels of {v}")
66
65
  levels[v] = u[v].to_list()
67
66
  out = []
68
67
  for s in specs:
69
- dep, rhs = (t.strip() for t in s.second_stage.split("~", 1))
70
- fe = tuple(parse_fe(s.fixed_effects))
68
+ dep, rhs = s.dep, s.rhs
69
+ fe = tuple(parse_fe(s.fe))
71
70
  for d in fe:
72
71
  if "[" in d:
73
- raise ValueError(f"{s.formula}: varying slopes are not available for logit models")
72
+ raise ValueError(f"{s.key}: varying slopes are not available for logit models")
74
73
  _, cols = compile_design(dep, rhs, schema, levels, intercept=intercept and not fe)
75
74
  if not intercept:
76
75
  cols = [(nm, ex) for nm, ex in cols if nm != "Intercept"]
77
- out.append(_Spec(fml=s.formula, depvar=dep, y=_outcome_expr(dep, schema), x=cols, fe=fe))
76
+ out.append(_Spec(fml=s.key, depvar=dep, y=_outcome_expr(dep, schema), x=cols, fe=fe))
78
77
  return out
@@ -37,6 +37,31 @@ from scipy.special import expit, ndtr, ndtri
37
37
  # average, and the parameters few enough for a per-thread accumulator each
38
38
  SPARSE_DENSITY = 0.5
39
39
  SPARSE_MAX_PARAMS = 1200
40
+ # The conditional and fixed-effects logits' numba passes (`_clogit_pass`,
41
+ # `_cfe_units`) split a block's choice sets or units into this many parts per
42
+ # thread, handed out as threads come free (`_parts`), and keep the parts'
43
+ # information accumulators together to at most PARTS_MB (but at least one
44
+ # part per thread). The conditional logit's is used up to CLOGIT_KERNEL_MAX_P
45
+ # parameters: it adds each
46
+ # row into a P x P accumulator per thread, which is fast while that fits in a
47
+ # core's L2 cache (0.5 MB at 256; current cores have 1-2.5 MB) and several
48
+ # times slower once it does not, where BLAS's blocked products are not
49
+ # (measured, 4 threads: 0.17 of the NumPy pass's time at P = 128, 0.43 at
50
+ # 256, 0.7-0.85 at 384-512, 2.3 at 768 with a 2.5 MB L2). Above it the pass
51
+ # is NumPy's (`_Conditional._accumulate_blas`).
52
+ PARTS_PER_THREAD = 4
53
+ PARTS_MB = 64.0
54
+ # the fixed-effects logit's parts take runs of this many consecutive units
55
+ CFE_RUN = 64
56
+ CLOGIT_KERNEL_MAX_P = 256
57
+
58
+
59
+ def _parts(n_items: int, P: int) -> int:
60
+ """How many parts to split `n_items` sets or units into, for a pass
61
+ whose parts each keep a P x P accumulator (see PARTS_PER_THREAD)."""
62
+ threads = nb.get_num_threads()
63
+ cap = max(threads, int(PARTS_MB * 1e6 // (8 * max(P * P, 1))))
64
+ return max(1, min(n_items, PARTS_PER_THREAD * threads, cap))
40
65
 
41
66
 
42
67
  def _gram(X, v):
@@ -91,6 +116,93 @@ def _mnl_info_sparse(X, P, w, n_threads, out):
91
116
  out += acc[t]
92
117
 
93
118
 
119
+ @nb.njit(parallel=True, cache=True)
120
+ def _clogit_pass(X, cols, theta, y, w, off, starts, hessian, max_size, ll_t, g_t, H_t):
121
+ """The conditional logit's log-likelihood, gradient and information over
122
+ the choice sets of a block, in one pass over its rows: the sets are
123
+ split into ``len(ll_t)`` parts, and part t's sums are written to
124
+ ``ll_t[t]``, ``g_t[t]`` and (with `hessian`) ``H_t[t]``, for the caller
125
+ to add up (`_Conditional.accumulate_full`). `cols` are the columns of
126
+ `X` in the model; `w` and `off` are empty when there are no weights or
127
+ offset.
128
+
129
+ The only parallel loop is the one over the parts: the accumulators are
130
+ allocated and summed by the caller, since with `parallel_chunksize`
131
+ numba would hand out every element of an array operation here as a
132
+ chunk of its own.
133
+
134
+ Per set: p = softmax(eta), the log-likelihood eta_chosen - log sum
135
+ exp(eta), the gradient sum (y - p) x, and the information
136
+ sum p (x - xbar)(x - xbar)' with xbar = sum p x (the within-set
137
+ covariance, centred before the products: the same matrix as
138
+ sum p x x' - xbar xbar', without the cancellation). Each is weighted by
139
+ the set's weight. Only the upper triangle of the information is filled."""
140
+ n_sets = len(starts) - 1
141
+ P = len(cols)
142
+ n_parts = len(ll_t)
143
+ has_w, has_off = len(w) > 0, len(off) > 0
144
+ step = (n_sets + n_parts - 1) // n_parts
145
+ for t in nb.prange(n_parts):
146
+ # local accumulators, copied out at the end: rows of the shared
147
+ # arrays would share cache lines between threads
148
+ eta = np.empty(max_size)
149
+ xs = np.empty((max_size, P)) # the set's rows, read once
150
+ xbar = np.empty(P)
151
+ d = np.empty(P)
152
+ g = np.zeros(P)
153
+ Ht = np.zeros((P, P) if hessian else (0, 0))
154
+ llt = 0.0
155
+ for s in range(t * step, min(n_sets, (t + 1) * step)):
156
+ a, b = starts[s], starts[s + 1]
157
+ ws = w[a] if has_w else 1.0
158
+ m = -np.inf
159
+ chosen = 0.0
160
+ for i in range(a, b):
161
+ e = off[i] if has_off else 0.0
162
+ for k in range(P):
163
+ x = X[i, cols[k]]
164
+ xs[i - a, k] = x
165
+ e += x * theta[k]
166
+ eta[i - a] = e
167
+ if y[i] == 1:
168
+ chosen += e
169
+ if e > m:
170
+ m = e
171
+ S = 0.0
172
+ for i in range(a, b):
173
+ eta[i - a] = np.exp(eta[i - a] - m)
174
+ S += eta[i - a]
175
+ llt += ws * (chosen - m - np.log(S))
176
+ for k in range(P):
177
+ xbar[k] = 0.0
178
+ for i in range(a, b):
179
+ p = eta[i - a] / S
180
+ eta[i - a] = p
181
+ r = ws * ((1.0 if y[i] == 1 else 0.0) - p)
182
+ for k in range(P):
183
+ x = xs[i - a, k]
184
+ g[k] += r * x
185
+ xbar[k] += p * x
186
+ if hessian:
187
+ for i in range(a, b):
188
+ p = ws * eta[i - a]
189
+ for k in range(P):
190
+ d[k] = xs[i - a, k] - xbar[k]
191
+ # the full square (a fixed-length inner loop, which
192
+ # vectorizes), though only the upper triangle is used
193
+ for k in range(P):
194
+ v = p * d[k]
195
+ for l in range(P):
196
+ Ht[k, l] += v * d[l]
197
+ ll_t[t] = llt
198
+ for k in range(P):
199
+ g_t[t, k] = g[k]
200
+ if hessian:
201
+ for k in range(P):
202
+ for l in range(P):
203
+ H_t[t, k, l] = Ht[k, l]
204
+
205
+
94
206
  def _bincount(idx, weights, n):
95
207
  return np.bincount(idx, weights=weights, minlength=n)[:n]
96
208
 
@@ -530,6 +642,43 @@ class _Conditional(_Model):
530
642
  return e / np.repeat(S, sizes), m, S
531
643
 
532
644
  def accumulate(self, theta, X, y, w, starts, hessian, offset=None):
645
+ return self.accumulate_full(theta, X, np.arange(X.shape[1]), y, w, starts, hessian,
646
+ offset)
647
+
648
+ def accumulate_full(self, theta, X, cols, y, w, starts, hessian, offset=None):
649
+ """`accumulate` on the columns `cols` of `X`: a whole block, by
650
+ numba's pass, which holds no row-sized temporaries and reads the
651
+ columns in place rather than copying them; above
652
+ CLOGIT_KERNEL_MAX_P parameters, by NumPy's."""
653
+ P = len(cols)
654
+ if P > CLOGIT_KERNEL_MAX_P:
655
+ Xc = X if P == X.shape[1] and np.array_equal(cols, np.arange(P)) else X[:, cols]
656
+ return self._accumulate_blas(theta, Xc, y, w, starts, hessian, offset)
657
+ empty = np.empty(0)
658
+ starts = np.asarray(starts, np.int64)
659
+ # parts handed out one at a time as threads come free, so that a
660
+ # thread the machine slows (another process, the store's reader, a
661
+ # slower core) does not hold up the rest; their accumulators are
662
+ # capped together, at one part per thread
663
+ n_parts = _parts(len(starts) - 1, P)
664
+ ll_t, g_t = np.zeros(n_parts), np.zeros((n_parts, P))
665
+ H_t = np.zeros((n_parts, P, P) if hessian else (n_parts, 0, 0))
666
+ with nb.parallel_chunksize(1):
667
+ _clogit_pass(
668
+ np.ascontiguousarray(X, np.float64), np.asarray(cols, np.int64),
669
+ np.ascontiguousarray(theta, np.float64), y,
670
+ empty if w is None else np.ascontiguousarray(w, np.float64),
671
+ empty if offset is None else np.ascontiguousarray(offset, np.float64),
672
+ starts, bool(hessian), int(np.diff(starts).max()) if len(starts) > 1 else 1,
673
+ ll_t, g_t, H_t)
674
+ ll, g, H = ll_t.sum(), g_t.sum(axis=0), H_t.sum(axis=0)
675
+ if not hessian:
676
+ return float(ll), g, None
677
+ return float(ll), g, np.triu(H) + np.triu(H, 1).T
678
+
679
+ def _accumulate_blas(self, theta, X, y, w, starts, hessian, offset=None):
680
+ """`accumulate` by NumPy array operations, the information by BLAS's
681
+ Gram products: for a wide design (`CLOGIT_KERNEL_MAX_P`)."""
533
682
  st = starts[:-1]
534
683
  eta = X @ theta
535
684
  if offset is not None:
@@ -560,7 +709,7 @@ class _Conditional(_Model):
560
709
 
561
710
  # ------------------------------------------------------------------ conditional FE logit
562
711
  @nb.njit(parallel=True, cache=True)
563
- def _cfe_units(X, eta, y, w, starts, hessian, n_threads, ll_u, g_u, out):
712
+ def _cfe_units(X, eta, y, w, starts, hessian, ll_u, g_u, acc):
564
713
  """Chamberlain's conditional likelihood, unit by unit (the rows of unit u
565
714
  are starts[u]:starts[u+1]): with k the unit's number of 1s among its T
566
715
  rows, log P(y | k) = sum y eta - log e_k(exp(eta)), e_k the elementary
@@ -573,68 +722,81 @@ def _cfe_units(X, eta, y, w, starts, hessian, n_threads, ll_u, g_u, out):
573
722
  unit's largest, and a unit with k > T / 2 is fitted as its complement
574
723
  (1 - y, -x, -eta), so that k <= T / 2. Units with k = 0 or k = T carry no
575
724
  information and are skipped. Writes each unit's (weighted) log-likelihood
576
- and gradient; the information is summed into `out` over `n_threads`
577
- blocks of units."""
725
+ and gradient; the information is summed over ``len(acc)`` parts of the
726
+ units, part t's into ``acc[t]``, for the caller to add up.
727
+
728
+ A unit's work grows with T k P^2, so units differ in cost: the parts are
729
+ more than the threads, handed out as threads come free (`_CFELogit._units`
730
+ runs this under ``parallel_chunksize(1)``), and each takes every
731
+ ``len(acc)``-th run of CFE_RUN units, so that each has a mix of short and
732
+ long panels. The only parallel loop is the
733
+ one over the parts, so the accumulators are allocated and summed by the
734
+ caller (numba would otherwise hand out every element of an array
735
+ operation here as a chunk of its own)."""
578
736
  U, P = len(starts) - 1, X.shape[1]
579
- acc = np.zeros((n_threads, P, P))
580
- step = (U + n_threads - 1) // n_threads
581
- for th in nb.prange(n_threads):
582
- # (a thread's own accumulator, copied out at the end: the threads'
737
+ n_parts = len(acc)
738
+ for th in nb.prange(n_parts):
739
+ # (a part's own accumulator, copied out at the end: the parts'
583
740
  # slices of `acc` are too close together to add into unit by unit)
584
741
  a_ = np.zeros((P, P))
585
- for u in range(th * step, min(U, (th + 1) * step)):
586
- r0, r1 = starts[u], starts[u + 1]
587
- T = r1 - r0
588
- k = 0
589
- for r in range(r0, r1):
590
- k += 1 if y[r] > 0 else 0
591
- ll_u[u] = 0.0
592
- for p in range(P):
593
- g_u[u, p] = 0.0
594
- if k == 0 or k == T:
595
- continue
596
- s = 1.0
597
- if 2 * k > T:
598
- s, k = -1.0, T - k
599
- c = -np.inf
600
- for r in range(r0, r1):
601
- c = max(c, s * eta[r])
602
- e = np.zeros(k + 1)
603
- G = np.zeros((k + 1, P))
604
- Hm = np.zeros((k + 1, P, P)) if hessian else np.zeros((1, 1, 1))
605
- e[0] = 1.0
606
- obs = 0.0
607
- for r in range(r0, r1):
608
- z = np.exp(s * eta[r] - c)
609
- yr = (y[r] > 0) if s > 0 else (y[r] <= 0)
610
- if yr:
611
- obs += s * eta[r]
612
- for p in range(P):
613
- g_u[u, p] += s * X[r, p]
614
- for j in range(k, 0, -1):
615
- if hessian:
742
+ # runs of CFE_RUN units, every n_parts-th run: data in order of panel
743
+ # length (a cohort, an entry date) would otherwise put the longest,
744
+ # costliest units together in the last parts, while single units
745
+ # dealt out would scatter the reads
746
+ for u0 in range(th * CFE_RUN, U, n_parts * CFE_RUN):
747
+ for u in range(u0, min(U, u0 + CFE_RUN)):
748
+ r0, r1 = starts[u], starts[u + 1]
749
+ T = r1 - r0
750
+ k = 0
751
+ for r in range(r0, r1):
752
+ k += 1 if y[r] > 0 else 0
753
+ ll_u[u] = 0.0
754
+ for p in range(P):
755
+ g_u[u, p] = 0.0
756
+ if k == 0 or k == T:
757
+ continue
758
+ s = 1.0
759
+ if 2 * k > T:
760
+ s, k = -1.0, T - k
761
+ c = -np.inf
762
+ for r in range(r0, r1):
763
+ c = max(c, s * eta[r])
764
+ e = np.zeros(k + 1)
765
+ G = np.zeros((k + 1, P))
766
+ Hm = np.zeros((k + 1, P, P)) if hessian else np.zeros((1, 1, 1))
767
+ e[0] = 1.0
768
+ obs = 0.0
769
+ for r in range(r0, r1):
770
+ z = np.exp(s * eta[r] - c)
771
+ yr = (y[r] > 0) if s > 0 else (y[r] <= 0)
772
+ if yr:
773
+ obs += s * eta[r]
616
774
  for p in range(P):
617
- xp = s * X[r, p]
618
- for q in range(P):
619
- xq = s * X[r, q]
620
- Hm[j, p, q] += z * (Hm[j - 1, p, q] + xp * G[j - 1, q]
621
- + G[j - 1, p] * xq + xp * xq * e[j - 1])
622
- for p in range(P):
623
- G[j, p] += z * (G[j - 1, p] + s * X[r, p] * e[j - 1])
624
- e[j] += z * e[j - 1]
625
- wu = w[r0]
626
- ll_u[u] = wu * (obs - np.log(e[k]) - k * c)
627
- for p in range(P):
628
- m = G[k, p] / e[k]
629
- g_u[u, p] = wu * (g_u[u, p] - m)
630
- if hessian:
775
+ g_u[u, p] += s * X[r, p]
776
+ for j in range(k, 0, -1):
777
+ if hessian:
778
+ for p in range(P):
779
+ xp = s * X[r, p]
780
+ for q in range(P):
781
+ xq = s * X[r, q]
782
+ Hm[j, p, q] += z * (Hm[j - 1, p, q] + xp * G[j - 1, q]
783
+ + G[j - 1, p] * xq + xp * xq * e[j - 1])
784
+ for p in range(P):
785
+ G[j, p] += z * (G[j - 1, p] + s * X[r, p] * e[j - 1])
786
+ e[j] += z * e[j - 1]
787
+ wu = w[r0]
788
+ ll_u[u] = wu * (obs - np.log(e[k]) - k * c)
631
789
  for p in range(P):
632
- mp = G[k, p] / e[k]
633
- for q in range(P):
634
- a_[p, q] += wu * (Hm[k, p, q] / e[k] - mp * G[k, q] / e[k])
635
- acc[th] = a_
636
- for th in range(n_threads):
637
- out += acc[th]
790
+ m = G[k, p] / e[k]
791
+ g_u[u, p] = wu * (g_u[u, p] - m)
792
+ if hessian:
793
+ for p in range(P):
794
+ mp = G[k, p] / e[k]
795
+ for q in range(P):
796
+ a_[p, q] += wu * (Hm[k, p, q] / e[k] - mp * G[k, q] / e[k])
797
+ for p in range(P):
798
+ for q in range(P):
799
+ acc[th, p, q] = a_[p, q]
638
800
 
639
801
 
640
802
  class _CFELogit(_Model):
@@ -673,12 +835,12 @@ class _CFELogit(_Model):
673
835
  eta = eta + offset
674
836
  U, P = len(starts) - 1, X.shape[1]
675
837
  ll_u, g_u = np.empty(U), np.empty((U, P))
676
- H = np.zeros((P, P))
838
+ acc = np.zeros((_parts(U, P), P, P))
677
839
  ww = np.ones(len(y)) if w is None else np.ascontiguousarray(w, dtype=np.float64)
678
- _cfe_units(np.ascontiguousarray(X), np.ascontiguousarray(eta), y, ww,
679
- np.ascontiguousarray(starts, dtype=np.int64), hessian,
680
- nb.get_num_threads(), ll_u, g_u, H)
681
- return ll_u, g_u, H
840
+ with nb.parallel_chunksize(1):
841
+ _cfe_units(np.ascontiguousarray(X), np.ascontiguousarray(eta), y, ww,
842
+ np.ascontiguousarray(starts, dtype=np.int64), hessian, ll_u, g_u, acc)
843
+ return ll_u, g_u, acc.sum(axis=0)
682
844
 
683
845
  def accumulate(self, theta, X, y, w, starts, hessian, offset=None):
684
846
  ll_u, g_u, H = self._units(theta, X, y, w, starts, hessian, offset)
@@ -129,7 +129,19 @@ class _Driver:
129
129
  m = self.model
130
130
  ll, g, H = 0.0, np.zeros(m.P), (np.zeros((m.P, m.P)) if hessian else None)
131
131
  rows = self._rows()
132
+ full = getattr(m, "accumulate_full", None)
132
133
  for blk in self.store:
134
+ if full is not None and firth_A is None:
135
+ # a numba pass over the whole block, on its columns in place
136
+ l, gg, hh = full(theta, blk["X"], m.cols, blk["y"], blk.get("w"),
137
+ blk.get("starts"), hessian, offset=blk.get("off"))
138
+ if not np.isfinite(l):
139
+ return -np.inf, None, None
140
+ ll += l
141
+ g += gg
142
+ if hessian:
143
+ H += hh
144
+ continue
133
145
  for ch in _chunks(blk, rows):
134
146
  X = _design(ch, m.cols, self.K0)
135
147
  l, gg, hh = m.accumulate(theta, X, ch["y"], ch.get("w"), ch.get("starts"),