logit-stream 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. {logit_stream-0.2.0 → logit_stream-0.3.0}/CHANGELOG.md +19 -0
  2. {logit_stream-0.2.0 → logit_stream-0.3.0}/CITATION.cff +2 -2
  3. {logit_stream-0.2.0 → logit_stream-0.3.0}/PKG-INFO +3 -2
  4. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/__init__.py +1 -1
  5. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/models.py +225 -63
  6. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/newton.py +12 -0
  7. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/store.py +150 -58
  8. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream.egg-info/PKG-INFO +3 -2
  9. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream.egg-info/SOURCES.txt +1 -0
  10. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream.egg-info/requires.txt +2 -1
  11. {logit_stream-0.2.0 → logit_stream-0.3.0}/pyproject.toml +5 -2
  12. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_cfelogit.py +18 -0
  13. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_conditional.py +22 -0
  14. logit_stream-0.3.0/tests/test_resources.py +135 -0
  15. {logit_stream-0.2.0 → logit_stream-0.3.0}/LICENSE.md +0 -0
  16. {logit_stream-0.2.0 → logit_stream-0.3.0}/MANIFEST.in +0 -0
  17. {logit_stream-0.2.0 → logit_stream-0.3.0}/README.md +0 -0
  18. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/_choice.py +0 -0
  19. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/_frontend.py +0 -0
  20. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/_limits.py +0 -0
  21. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/api.py +0 -0
  22. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/__init__.py +0 -0
  23. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/_common.py +0 -0
  24. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/fe.py +0 -0
  25. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/meat.py +0 -0
  26. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/mixed.py +0 -0
  27. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/models.py +0 -0
  28. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/nested.py +0 -0
  29. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/random_effects.py +0 -0
  30. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/store.py +0 -0
  31. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/estimator.py +0 -0
  32. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/fe.py +0 -0
  33. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/mixed.py +0 -0
  34. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/multi_fe.py +0 -0
  35. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/nested.py +0 -0
  36. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/optimize.py +0 -0
  37. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/profile.py +0 -0
  38. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/py.typed +0 -0
  39. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/random_effects/__init__.py +0 -0
  40. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/random_effects/fit.py +0 -0
  41. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/random_effects/linalg.py +0 -0
  42. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/random_effects/model.py +0 -0
  43. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/random_effects/sigma.py +0 -0
  44. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/results.py +0 -0
  45. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/simulate.py +0 -0
  46. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream.egg-info/dependency_links.txt +0 -0
  47. {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream.egg-info/top_level.txt +0 -0
  48. {logit_stream-0.2.0 → logit_stream-0.3.0}/setup.cfg +0 -0
  49. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_api.py +0 -0
  50. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_clustering.py +0 -0
  51. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_device.py +0 -0
  52. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_firth_and_fit.py +0 -0
  53. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_fixed_effects.py +0 -0
  54. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_headroom.py +0 -0
  55. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_index_dtype.py +0 -0
  56. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_kernels.py +0 -0
  57. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_mixed.py +0 -0
  58. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_multi_fe.py +0 -0
  59. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_multinomial.py +0 -0
  60. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_nested.py +0 -0
  61. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_offset_start.py +0 -0
  62. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_ordered.py +0 -0
  63. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_probit.py +0 -0
  64. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_profile.py +0 -0
  65. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_random.py +0 -0
  66. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_simulate.py +0 -0
  67. {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_weights_scale.py +0 -0
@@ -1,5 +1,24 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.3.0
4
+
5
+ - **The conditional logit is 2.3-2.8 times faster.** Its log-likelihood, gradient and information
6
+ are computed in one parallel numba pass over each block's choice sets, giving the same
7
+ estimates (4 million sets: 13 s, now 5 s; 8 million: 26 s, now 9-12 s). Designs of more than
8
+ 256 parameters keep the previous BLAS-based pass, which is faster there.
9
+ - **The fixed-effects logit (`cfelogit_stream`) balances work across threads better.** Panels
10
+ are dealt out in small pieces to more parts than threads, so a run of long panels no longer
11
+ leaves threads idle: 0.55 of the time on panels sorted by length, 0.8-1.0 on shuffled
12
+ ones, with the same results.
13
+ - **Lower memory for conditional, mixed and fixed-effects logit.** Choice sets are now checked
14
+ and counted a block at a time instead of from a table of every set, so memory no longer grows
15
+ with the number of sets (8 million sets: peak 1.71 GB down to 1.33 GB). Same errors, same
16
+ results.
17
+ - **Memory tests.** `pytest -m resource` fits the multinomial, ordered and conditional logit
18
+ on data larger than a fixed memory limit; CI runs them on Linux, macOS and Windows. The
19
+ default `pytest` run leaves them out.
20
+ - Requires hdfe-stream 0.9.0 or later, which reads Parquet in less memory.
21
+
3
22
  ## 0.2.0
4
23
 
5
24
  - **Fit resources (CPUs and memory) are constrained to your job's limits automatically.** On a
@@ -7,8 +7,8 @@ authors:
7
7
  - family-names: Tucker
8
8
  given-names: Lee
9
9
  email: lee.c.tucker@census.gov
10
- version: 0.2.0
11
- date-released: 2026-10-31
10
+ version: 0.3.0
11
+ date-released: 2026-11-05
12
12
  license: MIT
13
13
  repository-code: "https://github.com/leetucker/logit-stream"
14
14
  url: "https://github.com/leetucker/logit-stream"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: logit-stream
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects
5
5
  Author: Lee Tucker
6
6
  License-Expression: MIT
@@ -25,7 +25,7 @@ Classifier: Typing :: Typed
25
25
  Requires-Python: >=3.10
26
26
  Description-Content-Type: text/markdown
27
27
  License-File: LICENSE.md
28
- Requires-Dist: hdfe-stream[formula]>=0.8.0
28
+ Requires-Dist: hdfe-stream[formula]>=0.9.0
29
29
  Requires-Dist: stream-limits>=0.1.1
30
30
  Requires-Dist: polars!=1.41.*,>=1.37
31
31
  Requires-Dist: numpy>=1.24
@@ -49,6 +49,7 @@ Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
49
49
  Requires-Dist: sphinx-design>=0.6; extra == "docs"
50
50
  Provides-Extra: test
51
51
  Requires-Dist: pytest>=7.0; extra == "test"
52
+ Requires-Dist: stream-limits[testing]>=0.2.1; extra == "test"
52
53
  Requires-Dist: scikit-learn>=1.5; extra == "test"
53
54
  Requires-Dist: statsmodels>=0.14; extra == "test"
54
55
  Requires-Dist: pandas>=2.0; extra == "test"
@@ -62,7 +62,7 @@ def _version():
62
62
  from importlib.metadata import version
63
63
  return version("logit-stream")
64
64
  except Exception: # noqa: BLE001 -- not installed
65
- return "0.2.0"
65
+ return "0.3.0"
66
66
 
67
67
 
68
68
  __version__ = _version()
@@ -37,6 +37,31 @@ from scipy.special import expit, ndtr, ndtri
37
37
  # average, and the parameters few enough for a per-thread accumulator each
38
38
  SPARSE_DENSITY = 0.5
39
39
  SPARSE_MAX_PARAMS = 1200
40
+ # The conditional and fixed-effects logits' numba passes (`_clogit_pass`,
41
+ # `_cfe_units`) split a block's choice sets or units into this many parts per
42
+ # thread, handed out as threads come free (`_parts`), and keep the parts'
43
+ # information accumulators together to at most PARTS_MB (but at least one
44
+ # part per thread). The conditional logit's is used up to CLOGIT_KERNEL_MAX_P
45
+ # parameters: it adds each
46
+ # row into a P x P accumulator per thread, which is fast while that fits in a
47
+ # core's L2 cache (0.5 MB at 256; current cores have 1-2.5 MB) and several
48
+ # times slower once it does not, where BLAS's blocked products are not
49
+ # (measured, 4 threads: 0.17 of the NumPy pass's time at P = 128, 0.43 at
50
+ # 256, 0.7-0.85 at 384-512, 2.3 at 768 with a 2.5 MB L2). Above it the pass
51
+ # is NumPy's (`_Conditional._accumulate_blas`).
52
+ PARTS_PER_THREAD = 4
53
+ PARTS_MB = 64.0
54
+ # the fixed-effects logit's parts take runs of this many consecutive units
55
+ CFE_RUN = 64
56
+ CLOGIT_KERNEL_MAX_P = 256
57
+
58
+
59
+ def _parts(n_items: int, P: int) -> int:
60
+ """How many parts to split `n_items` sets or units into, for a pass
61
+ whose parts each keep a P x P accumulator (see PARTS_PER_THREAD)."""
62
+ threads = nb.get_num_threads()
63
+ cap = max(threads, int(PARTS_MB * 1e6 // (8 * max(P * P, 1))))
64
+ return max(1, min(n_items, PARTS_PER_THREAD * threads, cap))
40
65
 
41
66
 
42
67
  def _gram(X, v):
@@ -91,6 +116,93 @@ def _mnl_info_sparse(X, P, w, n_threads, out):
91
116
  out += acc[t]
92
117
 
93
118
 
119
+ @nb.njit(parallel=True, cache=True)
120
+ def _clogit_pass(X, cols, theta, y, w, off, starts, hessian, max_size, ll_t, g_t, H_t):
121
+ """The conditional logit's log-likelihood, gradient and information over
122
+ the choice sets of a block, in one pass over its rows: the sets are
123
+ split into ``len(ll_t)`` parts, and part t's sums are written to
124
+ ``ll_t[t]``, ``g_t[t]`` and (with `hessian`) ``H_t[t]``, for the caller
125
+ to add up (`_Conditional.accumulate_full`). `cols` are the columns of
126
+ `X` in the model; `w` and `off` are empty when there are no weights or
127
+ offset.
128
+
129
+ The only parallel loop is the one over the parts: the accumulators are
130
+ allocated and summed by the caller, since with `parallel_chunksize`
131
+ numba would hand out every element of an array operation here as a
132
+ chunk of its own.
133
+
134
+ Per set: p = softmax(eta), the log-likelihood eta_chosen - log sum
135
+ exp(eta), the gradient sum (y - p) x, and the information
136
+ sum p (x - xbar)(x - xbar)' with xbar = sum p x (the within-set
137
+ covariance, centred before the products: the same matrix as
138
+ sum p x x' - xbar xbar', without the cancellation). Each is weighted by
139
+ the set's weight. Only the upper triangle of the information is filled."""
140
+ n_sets = len(starts) - 1
141
+ P = len(cols)
142
+ n_parts = len(ll_t)
143
+ has_w, has_off = len(w) > 0, len(off) > 0
144
+ step = (n_sets + n_parts - 1) // n_parts
145
+ for t in nb.prange(n_parts):
146
+ # local accumulators, copied out at the end: rows of the shared
147
+ # arrays would share cache lines between threads
148
+ eta = np.empty(max_size)
149
+ xs = np.empty((max_size, P)) # the set's rows, read once
150
+ xbar = np.empty(P)
151
+ d = np.empty(P)
152
+ g = np.zeros(P)
153
+ Ht = np.zeros((P, P) if hessian else (0, 0))
154
+ llt = 0.0
155
+ for s in range(t * step, min(n_sets, (t + 1) * step)):
156
+ a, b = starts[s], starts[s + 1]
157
+ ws = w[a] if has_w else 1.0
158
+ m = -np.inf
159
+ chosen = 0.0
160
+ for i in range(a, b):
161
+ e = off[i] if has_off else 0.0
162
+ for k in range(P):
163
+ x = X[i, cols[k]]
164
+ xs[i - a, k] = x
165
+ e += x * theta[k]
166
+ eta[i - a] = e
167
+ if y[i] == 1:
168
+ chosen += e
169
+ if e > m:
170
+ m = e
171
+ S = 0.0
172
+ for i in range(a, b):
173
+ eta[i - a] = np.exp(eta[i - a] - m)
174
+ S += eta[i - a]
175
+ llt += ws * (chosen - m - np.log(S))
176
+ for k in range(P):
177
+ xbar[k] = 0.0
178
+ for i in range(a, b):
179
+ p = eta[i - a] / S
180
+ eta[i - a] = p
181
+ r = ws * ((1.0 if y[i] == 1 else 0.0) - p)
182
+ for k in range(P):
183
+ x = xs[i - a, k]
184
+ g[k] += r * x
185
+ xbar[k] += p * x
186
+ if hessian:
187
+ for i in range(a, b):
188
+ p = ws * eta[i - a]
189
+ for k in range(P):
190
+ d[k] = xs[i - a, k] - xbar[k]
191
+ # the full square (a fixed-length inner loop, which
192
+ # vectorizes), though only the upper triangle is used
193
+ for k in range(P):
194
+ v = p * d[k]
195
+ for l in range(P):
196
+ Ht[k, l] += v * d[l]
197
+ ll_t[t] = llt
198
+ for k in range(P):
199
+ g_t[t, k] = g[k]
200
+ if hessian:
201
+ for k in range(P):
202
+ for l in range(P):
203
+ H_t[t, k, l] = Ht[k, l]
204
+
205
+
94
206
  def _bincount(idx, weights, n):
95
207
  return np.bincount(idx, weights=weights, minlength=n)[:n]
96
208
 
@@ -530,6 +642,43 @@ class _Conditional(_Model):
530
642
  return e / np.repeat(S, sizes), m, S
531
643
 
532
644
  def accumulate(self, theta, X, y, w, starts, hessian, offset=None):
645
+ return self.accumulate_full(theta, X, np.arange(X.shape[1]), y, w, starts, hessian,
646
+ offset)
647
+
648
+ def accumulate_full(self, theta, X, cols, y, w, starts, hessian, offset=None):
649
+ """`accumulate` on the columns `cols` of `X`: a whole block, by
650
+ numba's pass, which holds no row-sized temporaries and reads the
651
+ columns in place rather than copying them; above
652
+ CLOGIT_KERNEL_MAX_P parameters, by NumPy's."""
653
+ P = len(cols)
654
+ if P > CLOGIT_KERNEL_MAX_P:
655
+ Xc = X if P == X.shape[1] and np.array_equal(cols, np.arange(P)) else X[:, cols]
656
+ return self._accumulate_blas(theta, Xc, y, w, starts, hessian, offset)
657
+ empty = np.empty(0)
658
+ starts = np.asarray(starts, np.int64)
659
+ # parts handed out one at a time as threads come free, so that a
660
+ # thread the machine slows (another process, the store's reader, a
661
+ # slower core) does not hold up the rest; their accumulators are
662
+ # capped together, at one part per thread
663
+ n_parts = _parts(len(starts) - 1, P)
664
+ ll_t, g_t = np.zeros(n_parts), np.zeros((n_parts, P))
665
+ H_t = np.zeros((n_parts, P, P) if hessian else (n_parts, 0, 0))
666
+ with nb.parallel_chunksize(1):
667
+ _clogit_pass(
668
+ np.ascontiguousarray(X, np.float64), np.asarray(cols, np.int64),
669
+ np.ascontiguousarray(theta, np.float64), y,
670
+ empty if w is None else np.ascontiguousarray(w, np.float64),
671
+ empty if offset is None else np.ascontiguousarray(offset, np.float64),
672
+ starts, bool(hessian), int(np.diff(starts).max()) if len(starts) > 1 else 1,
673
+ ll_t, g_t, H_t)
674
+ ll, g, H = ll_t.sum(), g_t.sum(axis=0), H_t.sum(axis=0)
675
+ if not hessian:
676
+ return float(ll), g, None
677
+ return float(ll), g, np.triu(H) + np.triu(H, 1).T
678
+
679
+ def _accumulate_blas(self, theta, X, y, w, starts, hessian, offset=None):
680
+ """`accumulate` by NumPy array operations, the information by BLAS's
681
+ Gram products: for a wide design (`CLOGIT_KERNEL_MAX_P`)."""
533
682
  st = starts[:-1]
534
683
  eta = X @ theta
535
684
  if offset is not None:
@@ -560,7 +709,7 @@ class _Conditional(_Model):
560
709
 
561
710
  # ------------------------------------------------------------------ conditional FE logit
562
711
  @nb.njit(parallel=True, cache=True)
563
- def _cfe_units(X, eta, y, w, starts, hessian, n_threads, ll_u, g_u, out):
712
+ def _cfe_units(X, eta, y, w, starts, hessian, ll_u, g_u, acc):
564
713
  """Chamberlain's conditional likelihood, unit by unit (the rows of unit u
565
714
  are starts[u]:starts[u+1]): with k the unit's number of 1s among its T
566
715
  rows, log P(y | k) = sum y eta - log e_k(exp(eta)), e_k the elementary
@@ -573,68 +722,81 @@ def _cfe_units(X, eta, y, w, starts, hessian, n_threads, ll_u, g_u, out):
573
722
  unit's largest, and a unit with k > T / 2 is fitted as its complement
574
723
  (1 - y, -x, -eta), so that k <= T / 2. Units with k = 0 or k = T carry no
575
724
  information and are skipped. Writes each unit's (weighted) log-likelihood
576
- and gradient; the information is summed into `out` over `n_threads`
577
- blocks of units."""
725
+ and gradient; the information is summed over ``len(acc)`` parts of the
726
+ units, part t's into ``acc[t]``, for the caller to add up.
727
+
728
+ A unit's work grows with T k P^2, so units differ in cost: the parts are
729
+ more than the threads, handed out as threads come free (`_CFELogit._units`
730
+ runs this under ``parallel_chunksize(1)``), and each takes every
731
+ ``len(acc)``-th run of CFE_RUN units, so that each has a mix of short and
732
+ long panels. The only parallel loop is the
733
+ one over the parts, so the accumulators are allocated and summed by the
734
+ caller (numba would otherwise hand out every element of an array
735
+ operation here as a chunk of its own)."""
578
736
  U, P = len(starts) - 1, X.shape[1]
579
- acc = np.zeros((n_threads, P, P))
580
- step = (U + n_threads - 1) // n_threads
581
- for th in nb.prange(n_threads):
582
- # (a thread's own accumulator, copied out at the end: the threads'
737
+ n_parts = len(acc)
738
+ for th in nb.prange(n_parts):
739
+ # (a part's own accumulator, copied out at the end: the parts'
583
740
  # slices of `acc` are too close together to add into unit by unit)
584
741
  a_ = np.zeros((P, P))
585
- for u in range(th * step, min(U, (th + 1) * step)):
586
- r0, r1 = starts[u], starts[u + 1]
587
- T = r1 - r0
588
- k = 0
589
- for r in range(r0, r1):
590
- k += 1 if y[r] > 0 else 0
591
- ll_u[u] = 0.0
592
- for p in range(P):
593
- g_u[u, p] = 0.0
594
- if k == 0 or k == T:
595
- continue
596
- s = 1.0
597
- if 2 * k > T:
598
- s, k = -1.0, T - k
599
- c = -np.inf
600
- for r in range(r0, r1):
601
- c = max(c, s * eta[r])
602
- e = np.zeros(k + 1)
603
- G = np.zeros((k + 1, P))
604
- Hm = np.zeros((k + 1, P, P)) if hessian else np.zeros((1, 1, 1))
605
- e[0] = 1.0
606
- obs = 0.0
607
- for r in range(r0, r1):
608
- z = np.exp(s * eta[r] - c)
609
- yr = (y[r] > 0) if s > 0 else (y[r] <= 0)
610
- if yr:
611
- obs += s * eta[r]
612
- for p in range(P):
613
- g_u[u, p] += s * X[r, p]
614
- for j in range(k, 0, -1):
615
- if hessian:
742
+ # runs of CFE_RUN units, every n_parts-th run: data in order of panel
743
+ # length (a cohort, an entry date) would otherwise put the longest,
744
+ # costliest units together in the last parts, while single units
745
+ # dealt out would scatter the reads
746
+ for u0 in range(th * CFE_RUN, U, n_parts * CFE_RUN):
747
+ for u in range(u0, min(U, u0 + CFE_RUN)):
748
+ r0, r1 = starts[u], starts[u + 1]
749
+ T = r1 - r0
750
+ k = 0
751
+ for r in range(r0, r1):
752
+ k += 1 if y[r] > 0 else 0
753
+ ll_u[u] = 0.0
754
+ for p in range(P):
755
+ g_u[u, p] = 0.0
756
+ if k == 0 or k == T:
757
+ continue
758
+ s = 1.0
759
+ if 2 * k > T:
760
+ s, k = -1.0, T - k
761
+ c = -np.inf
762
+ for r in range(r0, r1):
763
+ c = max(c, s * eta[r])
764
+ e = np.zeros(k + 1)
765
+ G = np.zeros((k + 1, P))
766
+ Hm = np.zeros((k + 1, P, P)) if hessian else np.zeros((1, 1, 1))
767
+ e[0] = 1.0
768
+ obs = 0.0
769
+ for r in range(r0, r1):
770
+ z = np.exp(s * eta[r] - c)
771
+ yr = (y[r] > 0) if s > 0 else (y[r] <= 0)
772
+ if yr:
773
+ obs += s * eta[r]
616
774
  for p in range(P):
617
- xp = s * X[r, p]
618
- for q in range(P):
619
- xq = s * X[r, q]
620
- Hm[j, p, q] += z * (Hm[j - 1, p, q] + xp * G[j - 1, q]
621
- + G[j - 1, p] * xq + xp * xq * e[j - 1])
622
- for p in range(P):
623
- G[j, p] += z * (G[j - 1, p] + s * X[r, p] * e[j - 1])
624
- e[j] += z * e[j - 1]
625
- wu = w[r0]
626
- ll_u[u] = wu * (obs - np.log(e[k]) - k * c)
627
- for p in range(P):
628
- m = G[k, p] / e[k]
629
- g_u[u, p] = wu * (g_u[u, p] - m)
630
- if hessian:
775
+ g_u[u, p] += s * X[r, p]
776
+ for j in range(k, 0, -1):
777
+ if hessian:
778
+ for p in range(P):
779
+ xp = s * X[r, p]
780
+ for q in range(P):
781
+ xq = s * X[r, q]
782
+ Hm[j, p, q] += z * (Hm[j - 1, p, q] + xp * G[j - 1, q]
783
+ + G[j - 1, p] * xq + xp * xq * e[j - 1])
784
+ for p in range(P):
785
+ G[j, p] += z * (G[j - 1, p] + s * X[r, p] * e[j - 1])
786
+ e[j] += z * e[j - 1]
787
+ wu = w[r0]
788
+ ll_u[u] = wu * (obs - np.log(e[k]) - k * c)
631
789
  for p in range(P):
632
- mp = G[k, p] / e[k]
633
- for q in range(P):
634
- a_[p, q] += wu * (Hm[k, p, q] / e[k] - mp * G[k, q] / e[k])
635
- acc[th] = a_
636
- for th in range(n_threads):
637
- out += acc[th]
790
+ m = G[k, p] / e[k]
791
+ g_u[u, p] = wu * (g_u[u, p] - m)
792
+ if hessian:
793
+ for p in range(P):
794
+ mp = G[k, p] / e[k]
795
+ for q in range(P):
796
+ a_[p, q] += wu * (Hm[k, p, q] / e[k] - mp * G[k, q] / e[k])
797
+ for p in range(P):
798
+ for q in range(P):
799
+ acc[th, p, q] = a_[p, q]
638
800
 
639
801
 
640
802
  class _CFELogit(_Model):
@@ -673,12 +835,12 @@ class _CFELogit(_Model):
673
835
  eta = eta + offset
674
836
  U, P = len(starts) - 1, X.shape[1]
675
837
  ll_u, g_u = np.empty(U), np.empty((U, P))
676
- H = np.zeros((P, P))
838
+ acc = np.zeros((_parts(U, P), P, P))
677
839
  ww = np.ones(len(y)) if w is None else np.ascontiguousarray(w, dtype=np.float64)
678
- _cfe_units(np.ascontiguousarray(X), np.ascontiguousarray(eta), y, ww,
679
- np.ascontiguousarray(starts, dtype=np.int64), hessian,
680
- nb.get_num_threads(), ll_u, g_u, H)
681
- return ll_u, g_u, H
840
+ with nb.parallel_chunksize(1):
841
+ _cfe_units(np.ascontiguousarray(X), np.ascontiguousarray(eta), y, ww,
842
+ np.ascontiguousarray(starts, dtype=np.int64), hessian, ll_u, g_u, acc)
843
+ return ll_u, g_u, acc.sum(axis=0)
682
844
 
683
845
  def accumulate(self, theta, X, y, w, starts, hessian, offset=None):
684
846
  ll_u, g_u, H = self._units(theta, X, y, w, starts, hessian, offset)
@@ -129,7 +129,19 @@ class _Driver:
129
129
  m = self.model
130
130
  ll, g, H = 0.0, np.zeros(m.P), (np.zeros((m.P, m.P)) if hessian else None)
131
131
  rows = self._rows()
132
+ full = getattr(m, "accumulate_full", None)
132
133
  for blk in self.store:
134
+ if full is not None and firth_A is None:
135
+ # a numba pass over the whole block, on its columns in place
136
+ l, gg, hh = full(theta, blk["X"], m.cols, blk["y"], blk.get("w"),
137
+ blk.get("starts"), hessian, offset=blk.get("off"))
138
+ if not np.isfinite(l):
139
+ return -np.inf, None, None
140
+ ll += l
141
+ g += gg
142
+ if hessian:
143
+ H += hh
144
+ continue
133
145
  for ch in _chunks(blk, rows):
134
146
  X = _design(ch, m.cols, self.K0)
135
147
  l, gg, hh = m.accumulate(theta, X, ch["y"], ch.get("w"), ch.get("starts"),
@@ -138,21 +138,39 @@ def _build(lf: pl.LazyFrame, spec, kind: str, run_dir: Path, engine: PolarsEngin
138
138
  cols = list(dict.fromkeys([*roots, *([weights] if weights else []), *raw]))
139
139
  rows = block_rows(block_mb, K)
140
140
  design_path = run_dir / "sample.parquet"
141
+ log("pass 0: writing the sample")
142
+ write_sample(engine, lf, usable, cols, design_path, rows, ROW)
141
143
  # A choice set with an incomplete alternative is dropped whole (Stata's
142
144
  # cmclogit default): fitted on its other alternatives, it would be a
143
- # different choice set. altwise=True keeps them.
145
+ # different choice set. altwise=True keeps them. Whether there is one is
146
+ # the source's rows with a choice set against the sample's (a count of
147
+ # the key columns); which sets, and a second copy of the sample without
148
+ # them, only when there is. (Filtering the source for the incomplete
149
+ # rows, as this did, holds memory in proportion to the source in Polars'
150
+ # streaming engine even when none match: 0.3 GB on 21.6 million rows.)
144
151
  incomplete = None
145
152
  if kind in ("clogit", "mixlogit") and not altwise:
146
153
  skey = list(dict.fromkeys([*(panel or []), *case]))
147
- bad = engine.collect(lf.filter(~usable).select(skey).drop_nulls().unique(),
148
- "incomplete choice sets")
149
- if bad.height:
150
- incomplete = (bad, skey)
151
- warn(f"{spec.fml}: {bad.height:,} choice sets dropped because some of their "
152
- "alternatives have missing values (altwise=True keeps their complete "
153
- "alternatives)")
154
- log("pass 0: writing the sample")
155
- write_sample(engine, lf, usable, cols, design_path, rows, ROW, drop=incomplete)
154
+ keyed = engine.collect(
155
+ lf.filter(pl.all_horizontal([pl.col(c).is_not_null() for c in skey]))
156
+ .select(pl.len()), "rows with a choice set").item()
157
+ kept = engine.collect(pl.scan_parquet(design_path).select(pl.len()),
158
+ "sample rows").item()
159
+ if keyed > kept:
160
+ bad = engine.collect(lf.filter(~usable).select(skey).drop_nulls().unique(),
161
+ "incomplete choice sets")
162
+ if bad.height:
163
+ incomplete = (bad, skey)
164
+ warn(f"{spec.fml}: {bad.height:,} choice sets dropped because some of their "
165
+ "alternatives have missing values (altwise=True keeps their complete "
166
+ "alternatives)")
167
+ every = run_dir / "sample_all.parquet"
168
+ design_path.rename(every)
169
+ engine.sink(pl.scan_parquet(every).join(bad.lazy(), on=skey, how="anti",
170
+ maintain_order="left"),
171
+ str(design_path), "pass 0 sample (complete sets)",
172
+ row_group_size=min(rows, 250_000))
173
+ every.unlink()
156
174
  evaluated = [pl.col(ROW), yexpr.alias(Y), *xexprs, *wexpr,
157
175
  *([offset.alias(OFF)] if offset is not None else []),
158
176
  *[pl.col(c) for c in raw if c not in (Y, W)]]
@@ -271,6 +289,8 @@ def _build(lf: pl.LazyFrame, spec, kind: str, run_dir: Path, engine: PolarsEngin
271
289
  "pidx": pidx,
272
290
  "row": frame[ROW].to_numpy().astype(np.int64)})
273
291
 
292
+ convert.store = store # for _case_order's weight rescaling
293
+
274
294
  n_cases, wcases, n_panels = 0, 0.0, 0
275
295
  units = {}
276
296
  log(f"pass 0: writing {n_obs:,} rows as blocks of {rows:,}"
@@ -280,7 +300,8 @@ def _build(lf: pl.LazyFrame, spec, kind: str, run_dir: Path, engine: PolarsEngin
280
300
  n_cases, wcases, n_panels = _case_order(engine, design, design_path, case, weights,
281
301
  spec, rows, convert, run_dir, log, panel,
282
302
  ccols, wscale if weights_type != "fweights"
283
- else None, kind=kind, info=units)
303
+ else None, kind=kind, info=units,
304
+ evaluated=evaluated)
284
305
  else:
285
306
  for frame in iter_batches(design_path, rows):
286
307
  convert(frame)
@@ -294,7 +315,8 @@ def _build(lf: pl.LazyFrame, spec, kind: str, run_dir: Path, engine: PolarsEngin
294
315
 
295
316
 
296
317
  def _case_order(engine, design, design_path, case, weights, spec, rows, convert, run_dir,
297
- log, panel=None, cluster_cols=(), wscale=None, kind="clogit", info=None):
318
+ log, panel=None, cluster_cols=(), wscale=None, kind="clogit", info=None,
319
+ evaluated=()):
298
320
  """Write the conditional logit's blocks so that each holds whole choice
299
321
  sets, each a run of consecutive rows; return the number of sets, their
300
322
  summed weight and the number of panels.
@@ -308,7 +330,16 @@ def _case_order(engine, design, design_path, case, weights, spec, rows, convert,
308
330
 
309
331
  Long-format choice data usually arrives grouped by choice set already,
310
332
  and then the rows are written in their order; otherwise they are sorted
311
- by set in buckets (`hdfe_stream.core.store.iter_groups`)."""
333
+ by set in buckets (`hdfe_stream.core.store.iter_groups`).
334
+
335
+ The sets are checked (one chosen alternative; weights and clusters
336
+ constant within each) and summed (their weights; the fixed-effects
337
+ logit's units) a block at a time, as the blocks are written: a summary
338
+ of every set at once would hold a row per set in memory. Analytic
339
+ weights, which are rescaled to average 1 over the sets, are therefore
340
+ written as they are and rescaled once the sum is known
341
+ (`_rescale_weights`). `evaluated` are the sample's expressions, as
342
+ `convert` evaluates them, for the outcome and the weights."""
312
343
  skey = list(dict.fromkeys([*(panel or []), *case]))
313
344
  unit = list(panel) if panel else skey
314
345
 
@@ -316,62 +347,123 @@ def _case_order(engine, design, design_path, case, weights, spec, rows, convert,
316
347
  return pl.any_horizontal([(pl.col(c) != pl.col(c).shift()).fill_null(True)
317
348
  for c in cols])
318
349
 
350
+ def key(cols):
351
+ # one column's distinct values are counted with little memory; a
352
+ # struct's with more, but still far less than a row per set
353
+ return pl.col(cols[0]) if len(cols) == 1 else pl.struct(cols)
354
+
319
355
  changed, uchanged = runs_of(skey), runs_of(unit)
320
- per = engine.collect(design.group_by(skey).agg(
321
- pl.col(Y).sum().alias("chosen"), pl.len().alias("size"),
322
- *([(pl.col(W).max() - pl.col(W).min()).alias("wspread"),
323
- pl.col(W).first().alias("wcase")] if weights else []),
324
- *[pl.col(c).n_unique().alias(f"__ls_nu_{i}") for i, c in enumerate(cluster_cols)]),
325
- "choice sets")
326
- n_cases = per.height
327
- wcases = float(per["wcase"].sum()) if weights else float(n_cases)
328
- if weights and wscale is not None: # analytic weights: average 1 over the sets
329
- wscale[0] = n_cases / wcases
330
- wcases = float(n_cases)
331
- n_panels = per.select(panel).unique().height if panel else 0
356
+ n_cases = None
357
+ if not panel and len(skey) == 1:
358
+ # sorted by set, as long-format choice data usually are: each set's
359
+ # rows are then consecutive, and the sets are the runs of the key,
360
+ # counted without a table of the keys (about 30 bytes a set)
361
+ k = pl.col(skey[0])
362
+ order = engine.collect(design.select(
363
+ runs_of(skey).sum().alias("runs"),
364
+ (k < k.shift()).fill_null(False).any().alias("down")), "choice set order")
365
+ if not order["down"].item():
366
+ n_cases = int(order["runs"].item())
367
+ if n_cases is None:
368
+ counts = engine.collect(design.select(
369
+ key(skey).n_unique().alias("sets"),
370
+ *([key(list(panel)).n_unique().alias("panels")] if panel else [])), "choice sets")
371
+ n_cases = int(counts["sets"].item())
372
+ n_panels = int(counts["panels"].item()) if panel else 0
373
+
374
+ need = {Y, *([W] if weights else [])}
375
+ ev = [e for e in evaluated if e.meta.output_name() in need]
376
+ run = "__ls_set"
377
+ agg = [pl.col(Y).sum().alias("chosen"), pl.len().alias("size"),
378
+ *([(pl.col(W).max() - pl.col(W).min()).alias("wspread"),
379
+ pl.col(W).first().alias("wcase")] if weights else []),
380
+ *[pl.col(c).n_unique().alias(f"__ls_nu_{i}") for i, c in enumerate(cluster_cols)]]
381
+ wsum_sets = 0.0
382
+ bad_sets, bad_eg = 0, []
383
+ wspread = False
384
+ spread = [[0, []] for _ in cluster_cols]
385
+ cfe = {"units": 0, "rows": 0, "wunits": 0.0, "loglik_null": 0.0}
332
386
  if kind == "cfelogit":
333
- # units whose outcome never varies carry no information (their rows stay,
334
- # and the likelihood skips them); the null, every arrangement of a unit's
335
- # 1s equally likely, is -sum log C(T, k)
336
387
  from scipy.special import gammaln
337
- T, k = per["size"].to_numpy().astype(float), per["chosen"].to_numpy()
338
- live = (k > 0) & (k < T)
339
- wu = per["wcase"].to_numpy() if weights else np.ones(len(T))
340
- if weights and wscale is not None: # average 1 over the informative units
341
- wscale[0] = live.sum() / wu[live].sum()
342
- wu = wu * (wscale[0] if weights and wscale is not None else 1.0)
343
- lc = gammaln(T + 1) - gammaln(k + 1) - gammaln(T - k + 1)
344
- info.update(units=int(live.sum()), dropped=int((~live).sum()),
345
- rows=int(T[live].sum()), wunits=float(wu[live].sum()),
346
- loglik_null=float(-(lc[live] * wu[live]).sum()))
347
- bad = per.filter(pl.col("chosen") != 1) if kind != "cfelogit" else per.head(0)
348
- if bad.height:
349
- raise ValueError(
350
- f"{spec.fml}: every choice set must have exactly one chosen alternative "
351
- f"(outcome 1); {bad.height:,} of {n_cases:,} do not (e.g. "
352
- f"{bad.select(skey).head(3).to_dicts()}). Rows dropped for missing values can "
353
- "remove a set's chosen alternative.")
354
- if weights and per["wspread"].max() > 0:
355
- raise ValueError("weights must be constant within each choice set (a choice set is "
356
- "one observation)")
357
- for i, c in enumerate(cluster_cols):
358
- spread = per.filter(pl.col(f"__ls_nu_{i}") > 1)
359
- if spread.height:
360
- raise ValueError(f"cluster variable {c!r} must be constant within each choice set "
361
- f"(a choice set is one observation); {spread.height:,} sets "
362
- f"have more than one value (e.g. "
363
- f"{spread.select(skey).head(3).to_dicts()})")
364
- del per
365
388
 
366
389
  # blocks of whole units (sets, or panels with their sets consecutive)
367
390
  for body in iter_groups(engine, design_path, unit, rows, run_dir,
368
391
  n_panels if panel else n_cases, [*skey, ROW],
369
392
  also=[(skey, n_cases)] if panel else (), log=log):
370
- starts = np.append(np.flatnonzero(body.select(changed.alias("b"))["b"].to_numpy()),
371
- body.height).astype(np.int64)
393
+ b = body.select(changed.alias("b"))["b"].to_numpy()
394
+ starts = np.append(np.flatnonzero(b), body.height).astype(np.int64)
395
+ # this block's sets, in their order
396
+ per = (body.select(*skey, *[pl.col(c) for c in cluster_cols if c not in skey], *ev)
397
+ .with_columns(pl.Series(run, np.cumsum(b)))
398
+ .group_by(run, maintain_order=True)
399
+ .agg(*[pl.col(c).first() for c in skey], *agg))
400
+ if weights:
401
+ wsum_sets += float(per["wcase"].sum())
402
+ wspread = wspread or per["wspread"].max() > 0
403
+ if kind == "cfelogit":
404
+ # units whose outcome never varies carry no information (their rows
405
+ # stay, and the likelihood skips them); the null, every arrangement
406
+ # of a unit's 1s equally likely, is -sum log C(T, k)
407
+ T, k = per["size"].to_numpy().astype(float), per["chosen"].to_numpy()
408
+ live = (k > 0) & (k < T)
409
+ wu = per["wcase"].to_numpy() if weights else np.ones(len(T))
410
+ lc = gammaln(T + 1) - gammaln(k + 1) - gammaln(T - k + 1)
411
+ cfe["units"] += int(live.sum())
412
+ cfe["rows"] += int(T[live].sum())
413
+ cfe["wunits"] += float(wu[live].sum())
414
+ cfe["loglik_null"] -= float((lc[live] * wu[live]).sum())
415
+ else:
416
+ bad = per.filter(pl.col("chosen") != 1)
417
+ bad_sets += bad.height
418
+ if len(bad_eg) < 3:
419
+ bad_eg += bad.select(skey).head(3 - len(bad_eg)).to_dicts()
420
+ for i in range(len(cluster_cols)):
421
+ sp = per.filter(pl.col(f"__ls_nu_{i}") > 1)
422
+ spread[i][0] += sp.height
423
+ if len(spread[i][1]) < 3:
424
+ spread[i][1] += sp.select(skey).head(3 - len(spread[i][1])).to_dicts()
425
+ del per
372
426
  pstarts = None
373
427
  if panel:
374
428
  pstarts = np.append(np.flatnonzero(body.select(uchanged.alias("b"))["b"]
375
429
  .to_numpy()), body.height).astype(np.int64)
376
430
  convert(body, starts, pstarts)
431
+
432
+ if bad_sets:
433
+ raise ValueError(
434
+ f"{spec.fml}: every choice set must have exactly one chosen alternative "
435
+ f"(outcome 1); {bad_sets:,} of {n_cases:,} do not (e.g. {bad_eg}). Rows dropped "
436
+ "for missing values can remove a set's chosen alternative.")
437
+ if wspread:
438
+ raise ValueError("weights must be constant within each choice set (a choice set is "
439
+ "one observation)")
440
+ for c, (n_spread, eg) in zip(cluster_cols, spread):
441
+ if n_spread:
442
+ raise ValueError(f"cluster variable {c!r} must be constant within each choice set "
443
+ f"(a choice set is one observation); {n_spread:,} sets "
444
+ f"have more than one value (e.g. {eg})")
445
+
446
+ wcases = wsum_sets if weights else float(n_cases)
447
+ if weights and wscale is not None: # analytic weights: average 1 over the sets
448
+ wscale[0] = n_cases / wcases
449
+ wcases = float(n_cases)
450
+ if kind == "cfelogit":
451
+ scale = 1.0
452
+ if weights and wscale is not None: # average 1 over the informative units
453
+ scale = wscale[0] = cfe["units"] / cfe["wunits"]
454
+ info.update(units=cfe["units"], dropped=n_cases - cfe["units"], rows=cfe["rows"],
455
+ wunits=cfe["wunits"] * scale, loglik_null=cfe["loglik_null"] * scale)
456
+ if weights and wscale is not None and wscale[0] != 1.0:
457
+ _rescale_weights(convert.store, wscale[0])
377
458
  return n_cases, wcases, n_panels
459
+
460
+
461
+ def _rescale_weights(store, factor: float) -> None:
462
+ """Multiply the weights (field ``w``) of every block of `store` by
463
+ `factor`, in memory or in their files."""
464
+ for blk in store.blocks:
465
+ if store.in_memory:
466
+ if "w" in blk.arrays:
467
+ blk.arrays["w"] = blk.arrays["w"] * factor
468
+ elif "w" in blk.files:
469
+ np.save(blk.files["w"], np.load(blk.files["w"]) * factor)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: logit-stream
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects
5
5
  Author: Lee Tucker
6
6
  License-Expression: MIT
@@ -25,7 +25,7 @@ Classifier: Typing :: Typed
25
25
  Requires-Python: >=3.10
26
26
  Description-Content-Type: text/markdown
27
27
  License-File: LICENSE.md
28
- Requires-Dist: hdfe-stream[formula]>=0.8.0
28
+ Requires-Dist: hdfe-stream[formula]>=0.9.0
29
29
  Requires-Dist: stream-limits>=0.1.1
30
30
  Requires-Dist: polars!=1.41.*,>=1.37
31
31
  Requires-Dist: numpy>=1.24
@@ -49,6 +49,7 @@ Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
49
49
  Requires-Dist: sphinx-design>=0.6; extra == "docs"
50
50
  Provides-Extra: test
51
51
  Requires-Dist: pytest>=7.0; extra == "test"
52
+ Requires-Dist: stream-limits[testing]>=0.2.1; extra == "test"
52
53
  Requires-Dist: scikit-learn>=1.5; extra == "test"
53
54
  Requires-Dist: statsmodels>=0.14; extra == "test"
54
55
  Requires-Dist: pandas>=2.0; extra == "test"
@@ -60,5 +60,6 @@ tests/test_ordered.py
60
60
  tests/test_probit.py
61
61
  tests/test_profile.py
62
62
  tests/test_random.py
63
+ tests/test_resources.py
63
64
  tests/test_simulate.py
64
65
  tests/test_weights_scale.py
@@ -1,4 +1,4 @@
1
- hdfe-stream[formula]>=0.8.0
1
+ hdfe-stream[formula]>=0.9.0
2
2
  stream-limits>=0.1.1
3
3
  polars!=1.41.*,>=1.37
4
4
  numpy>=1.24
@@ -25,6 +25,7 @@ sphinx-design>=0.6
25
25
 
26
26
  [test]
27
27
  pytest>=7.0
28
+ stream-limits[testing]>=0.2.1
28
29
  scikit-learn>=1.5
29
30
  statsmodels>=0.14
30
31
  pandas>=2.0
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "logit-stream"
7
- version = "0.2.0"
7
+ version = "0.3.0"
8
8
  description = "Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects"
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -30,7 +30,7 @@ requires-python = ">=3.10"
30
30
  dependencies = [
31
31
  # the streaming engine, the result base classes and the formula front end;
32
32
  # binary logit is hdfe-stream's feglm_stream
33
- "hdfe-stream[formula]>=0.8.0", # core.resources and core.profile are new in 0.8.0
33
+ "hdfe-stream[formula]>=0.9.0", # 0.9.0: lower-memory Parquet batch reads (iter_groups)
34
34
  "stream-limits>=0.1.1", # thread and memory limits for jobs on a scheduler (0.1.1: Windows memory)
35
35
  "polars>=1.37,!=1.41.*",
36
36
  "numpy>=1.24",
@@ -73,6 +73,7 @@ docs = [
73
73
  # the test suite compares against scikit-learn and statsmodels
74
74
  test = [
75
75
  "pytest>=7.0",
76
+ "stream-limits[testing]>=0.2.1", # resource_runner, for tests/test_resources.py
76
77
  "scikit-learn>=1.5",
77
78
  "statsmodels>=0.14",
78
79
  "pandas>=2.0",
@@ -86,6 +87,8 @@ include = ["logit_stream*"]
86
87
 
87
88
  [tool.pytest.ini_options]
88
89
  testpaths = ["tests"]
90
+ # memory tests (tests/test_resources.py) are slow: run them with `pytest -m resource`
91
+ addopts = "-m 'not resource'"
89
92
  filterwarnings = [
90
93
  "ignore::DeprecationWarning",
91
94
  "ignore::FutureWarning",
@@ -29,6 +29,24 @@ def panel():
29
29
  return _panel()
30
30
 
31
31
 
32
+ def test_analytic_weights_scale_free(panel):
33
+ """Analytic weights average 1 over the informative units, so scaling
34
+ them changes nothing; and the units' sums, taken a block at a time, do
35
+ not depend on how the rows are divided into blocks."""
36
+ w = panel.with_columns(w=(pl.col("i") % 4 + 1).cast(pl.Float64))
37
+ w = w.with_columns(w10=10 * pl.col("w"))
38
+ a = cfelogit_stream("y ~ x1 + x2 | i", w, weights="w", verbose=False, tol=1e-14)
39
+ b = cfelogit_stream("y ~ x1 + x2 | i", w, weights="w10", verbose=False, tol=1e-14)
40
+ c = cfelogit_stream("y ~ x1 + x2 | i", w, weights="w", verbose=False, tol=1e-14,
41
+ block_mb=0.005)
42
+ assert c.diagnostics["store"]["blocks"] > 5
43
+ for r in (b, c):
44
+ np.testing.assert_allclose(r.beta, a.beta, rtol=1e-10)
45
+ np.testing.assert_allclose(r.se, a.se, rtol=1e-9)
46
+ assert r.loglik == pytest.approx(a.loglik, rel=1e-12)
47
+ assert r.loglik_null == pytest.approx(a.loglik_null, rel=1e-12)
48
+
49
+
32
50
  def test_matches_statsmodels(panel):
33
51
  cm = pytest.importorskip("statsmodels.discrete.conditional_models")
34
52
  f = cfelogit_stream("y ~ x1 + x2 | i", panel, verbose=False, tol=1e-14)
@@ -91,6 +91,28 @@ def test_choice_set_validation(choice_data):
91
91
  clogit_stream("chosen ~ cost", w, case="case", weights="w", verbose=False)
92
92
 
93
93
 
94
+ def test_sets_checked_and_weighted_block_by_block(choice_data):
95
+ """The sets are checked and their weights summed a block at a time: the
96
+ error counts every bad set, in any block, and the analytic weights,
97
+ rescaled once the sum is known, give the same fit however the rows are
98
+ divided into blocks and wherever the blocks are kept."""
99
+ two = choice_data.with_columns(
100
+ pl.when(pl.col("case") % 100 == 0).then(1).otherwise(pl.col("chosen"))
101
+ .cast(pl.Int8).alias("chosen"))
102
+ n_bad = two.group_by("case").agg(pl.col("chosen").sum()).filter(pl.col("chosen") != 1).height
103
+ with pytest.raises(ValueError, match=f"{n_bad:,} of 3,000 do not"):
104
+ clogit_stream("chosen ~ cost", two, case="case", verbose=False, block_mb=0.02)
105
+ w = choice_data.with_columns((pl.col("case") % 7 + 1).cast(pl.Float64).alias("w"))
106
+ kw = dict(case="case", weights="w", verbose=False, tol=1e-14)
107
+ a = clogit_stream(FML, w, **kw)
108
+ for opts in (dict(block_mb=0.02), dict(storage="memory", block_mb=0.02)):
109
+ b = clogit_stream(FML, w.sample(fraction=1.0, shuffle=True, seed=3), **kw, **opts)
110
+ assert b.diagnostics["store"]["blocks"] > 5
111
+ np.testing.assert_allclose(b.beta, a.beta, rtol=1e-10)
112
+ np.testing.assert_allclose(b.se, a.se, rtol=1e-10)
113
+ assert b.loglik == pytest.approx(a.loglik, rel=1e-12)
114
+
115
+
94
116
  def test_weights_are_per_choice_set(choice_data):
95
117
  d = choice_data.filter(pl.col("case") < 1500)
96
118
  w = d.with_columns((pl.col("case") % 3 + 1).cast(pl.Float64).alias("w"))
@@ -0,0 +1,135 @@
1
+ """Memory tests: the streamed fits stay inside a memory limit on data that
2
+ would not fit in it.
3
+
4
+ Run with ``pytest -m resource`` (the default run leaves them out: they take
5
+ about three minutes). Each fit runs in a child process under stream-limits'
6
+ ``resource_runner``: a watchdog on every platform, or a kernel limit when
7
+ ``STREAM_LIMITS_TEST_CGROUP_PARENT`` names a cgroup set up for it.
8
+
9
+ The binary logit is hdfe-stream's ``feglm_stream``, whose memory tests are
10
+ hdfe-stream's; these are this package's own models: the multinomial and
11
+ ordered logit with a fixed effect absorbed, and the conditional logit. With
12
+ ``max_memory_gb=1.5`` the fits hold 0.7-1.2 GB of private memory (not
13
+ counting memory-mapped scratch files, which the kernel can reclaim), most of
14
+ it buffers whose sizes are capped. The limit is 1.6 GB. Measured on Linux
15
+ (WSL2, 2 threads):
16
+
17
+ - The multinomial and ordered logit, on 20 million people (1.05 GB in
18
+ memory): 0.94 and 0.73 GB, about the same as on 2.5 million. A fit that
19
+ held the data would need at least 1.8 GB.
20
+ - The conditional logit, on 8 million choice sets (21.6 million rows,
21
+ 0.98 GB): 1.0-1.3 GB from run to run, about what it held on 4 million
22
+ before its pass 0 checked the sets a block at a time (1.17-1.22 GB). A
23
+ fit that held the data would need at least 2 GB (one stopped at 1.63).
24
+
25
+ (A smaller budget is not a smaller fit: the buffers are there before the
26
+ multinomial logit checks that the fixed effect's blocks fit, and with
27
+ ``max_memory_gb=1.0`` it finds no room for them and stops.)
28
+
29
+ The limit is on private memory, which stream-limits measures on Linux and
30
+ Windows. On macOS it measures resident memory, which counts the mapped
31
+ scratch files, so there the limit is a looser safety net.
32
+ On Windows, private memory is committed memory: a buffer or an allocator
33
+ segment counts in full once allocated, touched or not (an hdfe-stream logit
34
+ fit had 1.61 GB committed while 1.14 GB was resident). Measured in CI
35
+ (GitHub's windows-latest, 2 threads): multinomial 1.70 GB, ordered 1.76 GB,
36
+ conditional 1.79 GB; the limit there is 2.1 GB, which a fit holding the data
37
+ (at least 2.7 GB) would still exceed.
38
+ """
39
+
40
+ from __future__ import annotations
41
+
42
+ import json
43
+ import sys
44
+
45
+ import numpy as np
46
+ import pytest
47
+
48
+ from logit_stream.simulate import RACES, simulate_choices, simulate_people, truth_people
49
+
50
+ pytestmark = pytest.mark.resource
51
+
52
+ N_PEOPLE = 20_000_000
53
+ N_CASES = 8_000_000 # 21.6 million rows
54
+ BUDGET_GB = 1.5 # the fit's max_memory_gb
55
+ LIMIT_GB = {"darwin": 2.6, "win32": 2.1}.get(sys.platform, 1.6)
56
+ X = "age10 + female + log_income + owner + urban"
57
+ XS = X.split(" + ")
58
+
59
+
60
+ @pytest.fixture(scope="module")
61
+ def people(tmp_path_factory):
62
+ path = tmp_path_factory.mktemp("big") / "people.parquet"
63
+ d = simulate_people(N_PEOPLE, seed=1)
64
+ d.write_parquet(path)
65
+ assert d.estimated_size() > 1.0e9
66
+ return path
67
+
68
+
69
+ @pytest.fixture(scope="module")
70
+ def choices(tmp_path_factory):
71
+ path = tmp_path_factory.mktemp("big") / "choices.parquet"
72
+ d = simulate_choices(N_CASES, seed=1)
73
+ d.write_parquet(path)
74
+ assert d.estimated_size() > 0.95e9
75
+ return path
76
+
77
+
78
+ def _fit_in_child(resource_runner, call: str, path, workdir) -> dict:
79
+ code = f"""
80
+ import json
81
+ from logit_stream import clogit_stream, mlogit_stream, ologit_stream
82
+ r = {call}
83
+ print(json.dumps(dict(converged=bool(r.converged),
84
+ coef={{k: float(v) for k, v in r.coef().items()}})))
85
+ """
86
+ code = code.replace("SRC", repr(str(path))).replace("WORK", repr(str(workdir)))
87
+ result = resource_runner([sys.executable, "-c", code], memory_gb=LIMIT_GB,
88
+ threads=2, timeout=900)
89
+ report = {k: result.as_dict()[k] for k in ("status", "mode", "memory_measure",
90
+ "peak_private_bytes", "peak_rss_bytes",
91
+ "elapsed_s")}
92
+ print(json.dumps(report))
93
+ assert result.status == "ok", (report, result.stderr[-3000:])
94
+ if result.kernel:
95
+ assert result.kernel["events"].get("oom_kill", 0) == 0
96
+ out = json.loads(result.stdout.strip().splitlines()[-1])
97
+ assert out["converged"]
98
+ return out["coef"]
99
+
100
+
101
+ # The checks against the simulated coefficients are sanity checks only:
102
+ # agreement with statsmodels and scikit-learn is the rest of the suite's job.
103
+
104
+ def test_multinomial_logit_streams_inside_the_limit(resource_runner, people, tmp_path):
105
+ coef = _fit_in_child(
106
+ resource_runner,
107
+ f"mlogit_stream('race ~ {X} | puma', SRC, workdir=WORK, verbose=False, "
108
+ f"max_memory_gb={BUDGET_GB})",
109
+ people, tmp_path / "work")
110
+ # the base level is the first in sort order (AIAN); the truth's is White
111
+ truth = truth_people()["race"]
112
+ for race in RACES[1:]:
113
+ for x in XS:
114
+ fit = coef.get(f"{race}:{x}", 0.0) - coef[f"White:{x}"]
115
+ assert abs(fit - truth[f"{race}:{x}"]) < 0.1, (race, x, fit)
116
+
117
+
118
+ def test_ordered_logit_streams_inside_the_limit(resource_runner, people, tmp_path):
119
+ coef = _fit_in_child(
120
+ resource_runner,
121
+ f"ologit_stream('education ~ {X} | puma', SRC, workdir=WORK, verbose=False, "
122
+ f"max_memory_gb={BUDGET_GB})",
123
+ people, tmp_path / "work")
124
+ truth = truth_people()["education"]
125
+ np.testing.assert_allclose([coef[x] for x in XS], [truth[x] for x in XS], atol=0.03)
126
+
127
+
128
+ def test_conditional_logit_streams_inside_the_limit(resource_runner, choices, tmp_path):
129
+ coef = _fit_in_child(
130
+ resource_runner,
131
+ "clogit_stream('chosen ~ cost + time + C(alt) + income:C(alt)', SRC, case='case', "
132
+ f"workdir=WORK, verbose=False, max_memory_gb={BUDGET_GB})",
133
+ choices, tmp_path / "work")
134
+ # simulate_choices' defaults
135
+ np.testing.assert_allclose([coef["cost"], coef["time"]], [-1.1, -1.3], atol=0.03)
File without changes
File without changes
File without changes
File without changes