logit-stream 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {logit_stream-0.2.0 → logit_stream-0.3.0}/CHANGELOG.md +19 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/CITATION.cff +2 -2
- {logit_stream-0.2.0 → logit_stream-0.3.0}/PKG-INFO +3 -2
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/__init__.py +1 -1
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/models.py +225 -63
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/newton.py +12 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/store.py +150 -58
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream.egg-info/PKG-INFO +3 -2
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream.egg-info/SOURCES.txt +1 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream.egg-info/requires.txt +2 -1
- {logit_stream-0.2.0 → logit_stream-0.3.0}/pyproject.toml +5 -2
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_cfelogit.py +18 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_conditional.py +22 -0
- logit_stream-0.3.0/tests/test_resources.py +135 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/LICENSE.md +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/MANIFEST.in +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/README.md +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/_choice.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/_frontend.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/_limits.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/api.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/__init__.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/_common.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/fe.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/meat.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/mixed.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/models.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/nested.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/random_effects.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/device/store.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/estimator.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/fe.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/mixed.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/multi_fe.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/nested.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/optimize.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/profile.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/py.typed +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/random_effects/__init__.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/random_effects/fit.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/random_effects/linalg.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/random_effects/model.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/random_effects/sigma.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/results.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream/simulate.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream.egg-info/dependency_links.txt +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/logit_stream.egg-info/top_level.txt +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/setup.cfg +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_api.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_clustering.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_device.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_firth_and_fit.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_fixed_effects.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_headroom.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_index_dtype.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_kernels.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_mixed.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_multi_fe.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_multinomial.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_nested.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_offset_start.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_ordered.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_probit.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_profile.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_random.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_simulate.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.0}/tests/test_weights_scale.py +0 -0
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.0
|
|
4
|
+
|
|
5
|
+
- **The conditional logit is 2.3-2.8 times faster.** Its log-likelihood, gradient and information
|
|
6
|
+
are computed in one parallel numba pass over each block's choice sets, giving the same
|
|
7
|
+
estimates (4 million sets: 13 s, now 5 s; 8 million: 26 s, now 9-12 s). Designs of more than
|
|
8
|
+
256 parameters keep the previous BLAS-based pass, which is faster there.
|
|
9
|
+
- **The fixed-effects logit (`cfelogit_stream`) balances work across threads better.** Panels
|
|
10
|
+
are dealt out in small pieces to more parts than threads, so a run of long panels no longer
|
|
11
|
+
leaves threads idle: 0.55 of the time on panels sorted by length, 0.8-1.0 on shuffled
|
|
12
|
+
ones, with the same results.
|
|
13
|
+
- **Lower memory for conditional, mixed and fixed-effects logit.** Choice sets are now checked
|
|
14
|
+
and counted a block at a time instead of from a table of every set, so memory no longer grows
|
|
15
|
+
with the number of sets (8 million sets: peak 1.71 GB down to 1.33 GB). Same errors, same
|
|
16
|
+
results.
|
|
17
|
+
- **Memory tests.** `pytest -m resource` fits the multinomial, ordered and conditional logit
|
|
18
|
+
on data larger than a fixed memory limit; CI runs them on Linux, macOS and Windows. The
|
|
19
|
+
default `pytest` run leaves them out.
|
|
20
|
+
- Requires hdfe-stream 0.9.0 or later, which reads Parquet in less memory.
|
|
21
|
+
|
|
3
22
|
## 0.2.0
|
|
4
23
|
|
|
5
24
|
- **Fit resources (CPUs and memory) are constrained to your job's limits automatically.** On a
|
|
@@ -7,8 +7,8 @@ authors:
|
|
|
7
7
|
- family-names: Tucker
|
|
8
8
|
given-names: Lee
|
|
9
9
|
email: lee.c.tucker@census.gov
|
|
10
|
-
version: 0.
|
|
11
|
-
date-released: 2026-
|
|
10
|
+
version: 0.3.0
|
|
11
|
+
date-released: 2026-11-05
|
|
12
12
|
license: MIT
|
|
13
13
|
repository-code: "https://github.com/leetucker/logit-stream"
|
|
14
14
|
url: "https://github.com/leetucker/logit-stream"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: logit-stream
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects
|
|
5
5
|
Author: Lee Tucker
|
|
6
6
|
License-Expression: MIT
|
|
@@ -25,7 +25,7 @@ Classifier: Typing :: Typed
|
|
|
25
25
|
Requires-Python: >=3.10
|
|
26
26
|
Description-Content-Type: text/markdown
|
|
27
27
|
License-File: LICENSE.md
|
|
28
|
-
Requires-Dist: hdfe-stream[formula]>=0.
|
|
28
|
+
Requires-Dist: hdfe-stream[formula]>=0.9.0
|
|
29
29
|
Requires-Dist: stream-limits>=0.1.1
|
|
30
30
|
Requires-Dist: polars!=1.41.*,>=1.37
|
|
31
31
|
Requires-Dist: numpy>=1.24
|
|
@@ -49,6 +49,7 @@ Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
|
|
|
49
49
|
Requires-Dist: sphinx-design>=0.6; extra == "docs"
|
|
50
50
|
Provides-Extra: test
|
|
51
51
|
Requires-Dist: pytest>=7.0; extra == "test"
|
|
52
|
+
Requires-Dist: stream-limits[testing]>=0.2.1; extra == "test"
|
|
52
53
|
Requires-Dist: scikit-learn>=1.5; extra == "test"
|
|
53
54
|
Requires-Dist: statsmodels>=0.14; extra == "test"
|
|
54
55
|
Requires-Dist: pandas>=2.0; extra == "test"
|
|
@@ -37,6 +37,31 @@ from scipy.special import expit, ndtr, ndtri
|
|
|
37
37
|
# average, and the parameters few enough for a per-thread accumulator each
|
|
38
38
|
SPARSE_DENSITY = 0.5
|
|
39
39
|
SPARSE_MAX_PARAMS = 1200
|
|
40
|
+
# The conditional and fixed-effects logits' numba passes (`_clogit_pass`,
|
|
41
|
+
# `_cfe_units`) split a block's choice sets or units into this many parts per
|
|
42
|
+
# thread, handed out as threads come free (`_parts`), and keep the parts'
|
|
43
|
+
# information accumulators together to at most PARTS_MB (but at least one
|
|
44
|
+
# part per thread). The conditional logit's is used up to CLOGIT_KERNEL_MAX_P
|
|
45
|
+
# parameters: it adds each
|
|
46
|
+
# row into a P x P accumulator per thread, which is fast while that fits in a
|
|
47
|
+
# core's L2 cache (0.5 MB at 256; current cores have 1-2.5 MB) and several
|
|
48
|
+
# times slower once it does not, where BLAS's blocked products are not
|
|
49
|
+
# (measured, 4 threads: 0.17 of the NumPy pass's time at P = 128, 0.43 at
|
|
50
|
+
# 256, 0.7-0.85 at 384-512, 2.3 at 768 with a 2.5 MB L2). Above it the pass
|
|
51
|
+
# is NumPy's (`_Conditional._accumulate_blas`).
|
|
52
|
+
PARTS_PER_THREAD = 4
|
|
53
|
+
PARTS_MB = 64.0
|
|
54
|
+
# the fixed-effects logit's parts take runs of this many consecutive units
|
|
55
|
+
CFE_RUN = 64
|
|
56
|
+
CLOGIT_KERNEL_MAX_P = 256
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _parts(n_items: int, P: int) -> int:
|
|
60
|
+
"""How many parts to split `n_items` sets or units into, for a pass
|
|
61
|
+
whose parts each keep a P x P accumulator (see PARTS_PER_THREAD)."""
|
|
62
|
+
threads = nb.get_num_threads()
|
|
63
|
+
cap = max(threads, int(PARTS_MB * 1e6 // (8 * max(P * P, 1))))
|
|
64
|
+
return max(1, min(n_items, PARTS_PER_THREAD * threads, cap))
|
|
40
65
|
|
|
41
66
|
|
|
42
67
|
def _gram(X, v):
|
|
@@ -91,6 +116,93 @@ def _mnl_info_sparse(X, P, w, n_threads, out):
|
|
|
91
116
|
out += acc[t]
|
|
92
117
|
|
|
93
118
|
|
|
119
|
+
@nb.njit(parallel=True, cache=True)
|
|
120
|
+
def _clogit_pass(X, cols, theta, y, w, off, starts, hessian, max_size, ll_t, g_t, H_t):
|
|
121
|
+
"""The conditional logit's log-likelihood, gradient and information over
|
|
122
|
+
the choice sets of a block, in one pass over its rows: the sets are
|
|
123
|
+
split into ``len(ll_t)`` parts, and part t's sums are written to
|
|
124
|
+
``ll_t[t]``, ``g_t[t]`` and (with `hessian`) ``H_t[t]``, for the caller
|
|
125
|
+
to add up (`_Conditional.accumulate_full`). `cols` are the columns of
|
|
126
|
+
`X` in the model; `w` and `off` are empty when there are no weights or
|
|
127
|
+
offset.
|
|
128
|
+
|
|
129
|
+
The only parallel loop is the one over the parts: the accumulators are
|
|
130
|
+
allocated and summed by the caller, since with `parallel_chunksize`
|
|
131
|
+
numba would hand out every element of an array operation here as a
|
|
132
|
+
chunk of its own.
|
|
133
|
+
|
|
134
|
+
Per set: p = softmax(eta), the log-likelihood eta_chosen - log sum
|
|
135
|
+
exp(eta), the gradient sum (y - p) x, and the information
|
|
136
|
+
sum p (x - xbar)(x - xbar)' with xbar = sum p x (the within-set
|
|
137
|
+
covariance, centred before the products: the same matrix as
|
|
138
|
+
sum p x x' - xbar xbar', without the cancellation). Each is weighted by
|
|
139
|
+
the set's weight. Only the upper triangle of the information is filled."""
|
|
140
|
+
n_sets = len(starts) - 1
|
|
141
|
+
P = len(cols)
|
|
142
|
+
n_parts = len(ll_t)
|
|
143
|
+
has_w, has_off = len(w) > 0, len(off) > 0
|
|
144
|
+
step = (n_sets + n_parts - 1) // n_parts
|
|
145
|
+
for t in nb.prange(n_parts):
|
|
146
|
+
# local accumulators, copied out at the end: rows of the shared
|
|
147
|
+
# arrays would share cache lines between threads
|
|
148
|
+
eta = np.empty(max_size)
|
|
149
|
+
xs = np.empty((max_size, P)) # the set's rows, read once
|
|
150
|
+
xbar = np.empty(P)
|
|
151
|
+
d = np.empty(P)
|
|
152
|
+
g = np.zeros(P)
|
|
153
|
+
Ht = np.zeros((P, P) if hessian else (0, 0))
|
|
154
|
+
llt = 0.0
|
|
155
|
+
for s in range(t * step, min(n_sets, (t + 1) * step)):
|
|
156
|
+
a, b = starts[s], starts[s + 1]
|
|
157
|
+
ws = w[a] if has_w else 1.0
|
|
158
|
+
m = -np.inf
|
|
159
|
+
chosen = 0.0
|
|
160
|
+
for i in range(a, b):
|
|
161
|
+
e = off[i] if has_off else 0.0
|
|
162
|
+
for k in range(P):
|
|
163
|
+
x = X[i, cols[k]]
|
|
164
|
+
xs[i - a, k] = x
|
|
165
|
+
e += x * theta[k]
|
|
166
|
+
eta[i - a] = e
|
|
167
|
+
if y[i] == 1:
|
|
168
|
+
chosen += e
|
|
169
|
+
if e > m:
|
|
170
|
+
m = e
|
|
171
|
+
S = 0.0
|
|
172
|
+
for i in range(a, b):
|
|
173
|
+
eta[i - a] = np.exp(eta[i - a] - m)
|
|
174
|
+
S += eta[i - a]
|
|
175
|
+
llt += ws * (chosen - m - np.log(S))
|
|
176
|
+
for k in range(P):
|
|
177
|
+
xbar[k] = 0.0
|
|
178
|
+
for i in range(a, b):
|
|
179
|
+
p = eta[i - a] / S
|
|
180
|
+
eta[i - a] = p
|
|
181
|
+
r = ws * ((1.0 if y[i] == 1 else 0.0) - p)
|
|
182
|
+
for k in range(P):
|
|
183
|
+
x = xs[i - a, k]
|
|
184
|
+
g[k] += r * x
|
|
185
|
+
xbar[k] += p * x
|
|
186
|
+
if hessian:
|
|
187
|
+
for i in range(a, b):
|
|
188
|
+
p = ws * eta[i - a]
|
|
189
|
+
for k in range(P):
|
|
190
|
+
d[k] = xs[i - a, k] - xbar[k]
|
|
191
|
+
# the full square (a fixed-length inner loop, which
|
|
192
|
+
# vectorizes), though only the upper triangle is used
|
|
193
|
+
for k in range(P):
|
|
194
|
+
v = p * d[k]
|
|
195
|
+
for l in range(P):
|
|
196
|
+
Ht[k, l] += v * d[l]
|
|
197
|
+
ll_t[t] = llt
|
|
198
|
+
for k in range(P):
|
|
199
|
+
g_t[t, k] = g[k]
|
|
200
|
+
if hessian:
|
|
201
|
+
for k in range(P):
|
|
202
|
+
for l in range(P):
|
|
203
|
+
H_t[t, k, l] = Ht[k, l]
|
|
204
|
+
|
|
205
|
+
|
|
94
206
|
def _bincount(idx, weights, n):
|
|
95
207
|
return np.bincount(idx, weights=weights, minlength=n)[:n]
|
|
96
208
|
|
|
@@ -530,6 +642,43 @@ class _Conditional(_Model):
|
|
|
530
642
|
return e / np.repeat(S, sizes), m, S
|
|
531
643
|
|
|
532
644
|
def accumulate(self, theta, X, y, w, starts, hessian, offset=None):
|
|
645
|
+
return self.accumulate_full(theta, X, np.arange(X.shape[1]), y, w, starts, hessian,
|
|
646
|
+
offset)
|
|
647
|
+
|
|
648
|
+
def accumulate_full(self, theta, X, cols, y, w, starts, hessian, offset=None):
|
|
649
|
+
"""`accumulate` on the columns `cols` of `X`: a whole block, by
|
|
650
|
+
numba's pass, which holds no row-sized temporaries and reads the
|
|
651
|
+
columns in place rather than copying them; above
|
|
652
|
+
CLOGIT_KERNEL_MAX_P parameters, by NumPy's."""
|
|
653
|
+
P = len(cols)
|
|
654
|
+
if P > CLOGIT_KERNEL_MAX_P:
|
|
655
|
+
Xc = X if P == X.shape[1] and np.array_equal(cols, np.arange(P)) else X[:, cols]
|
|
656
|
+
return self._accumulate_blas(theta, Xc, y, w, starts, hessian, offset)
|
|
657
|
+
empty = np.empty(0)
|
|
658
|
+
starts = np.asarray(starts, np.int64)
|
|
659
|
+
# parts handed out one at a time as threads come free, so that a
|
|
660
|
+
# thread the machine slows (another process, the store's reader, a
|
|
661
|
+
# slower core) does not hold up the rest; their accumulators are
|
|
662
|
+
# capped together, at one part per thread
|
|
663
|
+
n_parts = _parts(len(starts) - 1, P)
|
|
664
|
+
ll_t, g_t = np.zeros(n_parts), np.zeros((n_parts, P))
|
|
665
|
+
H_t = np.zeros((n_parts, P, P) if hessian else (n_parts, 0, 0))
|
|
666
|
+
with nb.parallel_chunksize(1):
|
|
667
|
+
_clogit_pass(
|
|
668
|
+
np.ascontiguousarray(X, np.float64), np.asarray(cols, np.int64),
|
|
669
|
+
np.ascontiguousarray(theta, np.float64), y,
|
|
670
|
+
empty if w is None else np.ascontiguousarray(w, np.float64),
|
|
671
|
+
empty if offset is None else np.ascontiguousarray(offset, np.float64),
|
|
672
|
+
starts, bool(hessian), int(np.diff(starts).max()) if len(starts) > 1 else 1,
|
|
673
|
+
ll_t, g_t, H_t)
|
|
674
|
+
ll, g, H = ll_t.sum(), g_t.sum(axis=0), H_t.sum(axis=0)
|
|
675
|
+
if not hessian:
|
|
676
|
+
return float(ll), g, None
|
|
677
|
+
return float(ll), g, np.triu(H) + np.triu(H, 1).T
|
|
678
|
+
|
|
679
|
+
def _accumulate_blas(self, theta, X, y, w, starts, hessian, offset=None):
|
|
680
|
+
"""`accumulate` by NumPy array operations, the information by BLAS's
|
|
681
|
+
Gram products: for a wide design (`CLOGIT_KERNEL_MAX_P`)."""
|
|
533
682
|
st = starts[:-1]
|
|
534
683
|
eta = X @ theta
|
|
535
684
|
if offset is not None:
|
|
@@ -560,7 +709,7 @@ class _Conditional(_Model):
|
|
|
560
709
|
|
|
561
710
|
# ------------------------------------------------------------------ conditional FE logit
|
|
562
711
|
@nb.njit(parallel=True, cache=True)
|
|
563
|
-
def _cfe_units(X, eta, y, w, starts, hessian,
|
|
712
|
+
def _cfe_units(X, eta, y, w, starts, hessian, ll_u, g_u, acc):
|
|
564
713
|
"""Chamberlain's conditional likelihood, unit by unit (the rows of unit u
|
|
565
714
|
are starts[u]:starts[u+1]): with k the unit's number of 1s among its T
|
|
566
715
|
rows, log P(y | k) = sum y eta - log e_k(exp(eta)), e_k the elementary
|
|
@@ -573,68 +722,81 @@ def _cfe_units(X, eta, y, w, starts, hessian, n_threads, ll_u, g_u, out):
|
|
|
573
722
|
unit's largest, and a unit with k > T / 2 is fitted as its complement
|
|
574
723
|
(1 - y, -x, -eta), so that k <= T / 2. Units with k = 0 or k = T carry no
|
|
575
724
|
information and are skipped. Writes each unit's (weighted) log-likelihood
|
|
576
|
-
and gradient; the information is summed
|
|
577
|
-
|
|
725
|
+
and gradient; the information is summed over ``len(acc)`` parts of the
|
|
726
|
+
units, part t's into ``acc[t]``, for the caller to add up.
|
|
727
|
+
|
|
728
|
+
A unit's work grows with T k P^2, so units differ in cost: the parts are
|
|
729
|
+
more than the threads, handed out as threads come free (`_CFELogit._units`
|
|
730
|
+
runs this under ``parallel_chunksize(1)``), and each takes every
|
|
731
|
+
``len(acc)``-th run of CFE_RUN units, so that each has a mix of short and
|
|
732
|
+
long panels. The only parallel loop is the
|
|
733
|
+
one over the parts, so the accumulators are allocated and summed by the
|
|
734
|
+
caller (numba would otherwise hand out every element of an array
|
|
735
|
+
operation here as a chunk of its own)."""
|
|
578
736
|
U, P = len(starts) - 1, X.shape[1]
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
# (a thread's own accumulator, copied out at the end: the threads'
|
|
737
|
+
n_parts = len(acc)
|
|
738
|
+
for th in nb.prange(n_parts):
|
|
739
|
+
# (a part's own accumulator, copied out at the end: the parts'
|
|
583
740
|
# slices of `acc` are too close together to add into unit by unit)
|
|
584
741
|
a_ = np.zeros((P, P))
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
if
|
|
742
|
+
# runs of CFE_RUN units, every n_parts-th run: data in order of panel
|
|
743
|
+
# length (a cohort, an entry date) would otherwise put the longest,
|
|
744
|
+
# costliest units together in the last parts, while single units
|
|
745
|
+
# dealt out would scatter the reads
|
|
746
|
+
for u0 in range(th * CFE_RUN, U, n_parts * CFE_RUN):
|
|
747
|
+
for u in range(u0, min(U, u0 + CFE_RUN)):
|
|
748
|
+
r0, r1 = starts[u], starts[u + 1]
|
|
749
|
+
T = r1 - r0
|
|
750
|
+
k = 0
|
|
751
|
+
for r in range(r0, r1):
|
|
752
|
+
k += 1 if y[r] > 0 else 0
|
|
753
|
+
ll_u[u] = 0.0
|
|
754
|
+
for p in range(P):
|
|
755
|
+
g_u[u, p] = 0.0
|
|
756
|
+
if k == 0 or k == T:
|
|
757
|
+
continue
|
|
758
|
+
s = 1.0
|
|
759
|
+
if 2 * k > T:
|
|
760
|
+
s, k = -1.0, T - k
|
|
761
|
+
c = -np.inf
|
|
762
|
+
for r in range(r0, r1):
|
|
763
|
+
c = max(c, s * eta[r])
|
|
764
|
+
e = np.zeros(k + 1)
|
|
765
|
+
G = np.zeros((k + 1, P))
|
|
766
|
+
Hm = np.zeros((k + 1, P, P)) if hessian else np.zeros((1, 1, 1))
|
|
767
|
+
e[0] = 1.0
|
|
768
|
+
obs = 0.0
|
|
769
|
+
for r in range(r0, r1):
|
|
770
|
+
z = np.exp(s * eta[r] - c)
|
|
771
|
+
yr = (y[r] > 0) if s > 0 else (y[r] <= 0)
|
|
772
|
+
if yr:
|
|
773
|
+
obs += s * eta[r]
|
|
616
774
|
for p in range(P):
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
775
|
+
g_u[u, p] += s * X[r, p]
|
|
776
|
+
for j in range(k, 0, -1):
|
|
777
|
+
if hessian:
|
|
778
|
+
for p in range(P):
|
|
779
|
+
xp = s * X[r, p]
|
|
780
|
+
for q in range(P):
|
|
781
|
+
xq = s * X[r, q]
|
|
782
|
+
Hm[j, p, q] += z * (Hm[j - 1, p, q] + xp * G[j - 1, q]
|
|
783
|
+
+ G[j - 1, p] * xq + xp * xq * e[j - 1])
|
|
784
|
+
for p in range(P):
|
|
785
|
+
G[j, p] += z * (G[j - 1, p] + s * X[r, p] * e[j - 1])
|
|
786
|
+
e[j] += z * e[j - 1]
|
|
787
|
+
wu = w[r0]
|
|
788
|
+
ll_u[u] = wu * (obs - np.log(e[k]) - k * c)
|
|
631
789
|
for p in range(P):
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
790
|
+
m = G[k, p] / e[k]
|
|
791
|
+
g_u[u, p] = wu * (g_u[u, p] - m)
|
|
792
|
+
if hessian:
|
|
793
|
+
for p in range(P):
|
|
794
|
+
mp = G[k, p] / e[k]
|
|
795
|
+
for q in range(P):
|
|
796
|
+
a_[p, q] += wu * (Hm[k, p, q] / e[k] - mp * G[k, q] / e[k])
|
|
797
|
+
for p in range(P):
|
|
798
|
+
for q in range(P):
|
|
799
|
+
acc[th, p, q] = a_[p, q]
|
|
638
800
|
|
|
639
801
|
|
|
640
802
|
class _CFELogit(_Model):
|
|
@@ -673,12 +835,12 @@ class _CFELogit(_Model):
|
|
|
673
835
|
eta = eta + offset
|
|
674
836
|
U, P = len(starts) - 1, X.shape[1]
|
|
675
837
|
ll_u, g_u = np.empty(U), np.empty((U, P))
|
|
676
|
-
|
|
838
|
+
acc = np.zeros((_parts(U, P), P, P))
|
|
677
839
|
ww = np.ones(len(y)) if w is None else np.ascontiguousarray(w, dtype=np.float64)
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
return ll_u, g_u,
|
|
840
|
+
with nb.parallel_chunksize(1):
|
|
841
|
+
_cfe_units(np.ascontiguousarray(X), np.ascontiguousarray(eta), y, ww,
|
|
842
|
+
np.ascontiguousarray(starts, dtype=np.int64), hessian, ll_u, g_u, acc)
|
|
843
|
+
return ll_u, g_u, acc.sum(axis=0)
|
|
682
844
|
|
|
683
845
|
def accumulate(self, theta, X, y, w, starts, hessian, offset=None):
|
|
684
846
|
ll_u, g_u, H = self._units(theta, X, y, w, starts, hessian, offset)
|
|
@@ -129,7 +129,19 @@ class _Driver:
|
|
|
129
129
|
m = self.model
|
|
130
130
|
ll, g, H = 0.0, np.zeros(m.P), (np.zeros((m.P, m.P)) if hessian else None)
|
|
131
131
|
rows = self._rows()
|
|
132
|
+
full = getattr(m, "accumulate_full", None)
|
|
132
133
|
for blk in self.store:
|
|
134
|
+
if full is not None and firth_A is None:
|
|
135
|
+
# a numba pass over the whole block, on its columns in place
|
|
136
|
+
l, gg, hh = full(theta, blk["X"], m.cols, blk["y"], blk.get("w"),
|
|
137
|
+
blk.get("starts"), hessian, offset=blk.get("off"))
|
|
138
|
+
if not np.isfinite(l):
|
|
139
|
+
return -np.inf, None, None
|
|
140
|
+
ll += l
|
|
141
|
+
g += gg
|
|
142
|
+
if hessian:
|
|
143
|
+
H += hh
|
|
144
|
+
continue
|
|
133
145
|
for ch in _chunks(blk, rows):
|
|
134
146
|
X = _design(ch, m.cols, self.K0)
|
|
135
147
|
l, gg, hh = m.accumulate(theta, X, ch["y"], ch.get("w"), ch.get("starts"),
|
|
@@ -138,21 +138,39 @@ def _build(lf: pl.LazyFrame, spec, kind: str, run_dir: Path, engine: PolarsEngin
|
|
|
138
138
|
cols = list(dict.fromkeys([*roots, *([weights] if weights else []), *raw]))
|
|
139
139
|
rows = block_rows(block_mb, K)
|
|
140
140
|
design_path = run_dir / "sample.parquet"
|
|
141
|
+
log("pass 0: writing the sample")
|
|
142
|
+
write_sample(engine, lf, usable, cols, design_path, rows, ROW)
|
|
141
143
|
# A choice set with an incomplete alternative is dropped whole (Stata's
|
|
142
144
|
# cmclogit default): fitted on its other alternatives, it would be a
|
|
143
|
-
# different choice set. altwise=True keeps them.
|
|
145
|
+
# different choice set. altwise=True keeps them. Whether there is one is
|
|
146
|
+
# the source's rows with a choice set against the sample's (a count of
|
|
147
|
+
# the key columns); which sets, and a second copy of the sample without
|
|
148
|
+
# them, only when there is. (Filtering the source for the incomplete
|
|
149
|
+
# rows, as this did, holds memory in proportion to the source in Polars'
|
|
150
|
+
# streaming engine even when none match: 0.3 GB on 21.6 million rows.)
|
|
144
151
|
incomplete = None
|
|
145
152
|
if kind in ("clogit", "mixlogit") and not altwise:
|
|
146
153
|
skey = list(dict.fromkeys([*(panel or []), *case]))
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
154
|
+
keyed = engine.collect(
|
|
155
|
+
lf.filter(pl.all_horizontal([pl.col(c).is_not_null() for c in skey]))
|
|
156
|
+
.select(pl.len()), "rows with a choice set").item()
|
|
157
|
+
kept = engine.collect(pl.scan_parquet(design_path).select(pl.len()),
|
|
158
|
+
"sample rows").item()
|
|
159
|
+
if keyed > kept:
|
|
160
|
+
bad = engine.collect(lf.filter(~usable).select(skey).drop_nulls().unique(),
|
|
161
|
+
"incomplete choice sets")
|
|
162
|
+
if bad.height:
|
|
163
|
+
incomplete = (bad, skey)
|
|
164
|
+
warn(f"{spec.fml}: {bad.height:,} choice sets dropped because some of their "
|
|
165
|
+
"alternatives have missing values (altwise=True keeps their complete "
|
|
166
|
+
"alternatives)")
|
|
167
|
+
every = run_dir / "sample_all.parquet"
|
|
168
|
+
design_path.rename(every)
|
|
169
|
+
engine.sink(pl.scan_parquet(every).join(bad.lazy(), on=skey, how="anti",
|
|
170
|
+
maintain_order="left"),
|
|
171
|
+
str(design_path), "pass 0 sample (complete sets)",
|
|
172
|
+
row_group_size=min(rows, 250_000))
|
|
173
|
+
every.unlink()
|
|
156
174
|
evaluated = [pl.col(ROW), yexpr.alias(Y), *xexprs, *wexpr,
|
|
157
175
|
*([offset.alias(OFF)] if offset is not None else []),
|
|
158
176
|
*[pl.col(c) for c in raw if c not in (Y, W)]]
|
|
@@ -271,6 +289,8 @@ def _build(lf: pl.LazyFrame, spec, kind: str, run_dir: Path, engine: PolarsEngin
|
|
|
271
289
|
"pidx": pidx,
|
|
272
290
|
"row": frame[ROW].to_numpy().astype(np.int64)})
|
|
273
291
|
|
|
292
|
+
convert.store = store # for _case_order's weight rescaling
|
|
293
|
+
|
|
274
294
|
n_cases, wcases, n_panels = 0, 0.0, 0
|
|
275
295
|
units = {}
|
|
276
296
|
log(f"pass 0: writing {n_obs:,} rows as blocks of {rows:,}"
|
|
@@ -280,7 +300,8 @@ def _build(lf: pl.LazyFrame, spec, kind: str, run_dir: Path, engine: PolarsEngin
|
|
|
280
300
|
n_cases, wcases, n_panels = _case_order(engine, design, design_path, case, weights,
|
|
281
301
|
spec, rows, convert, run_dir, log, panel,
|
|
282
302
|
ccols, wscale if weights_type != "fweights"
|
|
283
|
-
else None, kind=kind, info=units
|
|
303
|
+
else None, kind=kind, info=units,
|
|
304
|
+
evaluated=evaluated)
|
|
284
305
|
else:
|
|
285
306
|
for frame in iter_batches(design_path, rows):
|
|
286
307
|
convert(frame)
|
|
@@ -294,7 +315,8 @@ def _build(lf: pl.LazyFrame, spec, kind: str, run_dir: Path, engine: PolarsEngin
|
|
|
294
315
|
|
|
295
316
|
|
|
296
317
|
def _case_order(engine, design, design_path, case, weights, spec, rows, convert, run_dir,
|
|
297
|
-
log, panel=None, cluster_cols=(), wscale=None, kind="clogit", info=None
|
|
318
|
+
log, panel=None, cluster_cols=(), wscale=None, kind="clogit", info=None,
|
|
319
|
+
evaluated=()):
|
|
298
320
|
"""Write the conditional logit's blocks so that each holds whole choice
|
|
299
321
|
sets, each a run of consecutive rows; return the number of sets, their
|
|
300
322
|
summed weight and the number of panels.
|
|
@@ -308,7 +330,16 @@ def _case_order(engine, design, design_path, case, weights, spec, rows, convert,
|
|
|
308
330
|
|
|
309
331
|
Long-format choice data usually arrives grouped by choice set already,
|
|
310
332
|
and then the rows are written in their order; otherwise they are sorted
|
|
311
|
-
by set in buckets (`hdfe_stream.core.store.iter_groups`).
|
|
333
|
+
by set in buckets (`hdfe_stream.core.store.iter_groups`).
|
|
334
|
+
|
|
335
|
+
The sets are checked (one chosen alternative; weights and clusters
|
|
336
|
+
constant within each) and summed (their weights; the fixed-effects
|
|
337
|
+
logit's units) a block at a time, as the blocks are written: a summary
|
|
338
|
+
of every set at once would hold a row per set in memory. Analytic
|
|
339
|
+
weights, which are rescaled to average 1 over the sets, are therefore
|
|
340
|
+
written as they are and rescaled once the sum is known
|
|
341
|
+
(`_rescale_weights`). `evaluated` are the sample's expressions, as
|
|
342
|
+
`convert` evaluates them, for the outcome and the weights."""
|
|
312
343
|
skey = list(dict.fromkeys([*(panel or []), *case]))
|
|
313
344
|
unit = list(panel) if panel else skey
|
|
314
345
|
|
|
@@ -316,62 +347,123 @@ def _case_order(engine, design, design_path, case, weights, spec, rows, convert,
|
|
|
316
347
|
return pl.any_horizontal([(pl.col(c) != pl.col(c).shift()).fill_null(True)
|
|
317
348
|
for c in cols])
|
|
318
349
|
|
|
350
|
+
def key(cols):
|
|
351
|
+
# one column's distinct values are counted with little memory; a
|
|
352
|
+
# struct's with more, but still far less than a row per set
|
|
353
|
+
return pl.col(cols[0]) if len(cols) == 1 else pl.struct(cols)
|
|
354
|
+
|
|
319
355
|
changed, uchanged = runs_of(skey), runs_of(unit)
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
356
|
+
n_cases = None
|
|
357
|
+
if not panel and len(skey) == 1:
|
|
358
|
+
# sorted by set, as long-format choice data usually are: each set's
|
|
359
|
+
# rows are then consecutive, and the sets are the runs of the key,
|
|
360
|
+
# counted without a table of the keys (about 30 bytes a set)
|
|
361
|
+
k = pl.col(skey[0])
|
|
362
|
+
order = engine.collect(design.select(
|
|
363
|
+
runs_of(skey).sum().alias("runs"),
|
|
364
|
+
(k < k.shift()).fill_null(False).any().alias("down")), "choice set order")
|
|
365
|
+
if not order["down"].item():
|
|
366
|
+
n_cases = int(order["runs"].item())
|
|
367
|
+
if n_cases is None:
|
|
368
|
+
counts = engine.collect(design.select(
|
|
369
|
+
key(skey).n_unique().alias("sets"),
|
|
370
|
+
*([key(list(panel)).n_unique().alias("panels")] if panel else [])), "choice sets")
|
|
371
|
+
n_cases = int(counts["sets"].item())
|
|
372
|
+
n_panels = int(counts["panels"].item()) if panel else 0
|
|
373
|
+
|
|
374
|
+
need = {Y, *([W] if weights else [])}
|
|
375
|
+
ev = [e for e in evaluated if e.meta.output_name() in need]
|
|
376
|
+
run = "__ls_set"
|
|
377
|
+
agg = [pl.col(Y).sum().alias("chosen"), pl.len().alias("size"),
|
|
378
|
+
*([(pl.col(W).max() - pl.col(W).min()).alias("wspread"),
|
|
379
|
+
pl.col(W).first().alias("wcase")] if weights else []),
|
|
380
|
+
*[pl.col(c).n_unique().alias(f"__ls_nu_{i}") for i, c in enumerate(cluster_cols)]]
|
|
381
|
+
wsum_sets = 0.0
|
|
382
|
+
bad_sets, bad_eg = 0, []
|
|
383
|
+
wspread = False
|
|
384
|
+
spread = [[0, []] for _ in cluster_cols]
|
|
385
|
+
cfe = {"units": 0, "rows": 0, "wunits": 0.0, "loglik_null": 0.0}
|
|
332
386
|
if kind == "cfelogit":
|
|
333
|
-
# units whose outcome never varies carry no information (their rows stay,
|
|
334
|
-
# and the likelihood skips them); the null, every arrangement of a unit's
|
|
335
|
-
# 1s equally likely, is -sum log C(T, k)
|
|
336
387
|
from scipy.special import gammaln
|
|
337
|
-
T, k = per["size"].to_numpy().astype(float), per["chosen"].to_numpy()
|
|
338
|
-
live = (k > 0) & (k < T)
|
|
339
|
-
wu = per["wcase"].to_numpy() if weights else np.ones(len(T))
|
|
340
|
-
if weights and wscale is not None: # average 1 over the informative units
|
|
341
|
-
wscale[0] = live.sum() / wu[live].sum()
|
|
342
|
-
wu = wu * (wscale[0] if weights and wscale is not None else 1.0)
|
|
343
|
-
lc = gammaln(T + 1) - gammaln(k + 1) - gammaln(T - k + 1)
|
|
344
|
-
info.update(units=int(live.sum()), dropped=int((~live).sum()),
|
|
345
|
-
rows=int(T[live].sum()), wunits=float(wu[live].sum()),
|
|
346
|
-
loglik_null=float(-(lc[live] * wu[live]).sum()))
|
|
347
|
-
bad = per.filter(pl.col("chosen") != 1) if kind != "cfelogit" else per.head(0)
|
|
348
|
-
if bad.height:
|
|
349
|
-
raise ValueError(
|
|
350
|
-
f"{spec.fml}: every choice set must have exactly one chosen alternative "
|
|
351
|
-
f"(outcome 1); {bad.height:,} of {n_cases:,} do not (e.g. "
|
|
352
|
-
f"{bad.select(skey).head(3).to_dicts()}). Rows dropped for missing values can "
|
|
353
|
-
"remove a set's chosen alternative.")
|
|
354
|
-
if weights and per["wspread"].max() > 0:
|
|
355
|
-
raise ValueError("weights must be constant within each choice set (a choice set is "
|
|
356
|
-
"one observation)")
|
|
357
|
-
for i, c in enumerate(cluster_cols):
|
|
358
|
-
spread = per.filter(pl.col(f"__ls_nu_{i}") > 1)
|
|
359
|
-
if spread.height:
|
|
360
|
-
raise ValueError(f"cluster variable {c!r} must be constant within each choice set "
|
|
361
|
-
f"(a choice set is one observation); {spread.height:,} sets "
|
|
362
|
-
f"have more than one value (e.g. "
|
|
363
|
-
f"{spread.select(skey).head(3).to_dicts()})")
|
|
364
|
-
del per
|
|
365
388
|
|
|
366
389
|
# blocks of whole units (sets, or panels with their sets consecutive)
|
|
367
390
|
for body in iter_groups(engine, design_path, unit, rows, run_dir,
|
|
368
391
|
n_panels if panel else n_cases, [*skey, ROW],
|
|
369
392
|
also=[(skey, n_cases)] if panel else (), log=log):
|
|
370
|
-
|
|
371
|
-
|
|
393
|
+
b = body.select(changed.alias("b"))["b"].to_numpy()
|
|
394
|
+
starts = np.append(np.flatnonzero(b), body.height).astype(np.int64)
|
|
395
|
+
# this block's sets, in their order
|
|
396
|
+
per = (body.select(*skey, *[pl.col(c) for c in cluster_cols if c not in skey], *ev)
|
|
397
|
+
.with_columns(pl.Series(run, np.cumsum(b)))
|
|
398
|
+
.group_by(run, maintain_order=True)
|
|
399
|
+
.agg(*[pl.col(c).first() for c in skey], *agg))
|
|
400
|
+
if weights:
|
|
401
|
+
wsum_sets += float(per["wcase"].sum())
|
|
402
|
+
wspread = wspread or per["wspread"].max() > 0
|
|
403
|
+
if kind == "cfelogit":
|
|
404
|
+
# units whose outcome never varies carry no information (their rows
|
|
405
|
+
# stay, and the likelihood skips them); the null, every arrangement
|
|
406
|
+
# of a unit's 1s equally likely, is -sum log C(T, k)
|
|
407
|
+
T, k = per["size"].to_numpy().astype(float), per["chosen"].to_numpy()
|
|
408
|
+
live = (k > 0) & (k < T)
|
|
409
|
+
wu = per["wcase"].to_numpy() if weights else np.ones(len(T))
|
|
410
|
+
lc = gammaln(T + 1) - gammaln(k + 1) - gammaln(T - k + 1)
|
|
411
|
+
cfe["units"] += int(live.sum())
|
|
412
|
+
cfe["rows"] += int(T[live].sum())
|
|
413
|
+
cfe["wunits"] += float(wu[live].sum())
|
|
414
|
+
cfe["loglik_null"] -= float((lc[live] * wu[live]).sum())
|
|
415
|
+
else:
|
|
416
|
+
bad = per.filter(pl.col("chosen") != 1)
|
|
417
|
+
bad_sets += bad.height
|
|
418
|
+
if len(bad_eg) < 3:
|
|
419
|
+
bad_eg += bad.select(skey).head(3 - len(bad_eg)).to_dicts()
|
|
420
|
+
for i in range(len(cluster_cols)):
|
|
421
|
+
sp = per.filter(pl.col(f"__ls_nu_{i}") > 1)
|
|
422
|
+
spread[i][0] += sp.height
|
|
423
|
+
if len(spread[i][1]) < 3:
|
|
424
|
+
spread[i][1] += sp.select(skey).head(3 - len(spread[i][1])).to_dicts()
|
|
425
|
+
del per
|
|
372
426
|
pstarts = None
|
|
373
427
|
if panel:
|
|
374
428
|
pstarts = np.append(np.flatnonzero(body.select(uchanged.alias("b"))["b"]
|
|
375
429
|
.to_numpy()), body.height).astype(np.int64)
|
|
376
430
|
convert(body, starts, pstarts)
|
|
431
|
+
|
|
432
|
+
if bad_sets:
|
|
433
|
+
raise ValueError(
|
|
434
|
+
f"{spec.fml}: every choice set must have exactly one chosen alternative "
|
|
435
|
+
f"(outcome 1); {bad_sets:,} of {n_cases:,} do not (e.g. {bad_eg}). Rows dropped "
|
|
436
|
+
"for missing values can remove a set's chosen alternative.")
|
|
437
|
+
if wspread:
|
|
438
|
+
raise ValueError("weights must be constant within each choice set (a choice set is "
|
|
439
|
+
"one observation)")
|
|
440
|
+
for c, (n_spread, eg) in zip(cluster_cols, spread):
|
|
441
|
+
if n_spread:
|
|
442
|
+
raise ValueError(f"cluster variable {c!r} must be constant within each choice set "
|
|
443
|
+
f"(a choice set is one observation); {n_spread:,} sets "
|
|
444
|
+
f"have more than one value (e.g. {eg})")
|
|
445
|
+
|
|
446
|
+
wcases = wsum_sets if weights else float(n_cases)
|
|
447
|
+
if weights and wscale is not None: # analytic weights: average 1 over the sets
|
|
448
|
+
wscale[0] = n_cases / wcases
|
|
449
|
+
wcases = float(n_cases)
|
|
450
|
+
if kind == "cfelogit":
|
|
451
|
+
scale = 1.0
|
|
452
|
+
if weights and wscale is not None: # average 1 over the informative units
|
|
453
|
+
scale = wscale[0] = cfe["units"] / cfe["wunits"]
|
|
454
|
+
info.update(units=cfe["units"], dropped=n_cases - cfe["units"], rows=cfe["rows"],
|
|
455
|
+
wunits=cfe["wunits"] * scale, loglik_null=cfe["loglik_null"] * scale)
|
|
456
|
+
if weights and wscale is not None and wscale[0] != 1.0:
|
|
457
|
+
_rescale_weights(convert.store, wscale[0])
|
|
377
458
|
return n_cases, wcases, n_panels
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _rescale_weights(store, factor: float) -> None:
|
|
462
|
+
"""Multiply the weights (field ``w``) of every block of `store` by
|
|
463
|
+
`factor`, in memory or in their files."""
|
|
464
|
+
for blk in store.blocks:
|
|
465
|
+
if store.in_memory:
|
|
466
|
+
if "w" in blk.arrays:
|
|
467
|
+
blk.arrays["w"] = blk.arrays["w"] * factor
|
|
468
|
+
elif "w" in blk.files:
|
|
469
|
+
np.save(blk.files["w"], np.load(blk.files["w"]) * factor)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: logit-stream
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects
|
|
5
5
|
Author: Lee Tucker
|
|
6
6
|
License-Expression: MIT
|
|
@@ -25,7 +25,7 @@ Classifier: Typing :: Typed
|
|
|
25
25
|
Requires-Python: >=3.10
|
|
26
26
|
Description-Content-Type: text/markdown
|
|
27
27
|
License-File: LICENSE.md
|
|
28
|
-
Requires-Dist: hdfe-stream[formula]>=0.
|
|
28
|
+
Requires-Dist: hdfe-stream[formula]>=0.9.0
|
|
29
29
|
Requires-Dist: stream-limits>=0.1.1
|
|
30
30
|
Requires-Dist: polars!=1.41.*,>=1.37
|
|
31
31
|
Requires-Dist: numpy>=1.24
|
|
@@ -49,6 +49,7 @@ Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
|
|
|
49
49
|
Requires-Dist: sphinx-design>=0.6; extra == "docs"
|
|
50
50
|
Provides-Extra: test
|
|
51
51
|
Requires-Dist: pytest>=7.0; extra == "test"
|
|
52
|
+
Requires-Dist: stream-limits[testing]>=0.2.1; extra == "test"
|
|
52
53
|
Requires-Dist: scikit-learn>=1.5; extra == "test"
|
|
53
54
|
Requires-Dist: statsmodels>=0.14; extra == "test"
|
|
54
55
|
Requires-Dist: pandas>=2.0; extra == "test"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
hdfe-stream[formula]>=0.
|
|
1
|
+
hdfe-stream[formula]>=0.9.0
|
|
2
2
|
stream-limits>=0.1.1
|
|
3
3
|
polars!=1.41.*,>=1.37
|
|
4
4
|
numpy>=1.24
|
|
@@ -25,6 +25,7 @@ sphinx-design>=0.6
|
|
|
25
25
|
|
|
26
26
|
[test]
|
|
27
27
|
pytest>=7.0
|
|
28
|
+
stream-limits[testing]>=0.2.1
|
|
28
29
|
scikit-learn>=1.5
|
|
29
30
|
statsmodels>=0.14
|
|
30
31
|
pandas>=2.0
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "logit-stream"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -30,7 +30,7 @@ requires-python = ">=3.10"
|
|
|
30
30
|
dependencies = [
|
|
31
31
|
# the streaming engine, the result base classes and the formula front end;
|
|
32
32
|
# binary logit is hdfe-stream's feglm_stream
|
|
33
|
-
"hdfe-stream[formula]>=0.
|
|
33
|
+
"hdfe-stream[formula]>=0.9.0", # 0.9.0: lower-memory Parquet batch reads (iter_groups)
|
|
34
34
|
"stream-limits>=0.1.1", # thread and memory limits for jobs on a scheduler (0.1.1: Windows memory)
|
|
35
35
|
"polars>=1.37,!=1.41.*",
|
|
36
36
|
"numpy>=1.24",
|
|
@@ -73,6 +73,7 @@ docs = [
|
|
|
73
73
|
# the test suite compares against scikit-learn and statsmodels
|
|
74
74
|
test = [
|
|
75
75
|
"pytest>=7.0",
|
|
76
|
+
"stream-limits[testing]>=0.2.1", # resource_runner, for tests/test_resources.py
|
|
76
77
|
"scikit-learn>=1.5",
|
|
77
78
|
"statsmodels>=0.14",
|
|
78
79
|
"pandas>=2.0",
|
|
@@ -86,6 +87,8 @@ include = ["logit_stream*"]
|
|
|
86
87
|
|
|
87
88
|
[tool.pytest.ini_options]
|
|
88
89
|
testpaths = ["tests"]
|
|
90
|
+
# memory tests (tests/test_resources.py) are slow: run them with `pytest -m resource`
|
|
91
|
+
addopts = "-m 'not resource'"
|
|
89
92
|
filterwarnings = [
|
|
90
93
|
"ignore::DeprecationWarning",
|
|
91
94
|
"ignore::FutureWarning",
|
|
@@ -29,6 +29,24 @@ def panel():
|
|
|
29
29
|
return _panel()
|
|
30
30
|
|
|
31
31
|
|
|
32
|
+
def test_analytic_weights_scale_free(panel):
|
|
33
|
+
"""Analytic weights average 1 over the informative units, so scaling
|
|
34
|
+
them changes nothing; and the units' sums, taken a block at a time, do
|
|
35
|
+
not depend on how the rows are divided into blocks."""
|
|
36
|
+
w = panel.with_columns(w=(pl.col("i") % 4 + 1).cast(pl.Float64))
|
|
37
|
+
w = w.with_columns(w10=10 * pl.col("w"))
|
|
38
|
+
a = cfelogit_stream("y ~ x1 + x2 | i", w, weights="w", verbose=False, tol=1e-14)
|
|
39
|
+
b = cfelogit_stream("y ~ x1 + x2 | i", w, weights="w10", verbose=False, tol=1e-14)
|
|
40
|
+
c = cfelogit_stream("y ~ x1 + x2 | i", w, weights="w", verbose=False, tol=1e-14,
|
|
41
|
+
block_mb=0.005)
|
|
42
|
+
assert c.diagnostics["store"]["blocks"] > 5
|
|
43
|
+
for r in (b, c):
|
|
44
|
+
np.testing.assert_allclose(r.beta, a.beta, rtol=1e-10)
|
|
45
|
+
np.testing.assert_allclose(r.se, a.se, rtol=1e-9)
|
|
46
|
+
assert r.loglik == pytest.approx(a.loglik, rel=1e-12)
|
|
47
|
+
assert r.loglik_null == pytest.approx(a.loglik_null, rel=1e-12)
|
|
48
|
+
|
|
49
|
+
|
|
32
50
|
def test_matches_statsmodels(panel):
|
|
33
51
|
cm = pytest.importorskip("statsmodels.discrete.conditional_models")
|
|
34
52
|
f = cfelogit_stream("y ~ x1 + x2 | i", panel, verbose=False, tol=1e-14)
|
|
@@ -91,6 +91,28 @@ def test_choice_set_validation(choice_data):
|
|
|
91
91
|
clogit_stream("chosen ~ cost", w, case="case", weights="w", verbose=False)
|
|
92
92
|
|
|
93
93
|
|
|
94
|
+
def test_sets_checked_and_weighted_block_by_block(choice_data):
|
|
95
|
+
"""The sets are checked and their weights summed a block at a time: the
|
|
96
|
+
error counts every bad set, in any block, and the analytic weights,
|
|
97
|
+
rescaled once the sum is known, give the same fit however the rows are
|
|
98
|
+
divided into blocks and wherever the blocks are kept."""
|
|
99
|
+
two = choice_data.with_columns(
|
|
100
|
+
pl.when(pl.col("case") % 100 == 0).then(1).otherwise(pl.col("chosen"))
|
|
101
|
+
.cast(pl.Int8).alias("chosen"))
|
|
102
|
+
n_bad = two.group_by("case").agg(pl.col("chosen").sum()).filter(pl.col("chosen") != 1).height
|
|
103
|
+
with pytest.raises(ValueError, match=f"{n_bad:,} of 3,000 do not"):
|
|
104
|
+
clogit_stream("chosen ~ cost", two, case="case", verbose=False, block_mb=0.02)
|
|
105
|
+
w = choice_data.with_columns((pl.col("case") % 7 + 1).cast(pl.Float64).alias("w"))
|
|
106
|
+
kw = dict(case="case", weights="w", verbose=False, tol=1e-14)
|
|
107
|
+
a = clogit_stream(FML, w, **kw)
|
|
108
|
+
for opts in (dict(block_mb=0.02), dict(storage="memory", block_mb=0.02)):
|
|
109
|
+
b = clogit_stream(FML, w.sample(fraction=1.0, shuffle=True, seed=3), **kw, **opts)
|
|
110
|
+
assert b.diagnostics["store"]["blocks"] > 5
|
|
111
|
+
np.testing.assert_allclose(b.beta, a.beta, rtol=1e-10)
|
|
112
|
+
np.testing.assert_allclose(b.se, a.se, rtol=1e-10)
|
|
113
|
+
assert b.loglik == pytest.approx(a.loglik, rel=1e-12)
|
|
114
|
+
|
|
115
|
+
|
|
94
116
|
def test_weights_are_per_choice_set(choice_data):
|
|
95
117
|
d = choice_data.filter(pl.col("case") < 1500)
|
|
96
118
|
w = d.with_columns((pl.col("case") % 3 + 1).cast(pl.Float64).alias("w"))
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Memory tests: the streamed fits stay inside a memory limit on data that
|
|
2
|
+
would not fit in it.
|
|
3
|
+
|
|
4
|
+
Run with ``pytest -m resource`` (the default run leaves them out: they take
|
|
5
|
+
about three minutes). Each fit runs in a child process under stream-limits'
|
|
6
|
+
``resource_runner``: a watchdog on every platform, or a kernel limit when
|
|
7
|
+
``STREAM_LIMITS_TEST_CGROUP_PARENT`` names a cgroup set up for it.
|
|
8
|
+
|
|
9
|
+
The binary logit is hdfe-stream's ``feglm_stream``, whose memory tests are
|
|
10
|
+
hdfe-stream's; these are this package's own models: the multinomial and
|
|
11
|
+
ordered logit with a fixed effect absorbed, and the conditional logit. With
|
|
12
|
+
``max_memory_gb=1.5`` the fits hold 0.7-1.2 GB of private memory (not
|
|
13
|
+
counting memory-mapped scratch files, which the kernel can reclaim), most of
|
|
14
|
+
it buffers whose sizes are capped. The limit is 1.6 GB. Measured on Linux
|
|
15
|
+
(WSL2, 2 threads):
|
|
16
|
+
|
|
17
|
+
- The multinomial and ordered logit, on 20 million people (1.05 GB in
|
|
18
|
+
memory): 0.94 and 0.73 GB, about the same as on 2.5 million. A fit that
|
|
19
|
+
held the data would need at least 1.8 GB.
|
|
20
|
+
- The conditional logit, on 8 million choice sets (21.6 million rows,
|
|
21
|
+
0.98 GB): 1.0-1.3 GB from run to run, about what it held on 4 million
|
|
22
|
+
before its pass 0 checked the sets a block at a time (1.17-1.22 GB). A
|
|
23
|
+
fit that held the data would need at least 2 GB (one stopped at 1.63).
|
|
24
|
+
|
|
25
|
+
(A smaller budget is not a smaller fit: the buffers are there before the
|
|
26
|
+
multinomial logit checks that the fixed effect's blocks fit, and with
|
|
27
|
+
``max_memory_gb=1.0`` it finds no room for them and stops.)
|
|
28
|
+
|
|
29
|
+
The limit is on private memory, which stream-limits measures on Linux and
|
|
30
|
+
Windows. On macOS it measures resident memory, which counts the mapped
|
|
31
|
+
scratch files, so there the limit is a looser safety net.
|
|
32
|
+
On Windows, private memory is committed memory: a buffer or an allocator
|
|
33
|
+
segment counts in full once allocated, touched or not (an hdfe-stream logit
|
|
34
|
+
fit had 1.61 GB committed while 1.14 GB was resident). Measured in CI
|
|
35
|
+
(GitHub's windows-latest, 2 threads): multinomial 1.70 GB, ordered 1.76 GB,
|
|
36
|
+
conditional 1.79 GB; the limit there is 2.1 GB, which a fit holding the data
|
|
37
|
+
(at least 2.7 GB) would still exceed.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import json
|
|
43
|
+
import sys
|
|
44
|
+
|
|
45
|
+
import numpy as np
|
|
46
|
+
import pytest
|
|
47
|
+
|
|
48
|
+
from logit_stream.simulate import RACES, simulate_choices, simulate_people, truth_people
|
|
49
|
+
|
|
50
|
+
pytestmark = pytest.mark.resource
|
|
51
|
+
|
|
52
|
+
N_PEOPLE = 20_000_000
|
|
53
|
+
N_CASES = 8_000_000 # 21.6 million rows
|
|
54
|
+
BUDGET_GB = 1.5 # the fit's max_memory_gb
|
|
55
|
+
LIMIT_GB = {"darwin": 2.6, "win32": 2.1}.get(sys.platform, 1.6)
|
|
56
|
+
X = "age10 + female + log_income + owner + urban"
|
|
57
|
+
XS = X.split(" + ")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@pytest.fixture(scope="module")
|
|
61
|
+
def people(tmp_path_factory):
|
|
62
|
+
path = tmp_path_factory.mktemp("big") / "people.parquet"
|
|
63
|
+
d = simulate_people(N_PEOPLE, seed=1)
|
|
64
|
+
d.write_parquet(path)
|
|
65
|
+
assert d.estimated_size() > 1.0e9
|
|
66
|
+
return path
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@pytest.fixture(scope="module")
|
|
70
|
+
def choices(tmp_path_factory):
|
|
71
|
+
path = tmp_path_factory.mktemp("big") / "choices.parquet"
|
|
72
|
+
d = simulate_choices(N_CASES, seed=1)
|
|
73
|
+
d.write_parquet(path)
|
|
74
|
+
assert d.estimated_size() > 0.95e9
|
|
75
|
+
return path
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _fit_in_child(resource_runner, call: str, path, workdir) -> dict:
|
|
79
|
+
code = f"""
|
|
80
|
+
import json
|
|
81
|
+
from logit_stream import clogit_stream, mlogit_stream, ologit_stream
|
|
82
|
+
r = {call}
|
|
83
|
+
print(json.dumps(dict(converged=bool(r.converged),
|
|
84
|
+
coef={{k: float(v) for k, v in r.coef().items()}})))
|
|
85
|
+
"""
|
|
86
|
+
code = code.replace("SRC", repr(str(path))).replace("WORK", repr(str(workdir)))
|
|
87
|
+
result = resource_runner([sys.executable, "-c", code], memory_gb=LIMIT_GB,
|
|
88
|
+
threads=2, timeout=900)
|
|
89
|
+
report = {k: result.as_dict()[k] for k in ("status", "mode", "memory_measure",
|
|
90
|
+
"peak_private_bytes", "peak_rss_bytes",
|
|
91
|
+
"elapsed_s")}
|
|
92
|
+
print(json.dumps(report))
|
|
93
|
+
assert result.status == "ok", (report, result.stderr[-3000:])
|
|
94
|
+
if result.kernel:
|
|
95
|
+
assert result.kernel["events"].get("oom_kill", 0) == 0
|
|
96
|
+
out = json.loads(result.stdout.strip().splitlines()[-1])
|
|
97
|
+
assert out["converged"]
|
|
98
|
+
return out["coef"]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
# The checks against the simulated coefficients are sanity checks only:
|
|
102
|
+
# agreement with statsmodels and scikit-learn is the rest of the suite's job.
|
|
103
|
+
|
|
104
|
+
def test_multinomial_logit_streams_inside_the_limit(resource_runner, people, tmp_path):
|
|
105
|
+
coef = _fit_in_child(
|
|
106
|
+
resource_runner,
|
|
107
|
+
f"mlogit_stream('race ~ {X} | puma', SRC, workdir=WORK, verbose=False, "
|
|
108
|
+
f"max_memory_gb={BUDGET_GB})",
|
|
109
|
+
people, tmp_path / "work")
|
|
110
|
+
# the base level is the first in sort order (AIAN); the truth's is White
|
|
111
|
+
truth = truth_people()["race"]
|
|
112
|
+
for race in RACES[1:]:
|
|
113
|
+
for x in XS:
|
|
114
|
+
fit = coef.get(f"{race}:{x}", 0.0) - coef[f"White:{x}"]
|
|
115
|
+
assert abs(fit - truth[f"{race}:{x}"]) < 0.1, (race, x, fit)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_ordered_logit_streams_inside_the_limit(resource_runner, people, tmp_path):
|
|
119
|
+
coef = _fit_in_child(
|
|
120
|
+
resource_runner,
|
|
121
|
+
f"ologit_stream('education ~ {X} | puma', SRC, workdir=WORK, verbose=False, "
|
|
122
|
+
f"max_memory_gb={BUDGET_GB})",
|
|
123
|
+
people, tmp_path / "work")
|
|
124
|
+
truth = truth_people()["education"]
|
|
125
|
+
np.testing.assert_allclose([coef[x] for x in XS], [truth[x] for x in XS], atol=0.03)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def test_conditional_logit_streams_inside_the_limit(resource_runner, choices, tmp_path):
|
|
129
|
+
coef = _fit_in_child(
|
|
130
|
+
resource_runner,
|
|
131
|
+
"clogit_stream('chosen ~ cost + time + C(alt) + income:C(alt)', SRC, case='case', "
|
|
132
|
+
f"workdir=WORK, verbose=False, max_memory_gb={BUDGET_GB})",
|
|
133
|
+
choices, tmp_path / "work")
|
|
134
|
+
# simulate_choices' defaults
|
|
135
|
+
np.testing.assert_allclose([coef["cost"], coef["time"]], [-1.1, -1.3], atol=0.03)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|