logit-stream 0.2.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {logit_stream-0.2.0 → logit_stream-0.3.1}/CHANGELOG.md +25 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/CITATION.cff +2 -2
- {logit_stream-0.2.0 → logit_stream-0.3.1}/PKG-INFO +4 -6
- {logit_stream-0.2.0 → logit_stream-0.3.1}/README.md +1 -4
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/__init__.py +1 -1
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/_frontend.py +9 -10
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/models.py +225 -63
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/newton.py +12 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/store.py +150 -58
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream.egg-info/PKG-INFO +4 -6
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream.egg-info/SOURCES.txt +1 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream.egg-info/requires.txt +2 -1
- {logit_stream-0.2.0 → logit_stream-0.3.1}/pyproject.toml +5 -2
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_cfelogit.py +18 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_conditional.py +22 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_multinomial.py +4 -2
- logit_stream-0.3.1/tests/test_resources.py +135 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/LICENSE.md +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/MANIFEST.in +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/_choice.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/_limits.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/api.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/__init__.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/_common.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/fe.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/meat.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/mixed.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/models.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/nested.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/random_effects.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/device/store.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/estimator.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/fe.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/mixed.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/multi_fe.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/nested.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/optimize.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/profile.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/py.typed +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/random_effects/__init__.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/random_effects/fit.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/random_effects/linalg.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/random_effects/model.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/random_effects/sigma.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/results.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream/simulate.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream.egg-info/dependency_links.txt +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/logit_stream.egg-info/top_level.txt +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/setup.cfg +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_api.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_clustering.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_device.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_firth_and_fit.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_fixed_effects.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_headroom.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_index_dtype.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_kernels.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_mixed.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_multi_fe.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_nested.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_offset_start.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_ordered.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_probit.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_profile.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_random.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_simulate.py +0 -0
- {logit_stream-0.2.0 → logit_stream-0.3.1}/tests/test_weights_scale.py +0 -0
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.1
|
|
4
|
+
|
|
5
|
+
- **Works with pyfixest 0.60 and 0.70.** Formulas are read through hdfe-stream's own formula
|
|
6
|
+
parser, so either pyfixest version can be installed.
|
|
7
|
+
- Requires hdfe-stream 0.10.0 or later.
|
|
8
|
+
|
|
9
|
+
## 0.3.0
|
|
10
|
+
|
|
11
|
+
- **The conditional logit is 2.3-2.8 times faster.** Its log-likelihood, gradient and information
|
|
12
|
+
are computed in one parallel numba pass over each block's choice sets, giving the same
|
|
13
|
+
estimates (4 million sets: 13 s, now 5 s; 8 million: 26 s, now 9-12 s). Designs of more than
|
|
14
|
+
256 parameters keep the previous BLAS-based pass, which is faster there.
|
|
15
|
+
- **The fixed-effects logit (`cfelogit_stream`) balances work across threads better.** Panels
|
|
16
|
+
are dealt out in small pieces to more parts than threads, so a run of long panels no longer
|
|
17
|
+
leaves threads idle: 0.55 of the time on panels sorted by length, 0.8-1.0 on shuffled
|
|
18
|
+
ones, with the same results.
|
|
19
|
+
- **Lower memory for conditional, mixed and fixed-effects logit.** Choice sets are now checked
|
|
20
|
+
and counted a block at a time instead of from a table of every set, so memory no longer grows
|
|
21
|
+
with the number of sets (8 million sets: peak 1.71 GB down to 1.33 GB). Same errors, same
|
|
22
|
+
results.
|
|
23
|
+
- **Memory tests.** `pytest -m resource` fits the multinomial, ordered and conditional logit
|
|
24
|
+
on data larger than a fixed memory limit; CI runs them on Linux, macOS and Windows. The
|
|
25
|
+
default `pytest` run leaves them out.
|
|
26
|
+
- Requires hdfe-stream 0.9.0 or later, which reads Parquet in less memory.
|
|
27
|
+
|
|
3
28
|
## 0.2.0
|
|
4
29
|
|
|
5
30
|
- **Fit resources (CPUs and memory) are constrained to your job's limits automatically.** On a
|
|
@@ -7,8 +7,8 @@ authors:
|
|
|
7
7
|
- family-names: Tucker
|
|
8
8
|
given-names: Lee
|
|
9
9
|
email: lee.c.tucker@census.gov
|
|
10
|
-
version: 0.
|
|
11
|
-
date-released: 2026-
|
|
10
|
+
version: 0.3.1
|
|
11
|
+
date-released: 2026-11-09
|
|
12
12
|
license: MIT
|
|
13
13
|
repository-code: "https://github.com/leetucker/logit-stream"
|
|
14
14
|
url: "https://github.com/leetucker/logit-stream"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: logit-stream
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Out-of-core logit models by streamed Newton: binary, multinomial, ordered and conditional logit, with absorbed fixed effects
|
|
5
5
|
Author: Lee Tucker
|
|
6
6
|
License-Expression: MIT
|
|
@@ -25,7 +25,7 @@ Classifier: Typing :: Typed
|
|
|
25
25
|
Requires-Python: >=3.10
|
|
26
26
|
Description-Content-Type: text/markdown
|
|
27
27
|
License-File: LICENSE.md
|
|
28
|
-
Requires-Dist: hdfe-stream[formula]>=0.
|
|
28
|
+
Requires-Dist: hdfe-stream[formula]>=0.10.0
|
|
29
29
|
Requires-Dist: stream-limits>=0.1.1
|
|
30
30
|
Requires-Dist: polars!=1.41.*,>=1.37
|
|
31
31
|
Requires-Dist: numpy>=1.24
|
|
@@ -49,6 +49,7 @@ Requires-Dist: sphinx-copybutton>=0.5; extra == "docs"
|
|
|
49
49
|
Requires-Dist: sphinx-design>=0.6; extra == "docs"
|
|
50
50
|
Provides-Extra: test
|
|
51
51
|
Requires-Dist: pytest>=7.0; extra == "test"
|
|
52
|
+
Requires-Dist: stream-limits[testing]>=0.2.1; extra == "test"
|
|
52
53
|
Requires-Dist: scikit-learn>=1.5; extra == "test"
|
|
53
54
|
Requires-Dist: statsmodels>=0.14; extra == "test"
|
|
54
55
|
Requires-Dist: pandas>=2.0; extra == "test"
|
|
@@ -108,10 +109,7 @@ data.
|
|
|
108
109
|
pip install logit-stream
|
|
109
110
|
```
|
|
110
111
|
|
|
111
|
-
Python 3.10 or later.
|
|
112
|
-
formula extra brings in) has no wheels yet: it builds from source with a Rust
|
|
113
|
-
toolchain and `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` set; everything else
|
|
114
|
-
installs from wheels.
|
|
112
|
+
Python 3.10 or later.
|
|
115
113
|
|
|
116
114
|
## Benchmarks
|
|
117
115
|
|
|
@@ -52,10 +52,7 @@ data.
|
|
|
52
52
|
pip install logit-stream
|
|
53
53
|
```
|
|
54
54
|
|
|
55
|
-
Python 3.10 or later.
|
|
56
|
-
formula extra brings in) has no wheels yet: it builds from source with a Rust
|
|
57
|
-
toolchain and `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1` set; everything else
|
|
58
|
-
installs from wheels.
|
|
55
|
+
Python 3.10 or later.
|
|
59
56
|
|
|
60
57
|
## Benchmarks
|
|
61
58
|
|
|
@@ -19,7 +19,7 @@ from hdfe_stream.core.formula import (
|
|
|
19
19
|
categorical_vars,
|
|
20
20
|
compile_design,
|
|
21
21
|
numeric_expr,
|
|
22
|
-
|
|
22
|
+
parse_formula,
|
|
23
23
|
)
|
|
24
24
|
|
|
25
25
|
|
|
@@ -52,27 +52,26 @@ def _plan(fml: str, lf: pl.LazyFrame, intercept: bool, collect) -> list[_Spec]:
|
|
|
52
52
|
categorical coding is still reduced rank, as with it.
|
|
53
53
|
`collect(lf, what)` runs a query (so that the GPU engine applies).
|
|
54
54
|
"""
|
|
55
|
-
Formula, _ = pyfixest_formula_api()
|
|
56
55
|
schema = lf.collect_schema()
|
|
57
|
-
specs =
|
|
56
|
+
specs = parse_formula(fml)
|
|
58
57
|
cats = set()
|
|
59
58
|
for s in specs:
|
|
60
|
-
if s.
|
|
61
|
-
raise ValueError(f"{s.
|
|
62
|
-
cats |= categorical_vars(s.
|
|
59
|
+
if s.iv is not None:
|
|
60
|
+
raise ValueError(f"{s.key}: IV formulas are not available for logit models")
|
|
61
|
+
cats |= categorical_vars(s.rhs, schema)
|
|
63
62
|
levels = {}
|
|
64
63
|
for v in sorted(cats):
|
|
65
64
|
u = collect(lf.select(pl.col(v).drop_nulls().unique().sort()), f"levels of {v}")
|
|
66
65
|
levels[v] = u[v].to_list()
|
|
67
66
|
out = []
|
|
68
67
|
for s in specs:
|
|
69
|
-
dep, rhs =
|
|
70
|
-
fe = tuple(parse_fe(s.
|
|
68
|
+
dep, rhs = s.dep, s.rhs
|
|
69
|
+
fe = tuple(parse_fe(s.fe))
|
|
71
70
|
for d in fe:
|
|
72
71
|
if "[" in d:
|
|
73
|
-
raise ValueError(f"{s.
|
|
72
|
+
raise ValueError(f"{s.key}: varying slopes are not available for logit models")
|
|
74
73
|
_, cols = compile_design(dep, rhs, schema, levels, intercept=intercept and not fe)
|
|
75
74
|
if not intercept:
|
|
76
75
|
cols = [(nm, ex) for nm, ex in cols if nm != "Intercept"]
|
|
77
|
-
out.append(_Spec(fml=s.
|
|
76
|
+
out.append(_Spec(fml=s.key, depvar=dep, y=_outcome_expr(dep, schema), x=cols, fe=fe))
|
|
78
77
|
return out
|
|
@@ -37,6 +37,31 @@ from scipy.special import expit, ndtr, ndtri
|
|
|
37
37
|
# average, and the parameters few enough for a per-thread accumulator each
|
|
38
38
|
SPARSE_DENSITY = 0.5
|
|
39
39
|
SPARSE_MAX_PARAMS = 1200
|
|
40
|
+
# The conditional and fixed-effects logits' numba passes (`_clogit_pass`,
|
|
41
|
+
# `_cfe_units`) split a block's choice sets or units into this many parts per
|
|
42
|
+
# thread, handed out as threads come free (`_parts`), and keep the parts'
|
|
43
|
+
# information accumulators together to at most PARTS_MB (but at least one
|
|
44
|
+
# part per thread). The conditional logit's is used up to CLOGIT_KERNEL_MAX_P
|
|
45
|
+
# parameters: it adds each
|
|
46
|
+
# row into a P x P accumulator per thread, which is fast while that fits in a
|
|
47
|
+
# core's L2 cache (0.5 MB at 256; current cores have 1-2.5 MB) and several
|
|
48
|
+
# times slower once it does not, where BLAS's blocked products are not
|
|
49
|
+
# (measured, 4 threads: 0.17 of the NumPy pass's time at P = 128, 0.43 at
|
|
50
|
+
# 256, 0.7-0.85 at 384-512, 2.3 at 768 with a 2.5 MB L2). Above it the pass
|
|
51
|
+
# is NumPy's (`_Conditional._accumulate_blas`).
|
|
52
|
+
PARTS_PER_THREAD = 4
|
|
53
|
+
PARTS_MB = 64.0
|
|
54
|
+
# the fixed-effects logit's parts take runs of this many consecutive units
|
|
55
|
+
CFE_RUN = 64
|
|
56
|
+
CLOGIT_KERNEL_MAX_P = 256
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _parts(n_items: int, P: int) -> int:
|
|
60
|
+
"""How many parts to split `n_items` sets or units into, for a pass
|
|
61
|
+
whose parts each keep a P x P accumulator (see PARTS_PER_THREAD)."""
|
|
62
|
+
threads = nb.get_num_threads()
|
|
63
|
+
cap = max(threads, int(PARTS_MB * 1e6 // (8 * max(P * P, 1))))
|
|
64
|
+
return max(1, min(n_items, PARTS_PER_THREAD * threads, cap))
|
|
40
65
|
|
|
41
66
|
|
|
42
67
|
def _gram(X, v):
|
|
@@ -91,6 +116,93 @@ def _mnl_info_sparse(X, P, w, n_threads, out):
|
|
|
91
116
|
out += acc[t]
|
|
92
117
|
|
|
93
118
|
|
|
119
|
+
@nb.njit(parallel=True, cache=True)
|
|
120
|
+
def _clogit_pass(X, cols, theta, y, w, off, starts, hessian, max_size, ll_t, g_t, H_t):
|
|
121
|
+
"""The conditional logit's log-likelihood, gradient and information over
|
|
122
|
+
the choice sets of a block, in one pass over its rows: the sets are
|
|
123
|
+
split into ``len(ll_t)`` parts, and part t's sums are written to
|
|
124
|
+
``ll_t[t]``, ``g_t[t]`` and (with `hessian`) ``H_t[t]``, for the caller
|
|
125
|
+
to add up (`_Conditional.accumulate_full`). `cols` are the columns of
|
|
126
|
+
`X` in the model; `w` and `off` are empty when there are no weights or
|
|
127
|
+
offset.
|
|
128
|
+
|
|
129
|
+
The only parallel loop is the one over the parts: the accumulators are
|
|
130
|
+
allocated and summed by the caller, since with `parallel_chunksize`
|
|
131
|
+
numba would hand out every element of an array operation here as a
|
|
132
|
+
chunk of its own.
|
|
133
|
+
|
|
134
|
+
Per set: p = softmax(eta), the log-likelihood eta_chosen - log sum
|
|
135
|
+
exp(eta), the gradient sum (y - p) x, and the information
|
|
136
|
+
sum p (x - xbar)(x - xbar)' with xbar = sum p x (the within-set
|
|
137
|
+
covariance, centred before the products: the same matrix as
|
|
138
|
+
sum p x x' - xbar xbar', without the cancellation). Each is weighted by
|
|
139
|
+
the set's weight. Only the upper triangle of the information is filled."""
|
|
140
|
+
n_sets = len(starts) - 1
|
|
141
|
+
P = len(cols)
|
|
142
|
+
n_parts = len(ll_t)
|
|
143
|
+
has_w, has_off = len(w) > 0, len(off) > 0
|
|
144
|
+
step = (n_sets + n_parts - 1) // n_parts
|
|
145
|
+
for t in nb.prange(n_parts):
|
|
146
|
+
# local accumulators, copied out at the end: rows of the shared
|
|
147
|
+
# arrays would share cache lines between threads
|
|
148
|
+
eta = np.empty(max_size)
|
|
149
|
+
xs = np.empty((max_size, P)) # the set's rows, read once
|
|
150
|
+
xbar = np.empty(P)
|
|
151
|
+
d = np.empty(P)
|
|
152
|
+
g = np.zeros(P)
|
|
153
|
+
Ht = np.zeros((P, P) if hessian else (0, 0))
|
|
154
|
+
llt = 0.0
|
|
155
|
+
for s in range(t * step, min(n_sets, (t + 1) * step)):
|
|
156
|
+
a, b = starts[s], starts[s + 1]
|
|
157
|
+
ws = w[a] if has_w else 1.0
|
|
158
|
+
m = -np.inf
|
|
159
|
+
chosen = 0.0
|
|
160
|
+
for i in range(a, b):
|
|
161
|
+
e = off[i] if has_off else 0.0
|
|
162
|
+
for k in range(P):
|
|
163
|
+
x = X[i, cols[k]]
|
|
164
|
+
xs[i - a, k] = x
|
|
165
|
+
e += x * theta[k]
|
|
166
|
+
eta[i - a] = e
|
|
167
|
+
if y[i] == 1:
|
|
168
|
+
chosen += e
|
|
169
|
+
if e > m:
|
|
170
|
+
m = e
|
|
171
|
+
S = 0.0
|
|
172
|
+
for i in range(a, b):
|
|
173
|
+
eta[i - a] = np.exp(eta[i - a] - m)
|
|
174
|
+
S += eta[i - a]
|
|
175
|
+
llt += ws * (chosen - m - np.log(S))
|
|
176
|
+
for k in range(P):
|
|
177
|
+
xbar[k] = 0.0
|
|
178
|
+
for i in range(a, b):
|
|
179
|
+
p = eta[i - a] / S
|
|
180
|
+
eta[i - a] = p
|
|
181
|
+
r = ws * ((1.0 if y[i] == 1 else 0.0) - p)
|
|
182
|
+
for k in range(P):
|
|
183
|
+
x = xs[i - a, k]
|
|
184
|
+
g[k] += r * x
|
|
185
|
+
xbar[k] += p * x
|
|
186
|
+
if hessian:
|
|
187
|
+
for i in range(a, b):
|
|
188
|
+
p = ws * eta[i - a]
|
|
189
|
+
for k in range(P):
|
|
190
|
+
d[k] = xs[i - a, k] - xbar[k]
|
|
191
|
+
# the full square (a fixed-length inner loop, which
|
|
192
|
+
# vectorizes), though only the upper triangle is used
|
|
193
|
+
for k in range(P):
|
|
194
|
+
v = p * d[k]
|
|
195
|
+
for l in range(P):
|
|
196
|
+
Ht[k, l] += v * d[l]
|
|
197
|
+
ll_t[t] = llt
|
|
198
|
+
for k in range(P):
|
|
199
|
+
g_t[t, k] = g[k]
|
|
200
|
+
if hessian:
|
|
201
|
+
for k in range(P):
|
|
202
|
+
for l in range(P):
|
|
203
|
+
H_t[t, k, l] = Ht[k, l]
|
|
204
|
+
|
|
205
|
+
|
|
94
206
|
def _bincount(idx, weights, n):
|
|
95
207
|
return np.bincount(idx, weights=weights, minlength=n)[:n]
|
|
96
208
|
|
|
@@ -530,6 +642,43 @@ class _Conditional(_Model):
|
|
|
530
642
|
return e / np.repeat(S, sizes), m, S
|
|
531
643
|
|
|
532
644
|
def accumulate(self, theta, X, y, w, starts, hessian, offset=None):
|
|
645
|
+
return self.accumulate_full(theta, X, np.arange(X.shape[1]), y, w, starts, hessian,
|
|
646
|
+
offset)
|
|
647
|
+
|
|
648
|
+
def accumulate_full(self, theta, X, cols, y, w, starts, hessian, offset=None):
|
|
649
|
+
"""`accumulate` on the columns `cols` of `X`: a whole block, by
|
|
650
|
+
numba's pass, which holds no row-sized temporaries and reads the
|
|
651
|
+
columns in place rather than copying them; above
|
|
652
|
+
CLOGIT_KERNEL_MAX_P parameters, by NumPy's."""
|
|
653
|
+
P = len(cols)
|
|
654
|
+
if P > CLOGIT_KERNEL_MAX_P:
|
|
655
|
+
Xc = X if P == X.shape[1] and np.array_equal(cols, np.arange(P)) else X[:, cols]
|
|
656
|
+
return self._accumulate_blas(theta, Xc, y, w, starts, hessian, offset)
|
|
657
|
+
empty = np.empty(0)
|
|
658
|
+
starts = np.asarray(starts, np.int64)
|
|
659
|
+
# parts handed out one at a time as threads come free, so that a
|
|
660
|
+
# thread the machine slows (another process, the store's reader, a
|
|
661
|
+
# slower core) does not hold up the rest; their accumulators are
|
|
662
|
+
# capped together, at one part per thread
|
|
663
|
+
n_parts = _parts(len(starts) - 1, P)
|
|
664
|
+
ll_t, g_t = np.zeros(n_parts), np.zeros((n_parts, P))
|
|
665
|
+
H_t = np.zeros((n_parts, P, P) if hessian else (n_parts, 0, 0))
|
|
666
|
+
with nb.parallel_chunksize(1):
|
|
667
|
+
_clogit_pass(
|
|
668
|
+
np.ascontiguousarray(X, np.float64), np.asarray(cols, np.int64),
|
|
669
|
+
np.ascontiguousarray(theta, np.float64), y,
|
|
670
|
+
empty if w is None else np.ascontiguousarray(w, np.float64),
|
|
671
|
+
empty if offset is None else np.ascontiguousarray(offset, np.float64),
|
|
672
|
+
starts, bool(hessian), int(np.diff(starts).max()) if len(starts) > 1 else 1,
|
|
673
|
+
ll_t, g_t, H_t)
|
|
674
|
+
ll, g, H = ll_t.sum(), g_t.sum(axis=0), H_t.sum(axis=0)
|
|
675
|
+
if not hessian:
|
|
676
|
+
return float(ll), g, None
|
|
677
|
+
return float(ll), g, np.triu(H) + np.triu(H, 1).T
|
|
678
|
+
|
|
679
|
+
def _accumulate_blas(self, theta, X, y, w, starts, hessian, offset=None):
|
|
680
|
+
"""`accumulate` by NumPy array operations, the information by BLAS's
|
|
681
|
+
Gram products: for a wide design (`CLOGIT_KERNEL_MAX_P`)."""
|
|
533
682
|
st = starts[:-1]
|
|
534
683
|
eta = X @ theta
|
|
535
684
|
if offset is not None:
|
|
@@ -560,7 +709,7 @@ class _Conditional(_Model):
|
|
|
560
709
|
|
|
561
710
|
# ------------------------------------------------------------------ conditional FE logit
|
|
562
711
|
@nb.njit(parallel=True, cache=True)
|
|
563
|
-
def _cfe_units(X, eta, y, w, starts, hessian,
|
|
712
|
+
def _cfe_units(X, eta, y, w, starts, hessian, ll_u, g_u, acc):
|
|
564
713
|
"""Chamberlain's conditional likelihood, unit by unit (the rows of unit u
|
|
565
714
|
are starts[u]:starts[u+1]): with k the unit's number of 1s among its T
|
|
566
715
|
rows, log P(y | k) = sum y eta - log e_k(exp(eta)), e_k the elementary
|
|
@@ -573,68 +722,81 @@ def _cfe_units(X, eta, y, w, starts, hessian, n_threads, ll_u, g_u, out):
|
|
|
573
722
|
unit's largest, and a unit with k > T / 2 is fitted as its complement
|
|
574
723
|
(1 - y, -x, -eta), so that k <= T / 2. Units with k = 0 or k = T carry no
|
|
575
724
|
information and are skipped. Writes each unit's (weighted) log-likelihood
|
|
576
|
-
and gradient; the information is summed
|
|
577
|
-
|
|
725
|
+
and gradient; the information is summed over ``len(acc)`` parts of the
|
|
726
|
+
units, part t's into ``acc[t]``, for the caller to add up.
|
|
727
|
+
|
|
728
|
+
A unit's work grows with T k P^2, so units differ in cost: the parts are
|
|
729
|
+
more than the threads, handed out as threads come free (`_CFELogit._units`
|
|
730
|
+
runs this under ``parallel_chunksize(1)``), and each takes every
|
|
731
|
+
``len(acc)``-th run of CFE_RUN units, so that each has a mix of short and
|
|
732
|
+
long panels. The only parallel loop is the
|
|
733
|
+
one over the parts, so the accumulators are allocated and summed by the
|
|
734
|
+
caller (numba would otherwise hand out every element of an array
|
|
735
|
+
operation here as a chunk of its own)."""
|
|
578
736
|
U, P = len(starts) - 1, X.shape[1]
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
# (a thread's own accumulator, copied out at the end: the threads'
|
|
737
|
+
n_parts = len(acc)
|
|
738
|
+
for th in nb.prange(n_parts):
|
|
739
|
+
# (a part's own accumulator, copied out at the end: the parts'
|
|
583
740
|
# slices of `acc` are too close together to add into unit by unit)
|
|
584
741
|
a_ = np.zeros((P, P))
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
if
|
|
742
|
+
# runs of CFE_RUN units, every n_parts-th run: data in order of panel
|
|
743
|
+
# length (a cohort, an entry date) would otherwise put the longest,
|
|
744
|
+
# costliest units together in the last parts, while single units
|
|
745
|
+
# dealt out would scatter the reads
|
|
746
|
+
for u0 in range(th * CFE_RUN, U, n_parts * CFE_RUN):
|
|
747
|
+
for u in range(u0, min(U, u0 + CFE_RUN)):
|
|
748
|
+
r0, r1 = starts[u], starts[u + 1]
|
|
749
|
+
T = r1 - r0
|
|
750
|
+
k = 0
|
|
751
|
+
for r in range(r0, r1):
|
|
752
|
+
k += 1 if y[r] > 0 else 0
|
|
753
|
+
ll_u[u] = 0.0
|
|
754
|
+
for p in range(P):
|
|
755
|
+
g_u[u, p] = 0.0
|
|
756
|
+
if k == 0 or k == T:
|
|
757
|
+
continue
|
|
758
|
+
s = 1.0
|
|
759
|
+
if 2 * k > T:
|
|
760
|
+
s, k = -1.0, T - k
|
|
761
|
+
c = -np.inf
|
|
762
|
+
for r in range(r0, r1):
|
|
763
|
+
c = max(c, s * eta[r])
|
|
764
|
+
e = np.zeros(k + 1)
|
|
765
|
+
G = np.zeros((k + 1, P))
|
|
766
|
+
Hm = np.zeros((k + 1, P, P)) if hessian else np.zeros((1, 1, 1))
|
|
767
|
+
e[0] = 1.0
|
|
768
|
+
obs = 0.0
|
|
769
|
+
for r in range(r0, r1):
|
|
770
|
+
z = np.exp(s * eta[r] - c)
|
|
771
|
+
yr = (y[r] > 0) if s > 0 else (y[r] <= 0)
|
|
772
|
+
if yr:
|
|
773
|
+
obs += s * eta[r]
|
|
616
774
|
for p in range(P):
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
775
|
+
g_u[u, p] += s * X[r, p]
|
|
776
|
+
for j in range(k, 0, -1):
|
|
777
|
+
if hessian:
|
|
778
|
+
for p in range(P):
|
|
779
|
+
xp = s * X[r, p]
|
|
780
|
+
for q in range(P):
|
|
781
|
+
xq = s * X[r, q]
|
|
782
|
+
Hm[j, p, q] += z * (Hm[j - 1, p, q] + xp * G[j - 1, q]
|
|
783
|
+
+ G[j - 1, p] * xq + xp * xq * e[j - 1])
|
|
784
|
+
for p in range(P):
|
|
785
|
+
G[j, p] += z * (G[j - 1, p] + s * X[r, p] * e[j - 1])
|
|
786
|
+
e[j] += z * e[j - 1]
|
|
787
|
+
wu = w[r0]
|
|
788
|
+
ll_u[u] = wu * (obs - np.log(e[k]) - k * c)
|
|
631
789
|
for p in range(P):
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
790
|
+
m = G[k, p] / e[k]
|
|
791
|
+
g_u[u, p] = wu * (g_u[u, p] - m)
|
|
792
|
+
if hessian:
|
|
793
|
+
for p in range(P):
|
|
794
|
+
mp = G[k, p] / e[k]
|
|
795
|
+
for q in range(P):
|
|
796
|
+
a_[p, q] += wu * (Hm[k, p, q] / e[k] - mp * G[k, q] / e[k])
|
|
797
|
+
for p in range(P):
|
|
798
|
+
for q in range(P):
|
|
799
|
+
acc[th, p, q] = a_[p, q]
|
|
638
800
|
|
|
639
801
|
|
|
640
802
|
class _CFELogit(_Model):
|
|
@@ -673,12 +835,12 @@ class _CFELogit(_Model):
|
|
|
673
835
|
eta = eta + offset
|
|
674
836
|
U, P = len(starts) - 1, X.shape[1]
|
|
675
837
|
ll_u, g_u = np.empty(U), np.empty((U, P))
|
|
676
|
-
|
|
838
|
+
acc = np.zeros((_parts(U, P), P, P))
|
|
677
839
|
ww = np.ones(len(y)) if w is None else np.ascontiguousarray(w, dtype=np.float64)
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
return ll_u, g_u,
|
|
840
|
+
with nb.parallel_chunksize(1):
|
|
841
|
+
_cfe_units(np.ascontiguousarray(X), np.ascontiguousarray(eta), y, ww,
|
|
842
|
+
np.ascontiguousarray(starts, dtype=np.int64), hessian, ll_u, g_u, acc)
|
|
843
|
+
return ll_u, g_u, acc.sum(axis=0)
|
|
682
844
|
|
|
683
845
|
def accumulate(self, theta, X, y, w, starts, hessian, offset=None):
|
|
684
846
|
ll_u, g_u, H = self._units(theta, X, y, w, starts, hessian, offset)
|
|
@@ -129,7 +129,19 @@ class _Driver:
|
|
|
129
129
|
m = self.model
|
|
130
130
|
ll, g, H = 0.0, np.zeros(m.P), (np.zeros((m.P, m.P)) if hessian else None)
|
|
131
131
|
rows = self._rows()
|
|
132
|
+
full = getattr(m, "accumulate_full", None)
|
|
132
133
|
for blk in self.store:
|
|
134
|
+
if full is not None and firth_A is None:
|
|
135
|
+
# a numba pass over the whole block, on its columns in place
|
|
136
|
+
l, gg, hh = full(theta, blk["X"], m.cols, blk["y"], blk.get("w"),
|
|
137
|
+
blk.get("starts"), hessian, offset=blk.get("off"))
|
|
138
|
+
if not np.isfinite(l):
|
|
139
|
+
return -np.inf, None, None
|
|
140
|
+
ll += l
|
|
141
|
+
g += gg
|
|
142
|
+
if hessian:
|
|
143
|
+
H += hh
|
|
144
|
+
continue
|
|
133
145
|
for ch in _chunks(blk, rows):
|
|
134
146
|
X = _design(ch, m.cols, self.K0)
|
|
135
147
|
l, gg, hh = m.accumulate(theta, X, ch["y"], ch.get("w"), ch.get("starts"),
|