crispyx 0.2.3__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {crispyx-0.2.3/src/crispyx.egg-info → crispyx-0.2.4}/PKG-INFO +2 -2
- {crispyx-0.2.3 → crispyx-0.2.4}/pyproject.toml +1 -1
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_irls.py +19 -6
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_kernels.py +327 -9
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_namespaces.py +4 -3
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_version.py +1 -1
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/batch.py +3 -3
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/data.py +38 -1
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/de.py +83 -76
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/glm.py +203 -54
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/qc.py +2 -2
- {crispyx-0.2.3 → crispyx-0.2.4/src/crispyx.egg-info}/PKG-INFO +2 -2
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx.egg-info/requires.txt +1 -1
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_batch_process.py +26 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_glm_structured.py +50 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_nb_glm.py +20 -7
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_preflight_dispatch.py +8 -1
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_read_backed.py +21 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_two_group_deseq2.py +85 -2
- {crispyx-0.2.3 → crispyx-0.2.4}/LICENSE +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/README.md +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/setup.cfg +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/__init__.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_checkpoint.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_disk.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_grouping.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_h5codec.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_memory.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_messages.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_preflight.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_provenance.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_size_factors.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_statistics.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/dimred.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/hvg.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/plotting.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/profiling.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/pseudobulk.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/residual.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/sample.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx.egg-info/SOURCES.txt +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx.egg-info/dependency_links.txt +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx.egg-info/top_level.txt +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_cgroup_memory_budget.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_compare_controls.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_compress_h5ad.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_concat_h5ad.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_convert_format.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_data_helpers.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_de_edge_cases.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_de_lowexpr_filter.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_de_resume.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_dimred.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_disk_dispatch.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_downsample_counts.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_groupby_alias.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_hvg.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_in_memory_input.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_irls.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_memory_dispatch.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_messages.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_normalisation.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_normalized_effects.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_normalized_effects_se.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_obs_writes.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_pickle.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_plotting.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_profiling.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_pseudobulk_batch.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_pseudobulk_memory.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_pseudobulk_profiles.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_qc_parity.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_rank_genes_groups_outputs.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_residual_effects.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_result_provenance.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_scanpy_parity.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_standardize.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_stream_on_fast_axis.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_subsample.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_temp_file_cleanup.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_wilcoxon_dispatch.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_wilcoxon_labels.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_wilcoxon_stratified.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_workflow.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: crispyx
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: Memory-efficient streaming analysis of large-scale CRISPR and Perturb-seq screens on disk-backed AnnData files
|
|
5
5
|
Author: Jin-Hong Du
|
|
6
6
|
License: Modified MIT License
|
|
@@ -105,7 +105,7 @@ Requires-Dist: numpy>=1.23
|
|
|
105
105
|
Requires-Dist: numba>=0.59
|
|
106
106
|
Requires-Dist: pandas>=1.5
|
|
107
107
|
Requires-Dist: scipy>=1.10
|
|
108
|
-
Requires-Dist: h5py>=3.
|
|
108
|
+
Requires-Dist: h5py>=3.10
|
|
109
109
|
Requires-Dist: joblib>=1.4
|
|
110
110
|
Requires-Dist: scikit-learn>=1.0
|
|
111
111
|
Requires-Dist: scikit-misc>=0.1.4
|
|
@@ -404,7 +404,18 @@ class OneHotGroups:
|
|
|
404
404
|
Returns shape ``(n_groups, n_genes)``. This is the diagonal of
|
|
405
405
|
``G' W G`` when ``values`` are the IRLS weights.
|
|
406
406
|
"""
|
|
407
|
-
return
|
|
407
|
+
return self.sums_sorted(self.sort(values))
|
|
408
|
+
|
|
409
|
+
def sums_sorted(self, values_sorted: np.ndarray) -> np.ndarray:
|
|
410
|
+
""":meth:`sums` of values already in :meth:`sort` order: a NumPy sum
|
|
411
|
+
per group's contiguous slice, which releases the GIL (a sparse
|
|
412
|
+
product does not, and serialised the IRLS's threads)."""
|
|
413
|
+
out = np.zeros((self.n_groups, values_sorted.shape[1]), dtype=np.float64)
|
|
414
|
+
for k in range(self.n_groups):
|
|
415
|
+
start, stop = self._starts[k], self._ends[k]
|
|
416
|
+
if stop > start:
|
|
417
|
+
values_sorted[start:stop].sum(axis=0, out=out[k])
|
|
418
|
+
return out
|
|
408
419
|
|
|
409
420
|
def cross(self, design_sorted: np.ndarray, weights_sorted: np.ndarray) -> np.ndarray:
|
|
410
421
|
"""``X' W G`` per gene, shape ``(n_groups, n_features, n_genes)``.
|
|
@@ -427,11 +438,13 @@ class OneHotGroups:
|
|
|
427
438
|
return out
|
|
428
439
|
|
|
429
440
|
def expand(self, per_group: np.ndarray) -> np.ndarray:
|
|
430
|
-
"""Scatter ``(n_genes, n_groups)`` values back to ``(n_samples, n_genes)``.
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
441
|
+
"""Scatter ``(n_genes, n_groups)`` values back to ``(n_samples, n_genes)``.
|
|
442
|
+
|
|
443
|
+
One ``take`` from the values with a zero row appended, which a cell
|
|
444
|
+
in no group (label ``-1``) picks."""
|
|
445
|
+
padded = np.zeros((self.n_groups + 1, per_group.shape[0]), dtype=np.float64)
|
|
446
|
+
padded[:-1] = per_group.T
|
|
447
|
+
return padded.take(self.labels, axis=0)
|
|
435
448
|
|
|
436
449
|
|
|
437
450
|
def _arrowhead_blocks(
|
|
@@ -115,6 +115,11 @@ def _neg_half_log_det_numba(xtwx: np.ndarray) -> float:
|
|
|
115
115
|
return -log_det # -0.5 * (2 * sum(log diag(L)))
|
|
116
116
|
|
|
117
117
|
|
|
118
|
+
#: Counts below this are tallied per gene for the NB likelihood's lgamma
|
|
119
|
+
#: terms (_nb_posterior_with_cache_numba); _lgamma_ratio_term sums them exactly.
|
|
120
|
+
_SMALL_COUNTS = 32
|
|
121
|
+
|
|
122
|
+
|
|
118
123
|
@nb.njit(cache=True)
|
|
119
124
|
def _brent_minimize_numba(
|
|
120
125
|
Y_g: np.ndarray,
|
|
@@ -122,6 +127,7 @@ def _brent_minimize_numba(
|
|
|
122
127
|
log_trend_g: float,
|
|
123
128
|
prior_var: float,
|
|
124
129
|
gammaln_y_plus_1_sum: float,
|
|
130
|
+
small_counts: np.ndarray,
|
|
125
131
|
x_indptr: np.ndarray,
|
|
126
132
|
x_indices: np.ndarray,
|
|
127
133
|
x_data: np.ndarray,
|
|
@@ -142,7 +148,8 @@ def _brent_minimize_numba(
|
|
|
142
148
|
# Initial setup
|
|
143
149
|
x = w = v = a + golden * (b - a)
|
|
144
150
|
fx = fw = fv = -_nb_posterior_with_cache_numba(
|
|
145
|
-
Y_g, mu_g, x, log_trend_g, prior_var, gammaln_y_plus_1_sum, x_indptr, x_indices, x_data,
|
|
151
|
+
Y_g, mu_g, x, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
|
|
152
|
+
xtwx,
|
|
146
153
|
)
|
|
147
154
|
|
|
148
155
|
d = 0.0 # Distance to next point
|
|
@@ -198,7 +205,8 @@ def _brent_minimize_numba(
|
|
|
198
205
|
u = x + (tol1 if d > 0 else -tol1)
|
|
199
206
|
|
|
200
207
|
fu = -_nb_posterior_with_cache_numba(
|
|
201
|
-
Y_g, mu_g, u, log_trend_g, prior_var, gammaln_y_plus_1_sum, x_indptr, x_indices, x_data,
|
|
208
|
+
Y_g, mu_g, u, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
|
|
209
|
+
xtwx,
|
|
202
210
|
)
|
|
203
211
|
|
|
204
212
|
# Update a, b, v, w, x
|
|
@@ -240,6 +248,7 @@ def _nb_posterior_with_cache_numba(
|
|
|
240
248
|
log_trend_g: float,
|
|
241
249
|
prior_var: float,
|
|
242
250
|
gammaln_y_plus_1_sum: float,
|
|
251
|
+
small_counts: np.ndarray,
|
|
243
252
|
x_indptr: np.ndarray,
|
|
244
253
|
x_indices: np.ndarray,
|
|
245
254
|
x_data: np.ndarray,
|
|
@@ -251,7 +260,10 @@ def _nb_posterior_with_cache_numba(
|
|
|
251
260
|
alpha mu)``, is accumulated into the work buffer ``xtwx`` (``(0, 0)``
|
|
252
261
|
for no adjustment) in the same pass over the cells as the likelihood;
|
|
253
262
|
the design ``X`` is given in CSR form, so a one-hot batch design costs a
|
|
254
|
-
few products per cell rather than ``n_features**2``.
|
|
263
|
+
few products per cell rather than ``n_features**2``. ``small_counts[k]``
|
|
264
|
+
is the number of cells with count ``k`` (below its length): their
|
|
265
|
+
``lgamma(y + r) - lgamma(r)`` terms depend on the count only, so each
|
|
266
|
+
value's is formed once, by recurrence, rather than once per cell.
|
|
255
267
|
"""
|
|
256
268
|
n_cells = Y_g.shape[0]
|
|
257
269
|
alpha = math.exp(log_alpha)
|
|
@@ -260,14 +272,20 @@ def _nb_posterior_with_cache_numba(
|
|
|
260
272
|
cox_reid = xtwx.shape[0] > 0
|
|
261
273
|
xtwx[:, :] = 0.0
|
|
262
274
|
|
|
275
|
+
# lgamma(y + r) - lgamma(r) - y log r, without libm's lgamma (see
|
|
276
|
+
# _lgamma_ratio_term): sum(log1p(i / r), i < y) for the small counts.
|
|
263
277
|
ll = 0.0
|
|
278
|
+
term = 0.0
|
|
279
|
+
for k in range(1, small_counts.shape[0]):
|
|
280
|
+
term += math.log1p((k - 1) / r)
|
|
281
|
+
ll += small_counts[k] * term
|
|
282
|
+
n_small = small_counts.shape[0]
|
|
264
283
|
for i in range(n_cells):
|
|
265
284
|
y = Y_g[i]
|
|
266
285
|
mu_i = mu_g[i]
|
|
267
286
|
r_plus_mu = r + mu_i + 1e-12
|
|
268
|
-
# lgamma(y + r) - lgamma(r), without libm's lgamma (see _lgamma_ratio_term)
|
|
269
287
|
ll += (
|
|
270
|
-
_lgamma_ratio_term(y, r) + y * log_r
|
|
288
|
+
(_lgamma_ratio_term(y, r) if y >= n_small else 0.0) + y * log_r
|
|
271
289
|
+ r * (log_r - math.log(r_plus_mu))
|
|
272
290
|
+ y * math.log(mu_i / r_plus_mu + 1e-12)
|
|
273
291
|
)
|
|
@@ -340,13 +358,17 @@ def _nb_map_grid_search_with_refinement_numba(
|
|
|
340
358
|
Y_g = Y[:, g]
|
|
341
359
|
mu_g = mu[:, g]
|
|
342
360
|
xtwx = np.empty((n_features, n_features))
|
|
361
|
+
small_counts = np.zeros(_SMALL_COUNTS)
|
|
362
|
+
for i in range(Y_g.shape[0]):
|
|
363
|
+
if Y_g[i] < _SMALL_COUNTS:
|
|
364
|
+
small_counts[int(Y_g[i])] += 1.0
|
|
343
365
|
step = (hi[g] - lo[g]) / (n_grid - 1)
|
|
344
366
|
# sum(lgamma(y + 1)) is constant in alpha, so it is left out (0.0).
|
|
345
367
|
best_posterior = -np.inf
|
|
346
368
|
best_k = 0
|
|
347
369
|
for k in range(n_grid):
|
|
348
370
|
posterior = _nb_posterior_with_cache_numba(
|
|
349
|
-
Y_g, mu_g, lo[g] + k * step, log_trend_g, prior_var, 0.0,
|
|
371
|
+
Y_g, mu_g, lo[g] + k * step, log_trend_g, prior_var, 0.0, small_counts,
|
|
350
372
|
x_indptr, x_indices, x_data, xtwx,
|
|
351
373
|
)
|
|
352
374
|
if posterior > best_posterior:
|
|
@@ -355,11 +377,12 @@ def _nb_map_grid_search_with_refinement_numba(
|
|
|
355
377
|
best = lo[g] + best_k * step
|
|
356
378
|
if 0 < best_k < n_grid - 1 and 2 * step > tol:
|
|
357
379
|
refined = _brent_minimize_numba(
|
|
358
|
-
Y_g, mu_g, log_trend_g, prior_var, 0.0, x_indptr, x_indices, x_data, xtwx,
|
|
380
|
+
Y_g, mu_g, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data, xtwx,
|
|
359
381
|
best - step, best + step, tol=tol, max_iter=max_refine_iter,
|
|
360
382
|
)
|
|
361
383
|
refined_posterior = _nb_posterior_with_cache_numba(
|
|
362
|
-
Y_g, mu_g, refined, log_trend_g, prior_var, 0.0, x_indptr, x_indices, x_data,
|
|
384
|
+
Y_g, mu_g, refined, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data,
|
|
385
|
+
xtwx,
|
|
363
386
|
)
|
|
364
387
|
if refined_posterior >= best_posterior:
|
|
365
388
|
best = refined
|
|
@@ -1682,9 +1705,13 @@ def _two_group_lfc(genes, dispersion, a0, b0, t_ptr, t_y, t_s, s_t, min_mu, cent
|
|
|
1682
1705
|
moved = max(abs(na - a), abs(nb_ - b))
|
|
1683
1706
|
a = na
|
|
1684
1707
|
b = nb_
|
|
1685
|
-
|
|
1708
|
+
# Converged on the full Newton step: the step actually taken also
|
|
1709
|
+
# shrinks when the line search gives up or a bound clips it.
|
|
1710
|
+
if max(abs(da), abs(db)) < tol:
|
|
1686
1711
|
converged[i] = True
|
|
1687
1712
|
break
|
|
1713
|
+
if moved < tol:
|
|
1714
|
+
break
|
|
1688
1715
|
k_t = math.exp(b)
|
|
1689
1716
|
w_t = 0.0
|
|
1690
1717
|
for j in range(s_t.shape[0]):
|
|
@@ -1733,3 +1760,294 @@ def _accumulate_control_summary(data, indices, indptr, bin_of_row, s_rows, centr
|
|
|
1733
1760
|
big_values[n_big] = y
|
|
1734
1761
|
n_big += 1
|
|
1735
1762
|
return n_big
|
|
1763
|
+
|
|
1764
|
+
|
|
1765
|
+
# =============================================================================
|
|
1766
|
+
# Structured NB IRLS at a fixed dispersion, one gene per thread
|
|
1767
|
+
# =============================================================================
|
|
1768
|
+
#
|
|
1769
|
+
# StructuredGLMBatchFitter._irls, gene by gene: the same start, Newton step
|
|
1770
|
+
# (arrowhead Hessian solved through the Schur complement of the group block),
|
|
1771
|
+
# step halving, convergence rules and standard errors, in one pass over a
|
|
1772
|
+
# gene's cells per quantity and no temporary (cells x genes) arrays. The NumPy
|
|
1773
|
+
# fitter's batches allocate such arrays at every step, and on many threads
|
|
1774
|
+
# they wait on the allocator and the GIL; this runs on all of them.
|
|
1775
|
+
|
|
1776
|
+
|
|
1777
|
+
@nb.njit(cache=True)
|
|
1778
|
+
def _structured_predictor(X, labels, offset, bf, bg, eta_lo, eta_hi, eta, mu):
|
|
1779
|
+
"""eta = clip(offset + X bf + bg[label], eta_lo, eta_hi) and mu = exp(eta)."""
|
|
1780
|
+
n, dx = X.shape
|
|
1781
|
+
for i in range(n):
|
|
1782
|
+
v = offset[i]
|
|
1783
|
+
for j in range(dx):
|
|
1784
|
+
v += X[i, j] * bf[j]
|
|
1785
|
+
if labels[i] >= 0:
|
|
1786
|
+
v += bg[labels[i]]
|
|
1787
|
+
v = min(max(v, eta_lo), eta_hi)
|
|
1788
|
+
eta[i] = v
|
|
1789
|
+
mu[i] = math.exp(v)
|
|
1790
|
+
|
|
1791
|
+
|
|
1792
|
+
@nb.njit(cache=True)
|
|
1793
|
+
def _structured_deviance(y, eta, mu, size, ylogy):
|
|
1794
|
+
"""2 (sum y log y - sum y eta + sum (y + r) log1p((mu - y) / (y + r))), as Deviance.total."""
|
|
1795
|
+
cross = 0.0
|
|
1796
|
+
tail = 0.0
|
|
1797
|
+
for i in range(y.shape[0]):
|
|
1798
|
+
cross += y[i] * eta[i]
|
|
1799
|
+
yr = y[i] + size
|
|
1800
|
+
tail += yr * math.log1p((mu[i] - y[i]) / yr)
|
|
1801
|
+
return 2.0 * (ylogy - cross + tail)
|
|
1802
|
+
|
|
1803
|
+
|
|
1804
|
+
@nb.njit(cache=True)
|
|
1805
|
+
def _structured_blocks(X, labels, w, wz, with_rhs, gram, cross, diag, rx, rg, ridge, ridge_group):
|
|
1806
|
+
"""X'WX (+ ridge, one value per column), X'WG and diag(G'WG)
|
|
1807
|
+
(+ ridge_group) for the weights w, and with_rhs, X'Wz and G'Wz for
|
|
1808
|
+
wz = w z."""
|
|
1809
|
+
n, dx = X.shape
|
|
1810
|
+
gram[:, :] = 0.0
|
|
1811
|
+
cross[:, :] = 0.0
|
|
1812
|
+
diag[:] = 0.0
|
|
1813
|
+
rx[:] = 0.0
|
|
1814
|
+
rg[:] = 0.0
|
|
1815
|
+
for i in range(n):
|
|
1816
|
+
wi = w[i]
|
|
1817
|
+
k = labels[i]
|
|
1818
|
+
for j in range(dx):
|
|
1819
|
+
xw = X[i, j] * wi
|
|
1820
|
+
for l in range(j + 1):
|
|
1821
|
+
gram[j, l] += xw * X[i, l]
|
|
1822
|
+
if k >= 0:
|
|
1823
|
+
cross[j, k] += xw
|
|
1824
|
+
if with_rhs:
|
|
1825
|
+
rx[j] += X[i, j] * wz[i]
|
|
1826
|
+
if k >= 0:
|
|
1827
|
+
diag[k] += wi
|
|
1828
|
+
if with_rhs:
|
|
1829
|
+
rg[k] += wz[i]
|
|
1830
|
+
for j in range(dx):
|
|
1831
|
+
for l in range(j):
|
|
1832
|
+
gram[l, j] = gram[j, l]
|
|
1833
|
+
gram[j, j] += ridge[j]
|
|
1834
|
+
for k in range(diag.shape[0]):
|
|
1835
|
+
diag[k] += ridge_group
|
|
1836
|
+
|
|
1837
|
+
|
|
1838
|
+
@nb.njit(cache=True)
|
|
1839
|
+
def _solve_small(A, B):
|
|
1840
|
+
"""A^-1 B for a small square A, by Gaussian elimination with partial
|
|
1841
|
+
pivoting (as LAPACK's gesv); NaN when A is singular."""
|
|
1842
|
+
m = A.shape[0]
|
|
1843
|
+
a = A.copy()
|
|
1844
|
+
x = B.copy()
|
|
1845
|
+
for c in range(m):
|
|
1846
|
+
p = c
|
|
1847
|
+
for r in range(c + 1, m):
|
|
1848
|
+
if abs(a[r, c]) > abs(a[p, c]):
|
|
1849
|
+
p = r
|
|
1850
|
+
if a[p, c] == 0.0:
|
|
1851
|
+
x[:, :] = np.nan
|
|
1852
|
+
return x
|
|
1853
|
+
if p != c:
|
|
1854
|
+
for j in range(m):
|
|
1855
|
+
t = a[c, j]
|
|
1856
|
+
a[c, j] = a[p, j]
|
|
1857
|
+
a[p, j] = t
|
|
1858
|
+
for j in range(x.shape[1]):
|
|
1859
|
+
t = x[c, j]
|
|
1860
|
+
x[c, j] = x[p, j]
|
|
1861
|
+
x[p, j] = t
|
|
1862
|
+
for r in range(c + 1, m):
|
|
1863
|
+
f = a[r, c] / a[c, c]
|
|
1864
|
+
if f != 0.0:
|
|
1865
|
+
for j in range(c, m):
|
|
1866
|
+
a[r, j] -= f * a[c, j]
|
|
1867
|
+
for j in range(x.shape[1]):
|
|
1868
|
+
x[r, j] -= f * x[c, j]
|
|
1869
|
+
for c in range(m - 1, -1, -1):
|
|
1870
|
+
for j in range(x.shape[1]):
|
|
1871
|
+
v = x[c, j]
|
|
1872
|
+
for k in range(c + 1, m):
|
|
1873
|
+
v -= a[c, k] * x[k, j]
|
|
1874
|
+
x[c, j] = v / a[c, c]
|
|
1875
|
+
return x
|
|
1876
|
+
|
|
1877
|
+
|
|
1878
|
+
@nb.njit(cache=True)
|
|
1879
|
+
def _schur(gram, cross, diag):
|
|
1880
|
+
"""S = C - B D^-1 B'."""
|
|
1881
|
+
dx, a = cross.shape
|
|
1882
|
+
S = gram.copy()
|
|
1883
|
+
for j in range(dx):
|
|
1884
|
+
for l in range(dx):
|
|
1885
|
+
v = 0.0
|
|
1886
|
+
for k in range(a):
|
|
1887
|
+
v += cross[j, k] * cross[l, k] / diag[k]
|
|
1888
|
+
S[j, l] -= v
|
|
1889
|
+
return S
|
|
1890
|
+
|
|
1891
|
+
|
|
1892
|
+
@nb.njit(parallel=True, cache=True)
|
|
1893
|
+
def _structured_nb_irls(X, labels, offset, Y, alpha, n_groups, const_col, const_val, mean_exp_offset,
|
|
1894
|
+
eta_lo, eta_hi, wald_lo, floored, max_iter, tol, ridge, ridge_group, clip_group,
|
|
1895
|
+
max_halvings, eps):
|
|
1896
|
+
"""StructuredGLMBatchFitter._irls and its standard errors at the fixed NB
|
|
1897
|
+
dispersions alpha, for each column of Y (Fortran order), one gene per
|
|
1898
|
+
thread; ridge is per covariate column, and n_groups may be 0 (a dense
|
|
1899
|
+
design). Returns coef (genes, dx + a: covariates, then groups), se,
|
|
1900
|
+
deviance, converged and n_iter."""
|
|
1901
|
+
n, dx = X.shape
|
|
1902
|
+
n_genes = Y.shape[1]
|
|
1903
|
+
a = n_groups
|
|
1904
|
+
coef = np.zeros((n_genes, dx + a))
|
|
1905
|
+
se = np.full((n_genes, dx + a), np.inf)
|
|
1906
|
+
dev_out = np.full(n_genes, np.nan)
|
|
1907
|
+
converged = np.zeros(n_genes, dtype=np.bool_)
|
|
1908
|
+
n_iter = np.zeros(n_genes, dtype=np.int32)
|
|
1909
|
+
for g in nb.prange(n_genes):
|
|
1910
|
+
y = Y[:, g]
|
|
1911
|
+
ag = alpha[g]
|
|
1912
|
+
size = 1.0 / min(max(ag, eps), 1e10)
|
|
1913
|
+
ylogy = 0.0
|
|
1914
|
+
total = 0.0
|
|
1915
|
+
for i in range(n):
|
|
1916
|
+
total += y[i]
|
|
1917
|
+
if y[i] > 0.0:
|
|
1918
|
+
ylogy += y[i] * math.log(y[i])
|
|
1919
|
+
bf = np.zeros(dx)
|
|
1920
|
+
bg = np.zeros(a)
|
|
1921
|
+
if const_col >= 0:
|
|
1922
|
+
bf[const_col] = math.log(max(total / n, 1e-3) / mean_exp_offset) / const_val
|
|
1923
|
+
eta = np.empty(n)
|
|
1924
|
+
mu = np.empty(n)
|
|
1925
|
+
_structured_predictor(X, labels, offset, bf, bg, eta_lo, eta_hi, eta, mu)
|
|
1926
|
+
dev = _structured_deviance(y, eta, mu, size, ylogy)
|
|
1927
|
+
w = np.empty(n)
|
|
1928
|
+
wz = np.empty(n)
|
|
1929
|
+
new_eta = np.empty(n)
|
|
1930
|
+
new_mu = np.empty(n)
|
|
1931
|
+
half_eta = np.empty(n)
|
|
1932
|
+
half_mu = np.empty(n)
|
|
1933
|
+
gram = np.empty((dx, dx))
|
|
1934
|
+
cross = np.empty((dx, a))
|
|
1935
|
+
diag = np.empty(a)
|
|
1936
|
+
rx = np.empty(dx)
|
|
1937
|
+
rg = np.empty(a)
|
|
1938
|
+
conv = False
|
|
1939
|
+
it = 0
|
|
1940
|
+
for iteration in range(1, max_iter + 1):
|
|
1941
|
+
# The Newton step at (eta, mu): weights, working response, normal equations.
|
|
1942
|
+
for i in range(n):
|
|
1943
|
+
wi = mu[i] / (1.0 + ag * mu[i])
|
|
1944
|
+
w[i] = wi
|
|
1945
|
+
wz[i] = wi * (eta[i] - offset[i] + (y[i] - mu[i]) / max(mu[i], eps))
|
|
1946
|
+
_structured_blocks(X, labels, w, wz, True, gram, cross, diag, rx, rg, ridge, ridge_group)
|
|
1947
|
+
S = _schur(gram, cross, diag)
|
|
1948
|
+
rhs = np.empty((dx, 1))
|
|
1949
|
+
for j in range(dx):
|
|
1950
|
+
v = rx[j]
|
|
1951
|
+
for k in range(a):
|
|
1952
|
+
v -= cross[j, k] / diag[k] * rg[k]
|
|
1953
|
+
rhs[j, 0] = v
|
|
1954
|
+
full_f = _solve_small(S, rhs)[:, 0].copy()
|
|
1955
|
+
full_g = np.empty(a)
|
|
1956
|
+
for k in range(a):
|
|
1957
|
+
v = rg[k]
|
|
1958
|
+
for j in range(dx):
|
|
1959
|
+
v -= cross[j, k] * full_f[j]
|
|
1960
|
+
full_g[k] = min(max(v / diag[k], -clip_group), clip_group)
|
|
1961
|
+
_structured_predictor(X, labels, offset, full_f, full_g, eta_lo, eta_hi, new_eta, new_mu)
|
|
1962
|
+
new_dev = _structured_deviance(y, new_eta, new_mu, size, ylogy)
|
|
1963
|
+
new_f = full_f.copy()
|
|
1964
|
+
new_g = full_g.copy()
|
|
1965
|
+
|
|
1966
|
+
# A gene resting on the mean floor takes the full step and is
|
|
1967
|
+
# judged by the deviance ratio (as the NumPy fitter).
|
|
1968
|
+
on_floor = False
|
|
1969
|
+
if floored:
|
|
1970
|
+
for i in range(n):
|
|
1971
|
+
if eta[i] <= eta_lo:
|
|
1972
|
+
on_floor = True
|
|
1973
|
+
break
|
|
1974
|
+
ceiling = dev + 1e-9 * abs(dev)
|
|
1975
|
+
rose = new_dev > ceiling
|
|
1976
|
+
worse = rose and not on_floor
|
|
1977
|
+
scale = max(abs(dev), 1e-8)
|
|
1978
|
+
excess = (new_dev - dev) / scale
|
|
1979
|
+
step = 1.0
|
|
1980
|
+
for _h in range(max_halvings):
|
|
1981
|
+
if not worse:
|
|
1982
|
+
break
|
|
1983
|
+
step *= 0.5
|
|
1984
|
+
half_f = bf + step * (full_f - bf)
|
|
1985
|
+
half_g = bg + step * (full_g - bg)
|
|
1986
|
+
_structured_predictor(X, labels, offset, half_f, half_g, eta_lo, eta_hi, half_eta, half_mu)
|
|
1987
|
+
half_dev = _structured_deviance(y, half_eta, half_mu, size, ylogy)
|
|
1988
|
+
excess = min(excess, (half_dev - dev) / scale)
|
|
1989
|
+
if half_dev <= ceiling:
|
|
1990
|
+
new_f = half_f
|
|
1991
|
+
new_g = half_g
|
|
1992
|
+
new_eta[:] = half_eta
|
|
1993
|
+
new_mu[:] = half_mu
|
|
1994
|
+
new_dev = half_dev
|
|
1995
|
+
worse = False
|
|
1996
|
+
fixed_point = rose and on_floor
|
|
1997
|
+
if worse:
|
|
1998
|
+
new_f = bf.copy()
|
|
1999
|
+
new_g = bg.copy()
|
|
2000
|
+
new_eta[:] = eta
|
|
2001
|
+
new_mu[:] = mu
|
|
2002
|
+
new_dev = dev
|
|
2003
|
+
relative_change = abs(new_dev - dev) / scale
|
|
2004
|
+
deviance_ratio = abs(new_dev - dev) / (abs(new_dev) + 0.1)
|
|
2005
|
+
coef_change = 0.0
|
|
2006
|
+
for j in range(dx):
|
|
2007
|
+
coef_change = max(coef_change, abs(new_f[j] - bf[j]))
|
|
2008
|
+
for k in range(a):
|
|
2009
|
+
coef_change = max(coef_change, abs(new_g[k] - bg[k]))
|
|
2010
|
+
bf = new_f
|
|
2011
|
+
bg = new_g
|
|
2012
|
+
eta[:] = new_eta
|
|
2013
|
+
mu[:] = new_mu
|
|
2014
|
+
dev = new_dev
|
|
2015
|
+
it = iteration
|
|
2016
|
+
if fixed_point:
|
|
2017
|
+
conv = deviance_ratio < tol and coef_change < tol
|
|
2018
|
+
elif worse:
|
|
2019
|
+
conv = excess < tol
|
|
2020
|
+
else:
|
|
2021
|
+
conv = relative_change < tol and coef_change < tol
|
|
2022
|
+
if conv or worse:
|
|
2023
|
+
break
|
|
2024
|
+
|
|
2025
|
+
# Standard errors from the blocks at the fit, with the unfloored mean.
|
|
2026
|
+
if floored:
|
|
2027
|
+
_structured_predictor(X, labels, offset, bf, bg, wald_lo, eta_hi, half_eta, half_mu)
|
|
2028
|
+
for i in range(n):
|
|
2029
|
+
w[i] = half_mu[i] / (1.0 + ag * half_mu[i])
|
|
2030
|
+
else:
|
|
2031
|
+
for i in range(n):
|
|
2032
|
+
w[i] = mu[i] / (1.0 + ag * mu[i])
|
|
2033
|
+
_structured_blocks(X, labels, w, wz, False, gram, cross, diag, rx, rg, ridge, ridge_group)
|
|
2034
|
+
S = _schur(gram, cross, diag)
|
|
2035
|
+
rhs = np.zeros((dx, dx + a))
|
|
2036
|
+
for j in range(dx):
|
|
2037
|
+
rhs[j, j] = 1.0
|
|
2038
|
+
for k in range(a):
|
|
2039
|
+
rhs[j, dx + k] = cross[j, k]
|
|
2040
|
+
solved = _solve_small(S, rhs)
|
|
2041
|
+
for j in range(dx):
|
|
2042
|
+
coef[g, j] = bf[j]
|
|
2043
|
+
se[g, j] = math.sqrt(max(solved[j, j], 0.0))
|
|
2044
|
+
for k in range(a):
|
|
2045
|
+
quad = 0.0
|
|
2046
|
+
for j in range(dx):
|
|
2047
|
+
quad += cross[j, k] * solved[j, dx + k]
|
|
2048
|
+
coef[g, dx + k] = bg[k]
|
|
2049
|
+
se[g, dx + k] = math.sqrt(max(1.0 / diag[k] + quad / (diag[k] * diag[k]), 0.0))
|
|
2050
|
+
dev_out[g] = dev
|
|
2051
|
+
converged[g] = conv
|
|
2052
|
+
n_iter[g] = it
|
|
2053
|
+
return coef, se, dev_out, converged, n_iter
|
|
@@ -20,6 +20,7 @@ from .data import (
|
|
|
20
20
|
concat_h5ad,
|
|
21
21
|
downsample_counts,
|
|
22
22
|
normalize_total_log1p,
|
|
23
|
+
read_backed,
|
|
23
24
|
resolve_data_path,
|
|
24
25
|
)
|
|
25
26
|
from .de import (
|
|
@@ -504,7 +505,7 @@ class _PreprocessingNamespace:
|
|
|
504
505
|
from .dimred import pca as _pca
|
|
505
506
|
|
|
506
507
|
if isinstance(data, (str, Path)):
|
|
507
|
-
adata =
|
|
508
|
+
adata = read_backed(data)
|
|
508
509
|
else:
|
|
509
510
|
adata = data
|
|
510
511
|
|
|
@@ -562,7 +563,7 @@ class _PreprocessingNamespace:
|
|
|
562
563
|
from .dimred import neighbors as _neighbors
|
|
563
564
|
|
|
564
565
|
if isinstance(data, (str, Path)):
|
|
565
|
-
adata =
|
|
566
|
+
adata = read_backed(data)
|
|
566
567
|
else:
|
|
567
568
|
adata = data
|
|
568
569
|
|
|
@@ -762,7 +763,7 @@ class _ToolsNamespace:
|
|
|
762
763
|
from .dimred import umap as _umap
|
|
763
764
|
|
|
764
765
|
if isinstance(data, (str, Path)):
|
|
765
|
-
adata =
|
|
766
|
+
adata = read_backed(data)
|
|
766
767
|
else:
|
|
767
768
|
adata = data
|
|
768
769
|
|
|
@@ -682,7 +682,7 @@ def batch_process(
|
|
|
682
682
|
# width that output was written with. Skipped under force=True,
|
|
683
683
|
# which reads nothing from the existing output -- including when
|
|
684
684
|
# it is too damaged to open, the case force exists for.
|
|
685
|
-
existing =
|
|
685
|
+
existing = read_backed(resolved_output)
|
|
686
686
|
try:
|
|
687
687
|
stored_chunk_size = existing.uns.get("chunk_size")
|
|
688
688
|
finally:
|
|
@@ -735,7 +735,7 @@ def batch_process(
|
|
|
735
735
|
# tell a finished result from one whose run was killed. Only the
|
|
736
736
|
# completion marker, written after the last chunk, can.
|
|
737
737
|
if resolved_output.exists() and not force:
|
|
738
|
-
existing =
|
|
738
|
+
existing = read_backed(resolved_output)
|
|
739
739
|
try:
|
|
740
740
|
complete = _run_is_complete(existing)
|
|
741
741
|
finally:
|
|
@@ -792,7 +792,7 @@ def batch_process(
|
|
|
792
792
|
f"gene chunk {last_completed_chunk} by scanning {resolved_output.name}",
|
|
793
793
|
)
|
|
794
794
|
if resume and last_completed_chunk >= 0 and resolved_output.exists():
|
|
795
|
-
existing =
|
|
795
|
+
existing = read_backed(resolved_output)
|
|
796
796
|
try:
|
|
797
797
|
# A finished output rejected above fails this too, so it is
|
|
798
798
|
# recomputed rather than "resumed" with nothing left to do.
|
|
@@ -423,6 +423,43 @@ def read_backed(path: str | Path, mode: Literal["r", "r+"] = "r") -> ad.AnnData:
|
|
|
423
423
|
return adata
|
|
424
424
|
|
|
425
425
|
|
|
426
|
+
def read_rows_csc(path: str | Path, rows: np.ndarray, chunk_rows: int = 4096) -> sp.csc_matrix:
|
|
427
|
+
"""The rows ``rows`` (ascending) of ``path``'s ``X`` as an in-memory CSC
|
|
428
|
+
matrix, in the file's dtype, without stored zeros.
|
|
429
|
+
|
|
430
|
+
Built in two passes over the rows, ``chunk_rows`` at a time -- the
|
|
431
|
+
column counts, then the non-zeros scattered into place (as
|
|
432
|
+
:func:`convert_to_csc` transposes) -- so memory peaks at the result plus
|
|
433
|
+
one chunk, where reading CSR and converting holds both and more.
|
|
434
|
+
"""
|
|
435
|
+
rows = np.asarray(rows)
|
|
436
|
+
starts = range(0, rows.size, chunk_rows)
|
|
437
|
+
backed = read_backed(path)
|
|
438
|
+
try:
|
|
439
|
+
n_cols = backed.n_vars
|
|
440
|
+
|
|
441
|
+
def chunks():
|
|
442
|
+
for start in starts:
|
|
443
|
+
chunk = sp.csr_matrix(backed.X[rows[start:start + chunk_rows], :])
|
|
444
|
+
chunk.eliminate_zeros()
|
|
445
|
+
yield start, chunk
|
|
446
|
+
|
|
447
|
+
counts = np.zeros(n_cols, dtype=np.int64)
|
|
448
|
+
dtype = None
|
|
449
|
+
for _, chunk in chunks():
|
|
450
|
+
counts += np.bincount(chunk.indices, minlength=n_cols)
|
|
451
|
+
dtype = chunk.dtype
|
|
452
|
+
indptr = np.concatenate([[0], np.cumsum(counts)])
|
|
453
|
+
data = np.empty(int(indptr[-1]), dtype=dtype or np.float64)
|
|
454
|
+
indices = np.empty(int(indptr[-1]), dtype=np.int32 if rows.size <= np.iinfo(np.int32).max else np.int64)
|
|
455
|
+
offset = indptr[:-1].copy()
|
|
456
|
+
for start, chunk in chunks():
|
|
457
|
+
_scatter_band(chunk.indptr, chunk.indices, chunk.data, start, 0, n_cols, offset, data, indices)
|
|
458
|
+
finally:
|
|
459
|
+
backed.file.close()
|
|
460
|
+
return sp.csc_matrix((data, indices, indptr), shape=(rows.size, n_cols))
|
|
461
|
+
|
|
462
|
+
|
|
426
463
|
# Read into memory by read_backed, as anndata's backed mode does.
|
|
427
464
|
_IN_MEMORY_SLOTS = ("obs", "var", "uns", "obsm", "varm", "obsp", "varp")
|
|
428
465
|
|
|
@@ -4847,7 +4884,7 @@ def standardize_dataset(
|
|
|
4847
4884
|
}
|
|
4848
4885
|
|
|
4849
4886
|
# Read obs/var metadata only (without loading X into memory)
|
|
4850
|
-
adata =
|
|
4887
|
+
adata = read_backed(dataset_path)
|
|
4851
4888
|
obs_df = adata.obs.copy()
|
|
4852
4889
|
var_df = adata.var.copy()
|
|
4853
4890
|
uns_dict = dict(adata.uns) # shallow copy of uns
|