crispyx 0.2.3__tar.gz → 0.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {crispyx-0.2.3/src/crispyx.egg-info → crispyx-0.2.4}/PKG-INFO +2 -2
  2. {crispyx-0.2.3 → crispyx-0.2.4}/pyproject.toml +1 -1
  3. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_irls.py +19 -6
  4. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_kernels.py +327 -9
  5. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_namespaces.py +4 -3
  6. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_version.py +1 -1
  7. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/batch.py +3 -3
  8. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/data.py +38 -1
  9. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/de.py +83 -76
  10. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/glm.py +203 -54
  11. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/qc.py +2 -2
  12. {crispyx-0.2.3 → crispyx-0.2.4/src/crispyx.egg-info}/PKG-INFO +2 -2
  13. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx.egg-info/requires.txt +1 -1
  14. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_batch_process.py +26 -0
  15. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_glm_structured.py +50 -0
  16. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_nb_glm.py +20 -7
  17. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_preflight_dispatch.py +8 -1
  18. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_read_backed.py +21 -0
  19. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_two_group_deseq2.py +85 -2
  20. {crispyx-0.2.3 → crispyx-0.2.4}/LICENSE +0 -0
  21. {crispyx-0.2.3 → crispyx-0.2.4}/README.md +0 -0
  22. {crispyx-0.2.3 → crispyx-0.2.4}/setup.cfg +0 -0
  23. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/__init__.py +0 -0
  24. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_checkpoint.py +0 -0
  25. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_disk.py +0 -0
  26. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_grouping.py +0 -0
  27. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_h5codec.py +0 -0
  28. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_memory.py +0 -0
  29. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_messages.py +0 -0
  30. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_preflight.py +0 -0
  31. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_provenance.py +0 -0
  32. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_size_factors.py +0 -0
  33. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/_statistics.py +0 -0
  34. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/dimred.py +0 -0
  35. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/hvg.py +0 -0
  36. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/plotting.py +0 -0
  37. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/profiling.py +0 -0
  38. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/pseudobulk.py +0 -0
  39. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/residual.py +0 -0
  40. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx/sample.py +0 -0
  41. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx.egg-info/SOURCES.txt +0 -0
  42. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx.egg-info/dependency_links.txt +0 -0
  43. {crispyx-0.2.3 → crispyx-0.2.4}/src/crispyx.egg-info/top_level.txt +0 -0
  44. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_cgroup_memory_budget.py +0 -0
  45. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_compare_controls.py +0 -0
  46. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_compress_h5ad.py +0 -0
  47. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_concat_h5ad.py +0 -0
  48. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_convert_format.py +0 -0
  49. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_data_helpers.py +0 -0
  50. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_de_edge_cases.py +0 -0
  51. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_de_lowexpr_filter.py +0 -0
  52. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_de_resume.py +0 -0
  53. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_dimred.py +0 -0
  54. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_disk_dispatch.py +0 -0
  55. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_downsample_counts.py +0 -0
  56. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_groupby_alias.py +0 -0
  57. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_hvg.py +0 -0
  58. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_in_memory_input.py +0 -0
  59. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_irls.py +0 -0
  60. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_memory_dispatch.py +0 -0
  61. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_messages.py +0 -0
  62. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_normalisation.py +0 -0
  63. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_normalized_effects.py +0 -0
  64. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_normalized_effects_se.py +0 -0
  65. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_obs_writes.py +0 -0
  66. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_pickle.py +0 -0
  67. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_plotting.py +0 -0
  68. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_profiling.py +0 -0
  69. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_pseudobulk_batch.py +0 -0
  70. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_pseudobulk_memory.py +0 -0
  71. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_pseudobulk_profiles.py +0 -0
  72. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_qc_parity.py +0 -0
  73. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_rank_genes_groups_outputs.py +0 -0
  74. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_residual_effects.py +0 -0
  75. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_result_provenance.py +0 -0
  76. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_scanpy_parity.py +0 -0
  77. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_standardize.py +0 -0
  78. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_stream_on_fast_axis.py +0 -0
  79. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_subsample.py +0 -0
  80. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_temp_file_cleanup.py +0 -0
  81. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_wilcoxon_dispatch.py +0 -0
  82. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_wilcoxon_labels.py +0 -0
  83. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_wilcoxon_stratified.py +0 -0
  84. {crispyx-0.2.3 → crispyx-0.2.4}/tests/test_workflow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: crispyx
3
- Version: 0.2.3
3
+ Version: 0.2.4
4
4
  Summary: Memory-efficient streaming analysis of large-scale CRISPR and Perturb-seq screens on disk-backed AnnData files
5
5
  Author: Jin-Hong Du
6
6
  License: Modified MIT License
@@ -105,7 +105,7 @@ Requires-Dist: numpy>=1.23
105
105
  Requires-Dist: numba>=0.59
106
106
  Requires-Dist: pandas>=1.5
107
107
  Requires-Dist: scipy>=1.10
108
- Requires-Dist: h5py>=3.0
108
+ Requires-Dist: h5py>=3.10
109
109
  Requires-Dist: joblib>=1.4
110
110
  Requires-Dist: scikit-learn>=1.0
111
111
  Requires-Dist: scikit-misc>=0.1.4
@@ -39,7 +39,7 @@ dependencies = [
39
39
  "numba>=0.59",
40
40
  "pandas>=1.5",
41
41
  "scipy>=1.10",
42
- "h5py>=3.0",
42
+ "h5py>=3.10", # 3.9 segfaults copying committed (named) datatypes
43
43
  "joblib>=1.4",
44
44
  "scikit-learn>=1.0",
45
45
  "scikit-misc>=0.1.4",
@@ -404,7 +404,18 @@ class OneHotGroups:
404
404
  Returns shape ``(n_groups, n_genes)``. This is the diagonal of
405
405
  ``G' W G`` when ``values`` are the IRLS weights.
406
406
  """
407
- return np.asarray(self.matrix.T @ values)
407
+ return self.sums_sorted(self.sort(values))
408
+
409
+ def sums_sorted(self, values_sorted: np.ndarray) -> np.ndarray:
410
+ """:meth:`sums` of values already in :meth:`sort` order: a NumPy sum
411
+ per group's contiguous slice, which releases the GIL (a sparse
412
+ product does not, and serialised the IRLS's threads)."""
413
+ out = np.zeros((self.n_groups, values_sorted.shape[1]), dtype=np.float64)
414
+ for k in range(self.n_groups):
415
+ start, stop = self._starts[k], self._ends[k]
416
+ if stop > start:
417
+ values_sorted[start:stop].sum(axis=0, out=out[k])
418
+ return out
408
419
 
409
420
  def cross(self, design_sorted: np.ndarray, weights_sorted: np.ndarray) -> np.ndarray:
410
421
  """``X' W G`` per gene, shape ``(n_groups, n_features, n_genes)``.
@@ -427,11 +438,13 @@ class OneHotGroups:
427
438
  return out
428
439
 
429
440
  def expand(self, per_group: np.ndarray) -> np.ndarray:
430
- """Scatter ``(n_genes, n_groups)`` values back to ``(n_samples, n_genes)``."""
431
- out = np.zeros((self.n_samples, per_group.shape[0]), dtype=np.float64)
432
- assigned = self.labels >= 0
433
- out[assigned] = per_group.T[self.labels[assigned]]
434
- return out
441
+ """Scatter ``(n_genes, n_groups)`` values back to ``(n_samples, n_genes)``.
442
+
443
+ One ``take`` from the values with a zero row appended, which a cell
444
+ in no group (label ``-1``) picks."""
445
+ padded = np.zeros((self.n_groups + 1, per_group.shape[0]), dtype=np.float64)
446
+ padded[:-1] = per_group.T
447
+ return padded.take(self.labels, axis=0)
435
448
 
436
449
 
437
450
  def _arrowhead_blocks(
@@ -115,6 +115,11 @@ def _neg_half_log_det_numba(xtwx: np.ndarray) -> float:
115
115
  return -log_det # -0.5 * (2 * sum(log diag(L)))
116
116
 
117
117
 
118
+ #: Counts below this are tallied per gene for the NB likelihood's lgamma
119
+ #: terms (_nb_posterior_with_cache_numba); _lgamma_ratio_term sums them exactly.
120
+ _SMALL_COUNTS = 32
121
+
122
+
118
123
  @nb.njit(cache=True)
119
124
  def _brent_minimize_numba(
120
125
  Y_g: np.ndarray,
@@ -122,6 +127,7 @@ def _brent_minimize_numba(
122
127
  log_trend_g: float,
123
128
  prior_var: float,
124
129
  gammaln_y_plus_1_sum: float,
130
+ small_counts: np.ndarray,
125
131
  x_indptr: np.ndarray,
126
132
  x_indices: np.ndarray,
127
133
  x_data: np.ndarray,
@@ -142,7 +148,8 @@ def _brent_minimize_numba(
142
148
  # Initial setup
143
149
  x = w = v = a + golden * (b - a)
144
150
  fx = fw = fv = -_nb_posterior_with_cache_numba(
145
- Y_g, mu_g, x, log_trend_g, prior_var, gammaln_y_plus_1_sum, x_indptr, x_indices, x_data, xtwx
151
+ Y_g, mu_g, x, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
152
+ xtwx,
146
153
  )
147
154
 
148
155
  d = 0.0 # Distance to next point
@@ -198,7 +205,8 @@ def _brent_minimize_numba(
198
205
  u = x + (tol1 if d > 0 else -tol1)
199
206
 
200
207
  fu = -_nb_posterior_with_cache_numba(
201
- Y_g, mu_g, u, log_trend_g, prior_var, gammaln_y_plus_1_sum, x_indptr, x_indices, x_data, xtwx
208
+ Y_g, mu_g, u, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
209
+ xtwx,
202
210
  )
203
211
 
204
212
  # Update a, b, v, w, x
@@ -240,6 +248,7 @@ def _nb_posterior_with_cache_numba(
240
248
  log_trend_g: float,
241
249
  prior_var: float,
242
250
  gammaln_y_plus_1_sum: float,
251
+ small_counts: np.ndarray,
243
252
  x_indptr: np.ndarray,
244
253
  x_indices: np.ndarray,
245
254
  x_data: np.ndarray,
@@ -251,7 +260,10 @@ def _nb_posterior_with_cache_numba(
251
260
  alpha mu)``, is accumulated into the work buffer ``xtwx`` (``(0, 0)``
252
261
  for no adjustment) in the same pass over the cells as the likelihood;
253
262
  the design ``X`` is given in CSR form, so a one-hot batch design costs a
254
- few products per cell rather than ``n_features**2``.
263
+ few products per cell rather than ``n_features**2``. ``small_counts[k]``
264
+ is the number of cells with count ``k`` (below its length): their
265
+ ``lgamma(y + r) - lgamma(r)`` terms depend on the count only, so each
266
+ value's is formed once, by recurrence, rather than once per cell.
255
267
  """
256
268
  n_cells = Y_g.shape[0]
257
269
  alpha = math.exp(log_alpha)
@@ -260,14 +272,20 @@ def _nb_posterior_with_cache_numba(
260
272
  cox_reid = xtwx.shape[0] > 0
261
273
  xtwx[:, :] = 0.0
262
274
 
275
+ # lgamma(y + r) - lgamma(r) - y log r, without libm's lgamma (see
276
+ # _lgamma_ratio_term): sum(log1p(i / r), i < y) for the small counts.
263
277
  ll = 0.0
278
+ term = 0.0
279
+ for k in range(1, small_counts.shape[0]):
280
+ term += math.log1p((k - 1) / r)
281
+ ll += small_counts[k] * term
282
+ n_small = small_counts.shape[0]
264
283
  for i in range(n_cells):
265
284
  y = Y_g[i]
266
285
  mu_i = mu_g[i]
267
286
  r_plus_mu = r + mu_i + 1e-12
268
- # lgamma(y + r) - lgamma(r), without libm's lgamma (see _lgamma_ratio_term)
269
287
  ll += (
270
- _lgamma_ratio_term(y, r) + y * log_r
288
+ (_lgamma_ratio_term(y, r) if y >= n_small else 0.0) + y * log_r
271
289
  + r * (log_r - math.log(r_plus_mu))
272
290
  + y * math.log(mu_i / r_plus_mu + 1e-12)
273
291
  )
@@ -340,13 +358,17 @@ def _nb_map_grid_search_with_refinement_numba(
340
358
  Y_g = Y[:, g]
341
359
  mu_g = mu[:, g]
342
360
  xtwx = np.empty((n_features, n_features))
361
+ small_counts = np.zeros(_SMALL_COUNTS)
362
+ for i in range(Y_g.shape[0]):
363
+ if Y_g[i] < _SMALL_COUNTS:
364
+ small_counts[int(Y_g[i])] += 1.0
343
365
  step = (hi[g] - lo[g]) / (n_grid - 1)
344
366
  # sum(lgamma(y + 1)) is constant in alpha, so it is left out (0.0).
345
367
  best_posterior = -np.inf
346
368
  best_k = 0
347
369
  for k in range(n_grid):
348
370
  posterior = _nb_posterior_with_cache_numba(
349
- Y_g, mu_g, lo[g] + k * step, log_trend_g, prior_var, 0.0,
371
+ Y_g, mu_g, lo[g] + k * step, log_trend_g, prior_var, 0.0, small_counts,
350
372
  x_indptr, x_indices, x_data, xtwx,
351
373
  )
352
374
  if posterior > best_posterior:
@@ -355,11 +377,12 @@ def _nb_map_grid_search_with_refinement_numba(
355
377
  best = lo[g] + best_k * step
356
378
  if 0 < best_k < n_grid - 1 and 2 * step > tol:
357
379
  refined = _brent_minimize_numba(
358
- Y_g, mu_g, log_trend_g, prior_var, 0.0, x_indptr, x_indices, x_data, xtwx,
380
+ Y_g, mu_g, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data, xtwx,
359
381
  best - step, best + step, tol=tol, max_iter=max_refine_iter,
360
382
  )
361
383
  refined_posterior = _nb_posterior_with_cache_numba(
362
- Y_g, mu_g, refined, log_trend_g, prior_var, 0.0, x_indptr, x_indices, x_data, xtwx,
384
+ Y_g, mu_g, refined, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data,
385
+ xtwx,
363
386
  )
364
387
  if refined_posterior >= best_posterior:
365
388
  best = refined
@@ -1682,9 +1705,13 @@ def _two_group_lfc(genes, dispersion, a0, b0, t_ptr, t_y, t_s, s_t, min_mu, cent
1682
1705
  moved = max(abs(na - a), abs(nb_ - b))
1683
1706
  a = na
1684
1707
  b = nb_
1685
- if moved < tol:
1708
+ # Converged on the full Newton step: the step actually taken also
1709
+ # shrinks when the line search gives up or a bound clips it.
1710
+ if max(abs(da), abs(db)) < tol:
1686
1711
  converged[i] = True
1687
1712
  break
1713
+ if moved < tol:
1714
+ break
1688
1715
  k_t = math.exp(b)
1689
1716
  w_t = 0.0
1690
1717
  for j in range(s_t.shape[0]):
@@ -1733,3 +1760,294 @@ def _accumulate_control_summary(data, indices, indptr, bin_of_row, s_rows, centr
1733
1760
  big_values[n_big] = y
1734
1761
  n_big += 1
1735
1762
  return n_big
1763
+
1764
+
1765
+ # =============================================================================
1766
+ # Structured NB IRLS at a fixed dispersion, one gene per thread
1767
+ # =============================================================================
1768
+ #
1769
+ # StructuredGLMBatchFitter._irls, gene by gene: the same start, Newton step
1770
+ # (arrowhead Hessian solved through the Schur complement of the group block),
1771
+ # step halving, convergence rules and standard errors, in one pass over a
1772
+ # gene's cells per quantity and no temporary (cells x genes) arrays. The NumPy
1773
+ # fitter's batches allocate such arrays at every step, and on many threads
1774
+ # they wait on the allocator and the GIL; this runs on all of them.
1775
+
1776
+
1777
+ @nb.njit(cache=True)
1778
+ def _structured_predictor(X, labels, offset, bf, bg, eta_lo, eta_hi, eta, mu):
1779
+ """eta = clip(offset + X bf + bg[label], eta_lo, eta_hi) and mu = exp(eta)."""
1780
+ n, dx = X.shape
1781
+ for i in range(n):
1782
+ v = offset[i]
1783
+ for j in range(dx):
1784
+ v += X[i, j] * bf[j]
1785
+ if labels[i] >= 0:
1786
+ v += bg[labels[i]]
1787
+ v = min(max(v, eta_lo), eta_hi)
1788
+ eta[i] = v
1789
+ mu[i] = math.exp(v)
1790
+
1791
+
1792
+ @nb.njit(cache=True)
1793
+ def _structured_deviance(y, eta, mu, size, ylogy):
1794
+ """2 (sum y log y - sum y eta + sum (y + r) log1p((mu - y) / (y + r))), as Deviance.total."""
1795
+ cross = 0.0
1796
+ tail = 0.0
1797
+ for i in range(y.shape[0]):
1798
+ cross += y[i] * eta[i]
1799
+ yr = y[i] + size
1800
+ tail += yr * math.log1p((mu[i] - y[i]) / yr)
1801
+ return 2.0 * (ylogy - cross + tail)
1802
+
1803
+
1804
+ @nb.njit(cache=True)
1805
+ def _structured_blocks(X, labels, w, wz, with_rhs, gram, cross, diag, rx, rg, ridge, ridge_group):
1806
+ """X'WX (+ ridge, one value per column), X'WG and diag(G'WG)
1807
+ (+ ridge_group) for the weights w, and with_rhs, X'Wz and G'Wz for
1808
+ wz = w z."""
1809
+ n, dx = X.shape
1810
+ gram[:, :] = 0.0
1811
+ cross[:, :] = 0.0
1812
+ diag[:] = 0.0
1813
+ rx[:] = 0.0
1814
+ rg[:] = 0.0
1815
+ for i in range(n):
1816
+ wi = w[i]
1817
+ k = labels[i]
1818
+ for j in range(dx):
1819
+ xw = X[i, j] * wi
1820
+ for l in range(j + 1):
1821
+ gram[j, l] += xw * X[i, l]
1822
+ if k >= 0:
1823
+ cross[j, k] += xw
1824
+ if with_rhs:
1825
+ rx[j] += X[i, j] * wz[i]
1826
+ if k >= 0:
1827
+ diag[k] += wi
1828
+ if with_rhs:
1829
+ rg[k] += wz[i]
1830
+ for j in range(dx):
1831
+ for l in range(j):
1832
+ gram[l, j] = gram[j, l]
1833
+ gram[j, j] += ridge[j]
1834
+ for k in range(diag.shape[0]):
1835
+ diag[k] += ridge_group
1836
+
1837
+
1838
+ @nb.njit(cache=True)
1839
+ def _solve_small(A, B):
1840
+ """A^-1 B for a small square A, by Gaussian elimination with partial
1841
+ pivoting (as LAPACK's gesv); NaN when A is singular."""
1842
+ m = A.shape[0]
1843
+ a = A.copy()
1844
+ x = B.copy()
1845
+ for c in range(m):
1846
+ p = c
1847
+ for r in range(c + 1, m):
1848
+ if abs(a[r, c]) > abs(a[p, c]):
1849
+ p = r
1850
+ if a[p, c] == 0.0:
1851
+ x[:, :] = np.nan
1852
+ return x
1853
+ if p != c:
1854
+ for j in range(m):
1855
+ t = a[c, j]
1856
+ a[c, j] = a[p, j]
1857
+ a[p, j] = t
1858
+ for j in range(x.shape[1]):
1859
+ t = x[c, j]
1860
+ x[c, j] = x[p, j]
1861
+ x[p, j] = t
1862
+ for r in range(c + 1, m):
1863
+ f = a[r, c] / a[c, c]
1864
+ if f != 0.0:
1865
+ for j in range(c, m):
1866
+ a[r, j] -= f * a[c, j]
1867
+ for j in range(x.shape[1]):
1868
+ x[r, j] -= f * x[c, j]
1869
+ for c in range(m - 1, -1, -1):
1870
+ for j in range(x.shape[1]):
1871
+ v = x[c, j]
1872
+ for k in range(c + 1, m):
1873
+ v -= a[c, k] * x[k, j]
1874
+ x[c, j] = v / a[c, c]
1875
+ return x
1876
+
1877
+
1878
+ @nb.njit(cache=True)
1879
+ def _schur(gram, cross, diag):
1880
+ """S = C - B D^-1 B'."""
1881
+ dx, a = cross.shape
1882
+ S = gram.copy()
1883
+ for j in range(dx):
1884
+ for l in range(dx):
1885
+ v = 0.0
1886
+ for k in range(a):
1887
+ v += cross[j, k] * cross[l, k] / diag[k]
1888
+ S[j, l] -= v
1889
+ return S
1890
+
1891
+
1892
+ @nb.njit(parallel=True, cache=True)
1893
+ def _structured_nb_irls(X, labels, offset, Y, alpha, n_groups, const_col, const_val, mean_exp_offset,
1894
+ eta_lo, eta_hi, wald_lo, floored, max_iter, tol, ridge, ridge_group, clip_group,
1895
+ max_halvings, eps):
1896
+ """StructuredGLMBatchFitter._irls and its standard errors at the fixed NB
1897
+ dispersions alpha, for each column of Y (Fortran order), one gene per
1898
+ thread; ridge is per covariate column, and n_groups may be 0 (a dense
1899
+ design). Returns coef (genes, dx + a: covariates, then groups), se,
1900
+ deviance, converged and n_iter."""
1901
+ n, dx = X.shape
1902
+ n_genes = Y.shape[1]
1903
+ a = n_groups
1904
+ coef = np.zeros((n_genes, dx + a))
1905
+ se = np.full((n_genes, dx + a), np.inf)
1906
+ dev_out = np.full(n_genes, np.nan)
1907
+ converged = np.zeros(n_genes, dtype=np.bool_)
1908
+ n_iter = np.zeros(n_genes, dtype=np.int32)
1909
+ for g in nb.prange(n_genes):
1910
+ y = Y[:, g]
1911
+ ag = alpha[g]
1912
+ size = 1.0 / min(max(ag, eps), 1e10)
1913
+ ylogy = 0.0
1914
+ total = 0.0
1915
+ for i in range(n):
1916
+ total += y[i]
1917
+ if y[i] > 0.0:
1918
+ ylogy += y[i] * math.log(y[i])
1919
+ bf = np.zeros(dx)
1920
+ bg = np.zeros(a)
1921
+ if const_col >= 0:
1922
+ bf[const_col] = math.log(max(total / n, 1e-3) / mean_exp_offset) / const_val
1923
+ eta = np.empty(n)
1924
+ mu = np.empty(n)
1925
+ _structured_predictor(X, labels, offset, bf, bg, eta_lo, eta_hi, eta, mu)
1926
+ dev = _structured_deviance(y, eta, mu, size, ylogy)
1927
+ w = np.empty(n)
1928
+ wz = np.empty(n)
1929
+ new_eta = np.empty(n)
1930
+ new_mu = np.empty(n)
1931
+ half_eta = np.empty(n)
1932
+ half_mu = np.empty(n)
1933
+ gram = np.empty((dx, dx))
1934
+ cross = np.empty((dx, a))
1935
+ diag = np.empty(a)
1936
+ rx = np.empty(dx)
1937
+ rg = np.empty(a)
1938
+ conv = False
1939
+ it = 0
1940
+ for iteration in range(1, max_iter + 1):
1941
+ # The Newton step at (eta, mu): weights, working response, normal equations.
1942
+ for i in range(n):
1943
+ wi = mu[i] / (1.0 + ag * mu[i])
1944
+ w[i] = wi
1945
+ wz[i] = wi * (eta[i] - offset[i] + (y[i] - mu[i]) / max(mu[i], eps))
1946
+ _structured_blocks(X, labels, w, wz, True, gram, cross, diag, rx, rg, ridge, ridge_group)
1947
+ S = _schur(gram, cross, diag)
1948
+ rhs = np.empty((dx, 1))
1949
+ for j in range(dx):
1950
+ v = rx[j]
1951
+ for k in range(a):
1952
+ v -= cross[j, k] / diag[k] * rg[k]
1953
+ rhs[j, 0] = v
1954
+ full_f = _solve_small(S, rhs)[:, 0].copy()
1955
+ full_g = np.empty(a)
1956
+ for k in range(a):
1957
+ v = rg[k]
1958
+ for j in range(dx):
1959
+ v -= cross[j, k] * full_f[j]
1960
+ full_g[k] = min(max(v / diag[k], -clip_group), clip_group)
1961
+ _structured_predictor(X, labels, offset, full_f, full_g, eta_lo, eta_hi, new_eta, new_mu)
1962
+ new_dev = _structured_deviance(y, new_eta, new_mu, size, ylogy)
1963
+ new_f = full_f.copy()
1964
+ new_g = full_g.copy()
1965
+
1966
+ # A gene resting on the mean floor takes the full step and is
1967
+ # judged by the deviance ratio (as the NumPy fitter).
1968
+ on_floor = False
1969
+ if floored:
1970
+ for i in range(n):
1971
+ if eta[i] <= eta_lo:
1972
+ on_floor = True
1973
+ break
1974
+ ceiling = dev + 1e-9 * abs(dev)
1975
+ rose = new_dev > ceiling
1976
+ worse = rose and not on_floor
1977
+ scale = max(abs(dev), 1e-8)
1978
+ excess = (new_dev - dev) / scale
1979
+ step = 1.0
1980
+ for _h in range(max_halvings):
1981
+ if not worse:
1982
+ break
1983
+ step *= 0.5
1984
+ half_f = bf + step * (full_f - bf)
1985
+ half_g = bg + step * (full_g - bg)
1986
+ _structured_predictor(X, labels, offset, half_f, half_g, eta_lo, eta_hi, half_eta, half_mu)
1987
+ half_dev = _structured_deviance(y, half_eta, half_mu, size, ylogy)
1988
+ excess = min(excess, (half_dev - dev) / scale)
1989
+ if half_dev <= ceiling:
1990
+ new_f = half_f
1991
+ new_g = half_g
1992
+ new_eta[:] = half_eta
1993
+ new_mu[:] = half_mu
1994
+ new_dev = half_dev
1995
+ worse = False
1996
+ fixed_point = rose and on_floor
1997
+ if worse:
1998
+ new_f = bf.copy()
1999
+ new_g = bg.copy()
2000
+ new_eta[:] = eta
2001
+ new_mu[:] = mu
2002
+ new_dev = dev
2003
+ relative_change = abs(new_dev - dev) / scale
2004
+ deviance_ratio = abs(new_dev - dev) / (abs(new_dev) + 0.1)
2005
+ coef_change = 0.0
2006
+ for j in range(dx):
2007
+ coef_change = max(coef_change, abs(new_f[j] - bf[j]))
2008
+ for k in range(a):
2009
+ coef_change = max(coef_change, abs(new_g[k] - bg[k]))
2010
+ bf = new_f
2011
+ bg = new_g
2012
+ eta[:] = new_eta
2013
+ mu[:] = new_mu
2014
+ dev = new_dev
2015
+ it = iteration
2016
+ if fixed_point:
2017
+ conv = deviance_ratio < tol and coef_change < tol
2018
+ elif worse:
2019
+ conv = excess < tol
2020
+ else:
2021
+ conv = relative_change < tol and coef_change < tol
2022
+ if conv or worse:
2023
+ break
2024
+
2025
+ # Standard errors from the blocks at the fit, with the unfloored mean.
2026
+ if floored:
2027
+ _structured_predictor(X, labels, offset, bf, bg, wald_lo, eta_hi, half_eta, half_mu)
2028
+ for i in range(n):
2029
+ w[i] = half_mu[i] / (1.0 + ag * half_mu[i])
2030
+ else:
2031
+ for i in range(n):
2032
+ w[i] = mu[i] / (1.0 + ag * mu[i])
2033
+ _structured_blocks(X, labels, w, wz, False, gram, cross, diag, rx, rg, ridge, ridge_group)
2034
+ S = _schur(gram, cross, diag)
2035
+ rhs = np.zeros((dx, dx + a))
2036
+ for j in range(dx):
2037
+ rhs[j, j] = 1.0
2038
+ for k in range(a):
2039
+ rhs[j, dx + k] = cross[j, k]
2040
+ solved = _solve_small(S, rhs)
2041
+ for j in range(dx):
2042
+ coef[g, j] = bf[j]
2043
+ se[g, j] = math.sqrt(max(solved[j, j], 0.0))
2044
+ for k in range(a):
2045
+ quad = 0.0
2046
+ for j in range(dx):
2047
+ quad += cross[j, k] * solved[j, dx + k]
2048
+ coef[g, dx + k] = bg[k]
2049
+ se[g, dx + k] = math.sqrt(max(1.0 / diag[k] + quad / (diag[k] * diag[k]), 0.0))
2050
+ dev_out[g] = dev
2051
+ converged[g] = conv
2052
+ n_iter[g] = it
2053
+ return coef, se, dev_out, converged, n_iter
@@ -20,6 +20,7 @@ from .data import (
20
20
  concat_h5ad,
21
21
  downsample_counts,
22
22
  normalize_total_log1p,
23
+ read_backed,
23
24
  resolve_data_path,
24
25
  )
25
26
  from .de import (
@@ -504,7 +505,7 @@ class _PreprocessingNamespace:
504
505
  from .dimred import pca as _pca
505
506
 
506
507
  if isinstance(data, (str, Path)):
507
- adata = ad.read_h5ad(data, backed='r')
508
+ adata = read_backed(data)
508
509
  else:
509
510
  adata = data
510
511
 
@@ -562,7 +563,7 @@ class _PreprocessingNamespace:
562
563
  from .dimred import neighbors as _neighbors
563
564
 
564
565
  if isinstance(data, (str, Path)):
565
- adata = ad.read_h5ad(data, backed='r')
566
+ adata = read_backed(data)
566
567
  else:
567
568
  adata = data
568
569
 
@@ -762,7 +763,7 @@ class _ToolsNamespace:
762
763
  from .dimred import umap as _umap
763
764
 
764
765
  if isinstance(data, (str, Path)):
765
- adata = ad.read_h5ad(data, backed='r')
766
+ adata = read_backed(data)
766
767
  else:
767
768
  adata = data
768
769
 
@@ -2,4 +2,4 @@
2
2
  the distribution's metadata reads it at build time, so a source checkout
3
3
  reports the code it runs, whatever version is installed."""
4
4
 
5
- __version__ = "0.2.3"
5
+ __version__ = "0.2.4"
@@ -682,7 +682,7 @@ def batch_process(
682
682
  # width that output was written with. Skipped under force=True,
683
683
  # which reads nothing from the existing output -- including when
684
684
  # it is too damaged to open, the case force exists for.
685
- existing = ad.read_h5ad(resolved_output, backed="r")
685
+ existing = read_backed(resolved_output)
686
686
  try:
687
687
  stored_chunk_size = existing.uns.get("chunk_size")
688
688
  finally:
@@ -735,7 +735,7 @@ def batch_process(
735
735
  # tell a finished result from one whose run was killed. Only the
736
736
  # completion marker, written after the last chunk, can.
737
737
  if resolved_output.exists() and not force:
738
- existing = ad.read_h5ad(resolved_output, backed="r")
738
+ existing = read_backed(resolved_output)
739
739
  try:
740
740
  complete = _run_is_complete(existing)
741
741
  finally:
@@ -792,7 +792,7 @@ def batch_process(
792
792
  f"gene chunk {last_completed_chunk} by scanning {resolved_output.name}",
793
793
  )
794
794
  if resume and last_completed_chunk >= 0 and resolved_output.exists():
795
- existing = ad.read_h5ad(resolved_output, backed="r")
795
+ existing = read_backed(resolved_output)
796
796
  try:
797
797
  # A finished output rejected above fails this too, so it is
798
798
  # recomputed rather than "resumed" with nothing left to do.
@@ -423,6 +423,43 @@ def read_backed(path: str | Path, mode: Literal["r", "r+"] = "r") -> ad.AnnData:
423
423
  return adata
424
424
 
425
425
 
426
+ def read_rows_csc(path: str | Path, rows: np.ndarray, chunk_rows: int = 4096) -> sp.csc_matrix:
427
+ """The rows ``rows`` (ascending) of ``path``'s ``X`` as an in-memory CSC
428
+ matrix, in the file's dtype, without stored zeros.
429
+
430
+ Built in two passes over the rows, ``chunk_rows`` at a time -- the
431
+ column counts, then the non-zeros scattered into place (as
432
+ :func:`convert_to_csc` transposes) -- so memory peaks at the result plus
433
+ one chunk, where reading CSR and converting holds both and more.
434
+ """
435
+ rows = np.asarray(rows)
436
+ starts = range(0, rows.size, chunk_rows)
437
+ backed = read_backed(path)
438
+ try:
439
+ n_cols = backed.n_vars
440
+
441
+ def chunks():
442
+ for start in starts:
443
+ chunk = sp.csr_matrix(backed.X[rows[start:start + chunk_rows], :])
444
+ chunk.eliminate_zeros()
445
+ yield start, chunk
446
+
447
+ counts = np.zeros(n_cols, dtype=np.int64)
448
+ dtype = None
449
+ for _, chunk in chunks():
450
+ counts += np.bincount(chunk.indices, minlength=n_cols)
451
+ dtype = chunk.dtype
452
+ indptr = np.concatenate([[0], np.cumsum(counts)])
453
+ data = np.empty(int(indptr[-1]), dtype=dtype or np.float64)
454
+ indices = np.empty(int(indptr[-1]), dtype=np.int32 if rows.size <= np.iinfo(np.int32).max else np.int64)
455
+ offset = indptr[:-1].copy()
456
+ for start, chunk in chunks():
457
+ _scatter_band(chunk.indptr, chunk.indices, chunk.data, start, 0, n_cols, offset, data, indices)
458
+ finally:
459
+ backed.file.close()
460
+ return sp.csc_matrix((data, indices, indptr), shape=(rows.size, n_cols))
461
+
462
+
426
463
  # Read into memory by read_backed, as anndata's backed mode does.
427
464
  _IN_MEMORY_SLOTS = ("obs", "var", "uns", "obsm", "varm", "obsp", "varp")
428
465
 
@@ -4847,7 +4884,7 @@ def standardize_dataset(
4847
4884
  }
4848
4885
 
4849
4886
  # Read obs/var metadata only (without loading X into memory)
4850
- adata = ad.read_h5ad(dataset_path, backed='r')
4887
+ adata = read_backed(dataset_path)
4851
4888
  obs_df = adata.obs.copy()
4852
4889
  var_df = adata.var.copy()
4853
4890
  uns_dict = dict(adata.uns) # shallow copy of uns