crispyx 0.2.5__tar.gz → 0.2.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. {crispyx-0.2.5/src/crispyx.egg-info → crispyx-0.2.7}/PKG-INFO +1 -1
  2. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_checkpoint.py +2 -0
  3. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_grouping.py +22 -0
  4. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_kernels.py +272 -25
  5. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_namespaces.py +23 -9
  6. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_provenance.py +4 -2
  7. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_statistics.py +26 -2
  8. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_version.py +1 -1
  9. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/data.py +401 -358
  10. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/de.py +484 -252
  11. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/glm.py +32 -19
  12. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/qc.py +73 -5
  13. {crispyx-0.2.5 → crispyx-0.2.7/src/crispyx.egg-info}/PKG-INFO +1 -1
  14. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx.egg-info/SOURCES.txt +3 -0
  15. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_de_resume.py +37 -4
  16. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_glm_structured.py +31 -0
  17. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_nb_glm.py +4 -6
  18. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_nb_offset.py +74 -2
  19. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_qc_parity.py +78 -0
  20. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_shrink_lfc_arrays.py +53 -0
  21. crispyx-0.2.7/tests/test_sort_by_perturbation.py +149 -0
  22. crispyx-0.2.7/tests/test_statistics.py +44 -0
  23. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_temp_file_cleanup.py +6 -9
  24. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_two_group_deseq2.py +5 -6
  25. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_wilcoxon_dispatch.py +1 -1
  26. crispyx-0.2.7/tests/test_wilcoxon_hypergeometric.py +115 -0
  27. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_wilcoxon_stratified.py +17 -15
  28. {crispyx-0.2.5 → crispyx-0.2.7}/LICENSE +0 -0
  29. {crispyx-0.2.5 → crispyx-0.2.7}/README.md +0 -0
  30. {crispyx-0.2.5 → crispyx-0.2.7}/pyproject.toml +0 -0
  31. {crispyx-0.2.5 → crispyx-0.2.7}/setup.cfg +0 -0
  32. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/__init__.py +0 -0
  33. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_disk.py +0 -0
  34. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_h5codec.py +0 -0
  35. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_irls.py +0 -0
  36. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_memory.py +0 -0
  37. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_messages.py +0 -0
  38. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_preflight.py +0 -0
  39. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/_size_factors.py +0 -0
  40. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/batch.py +0 -0
  41. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/dimred.py +0 -0
  42. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/hvg.py +0 -0
  43. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/plotting.py +0 -0
  44. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/profiling.py +0 -0
  45. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/pseudobulk.py +0 -0
  46. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/residual.py +0 -0
  47. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx/sample.py +0 -0
  48. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx.egg-info/dependency_links.txt +0 -0
  49. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx.egg-info/requires.txt +0 -0
  50. {crispyx-0.2.5 → crispyx-0.2.7}/src/crispyx.egg-info/top_level.txt +0 -0
  51. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_assign_guides.py +0 -0
  52. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_atomic_outputs.py +0 -0
  53. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_batch_process.py +0 -0
  54. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_cgroup_memory_budget.py +0 -0
  55. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_check_data.py +0 -0
  56. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_collapse_genes.py +0 -0
  57. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_compare_controls.py +0 -0
  58. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_compress_h5ad.py +0 -0
  59. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_concat_h5ad.py +0 -0
  60. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_convert_format.py +0 -0
  61. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_data_helpers.py +0 -0
  62. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_de_edge_cases.py +0 -0
  63. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_de_lowexpr_filter.py +0 -0
  64. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_dimred.py +0 -0
  65. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_disk_dispatch.py +0 -0
  66. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_downsample_counts.py +0 -0
  67. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_groupby_alias.py +0 -0
  68. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_hvg.py +0 -0
  69. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_in_memory_input.py +0 -0
  70. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_irls.py +0 -0
  71. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_memory_dispatch.py +0 -0
  72. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_messages.py +0 -0
  73. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_normalisation.py +0 -0
  74. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_normalized_effects.py +0 -0
  75. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_normalized_effects_se.py +0 -0
  76. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_obs_writes.py +0 -0
  77. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_pickle.py +0 -0
  78. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_plotting.py +0 -0
  79. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_preflight_dispatch.py +0 -0
  80. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_profiling.py +0 -0
  81. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_pseudobulk_batch.py +0 -0
  82. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_pseudobulk_memory.py +0 -0
  83. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_pseudobulk_profiles.py +0 -0
  84. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_pseudobulk_sparse_output.py +0 -0
  85. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_rank_genes_groups_outputs.py +0 -0
  86. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_read_backed.py +0 -0
  87. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_recover_counts.py +0 -0
  88. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_residual_effects.py +0 -0
  89. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_result_provenance.py +0 -0
  90. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_scanpy_parity.py +0 -0
  91. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_standardize.py +0 -0
  92. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_stream_on_fast_axis.py +0 -0
  93. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_subsample.py +0 -0
  94. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_wilcoxon_labels.py +0 -0
  95. {crispyx-0.2.5 → crispyx-0.2.7}/tests/test_workflow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: crispyx
3
- Version: 0.2.5
3
+ Version: 0.2.7
4
4
  Summary: Memory-efficient streaming analysis of large-scale CRISPR and Perturb-seq screens on disk-backed AnnData files
5
5
  Author: Jin-Hong Du
6
6
  License: Modified MIT License
@@ -253,6 +253,8 @@ def fingerprint_difference(previous: dict, current: dict) -> str | None:
253
253
  return "the input file has changed since"
254
254
  if previous.get("inputs") != current.get("inputs"):
255
255
  return "an input file has changed since"
256
+ if previous.get("crispyx") != current.get("crispyx"):
257
+ return f"it was written by crispyx {previous.get('crispyx', 'before 0.2.6')}"
256
258
  old_params = previous.get("params", {})
257
259
  new_params = current.get("params", {})
258
260
  for name in sorted(set(old_params) | set(new_params)):
@@ -103,3 +103,25 @@ def cell_labels(obs: pd.DataFrame, labels: str | pd.Series) -> tuple[pd.Series,
103
103
  def label_strings(labels: pd.Series) -> np.ndarray:
104
104
  """Per-cell labels as strings, missing ones as :data:`UNLABELLED`."""
105
105
  return labels.astype(object).where(labels.notna(), UNLABELLED).astype(str).to_numpy()
106
+
107
+
108
+ def contiguous_ranges(labels: np.ndarray) -> dict[str, slice]:
109
+ """Each label whose cells form one contiguous run of rows, as that run's
110
+ ``slice``. Labels split over several runs are left out, so a file is
111
+ sorted by ``labels`` exactly when every distinct label is a key.
112
+
113
+ Derived from the labels rather than stored with the file: a map stored
114
+ in ``uns`` is one HDF5 dataset per label, which every backed open reads
115
+ (over a second for a genome-wide screen) and which can go stale."""
116
+ codes, uniques = pd.factorize(np.asarray(labels), use_na_sentinel=False)
117
+ if not codes.size:
118
+ return {}
119
+ starts = np.flatnonzero(np.r_[True, codes[1:] != codes[:-1]])
120
+ stops = np.r_[starts[1:], codes.size]
121
+ run_codes = codes[starts]
122
+ single = np.bincount(run_codes, minlength=len(uniques)) == 1
123
+ return {
124
+ str(uniques[code]): slice(int(start), int(stop))
125
+ for code, start, stop in zip(run_codes, starts, stops)
126
+ if single[code]
127
+ }
@@ -1,8 +1,8 @@
1
- """Numba-accelerated kernels for NB-GLM differential expression.
1
+ """Numba-accelerated kernels for differential expression.
2
2
 
3
- This module contains all Numba JIT-compiled functions used by the GLM module.
4
- Separating these kernels improves code organization and allows for easier
5
- maintenance of the performance-critical code paths.
3
+ It holds the Numba JIT-compiled functions of the NB-GLM fits, the Wilcoxon
4
+ rank sums and the t-test's per-group moments, kept apart from the Python
5
+ modules that call them.
6
6
  """
7
7
 
8
8
  from __future__ import annotations
@@ -94,8 +94,8 @@ def _nb_loglik_grid_numba(
94
94
  def _neg_half_log_det_numba(xtwx: np.ndarray) -> float:
95
95
  """``-0.5 * log det(xtwx)`` from a Cholesky factorisation in place.
96
96
 
97
- Zero for an empty matrix (no Cox-Reid adjustment), or one that is not
98
- positive definite (as the gene-wise Cox-Reid step treats it).
97
+ Zero for an empty matrix (no Cox-Reid adjustment); NaN for one that is
98
+ not positive definite.
99
99
  """
100
100
  log_det = 0.0
101
101
  for j in range(xtwx.shape[0]):
@@ -103,7 +103,7 @@ def _neg_half_log_det_numba(xtwx: np.ndarray) -> float:
103
103
  for k in range(j):
104
104
  d -= xtwx[j, k] * xtwx[j, k]
105
105
  if d <= 0.0:
106
- return 0.0
106
+ return np.nan
107
107
  ljj = math.sqrt(d)
108
108
  xtwx[j, j] = ljj
109
109
  log_det += math.log(ljj)
@@ -132,6 +132,9 @@ def _brent_minimize_numba(
132
132
  x_indices: np.ndarray,
133
133
  x_data: np.ndarray,
134
134
  xtwx: np.ndarray,
135
+ x_group: np.ndarray,
136
+ gdiag: np.ndarray,
137
+ gcross: np.ndarray,
135
138
  a: float,
136
139
  b: float,
137
140
  tol: float = 1e-5,
@@ -149,7 +152,7 @@ def _brent_minimize_numba(
149
152
  x = w = v = a + golden * (b - a)
150
153
  fx = fw = fv = -_nb_posterior_with_cache_numba(
151
154
  Y_g, mu_g, x, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
152
- xtwx,
155
+ xtwx, x_group, gdiag, gcross,
153
156
  )
154
157
 
155
158
  d = 0.0 # Distance to next point
@@ -206,7 +209,7 @@ def _brent_minimize_numba(
206
209
 
207
210
  fu = -_nb_posterior_with_cache_numba(
208
211
  Y_g, mu_g, u, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
209
- xtwx,
212
+ xtwx, x_group, gdiag, gcross,
210
213
  )
211
214
 
212
215
  # Update a, b, v, w, x
@@ -253,14 +256,23 @@ def _nb_posterior_with_cache_numba(
253
256
  x_indices: np.ndarray,
254
257
  x_data: np.ndarray,
255
258
  xtwx: np.ndarray,
259
+ x_group: np.ndarray,
260
+ gdiag: np.ndarray,
261
+ gcross: np.ndarray,
256
262
  ) -> float:
257
263
  """Cox-Reid adjusted NB log-likelihood plus log-prior for one gene.
258
264
 
259
- Uses the precomputed ``gammaln(Y+1)`` sum. ``XᵀWX``, ``W = mu / (1 +
260
- alpha mu)``, is accumulated into the work buffer ``xtwx`` (``(0, 0)``
261
- for no adjustment) in the same pass over the cells as the likelihood;
262
- the design ``X`` is given in CSR form, so a one-hot batch design costs a
263
- few products per cell rather than ``n_features**2``. ``small_counts[k]``
265
+ Uses the precomputed ``gammaln(Y+1)`` sum. The design is ``[F | G]``:
266
+ ``F`` in CSR form (``x_indptr``, ``x_indices``, ``x_data``), and ``G``
267
+ one-hot, cell ``i`` in column ``x_group[i]`` (-1 for none). With ``W =
268
+ mu / (1 + alpha mu)``, ``FᵀWF`` is accumulated into the work buffer
269
+ ``xtwx`` (``(0, 0)`` for no adjustment), the diagonal ``GᵀWG`` into
270
+ ``gdiag`` and ``GᵀWF`` into ``gcross``, in the same pass over the cells
271
+ as the likelihood; then ``log det(XᵀWX) = sum log gdiag + log det(FᵀWF -
272
+ GᵀWFᵀ diag(gdiag)⁻¹ GᵀWF)``, the Schur complement of the one-hot block,
273
+ so hundreds of stratum columns cost ``n_groups x n_F**2`` rather than a
274
+ factorisation of the whole ``XᵀWX``. The adjustment is zero when
275
+ ``XᵀWX`` is not positive definite. ``small_counts[k]``
264
276
  is the number of cells with count ``k`` (below its length): their
265
277
  ``lgamma(y + r) - lgamma(r)`` terms depend on the count only, so each
266
278
  value's is formed once, by recurrence, rather than once per cell.
@@ -271,6 +283,10 @@ def _nb_posterior_with_cache_numba(
271
283
  log_r = math.log(r)
272
284
  cox_reid = xtwx.shape[0] > 0
273
285
  xtwx[:, :] = 0.0
286
+ n_groups = gdiag.shape[0]
287
+ if n_groups:
288
+ gdiag[:] = 0.0
289
+ gcross[:, :] = 0.0
274
290
 
275
291
  # lgamma(y + r) - lgamma(r) - y log r, without libm's lgamma (see
276
292
  # _lgamma_ratio_term): sum(log1p(i / r), i < y) for the small counts.
@@ -296,10 +312,31 @@ def _nb_posterior_with_cache_numba(
296
312
  ja = x_indices[a]
297
313
  for b in range(x_indptr[i], x_indptr[i + 1]):
298
314
  xtwx[ja, x_indices[b]] += wa * x_data[b]
315
+ if n_groups and x_group[i] >= 0:
316
+ j = x_group[i]
317
+ gdiag[j] += w
318
+ for a in range(x_indptr[i], x_indptr[i + 1]):
319
+ gcross[j, x_indices[a]] += w * x_data[a]
299
320
  ll -= gammaln_y_plus_1_sum
300
321
 
301
322
  log_prior = -0.5 * (log_alpha - log_trend_g) ** 2 / prior_var
302
- return ll + log_prior + _neg_half_log_det_numba(xtwx)
323
+ if not cox_reid:
324
+ return ll + log_prior
325
+ adjustment = 0.0
326
+ for j in range(n_groups):
327
+ dj = gdiag[j]
328
+ if dj <= 0.0:
329
+ return ll + log_prior # XᵀWX is not positive definite
330
+ adjustment -= 0.5 * math.log(dj)
331
+ for a in range(xtwx.shape[0]):
332
+ ca = gcross[j, a] / dj
333
+ if ca != 0.0:
334
+ for b in range(xtwx.shape[0]):
335
+ xtwx[a, b] -= ca * gcross[j, b]
336
+ schur = _neg_half_log_det_numba(xtwx)
337
+ if math.isnan(schur):
338
+ return ll + log_prior
339
+ return ll + log_prior + adjustment + schur
303
340
 
304
341
 
305
342
  @nb.njit(parallel=True, cache=True)
@@ -315,6 +352,8 @@ def _nb_map_grid_search_with_refinement_numba(
315
352
  x_indices: np.ndarray,
316
353
  x_data: np.ndarray,
317
354
  n_features: int,
355
+ x_group: np.ndarray,
356
+ n_groups: int,
318
357
  tol: float = 1e-5,
319
358
  max_refine_iter: int = 50,
320
359
  ) -> np.ndarray:
@@ -339,8 +378,12 @@ def _nb_map_grid_search_with_refinement_numba(
339
378
  Variance of the log-normal prior (``inf`` for none: the gene-wise
340
379
  estimate).
341
380
  x_indptr, x_indices, x_data, n_features
342
- The design matrix in CSR form, for the Cox-Reid adjustment
343
- (``n_features=0`` for none).
381
+ The design's columns outside its one-hot block in CSR form, for the
382
+ Cox-Reid adjustment (``n_features=0`` for none).
383
+ x_group, n_groups
384
+ Each cell's column of the one-hot block (-1 for none), and the
385
+ block's width (0 for no block): see
386
+ :func:`_nb_posterior_with_cache_numba`.
344
387
  tol : float
345
388
  Tolerance for Brent convergence.
346
389
  max_refine_iter : int
@@ -358,6 +401,8 @@ def _nb_map_grid_search_with_refinement_numba(
358
401
  Y_g = Y[:, g]
359
402
  mu_g = mu[:, g]
360
403
  xtwx = np.empty((n_features, n_features))
404
+ gdiag = np.empty(n_groups)
405
+ gcross = np.empty((n_groups, n_features))
361
406
  small_counts = np.zeros(_SMALL_COUNTS)
362
407
  for i in range(Y_g.shape[0]):
363
408
  if Y_g[i] < _SMALL_COUNTS:
@@ -369,7 +414,7 @@ def _nb_map_grid_search_with_refinement_numba(
369
414
  for k in range(n_grid):
370
415
  posterior = _nb_posterior_with_cache_numba(
371
416
  Y_g, mu_g, lo[g] + k * step, log_trend_g, prior_var, 0.0, small_counts,
372
- x_indptr, x_indices, x_data, xtwx,
417
+ x_indptr, x_indices, x_data, xtwx, x_group, gdiag, gcross,
373
418
  )
374
419
  if posterior > best_posterior:
375
420
  best_posterior = posterior
@@ -378,11 +423,11 @@ def _nb_map_grid_search_with_refinement_numba(
378
423
  if 0 < best_k < n_grid - 1 and 2 * step > tol:
379
424
  refined = _brent_minimize_numba(
380
425
  Y_g, mu_g, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data, xtwx,
381
- best - step, best + step, tol=tol, max_iter=max_refine_iter,
426
+ x_group, gdiag, gcross, best - step, best + step, tol=tol, max_iter=max_refine_iter,
382
427
  )
383
428
  refined_posterior = _nb_posterior_with_cache_numba(
384
429
  Y_g, mu_g, refined, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data,
385
- xtwx,
430
+ xtwx, x_group, gdiag, gcross,
386
431
  )
387
432
  if refined_posterior >= best_posterior:
388
433
  best = refined
@@ -498,6 +543,148 @@ def _compute_ctrl_tie_sums(
498
543
  return ctrl_tie_sums
499
544
 
500
545
 
546
+ # The NB IRLS fits measure a deviance change relative to |deviance| plus
547
+ # this, DESeq2's ratio: a saturated fit's deviance near zero still converges.
548
+ _DEVIANCE_OFFSET = 0.1
549
+
550
+
551
+ # The rank-sum p-value treats the number of expressing cells that fall in the
552
+ # perturbation group as hypergeometric whenever its variance, summed over
553
+ # strata, is below this; above it the normal approximation is used.
554
+ _RANK_SUM_MIXTURE_MAX_VAR = 100.0
555
+ # The most components the strata's combined mixture keeps; beyond it,
556
+ # components with nearby means are merged.
557
+ _RANK_SUM_MIXTURE_MAX_COMPONENTS = 1 << 15
558
+
559
+
560
+ @nb.njit(cache=True)
561
+ def _merge_mixture(w: np.ndarray, m: np.ndarray, v: np.ndarray, n_bins: int) -> tuple:
562
+ """Merge a normal mixture's components into at most ``n_bins`` by their
563
+ mean (equal-width bins over its range), each bin a normal with the
564
+ bin's weight, mean and variance."""
565
+ lo = m.min()
566
+ span = m.max() - lo
567
+ bins = np.zeros(m.shape[0], dtype=np.int64)
568
+ if span > 0.0:
569
+ for i in range(m.shape[0]):
570
+ bins[i] = min(int((m[i] - lo) / span * n_bins), n_bins - 1)
571
+ bw = np.zeros(n_bins)
572
+ bm = np.zeros(n_bins)
573
+ for i in range(m.shape[0]):
574
+ bw[bins[i]] += w[i]
575
+ bm[bins[i]] += w[i] * m[i]
576
+ for b in range(n_bins):
577
+ if bw[b] > 0.0:
578
+ bm[b] /= bw[b]
579
+ bv = np.zeros(n_bins)
580
+ for i in range(m.shape[0]):
581
+ d = m[i] - bm[bins[i]]
582
+ bv[bins[i]] += w[i] * (v[i] + d * d)
583
+ keep = np.flatnonzero(bw > 0.0)
584
+ return bw[keep], bm[keep], bv[keep] / bw[keep]
585
+
586
+
587
+ @nb.njit(cache=True)
588
+ def _rank_sum_mixture_pvalue(
589
+ n1s: np.ndarray, ns: np.ndarray, ks: np.ndarray, nz_tie_sums: np.ndarray, deviation: float,
590
+ max_components: int = _RANK_SUM_MIXTURE_MAX_COMPONENTS,
591
+ ) -> float:
592
+ """Two-sided p-value of a (stratified) rank sum when most values are tied
593
+ at zero.
594
+
595
+ Stratum ``b`` has ``ns[b]`` cells, ``n1s[b]`` of them in the group and
596
+ ``ks[b]`` non-zero, whose ties give ``nz_tie_sums[b] = sum(t^3 - t)``;
597
+ ``deviation`` is ``sum_b (R_b - E_b)``. Under the permutation null the
598
+ group's count of non-zero cells ``J_b`` is hypergeometric: with few
599
+ non-zeros it is very skewed, which the normal approximation misses by
600
+ orders of magnitude (one expressing cell of 232 against 3 of 30,000
601
+ controls: p 3e-8 against a permutation p of 0.03). Here ``J_b`` keeps
602
+ its exact distribution, and given ``J_b = j`` the rank sum is normal
603
+ with the exact mean and variance of ``j`` draws without replacement from
604
+ the non-zero ranks. The strata's
605
+ mixtures are combined exactly up to ``max_components`` components;
606
+ past it, :func:`_merge_mixture` merges those with nearby means, keeping
607
+ each merged component's weight, mean and variance.
608
+ """
609
+ w = np.ones(1)
610
+ m = np.zeros(1)
611
+ v = np.zeros(1)
612
+ for b in range(n1s.shape[0]):
613
+ n1 = n1s[b]
614
+ n = ns[b]
615
+ k = ks[b]
616
+ z = n - k
617
+ if n1 == 0 or n1 == n:
618
+ continue
619
+ r0 = (z + 1.0) / 2.0
620
+ mean_nz = z + (k + 1.0) / 2.0
621
+ var_nz = ((k * k - 1.0) / 12.0 - nz_tie_sums[b] / (12.0 * k)) if k > 1 else 0.0
622
+ expected = n1 * (n + 1.0) / 2.0
623
+ lo = max(0, n1 - z)
624
+ hi = min(k, n1)
625
+ # Start at the mode and walk out by the ratio of successive terms.
626
+ mode = min(max(int((n1 + 1.0) * (k + 1.0) / (n + 2.0)), lo), hi)
627
+ log_mode = (math.lgamma(k + 1.0) - math.lgamma(mode + 1.0) - math.lgamma(k - mode + 1.0)
628
+ + math.lgamma(z + 1.0) - math.lgamma(n1 - mode + 1.0) - math.lgamma(z - n1 + mode + 1.0)
629
+ - math.lgamma(n + 1.0) + math.lgamma(n1 + 1.0) + math.lgamma(n - n1 + 1.0))
630
+ width = hi - lo + 1
631
+ pmf = np.zeros(width)
632
+ pmf[mode - lo] = math.exp(log_mode)
633
+ for j in range(mode, hi):
634
+ pmf[j + 1 - lo] = pmf[j - lo] * (k - j) * (n1 - j) / ((j + 1.0) * (z - n1 + j + 1.0))
635
+ if pmf[j + 1 - lo] < 1e-300:
636
+ break
637
+ for j in range(mode, lo, -1):
638
+ pmf[j - 1 - lo] = pmf[j - lo] * j * (z - n1 + j) / ((k - j + 1.0) * (n1 - j + 1.0))
639
+ if pmf[j - 1 - lo] < 1e-300:
640
+ break
641
+ c = 0
642
+ for i in range(width):
643
+ if pmf[i] >= 1e-300:
644
+ c += 1
645
+ if c == 1:
646
+ j = mode
647
+ m += (n1 - j) * r0 + j * mean_nz - expected
648
+ v += j * (k - j) / (k - 1.0) * var_nz if k > 1 else 0.0
649
+ continue
650
+ if w.shape[0] * c > max_components:
651
+ w, m, v = _merge_mixture(w, m, v, max(max_components // c, 1))
652
+ nw = np.empty(w.shape[0] * c)
653
+ nm = np.empty(w.shape[0] * c)
654
+ nv = np.empty(w.shape[0] * c)
655
+ out = 0
656
+ for i in range(width):
657
+ if pmf[i] < 1e-300:
658
+ continue
659
+ j = lo + i
660
+ cm = (n1 - j) * r0 + j * mean_nz - expected
661
+ cv = j * (k - j) / (k - 1.0) * var_nz if k > 1 else 0.0
662
+ for a in range(w.shape[0]):
663
+ nw[out] = w[a] * pmf[i]
664
+ nm[out] = m[a] + cm
665
+ nv[out] = v[a] + cv
666
+ out += 1
667
+ w, m, v = nw, nm, nv
668
+ d = abs(deviation)
669
+ tol = 1e-9 * (1.0 + d)
670
+ total = 0.0
671
+ for i in range(w.shape[0]):
672
+ if v[i] > 0.0:
673
+ sd = math.sqrt(2.0 * v[i])
674
+ total += w[i] * 0.5 * (math.erfc((d - m[i]) / sd) + math.erfc((d + m[i]) / sd))
675
+ elif m[i] >= d - tol or m[i] <= -d + tol:
676
+ total += w[i]
677
+ return min(total, 1.0)
678
+
679
+
680
+ @nb.njit(cache=True)
681
+ def _hypergeometric_variance(n1: float, n: float, k: float) -> float:
682
+ """Variance of the group's count of non-zero cells under the permutation null."""
683
+ if n <= 1.0:
684
+ return 0.0
685
+ return n1 * (k / n) * ((n - k) / n) * ((n - n1) / (n - 1.0))
686
+
687
+
501
688
  @nb.njit(cache=True)
502
689
  def _all_tied(n_zeros: int, ctrl_sorted: np.ndarray, pert_sorted: np.ndarray) -> bool:
503
690
  """Whether every value of a gene -- ``n_zeros`` zeros plus the sorted
@@ -551,6 +738,8 @@ def _rank_sum_pert_bsearch_numba(
551
738
  ``n_pert_zeros * zero_avg_rank`` is added by the caller).
552
739
  tie_corr : float
553
740
  Tie correction factor: 1 - sum(t^3 - t) / (n_total^3 - n_total).
741
+ tie_sum : float
742
+ ``sum(t^3 - t)`` over every tie group, the zeros included.
554
743
  """
555
744
  n_ctrl_nz = ctrl_sorted.shape[0]
556
745
  n_pert_nz = pert_sorted.shape[0]
@@ -614,7 +803,7 @@ def _rank_sum_pert_bsearch_numba(
614
803
  else:
615
804
  tie_corr = 1.0
616
805
 
617
- return rank_sum, tie_corr
806
+ return rank_sum, tie_corr, tie_sum
618
807
 
619
808
 
620
809
  @nb.njit(parallel=False, cache=True)
@@ -628,6 +817,7 @@ def _wilcoxon_single_pert_presorted(
628
817
  pert_dense: np.ndarray,
629
818
  valid_genes: np.ndarray,
630
819
  tie_correct: bool,
820
+ hypergeometric: bool,
631
821
  zero_threshold: float,
632
822
  u_stat_out: np.ndarray,
633
823
  z_score_out: np.ndarray,
@@ -673,6 +863,7 @@ def _wilcoxon_single_pert_presorted(
673
863
  tied = n_zeros == n_total
674
864
  rank_sum = 0.0
675
865
  tie_corr = 1.0
866
+ tie_sum = 0.0
676
867
  if not tied:
677
868
  # --- Binary-search ranking (always) ---
678
869
  # O(n_pert_nz * log(n_ctrl_nz)) — works for any zero fraction.
@@ -693,7 +884,7 @@ def _wilcoxon_single_pert_presorted(
693
884
  ctrl_sorted = ctrl_sorted_flat[start : start + n_ctrl_nz]
694
885
 
695
886
  tied = _all_tied(n_zeros, ctrl_sorted, pert_sorted)
696
- rank_sum_nz, tie_corr = _rank_sum_pert_bsearch_numba(
887
+ rank_sum_nz, tie_corr, tie_sum = _rank_sum_pert_bsearch_numba(
697
888
  ctrl_sorted, pert_sorted, n_zeros, ctrl_tie_sums[g]
698
889
  )
699
890
 
@@ -722,6 +913,13 @@ def _wilcoxon_single_pert_presorted(
722
913
  z = (rank_sum - expected) / std
723
914
  abs_z = abs(z)
724
915
  pval = math.erfc(abs_z / math.sqrt(2.0))
916
+ n_nonzero = n_total - n_zeros
917
+ if hypergeometric and _hypergeometric_variance(n_pert_f, n_total_f, float(n_nonzero)) < _RANK_SUM_MIXTURE_MAX_VAR:
918
+ zf = float(n_zeros)
919
+ pval = _rank_sum_mixture_pvalue(
920
+ np.array([n_pert]), np.array([n_total]), np.array([n_nonzero]),
921
+ np.array([tie_sum - (zf ** 3 - zf)]), rank_sum - expected,
922
+ )
725
923
  else:
726
924
  # An empty group: no comparison was made.
727
925
  z = np.nan
@@ -753,6 +951,7 @@ def _wilcoxon_batch_perts_presorted_numba(
753
951
  pert_row_offsets: np.ndarray,
754
952
  valid_masks: np.ndarray,
755
953
  tie_correct: bool,
954
+ hypergeometric: bool,
756
955
  zero_threshold: float,
757
956
  u_stat_out: np.ndarray,
758
957
  z_score_out: np.ndarray,
@@ -788,6 +987,9 @@ def _wilcoxon_batch_perts_presorted_numba(
788
987
  valid_masks : (n_perts, n_valid_genes)
789
988
  Boolean validity mask per (pert, gene).
790
989
  tie_correct : bool
990
+ hypergeometric : bool
991
+ Whether p-values come from :func:`_rank_sum_mixture_pvalue` where the
992
+ group's count of non-zero cells is skewed.
791
993
  zero_threshold : float
792
994
  u_stat_out : (n_perts, n_valid_genes)
793
995
  z_score_out : (n_perts, n_valid_genes)
@@ -811,6 +1013,7 @@ def _wilcoxon_batch_perts_presorted_numba(
811
1013
  pert_dense,
812
1014
  valid_masks[p_idx],
813
1015
  tie_correct,
1016
+ hypergeometric,
814
1017
  zero_threshold,
815
1018
  u_stat_out[p_idx],
816
1019
  z_score_out[p_idx],
@@ -833,6 +1036,7 @@ def _wilcoxon_stratified_single_pert(
833
1036
  seg_hi: int,
834
1037
  valid_genes: np.ndarray,
835
1038
  tie_correct: bool,
1039
+ hypergeometric: bool,
836
1040
  u_stat_out: np.ndarray,
837
1041
  z_score_out: np.ndarray,
838
1042
  pvalue_out: np.ndarray,
@@ -867,6 +1071,11 @@ def _wilcoxon_stratified_single_pert(
867
1071
  """
868
1072
  n_genes = all_pert_stacked.shape[1]
869
1073
  sqrt2 = math.sqrt(2.0)
1074
+ n_seg = seg_hi - seg_lo
1075
+ strata_n1 = np.empty(n_seg, dtype=np.int64)
1076
+ strata_n = np.empty(n_seg, dtype=np.int64)
1077
+ strata_k = np.empty(n_seg, dtype=np.int64)
1078
+ strata_ties = np.empty(n_seg, dtype=np.float64)
870
1079
 
871
1080
  for g in range(n_genes):
872
1081
  if not valid_genes[g]:
@@ -881,6 +1090,8 @@ def _wilcoxon_stratified_single_pert(
881
1090
  u_sum = 0.0 # sum_b U_b
882
1091
  n1n0_sum = 0.0 # sum_b n1_b * n0_b
883
1092
  tied = True # every contributing stratum has a single value
1093
+ var_j = 0.0 # sum_b Var(J_b): the group's count of non-zero cells
1094
+ n_used = 0
884
1095
 
885
1096
  for s in range(seg_lo, seg_hi):
886
1097
  b = seg_batch[s]
@@ -919,9 +1130,16 @@ def _wilcoxon_stratified_single_pert(
919
1130
 
920
1131
  if not _all_tied(n_zeros, ctrl_sorted, pert_sorted):
921
1132
  tied = False
922
- rank_sum_nz, tie_corr = _rank_sum_pert_bsearch_numba(
1133
+ rank_sum_nz, tie_corr, tie_sum = _rank_sum_pert_bsearch_numba(
923
1134
  ctrl_sorted, pert_sorted, n_zeros, ctrl_tie_sums[b, g]
924
1135
  )
1136
+ zf = float(n_zeros)
1137
+ strata_n1[n_used] = n1
1138
+ strata_n[n_used] = N
1139
+ strata_k[n_used] = N - n_zeros
1140
+ strata_ties[n_used] = tie_sum - (zf ** 3 - zf)
1141
+ n_used += 1
1142
+ var_j += _hypergeometric_variance(float(n1), float(N), float(N - n_zeros))
925
1143
  if not tie_correct:
926
1144
  tie_corr = 1.0
927
1145
 
@@ -943,6 +1161,10 @@ def _wilcoxon_stratified_single_pert(
943
1161
  if var > 0.0 and not tied:
944
1162
  z = num / math.sqrt(var)
945
1163
  pval = math.erfc(abs(z) / sqrt2)
1164
+ if hypergeometric and var_j < _RANK_SUM_MIXTURE_MAX_VAR:
1165
+ pval = _rank_sum_mixture_pvalue(
1166
+ strata_n1[:n_used], strata_n[:n_used], strata_k[:n_used], strata_ties[:n_used], num,
1167
+ )
946
1168
  else:
947
1169
  # No stratum to compare in, or every value tied within each:
948
1170
  # the rank test is undefined (with or without the tie
@@ -977,6 +1199,7 @@ def _wilcoxon_stratified_batch_perts_numba(
977
1199
  pert_ptr: np.ndarray,
978
1200
  valid_masks: np.ndarray,
979
1201
  tie_correct: bool,
1202
+ hypergeometric: bool,
980
1203
  u_stat_out: np.ndarray,
981
1204
  z_score_out: np.ndarray,
982
1205
  pvalue_out: np.ndarray,
@@ -1006,6 +1229,8 @@ def _wilcoxon_stratified_batch_perts_numba(
1006
1229
  perturbation ``p``.
1007
1230
  valid_masks : (n_perts, n_valid_genes)
1008
1231
  tie_correct : bool
1232
+ hypergeometric : bool
1233
+ As in :func:`_wilcoxon_batch_perts_presorted_numba`.
1009
1234
  u_stat_out, z_score_out, pvalue_out, effect_out : (n_perts, n_valid_genes)
1010
1235
  """
1011
1236
  n_perts = pert_ptr.shape[0] - 1
@@ -1024,6 +1249,7 @@ def _wilcoxon_stratified_batch_perts_numba(
1024
1249
  pert_ptr[p_idx + 1],
1025
1250
  valid_masks[p_idx],
1026
1251
  tie_correct,
1252
+ hypergeometric,
1027
1253
  u_stat_out[p_idx],
1028
1254
  z_score_out[p_idx],
1029
1255
  pvalue_out[p_idx],
@@ -1974,7 +2200,7 @@ def _structured_nb_irls(X, labels, offset, Y, alpha, n_groups, const_col, const_
1974
2200
  ceiling = dev + 1e-9 * abs(dev)
1975
2201
  rose = new_dev > ceiling
1976
2202
  worse = rose and not on_floor
1977
- scale = max(abs(dev), 1e-8)
2203
+ scale = abs(dev) + _DEVIANCE_OFFSET
1978
2204
  excess = (new_dev - dev) / scale
1979
2205
  step = 1.0
1980
2206
  for _h in range(max_halvings):
@@ -2001,7 +2227,7 @@ def _structured_nb_irls(X, labels, offset, Y, alpha, n_groups, const_col, const_
2001
2227
  new_mu[:] = mu
2002
2228
  new_dev = dev
2003
2229
  relative_change = abs(new_dev - dev) / scale
2004
- deviance_ratio = abs(new_dev - dev) / (abs(new_dev) + 0.1)
2230
+ deviance_ratio = abs(new_dev - dev) / (abs(new_dev) + _DEVIANCE_OFFSET)
2005
2231
  coef_change = 0.0
2006
2232
  for j in range(dx):
2007
2233
  coef_change = max(coef_change, abs(new_f[j] - bf[j]))
@@ -2051,3 +2277,24 @@ def _structured_nb_irls(X, labels, offset, Y, alpha, n_groups, const_col, const_
2051
2277
  converged[g] = conv
2052
2278
  n_iter[g] = it
2053
2279
  return coef, se, dev_out, converged, n_iter
2280
+
2281
+
2282
+ @nb.njit(cache=True, nogil=True)
2283
+ def _group_moments_csr(indptr, indices, data, codes, sums, sumsq, expr_counts, counts):
2284
+ """Add each group's per-gene sum and sum of squares (in float64), its
2285
+ stored-entry count and its cell count over CSR rows into the
2286
+ accumulators; row ``i`` is in group ``codes[i]``, ``-1`` rows left out.
2287
+
2288
+ One pass over the stored entries whatever the row order, and each group
2289
+ summed in row order -- the same on a file sorted by group as unsorted."""
2290
+ for row in range(codes.shape[0]):
2291
+ code = codes[row]
2292
+ if code < 0:
2293
+ continue
2294
+ counts[code] += 1
2295
+ for entry in range(indptr[row], indptr[row + 1]):
2296
+ gene = indices[entry]
2297
+ value = np.float64(data[entry])
2298
+ sums[code, gene] += value
2299
+ sumsq[code, gene] += value * value
2300
+ expr_counts[code, gene] += 1