crispyx 0.2.4__tar.gz → 0.2.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. {crispyx-0.2.4/src/crispyx.egg-info → crispyx-0.2.6}/PKG-INFO +5 -5
  2. {crispyx-0.2.4 → crispyx-0.2.6}/README.md +4 -4
  3. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/__init__.py +18 -0
  4. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_checkpoint.py +2 -0
  5. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_kernels.py +247 -21
  6. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_namespaces.py +111 -7
  7. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_provenance.py +3 -1
  8. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_version.py +1 -1
  9. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/data.py +1144 -507
  10. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/de.py +1355 -214
  11. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/glm.py +95 -123
  12. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/plotting.py +2 -1
  13. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/pseudobulk.py +85 -9
  14. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/qc.py +471 -6
  15. {crispyx-0.2.4 → crispyx-0.2.6/src/crispyx.egg-info}/PKG-INFO +5 -5
  16. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx.egg-info/SOURCES.txt +10 -0
  17. crispyx-0.2.6/tests/test_assign_guides.py +88 -0
  18. crispyx-0.2.6/tests/test_atomic_outputs.py +168 -0
  19. crispyx-0.2.6/tests/test_check_data.py +201 -0
  20. crispyx-0.2.6/tests/test_collapse_genes.py +122 -0
  21. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_concat_h5ad.py +1 -1
  22. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_convert_format.py +4 -1
  23. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_data_helpers.py +9 -0
  24. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_de_resume.py +37 -4
  25. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_glm_structured.py +31 -0
  26. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_nb_glm.py +27 -0
  27. crispyx-0.2.6/tests/test_nb_offset.py +458 -0
  28. crispyx-0.2.6/tests/test_pseudobulk_sparse_output.py +95 -0
  29. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_qc_parity.py +75 -0
  30. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_rank_genes_groups_outputs.py +17 -0
  31. crispyx-0.2.6/tests/test_recover_counts.py +106 -0
  32. crispyx-0.2.6/tests/test_shrink_lfc_arrays.py +132 -0
  33. crispyx-0.2.6/tests/test_sort_by_perturbation.py +58 -0
  34. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_stream_on_fast_axis.py +6 -5
  35. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_temp_file_cleanup.py +6 -9
  36. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_two_group_deseq2.py +16 -0
  37. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_wilcoxon_dispatch.py +1 -1
  38. crispyx-0.2.6/tests/test_wilcoxon_hypergeometric.py +115 -0
  39. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_wilcoxon_stratified.py +17 -15
  40. {crispyx-0.2.4 → crispyx-0.2.6}/LICENSE +0 -0
  41. {crispyx-0.2.4 → crispyx-0.2.6}/pyproject.toml +0 -0
  42. {crispyx-0.2.4 → crispyx-0.2.6}/setup.cfg +0 -0
  43. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_disk.py +0 -0
  44. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_grouping.py +0 -0
  45. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_h5codec.py +0 -0
  46. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_irls.py +0 -0
  47. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_memory.py +0 -0
  48. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_messages.py +0 -0
  49. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_preflight.py +0 -0
  50. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_size_factors.py +0 -0
  51. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_statistics.py +0 -0
  52. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/batch.py +0 -0
  53. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/dimred.py +0 -0
  54. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/hvg.py +0 -0
  55. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/profiling.py +0 -0
  56. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/residual.py +0 -0
  57. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/sample.py +0 -0
  58. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx.egg-info/dependency_links.txt +0 -0
  59. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx.egg-info/requires.txt +0 -0
  60. {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx.egg-info/top_level.txt +0 -0
  61. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_batch_process.py +0 -0
  62. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_cgroup_memory_budget.py +0 -0
  63. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_compare_controls.py +0 -0
  64. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_compress_h5ad.py +0 -0
  65. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_de_edge_cases.py +0 -0
  66. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_de_lowexpr_filter.py +0 -0
  67. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_dimred.py +0 -0
  68. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_disk_dispatch.py +0 -0
  69. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_downsample_counts.py +0 -0
  70. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_groupby_alias.py +0 -0
  71. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_hvg.py +0 -0
  72. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_in_memory_input.py +0 -0
  73. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_irls.py +0 -0
  74. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_memory_dispatch.py +0 -0
  75. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_messages.py +0 -0
  76. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_normalisation.py +0 -0
  77. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_normalized_effects.py +0 -0
  78. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_normalized_effects_se.py +0 -0
  79. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_obs_writes.py +0 -0
  80. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_pickle.py +0 -0
  81. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_plotting.py +0 -0
  82. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_preflight_dispatch.py +0 -0
  83. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_profiling.py +0 -0
  84. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_pseudobulk_batch.py +0 -0
  85. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_pseudobulk_memory.py +0 -0
  86. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_pseudobulk_profiles.py +0 -0
  87. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_read_backed.py +0 -0
  88. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_residual_effects.py +0 -0
  89. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_result_provenance.py +0 -0
  90. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_scanpy_parity.py +0 -0
  91. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_standardize.py +0 -0
  92. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_subsample.py +0 -0
  93. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_wilcoxon_labels.py +0 -0
  94. {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_workflow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: crispyx
3
- Version: 0.2.4
3
+ Version: 0.2.6
4
4
  Summary: Memory-efficient streaming analysis of large-scale CRISPR and Perturb-seq screens on disk-backed AnnData files
5
5
  Author: Jin-Hong Du
6
6
  License: Modified MIT License
@@ -145,7 +145,7 @@ Dynamic: license-file
145
145
 
146
146
  Genome-wide CRISPR screens routinely produce datasets with hundreds of thousands of cells and tens of thousands of genes. Standard single-cell analysis toolkits (Scanpy, Pertpy) load the entire count matrix into memory, requiring large RAM allocations and often making routine workflows impractical on laptops or shared compute nodes.
147
147
 
148
- **crispyx** solves this by streaming data directly from on-disk AnnData (`.h5ad`) files. Quality control, normalisation, pseudo-bulk aggregation, and differential expression all operate without materialising the full matrix.
148
+ **crispyx** runs quality control, normalisation, pseudo-bulk aggregation, differential expression (t-test, Wilcoxon, NB-GLM with covariates, covariate-adjusted residual effects), control checks and PCA/KNN on screens stored as `.h5ad` files, streaming from disk under a memory budget, so that million-cell screens run on a laptop or a cluster node. The API mirrors Scanpy. It starts from an `.h5ad` file; it does not read raw sequencing outputs, and it is not a general single-cell toolkit.
149
149
 
150
150
  **Documentation:** [crispyx.readthedocs.io](https://crispyx.readthedocs.io/en/latest/) — [installation](https://crispyx.readthedocs.io/en/latest/installation.html), [quick start](https://crispyx.readthedocs.io/en/latest/quickstart.html), [tutorials](https://crispyx.readthedocs.io/en/latest/notebooks/crispyx_tutorial.html) and [API reference](https://crispyx.readthedocs.io/en/latest/api.html).
151
151
 
@@ -202,18 +202,18 @@ For the full workflow (normalisation, PCA, pseudo-bulk, NB-GLM, LFC shrinkage, p
202
202
  - **Streaming QC & preprocessing** – Filter cells, perturbations, and genes; normalise and log-transform; select highly variable genes (`seurat_v3`/`mean_dispersion`, control-cells-only by default); CSC-aware streaming with `format_mismatch_policy`; all without loading the full matrix into memory
203
203
  - **Subsampling & downsampling** – Stratified or cluster-sampled cell subsampling (`cx.pp.subsample`, exact count or proportion per stratum, drop or keep small groups) and dependency-free per-cell count thinning (`cx.pp.downsample_counts`, the streaming equivalent of `scanpy.pp.downsample_counts`) for aligning dataset scale and sequencing depth before comparing screens
204
204
  - **Pseudo-bulk aggregation** – Absolute profiles over multiple grouping columns (for example, perturbation × batch), strict count sums or mean log1p expression, optional deterministic bootstrap sampling, and explicit within-batch effect calculation
205
- - **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`), and negative binomial GLM with covariates and apeGLM LFC shrinkage; multi-core support and adaptive memory management; per-condition low-expression filtering to exclude genes that are near-zero in both groups
205
+ - **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`), negative binomial GLM with covariates and apeGLM LFC shrinkage, and a fixed-offset NB test for many small perturbations against a large, stratified control set (`cx.tl.nb_offset_test`); multi-core support and adaptive memory management; per-condition low-expression filtering to exclude genes that are near-zero in both groups
206
206
  - **Covariate-adjusted effects** – `cx.tl.residual_effects` (Feng et al., 2026) fits each gene to covariates such as donor, lane or batch on the control cells only, and takes a perturbation's effect as the mean residual of its cells. Unlike the NB GLM, which fits the covariates on counts jointly with each perturbation, the control fit is shared by every perturbation, so a perturbation's own cells cannot shift its covariate adjustment, and all perturbations come from one streaming pass. On a 1.16-million-cell screen it reproduces the published effects and significance calls in under a minute.
207
207
  - **Storage-format aware** – Gene-streaming functions (`wilcoxon_test`, `batch_process`) can convert a CSR source once to a temporary, memory-bounded CSC copy beside the output instead of re-reading the whole matrix once per gene chunk; `format_mismatch_policy="auto"` (the default) measures the source's read throughput and does so only when the re-reads would really cost more than the conversion, so a small local file is streamed as-is and a large one on a network filesystem is converted; `convert_to_csc`/`convert_to_csr` honour `memory_limit_gb` and preserve the value dtype
208
208
  - **Dimension reduction** – Memory-efficient PCA and KNN graph construction on backed data
209
209
  - **Scanpy-compatible API & plotting** – Familiar `cx.pp`, `cx.pb`, `cx.tl`, and `cx.pl` namespaces; Scanpy-style rank genes plots, volcano, MA, PCA, UMAP, QC summaries, and overlap heatmaps
210
- - **Data preparation utilities** – Edit backed metadata without loading X; standardise gene names; normalise perturbation labels; auto-detect metadata columns
210
+ - **Data preparation utilities** – Edit backed metadata without loading X; normalise perturbation labels and take target genes out of guide names (`target_pattern`); assign guides from guide-capture counts (`cx.pp.assign_guides`); recover integer counts from data published as log1p-normalised values (`cx.pp.recover_counts`); sum the rows of a gene split over several (`cx.pp.collapse_genes`); concatenate multi-file screens; check a file for the mistakes that mislead an analysis (`cx.pp.check_data`); auto-detect metadata columns
211
211
  - **HPC-ready** – Resume/checkpoint for long-running jobs; configurable `memory_limit_gb`
212
212
  - **Disk-aware** – Estimates and warns about scratch-disk usage before large writes or CSC/CSR conversions, and `cx.estimate_disk_usage(...)` answers "how much disk will this need?" up front; the memory savings above assume the machine has enough free disk for streaming intermediates and output files
213
213
 
214
214
  ## Performance
215
215
 
216
- crispyx consistently outperforms Scanpy, Pertpy/PyDESeq2, and edgeR in both speed and memory across a range of CRISPR screen dataset sizes, with results matching Scanpy to Pearson *r* > 0.999:
216
+ Across 12 public CRISPR screens, crispyx completes every dataset where Scanpy, Pertpy/PyDESeq2 and edgeR run out of memory or time, with t-test and Wilcoxon results matching Scanpy to Pearson *r* > 0.999 and NB-GLM statistics agreeing with PyDESeq2 given the same size factors (z correlation 0.997–0.9999):
217
217
 
218
218
  <p align="center">
219
219
  <img src="docs/_static/fig2.png" width="800" alt="Benchmark results across 12 CRISPR screens: (a) dataset sizes, (b) completion status by method, (c) concordance with Scanpy, (d) runtime scaling, (e) peak memory scaling">
@@ -11,7 +11,7 @@
11
11
 
12
12
  Genome-wide CRISPR screens routinely produce datasets with hundreds of thousands of cells and tens of thousands of genes. Standard single-cell analysis toolkits (Scanpy, Pertpy) load the entire count matrix into memory, requiring large RAM allocations and often making routine workflows impractical on laptops or shared compute nodes.
13
13
 
14
- **crispyx** solves this by streaming data directly from on-disk AnnData (`.h5ad`) files. Quality control, normalisation, pseudo-bulk aggregation, and differential expression all operate without materialising the full matrix.
14
+ **crispyx** runs quality control, normalisation, pseudo-bulk aggregation, differential expression (t-test, Wilcoxon, NB-GLM with covariates, covariate-adjusted residual effects), control checks and PCA/KNN on screens stored as `.h5ad` files, streaming from disk under a memory budget, so that million-cell screens run on a laptop or a cluster node. The API mirrors Scanpy. It starts from an `.h5ad` file; it does not read raw sequencing outputs, and it is not a general single-cell toolkit.
15
15
 
16
16
  **Documentation:** [crispyx.readthedocs.io](https://crispyx.readthedocs.io/en/latest/) — [installation](https://crispyx.readthedocs.io/en/latest/installation.html), [quick start](https://crispyx.readthedocs.io/en/latest/quickstart.html), [tutorials](https://crispyx.readthedocs.io/en/latest/notebooks/crispyx_tutorial.html) and [API reference](https://crispyx.readthedocs.io/en/latest/api.html).
17
17
 
@@ -68,18 +68,18 @@ For the full workflow (normalisation, PCA, pseudo-bulk, NB-GLM, LFC shrinkage, p
68
68
  - **Streaming QC & preprocessing** – Filter cells, perturbations, and genes; normalise and log-transform; select highly variable genes (`seurat_v3`/`mean_dispersion`, control-cells-only by default); CSC-aware streaming with `format_mismatch_policy`; all without loading the full matrix into memory
69
69
  - **Subsampling & downsampling** – Stratified or cluster-sampled cell subsampling (`cx.pp.subsample`, exact count or proportion per stratum, drop or keep small groups) and dependency-free per-cell count thinning (`cx.pp.downsample_counts`, the streaming equivalent of `scanpy.pp.downsample_counts`) for aligning dataset scale and sequencing depth before comparing screens
70
70
  - **Pseudo-bulk aggregation** – Absolute profiles over multiple grouping columns (for example, perturbation × batch), strict count sums or mean log1p expression, optional deterministic bootstrap sampling, and explicit within-batch effect calculation
71
- - **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`), and negative binomial GLM with covariates and apeGLM LFC shrinkage; multi-core support and adaptive memory management; per-condition low-expression filtering to exclude genes that are near-zero in both groups
71
+ - **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`), negative binomial GLM with covariates and apeGLM LFC shrinkage, and a fixed-offset NB test for many small perturbations against a large, stratified control set (`cx.tl.nb_offset_test`); multi-core support and adaptive memory management; per-condition low-expression filtering to exclude genes that are near-zero in both groups
72
72
  - **Covariate-adjusted effects** – `cx.tl.residual_effects` (Feng et al., 2026) fits each gene to covariates such as donor, lane or batch on the control cells only, and takes a perturbation's effect as the mean residual of its cells. Unlike the NB GLM, which fits the covariates on counts jointly with each perturbation, the control fit is shared by every perturbation, so a perturbation's own cells cannot shift its covariate adjustment, and all perturbations come from one streaming pass. On a 1.16-million-cell screen it reproduces the published effects and significance calls in under a minute.
73
73
  - **Storage-format aware** – Gene-streaming functions (`wilcoxon_test`, `batch_process`) can convert a CSR source once to a temporary, memory-bounded CSC copy beside the output instead of re-reading the whole matrix once per gene chunk; `format_mismatch_policy="auto"` (the default) measures the source's read throughput and does so only when the re-reads would really cost more than the conversion, so a small local file is streamed as-is and a large one on a network filesystem is converted; `convert_to_csc`/`convert_to_csr` honour `memory_limit_gb` and preserve the value dtype
74
74
  - **Dimension reduction** – Memory-efficient PCA and KNN graph construction on backed data
75
75
  - **Scanpy-compatible API & plotting** – Familiar `cx.pp`, `cx.pb`, `cx.tl`, and `cx.pl` namespaces; Scanpy-style rank genes plots, volcano, MA, PCA, UMAP, QC summaries, and overlap heatmaps
76
- - **Data preparation utilities** – Edit backed metadata without loading X; standardise gene names; normalise perturbation labels; auto-detect metadata columns
76
+ - **Data preparation utilities** – Edit backed metadata without loading X; normalise perturbation labels and take target genes out of guide names (`target_pattern`); assign guides from guide-capture counts (`cx.pp.assign_guides`); recover integer counts from data published as log1p-normalised values (`cx.pp.recover_counts`); sum the rows of a gene split over several (`cx.pp.collapse_genes`); concatenate multi-file screens; check a file for the mistakes that mislead an analysis (`cx.pp.check_data`); auto-detect metadata columns
77
77
  - **HPC-ready** – Resume/checkpoint for long-running jobs; configurable `memory_limit_gb`
78
78
  - **Disk-aware** – Estimates and warns about scratch-disk usage before large writes or CSC/CSR conversions, and `cx.estimate_disk_usage(...)` answers "how much disk will this need?" up front; the memory savings above assume the machine has enough free disk for streaming intermediates and output files
79
79
 
80
80
  ## Performance
81
81
 
82
- crispyx consistently outperforms Scanpy, Pertpy/PyDESeq2, and edgeR in both speed and memory across a range of CRISPR screen dataset sizes, with results matching Scanpy to Pearson *r* > 0.999:
82
+ Across 12 public CRISPR screens, crispyx completes every dataset where Scanpy, Pertpy/PyDESeq2 and edgeR run out of memory or time, with t-test and Wilcoxon results matching Scanpy to Pearson *r* > 0.999 and NB-GLM statistics agreeing with PyDESeq2 given the same size factors (z correlation 0.997–0.9999):
83
83
 
84
84
  <p align="center">
85
85
  <img src="docs/_static/fig2.png" width="800" alt="Benchmark results across 12 CRISPR screens: (a) dataset sizes, (b) completion status by method, (c) concordance with Scanpy, (d) runtime scaling, (e) peak memory scaling">
@@ -41,6 +41,8 @@ from .data import (
41
41
  normalise_perturbation_labels,
42
42
  read_h5ad_ondisk,
43
43
  read_backed,
44
+ recover_counts,
45
+ collapse_genes,
44
46
  resolve_data_path,
45
47
  standardise_gene_names,
46
48
  write_filtered_subset,
@@ -50,7 +52,10 @@ from .data import (
50
52
  from .de import (
51
53
  RankGenesGroupsResult,
52
54
  nb_glm_test,
55
+ nb_offset_test,
53
56
  shrink_lfc,
57
+ shrink_lfc_arrays,
58
+ ShrunkLFC,
54
59
  t_test,
55
60
  wilcoxon_test,
56
61
  )
@@ -84,12 +89,16 @@ from .pseudobulk import (
84
89
  compute_pseudobulk_effects,
85
90
  )
86
91
  from .qc import (
92
+ Issue,
93
+ assign_guides,
94
+ check_perturbation_data,
87
95
  compare_controls,
88
96
  control_guide_correlation,
89
97
  filter_cells_by_gene_count,
90
98
  filter_genes_by_cell_count,
91
99
  filter_perturbations_by_cell_count,
92
100
  quality_control_summary,
101
+ raise_on_errors,
93
102
  )
94
103
  from .residual import residual_effects
95
104
  from .sample import subsample
@@ -128,6 +137,10 @@ __all__ = [
128
137
  "quality_control_summary",
129
138
  "compare_controls",
130
139
  "control_guide_correlation",
140
+ "assign_guides",
141
+ "check_perturbation_data",
142
+ "Issue",
143
+ "raise_on_errors",
131
144
  "plot_control_guide_correlation",
132
145
  "highly_variable_genes",
133
146
  # Subsampling
@@ -147,7 +160,10 @@ __all__ = [
147
160
  "t_test",
148
161
  "wilcoxon_test",
149
162
  "nb_glm_test",
163
+ "nb_offset_test",
150
164
  "shrink_lfc",
165
+ "shrink_lfc_arrays",
166
+ "ShrunkLFC",
151
167
  # Data utilities
152
168
  "AnnData",
153
169
  "ensure_gene_symbol_column",
@@ -156,6 +172,8 @@ __all__ = [
156
172
  "resolve_data_path",
157
173
  "normalize_total_log1p",
158
174
  "downsample_counts",
175
+ "recover_counts",
176
+ "collapse_genes",
159
177
  "convert_to_csc",
160
178
  "convert_to_csr",
161
179
  "compress_h5ad",
@@ -253,6 +253,8 @@ def fingerprint_difference(previous: dict, current: dict) -> str | None:
253
253
  return "the input file has changed since"
254
254
  if previous.get("inputs") != current.get("inputs"):
255
255
  return "an input file has changed since"
256
+ if previous.get("crispyx") != current.get("crispyx"):
257
+ return f"it was written by crispyx {previous.get('crispyx', 'before 0.2.6')}"
256
258
  old_params = previous.get("params", {})
257
259
  new_params = current.get("params", {})
258
260
  for name in sorted(set(old_params) | set(new_params)):
@@ -94,8 +94,8 @@ def _nb_loglik_grid_numba(
94
94
  def _neg_half_log_det_numba(xtwx: np.ndarray) -> float:
95
95
  """``-0.5 * log det(xtwx)`` from a Cholesky factorisation in place.
96
96
 
97
- Zero for an empty matrix (no Cox-Reid adjustment), or one that is not
98
- positive definite (as the gene-wise Cox-Reid step treats it).
97
+ Zero for an empty matrix (no Cox-Reid adjustment); NaN for one that is
98
+ not positive definite.
99
99
  """
100
100
  log_det = 0.0
101
101
  for j in range(xtwx.shape[0]):
@@ -103,7 +103,7 @@ def _neg_half_log_det_numba(xtwx: np.ndarray) -> float:
103
103
  for k in range(j):
104
104
  d -= xtwx[j, k] * xtwx[j, k]
105
105
  if d <= 0.0:
106
- return 0.0
106
+ return np.nan
107
107
  ljj = math.sqrt(d)
108
108
  xtwx[j, j] = ljj
109
109
  log_det += math.log(ljj)
@@ -132,6 +132,9 @@ def _brent_minimize_numba(
132
132
  x_indices: np.ndarray,
133
133
  x_data: np.ndarray,
134
134
  xtwx: np.ndarray,
135
+ x_group: np.ndarray,
136
+ gdiag: np.ndarray,
137
+ gcross: np.ndarray,
135
138
  a: float,
136
139
  b: float,
137
140
  tol: float = 1e-5,
@@ -149,7 +152,7 @@ def _brent_minimize_numba(
149
152
  x = w = v = a + golden * (b - a)
150
153
  fx = fw = fv = -_nb_posterior_with_cache_numba(
151
154
  Y_g, mu_g, x, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
152
- xtwx,
155
+ xtwx, x_group, gdiag, gcross,
153
156
  )
154
157
 
155
158
  d = 0.0 # Distance to next point
@@ -206,7 +209,7 @@ def _brent_minimize_numba(
206
209
 
207
210
  fu = -_nb_posterior_with_cache_numba(
208
211
  Y_g, mu_g, u, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
209
- xtwx,
212
+ xtwx, x_group, gdiag, gcross,
210
213
  )
211
214
 
212
215
  # Update a, b, v, w, x
@@ -253,14 +256,23 @@ def _nb_posterior_with_cache_numba(
253
256
  x_indices: np.ndarray,
254
257
  x_data: np.ndarray,
255
258
  xtwx: np.ndarray,
259
+ x_group: np.ndarray,
260
+ gdiag: np.ndarray,
261
+ gcross: np.ndarray,
256
262
  ) -> float:
257
263
  """Cox-Reid adjusted NB log-likelihood plus log-prior for one gene.
258
264
 
259
- Uses the precomputed ``gammaln(Y+1)`` sum. ``XᵀWX``, ``W = mu / (1 +
260
- alpha mu)``, is accumulated into the work buffer ``xtwx`` (``(0, 0)``
261
- for no adjustment) in the same pass over the cells as the likelihood;
262
- the design ``X`` is given in CSR form, so a one-hot batch design costs a
263
- few products per cell rather than ``n_features**2``. ``small_counts[k]``
265
+ Uses the precomputed ``gammaln(Y+1)`` sum. The design is ``[F | G]``:
266
+ ``F`` in CSR form (``x_indptr``, ``x_indices``, ``x_data``), and ``G``
267
+ one-hot, cell ``i`` in column ``x_group[i]`` (-1 for none). With ``W =
268
+ mu / (1 + alpha mu)``, ``FᵀWF`` is accumulated into the work buffer
269
+ ``xtwx`` (``(0, 0)`` for no adjustment), the diagonal ``GᵀWG`` into
270
+ ``gdiag`` and ``GᵀWF`` into ``gcross``, in the same pass over the cells
271
+ as the likelihood; then ``log det(XᵀWX) = sum log gdiag + log det(FᵀWF -
272
+ GᵀWFᵀ diag(gdiag)⁻¹ GᵀWF)``, the Schur complement of the one-hot block,
273
+ so hundreds of stratum columns cost ``n_groups x n_F**2`` rather than a
274
+ factorisation of the whole ``XᵀWX``. The adjustment is zero when
275
+ ``XᵀWX`` is not positive definite. ``small_counts[k]``
264
276
  is the number of cells with count ``k`` (below its length): their
265
277
  ``lgamma(y + r) - lgamma(r)`` terms depend on the count only, so each
266
278
  value's is formed once, by recurrence, rather than once per cell.
@@ -271,6 +283,10 @@ def _nb_posterior_with_cache_numba(
271
283
  log_r = math.log(r)
272
284
  cox_reid = xtwx.shape[0] > 0
273
285
  xtwx[:, :] = 0.0
286
+ n_groups = gdiag.shape[0]
287
+ if n_groups:
288
+ gdiag[:] = 0.0
289
+ gcross[:, :] = 0.0
274
290
 
275
291
  # lgamma(y + r) - lgamma(r) - y log r, without libm's lgamma (see
276
292
  # _lgamma_ratio_term): sum(log1p(i / r), i < y) for the small counts.
@@ -296,10 +312,31 @@ def _nb_posterior_with_cache_numba(
296
312
  ja = x_indices[a]
297
313
  for b in range(x_indptr[i], x_indptr[i + 1]):
298
314
  xtwx[ja, x_indices[b]] += wa * x_data[b]
315
+ if n_groups and x_group[i] >= 0:
316
+ j = x_group[i]
317
+ gdiag[j] += w
318
+ for a in range(x_indptr[i], x_indptr[i + 1]):
319
+ gcross[j, x_indices[a]] += w * x_data[a]
299
320
  ll -= gammaln_y_plus_1_sum
300
321
 
301
322
  log_prior = -0.5 * (log_alpha - log_trend_g) ** 2 / prior_var
302
- return ll + log_prior + _neg_half_log_det_numba(xtwx)
323
+ if not cox_reid:
324
+ return ll + log_prior
325
+ adjustment = 0.0
326
+ for j in range(n_groups):
327
+ dj = gdiag[j]
328
+ if dj <= 0.0:
329
+ return ll + log_prior # XᵀWX is not positive definite
330
+ adjustment -= 0.5 * math.log(dj)
331
+ for a in range(xtwx.shape[0]):
332
+ ca = gcross[j, a] / dj
333
+ if ca != 0.0:
334
+ for b in range(xtwx.shape[0]):
335
+ xtwx[a, b] -= ca * gcross[j, b]
336
+ schur = _neg_half_log_det_numba(xtwx)
337
+ if math.isnan(schur):
338
+ return ll + log_prior
339
+ return ll + log_prior + adjustment + schur
303
340
 
304
341
 
305
342
  @nb.njit(parallel=True, cache=True)
@@ -315,6 +352,8 @@ def _nb_map_grid_search_with_refinement_numba(
315
352
  x_indices: np.ndarray,
316
353
  x_data: np.ndarray,
317
354
  n_features: int,
355
+ x_group: np.ndarray,
356
+ n_groups: int,
318
357
  tol: float = 1e-5,
319
358
  max_refine_iter: int = 50,
320
359
  ) -> np.ndarray:
@@ -339,8 +378,12 @@ def _nb_map_grid_search_with_refinement_numba(
339
378
  Variance of the log-normal prior (``inf`` for none: the gene-wise
340
379
  estimate).
341
380
  x_indptr, x_indices, x_data, n_features
342
- The design matrix in CSR form, for the Cox-Reid adjustment
343
- (``n_features=0`` for none).
381
+ The design's columns outside its one-hot block in CSR form, for the
382
+ Cox-Reid adjustment (``n_features=0`` for none).
383
+ x_group, n_groups
384
+ Each cell's column of the one-hot block (-1 for none), and the
385
+ block's width (0 for no block): see
386
+ :func:`_nb_posterior_with_cache_numba`.
344
387
  tol : float
345
388
  Tolerance for Brent convergence.
346
389
  max_refine_iter : int
@@ -358,6 +401,8 @@ def _nb_map_grid_search_with_refinement_numba(
358
401
  Y_g = Y[:, g]
359
402
  mu_g = mu[:, g]
360
403
  xtwx = np.empty((n_features, n_features))
404
+ gdiag = np.empty(n_groups)
405
+ gcross = np.empty((n_groups, n_features))
361
406
  small_counts = np.zeros(_SMALL_COUNTS)
362
407
  for i in range(Y_g.shape[0]):
363
408
  if Y_g[i] < _SMALL_COUNTS:
@@ -369,7 +414,7 @@ def _nb_map_grid_search_with_refinement_numba(
369
414
  for k in range(n_grid):
370
415
  posterior = _nb_posterior_with_cache_numba(
371
416
  Y_g, mu_g, lo[g] + k * step, log_trend_g, prior_var, 0.0, small_counts,
372
- x_indptr, x_indices, x_data, xtwx,
417
+ x_indptr, x_indices, x_data, xtwx, x_group, gdiag, gcross,
373
418
  )
374
419
  if posterior > best_posterior:
375
420
  best_posterior = posterior
@@ -378,11 +423,11 @@ def _nb_map_grid_search_with_refinement_numba(
378
423
  if 0 < best_k < n_grid - 1 and 2 * step > tol:
379
424
  refined = _brent_minimize_numba(
380
425
  Y_g, mu_g, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data, xtwx,
381
- best - step, best + step, tol=tol, max_iter=max_refine_iter,
426
+ x_group, gdiag, gcross, best - step, best + step, tol=tol, max_iter=max_refine_iter,
382
427
  )
383
428
  refined_posterior = _nb_posterior_with_cache_numba(
384
429
  Y_g, mu_g, refined, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data,
385
- xtwx,
430
+ xtwx, x_group, gdiag, gcross,
386
431
  )
387
432
  if refined_posterior >= best_posterior:
388
433
  best = refined
@@ -498,6 +543,148 @@ def _compute_ctrl_tie_sums(
498
543
  return ctrl_tie_sums
499
544
 
500
545
 
546
+ # The NB IRLS fits measure a deviance change relative to |deviance| plus
547
+ # this, DESeq2's ratio: a saturated fit's deviance near zero still converges.
548
+ _DEVIANCE_OFFSET = 0.1
549
+
550
+
551
+ # The rank-sum p-value treats the number of expressing cells that fall in the
552
+ # perturbation group as hypergeometric whenever its variance, summed over
553
+ # strata, is below this; above it the normal approximation is used.
554
+ _RANK_SUM_MIXTURE_MAX_VAR = 100.0
555
+ # The most components the strata's combined mixture keeps; beyond it,
556
+ # components with nearby means are merged.
557
+ _RANK_SUM_MIXTURE_MAX_COMPONENTS = 1 << 15
558
+
559
+
560
+ @nb.njit(cache=True)
561
+ def _merge_mixture(w: np.ndarray, m: np.ndarray, v: np.ndarray, n_bins: int) -> tuple:
562
+ """Merge a normal mixture's components into at most ``n_bins`` by their
563
+ mean (equal-width bins over its range), each bin a normal with the
564
+ bin's weight, mean and variance."""
565
+ lo = m.min()
566
+ span = m.max() - lo
567
+ bins = np.zeros(m.shape[0], dtype=np.int64)
568
+ if span > 0.0:
569
+ for i in range(m.shape[0]):
570
+ bins[i] = min(int((m[i] - lo) / span * n_bins), n_bins - 1)
571
+ bw = np.zeros(n_bins)
572
+ bm = np.zeros(n_bins)
573
+ for i in range(m.shape[0]):
574
+ bw[bins[i]] += w[i]
575
+ bm[bins[i]] += w[i] * m[i]
576
+ for b in range(n_bins):
577
+ if bw[b] > 0.0:
578
+ bm[b] /= bw[b]
579
+ bv = np.zeros(n_bins)
580
+ for i in range(m.shape[0]):
581
+ d = m[i] - bm[bins[i]]
582
+ bv[bins[i]] += w[i] * (v[i] + d * d)
583
+ keep = np.flatnonzero(bw > 0.0)
584
+ return bw[keep], bm[keep], bv[keep] / bw[keep]
585
+
586
+
587
+ @nb.njit(cache=True)
588
+ def _rank_sum_mixture_pvalue(
589
+ n1s: np.ndarray, ns: np.ndarray, ks: np.ndarray, nz_tie_sums: np.ndarray, deviation: float,
590
+ max_components: int = _RANK_SUM_MIXTURE_MAX_COMPONENTS,
591
+ ) -> float:
592
+ """Two-sided p-value of a (stratified) rank sum when most values are tied
593
+ at zero.
594
+
595
+ Stratum ``b`` has ``ns[b]`` cells, ``n1s[b]`` of them in the group and
596
+ ``ks[b]`` non-zero, whose ties give ``nz_tie_sums[b] = sum(t^3 - t)``;
597
+ ``deviation`` is ``sum_b (R_b - E_b)``. Under the permutation null the
598
+ group's count of non-zero cells ``J_b`` is hypergeometric: with few
599
+ non-zeros it is very skewed, which the normal approximation misses by
600
+ orders of magnitude (one expressing cell of 232 against 3 of 30,000
601
+ controls: p 3e-8 against a permutation p of 0.03). Here ``J_b`` keeps
602
+ its exact distribution, and given ``J_b = j`` the rank sum is normal
603
+ with the exact mean and variance of ``j`` draws without replacement from
604
+ the non-zero ranks. The strata's
605
+ mixtures are combined exactly up to ``max_components`` components;
606
+ past it, :func:`_merge_mixture` merges those with nearby means, keeping
607
+ each merged component's weight, mean and variance.
608
+ """
609
+ w = np.ones(1)
610
+ m = np.zeros(1)
611
+ v = np.zeros(1)
612
+ for b in range(n1s.shape[0]):
613
+ n1 = n1s[b]
614
+ n = ns[b]
615
+ k = ks[b]
616
+ z = n - k
617
+ if n1 == 0 or n1 == n:
618
+ continue
619
+ r0 = (z + 1.0) / 2.0
620
+ mean_nz = z + (k + 1.0) / 2.0
621
+ var_nz = ((k * k - 1.0) / 12.0 - nz_tie_sums[b] / (12.0 * k)) if k > 1 else 0.0
622
+ expected = n1 * (n + 1.0) / 2.0
623
+ lo = max(0, n1 - z)
624
+ hi = min(k, n1)
625
+ # Start at the mode and walk out by the ratio of successive terms.
626
+ mode = min(max(int((n1 + 1.0) * (k + 1.0) / (n + 2.0)), lo), hi)
627
+ log_mode = (math.lgamma(k + 1.0) - math.lgamma(mode + 1.0) - math.lgamma(k - mode + 1.0)
628
+ + math.lgamma(z + 1.0) - math.lgamma(n1 - mode + 1.0) - math.lgamma(z - n1 + mode + 1.0)
629
+ - math.lgamma(n + 1.0) + math.lgamma(n1 + 1.0) + math.lgamma(n - n1 + 1.0))
630
+ width = hi - lo + 1
631
+ pmf = np.zeros(width)
632
+ pmf[mode - lo] = math.exp(log_mode)
633
+ for j in range(mode, hi):
634
+ pmf[j + 1 - lo] = pmf[j - lo] * (k - j) * (n1 - j) / ((j + 1.0) * (z - n1 + j + 1.0))
635
+ if pmf[j + 1 - lo] < 1e-300:
636
+ break
637
+ for j in range(mode, lo, -1):
638
+ pmf[j - 1 - lo] = pmf[j - lo] * j * (z - n1 + j) / ((k - j + 1.0) * (n1 - j + 1.0))
639
+ if pmf[j - 1 - lo] < 1e-300:
640
+ break
641
+ c = 0
642
+ for i in range(width):
643
+ if pmf[i] >= 1e-300:
644
+ c += 1
645
+ if c == 1:
646
+ j = mode
647
+ m += (n1 - j) * r0 + j * mean_nz - expected
648
+ v += j * (k - j) / (k - 1.0) * var_nz if k > 1 else 0.0
649
+ continue
650
+ if w.shape[0] * c > max_components:
651
+ w, m, v = _merge_mixture(w, m, v, max(max_components // c, 1))
652
+ nw = np.empty(w.shape[0] * c)
653
+ nm = np.empty(w.shape[0] * c)
654
+ nv = np.empty(w.shape[0] * c)
655
+ out = 0
656
+ for i in range(width):
657
+ if pmf[i] < 1e-300:
658
+ continue
659
+ j = lo + i
660
+ cm = (n1 - j) * r0 + j * mean_nz - expected
661
+ cv = j * (k - j) / (k - 1.0) * var_nz if k > 1 else 0.0
662
+ for a in range(w.shape[0]):
663
+ nw[out] = w[a] * pmf[i]
664
+ nm[out] = m[a] + cm
665
+ nv[out] = v[a] + cv
666
+ out += 1
667
+ w, m, v = nw, nm, nv
668
+ d = abs(deviation)
669
+ tol = 1e-9 * (1.0 + d)
670
+ total = 0.0
671
+ for i in range(w.shape[0]):
672
+ if v[i] > 0.0:
673
+ sd = math.sqrt(2.0 * v[i])
674
+ total += w[i] * 0.5 * (math.erfc((d - m[i]) / sd) + math.erfc((d + m[i]) / sd))
675
+ elif m[i] >= d - tol or m[i] <= -d + tol:
676
+ total += w[i]
677
+ return min(total, 1.0)
678
+
679
+
680
+ @nb.njit(cache=True)
681
+ def _hypergeometric_variance(n1: float, n: float, k: float) -> float:
682
+ """Variance of the group's count of non-zero cells under the permutation null."""
683
+ if n <= 1.0:
684
+ return 0.0
685
+ return n1 * (k / n) * ((n - k) / n) * ((n - n1) / (n - 1.0))
686
+
687
+
501
688
  @nb.njit(cache=True)
502
689
  def _all_tied(n_zeros: int, ctrl_sorted: np.ndarray, pert_sorted: np.ndarray) -> bool:
503
690
  """Whether every value of a gene -- ``n_zeros`` zeros plus the sorted
@@ -551,6 +738,8 @@ def _rank_sum_pert_bsearch_numba(
551
738
  ``n_pert_zeros * zero_avg_rank`` is added by the caller).
552
739
  tie_corr : float
553
740
  Tie correction factor: 1 - sum(t^3 - t) / (n_total^3 - n_total).
741
+ tie_sum : float
742
+ ``sum(t^3 - t)`` over every tie group, the zeros included.
554
743
  """
555
744
  n_ctrl_nz = ctrl_sorted.shape[0]
556
745
  n_pert_nz = pert_sorted.shape[0]
@@ -614,7 +803,7 @@ def _rank_sum_pert_bsearch_numba(
614
803
  else:
615
804
  tie_corr = 1.0
616
805
 
617
- return rank_sum, tie_corr
806
+ return rank_sum, tie_corr, tie_sum
618
807
 
619
808
 
620
809
  @nb.njit(parallel=False, cache=True)
@@ -628,6 +817,7 @@ def _wilcoxon_single_pert_presorted(
628
817
  pert_dense: np.ndarray,
629
818
  valid_genes: np.ndarray,
630
819
  tie_correct: bool,
820
+ hypergeometric: bool,
631
821
  zero_threshold: float,
632
822
  u_stat_out: np.ndarray,
633
823
  z_score_out: np.ndarray,
@@ -673,6 +863,7 @@ def _wilcoxon_single_pert_presorted(
673
863
  tied = n_zeros == n_total
674
864
  rank_sum = 0.0
675
865
  tie_corr = 1.0
866
+ tie_sum = 0.0
676
867
  if not tied:
677
868
  # --- Binary-search ranking (always) ---
678
869
  # O(n_pert_nz * log(n_ctrl_nz)) — works for any zero fraction.
@@ -693,7 +884,7 @@ def _wilcoxon_single_pert_presorted(
693
884
  ctrl_sorted = ctrl_sorted_flat[start : start + n_ctrl_nz]
694
885
 
695
886
  tied = _all_tied(n_zeros, ctrl_sorted, pert_sorted)
696
- rank_sum_nz, tie_corr = _rank_sum_pert_bsearch_numba(
887
+ rank_sum_nz, tie_corr, tie_sum = _rank_sum_pert_bsearch_numba(
697
888
  ctrl_sorted, pert_sorted, n_zeros, ctrl_tie_sums[g]
698
889
  )
699
890
 
@@ -722,6 +913,13 @@ def _wilcoxon_single_pert_presorted(
722
913
  z = (rank_sum - expected) / std
723
914
  abs_z = abs(z)
724
915
  pval = math.erfc(abs_z / math.sqrt(2.0))
916
+ n_nonzero = n_total - n_zeros
917
+ if hypergeometric and _hypergeometric_variance(n_pert_f, n_total_f, float(n_nonzero)) < _RANK_SUM_MIXTURE_MAX_VAR:
918
+ zf = float(n_zeros)
919
+ pval = _rank_sum_mixture_pvalue(
920
+ np.array([n_pert]), np.array([n_total]), np.array([n_nonzero]),
921
+ np.array([tie_sum - (zf ** 3 - zf)]), rank_sum - expected,
922
+ )
725
923
  else:
726
924
  # An empty group: no comparison was made.
727
925
  z = np.nan
@@ -753,6 +951,7 @@ def _wilcoxon_batch_perts_presorted_numba(
753
951
  pert_row_offsets: np.ndarray,
754
952
  valid_masks: np.ndarray,
755
953
  tie_correct: bool,
954
+ hypergeometric: bool,
756
955
  zero_threshold: float,
757
956
  u_stat_out: np.ndarray,
758
957
  z_score_out: np.ndarray,
@@ -788,6 +987,9 @@ def _wilcoxon_batch_perts_presorted_numba(
788
987
  valid_masks : (n_perts, n_valid_genes)
789
988
  Boolean validity mask per (pert, gene).
790
989
  tie_correct : bool
990
+ hypergeometric : bool
991
+ Whether p-values come from :func:`_rank_sum_mixture_pvalue` where the
992
+ group's count of non-zero cells is skewed.
791
993
  zero_threshold : float
792
994
  u_stat_out : (n_perts, n_valid_genes)
793
995
  z_score_out : (n_perts, n_valid_genes)
@@ -811,6 +1013,7 @@ def _wilcoxon_batch_perts_presorted_numba(
811
1013
  pert_dense,
812
1014
  valid_masks[p_idx],
813
1015
  tie_correct,
1016
+ hypergeometric,
814
1017
  zero_threshold,
815
1018
  u_stat_out[p_idx],
816
1019
  z_score_out[p_idx],
@@ -833,6 +1036,7 @@ def _wilcoxon_stratified_single_pert(
833
1036
  seg_hi: int,
834
1037
  valid_genes: np.ndarray,
835
1038
  tie_correct: bool,
1039
+ hypergeometric: bool,
836
1040
  u_stat_out: np.ndarray,
837
1041
  z_score_out: np.ndarray,
838
1042
  pvalue_out: np.ndarray,
@@ -867,6 +1071,11 @@ def _wilcoxon_stratified_single_pert(
867
1071
  """
868
1072
  n_genes = all_pert_stacked.shape[1]
869
1073
  sqrt2 = math.sqrt(2.0)
1074
+ n_seg = seg_hi - seg_lo
1075
+ strata_n1 = np.empty(n_seg, dtype=np.int64)
1076
+ strata_n = np.empty(n_seg, dtype=np.int64)
1077
+ strata_k = np.empty(n_seg, dtype=np.int64)
1078
+ strata_ties = np.empty(n_seg, dtype=np.float64)
870
1079
 
871
1080
  for g in range(n_genes):
872
1081
  if not valid_genes[g]:
@@ -881,6 +1090,8 @@ def _wilcoxon_stratified_single_pert(
881
1090
  u_sum = 0.0 # sum_b U_b
882
1091
  n1n0_sum = 0.0 # sum_b n1_b * n0_b
883
1092
  tied = True # every contributing stratum has a single value
1093
+ var_j = 0.0 # sum_b Var(J_b): the group's count of non-zero cells
1094
+ n_used = 0
884
1095
 
885
1096
  for s in range(seg_lo, seg_hi):
886
1097
  b = seg_batch[s]
@@ -919,9 +1130,16 @@ def _wilcoxon_stratified_single_pert(
919
1130
 
920
1131
  if not _all_tied(n_zeros, ctrl_sorted, pert_sorted):
921
1132
  tied = False
922
- rank_sum_nz, tie_corr = _rank_sum_pert_bsearch_numba(
1133
+ rank_sum_nz, tie_corr, tie_sum = _rank_sum_pert_bsearch_numba(
923
1134
  ctrl_sorted, pert_sorted, n_zeros, ctrl_tie_sums[b, g]
924
1135
  )
1136
+ zf = float(n_zeros)
1137
+ strata_n1[n_used] = n1
1138
+ strata_n[n_used] = N
1139
+ strata_k[n_used] = N - n_zeros
1140
+ strata_ties[n_used] = tie_sum - (zf ** 3 - zf)
1141
+ n_used += 1
1142
+ var_j += _hypergeometric_variance(float(n1), float(N), float(N - n_zeros))
925
1143
  if not tie_correct:
926
1144
  tie_corr = 1.0
927
1145
 
@@ -943,6 +1161,10 @@ def _wilcoxon_stratified_single_pert(
943
1161
  if var > 0.0 and not tied:
944
1162
  z = num / math.sqrt(var)
945
1163
  pval = math.erfc(abs(z) / sqrt2)
1164
+ if hypergeometric and var_j < _RANK_SUM_MIXTURE_MAX_VAR:
1165
+ pval = _rank_sum_mixture_pvalue(
1166
+ strata_n1[:n_used], strata_n[:n_used], strata_k[:n_used], strata_ties[:n_used], num,
1167
+ )
946
1168
  else:
947
1169
  # No stratum to compare in, or every value tied within each:
948
1170
  # the rank test is undefined (with or without the tie
@@ -977,6 +1199,7 @@ def _wilcoxon_stratified_batch_perts_numba(
977
1199
  pert_ptr: np.ndarray,
978
1200
  valid_masks: np.ndarray,
979
1201
  tie_correct: bool,
1202
+ hypergeometric: bool,
980
1203
  u_stat_out: np.ndarray,
981
1204
  z_score_out: np.ndarray,
982
1205
  pvalue_out: np.ndarray,
@@ -1006,6 +1229,8 @@ def _wilcoxon_stratified_batch_perts_numba(
1006
1229
  perturbation ``p``.
1007
1230
  valid_masks : (n_perts, n_valid_genes)
1008
1231
  tie_correct : bool
1232
+ hypergeometric : bool
1233
+ As in :func:`_wilcoxon_batch_perts_presorted_numba`.
1009
1234
  u_stat_out, z_score_out, pvalue_out, effect_out : (n_perts, n_valid_genes)
1010
1235
  """
1011
1236
  n_perts = pert_ptr.shape[0] - 1
@@ -1024,6 +1249,7 @@ def _wilcoxon_stratified_batch_perts_numba(
1024
1249
  pert_ptr[p_idx + 1],
1025
1250
  valid_masks[p_idx],
1026
1251
  tie_correct,
1252
+ hypergeometric,
1027
1253
  u_stat_out[p_idx],
1028
1254
  z_score_out[p_idx],
1029
1255
  pvalue_out[p_idx],
@@ -1974,7 +2200,7 @@ def _structured_nb_irls(X, labels, offset, Y, alpha, n_groups, const_col, const_
1974
2200
  ceiling = dev + 1e-9 * abs(dev)
1975
2201
  rose = new_dev > ceiling
1976
2202
  worse = rose and not on_floor
1977
- scale = max(abs(dev), 1e-8)
2203
+ scale = abs(dev) + _DEVIANCE_OFFSET
1978
2204
  excess = (new_dev - dev) / scale
1979
2205
  step = 1.0
1980
2206
  for _h in range(max_halvings):
@@ -2001,7 +2227,7 @@ def _structured_nb_irls(X, labels, offset, Y, alpha, n_groups, const_col, const_
2001
2227
  new_mu[:] = mu
2002
2228
  new_dev = dev
2003
2229
  relative_change = abs(new_dev - dev) / scale
2004
- deviance_ratio = abs(new_dev - dev) / (abs(new_dev) + 0.1)
2230
+ deviance_ratio = abs(new_dev - dev) / (abs(new_dev) + _DEVIANCE_OFFSET)
2005
2231
  coef_change = 0.0
2006
2232
  for j in range(dx):
2007
2233
  coef_change = max(coef_change, abs(new_f[j] - bf[j]))