crispyx 0.2.4__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {crispyx-0.2.4/src/crispyx.egg-info → crispyx-0.2.6}/PKG-INFO +5 -5
- {crispyx-0.2.4 → crispyx-0.2.6}/README.md +4 -4
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/__init__.py +18 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_checkpoint.py +2 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_kernels.py +247 -21
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_namespaces.py +111 -7
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_provenance.py +3 -1
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_version.py +1 -1
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/data.py +1144 -507
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/de.py +1355 -214
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/glm.py +95 -123
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/plotting.py +2 -1
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/pseudobulk.py +85 -9
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/qc.py +471 -6
- {crispyx-0.2.4 → crispyx-0.2.6/src/crispyx.egg-info}/PKG-INFO +5 -5
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx.egg-info/SOURCES.txt +10 -0
- crispyx-0.2.6/tests/test_assign_guides.py +88 -0
- crispyx-0.2.6/tests/test_atomic_outputs.py +168 -0
- crispyx-0.2.6/tests/test_check_data.py +201 -0
- crispyx-0.2.6/tests/test_collapse_genes.py +122 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_concat_h5ad.py +1 -1
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_convert_format.py +4 -1
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_data_helpers.py +9 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_de_resume.py +37 -4
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_glm_structured.py +31 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_nb_glm.py +27 -0
- crispyx-0.2.6/tests/test_nb_offset.py +458 -0
- crispyx-0.2.6/tests/test_pseudobulk_sparse_output.py +95 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_qc_parity.py +75 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_rank_genes_groups_outputs.py +17 -0
- crispyx-0.2.6/tests/test_recover_counts.py +106 -0
- crispyx-0.2.6/tests/test_shrink_lfc_arrays.py +132 -0
- crispyx-0.2.6/tests/test_sort_by_perturbation.py +58 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_stream_on_fast_axis.py +6 -5
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_temp_file_cleanup.py +6 -9
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_two_group_deseq2.py +16 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_wilcoxon_dispatch.py +1 -1
- crispyx-0.2.6/tests/test_wilcoxon_hypergeometric.py +115 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_wilcoxon_stratified.py +17 -15
- {crispyx-0.2.4 → crispyx-0.2.6}/LICENSE +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/pyproject.toml +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/setup.cfg +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_disk.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_grouping.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_h5codec.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_irls.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_memory.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_messages.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_preflight.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_size_factors.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/_statistics.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/batch.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/dimred.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/hvg.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/profiling.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/residual.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx/sample.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx.egg-info/dependency_links.txt +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx.egg-info/requires.txt +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/src/crispyx.egg-info/top_level.txt +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_batch_process.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_cgroup_memory_budget.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_compare_controls.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_compress_h5ad.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_de_edge_cases.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_de_lowexpr_filter.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_dimred.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_disk_dispatch.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_downsample_counts.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_groupby_alias.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_hvg.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_in_memory_input.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_irls.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_memory_dispatch.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_messages.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_normalisation.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_normalized_effects.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_normalized_effects_se.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_obs_writes.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_pickle.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_plotting.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_preflight_dispatch.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_profiling.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_pseudobulk_batch.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_pseudobulk_memory.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_pseudobulk_profiles.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_read_backed.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_residual_effects.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_result_provenance.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_scanpy_parity.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_standardize.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_subsample.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_wilcoxon_labels.py +0 -0
- {crispyx-0.2.4 → crispyx-0.2.6}/tests/test_workflow.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: crispyx
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.6
|
|
4
4
|
Summary: Memory-efficient streaming analysis of large-scale CRISPR and Perturb-seq screens on disk-backed AnnData files
|
|
5
5
|
Author: Jin-Hong Du
|
|
6
6
|
License: Modified MIT License
|
|
@@ -145,7 +145,7 @@ Dynamic: license-file
|
|
|
145
145
|
|
|
146
146
|
Genome-wide CRISPR screens routinely produce datasets with hundreds of thousands of cells and tens of thousands of genes. Standard single-cell analysis toolkits (Scanpy, Pertpy) load the entire count matrix into memory, requiring large RAM allocations and often making routine workflows impractical on laptops or shared compute nodes.
|
|
147
147
|
|
|
148
|
-
**crispyx**
|
|
148
|
+
**crispyx** runs quality control, normalisation, pseudo-bulk aggregation, differential expression (t-test, Wilcoxon, NB-GLM with covariates, covariate-adjusted residual effects), control checks and PCA/KNN on screens stored as `.h5ad` files, streaming from disk under a memory budget, so that million-cell screens run on a laptop or a cluster node. The API mirrors Scanpy. It starts from an `.h5ad` file; it does not read raw sequencing outputs, and it is not a general single-cell toolkit.
|
|
149
149
|
|
|
150
150
|
**Documentation:** [crispyx.readthedocs.io](https://crispyx.readthedocs.io/en/latest/) — [installation](https://crispyx.readthedocs.io/en/latest/installation.html), [quick start](https://crispyx.readthedocs.io/en/latest/quickstart.html), [tutorials](https://crispyx.readthedocs.io/en/latest/notebooks/crispyx_tutorial.html) and [API reference](https://crispyx.readthedocs.io/en/latest/api.html).
|
|
151
151
|
|
|
@@ -202,18 +202,18 @@ For the full workflow (normalisation, PCA, pseudo-bulk, NB-GLM, LFC shrinkage, p
|
|
|
202
202
|
- **Streaming QC & preprocessing** – Filter cells, perturbations, and genes; normalise and log-transform; select highly variable genes (`seurat_v3`/`mean_dispersion`, control-cells-only by default); CSC-aware streaming with `format_mismatch_policy`; all without loading the full matrix into memory
|
|
203
203
|
- **Subsampling & downsampling** – Stratified or cluster-sampled cell subsampling (`cx.pp.subsample`, exact count or proportion per stratum, drop or keep small groups) and dependency-free per-cell count thinning (`cx.pp.downsample_counts`, the streaming equivalent of `scanpy.pp.downsample_counts`) for aligning dataset scale and sequencing depth before comparing screens
|
|
204
204
|
- **Pseudo-bulk aggregation** – Absolute profiles over multiple grouping columns (for example, perturbation × batch), strict count sums or mean log1p expression, optional deterministic bootstrap sampling, and explicit within-batch effect calculation
|
|
205
|
-
- **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`),
|
|
205
|
+
- **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`), negative binomial GLM with covariates and apeGLM LFC shrinkage, and a fixed-offset NB test for many small perturbations against a large, stratified control set (`cx.tl.nb_offset_test`); multi-core support and adaptive memory management; per-condition low-expression filtering to exclude genes that are near-zero in both groups
|
|
206
206
|
- **Covariate-adjusted effects** – `cx.tl.residual_effects` (Feng et al., 2026) fits each gene to covariates such as donor, lane or batch on the control cells only, and takes a perturbation's effect as the mean residual of its cells. Unlike the NB GLM, which fits the covariates on counts jointly with each perturbation, the control fit is shared by every perturbation, so a perturbation's own cells cannot shift its covariate adjustment, and all perturbations come from one streaming pass. On a 1.16-million-cell screen it reproduces the published effects and significance calls in under a minute.
|
|
207
207
|
- **Storage-format aware** – Gene-streaming functions (`wilcoxon_test`, `batch_process`) can convert a CSR source once to a temporary, memory-bounded CSC copy beside the output instead of re-reading the whole matrix once per gene chunk; `format_mismatch_policy="auto"` (the default) measures the source's read throughput and does so only when the re-reads would really cost more than the conversion, so a small local file is streamed as-is and a large one on a network filesystem is converted; `convert_to_csc`/`convert_to_csr` honour `memory_limit_gb` and preserve the value dtype
|
|
208
208
|
- **Dimension reduction** – Memory-efficient PCA and KNN graph construction on backed data
|
|
209
209
|
- **Scanpy-compatible API & plotting** – Familiar `cx.pp`, `cx.pb`, `cx.tl`, and `cx.pl` namespaces; Scanpy-style rank genes plots, volcano, MA, PCA, UMAP, QC summaries, and overlap heatmaps
|
|
210
|
-
- **Data preparation utilities** – Edit backed metadata without loading X;
|
|
210
|
+
- **Data preparation utilities** – Edit backed metadata without loading X; normalise perturbation labels and take target genes out of guide names (`target_pattern`); assign guides from guide-capture counts (`cx.pp.assign_guides`); recover integer counts from data published as log1p-normalised values (`cx.pp.recover_counts`); sum the rows of a gene split over several (`cx.pp.collapse_genes`); concatenate multi-file screens; check a file for the mistakes that mislead an analysis (`cx.pp.check_data`); auto-detect metadata columns
|
|
211
211
|
- **HPC-ready** – Resume/checkpoint for long-running jobs; configurable `memory_limit_gb`
|
|
212
212
|
- **Disk-aware** – Estimates and warns about scratch-disk usage before large writes or CSC/CSR conversions, and `cx.estimate_disk_usage(...)` answers "how much disk will this need?" up front; the memory savings above assume the machine has enough free disk for streaming intermediates and output files
|
|
213
213
|
|
|
214
214
|
## Performance
|
|
215
215
|
|
|
216
|
-
crispyx
|
|
216
|
+
Across 12 public CRISPR screens, crispyx completes every dataset where Scanpy, Pertpy/PyDESeq2 and edgeR run out of memory or time, with t-test and Wilcoxon results matching Scanpy to Pearson *r* > 0.999 and NB-GLM statistics agreeing with PyDESeq2 given the same size factors (z correlation 0.997–0.9999):
|
|
217
217
|
|
|
218
218
|
<p align="center">
|
|
219
219
|
<img src="docs/_static/fig2.png" width="800" alt="Benchmark results across 12 CRISPR screens: (a) dataset sizes, (b) completion status by method, (c) concordance with Scanpy, (d) runtime scaling, (e) peak memory scaling">
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
Genome-wide CRISPR screens routinely produce datasets with hundreds of thousands of cells and tens of thousands of genes. Standard single-cell analysis toolkits (Scanpy, Pertpy) load the entire count matrix into memory, requiring large RAM allocations and often making routine workflows impractical on laptops or shared compute nodes.
|
|
13
13
|
|
|
14
|
-
**crispyx**
|
|
14
|
+
**crispyx** runs quality control, normalisation, pseudo-bulk aggregation, differential expression (t-test, Wilcoxon, NB-GLM with covariates, covariate-adjusted residual effects), control checks and PCA/KNN on screens stored as `.h5ad` files, streaming from disk under a memory budget, so that million-cell screens run on a laptop or a cluster node. The API mirrors Scanpy. It starts from an `.h5ad` file; it does not read raw sequencing outputs, and it is not a general single-cell toolkit.
|
|
15
15
|
|
|
16
16
|
**Documentation:** [crispyx.readthedocs.io](https://crispyx.readthedocs.io/en/latest/) — [installation](https://crispyx.readthedocs.io/en/latest/installation.html), [quick start](https://crispyx.readthedocs.io/en/latest/quickstart.html), [tutorials](https://crispyx.readthedocs.io/en/latest/notebooks/crispyx_tutorial.html) and [API reference](https://crispyx.readthedocs.io/en/latest/api.html).
|
|
17
17
|
|
|
@@ -68,18 +68,18 @@ For the full workflow (normalisation, PCA, pseudo-bulk, NB-GLM, LFC shrinkage, p
|
|
|
68
68
|
- **Streaming QC & preprocessing** – Filter cells, perturbations, and genes; normalise and log-transform; select highly variable genes (`seurat_v3`/`mean_dispersion`, control-cells-only by default); CSC-aware streaming with `format_mismatch_policy`; all without loading the full matrix into memory
|
|
69
69
|
- **Subsampling & downsampling** – Stratified or cluster-sampled cell subsampling (`cx.pp.subsample`, exact count or proportion per stratum, drop or keep small groups) and dependency-free per-cell count thinning (`cx.pp.downsample_counts`, the streaming equivalent of `scanpy.pp.downsample_counts`) for aligning dataset scale and sequencing depth before comparing screens
|
|
70
70
|
- **Pseudo-bulk aggregation** – Absolute profiles over multiple grouping columns (for example, perturbation × batch), strict count sums or mean log1p expression, optional deterministic bootstrap sampling, and explicit within-batch effect calculation
|
|
71
|
-
- **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`),
|
|
71
|
+
- **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`), negative binomial GLM with covariates and apeGLM LFC shrinkage, and a fixed-offset NB test for many small perturbations against a large, stratified control set (`cx.tl.nb_offset_test`); multi-core support and adaptive memory management; per-condition low-expression filtering to exclude genes that are near-zero in both groups
|
|
72
72
|
- **Covariate-adjusted effects** – `cx.tl.residual_effects` (Feng et al., 2026) fits each gene to covariates such as donor, lane or batch on the control cells only, and takes a perturbation's effect as the mean residual of its cells. Unlike the NB GLM, which fits the covariates on counts jointly with each perturbation, the control fit is shared by every perturbation, so a perturbation's own cells cannot shift its covariate adjustment, and all perturbations come from one streaming pass. On a 1.16-million-cell screen it reproduces the published effects and significance calls in under a minute.
|
|
73
73
|
- **Storage-format aware** – Gene-streaming functions (`wilcoxon_test`, `batch_process`) can convert a CSR source once to a temporary, memory-bounded CSC copy beside the output instead of re-reading the whole matrix once per gene chunk; `format_mismatch_policy="auto"` (the default) measures the source's read throughput and does so only when the re-reads would really cost more than the conversion, so a small local file is streamed as-is and a large one on a network filesystem is converted; `convert_to_csc`/`convert_to_csr` honour `memory_limit_gb` and preserve the value dtype
|
|
74
74
|
- **Dimension reduction** – Memory-efficient PCA and KNN graph construction on backed data
|
|
75
75
|
- **Scanpy-compatible API & plotting** – Familiar `cx.pp`, `cx.pb`, `cx.tl`, and `cx.pl` namespaces; Scanpy-style rank genes plots, volcano, MA, PCA, UMAP, QC summaries, and overlap heatmaps
|
|
76
|
-
- **Data preparation utilities** – Edit backed metadata without loading X;
|
|
76
|
+
- **Data preparation utilities** – Edit backed metadata without loading X; normalise perturbation labels and take target genes out of guide names (`target_pattern`); assign guides from guide-capture counts (`cx.pp.assign_guides`); recover integer counts from data published as log1p-normalised values (`cx.pp.recover_counts`); sum the rows of a gene split over several (`cx.pp.collapse_genes`); concatenate multi-file screens; check a file for the mistakes that mislead an analysis (`cx.pp.check_data`); auto-detect metadata columns
|
|
77
77
|
- **HPC-ready** – Resume/checkpoint for long-running jobs; configurable `memory_limit_gb`
|
|
78
78
|
- **Disk-aware** – Estimates and warns about scratch-disk usage before large writes or CSC/CSR conversions, and `cx.estimate_disk_usage(...)` answers "how much disk will this need?" up front; the memory savings above assume the machine has enough free disk for streaming intermediates and output files
|
|
79
79
|
|
|
80
80
|
## Performance
|
|
81
81
|
|
|
82
|
-
crispyx
|
|
82
|
+
Across 12 public CRISPR screens, crispyx completes every dataset where Scanpy, Pertpy/PyDESeq2 and edgeR run out of memory or time, with t-test and Wilcoxon results matching Scanpy to Pearson *r* > 0.999 and NB-GLM statistics agreeing with PyDESeq2 given the same size factors (z correlation 0.997–0.9999):
|
|
83
83
|
|
|
84
84
|
<p align="center">
|
|
85
85
|
<img src="docs/_static/fig2.png" width="800" alt="Benchmark results across 12 CRISPR screens: (a) dataset sizes, (b) completion status by method, (c) concordance with Scanpy, (d) runtime scaling, (e) peak memory scaling">
|
|
@@ -41,6 +41,8 @@ from .data import (
|
|
|
41
41
|
normalise_perturbation_labels,
|
|
42
42
|
read_h5ad_ondisk,
|
|
43
43
|
read_backed,
|
|
44
|
+
recover_counts,
|
|
45
|
+
collapse_genes,
|
|
44
46
|
resolve_data_path,
|
|
45
47
|
standardise_gene_names,
|
|
46
48
|
write_filtered_subset,
|
|
@@ -50,7 +52,10 @@ from .data import (
|
|
|
50
52
|
from .de import (
|
|
51
53
|
RankGenesGroupsResult,
|
|
52
54
|
nb_glm_test,
|
|
55
|
+
nb_offset_test,
|
|
53
56
|
shrink_lfc,
|
|
57
|
+
shrink_lfc_arrays,
|
|
58
|
+
ShrunkLFC,
|
|
54
59
|
t_test,
|
|
55
60
|
wilcoxon_test,
|
|
56
61
|
)
|
|
@@ -84,12 +89,16 @@ from .pseudobulk import (
|
|
|
84
89
|
compute_pseudobulk_effects,
|
|
85
90
|
)
|
|
86
91
|
from .qc import (
|
|
92
|
+
Issue,
|
|
93
|
+
assign_guides,
|
|
94
|
+
check_perturbation_data,
|
|
87
95
|
compare_controls,
|
|
88
96
|
control_guide_correlation,
|
|
89
97
|
filter_cells_by_gene_count,
|
|
90
98
|
filter_genes_by_cell_count,
|
|
91
99
|
filter_perturbations_by_cell_count,
|
|
92
100
|
quality_control_summary,
|
|
101
|
+
raise_on_errors,
|
|
93
102
|
)
|
|
94
103
|
from .residual import residual_effects
|
|
95
104
|
from .sample import subsample
|
|
@@ -128,6 +137,10 @@ __all__ = [
|
|
|
128
137
|
"quality_control_summary",
|
|
129
138
|
"compare_controls",
|
|
130
139
|
"control_guide_correlation",
|
|
140
|
+
"assign_guides",
|
|
141
|
+
"check_perturbation_data",
|
|
142
|
+
"Issue",
|
|
143
|
+
"raise_on_errors",
|
|
131
144
|
"plot_control_guide_correlation",
|
|
132
145
|
"highly_variable_genes",
|
|
133
146
|
# Subsampling
|
|
@@ -147,7 +160,10 @@ __all__ = [
|
|
|
147
160
|
"t_test",
|
|
148
161
|
"wilcoxon_test",
|
|
149
162
|
"nb_glm_test",
|
|
163
|
+
"nb_offset_test",
|
|
150
164
|
"shrink_lfc",
|
|
165
|
+
"shrink_lfc_arrays",
|
|
166
|
+
"ShrunkLFC",
|
|
151
167
|
# Data utilities
|
|
152
168
|
"AnnData",
|
|
153
169
|
"ensure_gene_symbol_column",
|
|
@@ -156,6 +172,8 @@ __all__ = [
|
|
|
156
172
|
"resolve_data_path",
|
|
157
173
|
"normalize_total_log1p",
|
|
158
174
|
"downsample_counts",
|
|
175
|
+
"recover_counts",
|
|
176
|
+
"collapse_genes",
|
|
159
177
|
"convert_to_csc",
|
|
160
178
|
"convert_to_csr",
|
|
161
179
|
"compress_h5ad",
|
|
@@ -253,6 +253,8 @@ def fingerprint_difference(previous: dict, current: dict) -> str | None:
|
|
|
253
253
|
return "the input file has changed since"
|
|
254
254
|
if previous.get("inputs") != current.get("inputs"):
|
|
255
255
|
return "an input file has changed since"
|
|
256
|
+
if previous.get("crispyx") != current.get("crispyx"):
|
|
257
|
+
return f"it was written by crispyx {previous.get('crispyx', 'before 0.2.6')}"
|
|
256
258
|
old_params = previous.get("params", {})
|
|
257
259
|
new_params = current.get("params", {})
|
|
258
260
|
for name in sorted(set(old_params) | set(new_params)):
|
|
@@ -94,8 +94,8 @@ def _nb_loglik_grid_numba(
|
|
|
94
94
|
def _neg_half_log_det_numba(xtwx: np.ndarray) -> float:
|
|
95
95
|
"""``-0.5 * log det(xtwx)`` from a Cholesky factorisation in place.
|
|
96
96
|
|
|
97
|
-
Zero for an empty matrix (no Cox-Reid adjustment)
|
|
98
|
-
positive definite
|
|
97
|
+
Zero for an empty matrix (no Cox-Reid adjustment); NaN for one that is
|
|
98
|
+
not positive definite.
|
|
99
99
|
"""
|
|
100
100
|
log_det = 0.0
|
|
101
101
|
for j in range(xtwx.shape[0]):
|
|
@@ -103,7 +103,7 @@ def _neg_half_log_det_numba(xtwx: np.ndarray) -> float:
|
|
|
103
103
|
for k in range(j):
|
|
104
104
|
d -= xtwx[j, k] * xtwx[j, k]
|
|
105
105
|
if d <= 0.0:
|
|
106
|
-
return
|
|
106
|
+
return np.nan
|
|
107
107
|
ljj = math.sqrt(d)
|
|
108
108
|
xtwx[j, j] = ljj
|
|
109
109
|
log_det += math.log(ljj)
|
|
@@ -132,6 +132,9 @@ def _brent_minimize_numba(
|
|
|
132
132
|
x_indices: np.ndarray,
|
|
133
133
|
x_data: np.ndarray,
|
|
134
134
|
xtwx: np.ndarray,
|
|
135
|
+
x_group: np.ndarray,
|
|
136
|
+
gdiag: np.ndarray,
|
|
137
|
+
gcross: np.ndarray,
|
|
135
138
|
a: float,
|
|
136
139
|
b: float,
|
|
137
140
|
tol: float = 1e-5,
|
|
@@ -149,7 +152,7 @@ def _brent_minimize_numba(
|
|
|
149
152
|
x = w = v = a + golden * (b - a)
|
|
150
153
|
fx = fw = fv = -_nb_posterior_with_cache_numba(
|
|
151
154
|
Y_g, mu_g, x, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
|
|
152
|
-
xtwx,
|
|
155
|
+
xtwx, x_group, gdiag, gcross,
|
|
153
156
|
)
|
|
154
157
|
|
|
155
158
|
d = 0.0 # Distance to next point
|
|
@@ -206,7 +209,7 @@ def _brent_minimize_numba(
|
|
|
206
209
|
|
|
207
210
|
fu = -_nb_posterior_with_cache_numba(
|
|
208
211
|
Y_g, mu_g, u, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
|
|
209
|
-
xtwx,
|
|
212
|
+
xtwx, x_group, gdiag, gcross,
|
|
210
213
|
)
|
|
211
214
|
|
|
212
215
|
# Update a, b, v, w, x
|
|
@@ -253,14 +256,23 @@ def _nb_posterior_with_cache_numba(
|
|
|
253
256
|
x_indices: np.ndarray,
|
|
254
257
|
x_data: np.ndarray,
|
|
255
258
|
xtwx: np.ndarray,
|
|
259
|
+
x_group: np.ndarray,
|
|
260
|
+
gdiag: np.ndarray,
|
|
261
|
+
gcross: np.ndarray,
|
|
256
262
|
) -> float:
|
|
257
263
|
"""Cox-Reid adjusted NB log-likelihood plus log-prior for one gene.
|
|
258
264
|
|
|
259
|
-
Uses the precomputed ``gammaln(Y+1)`` sum.
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
265
|
+
Uses the precomputed ``gammaln(Y+1)`` sum. The design is ``[F | G]``:
|
|
266
|
+
``F`` in CSR form (``x_indptr``, ``x_indices``, ``x_data``), and ``G``
|
|
267
|
+
one-hot, cell ``i`` in column ``x_group[i]`` (-1 for none). With ``W =
|
|
268
|
+
mu / (1 + alpha mu)``, ``FᵀWF`` is accumulated into the work buffer
|
|
269
|
+
``xtwx`` (``(0, 0)`` for no adjustment), the diagonal ``GᵀWG`` into
|
|
270
|
+
``gdiag`` and ``GᵀWF`` into ``gcross``, in the same pass over the cells
|
|
271
|
+
as the likelihood; then ``log det(XᵀWX) = sum log gdiag + log det(FᵀWF -
|
|
272
|
+
GᵀWFᵀ diag(gdiag)⁻¹ GᵀWF)``, the Schur complement of the one-hot block,
|
|
273
|
+
so hundreds of stratum columns cost ``n_groups x n_F**2`` rather than a
|
|
274
|
+
factorisation of the whole ``XᵀWX``. The adjustment is zero when
|
|
275
|
+
``XᵀWX`` is not positive definite. ``small_counts[k]``
|
|
264
276
|
is the number of cells with count ``k`` (below its length): their
|
|
265
277
|
``lgamma(y + r) - lgamma(r)`` terms depend on the count only, so each
|
|
266
278
|
value's is formed once, by recurrence, rather than once per cell.
|
|
@@ -271,6 +283,10 @@ def _nb_posterior_with_cache_numba(
|
|
|
271
283
|
log_r = math.log(r)
|
|
272
284
|
cox_reid = xtwx.shape[0] > 0
|
|
273
285
|
xtwx[:, :] = 0.0
|
|
286
|
+
n_groups = gdiag.shape[0]
|
|
287
|
+
if n_groups:
|
|
288
|
+
gdiag[:] = 0.0
|
|
289
|
+
gcross[:, :] = 0.0
|
|
274
290
|
|
|
275
291
|
# lgamma(y + r) - lgamma(r) - y log r, without libm's lgamma (see
|
|
276
292
|
# _lgamma_ratio_term): sum(log1p(i / r), i < y) for the small counts.
|
|
@@ -296,10 +312,31 @@ def _nb_posterior_with_cache_numba(
|
|
|
296
312
|
ja = x_indices[a]
|
|
297
313
|
for b in range(x_indptr[i], x_indptr[i + 1]):
|
|
298
314
|
xtwx[ja, x_indices[b]] += wa * x_data[b]
|
|
315
|
+
if n_groups and x_group[i] >= 0:
|
|
316
|
+
j = x_group[i]
|
|
317
|
+
gdiag[j] += w
|
|
318
|
+
for a in range(x_indptr[i], x_indptr[i + 1]):
|
|
319
|
+
gcross[j, x_indices[a]] += w * x_data[a]
|
|
299
320
|
ll -= gammaln_y_plus_1_sum
|
|
300
321
|
|
|
301
322
|
log_prior = -0.5 * (log_alpha - log_trend_g) ** 2 / prior_var
|
|
302
|
-
|
|
323
|
+
if not cox_reid:
|
|
324
|
+
return ll + log_prior
|
|
325
|
+
adjustment = 0.0
|
|
326
|
+
for j in range(n_groups):
|
|
327
|
+
dj = gdiag[j]
|
|
328
|
+
if dj <= 0.0:
|
|
329
|
+
return ll + log_prior # XᵀWX is not positive definite
|
|
330
|
+
adjustment -= 0.5 * math.log(dj)
|
|
331
|
+
for a in range(xtwx.shape[0]):
|
|
332
|
+
ca = gcross[j, a] / dj
|
|
333
|
+
if ca != 0.0:
|
|
334
|
+
for b in range(xtwx.shape[0]):
|
|
335
|
+
xtwx[a, b] -= ca * gcross[j, b]
|
|
336
|
+
schur = _neg_half_log_det_numba(xtwx)
|
|
337
|
+
if math.isnan(schur):
|
|
338
|
+
return ll + log_prior
|
|
339
|
+
return ll + log_prior + adjustment + schur
|
|
303
340
|
|
|
304
341
|
|
|
305
342
|
@nb.njit(parallel=True, cache=True)
|
|
@@ -315,6 +352,8 @@ def _nb_map_grid_search_with_refinement_numba(
|
|
|
315
352
|
x_indices: np.ndarray,
|
|
316
353
|
x_data: np.ndarray,
|
|
317
354
|
n_features: int,
|
|
355
|
+
x_group: np.ndarray,
|
|
356
|
+
n_groups: int,
|
|
318
357
|
tol: float = 1e-5,
|
|
319
358
|
max_refine_iter: int = 50,
|
|
320
359
|
) -> np.ndarray:
|
|
@@ -339,8 +378,12 @@ def _nb_map_grid_search_with_refinement_numba(
|
|
|
339
378
|
Variance of the log-normal prior (``inf`` for none: the gene-wise
|
|
340
379
|
estimate).
|
|
341
380
|
x_indptr, x_indices, x_data, n_features
|
|
342
|
-
The design
|
|
343
|
-
(``n_features=0`` for none).
|
|
381
|
+
The design's columns outside its one-hot block in CSR form, for the
|
|
382
|
+
Cox-Reid adjustment (``n_features=0`` for none).
|
|
383
|
+
x_group, n_groups
|
|
384
|
+
Each cell's column of the one-hot block (-1 for none), and the
|
|
385
|
+
block's width (0 for no block): see
|
|
386
|
+
:func:`_nb_posterior_with_cache_numba`.
|
|
344
387
|
tol : float
|
|
345
388
|
Tolerance for Brent convergence.
|
|
346
389
|
max_refine_iter : int
|
|
@@ -358,6 +401,8 @@ def _nb_map_grid_search_with_refinement_numba(
|
|
|
358
401
|
Y_g = Y[:, g]
|
|
359
402
|
mu_g = mu[:, g]
|
|
360
403
|
xtwx = np.empty((n_features, n_features))
|
|
404
|
+
gdiag = np.empty(n_groups)
|
|
405
|
+
gcross = np.empty((n_groups, n_features))
|
|
361
406
|
small_counts = np.zeros(_SMALL_COUNTS)
|
|
362
407
|
for i in range(Y_g.shape[0]):
|
|
363
408
|
if Y_g[i] < _SMALL_COUNTS:
|
|
@@ -369,7 +414,7 @@ def _nb_map_grid_search_with_refinement_numba(
|
|
|
369
414
|
for k in range(n_grid):
|
|
370
415
|
posterior = _nb_posterior_with_cache_numba(
|
|
371
416
|
Y_g, mu_g, lo[g] + k * step, log_trend_g, prior_var, 0.0, small_counts,
|
|
372
|
-
x_indptr, x_indices, x_data, xtwx,
|
|
417
|
+
x_indptr, x_indices, x_data, xtwx, x_group, gdiag, gcross,
|
|
373
418
|
)
|
|
374
419
|
if posterior > best_posterior:
|
|
375
420
|
best_posterior = posterior
|
|
@@ -378,11 +423,11 @@ def _nb_map_grid_search_with_refinement_numba(
|
|
|
378
423
|
if 0 < best_k < n_grid - 1 and 2 * step > tol:
|
|
379
424
|
refined = _brent_minimize_numba(
|
|
380
425
|
Y_g, mu_g, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data, xtwx,
|
|
381
|
-
best - step, best + step, tol=tol, max_iter=max_refine_iter,
|
|
426
|
+
x_group, gdiag, gcross, best - step, best + step, tol=tol, max_iter=max_refine_iter,
|
|
382
427
|
)
|
|
383
428
|
refined_posterior = _nb_posterior_with_cache_numba(
|
|
384
429
|
Y_g, mu_g, refined, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data,
|
|
385
|
-
xtwx,
|
|
430
|
+
xtwx, x_group, gdiag, gcross,
|
|
386
431
|
)
|
|
387
432
|
if refined_posterior >= best_posterior:
|
|
388
433
|
best = refined
|
|
@@ -498,6 +543,148 @@ def _compute_ctrl_tie_sums(
|
|
|
498
543
|
return ctrl_tie_sums
|
|
499
544
|
|
|
500
545
|
|
|
546
|
+
# The NB IRLS fits measure a deviance change relative to |deviance| plus
|
|
547
|
+
# this, DESeq2's ratio: a saturated fit's deviance near zero still converges.
|
|
548
|
+
_DEVIANCE_OFFSET = 0.1
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
# The rank-sum p-value treats the number of expressing cells that fall in the
|
|
552
|
+
# perturbation group as hypergeometric whenever its variance, summed over
|
|
553
|
+
# strata, is below this; above it the normal approximation is used.
|
|
554
|
+
_RANK_SUM_MIXTURE_MAX_VAR = 100.0
|
|
555
|
+
# The most components the strata's combined mixture keeps; beyond it,
|
|
556
|
+
# components with nearby means are merged.
|
|
557
|
+
_RANK_SUM_MIXTURE_MAX_COMPONENTS = 1 << 15
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
@nb.njit(cache=True)
|
|
561
|
+
def _merge_mixture(w: np.ndarray, m: np.ndarray, v: np.ndarray, n_bins: int) -> tuple:
|
|
562
|
+
"""Merge a normal mixture's components into at most ``n_bins`` by their
|
|
563
|
+
mean (equal-width bins over its range), each bin a normal with the
|
|
564
|
+
bin's weight, mean and variance."""
|
|
565
|
+
lo = m.min()
|
|
566
|
+
span = m.max() - lo
|
|
567
|
+
bins = np.zeros(m.shape[0], dtype=np.int64)
|
|
568
|
+
if span > 0.0:
|
|
569
|
+
for i in range(m.shape[0]):
|
|
570
|
+
bins[i] = min(int((m[i] - lo) / span * n_bins), n_bins - 1)
|
|
571
|
+
bw = np.zeros(n_bins)
|
|
572
|
+
bm = np.zeros(n_bins)
|
|
573
|
+
for i in range(m.shape[0]):
|
|
574
|
+
bw[bins[i]] += w[i]
|
|
575
|
+
bm[bins[i]] += w[i] * m[i]
|
|
576
|
+
for b in range(n_bins):
|
|
577
|
+
if bw[b] > 0.0:
|
|
578
|
+
bm[b] /= bw[b]
|
|
579
|
+
bv = np.zeros(n_bins)
|
|
580
|
+
for i in range(m.shape[0]):
|
|
581
|
+
d = m[i] - bm[bins[i]]
|
|
582
|
+
bv[bins[i]] += w[i] * (v[i] + d * d)
|
|
583
|
+
keep = np.flatnonzero(bw > 0.0)
|
|
584
|
+
return bw[keep], bm[keep], bv[keep] / bw[keep]
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
@nb.njit(cache=True)
|
|
588
|
+
def _rank_sum_mixture_pvalue(
|
|
589
|
+
n1s: np.ndarray, ns: np.ndarray, ks: np.ndarray, nz_tie_sums: np.ndarray, deviation: float,
|
|
590
|
+
max_components: int = _RANK_SUM_MIXTURE_MAX_COMPONENTS,
|
|
591
|
+
) -> float:
|
|
592
|
+
"""Two-sided p-value of a (stratified) rank sum when most values are tied
|
|
593
|
+
at zero.
|
|
594
|
+
|
|
595
|
+
Stratum ``b`` has ``ns[b]`` cells, ``n1s[b]`` of them in the group and
|
|
596
|
+
``ks[b]`` non-zero, whose ties give ``nz_tie_sums[b] = sum(t^3 - t)``;
|
|
597
|
+
``deviation`` is ``sum_b (R_b - E_b)``. Under the permutation null the
|
|
598
|
+
group's count of non-zero cells ``J_b`` is hypergeometric: with few
|
|
599
|
+
non-zeros it is very skewed, which the normal approximation misses by
|
|
600
|
+
orders of magnitude (one expressing cell of 232 against 3 of 30,000
|
|
601
|
+
controls: p 3e-8 against a permutation p of 0.03). Here ``J_b`` keeps
|
|
602
|
+
its exact distribution, and given ``J_b = j`` the rank sum is normal
|
|
603
|
+
with the exact mean and variance of ``j`` draws without replacement from
|
|
604
|
+
the non-zero ranks. The strata's
|
|
605
|
+
mixtures are combined exactly up to ``max_components`` components;
|
|
606
|
+
past it, :func:`_merge_mixture` merges those with nearby means, keeping
|
|
607
|
+
each merged component's weight, mean and variance.
|
|
608
|
+
"""
|
|
609
|
+
w = np.ones(1)
|
|
610
|
+
m = np.zeros(1)
|
|
611
|
+
v = np.zeros(1)
|
|
612
|
+
for b in range(n1s.shape[0]):
|
|
613
|
+
n1 = n1s[b]
|
|
614
|
+
n = ns[b]
|
|
615
|
+
k = ks[b]
|
|
616
|
+
z = n - k
|
|
617
|
+
if n1 == 0 or n1 == n:
|
|
618
|
+
continue
|
|
619
|
+
r0 = (z + 1.0) / 2.0
|
|
620
|
+
mean_nz = z + (k + 1.0) / 2.0
|
|
621
|
+
var_nz = ((k * k - 1.0) / 12.0 - nz_tie_sums[b] / (12.0 * k)) if k > 1 else 0.0
|
|
622
|
+
expected = n1 * (n + 1.0) / 2.0
|
|
623
|
+
lo = max(0, n1 - z)
|
|
624
|
+
hi = min(k, n1)
|
|
625
|
+
# Start at the mode and walk out by the ratio of successive terms.
|
|
626
|
+
mode = min(max(int((n1 + 1.0) * (k + 1.0) / (n + 2.0)), lo), hi)
|
|
627
|
+
log_mode = (math.lgamma(k + 1.0) - math.lgamma(mode + 1.0) - math.lgamma(k - mode + 1.0)
|
|
628
|
+
+ math.lgamma(z + 1.0) - math.lgamma(n1 - mode + 1.0) - math.lgamma(z - n1 + mode + 1.0)
|
|
629
|
+
- math.lgamma(n + 1.0) + math.lgamma(n1 + 1.0) + math.lgamma(n - n1 + 1.0))
|
|
630
|
+
width = hi - lo + 1
|
|
631
|
+
pmf = np.zeros(width)
|
|
632
|
+
pmf[mode - lo] = math.exp(log_mode)
|
|
633
|
+
for j in range(mode, hi):
|
|
634
|
+
pmf[j + 1 - lo] = pmf[j - lo] * (k - j) * (n1 - j) / ((j + 1.0) * (z - n1 + j + 1.0))
|
|
635
|
+
if pmf[j + 1 - lo] < 1e-300:
|
|
636
|
+
break
|
|
637
|
+
for j in range(mode, lo, -1):
|
|
638
|
+
pmf[j - 1 - lo] = pmf[j - lo] * j * (z - n1 + j) / ((k - j + 1.0) * (n1 - j + 1.0))
|
|
639
|
+
if pmf[j - 1 - lo] < 1e-300:
|
|
640
|
+
break
|
|
641
|
+
c = 0
|
|
642
|
+
for i in range(width):
|
|
643
|
+
if pmf[i] >= 1e-300:
|
|
644
|
+
c += 1
|
|
645
|
+
if c == 1:
|
|
646
|
+
j = mode
|
|
647
|
+
m += (n1 - j) * r0 + j * mean_nz - expected
|
|
648
|
+
v += j * (k - j) / (k - 1.0) * var_nz if k > 1 else 0.0
|
|
649
|
+
continue
|
|
650
|
+
if w.shape[0] * c > max_components:
|
|
651
|
+
w, m, v = _merge_mixture(w, m, v, max(max_components // c, 1))
|
|
652
|
+
nw = np.empty(w.shape[0] * c)
|
|
653
|
+
nm = np.empty(w.shape[0] * c)
|
|
654
|
+
nv = np.empty(w.shape[0] * c)
|
|
655
|
+
out = 0
|
|
656
|
+
for i in range(width):
|
|
657
|
+
if pmf[i] < 1e-300:
|
|
658
|
+
continue
|
|
659
|
+
j = lo + i
|
|
660
|
+
cm = (n1 - j) * r0 + j * mean_nz - expected
|
|
661
|
+
cv = j * (k - j) / (k - 1.0) * var_nz if k > 1 else 0.0
|
|
662
|
+
for a in range(w.shape[0]):
|
|
663
|
+
nw[out] = w[a] * pmf[i]
|
|
664
|
+
nm[out] = m[a] + cm
|
|
665
|
+
nv[out] = v[a] + cv
|
|
666
|
+
out += 1
|
|
667
|
+
w, m, v = nw, nm, nv
|
|
668
|
+
d = abs(deviation)
|
|
669
|
+
tol = 1e-9 * (1.0 + d)
|
|
670
|
+
total = 0.0
|
|
671
|
+
for i in range(w.shape[0]):
|
|
672
|
+
if v[i] > 0.0:
|
|
673
|
+
sd = math.sqrt(2.0 * v[i])
|
|
674
|
+
total += w[i] * 0.5 * (math.erfc((d - m[i]) / sd) + math.erfc((d + m[i]) / sd))
|
|
675
|
+
elif m[i] >= d - tol or m[i] <= -d + tol:
|
|
676
|
+
total += w[i]
|
|
677
|
+
return min(total, 1.0)
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
@nb.njit(cache=True)
|
|
681
|
+
def _hypergeometric_variance(n1: float, n: float, k: float) -> float:
|
|
682
|
+
"""Variance of the group's count of non-zero cells under the permutation null."""
|
|
683
|
+
if n <= 1.0:
|
|
684
|
+
return 0.0
|
|
685
|
+
return n1 * (k / n) * ((n - k) / n) * ((n - n1) / (n - 1.0))
|
|
686
|
+
|
|
687
|
+
|
|
501
688
|
@nb.njit(cache=True)
|
|
502
689
|
def _all_tied(n_zeros: int, ctrl_sorted: np.ndarray, pert_sorted: np.ndarray) -> bool:
|
|
503
690
|
"""Whether every value of a gene -- ``n_zeros`` zeros plus the sorted
|
|
@@ -551,6 +738,8 @@ def _rank_sum_pert_bsearch_numba(
|
|
|
551
738
|
``n_pert_zeros * zero_avg_rank`` is added by the caller).
|
|
552
739
|
tie_corr : float
|
|
553
740
|
Tie correction factor: 1 - sum(t^3 - t) / (n_total^3 - n_total).
|
|
741
|
+
tie_sum : float
|
|
742
|
+
``sum(t^3 - t)`` over every tie group, the zeros included.
|
|
554
743
|
"""
|
|
555
744
|
n_ctrl_nz = ctrl_sorted.shape[0]
|
|
556
745
|
n_pert_nz = pert_sorted.shape[0]
|
|
@@ -614,7 +803,7 @@ def _rank_sum_pert_bsearch_numba(
|
|
|
614
803
|
else:
|
|
615
804
|
tie_corr = 1.0
|
|
616
805
|
|
|
617
|
-
return rank_sum, tie_corr
|
|
806
|
+
return rank_sum, tie_corr, tie_sum
|
|
618
807
|
|
|
619
808
|
|
|
620
809
|
@nb.njit(parallel=False, cache=True)
|
|
@@ -628,6 +817,7 @@ def _wilcoxon_single_pert_presorted(
|
|
|
628
817
|
pert_dense: np.ndarray,
|
|
629
818
|
valid_genes: np.ndarray,
|
|
630
819
|
tie_correct: bool,
|
|
820
|
+
hypergeometric: bool,
|
|
631
821
|
zero_threshold: float,
|
|
632
822
|
u_stat_out: np.ndarray,
|
|
633
823
|
z_score_out: np.ndarray,
|
|
@@ -673,6 +863,7 @@ def _wilcoxon_single_pert_presorted(
|
|
|
673
863
|
tied = n_zeros == n_total
|
|
674
864
|
rank_sum = 0.0
|
|
675
865
|
tie_corr = 1.0
|
|
866
|
+
tie_sum = 0.0
|
|
676
867
|
if not tied:
|
|
677
868
|
# --- Binary-search ranking (always) ---
|
|
678
869
|
# O(n_pert_nz * log(n_ctrl_nz)) — works for any zero fraction.
|
|
@@ -693,7 +884,7 @@ def _wilcoxon_single_pert_presorted(
|
|
|
693
884
|
ctrl_sorted = ctrl_sorted_flat[start : start + n_ctrl_nz]
|
|
694
885
|
|
|
695
886
|
tied = _all_tied(n_zeros, ctrl_sorted, pert_sorted)
|
|
696
|
-
rank_sum_nz, tie_corr = _rank_sum_pert_bsearch_numba(
|
|
887
|
+
rank_sum_nz, tie_corr, tie_sum = _rank_sum_pert_bsearch_numba(
|
|
697
888
|
ctrl_sorted, pert_sorted, n_zeros, ctrl_tie_sums[g]
|
|
698
889
|
)
|
|
699
890
|
|
|
@@ -722,6 +913,13 @@ def _wilcoxon_single_pert_presorted(
|
|
|
722
913
|
z = (rank_sum - expected) / std
|
|
723
914
|
abs_z = abs(z)
|
|
724
915
|
pval = math.erfc(abs_z / math.sqrt(2.0))
|
|
916
|
+
n_nonzero = n_total - n_zeros
|
|
917
|
+
if hypergeometric and _hypergeometric_variance(n_pert_f, n_total_f, float(n_nonzero)) < _RANK_SUM_MIXTURE_MAX_VAR:
|
|
918
|
+
zf = float(n_zeros)
|
|
919
|
+
pval = _rank_sum_mixture_pvalue(
|
|
920
|
+
np.array([n_pert]), np.array([n_total]), np.array([n_nonzero]),
|
|
921
|
+
np.array([tie_sum - (zf ** 3 - zf)]), rank_sum - expected,
|
|
922
|
+
)
|
|
725
923
|
else:
|
|
726
924
|
# An empty group: no comparison was made.
|
|
727
925
|
z = np.nan
|
|
@@ -753,6 +951,7 @@ def _wilcoxon_batch_perts_presorted_numba(
|
|
|
753
951
|
pert_row_offsets: np.ndarray,
|
|
754
952
|
valid_masks: np.ndarray,
|
|
755
953
|
tie_correct: bool,
|
|
954
|
+
hypergeometric: bool,
|
|
756
955
|
zero_threshold: float,
|
|
757
956
|
u_stat_out: np.ndarray,
|
|
758
957
|
z_score_out: np.ndarray,
|
|
@@ -788,6 +987,9 @@ def _wilcoxon_batch_perts_presorted_numba(
|
|
|
788
987
|
valid_masks : (n_perts, n_valid_genes)
|
|
789
988
|
Boolean validity mask per (pert, gene).
|
|
790
989
|
tie_correct : bool
|
|
990
|
+
hypergeometric : bool
|
|
991
|
+
Whether p-values come from :func:`_rank_sum_mixture_pvalue` where the
|
|
992
|
+
group's count of non-zero cells is skewed.
|
|
791
993
|
zero_threshold : float
|
|
792
994
|
u_stat_out : (n_perts, n_valid_genes)
|
|
793
995
|
z_score_out : (n_perts, n_valid_genes)
|
|
@@ -811,6 +1013,7 @@ def _wilcoxon_batch_perts_presorted_numba(
|
|
|
811
1013
|
pert_dense,
|
|
812
1014
|
valid_masks[p_idx],
|
|
813
1015
|
tie_correct,
|
|
1016
|
+
hypergeometric,
|
|
814
1017
|
zero_threshold,
|
|
815
1018
|
u_stat_out[p_idx],
|
|
816
1019
|
z_score_out[p_idx],
|
|
@@ -833,6 +1036,7 @@ def _wilcoxon_stratified_single_pert(
|
|
|
833
1036
|
seg_hi: int,
|
|
834
1037
|
valid_genes: np.ndarray,
|
|
835
1038
|
tie_correct: bool,
|
|
1039
|
+
hypergeometric: bool,
|
|
836
1040
|
u_stat_out: np.ndarray,
|
|
837
1041
|
z_score_out: np.ndarray,
|
|
838
1042
|
pvalue_out: np.ndarray,
|
|
@@ -867,6 +1071,11 @@ def _wilcoxon_stratified_single_pert(
|
|
|
867
1071
|
"""
|
|
868
1072
|
n_genes = all_pert_stacked.shape[1]
|
|
869
1073
|
sqrt2 = math.sqrt(2.0)
|
|
1074
|
+
n_seg = seg_hi - seg_lo
|
|
1075
|
+
strata_n1 = np.empty(n_seg, dtype=np.int64)
|
|
1076
|
+
strata_n = np.empty(n_seg, dtype=np.int64)
|
|
1077
|
+
strata_k = np.empty(n_seg, dtype=np.int64)
|
|
1078
|
+
strata_ties = np.empty(n_seg, dtype=np.float64)
|
|
870
1079
|
|
|
871
1080
|
for g in range(n_genes):
|
|
872
1081
|
if not valid_genes[g]:
|
|
@@ -881,6 +1090,8 @@ def _wilcoxon_stratified_single_pert(
|
|
|
881
1090
|
u_sum = 0.0 # sum_b U_b
|
|
882
1091
|
n1n0_sum = 0.0 # sum_b n1_b * n0_b
|
|
883
1092
|
tied = True # every contributing stratum has a single value
|
|
1093
|
+
var_j = 0.0 # sum_b Var(J_b): the group's count of non-zero cells
|
|
1094
|
+
n_used = 0
|
|
884
1095
|
|
|
885
1096
|
for s in range(seg_lo, seg_hi):
|
|
886
1097
|
b = seg_batch[s]
|
|
@@ -919,9 +1130,16 @@ def _wilcoxon_stratified_single_pert(
|
|
|
919
1130
|
|
|
920
1131
|
if not _all_tied(n_zeros, ctrl_sorted, pert_sorted):
|
|
921
1132
|
tied = False
|
|
922
|
-
rank_sum_nz, tie_corr = _rank_sum_pert_bsearch_numba(
|
|
1133
|
+
rank_sum_nz, tie_corr, tie_sum = _rank_sum_pert_bsearch_numba(
|
|
923
1134
|
ctrl_sorted, pert_sorted, n_zeros, ctrl_tie_sums[b, g]
|
|
924
1135
|
)
|
|
1136
|
+
zf = float(n_zeros)
|
|
1137
|
+
strata_n1[n_used] = n1
|
|
1138
|
+
strata_n[n_used] = N
|
|
1139
|
+
strata_k[n_used] = N - n_zeros
|
|
1140
|
+
strata_ties[n_used] = tie_sum - (zf ** 3 - zf)
|
|
1141
|
+
n_used += 1
|
|
1142
|
+
var_j += _hypergeometric_variance(float(n1), float(N), float(N - n_zeros))
|
|
925
1143
|
if not tie_correct:
|
|
926
1144
|
tie_corr = 1.0
|
|
927
1145
|
|
|
@@ -943,6 +1161,10 @@ def _wilcoxon_stratified_single_pert(
|
|
|
943
1161
|
if var > 0.0 and not tied:
|
|
944
1162
|
z = num / math.sqrt(var)
|
|
945
1163
|
pval = math.erfc(abs(z) / sqrt2)
|
|
1164
|
+
if hypergeometric and var_j < _RANK_SUM_MIXTURE_MAX_VAR:
|
|
1165
|
+
pval = _rank_sum_mixture_pvalue(
|
|
1166
|
+
strata_n1[:n_used], strata_n[:n_used], strata_k[:n_used], strata_ties[:n_used], num,
|
|
1167
|
+
)
|
|
946
1168
|
else:
|
|
947
1169
|
# No stratum to compare in, or every value tied within each:
|
|
948
1170
|
# the rank test is undefined (with or without the tie
|
|
@@ -977,6 +1199,7 @@ def _wilcoxon_stratified_batch_perts_numba(
|
|
|
977
1199
|
pert_ptr: np.ndarray,
|
|
978
1200
|
valid_masks: np.ndarray,
|
|
979
1201
|
tie_correct: bool,
|
|
1202
|
+
hypergeometric: bool,
|
|
980
1203
|
u_stat_out: np.ndarray,
|
|
981
1204
|
z_score_out: np.ndarray,
|
|
982
1205
|
pvalue_out: np.ndarray,
|
|
@@ -1006,6 +1229,8 @@ def _wilcoxon_stratified_batch_perts_numba(
|
|
|
1006
1229
|
perturbation ``p``.
|
|
1007
1230
|
valid_masks : (n_perts, n_valid_genes)
|
|
1008
1231
|
tie_correct : bool
|
|
1232
|
+
hypergeometric : bool
|
|
1233
|
+
As in :func:`_wilcoxon_batch_perts_presorted_numba`.
|
|
1009
1234
|
u_stat_out, z_score_out, pvalue_out, effect_out : (n_perts, n_valid_genes)
|
|
1010
1235
|
"""
|
|
1011
1236
|
n_perts = pert_ptr.shape[0] - 1
|
|
@@ -1024,6 +1249,7 @@ def _wilcoxon_stratified_batch_perts_numba(
|
|
|
1024
1249
|
pert_ptr[p_idx + 1],
|
|
1025
1250
|
valid_masks[p_idx],
|
|
1026
1251
|
tie_correct,
|
|
1252
|
+
hypergeometric,
|
|
1027
1253
|
u_stat_out[p_idx],
|
|
1028
1254
|
z_score_out[p_idx],
|
|
1029
1255
|
pvalue_out[p_idx],
|
|
@@ -1974,7 +2200,7 @@ def _structured_nb_irls(X, labels, offset, Y, alpha, n_groups, const_col, const_
|
|
|
1974
2200
|
ceiling = dev + 1e-9 * abs(dev)
|
|
1975
2201
|
rose = new_dev > ceiling
|
|
1976
2202
|
worse = rose and not on_floor
|
|
1977
|
-
scale =
|
|
2203
|
+
scale = abs(dev) + _DEVIANCE_OFFSET
|
|
1978
2204
|
excess = (new_dev - dev) / scale
|
|
1979
2205
|
step = 1.0
|
|
1980
2206
|
for _h in range(max_halvings):
|
|
@@ -2001,7 +2227,7 @@ def _structured_nb_irls(X, labels, offset, Y, alpha, n_groups, const_col, const_
|
|
|
2001
2227
|
new_mu[:] = mu
|
|
2002
2228
|
new_dev = dev
|
|
2003
2229
|
relative_change = abs(new_dev - dev) / scale
|
|
2004
|
-
deviance_ratio = abs(new_dev - dev) / (abs(new_dev) +
|
|
2230
|
+
deviance_ratio = abs(new_dev - dev) / (abs(new_dev) + _DEVIANCE_OFFSET)
|
|
2005
2231
|
coef_change = 0.0
|
|
2006
2232
|
for j in range(dx):
|
|
2007
2233
|
coef_change = max(coef_change, abs(new_f[j] - bf[j]))
|