crispyx 0.2.3__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {crispyx-0.2.3/src/crispyx.egg-info → crispyx-0.2.5}/PKG-INFO +6 -6
- {crispyx-0.2.3 → crispyx-0.2.5}/README.md +4 -4
- {crispyx-0.2.3 → crispyx-0.2.5}/pyproject.toml +1 -1
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/__init__.py +18 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_irls.py +19 -6
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_kernels.py +327 -9
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_namespaces.py +94 -3
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_version.py +1 -1
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/batch.py +3 -3
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/data.py +796 -220
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/de.py +1055 -160
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/glm.py +266 -158
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/plotting.py +2 -1
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/pseudobulk.py +85 -9
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/qc.py +395 -3
- {crispyx-0.2.3 → crispyx-0.2.5/src/crispyx.egg-info}/PKG-INFO +6 -6
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx.egg-info/SOURCES.txt +8 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx.egg-info/requires.txt +1 -1
- crispyx-0.2.5/tests/test_assign_guides.py +88 -0
- crispyx-0.2.5/tests/test_atomic_outputs.py +168 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_batch_process.py +26 -0
- crispyx-0.2.5/tests/test_check_data.py +201 -0
- crispyx-0.2.5/tests/test_collapse_genes.py +122 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_concat_h5ad.py +1 -1
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_convert_format.py +4 -1
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_data_helpers.py +9 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_glm_structured.py +50 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_nb_glm.py +47 -7
- crispyx-0.2.5/tests/test_nb_offset.py +386 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_preflight_dispatch.py +8 -1
- crispyx-0.2.5/tests/test_pseudobulk_sparse_output.py +95 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_rank_genes_groups_outputs.py +17 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_read_backed.py +21 -0
- crispyx-0.2.5/tests/test_recover_counts.py +106 -0
- crispyx-0.2.5/tests/test_shrink_lfc_arrays.py +79 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_stream_on_fast_axis.py +6 -5
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_two_group_deseq2.py +101 -2
- {crispyx-0.2.3 → crispyx-0.2.5}/LICENSE +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/setup.cfg +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_checkpoint.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_disk.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_grouping.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_h5codec.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_memory.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_messages.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_preflight.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_provenance.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_size_factors.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/_statistics.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/dimred.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/hvg.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/profiling.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/residual.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx/sample.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx.egg-info/dependency_links.txt +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/src/crispyx.egg-info/top_level.txt +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_cgroup_memory_budget.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_compare_controls.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_compress_h5ad.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_de_edge_cases.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_de_lowexpr_filter.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_de_resume.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_dimred.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_disk_dispatch.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_downsample_counts.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_groupby_alias.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_hvg.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_in_memory_input.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_irls.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_memory_dispatch.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_messages.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_normalisation.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_normalized_effects.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_normalized_effects_se.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_obs_writes.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_pickle.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_plotting.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_profiling.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_pseudobulk_batch.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_pseudobulk_memory.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_pseudobulk_profiles.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_qc_parity.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_residual_effects.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_result_provenance.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_scanpy_parity.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_standardize.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_subsample.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_temp_file_cleanup.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_wilcoxon_dispatch.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_wilcoxon_labels.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_wilcoxon_stratified.py +0 -0
- {crispyx-0.2.3 → crispyx-0.2.5}/tests/test_workflow.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: crispyx
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: Memory-efficient streaming analysis of large-scale CRISPR and Perturb-seq screens on disk-backed AnnData files
|
|
5
5
|
Author: Jin-Hong Du
|
|
6
6
|
License: Modified MIT License
|
|
@@ -105,7 +105,7 @@ Requires-Dist: numpy>=1.23
|
|
|
105
105
|
Requires-Dist: numba>=0.59
|
|
106
106
|
Requires-Dist: pandas>=1.5
|
|
107
107
|
Requires-Dist: scipy>=1.10
|
|
108
|
-
Requires-Dist: h5py>=3.
|
|
108
|
+
Requires-Dist: h5py>=3.10
|
|
109
109
|
Requires-Dist: joblib>=1.4
|
|
110
110
|
Requires-Dist: scikit-learn>=1.0
|
|
111
111
|
Requires-Dist: scikit-misc>=0.1.4
|
|
@@ -145,7 +145,7 @@ Dynamic: license-file
|
|
|
145
145
|
|
|
146
146
|
Genome-wide CRISPR screens routinely produce datasets with hundreds of thousands of cells and tens of thousands of genes. Standard single-cell analysis toolkits (Scanpy, Pertpy) load the entire count matrix into memory, requiring large RAM allocations and often making routine workflows impractical on laptops or shared compute nodes.
|
|
147
147
|
|
|
148
|
-
**crispyx**
|
|
148
|
+
**crispyx** runs quality control, normalisation, pseudo-bulk aggregation, differential expression (t-test, Wilcoxon, NB-GLM with covariates, covariate-adjusted residual effects), control checks and PCA/KNN on screens stored as `.h5ad` files, streaming from disk under a memory budget, so that million-cell screens run on a laptop or a cluster node. The API mirrors Scanpy. It starts from an `.h5ad` file; it does not read raw sequencing outputs, and it is not a general single-cell toolkit.
|
|
149
149
|
|
|
150
150
|
**Documentation:** [crispyx.readthedocs.io](https://crispyx.readthedocs.io/en/latest/) — [installation](https://crispyx.readthedocs.io/en/latest/installation.html), [quick start](https://crispyx.readthedocs.io/en/latest/quickstart.html), [tutorials](https://crispyx.readthedocs.io/en/latest/notebooks/crispyx_tutorial.html) and [API reference](https://crispyx.readthedocs.io/en/latest/api.html).
|
|
151
151
|
|
|
@@ -202,18 +202,18 @@ For the full workflow (normalisation, PCA, pseudo-bulk, NB-GLM, LFC shrinkage, p
|
|
|
202
202
|
- **Streaming QC & preprocessing** – Filter cells, perturbations, and genes; normalise and log-transform; select highly variable genes (`seurat_v3`/`mean_dispersion`, control-cells-only by default); CSC-aware streaming with `format_mismatch_policy`; all without loading the full matrix into memory
|
|
203
203
|
- **Subsampling & downsampling** – Stratified or cluster-sampled cell subsampling (`cx.pp.subsample`, exact count or proportion per stratum, drop or keep small groups) and dependency-free per-cell count thinning (`cx.pp.downsample_counts`, the streaming equivalent of `scanpy.pp.downsample_counts`) for aligning dataset scale and sequencing depth before comparing screens
|
|
204
204
|
- **Pseudo-bulk aggregation** – Absolute profiles over multiple grouping columns (for example, perturbation × batch), strict count sums or mean log1p expression, optional deterministic bootstrap sampling, and explicit within-batch effect calculation
|
|
205
|
-
- **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`),
|
|
205
|
+
- **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`), negative binomial GLM with covariates and apeGLM LFC shrinkage, and a fixed-offset NB test for many small perturbations against a large, stratified control set (`cx.tl.nb_offset_test`); multi-core support and adaptive memory management; per-condition low-expression filtering to exclude genes that are near-zero in both groups
|
|
206
206
|
- **Covariate-adjusted effects** – `cx.tl.residual_effects` (Feng et al., 2026) fits each gene to covariates such as donor, lane or batch on the control cells only, and takes a perturbation's effect as the mean residual of its cells. Unlike the NB GLM, which fits the covariates on counts jointly with each perturbation, the control fit is shared by every perturbation, so a perturbation's own cells cannot shift its covariate adjustment, and all perturbations come from one streaming pass. On a 1.16-million-cell screen it reproduces the published effects and significance calls in under a minute.
|
|
207
207
|
- **Storage-format aware** – Gene-streaming functions (`wilcoxon_test`, `batch_process`) can convert a CSR source once to a temporary, memory-bounded CSC copy beside the output instead of re-reading the whole matrix once per gene chunk; `format_mismatch_policy="auto"` (the default) measures the source's read throughput and does so only when the re-reads would really cost more than the conversion, so a small local file is streamed as-is and a large one on a network filesystem is converted; `convert_to_csc`/`convert_to_csr` honour `memory_limit_gb` and preserve the value dtype
|
|
208
208
|
- **Dimension reduction** – Memory-efficient PCA and KNN graph construction on backed data
|
|
209
209
|
- **Scanpy-compatible API & plotting** – Familiar `cx.pp`, `cx.pb`, `cx.tl`, and `cx.pl` namespaces; Scanpy-style rank genes plots, volcano, MA, PCA, UMAP, QC summaries, and overlap heatmaps
|
|
210
|
-
- **Data preparation utilities** – Edit backed metadata without loading X;
|
|
210
|
+
- **Data preparation utilities** – Edit backed metadata without loading X; normalise perturbation labels and take target genes out of guide names (`target_pattern`); assign guides from guide-capture counts (`cx.pp.assign_guides`); recover integer counts from data published as log1p-normalised values (`cx.pp.recover_counts`); sum the rows of a gene split over several (`cx.pp.collapse_genes`); concatenate multi-file screens; check a file for the mistakes that mislead an analysis (`cx.pp.check_data`); auto-detect metadata columns
|
|
211
211
|
- **HPC-ready** – Resume/checkpoint for long-running jobs; configurable `memory_limit_gb`
|
|
212
212
|
- **Disk-aware** – Estimates and warns about scratch-disk usage before large writes or CSC/CSR conversions, and `cx.estimate_disk_usage(...)` answers "how much disk will this need?" up front; the memory savings above assume the machine has enough free disk for streaming intermediates and output files
|
|
213
213
|
|
|
214
214
|
## Performance
|
|
215
215
|
|
|
216
|
-
crispyx
|
|
216
|
+
Across 12 public CRISPR screens, crispyx completes every dataset where Scanpy, Pertpy/PyDESeq2 and edgeR run out of memory or time, with t-test and Wilcoxon results matching Scanpy to Pearson *r* > 0.999 and NB-GLM statistics agreeing with PyDESeq2 given the same size factors (z correlation 0.997–0.9999):
|
|
217
217
|
|
|
218
218
|
<p align="center">
|
|
219
219
|
<img src="docs/_static/fig2.png" width="800" alt="Benchmark results across 12 CRISPR screens: (a) dataset sizes, (b) completion status by method, (c) concordance with Scanpy, (d) runtime scaling, (e) peak memory scaling">
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
Genome-wide CRISPR screens routinely produce datasets with hundreds of thousands of cells and tens of thousands of genes. Standard single-cell analysis toolkits (Scanpy, Pertpy) load the entire count matrix into memory, requiring large RAM allocations and often making routine workflows impractical on laptops or shared compute nodes.
|
|
13
13
|
|
|
14
|
-
**crispyx**
|
|
14
|
+
**crispyx** runs quality control, normalisation, pseudo-bulk aggregation, differential expression (t-test, Wilcoxon, NB-GLM with covariates, covariate-adjusted residual effects), control checks and PCA/KNN on screens stored as `.h5ad` files, streaming from disk under a memory budget, so that million-cell screens run on a laptop or a cluster node. The API mirrors Scanpy. It starts from an `.h5ad` file; it does not read raw sequencing outputs, and it is not a general single-cell toolkit.
|
|
15
15
|
|
|
16
16
|
**Documentation:** [crispyx.readthedocs.io](https://crispyx.readthedocs.io/en/latest/) — [installation](https://crispyx.readthedocs.io/en/latest/installation.html), [quick start](https://crispyx.readthedocs.io/en/latest/quickstart.html), [tutorials](https://crispyx.readthedocs.io/en/latest/notebooks/crispyx_tutorial.html) and [API reference](https://crispyx.readthedocs.io/en/latest/api.html).
|
|
17
17
|
|
|
@@ -68,18 +68,18 @@ For the full workflow (normalisation, PCA, pseudo-bulk, NB-GLM, LFC shrinkage, p
|
|
|
68
68
|
- **Streaming QC & preprocessing** – Filter cells, perturbations, and genes; normalise and log-transform; select highly variable genes (`seurat_v3`/`mean_dispersion`, control-cells-only by default); CSC-aware streaming with `format_mismatch_policy`; all without loading the full matrix into memory
|
|
69
69
|
- **Subsampling & downsampling** – Stratified or cluster-sampled cell subsampling (`cx.pp.subsample`, exact count or proportion per stratum, drop or keep small groups) and dependency-free per-cell count thinning (`cx.pp.downsample_counts`, the streaming equivalent of `scanpy.pp.downsample_counts`) for aligning dataset scale and sequencing depth before comparing screens
|
|
70
70
|
- **Pseudo-bulk aggregation** – Absolute profiles over multiple grouping columns (for example, perturbation × batch), strict count sums or mean log1p expression, optional deterministic bootstrap sampling, and explicit within-batch effect calculation
|
|
71
|
-
- **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`),
|
|
71
|
+
- **Differential expression** – t-test, Wilcoxon rank-sum (including batch-stratified / van Elteren test via `batch_column`), negative binomial GLM with covariates and apeGLM LFC shrinkage, and a fixed-offset NB test for many small perturbations against a large, stratified control set (`cx.tl.nb_offset_test`); multi-core support and adaptive memory management; per-condition low-expression filtering to exclude genes that are near-zero in both groups
|
|
72
72
|
- **Covariate-adjusted effects** – `cx.tl.residual_effects` (Feng et al., 2026) fits each gene to covariates such as donor, lane or batch on the control cells only, and takes a perturbation's effect as the mean residual of its cells. Unlike the NB GLM, which fits the covariates on counts jointly with each perturbation, the control fit is shared by every perturbation, so a perturbation's own cells cannot shift its covariate adjustment, and all perturbations come from one streaming pass. On a 1.16-million-cell screen it reproduces the published effects and significance calls in under a minute.
|
|
73
73
|
- **Storage-format aware** – Gene-streaming functions (`wilcoxon_test`, `batch_process`) can convert a CSR source once to a temporary, memory-bounded CSC copy beside the output instead of re-reading the whole matrix once per gene chunk; `format_mismatch_policy="auto"` (the default) measures the source's read throughput and does so only when the re-reads would really cost more than the conversion, so a small local file is streamed as-is and a large one on a network filesystem is converted; `convert_to_csc`/`convert_to_csr` honour `memory_limit_gb` and preserve the value dtype
|
|
74
74
|
- **Dimension reduction** – Memory-efficient PCA and KNN graph construction on backed data
|
|
75
75
|
- **Scanpy-compatible API & plotting** – Familiar `cx.pp`, `cx.pb`, `cx.tl`, and `cx.pl` namespaces; Scanpy-style rank genes plots, volcano, MA, PCA, UMAP, QC summaries, and overlap heatmaps
|
|
76
|
-
- **Data preparation utilities** – Edit backed metadata without loading X;
|
|
76
|
+
- **Data preparation utilities** – Edit backed metadata without loading X; normalise perturbation labels and take target genes out of guide names (`target_pattern`); assign guides from guide-capture counts (`cx.pp.assign_guides`); recover integer counts from data published as log1p-normalised values (`cx.pp.recover_counts`); sum the rows of a gene split over several (`cx.pp.collapse_genes`); concatenate multi-file screens; check a file for the mistakes that mislead an analysis (`cx.pp.check_data`); auto-detect metadata columns
|
|
77
77
|
- **HPC-ready** – Resume/checkpoint for long-running jobs; configurable `memory_limit_gb`
|
|
78
78
|
- **Disk-aware** – Estimates and warns about scratch-disk usage before large writes or CSC/CSR conversions, and `cx.estimate_disk_usage(...)` answers "how much disk will this need?" up front; the memory savings above assume the machine has enough free disk for streaming intermediates and output files
|
|
79
79
|
|
|
80
80
|
## Performance
|
|
81
81
|
|
|
82
|
-
crispyx
|
|
82
|
+
Across 12 public CRISPR screens, crispyx completes every dataset where Scanpy, Pertpy/PyDESeq2 and edgeR run out of memory or time, with t-test and Wilcoxon results matching Scanpy to Pearson *r* > 0.999 and NB-GLM statistics agreeing with PyDESeq2 given the same size factors (z correlation 0.997–0.9999):
|
|
83
83
|
|
|
84
84
|
<p align="center">
|
|
85
85
|
<img src="docs/_static/fig2.png" width="800" alt="Benchmark results across 12 CRISPR screens: (a) dataset sizes, (b) completion status by method, (c) concordance with Scanpy, (d) runtime scaling, (e) peak memory scaling">
|
|
@@ -41,6 +41,8 @@ from .data import (
|
|
|
41
41
|
normalise_perturbation_labels,
|
|
42
42
|
read_h5ad_ondisk,
|
|
43
43
|
read_backed,
|
|
44
|
+
recover_counts,
|
|
45
|
+
collapse_genes,
|
|
44
46
|
resolve_data_path,
|
|
45
47
|
standardise_gene_names,
|
|
46
48
|
write_filtered_subset,
|
|
@@ -50,7 +52,10 @@ from .data import (
|
|
|
50
52
|
from .de import (
|
|
51
53
|
RankGenesGroupsResult,
|
|
52
54
|
nb_glm_test,
|
|
55
|
+
nb_offset_test,
|
|
53
56
|
shrink_lfc,
|
|
57
|
+
shrink_lfc_arrays,
|
|
58
|
+
ShrunkLFC,
|
|
54
59
|
t_test,
|
|
55
60
|
wilcoxon_test,
|
|
56
61
|
)
|
|
@@ -84,12 +89,16 @@ from .pseudobulk import (
|
|
|
84
89
|
compute_pseudobulk_effects,
|
|
85
90
|
)
|
|
86
91
|
from .qc import (
|
|
92
|
+
Issue,
|
|
93
|
+
assign_guides,
|
|
94
|
+
check_perturbation_data,
|
|
87
95
|
compare_controls,
|
|
88
96
|
control_guide_correlation,
|
|
89
97
|
filter_cells_by_gene_count,
|
|
90
98
|
filter_genes_by_cell_count,
|
|
91
99
|
filter_perturbations_by_cell_count,
|
|
92
100
|
quality_control_summary,
|
|
101
|
+
raise_on_errors,
|
|
93
102
|
)
|
|
94
103
|
from .residual import residual_effects
|
|
95
104
|
from .sample import subsample
|
|
@@ -128,6 +137,10 @@ __all__ = [
|
|
|
128
137
|
"quality_control_summary",
|
|
129
138
|
"compare_controls",
|
|
130
139
|
"control_guide_correlation",
|
|
140
|
+
"assign_guides",
|
|
141
|
+
"check_perturbation_data",
|
|
142
|
+
"Issue",
|
|
143
|
+
"raise_on_errors",
|
|
131
144
|
"plot_control_guide_correlation",
|
|
132
145
|
"highly_variable_genes",
|
|
133
146
|
# Subsampling
|
|
@@ -147,7 +160,10 @@ __all__ = [
|
|
|
147
160
|
"t_test",
|
|
148
161
|
"wilcoxon_test",
|
|
149
162
|
"nb_glm_test",
|
|
163
|
+
"nb_offset_test",
|
|
150
164
|
"shrink_lfc",
|
|
165
|
+
"shrink_lfc_arrays",
|
|
166
|
+
"ShrunkLFC",
|
|
151
167
|
# Data utilities
|
|
152
168
|
"AnnData",
|
|
153
169
|
"ensure_gene_symbol_column",
|
|
@@ -156,6 +172,8 @@ __all__ = [
|
|
|
156
172
|
"resolve_data_path",
|
|
157
173
|
"normalize_total_log1p",
|
|
158
174
|
"downsample_counts",
|
|
175
|
+
"recover_counts",
|
|
176
|
+
"collapse_genes",
|
|
159
177
|
"convert_to_csc",
|
|
160
178
|
"convert_to_csr",
|
|
161
179
|
"compress_h5ad",
|
|
@@ -404,7 +404,18 @@ class OneHotGroups:
|
|
|
404
404
|
Returns shape ``(n_groups, n_genes)``. This is the diagonal of
|
|
405
405
|
``G' W G`` when ``values`` are the IRLS weights.
|
|
406
406
|
"""
|
|
407
|
-
return
|
|
407
|
+
return self.sums_sorted(self.sort(values))
|
|
408
|
+
|
|
409
|
+
def sums_sorted(self, values_sorted: np.ndarray) -> np.ndarray:
|
|
410
|
+
""":meth:`sums` of values already in :meth:`sort` order: a NumPy sum
|
|
411
|
+
per group's contiguous slice, which releases the GIL (a sparse
|
|
412
|
+
product does not, and serialised the IRLS's threads)."""
|
|
413
|
+
out = np.zeros((self.n_groups, values_sorted.shape[1]), dtype=np.float64)
|
|
414
|
+
for k in range(self.n_groups):
|
|
415
|
+
start, stop = self._starts[k], self._ends[k]
|
|
416
|
+
if stop > start:
|
|
417
|
+
values_sorted[start:stop].sum(axis=0, out=out[k])
|
|
418
|
+
return out
|
|
408
419
|
|
|
409
420
|
def cross(self, design_sorted: np.ndarray, weights_sorted: np.ndarray) -> np.ndarray:
|
|
410
421
|
"""``X' W G`` per gene, shape ``(n_groups, n_features, n_genes)``.
|
|
@@ -427,11 +438,13 @@ class OneHotGroups:
|
|
|
427
438
|
return out
|
|
428
439
|
|
|
429
440
|
def expand(self, per_group: np.ndarray) -> np.ndarray:
|
|
430
|
-
"""Scatter ``(n_genes, n_groups)`` values back to ``(n_samples, n_genes)``.
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
441
|
+
"""Scatter ``(n_genes, n_groups)`` values back to ``(n_samples, n_genes)``.
|
|
442
|
+
|
|
443
|
+
One ``take`` from the values with a zero row appended, which a cell
|
|
444
|
+
in no group (label ``-1``) picks."""
|
|
445
|
+
padded = np.zeros((self.n_groups + 1, per_group.shape[0]), dtype=np.float64)
|
|
446
|
+
padded[:-1] = per_group.T
|
|
447
|
+
return padded.take(self.labels, axis=0)
|
|
435
448
|
|
|
436
449
|
|
|
437
450
|
def _arrowhead_blocks(
|
|
@@ -115,6 +115,11 @@ def _neg_half_log_det_numba(xtwx: np.ndarray) -> float:
|
|
|
115
115
|
return -log_det # -0.5 * (2 * sum(log diag(L)))
|
|
116
116
|
|
|
117
117
|
|
|
118
|
+
#: Counts below this are tallied per gene for the NB likelihood's lgamma
|
|
119
|
+
#: terms (_nb_posterior_with_cache_numba); _lgamma_ratio_term sums them exactly.
|
|
120
|
+
_SMALL_COUNTS = 32
|
|
121
|
+
|
|
122
|
+
|
|
118
123
|
@nb.njit(cache=True)
|
|
119
124
|
def _brent_minimize_numba(
|
|
120
125
|
Y_g: np.ndarray,
|
|
@@ -122,6 +127,7 @@ def _brent_minimize_numba(
|
|
|
122
127
|
log_trend_g: float,
|
|
123
128
|
prior_var: float,
|
|
124
129
|
gammaln_y_plus_1_sum: float,
|
|
130
|
+
small_counts: np.ndarray,
|
|
125
131
|
x_indptr: np.ndarray,
|
|
126
132
|
x_indices: np.ndarray,
|
|
127
133
|
x_data: np.ndarray,
|
|
@@ -142,7 +148,8 @@ def _brent_minimize_numba(
|
|
|
142
148
|
# Initial setup
|
|
143
149
|
x = w = v = a + golden * (b - a)
|
|
144
150
|
fx = fw = fv = -_nb_posterior_with_cache_numba(
|
|
145
|
-
Y_g, mu_g, x, log_trend_g, prior_var, gammaln_y_plus_1_sum, x_indptr, x_indices, x_data,
|
|
151
|
+
Y_g, mu_g, x, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
|
|
152
|
+
xtwx,
|
|
146
153
|
)
|
|
147
154
|
|
|
148
155
|
d = 0.0 # Distance to next point
|
|
@@ -198,7 +205,8 @@ def _brent_minimize_numba(
|
|
|
198
205
|
u = x + (tol1 if d > 0 else -tol1)
|
|
199
206
|
|
|
200
207
|
fu = -_nb_posterior_with_cache_numba(
|
|
201
|
-
Y_g, mu_g, u, log_trend_g, prior_var, gammaln_y_plus_1_sum, x_indptr, x_indices, x_data,
|
|
208
|
+
Y_g, mu_g, u, log_trend_g, prior_var, gammaln_y_plus_1_sum, small_counts, x_indptr, x_indices, x_data,
|
|
209
|
+
xtwx,
|
|
202
210
|
)
|
|
203
211
|
|
|
204
212
|
# Update a, b, v, w, x
|
|
@@ -240,6 +248,7 @@ def _nb_posterior_with_cache_numba(
|
|
|
240
248
|
log_trend_g: float,
|
|
241
249
|
prior_var: float,
|
|
242
250
|
gammaln_y_plus_1_sum: float,
|
|
251
|
+
small_counts: np.ndarray,
|
|
243
252
|
x_indptr: np.ndarray,
|
|
244
253
|
x_indices: np.ndarray,
|
|
245
254
|
x_data: np.ndarray,
|
|
@@ -251,7 +260,10 @@ def _nb_posterior_with_cache_numba(
|
|
|
251
260
|
alpha mu)``, is accumulated into the work buffer ``xtwx`` (``(0, 0)``
|
|
252
261
|
for no adjustment) in the same pass over the cells as the likelihood;
|
|
253
262
|
the design ``X`` is given in CSR form, so a one-hot batch design costs a
|
|
254
|
-
few products per cell rather than ``n_features**2``.
|
|
263
|
+
few products per cell rather than ``n_features**2``. ``small_counts[k]``
|
|
264
|
+
is the number of cells with count ``k`` (below its length): their
|
|
265
|
+
``lgamma(y + r) - lgamma(r)`` terms depend on the count only, so each
|
|
266
|
+
value's is formed once, by recurrence, rather than once per cell.
|
|
255
267
|
"""
|
|
256
268
|
n_cells = Y_g.shape[0]
|
|
257
269
|
alpha = math.exp(log_alpha)
|
|
@@ -260,14 +272,20 @@ def _nb_posterior_with_cache_numba(
|
|
|
260
272
|
cox_reid = xtwx.shape[0] > 0
|
|
261
273
|
xtwx[:, :] = 0.0
|
|
262
274
|
|
|
275
|
+
# lgamma(y + r) - lgamma(r) - y log r, without libm's lgamma (see
|
|
276
|
+
# _lgamma_ratio_term): sum(log1p(i / r), i < y) for the small counts.
|
|
263
277
|
ll = 0.0
|
|
278
|
+
term = 0.0
|
|
279
|
+
for k in range(1, small_counts.shape[0]):
|
|
280
|
+
term += math.log1p((k - 1) / r)
|
|
281
|
+
ll += small_counts[k] * term
|
|
282
|
+
n_small = small_counts.shape[0]
|
|
264
283
|
for i in range(n_cells):
|
|
265
284
|
y = Y_g[i]
|
|
266
285
|
mu_i = mu_g[i]
|
|
267
286
|
r_plus_mu = r + mu_i + 1e-12
|
|
268
|
-
# lgamma(y + r) - lgamma(r), without libm's lgamma (see _lgamma_ratio_term)
|
|
269
287
|
ll += (
|
|
270
|
-
_lgamma_ratio_term(y, r) + y * log_r
|
|
288
|
+
(_lgamma_ratio_term(y, r) if y >= n_small else 0.0) + y * log_r
|
|
271
289
|
+ r * (log_r - math.log(r_plus_mu))
|
|
272
290
|
+ y * math.log(mu_i / r_plus_mu + 1e-12)
|
|
273
291
|
)
|
|
@@ -340,13 +358,17 @@ def _nb_map_grid_search_with_refinement_numba(
|
|
|
340
358
|
Y_g = Y[:, g]
|
|
341
359
|
mu_g = mu[:, g]
|
|
342
360
|
xtwx = np.empty((n_features, n_features))
|
|
361
|
+
small_counts = np.zeros(_SMALL_COUNTS)
|
|
362
|
+
for i in range(Y_g.shape[0]):
|
|
363
|
+
if Y_g[i] < _SMALL_COUNTS:
|
|
364
|
+
small_counts[int(Y_g[i])] += 1.0
|
|
343
365
|
step = (hi[g] - lo[g]) / (n_grid - 1)
|
|
344
366
|
# sum(lgamma(y + 1)) is constant in alpha, so it is left out (0.0).
|
|
345
367
|
best_posterior = -np.inf
|
|
346
368
|
best_k = 0
|
|
347
369
|
for k in range(n_grid):
|
|
348
370
|
posterior = _nb_posterior_with_cache_numba(
|
|
349
|
-
Y_g, mu_g, lo[g] + k * step, log_trend_g, prior_var, 0.0,
|
|
371
|
+
Y_g, mu_g, lo[g] + k * step, log_trend_g, prior_var, 0.0, small_counts,
|
|
350
372
|
x_indptr, x_indices, x_data, xtwx,
|
|
351
373
|
)
|
|
352
374
|
if posterior > best_posterior:
|
|
@@ -355,11 +377,12 @@ def _nb_map_grid_search_with_refinement_numba(
|
|
|
355
377
|
best = lo[g] + best_k * step
|
|
356
378
|
if 0 < best_k < n_grid - 1 and 2 * step > tol:
|
|
357
379
|
refined = _brent_minimize_numba(
|
|
358
|
-
Y_g, mu_g, log_trend_g, prior_var, 0.0, x_indptr, x_indices, x_data, xtwx,
|
|
380
|
+
Y_g, mu_g, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data, xtwx,
|
|
359
381
|
best - step, best + step, tol=tol, max_iter=max_refine_iter,
|
|
360
382
|
)
|
|
361
383
|
refined_posterior = _nb_posterior_with_cache_numba(
|
|
362
|
-
Y_g, mu_g, refined, log_trend_g, prior_var, 0.0, x_indptr, x_indices, x_data,
|
|
384
|
+
Y_g, mu_g, refined, log_trend_g, prior_var, 0.0, small_counts, x_indptr, x_indices, x_data,
|
|
385
|
+
xtwx,
|
|
363
386
|
)
|
|
364
387
|
if refined_posterior >= best_posterior:
|
|
365
388
|
best = refined
|
|
@@ -1682,9 +1705,13 @@ def _two_group_lfc(genes, dispersion, a0, b0, t_ptr, t_y, t_s, s_t, min_mu, cent
|
|
|
1682
1705
|
moved = max(abs(na - a), abs(nb_ - b))
|
|
1683
1706
|
a = na
|
|
1684
1707
|
b = nb_
|
|
1685
|
-
|
|
1708
|
+
# Converged on the full Newton step: the step actually taken also
|
|
1709
|
+
# shrinks when the line search gives up or a bound clips it.
|
|
1710
|
+
if max(abs(da), abs(db)) < tol:
|
|
1686
1711
|
converged[i] = True
|
|
1687
1712
|
break
|
|
1713
|
+
if moved < tol:
|
|
1714
|
+
break
|
|
1688
1715
|
k_t = math.exp(b)
|
|
1689
1716
|
w_t = 0.0
|
|
1690
1717
|
for j in range(s_t.shape[0]):
|
|
@@ -1733,3 +1760,294 @@ def _accumulate_control_summary(data, indices, indptr, bin_of_row, s_rows, centr
|
|
|
1733
1760
|
big_values[n_big] = y
|
|
1734
1761
|
n_big += 1
|
|
1735
1762
|
return n_big
|
|
1763
|
+
|
|
1764
|
+
|
|
1765
|
+
# =============================================================================
|
|
1766
|
+
# Structured NB IRLS at a fixed dispersion, one gene per thread
|
|
1767
|
+
# =============================================================================
|
|
1768
|
+
#
|
|
1769
|
+
# StructuredGLMBatchFitter._irls, gene by gene: the same start, Newton step
|
|
1770
|
+
# (arrowhead Hessian solved through the Schur complement of the group block),
|
|
1771
|
+
# step halving, convergence rules and standard errors, in one pass over a
|
|
1772
|
+
# gene's cells per quantity and no temporary (cells x genes) arrays. The NumPy
|
|
1773
|
+
# fitter's batches allocate such arrays at every step, and on many threads
|
|
1774
|
+
# they wait on the allocator and the GIL; this runs on all of them.
|
|
1775
|
+
|
|
1776
|
+
|
|
1777
|
+
@nb.njit(cache=True)
|
|
1778
|
+
def _structured_predictor(X, labels, offset, bf, bg, eta_lo, eta_hi, eta, mu):
|
|
1779
|
+
"""eta = clip(offset + X bf + bg[label], eta_lo, eta_hi) and mu = exp(eta)."""
|
|
1780
|
+
n, dx = X.shape
|
|
1781
|
+
for i in range(n):
|
|
1782
|
+
v = offset[i]
|
|
1783
|
+
for j in range(dx):
|
|
1784
|
+
v += X[i, j] * bf[j]
|
|
1785
|
+
if labels[i] >= 0:
|
|
1786
|
+
v += bg[labels[i]]
|
|
1787
|
+
v = min(max(v, eta_lo), eta_hi)
|
|
1788
|
+
eta[i] = v
|
|
1789
|
+
mu[i] = math.exp(v)
|
|
1790
|
+
|
|
1791
|
+
|
|
1792
|
+
@nb.njit(cache=True)
|
|
1793
|
+
def _structured_deviance(y, eta, mu, size, ylogy):
|
|
1794
|
+
"""2 (sum y log y - sum y eta + sum (y + r) log1p((mu - y) / (y + r))), as Deviance.total."""
|
|
1795
|
+
cross = 0.0
|
|
1796
|
+
tail = 0.0
|
|
1797
|
+
for i in range(y.shape[0]):
|
|
1798
|
+
cross += y[i] * eta[i]
|
|
1799
|
+
yr = y[i] + size
|
|
1800
|
+
tail += yr * math.log1p((mu[i] - y[i]) / yr)
|
|
1801
|
+
return 2.0 * (ylogy - cross + tail)
|
|
1802
|
+
|
|
1803
|
+
|
|
1804
|
+
@nb.njit(cache=True)
|
|
1805
|
+
def _structured_blocks(X, labels, w, wz, with_rhs, gram, cross, diag, rx, rg, ridge, ridge_group):
|
|
1806
|
+
"""X'WX (+ ridge, one value per column), X'WG and diag(G'WG)
|
|
1807
|
+
(+ ridge_group) for the weights w, and with_rhs, X'Wz and G'Wz for
|
|
1808
|
+
wz = w z."""
|
|
1809
|
+
n, dx = X.shape
|
|
1810
|
+
gram[:, :] = 0.0
|
|
1811
|
+
cross[:, :] = 0.0
|
|
1812
|
+
diag[:] = 0.0
|
|
1813
|
+
rx[:] = 0.0
|
|
1814
|
+
rg[:] = 0.0
|
|
1815
|
+
for i in range(n):
|
|
1816
|
+
wi = w[i]
|
|
1817
|
+
k = labels[i]
|
|
1818
|
+
for j in range(dx):
|
|
1819
|
+
xw = X[i, j] * wi
|
|
1820
|
+
for l in range(j + 1):
|
|
1821
|
+
gram[j, l] += xw * X[i, l]
|
|
1822
|
+
if k >= 0:
|
|
1823
|
+
cross[j, k] += xw
|
|
1824
|
+
if with_rhs:
|
|
1825
|
+
rx[j] += X[i, j] * wz[i]
|
|
1826
|
+
if k >= 0:
|
|
1827
|
+
diag[k] += wi
|
|
1828
|
+
if with_rhs:
|
|
1829
|
+
rg[k] += wz[i]
|
|
1830
|
+
for j in range(dx):
|
|
1831
|
+
for l in range(j):
|
|
1832
|
+
gram[l, j] = gram[j, l]
|
|
1833
|
+
gram[j, j] += ridge[j]
|
|
1834
|
+
for k in range(diag.shape[0]):
|
|
1835
|
+
diag[k] += ridge_group
|
|
1836
|
+
|
|
1837
|
+
|
|
1838
|
+
@nb.njit(cache=True)
|
|
1839
|
+
def _solve_small(A, B):
|
|
1840
|
+
"""A^-1 B for a small square A, by Gaussian elimination with partial
|
|
1841
|
+
pivoting (as LAPACK's gesv); NaN when A is singular."""
|
|
1842
|
+
m = A.shape[0]
|
|
1843
|
+
a = A.copy()
|
|
1844
|
+
x = B.copy()
|
|
1845
|
+
for c in range(m):
|
|
1846
|
+
p = c
|
|
1847
|
+
for r in range(c + 1, m):
|
|
1848
|
+
if abs(a[r, c]) > abs(a[p, c]):
|
|
1849
|
+
p = r
|
|
1850
|
+
if a[p, c] == 0.0:
|
|
1851
|
+
x[:, :] = np.nan
|
|
1852
|
+
return x
|
|
1853
|
+
if p != c:
|
|
1854
|
+
for j in range(m):
|
|
1855
|
+
t = a[c, j]
|
|
1856
|
+
a[c, j] = a[p, j]
|
|
1857
|
+
a[p, j] = t
|
|
1858
|
+
for j in range(x.shape[1]):
|
|
1859
|
+
t = x[c, j]
|
|
1860
|
+
x[c, j] = x[p, j]
|
|
1861
|
+
x[p, j] = t
|
|
1862
|
+
for r in range(c + 1, m):
|
|
1863
|
+
f = a[r, c] / a[c, c]
|
|
1864
|
+
if f != 0.0:
|
|
1865
|
+
for j in range(c, m):
|
|
1866
|
+
a[r, j] -= f * a[c, j]
|
|
1867
|
+
for j in range(x.shape[1]):
|
|
1868
|
+
x[r, j] -= f * x[c, j]
|
|
1869
|
+
for c in range(m - 1, -1, -1):
|
|
1870
|
+
for j in range(x.shape[1]):
|
|
1871
|
+
v = x[c, j]
|
|
1872
|
+
for k in range(c + 1, m):
|
|
1873
|
+
v -= a[c, k] * x[k, j]
|
|
1874
|
+
x[c, j] = v / a[c, c]
|
|
1875
|
+
return x
|
|
1876
|
+
|
|
1877
|
+
|
|
1878
|
+
@nb.njit(cache=True)
|
|
1879
|
+
def _schur(gram, cross, diag):
|
|
1880
|
+
"""S = C - B D^-1 B'."""
|
|
1881
|
+
dx, a = cross.shape
|
|
1882
|
+
S = gram.copy()
|
|
1883
|
+
for j in range(dx):
|
|
1884
|
+
for l in range(dx):
|
|
1885
|
+
v = 0.0
|
|
1886
|
+
for k in range(a):
|
|
1887
|
+
v += cross[j, k] * cross[l, k] / diag[k]
|
|
1888
|
+
S[j, l] -= v
|
|
1889
|
+
return S
|
|
1890
|
+
|
|
1891
|
+
|
|
1892
|
+
@nb.njit(parallel=True, cache=True)
|
|
1893
|
+
def _structured_nb_irls(X, labels, offset, Y, alpha, n_groups, const_col, const_val, mean_exp_offset,
|
|
1894
|
+
eta_lo, eta_hi, wald_lo, floored, max_iter, tol, ridge, ridge_group, clip_group,
|
|
1895
|
+
max_halvings, eps):
|
|
1896
|
+
"""StructuredGLMBatchFitter._irls and its standard errors at the fixed NB
|
|
1897
|
+
dispersions alpha, for each column of Y (Fortran order), one gene per
|
|
1898
|
+
thread; ridge is per covariate column, and n_groups may be 0 (a dense
|
|
1899
|
+
design). Returns coef (genes, dx + a: covariates, then groups), se,
|
|
1900
|
+
deviance, converged and n_iter."""
|
|
1901
|
+
n, dx = X.shape
|
|
1902
|
+
n_genes = Y.shape[1]
|
|
1903
|
+
a = n_groups
|
|
1904
|
+
coef = np.zeros((n_genes, dx + a))
|
|
1905
|
+
se = np.full((n_genes, dx + a), np.inf)
|
|
1906
|
+
dev_out = np.full(n_genes, np.nan)
|
|
1907
|
+
converged = np.zeros(n_genes, dtype=np.bool_)
|
|
1908
|
+
n_iter = np.zeros(n_genes, dtype=np.int32)
|
|
1909
|
+
for g in nb.prange(n_genes):
|
|
1910
|
+
y = Y[:, g]
|
|
1911
|
+
ag = alpha[g]
|
|
1912
|
+
size = 1.0 / min(max(ag, eps), 1e10)
|
|
1913
|
+
ylogy = 0.0
|
|
1914
|
+
total = 0.0
|
|
1915
|
+
for i in range(n):
|
|
1916
|
+
total += y[i]
|
|
1917
|
+
if y[i] > 0.0:
|
|
1918
|
+
ylogy += y[i] * math.log(y[i])
|
|
1919
|
+
bf = np.zeros(dx)
|
|
1920
|
+
bg = np.zeros(a)
|
|
1921
|
+
if const_col >= 0:
|
|
1922
|
+
bf[const_col] = math.log(max(total / n, 1e-3) / mean_exp_offset) / const_val
|
|
1923
|
+
eta = np.empty(n)
|
|
1924
|
+
mu = np.empty(n)
|
|
1925
|
+
_structured_predictor(X, labels, offset, bf, bg, eta_lo, eta_hi, eta, mu)
|
|
1926
|
+
dev = _structured_deviance(y, eta, mu, size, ylogy)
|
|
1927
|
+
w = np.empty(n)
|
|
1928
|
+
wz = np.empty(n)
|
|
1929
|
+
new_eta = np.empty(n)
|
|
1930
|
+
new_mu = np.empty(n)
|
|
1931
|
+
half_eta = np.empty(n)
|
|
1932
|
+
half_mu = np.empty(n)
|
|
1933
|
+
gram = np.empty((dx, dx))
|
|
1934
|
+
cross = np.empty((dx, a))
|
|
1935
|
+
diag = np.empty(a)
|
|
1936
|
+
rx = np.empty(dx)
|
|
1937
|
+
rg = np.empty(a)
|
|
1938
|
+
conv = False
|
|
1939
|
+
it = 0
|
|
1940
|
+
for iteration in range(1, max_iter + 1):
|
|
1941
|
+
# The Newton step at (eta, mu): weights, working response, normal equations.
|
|
1942
|
+
for i in range(n):
|
|
1943
|
+
wi = mu[i] / (1.0 + ag * mu[i])
|
|
1944
|
+
w[i] = wi
|
|
1945
|
+
wz[i] = wi * (eta[i] - offset[i] + (y[i] - mu[i]) / max(mu[i], eps))
|
|
1946
|
+
_structured_blocks(X, labels, w, wz, True, gram, cross, diag, rx, rg, ridge, ridge_group)
|
|
1947
|
+
S = _schur(gram, cross, diag)
|
|
1948
|
+
rhs = np.empty((dx, 1))
|
|
1949
|
+
for j in range(dx):
|
|
1950
|
+
v = rx[j]
|
|
1951
|
+
for k in range(a):
|
|
1952
|
+
v -= cross[j, k] / diag[k] * rg[k]
|
|
1953
|
+
rhs[j, 0] = v
|
|
1954
|
+
full_f = _solve_small(S, rhs)[:, 0].copy()
|
|
1955
|
+
full_g = np.empty(a)
|
|
1956
|
+
for k in range(a):
|
|
1957
|
+
v = rg[k]
|
|
1958
|
+
for j in range(dx):
|
|
1959
|
+
v -= cross[j, k] * full_f[j]
|
|
1960
|
+
full_g[k] = min(max(v / diag[k], -clip_group), clip_group)
|
|
1961
|
+
_structured_predictor(X, labels, offset, full_f, full_g, eta_lo, eta_hi, new_eta, new_mu)
|
|
1962
|
+
new_dev = _structured_deviance(y, new_eta, new_mu, size, ylogy)
|
|
1963
|
+
new_f = full_f.copy()
|
|
1964
|
+
new_g = full_g.copy()
|
|
1965
|
+
|
|
1966
|
+
# A gene resting on the mean floor takes the full step and is
|
|
1967
|
+
# judged by the deviance ratio (as the NumPy fitter).
|
|
1968
|
+
on_floor = False
|
|
1969
|
+
if floored:
|
|
1970
|
+
for i in range(n):
|
|
1971
|
+
if eta[i] <= eta_lo:
|
|
1972
|
+
on_floor = True
|
|
1973
|
+
break
|
|
1974
|
+
ceiling = dev + 1e-9 * abs(dev)
|
|
1975
|
+
rose = new_dev > ceiling
|
|
1976
|
+
worse = rose and not on_floor
|
|
1977
|
+
scale = max(abs(dev), 1e-8)
|
|
1978
|
+
excess = (new_dev - dev) / scale
|
|
1979
|
+
step = 1.0
|
|
1980
|
+
for _h in range(max_halvings):
|
|
1981
|
+
if not worse:
|
|
1982
|
+
break
|
|
1983
|
+
step *= 0.5
|
|
1984
|
+
half_f = bf + step * (full_f - bf)
|
|
1985
|
+
half_g = bg + step * (full_g - bg)
|
|
1986
|
+
_structured_predictor(X, labels, offset, half_f, half_g, eta_lo, eta_hi, half_eta, half_mu)
|
|
1987
|
+
half_dev = _structured_deviance(y, half_eta, half_mu, size, ylogy)
|
|
1988
|
+
excess = min(excess, (half_dev - dev) / scale)
|
|
1989
|
+
if half_dev <= ceiling:
|
|
1990
|
+
new_f = half_f
|
|
1991
|
+
new_g = half_g
|
|
1992
|
+
new_eta[:] = half_eta
|
|
1993
|
+
new_mu[:] = half_mu
|
|
1994
|
+
new_dev = half_dev
|
|
1995
|
+
worse = False
|
|
1996
|
+
fixed_point = rose and on_floor
|
|
1997
|
+
if worse:
|
|
1998
|
+
new_f = bf.copy()
|
|
1999
|
+
new_g = bg.copy()
|
|
2000
|
+
new_eta[:] = eta
|
|
2001
|
+
new_mu[:] = mu
|
|
2002
|
+
new_dev = dev
|
|
2003
|
+
relative_change = abs(new_dev - dev) / scale
|
|
2004
|
+
deviance_ratio = abs(new_dev - dev) / (abs(new_dev) + 0.1)
|
|
2005
|
+
coef_change = 0.0
|
|
2006
|
+
for j in range(dx):
|
|
2007
|
+
coef_change = max(coef_change, abs(new_f[j] - bf[j]))
|
|
2008
|
+
for k in range(a):
|
|
2009
|
+
coef_change = max(coef_change, abs(new_g[k] - bg[k]))
|
|
2010
|
+
bf = new_f
|
|
2011
|
+
bg = new_g
|
|
2012
|
+
eta[:] = new_eta
|
|
2013
|
+
mu[:] = new_mu
|
|
2014
|
+
dev = new_dev
|
|
2015
|
+
it = iteration
|
|
2016
|
+
if fixed_point:
|
|
2017
|
+
conv = deviance_ratio < tol and coef_change < tol
|
|
2018
|
+
elif worse:
|
|
2019
|
+
conv = excess < tol
|
|
2020
|
+
else:
|
|
2021
|
+
conv = relative_change < tol and coef_change < tol
|
|
2022
|
+
if conv or worse:
|
|
2023
|
+
break
|
|
2024
|
+
|
|
2025
|
+
# Standard errors from the blocks at the fit, with the unfloored mean.
|
|
2026
|
+
if floored:
|
|
2027
|
+
_structured_predictor(X, labels, offset, bf, bg, wald_lo, eta_hi, half_eta, half_mu)
|
|
2028
|
+
for i in range(n):
|
|
2029
|
+
w[i] = half_mu[i] / (1.0 + ag * half_mu[i])
|
|
2030
|
+
else:
|
|
2031
|
+
for i in range(n):
|
|
2032
|
+
w[i] = mu[i] / (1.0 + ag * mu[i])
|
|
2033
|
+
_structured_blocks(X, labels, w, wz, False, gram, cross, diag, rx, rg, ridge, ridge_group)
|
|
2034
|
+
S = _schur(gram, cross, diag)
|
|
2035
|
+
rhs = np.zeros((dx, dx + a))
|
|
2036
|
+
for j in range(dx):
|
|
2037
|
+
rhs[j, j] = 1.0
|
|
2038
|
+
for k in range(a):
|
|
2039
|
+
rhs[j, dx + k] = cross[j, k]
|
|
2040
|
+
solved = _solve_small(S, rhs)
|
|
2041
|
+
for j in range(dx):
|
|
2042
|
+
coef[g, j] = bf[j]
|
|
2043
|
+
se[g, j] = math.sqrt(max(solved[j, j], 0.0))
|
|
2044
|
+
for k in range(a):
|
|
2045
|
+
quad = 0.0
|
|
2046
|
+
for j in range(dx):
|
|
2047
|
+
quad += cross[j, k] * solved[j, dx + k]
|
|
2048
|
+
coef[g, dx + k] = bg[k]
|
|
2049
|
+
se[g, dx + k] = math.sqrt(max(1.0 / diag[k] + quad / (diag[k] * diag[k]), 0.0))
|
|
2050
|
+
dev_out[g] = dev
|
|
2051
|
+
converged[g] = conv
|
|
2052
|
+
n_iter[g] = it
|
|
2053
|
+
return coef, se, dev_out, converged, n_iter
|