illico 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- illico/__init__.py +3 -0
- illico/asymptotic_wilcoxon.py +169 -0
- illico/ovo/__init__.py +84 -0
- illico/ovo/dense_ovo.py +129 -0
- illico/ovo/sparse_ovo.py +247 -0
- illico/ovr/__init__.py +87 -0
- illico/ovr/dense_ovr.py +78 -0
- illico/ovr/sparse_ovr.py +201 -0
- illico/utils/__init__.py +0 -0
- illico/utils/compile.py +100 -0
- illico/utils/groups.py +57 -0
- illico/utils/math.py +278 -0
- illico/utils/memory.py +37 -0
- illico/utils/ranking.py +274 -0
- illico/utils/sparse/__init__.py +0 -0
- illico/utils/sparse/csc.py +203 -0
- illico/utils/sparse/csr.py +278 -0
- illico/utils/type.py +35 -0
- illico-0.1.0.dist-info/METADATA +277 -0
- illico-0.1.0.dist-info/RECORD +22 -0
- illico-0.1.0.dist-info/WHEEL +4 -0
- illico-0.1.0.dist-info/licenses/LICENSE +13 -0
illico/__init__.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import gc
|
|
2
|
+
import math
|
|
3
|
+
|
|
4
|
+
import anndata as ad
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
from joblib import Parallel, delayed
|
|
8
|
+
from loguru import logger
|
|
9
|
+
from scipy import sparse
|
|
10
|
+
from tqdm.auto import tqdm
|
|
11
|
+
|
|
12
|
+
from illico.ovo import ovo_mwu_over_contiguous_col_chunk
|
|
13
|
+
from illico.ovr import ovr_mwu_over_col_contiguous_chunk
|
|
14
|
+
from illico.utils.compile import _precompile
|
|
15
|
+
from illico.utils.groups import encode_and_count_groups
|
|
16
|
+
from illico.utils.memory import _log_memory_usage
|
|
17
|
+
from illico.utils.ranking import check_indices_sorted_per_parcel
|
|
18
|
+
|
|
19
|
+
# from illico.utils.math import _warn_log1p
|
|
20
|
+
|
|
21
|
+
__all__ = ["asymptotic_wilcoxon"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def asymptotic_wilcoxon(
|
|
25
|
+
adata: ad.AnnData,
|
|
26
|
+
is_log1p: bool,
|
|
27
|
+
group_keys: str,
|
|
28
|
+
reference_group: str | None = None,
|
|
29
|
+
n_threads: int = 1,
|
|
30
|
+
batch_size: int = 256,
|
|
31
|
+
alternative: str = "two-sided",
|
|
32
|
+
use_continuity: bool = True,
|
|
33
|
+
layer: str | None = None,
|
|
34
|
+
precompile: bool = True,
|
|
35
|
+
):
|
|
36
|
+
"""Perform asymptotic Mann-Whitney tests for differential gene expression.
|
|
37
|
+
|
|
38
|
+
Mann-Whitney test is the same as Wilcoxon rank-sum test.
|
|
39
|
+
|
|
40
|
+
This function takes as input an AnnData object of shape (n-cells, n-genes) with a group
|
|
41
|
+
(e.g., perturbation) variable stored in .obs. It performs either one-versus-rest (OVR) or
|
|
42
|
+
one-versus-one (OVO) Wilcoxon-Mann-Whitney tests for each gene, depending on whether a
|
|
43
|
+
reference group is provided.
|
|
44
|
+
|
|
45
|
+
Parameters
|
|
46
|
+
----------
|
|
47
|
+
adata:
|
|
48
|
+
Annotated data matrix of shape (n-cells, n-genes).
|
|
49
|
+
is_log1p
|
|
50
|
+
Whether the data is log1p transformed.
|
|
51
|
+
group_keys
|
|
52
|
+
Key in `adata.obs` specifying the group variable.
|
|
53
|
+
reference_group
|
|
54
|
+
Name of the reference group for OVO tests. If `None`, OVR tests are performed.
|
|
55
|
+
n_threads
|
|
56
|
+
Number of threads to use for parallel computation.
|
|
57
|
+
batch_size
|
|
58
|
+
Number of genes to process in each batch.
|
|
59
|
+
alternative
|
|
60
|
+
Type of alternative hypothesis. One of 'two-sided', 'less', or 'greater'.
|
|
61
|
+
use_continuity
|
|
62
|
+
Whether to apply continuity correction.
|
|
63
|
+
layer
|
|
64
|
+
Layer in `adata.layers` to use for the data. If `None`, uses `adata.X`.
|
|
65
|
+
precompile
|
|
66
|
+
Whether to precompile necessary functions for performance.
|
|
67
|
+
|
|
68
|
+
Returns
|
|
69
|
+
-------
|
|
70
|
+
pd.DataFrame
|
|
71
|
+
A DataFrame with MultiIndex (group, gene) containing p-values, statistics, and fold changes.
|
|
72
|
+
|
|
73
|
+
Author: Rémy Dubois
|
|
74
|
+
"""
|
|
75
|
+
# TODO: add a sparsity warning inviting user to convert to sparse if possible
|
|
76
|
+
# TODO: rename group_keys
|
|
77
|
+
|
|
78
|
+
if layer is not None:
|
|
79
|
+
logger.info(f"Using layer '{layer}' for differential expression.")
|
|
80
|
+
X = adata.layers[layer]
|
|
81
|
+
else:
|
|
82
|
+
X = adata.X
|
|
83
|
+
|
|
84
|
+
# Check that the input CSR is sorted.
|
|
85
|
+
if isinstance(X, sparse.csr_matrix):
|
|
86
|
+
if not check_indices_sorted_per_parcel(X.indices, X.indptr):
|
|
87
|
+
raise ValueError(
|
|
88
|
+
"Input data matrix indices are not sorted. This is very unusual and may lead to incorrect results. "
|
|
89
|
+
"This can be the result of operations like `adata[:, np.random.choice(…)]` that do not preserve sorting."
|
|
90
|
+
"Please make sure that indices used to chunk the adata or the expression matrix have been sorted "
|
|
91
|
+
"prior to computing DE genes."
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
if precompile:
|
|
95
|
+
_precompile(X, reference_group)
|
|
96
|
+
|
|
97
|
+
# Process the groups information
|
|
98
|
+
raw_groups = adata.obs[group_keys].tolist()
|
|
99
|
+
unique_raw_groups, group_container = encode_and_count_groups(groups=raw_groups, ref_group=reference_group)
|
|
100
|
+
logger.info(
|
|
101
|
+
f"Found {group_container.counts.size} unique groups (min size: {group_container.counts.min()} cells; max size: {group_container.counts.max()} cells), with reference group: {reference_group}"
|
|
102
|
+
)
|
|
103
|
+
_, n_genes = X.shape
|
|
104
|
+
|
|
105
|
+
# Allocate the results dataframes
|
|
106
|
+
cols = pd.Series(adata.var_names, name="feature", dtype=str)
|
|
107
|
+
rows = pd.Series(unique_raw_groups, name="pert", dtype=str)
|
|
108
|
+
results = np.empty((len(rows), len(cols), 3), dtype=np.float64)
|
|
109
|
+
|
|
110
|
+
# Adapt batch size to leverage multithreading regarding the number of genes, if requested
|
|
111
|
+
batch_size = min(batch_size, math.ceil(n_genes / n_threads))
|
|
112
|
+
logger.trace(f"Using batch size of {batch_size} for {n_threads} threads and {n_genes} genes.")
|
|
113
|
+
|
|
114
|
+
# Compute estimated mem footprint
|
|
115
|
+
_log_memory_usage(adata, group_container, batch_size, n_threads)
|
|
116
|
+
|
|
117
|
+
# Generate batches of columns
|
|
118
|
+
bounds = np.append(np.arange(0, n_genes, batch_size), n_genes)
|
|
119
|
+
iterator = list(zip(bounds[:-1], bounds[1:]))
|
|
120
|
+
n_tests = n_genes * group_container.counts.size
|
|
121
|
+
logger.trace(f"Performing a total of {n_tests:,d} tests.")
|
|
122
|
+
with Parallel(n_threads, prefer="threads", return_as="generator_unordered") as pool:
|
|
123
|
+
with tqdm(total=n_tests, smoothing=0.0, unit="it", unit_scale=True, unit_divisor=1000) as pbar:
|
|
124
|
+
if reference_group is None: # ovr use case
|
|
125
|
+
pbar.set_description("Running one-versus-all MannWhitney-U tests")
|
|
126
|
+
op = delayed(lambda *args: (ovr_mwu_over_col_contiguous_chunk(*args), args))
|
|
127
|
+
else: # ovo use case
|
|
128
|
+
pbar.set_description("Running one-versus-ref MannWhitney-U tests")
|
|
129
|
+
op = delayed(lambda *args: (ovo_mwu_over_contiguous_col_chunk(*args), args))
|
|
130
|
+
for (pv, ustat, fc), args in pool(
|
|
131
|
+
op(X, lb, ub, group_container, is_log1p, use_continuity, alternative) for lb, ub in iterator
|
|
132
|
+
):
|
|
133
|
+
# progress.update(task, advance=group_container.counts.size * (args[2] - args[1])) # refresh after processing is done
|
|
134
|
+
pbar.update(group_container.counts.size * (args[2] - args[1]))
|
|
135
|
+
col_chunk = slice(*args[1:3])
|
|
136
|
+
results[:, col_chunk, 0] = pv
|
|
137
|
+
results[:, col_chunk, 1] = ustat
|
|
138
|
+
results[:, col_chunk, 2] = fc
|
|
139
|
+
|
|
140
|
+
# Cleanup memory
|
|
141
|
+
del pv, ustat, fc
|
|
142
|
+
gc.collect()
|
|
143
|
+
|
|
144
|
+
# Return a pd.DataFrame to index results
|
|
145
|
+
results = pd.DataFrame(
|
|
146
|
+
data=results.reshape(-1, 3),
|
|
147
|
+
index=pd.MultiIndex.from_product([rows, cols], names=["pert", "feature"]),
|
|
148
|
+
columns=["p_value", "statistic", "fold_change"],
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
"""
|
|
152
|
+
This case is harder than the OVR use case, because:
|
|
153
|
+
1. If input is dense, then it's easy we can slice easily over rows (or group of rows) or columns and parallelize the way we want
|
|
154
|
+
2. If input is CSR, we can easily slice the perturbations, and easily slice contiguous columns
|
|
155
|
+
3. If input is CSC, we can easily slice along the columns, BUT NOT ALONG THE PERTURBATIONS because perturbations are non-contiguous along axis 0
|
|
156
|
+
|
|
157
|
+
So the answer is simple: if we want a unified parallelism scheme, it has to be along the columns. Now:
|
|
158
|
+
- For the dense use case, there is no problem.
|
|
159
|
+
- For the sparse use case, the OVO kernel requires data to be CSC, so:
|
|
160
|
+
a) If input is CSC, it means we do overall: big CSC -> vertical chunk of CSC -> horizontal chunk of vertical chunk of CSC -> test
|
|
161
|
+
b) If input is CSR, it means we do: big CSR -> vertical chunk of CSR -> horizontal chunk of vertical chunk of CSR -> CSC -> test
|
|
162
|
+
For a), the second chunking is suboptimal because we chunk along 0 a CSC matrix, but because it has little columns this is fast.
|
|
163
|
+
For b), this is a lot of conversion but I believe they all happen on small matrices so this should be fine.
|
|
164
|
+
|
|
165
|
+
Conclusion: runtime on H1 is ~25 seconds for CSC, 25 seconds for CSR so still very nice.
|
|
166
|
+
TO KEEP IN MIND: there is **no** way to optimize any other parallelism than column-based actually, the current idea (parallelizing over the
|
|
167
|
+
perturbations) will be very slow, or require a brand new CSR allocation (which we don't want), to function.
|
|
168
|
+
"""
|
|
169
|
+
return results
|
illico/ovo/__init__.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from typing import Literal
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
from scipy import sparse as py_sparse
|
|
5
|
+
|
|
6
|
+
from illico.ovo.dense_ovo import dense_ovo_mwu_kernel_over_contiguous_col_chunk
|
|
7
|
+
from illico.ovo.sparse_ovo import (
|
|
8
|
+
csc_ovo_mwu_kernel_over_contiguous_col_chunk,
|
|
9
|
+
csr_ovo_mwu_kernel_over_contiguous_col_chunk,
|
|
10
|
+
)
|
|
11
|
+
from illico.utils.groups import GroupContainer
|
|
12
|
+
from illico.utils.sparse.csc import CSCMatrix
|
|
13
|
+
from illico.utils.sparse.csr import CSRMatrix
|
|
14
|
+
from illico.utils.type import scipy_to_nb
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def ovo_mwu_over_contiguous_col_chunk(
|
|
18
|
+
X: np.ndarray | py_sparse.csc_matrix | py_sparse.csr_matrix,
|
|
19
|
+
chunk_lb: int,
|
|
20
|
+
chunk_ub: int,
|
|
21
|
+
group_container: GroupContainer,
|
|
22
|
+
is_log1p: bool,
|
|
23
|
+
use_continuity: bool = True,
|
|
24
|
+
alternative: Literal["two-sided", "less", "greater"] = "two-sided",
|
|
25
|
+
):
|
|
26
|
+
"""Dispatcher for the OVO ranksum test. Routes toward the optimized
|
|
27
|
+
implementation depending on input data format.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
X (np.ndarray | py_sparse.csc_matrix | py_sparse.csr_matrix): Input expression count matrix.
|
|
31
|
+
chunk_lb (int): Vertical slicing (column) lower bound
|
|
32
|
+
chunk_ub (int): Vertical slicing (column) upper bound
|
|
33
|
+
group_container (GroupContainer): GroupContainer
|
|
34
|
+
is_log1p (bool): User-indicated flag telling if expression counts underwent log1p transfo.
|
|
35
|
+
use_continuity (bool, optional): Whether to use continuity correction when computing p-values.
|
|
36
|
+
alternative (Literal["two-sided", "less", "greater"]): Type of alternative hypothesis.
|
|
37
|
+
|
|
38
|
+
Raises:
|
|
39
|
+
ValueError: If input is neither dense, csc or csr.
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
tuple[np.ndarray]: Two-sided p-values, U-statistics and fold change.
|
|
43
|
+
Each np.ndarray of shape (n_groups, n_genes)
|
|
44
|
+
|
|
45
|
+
Author: Rémy Dubois
|
|
46
|
+
"""
|
|
47
|
+
# Convert to numba friendly dtype
|
|
48
|
+
X_nb = scipy_to_nb(X)
|
|
49
|
+
|
|
50
|
+
# Check all possible input types
|
|
51
|
+
if isinstance(X, np.ndarray):
|
|
52
|
+
pvalues, statistics, fold_change = dense_ovo_mwu_kernel_over_contiguous_col_chunk(
|
|
53
|
+
X,
|
|
54
|
+
chunk_lb,
|
|
55
|
+
chunk_ub,
|
|
56
|
+
group_container,
|
|
57
|
+
is_log1p=is_log1p,
|
|
58
|
+
use_continuity=use_continuity,
|
|
59
|
+
alternative=alternative,
|
|
60
|
+
)
|
|
61
|
+
elif isinstance(X_nb, CSCMatrix):
|
|
62
|
+
pvalues, statistics, fold_change = csc_ovo_mwu_kernel_over_contiguous_col_chunk(
|
|
63
|
+
X_nb,
|
|
64
|
+
chunk_lb,
|
|
65
|
+
chunk_ub,
|
|
66
|
+
group_container,
|
|
67
|
+
is_log1p=is_log1p,
|
|
68
|
+
use_continuity=use_continuity,
|
|
69
|
+
alternative=alternative,
|
|
70
|
+
)
|
|
71
|
+
elif isinstance(X_nb, CSRMatrix):
|
|
72
|
+
pvalues, statistics, fold_change = csr_ovo_mwu_kernel_over_contiguous_col_chunk(
|
|
73
|
+
X_nb,
|
|
74
|
+
chunk_lb,
|
|
75
|
+
chunk_ub,
|
|
76
|
+
group_container,
|
|
77
|
+
is_log1p=is_log1p,
|
|
78
|
+
use_continuity=use_continuity,
|
|
79
|
+
alternative=alternative,
|
|
80
|
+
)
|
|
81
|
+
else:
|
|
82
|
+
raise ValueError(X_nb)
|
|
83
|
+
|
|
84
|
+
return pvalues, statistics, fold_change
|
illico/ovo/dense_ovo.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
from typing import Literal
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
from numba import njit
|
|
5
|
+
|
|
6
|
+
from illico.utils.groups import GroupContainer
|
|
7
|
+
from illico.utils.math import chunk_and_fortranize, compute_pval, dense_fold_change
|
|
8
|
+
from illico.utils.ranking import (
|
|
9
|
+
_sort_along_axis_inplace,
|
|
10
|
+
rank_sum_and_ties_from_sorted,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@njit(nogil=True, fastmath=True, parallel=False, cache=False)
|
|
15
|
+
def dense_ovo_mwu_kernel(
|
|
16
|
+
sorted_ref_data: np.ndarray,
|
|
17
|
+
sorted_tgt_data: np.ndarray,
|
|
18
|
+
use_continuity: bool = True,
|
|
19
|
+
alternative: Literal["two-sided", "less", "greater"] = "two-sided",
|
|
20
|
+
) -> tuple[np.ndarray]:
|
|
21
|
+
"""Sequentially perform OVO tests on columns between sorted ref and sorted perturbed data.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
sorted_ref_data (np.ndarray): Vertically sorted reference data.
|
|
25
|
+
sorted_tgt_data (_type_): Vertically sorted perturbed data.
|
|
26
|
+
use_continuity (bool, optional): Apply continuity factor or not . Defaults to True.
|
|
27
|
+
alternative (Literal["two-sided", "less", "greater"]): Type of alternative hypothesis
|
|
28
|
+
|
|
29
|
+
Returns:
|
|
30
|
+
tuple[np.ndarray]: two-sided p-values, U-statistics. Each of shape (n_genes,).
|
|
31
|
+
|
|
32
|
+
Author: Rémy Dubois
|
|
33
|
+
"""
|
|
34
|
+
n_ref, ncols = sorted_ref_data.shape
|
|
35
|
+
n_tgt, _ = sorted_tgt_data.shape
|
|
36
|
+
|
|
37
|
+
U_statistics = np.empty(ncols, dtype=np.float64)
|
|
38
|
+
pvals = np.empty(ncols, dtype=np.float64)
|
|
39
|
+
n = n_ref + n_tgt
|
|
40
|
+
mu = n_ref * n_tgt / 2.0
|
|
41
|
+
for j in range(ncols):
|
|
42
|
+
R1, tie_sum = rank_sum_and_ties_from_sorted(sorted_ref_data[:, j], sorted_tgt_data[:, j])
|
|
43
|
+
|
|
44
|
+
# Compute U-stat
|
|
45
|
+
U1 = n_ref * n_tgt + n_tgt * (n_tgt + 1) / 2 - R1
|
|
46
|
+
|
|
47
|
+
pvals[j] = compute_pval(
|
|
48
|
+
n_ref=n_ref,
|
|
49
|
+
n_tgt=n_tgt,
|
|
50
|
+
n=n,
|
|
51
|
+
tie_sum=tie_sum,
|
|
52
|
+
U=U1,
|
|
53
|
+
mu=mu,
|
|
54
|
+
contin_corr=0.5 if use_continuity else 0.0,
|
|
55
|
+
alternative=alternative,
|
|
56
|
+
)
|
|
57
|
+
U_statistics[j] = U1
|
|
58
|
+
|
|
59
|
+
return pvals, U_statistics
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@njit(nogil=True, fastmath=True, cache=False, boundscheck=False)
|
|
63
|
+
def dense_ovo_mwu_kernel_over_contiguous_col_chunk(
|
|
64
|
+
X: np.ndarray,
|
|
65
|
+
chunk_lb: int,
|
|
66
|
+
chunk_ub: int,
|
|
67
|
+
grpc: GroupContainer,
|
|
68
|
+
is_log1p: bool,
|
|
69
|
+
use_continuity: bool = True,
|
|
70
|
+
alternative: Literal["two-sided", "less", "greater"] = "two-sided",
|
|
71
|
+
) -> tuple[np.ndarray]:
|
|
72
|
+
"""Perform OVO tests group-wise and gene(col)-wise.
|
|
73
|
+
|
|
74
|
+
Update: There is no need to fortranize the whole chunk at once, it can be done group by group within the loop. The memo
|
|
75
|
+
|
|
76
|
+
Memory footprint investigations:
|
|
77
|
+
1. ad.read_h5ad allocates memory in such a weird way that zeros are not properly assigned. Most likely this differs on unix systems. All investigations done on my MBP were so weird because of that.
|
|
78
|
+
Tests were so weird because during the wilcoxon test, values were accessed and re-ordered and all those things. As a result, some memory allocation was happening within my functions although
|
|
79
|
+
I was not allocating anything.
|
|
80
|
+
2. np.asfortranarray(X[:, chunk_lb:chunk_ub][grpc.indices]) seems to allocate 2x the memory needed for the chunk. So if chunk is 2GB, this line allocates 4GB temporarily.
|
|
81
|
+
3.
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
X (np.ndarray): Input dense expression matrix of shape (n_cells, n_genes)
|
|
86
|
+
chunk_lb (int): Lower bound of the vertical slicing
|
|
87
|
+
chunk_ub (int): Upper bound of the vertical slicing
|
|
88
|
+
grpc (GroupContainer): GroupContainer, contains information about which group each row belongs to.
|
|
89
|
+
use_continuity (bool, optional): Apply continuity factor or not. Defaults to True.
|
|
90
|
+
is_log1p (bool, optional): User-indicated flag telling if data underwent log1p transform or not. Defaults to False.
|
|
91
|
+
|
|
92
|
+
Raises:
|
|
93
|
+
ValueError: If bounds are not intelligible.
|
|
94
|
+
|
|
95
|
+
Returns:
|
|
96
|
+
tuple[np.ndarray]: two-sided p-values, U-statistics, fold change. Each
|
|
97
|
+
of shape (n_groups, chunk_ub - chunk_lb).
|
|
98
|
+
|
|
99
|
+
Author: Rémy Dubois
|
|
100
|
+
"""
|
|
101
|
+
chunk = X[:, chunk_lb:chunk_ub]
|
|
102
|
+
n_groups = grpc.counts.size
|
|
103
|
+
|
|
104
|
+
ref_indices = grpc.indices[grpc.indptr[grpc.encoded_ref_group] : grpc.indptr[grpc.encoded_ref_group + 1]]
|
|
105
|
+
# TODO: still have to benchmark speedup of F order
|
|
106
|
+
ref_chunk = chunk_and_fortranize(X, chunk_lb, chunk_ub, ref_indices)
|
|
107
|
+
_sort_along_axis_inplace(ref_chunk, axis=0)
|
|
108
|
+
|
|
109
|
+
pvalues = np.empty((n_groups, chunk_ub - chunk_lb), dtype=np.float64)
|
|
110
|
+
statistics = np.empty((n_groups, chunk_ub - chunk_lb), dtype=np.float64)
|
|
111
|
+
for group_id in range(n_groups):
|
|
112
|
+
if group_id == grpc.encoded_ref_group:
|
|
113
|
+
continue
|
|
114
|
+
tgt_indices = grpc.indices[grpc.indptr[group_id] : grpc.indptr[group_id + 1]]
|
|
115
|
+
# tgt_chunk = np.asfortranarray(chunk[tgt_indices, :])
|
|
116
|
+
tgt_chunk = chunk_and_fortranize(X, chunk_lb, chunk_ub, tgt_indices)
|
|
117
|
+
_sort_along_axis_inplace(tgt_chunk, axis=0)
|
|
118
|
+
|
|
119
|
+
pvalues[group_id], statistics[group_id] = dense_ovo_mwu_kernel(
|
|
120
|
+
sorted_ref_data=ref_chunk,
|
|
121
|
+
sorted_tgt_data=tgt_chunk,
|
|
122
|
+
use_continuity=use_continuity,
|
|
123
|
+
alternative=alternative,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
# Compute fold change
|
|
127
|
+
fc = dense_fold_change(chunk, grpc, is_log1p=is_log1p)
|
|
128
|
+
|
|
129
|
+
return pvalues, statistics, fc
|
illico/ovo/sparse_ovo.py
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
from typing import Literal
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
from numba import njit
|
|
5
|
+
|
|
6
|
+
from illico.utils.groups import GroupContainer
|
|
7
|
+
from illico.utils.math import compute_pval, diff
|
|
8
|
+
from illico.utils.ranking import (
|
|
9
|
+
_sort_csc_columns_inplace,
|
|
10
|
+
rank_sum_and_ties_from_sorted,
|
|
11
|
+
)
|
|
12
|
+
from illico.utils.sparse.csc import csc_get_contig_cols_into_csr
|
|
13
|
+
from illico.utils.sparse.csr import (
|
|
14
|
+
csr_fold_change,
|
|
15
|
+
csr_get_contig_cols_into_csr,
|
|
16
|
+
csr_get_rows_into_csc,
|
|
17
|
+
)
|
|
18
|
+
from illico.utils.type import CSCMatrix, CSRMatrix
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@njit(nogil=True, fastmath=True, cache=False)
|
|
22
|
+
def single_group_sparse_ovo_mwu_kernel(
|
|
23
|
+
sorted_ref_data: CSCMatrix,
|
|
24
|
+
sorted_tgt_data: CSCMatrix,
|
|
25
|
+
use_continuity: bool = True,
|
|
26
|
+
alternative: Literal["two-sided", "less", "greater"] = "two-sided",
|
|
27
|
+
) -> tuple[np.ndarray]:
|
|
28
|
+
"""Perform OVO tests gene wise using the two **sorted** CSC matrix given as input.
|
|
29
|
+
|
|
30
|
+
The test performed is the equivalent of:
|
|
31
|
+
`scipy.stats.mannwhitneyu(sorted_ref_data.toarray(), sorted_tgt_data.toarray(), use_continuity=True)`
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
sorted_ref_data (CSCMatrix): Reference data stored in CSC, sorted column-wise
|
|
35
|
+
sorted_tgt_data (CSCMatrix): Perturbed data stored in CSC, sorted column-wise
|
|
36
|
+
use_continuity (bool, optional): Apply continuity factor or not. Defaults to True.
|
|
37
|
+
alternative (Literal["two-sided", "less", "greater"]): Type of alternative hypothesis
|
|
38
|
+
|
|
39
|
+
Raises:
|
|
40
|
+
ValueError: If shape mismatche
|
|
41
|
+
|
|
42
|
+
Returns:
|
|
43
|
+
tuple[np.ndarray]: two-sided p-values, U-statistics. Each of shape (n_genes,).
|
|
44
|
+
|
|
45
|
+
Author: Rémy Dubois
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
n_ref, n_cols_ref = sorted_ref_data.shape
|
|
49
|
+
n_tgt, n_cols_tgt = sorted_tgt_data.shape
|
|
50
|
+
# TODO: make this check everywhere or nowhere
|
|
51
|
+
if n_cols_ref != n_cols_tgt:
|
|
52
|
+
raise ValueError("Uneven number of columns between ref and perturbed.")
|
|
53
|
+
|
|
54
|
+
# Allocate placeholders
|
|
55
|
+
n_zeros_tgt = (n_tgt - diff(sorted_tgt_data.indptr)).astype(np.int64)
|
|
56
|
+
n_zeros_ref = (n_ref - diff(sorted_ref_data.indptr)).astype(np.int64)
|
|
57
|
+
U_statistics = np.empty(n_cols_ref, dtype=np.float64)
|
|
58
|
+
pvals = np.empty(n_cols_ref, dtype=np.float64)
|
|
59
|
+
n = n_ref + n_tgt
|
|
60
|
+
mu = n_ref * n_tgt / 2.0
|
|
61
|
+
for j in range(n_cols_ref):
|
|
62
|
+
n_zeros_combined = n_zeros_ref[j] + n_zeros_tgt[j]
|
|
63
|
+
# Get the bounds
|
|
64
|
+
lbt, ubt = sorted_tgt_data.indptr[j], sorted_tgt_data.indptr[j + 1]
|
|
65
|
+
lbr, ubr = sorted_ref_data.indptr[j], sorted_ref_data.indptr[j + 1]
|
|
66
|
+
|
|
67
|
+
# Compute ranksum and tie sum for non zero values
|
|
68
|
+
ranksum, tie_sum = rank_sum_and_ties_from_sorted(sorted_ref_data.data[lbr:ubr], sorted_tgt_data.data[lbt:ubt])
|
|
69
|
+
|
|
70
|
+
# Offset the ranks of the number of zeros in ref and perturbed
|
|
71
|
+
ranksum += n_zeros_combined * (ubt - lbt)
|
|
72
|
+
|
|
73
|
+
# Compute ranksum
|
|
74
|
+
n0 = n_zeros_tgt[j]
|
|
75
|
+
R1_nz = ranksum # Sum ranks
|
|
76
|
+
R1 = R1_nz + n0 * (n_zeros_ref[j] + n0 + 1) / 2.0 # Add sumranks of zeros
|
|
77
|
+
|
|
78
|
+
# Compute U-stat
|
|
79
|
+
U1 = n_ref * n_tgt + n_tgt * (n_tgt + 1) / 2 - R1
|
|
80
|
+
|
|
81
|
+
# Compute sigma
|
|
82
|
+
tie_sum += n_zeros_combined**3 - n_zeros_combined
|
|
83
|
+
pvals[j] = compute_pval(
|
|
84
|
+
n_ref=n_ref,
|
|
85
|
+
n_tgt=n_tgt,
|
|
86
|
+
n=n,
|
|
87
|
+
tie_sum=tie_sum,
|
|
88
|
+
U=U1,
|
|
89
|
+
mu=mu,
|
|
90
|
+
contin_corr=0.5 if use_continuity else 0.0,
|
|
91
|
+
alternative=alternative,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
# Regardless of the alternative, always return U1 like scipy
|
|
95
|
+
U_statistics[j] = U1
|
|
96
|
+
|
|
97
|
+
return pvals, U_statistics
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@njit(nogil=True, fastmath=True, cache=False)
|
|
101
|
+
def multi_group_sparse_ovo_mwu_kernel(
|
|
102
|
+
X: CSRMatrix,
|
|
103
|
+
grpc: GroupContainer,
|
|
104
|
+
ref_group_id: int,
|
|
105
|
+
use_continuity: bool = True,
|
|
106
|
+
alternative: Literal["two-sided", "less", "greater"] = "two-sided",
|
|
107
|
+
) -> tuple[np.ndarray]:
|
|
108
|
+
"""Sequentially perform group-wise OVO tests along the columns on a CSR matrix.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
X (CSRMatrix): Input CSR matrix of shape (n_cells, n_genes)
|
|
112
|
+
grpc (GroupContainer): GroupContainer
|
|
113
|
+
ref_group_id (int): Encoded reference group ID.
|
|
114
|
+
use_continuity (bool, optional): Whether to use continuity correction when computing p-values. Defaults to True.
|
|
115
|
+
alternative (Literal["two-sided", "less", "greater"]): Type of alternative hypothesis
|
|
116
|
+
|
|
117
|
+
Returns:
|
|
118
|
+
tuple[np.ndarray]: two-sided p-values, U-statistics. Each of shape (n_groups, n_genes).
|
|
119
|
+
|
|
120
|
+
Author: Rémy Dubois
|
|
121
|
+
"""
|
|
122
|
+
# Avoid accessing attributes in the for loop
|
|
123
|
+
group_indices = grpc.indices
|
|
124
|
+
group_indptr = grpc.indptr
|
|
125
|
+
# Now, get the ref group, and convert it back to CSC because the test requires CSC data
|
|
126
|
+
n_groups = group_indptr.size - 1
|
|
127
|
+
ref_indices = group_indices[group_indptr[ref_group_id] : group_indptr[ref_group_id + 1]]
|
|
128
|
+
csc_X_ref = csr_get_rows_into_csc(X, ref_indices)
|
|
129
|
+
_sort_csc_columns_inplace(csc_matrix=csc_X_ref)
|
|
130
|
+
|
|
131
|
+
# Now go through all the groups one by one
|
|
132
|
+
pvalues = np.empty((n_groups, X.shape[1]), dtype=np.float64)
|
|
133
|
+
statistics = np.empty((n_groups, X.shape[1]), dtype=np.float64)
|
|
134
|
+
for group_id in range(group_indptr.size - 1):
|
|
135
|
+
if group_id == ref_group_id:
|
|
136
|
+
pvalues[group_id, :] = 1.0
|
|
137
|
+
statistics[group_id, :] = -1.0
|
|
138
|
+
continue
|
|
139
|
+
|
|
140
|
+
tgt_idxs = group_indices[group_indptr[group_id] : group_indptr[group_id + 1]]
|
|
141
|
+
X_tgt = csr_get_rows_into_csc(X, tgt_idxs)
|
|
142
|
+
_sort_csc_columns_inplace(X_tgt)
|
|
143
|
+
pvalue, statistic = single_group_sparse_ovo_mwu_kernel(
|
|
144
|
+
sorted_ref_data=csc_X_ref, sorted_tgt_data=X_tgt, use_continuity=use_continuity, alternative=alternative
|
|
145
|
+
)
|
|
146
|
+
pvalues[group_id, :] = pvalue
|
|
147
|
+
statistics[group_id, :] = statistic
|
|
148
|
+
|
|
149
|
+
return pvalues, statistics
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# Not jitting this and sorting all the cells at once is 1.5x slower. Ideally, it would be faster to sort only groups one by one but
|
|
153
|
+
# doubt this would be enough faster (think of mergesort) => It is twice faster, so i dont think it will bridge the gap
|
|
154
|
+
@njit(nogil=True, fastmath=True, cache=False) # This requires too many caching, too dangerous
|
|
155
|
+
def csc_ovo_mwu_kernel_over_contiguous_col_chunk(
|
|
156
|
+
X: CSCMatrix,
|
|
157
|
+
chunk_lb: int,
|
|
158
|
+
chunk_ub: int,
|
|
159
|
+
grpc: GroupContainer,
|
|
160
|
+
is_log1p: bool,
|
|
161
|
+
use_continuity: bool = True,
|
|
162
|
+
alternative: Literal["two-sided", "less", "greater"] = "two-sided",
|
|
163
|
+
):
|
|
164
|
+
"""Perform OVO test over contiguous chunk of column of a CSC matrix.
|
|
165
|
+
|
|
166
|
+
Args:
|
|
167
|
+
X (CSCMatrix): Input matrix of shape (n_cells, n_genes)
|
|
168
|
+
chunk_lb (int): Lower bound of the vertical slicing
|
|
169
|
+
chunk_ub (int): Upper bound of the vertical slicing
|
|
170
|
+
grpc (GroupContainer): GroupContainer
|
|
171
|
+
is_log1p (bool): User-indicated flag telling if data underwent log1p transform.
|
|
172
|
+
|
|
173
|
+
Raises:
|
|
174
|
+
ValueError: If chunk bounds are inintelligible.
|
|
175
|
+
|
|
176
|
+
Returns:
|
|
177
|
+
tuple[np.ndarray]: two-sided p-values, U-statistics, fold change. Each
|
|
178
|
+
of shape (n_groups, chunk_ub - chunk_lb).
|
|
179
|
+
|
|
180
|
+
Author: Rémy Dubois
|
|
181
|
+
"""
|
|
182
|
+
if chunk_lb < 0 or chunk_ub > X.shape[1] or chunk_lb > chunk_ub:
|
|
183
|
+
raise ValueError((chunk_lb, chunk_ub))
|
|
184
|
+
|
|
185
|
+
# This copies the data, so all that follow can happen in-place
|
|
186
|
+
csr_chunk = csc_get_contig_cols_into_csr(X, chunk_lb, chunk_ub)
|
|
187
|
+
pvalues, statistics = multi_group_sparse_ovo_mwu_kernel(
|
|
188
|
+
X=csr_chunk,
|
|
189
|
+
grpc=grpc,
|
|
190
|
+
ref_group_id=grpc.encoded_ref_group,
|
|
191
|
+
use_continuity=use_continuity,
|
|
192
|
+
alternative=alternative,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
# Compute fold change
|
|
196
|
+
fold_change = csr_fold_change(csr_chunk, grpc, is_log1p=is_log1p)
|
|
197
|
+
|
|
198
|
+
return pvalues, statistics, fold_change
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
# Real scale tests on whole H1 showed 24secs on 8 threads and 2min45s on 1, so a speedup of 165 / 24 = 6.875x
|
|
202
|
+
@njit(nogil=True, fastmath=True, cache=False)
|
|
203
|
+
def csr_ovo_mwu_kernel_over_contiguous_col_chunk(
|
|
204
|
+
X: CSRMatrix,
|
|
205
|
+
chunk_lb: int,
|
|
206
|
+
chunk_ub: int,
|
|
207
|
+
grpc: GroupContainer,
|
|
208
|
+
is_log1p: bool,
|
|
209
|
+
use_continuity: bool = True,
|
|
210
|
+
alternative: Literal["two-sided", "less", "greater"] = "two-sided",
|
|
211
|
+
):
|
|
212
|
+
"""Perform OVO test over contiguous chunk of column of a CSR matrix.
|
|
213
|
+
|
|
214
|
+
Args:
|
|
215
|
+
X (CSRMatrix): Input matrix of shape (n_cells, n_genes)
|
|
216
|
+
chunk_lb (int): Lower bound of the vertical slicing
|
|
217
|
+
chunk_ub (int): Upper bound of the vertical slicing
|
|
218
|
+
grpc (GroupContainer): GroupContainer
|
|
219
|
+
is_log1p (bool): User-indicated flag telling if data underwent log1p transform.
|
|
220
|
+
use_continuity (bool): Whether to use continuity correction when computing p-values.
|
|
221
|
+
alternative (Literal["two-sided", "less", "greater"]): Type of alternative hypothesis
|
|
222
|
+
|
|
223
|
+
Raises:
|
|
224
|
+
ValueError: If chunk bounds are inintelligible.
|
|
225
|
+
|
|
226
|
+
Returns:
|
|
227
|
+
tuple[np.ndarray]: two-sided p-values, U-statistics, fold change. Each
|
|
228
|
+
of shape (n_groups, chunk_ub - chunk_lb).
|
|
229
|
+
|
|
230
|
+
Author: Rémy Dubois
|
|
231
|
+
"""
|
|
232
|
+
if chunk_lb < 0 or chunk_ub > X.shape[1] or chunk_lb > chunk_ub:
|
|
233
|
+
raise ValueError((chunk_lb, chunk_ub))
|
|
234
|
+
|
|
235
|
+
# This copies the data, so all that follow can happen in-place
|
|
236
|
+
csr_chunk = csr_get_contig_cols_into_csr(X, chunk_lb, chunk_ub)
|
|
237
|
+
pvalues, statistics = multi_group_sparse_ovo_mwu_kernel(
|
|
238
|
+
X=csr_chunk,
|
|
239
|
+
grpc=grpc,
|
|
240
|
+
ref_group_id=grpc.encoded_ref_group,
|
|
241
|
+
use_continuity=use_continuity,
|
|
242
|
+
alternative=alternative,
|
|
243
|
+
)
|
|
244
|
+
# Compute fold change
|
|
245
|
+
fold_change = csr_fold_change(csr_chunk, grpc, is_log1p=is_log1p)
|
|
246
|
+
|
|
247
|
+
return pvalues, statistics, fold_change
|