illico 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
illico/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ from illico.asymptotic_wilcoxon import asymptotic_wilcoxon
2
+
3
+ __all__ = ["asymptotic_wilcoxon"]
@@ -0,0 +1,169 @@
1
+ import gc
2
+ import math
3
+
4
+ import anndata as ad
5
+ import numpy as np
6
+ import pandas as pd
7
+ from joblib import Parallel, delayed
8
+ from loguru import logger
9
+ from scipy import sparse
10
+ from tqdm.auto import tqdm
11
+
12
+ from illico.ovo import ovo_mwu_over_contiguous_col_chunk
13
+ from illico.ovr import ovr_mwu_over_col_contiguous_chunk
14
+ from illico.utils.compile import _precompile
15
+ from illico.utils.groups import encode_and_count_groups
16
+ from illico.utils.memory import _log_memory_usage
17
+ from illico.utils.ranking import check_indices_sorted_per_parcel
18
+
19
+ # from illico.utils.math import _warn_log1p
20
+
21
+ __all__ = ["asymptotic_wilcoxon"]
22
+
23
+
24
+ def asymptotic_wilcoxon(
25
+ adata: ad.AnnData,
26
+ is_log1p: bool,
27
+ group_keys: str,
28
+ reference_group: str | None = None,
29
+ n_threads: int = 1,
30
+ batch_size: int = 256,
31
+ alternative: str = "two-sided",
32
+ use_continuity: bool = True,
33
+ layer: str | None = None,
34
+ precompile: bool = True,
35
+ ):
36
+ """Perform asymptotic Mann-Whitney tests for differential gene expression.
37
+
38
+ Mann-Whitney test is the same as Wilcoxon rank-sum test.
39
+
40
+ This function takes as input an AnnData object of shape (n-cells, n-genes) with a group
41
+ (e.g., perturbation) variable stored in .obs. It performs either one-versus-rest (OVR) or
42
+ one-versus-one (OVO) Wilcoxon-Mann-Whitney tests for each gene, depending on whether a
43
+ reference group is provided.
44
+
45
+ Parameters
46
+ ----------
47
+ adata:
48
+ Annotated data matrix of shape (n-cells, n-genes).
49
+ is_log1p
50
+ Whether the data is log1p transformed.
51
+ group_keys
52
+ Key in `adata.obs` specifying the group variable.
53
+ reference_group
54
+ Name of the reference group for OVO tests. If `None`, OVR tests are performed.
55
+ n_threads
56
+ Number of threads to use for parallel computation.
57
+ batch_size
58
+ Number of genes to process in each batch.
59
+ alternative
60
+ Type of alternative hypothesis. One of 'two-sided', 'less', or 'greater'.
61
+ use_continuity
62
+ Whether to apply continuity correction.
63
+ layer
64
+ Layer in `adata.layers` to use for the data. If `None`, uses `adata.X`.
65
+ precompile
66
+ Whether to precompile necessary functions for performance.
67
+
68
+ Returns
69
+ -------
70
+ pd.DataFrame
71
+ A DataFrame with MultiIndex (group, gene) containing p-values, statistics, and fold changes.
72
+
73
+ Author: Rémy Dubois
74
+ """
75
+ # TODO: add a sparsity warning inviting user to convert to sparse if possible
76
+ # TODO: rename group_keys
77
+
78
+ if layer is not None:
79
+ logger.info(f"Using layer '{layer}' for differential expression.")
80
+ X = adata.layers[layer]
81
+ else:
82
+ X = adata.X
83
+
84
+ # Check that the input CSR is sorted.
85
+ if isinstance(X, sparse.csr_matrix):
86
+ if not check_indices_sorted_per_parcel(X.indices, X.indptr):
87
+ raise ValueError(
88
+ "Input data matrix indices are not sorted. This is very unusual and may lead to incorrect results. "
89
+ "This can be the result of operations like `adata[:, np.random.choice(…)]` that do not preserve sorting."
90
+ "Please make sure that indices used to chunk the adata or the expression matrix have been sorted "
91
+ "prior to computing DE genes."
92
+ )
93
+
94
+ if precompile:
95
+ _precompile(X, reference_group)
96
+
97
+ # Process the groups information
98
+ raw_groups = adata.obs[group_keys].tolist()
99
+ unique_raw_groups, group_container = encode_and_count_groups(groups=raw_groups, ref_group=reference_group)
100
+ logger.info(
101
+ f"Found {group_container.counts.size} unique groups (min size: {group_container.counts.min()} cells; max size: {group_container.counts.max()} cells), with reference group: {reference_group}"
102
+ )
103
+ _, n_genes = X.shape
104
+
105
+ # Allocate the results dataframes
106
+ cols = pd.Series(adata.var_names, name="feature", dtype=str)
107
+ rows = pd.Series(unique_raw_groups, name="pert", dtype=str)
108
+ results = np.empty((len(rows), len(cols), 3), dtype=np.float64)
109
+
110
+ # Adapt batch size to leverage multithreading regarding the number of genes, if requested
111
+ batch_size = min(batch_size, math.ceil(n_genes / n_threads))
112
+ logger.trace(f"Using batch size of {batch_size} for {n_threads} threads and {n_genes} genes.")
113
+
114
+ # Compute estimated mem footprint
115
+ _log_memory_usage(adata, group_container, batch_size, n_threads)
116
+
117
+ # Generate batches of columns
118
+ bounds = np.append(np.arange(0, n_genes, batch_size), n_genes)
119
+ iterator = list(zip(bounds[:-1], bounds[1:]))
120
+ n_tests = n_genes * group_container.counts.size
121
+ logger.trace(f"Performing a total of {n_tests:,d} tests.")
122
+ with Parallel(n_threads, prefer="threads", return_as="generator_unordered") as pool:
123
+ with tqdm(total=n_tests, smoothing=0.0, unit="it", unit_scale=True, unit_divisor=1000) as pbar:
124
+ if reference_group is None: # ovr use case
125
+ pbar.set_description("Running one-versus-all MannWhitney-U tests")
126
+ op = delayed(lambda *args: (ovr_mwu_over_col_contiguous_chunk(*args), args))
127
+ else: # ovo use case
128
+ pbar.set_description("Running one-versus-ref MannWhitney-U tests")
129
+ op = delayed(lambda *args: (ovo_mwu_over_contiguous_col_chunk(*args), args))
130
+ for (pv, ustat, fc), args in pool(
131
+ op(X, lb, ub, group_container, is_log1p, use_continuity, alternative) for lb, ub in iterator
132
+ ):
133
+ # progress.update(task, advance=group_container.counts.size * (args[2] - args[1])) # refresh after processing is done
134
+ pbar.update(group_container.counts.size * (args[2] - args[1]))
135
+ col_chunk = slice(*args[1:3])
136
+ results[:, col_chunk, 0] = pv
137
+ results[:, col_chunk, 1] = ustat
138
+ results[:, col_chunk, 2] = fc
139
+
140
+ # Cleanup memory
141
+ del pv, ustat, fc
142
+ gc.collect()
143
+
144
+ # Return a pd.DataFrame to index results
145
+ results = pd.DataFrame(
146
+ data=results.reshape(-1, 3),
147
+ index=pd.MultiIndex.from_product([rows, cols], names=["pert", "feature"]),
148
+ columns=["p_value", "statistic", "fold_change"],
149
+ )
150
+
151
+ """
152
+ This case is harder than the OVR use case, because:
153
+ 1. If input is dense, then it's easy we can slice easily over rows (or group of rows) or columns and parallelize the way we want
154
+ 2. If input is CSR, we can easily slice the perturbations, and easily slice contiguous columns
155
+ 3. If input is CSC, we can easily slice along the columns, BUT NOT ALONG THE PERTURBATIONS because perturbations are non-contiguous along axis 0
156
+
157
+ So the answer is simple: if we want a unified parallelism scheme, it has to be along the columns. Now:
158
+ - For the dense use case, there is no problem.
159
+ - For the sparse use case, the OVO kernel requires data to be CSC, so:
160
+ a) If input is CSC, it means we do overall: big CSC -> vertical chunk of CSC -> horizontal chunk of vertical chunk of CSC -> test
161
+ b) If input is CSR, it means we do: big CSR -> vertical chunk of CSR -> horizontal chunk of vertical chunk of CSR -> CSC -> test
162
+ For a), the second chunking is suboptimal because we chunk along 0 a CSC matrix, but because it has little columns this is fast.
163
+ For b), this is a lot of conversion but I believe they all happen on small matrices so this should be fine.
164
+
165
+ Conclusion: runtime on H1 is ~25 seconds for CSC, 25 seconds for CSR so still very nice.
166
+ TO KEEP IN MIND: there is **no** way to optimize any other parallelism than column-based actually, the current idea (parallelizing over the
167
+ perturbations) will be very slow, or require a brand new CSR allocation (which we don't want), to function.
168
+ """
169
+ return results
illico/ovo/__init__.py ADDED
@@ -0,0 +1,84 @@
1
+ from typing import Literal
2
+
3
+ import numpy as np
4
+ from scipy import sparse as py_sparse
5
+
6
+ from illico.ovo.dense_ovo import dense_ovo_mwu_kernel_over_contiguous_col_chunk
7
+ from illico.ovo.sparse_ovo import (
8
+ csc_ovo_mwu_kernel_over_contiguous_col_chunk,
9
+ csr_ovo_mwu_kernel_over_contiguous_col_chunk,
10
+ )
11
+ from illico.utils.groups import GroupContainer
12
+ from illico.utils.sparse.csc import CSCMatrix
13
+ from illico.utils.sparse.csr import CSRMatrix
14
+ from illico.utils.type import scipy_to_nb
15
+
16
+
17
+ def ovo_mwu_over_contiguous_col_chunk(
18
+ X: np.ndarray | py_sparse.csc_matrix | py_sparse.csr_matrix,
19
+ chunk_lb: int,
20
+ chunk_ub: int,
21
+ group_container: GroupContainer,
22
+ is_log1p: bool,
23
+ use_continuity: bool = True,
24
+ alternative: Literal["two-sided", "less", "greater"] = "two-sided",
25
+ ):
26
+ """Dispatcher for the OVO ranksum test. Routes toward the optimized
27
+ implementation depending on input data format.
28
+
29
+ Args:
30
+ X (np.ndarray | py_sparse.csc_matrix | py_sparse.csr_matrix): Input expression count matrix.
31
+ chunk_lb (int): Vertical slicing (column) lower bound
32
+ chunk_ub (int): Vertical slicing (column) upper bound
33
+ group_container (GroupContainer): GroupContainer
34
+ is_log1p (bool): User-indicated flag telling if expression counts underwent log1p transfo.
35
+ use_continuity (bool, optional): Whether to use continuity correction when computing p-values.
36
+ alternative (Literal["two-sided", "less", "greater"]): Type of alternative hypothesis.
37
+
38
+ Raises:
39
+ ValueError: If input is neither dense, csc or csr.
40
+
41
+ Returns:
42
+ tuple[np.ndarray]: Two-sided p-values, U-statistics and fold change.
43
+ Each np.ndarray of shape (n_groups, n_genes)
44
+
45
+ Author: Rémy Dubois
46
+ """
47
+ # Convert to numba friendly dtype
48
+ X_nb = scipy_to_nb(X)
49
+
50
+ # Check all possible input types
51
+ if isinstance(X, np.ndarray):
52
+ pvalues, statistics, fold_change = dense_ovo_mwu_kernel_over_contiguous_col_chunk(
53
+ X,
54
+ chunk_lb,
55
+ chunk_ub,
56
+ group_container,
57
+ is_log1p=is_log1p,
58
+ use_continuity=use_continuity,
59
+ alternative=alternative,
60
+ )
61
+ elif isinstance(X_nb, CSCMatrix):
62
+ pvalues, statistics, fold_change = csc_ovo_mwu_kernel_over_contiguous_col_chunk(
63
+ X_nb,
64
+ chunk_lb,
65
+ chunk_ub,
66
+ group_container,
67
+ is_log1p=is_log1p,
68
+ use_continuity=use_continuity,
69
+ alternative=alternative,
70
+ )
71
+ elif isinstance(X_nb, CSRMatrix):
72
+ pvalues, statistics, fold_change = csr_ovo_mwu_kernel_over_contiguous_col_chunk(
73
+ X_nb,
74
+ chunk_lb,
75
+ chunk_ub,
76
+ group_container,
77
+ is_log1p=is_log1p,
78
+ use_continuity=use_continuity,
79
+ alternative=alternative,
80
+ )
81
+ else:
82
+ raise ValueError(X_nb)
83
+
84
+ return pvalues, statistics, fold_change
@@ -0,0 +1,129 @@
1
+ from typing import Literal
2
+
3
+ import numpy as np
4
+ from numba import njit
5
+
6
+ from illico.utils.groups import GroupContainer
7
+ from illico.utils.math import chunk_and_fortranize, compute_pval, dense_fold_change
8
+ from illico.utils.ranking import (
9
+ _sort_along_axis_inplace,
10
+ rank_sum_and_ties_from_sorted,
11
+ )
12
+
13
+
14
+ @njit(nogil=True, fastmath=True, parallel=False, cache=False)
15
+ def dense_ovo_mwu_kernel(
16
+ sorted_ref_data: np.ndarray,
17
+ sorted_tgt_data: np.ndarray,
18
+ use_continuity: bool = True,
19
+ alternative: Literal["two-sided", "less", "greater"] = "two-sided",
20
+ ) -> tuple[np.ndarray]:
21
+ """Sequentially perform OVO tests on columns between sorted ref and sorted perturbed data.
22
+
23
+ Args:
24
+ sorted_ref_data (np.ndarray): Vertically sorted reference data.
25
+ sorted_tgt_data (_type_): Vertically sorted perturbed data.
26
+ use_continuity (bool, optional): Apply continuity factor or not . Defaults to True.
27
+ alternative (Literal["two-sided", "less", "greater"]): Type of alternative hypothesis
28
+
29
+ Returns:
30
+ tuple[np.ndarray]: two-sided p-values, U-statistics. Each of shape (n_genes,).
31
+
32
+ Author: Rémy Dubois
33
+ """
34
+ n_ref, ncols = sorted_ref_data.shape
35
+ n_tgt, _ = sorted_tgt_data.shape
36
+
37
+ U_statistics = np.empty(ncols, dtype=np.float64)
38
+ pvals = np.empty(ncols, dtype=np.float64)
39
+ n = n_ref + n_tgt
40
+ mu = n_ref * n_tgt / 2.0
41
+ for j in range(ncols):
42
+ R1, tie_sum = rank_sum_and_ties_from_sorted(sorted_ref_data[:, j], sorted_tgt_data[:, j])
43
+
44
+ # Compute U-stat
45
+ U1 = n_ref * n_tgt + n_tgt * (n_tgt + 1) / 2 - R1
46
+
47
+ pvals[j] = compute_pval(
48
+ n_ref=n_ref,
49
+ n_tgt=n_tgt,
50
+ n=n,
51
+ tie_sum=tie_sum,
52
+ U=U1,
53
+ mu=mu,
54
+ contin_corr=0.5 if use_continuity else 0.0,
55
+ alternative=alternative,
56
+ )
57
+ U_statistics[j] = U1
58
+
59
+ return pvals, U_statistics
60
+
61
+
62
+ @njit(nogil=True, fastmath=True, cache=False, boundscheck=False)
63
+ def dense_ovo_mwu_kernel_over_contiguous_col_chunk(
64
+ X: np.ndarray,
65
+ chunk_lb: int,
66
+ chunk_ub: int,
67
+ grpc: GroupContainer,
68
+ is_log1p: bool,
69
+ use_continuity: bool = True,
70
+ alternative: Literal["two-sided", "less", "greater"] = "two-sided",
71
+ ) -> tuple[np.ndarray]:
72
+ """Perform OVO tests group-wise and gene(col)-wise.
73
+
74
+ Update: There is no need to fortranize the whole chunk at once, it can be done group by group within the loop. The memo
75
+
76
+ Memory footprint investigations:
77
+ 1. ad.read_h5ad allocates memory in such a weird way that zeros are not properly assigned. Most likely this differs on unix systems. All investigations done on my MBP were so weird because of that.
78
+ Tests were so weird because during the wilcoxon test, values were accessed and re-ordered and all those things. As a result, some memory allocation was happening within my functions although
79
+ I was not allocating anything.
80
+ 2. np.asfortranarray(X[:, chunk_lb:chunk_ub][grpc.indices]) seems to allocate 2x the memory needed for the chunk. So if chunk is 2GB, this line allocates 4GB temporarily.
81
+ 3.
82
+
83
+
84
+ Args:
85
+ X (np.ndarray): Input dense expression matrix of shape (n_cells, n_genes)
86
+ chunk_lb (int): Lower bound of the vertical slicing
87
+ chunk_ub (int): Upper bound of the vertical slicing
88
+ grpc (GroupContainer): GroupContainer, contains information about which group each row belongs to.
89
+ use_continuity (bool, optional): Apply continuity factor or not. Defaults to True.
90
+ is_log1p (bool, optional): User-indicated flag telling if data underwent log1p transform or not. Defaults to False.
91
+
92
+ Raises:
93
+ ValueError: If bounds are not intelligible.
94
+
95
+ Returns:
96
+ tuple[np.ndarray]: two-sided p-values, U-statistics, fold change. Each
97
+ of shape (n_groups, chunk_ub - chunk_lb).
98
+
99
+ Author: Rémy Dubois
100
+ """
101
+ chunk = X[:, chunk_lb:chunk_ub]
102
+ n_groups = grpc.counts.size
103
+
104
+ ref_indices = grpc.indices[grpc.indptr[grpc.encoded_ref_group] : grpc.indptr[grpc.encoded_ref_group + 1]]
105
+ # TODO: still have to benchmark speedup of F order
106
+ ref_chunk = chunk_and_fortranize(X, chunk_lb, chunk_ub, ref_indices)
107
+ _sort_along_axis_inplace(ref_chunk, axis=0)
108
+
109
+ pvalues = np.empty((n_groups, chunk_ub - chunk_lb), dtype=np.float64)
110
+ statistics = np.empty((n_groups, chunk_ub - chunk_lb), dtype=np.float64)
111
+ for group_id in range(n_groups):
112
+ if group_id == grpc.encoded_ref_group:
113
+ continue
114
+ tgt_indices = grpc.indices[grpc.indptr[group_id] : grpc.indptr[group_id + 1]]
115
+ # tgt_chunk = np.asfortranarray(chunk[tgt_indices, :])
116
+ tgt_chunk = chunk_and_fortranize(X, chunk_lb, chunk_ub, tgt_indices)
117
+ _sort_along_axis_inplace(tgt_chunk, axis=0)
118
+
119
+ pvalues[group_id], statistics[group_id] = dense_ovo_mwu_kernel(
120
+ sorted_ref_data=ref_chunk,
121
+ sorted_tgt_data=tgt_chunk,
122
+ use_continuity=use_continuity,
123
+ alternative=alternative,
124
+ )
125
+
126
+ # Compute fold change
127
+ fc = dense_fold_change(chunk, grpc, is_log1p=is_log1p)
128
+
129
+ return pvalues, statistics, fc
@@ -0,0 +1,247 @@
1
+ from typing import Literal
2
+
3
+ import numpy as np
4
+ from numba import njit
5
+
6
+ from illico.utils.groups import GroupContainer
7
+ from illico.utils.math import compute_pval, diff
8
+ from illico.utils.ranking import (
9
+ _sort_csc_columns_inplace,
10
+ rank_sum_and_ties_from_sorted,
11
+ )
12
+ from illico.utils.sparse.csc import csc_get_contig_cols_into_csr
13
+ from illico.utils.sparse.csr import (
14
+ csr_fold_change,
15
+ csr_get_contig_cols_into_csr,
16
+ csr_get_rows_into_csc,
17
+ )
18
+ from illico.utils.type import CSCMatrix, CSRMatrix
19
+
20
+
21
+ @njit(nogil=True, fastmath=True, cache=False)
22
+ def single_group_sparse_ovo_mwu_kernel(
23
+ sorted_ref_data: CSCMatrix,
24
+ sorted_tgt_data: CSCMatrix,
25
+ use_continuity: bool = True,
26
+ alternative: Literal["two-sided", "less", "greater"] = "two-sided",
27
+ ) -> tuple[np.ndarray]:
28
+ """Perform OVO tests gene wise using the two **sorted** CSC matrix given as input.
29
+
30
+ The test performed is the equivalent of:
31
+ `scipy.stats.mannwhitneyu(sorted_ref_data.toarray(), sorted_tgt_data.toarray(), use_continuity=True)`
32
+
33
+ Args:
34
+ sorted_ref_data (CSCMatrix): Reference data stored in CSC, sorted column-wise
35
+ sorted_tgt_data (CSCMatrix): Perturbed data stored in CSC, sorted column-wise
36
+ use_continuity (bool, optional): Apply continuity factor or not. Defaults to True.
37
+ alternative (Literal["two-sided", "less", "greater"]): Type of alternative hypothesis
38
+
39
+ Raises:
40
+ ValueError: If shape mismatche
41
+
42
+ Returns:
43
+ tuple[np.ndarray]: two-sided p-values, U-statistics. Each of shape (n_genes,).
44
+
45
+ Author: Rémy Dubois
46
+ """
47
+
48
+ n_ref, n_cols_ref = sorted_ref_data.shape
49
+ n_tgt, n_cols_tgt = sorted_tgt_data.shape
50
+ # TODO: make this check everywhere or nowhere
51
+ if n_cols_ref != n_cols_tgt:
52
+ raise ValueError("Uneven number of columns between ref and perturbed.")
53
+
54
+ # Allocate placeholders
55
+ n_zeros_tgt = (n_tgt - diff(sorted_tgt_data.indptr)).astype(np.int64)
56
+ n_zeros_ref = (n_ref - diff(sorted_ref_data.indptr)).astype(np.int64)
57
+ U_statistics = np.empty(n_cols_ref, dtype=np.float64)
58
+ pvals = np.empty(n_cols_ref, dtype=np.float64)
59
+ n = n_ref + n_tgt
60
+ mu = n_ref * n_tgt / 2.0
61
+ for j in range(n_cols_ref):
62
+ n_zeros_combined = n_zeros_ref[j] + n_zeros_tgt[j]
63
+ # Get the bounds
64
+ lbt, ubt = sorted_tgt_data.indptr[j], sorted_tgt_data.indptr[j + 1]
65
+ lbr, ubr = sorted_ref_data.indptr[j], sorted_ref_data.indptr[j + 1]
66
+
67
+ # Compute ranksum and tie sum for non zero values
68
+ ranksum, tie_sum = rank_sum_and_ties_from_sorted(sorted_ref_data.data[lbr:ubr], sorted_tgt_data.data[lbt:ubt])
69
+
70
+ # Offset the ranks of the number of zeros in ref and perturbed
71
+ ranksum += n_zeros_combined * (ubt - lbt)
72
+
73
+ # Compute ranksum
74
+ n0 = n_zeros_tgt[j]
75
+ R1_nz = ranksum # Sum ranks
76
+ R1 = R1_nz + n0 * (n_zeros_ref[j] + n0 + 1) / 2.0 # Add sumranks of zeros
77
+
78
+ # Compute U-stat
79
+ U1 = n_ref * n_tgt + n_tgt * (n_tgt + 1) / 2 - R1
80
+
81
+ # Compute sigma
82
+ tie_sum += n_zeros_combined**3 - n_zeros_combined
83
+ pvals[j] = compute_pval(
84
+ n_ref=n_ref,
85
+ n_tgt=n_tgt,
86
+ n=n,
87
+ tie_sum=tie_sum,
88
+ U=U1,
89
+ mu=mu,
90
+ contin_corr=0.5 if use_continuity else 0.0,
91
+ alternative=alternative,
92
+ )
93
+
94
+ # Regardless of the alternative, always return U1 like scipy
95
+ U_statistics[j] = U1
96
+
97
+ return pvals, U_statistics
98
+
99
+
100
+ @njit(nogil=True, fastmath=True, cache=False)
101
+ def multi_group_sparse_ovo_mwu_kernel(
102
+ X: CSRMatrix,
103
+ grpc: GroupContainer,
104
+ ref_group_id: int,
105
+ use_continuity: bool = True,
106
+ alternative: Literal["two-sided", "less", "greater"] = "two-sided",
107
+ ) -> tuple[np.ndarray]:
108
+ """Sequentially perform group-wise OVO tests along the columns on a CSR matrix.
109
+
110
+ Args:
111
+ X (CSRMatrix): Input CSR matrix of shape (n_cells, n_genes)
112
+ grpc (GroupContainer): GroupContainer
113
+ ref_group_id (int): Encoded reference group ID.
114
+ use_continuity (bool, optional): Whether to use continuity correction when computing p-values. Defaults to True.
115
+ alternative (Literal["two-sided", "less", "greater"]): Type of alternative hypothesis
116
+
117
+ Returns:
118
+ tuple[np.ndarray]: two-sided p-values, U-statistics. Each of shape (n_groups, n_genes).
119
+
120
+ Author: Rémy Dubois
121
+ """
122
+ # Avoid accessing attributes in the for loop
123
+ group_indices = grpc.indices
124
+ group_indptr = grpc.indptr
125
+ # Now, get the ref group, and convert it back to CSC because the test requires CSC data
126
+ n_groups = group_indptr.size - 1
127
+ ref_indices = group_indices[group_indptr[ref_group_id] : group_indptr[ref_group_id + 1]]
128
+ csc_X_ref = csr_get_rows_into_csc(X, ref_indices)
129
+ _sort_csc_columns_inplace(csc_matrix=csc_X_ref)
130
+
131
+ # Now go through all the groups one by one
132
+ pvalues = np.empty((n_groups, X.shape[1]), dtype=np.float64)
133
+ statistics = np.empty((n_groups, X.shape[1]), dtype=np.float64)
134
+ for group_id in range(group_indptr.size - 1):
135
+ if group_id == ref_group_id:
136
+ pvalues[group_id, :] = 1.0
137
+ statistics[group_id, :] = -1.0
138
+ continue
139
+
140
+ tgt_idxs = group_indices[group_indptr[group_id] : group_indptr[group_id + 1]]
141
+ X_tgt = csr_get_rows_into_csc(X, tgt_idxs)
142
+ _sort_csc_columns_inplace(X_tgt)
143
+ pvalue, statistic = single_group_sparse_ovo_mwu_kernel(
144
+ sorted_ref_data=csc_X_ref, sorted_tgt_data=X_tgt, use_continuity=use_continuity, alternative=alternative
145
+ )
146
+ pvalues[group_id, :] = pvalue
147
+ statistics[group_id, :] = statistic
148
+
149
+ return pvalues, statistics
150
+
151
+
152
+ # Not jitting this and sorting all the cells at once is 1.5x slower. Ideally, it would be faster to sort only groups one by one but
153
+ # doubt this would be enough faster (think of mergesort) => It is twice faster, so i dont think it will bridge the gap
154
+ @njit(nogil=True, fastmath=True, cache=False) # This requires too many caching, too dangerous
155
+ def csc_ovo_mwu_kernel_over_contiguous_col_chunk(
156
+ X: CSCMatrix,
157
+ chunk_lb: int,
158
+ chunk_ub: int,
159
+ grpc: GroupContainer,
160
+ is_log1p: bool,
161
+ use_continuity: bool = True,
162
+ alternative: Literal["two-sided", "less", "greater"] = "two-sided",
163
+ ):
164
+ """Perform OVO test over contiguous chunk of column of a CSC matrix.
165
+
166
+ Args:
167
+ X (CSCMatrix): Input matrix of shape (n_cells, n_genes)
168
+ chunk_lb (int): Lower bound of the vertical slicing
169
+ chunk_ub (int): Upper bound of the vertical slicing
170
+ grpc (GroupContainer): GroupContainer
171
+ is_log1p (bool): User-indicated flag telling if data underwent log1p transform.
172
+
173
+ Raises:
174
+ ValueError: If chunk bounds are inintelligible.
175
+
176
+ Returns:
177
+ tuple[np.ndarray]: two-sided p-values, U-statistics, fold change. Each
178
+ of shape (n_groups, chunk_ub - chunk_lb).
179
+
180
+ Author: Rémy Dubois
181
+ """
182
+ if chunk_lb < 0 or chunk_ub > X.shape[1] or chunk_lb > chunk_ub:
183
+ raise ValueError((chunk_lb, chunk_ub))
184
+
185
+ # This copies the data, so all that follow can happen in-place
186
+ csr_chunk = csc_get_contig_cols_into_csr(X, chunk_lb, chunk_ub)
187
+ pvalues, statistics = multi_group_sparse_ovo_mwu_kernel(
188
+ X=csr_chunk,
189
+ grpc=grpc,
190
+ ref_group_id=grpc.encoded_ref_group,
191
+ use_continuity=use_continuity,
192
+ alternative=alternative,
193
+ )
194
+
195
+ # Compute fold change
196
+ fold_change = csr_fold_change(csr_chunk, grpc, is_log1p=is_log1p)
197
+
198
+ return pvalues, statistics, fold_change
199
+
200
+
201
+ # Real scale tests on whole H1 showed 24secs on 8 threads and 2min45s on 1, so a speedup of 165 / 24 = 6.875x
202
+ @njit(nogil=True, fastmath=True, cache=False)
203
+ def csr_ovo_mwu_kernel_over_contiguous_col_chunk(
204
+ X: CSRMatrix,
205
+ chunk_lb: int,
206
+ chunk_ub: int,
207
+ grpc: GroupContainer,
208
+ is_log1p: bool,
209
+ use_continuity: bool = True,
210
+ alternative: Literal["two-sided", "less", "greater"] = "two-sided",
211
+ ):
212
+ """Perform OVO test over contiguous chunk of column of a CSR matrix.
213
+
214
+ Args:
215
+ X (CSRMatrix): Input matrix of shape (n_cells, n_genes)
216
+ chunk_lb (int): Lower bound of the vertical slicing
217
+ chunk_ub (int): Upper bound of the vertical slicing
218
+ grpc (GroupContainer): GroupContainer
219
+ is_log1p (bool): User-indicated flag telling if data underwent log1p transform.
220
+ use_continuity (bool): Whether to use continuity correction when computing p-values.
221
+ alternative (Literal["two-sided", "less", "greater"]): Type of alternative hypothesis
222
+
223
+ Raises:
224
+ ValueError: If chunk bounds are inintelligible.
225
+
226
+ Returns:
227
+ tuple[np.ndarray]: two-sided p-values, U-statistics, fold change. Each
228
+ of shape (n_groups, chunk_ub - chunk_lb).
229
+
230
+ Author: Rémy Dubois
231
+ """
232
+ if chunk_lb < 0 or chunk_ub > X.shape[1] or chunk_lb > chunk_ub:
233
+ raise ValueError((chunk_lb, chunk_ub))
234
+
235
+ # This copies the data, so all that follow can happen in-place
236
+ csr_chunk = csr_get_contig_cols_into_csr(X, chunk_lb, chunk_ub)
237
+ pvalues, statistics = multi_group_sparse_ovo_mwu_kernel(
238
+ X=csr_chunk,
239
+ grpc=grpc,
240
+ ref_group_id=grpc.encoded_ref_group,
241
+ use_continuity=use_continuity,
242
+ alternative=alternative,
243
+ )
244
+ # Compute fold change
245
+ fold_change = csr_fold_change(csr_chunk, grpc, is_log1p=is_log1p)
246
+
247
+ return pvalues, statistics, fold_change