clone2vec 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clone2vec/__init__.py +23 -0
- clone2vec/associations.py +1117 -0
- clone2vec/catboost.py +622 -0
- clone2vec/datasets.py +94 -0
- clone2vec/eigenvalues.py +187 -0
- clone2vec/embeddings.py +1204 -0
- clone2vec/integration.py +511 -0
- clone2vec/plotting.py +3540 -0
- clone2vec/preprocessing.py +660 -0
- clone2vec/seurat.py +720 -0
- clone2vec/tools.py +633 -0
- clone2vec/utils.py +618 -0
- clone2vec-0.1.0.dist-info/METADATA +86 -0
- clone2vec-0.1.0.dist-info/RECORD +16 -0
- clone2vec-0.1.0.dist-info/WHEEL +5 -0
- clone2vec-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1117 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import scanpy as sc
|
|
4
|
+
import numpy as np
|
|
5
|
+
import scipy.sparse as sp
|
|
6
|
+
import statsmodels.api as sm
|
|
7
|
+
import statsmodels.formula.api as smf
|
|
8
|
+
import scipy.stats as stats
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
from contextlib import nullcontext
|
|
15
|
+
from joblib import Parallel, delayed
|
|
16
|
+
from typing import Literal
|
|
17
|
+
from tqdm import tqdm
|
|
18
|
+
|
|
19
|
+
from .utils import _nan_mask
|
|
20
|
+
|
|
21
|
+
logg = sc.logging
|
|
22
|
+
|
|
23
|
+
def _clr(x: np.ndarray, epsilon: float = 1e-3) -> np.ndarray:
|
|
24
|
+
"""
|
|
25
|
+
Center Log-Ratio (CLR) transformation.
|
|
26
|
+
|
|
27
|
+
Parameters
|
|
28
|
+
----------
|
|
29
|
+
x : np.ndarray
|
|
30
|
+
Input array with shape (n_samples, n_features).
|
|
31
|
+
epsilon : float, optional
|
|
32
|
+
Small constant to avoid log(0) and division by zero, by default 1e-3.
|
|
33
|
+
|
|
34
|
+
Returns
|
|
35
|
+
-------
|
|
36
|
+
np.ndarray
|
|
37
|
+
CLR-transformed array with shape (n_samples, n_features).
|
|
38
|
+
"""
|
|
39
|
+
if not np.allclose(x, x.astype(int)):
|
|
40
|
+
if epsilon == 1e-3:
|
|
41
|
+
logg.warning("Pseudocount 0.001 is used for integer counts as an input for CLR, it might skew proportions.")
|
|
42
|
+
|
|
43
|
+
if isinstance(x, pd.DataFrame):
|
|
44
|
+
x_cols = x.columns
|
|
45
|
+
x_rows = x.index
|
|
46
|
+
x = x.values.copy()
|
|
47
|
+
|
|
48
|
+
x_pseudo = x + epsilon
|
|
49
|
+
log_geo_mean = np.mean(np.log(x_pseudo), axis=1, keepdims=True)
|
|
50
|
+
clr_transformed = np.log(x_pseudo) - log_geo_mean
|
|
51
|
+
|
|
52
|
+
return pd.DataFrame(clr_transformed, columns=x_cols, index=x_rows)
|
|
53
|
+
else:
|
|
54
|
+
x_pseudo = x + epsilon
|
|
55
|
+
log_geo_mean = np.mean(np.log(x_pseudo), axis=1, keepdims=True)
|
|
56
|
+
clr_transformed = np.log(x_pseudo) - log_geo_mean
|
|
57
|
+
|
|
58
|
+
return clr_transformed
|
|
59
|
+
|
|
60
|
+
def _fast_corr(
|
|
61
|
+
X: np.ndarray | sp.spmatrix,
|
|
62
|
+
Y: np.ndarray,
|
|
63
|
+
method: Literal["pearson", "spearman"] = "pearson",
|
|
64
|
+
significance: bool = True,
|
|
65
|
+
slope: bool = True,
|
|
66
|
+
batch_size: int = 1000,
|
|
67
|
+
progress_bar: bool = True,
|
|
68
|
+
weights: np.ndarray | None = None,
|
|
69
|
+
) -> dict[str, np.ndarray]:
|
|
70
|
+
"""
|
|
71
|
+
Calculate the correlation matrix between X and Y, gracefully handling
|
|
72
|
+
columns with zero variance by setting the correlation to NaN.
|
|
73
|
+
|
|
74
|
+
Supports sparse X by converting column-wise dense batches for computation.
|
|
75
|
+
|
|
76
|
+
Parameters
|
|
77
|
+
----------
|
|
78
|
+
X : np.ndarray | scipy.sparse.spmatrix
|
|
79
|
+
Dataset with shape (n_samples, n_features). If sparse, columns are
|
|
80
|
+
processed in dense batches to limit memory usage.
|
|
81
|
+
Y : np.ndarray
|
|
82
|
+
Response matrix with shape (n_samples, n_responses). Assumed dense.
|
|
83
|
+
method : Literal["pearson", "spearman"], optional
|
|
84
|
+
Correlation method to use, by default "pearson".
|
|
85
|
+
significance : bool
|
|
86
|
+
If True, also returns a matrix of p-values corresponding to the
|
|
87
|
+
correlation matrix under the assumption of a t-distribution with
|
|
88
|
+
degrees of freedom equal to the number of samples minus two.
|
|
89
|
+
slope : bool
|
|
90
|
+
If True, also returns a matrix of slopes corresponding to the
|
|
91
|
+
correlation matrix.
|
|
92
|
+
batch_size : int, optional
|
|
93
|
+
Number of X columns to process per batch when X is sparse, by default 1000.
|
|
94
|
+
progress_bar : bool, optional
|
|
95
|
+
Whether to display a progress bar for batch processing when X is sparse,
|
|
96
|
+
by default True.
|
|
97
|
+
weights : np.ndarray | None, optional
|
|
98
|
+
Per-sample weights of shape (n_samples,). When provided, compute
|
|
99
|
+
weighted correlation. Typically used with ``binomial=True`` where weights
|
|
100
|
+
are the total counts per cell, by default None.
|
|
101
|
+
|
|
102
|
+
Returns
|
|
103
|
+
-------
|
|
104
|
+
dict[str, np.ndarray]
|
|
105
|
+
Dictionary with correlation matrix, p-values, and slopes.
|
|
106
|
+
"""
|
|
107
|
+
if isinstance(X, np.ndarray) and X.ndim == 1:
|
|
108
|
+
X = X[:, None].copy()
|
|
109
|
+
if Y.ndim == 1:
|
|
110
|
+
Y = Y[:, None].copy()
|
|
111
|
+
|
|
112
|
+
if sp.issparse(Y):
|
|
113
|
+
logg.warning("Sparse input for the response matrix Y isn't supported, converting to dense for correlation calculation.")
|
|
114
|
+
Y = Y.toarray()
|
|
115
|
+
|
|
116
|
+
if X.shape[0] != Y.shape[0]:
|
|
117
|
+
raise ValueError("X and Y must have the same number of samples.")
|
|
118
|
+
|
|
119
|
+
if weights is not None:
|
|
120
|
+
weights = np.asarray(weights, dtype=float).ravel()
|
|
121
|
+
if weights.shape[0] != X.shape[0]:
|
|
122
|
+
raise ValueError("weights must have the same length as the number of samples.")
|
|
123
|
+
|
|
124
|
+
if method == "spearman":
|
|
125
|
+
Y_data = stats.rankdata(Y, axis=0)
|
|
126
|
+
else:
|
|
127
|
+
Y_data = Y
|
|
128
|
+
|
|
129
|
+
# ---- weighted vs unweighted centering / norms ----
|
|
130
|
+
if weights is not None:
|
|
131
|
+
w = weights[:, None] # (n, 1)
|
|
132
|
+
W = w.sum()
|
|
133
|
+
Y_mean = (w * Y_data).sum(axis=0) / W
|
|
134
|
+
Y_centered = Y_data - Y_mean
|
|
135
|
+
norm_Y = np.sqrt((w * Y_centered ** 2).sum(axis=0))
|
|
136
|
+
else:
|
|
137
|
+
Y_centered = Y_data - np.mean(Y_data, axis=0)
|
|
138
|
+
norm_Y = np.linalg.norm(Y_centered, axis=0)
|
|
139
|
+
|
|
140
|
+
n_features = X.shape[1]
|
|
141
|
+
n_responses = Y.shape[1]
|
|
142
|
+
R = np.full((n_features, n_responses), np.nan)
|
|
143
|
+
|
|
144
|
+
if not sp.issparse(X):
|
|
145
|
+
X_data = X
|
|
146
|
+
if method == "spearman":
|
|
147
|
+
X_data = stats.rankdata(X_data, axis=0)
|
|
148
|
+
|
|
149
|
+
if weights is not None:
|
|
150
|
+
X_mean = (w * X_data).sum(axis=0) / W
|
|
151
|
+
X_centered = X_data - X_mean
|
|
152
|
+
norm_X = np.sqrt((w * X_centered ** 2).sum(axis=0))
|
|
153
|
+
else:
|
|
154
|
+
X_centered = X_data - np.mean(X_data, axis=0)
|
|
155
|
+
norm_X = np.linalg.norm(X_centered, axis=0)
|
|
156
|
+
|
|
157
|
+
denominator = norm_X[:, None] * norm_Y
|
|
158
|
+
|
|
159
|
+
if weights is not None:
|
|
160
|
+
numerator = (w * X_centered).T @ Y_centered
|
|
161
|
+
else:
|
|
162
|
+
numerator = X_centered.T @ Y_centered
|
|
163
|
+
np.divide(numerator, denominator, where=(denominator != 0), out=R)
|
|
164
|
+
|
|
165
|
+
results = {"r": R}
|
|
166
|
+
|
|
167
|
+
if significance:
|
|
168
|
+
n = X.shape[0]
|
|
169
|
+
if weights is not None:
|
|
170
|
+
n_eff = W ** 2 / (weights ** 2).sum()
|
|
171
|
+
else:
|
|
172
|
+
n_eff = float(n)
|
|
173
|
+
if n_eff <= 2:
|
|
174
|
+
P = np.full_like(R, np.nan)
|
|
175
|
+
results["pvalue"] = P
|
|
176
|
+
else:
|
|
177
|
+
df = n_eff - 2
|
|
178
|
+
T = np.nan_to_num(R) * np.sqrt(df / (1 - np.nan_to_num(R) ** 2))
|
|
179
|
+
P = 2 * stats.t.sf(np.abs(T), df)
|
|
180
|
+
P[np.isnan(R)] = np.nan
|
|
181
|
+
results["pvalue"] = P
|
|
182
|
+
|
|
183
|
+
if slope:
|
|
184
|
+
if weights is not None:
|
|
185
|
+
std_X = np.sqrt((w * (X_data - X_mean) ** 2).sum(axis=0) / W)
|
|
186
|
+
std_Y = np.sqrt((w * (Y_data - Y_mean) ** 2).sum(axis=0) / W)
|
|
187
|
+
else:
|
|
188
|
+
std_X = np.std(X_data, axis=0)
|
|
189
|
+
std_Y = np.std(Y_data, axis=0)
|
|
190
|
+
|
|
191
|
+
std_ratio = np.full_like(R, np.nan)
|
|
192
|
+
np.divide(std_Y, std_X[:, None], out=std_ratio, where=(std_X[:, None] != 0))
|
|
193
|
+
|
|
194
|
+
S = R * std_ratio
|
|
195
|
+
results["slope"] = S
|
|
196
|
+
|
|
197
|
+
return results
|
|
198
|
+
else:
|
|
199
|
+
X_sparse = X.tocsc()
|
|
200
|
+
|
|
201
|
+
col_indices = range(0, n_features, max(1, int(batch_size)))
|
|
202
|
+
|
|
203
|
+
if sc.settings.verbosity.value >= 2:
|
|
204
|
+
prefix = " "
|
|
205
|
+
else:
|
|
206
|
+
prefix = ""
|
|
207
|
+
cm = tqdm(
|
|
208
|
+
col_indices,
|
|
209
|
+
desc=prefix + "correlation batches",
|
|
210
|
+
leave=True,
|
|
211
|
+
file=sys.stdout,
|
|
212
|
+
) if progress_bar else nullcontext(col_indices)
|
|
213
|
+
|
|
214
|
+
if slope:
|
|
215
|
+
if weights is not None:
|
|
216
|
+
std_Y = np.sqrt((w * (Y_data - Y_mean) ** 2).sum(axis=0) / W)
|
|
217
|
+
else:
|
|
218
|
+
std_Y = np.std(Y_data, axis=0)
|
|
219
|
+
S = np.full_like(R, np.nan)
|
|
220
|
+
|
|
221
|
+
if significance:
|
|
222
|
+
n = X.shape[0]
|
|
223
|
+
P = np.full_like(R, np.nan)
|
|
224
|
+
if weights is not None:
|
|
225
|
+
n_eff = W ** 2 / (weights ** 2).sum()
|
|
226
|
+
else:
|
|
227
|
+
n_eff = float(n)
|
|
228
|
+
df = n_eff - 2
|
|
229
|
+
|
|
230
|
+
with cm as iterator:
|
|
231
|
+
for start in iterator:
|
|
232
|
+
end = min(start + batch_size, n_features)
|
|
233
|
+
cols = np.arange(start, end)
|
|
234
|
+
|
|
235
|
+
X_batch = X_sparse[:, cols].toarray()
|
|
236
|
+
if method == "spearman":
|
|
237
|
+
X_batch = stats.rankdata(X_batch, axis=0)
|
|
238
|
+
|
|
239
|
+
if weights is not None:
|
|
240
|
+
X_mean_b = (w * X_batch).sum(axis=0) / W
|
|
241
|
+
X_centered_b = X_batch - X_mean_b
|
|
242
|
+
norm_X_b = np.sqrt((w * X_centered_b ** 2).sum(axis=0))
|
|
243
|
+
else:
|
|
244
|
+
X_centered_b = X_batch - np.mean(X_batch, axis=0)
|
|
245
|
+
norm_X_b = np.linalg.norm(X_centered_b, axis=0)
|
|
246
|
+
|
|
247
|
+
denominator_b = norm_X_b[:, None] * norm_Y
|
|
248
|
+
|
|
249
|
+
if weights is not None:
|
|
250
|
+
numerator_b = (w * X_centered_b).T @ Y_centered
|
|
251
|
+
else:
|
|
252
|
+
numerator_b = X_centered_b.T @ Y_centered
|
|
253
|
+
R_b = np.full(denominator_b.shape, np.nan)
|
|
254
|
+
np.divide(numerator_b, denominator_b, where=(denominator_b != 0), out=R_b)
|
|
255
|
+
R[start:end, :] = R_b
|
|
256
|
+
|
|
257
|
+
if slope:
|
|
258
|
+
if weights is not None:
|
|
259
|
+
std_X_b = np.sqrt((w * (X_batch - X_mean_b) ** 2).sum(axis=0) / W)
|
|
260
|
+
else:
|
|
261
|
+
std_X_b = np.std(X_batch, axis=0)
|
|
262
|
+
std_ratio_b = np.full_like(R_b, np.nan)
|
|
263
|
+
np.divide(std_Y, std_X_b[:, None], out=std_ratio_b, where=(std_X_b[:, None] != 0))
|
|
264
|
+
S_b = R_b * std_ratio_b
|
|
265
|
+
S[start:end, :] = S_b
|
|
266
|
+
|
|
267
|
+
if significance and n_eff > 2:
|
|
268
|
+
T_b = np.nan_to_num(R_b) * np.sqrt(df / (1 - np.nan_to_num(R_b) ** 2))
|
|
269
|
+
P_b = 2 * stats.t.sf(np.abs(T_b), df)
|
|
270
|
+
P_b[np.isnan(R_b)] = np.nan
|
|
271
|
+
P[start:end, :] = P_b
|
|
272
|
+
|
|
273
|
+
results = {"r": R}
|
|
274
|
+
if significance:
|
|
275
|
+
results["pvalue"] = P
|
|
276
|
+
if slope:
|
|
277
|
+
results["slope"] = S
|
|
278
|
+
|
|
279
|
+
return results
|
|
280
|
+
|
|
281
|
+
def _fast_corr_cols(
|
|
282
|
+
X: np.ndarray | sp.spmatrix,
|
|
283
|
+
Y: np.ndarray | sp.spmatrix,
|
|
284
|
+
method: Literal["pearson", "spearman"] = "pearson",
|
|
285
|
+
significance: bool = True,
|
|
286
|
+
slope: bool = True,
|
|
287
|
+
batch_size: int = 1000,
|
|
288
|
+
progress_bar: bool = True,
|
|
289
|
+
weights: np.ndarray | None = None,
|
|
290
|
+
) -> dict[str, np.ndarray]:
|
|
291
|
+
"""
|
|
292
|
+
Calculate correlation statistics between corresponding columns of X and Y
|
|
293
|
+
(1<->1, 2<->2, ...), gracefully handling zero-variance columns by setting
|
|
294
|
+
the result to NaN. Supports sparse inputs similarly to `_fast_corr`.
|
|
295
|
+
|
|
296
|
+
Parameters
|
|
297
|
+
----------
|
|
298
|
+
X : np.ndarray | scipy.sparse.spmatrix
|
|
299
|
+
Dataset with shape (n_samples, n_features).
|
|
300
|
+
Y : np.ndarray | scipy.sparse.spmatrix
|
|
301
|
+
Another dataset with the same shape as X.
|
|
302
|
+
method : Literal["pearson", "spearman"], optional
|
|
303
|
+
Statistic to compute, by default "pearson".
|
|
304
|
+
significance : bool, optional
|
|
305
|
+
If True, also returns p-values (for Pearson/Spearman via t-test,
|
|
306
|
+
for GAM via LRT/Wald), by default True.
|
|
307
|
+
slope : bool, optional
|
|
308
|
+
If True, also returns slopes (for Pearson/Spearman as std ratio times r;
|
|
309
|
+
for GAM the endpoints change), by default True.
|
|
310
|
+
batch_size : int, optional
|
|
311
|
+
Number of columns to process per batch when X (or Y) is sparse,
|
|
312
|
+
by default 1000.
|
|
313
|
+
progress_bar : bool, optional
|
|
314
|
+
Whether to display a progress bar for batch processing when inputs are
|
|
315
|
+
sparse or when method=="gam", by default True.
|
|
316
|
+
weights : np.ndarray | None, optional
|
|
317
|
+
Per-sample weights of shape (n_samples,). When provided, compute
|
|
318
|
+
weighted correlation, by default None.
|
|
319
|
+
|
|
320
|
+
Returns
|
|
321
|
+
-------
|
|
322
|
+
dict[str, np.ndarray]
|
|
323
|
+
Dictionary with statistics. For Pearson/Spearman: keys "r",
|
|
324
|
+
optionally "pvalue" and "slope". For GAM: keys "r2",
|
|
325
|
+
"amplitude", optionally "pvalue" and "slope".
|
|
326
|
+
"""
|
|
327
|
+
if isinstance(X, np.ndarray) and X.ndim == 1:
|
|
328
|
+
X = X[:, None].copy()
|
|
329
|
+
if isinstance(Y, np.ndarray) and Y.ndim == 1:
|
|
330
|
+
Y = Y[:, None].copy()
|
|
331
|
+
|
|
332
|
+
if X.shape != Y.shape:
|
|
333
|
+
raise ValueError("X and Y must have the same shape for column-wise correlation.")
|
|
334
|
+
|
|
335
|
+
n_samples, n_cols = X.shape
|
|
336
|
+
|
|
337
|
+
if weights is not None:
|
|
338
|
+
weights = np.asarray(weights, dtype=float).ravel()
|
|
339
|
+
if weights.shape[0] != n_samples:
|
|
340
|
+
raise ValueError("weights must have the same length as the number of samples.")
|
|
341
|
+
|
|
342
|
+
if method in ["pearson", "spearman"]:
|
|
343
|
+
results = {}
|
|
344
|
+
|
|
345
|
+
if (not sp.issparse(X)) and (not sp.issparse(Y)):
|
|
346
|
+
X_data = X
|
|
347
|
+
Y_data = Y
|
|
348
|
+
if method == "spearman":
|
|
349
|
+
X_data = stats.rankdata(X_data, axis=0)
|
|
350
|
+
Y_data = stats.rankdata(Y_data, axis=0)
|
|
351
|
+
|
|
352
|
+
if weights is not None:
|
|
353
|
+
w = weights[:, None]
|
|
354
|
+
W = w.sum()
|
|
355
|
+
X_mean = (w * X_data).sum(axis=0) / W
|
|
356
|
+
Y_mean = (w * Y_data).sum(axis=0) / W
|
|
357
|
+
X_centered = X_data - X_mean
|
|
358
|
+
Y_centered = Y_data - Y_mean
|
|
359
|
+
norm_X = np.sqrt((w * X_centered ** 2).sum(axis=0))
|
|
360
|
+
norm_Y = np.sqrt((w * Y_centered ** 2).sum(axis=0))
|
|
361
|
+
else:
|
|
362
|
+
X_centered = X_data - np.mean(X_data, axis=0)
|
|
363
|
+
Y_centered = Y_data - np.mean(Y_data, axis=0)
|
|
364
|
+
norm_X = np.linalg.norm(X_centered, axis=0)
|
|
365
|
+
norm_Y = np.linalg.norm(Y_centered, axis=0)
|
|
366
|
+
denom = norm_X * norm_Y
|
|
367
|
+
|
|
368
|
+
if weights is not None:
|
|
369
|
+
numer = (w * X_centered * Y_centered).sum(axis=0)
|
|
370
|
+
else:
|
|
371
|
+
numer = np.sum(X_centered * Y_centered, axis=0)
|
|
372
|
+
R = np.full(n_cols, np.nan)
|
|
373
|
+
np.divide(numer, denom, where=(denom != 0), out=R)
|
|
374
|
+
|
|
375
|
+
results["r"] = R
|
|
376
|
+
|
|
377
|
+
if significance:
|
|
378
|
+
n = n_samples
|
|
379
|
+
if weights is not None:
|
|
380
|
+
n_eff = W ** 2 / (weights ** 2).sum()
|
|
381
|
+
else:
|
|
382
|
+
n_eff = float(n)
|
|
383
|
+
if n_eff <= 2:
|
|
384
|
+
P = np.full_like(R, np.nan)
|
|
385
|
+
else:
|
|
386
|
+
df = n_eff - 2
|
|
387
|
+
T = np.nan_to_num(R) * np.sqrt(df / (1 - np.nan_to_num(R) ** 2))
|
|
388
|
+
P = 2 * stats.t.sf(np.abs(T), df)
|
|
389
|
+
P[np.isnan(R)] = np.nan
|
|
390
|
+
results["pvalue"] = P
|
|
391
|
+
|
|
392
|
+
if slope:
|
|
393
|
+
if weights is not None:
|
|
394
|
+
std_X = np.sqrt((w * (X_data - X_mean) ** 2).sum(axis=0) / W)
|
|
395
|
+
std_Y = np.sqrt((w * (Y_data - Y_mean) ** 2).sum(axis=0) / W)
|
|
396
|
+
else:
|
|
397
|
+
std_X = np.std(X_data, axis=0)
|
|
398
|
+
std_Y = np.std(Y_data, axis=0)
|
|
399
|
+
std_ratio = np.full_like(R, np.nan)
|
|
400
|
+
np.divide(std_Y, std_X, out=std_ratio, where=(std_X != 0))
|
|
401
|
+
S = R * std_ratio
|
|
402
|
+
results["slope"] = S
|
|
403
|
+
|
|
404
|
+
return results
|
|
405
|
+
|
|
406
|
+
X_sparse = X.tocsc() if sp.issparse(X) else None
|
|
407
|
+
Y_sparse = Y.tocsc() if sp.issparse(Y) else None
|
|
408
|
+
|
|
409
|
+
R = np.full(n_cols, np.nan)
|
|
410
|
+
if significance:
|
|
411
|
+
n = n_samples
|
|
412
|
+
P = np.full(n_cols, np.nan)
|
|
413
|
+
if weights is not None:
|
|
414
|
+
W = weights.sum()
|
|
415
|
+
n_eff = W ** 2 / (weights ** 2).sum()
|
|
416
|
+
else:
|
|
417
|
+
n_eff = float(n)
|
|
418
|
+
df = n_eff - 2
|
|
419
|
+
if slope:
|
|
420
|
+
S = np.full(n_cols, np.nan)
|
|
421
|
+
|
|
422
|
+
col_indices = range(0, n_cols, max(1, int(batch_size)))
|
|
423
|
+
if sc.settings.verbosity.value >= 2:
|
|
424
|
+
prefix = " "
|
|
425
|
+
else:
|
|
426
|
+
prefix = ""
|
|
427
|
+
cm = tqdm(
|
|
428
|
+
total=len(col_indices),
|
|
429
|
+
desc=prefix + "correlation batches",
|
|
430
|
+
leave=True,
|
|
431
|
+
file=sys.stdout,
|
|
432
|
+
) if progress_bar else nullcontext()
|
|
433
|
+
with cm as pbar:
|
|
434
|
+
for start in col_indices:
|
|
435
|
+
end = min(start + batch_size, n_cols)
|
|
436
|
+
cols = np.arange(start, end)
|
|
437
|
+
|
|
438
|
+
X_batch = (X_sparse[:, cols].toarray() if X_sparse is not None else X[:, cols])
|
|
439
|
+
Y_batch = (Y_sparse[:, cols].toarray() if Y_sparse is not None else Y[:, cols])
|
|
440
|
+
|
|
441
|
+
if method == "spearman":
|
|
442
|
+
X_batch = stats.rankdata(X_batch, axis=0)
|
|
443
|
+
Y_batch = stats.rankdata(Y_batch, axis=0)
|
|
444
|
+
|
|
445
|
+
if weights is not None:
|
|
446
|
+
w = weights[:, None]
|
|
447
|
+
W_val = w.sum()
|
|
448
|
+
X_mean_b = (w * X_batch).sum(axis=0) / W_val
|
|
449
|
+
Y_mean_b = (w * Y_batch).sum(axis=0) / W_val
|
|
450
|
+
X_centered_b = X_batch - X_mean_b
|
|
451
|
+
Y_centered_b = Y_batch - Y_mean_b
|
|
452
|
+
norm_X_b = np.sqrt((w * X_centered_b ** 2).sum(axis=0))
|
|
453
|
+
norm_Y_b = np.sqrt((w * Y_centered_b ** 2).sum(axis=0))
|
|
454
|
+
else:
|
|
455
|
+
X_centered_b = X_batch - np.mean(X_batch, axis=0)
|
|
456
|
+
Y_centered_b = Y_batch - np.mean(Y_batch, axis=0)
|
|
457
|
+
norm_X_b = np.linalg.norm(X_centered_b, axis=0)
|
|
458
|
+
norm_Y_b = np.linalg.norm(Y_centered_b, axis=0)
|
|
459
|
+
|
|
460
|
+
denom_b = norm_X_b * norm_Y_b
|
|
461
|
+
if weights is not None:
|
|
462
|
+
numer_b = (w * X_centered_b * Y_centered_b).sum(axis=0)
|
|
463
|
+
else:
|
|
464
|
+
numer_b = np.sum(X_centered_b * Y_centered_b, axis=0)
|
|
465
|
+
R_b = np.full(len(cols), np.nan)
|
|
466
|
+
np.divide(numer_b, denom_b, where=(denom_b != 0), out=R_b)
|
|
467
|
+
R[start:end] = R_b
|
|
468
|
+
|
|
469
|
+
if slope:
|
|
470
|
+
if weights is not None:
|
|
471
|
+
std_X_b = np.sqrt((w * (X_batch - X_mean_b) ** 2).sum(axis=0) / W_val)
|
|
472
|
+
std_Y_b = np.sqrt((w * (Y_batch - Y_mean_b) ** 2).sum(axis=0) / W_val)
|
|
473
|
+
else:
|
|
474
|
+
std_X_b = np.std(X_batch, axis=0)
|
|
475
|
+
std_Y_b = np.std(Y_batch, axis=0)
|
|
476
|
+
std_ratio_b = np.full(len(cols), np.nan)
|
|
477
|
+
np.divide(std_Y_b, std_X_b, out=std_ratio_b, where=(std_X_b != 0))
|
|
478
|
+
S_b = R_b * std_ratio_b
|
|
479
|
+
S[start:end] = S_b
|
|
480
|
+
|
|
481
|
+
if significance and n_eff > 2:
|
|
482
|
+
T_b = np.nan_to_num(R_b) * np.sqrt(df / (1 - np.nan_to_num(R_b) ** 2))
|
|
483
|
+
P_b = 2 * stats.t.sf(np.abs(T_b), df)
|
|
484
|
+
P_b[np.isnan(R_b)] = np.nan
|
|
485
|
+
P[start:end] = P_b
|
|
486
|
+
if progress_bar:
|
|
487
|
+
pbar.update(1)
|
|
488
|
+
|
|
489
|
+
results = {"r": R}
|
|
490
|
+
if significance:
|
|
491
|
+
results["pvalue"] = P
|
|
492
|
+
if slope:
|
|
493
|
+
results["slope"] = S
|
|
494
|
+
return results
|
|
495
|
+
|
|
496
|
+
else:
|
|
497
|
+
raise ValueError("method must be 'pearson' or 'spearman'")
|
|
498
|
+
|
|
499
|
+
def _calculate_gam_stats(
|
|
500
|
+
x: np.array,
|
|
501
|
+
y: np.array,
|
|
502
|
+
spline_df: int = 5,
|
|
503
|
+
spline_function: str = "bs",
|
|
504
|
+
p_method: Literal["lrt", "wald"] = "lrt",
|
|
505
|
+
random_state: int | None = 42,
|
|
506
|
+
binomial: bool = False,
|
|
507
|
+
totals: np.array | None = None,
|
|
508
|
+
**spline_kwargs,
|
|
509
|
+
) -> dict[str, np.ndarray]:
|
|
510
|
+
"""
|
|
511
|
+
Calculate GAM statistics for a single pair of columns in X and Y.
|
|
512
|
+
|
|
513
|
+
Parameters
|
|
514
|
+
----------
|
|
515
|
+
x : np.array
|
|
516
|
+
Input feature vector.
|
|
517
|
+
y : np.array
|
|
518
|
+
Response vector. When ``binomial=True`` this should be proportions
|
|
519
|
+
(successes / totals) in the range [0, 1].
|
|
520
|
+
spline_df : int, optional
|
|
521
|
+
Degrees of freedom for the spline, by default 5.
|
|
522
|
+
spline_function : str, optional
|
|
523
|
+
Spline function to use, by default "bs".
|
|
524
|
+
p_method : Literal["lrt", "wald"], optional
|
|
525
|
+
Method to use for p-value calculation, by default "lrt".
|
|
526
|
+
random_state : int | None, optional
|
|
527
|
+
Seed for NumPy RNG to ensure reproducible behavior in stochastic routines, by default 42.
|
|
528
|
+
binomial : bool, optional
|
|
529
|
+
If True, use a Binomial GLM instead of Gaussian. ``y`` should be
|
|
530
|
+
proportions and ``totals`` the per-observation trial counts, by default False.
|
|
531
|
+
totals : np.array | None, optional
|
|
532
|
+
Per-observation trial counts (used only when ``binomial=True``), by default None.
|
|
533
|
+
**spline_kwargs : dict, optional
|
|
534
|
+
Additional keyword arguments to pass to the spline formula.
|
|
535
|
+
|
|
536
|
+
Returns
|
|
537
|
+
-------
|
|
538
|
+
dict[str, np.ndarray]
|
|
539
|
+
Dictionary with GAM statistics.
|
|
540
|
+
"""
|
|
541
|
+
if random_state is not None:
|
|
542
|
+
try:
|
|
543
|
+
np.random.seed(int(random_state))
|
|
544
|
+
except Exception:
|
|
545
|
+
pass
|
|
546
|
+
|
|
547
|
+
if np.allclose(x, x[0]):
|
|
548
|
+
results = {"r2": np.nan, "amplitude": np.nan, "slope": np.nan, "pvalue": np.nan}
|
|
549
|
+
return results
|
|
550
|
+
|
|
551
|
+
if len(x) != len(y):
|
|
552
|
+
logg.warning("x and y must have the same number of samples. Return NaN.")
|
|
553
|
+
results = {"r2": np.nan, "amplitude": np.nan, "slope": np.nan, "pvalue": np.nan}
|
|
554
|
+
return results
|
|
555
|
+
|
|
556
|
+
try:
|
|
557
|
+
results = {}
|
|
558
|
+
|
|
559
|
+
kwargs_str = ""
|
|
560
|
+
for k, v in spline_kwargs.items():
|
|
561
|
+
kwargs_str += f", {k}={v}"
|
|
562
|
+
|
|
563
|
+
if binomial:
|
|
564
|
+
if totals is None:
|
|
565
|
+
raise ValueError("totals must be provided when binomial=True")
|
|
566
|
+
family = sm.families.Binomial()
|
|
567
|
+
gam_fit = smf.glm(
|
|
568
|
+
formula=f"y ~ {spline_function}(x, df={spline_df}{kwargs_str})",
|
|
569
|
+
data={"x": x, "y": y},
|
|
570
|
+
family=family,
|
|
571
|
+
freq_weights=totals.astype(float),
|
|
572
|
+
).fit(random_state=random_state)
|
|
573
|
+
else:
|
|
574
|
+
family = sm.families.Gaussian()
|
|
575
|
+
gam_fit = smf.glm(
|
|
576
|
+
formula=f"y ~ {spline_function}(x, df={spline_df}{kwargs_str})",
|
|
577
|
+
data={"x": x, "y": y},
|
|
578
|
+
family=family,
|
|
579
|
+
).fit(random_state=random_state)
|
|
580
|
+
results["r2"] = gam_fit.pseudo_rsquared()
|
|
581
|
+
|
|
582
|
+
fit_prediction = gam_fit.predict({"x": x}).values
|
|
583
|
+
effect_size = fit_prediction.max() - fit_prediction.min()
|
|
584
|
+
results["amplitude"] = effect_size
|
|
585
|
+
|
|
586
|
+
endpoints_change = gam_fit.predict({"x": [x.max(), x.min()]}).values
|
|
587
|
+
endpoints_change = endpoints_change[0] - endpoints_change[1]
|
|
588
|
+
results["slope"] = endpoints_change
|
|
589
|
+
|
|
590
|
+
if p_method not in ["lrt", "wald"]:
|
|
591
|
+
logg.warning(f"p_method must be 'lrt' or 'wald', got {p_method}. Default to 'lrt'.")
|
|
592
|
+
p_method = "lrt"
|
|
593
|
+
|
|
594
|
+
if p_method == "lrt":
|
|
595
|
+
if binomial:
|
|
596
|
+
const_fit = smf.glm(
|
|
597
|
+
formula="y ~ 1",
|
|
598
|
+
data={"y": y},
|
|
599
|
+
family=family,
|
|
600
|
+
freq_weights=totals.astype(float),
|
|
601
|
+
).fit(random_state=random_state)
|
|
602
|
+
else:
|
|
603
|
+
x_const = sm.add_constant(np.zeros_like(x))
|
|
604
|
+
const_fit = sm.OLS(y, x_const).fit(random_state=random_state)
|
|
605
|
+
lrt = 2 * (gam_fit.llf - const_fit.llf)
|
|
606
|
+
df_d = gam_fit.df_model - const_fit.df_model
|
|
607
|
+
p = stats.chi2.sf(lrt, df_d)
|
|
608
|
+
results["pvalue"] = p
|
|
609
|
+
else:
|
|
610
|
+
wald_stat = gam_fit.wald_test_terms().summary_frame()
|
|
611
|
+
wald_p = wald_stat.iloc[wald_stat.index != "Intercept"]["P>chi2"].values[0]
|
|
612
|
+
results["pvalue"] = wald_p
|
|
613
|
+
|
|
614
|
+
return results
|
|
615
|
+
except Exception as e:
|
|
616
|
+
logg.warning(f"Error fitting GAM: {e}. Return NaN.")
|
|
617
|
+
results = {"r2": np.nan, "amplitude": np.nan, "slope": np.nan, "pvalue": np.nan}
|
|
618
|
+
return results
|
|
619
|
+
|
|
620
|
+
def _gam(
|
|
621
|
+
X: np.ndarray,
|
|
622
|
+
Y: np.ndarray,
|
|
623
|
+
spline_df: int = 5,
|
|
624
|
+
spline_degree: int = 3,
|
|
625
|
+
n_jobs: int = -1,
|
|
626
|
+
p_method: Literal["lrt", "wald"] = "lrt",
|
|
627
|
+
progress_bar: bool = True,
|
|
628
|
+
random_state: int | None = 42,
|
|
629
|
+
binomial: bool = False,
|
|
630
|
+
totals: np.ndarray | None = None,
|
|
631
|
+
**spline_kwargs,
|
|
632
|
+
) -> dict[str, np.ndarray]:
|
|
633
|
+
"""
|
|
634
|
+
Calculate GAM (generalized additive model) statistics for each pair of
|
|
635
|
+
columns in X and Y.
|
|
636
|
+
|
|
637
|
+
Parameters
|
|
638
|
+
----------
|
|
639
|
+
X : np.ndarray
|
|
640
|
+
Dataset with shape (n_samples, n_features).
|
|
641
|
+
Y : np.ndarray
|
|
642
|
+
Response matrix with shape (n_samples, n_responses).
|
|
643
|
+
When ``binomial=True`` these should be proportions.
|
|
644
|
+
spline_df : int, optional
|
|
645
|
+
Degrees of freedom for the spline, by default 5.
|
|
646
|
+
spline_degree : int, optional
|
|
647
|
+
Degree of the spline, by default 3.
|
|
648
|
+
n_jobs : int, optional
|
|
649
|
+
Number of parallel jobs to run, by default -1.
|
|
650
|
+
p_method : Literal["lrt", "wald"], optional
|
|
651
|
+
Method to use for p-value calculation, by default "lrt".
|
|
652
|
+
progress_bar : bool, optional
|
|
653
|
+
If True, display a progress bar, by default True.
|
|
654
|
+
random_state : int | None, optional
|
|
655
|
+
Base seed used to derive a deterministic per-pair seed, ensuring reproducibility, by default 42.
|
|
656
|
+
binomial : bool, optional
|
|
657
|
+
If True, use a Binomial GLM instead of Gaussian, by default False.
|
|
658
|
+
totals : np.ndarray | None, optional
|
|
659
|
+
Per-observation trial counts (used only when ``binomial=True``), by default None.
|
|
660
|
+
**spline_kwargs : dict, optional
|
|
661
|
+
Additional keyword arguments to pass to the spline formula.
|
|
662
|
+
|
|
663
|
+
Returns
|
|
664
|
+
-------
|
|
665
|
+
dict[str, np.ndarray]
|
|
666
|
+
Dictionary with GAM statistics for each pair of columns in X and Y.
|
|
667
|
+
"""
|
|
668
|
+
from itertools import product
|
|
669
|
+
|
|
670
|
+
if sp.issparse(X):
|
|
671
|
+
X_sparse = X.tocsc()
|
|
672
|
+
|
|
673
|
+
# Consistency with fast_corr function
|
|
674
|
+
if sp.issparse(Y):
|
|
675
|
+
logg.warning("Sparse input for the response matrix Y isn't supported, converting to dense for correlation calculation.")
|
|
676
|
+
Y = Y.toarray()
|
|
677
|
+
|
|
678
|
+
X_col_n = X.shape[1]
|
|
679
|
+
Y_col_n = Y.shape[1]
|
|
680
|
+
|
|
681
|
+
tasks = product(np.arange(X_col_n), np.arange(Y_col_n))
|
|
682
|
+
|
|
683
|
+
if sc.settings.verbosity.value >= 2:
|
|
684
|
+
prefix = " "
|
|
685
|
+
progress_bar = True
|
|
686
|
+
else:
|
|
687
|
+
prefix = ""
|
|
688
|
+
|
|
689
|
+
if progress_bar:
|
|
690
|
+
tasks = tqdm(
|
|
691
|
+
tasks,
|
|
692
|
+
desc=prefix + "fitting GAMs",
|
|
693
|
+
total=(X_col_n * Y_col_n),
|
|
694
|
+
file=sys.stdout,
|
|
695
|
+
)
|
|
696
|
+
|
|
697
|
+
if not sp.issparse(X):
|
|
698
|
+
results = Parallel(n_jobs=n_jobs)(
|
|
699
|
+
delayed(_calculate_gam_stats)(
|
|
700
|
+
X[:, x],
|
|
701
|
+
Y[:, y],
|
|
702
|
+
spline_df=spline_df,
|
|
703
|
+
p_method=p_method,
|
|
704
|
+
random_state=(None if random_state is None else int(random_state) + x * Y_col_n + y),
|
|
705
|
+
degree=spline_degree,
|
|
706
|
+
binomial=binomial,
|
|
707
|
+
totals=totals,
|
|
708
|
+
**spline_kwargs,
|
|
709
|
+
) for x, y in tasks
|
|
710
|
+
)
|
|
711
|
+
else:
|
|
712
|
+
results = Parallel(n_jobs=n_jobs)(
|
|
713
|
+
delayed(_calculate_gam_stats)(
|
|
714
|
+
X_sparse[:, x].toarray().flatten(),
|
|
715
|
+
Y[:, y],
|
|
716
|
+
spline_df=spline_df,
|
|
717
|
+
p_method=p_method,
|
|
718
|
+
random_state=(None if random_state is None else int(random_state) + x * Y_col_n + y),
|
|
719
|
+
degree=spline_degree,
|
|
720
|
+
binomial=binomial,
|
|
721
|
+
totals=totals,
|
|
722
|
+
**spline_kwargs,
|
|
723
|
+
) for x, y in tasks
|
|
724
|
+
)
|
|
725
|
+
|
|
726
|
+
results = {
|
|
727
|
+
key: np.array([r[key] for r in results]).reshape(X_col_n, Y_col_n)
|
|
728
|
+
for key in results[0]
|
|
729
|
+
}
|
|
730
|
+
return results
|
|
731
|
+
|
|
732
|
+
def _adjust_pvalues(
|
|
733
|
+
pvalues: np.ndarray,
|
|
734
|
+
method: str = "fdr_bh",
|
|
735
|
+
) -> np.ndarray:
|
|
736
|
+
"""
|
|
737
|
+
Adjust p-values using the specified method.
|
|
738
|
+
NaN values in `pvalues` will be left as NaN in the output.
|
|
739
|
+
|
|
740
|
+
Parameters
|
|
741
|
+
----------
|
|
742
|
+
pvalues : np.ndarray
|
|
743
|
+
Array of p-values to adjust. Can be any-dimensional.
|
|
744
|
+
method : str, optional
|
|
745
|
+
Method for adjustment, will be passed to `statsmodels.stats.multitest.multipletests`,
|
|
746
|
+
default is "fdr_bh".
|
|
747
|
+
|
|
748
|
+
Returns
|
|
749
|
+
-------
|
|
750
|
+
np.ndarray
|
|
751
|
+
Adjusted p-values. Has the same shape as `pvalues`.
|
|
752
|
+
"""
|
|
753
|
+
from statsmodels.stats.multitest import multipletests
|
|
754
|
+
|
|
755
|
+
dims = pvalues.shape
|
|
756
|
+
pvalues_flat = pvalues.flatten()
|
|
757
|
+
nan_mask = np.isnan(pvalues_flat)
|
|
758
|
+
fdrs_flat = np.full_like(pvalues_flat, np.nan)
|
|
759
|
+
|
|
760
|
+
fdrs_flat[~nan_mask] = multipletests(pvalues_flat[~nan_mask], method=method)[1]
|
|
761
|
+
|
|
762
|
+
return fdrs_flat.reshape(dims)
|
|
763
|
+
|
|
764
|
+
|
|
765
|
+
def graph_associations(
|
|
766
|
+
adata: sc.AnnData,
|
|
767
|
+
layer: None | str = None,
|
|
768
|
+
graph_key: str = "connectivities",
|
|
769
|
+
n_pcs: int = 50,
|
|
770
|
+
adj_method: str = "fdr_bh",
|
|
771
|
+
key_added: str = "X_gPCA",
|
|
772
|
+
) -> None:
|
|
773
|
+
"""
|
|
774
|
+
Computes the supervised principal components analysis (sPCA) of the data
|
|
775
|
+
[10.1016/j.patcog.2010.12.015]. Shortly, it finds the axes that maximize the correlation
|
|
776
|
+
between the gene expression and the graph Laplacian (one can see it as an
|
|
777
|
+
autocorrelation-aware PCA).
|
|
778
|
+
|
|
779
|
+
Parameters
|
|
780
|
+
----------
|
|
781
|
+
adata : sc.AnnData
|
|
782
|
+
AnnData object containing the data.
|
|
783
|
+
layer : None | str
|
|
784
|
+
Layer to use for the analysis.
|
|
785
|
+
graph_key : str
|
|
786
|
+
Key to use for the graph.
|
|
787
|
+
n_pcs : int
|
|
788
|
+
Number of principal components to compute.
|
|
789
|
+
adj_method : str
|
|
790
|
+
Method for adjusting p-values.
|
|
791
|
+
key_added : str
|
|
792
|
+
Key to use for adding the results to the AnnData object.
|
|
793
|
+
|
|
794
|
+
Returns
|
|
795
|
+
-------
|
|
796
|
+
None
|
|
797
|
+
"""
|
|
798
|
+
from scipy.sparse.linalg import eigsh
|
|
799
|
+
import scipy.stats as stats
|
|
800
|
+
|
|
801
|
+
start = logg.info(f"computing graph associations")
|
|
802
|
+
|
|
803
|
+
if layer is None:
|
|
804
|
+
layer = "X"
|
|
805
|
+
use_X = True
|
|
806
|
+
else:
|
|
807
|
+
use_X = False
|
|
808
|
+
|
|
809
|
+
if use_X:
|
|
810
|
+
X = adata.X
|
|
811
|
+
else:
|
|
812
|
+
X = adata.layers[layer]
|
|
813
|
+
|
|
814
|
+
if sp.issparse(X):
|
|
815
|
+
logg.warning("provided matrix is sparse, make sure that you scaled data before passing to the function")
|
|
816
|
+
X = X.todense()
|
|
817
|
+
|
|
818
|
+
total_variance = np.sum(X ** 2)
|
|
819
|
+
|
|
820
|
+
spcov = X.T @ adata.obsp[graph_key] @ X
|
|
821
|
+
evals, evecs = eigsh(spcov, k=n_pcs, which="LA")
|
|
822
|
+
evals, evecs = evals[::-1], evecs[:, ::-1]
|
|
823
|
+
|
|
824
|
+
adata.varm["gPCs"] = evecs
|
|
825
|
+
adata.obsm[key_added] = X @ evecs
|
|
826
|
+
adata.uns["gpca"] = {
|
|
827
|
+
"evals": evals,
|
|
828
|
+
"varexpl": evals / total_variance,
|
|
829
|
+
"graph_key": graph_key,
|
|
830
|
+
"n_pcs": n_pcs,
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
# Moran's I
|
|
834
|
+
numerators = np.diag(spcov)
|
|
835
|
+
denominators = np.sum(X ** 2, axis=0)
|
|
836
|
+
w = adata.obsp[graph_key]
|
|
837
|
+
W = w.sum()
|
|
838
|
+
N = len(adata)
|
|
839
|
+
|
|
840
|
+
morans_i = (N / W) * (numerators / denominators)
|
|
841
|
+
expected_I = -1.0 / (N - 1)
|
|
842
|
+
S1 = 0.5 * (w + w.T).power(2).sum()
|
|
843
|
+
deg_row = np.array(w.sum(axis=1))
|
|
844
|
+
deg_col = np.array(w.sum(axis=0)).T
|
|
845
|
+
S2 = np.sum((deg_row + deg_col) ** 2)
|
|
846
|
+
W_sq = W ** 2
|
|
847
|
+
var_I = (
|
|
848
|
+
(N * ((N**2 - 3*N + 3) * S1 - N * S2 + 3 * W_sq)) /
|
|
849
|
+
((N - 1) * (N - 2) * (N - 3) * W_sq)
|
|
850
|
+
) - (expected_I ** 2)
|
|
851
|
+
with np.errstate(invalid="ignore"):
|
|
852
|
+
z_scores = (morans_i - expected_I) / np.sqrt(var_I)
|
|
853
|
+
p_values = stats.norm.sf(z_scores)
|
|
854
|
+
|
|
855
|
+
adata.var["morans_i"] = morans_i
|
|
856
|
+
adata.var["morans_i_pval"] = p_values
|
|
857
|
+
adata.var["morans_i_p_adj"] = _adjust_pvalues(p_values, method=adj_method)
|
|
858
|
+
|
|
859
|
+
lines = [
|
|
860
|
+
"added",
|
|
861
|
+
f" .obsm['{key_added}'] {n_pcs} graph-guided principal components",
|
|
862
|
+
f" .varm['gPCs'] matrix of loadings",
|
|
863
|
+
f" .uns['gPCA'] eigenvalues and variance explained",
|
|
864
|
+
f" .var['morans_i'] Moran's I statistics",
|
|
865
|
+
f" .var['morans_i_pval'] p-values",
|
|
866
|
+
f" .var['morans_i_p_adj'] adjusted p-values",
|
|
867
|
+
]
|
|
868
|
+
logg.info(" finished ({time_passed})", deep="\n".join(lines), time=start)
|
|
869
|
+
|
|
870
|
+
def associations(
|
|
871
|
+
adata,
|
|
872
|
+
response_key: str | list[str],
|
|
873
|
+
response_field: Literal["obs", "obsm"] = "obsm",
|
|
874
|
+
response_transform: None | Literal["logit", "log1p", "sqrt", "clr"] = None,
|
|
875
|
+
use_rep: Literal["X", "layers"] | None = None,
|
|
876
|
+
use_raw: bool | None = None,
|
|
877
|
+
layers: list[str] | str | None = None,
|
|
878
|
+
method: Literal["pearson", "spearman", "gam"] = "pearson",
|
|
879
|
+
p_adjustment_method: str = "fdr_bh",
|
|
880
|
+
concat_layers: bool = False,
|
|
881
|
+
random_state: int = 42,
|
|
882
|
+
progress_bar: bool = False,
|
|
883
|
+
mask_key: str | None | Literal[False] = None,
|
|
884
|
+
clr_pseudocount: float = 1e-3,
|
|
885
|
+
binomial: bool = False,
|
|
886
|
+
**gam_kwargs,
|
|
887
|
+
) -> None:
|
|
888
|
+
"""
|
|
889
|
+
Calculate correlation between features in `adata` and a response variable.
|
|
890
|
+
Correlation coefficients, p-values, and False Discovery Rates (FDRs) are computed for each feature.
|
|
891
|
+
Results are stored in `adata.varm` with keys "{X_name}:corr", "{X_name}:pvalue", and "{X_name}:FDR",
|
|
892
|
+
where {X_name} is the name of the feature representation (e.g., "X", "X_raw", or layer names).
|
|
893
|
+
If `p_adjustment_method` is None, no correction is performed.
|
|
894
|
+
|
|
895
|
+
Parameters
|
|
896
|
+
----------
|
|
897
|
+
adata : AnnData
|
|
898
|
+
Annotated data matrix at the cell level.
|
|
899
|
+
response_key : str | list[str] | None
|
|
900
|
+
Key in `adata.obs` or `adata.obsm` containing the response variable. Might be a list
|
|
901
|
+
only if `response_field='obs'`, in this case the response variable is a multi-column DataFrame.
|
|
902
|
+
response_field : Literal["obs", "obsm"], optional
|
|
903
|
+
Whether the response variable is in `adata.obs` or `adata.obsm`, by default "obsm".
|
|
904
|
+
response_transform : None | Literal["logit", "log1p", "sqrt", "clr"], optional
|
|
905
|
+
Transform to apply to the response variable before correlation calculation, by default None.
|
|
906
|
+
use_rep : Literal["X", "layers"] | None, optional
|
|
907
|
+
Whether to use `adata.X` or `adata.layers` for feature representations, if None, uses
|
|
908
|
+
`adata.X` if `layers` is None, otherwise `adata.layers`, by default None.
|
|
909
|
+
use_raw : bool | None, optional
|
|
910
|
+
Whether to use `adata.raw.X` as the feature representation, by default None.
|
|
911
|
+
layers : list[str] | str | None, optional
|
|
912
|
+
Layers in `adata.layers` to use for feature representations if `use_rep='layers'`. If None and
|
|
913
|
+
use_rep="layers", uses all layers, by default None.
|
|
914
|
+
method : Literal["pearson", "spearman", "gam"], optional
|
|
915
|
+
Method for correlation calculation, by default "pearson".
|
|
916
|
+
p_adjustment_method : str, optional
|
|
917
|
+
Method for multiple testing correction, passed to `statsmodels.stats.multitest.multipletests`,
|
|
918
|
+
default is "fdr_bh".
|
|
919
|
+
random_state : int, optional
|
|
920
|
+
Random seed for GAM model training, by default 42.
|
|
921
|
+
progress_bar : bool, optional
|
|
922
|
+
Whether to show a progress bar, by default False.
|
|
923
|
+
mask_key : str | None | Literal[False], optional
|
|
924
|
+
Key in `adata.obs` or `adata.obsm` containing a boolean mask to filter cells, by default None.
|
|
925
|
+
clr_pseudocount : float, optional
|
|
926
|
+
Pseudocount to add to expression values before CLR transformation, by default 1e-3.
|
|
927
|
+
binomial : bool, optional
|
|
928
|
+
If True, treat the response as count data and use binomial modeling. The response
|
|
929
|
+
matrix is interpreted as integer counts; proportions and row totals are computed
|
|
930
|
+
internally. For ``method='gam'``, a Binomial GLM is fitted (with ``freq_weights=totals``).
|
|
931
|
+
For ``method='pearson'`` or ``'spearman'``, weighted correlation is computed on the
|
|
932
|
+
proportions, with weights equal to row totals, by default False.
|
|
933
|
+
**gam_kwargs : dict, optional
|
|
934
|
+
Additional keyword arguments to pass to `gam`.
|
|
935
|
+
|
|
936
|
+
Returns
|
|
937
|
+
-------
|
|
938
|
+
None
|
|
939
|
+
The `adata` object is modified in place with correlation results stored in `adata.varm`.
|
|
940
|
+
"""
|
|
941
|
+
if mask_key is None and "mask_key" in adata.uns.keys():
|
|
942
|
+
mask_key = adata.uns["mask_key"]
|
|
943
|
+
|
|
944
|
+
if use_rep is None:
|
|
945
|
+
if layers is None:
|
|
946
|
+
use_rep = "X"
|
|
947
|
+
else:
|
|
948
|
+
use_rep = "layers"
|
|
949
|
+
if use_raw and use_rep == "layers":
|
|
950
|
+
logg.warning("use_raw=True is not supported when use_rep='layers'. Setting use_raw=False.")
|
|
951
|
+
use_raw = False
|
|
952
|
+
if (use_raw is None) and (use_rep == "X"):
|
|
953
|
+
if adata.raw:
|
|
954
|
+
logg.warning("Using raw data when use_raw is None and adata.raw is not None. Setting use_raw=True.")
|
|
955
|
+
use_raw = True
|
|
956
|
+
if use_rep == "X" and layers:
|
|
957
|
+
logg.warning("Using layers when use_rep='X' is not supported. Setting use_rep='layers'.")
|
|
958
|
+
use_rep = "layers"
|
|
959
|
+
|
|
960
|
+
start_lines = "\n"
|
|
961
|
+
if use_raw:
|
|
962
|
+
Xs = [adata.raw.X]
|
|
963
|
+
masks = [~_nan_mask(adata.raw.X)]
|
|
964
|
+
X_names = ["X_raw"]
|
|
965
|
+
var_names = adata.raw.var_names.values
|
|
966
|
+
start_lines += " using `adata.raw.X` as features\n"
|
|
967
|
+
elif use_rep == "X":
|
|
968
|
+
Xs = [adata.X]
|
|
969
|
+
masks = [~_nan_mask(adata.X)]
|
|
970
|
+
X_names = ["X"]
|
|
971
|
+
var_names = adata.var_names.values
|
|
972
|
+
start_lines += " using `adata.X` as features\n"
|
|
973
|
+
else:
|
|
974
|
+
if layers is None:
|
|
975
|
+
layers = list(adata.layers.keys())
|
|
976
|
+
elif isinstance(layers, str):
|
|
977
|
+
layers = [layers]
|
|
978
|
+
Xs = [adata.layers[layer] for layer in layers]
|
|
979
|
+
masks = [~_nan_mask(X) for X in Xs]
|
|
980
|
+
if mask_key:
|
|
981
|
+
masks = [masks[i] & adata.obsm[mask_key][layer].values for i, layer in enumerate(layers)]
|
|
982
|
+
if concat_layers:
|
|
983
|
+
if sp.issparse(Xs[0]):
|
|
984
|
+
Xs = [sp.hstack(Xs, format="csr")]
|
|
985
|
+
else:
|
|
986
|
+
Xs = [np.concatenate(Xs, axis=0)]
|
|
987
|
+
masks = [np.concatenate(masks, axis=0)]
|
|
988
|
+
X_names = ["multi_layer"]
|
|
989
|
+
else:
|
|
990
|
+
X_names = [layer for layer in layers]
|
|
991
|
+
var_names = adata.var_names.values
|
|
992
|
+
if concat_layers:
|
|
993
|
+
start_lines += f" using layers {layers} (concatenated) as features\n"
|
|
994
|
+
else:
|
|
995
|
+
start_lines += f" using layers {layers} as features\n"
|
|
996
|
+
|
|
997
|
+
Xs = [X[mask] for X, mask in zip(Xs, masks)]
|
|
998
|
+
|
|
999
|
+
if response_field == "obs":
|
|
1000
|
+
if isinstance(response_key, str):
|
|
1001
|
+
response_key_added = response_key
|
|
1002
|
+
Y = adata.obs[response_key].values[:, None]
|
|
1003
|
+
y_names = [response_key]
|
|
1004
|
+
start_lines += f" using `adata.obs['{response_key}']` as response"
|
|
1005
|
+
else:
|
|
1006
|
+
response_key_added = "+".join(response_key)
|
|
1007
|
+
Y = adata.obs[response_key].values
|
|
1008
|
+
y_names = response_key.columns.values.astype(str)
|
|
1009
|
+
start_lines += f" using `adata.obs[{y_names}]` as response"
|
|
1010
|
+
elif response_field == "obsm":
|
|
1011
|
+
if isinstance(response_key, list):
|
|
1012
|
+
raise ValueError("response_key must be a single string when response_field='obsm'")
|
|
1013
|
+
try:
|
|
1014
|
+
response_key_added = response_key
|
|
1015
|
+
y = adata.obsm[response_key]
|
|
1016
|
+
if isinstance(y, pd.DataFrame):
|
|
1017
|
+
y_names = y.columns.values.astype(str)
|
|
1018
|
+
y = y.values
|
|
1019
|
+
else:
|
|
1020
|
+
y_names = [f"{response_key}:{i}" for i in range(y.shape[1])]
|
|
1021
|
+
start_lines += f" using `adata.obsm['{response_key}']` as response"
|
|
1022
|
+
except KeyError:
|
|
1023
|
+
raise KeyError(f"Response key '{response_key}' not found in adata.obsm. Please provide a valid response key")
|
|
1024
|
+
else:
|
|
1025
|
+
raise TypeError("response_field must be 'obs' or 'obsm'.")
|
|
1026
|
+
|
|
1027
|
+
start = logg.info("computing feature–response associations", deep=start_lines)
|
|
1028
|
+
if X_names == ["multi_layer"]:
|
|
1029
|
+
Y = np.tile(y, (len(layers), 1))
|
|
1030
|
+
else:
|
|
1031
|
+
Y = y
|
|
1032
|
+
|
|
1033
|
+
if binomial and response_transform is not None:
|
|
1034
|
+
logg.warning("response_transform is ignored when binomial=True.")
|
|
1035
|
+
response_transform = None
|
|
1036
|
+
|
|
1037
|
+
if response_transform == "logit":
|
|
1038
|
+
Y = np.clip(Y, 0.01, 0.99).copy()
|
|
1039
|
+
Y = np.log(Y / (1 - Y))
|
|
1040
|
+
elif response_transform == "log1p":
|
|
1041
|
+
Y = np.log1p(Y).copy()
|
|
1042
|
+
elif response_transform == "sqrt":
|
|
1043
|
+
Y = np.sqrt(Y).copy()
|
|
1044
|
+
elif response_transform == "clr":
|
|
1045
|
+
Y = _clr(Y, epsilon=clr_pseudocount).copy()
|
|
1046
|
+
elif response_transform:
|
|
1047
|
+
logg.warning("response_transform must be None, 'logit', 'log1p', or 'sqrt'. Setting response_transform=None")
|
|
1048
|
+
response_transform = None
|
|
1049
|
+
|
|
1050
|
+
row_totals = None
|
|
1051
|
+
if binomial:
|
|
1052
|
+
Y = np.asarray(Y, dtype=float)
|
|
1053
|
+
row_totals = Y.sum(axis=1)
|
|
1054
|
+
safe_totals = np.where(row_totals > 0, row_totals, 1.0)
|
|
1055
|
+
Y = Y / safe_totals[:, None]
|
|
1056
|
+
start_lines += "\n binomial=True: response treated as counts, converted to proportions"
|
|
1057
|
+
|
|
1058
|
+
results = {}
|
|
1059
|
+
|
|
1060
|
+
for i, (X, X_name) in enumerate(list(zip(Xs, X_names))):
|
|
1061
|
+
Y_masked = Y[masks[i]]
|
|
1062
|
+
if method in ["pearson", "spearman"]:
|
|
1063
|
+
corr_kwargs = dict(method=method, significance=True, progress_bar=progress_bar)
|
|
1064
|
+
if binomial and row_totals is not None:
|
|
1065
|
+
corr_kwargs["weights"] = row_totals[masks[i]]
|
|
1066
|
+
X_res = _fast_corr(X, Y_masked, **corr_kwargs)
|
|
1067
|
+
else:
|
|
1068
|
+
gam_extra = dict(random_state=random_state, progress_bar=progress_bar)
|
|
1069
|
+
if binomial and row_totals is not None:
|
|
1070
|
+
gam_extra["binomial"] = True
|
|
1071
|
+
gam_extra["totals"] = row_totals[masks[i]]
|
|
1072
|
+
X_res = _gam(X, Y_masked, **gam_extra, **gam_kwargs)
|
|
1073
|
+
results[X_name] = X_res
|
|
1074
|
+
logg.info(f" computed '{method}' statistics for '{X_name}' features")
|
|
1075
|
+
|
|
1076
|
+
Ps = np.array([results[X_name]["pvalue"] for X_name in X_names])
|
|
1077
|
+
FDRs = _adjust_pvalues(Ps, method=p_adjustment_method)
|
|
1078
|
+
for i, X_name in enumerate(X_names):
|
|
1079
|
+
results[X_name]["p_adj"] = FDRs[i]
|
|
1080
|
+
|
|
1081
|
+
for X_name in X_names:
|
|
1082
|
+
for key in results[X_name]:
|
|
1083
|
+
if use_raw:
|
|
1084
|
+
adata.raw.varm[f"{response_key_added}:{X_name}:{method}:{key}"] = pd.DataFrame(
|
|
1085
|
+
results[X_name][key],
|
|
1086
|
+
index=var_names,
|
|
1087
|
+
columns=y_names,
|
|
1088
|
+
)
|
|
1089
|
+
else:
|
|
1090
|
+
adata.varm[f"{response_key_added}:{X_name}:{method}:{key}"] = pd.DataFrame(
|
|
1091
|
+
results[X_name][key],
|
|
1092
|
+
index=var_names,
|
|
1093
|
+
columns=y_names,
|
|
1094
|
+
)
|
|
1095
|
+
|
|
1096
|
+
lines = ["added"]
|
|
1097
|
+
for key in results[X_name]:
|
|
1098
|
+
if key.endswith("pvalue"):
|
|
1099
|
+
postfix = " with uncorrected p-values"
|
|
1100
|
+
elif key.endswith("p_adj"):
|
|
1101
|
+
postfix = " with corrected p-values"
|
|
1102
|
+
elif key.endswith("r"):
|
|
1103
|
+
postfix = " with correlation coefficients"
|
|
1104
|
+
elif key.endswith("slope"):
|
|
1105
|
+
postfix = " with regression slopes coefficients"
|
|
1106
|
+
elif key.endswith("r2"):
|
|
1107
|
+
postfix = " with models R2 coefficients"
|
|
1108
|
+
elif key.endswith("amplitude"):
|
|
1109
|
+
postfix = " with amplitudes of fitted GAMs"
|
|
1110
|
+
else:
|
|
1111
|
+
postfix = ""
|
|
1112
|
+
|
|
1113
|
+
if use_raw:
|
|
1114
|
+
lines.append(f" .raw.varm['<response_key>:<feature matrix>:{method}:{key}'] matrix{postfix}")
|
|
1115
|
+
else:
|
|
1116
|
+
lines.append(f" .varm['<response_key>:<feature matrix>:{method}:{key}'] matrix{postfix}")
|
|
1117
|
+
logg.info(" finished ({time_passed})", deep="\n".join(lines), time=start)
|