clone2vec 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1117 @@
1
+ from __future__ import annotations
2
+
3
+ import scanpy as sc
4
+ import numpy as np
5
+ import scipy.sparse as sp
6
+ import statsmodels.api as sm
7
+ import statsmodels.formula.api as smf
8
+ import scipy.stats as stats
9
+ import pandas as pd
10
+
11
+ import os
12
+ import sys
13
+
14
+ from contextlib import nullcontext
15
+ from joblib import Parallel, delayed
16
+ from typing import Literal
17
+ from tqdm import tqdm
18
+
19
+ from .utils import _nan_mask
20
+
21
+ logg = sc.logging
22
+
23
+ def _clr(x: np.ndarray, epsilon: float = 1e-3) -> np.ndarray:
24
+ """
25
+ Center Log-Ratio (CLR) transformation.
26
+
27
+ Parameters
28
+ ----------
29
+ x : np.ndarray
30
+ Input array with shape (n_samples, n_features).
31
+ epsilon : float, optional
32
+ Small constant to avoid log(0) and division by zero, by default 1e-3.
33
+
34
+ Returns
35
+ -------
36
+ np.ndarray
37
+ CLR-transformed array with shape (n_samples, n_features).
38
+ """
39
+ if not np.allclose(x, x.astype(int)):
40
+ if epsilon == 1e-3:
41
+ logg.warning("Pseudocount 0.001 is used for integer counts as an input for CLR, it might skew proportions.")
42
+
43
+ if isinstance(x, pd.DataFrame):
44
+ x_cols = x.columns
45
+ x_rows = x.index
46
+ x = x.values.copy()
47
+
48
+ x_pseudo = x + epsilon
49
+ log_geo_mean = np.mean(np.log(x_pseudo), axis=1, keepdims=True)
50
+ clr_transformed = np.log(x_pseudo) - log_geo_mean
51
+
52
+ return pd.DataFrame(clr_transformed, columns=x_cols, index=x_rows)
53
+ else:
54
+ x_pseudo = x + epsilon
55
+ log_geo_mean = np.mean(np.log(x_pseudo), axis=1, keepdims=True)
56
+ clr_transformed = np.log(x_pseudo) - log_geo_mean
57
+
58
+ return clr_transformed
59
+
60
+ def _fast_corr(
61
+ X: np.ndarray | sp.spmatrix,
62
+ Y: np.ndarray,
63
+ method: Literal["pearson", "spearman"] = "pearson",
64
+ significance: bool = True,
65
+ slope: bool = True,
66
+ batch_size: int = 1000,
67
+ progress_bar: bool = True,
68
+ weights: np.ndarray | None = None,
69
+ ) -> dict[str, np.ndarray]:
70
+ """
71
+ Calculate the correlation matrix between X and Y, gracefully handling
72
+ columns with zero variance by setting the correlation to NaN.
73
+
74
+ Supports sparse X by converting column-wise dense batches for computation.
75
+
76
+ Parameters
77
+ ----------
78
+ X : np.ndarray | scipy.sparse.spmatrix
79
+ Dataset with shape (n_samples, n_features). If sparse, columns are
80
+ processed in dense batches to limit memory usage.
81
+ Y : np.ndarray
82
+ Response matrix with shape (n_samples, n_responses). Assumed dense.
83
+ method : Literal["pearson", "spearman"], optional
84
+ Correlation method to use, by default "pearson".
85
+ significance : bool
86
+ If True, also returns a matrix of p-values corresponding to the
87
+ correlation matrix under the assumption of a t-distribution with
88
+ degrees of freedom equal to the number of samples minus two.
89
+ slope : bool
90
+ If True, also returns a matrix of slopes corresponding to the
91
+ correlation matrix.
92
+ batch_size : int, optional
93
+ Number of X columns to process per batch when X is sparse, by default 1000.
94
+ progress_bar : bool, optional
95
+ Whether to display a progress bar for batch processing when X is sparse,
96
+ by default True.
97
+ weights : np.ndarray | None, optional
98
+ Per-sample weights of shape (n_samples,). When provided, compute
99
+ weighted correlation. Typically used with ``binomial=True`` where weights
100
+ are the total counts per cell, by default None.
101
+
102
+ Returns
103
+ -------
104
+ dict[str, np.ndarray]
105
+ Dictionary with correlation matrix, p-values, and slopes.
106
+ """
107
+ if isinstance(X, np.ndarray) and X.ndim == 1:
108
+ X = X[:, None].copy()
109
+ if Y.ndim == 1:
110
+ Y = Y[:, None].copy()
111
+
112
+ if sp.issparse(Y):
113
+ logg.warning("Sparse input for the response matrix Y isn't supported, converting to dense for correlation calculation.")
114
+ Y = Y.toarray()
115
+
116
+ if X.shape[0] != Y.shape[0]:
117
+ raise ValueError("X and Y must have the same number of samples.")
118
+
119
+ if weights is not None:
120
+ weights = np.asarray(weights, dtype=float).ravel()
121
+ if weights.shape[0] != X.shape[0]:
122
+ raise ValueError("weights must have the same length as the number of samples.")
123
+
124
+ if method == "spearman":
125
+ Y_data = stats.rankdata(Y, axis=0)
126
+ else:
127
+ Y_data = Y
128
+
129
+ # ---- weighted vs unweighted centering / norms ----
130
+ if weights is not None:
131
+ w = weights[:, None] # (n, 1)
132
+ W = w.sum()
133
+ Y_mean = (w * Y_data).sum(axis=0) / W
134
+ Y_centered = Y_data - Y_mean
135
+ norm_Y = np.sqrt((w * Y_centered ** 2).sum(axis=0))
136
+ else:
137
+ Y_centered = Y_data - np.mean(Y_data, axis=0)
138
+ norm_Y = np.linalg.norm(Y_centered, axis=0)
139
+
140
+ n_features = X.shape[1]
141
+ n_responses = Y.shape[1]
142
+ R = np.full((n_features, n_responses), np.nan)
143
+
144
+ if not sp.issparse(X):
145
+ X_data = X
146
+ if method == "spearman":
147
+ X_data = stats.rankdata(X_data, axis=0)
148
+
149
+ if weights is not None:
150
+ X_mean = (w * X_data).sum(axis=0) / W
151
+ X_centered = X_data - X_mean
152
+ norm_X = np.sqrt((w * X_centered ** 2).sum(axis=0))
153
+ else:
154
+ X_centered = X_data - np.mean(X_data, axis=0)
155
+ norm_X = np.linalg.norm(X_centered, axis=0)
156
+
157
+ denominator = norm_X[:, None] * norm_Y
158
+
159
+ if weights is not None:
160
+ numerator = (w * X_centered).T @ Y_centered
161
+ else:
162
+ numerator = X_centered.T @ Y_centered
163
+ np.divide(numerator, denominator, where=(denominator != 0), out=R)
164
+
165
+ results = {"r": R}
166
+
167
+ if significance:
168
+ n = X.shape[0]
169
+ if weights is not None:
170
+ n_eff = W ** 2 / (weights ** 2).sum()
171
+ else:
172
+ n_eff = float(n)
173
+ if n_eff <= 2:
174
+ P = np.full_like(R, np.nan)
175
+ results["pvalue"] = P
176
+ else:
177
+ df = n_eff - 2
178
+ T = np.nan_to_num(R) * np.sqrt(df / (1 - np.nan_to_num(R) ** 2))
179
+ P = 2 * stats.t.sf(np.abs(T), df)
180
+ P[np.isnan(R)] = np.nan
181
+ results["pvalue"] = P
182
+
183
+ if slope:
184
+ if weights is not None:
185
+ std_X = np.sqrt((w * (X_data - X_mean) ** 2).sum(axis=0) / W)
186
+ std_Y = np.sqrt((w * (Y_data - Y_mean) ** 2).sum(axis=0) / W)
187
+ else:
188
+ std_X = np.std(X_data, axis=0)
189
+ std_Y = np.std(Y_data, axis=0)
190
+
191
+ std_ratio = np.full_like(R, np.nan)
192
+ np.divide(std_Y, std_X[:, None], out=std_ratio, where=(std_X[:, None] != 0))
193
+
194
+ S = R * std_ratio
195
+ results["slope"] = S
196
+
197
+ return results
198
+ else:
199
+ X_sparse = X.tocsc()
200
+
201
+ col_indices = range(0, n_features, max(1, int(batch_size)))
202
+
203
+ if sc.settings.verbosity.value >= 2:
204
+ prefix = " "
205
+ else:
206
+ prefix = ""
207
+ cm = tqdm(
208
+ col_indices,
209
+ desc=prefix + "correlation batches",
210
+ leave=True,
211
+ file=sys.stdout,
212
+ ) if progress_bar else nullcontext(col_indices)
213
+
214
+ if slope:
215
+ if weights is not None:
216
+ std_Y = np.sqrt((w * (Y_data - Y_mean) ** 2).sum(axis=0) / W)
217
+ else:
218
+ std_Y = np.std(Y_data, axis=0)
219
+ S = np.full_like(R, np.nan)
220
+
221
+ if significance:
222
+ n = X.shape[0]
223
+ P = np.full_like(R, np.nan)
224
+ if weights is not None:
225
+ n_eff = W ** 2 / (weights ** 2).sum()
226
+ else:
227
+ n_eff = float(n)
228
+ df = n_eff - 2
229
+
230
+ with cm as iterator:
231
+ for start in iterator:
232
+ end = min(start + batch_size, n_features)
233
+ cols = np.arange(start, end)
234
+
235
+ X_batch = X_sparse[:, cols].toarray()
236
+ if method == "spearman":
237
+ X_batch = stats.rankdata(X_batch, axis=0)
238
+
239
+ if weights is not None:
240
+ X_mean_b = (w * X_batch).sum(axis=0) / W
241
+ X_centered_b = X_batch - X_mean_b
242
+ norm_X_b = np.sqrt((w * X_centered_b ** 2).sum(axis=0))
243
+ else:
244
+ X_centered_b = X_batch - np.mean(X_batch, axis=0)
245
+ norm_X_b = np.linalg.norm(X_centered_b, axis=0)
246
+
247
+ denominator_b = norm_X_b[:, None] * norm_Y
248
+
249
+ if weights is not None:
250
+ numerator_b = (w * X_centered_b).T @ Y_centered
251
+ else:
252
+ numerator_b = X_centered_b.T @ Y_centered
253
+ R_b = np.full(denominator_b.shape, np.nan)
254
+ np.divide(numerator_b, denominator_b, where=(denominator_b != 0), out=R_b)
255
+ R[start:end, :] = R_b
256
+
257
+ if slope:
258
+ if weights is not None:
259
+ std_X_b = np.sqrt((w * (X_batch - X_mean_b) ** 2).sum(axis=0) / W)
260
+ else:
261
+ std_X_b = np.std(X_batch, axis=0)
262
+ std_ratio_b = np.full_like(R_b, np.nan)
263
+ np.divide(std_Y, std_X_b[:, None], out=std_ratio_b, where=(std_X_b[:, None] != 0))
264
+ S_b = R_b * std_ratio_b
265
+ S[start:end, :] = S_b
266
+
267
+ if significance and n_eff > 2:
268
+ T_b = np.nan_to_num(R_b) * np.sqrt(df / (1 - np.nan_to_num(R_b) ** 2))
269
+ P_b = 2 * stats.t.sf(np.abs(T_b), df)
270
+ P_b[np.isnan(R_b)] = np.nan
271
+ P[start:end, :] = P_b
272
+
273
+ results = {"r": R}
274
+ if significance:
275
+ results["pvalue"] = P
276
+ if slope:
277
+ results["slope"] = S
278
+
279
+ return results
280
+
281
+ def _fast_corr_cols(
282
+ X: np.ndarray | sp.spmatrix,
283
+ Y: np.ndarray | sp.spmatrix,
284
+ method: Literal["pearson", "spearman"] = "pearson",
285
+ significance: bool = True,
286
+ slope: bool = True,
287
+ batch_size: int = 1000,
288
+ progress_bar: bool = True,
289
+ weights: np.ndarray | None = None,
290
+ ) -> dict[str, np.ndarray]:
291
+ """
292
+ Calculate correlation statistics between corresponding columns of X and Y
293
+ (1<->1, 2<->2, ...), gracefully handling zero-variance columns by setting
294
+ the result to NaN. Supports sparse inputs similarly to `_fast_corr`.
295
+
296
+ Parameters
297
+ ----------
298
+ X : np.ndarray | scipy.sparse.spmatrix
299
+ Dataset with shape (n_samples, n_features).
300
+ Y : np.ndarray | scipy.sparse.spmatrix
301
+ Another dataset with the same shape as X.
302
+ method : Literal["pearson", "spearman"], optional
303
+ Statistic to compute, by default "pearson".
304
+ significance : bool, optional
305
+ If True, also returns p-values (for Pearson/Spearman via t-test,
306
+ for GAM via LRT/Wald), by default True.
307
+ slope : bool, optional
308
+ If True, also returns slopes (for Pearson/Spearman as std ratio times r;
309
+ for GAM the endpoints change), by default True.
310
+ batch_size : int, optional
311
+ Number of columns to process per batch when X (or Y) is sparse,
312
+ by default 1000.
313
+ progress_bar : bool, optional
314
+ Whether to display a progress bar for batch processing when inputs are
315
+ sparse or when method=="gam", by default True.
316
+ weights : np.ndarray | None, optional
317
+ Per-sample weights of shape (n_samples,). When provided, compute
318
+ weighted correlation, by default None.
319
+
320
+ Returns
321
+ -------
322
+ dict[str, np.ndarray]
323
+ Dictionary with statistics. For Pearson/Spearman: keys "r",
324
+ optionally "pvalue" and "slope". For GAM: keys "r2",
325
+ "amplitude", optionally "pvalue" and "slope".
326
+ """
327
+ if isinstance(X, np.ndarray) and X.ndim == 1:
328
+ X = X[:, None].copy()
329
+ if isinstance(Y, np.ndarray) and Y.ndim == 1:
330
+ Y = Y[:, None].copy()
331
+
332
+ if X.shape != Y.shape:
333
+ raise ValueError("X and Y must have the same shape for column-wise correlation.")
334
+
335
+ n_samples, n_cols = X.shape
336
+
337
+ if weights is not None:
338
+ weights = np.asarray(weights, dtype=float).ravel()
339
+ if weights.shape[0] != n_samples:
340
+ raise ValueError("weights must have the same length as the number of samples.")
341
+
342
+ if method in ["pearson", "spearman"]:
343
+ results = {}
344
+
345
+ if (not sp.issparse(X)) and (not sp.issparse(Y)):
346
+ X_data = X
347
+ Y_data = Y
348
+ if method == "spearman":
349
+ X_data = stats.rankdata(X_data, axis=0)
350
+ Y_data = stats.rankdata(Y_data, axis=0)
351
+
352
+ if weights is not None:
353
+ w = weights[:, None]
354
+ W = w.sum()
355
+ X_mean = (w * X_data).sum(axis=0) / W
356
+ Y_mean = (w * Y_data).sum(axis=0) / W
357
+ X_centered = X_data - X_mean
358
+ Y_centered = Y_data - Y_mean
359
+ norm_X = np.sqrt((w * X_centered ** 2).sum(axis=0))
360
+ norm_Y = np.sqrt((w * Y_centered ** 2).sum(axis=0))
361
+ else:
362
+ X_centered = X_data - np.mean(X_data, axis=0)
363
+ Y_centered = Y_data - np.mean(Y_data, axis=0)
364
+ norm_X = np.linalg.norm(X_centered, axis=0)
365
+ norm_Y = np.linalg.norm(Y_centered, axis=0)
366
+ denom = norm_X * norm_Y
367
+
368
+ if weights is not None:
369
+ numer = (w * X_centered * Y_centered).sum(axis=0)
370
+ else:
371
+ numer = np.sum(X_centered * Y_centered, axis=0)
372
+ R = np.full(n_cols, np.nan)
373
+ np.divide(numer, denom, where=(denom != 0), out=R)
374
+
375
+ results["r"] = R
376
+
377
+ if significance:
378
+ n = n_samples
379
+ if weights is not None:
380
+ n_eff = W ** 2 / (weights ** 2).sum()
381
+ else:
382
+ n_eff = float(n)
383
+ if n_eff <= 2:
384
+ P = np.full_like(R, np.nan)
385
+ else:
386
+ df = n_eff - 2
387
+ T = np.nan_to_num(R) * np.sqrt(df / (1 - np.nan_to_num(R) ** 2))
388
+ P = 2 * stats.t.sf(np.abs(T), df)
389
+ P[np.isnan(R)] = np.nan
390
+ results["pvalue"] = P
391
+
392
+ if slope:
393
+ if weights is not None:
394
+ std_X = np.sqrt((w * (X_data - X_mean) ** 2).sum(axis=0) / W)
395
+ std_Y = np.sqrt((w * (Y_data - Y_mean) ** 2).sum(axis=0) / W)
396
+ else:
397
+ std_X = np.std(X_data, axis=0)
398
+ std_Y = np.std(Y_data, axis=0)
399
+ std_ratio = np.full_like(R, np.nan)
400
+ np.divide(std_Y, std_X, out=std_ratio, where=(std_X != 0))
401
+ S = R * std_ratio
402
+ results["slope"] = S
403
+
404
+ return results
405
+
406
+ X_sparse = X.tocsc() if sp.issparse(X) else None
407
+ Y_sparse = Y.tocsc() if sp.issparse(Y) else None
408
+
409
+ R = np.full(n_cols, np.nan)
410
+ if significance:
411
+ n = n_samples
412
+ P = np.full(n_cols, np.nan)
413
+ if weights is not None:
414
+ W = weights.sum()
415
+ n_eff = W ** 2 / (weights ** 2).sum()
416
+ else:
417
+ n_eff = float(n)
418
+ df = n_eff - 2
419
+ if slope:
420
+ S = np.full(n_cols, np.nan)
421
+
422
+ col_indices = range(0, n_cols, max(1, int(batch_size)))
423
+ if sc.settings.verbosity.value >= 2:
424
+ prefix = " "
425
+ else:
426
+ prefix = ""
427
+ cm = tqdm(
428
+ total=len(col_indices),
429
+ desc=prefix + "correlation batches",
430
+ leave=True,
431
+ file=sys.stdout,
432
+ ) if progress_bar else nullcontext()
433
+ with cm as pbar:
434
+ for start in col_indices:
435
+ end = min(start + batch_size, n_cols)
436
+ cols = np.arange(start, end)
437
+
438
+ X_batch = (X_sparse[:, cols].toarray() if X_sparse is not None else X[:, cols])
439
+ Y_batch = (Y_sparse[:, cols].toarray() if Y_sparse is not None else Y[:, cols])
440
+
441
+ if method == "spearman":
442
+ X_batch = stats.rankdata(X_batch, axis=0)
443
+ Y_batch = stats.rankdata(Y_batch, axis=0)
444
+
445
+ if weights is not None:
446
+ w = weights[:, None]
447
+ W_val = w.sum()
448
+ X_mean_b = (w * X_batch).sum(axis=0) / W_val
449
+ Y_mean_b = (w * Y_batch).sum(axis=0) / W_val
450
+ X_centered_b = X_batch - X_mean_b
451
+ Y_centered_b = Y_batch - Y_mean_b
452
+ norm_X_b = np.sqrt((w * X_centered_b ** 2).sum(axis=0))
453
+ norm_Y_b = np.sqrt((w * Y_centered_b ** 2).sum(axis=0))
454
+ else:
455
+ X_centered_b = X_batch - np.mean(X_batch, axis=0)
456
+ Y_centered_b = Y_batch - np.mean(Y_batch, axis=0)
457
+ norm_X_b = np.linalg.norm(X_centered_b, axis=0)
458
+ norm_Y_b = np.linalg.norm(Y_centered_b, axis=0)
459
+
460
+ denom_b = norm_X_b * norm_Y_b
461
+ if weights is not None:
462
+ numer_b = (w * X_centered_b * Y_centered_b).sum(axis=0)
463
+ else:
464
+ numer_b = np.sum(X_centered_b * Y_centered_b, axis=0)
465
+ R_b = np.full(len(cols), np.nan)
466
+ np.divide(numer_b, denom_b, where=(denom_b != 0), out=R_b)
467
+ R[start:end] = R_b
468
+
469
+ if slope:
470
+ if weights is not None:
471
+ std_X_b = np.sqrt((w * (X_batch - X_mean_b) ** 2).sum(axis=0) / W_val)
472
+ std_Y_b = np.sqrt((w * (Y_batch - Y_mean_b) ** 2).sum(axis=0) / W_val)
473
+ else:
474
+ std_X_b = np.std(X_batch, axis=0)
475
+ std_Y_b = np.std(Y_batch, axis=0)
476
+ std_ratio_b = np.full(len(cols), np.nan)
477
+ np.divide(std_Y_b, std_X_b, out=std_ratio_b, where=(std_X_b != 0))
478
+ S_b = R_b * std_ratio_b
479
+ S[start:end] = S_b
480
+
481
+ if significance and n_eff > 2:
482
+ T_b = np.nan_to_num(R_b) * np.sqrt(df / (1 - np.nan_to_num(R_b) ** 2))
483
+ P_b = 2 * stats.t.sf(np.abs(T_b), df)
484
+ P_b[np.isnan(R_b)] = np.nan
485
+ P[start:end] = P_b
486
+ if progress_bar:
487
+ pbar.update(1)
488
+
489
+ results = {"r": R}
490
+ if significance:
491
+ results["pvalue"] = P
492
+ if slope:
493
+ results["slope"] = S
494
+ return results
495
+
496
+ else:
497
+ raise ValueError("method must be 'pearson' or 'spearman'")
498
+
499
+ def _calculate_gam_stats(
500
+ x: np.array,
501
+ y: np.array,
502
+ spline_df: int = 5,
503
+ spline_function: str = "bs",
504
+ p_method: Literal["lrt", "wald"] = "lrt",
505
+ random_state: int | None = 42,
506
+ binomial: bool = False,
507
+ totals: np.array | None = None,
508
+ **spline_kwargs,
509
+ ) -> dict[str, np.ndarray]:
510
+ """
511
+ Calculate GAM statistics for a single pair of columns in X and Y.
512
+
513
+ Parameters
514
+ ----------
515
+ x : np.array
516
+ Input feature vector.
517
+ y : np.array
518
+ Response vector. When ``binomial=True`` this should be proportions
519
+ (successes / totals) in the range [0, 1].
520
+ spline_df : int, optional
521
+ Degrees of freedom for the spline, by default 5.
522
+ spline_function : str, optional
523
+ Spline function to use, by default "bs".
524
+ p_method : Literal["lrt", "wald"], optional
525
+ Method to use for p-value calculation, by default "lrt".
526
+ random_state : int | None, optional
527
+ Seed for NumPy RNG to ensure reproducible behavior in stochastic routines, by default 42.
528
+ binomial : bool, optional
529
+ If True, use a Binomial GLM instead of Gaussian. ``y`` should be
530
+ proportions and ``totals`` the per-observation trial counts, by default False.
531
+ totals : np.array | None, optional
532
+ Per-observation trial counts (used only when ``binomial=True``), by default None.
533
+ **spline_kwargs : dict, optional
534
+ Additional keyword arguments to pass to the spline formula.
535
+
536
+ Returns
537
+ -------
538
+ dict[str, np.ndarray]
539
+ Dictionary with GAM statistics.
540
+ """
541
+ if random_state is not None:
542
+ try:
543
+ np.random.seed(int(random_state))
544
+ except Exception:
545
+ pass
546
+
547
+ if np.allclose(x, x[0]):
548
+ results = {"r2": np.nan, "amplitude": np.nan, "slope": np.nan, "pvalue": np.nan}
549
+ return results
550
+
551
+ if len(x) != len(y):
552
+ logg.warning("x and y must have the same number of samples. Return NaN.")
553
+ results = {"r2": np.nan, "amplitude": np.nan, "slope": np.nan, "pvalue": np.nan}
554
+ return results
555
+
556
+ try:
557
+ results = {}
558
+
559
+ kwargs_str = ""
560
+ for k, v in spline_kwargs.items():
561
+ kwargs_str += f", {k}={v}"
562
+
563
+ if binomial:
564
+ if totals is None:
565
+ raise ValueError("totals must be provided when binomial=True")
566
+ family = sm.families.Binomial()
567
+ gam_fit = smf.glm(
568
+ formula=f"y ~ {spline_function}(x, df={spline_df}{kwargs_str})",
569
+ data={"x": x, "y": y},
570
+ family=family,
571
+ freq_weights=totals.astype(float),
572
+ ).fit(random_state=random_state)
573
+ else:
574
+ family = sm.families.Gaussian()
575
+ gam_fit = smf.glm(
576
+ formula=f"y ~ {spline_function}(x, df={spline_df}{kwargs_str})",
577
+ data={"x": x, "y": y},
578
+ family=family,
579
+ ).fit(random_state=random_state)
580
+ results["r2"] = gam_fit.pseudo_rsquared()
581
+
582
+ fit_prediction = gam_fit.predict({"x": x}).values
583
+ effect_size = fit_prediction.max() - fit_prediction.min()
584
+ results["amplitude"] = effect_size
585
+
586
+ endpoints_change = gam_fit.predict({"x": [x.max(), x.min()]}).values
587
+ endpoints_change = endpoints_change[0] - endpoints_change[1]
588
+ results["slope"] = endpoints_change
589
+
590
+ if p_method not in ["lrt", "wald"]:
591
+ logg.warning(f"p_method must be 'lrt' or 'wald', got {p_method}. Default to 'lrt'.")
592
+ p_method = "lrt"
593
+
594
+ if p_method == "lrt":
595
+ if binomial:
596
+ const_fit = smf.glm(
597
+ formula="y ~ 1",
598
+ data={"y": y},
599
+ family=family,
600
+ freq_weights=totals.astype(float),
601
+ ).fit(random_state=random_state)
602
+ else:
603
+ x_const = sm.add_constant(np.zeros_like(x))
604
+ const_fit = sm.OLS(y, x_const).fit(random_state=random_state)
605
+ lrt = 2 * (gam_fit.llf - const_fit.llf)
606
+ df_d = gam_fit.df_model - const_fit.df_model
607
+ p = stats.chi2.sf(lrt, df_d)
608
+ results["pvalue"] = p
609
+ else:
610
+ wald_stat = gam_fit.wald_test_terms().summary_frame()
611
+ wald_p = wald_stat.iloc[wald_stat.index != "Intercept"]["P>chi2"].values[0]
612
+ results["pvalue"] = wald_p
613
+
614
+ return results
615
+ except Exception as e:
616
+ logg.warning(f"Error fitting GAM: {e}. Return NaN.")
617
+ results = {"r2": np.nan, "amplitude": np.nan, "slope": np.nan, "pvalue": np.nan}
618
+ return results
619
+
620
+ def _gam(
621
+ X: np.ndarray,
622
+ Y: np.ndarray,
623
+ spline_df: int = 5,
624
+ spline_degree: int = 3,
625
+ n_jobs: int = -1,
626
+ p_method: Literal["lrt", "wald"] = "lrt",
627
+ progress_bar: bool = True,
628
+ random_state: int | None = 42,
629
+ binomial: bool = False,
630
+ totals: np.ndarray | None = None,
631
+ **spline_kwargs,
632
+ ) -> dict[str, np.ndarray]:
633
+ """
634
+ Calculate GAM (generalized additive model) statistics for each pair of
635
+ columns in X and Y.
636
+
637
+ Parameters
638
+ ----------
639
+ X : np.ndarray
640
+ Dataset with shape (n_samples, n_features).
641
+ Y : np.ndarray
642
+ Response matrix with shape (n_samples, n_responses).
643
+ When ``binomial=True`` these should be proportions.
644
+ spline_df : int, optional
645
+ Degrees of freedom for the spline, by default 5.
646
+ spline_degree : int, optional
647
+ Degree of the spline, by default 3.
648
+ n_jobs : int, optional
649
+ Number of parallel jobs to run, by default -1.
650
+ p_method : Literal["lrt", "wald"], optional
651
+ Method to use for p-value calculation, by default "lrt".
652
+ progress_bar : bool, optional
653
+ If True, display a progress bar, by default True.
654
+ random_state : int | None, optional
655
+ Base seed used to derive a deterministic per-pair seed, ensuring reproducibility, by default 42.
656
+ binomial : bool, optional
657
+ If True, use a Binomial GLM instead of Gaussian, by default False.
658
+ totals : np.ndarray | None, optional
659
+ Per-observation trial counts (used only when ``binomial=True``), by default None.
660
+ **spline_kwargs : dict, optional
661
+ Additional keyword arguments to pass to the spline formula.
662
+
663
+ Returns
664
+ -------
665
+ dict[str, np.ndarray]
666
+ Dictionary with GAM statistics for each pair of columns in X and Y.
667
+ """
668
+ from itertools import product
669
+
670
+ if sp.issparse(X):
671
+ X_sparse = X.tocsc()
672
+
673
+ # Consistency with fast_corr function
674
+ if sp.issparse(Y):
675
+ logg.warning("Sparse input for the response matrix Y isn't supported, converting to dense for correlation calculation.")
676
+ Y = Y.toarray()
677
+
678
+ X_col_n = X.shape[1]
679
+ Y_col_n = Y.shape[1]
680
+
681
+ tasks = product(np.arange(X_col_n), np.arange(Y_col_n))
682
+
683
+ if sc.settings.verbosity.value >= 2:
684
+ prefix = " "
685
+ progress_bar = True
686
+ else:
687
+ prefix = ""
688
+
689
+ if progress_bar:
690
+ tasks = tqdm(
691
+ tasks,
692
+ desc=prefix + "fitting GAMs",
693
+ total=(X_col_n * Y_col_n),
694
+ file=sys.stdout,
695
+ )
696
+
697
+ if not sp.issparse(X):
698
+ results = Parallel(n_jobs=n_jobs)(
699
+ delayed(_calculate_gam_stats)(
700
+ X[:, x],
701
+ Y[:, y],
702
+ spline_df=spline_df,
703
+ p_method=p_method,
704
+ random_state=(None if random_state is None else int(random_state) + x * Y_col_n + y),
705
+ degree=spline_degree,
706
+ binomial=binomial,
707
+ totals=totals,
708
+ **spline_kwargs,
709
+ ) for x, y in tasks
710
+ )
711
+ else:
712
+ results = Parallel(n_jobs=n_jobs)(
713
+ delayed(_calculate_gam_stats)(
714
+ X_sparse[:, x].toarray().flatten(),
715
+ Y[:, y],
716
+ spline_df=spline_df,
717
+ p_method=p_method,
718
+ random_state=(None if random_state is None else int(random_state) + x * Y_col_n + y),
719
+ degree=spline_degree,
720
+ binomial=binomial,
721
+ totals=totals,
722
+ **spline_kwargs,
723
+ ) for x, y in tasks
724
+ )
725
+
726
+ results = {
727
+ key: np.array([r[key] for r in results]).reshape(X_col_n, Y_col_n)
728
+ for key in results[0]
729
+ }
730
+ return results
731
+
732
+ def _adjust_pvalues(
733
+ pvalues: np.ndarray,
734
+ method: str = "fdr_bh",
735
+ ) -> np.ndarray:
736
+ """
737
+ Adjust p-values using the specified method.
738
+ NaN values in `pvalues` will be left as NaN in the output.
739
+
740
+ Parameters
741
+ ----------
742
+ pvalues : np.ndarray
743
+ Array of p-values to adjust. Can be any-dimensional.
744
+ method : str, optional
745
+ Method for adjustment, will be passed to `statsmodels.stats.multitest.multipletests`,
746
+ default is "fdr_bh".
747
+
748
+ Returns
749
+ -------
750
+ np.ndarray
751
+ Adjusted p-values. Has the same shape as `pvalues`.
752
+ """
753
+ from statsmodels.stats.multitest import multipletests
754
+
755
+ dims = pvalues.shape
756
+ pvalues_flat = pvalues.flatten()
757
+ nan_mask = np.isnan(pvalues_flat)
758
+ fdrs_flat = np.full_like(pvalues_flat, np.nan)
759
+
760
+ fdrs_flat[~nan_mask] = multipletests(pvalues_flat[~nan_mask], method=method)[1]
761
+
762
+ return fdrs_flat.reshape(dims)
763
+
764
+
765
+ def graph_associations(
766
+ adata: sc.AnnData,
767
+ layer: None | str = None,
768
+ graph_key: str = "connectivities",
769
+ n_pcs: int = 50,
770
+ adj_method: str = "fdr_bh",
771
+ key_added: str = "X_gPCA",
772
+ ) -> None:
773
+ """
774
+ Computes the supervised principal components analysis (sPCA) of the data
775
+ [10.1016/j.patcog.2010.12.015]. Shortly, it finds the axes that maximize the correlation
776
+ between the gene expression and the graph Laplacian (one can see it as an
777
+ autocorrelation-aware PCA).
778
+
779
+ Parameters
780
+ ----------
781
+ adata : sc.AnnData
782
+ AnnData object containing the data.
783
+ layer : None | str
784
+ Layer to use for the analysis.
785
+ graph_key : str
786
+ Key to use for the graph.
787
+ n_pcs : int
788
+ Number of principal components to compute.
789
+ adj_method : str
790
+ Method for adjusting p-values.
791
+ key_added : str
792
+ Key to use for adding the results to the AnnData object.
793
+
794
+ Returns
795
+ -------
796
+ None
797
+ """
798
+ from scipy.sparse.linalg import eigsh
799
+ import scipy.stats as stats
800
+
801
+ start = logg.info(f"computing graph associations")
802
+
803
+ if layer is None:
804
+ layer = "X"
805
+ use_X = True
806
+ else:
807
+ use_X = False
808
+
809
+ if use_X:
810
+ X = adata.X
811
+ else:
812
+ X = adata.layers[layer]
813
+
814
+ if sp.issparse(X):
815
+ logg.warning("provided matrix is sparse, make sure that you scaled data before passing to the function")
816
+ X = X.todense()
817
+
818
+ total_variance = np.sum(X ** 2)
819
+
820
+ spcov = X.T @ adata.obsp[graph_key] @ X
821
+ evals, evecs = eigsh(spcov, k=n_pcs, which="LA")
822
+ evals, evecs = evals[::-1], evecs[:, ::-1]
823
+
824
+ adata.varm["gPCs"] = evecs
825
+ adata.obsm[key_added] = X @ evecs
826
+ adata.uns["gpca"] = {
827
+ "evals": evals,
828
+ "varexpl": evals / total_variance,
829
+ "graph_key": graph_key,
830
+ "n_pcs": n_pcs,
831
+ }
832
+
833
+ # Moran's I
834
+ numerators = np.diag(spcov)
835
+ denominators = np.sum(X ** 2, axis=0)
836
+ w = adata.obsp[graph_key]
837
+ W = w.sum()
838
+ N = len(adata)
839
+
840
+ morans_i = (N / W) * (numerators / denominators)
841
+ expected_I = -1.0 / (N - 1)
842
+ S1 = 0.5 * (w + w.T).power(2).sum()
843
+ deg_row = np.array(w.sum(axis=1))
844
+ deg_col = np.array(w.sum(axis=0)).T
845
+ S2 = np.sum((deg_row + deg_col) ** 2)
846
+ W_sq = W ** 2
847
+ var_I = (
848
+ (N * ((N**2 - 3*N + 3) * S1 - N * S2 + 3 * W_sq)) /
849
+ ((N - 1) * (N - 2) * (N - 3) * W_sq)
850
+ ) - (expected_I ** 2)
851
+ with np.errstate(invalid="ignore"):
852
+ z_scores = (morans_i - expected_I) / np.sqrt(var_I)
853
+ p_values = stats.norm.sf(z_scores)
854
+
855
+ adata.var["morans_i"] = morans_i
856
+ adata.var["morans_i_pval"] = p_values
857
+ adata.var["morans_i_p_adj"] = _adjust_pvalues(p_values, method=adj_method)
858
+
859
+ lines = [
860
+ "added",
861
+ f" .obsm['{key_added}'] {n_pcs} graph-guided principal components",
862
+ f" .varm['gPCs'] matrix of loadings",
863
+ f" .uns['gPCA'] eigenvalues and variance explained",
864
+ f" .var['morans_i'] Moran's I statistics",
865
+ f" .var['morans_i_pval'] p-values",
866
+ f" .var['morans_i_p_adj'] adjusted p-values",
867
+ ]
868
+ logg.info(" finished ({time_passed})", deep="\n".join(lines), time=start)
869
+
870
+ def associations(
871
+ adata,
872
+ response_key: str | list[str],
873
+ response_field: Literal["obs", "obsm"] = "obsm",
874
+ response_transform: None | Literal["logit", "log1p", "sqrt", "clr"] = None,
875
+ use_rep: Literal["X", "layers"] | None = None,
876
+ use_raw: bool | None = None,
877
+ layers: list[str] | str | None = None,
878
+ method: Literal["pearson", "spearman", "gam"] = "pearson",
879
+ p_adjustment_method: str = "fdr_bh",
880
+ concat_layers: bool = False,
881
+ random_state: int = 42,
882
+ progress_bar: bool = False,
883
+ mask_key: str | None | Literal[False] = None,
884
+ clr_pseudocount: float = 1e-3,
885
+ binomial: bool = False,
886
+ **gam_kwargs,
887
+ ) -> None:
888
+ """
889
+ Calculate correlation between features in `adata` and a response variable.
890
+ Correlation coefficients, p-values, and False Discovery Rates (FDRs) are computed for each feature.
891
+ Results are stored in `adata.varm` with keys "{X_name}:corr", "{X_name}:pvalue", and "{X_name}:FDR",
892
+ where {X_name} is the name of the feature representation (e.g., "X", "X_raw", or layer names).
893
+ If `p_adjustment_method` is None, no correction is performed.
894
+
895
+ Parameters
896
+ ----------
897
+ adata : AnnData
898
+ Annotated data matrix at the cell level.
899
+ response_key : str | list[str] | None
900
+ Key in `adata.obs` or `adata.obsm` containing the response variable. Might be a list
901
+ only if `response_field='obs'`, in this case the response variable is a multi-column DataFrame.
902
+ response_field : Literal["obs", "obsm"], optional
903
+ Whether the response variable is in `adata.obs` or `adata.obsm`, by default "obsm".
904
+ response_transform : None | Literal["logit", "log1p", "sqrt", "clr"], optional
905
+ Transform to apply to the response variable before correlation calculation, by default None.
906
+ use_rep : Literal["X", "layers"] | None, optional
907
+ Whether to use `adata.X` or `adata.layers` for feature representations, if None, uses
908
+ `adata.X` if `layers` is None, otherwise `adata.layers`, by default None.
909
+ use_raw : bool | None, optional
910
+ Whether to use `adata.raw.X` as the feature representation, by default None.
911
+ layers : list[str] | str | None, optional
912
+ Layers in `adata.layers` to use for feature representations if `use_rep='layers'`. If None and
913
+ use_rep="layers", uses all layers, by default None.
914
+ method : Literal["pearson", "spearman", "gam"], optional
915
+ Method for correlation calculation, by default "pearson".
916
+ p_adjustment_method : str, optional
917
+ Method for multiple testing correction, passed to `statsmodels.stats.multitest.multipletests`,
918
+ default is "fdr_bh".
919
+ random_state : int, optional
920
+ Random seed for GAM model training, by default 42.
921
+ progress_bar : bool, optional
922
+ Whether to show a progress bar, by default False.
923
+ mask_key : str | None | Literal[False], optional
924
+ Key in `adata.obs` or `adata.obsm` containing a boolean mask to filter cells, by default None.
925
+ clr_pseudocount : float, optional
926
+ Pseudocount to add to expression values before CLR transformation, by default 1e-3.
927
+ binomial : bool, optional
928
+ If True, treat the response as count data and use binomial modeling. The response
929
+ matrix is interpreted as integer counts; proportions and row totals are computed
930
+ internally. For ``method='gam'``, a Binomial GLM is fitted (with ``freq_weights=totals``).
931
+ For ``method='pearson'`` or ``'spearman'``, weighted correlation is computed on the
932
+ proportions, with weights equal to row totals, by default False.
933
+ **gam_kwargs : dict, optional
934
+ Additional keyword arguments to pass to `gam`.
935
+
936
+ Returns
937
+ -------
938
+ None
939
+ The `adata` object is modified in place with correlation results stored in `adata.varm`.
940
+ """
941
+ if mask_key is None and "mask_key" in adata.uns.keys():
942
+ mask_key = adata.uns["mask_key"]
943
+
944
+ if use_rep is None:
945
+ if layers is None:
946
+ use_rep = "X"
947
+ else:
948
+ use_rep = "layers"
949
+ if use_raw and use_rep == "layers":
950
+ logg.warning("use_raw=True is not supported when use_rep='layers'. Setting use_raw=False.")
951
+ use_raw = False
952
+ if (use_raw is None) and (use_rep == "X"):
953
+ if adata.raw:
954
+ logg.warning("Using raw data when use_raw is None and adata.raw is not None. Setting use_raw=True.")
955
+ use_raw = True
956
+ if use_rep == "X" and layers:
957
+ logg.warning("Using layers when use_rep='X' is not supported. Setting use_rep='layers'.")
958
+ use_rep = "layers"
959
+
960
+ start_lines = "\n"
961
+ if use_raw:
962
+ Xs = [adata.raw.X]
963
+ masks = [~_nan_mask(adata.raw.X)]
964
+ X_names = ["X_raw"]
965
+ var_names = adata.raw.var_names.values
966
+ start_lines += " using `adata.raw.X` as features\n"
967
+ elif use_rep == "X":
968
+ Xs = [adata.X]
969
+ masks = [~_nan_mask(adata.X)]
970
+ X_names = ["X"]
971
+ var_names = adata.var_names.values
972
+ start_lines += " using `adata.X` as features\n"
973
+ else:
974
+ if layers is None:
975
+ layers = list(adata.layers.keys())
976
+ elif isinstance(layers, str):
977
+ layers = [layers]
978
+ Xs = [adata.layers[layer] for layer in layers]
979
+ masks = [~_nan_mask(X) for X in Xs]
980
+ if mask_key:
981
+ masks = [masks[i] & adata.obsm[mask_key][layer].values for i, layer in enumerate(layers)]
982
+ if concat_layers:
983
+ if sp.issparse(Xs[0]):
984
+ Xs = [sp.hstack(Xs, format="csr")]
985
+ else:
986
+ Xs = [np.concatenate(Xs, axis=0)]
987
+ masks = [np.concatenate(masks, axis=0)]
988
+ X_names = ["multi_layer"]
989
+ else:
990
+ X_names = [layer for layer in layers]
991
+ var_names = adata.var_names.values
992
+ if concat_layers:
993
+ start_lines += f" using layers {layers} (concatenated) as features\n"
994
+ else:
995
+ start_lines += f" using layers {layers} as features\n"
996
+
997
+ Xs = [X[mask] for X, mask in zip(Xs, masks)]
998
+
999
+ if response_field == "obs":
1000
+ if isinstance(response_key, str):
1001
+ response_key_added = response_key
1002
+ Y = adata.obs[response_key].values[:, None]
1003
+ y_names = [response_key]
1004
+ start_lines += f" using `adata.obs['{response_key}']` as response"
1005
+ else:
1006
+ response_key_added = "+".join(response_key)
1007
+ Y = adata.obs[response_key].values
1008
+ y_names = response_key.columns.values.astype(str)
1009
+ start_lines += f" using `adata.obs[{y_names}]` as response"
1010
+ elif response_field == "obsm":
1011
+ if isinstance(response_key, list):
1012
+ raise ValueError("response_key must be a single string when response_field='obsm'")
1013
+ try:
1014
+ response_key_added = response_key
1015
+ y = adata.obsm[response_key]
1016
+ if isinstance(y, pd.DataFrame):
1017
+ y_names = y.columns.values.astype(str)
1018
+ y = y.values
1019
+ else:
1020
+ y_names = [f"{response_key}:{i}" for i in range(y.shape[1])]
1021
+ start_lines += f" using `adata.obsm['{response_key}']` as response"
1022
+ except KeyError:
1023
+ raise KeyError(f"Response key '{response_key}' not found in adata.obsm. Please provide a valid response key")
1024
+ else:
1025
+ raise TypeError("response_field must be 'obs' or 'obsm'.")
1026
+
1027
+ start = logg.info("computing feature–response associations", deep=start_lines)
1028
+ if X_names == ["multi_layer"]:
1029
+ Y = np.tile(y, (len(layers), 1))
1030
+ else:
1031
+ Y = y
1032
+
1033
+ if binomial and response_transform is not None:
1034
+ logg.warning("response_transform is ignored when binomial=True.")
1035
+ response_transform = None
1036
+
1037
+ if response_transform == "logit":
1038
+ Y = np.clip(Y, 0.01, 0.99).copy()
1039
+ Y = np.log(Y / (1 - Y))
1040
+ elif response_transform == "log1p":
1041
+ Y = np.log1p(Y).copy()
1042
+ elif response_transform == "sqrt":
1043
+ Y = np.sqrt(Y).copy()
1044
+ elif response_transform == "clr":
1045
+ Y = _clr(Y, epsilon=clr_pseudocount).copy()
1046
+ elif response_transform:
1047
+ logg.warning("response_transform must be None, 'logit', 'log1p', or 'sqrt'. Setting response_transform=None")
1048
+ response_transform = None
1049
+
1050
+ row_totals = None
1051
+ if binomial:
1052
+ Y = np.asarray(Y, dtype=float)
1053
+ row_totals = Y.sum(axis=1)
1054
+ safe_totals = np.where(row_totals > 0, row_totals, 1.0)
1055
+ Y = Y / safe_totals[:, None]
1056
+ start_lines += "\n binomial=True: response treated as counts, converted to proportions"
1057
+
1058
+ results = {}
1059
+
1060
+ for i, (X, X_name) in enumerate(list(zip(Xs, X_names))):
1061
+ Y_masked = Y[masks[i]]
1062
+ if method in ["pearson", "spearman"]:
1063
+ corr_kwargs = dict(method=method, significance=True, progress_bar=progress_bar)
1064
+ if binomial and row_totals is not None:
1065
+ corr_kwargs["weights"] = row_totals[masks[i]]
1066
+ X_res = _fast_corr(X, Y_masked, **corr_kwargs)
1067
+ else:
1068
+ gam_extra = dict(random_state=random_state, progress_bar=progress_bar)
1069
+ if binomial and row_totals is not None:
1070
+ gam_extra["binomial"] = True
1071
+ gam_extra["totals"] = row_totals[masks[i]]
1072
+ X_res = _gam(X, Y_masked, **gam_extra, **gam_kwargs)
1073
+ results[X_name] = X_res
1074
+ logg.info(f" computed '{method}' statistics for '{X_name}' features")
1075
+
1076
+ Ps = np.array([results[X_name]["pvalue"] for X_name in X_names])
1077
+ FDRs = _adjust_pvalues(Ps, method=p_adjustment_method)
1078
+ for i, X_name in enumerate(X_names):
1079
+ results[X_name]["p_adj"] = FDRs[i]
1080
+
1081
+ for X_name in X_names:
1082
+ for key in results[X_name]:
1083
+ if use_raw:
1084
+ adata.raw.varm[f"{response_key_added}:{X_name}:{method}:{key}"] = pd.DataFrame(
1085
+ results[X_name][key],
1086
+ index=var_names,
1087
+ columns=y_names,
1088
+ )
1089
+ else:
1090
+ adata.varm[f"{response_key_added}:{X_name}:{method}:{key}"] = pd.DataFrame(
1091
+ results[X_name][key],
1092
+ index=var_names,
1093
+ columns=y_names,
1094
+ )
1095
+
1096
+ lines = ["added"]
1097
+ for key in results[X_name]:
1098
+ if key.endswith("pvalue"):
1099
+ postfix = " with uncorrected p-values"
1100
+ elif key.endswith("p_adj"):
1101
+ postfix = " with corrected p-values"
1102
+ elif key.endswith("r"):
1103
+ postfix = " with correlation coefficients"
1104
+ elif key.endswith("slope"):
1105
+ postfix = " with regression slopes coefficients"
1106
+ elif key.endswith("r2"):
1107
+ postfix = " with models R2 coefficients"
1108
+ elif key.endswith("amplitude"):
1109
+ postfix = " with amplitudes of fitted GAMs"
1110
+ else:
1111
+ postfix = ""
1112
+
1113
+ if use_raw:
1114
+ lines.append(f" .raw.varm['<response_key>:<feature matrix>:{method}:{key}'] matrix{postfix}")
1115
+ else:
1116
+ lines.append(f" .varm['<response_key>:<feature matrix>:{method}:{key}'] matrix{postfix}")
1117
+ logg.info(" finished ({time_passed})", deep="\n".join(lines), time=start)