threadfin 4.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- threadfin/__init__.py +86 -0
- threadfin/_utils.py +260 -0
- threadfin/api.py +658 -0
- threadfin/bcrgraph.py +255 -0
- threadfin/clones.py +653 -0
- threadfin/core.py +531 -0
- threadfin/dandelion.py +458 -0
- threadfin/density.py +409 -0
- threadfin/dynamics.py +275 -0
- threadfin/export.py +215 -0
- threadfin/expression.py +121 -0
- threadfin/heritability.py +224 -0
- threadfin/integrate.py +350 -0
- threadfin/io.py +280 -0
- threadfin/maptune.py +743 -0
- threadfin/metrics.py +137 -0
- threadfin/migration.py +165 -0
- threadfin/phylo.py +834 -0
- threadfin/plotting.py +535 -0
- threadfin/pp.py +253 -0
- threadfin/profiles.py +603 -0
- threadfin/programmes.py +1028 -0
- threadfin/programs.py +133 -0
- threadfin/reclustering.py +68 -0
- threadfin/sequence.py +379 -0
- threadfin/simulate.py +206 -0
- threadfin/specificity.py +355 -0
- threadfin/stats.py +476 -0
- threadfin/tl.py +62 -0
- threadfin-4.5.0.dist-info/METADATA +288 -0
- threadfin-4.5.0.dist-info/RECORD +34 -0
- threadfin-4.5.0.dist-info/WHEEL +5 -0
- threadfin-4.5.0.dist-info/licenses/LICENSE +21 -0
- threadfin-4.5.0.dist-info/top_level.txt +1 -0
threadfin/programmes.py
ADDED
|
@@ -0,0 +1,1028 @@
|
|
|
1
|
+
"""Clonal programmes: groups of clones with similar state profiles.
|
|
2
|
+
|
|
3
|
+
A *clonal programme* is a set of genetically distinct clones whose cells,
|
|
4
|
+
relative to their sampling context, occupy similar transcriptional states
|
|
5
|
+
(for example clones biased towards plasmablast output, or clones that stay
|
|
6
|
+
in the germinal centre). Programmes are found by Leiden community detection
|
|
7
|
+
on a k-nearest-neighbour graph of clone profiles
|
|
8
|
+
(:func:`threadfin.tl.clone_profiles`).
|
|
9
|
+
|
|
10
|
+
Community detection always returns *some* partition, even when clones vary
|
|
11
|
+
continuously without forming groups. Two safeguards address this:
|
|
12
|
+
|
|
13
|
+
1. **Split test.** The Leiden communities are arranged in a tree (average
|
|
14
|
+
linkage of their centroids) and tested top-down: at every node, the clones
|
|
15
|
+
below it are compared with a single multivariate Gaussian fitted to them
|
|
16
|
+
(same mean and covariance). The statistic is the 2-means cluster index
|
|
17
|
+
(within / total sum of squares, as in SigClust, Liu et al. 2008); the
|
|
18
|
+
null distribution comes from re-clustering Gaussian samples, so the test
|
|
19
|
+
repeats the 2-means search, but does not repeat the full data-dependent
|
|
20
|
+
Leiden/tree/resolution selection. This is an approximate diagnostic. A
|
|
21
|
+
node is split only if p <= alpha * (n_node - 1) / (n_all - 1), with
|
|
22
|
+
n the number of clones. This allocation follows hierarchical-testing
|
|
23
|
+
work (Meinshausen 2008; sc-SHC, Grabski et al. 2023), but this implementation
|
|
24
|
+
does not establish family-wise error control for the selected tree. Communities below a non-significant
|
|
25
|
+
node are merged. When even the root is not significant, the clones form
|
|
26
|
+
a single programme: they differ (see the coherence test) but along a
|
|
27
|
+
continuum rather than in distinct groups.
|
|
28
|
+
2. **Bootstrap stability.** Cells are resampled within clones (Poisson
|
|
29
|
+
bootstrap), profiles are re-estimated and the whole procedure (graph,
|
|
30
|
+
Leiden, split test) is repeated; each programme is matched to its best
|
|
31
|
+
bootstrap counterpart by Jaccard overlap (Hennig 2007, "clusterboot").
|
|
32
|
+
Programmes with mean Jaccard >= 0.75 are stable; below 0.6 they should
|
|
33
|
+
not be interpreted.
|
|
34
|
+
|
|
35
|
+
The Leiden resolution is chosen as the finest one whose size-weighted
|
|
36
|
+
stability reaches the threshold, instead of being fixed by hand.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import warnings
|
|
42
|
+
|
|
43
|
+
import numpy as np
|
|
44
|
+
import pandas as pd
|
|
45
|
+
|
|
46
|
+
from ._utils import get_uns, h5ad_safe, log, require_obs, stable_categorical
|
|
47
|
+
from .profiles import get_profiles, model_from_adata, project_features
|
|
48
|
+
|
|
49
|
+
PROGRAMME_KEY = "programmes"
|
|
50
|
+
DEFAULT_RESOLUTIONS = (0.2, 0.4, 0.6, 0.8, 1.0, 1.4, 2.0)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# --------------------------------------------------------------------------- graph + Leiden
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def knn_graph(features: np.ndarray, n_neighbors: int = 15, reliability: np.ndarray | None = None):
|
|
57
|
+
"""Weighted undirected kNN graph (igraph) with self-tuned Gaussian weights.
|
|
58
|
+
|
|
59
|
+
``reliability`` scales every edge by ``sqrt(rel_i * rel_j)``, so a clone seen in few cells still
|
|
60
|
+
takes part in the clustering but pulls on it only as hard as its profile deserves. This is the
|
|
61
|
+
alternative to a reliability threshold: nothing is removed, and a community of noisy profiles is
|
|
62
|
+
held together by weak edges that Leiden will not keep as a separate module.
|
|
63
|
+
"""
|
|
64
|
+
import igraph as ig
|
|
65
|
+
from sklearn.neighbors import NearestNeighbors
|
|
66
|
+
|
|
67
|
+
n = features.shape[0]
|
|
68
|
+
k = int(min(n_neighbors, n - 1))
|
|
69
|
+
if k < 1:
|
|
70
|
+
raise ValueError("Need at least two clones to build a graph.")
|
|
71
|
+
dist, idx = NearestNeighbors(n_neighbors=k + 1).fit(features).kneighbors(features)
|
|
72
|
+
dist, idx = dist[:, 1:], idx[:, 1:] # drop self
|
|
73
|
+
sigma = np.maximum(dist[:, -1:], 1e-12) # distance to the k-th neighbour
|
|
74
|
+
w = np.exp(-(dist**2) / sigma**2)
|
|
75
|
+
rows = np.repeat(np.arange(n), k)
|
|
76
|
+
cols = idx.ravel()
|
|
77
|
+
lo, hi = np.minimum(rows, cols), np.maximum(rows, cols)
|
|
78
|
+
# merge (i,j)/(j,i) duplicates, keeping the larger weight
|
|
79
|
+
key = lo.astype(np.int64) * n + hi
|
|
80
|
+
order = np.lexsort((-w.ravel(), key))
|
|
81
|
+
key_sorted = key[order]
|
|
82
|
+
first = np.ones(key_sorted.size, dtype=bool)
|
|
83
|
+
first[1:] = key_sorted[1:] != key_sorted[:-1]
|
|
84
|
+
sel = order[first]
|
|
85
|
+
weights = w.ravel()[sel]
|
|
86
|
+
if reliability is not None:
|
|
87
|
+
r = np.clip(np.asarray(reliability, dtype=float), 1e-6, 1.0)
|
|
88
|
+
weights = weights * np.sqrt(r[lo[sel]] * r[hi[sel]])
|
|
89
|
+
g = ig.Graph(n=n, edges=list(zip(lo[sel].tolist(), hi[sel].tolist())), directed=False)
|
|
90
|
+
g.es["weight"] = weights.tolist()
|
|
91
|
+
return g
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def leiden(graph, resolution: float, seed: int = 0) -> np.ndarray:
|
|
95
|
+
"""Leiden partition (RB configuration model) -> integer labels."""
|
|
96
|
+
import leidenalg
|
|
97
|
+
|
|
98
|
+
part = leidenalg.find_partition(
|
|
99
|
+
graph,
|
|
100
|
+
leidenalg.RBConfigurationVertexPartition,
|
|
101
|
+
weights="weight",
|
|
102
|
+
resolution_parameter=float(resolution),
|
|
103
|
+
seed=int(seed),
|
|
104
|
+
)
|
|
105
|
+
return np.asarray(part.membership, dtype=np.int64)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _leiden_many(tasks, n_jobs: int = 1) -> list:
|
|
109
|
+
"""Leiden partitions for ``(graph, resolution, seed)`` tasks, in parallel processes when ``n_jobs > 1``.
|
|
110
|
+
|
|
111
|
+
Each partition depends only on its graph, resolution and seed, so the result is the same for any ``n_jobs``.
|
|
112
|
+
"""
|
|
113
|
+
if n_jobs > 1 and len(tasks) > 1:
|
|
114
|
+
from joblib import Parallel, delayed
|
|
115
|
+
|
|
116
|
+
return Parallel(n_jobs=min(n_jobs, len(tasks)))(delayed(leiden)(g, r, s) for g, r, s in tasks)
|
|
117
|
+
return [leiden(g, r, s) for g, r, s in tasks]
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def jaccard_match(ref: np.ndarray, other: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
|
|
121
|
+
"""Best-match Jaccard of every reference cluster, and per-item agreement.
|
|
122
|
+
|
|
123
|
+
Returns ``(jaccard_per_ref_cluster, agrees)``; ``agrees[i]`` is True when
|
|
124
|
+
item ``i`` sits in the bootstrap cluster that best matches its reference
|
|
125
|
+
cluster.
|
|
126
|
+
"""
|
|
127
|
+
import scipy.sparse as sp
|
|
128
|
+
|
|
129
|
+
kr, ko = int(ref.max()) + 1, int(other.max()) + 1
|
|
130
|
+
m = sp.coo_matrix((np.ones(ref.size), (ref, other)), shape=(kr, ko)).toarray()
|
|
131
|
+
size_r, size_o = m.sum(axis=1), m.sum(axis=0)
|
|
132
|
+
union = size_r[:, None] + size_o[None, :] - m
|
|
133
|
+
jac = np.divide(m, union, out=np.zeros_like(m), where=union > 0)
|
|
134
|
+
best = jac.argmax(axis=1)
|
|
135
|
+
return jac.max(axis=1), other == best[ref]
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# --------------------------------------------------------------------------- split test
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _two_means_index(x: np.ndarray, rng, extra_split=None, n_random: int = 2, n_iter: int = 30) -> float:
|
|
142
|
+
"""Smallest 2-means cluster index (within / total sum of squares) over a few starts.
|
|
143
|
+
|
|
144
|
+
Starts: the sign of the first principal component, ``n_random`` random
|
|
145
|
+
pairs of centres and, if given, ``extra_split`` (a boolean labelling).
|
|
146
|
+
"""
|
|
147
|
+
xc = x - x.mean(axis=0)
|
|
148
|
+
tss = float(np.einsum("ij,ij->", xc, xc))
|
|
149
|
+
if xc.shape[0] < 3 or tss <= 0:
|
|
150
|
+
return 1.0
|
|
151
|
+
_, _, vt = np.linalg.svd(xc, full_matrices=False)
|
|
152
|
+
starts = [xc @ vt[0] > 0]
|
|
153
|
+
for _ in range(n_random):
|
|
154
|
+
a, b = rng.choice(xc.shape[0], size=2, replace=False)
|
|
155
|
+
starts.append(((xc - xc[a]) ** 2).sum(1) < ((xc - xc[b]) ** 2).sum(1))
|
|
156
|
+
if extra_split is not None:
|
|
157
|
+
starts.append(np.asarray(extra_split, dtype=bool))
|
|
158
|
+
best = np.inf
|
|
159
|
+
for lab in starts:
|
|
160
|
+
for _ in range(n_iter): # Lloyd iterations for two centres
|
|
161
|
+
if lab.all() or not lab.any():
|
|
162
|
+
break
|
|
163
|
+
c1, c0 = xc[lab].mean(axis=0), xc[~lab].mean(axis=0)
|
|
164
|
+
new = xc @ (c1 - c0) > 0.5 * (c1 @ c1 - c0 @ c0)
|
|
165
|
+
if np.array_equal(new, lab):
|
|
166
|
+
break
|
|
167
|
+
lab = new
|
|
168
|
+
if lab.all() or not lab.any():
|
|
169
|
+
continue
|
|
170
|
+
wss = tss - lab.sum() * np.sum(xc[lab].mean(0) ** 2) - (~lab).sum() * np.sum(xc[~lab].mean(0) ** 2)
|
|
171
|
+
best = min(best, wss)
|
|
172
|
+
return float(best / tss) if np.isfinite(best) else 1.0
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _two_means_labels(x: np.ndarray, rng, extra_split=None, n_random: int = 2, n_iter: int = 30):
|
|
176
|
+
"""The labelling attaining the smallest 2-means cluster index (same starts as :func:`_two_means_index`)."""
|
|
177
|
+
xc = x - x.mean(axis=0)
|
|
178
|
+
tss = float(np.einsum("ij,ij->", xc, xc))
|
|
179
|
+
if xc.shape[0] < 3 or tss <= 0:
|
|
180
|
+
return None
|
|
181
|
+
_, _, vt = np.linalg.svd(xc, full_matrices=False)
|
|
182
|
+
starts = [xc @ vt[0] > 0]
|
|
183
|
+
for _ in range(n_random):
|
|
184
|
+
a, b = rng.choice(xc.shape[0], size=2, replace=False)
|
|
185
|
+
starts.append(((xc - xc[a]) ** 2).sum(1) < ((xc - xc[b]) ** 2).sum(1))
|
|
186
|
+
if extra_split is not None:
|
|
187
|
+
starts.append(np.asarray(extra_split, dtype=bool))
|
|
188
|
+
best, best_lab = np.inf, None
|
|
189
|
+
for lab in starts:
|
|
190
|
+
for _ in range(n_iter):
|
|
191
|
+
if lab.all() or not lab.any():
|
|
192
|
+
break
|
|
193
|
+
c1, c0 = xc[lab].mean(axis=0), xc[~lab].mean(axis=0)
|
|
194
|
+
new = xc @ (c1 - c0) > 0.5 * (c1 @ c1 - c0 @ c0)
|
|
195
|
+
if np.array_equal(new, lab):
|
|
196
|
+
break
|
|
197
|
+
lab = new
|
|
198
|
+
if lab.all() or not lab.any():
|
|
199
|
+
continue
|
|
200
|
+
wss = tss - lab.sum() * np.sum(xc[lab].mean(0) ** 2) - (~lab).sum() * np.sum(xc[~lab].mean(0) ** 2)
|
|
201
|
+
if wss < best:
|
|
202
|
+
best, best_lab = wss, lab.copy()
|
|
203
|
+
return best_lab
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def split_test(x: np.ndarray, split=None, *, n_null: int = 100, random_state=0, return_null: bool = False,
|
|
207
|
+
null: str = "total") -> dict:
|
|
208
|
+
"""Is ``x`` (clones x features) better described by two clusters than by one Gaussian?
|
|
209
|
+
|
|
210
|
+
The 2-means cluster index of ``x`` is compared with its distribution in ``n_null`` Gaussian samples.
|
|
211
|
+
``null`` chooses that Gaussian's covariance. **Both options have been calibrated on real clone profiles
|
|
212
|
+
with a planted two-group structure; the measured numbers are quoted below, and neither setting is satisfactory:**
|
|
213
|
+
|
|
214
|
+
``"total"`` (default)
|
|
215
|
+
the sample covariance of ``x`` itself (SigClust-style). Valid but very conservative: the false-positive
|
|
216
|
+
rate at zero separation is 0 of 30 simulations, and so is the detection rate for two groups separated by
|
|
217
|
+
four within-group standard deviations. A split inflates the covariance along the splitting direction and
|
|
218
|
+
the null inherits that inflation, so the test partly cancels itself. **A non-significant result from
|
|
219
|
+
this test is therefore not evidence of a continuum.**
|
|
220
|
+
``"within"``
|
|
221
|
+
the covariance pooled *within* the two candidate groups. **Invalid — do not use for inference.** It was
|
|
222
|
+
tried as a fix and measured: on mouse_rbd clone profiles it reports a split in 30 of 30 simulations with
|
|
223
|
+
*no* planted structure (false-positive rate 1.0), because removing the candidate split from the
|
|
224
|
+
covariance estimate is circular. Kept only so that the calibration can be reproduced.
|
|
225
|
+
|
|
226
|
+
The p-value uses a normal fit to the null indices (so that it can be smaller than ``1 / n_null``); the
|
|
227
|
+
empirical p-value is also returned (and, with ``return_null``, the null indices themselves).
|
|
228
|
+
"""
|
|
229
|
+
from scipy.stats import norm
|
|
230
|
+
|
|
231
|
+
if null not in ("within", "total"):
|
|
232
|
+
raise ValueError("null must be 'within' or 'total'.")
|
|
233
|
+
rng = np.random.default_rng(random_state)
|
|
234
|
+
x = np.asarray(x, dtype=float)
|
|
235
|
+
observed = _two_means_index(x, rng, extra_split=split)
|
|
236
|
+
if null == "within":
|
|
237
|
+
lab = _two_means_labels(x, rng, extra_split=split)
|
|
238
|
+
if lab is None or lab.all() or not lab.any():
|
|
239
|
+
cov = np.atleast_2d(np.cov(x, rowvar=False))
|
|
240
|
+
else:
|
|
241
|
+
centred = x.copy()
|
|
242
|
+
centred[lab] -= x[lab].mean(axis=0)
|
|
243
|
+
centred[~lab] -= x[~lab].mean(axis=0)
|
|
244
|
+
dof = max(x.shape[0] - 2, 1)
|
|
245
|
+
cov = np.atleast_2d(centred.T @ centred / dof)
|
|
246
|
+
else:
|
|
247
|
+
cov = np.atleast_2d(np.cov(x, rowvar=False))
|
|
248
|
+
w, v = np.linalg.eigh(cov)
|
|
249
|
+
root = v * np.sqrt(np.clip(w, 0, None))
|
|
250
|
+
null = np.array([_two_means_index(rng.standard_normal(x.shape) @ root.T, rng) for _ in range(n_null)])
|
|
251
|
+
sd = float(null.std(ddof=1)) if n_null > 1 else np.nan
|
|
252
|
+
p_norm = float(norm.cdf((observed - null.mean()) / sd)) if np.isfinite(sd) and sd > 0 else float("nan")
|
|
253
|
+
p_emp = float((1 + np.sum(null <= observed)) / (n_null + 1))
|
|
254
|
+
if not np.isfinite(p_norm):
|
|
255
|
+
p_norm = p_emp # A degenerate null must not turn identical data into p=0.
|
|
256
|
+
out = {"cluster_index": observed, "null_mean": float(null.mean()), "null_sd": sd,
|
|
257
|
+
"pvalue": p_norm, "pvalue_empirical": p_emp}
|
|
258
|
+
if return_null:
|
|
259
|
+
out["null"] = null
|
|
260
|
+
return out
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _merge_by_split_test(feats, labels, alpha, n_null, rng, null="total"):
|
|
264
|
+
"""Test the community tree top-down and merge communities below non-significant nodes.
|
|
265
|
+
|
|
266
|
+
Returns ``(merged_labels, tests)`` with merged labels ``0..K-1`` and one
|
|
267
|
+
row per tested node.
|
|
268
|
+
"""
|
|
269
|
+
from scipy.cluster.hierarchy import linkage, to_tree
|
|
270
|
+
|
|
271
|
+
names = np.unique(labels)
|
|
272
|
+
if names.size < 2:
|
|
273
|
+
return np.zeros(labels.size, dtype=np.int64), pd.DataFrame()
|
|
274
|
+
centroids = np.vstack([feats[labels == k].mean(axis=0) for k in names])
|
|
275
|
+
root = to_tree(linkage(centroids, method="average"))
|
|
276
|
+
total = labels.size
|
|
277
|
+
rows, groups = [], []
|
|
278
|
+
|
|
279
|
+
def visit(node):
|
|
280
|
+
if node.is_leaf():
|
|
281
|
+
groups.append([names[node.id]])
|
|
282
|
+
return
|
|
283
|
+
a = [names[i] for i in node.left.pre_order()]
|
|
284
|
+
b = [names[i] for i in node.right.pre_order()]
|
|
285
|
+
inside = np.isin(labels, a + b)
|
|
286
|
+
res = split_test(feats[inside], np.isin(labels[inside], a), n_null=n_null, null=null,
|
|
287
|
+
random_state=int(rng.integers(2**31)))
|
|
288
|
+
level = alpha * (inside.sum() - 1) / (total - 1)
|
|
289
|
+
# communities as comma-separated text, so the table can be written to h5ad with the AnnData
|
|
290
|
+
rows.append({"communities_left": ",".join(str(int(c)) for c in a),
|
|
291
|
+
"communities_right": ",".join(str(int(c)) for c in b),
|
|
292
|
+
"n_communities_left": len(a), "n_communities_right": len(b),
|
|
293
|
+
"n_clones": int(inside.sum()), **res,
|
|
294
|
+
"alpha_node": float(level), "split": bool(res["pvalue"] <= level)})
|
|
295
|
+
if res["pvalue"] <= level:
|
|
296
|
+
visit(node.left)
|
|
297
|
+
visit(node.right)
|
|
298
|
+
else:
|
|
299
|
+
groups.append(a + b)
|
|
300
|
+
|
|
301
|
+
visit(root)
|
|
302
|
+
merged = np.empty(labels.size, dtype=np.int64)
|
|
303
|
+
for g, members in enumerate(groups):
|
|
304
|
+
merged[np.isin(labels, members)] = g
|
|
305
|
+
tests = pd.DataFrame(rows)
|
|
306
|
+
if not tests.empty:
|
|
307
|
+
to_group = {int(c): g for g, members in enumerate(groups) for c in members}
|
|
308
|
+
for side in ("left", "right"):
|
|
309
|
+
tests[f"groups_{side}"] = [sorted({to_group[int(c)] for c in cs.split(",")})
|
|
310
|
+
for cs in tests[f"communities_{side}"]]
|
|
311
|
+
return merged, tests
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
# --------------------------------------------------------------------------- bootstrap
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _bootstrap_features(model, adata, eligible_codes, n_boot, rng):
|
|
318
|
+
"""Feature matrices of the eligible clones under ``n_boot`` Poisson bootstraps."""
|
|
319
|
+
n_all = len(model.clone_index)
|
|
320
|
+
remap = -np.ones(n_all, dtype=np.int64)
|
|
321
|
+
remap[eligible_codes] = np.arange(eligible_codes.size)
|
|
322
|
+
groups = np.where(model.clone_codes >= 0, remap[np.maximum(model.clone_codes, 0)], -1)
|
|
323
|
+
in_group = groups >= 0
|
|
324
|
+
out = []
|
|
325
|
+
for _ in range(n_boot):
|
|
326
|
+
w = rng.poisson(1.0, size=groups.size).astype(float)
|
|
327
|
+
# a clone that draws no cells keeps its original cells (avoid empty clones)
|
|
328
|
+
tot = np.bincount(groups[in_group], weights=w[in_group], minlength=eligible_codes.size)
|
|
329
|
+
empty = np.flatnonzero(tot == 0)
|
|
330
|
+
if empty.size:
|
|
331
|
+
w[np.isin(groups, empty)] = 1.0
|
|
332
|
+
blup, _, _ = model.group_blups(groups, eligible_codes.size, weights=w)
|
|
333
|
+
out.append(project_features(adata, blup))
|
|
334
|
+
return out
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
# --------------------------------------------------------------------------- public API
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def embed_clones(
|
|
342
|
+
adata,
|
|
343
|
+
*,
|
|
344
|
+
n_neighbors: int | str = "auto",
|
|
345
|
+
min_dist: float | str = "auto",
|
|
346
|
+
spread: float = 1.0,
|
|
347
|
+
min_reliability: float = 0.0,
|
|
348
|
+
coherence_target: float | str = "auto",
|
|
349
|
+
tune_kwargs: dict | None = None,
|
|
350
|
+
random_state: int = 0,
|
|
351
|
+
n_jobs: int | None = 1,
|
|
352
|
+
verbose: bool = True,
|
|
353
|
+
) -> pd.DataFrame:
|
|
354
|
+
"""Lay the clone profiles out in two dimensions — **every** profiled clone, by default.
|
|
355
|
+
|
|
356
|
+
This is the clone map the rest of the package is built around, and it is deliberately separate from
|
|
357
|
+
:func:`find_programmes`. Programme discovery is a clustering problem and needs reasonably estimated
|
|
358
|
+
profiles, so it keeps its ``min_reliability`` gate. The map is a *display* of the clones that exist,
|
|
359
|
+
and filtering it by reliability throws away the data the map is meant to show: on GSE287123 the 0.5
|
|
360
|
+
gate removes 75 % (RBD) and 90 % (NP-OVA) of profiled clones, and on GSE246382 it removes all of
|
|
361
|
+
them, which is why that dataset appeared to have no clone map at all.
|
|
362
|
+
|
|
363
|
+
The map does not change any statistic: profiles, coherence, programmes and tests are computed in
|
|
364
|
+
profile space, never on the map.
|
|
365
|
+
|
|
366
|
+
Parameters
|
|
367
|
+
----------
|
|
368
|
+
n_neighbors, min_dist
|
|
369
|
+
UMAP settings. ``"auto"`` (the default for both) chooses them per dataset with
|
|
370
|
+
:func:`threadfin.tl.tune_clone_map`: a ladder of settings, each laid out with several seeds and
|
|
371
|
+
smoothed over its neighbours, finds the most faithful layout that is no more fragmented than the
|
|
372
|
+
clone profiles themselves, so a continuum of clones is drawn as one connected picture. Pass numbers to set them
|
|
373
|
+
by hand (``n_neighbors=50, min_dist=0.8`` reproduces the fixed default of Threadfin 4.2); a
|
|
374
|
+
number for one and ``"auto"`` for the other tunes only the second.
|
|
375
|
+
spread
|
|
376
|
+
UMAP ``spread``.
|
|
377
|
+
min_reliability
|
|
378
|
+
Reliability floor. ``0.0`` (the default) keeps every clone that has a profile; raise it only to
|
|
379
|
+
reproduce an older filtered layout.
|
|
380
|
+
coherence_target, tune_kwargs
|
|
381
|
+
Passed to :func:`threadfin.tl.tune_clone_map` when a setting is ``"auto"``.
|
|
382
|
+
random_state
|
|
383
|
+
Seed for the tuner and for the final layout.
|
|
384
|
+
n_jobs
|
|
385
|
+
Processes for the layouts of the automatic search (``-1``: all available cores). The chosen
|
|
386
|
+
settings and the map do not depend on it.
|
|
387
|
+
|
|
388
|
+
Returns
|
|
389
|
+
-------
|
|
390
|
+
The clone table with ``x`` and ``y`` added. ``adata.uns['threadfin']['profiles']['map_params']``
|
|
391
|
+
records the settings, how they were chosen (``"tuning"``, with the full optimisation history), the
|
|
392
|
+
quality of the final map (``"quality"``: coherence, local and global fidelity) and how many clones
|
|
393
|
+
were laid out.
|
|
394
|
+
"""
|
|
395
|
+
from .maptune import layout_clone_map, tune_clone_map
|
|
396
|
+
|
|
397
|
+
prof = get_profiles(adata)
|
|
398
|
+
table, feats_all = prof["clone_table"], prof["features"]
|
|
399
|
+
use = table.index[table["reliability"] >= min_reliability]
|
|
400
|
+
if use.size < 15:
|
|
401
|
+
raise ValueError(
|
|
402
|
+
f"Only {use.size} clones to embed (need 15). Profile more clones, or lower min_cells in "
|
|
403
|
+
"clone_profiles.")
|
|
404
|
+
for name, value in (("n_neighbors", n_neighbors), ("min_dist", min_dist)):
|
|
405
|
+
if isinstance(value, str) and value != "auto":
|
|
406
|
+
raise ValueError(f"{name} must be 'auto' or a number, got {value!r}.")
|
|
407
|
+
feats = feats_all.loc[use].to_numpy()
|
|
408
|
+
tuning, repulsion = None, 1.0
|
|
409
|
+
if n_neighbors == "auto" or min_dist == "auto":
|
|
410
|
+
tuning = tune_clone_map(feats, n_neighbors=n_neighbors, min_dist=min_dist, spread=spread,
|
|
411
|
+
coherence_target=coherence_target, random_state=random_state,
|
|
412
|
+
n_jobs=n_jobs, verbose=verbose, **(tune_kwargs or {}))
|
|
413
|
+
n_neighbors, min_dist, repulsion = tuning.n_neighbors, tuning.min_dist, tuning.repulsion_strength
|
|
414
|
+
|
|
415
|
+
xy, quality, setting, attempts = layout_clone_map(
|
|
416
|
+
feats, n_neighbors=n_neighbors, min_dist=min_dist, repulsion_strength=repulsion, spread=spread,
|
|
417
|
+
coherence_target=coherence_target, tuning=tuning, random_state=random_state)
|
|
418
|
+
n_neighbors, min_dist, repulsion, layout_seed = setting
|
|
419
|
+
table["x"], table["y"] = np.nan, np.nan
|
|
420
|
+
table.loc[use, "x"], table.loc[use, "y"] = xy[:, 0], xy[:, 1]
|
|
421
|
+
prof["clone_table"] = table
|
|
422
|
+
prof["map_params"] = {"n_neighbors": int(n_neighbors), "min_dist": float(min_dist),
|
|
423
|
+
"repulsion_strength": float(repulsion), "spread": float(spread),
|
|
424
|
+
"min_reliability": float(min_reliability), "random_state": int(random_state),
|
|
425
|
+
"layout_seed": int(layout_seed), "n_clones": int(use.size),
|
|
426
|
+
"n_profiled": int(len(table)),
|
|
427
|
+
"chosen_by": "tune_clone_map" if tuning is not None else "user",
|
|
428
|
+
"quality": {k: float(v) for k, v in quality.items()},
|
|
429
|
+
"attempts": pd.DataFrame(attempts)}
|
|
430
|
+
if tuning is not None:
|
|
431
|
+
prof["map_params"]["tuning"] = tuning.to_uns()
|
|
432
|
+
rep_txt = "" if repulsion == 1.0 else f", repulsion_strength {repulsion:g}"
|
|
433
|
+
log(f"clone map: {use.size:,} of {len(table):,} profiled clones laid out (n_neighbors {int(n_neighbors)}, "
|
|
434
|
+
f"min_dist {float(min_dist):.2f}{rep_txt}{', chosen automatically' if tuning is not None else ''}); "
|
|
435
|
+
f"{100 * quality['coherence']:.1f}% of clones in one connected piece.", verbose)
|
|
436
|
+
if tuning is None and quality["coherence"] < quality["coherence_target"] - 0.005:
|
|
437
|
+
log(f" the map is more fragmented than the clone profiles (which keep "
|
|
438
|
+
f"{100 * quality['coherence_target']:.1f}% of clones connected); n_neighbors='auto', "
|
|
439
|
+
"min_dist='auto' chooses settings that keep it together.", verbose)
|
|
440
|
+
return table
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def find_programmes(
|
|
444
|
+
adata,
|
|
445
|
+
*,
|
|
446
|
+
resolution: float | str = "auto",
|
|
447
|
+
resolutions=DEFAULT_RESOLUTIONS,
|
|
448
|
+
n_neighbors: int = 15,
|
|
449
|
+
min_reliability: float = 0.0,
|
|
450
|
+
weight_by_reliability: bool = True,
|
|
451
|
+
n_boot: int = 30,
|
|
452
|
+
stability_threshold: float = 0.75,
|
|
453
|
+
min_programme_size: int = 5,
|
|
454
|
+
min_clones: int = 30,
|
|
455
|
+
test_splits: bool = True,
|
|
456
|
+
alpha: float = 0.05,
|
|
457
|
+
n_null: int = 100,
|
|
458
|
+
split_null: str = "total",
|
|
459
|
+
assign_remaining: bool = True,
|
|
460
|
+
min_posterior: float = 0.7,
|
|
461
|
+
random_state: int = 0,
|
|
462
|
+
n_jobs: int | None = 1,
|
|
463
|
+
key_added: str = "clone_programme",
|
|
464
|
+
verbose: bool = True,
|
|
465
|
+
) -> pd.DataFrame:
|
|
466
|
+
"""Group clones into clonal programmes, with bootstrap stability.
|
|
467
|
+
|
|
468
|
+
Parameters
|
|
469
|
+
----------
|
|
470
|
+
adata
|
|
471
|
+
AnnData after :func:`threadfin.tl.clone_profiles`.
|
|
472
|
+
resolution
|
|
473
|
+
Leiden resolution, or ``"auto"`` to scan ``resolutions`` and keep the
|
|
474
|
+
finest partition whose size-weighted stability is at least
|
|
475
|
+
``stability_threshold`` (falling back to the most stable partition).
|
|
476
|
+
n_neighbors
|
|
477
|
+
Neighbours per clone in the clone graph.
|
|
478
|
+
weight_by_reliability
|
|
479
|
+
Scale each graph edge by ``sqrt(rel_i * rel_j)`` so that every clone is clustered but a
|
|
480
|
+
thinly observed one pulls weakly. This is what makes ``min_reliability=0`` safe: on a
|
|
481
|
+
simulated one-programme continuum of 3,000 clones with a median of one cell each, an
|
|
482
|
+
unweighted unfiltered run invents four programmes at bootstrap stability 0.78-0.86, while the
|
|
483
|
+
weighted run recovers the single true programme.
|
|
484
|
+
min_reliability
|
|
485
|
+
Reliability floor for clustering; ``0.0`` (the default) clusters every profiled clone.
|
|
486
|
+
Threadfin does not filter clones. Every clone, and every cell in it, is measured data, and
|
|
487
|
+
the profiles are already **shrunk**: a clone seen in few cells has its profile pulled towards the
|
|
488
|
+
population mean by the variance-component model, so its influence is down-weighted in proportion to
|
|
489
|
+
how little was seen. Cutting on reliability *on top of* that shrinkage discards the same information
|
|
490
|
+
twice, and it discards it silently — on GSE287123 a 0.5 floor removes 75 % (RBD) and 90 % (NP-OVA) of
|
|
491
|
+
profiled clones, and on GSE246382 it removes all of them, which is how that dataset came to look as
|
|
492
|
+
though it had no clone structure when its map separates the sorted gate at p < 0.001.
|
|
493
|
+
n_boot
|
|
494
|
+
Bootstrap replicates for stability (0 disables stability, which is
|
|
495
|
+
only allowed with a fixed ``resolution``).
|
|
496
|
+
min_programme_size
|
|
497
|
+
Groups with fewer clones are not reported as programmes (their clones
|
|
498
|
+
stay unassigned); resolutions that leave more than 10% of clones in
|
|
499
|
+
such groups are not selected.
|
|
500
|
+
min_clones
|
|
501
|
+
Minimum number of reliable clones; with fewer, programme detection is
|
|
502
|
+
refused (report the coherence test only).
|
|
503
|
+
test_splits, alpha, n_null
|
|
504
|
+
Merge communities that are not significantly better described as two
|
|
505
|
+
groups than as one Gaussian (see the module docstring); ``alpha`` is
|
|
506
|
+
a nominal hierarchical threshold (not a validated family-wise error guarantee) and ``n_null`` the
|
|
507
|
+
number of Gaussian null samples per test. The node tests are stored
|
|
508
|
+
in ``adata.uns['threadfin']['programmes']['split_tests']``. A single
|
|
509
|
+
programme means "no distinct programmes".
|
|
510
|
+
assign_remaining
|
|
511
|
+
Programmes are *defined* on reliable ("core") clones only. With this
|
|
512
|
+
option every other profiled clone is then *assigned* to the most
|
|
513
|
+
likely programme by a Gaussian classifier whose noise term shrinks
|
|
514
|
+
with clone size (see ``_assign_remaining``); assignments with
|
|
515
|
+
posterior >= ``min_posterior`` go to ``<key_added>_assigned``.
|
|
516
|
+
n_jobs
|
|
517
|
+
Processes for the Leiden runs on the bootstrap graphs (``-1``: all
|
|
518
|
+
available cores). Programmes and stabilities are identical for any
|
|
519
|
+
value; only the run time changes.
|
|
520
|
+
key_added
|
|
521
|
+
``obs`` column receiving each cell's programme (cells of clones that
|
|
522
|
+
were not clustered get NaN).
|
|
523
|
+
|
|
524
|
+
Returns
|
|
525
|
+
-------
|
|
526
|
+
Programme summary: ``n_clones``, ``n_cells``, ``stability`` (mean
|
|
527
|
+
bootstrap Jaccard) and ``mean_reliability`` per programme. Clone-level
|
|
528
|
+
assignments (with per-clone bootstrap ``confidence``) are added to
|
|
529
|
+
``adata.uns['threadfin']['profiles']['clone_table']``.
|
|
530
|
+
"""
|
|
531
|
+
prof = get_profiles(adata)
|
|
532
|
+
table = prof["clone_table"]
|
|
533
|
+
feats_all = prof["features"]
|
|
534
|
+
eligible = table.index[table["reliability"] >= min_reliability]
|
|
535
|
+
if eligible.size < min_clones:
|
|
536
|
+
raise ValueError(
|
|
537
|
+
f"Only {eligible.size} clones reach reliability {min_reliability} (need {min_clones}); "
|
|
538
|
+
"too few expanded clones to define programmes. Report the coherence test only, or "
|
|
539
|
+
"lower min_reliability."
|
|
540
|
+
)
|
|
541
|
+
feats = feats_all.loc[eligible].to_numpy()
|
|
542
|
+
rel = table.loc[eligible, "reliability"].to_numpy() if weight_by_reliability else None
|
|
543
|
+
graph = knn_graph(feats, n_neighbors, rel)
|
|
544
|
+
|
|
545
|
+
auto = isinstance(resolution, str)
|
|
546
|
+
if auto and resolution != "auto":
|
|
547
|
+
raise ValueError("resolution must be a number or 'auto'.")
|
|
548
|
+
if auto and n_boot < 5:
|
|
549
|
+
raise ValueError("resolution='auto' needs n_boot >= 5 bootstrap replicates.")
|
|
550
|
+
res_grid = list(resolutions) if auto else [float(resolution)]
|
|
551
|
+
|
|
552
|
+
rng = np.random.default_rng(random_state)
|
|
553
|
+
boot_graphs, boot_feats = [], []
|
|
554
|
+
if n_boot > 0:
|
|
555
|
+
model = model_from_adata(adata)
|
|
556
|
+
code_of = {str(c): i for i, c in enumerate(model.clone_index.astype(str))}
|
|
557
|
+
eligible_codes = np.array([code_of[c] for c in eligible], dtype=np.int64)
|
|
558
|
+
boot_feats = _bootstrap_features(model, adata, eligible_codes, n_boot, rng)
|
|
559
|
+
boot_graphs = [knn_graph(f, n_neighbors, rel) for f in boot_feats]
|
|
560
|
+
|
|
561
|
+
# every Leiden partition of the scan (reference and bootstrap graphs at every resolution), computed once
|
|
562
|
+
from .maptune import resolve_n_jobs
|
|
563
|
+
|
|
564
|
+
tasks = [(graph, res, random_state) for res in res_grid]
|
|
565
|
+
tasks += [(bg, res, random_state + b + 1) for res in res_grid for b, bg in enumerate(boot_graphs)]
|
|
566
|
+
parts = iter(_leiden_many(tasks, resolve_n_jobs(n_jobs)))
|
|
567
|
+
ref_part = {res: next(parts) for res in res_grid}
|
|
568
|
+
boot_part = {(res, b): next(parts) for res in res_grid for b in range(len(boot_graphs))}
|
|
569
|
+
|
|
570
|
+
scan_rows, partitions = [], {}
|
|
571
|
+
for res in res_grid:
|
|
572
|
+
ref = ref_part[res]
|
|
573
|
+
k = int(ref.max()) + 1
|
|
574
|
+
jac_sum = np.zeros(k)
|
|
575
|
+
agree = np.zeros(ref.size)
|
|
576
|
+
for b in range(len(boot_graphs)):
|
|
577
|
+
jac, ok = jaccard_match(ref, boot_part[(res, b)])
|
|
578
|
+
jac_sum += jac
|
|
579
|
+
agree += ok
|
|
580
|
+
nb = max(len(boot_graphs), 1)
|
|
581
|
+
jac_mean = jac_sum / nb if boot_graphs else np.full(k, np.nan)
|
|
582
|
+
sizes = np.bincount(ref, minlength=k)
|
|
583
|
+
big = sizes >= min_programme_size
|
|
584
|
+
frac_small = float(sizes[~big].sum() / sizes.sum())
|
|
585
|
+
weighted = (float(np.sum(jac_mean[big] * sizes[big]) / max(sizes[big].sum(), 1))
|
|
586
|
+
if boot_graphs and big.any() else np.nan)
|
|
587
|
+
partitions[res] = (ref, jac_mean, agree / nb if boot_graphs else np.full(ref.size, np.nan))
|
|
588
|
+
scan_rows.append({
|
|
589
|
+
"resolution": res, "n_programmes": int(big.sum()), "n_groups": k,
|
|
590
|
+
"frac_clones_in_small_groups": frac_small, "stability_weighted": weighted,
|
|
591
|
+
"stability_min": float(np.min(jac_mean[big])) if boot_graphs and big.any() else np.nan,
|
|
592
|
+
})
|
|
593
|
+
scan = pd.DataFrame(scan_rows)
|
|
594
|
+
|
|
595
|
+
caution = None # raised only if programmes survive the split test (below)
|
|
596
|
+
if auto:
|
|
597
|
+
multi = scan[(scan["n_programmes"] >= 2) & (scan["frac_clones_in_small_groups"] <= 0.10)]
|
|
598
|
+
stable = multi[multi["stability_weighted"] >= stability_threshold]
|
|
599
|
+
if not stable.empty:
|
|
600
|
+
best = stable.sort_values(["n_programmes", "stability_weighted"], ascending=False).iloc[0]
|
|
601
|
+
elif not multi.empty:
|
|
602
|
+
best = multi.sort_values("stability_weighted", ascending=False).iloc[0]
|
|
603
|
+
if boot_graphs: # without bootstrap replicates there is no stability to fall short of
|
|
604
|
+
caution = (f"No resolution reached stability {stability_threshold}; using the most stable "
|
|
605
|
+
f"partition (stability {best['stability_weighted']:.2f}). Interpret programmes "
|
|
606
|
+
"cautiously.")
|
|
607
|
+
else:
|
|
608
|
+
best = scan.iloc[0]
|
|
609
|
+
chosen = float(best["resolution"])
|
|
610
|
+
else:
|
|
611
|
+
chosen = res_grid[0]
|
|
612
|
+
|
|
613
|
+
ref, jac_mean, confidence = partitions[chosen]
|
|
614
|
+
# groups below the minimum size are not programmes: their clones stay unassigned
|
|
615
|
+
sizes = np.bincount(ref)
|
|
616
|
+
keep_mask = sizes[ref] >= min_programme_size
|
|
617
|
+
eligible = eligible[keep_mask]
|
|
618
|
+
feats = feats[keep_mask]
|
|
619
|
+
confidence = confidence[keep_mask]
|
|
620
|
+
ref = ref[keep_mask]
|
|
621
|
+
split_tests = pd.DataFrame()
|
|
622
|
+
if test_splits:
|
|
623
|
+
ref, split_tests = _merge_by_split_test(feats, ref, alpha, n_null, rng, split_null)
|
|
624
|
+
if boot_graphs:
|
|
625
|
+
# stability of the final programmes: repeat Leiden *and* the split test in every bootstrap
|
|
626
|
+
jac_mean = np.zeros(int(ref.max()) + 1)
|
|
627
|
+
agree = np.zeros(ref.size)
|
|
628
|
+
for b, bf in enumerate(boot_feats):
|
|
629
|
+
bl = boot_part[(chosen, b)][keep_mask]
|
|
630
|
+
bl, _ = _merge_by_split_test(bf[keep_mask], bl, alpha, max(n_null // 4, 20), rng, split_null)
|
|
631
|
+
jac, ok = jaccard_match(ref, bl)
|
|
632
|
+
jac_mean += jac
|
|
633
|
+
agree += ok
|
|
634
|
+
jac_mean /= len(boot_graphs)
|
|
635
|
+
confidence = agree / len(boot_graphs)
|
|
636
|
+
else:
|
|
637
|
+
jac_mean = np.full(int(ref.max()) + 1, np.nan)
|
|
638
|
+
labels = stable_categorical(ref, prefix="P")
|
|
639
|
+
raw_to_label = dict(zip(ref, labels))
|
|
640
|
+
|
|
641
|
+
# write clone-level results
|
|
642
|
+
table = table.copy()
|
|
643
|
+
table[key_added] = pd.Categorical([None] * len(table), categories=labels.categories)
|
|
644
|
+
table.loc[eligible, key_added] = np.asarray(labels)
|
|
645
|
+
table["confidence"] = np.nan
|
|
646
|
+
table.loc[eligible, "confidence"] = confidence
|
|
647
|
+
table["core"] = table.index.isin(eligible)
|
|
648
|
+
if assign_remaining:
|
|
649
|
+
model = model if n_boot > 0 else model_from_adata(adata)
|
|
650
|
+
prof["clone_table"] = table
|
|
651
|
+
best_lab, post = _assign_remaining(adata, model, eligible, np.asarray(labels, dtype=str))
|
|
652
|
+
assigned = pd.Series(np.where(post >= min_posterior, best_lab, None), index=table.index, dtype=object)
|
|
653
|
+
assigned.loc[eligible] = np.asarray(labels, dtype=str) # core clones keep their own programme
|
|
654
|
+
table[f"{key_added}_assigned"] = pd.Categorical(assigned, categories=labels.categories)
|
|
655
|
+
table["assignment_posterior"] = post
|
|
656
|
+
prof["clone_table"] = table
|
|
657
|
+
|
|
658
|
+
# map to cells
|
|
659
|
+
clone_key = prof["params"]["clone_key"]
|
|
660
|
+
cell_clone = adata.obs[clone_key].astype("object")
|
|
661
|
+
for col in [key_added] + ([f"{key_added}_assigned"] if assign_remaining else []):
|
|
662
|
+
mapping = table[col].dropna().astype(str)
|
|
663
|
+
adata.obs[col] = pd.Categorical(
|
|
664
|
+
cell_clone.map(lambda c, m=mapping: m.get(str(c)) if pd.notna(c) else None),
|
|
665
|
+
categories=labels.categories,
|
|
666
|
+
)
|
|
667
|
+
|
|
668
|
+
stab = {raw_to_label[r]: float(jac_mean[r]) for r in np.unique(ref)}
|
|
669
|
+
elig_tab = table.loc[eligible]
|
|
670
|
+
summary = (
|
|
671
|
+
elig_tab.groupby(key_added, observed=True)
|
|
672
|
+
.agg(n_clones=("n_cells", "size"), n_cells=("n_cells", "sum"),
|
|
673
|
+
mean_reliability=("reliability", "mean"), mean_confidence=("confidence", "mean"))
|
|
674
|
+
)
|
|
675
|
+
summary.index = pd.Index(summary.index.astype(str), name="programme")
|
|
676
|
+
summary["stability"] = [stab[p] for p in summary.index]
|
|
677
|
+
if assign_remaining:
|
|
678
|
+
n_assigned = table[f"{key_added}_assigned"].astype(str).value_counts()
|
|
679
|
+
summary["n_clones_with_assigned"] = [int(n_assigned.get(p, 0)) for p in summary.index]
|
|
680
|
+
if not split_tests.empty: # name the two sides of every tested node by the final programmes
|
|
681
|
+
split_tests = split_tests.copy()
|
|
682
|
+
for side in ("left", "right"):
|
|
683
|
+
split_tests[f"programmes_{side}"] = [", ".join(sorted({str(raw_to_label[g]) for g in gs if g in raw_to_label},
|
|
684
|
+
key=lambda t: (len(t), t)))
|
|
685
|
+
for gs in split_tests.pop(f"groups_{side}")]
|
|
686
|
+
get_uns(adata)[PROGRAMME_KEY] = {
|
|
687
|
+
"summary": summary,
|
|
688
|
+
"resolution_scan": h5ad_safe(scan),
|
|
689
|
+
"split_tests": h5ad_safe(split_tests),
|
|
690
|
+
"params": {
|
|
691
|
+
"resolution": chosen, "auto": auto, "n_neighbors": int(n_neighbors),
|
|
692
|
+
"weight_by_reliability": bool(weight_by_reliability),
|
|
693
|
+
"test_splits": bool(test_splits), "alpha": float(alpha), "n_null": int(n_null),
|
|
694
|
+
"split_null": str(split_null),
|
|
695
|
+
"min_reliability": float(min_reliability), "n_boot": int(n_boot),
|
|
696
|
+
"stability_threshold": float(stability_threshold), "key_added": key_added,
|
|
697
|
+
"min_programme_size": int(min_programme_size),
|
|
698
|
+
"assign_remaining": bool(assign_remaining), "min_posterior": float(min_posterior),
|
|
699
|
+
"random_state": int(random_state),
|
|
700
|
+
},
|
|
701
|
+
}
|
|
702
|
+
n_stable = int((summary["stability"] >= stability_threshold).sum()) if n_boot else 0
|
|
703
|
+
if caution and summary.shape[0] >= 2 and n_stable < summary.shape[0]:
|
|
704
|
+
# with a single programme left there is nothing to interpret, and the log says so
|
|
705
|
+
warnings.warn(caution, stacklevel=2)
|
|
706
|
+
extra = (f"; {int(table[f'{key_added}_assigned'].notna().sum())} of {len(table)} profiled clones "
|
|
707
|
+
f"assigned with posterior >= {min_posterior}" if assign_remaining else "")
|
|
708
|
+
if summary.shape[0] == 1:
|
|
709
|
+
log(f"programmes: no significant split among {eligible.size} core clones (resolution {chosen}): the "
|
|
710
|
+
"clones differ, but along a continuum rather than in distinct programmes.", verbose)
|
|
711
|
+
else:
|
|
712
|
+
n_merged = int((~split_tests["split"]).sum()) if not split_tests.empty else 0
|
|
713
|
+
merged_txt = f" after merging {n_merged} non-significant split(s)" if n_merged else ""
|
|
714
|
+
log(
|
|
715
|
+
f"programmes: {summary.shape[0]} programmes over {eligible.size} core clones "
|
|
716
|
+
f"(resolution {chosen}{merged_txt}); {n_stable} with bootstrap stability >= {stability_threshold}{extra}.",
|
|
717
|
+
verbose,
|
|
718
|
+
)
|
|
719
|
+
return summary
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
# --------------------------------------------------------------------------- posterior assignment
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
def _assign_remaining(adata, model, core_ids, core_labels):
|
|
726
|
+
"""Assign every profiled clone to a programme with a Gaussian classifier.
|
|
727
|
+
|
|
728
|
+
Each programme ``P`` is a Gaussian over (unshrunk) clone mean profiles
|
|
729
|
+
with centre ``mu_P`` and between-clone variance ``t_P`` estimated from its
|
|
730
|
+
core clones (sampling noise removed). A clone with ``n`` cells is observed
|
|
731
|
+
with extra noise ``sigma2 / n``, so small clones get flatter posteriors:
|
|
732
|
+
|
|
733
|
+
p(P | clone) ~ pi_P * N(mean_clone; mu_P, t_P + sigma2 / n)
|
|
734
|
+
|
|
735
|
+
Returns the most likely programme and its posterior for every row of the
|
|
736
|
+
clone table.
|
|
737
|
+
"""
|
|
738
|
+
import scipy.sparse as sp
|
|
739
|
+
from scipy.special import logsumexp
|
|
740
|
+
|
|
741
|
+
prof = get_profiles(adata)
|
|
742
|
+
table = prof["clone_table"]
|
|
743
|
+
code_of = {str(c): i for i, c in enumerate(model.clone_index.astype(str))}
|
|
744
|
+
rows = np.array([code_of[c] for c in table.index], dtype=np.int64)
|
|
745
|
+
g = model.clone_codes
|
|
746
|
+
ok = np.flatnonzero(g >= 0)
|
|
747
|
+
member = sp.csr_matrix((np.ones(ok.size), (g[ok], ok)), shape=(len(model.clone_index), g.size))
|
|
748
|
+
n = np.asarray(member.sum(axis=1)).ravel()
|
|
749
|
+
means = np.asarray(member @ model.residuals) / np.maximum(n, 1)[:, None] - model.vc.grand_mean
|
|
750
|
+
means, n = means[rows], n[rows]
|
|
751
|
+
red = prof.get("reduction")
|
|
752
|
+
if red is None:
|
|
753
|
+
x, noise = means, model.vc.sigma2
|
|
754
|
+
else:
|
|
755
|
+
comps = np.asarray(red["components"])
|
|
756
|
+
x = (means - np.asarray(red["raw_mean"])) @ comps.T
|
|
757
|
+
noise = (comps**2) @ model.vc.sigma2
|
|
758
|
+
|
|
759
|
+
core_pos = table.index.get_indexer(core_ids)
|
|
760
|
+
names = pd.Index(pd.unique(np.asarray(core_labels, dtype=str)))
|
|
761
|
+
lab = names.get_indexer(np.asarray(core_labels, dtype=str))
|
|
762
|
+
floor = 1e-6 * float(noise.mean()) + 1e-12
|
|
763
|
+
mu, t, prior = [], [], []
|
|
764
|
+
for k in range(len(names)):
|
|
765
|
+
members = core_pos[lab == k]
|
|
766
|
+
xm = x[members]
|
|
767
|
+
mu.append(xm.mean(axis=0))
|
|
768
|
+
t.append(np.maximum(xm.var(axis=0) - (noise[None, :] / n[members][:, None]).mean(axis=0), floor))
|
|
769
|
+
prior.append(members.size)
|
|
770
|
+
mu, t = np.vstack(mu), np.vstack(t)
|
|
771
|
+
prior = np.log(np.asarray(prior, dtype=float) / np.sum(prior))
|
|
772
|
+
var = t[None, :, :] + noise[None, None, :] / np.maximum(n, 1)[:, None, None]
|
|
773
|
+
ll = -0.5 * (((x[:, None, :] - mu[None, :, :]) ** 2) / var + np.log(var)).sum(axis=2) + prior[None, :]
|
|
774
|
+
post = np.exp(ll - logsumexp(ll, axis=1, keepdims=True))
|
|
775
|
+
return np.asarray(names)[post.argmax(axis=1)], post.max(axis=1)
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def get_programmes(adata) -> dict:
|
|
779
|
+
"""Stored programme results (raises if :func:`find_programmes` was not run)."""
|
|
780
|
+
out = get_uns(adata).get(PROGRAMME_KEY)
|
|
781
|
+
if out is None:
|
|
782
|
+
raise ValueError("Run threadfin.tl.find_programmes() (or threadfin.run()) first.")
|
|
783
|
+
return out
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
def programme_composition(adata, state_key: str, *, key: str | None = None) -> pd.DataFrame:
|
|
787
|
+
"""Fraction of each programme's cells in each cell state (rows sum to 1)."""
|
|
788
|
+
key = key or get_programmes(adata)["params"]["key_added"]
|
|
789
|
+
df = adata.obs[[key, state_key]].dropna()
|
|
790
|
+
tab = pd.crosstab(df[key], df[state_key])
|
|
791
|
+
return tab.div(tab.sum(axis=1), axis=0)
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
def _clone_expression(adata, *, clone_key, cell_mask, layer, context_key, state_key, state, min_cells_in_state,
|
|
795
|
+
min_frac_expressed, caller):
|
|
796
|
+
"""Mean context-centred expression of each clone's cells (optionally only its cells of one state).
|
|
797
|
+
|
|
798
|
+
Returns ``(cells, clone_codes, clone_index, genes, clone_means)``: the cells used, their clone codes, the
|
|
799
|
+
clone ids, the indices of the genes expressed in at least ``min_frac_expressed`` of those cells, and the
|
|
800
|
+
clones x genes matrix of means, each minus the mean of the same-state cells of the clone's contexts.
|
|
801
|
+
"""
|
|
802
|
+
import scipy.sparse as sp
|
|
803
|
+
|
|
804
|
+
from ._utils import codes
|
|
805
|
+
|
|
806
|
+
x = adata.layers[layer] if layer is not None else adata.X
|
|
807
|
+
x = sp.csr_matrix(x) if not sp.issparse(x) else x.tocsr()
|
|
808
|
+
in_state = np.ones(adata.n_obs, dtype=bool)
|
|
809
|
+
if state_key is not None:
|
|
810
|
+
require_obs(adata, state_key, context=f"{caller}(state_key=...)")
|
|
811
|
+
levels = adata.obs[state_key].dropna().astype(str)
|
|
812
|
+
if state is None or str(state) not in set(levels):
|
|
813
|
+
raise ValueError(f"state={state!r} is not a level of '{state_key}'; levels: "
|
|
814
|
+
f"{', '.join(sorted(set(levels))[:12])}.")
|
|
815
|
+
in_state = (adata.obs[state_key].astype(str) == str(state)).to_numpy()
|
|
816
|
+
cells = np.flatnonzero(np.asarray(cell_mask) & adata.obs[clone_key].notna().to_numpy() & in_state)
|
|
817
|
+
clone_codes, clone_index = codes(adata.obs[clone_key].iloc[cells].astype(str))
|
|
818
|
+
if min_cells_in_state > 1: # keep clones with enough cells in the state
|
|
819
|
+
counts = np.bincount(clone_codes, minlength=len(clone_index))
|
|
820
|
+
cells = cells[(counts >= min_cells_in_state)[clone_codes]]
|
|
821
|
+
clone_codes, clone_index = codes(adata.obs[clone_key].iloc[cells].astype(str))
|
|
822
|
+
n_clones = len(clone_index)
|
|
823
|
+
member = sp.csr_matrix((np.ones(cells.size), (clone_codes, np.arange(cells.size))), shape=(n_clones, cells.size))
|
|
824
|
+
member = sp.diags(1.0 / np.maximum(np.asarray(member.sum(axis=1)).ravel(), 1)) @ member
|
|
825
|
+
expressed = np.asarray((x[cells] > 0).mean(axis=0)).ravel() >= min_frac_expressed
|
|
826
|
+
genes = np.flatnonzero(expressed)
|
|
827
|
+
clone_means = np.asarray((member @ x[cells][:, genes]).toarray())
|
|
828
|
+
if context_key is not None:
|
|
829
|
+
# subtract, for every clone, the mean expression of the contexts its cells came from
|
|
830
|
+
ctx, ctx_index = codes(adata.obs[context_key])
|
|
831
|
+
ok = np.flatnonzero((ctx >= 0) & in_state) # within a state, centre on that state's cells
|
|
832
|
+
cmember = sp.csr_matrix((np.ones(ok.size), (ctx[ok], ok)), shape=(len(ctx_index), adata.n_obs))
|
|
833
|
+
cmember = sp.diags(1.0 / np.maximum(np.asarray(cmember.sum(axis=1)).ravel(), 1)) @ cmember
|
|
834
|
+
ctx_means = np.asarray((cmember @ x[:, genes]).toarray())
|
|
835
|
+
cell_ctx = sp.csr_matrix((np.ones(cells.size), (np.arange(cells.size), np.maximum(ctx[cells], 0))),
|
|
836
|
+
shape=(cells.size, len(ctx_index)))
|
|
837
|
+
clone_means -= np.asarray((member @ cell_ctx).toarray()) @ ctx_means
|
|
838
|
+
return cells, clone_codes, clone_index, genes, clone_means
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
def programme_markers(
|
|
842
|
+
adata,
|
|
843
|
+
*,
|
|
844
|
+
programme_key: str = "clone_programme",
|
|
845
|
+
layer: str | None = "log_norm",
|
|
846
|
+
context_key: str | None = "auto",
|
|
847
|
+
min_frac_expressed: float = 0.05,
|
|
848
|
+
n_top: int = 25,
|
|
849
|
+
state_key: str | None = None,
|
|
850
|
+
state: str | None = None,
|
|
851
|
+
min_cells_in_state: int = 1,
|
|
852
|
+
verbose: bool = True,
|
|
853
|
+
) -> pd.DataFrame:
|
|
854
|
+
"""Genes that distinguish programmes, tested with clones as replicates.
|
|
855
|
+
|
|
856
|
+
Every clone is summarised by the mean context-centred log expression of
|
|
857
|
+
its cells; for each programme, clones inside are compared with clones in
|
|
858
|
+
other programmes by a Mann-Whitney test per gene (BH FDR). Testing across
|
|
859
|
+
clones avoids the pseudoreplication of cell-level marker tests, where one
|
|
860
|
+
expanded clone contributes hundreds of correlated cells.
|
|
861
|
+
|
|
862
|
+
Programmes usually differ in which cell states their clones contain, so the genes found this way are
|
|
863
|
+
largely the markers of those states. To ask what differs between programmes *within one state* — for
|
|
864
|
+
example between the germinal-centre B cells of clones from two programmes — pass ``state_key`` and
|
|
865
|
+
``state``: each clone is then summarised by its cells in that state only (clones with at least
|
|
866
|
+
``min_cells_in_state`` such cells), centred on the cells of the same state in its context.
|
|
867
|
+
|
|
868
|
+
Returns
|
|
869
|
+
-------
|
|
870
|
+
Tidy DataFrame ``[programme, gene, mean_diff, auc, pvalue, fdr, rank]`` (and ``state`` when given)
|
|
871
|
+
with the top ``n_top`` up-regulated genes per programme.
|
|
872
|
+
"""
|
|
873
|
+
import scipy.sparse as sp
|
|
874
|
+
from scipy.stats import rankdata
|
|
875
|
+
|
|
876
|
+
from ._utils import bh_fdr, codes
|
|
877
|
+
|
|
878
|
+
prof = get_profiles(adata)
|
|
879
|
+
clone_key = prof["params"]["clone_key"]
|
|
880
|
+
if context_key == "auto":
|
|
881
|
+
context_key = prof["params"].get("context_key")
|
|
882
|
+
lab = adata.obs[programme_key]
|
|
883
|
+
cells, clone_codes, clone_index, genes, clone_means = _clone_expression(
|
|
884
|
+
adata, clone_key=clone_key, cell_mask=lab.notna().to_numpy(), layer=layer, context_key=context_key,
|
|
885
|
+
state_key=state_key, state=state, min_cells_in_state=min_cells_in_state,
|
|
886
|
+
min_frac_expressed=min_frac_expressed, caller="programme_markers")
|
|
887
|
+
n_clones = len(clone_index)
|
|
888
|
+
if n_clones < 6:
|
|
889
|
+
raise ValueError(f"Only {n_clones} clones have cells{' in ' + repr(state) if state_key else ''}; "
|
|
890
|
+
"programme markers need at least 3 clones inside and 3 outside a programme.")
|
|
891
|
+
clone_prog = (pd.Series(lab.iloc[cells].astype(str).to_numpy()).groupby(clone_codes).first()).to_numpy()
|
|
892
|
+
|
|
893
|
+
ranks = np.apply_along_axis(rankdata, 0, clone_means)
|
|
894
|
+
rows = []
|
|
895
|
+
for prog in sorted(pd.unique(clone_prog), key=lambda s: (len(s), s)):
|
|
896
|
+
inside = clone_prog == prog
|
|
897
|
+
n1, n2 = int(inside.sum()), int((~inside).sum())
|
|
898
|
+
if n1 < 3 or n2 < 3:
|
|
899
|
+
continue
|
|
900
|
+
u = ranks[inside].sum(axis=0) - n1 * (n1 + 1) / 2
|
|
901
|
+
auc = u / (n1 * n2)
|
|
902
|
+
# normal approximation to the Mann-Whitney U (ties ignored)
|
|
903
|
+
z = (u - n1 * n2 / 2) / np.sqrt(n1 * n2 * (n1 + n2 + 1) / 12)
|
|
904
|
+
from scipy.stats import norm
|
|
905
|
+
|
|
906
|
+
pvals = 2 * norm.sf(np.abs(z))
|
|
907
|
+
diff = clone_means[inside].mean(axis=0) - clone_means[~inside].mean(axis=0)
|
|
908
|
+
tab = pd.DataFrame({"programme": prog, "gene": adata.var_names[genes], "mean_diff": diff,
|
|
909
|
+
"auc": auc, "pvalue": pvals})
|
|
910
|
+
if state_key is not None:
|
|
911
|
+
tab.insert(1, "state", str(state))
|
|
912
|
+
tab["fdr"] = bh_fdr(tab["pvalue"].to_numpy())
|
|
913
|
+
tab = tab[tab["mean_diff"] > 0].sort_values(["fdr", "mean_diff"], ascending=[True, False]).head(n_top)
|
|
914
|
+
tab["rank"] = np.arange(1, len(tab) + 1)
|
|
915
|
+
rows.append(tab)
|
|
916
|
+
out = pd.concat(rows, ignore_index=True) if rows else pd.DataFrame(
|
|
917
|
+
columns=["programme", "gene", "mean_diff", "auc", "pvalue", "fdr", "rank"])
|
|
918
|
+
get_uns(adata)["programme_markers"] = h5ad_safe(out)
|
|
919
|
+
where = f" within '{state}'" if state_key is not None else ""
|
|
920
|
+
log(f"programme markers{where}: {n_clones} clones as replicates, {genes.size} genes tested.", verbose)
|
|
921
|
+
return out
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
def landscape_genes(
|
|
925
|
+
adata,
|
|
926
|
+
*,
|
|
927
|
+
layer: str | None = "log_norm",
|
|
928
|
+
context_key: str | None = "auto",
|
|
929
|
+
n_neighbors: int = 15,
|
|
930
|
+
min_frac_expressed: float = 0.05,
|
|
931
|
+
state_key: str | None = None,
|
|
932
|
+
state: str | None = None,
|
|
933
|
+
min_cells_in_state: int = 1,
|
|
934
|
+
n_top: int | None = 50,
|
|
935
|
+
verbose: bool = True,
|
|
936
|
+
) -> pd.DataFrame:
|
|
937
|
+
"""Genes whose expression changes smoothly across the clones, with no programmes needed.
|
|
938
|
+
|
|
939
|
+
Programme markers need distinct programmes; when clones vary along a continuum (a single programme),
|
|
940
|
+
this asks instead which genes follow the landscape of clone profiles. Every profiled clone is summarised
|
|
941
|
+
by the mean context-centred log expression of its cells (as in :func:`programme_markers`; with
|
|
942
|
+
``state_key``/``state``, of its cells in that state only). Clones are linked to their ``n_neighbors``
|
|
943
|
+
nearest clones in profile space, and each gene is scored by Moran's I of the clone means on that graph:
|
|
944
|
+
I > 0 when neighbouring clones express the gene alike, about 0 when the gene ignores the landscape. P values
|
|
945
|
+
use the normal approximation under randomisation (Cliff and Ord), corrected by Benjamini-Hochberg.
|
|
946
|
+
``corr_axis1`` is the Spearman correlation of the gene with the first component of the clone profiles,
|
|
947
|
+
the main axis along which clones differ, and gives the direction of a gradient.
|
|
948
|
+
|
|
949
|
+
This describes the landscape rather than testing an independent hypothesis: the profiles are built from
|
|
950
|
+
the same expression, so the genes that make clones differ are expected to score. Use it to name what
|
|
951
|
+
varies across clones; test annotations that the profiles did not use with :func:`profile_association`.
|
|
952
|
+
|
|
953
|
+
With hundreds of clones most expressed genes reach FDR < 0.05, so rank genes by Moran's I (0 to 1 in
|
|
954
|
+
practice) rather than counting them. The top genes often lie at one end of the main axis; ``n_top=None``
|
|
955
|
+
returns every gene that passes, so both ends can be read from ``corr_axis1``.
|
|
956
|
+
|
|
957
|
+
Returns
|
|
958
|
+
-------
|
|
959
|
+
DataFrame ``[gene, morans_i, z, pvalue, fdr, corr_axis1, rank]`` of the ``n_top`` genes with the largest
|
|
960
|
+
Moran's I among those with FDR < 0.05 (fewer if fewer pass; all of them with ``n_top=None``), also stored
|
|
961
|
+
in ``adata.uns['threadfin']['landscape_genes']``.
|
|
962
|
+
"""
|
|
963
|
+
import scipy.sparse as sp
|
|
964
|
+
from scipy.stats import norm, rankdata
|
|
965
|
+
from sklearn.neighbors import NearestNeighbors
|
|
966
|
+
|
|
967
|
+
from ._utils import bh_fdr, require_positive_int
|
|
968
|
+
|
|
969
|
+
require_positive_int(n_neighbors, "n_neighbors")
|
|
970
|
+
prof = get_profiles(adata)
|
|
971
|
+
clone_key = prof["params"]["clone_key"]
|
|
972
|
+
if context_key == "auto":
|
|
973
|
+
context_key = prof["params"].get("context_key")
|
|
974
|
+
feats = prof["features"]
|
|
975
|
+
profiled = adata.obs[clone_key].astype(str).isin(set(feats.index.astype(str))).to_numpy()
|
|
976
|
+
_, _, clone_index, genes, clone_means = _clone_expression(
|
|
977
|
+
adata, clone_key=clone_key, cell_mask=profiled, layer=layer, context_key=context_key,
|
|
978
|
+
state_key=state_key, state=state, min_cells_in_state=min_cells_in_state,
|
|
979
|
+
min_frac_expressed=min_frac_expressed, caller="landscape_genes")
|
|
980
|
+
n = len(clone_index)
|
|
981
|
+
if n < n_neighbors + 2:
|
|
982
|
+
raise ValueError(f"Only {n} profiled clones{' with cells in ' + repr(state) if state_key else ''}; "
|
|
983
|
+
f"landscape_genes needs more than n_neighbors + 1 = {n_neighbors + 1}.")
|
|
984
|
+
f = feats.loc[clone_index.astype(str)].to_numpy(dtype=float)
|
|
985
|
+
nn = NearestNeighbors(n_neighbors=n_neighbors + 1).fit(f)
|
|
986
|
+
_, idx = nn.kneighbors(f)
|
|
987
|
+
rows = np.repeat(np.arange(n), n_neighbors)
|
|
988
|
+
w = sp.csr_matrix((np.ones(rows.size), (rows, idx[:, 1:].ravel())), shape=(n, n))
|
|
989
|
+
s0 = float(w.sum())
|
|
990
|
+
ws = w + w.T
|
|
991
|
+
s1 = 0.5 * float(ws.multiply(ws).sum())
|
|
992
|
+
s2 = float(((np.asarray(w.sum(axis=1)).ravel() + np.asarray(w.sum(axis=0)).ravel()) ** 2).sum())
|
|
993
|
+
|
|
994
|
+
z = clone_means - clone_means.mean(axis=0)
|
|
995
|
+
var = (z ** 2).sum(axis=0)
|
|
996
|
+
ok = var > 0
|
|
997
|
+
z, var, genes = z[:, ok], var[ok], genes[ok]
|
|
998
|
+
moran = (n / s0) * np.asarray((z * (w @ z)).sum(axis=0)).ravel() / var
|
|
999
|
+
expected = -1.0 / (n - 1)
|
|
1000
|
+
b2 = n * (z ** 4).sum(axis=0) / var ** 2 # kurtosis of each gene's clone means
|
|
1001
|
+
var_i = ((n * ((n ** 2 - 3 * n + 3) * s1 - n * s2 + 3 * s0 ** 2)
|
|
1002
|
+
- b2 * ((n ** 2 - n) * s1 - 2 * n * s2 + 6 * s0 ** 2))
|
|
1003
|
+
/ ((n - 1) * (n - 2) * (n - 3) * s0 ** 2) - expected ** 2)
|
|
1004
|
+
zscore = (moran - expected) / np.sqrt(np.maximum(var_i, 1e-12))
|
|
1005
|
+
pvals = norm.sf(zscore) # one-sided: smoother than random
|
|
1006
|
+
# the main axis of the clone profiles (their first principal component; the sign is arbitrary, so it is
|
|
1007
|
+
# fixed to make the most extreme clone positive), and each gene's rank correlation with it
|
|
1008
|
+
fc = f - f.mean(axis=0)
|
|
1009
|
+
u, sv, _ = np.linalg.svd(fc, full_matrices=False)
|
|
1010
|
+
axis1 = u[:, 0] * sv[0]
|
|
1011
|
+
axis1 = axis1 if axis1[np.argmax(np.abs(axis1))] >= 0 else -axis1
|
|
1012
|
+
ra = rankdata(axis1)
|
|
1013
|
+
rz = rankdata(z, axis=0) # columns of z are the genes kept above
|
|
1014
|
+
ra, rz = ra - ra.mean(), rz - rz.mean(axis=0)
|
|
1015
|
+
denom = np.sqrt((ra ** 2).sum() * (rz ** 2).sum(axis=0))
|
|
1016
|
+
corr = np.divide((ra[:, None] * rz).sum(axis=0), denom, out=np.full(rz.shape[1], np.nan), where=denom > 0)
|
|
1017
|
+
tab = pd.DataFrame({"gene": adata.var_names[genes], "morans_i": moran, "z": zscore, "pvalue": pvals,
|
|
1018
|
+
"fdr": bh_fdr(pvals), "corr_axis1": corr})
|
|
1019
|
+
out = tab[tab["fdr"] < 0.05].sort_values("morans_i", ascending=False)
|
|
1020
|
+
out = (out if n_top is None else out.head(n_top)).reset_index(drop=True)
|
|
1021
|
+
out["rank"] = np.arange(1, len(out) + 1)
|
|
1022
|
+
get_uns(adata)["landscape_genes"] = h5ad_safe(out)
|
|
1023
|
+
where = f" within '{state}'" if state_key is not None else ""
|
|
1024
|
+
# with many clones most genes reach significance; Moran's I itself says how closely a gene follows the landscape
|
|
1025
|
+
top = ", ".join(f"{g} ({i:.2f})" for g, i in zip(out["gene"].head(5), out["morans_i"].head(5)))
|
|
1026
|
+
log(f"landscape genes{where}: Moran's I over {n} clones, FDR < 0.05 for {int((tab['fdr'] < 0.05).sum())} of "
|
|
1027
|
+
f"{len(tab)} genes; highest: {top}.", verbose)
|
|
1028
|
+
return out
|