threadfin 4.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1028 @@
1
+ """Clonal programmes: groups of clones with similar state profiles.
2
+
3
+ A *clonal programme* is a set of genetically distinct clones whose cells,
4
+ relative to their sampling context, occupy similar transcriptional states
5
+ (for example clones biased towards plasmablast output, or clones that stay
6
+ in the germinal centre). Programmes are found by Leiden community detection
7
+ on a k-nearest-neighbour graph of clone profiles
8
+ (:func:`threadfin.tl.clone_profiles`).
9
+
10
+ Community detection always returns *some* partition, even when clones vary
11
+ continuously without forming groups. Two safeguards address this:
12
+
13
+ 1. **Split test.** The Leiden communities are arranged in a tree (average
14
+ linkage of their centroids) and tested top-down: at every node, the clones
15
+ below it are compared with a single multivariate Gaussian fitted to them
16
+ (same mean and covariance). The statistic is the 2-means cluster index
17
+ (within / total sum of squares, as in SigClust, Liu et al. 2008); the
18
+ null distribution comes from re-clustering Gaussian samples, so the test
19
+ repeats the 2-means search, but does not repeat the full data-dependent
20
+ Leiden/tree/resolution selection. This is an approximate diagnostic. A
21
+ node is split only if p <= alpha * (n_node - 1) / (n_all - 1), with
22
+ n the number of clones. This allocation follows hierarchical-testing
23
+ work (Meinshausen 2008; sc-SHC, Grabski et al. 2023), but this implementation
24
+ does not establish family-wise error control for the selected tree. Communities below a non-significant
25
+ node are merged. When even the root is not significant, the clones form
26
+ a single programme: they differ (see the coherence test) but along a
27
+ continuum rather than in distinct groups.
28
+ 2. **Bootstrap stability.** Cells are resampled within clones (Poisson
29
+ bootstrap), profiles are re-estimated and the whole procedure (graph,
30
+ Leiden, split test) is repeated; each programme is matched to its best
31
+ bootstrap counterpart by Jaccard overlap (Hennig 2007, "clusterboot").
32
+ Programmes with mean Jaccard >= 0.75 are stable; below 0.6 they should
33
+ not be interpreted.
34
+
35
+ The Leiden resolution is chosen as the finest one whose size-weighted
36
+ stability reaches the threshold, instead of being fixed by hand.
37
+ """
38
+
39
+ from __future__ import annotations
40
+
41
+ import warnings
42
+
43
+ import numpy as np
44
+ import pandas as pd
45
+
46
+ from ._utils import get_uns, h5ad_safe, log, require_obs, stable_categorical
47
+ from .profiles import get_profiles, model_from_adata, project_features
48
+
49
+ PROGRAMME_KEY = "programmes"
50
+ DEFAULT_RESOLUTIONS = (0.2, 0.4, 0.6, 0.8, 1.0, 1.4, 2.0)
51
+
52
+
53
+ # --------------------------------------------------------------------------- graph + Leiden
54
+
55
+
56
+ def knn_graph(features: np.ndarray, n_neighbors: int = 15, reliability: np.ndarray | None = None):
57
+ """Weighted undirected kNN graph (igraph) with self-tuned Gaussian weights.
58
+
59
+ ``reliability`` scales every edge by ``sqrt(rel_i * rel_j)``, so a clone seen in few cells still
60
+ takes part in the clustering but pulls on it only as hard as its profile deserves. This is the
61
+ alternative to a reliability threshold: nothing is removed, and a community of noisy profiles is
62
+ held together by weak edges that Leiden will not keep as a separate module.
63
+ """
64
+ import igraph as ig
65
+ from sklearn.neighbors import NearestNeighbors
66
+
67
+ n = features.shape[0]
68
+ k = int(min(n_neighbors, n - 1))
69
+ if k < 1:
70
+ raise ValueError("Need at least two clones to build a graph.")
71
+ dist, idx = NearestNeighbors(n_neighbors=k + 1).fit(features).kneighbors(features)
72
+ dist, idx = dist[:, 1:], idx[:, 1:] # drop self
73
+ sigma = np.maximum(dist[:, -1:], 1e-12) # distance to the k-th neighbour
74
+ w = np.exp(-(dist**2) / sigma**2)
75
+ rows = np.repeat(np.arange(n), k)
76
+ cols = idx.ravel()
77
+ lo, hi = np.minimum(rows, cols), np.maximum(rows, cols)
78
+ # merge (i,j)/(j,i) duplicates, keeping the larger weight
79
+ key = lo.astype(np.int64) * n + hi
80
+ order = np.lexsort((-w.ravel(), key))
81
+ key_sorted = key[order]
82
+ first = np.ones(key_sorted.size, dtype=bool)
83
+ first[1:] = key_sorted[1:] != key_sorted[:-1]
84
+ sel = order[first]
85
+ weights = w.ravel()[sel]
86
+ if reliability is not None:
87
+ r = np.clip(np.asarray(reliability, dtype=float), 1e-6, 1.0)
88
+ weights = weights * np.sqrt(r[lo[sel]] * r[hi[sel]])
89
+ g = ig.Graph(n=n, edges=list(zip(lo[sel].tolist(), hi[sel].tolist())), directed=False)
90
+ g.es["weight"] = weights.tolist()
91
+ return g
92
+
93
+
94
+ def leiden(graph, resolution: float, seed: int = 0) -> np.ndarray:
95
+ """Leiden partition (RB configuration model) -> integer labels."""
96
+ import leidenalg
97
+
98
+ part = leidenalg.find_partition(
99
+ graph,
100
+ leidenalg.RBConfigurationVertexPartition,
101
+ weights="weight",
102
+ resolution_parameter=float(resolution),
103
+ seed=int(seed),
104
+ )
105
+ return np.asarray(part.membership, dtype=np.int64)
106
+
107
+
108
+ def _leiden_many(tasks, n_jobs: int = 1) -> list:
109
+ """Leiden partitions for ``(graph, resolution, seed)`` tasks, in parallel processes when ``n_jobs > 1``.
110
+
111
+ Each partition depends only on its graph, resolution and seed, so the result is the same for any ``n_jobs``.
112
+ """
113
+ if n_jobs > 1 and len(tasks) > 1:
114
+ from joblib import Parallel, delayed
115
+
116
+ return Parallel(n_jobs=min(n_jobs, len(tasks)))(delayed(leiden)(g, r, s) for g, r, s in tasks)
117
+ return [leiden(g, r, s) for g, r, s in tasks]
118
+
119
+
120
+ def jaccard_match(ref: np.ndarray, other: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
121
+ """Best-match Jaccard of every reference cluster, and per-item agreement.
122
+
123
+ Returns ``(jaccard_per_ref_cluster, agrees)``; ``agrees[i]`` is True when
124
+ item ``i`` sits in the bootstrap cluster that best matches its reference
125
+ cluster.
126
+ """
127
+ import scipy.sparse as sp
128
+
129
+ kr, ko = int(ref.max()) + 1, int(other.max()) + 1
130
+ m = sp.coo_matrix((np.ones(ref.size), (ref, other)), shape=(kr, ko)).toarray()
131
+ size_r, size_o = m.sum(axis=1), m.sum(axis=0)
132
+ union = size_r[:, None] + size_o[None, :] - m
133
+ jac = np.divide(m, union, out=np.zeros_like(m), where=union > 0)
134
+ best = jac.argmax(axis=1)
135
+ return jac.max(axis=1), other == best[ref]
136
+
137
+
138
+ # --------------------------------------------------------------------------- split test
139
+
140
+
141
+ def _two_means_index(x: np.ndarray, rng, extra_split=None, n_random: int = 2, n_iter: int = 30) -> float:
142
+ """Smallest 2-means cluster index (within / total sum of squares) over a few starts.
143
+
144
+ Starts: the sign of the first principal component, ``n_random`` random
145
+ pairs of centres and, if given, ``extra_split`` (a boolean labelling).
146
+ """
147
+ xc = x - x.mean(axis=0)
148
+ tss = float(np.einsum("ij,ij->", xc, xc))
149
+ if xc.shape[0] < 3 or tss <= 0:
150
+ return 1.0
151
+ _, _, vt = np.linalg.svd(xc, full_matrices=False)
152
+ starts = [xc @ vt[0] > 0]
153
+ for _ in range(n_random):
154
+ a, b = rng.choice(xc.shape[0], size=2, replace=False)
155
+ starts.append(((xc - xc[a]) ** 2).sum(1) < ((xc - xc[b]) ** 2).sum(1))
156
+ if extra_split is not None:
157
+ starts.append(np.asarray(extra_split, dtype=bool))
158
+ best = np.inf
159
+ for lab in starts:
160
+ for _ in range(n_iter): # Lloyd iterations for two centres
161
+ if lab.all() or not lab.any():
162
+ break
163
+ c1, c0 = xc[lab].mean(axis=0), xc[~lab].mean(axis=0)
164
+ new = xc @ (c1 - c0) > 0.5 * (c1 @ c1 - c0 @ c0)
165
+ if np.array_equal(new, lab):
166
+ break
167
+ lab = new
168
+ if lab.all() or not lab.any():
169
+ continue
170
+ wss = tss - lab.sum() * np.sum(xc[lab].mean(0) ** 2) - (~lab).sum() * np.sum(xc[~lab].mean(0) ** 2)
171
+ best = min(best, wss)
172
+ return float(best / tss) if np.isfinite(best) else 1.0
173
+
174
+
175
+ def _two_means_labels(x: np.ndarray, rng, extra_split=None, n_random: int = 2, n_iter: int = 30):
176
+ """The labelling attaining the smallest 2-means cluster index (same starts as :func:`_two_means_index`)."""
177
+ xc = x - x.mean(axis=0)
178
+ tss = float(np.einsum("ij,ij->", xc, xc))
179
+ if xc.shape[0] < 3 or tss <= 0:
180
+ return None
181
+ _, _, vt = np.linalg.svd(xc, full_matrices=False)
182
+ starts = [xc @ vt[0] > 0]
183
+ for _ in range(n_random):
184
+ a, b = rng.choice(xc.shape[0], size=2, replace=False)
185
+ starts.append(((xc - xc[a]) ** 2).sum(1) < ((xc - xc[b]) ** 2).sum(1))
186
+ if extra_split is not None:
187
+ starts.append(np.asarray(extra_split, dtype=bool))
188
+ best, best_lab = np.inf, None
189
+ for lab in starts:
190
+ for _ in range(n_iter):
191
+ if lab.all() or not lab.any():
192
+ break
193
+ c1, c0 = xc[lab].mean(axis=0), xc[~lab].mean(axis=0)
194
+ new = xc @ (c1 - c0) > 0.5 * (c1 @ c1 - c0 @ c0)
195
+ if np.array_equal(new, lab):
196
+ break
197
+ lab = new
198
+ if lab.all() or not lab.any():
199
+ continue
200
+ wss = tss - lab.sum() * np.sum(xc[lab].mean(0) ** 2) - (~lab).sum() * np.sum(xc[~lab].mean(0) ** 2)
201
+ if wss < best:
202
+ best, best_lab = wss, lab.copy()
203
+ return best_lab
204
+
205
+
206
+ def split_test(x: np.ndarray, split=None, *, n_null: int = 100, random_state=0, return_null: bool = False,
207
+ null: str = "total") -> dict:
208
+ """Is ``x`` (clones x features) better described by two clusters than by one Gaussian?
209
+
210
+ The 2-means cluster index of ``x`` is compared with its distribution in ``n_null`` Gaussian samples.
211
+ ``null`` chooses that Gaussian's covariance. **Both options have been calibrated on real clone profiles
212
+ with a planted two-group structure; the measured numbers are quoted below, and neither setting is satisfactory:**
213
+
214
+ ``"total"`` (default)
215
+ the sample covariance of ``x`` itself (SigClust-style). Valid but very conservative: the false-positive
216
+ rate at zero separation is 0 of 30 simulations, and so is the detection rate for two groups separated by
217
+ four within-group standard deviations. A split inflates the covariance along the splitting direction and
218
+ the null inherits that inflation, so the test partly cancels itself. **A non-significant result from
219
+ this test is therefore not evidence of a continuum.**
220
+ ``"within"``
221
+ the covariance pooled *within* the two candidate groups. **Invalid — do not use for inference.** It was
222
+ tried as a fix and measured: on mouse_rbd clone profiles it reports a split in 30 of 30 simulations with
223
+ *no* planted structure (false-positive rate 1.0), because removing the candidate split from the
224
+ covariance estimate is circular. Kept only so that the calibration can be reproduced.
225
+
226
+ The p-value uses a normal fit to the null indices (so that it can be smaller than ``1 / n_null``); the
227
+ empirical p-value is also returned (and, with ``return_null``, the null indices themselves).
228
+ """
229
+ from scipy.stats import norm
230
+
231
+ if null not in ("within", "total"):
232
+ raise ValueError("null must be 'within' or 'total'.")
233
+ rng = np.random.default_rng(random_state)
234
+ x = np.asarray(x, dtype=float)
235
+ observed = _two_means_index(x, rng, extra_split=split)
236
+ if null == "within":
237
+ lab = _two_means_labels(x, rng, extra_split=split)
238
+ if lab is None or lab.all() or not lab.any():
239
+ cov = np.atleast_2d(np.cov(x, rowvar=False))
240
+ else:
241
+ centred = x.copy()
242
+ centred[lab] -= x[lab].mean(axis=0)
243
+ centred[~lab] -= x[~lab].mean(axis=0)
244
+ dof = max(x.shape[0] - 2, 1)
245
+ cov = np.atleast_2d(centred.T @ centred / dof)
246
+ else:
247
+ cov = np.atleast_2d(np.cov(x, rowvar=False))
248
+ w, v = np.linalg.eigh(cov)
249
+ root = v * np.sqrt(np.clip(w, 0, None))
250
+ null = np.array([_two_means_index(rng.standard_normal(x.shape) @ root.T, rng) for _ in range(n_null)])
251
+ sd = float(null.std(ddof=1)) if n_null > 1 else np.nan
252
+ p_norm = float(norm.cdf((observed - null.mean()) / sd)) if np.isfinite(sd) and sd > 0 else float("nan")
253
+ p_emp = float((1 + np.sum(null <= observed)) / (n_null + 1))
254
+ if not np.isfinite(p_norm):
255
+ p_norm = p_emp # A degenerate null must not turn identical data into p=0.
256
+ out = {"cluster_index": observed, "null_mean": float(null.mean()), "null_sd": sd,
257
+ "pvalue": p_norm, "pvalue_empirical": p_emp}
258
+ if return_null:
259
+ out["null"] = null
260
+ return out
261
+
262
+
263
+ def _merge_by_split_test(feats, labels, alpha, n_null, rng, null="total"):
264
+ """Test the community tree top-down and merge communities below non-significant nodes.
265
+
266
+ Returns ``(merged_labels, tests)`` with merged labels ``0..K-1`` and one
267
+ row per tested node.
268
+ """
269
+ from scipy.cluster.hierarchy import linkage, to_tree
270
+
271
+ names = np.unique(labels)
272
+ if names.size < 2:
273
+ return np.zeros(labels.size, dtype=np.int64), pd.DataFrame()
274
+ centroids = np.vstack([feats[labels == k].mean(axis=0) for k in names])
275
+ root = to_tree(linkage(centroids, method="average"))
276
+ total = labels.size
277
+ rows, groups = [], []
278
+
279
+ def visit(node):
280
+ if node.is_leaf():
281
+ groups.append([names[node.id]])
282
+ return
283
+ a = [names[i] for i in node.left.pre_order()]
284
+ b = [names[i] for i in node.right.pre_order()]
285
+ inside = np.isin(labels, a + b)
286
+ res = split_test(feats[inside], np.isin(labels[inside], a), n_null=n_null, null=null,
287
+ random_state=int(rng.integers(2**31)))
288
+ level = alpha * (inside.sum() - 1) / (total - 1)
289
+ # communities as comma-separated text, so the table can be written to h5ad with the AnnData
290
+ rows.append({"communities_left": ",".join(str(int(c)) for c in a),
291
+ "communities_right": ",".join(str(int(c)) for c in b),
292
+ "n_communities_left": len(a), "n_communities_right": len(b),
293
+ "n_clones": int(inside.sum()), **res,
294
+ "alpha_node": float(level), "split": bool(res["pvalue"] <= level)})
295
+ if res["pvalue"] <= level:
296
+ visit(node.left)
297
+ visit(node.right)
298
+ else:
299
+ groups.append(a + b)
300
+
301
+ visit(root)
302
+ merged = np.empty(labels.size, dtype=np.int64)
303
+ for g, members in enumerate(groups):
304
+ merged[np.isin(labels, members)] = g
305
+ tests = pd.DataFrame(rows)
306
+ if not tests.empty:
307
+ to_group = {int(c): g for g, members in enumerate(groups) for c in members}
308
+ for side in ("left", "right"):
309
+ tests[f"groups_{side}"] = [sorted({to_group[int(c)] for c in cs.split(",")})
310
+ for cs in tests[f"communities_{side}"]]
311
+ return merged, tests
312
+
313
+
314
+ # --------------------------------------------------------------------------- bootstrap
315
+
316
+
317
+ def _bootstrap_features(model, adata, eligible_codes, n_boot, rng):
318
+ """Feature matrices of the eligible clones under ``n_boot`` Poisson bootstraps."""
319
+ n_all = len(model.clone_index)
320
+ remap = -np.ones(n_all, dtype=np.int64)
321
+ remap[eligible_codes] = np.arange(eligible_codes.size)
322
+ groups = np.where(model.clone_codes >= 0, remap[np.maximum(model.clone_codes, 0)], -1)
323
+ in_group = groups >= 0
324
+ out = []
325
+ for _ in range(n_boot):
326
+ w = rng.poisson(1.0, size=groups.size).astype(float)
327
+ # a clone that draws no cells keeps its original cells (avoid empty clones)
328
+ tot = np.bincount(groups[in_group], weights=w[in_group], minlength=eligible_codes.size)
329
+ empty = np.flatnonzero(tot == 0)
330
+ if empty.size:
331
+ w[np.isin(groups, empty)] = 1.0
332
+ blup, _, _ = model.group_blups(groups, eligible_codes.size, weights=w)
333
+ out.append(project_features(adata, blup))
334
+ return out
335
+
336
+
337
+ # --------------------------------------------------------------------------- public API
338
+
339
+
340
+
341
+ def embed_clones(
342
+ adata,
343
+ *,
344
+ n_neighbors: int | str = "auto",
345
+ min_dist: float | str = "auto",
346
+ spread: float = 1.0,
347
+ min_reliability: float = 0.0,
348
+ coherence_target: float | str = "auto",
349
+ tune_kwargs: dict | None = None,
350
+ random_state: int = 0,
351
+ n_jobs: int | None = 1,
352
+ verbose: bool = True,
353
+ ) -> pd.DataFrame:
354
+ """Lay the clone profiles out in two dimensions — **every** profiled clone, by default.
355
+
356
+ This is the clone map the rest of the package is built around, and it is deliberately separate from
357
+ :func:`find_programmes`. Programme discovery is a clustering problem and needs reasonably estimated
358
+ profiles, so it keeps its ``min_reliability`` gate. The map is a *display* of the clones that exist,
359
+ and filtering it by reliability throws away the data the map is meant to show: on GSE287123 the 0.5
360
+ gate removes 75 % (RBD) and 90 % (NP-OVA) of profiled clones, and on GSE246382 it removes all of
361
+ them, which is why that dataset appeared to have no clone map at all.
362
+
363
+ The map does not change any statistic: profiles, coherence, programmes and tests are computed in
364
+ profile space, never on the map.
365
+
366
+ Parameters
367
+ ----------
368
+ n_neighbors, min_dist
369
+ UMAP settings. ``"auto"`` (the default for both) chooses them per dataset with
370
+ :func:`threadfin.tl.tune_clone_map`: a ladder of settings, each laid out with several seeds and
371
+ smoothed over its neighbours, finds the most faithful layout that is no more fragmented than the
372
+ clone profiles themselves, so a continuum of clones is drawn as one connected picture. Pass numbers to set them
373
+ by hand (``n_neighbors=50, min_dist=0.8`` reproduces the fixed default of Threadfin 4.2); a
374
+ number for one and ``"auto"`` for the other tunes only the second.
375
+ spread
376
+ UMAP ``spread``.
377
+ min_reliability
378
+ Reliability floor. ``0.0`` (the default) keeps every clone that has a profile; raise it only to
379
+ reproduce an older filtered layout.
380
+ coherence_target, tune_kwargs
381
+ Passed to :func:`threadfin.tl.tune_clone_map` when a setting is ``"auto"``.
382
+ random_state
383
+ Seed for the tuner and for the final layout.
384
+ n_jobs
385
+ Processes for the layouts of the automatic search (``-1``: all available cores). The chosen
386
+ settings and the map do not depend on it.
387
+
388
+ Returns
389
+ -------
390
+ The clone table with ``x`` and ``y`` added. ``adata.uns['threadfin']['profiles']['map_params']``
391
+ records the settings, how they were chosen (``"tuning"``, with the full optimisation history), the
392
+ quality of the final map (``"quality"``: coherence, local and global fidelity) and how many clones
393
+ were laid out.
394
+ """
395
+ from .maptune import layout_clone_map, tune_clone_map
396
+
397
+ prof = get_profiles(adata)
398
+ table, feats_all = prof["clone_table"], prof["features"]
399
+ use = table.index[table["reliability"] >= min_reliability]
400
+ if use.size < 15:
401
+ raise ValueError(
402
+ f"Only {use.size} clones to embed (need 15). Profile more clones, or lower min_cells in "
403
+ "clone_profiles.")
404
+ for name, value in (("n_neighbors", n_neighbors), ("min_dist", min_dist)):
405
+ if isinstance(value, str) and value != "auto":
406
+ raise ValueError(f"{name} must be 'auto' or a number, got {value!r}.")
407
+ feats = feats_all.loc[use].to_numpy()
408
+ tuning, repulsion = None, 1.0
409
+ if n_neighbors == "auto" or min_dist == "auto":
410
+ tuning = tune_clone_map(feats, n_neighbors=n_neighbors, min_dist=min_dist, spread=spread,
411
+ coherence_target=coherence_target, random_state=random_state,
412
+ n_jobs=n_jobs, verbose=verbose, **(tune_kwargs or {}))
413
+ n_neighbors, min_dist, repulsion = tuning.n_neighbors, tuning.min_dist, tuning.repulsion_strength
414
+
415
+ xy, quality, setting, attempts = layout_clone_map(
416
+ feats, n_neighbors=n_neighbors, min_dist=min_dist, repulsion_strength=repulsion, spread=spread,
417
+ coherence_target=coherence_target, tuning=tuning, random_state=random_state)
418
+ n_neighbors, min_dist, repulsion, layout_seed = setting
419
+ table["x"], table["y"] = np.nan, np.nan
420
+ table.loc[use, "x"], table.loc[use, "y"] = xy[:, 0], xy[:, 1]
421
+ prof["clone_table"] = table
422
+ prof["map_params"] = {"n_neighbors": int(n_neighbors), "min_dist": float(min_dist),
423
+ "repulsion_strength": float(repulsion), "spread": float(spread),
424
+ "min_reliability": float(min_reliability), "random_state": int(random_state),
425
+ "layout_seed": int(layout_seed), "n_clones": int(use.size),
426
+ "n_profiled": int(len(table)),
427
+ "chosen_by": "tune_clone_map" if tuning is not None else "user",
428
+ "quality": {k: float(v) for k, v in quality.items()},
429
+ "attempts": pd.DataFrame(attempts)}
430
+ if tuning is not None:
431
+ prof["map_params"]["tuning"] = tuning.to_uns()
432
+ rep_txt = "" if repulsion == 1.0 else f", repulsion_strength {repulsion:g}"
433
+ log(f"clone map: {use.size:,} of {len(table):,} profiled clones laid out (n_neighbors {int(n_neighbors)}, "
434
+ f"min_dist {float(min_dist):.2f}{rep_txt}{', chosen automatically' if tuning is not None else ''}); "
435
+ f"{100 * quality['coherence']:.1f}% of clones in one connected piece.", verbose)
436
+ if tuning is None and quality["coherence"] < quality["coherence_target"] - 0.005:
437
+ log(f" the map is more fragmented than the clone profiles (which keep "
438
+ f"{100 * quality['coherence_target']:.1f}% of clones connected); n_neighbors='auto', "
439
+ "min_dist='auto' chooses settings that keep it together.", verbose)
440
+ return table
441
+
442
+
443
+ def find_programmes(
444
+ adata,
445
+ *,
446
+ resolution: float | str = "auto",
447
+ resolutions=DEFAULT_RESOLUTIONS,
448
+ n_neighbors: int = 15,
449
+ min_reliability: float = 0.0,
450
+ weight_by_reliability: bool = True,
451
+ n_boot: int = 30,
452
+ stability_threshold: float = 0.75,
453
+ min_programme_size: int = 5,
454
+ min_clones: int = 30,
455
+ test_splits: bool = True,
456
+ alpha: float = 0.05,
457
+ n_null: int = 100,
458
+ split_null: str = "total",
459
+ assign_remaining: bool = True,
460
+ min_posterior: float = 0.7,
461
+ random_state: int = 0,
462
+ n_jobs: int | None = 1,
463
+ key_added: str = "clone_programme",
464
+ verbose: bool = True,
465
+ ) -> pd.DataFrame:
466
+ """Group clones into clonal programmes, with bootstrap stability.
467
+
468
+ Parameters
469
+ ----------
470
+ adata
471
+ AnnData after :func:`threadfin.tl.clone_profiles`.
472
+ resolution
473
+ Leiden resolution, or ``"auto"`` to scan ``resolutions`` and keep the
474
+ finest partition whose size-weighted stability is at least
475
+ ``stability_threshold`` (falling back to the most stable partition).
476
+ n_neighbors
477
+ Neighbours per clone in the clone graph.
478
+ weight_by_reliability
479
+ Scale each graph edge by ``sqrt(rel_i * rel_j)`` so that every clone is clustered but a
480
+ thinly observed one pulls weakly. This is what makes ``min_reliability=0`` safe: on a
481
+ simulated one-programme continuum of 3,000 clones with a median of one cell each, an
482
+ unweighted unfiltered run invents four programmes at bootstrap stability 0.78-0.86, while the
483
+ weighted run recovers the single true programme.
484
+ min_reliability
485
+ Reliability floor for clustering; ``0.0`` (the default) clusters every profiled clone.
486
+ Threadfin does not filter clones. Every clone, and every cell in it, is measured data, and
487
+ the profiles are already **shrunk**: a clone seen in few cells has its profile pulled towards the
488
+ population mean by the variance-component model, so its influence is down-weighted in proportion to
489
+ how little was seen. Cutting on reliability *on top of* that shrinkage discards the same information
490
+ twice, and it discards it silently — on GSE287123 a 0.5 floor removes 75 % (RBD) and 90 % (NP-OVA) of
491
+ profiled clones, and on GSE246382 it removes all of them, which is how that dataset came to look as
492
+ though it had no clone structure when its map separates the sorted gate at p < 0.001.
493
+ n_boot
494
+ Bootstrap replicates for stability (0 disables stability, which is
495
+ only allowed with a fixed ``resolution``).
496
+ min_programme_size
497
+ Groups with fewer clones are not reported as programmes (their clones
498
+ stay unassigned); resolutions that leave more than 10% of clones in
499
+ such groups are not selected.
500
+ min_clones
501
+ Minimum number of reliable clones; with fewer, programme detection is
502
+ refused (report the coherence test only).
503
+ test_splits, alpha, n_null
504
+ Merge communities that are not significantly better described as two
505
+ groups than as one Gaussian (see the module docstring); ``alpha`` is
506
+ a nominal hierarchical threshold (not a validated family-wise error guarantee) and ``n_null`` the
507
+ number of Gaussian null samples per test. The node tests are stored
508
+ in ``adata.uns['threadfin']['programmes']['split_tests']``. A single
509
+ programme means "no distinct programmes".
510
+ assign_remaining
511
+ Programmes are *defined* on reliable ("core") clones only. With this
512
+ option every other profiled clone is then *assigned* to the most
513
+ likely programme by a Gaussian classifier whose noise term shrinks
514
+ with clone size (see ``_assign_remaining``); assignments with
515
+ posterior >= ``min_posterior`` go to ``<key_added>_assigned``.
516
+ n_jobs
517
+ Processes for the Leiden runs on the bootstrap graphs (``-1``: all
518
+ available cores). Programmes and stabilities are identical for any
519
+ value; only the run time changes.
520
+ key_added
521
+ ``obs`` column receiving each cell's programme (cells of clones that
522
+ were not clustered get NaN).
523
+
524
+ Returns
525
+ -------
526
+ Programme summary: ``n_clones``, ``n_cells``, ``stability`` (mean
527
+ bootstrap Jaccard) and ``mean_reliability`` per programme. Clone-level
528
+ assignments (with per-clone bootstrap ``confidence``) are added to
529
+ ``adata.uns['threadfin']['profiles']['clone_table']``.
530
+ """
531
+ prof = get_profiles(adata)
532
+ table = prof["clone_table"]
533
+ feats_all = prof["features"]
534
+ eligible = table.index[table["reliability"] >= min_reliability]
535
+ if eligible.size < min_clones:
536
+ raise ValueError(
537
+ f"Only {eligible.size} clones reach reliability {min_reliability} (need {min_clones}); "
538
+ "too few expanded clones to define programmes. Report the coherence test only, or "
539
+ "lower min_reliability."
540
+ )
541
+ feats = feats_all.loc[eligible].to_numpy()
542
+ rel = table.loc[eligible, "reliability"].to_numpy() if weight_by_reliability else None
543
+ graph = knn_graph(feats, n_neighbors, rel)
544
+
545
+ auto = isinstance(resolution, str)
546
+ if auto and resolution != "auto":
547
+ raise ValueError("resolution must be a number or 'auto'.")
548
+ if auto and n_boot < 5:
549
+ raise ValueError("resolution='auto' needs n_boot >= 5 bootstrap replicates.")
550
+ res_grid = list(resolutions) if auto else [float(resolution)]
551
+
552
+ rng = np.random.default_rng(random_state)
553
+ boot_graphs, boot_feats = [], []
554
+ if n_boot > 0:
555
+ model = model_from_adata(adata)
556
+ code_of = {str(c): i for i, c in enumerate(model.clone_index.astype(str))}
557
+ eligible_codes = np.array([code_of[c] for c in eligible], dtype=np.int64)
558
+ boot_feats = _bootstrap_features(model, adata, eligible_codes, n_boot, rng)
559
+ boot_graphs = [knn_graph(f, n_neighbors, rel) for f in boot_feats]
560
+
561
+ # every Leiden partition of the scan (reference and bootstrap graphs at every resolution), computed once
562
+ from .maptune import resolve_n_jobs
563
+
564
+ tasks = [(graph, res, random_state) for res in res_grid]
565
+ tasks += [(bg, res, random_state + b + 1) for res in res_grid for b, bg in enumerate(boot_graphs)]
566
+ parts = iter(_leiden_many(tasks, resolve_n_jobs(n_jobs)))
567
+ ref_part = {res: next(parts) for res in res_grid}
568
+ boot_part = {(res, b): next(parts) for res in res_grid for b in range(len(boot_graphs))}
569
+
570
+ scan_rows, partitions = [], {}
571
+ for res in res_grid:
572
+ ref = ref_part[res]
573
+ k = int(ref.max()) + 1
574
+ jac_sum = np.zeros(k)
575
+ agree = np.zeros(ref.size)
576
+ for b in range(len(boot_graphs)):
577
+ jac, ok = jaccard_match(ref, boot_part[(res, b)])
578
+ jac_sum += jac
579
+ agree += ok
580
+ nb = max(len(boot_graphs), 1)
581
+ jac_mean = jac_sum / nb if boot_graphs else np.full(k, np.nan)
582
+ sizes = np.bincount(ref, minlength=k)
583
+ big = sizes >= min_programme_size
584
+ frac_small = float(sizes[~big].sum() / sizes.sum())
585
+ weighted = (float(np.sum(jac_mean[big] * sizes[big]) / max(sizes[big].sum(), 1))
586
+ if boot_graphs and big.any() else np.nan)
587
+ partitions[res] = (ref, jac_mean, agree / nb if boot_graphs else np.full(ref.size, np.nan))
588
+ scan_rows.append({
589
+ "resolution": res, "n_programmes": int(big.sum()), "n_groups": k,
590
+ "frac_clones_in_small_groups": frac_small, "stability_weighted": weighted,
591
+ "stability_min": float(np.min(jac_mean[big])) if boot_graphs and big.any() else np.nan,
592
+ })
593
+ scan = pd.DataFrame(scan_rows)
594
+
595
+ caution = None # raised only if programmes survive the split test (below)
596
+ if auto:
597
+ multi = scan[(scan["n_programmes"] >= 2) & (scan["frac_clones_in_small_groups"] <= 0.10)]
598
+ stable = multi[multi["stability_weighted"] >= stability_threshold]
599
+ if not stable.empty:
600
+ best = stable.sort_values(["n_programmes", "stability_weighted"], ascending=False).iloc[0]
601
+ elif not multi.empty:
602
+ best = multi.sort_values("stability_weighted", ascending=False).iloc[0]
603
+ if boot_graphs: # without bootstrap replicates there is no stability to fall short of
604
+ caution = (f"No resolution reached stability {stability_threshold}; using the most stable "
605
+ f"partition (stability {best['stability_weighted']:.2f}). Interpret programmes "
606
+ "cautiously.")
607
+ else:
608
+ best = scan.iloc[0]
609
+ chosen = float(best["resolution"])
610
+ else:
611
+ chosen = res_grid[0]
612
+
613
+ ref, jac_mean, confidence = partitions[chosen]
614
+ # groups below the minimum size are not programmes: their clones stay unassigned
615
+ sizes = np.bincount(ref)
616
+ keep_mask = sizes[ref] >= min_programme_size
617
+ eligible = eligible[keep_mask]
618
+ feats = feats[keep_mask]
619
+ confidence = confidence[keep_mask]
620
+ ref = ref[keep_mask]
621
+ split_tests = pd.DataFrame()
622
+ if test_splits:
623
+ ref, split_tests = _merge_by_split_test(feats, ref, alpha, n_null, rng, split_null)
624
+ if boot_graphs:
625
+ # stability of the final programmes: repeat Leiden *and* the split test in every bootstrap
626
+ jac_mean = np.zeros(int(ref.max()) + 1)
627
+ agree = np.zeros(ref.size)
628
+ for b, bf in enumerate(boot_feats):
629
+ bl = boot_part[(chosen, b)][keep_mask]
630
+ bl, _ = _merge_by_split_test(bf[keep_mask], bl, alpha, max(n_null // 4, 20), rng, split_null)
631
+ jac, ok = jaccard_match(ref, bl)
632
+ jac_mean += jac
633
+ agree += ok
634
+ jac_mean /= len(boot_graphs)
635
+ confidence = agree / len(boot_graphs)
636
+ else:
637
+ jac_mean = np.full(int(ref.max()) + 1, np.nan)
638
+ labels = stable_categorical(ref, prefix="P")
639
+ raw_to_label = dict(zip(ref, labels))
640
+
641
+ # write clone-level results
642
+ table = table.copy()
643
+ table[key_added] = pd.Categorical([None] * len(table), categories=labels.categories)
644
+ table.loc[eligible, key_added] = np.asarray(labels)
645
+ table["confidence"] = np.nan
646
+ table.loc[eligible, "confidence"] = confidence
647
+ table["core"] = table.index.isin(eligible)
648
+ if assign_remaining:
649
+ model = model if n_boot > 0 else model_from_adata(adata)
650
+ prof["clone_table"] = table
651
+ best_lab, post = _assign_remaining(adata, model, eligible, np.asarray(labels, dtype=str))
652
+ assigned = pd.Series(np.where(post >= min_posterior, best_lab, None), index=table.index, dtype=object)
653
+ assigned.loc[eligible] = np.asarray(labels, dtype=str) # core clones keep their own programme
654
+ table[f"{key_added}_assigned"] = pd.Categorical(assigned, categories=labels.categories)
655
+ table["assignment_posterior"] = post
656
+ prof["clone_table"] = table
657
+
658
+ # map to cells
659
+ clone_key = prof["params"]["clone_key"]
660
+ cell_clone = adata.obs[clone_key].astype("object")
661
+ for col in [key_added] + ([f"{key_added}_assigned"] if assign_remaining else []):
662
+ mapping = table[col].dropna().astype(str)
663
+ adata.obs[col] = pd.Categorical(
664
+ cell_clone.map(lambda c, m=mapping: m.get(str(c)) if pd.notna(c) else None),
665
+ categories=labels.categories,
666
+ )
667
+
668
+ stab = {raw_to_label[r]: float(jac_mean[r]) for r in np.unique(ref)}
669
+ elig_tab = table.loc[eligible]
670
+ summary = (
671
+ elig_tab.groupby(key_added, observed=True)
672
+ .agg(n_clones=("n_cells", "size"), n_cells=("n_cells", "sum"),
673
+ mean_reliability=("reliability", "mean"), mean_confidence=("confidence", "mean"))
674
+ )
675
+ summary.index = pd.Index(summary.index.astype(str), name="programme")
676
+ summary["stability"] = [stab[p] for p in summary.index]
677
+ if assign_remaining:
678
+ n_assigned = table[f"{key_added}_assigned"].astype(str).value_counts()
679
+ summary["n_clones_with_assigned"] = [int(n_assigned.get(p, 0)) for p in summary.index]
680
+ if not split_tests.empty: # name the two sides of every tested node by the final programmes
681
+ split_tests = split_tests.copy()
682
+ for side in ("left", "right"):
683
+ split_tests[f"programmes_{side}"] = [", ".join(sorted({str(raw_to_label[g]) for g in gs if g in raw_to_label},
684
+ key=lambda t: (len(t), t)))
685
+ for gs in split_tests.pop(f"groups_{side}")]
686
+ get_uns(adata)[PROGRAMME_KEY] = {
687
+ "summary": summary,
688
+ "resolution_scan": h5ad_safe(scan),
689
+ "split_tests": h5ad_safe(split_tests),
690
+ "params": {
691
+ "resolution": chosen, "auto": auto, "n_neighbors": int(n_neighbors),
692
+ "weight_by_reliability": bool(weight_by_reliability),
693
+ "test_splits": bool(test_splits), "alpha": float(alpha), "n_null": int(n_null),
694
+ "split_null": str(split_null),
695
+ "min_reliability": float(min_reliability), "n_boot": int(n_boot),
696
+ "stability_threshold": float(stability_threshold), "key_added": key_added,
697
+ "min_programme_size": int(min_programme_size),
698
+ "assign_remaining": bool(assign_remaining), "min_posterior": float(min_posterior),
699
+ "random_state": int(random_state),
700
+ },
701
+ }
702
+ n_stable = int((summary["stability"] >= stability_threshold).sum()) if n_boot else 0
703
+ if caution and summary.shape[0] >= 2 and n_stable < summary.shape[0]:
704
+ # with a single programme left there is nothing to interpret, and the log says so
705
+ warnings.warn(caution, stacklevel=2)
706
+ extra = (f"; {int(table[f'{key_added}_assigned'].notna().sum())} of {len(table)} profiled clones "
707
+ f"assigned with posterior >= {min_posterior}" if assign_remaining else "")
708
+ if summary.shape[0] == 1:
709
+ log(f"programmes: no significant split among {eligible.size} core clones (resolution {chosen}): the "
710
+ "clones differ, but along a continuum rather than in distinct programmes.", verbose)
711
+ else:
712
+ n_merged = int((~split_tests["split"]).sum()) if not split_tests.empty else 0
713
+ merged_txt = f" after merging {n_merged} non-significant split(s)" if n_merged else ""
714
+ log(
715
+ f"programmes: {summary.shape[0]} programmes over {eligible.size} core clones "
716
+ f"(resolution {chosen}{merged_txt}); {n_stable} with bootstrap stability >= {stability_threshold}{extra}.",
717
+ verbose,
718
+ )
719
+ return summary
720
+
721
+
722
+ # --------------------------------------------------------------------------- posterior assignment
723
+
724
+
725
+ def _assign_remaining(adata, model, core_ids, core_labels):
726
+ """Assign every profiled clone to a programme with a Gaussian classifier.
727
+
728
+ Each programme ``P`` is a Gaussian over (unshrunk) clone mean profiles
729
+ with centre ``mu_P`` and between-clone variance ``t_P`` estimated from its
730
+ core clones (sampling noise removed). A clone with ``n`` cells is observed
731
+ with extra noise ``sigma2 / n``, so small clones get flatter posteriors:
732
+
733
+ p(P | clone) ~ pi_P * N(mean_clone; mu_P, t_P + sigma2 / n)
734
+
735
+ Returns the most likely programme and its posterior for every row of the
736
+ clone table.
737
+ """
738
+ import scipy.sparse as sp
739
+ from scipy.special import logsumexp
740
+
741
+ prof = get_profiles(adata)
742
+ table = prof["clone_table"]
743
+ code_of = {str(c): i for i, c in enumerate(model.clone_index.astype(str))}
744
+ rows = np.array([code_of[c] for c in table.index], dtype=np.int64)
745
+ g = model.clone_codes
746
+ ok = np.flatnonzero(g >= 0)
747
+ member = sp.csr_matrix((np.ones(ok.size), (g[ok], ok)), shape=(len(model.clone_index), g.size))
748
+ n = np.asarray(member.sum(axis=1)).ravel()
749
+ means = np.asarray(member @ model.residuals) / np.maximum(n, 1)[:, None] - model.vc.grand_mean
750
+ means, n = means[rows], n[rows]
751
+ red = prof.get("reduction")
752
+ if red is None:
753
+ x, noise = means, model.vc.sigma2
754
+ else:
755
+ comps = np.asarray(red["components"])
756
+ x = (means - np.asarray(red["raw_mean"])) @ comps.T
757
+ noise = (comps**2) @ model.vc.sigma2
758
+
759
+ core_pos = table.index.get_indexer(core_ids)
760
+ names = pd.Index(pd.unique(np.asarray(core_labels, dtype=str)))
761
+ lab = names.get_indexer(np.asarray(core_labels, dtype=str))
762
+ floor = 1e-6 * float(noise.mean()) + 1e-12
763
+ mu, t, prior = [], [], []
764
+ for k in range(len(names)):
765
+ members = core_pos[lab == k]
766
+ xm = x[members]
767
+ mu.append(xm.mean(axis=0))
768
+ t.append(np.maximum(xm.var(axis=0) - (noise[None, :] / n[members][:, None]).mean(axis=0), floor))
769
+ prior.append(members.size)
770
+ mu, t = np.vstack(mu), np.vstack(t)
771
+ prior = np.log(np.asarray(prior, dtype=float) / np.sum(prior))
772
+ var = t[None, :, :] + noise[None, None, :] / np.maximum(n, 1)[:, None, None]
773
+ ll = -0.5 * (((x[:, None, :] - mu[None, :, :]) ** 2) / var + np.log(var)).sum(axis=2) + prior[None, :]
774
+ post = np.exp(ll - logsumexp(ll, axis=1, keepdims=True))
775
+ return np.asarray(names)[post.argmax(axis=1)], post.max(axis=1)
776
+
777
+
778
+ def get_programmes(adata) -> dict:
779
+ """Stored programme results (raises if :func:`find_programmes` was not run)."""
780
+ out = get_uns(adata).get(PROGRAMME_KEY)
781
+ if out is None:
782
+ raise ValueError("Run threadfin.tl.find_programmes() (or threadfin.run()) first.")
783
+ return out
784
+
785
+
786
+ def programme_composition(adata, state_key: str, *, key: str | None = None) -> pd.DataFrame:
787
+ """Fraction of each programme's cells in each cell state (rows sum to 1)."""
788
+ key = key or get_programmes(adata)["params"]["key_added"]
789
+ df = adata.obs[[key, state_key]].dropna()
790
+ tab = pd.crosstab(df[key], df[state_key])
791
+ return tab.div(tab.sum(axis=1), axis=0)
792
+
793
+
794
+ def _clone_expression(adata, *, clone_key, cell_mask, layer, context_key, state_key, state, min_cells_in_state,
795
+ min_frac_expressed, caller):
796
+ """Mean context-centred expression of each clone's cells (optionally only its cells of one state).
797
+
798
+ Returns ``(cells, clone_codes, clone_index, genes, clone_means)``: the cells used, their clone codes, the
799
+ clone ids, the indices of the genes expressed in at least ``min_frac_expressed`` of those cells, and the
800
+ clones x genes matrix of means, each minus the mean of the same-state cells of the clone's contexts.
801
+ """
802
+ import scipy.sparse as sp
803
+
804
+ from ._utils import codes
805
+
806
+ x = adata.layers[layer] if layer is not None else adata.X
807
+ x = sp.csr_matrix(x) if not sp.issparse(x) else x.tocsr()
808
+ in_state = np.ones(adata.n_obs, dtype=bool)
809
+ if state_key is not None:
810
+ require_obs(adata, state_key, context=f"{caller}(state_key=...)")
811
+ levels = adata.obs[state_key].dropna().astype(str)
812
+ if state is None or str(state) not in set(levels):
813
+ raise ValueError(f"state={state!r} is not a level of '{state_key}'; levels: "
814
+ f"{', '.join(sorted(set(levels))[:12])}.")
815
+ in_state = (adata.obs[state_key].astype(str) == str(state)).to_numpy()
816
+ cells = np.flatnonzero(np.asarray(cell_mask) & adata.obs[clone_key].notna().to_numpy() & in_state)
817
+ clone_codes, clone_index = codes(adata.obs[clone_key].iloc[cells].astype(str))
818
+ if min_cells_in_state > 1: # keep clones with enough cells in the state
819
+ counts = np.bincount(clone_codes, minlength=len(clone_index))
820
+ cells = cells[(counts >= min_cells_in_state)[clone_codes]]
821
+ clone_codes, clone_index = codes(adata.obs[clone_key].iloc[cells].astype(str))
822
+ n_clones = len(clone_index)
823
+ member = sp.csr_matrix((np.ones(cells.size), (clone_codes, np.arange(cells.size))), shape=(n_clones, cells.size))
824
+ member = sp.diags(1.0 / np.maximum(np.asarray(member.sum(axis=1)).ravel(), 1)) @ member
825
+ expressed = np.asarray((x[cells] > 0).mean(axis=0)).ravel() >= min_frac_expressed
826
+ genes = np.flatnonzero(expressed)
827
+ clone_means = np.asarray((member @ x[cells][:, genes]).toarray())
828
+ if context_key is not None:
829
+ # subtract, for every clone, the mean expression of the contexts its cells came from
830
+ ctx, ctx_index = codes(adata.obs[context_key])
831
+ ok = np.flatnonzero((ctx >= 0) & in_state) # within a state, centre on that state's cells
832
+ cmember = sp.csr_matrix((np.ones(ok.size), (ctx[ok], ok)), shape=(len(ctx_index), adata.n_obs))
833
+ cmember = sp.diags(1.0 / np.maximum(np.asarray(cmember.sum(axis=1)).ravel(), 1)) @ cmember
834
+ ctx_means = np.asarray((cmember @ x[:, genes]).toarray())
835
+ cell_ctx = sp.csr_matrix((np.ones(cells.size), (np.arange(cells.size), np.maximum(ctx[cells], 0))),
836
+ shape=(cells.size, len(ctx_index)))
837
+ clone_means -= np.asarray((member @ cell_ctx).toarray()) @ ctx_means
838
+ return cells, clone_codes, clone_index, genes, clone_means
839
+
840
+
841
+ def programme_markers(
842
+ adata,
843
+ *,
844
+ programme_key: str = "clone_programme",
845
+ layer: str | None = "log_norm",
846
+ context_key: str | None = "auto",
847
+ min_frac_expressed: float = 0.05,
848
+ n_top: int = 25,
849
+ state_key: str | None = None,
850
+ state: str | None = None,
851
+ min_cells_in_state: int = 1,
852
+ verbose: bool = True,
853
+ ) -> pd.DataFrame:
854
+ """Genes that distinguish programmes, tested with clones as replicates.
855
+
856
+ Every clone is summarised by the mean context-centred log expression of
857
+ its cells; for each programme, clones inside are compared with clones in
858
+ other programmes by a Mann-Whitney test per gene (BH FDR). Testing across
859
+ clones avoids the pseudoreplication of cell-level marker tests, where one
860
+ expanded clone contributes hundreds of correlated cells.
861
+
862
+ Programmes usually differ in which cell states their clones contain, so the genes found this way are
863
+ largely the markers of those states. To ask what differs between programmes *within one state* — for
864
+ example between the germinal-centre B cells of clones from two programmes — pass ``state_key`` and
865
+ ``state``: each clone is then summarised by its cells in that state only (clones with at least
866
+ ``min_cells_in_state`` such cells), centred on the cells of the same state in its context.
867
+
868
+ Returns
869
+ -------
870
+ Tidy DataFrame ``[programme, gene, mean_diff, auc, pvalue, fdr, rank]`` (and ``state`` when given)
871
+ with the top ``n_top`` up-regulated genes per programme.
872
+ """
873
+ import scipy.sparse as sp
874
+ from scipy.stats import rankdata
875
+
876
+ from ._utils import bh_fdr, codes
877
+
878
+ prof = get_profiles(adata)
879
+ clone_key = prof["params"]["clone_key"]
880
+ if context_key == "auto":
881
+ context_key = prof["params"].get("context_key")
882
+ lab = adata.obs[programme_key]
883
+ cells, clone_codes, clone_index, genes, clone_means = _clone_expression(
884
+ adata, clone_key=clone_key, cell_mask=lab.notna().to_numpy(), layer=layer, context_key=context_key,
885
+ state_key=state_key, state=state, min_cells_in_state=min_cells_in_state,
886
+ min_frac_expressed=min_frac_expressed, caller="programme_markers")
887
+ n_clones = len(clone_index)
888
+ if n_clones < 6:
889
+ raise ValueError(f"Only {n_clones} clones have cells{' in ' + repr(state) if state_key else ''}; "
890
+ "programme markers need at least 3 clones inside and 3 outside a programme.")
891
+ clone_prog = (pd.Series(lab.iloc[cells].astype(str).to_numpy()).groupby(clone_codes).first()).to_numpy()
892
+
893
+ ranks = np.apply_along_axis(rankdata, 0, clone_means)
894
+ rows = []
895
+ for prog in sorted(pd.unique(clone_prog), key=lambda s: (len(s), s)):
896
+ inside = clone_prog == prog
897
+ n1, n2 = int(inside.sum()), int((~inside).sum())
898
+ if n1 < 3 or n2 < 3:
899
+ continue
900
+ u = ranks[inside].sum(axis=0) - n1 * (n1 + 1) / 2
901
+ auc = u / (n1 * n2)
902
+ # normal approximation to the Mann-Whitney U (ties ignored)
903
+ z = (u - n1 * n2 / 2) / np.sqrt(n1 * n2 * (n1 + n2 + 1) / 12)
904
+ from scipy.stats import norm
905
+
906
+ pvals = 2 * norm.sf(np.abs(z))
907
+ diff = clone_means[inside].mean(axis=0) - clone_means[~inside].mean(axis=0)
908
+ tab = pd.DataFrame({"programme": prog, "gene": adata.var_names[genes], "mean_diff": diff,
909
+ "auc": auc, "pvalue": pvals})
910
+ if state_key is not None:
911
+ tab.insert(1, "state", str(state))
912
+ tab["fdr"] = bh_fdr(tab["pvalue"].to_numpy())
913
+ tab = tab[tab["mean_diff"] > 0].sort_values(["fdr", "mean_diff"], ascending=[True, False]).head(n_top)
914
+ tab["rank"] = np.arange(1, len(tab) + 1)
915
+ rows.append(tab)
916
+ out = pd.concat(rows, ignore_index=True) if rows else pd.DataFrame(
917
+ columns=["programme", "gene", "mean_diff", "auc", "pvalue", "fdr", "rank"])
918
+ get_uns(adata)["programme_markers"] = h5ad_safe(out)
919
+ where = f" within '{state}'" if state_key is not None else ""
920
+ log(f"programme markers{where}: {n_clones} clones as replicates, {genes.size} genes tested.", verbose)
921
+ return out
922
+
923
+
924
+ def landscape_genes(
925
+ adata,
926
+ *,
927
+ layer: str | None = "log_norm",
928
+ context_key: str | None = "auto",
929
+ n_neighbors: int = 15,
930
+ min_frac_expressed: float = 0.05,
931
+ state_key: str | None = None,
932
+ state: str | None = None,
933
+ min_cells_in_state: int = 1,
934
+ n_top: int | None = 50,
935
+ verbose: bool = True,
936
+ ) -> pd.DataFrame:
937
+ """Genes whose expression changes smoothly across the clones, with no programmes needed.
938
+
939
+ Programme markers need distinct programmes; when clones vary along a continuum (a single programme),
940
+ this asks instead which genes follow the landscape of clone profiles. Every profiled clone is summarised
941
+ by the mean context-centred log expression of its cells (as in :func:`programme_markers`; with
942
+ ``state_key``/``state``, of its cells in that state only). Clones are linked to their ``n_neighbors``
943
+ nearest clones in profile space, and each gene is scored by Moran's I of the clone means on that graph:
944
+ I > 0 when neighbouring clones express the gene alike, about 0 when the gene ignores the landscape. P values
945
+ use the normal approximation under randomisation (Cliff and Ord), corrected by Benjamini-Hochberg.
946
+ ``corr_axis1`` is the Spearman correlation of the gene with the first component of the clone profiles,
947
+ the main axis along which clones differ, and gives the direction of a gradient.
948
+
949
+ This describes the landscape rather than testing an independent hypothesis: the profiles are built from
950
+ the same expression, so the genes that make clones differ are expected to score. Use it to name what
951
+ varies across clones; test annotations that the profiles did not use with :func:`profile_association`.
952
+
953
+ With hundreds of clones most expressed genes reach FDR < 0.05, so rank genes by Moran's I (0 to 1 in
954
+ practice) rather than counting them. The top genes often lie at one end of the main axis; ``n_top=None``
955
+ returns every gene that passes, so both ends can be read from ``corr_axis1``.
956
+
957
+ Returns
958
+ -------
959
+ DataFrame ``[gene, morans_i, z, pvalue, fdr, corr_axis1, rank]`` of the ``n_top`` genes with the largest
960
+ Moran's I among those with FDR < 0.05 (fewer if fewer pass; all of them with ``n_top=None``), also stored
961
+ in ``adata.uns['threadfin']['landscape_genes']``.
962
+ """
963
+ import scipy.sparse as sp
964
+ from scipy.stats import norm, rankdata
965
+ from sklearn.neighbors import NearestNeighbors
966
+
967
+ from ._utils import bh_fdr, require_positive_int
968
+
969
+ require_positive_int(n_neighbors, "n_neighbors")
970
+ prof = get_profiles(adata)
971
+ clone_key = prof["params"]["clone_key"]
972
+ if context_key == "auto":
973
+ context_key = prof["params"].get("context_key")
974
+ feats = prof["features"]
975
+ profiled = adata.obs[clone_key].astype(str).isin(set(feats.index.astype(str))).to_numpy()
976
+ _, _, clone_index, genes, clone_means = _clone_expression(
977
+ adata, clone_key=clone_key, cell_mask=profiled, layer=layer, context_key=context_key,
978
+ state_key=state_key, state=state, min_cells_in_state=min_cells_in_state,
979
+ min_frac_expressed=min_frac_expressed, caller="landscape_genes")
980
+ n = len(clone_index)
981
+ if n < n_neighbors + 2:
982
+ raise ValueError(f"Only {n} profiled clones{' with cells in ' + repr(state) if state_key else ''}; "
983
+ f"landscape_genes needs more than n_neighbors + 1 = {n_neighbors + 1}.")
984
+ f = feats.loc[clone_index.astype(str)].to_numpy(dtype=float)
985
+ nn = NearestNeighbors(n_neighbors=n_neighbors + 1).fit(f)
986
+ _, idx = nn.kneighbors(f)
987
+ rows = np.repeat(np.arange(n), n_neighbors)
988
+ w = sp.csr_matrix((np.ones(rows.size), (rows, idx[:, 1:].ravel())), shape=(n, n))
989
+ s0 = float(w.sum())
990
+ ws = w + w.T
991
+ s1 = 0.5 * float(ws.multiply(ws).sum())
992
+ s2 = float(((np.asarray(w.sum(axis=1)).ravel() + np.asarray(w.sum(axis=0)).ravel()) ** 2).sum())
993
+
994
+ z = clone_means - clone_means.mean(axis=0)
995
+ var = (z ** 2).sum(axis=0)
996
+ ok = var > 0
997
+ z, var, genes = z[:, ok], var[ok], genes[ok]
998
+ moran = (n / s0) * np.asarray((z * (w @ z)).sum(axis=0)).ravel() / var
999
+ expected = -1.0 / (n - 1)
1000
+ b2 = n * (z ** 4).sum(axis=0) / var ** 2 # kurtosis of each gene's clone means
1001
+ var_i = ((n * ((n ** 2 - 3 * n + 3) * s1 - n * s2 + 3 * s0 ** 2)
1002
+ - b2 * ((n ** 2 - n) * s1 - 2 * n * s2 + 6 * s0 ** 2))
1003
+ / ((n - 1) * (n - 2) * (n - 3) * s0 ** 2) - expected ** 2)
1004
+ zscore = (moran - expected) / np.sqrt(np.maximum(var_i, 1e-12))
1005
+ pvals = norm.sf(zscore) # one-sided: smoother than random
1006
+ # the main axis of the clone profiles (their first principal component; the sign is arbitrary, so it is
1007
+ # fixed to make the most extreme clone positive), and each gene's rank correlation with it
1008
+ fc = f - f.mean(axis=0)
1009
+ u, sv, _ = np.linalg.svd(fc, full_matrices=False)
1010
+ axis1 = u[:, 0] * sv[0]
1011
+ axis1 = axis1 if axis1[np.argmax(np.abs(axis1))] >= 0 else -axis1
1012
+ ra = rankdata(axis1)
1013
+ rz = rankdata(z, axis=0) # columns of z are the genes kept above
1014
+ ra, rz = ra - ra.mean(), rz - rz.mean(axis=0)
1015
+ denom = np.sqrt((ra ** 2).sum() * (rz ** 2).sum(axis=0))
1016
+ corr = np.divide((ra[:, None] * rz).sum(axis=0), denom, out=np.full(rz.shape[1], np.nan), where=denom > 0)
1017
+ tab = pd.DataFrame({"gene": adata.var_names[genes], "morans_i": moran, "z": zscore, "pvalue": pvals,
1018
+ "fdr": bh_fdr(pvals), "corr_axis1": corr})
1019
+ out = tab[tab["fdr"] < 0.05].sort_values("morans_i", ascending=False)
1020
+ out = (out if n_top is None else out.head(n_top)).reset_index(drop=True)
1021
+ out["rank"] = np.arange(1, len(out) + 1)
1022
+ get_uns(adata)["landscape_genes"] = h5ad_safe(out)
1023
+ where = f" within '{state}'" if state_key is not None else ""
1024
+ # with many clones most genes reach significance; Moran's I itself says how closely a gene follows the landscape
1025
+ top = ", ".join(f"{g} ({i:.2f})" for g, i in zip(out["gene"].head(5), out["morans_i"].head(5)))
1026
+ log(f"landscape genes{where}: Moran's I over {n} clones, FDR < 0.05 for {int((tab['fdr'] < 0.05).sum())} of "
1027
+ f"{len(tab)} genes; highest: {top}.", verbose)
1028
+ return out