threadfin 4.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
threadfin/profiles.py ADDED
@@ -0,0 +1,603 @@
1
+ """Clone state profiles: variance components and shrunken clone effects.
2
+
3
+ The model
4
+ ---------
5
+ For cell ``i`` of clone ``c`` sampled in context ``s`` (a sample, donor or
6
+ batch), with expression embedding ``x_i`` (one row of e.g. a PCA), Threadfin
7
+ fits the one-way random-effects model, independently for every feature ``j``::
8
+
9
+ x_ij = mu_sj + u_cj + e_ij, u_cj ~ N(0, tau2_j), e_ij ~ N(0, sigma2_j)
10
+
11
+ * ``mu_s`` is the context mean, estimated from *all* cells of the context, so
12
+ a clone is described relative to the cells it was sampled with;
13
+ * ``tau2_j`` is the between-clone (clonally inherited) variance and
14
+ ``sigma2_j`` the within-clone variance;
15
+ * the **clonal intraclass correlation**
16
+ ``ICC = sum(tau2) / sum(tau2 + sigma2)`` is the fraction of transcriptional
17
+ variance explained by clone identity;
18
+ * each clone is represented by its BLUP (best linear unbiased predictor)
19
+ ``u_c = lambda_c * mean_i(x_i - mu_s)``, with
20
+ ``lambda_cj = n_c tau2_j / (n_c tau2_j + sigma2_j)``. Small clones are
21
+ shrunk towards their context, expanded clones keep their own profile, and
22
+ expression directions without clonal signal (``tau2_j = 0``) drop out;
23
+ * a clone's **reliability** is the tau2-weighted mean of ``lambda_cj``
24
+ (Spearman-Brown): the expected squared correlation between the estimated
25
+ and the true clone profile.
26
+
27
+ With ``representation="kernel"`` the same model is applied to random Fourier
28
+ features of the embedding (Rahimi & Recht 2007). Each clone is then summarised
29
+ by a shrunken *kernel mean embedding* of its cell distribution instead of its
30
+ centroid, so a clone split between two states is distinguished from a clone
31
+ sitting between them.
32
+
33
+ Variance components use the moment (ANOVA) estimators of the unbalanced
34
+ one-way design (Searle, Casella & McCulloch 1992, ch. 3).
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ from dataclasses import dataclass, field
40
+ import json
41
+ import hashlib
42
+
43
+ import numpy as np
44
+ import pandas as pd
45
+ import scipy.sparse as sp
46
+
47
+ from ._utils import (codes, ensure_actual, get_basis, get_uns, log, modal, require_complete_groups, require_obs,
48
+ require_positive_int)
49
+
50
+ PROFILE_KEY = "profiles"
51
+
52
+
53
+ # --------------------------------------------------------------------------- variance components
54
+
55
+
56
+ @dataclass
57
+ class VarianceComponents:
58
+ """Moment estimates of the one-way random-effects model, per feature."""
59
+
60
+ sigma2: np.ndarray
61
+ tau2: np.ndarray
62
+ n0: float
63
+ n_clones: int
64
+ n_cells: int
65
+ grand_mean: np.ndarray = field(repr=False)
66
+
67
+ @property
68
+ def icc_per_feature(self) -> np.ndarray:
69
+ """ICC of each feature (e.g. each principal component)."""
70
+ tot = self.tau2 + self.sigma2
71
+ return np.divide(self.tau2, tot, out=np.zeros_like(tot), where=tot > 0)
72
+
73
+ @property
74
+ def icc(self) -> float:
75
+ """Pooled clonal ICC: share of total variance explained by clone."""
76
+ tot = float(np.sum(self.tau2 + self.sigma2))
77
+ return float(np.sum(self.tau2) / tot) if tot > 0 else 0.0
78
+
79
+ def shrinkage(self, n) -> np.ndarray:
80
+ """Shrinkage factors ``lambda`` for clone sizes ``n``: ``(len(n), n_features)``."""
81
+ n = np.asarray(n, dtype=float)[:, None]
82
+ num = n * self.tau2[None, :]
83
+ den = num + self.sigma2[None, :]
84
+ return np.divide(num, den, out=np.zeros_like(num), where=den > 0)
85
+
86
+ def reliability(self, n) -> np.ndarray:
87
+ """tau2-weighted mean shrinkage (Spearman-Brown reliability) per clone."""
88
+ lam = self.shrinkage(n)
89
+ total = self.tau2.sum()
90
+ w = self.tau2 / total if total > 0 else np.zeros_like(self.tau2)
91
+ return lam @ w
92
+
93
+ def min_cells_for(self, target: float = 0.5) -> int:
94
+ """Smallest size reaching the actual feature-weighted reliability.
95
+
96
+ The pooled-ICC Spearman–Brown shortcut is not valid when features
97
+ have different noise-to-signal ratios. Return int32 max if no
98
+ finite supported size reaches the target.
99
+ """
100
+ if not np.isfinite(target) or not 0 < target < 1:
101
+ raise ValueError("target must be finite and strictly between 0 and 1.")
102
+ limit = int(np.iinfo(np.int32).max)
103
+ if self.tau2.sum() <= 0:
104
+ return limit
105
+ hi = 1
106
+ while self.reliability([hi])[0] < target and hi < limit:
107
+ hi = min(2 * hi, limit)
108
+ if self.reliability([hi])[0] < target:
109
+ return limit
110
+ lo = 1
111
+ while lo < hi:
112
+ mid = (lo + hi) // 2
113
+ if self.reliability([mid])[0] >= target:
114
+ hi = mid
115
+ else:
116
+ lo = mid + 1
117
+ return lo
118
+
119
+ def as_dict(self) -> dict:
120
+ """Plain dict (numpy arrays) suitable for ``adata.uns``."""
121
+ return {
122
+ "icc": self.icc,
123
+ "sigma2": np.asarray(self.sigma2),
124
+ "tau2": np.asarray(self.tau2),
125
+ "icc_per_feature": self.icc_per_feature,
126
+ "n0": float(self.n0),
127
+ "n_clones": int(self.n_clones),
128
+ "n_cells": int(self.n_cells),
129
+ }
130
+
131
+
132
+ def variance_components(residuals: np.ndarray, clone_codes: np.ndarray) -> VarianceComponents:
133
+ """ANOVA estimators of ``sigma2`` and ``tau2`` from clones with >= 2 cells.
134
+
135
+ Parameters
136
+ ----------
137
+ residuals
138
+ ``(n_cells, n_features)`` context-centred values.
139
+ clone_codes
140
+ Integer clone code per row; negative values are ignored. Only clones
141
+ with at least two rows contribute, because a singleton carries no
142
+ information about within-clone variance.
143
+ """
144
+ cc = np.asarray(clone_codes)
145
+ ok = cc >= 0
146
+ sizes = np.bincount(cc[ok]) if ok.any() else np.array([], dtype=int)
147
+ multi = np.flatnonzero(sizes >= 2)
148
+ if multi.size < 2:
149
+ raise ValueError("Need at least two clones with >= 2 cells to estimate variance components.")
150
+ # re-index the multi-cell clones as 0..C-1; every other row is dropped
151
+ remap = -np.ones(sizes.size, dtype=np.int64)
152
+ remap[multi] = np.arange(multi.size)
153
+ g = np.where(ok, remap[np.where(ok, cc, 0)], -1)
154
+ keep = g >= 0
155
+ r, g = residuals[keep], g[keep]
156
+ n_clones = multi.size
157
+ n_c = np.bincount(g, minlength=n_clones).astype(float)
158
+ n_total = float(n_c.sum())
159
+
160
+ # per-clone sums via a sparse indicator matrix (much faster than np.add.at)
161
+ member = sp.csr_matrix((np.ones(g.size), (g, np.arange(g.size))), shape=(n_clones, g.size))
162
+ sums = np.asarray(member @ r)
163
+ means = sums / n_c[:, None]
164
+ grand = r.mean(axis=0)
165
+ # sums of squares via sum(x^2) - sum(n_c * mean_c^2): one pass, no loops
166
+ ss_raw = np.einsum("ij,ij->j", r, r)
167
+ ss_between_raw = np.einsum("c,cj->j", n_c, means**2)
168
+ ss_within = ss_raw - ss_between_raw
169
+ ss_between = ss_between_raw - n_total * grand**2
170
+ ms_within = np.maximum(ss_within, 0.0) / (n_total - n_clones)
171
+ ms_between = np.maximum(ss_between, 0.0) / (n_clones - 1)
172
+ # effective clone size of an unbalanced design
173
+ n0 = (n_total - float(np.sum(n_c**2)) / n_total) / (n_clones - 1)
174
+ tau2 = np.maximum((ms_between - ms_within) / n0, 0.0)
175
+ return VarianceComponents(
176
+ sigma2=ms_within, tau2=tau2, n0=float(n0), n_clones=int(n_clones),
177
+ n_cells=int(n_total), grand_mean=grand,
178
+ )
179
+
180
+
181
+ # --------------------------------------------------------------------------- feature maps
182
+
183
+
184
+ def _center_by(x: np.ndarray, ctx: np.ndarray, finite: np.ndarray) -> np.ndarray:
185
+ """Subtract each context's mean (cells with no context: the global mean)."""
186
+ n_ctx = int(ctx.max()) + 1 if (ctx >= 0).any() else 0
187
+ valid = (ctx >= 0) & finite
188
+ member = sp.csr_matrix((np.ones(valid.sum()), (ctx[valid], np.flatnonzero(valid))), shape=(n_ctx, x.shape[0]))
189
+ cnt = np.asarray(member.sum(axis=1)).ravel()
190
+ mu = np.asarray(member @ x) / np.maximum(cnt, 1)[:, None]
191
+ fallback = x[finite].mean(axis=0)
192
+ return x - np.where((ctx >= 0)[:, None], mu[np.maximum(ctx, 0)], fallback)
193
+
194
+
195
+ def neighbour_average(x: np.ndarray, k: int, random_state: int = 0) -> np.ndarray:
196
+ """Average every cell with its ``k`` nearest cells (itself included).
197
+
198
+ Borrows strength from cells in the same local state, which denoises the
199
+ profiles of small clones. Used for programme profiles only: the
200
+ coherence and memory statistics are computed on unsmoothed cells.
201
+ """
202
+ n = x.shape[0]
203
+ k = int(min(k, n))
204
+ if n > 50_000:
205
+ from pynndescent import NNDescent
206
+
207
+ idx = NNDescent(x, n_neighbors=k, random_state=random_state).neighbor_graph[0]
208
+ else:
209
+ from sklearn.neighbors import NearestNeighbors
210
+
211
+ idx = NearestNeighbors(n_neighbors=k).fit(x).kneighbors(x, return_distance=False)
212
+ out = np.zeros_like(x)
213
+ for j in range(k): # column-wise accumulation keeps memory at O(n * features)
214
+ out += x[idx[:, j]]
215
+ return out / k
216
+
217
+
218
+ def median_bandwidth(x: np.ndarray, n_sample: int = 2000, rng=None) -> float:
219
+ """Median distance between random pairs of cells (Gretton et al. 2012)."""
220
+ rng = rng or np.random.default_rng(0)
221
+ n = x.shape[0]
222
+ idx = rng.choice(n, size=min(n, n_sample), replace=False)
223
+ d = np.linalg.norm(x[idx] - x[rng.permutation(idx)], axis=1)
224
+ return float(np.median(d[d > 0])) if np.any(d > 0) else 1.0
225
+
226
+
227
+ def local_bandwidth(x: np.ndarray, k: int = 30, n_sample: int = 2000, rng=None) -> float:
228
+ """Kernel bandwidth from the typical distance to a cell's k-th neighbour.
229
+
230
+ A local scale (about the width of one cell state) resolves finer
231
+ differences between clones' state distributions than the median
232
+ heuristic, but every cell's features are noisier, so only larger clones
233
+ reach a given reliability. Useful for datasets with many large clones.
234
+ """
235
+ from sklearn.neighbors import NearestNeighbors
236
+
237
+ rng = rng or np.random.default_rng(0)
238
+ n = x.shape[0]
239
+ idx = rng.choice(n, size=min(n, n_sample), replace=False)
240
+ k = int(min(k, n - 1))
241
+ dist = NearestNeighbors(n_neighbors=k + 1).fit(x).kneighbors(x[idx])[0][:, -1]
242
+ bw = float(np.median(dist))
243
+ return bw if bw > 0 else 1.0
244
+
245
+
246
+ def _random_fourier_features(
247
+ x: np.ndarray, n_features: int, bandwidth: float, rng: np.random.Generator
248
+ ) -> tuple[np.ndarray, float]:
249
+ """Gaussian-kernel random Fourier features with the given bandwidth."""
250
+ p = x.shape[1]
251
+ w = rng.normal(0.0, 1.0 / bandwidth, size=(p, n_features))
252
+ phase = rng.uniform(0.0, 2 * np.pi, size=n_features)
253
+ return np.sqrt(2.0 / n_features) * np.cos(x @ w + phase), bandwidth
254
+
255
+
256
+ # --------------------------------------------------------------------------- model object
257
+
258
+
259
+ class ProfileModel:
260
+ """Context-centred cell features plus the fitted variance components.
261
+
262
+ It holds cell-level matrices, so it is not stored in ``adata``; it is
263
+ rebuilt on demand from the parameters saved by :func:`clone_profiles`
264
+ (see :func:`model_from_adata`).
265
+ """
266
+
267
+ def __init__(self, residuals, clone_codes, clone_index, context_codes, vc, params):
268
+ self.residuals = residuals
269
+ self.clone_codes = clone_codes
270
+ self.clone_index = clone_index
271
+ self.context_codes = context_codes
272
+ self.vc = vc
273
+ self.params = params
274
+
275
+ def group_blups(self, group_codes: np.ndarray, n_groups: int, weights: np.ndarray | None = None):
276
+ """BLUPs for any grouping of cells (clones, time snapshots, bootstraps).
277
+
278
+ Parameters
279
+ ----------
280
+ group_codes
281
+ Group code per cell (negative = not in any group).
282
+ n_groups
283
+ Number of groups.
284
+ weights
285
+ Optional per-cell weights (e.g. Poisson bootstrap counts).
286
+
287
+ Returns
288
+ -------
289
+ ``(blup, n_eff, reliability)`` where ``n_eff`` is the weighted number
290
+ of cells per group.
291
+ """
292
+ g = np.asarray(group_codes)
293
+ ok = np.flatnonzero(g >= 0)
294
+ w = np.ones(g.size) if weights is None else np.asarray(weights, dtype=float)
295
+ member = sp.csr_matrix((w[ok], (g[ok], ok)), shape=(n_groups, g.size))
296
+ sums = np.asarray(member @ self.residuals)
297
+ n_eff = np.asarray(member.sum(axis=1)).ravel()
298
+ with np.errstate(invalid="ignore", divide="ignore"):
299
+ means = sums / n_eff[:, None]
300
+ means = np.nan_to_num(means - self.vc.grand_mean[None, :])
301
+ blup = self.vc.shrinkage(n_eff) * means
302
+ return blup, n_eff, self.vc.reliability(n_eff)
303
+
304
+
305
+ def _input_fingerprint(adata, basis, clone_key, context_key, donor_key=None):
306
+ """Bind saved projections to their cells, groups and numeric coordinates."""
307
+ digest = hashlib.sha256(np.ascontiguousarray(get_basis(adata, basis)).tobytes())
308
+ columns = list(dict.fromkeys([clone_key] + [k for k in (context_key, donor_key) if k]))
309
+ digest.update(pd.util.hash_pandas_object(adata.obs[columns], index=True).to_numpy().tobytes())
310
+ return digest.hexdigest()
311
+
312
+
313
+ def build_model(
314
+ adata,
315
+ *,
316
+ clone_key: str = "clone_id",
317
+ basis: str = "X_pca",
318
+ context_key: str | None = None,
319
+ representation: str = "mean",
320
+ n_features: int = 256,
321
+ bandwidth: float | str = "median",
322
+ smooth: int = 0,
323
+ random_state: int = 0,
324
+ feature_rng_state: str | None = None,
325
+ ) -> ProfileModel:
326
+ """Build the profile model: features, context centring, variance components."""
327
+ require_obs(adata, clone_key, context_key, context="clone profiles")
328
+ require_complete_groups(adata, context_key, context="clone profiles")
329
+ require_positive_int(n_features, "n_features")
330
+ if isinstance(smooth, (bool, np.bool_)) or not isinstance(smooth, (int, np.integer)) or smooth < 0:
331
+ raise ValueError("smooth must be a non-negative integer.")
332
+ x = get_basis(adata, basis)
333
+ finite = np.isfinite(x).all(axis=1)
334
+ if not finite.all():
335
+ x = np.where(finite[:, None], x, 0.0)
336
+ params = {
337
+ "clone_key": clone_key,
338
+ "basis": basis,
339
+ "context_key": context_key,
340
+ "representation": representation,
341
+ "n_features": int(n_features),
342
+ "input_sha256": _input_fingerprint(adata, basis, clone_key, context_key),
343
+ "smooth": int(smooth),
344
+ "random_state": int(random_state),
345
+ }
346
+
347
+ if representation not in ("mean", "kernel"):
348
+ raise ValueError("representation must be 'mean' or 'kernel'.")
349
+ if context_key is not None:
350
+ ctx, _ = codes(adata.obs[context_key])
351
+ else:
352
+ ctx = np.zeros(adata.n_obs, dtype=np.int64)
353
+
354
+ # 1) remove each context's mean, so a clone is compared with the cells it
355
+ # was sampled with (this also removes technical shifts between samples)
356
+ resid = _center_by(x, ctx, finite)
357
+ rng = np.random.default_rng(random_state)
358
+ if representation == "kernel":
359
+ # bandwidth from unsmoothed cells, so smoothed and unsmoothed models share one kernel
360
+ if bandwidth is None or bandwidth == "median":
361
+ bandwidth = median_bandwidth(resid[finite], rng=rng)
362
+ elif bandwidth == "local":
363
+ bandwidth = local_bandwidth(resid[finite], rng=rng)
364
+ if smooth and smooth > 1:
365
+ resid = np.where(finite[:, None], neighbour_average(np.where(finite[:, None], resid, 0.0), smooth,
366
+ random_state), 0.0)
367
+
368
+ # 2) "kernel": describe each cell by random Fourier features of its
369
+ # context-centred position, then centre again so that a clone's profile
370
+ # is the difference between its own cell distribution and its context's
371
+ if representation == "kernel":
372
+ if not np.isfinite(float(bandwidth)) or float(bandwidth) <= 0:
373
+ raise ValueError("bandwidth must be finite and positive.")
374
+ if feature_rng_state is not None:
375
+ rng.bit_generator.state = json.loads(feature_rng_state)
376
+ # Bandwidth estimation consumes RNG draws. Retain the exact subsequent
377
+ # state so bootstrap/snapshot features share the fitted RFF coordinates.
378
+ params["feature_rng_state"] = json.dumps(rng.bit_generator.state, sort_keys=True)
379
+ feats, bw = _random_fourier_features(resid, n_features, float(bandwidth), rng)
380
+ params["bandwidth"] = bw
381
+ resid = _center_by(feats, ctx, finite)
382
+
383
+ # 3) variance components from clones with >= 2 cells
384
+ clone_codes, clone_index = codes(adata.obs[clone_key])
385
+ clone_codes = np.where(finite, clone_codes, -1)
386
+ vc = variance_components(resid, clone_codes)
387
+ return ProfileModel(resid, clone_codes, clone_index, ctx, vc, params)
388
+
389
+
390
+ # --------------------------------------------------------------------------- public API
391
+
392
+
393
+ def clone_profiles(
394
+ adata,
395
+ *,
396
+ clone_key: str = "clone_id",
397
+ basis: str = "X_pca",
398
+ context_key: str | None = None,
399
+ donor_key: str | None = None,
400
+ representation: str = "mean",
401
+ n_features: int = 256,
402
+ n_components: int = 30,
403
+ bandwidth: float | str = "median",
404
+ smooth: int | None = None,
405
+ min_cells: int = 2,
406
+ random_state: int = 0,
407
+ verbose: bool = True,
408
+ ) -> pd.DataFrame:
409
+ """Estimate a shrunken, context-adjusted state profile for every clone.
410
+
411
+ Parameters
412
+ ----------
413
+ adata
414
+ AnnData with ``obs[clone_key]`` and an expression embedding.
415
+ clone_key
416
+ ``obs`` column with clone ids (missing = no BCR). Clone ids must be
417
+ unique across donors (see :func:`threadfin.define_clones`).
418
+ basis
419
+ ``obsm`` key of the embedding. Use a batch-integrated embedding built
420
+ without immunoglobulin genes (:func:`threadfin.pp.prepare_embedding`).
421
+ context_key
422
+ ``obs`` column of sampling contexts whose mean is removed before
423
+ profiling (e.g. sample, or donor x timepoint). Choose *nuisance*
424
+ strata only: biology you want to compare between clones (tissue,
425
+ sort gate, infection status) must not be a context.
426
+ donor_key
427
+ Optional donor column, recorded per clone for stratified tests.
428
+ representation
429
+ ``"mean"`` (shrunken centroid) or ``"kernel"`` (shrunken kernel mean
430
+ embedding from ``n_features`` random Fourier features, reduced to
431
+ ``n_components`` principal components). :func:`threadfin.run` uses
432
+ ``"kernel"``; this function keeps ``"mean"`` as its default so that
433
+ code written for earlier versions gives the same result.
434
+ bandwidth
435
+ Kernel bandwidth: ``"median"`` (median distance between cells, the
436
+ standard heuristic), ``"local"`` (distance to the 30th nearest
437
+ neighbour; sharper, but needs larger clones) or a number.
438
+ smooth
439
+ Average each cell with its ``smooth`` nearest cells before profiling
440
+ (denoises small clones by borrowing strength from their local state).
441
+ Default: 15 for ``"kernel"``, 0 for ``"mean"``. The coherence test and
442
+ clonal memory always use unsmoothed cells.
443
+ min_cells
444
+ Clones with fewer cells are not profiled. The default of 2 is **not** a quality filter --
445
+ Threadfin never filters on reliability -- it is what the clone map is *of*. Singleton clones
446
+ are 75 to 98 % of clones in these deposits, so including them turns the clone embedding back
447
+ into the cell UMAP it is supposed to summarise. Set 1 to include them; they are then drawn as
448
+ triangles rather than circles so a reader can see how much of a map rests on a single cell.
449
+ Threadfin does not filter clones. Every clone, and every cell in it, is measured data, and
450
+ the profiles are already **shrunk**: a clone seen in few cells has its profile pulled towards the
451
+ population mean by the variance-component model, so its influence is down-weighted in proportion to
452
+ how little was seen. Cutting on reliability *on top of* that shrinkage discards the same information
453
+ twice, and it discards it silently — on GSE287123 a 0.5 floor removes 75 % (RBD) and 90 % (NP-OVA) of
454
+ profiled clones, and on GSE246382 it removes all of them, which is how that dataset came to look as
455
+ though it had no clone structure when its map separates the sorted gate at p < 0.001.
456
+
457
+ Returns
458
+ -------
459
+ Clone table (one row per profiled clone) with ``n_cells``,
460
+ ``reliability`` and, when available, ``donor`` and ``n_contexts``.
461
+ Profiles and model parameters go to ``adata.uns['threadfin']['profiles']``.
462
+ """
463
+ ensure_actual(adata, verbose)
464
+ require_obs(adata, clone_key, context="clone profiling")
465
+ require_complete_groups(adata, context_key, donor_key, context="clone profiling")
466
+ if representation not in {"mean", "kernel"}:
467
+ raise ValueError("representation must be 'mean' or 'kernel'.")
468
+ require_positive_int(min_cells, "min_cells")
469
+ require_positive_int(n_components, "n_components")
470
+ labels = pd.Index(pd.unique(adata.obs[clone_key].dropna()))
471
+ if not labels.astype(str).is_unique:
472
+ raise ValueError("clone ids must remain unique when converted to strings.")
473
+ if donor_key is not None:
474
+ groups = adata.obs.dropna(subset=[clone_key]).groupby(clone_key, observed=True)[donor_key].nunique()
475
+ if groups.gt(1).any():
476
+ shared = groups.index[groups.gt(1)]
477
+ raise ValueError(
478
+ f"{len(shared):,} clone id(s) occur in more than one donor (e.g. {shared[0]!r}). A clone belongs to one "
479
+ f"individual: call clones within donors, or make the ids donor-private, e.g. "
480
+ f"adata.obs['{clone_key}'] = adata.obs['{donor_key}'].astype(str) + '|' + adata.obs['{clone_key}'].astype(str)")
481
+ if smooth is None:
482
+ smooth = 15 if representation == "kernel" else 0
483
+ model = build_model(
484
+ adata, clone_key=clone_key, basis=basis, context_key=context_key,
485
+ representation=representation, n_features=n_features, bandwidth=bandwidth,
486
+ smooth=smooth, random_state=random_state,
487
+ )
488
+ blup, n_eff, rel = model.group_blups(model.clone_codes, len(model.clone_index))
489
+ keep = n_eff >= min_cells
490
+ if keep.sum() < 3:
491
+ raise ValueError(
492
+ f"Only {int(keep.sum())} clones have >= {min_cells} cells; Threadfin needs at least 3 "
493
+ "expanded clones to estimate clone profiles. Include more BCR-matched cells, lower "
494
+ "min_cells when scientifically appropriate, or use a larger dataset."
495
+ )
496
+ ids = pd.Index(model.clone_index[keep].astype(str), name=clone_key)
497
+ feats = blup[keep]
498
+ names = [f"f{j}" for j in range(feats.shape[1])]
499
+ reduction = None
500
+ if representation == "kernel":
501
+ # PCA of the clone x feature matrix keeps the downstream graph compact
502
+ raw_mean = feats.mean(axis=0)
503
+ u, s, vt = np.linalg.svd(feats - raw_mean, full_matrices=False)
504
+ k = int(min(n_components, int((s > 1e-12).sum())))
505
+ feats = u[:, :k] * s[:k]
506
+ names = [f"kpc{j}" for j in range(k)]
507
+ reduction = {"components": vt[:k], "raw_mean": raw_mean}
508
+
509
+ table = pd.DataFrame({"n_cells": n_eff[keep].astype(int), "reliability": rel[keep]}, index=ids)
510
+ obs = adata.obs
511
+ if donor_key is not None:
512
+ require_obs(adata, donor_key)
513
+ sub = obs[[clone_key, donor_key]].dropna()
514
+ donor = sub.groupby(sub[clone_key].astype(str), observed=True)[donor_key].agg(modal)
515
+ table["donor"] = donor.reindex(table.index).astype(str).values
516
+ if context_key is not None:
517
+ sub = obs[[clone_key, context_key]].dropna()
518
+ nctx = sub.groupby(sub[clone_key].astype(str), observed=True)[context_key].nunique()
519
+ table["n_contexts"] = nctx.reindex(table.index).fillna(0).astype(int).values
520
+
521
+ vc = model.vc
522
+ get_uns(adata)[PROFILE_KEY] = {
523
+ "features": pd.DataFrame(feats, index=table.index, columns=names),
524
+ "clone_table": table,
525
+ "variance_components": vc.as_dict(),
526
+ "params": {**model.params, "donor_key": donor_key, "min_cells": int(min_cells),
527
+ "n_components": int(n_components),
528
+ "input_sha256": _input_fingerprint(adata, basis, clone_key, context_key, donor_key)},
529
+ "reduction": reduction,
530
+ }
531
+ log(
532
+ # the clonal ICC itself is reported by clonal_coherence, on unsmoothed cells; the variance components
533
+ # of the profile features (smoothed for kernel profiles) only set the shrinkage, so they are not quoted
534
+ f"clone profiles: {table.shape[0]} clones (>= {min_cells} cells, {vc.n_cells} cells in all). Profiles are "
535
+ f"shrunk, so a clone reaches reliability 0.5 at {vc.min_cells_for(0.5)} cells; nothing is filtered on that.",
536
+ verbose,
537
+ )
538
+ return table
539
+
540
+
541
+ def get_profiles(adata) -> dict:
542
+ """Stored profile results (raises if :func:`clone_profiles` was not run)."""
543
+ prof = get_uns(adata).get(PROFILE_KEY)
544
+ if prof is None:
545
+ raise ValueError("Run threadfin.tl.clone_profiles() (or threadfin.run()) first.")
546
+ return prof
547
+
548
+
549
+ def _feature_rng_state(adata) -> str:
550
+ """Recover old serialized models by matching their stored features.
551
+
552
+ Version 4.0 did not store the RNG state after bandwidth estimation.
553
+ Its numeric/median/local paths can be tested against the saved projection.
554
+ Refuse a mismatch instead of silently projecting a different feature map.
555
+ """
556
+ prof = get_profiles(adata)
557
+ p = prof["params"]
558
+ if p.get("feature_rng_state") is not None:
559
+ return p["feature_rng_state"]
560
+ for bandwidth in (p.get("bandwidth", "median"), "median", "local"):
561
+ model = build_model(
562
+ adata, clone_key=p["clone_key"], basis=p["basis"], context_key=p.get("context_key"),
563
+ representation="kernel", n_features=p["n_features"], bandwidth=bandwidth,
564
+ smooth=p.get("smooth", 0), random_state=p["random_state"],
565
+ )
566
+ if not np.isclose(model.params["bandwidth"], p.get("bandwidth", model.params["bandwidth"])):
567
+ continue
568
+ blup, _, _ = model.group_blups(model.clone_codes, len(model.clone_index))
569
+ index = pd.Index(model.clone_index.astype(str)).get_indexer(prof["features"].index)
570
+ if np.any(index < 0):
571
+ break
572
+ projected = project_features(adata, blup[index])
573
+ if np.allclose(projected, prof["features"].to_numpy(), rtol=1e-7, atol=1e-9):
574
+ p["feature_rng_state"] = model.params["feature_rng_state"]
575
+ return p["feature_rng_state"]
576
+ raise ValueError("Stored kernel features cannot be reproduced from the current cells and settings. "
577
+ "Rerun clone_profiles before bootstrap or snapshot analysis.")
578
+
579
+
580
+ def model_from_adata(adata, *, unsmoothed: bool = False, representation: str | None = None) -> ProfileModel:
581
+ """Rebuild the fitted feature map; optionally omit smoothing for statistics."""
582
+ p = get_profiles(adata)["params"]
583
+ if p.get("input_sha256") is not None:
584
+ current = _input_fingerprint(adata, p["basis"], p["clone_key"], p.get("context_key"), p.get("donor_key"))
585
+ if current != p["input_sha256"]:
586
+ raise ValueError("Cells, clone/context labels or embedding changed since clone_profiles; "
587
+ "refit profiles before downstream analysis.")
588
+ rep = representation or p["representation"]
589
+ state = _feature_rng_state(adata) if rep == "kernel" and p["representation"] == "kernel" else None
590
+ return build_model(
591
+ adata, clone_key=p["clone_key"], basis=p["basis"], context_key=p.get("context_key"),
592
+ representation=rep, n_features=p["n_features"],
593
+ bandwidth=p.get("bandwidth", "median"), smooth=0 if unsmoothed else p.get("smooth", 0),
594
+ random_state=p["random_state"], feature_rng_state=state,
595
+ )
596
+
597
+
598
+ def project_features(adata, blup: np.ndarray) -> np.ndarray:
599
+ """Map raw BLUPs (bootstrap replicates, time snapshots) into profile space."""
600
+ red = get_profiles(adata).get("reduction")
601
+ if red is None:
602
+ return blup
603
+ return (blup - np.asarray(red["raw_mean"])) @ np.asarray(red["components"]).T