threadfin 4.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- threadfin/__init__.py +86 -0
- threadfin/_utils.py +260 -0
- threadfin/api.py +658 -0
- threadfin/bcrgraph.py +255 -0
- threadfin/clones.py +653 -0
- threadfin/core.py +531 -0
- threadfin/dandelion.py +458 -0
- threadfin/density.py +409 -0
- threadfin/dynamics.py +275 -0
- threadfin/export.py +215 -0
- threadfin/expression.py +121 -0
- threadfin/heritability.py +224 -0
- threadfin/integrate.py +350 -0
- threadfin/io.py +280 -0
- threadfin/maptune.py +743 -0
- threadfin/metrics.py +137 -0
- threadfin/migration.py +165 -0
- threadfin/phylo.py +834 -0
- threadfin/plotting.py +535 -0
- threadfin/pp.py +253 -0
- threadfin/profiles.py +603 -0
- threadfin/programmes.py +1028 -0
- threadfin/programs.py +133 -0
- threadfin/reclustering.py +68 -0
- threadfin/sequence.py +379 -0
- threadfin/simulate.py +206 -0
- threadfin/specificity.py +355 -0
- threadfin/stats.py +476 -0
- threadfin/tl.py +62 -0
- threadfin-4.5.0.dist-info/METADATA +288 -0
- threadfin-4.5.0.dist-info/RECORD +34 -0
- threadfin-4.5.0.dist-info/WHEEL +5 -0
- threadfin-4.5.0.dist-info/licenses/LICENSE +21 -0
- threadfin-4.5.0.dist-info/top_level.txt +1 -0
threadfin/profiles.py
ADDED
|
@@ -0,0 +1,603 @@
|
|
|
1
|
+
"""Clone state profiles: variance components and shrunken clone effects.
|
|
2
|
+
|
|
3
|
+
The model
|
|
4
|
+
---------
|
|
5
|
+
For cell ``i`` of clone ``c`` sampled in context ``s`` (a sample, donor or
|
|
6
|
+
batch), with expression embedding ``x_i`` (one row of e.g. a PCA), Threadfin
|
|
7
|
+
fits the one-way random-effects model, independently for every feature ``j``::
|
|
8
|
+
|
|
9
|
+
x_ij = mu_sj + u_cj + e_ij, u_cj ~ N(0, tau2_j), e_ij ~ N(0, sigma2_j)
|
|
10
|
+
|
|
11
|
+
* ``mu_s`` is the context mean, estimated from *all* cells of the context, so
|
|
12
|
+
a clone is described relative to the cells it was sampled with;
|
|
13
|
+
* ``tau2_j`` is the between-clone (clonally inherited) variance and
|
|
14
|
+
``sigma2_j`` the within-clone variance;
|
|
15
|
+
* the **clonal intraclass correlation**
|
|
16
|
+
``ICC = sum(tau2) / sum(tau2 + sigma2)`` is the fraction of transcriptional
|
|
17
|
+
variance explained by clone identity;
|
|
18
|
+
* each clone is represented by its BLUP (best linear unbiased predictor)
|
|
19
|
+
``u_c = lambda_c * mean_i(x_i - mu_s)``, with
|
|
20
|
+
``lambda_cj = n_c tau2_j / (n_c tau2_j + sigma2_j)``. Small clones are
|
|
21
|
+
shrunk towards their context, expanded clones keep their own profile, and
|
|
22
|
+
expression directions without clonal signal (``tau2_j = 0``) drop out;
|
|
23
|
+
* a clone's **reliability** is the tau2-weighted mean of ``lambda_cj``
|
|
24
|
+
(Spearman-Brown): the expected squared correlation between the estimated
|
|
25
|
+
and the true clone profile.
|
|
26
|
+
|
|
27
|
+
With ``representation="kernel"`` the same model is applied to random Fourier
|
|
28
|
+
features of the embedding (Rahimi & Recht 2007). Each clone is then summarised
|
|
29
|
+
by a shrunken *kernel mean embedding* of its cell distribution instead of its
|
|
30
|
+
centroid, so a clone split between two states is distinguished from a clone
|
|
31
|
+
sitting between them.
|
|
32
|
+
|
|
33
|
+
Variance components use the moment (ANOVA) estimators of the unbalanced
|
|
34
|
+
one-way design (Searle, Casella & McCulloch 1992, ch. 3).
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
from dataclasses import dataclass, field
|
|
40
|
+
import json
|
|
41
|
+
import hashlib
|
|
42
|
+
|
|
43
|
+
import numpy as np
|
|
44
|
+
import pandas as pd
|
|
45
|
+
import scipy.sparse as sp
|
|
46
|
+
|
|
47
|
+
from ._utils import (codes, ensure_actual, get_basis, get_uns, log, modal, require_complete_groups, require_obs,
|
|
48
|
+
require_positive_int)
|
|
49
|
+
|
|
50
|
+
PROFILE_KEY = "profiles"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# --------------------------------------------------------------------------- variance components
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class VarianceComponents:
|
|
58
|
+
"""Moment estimates of the one-way random-effects model, per feature."""
|
|
59
|
+
|
|
60
|
+
sigma2: np.ndarray
|
|
61
|
+
tau2: np.ndarray
|
|
62
|
+
n0: float
|
|
63
|
+
n_clones: int
|
|
64
|
+
n_cells: int
|
|
65
|
+
grand_mean: np.ndarray = field(repr=False)
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def icc_per_feature(self) -> np.ndarray:
|
|
69
|
+
"""ICC of each feature (e.g. each principal component)."""
|
|
70
|
+
tot = self.tau2 + self.sigma2
|
|
71
|
+
return np.divide(self.tau2, tot, out=np.zeros_like(tot), where=tot > 0)
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def icc(self) -> float:
|
|
75
|
+
"""Pooled clonal ICC: share of total variance explained by clone."""
|
|
76
|
+
tot = float(np.sum(self.tau2 + self.sigma2))
|
|
77
|
+
return float(np.sum(self.tau2) / tot) if tot > 0 else 0.0
|
|
78
|
+
|
|
79
|
+
def shrinkage(self, n) -> np.ndarray:
|
|
80
|
+
"""Shrinkage factors ``lambda`` for clone sizes ``n``: ``(len(n), n_features)``."""
|
|
81
|
+
n = np.asarray(n, dtype=float)[:, None]
|
|
82
|
+
num = n * self.tau2[None, :]
|
|
83
|
+
den = num + self.sigma2[None, :]
|
|
84
|
+
return np.divide(num, den, out=np.zeros_like(num), where=den > 0)
|
|
85
|
+
|
|
86
|
+
def reliability(self, n) -> np.ndarray:
|
|
87
|
+
"""tau2-weighted mean shrinkage (Spearman-Brown reliability) per clone."""
|
|
88
|
+
lam = self.shrinkage(n)
|
|
89
|
+
total = self.tau2.sum()
|
|
90
|
+
w = self.tau2 / total if total > 0 else np.zeros_like(self.tau2)
|
|
91
|
+
return lam @ w
|
|
92
|
+
|
|
93
|
+
def min_cells_for(self, target: float = 0.5) -> int:
|
|
94
|
+
"""Smallest size reaching the actual feature-weighted reliability.
|
|
95
|
+
|
|
96
|
+
The pooled-ICC Spearman–Brown shortcut is not valid when features
|
|
97
|
+
have different noise-to-signal ratios. Return int32 max if no
|
|
98
|
+
finite supported size reaches the target.
|
|
99
|
+
"""
|
|
100
|
+
if not np.isfinite(target) or not 0 < target < 1:
|
|
101
|
+
raise ValueError("target must be finite and strictly between 0 and 1.")
|
|
102
|
+
limit = int(np.iinfo(np.int32).max)
|
|
103
|
+
if self.tau2.sum() <= 0:
|
|
104
|
+
return limit
|
|
105
|
+
hi = 1
|
|
106
|
+
while self.reliability([hi])[0] < target and hi < limit:
|
|
107
|
+
hi = min(2 * hi, limit)
|
|
108
|
+
if self.reliability([hi])[0] < target:
|
|
109
|
+
return limit
|
|
110
|
+
lo = 1
|
|
111
|
+
while lo < hi:
|
|
112
|
+
mid = (lo + hi) // 2
|
|
113
|
+
if self.reliability([mid])[0] >= target:
|
|
114
|
+
hi = mid
|
|
115
|
+
else:
|
|
116
|
+
lo = mid + 1
|
|
117
|
+
return lo
|
|
118
|
+
|
|
119
|
+
def as_dict(self) -> dict:
|
|
120
|
+
"""Plain dict (numpy arrays) suitable for ``adata.uns``."""
|
|
121
|
+
return {
|
|
122
|
+
"icc": self.icc,
|
|
123
|
+
"sigma2": np.asarray(self.sigma2),
|
|
124
|
+
"tau2": np.asarray(self.tau2),
|
|
125
|
+
"icc_per_feature": self.icc_per_feature,
|
|
126
|
+
"n0": float(self.n0),
|
|
127
|
+
"n_clones": int(self.n_clones),
|
|
128
|
+
"n_cells": int(self.n_cells),
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def variance_components(residuals: np.ndarray, clone_codes: np.ndarray) -> VarianceComponents:
|
|
133
|
+
"""ANOVA estimators of ``sigma2`` and ``tau2`` from clones with >= 2 cells.
|
|
134
|
+
|
|
135
|
+
Parameters
|
|
136
|
+
----------
|
|
137
|
+
residuals
|
|
138
|
+
``(n_cells, n_features)`` context-centred values.
|
|
139
|
+
clone_codes
|
|
140
|
+
Integer clone code per row; negative values are ignored. Only clones
|
|
141
|
+
with at least two rows contribute, because a singleton carries no
|
|
142
|
+
information about within-clone variance.
|
|
143
|
+
"""
|
|
144
|
+
cc = np.asarray(clone_codes)
|
|
145
|
+
ok = cc >= 0
|
|
146
|
+
sizes = np.bincount(cc[ok]) if ok.any() else np.array([], dtype=int)
|
|
147
|
+
multi = np.flatnonzero(sizes >= 2)
|
|
148
|
+
if multi.size < 2:
|
|
149
|
+
raise ValueError("Need at least two clones with >= 2 cells to estimate variance components.")
|
|
150
|
+
# re-index the multi-cell clones as 0..C-1; every other row is dropped
|
|
151
|
+
remap = -np.ones(sizes.size, dtype=np.int64)
|
|
152
|
+
remap[multi] = np.arange(multi.size)
|
|
153
|
+
g = np.where(ok, remap[np.where(ok, cc, 0)], -1)
|
|
154
|
+
keep = g >= 0
|
|
155
|
+
r, g = residuals[keep], g[keep]
|
|
156
|
+
n_clones = multi.size
|
|
157
|
+
n_c = np.bincount(g, minlength=n_clones).astype(float)
|
|
158
|
+
n_total = float(n_c.sum())
|
|
159
|
+
|
|
160
|
+
# per-clone sums via a sparse indicator matrix (much faster than np.add.at)
|
|
161
|
+
member = sp.csr_matrix((np.ones(g.size), (g, np.arange(g.size))), shape=(n_clones, g.size))
|
|
162
|
+
sums = np.asarray(member @ r)
|
|
163
|
+
means = sums / n_c[:, None]
|
|
164
|
+
grand = r.mean(axis=0)
|
|
165
|
+
# sums of squares via sum(x^2) - sum(n_c * mean_c^2): one pass, no loops
|
|
166
|
+
ss_raw = np.einsum("ij,ij->j", r, r)
|
|
167
|
+
ss_between_raw = np.einsum("c,cj->j", n_c, means**2)
|
|
168
|
+
ss_within = ss_raw - ss_between_raw
|
|
169
|
+
ss_between = ss_between_raw - n_total * grand**2
|
|
170
|
+
ms_within = np.maximum(ss_within, 0.0) / (n_total - n_clones)
|
|
171
|
+
ms_between = np.maximum(ss_between, 0.0) / (n_clones - 1)
|
|
172
|
+
# effective clone size of an unbalanced design
|
|
173
|
+
n0 = (n_total - float(np.sum(n_c**2)) / n_total) / (n_clones - 1)
|
|
174
|
+
tau2 = np.maximum((ms_between - ms_within) / n0, 0.0)
|
|
175
|
+
return VarianceComponents(
|
|
176
|
+
sigma2=ms_within, tau2=tau2, n0=float(n0), n_clones=int(n_clones),
|
|
177
|
+
n_cells=int(n_total), grand_mean=grand,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
# --------------------------------------------------------------------------- feature maps
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _center_by(x: np.ndarray, ctx: np.ndarray, finite: np.ndarray) -> np.ndarray:
|
|
185
|
+
"""Subtract each context's mean (cells with no context: the global mean)."""
|
|
186
|
+
n_ctx = int(ctx.max()) + 1 if (ctx >= 0).any() else 0
|
|
187
|
+
valid = (ctx >= 0) & finite
|
|
188
|
+
member = sp.csr_matrix((np.ones(valid.sum()), (ctx[valid], np.flatnonzero(valid))), shape=(n_ctx, x.shape[0]))
|
|
189
|
+
cnt = np.asarray(member.sum(axis=1)).ravel()
|
|
190
|
+
mu = np.asarray(member @ x) / np.maximum(cnt, 1)[:, None]
|
|
191
|
+
fallback = x[finite].mean(axis=0)
|
|
192
|
+
return x - np.where((ctx >= 0)[:, None], mu[np.maximum(ctx, 0)], fallback)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def neighbour_average(x: np.ndarray, k: int, random_state: int = 0) -> np.ndarray:
|
|
196
|
+
"""Average every cell with its ``k`` nearest cells (itself included).
|
|
197
|
+
|
|
198
|
+
Borrows strength from cells in the same local state, which denoises the
|
|
199
|
+
profiles of small clones. Used for programme profiles only: the
|
|
200
|
+
coherence and memory statistics are computed on unsmoothed cells.
|
|
201
|
+
"""
|
|
202
|
+
n = x.shape[0]
|
|
203
|
+
k = int(min(k, n))
|
|
204
|
+
if n > 50_000:
|
|
205
|
+
from pynndescent import NNDescent
|
|
206
|
+
|
|
207
|
+
idx = NNDescent(x, n_neighbors=k, random_state=random_state).neighbor_graph[0]
|
|
208
|
+
else:
|
|
209
|
+
from sklearn.neighbors import NearestNeighbors
|
|
210
|
+
|
|
211
|
+
idx = NearestNeighbors(n_neighbors=k).fit(x).kneighbors(x, return_distance=False)
|
|
212
|
+
out = np.zeros_like(x)
|
|
213
|
+
for j in range(k): # column-wise accumulation keeps memory at O(n * features)
|
|
214
|
+
out += x[idx[:, j]]
|
|
215
|
+
return out / k
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def median_bandwidth(x: np.ndarray, n_sample: int = 2000, rng=None) -> float:
|
|
219
|
+
"""Median distance between random pairs of cells (Gretton et al. 2012)."""
|
|
220
|
+
rng = rng or np.random.default_rng(0)
|
|
221
|
+
n = x.shape[0]
|
|
222
|
+
idx = rng.choice(n, size=min(n, n_sample), replace=False)
|
|
223
|
+
d = np.linalg.norm(x[idx] - x[rng.permutation(idx)], axis=1)
|
|
224
|
+
return float(np.median(d[d > 0])) if np.any(d > 0) else 1.0
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def local_bandwidth(x: np.ndarray, k: int = 30, n_sample: int = 2000, rng=None) -> float:
|
|
228
|
+
"""Kernel bandwidth from the typical distance to a cell's k-th neighbour.
|
|
229
|
+
|
|
230
|
+
A local scale (about the width of one cell state) resolves finer
|
|
231
|
+
differences between clones' state distributions than the median
|
|
232
|
+
heuristic, but every cell's features are noisier, so only larger clones
|
|
233
|
+
reach a given reliability. Useful for datasets with many large clones.
|
|
234
|
+
"""
|
|
235
|
+
from sklearn.neighbors import NearestNeighbors
|
|
236
|
+
|
|
237
|
+
rng = rng or np.random.default_rng(0)
|
|
238
|
+
n = x.shape[0]
|
|
239
|
+
idx = rng.choice(n, size=min(n, n_sample), replace=False)
|
|
240
|
+
k = int(min(k, n - 1))
|
|
241
|
+
dist = NearestNeighbors(n_neighbors=k + 1).fit(x).kneighbors(x[idx])[0][:, -1]
|
|
242
|
+
bw = float(np.median(dist))
|
|
243
|
+
return bw if bw > 0 else 1.0
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _random_fourier_features(
|
|
247
|
+
x: np.ndarray, n_features: int, bandwidth: float, rng: np.random.Generator
|
|
248
|
+
) -> tuple[np.ndarray, float]:
|
|
249
|
+
"""Gaussian-kernel random Fourier features with the given bandwidth."""
|
|
250
|
+
p = x.shape[1]
|
|
251
|
+
w = rng.normal(0.0, 1.0 / bandwidth, size=(p, n_features))
|
|
252
|
+
phase = rng.uniform(0.0, 2 * np.pi, size=n_features)
|
|
253
|
+
return np.sqrt(2.0 / n_features) * np.cos(x @ w + phase), bandwidth
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
# --------------------------------------------------------------------------- model object
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class ProfileModel:
|
|
260
|
+
"""Context-centred cell features plus the fitted variance components.
|
|
261
|
+
|
|
262
|
+
It holds cell-level matrices, so it is not stored in ``adata``; it is
|
|
263
|
+
rebuilt on demand from the parameters saved by :func:`clone_profiles`
|
|
264
|
+
(see :func:`model_from_adata`).
|
|
265
|
+
"""
|
|
266
|
+
|
|
267
|
+
def __init__(self, residuals, clone_codes, clone_index, context_codes, vc, params):
|
|
268
|
+
self.residuals = residuals
|
|
269
|
+
self.clone_codes = clone_codes
|
|
270
|
+
self.clone_index = clone_index
|
|
271
|
+
self.context_codes = context_codes
|
|
272
|
+
self.vc = vc
|
|
273
|
+
self.params = params
|
|
274
|
+
|
|
275
|
+
def group_blups(self, group_codes: np.ndarray, n_groups: int, weights: np.ndarray | None = None):
|
|
276
|
+
"""BLUPs for any grouping of cells (clones, time snapshots, bootstraps).
|
|
277
|
+
|
|
278
|
+
Parameters
|
|
279
|
+
----------
|
|
280
|
+
group_codes
|
|
281
|
+
Group code per cell (negative = not in any group).
|
|
282
|
+
n_groups
|
|
283
|
+
Number of groups.
|
|
284
|
+
weights
|
|
285
|
+
Optional per-cell weights (e.g. Poisson bootstrap counts).
|
|
286
|
+
|
|
287
|
+
Returns
|
|
288
|
+
-------
|
|
289
|
+
``(blup, n_eff, reliability)`` where ``n_eff`` is the weighted number
|
|
290
|
+
of cells per group.
|
|
291
|
+
"""
|
|
292
|
+
g = np.asarray(group_codes)
|
|
293
|
+
ok = np.flatnonzero(g >= 0)
|
|
294
|
+
w = np.ones(g.size) if weights is None else np.asarray(weights, dtype=float)
|
|
295
|
+
member = sp.csr_matrix((w[ok], (g[ok], ok)), shape=(n_groups, g.size))
|
|
296
|
+
sums = np.asarray(member @ self.residuals)
|
|
297
|
+
n_eff = np.asarray(member.sum(axis=1)).ravel()
|
|
298
|
+
with np.errstate(invalid="ignore", divide="ignore"):
|
|
299
|
+
means = sums / n_eff[:, None]
|
|
300
|
+
means = np.nan_to_num(means - self.vc.grand_mean[None, :])
|
|
301
|
+
blup = self.vc.shrinkage(n_eff) * means
|
|
302
|
+
return blup, n_eff, self.vc.reliability(n_eff)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _input_fingerprint(adata, basis, clone_key, context_key, donor_key=None):
|
|
306
|
+
"""Bind saved projections to their cells, groups and numeric coordinates."""
|
|
307
|
+
digest = hashlib.sha256(np.ascontiguousarray(get_basis(adata, basis)).tobytes())
|
|
308
|
+
columns = list(dict.fromkeys([clone_key] + [k for k in (context_key, donor_key) if k]))
|
|
309
|
+
digest.update(pd.util.hash_pandas_object(adata.obs[columns], index=True).to_numpy().tobytes())
|
|
310
|
+
return digest.hexdigest()
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def build_model(
|
|
314
|
+
adata,
|
|
315
|
+
*,
|
|
316
|
+
clone_key: str = "clone_id",
|
|
317
|
+
basis: str = "X_pca",
|
|
318
|
+
context_key: str | None = None,
|
|
319
|
+
representation: str = "mean",
|
|
320
|
+
n_features: int = 256,
|
|
321
|
+
bandwidth: float | str = "median",
|
|
322
|
+
smooth: int = 0,
|
|
323
|
+
random_state: int = 0,
|
|
324
|
+
feature_rng_state: str | None = None,
|
|
325
|
+
) -> ProfileModel:
|
|
326
|
+
"""Build the profile model: features, context centring, variance components."""
|
|
327
|
+
require_obs(adata, clone_key, context_key, context="clone profiles")
|
|
328
|
+
require_complete_groups(adata, context_key, context="clone profiles")
|
|
329
|
+
require_positive_int(n_features, "n_features")
|
|
330
|
+
if isinstance(smooth, (bool, np.bool_)) or not isinstance(smooth, (int, np.integer)) or smooth < 0:
|
|
331
|
+
raise ValueError("smooth must be a non-negative integer.")
|
|
332
|
+
x = get_basis(adata, basis)
|
|
333
|
+
finite = np.isfinite(x).all(axis=1)
|
|
334
|
+
if not finite.all():
|
|
335
|
+
x = np.where(finite[:, None], x, 0.0)
|
|
336
|
+
params = {
|
|
337
|
+
"clone_key": clone_key,
|
|
338
|
+
"basis": basis,
|
|
339
|
+
"context_key": context_key,
|
|
340
|
+
"representation": representation,
|
|
341
|
+
"n_features": int(n_features),
|
|
342
|
+
"input_sha256": _input_fingerprint(adata, basis, clone_key, context_key),
|
|
343
|
+
"smooth": int(smooth),
|
|
344
|
+
"random_state": int(random_state),
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
if representation not in ("mean", "kernel"):
|
|
348
|
+
raise ValueError("representation must be 'mean' or 'kernel'.")
|
|
349
|
+
if context_key is not None:
|
|
350
|
+
ctx, _ = codes(adata.obs[context_key])
|
|
351
|
+
else:
|
|
352
|
+
ctx = np.zeros(adata.n_obs, dtype=np.int64)
|
|
353
|
+
|
|
354
|
+
# 1) remove each context's mean, so a clone is compared with the cells it
|
|
355
|
+
# was sampled with (this also removes technical shifts between samples)
|
|
356
|
+
resid = _center_by(x, ctx, finite)
|
|
357
|
+
rng = np.random.default_rng(random_state)
|
|
358
|
+
if representation == "kernel":
|
|
359
|
+
# bandwidth from unsmoothed cells, so smoothed and unsmoothed models share one kernel
|
|
360
|
+
if bandwidth is None or bandwidth == "median":
|
|
361
|
+
bandwidth = median_bandwidth(resid[finite], rng=rng)
|
|
362
|
+
elif bandwidth == "local":
|
|
363
|
+
bandwidth = local_bandwidth(resid[finite], rng=rng)
|
|
364
|
+
if smooth and smooth > 1:
|
|
365
|
+
resid = np.where(finite[:, None], neighbour_average(np.where(finite[:, None], resid, 0.0), smooth,
|
|
366
|
+
random_state), 0.0)
|
|
367
|
+
|
|
368
|
+
# 2) "kernel": describe each cell by random Fourier features of its
|
|
369
|
+
# context-centred position, then centre again so that a clone's profile
|
|
370
|
+
# is the difference between its own cell distribution and its context's
|
|
371
|
+
if representation == "kernel":
|
|
372
|
+
if not np.isfinite(float(bandwidth)) or float(bandwidth) <= 0:
|
|
373
|
+
raise ValueError("bandwidth must be finite and positive.")
|
|
374
|
+
if feature_rng_state is not None:
|
|
375
|
+
rng.bit_generator.state = json.loads(feature_rng_state)
|
|
376
|
+
# Bandwidth estimation consumes RNG draws. Retain the exact subsequent
|
|
377
|
+
# state so bootstrap/snapshot features share the fitted RFF coordinates.
|
|
378
|
+
params["feature_rng_state"] = json.dumps(rng.bit_generator.state, sort_keys=True)
|
|
379
|
+
feats, bw = _random_fourier_features(resid, n_features, float(bandwidth), rng)
|
|
380
|
+
params["bandwidth"] = bw
|
|
381
|
+
resid = _center_by(feats, ctx, finite)
|
|
382
|
+
|
|
383
|
+
# 3) variance components from clones with >= 2 cells
|
|
384
|
+
clone_codes, clone_index = codes(adata.obs[clone_key])
|
|
385
|
+
clone_codes = np.where(finite, clone_codes, -1)
|
|
386
|
+
vc = variance_components(resid, clone_codes)
|
|
387
|
+
return ProfileModel(resid, clone_codes, clone_index, ctx, vc, params)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
# --------------------------------------------------------------------------- public API
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def clone_profiles(
|
|
394
|
+
adata,
|
|
395
|
+
*,
|
|
396
|
+
clone_key: str = "clone_id",
|
|
397
|
+
basis: str = "X_pca",
|
|
398
|
+
context_key: str | None = None,
|
|
399
|
+
donor_key: str | None = None,
|
|
400
|
+
representation: str = "mean",
|
|
401
|
+
n_features: int = 256,
|
|
402
|
+
n_components: int = 30,
|
|
403
|
+
bandwidth: float | str = "median",
|
|
404
|
+
smooth: int | None = None,
|
|
405
|
+
min_cells: int = 2,
|
|
406
|
+
random_state: int = 0,
|
|
407
|
+
verbose: bool = True,
|
|
408
|
+
) -> pd.DataFrame:
|
|
409
|
+
"""Estimate a shrunken, context-adjusted state profile for every clone.
|
|
410
|
+
|
|
411
|
+
Parameters
|
|
412
|
+
----------
|
|
413
|
+
adata
|
|
414
|
+
AnnData with ``obs[clone_key]`` and an expression embedding.
|
|
415
|
+
clone_key
|
|
416
|
+
``obs`` column with clone ids (missing = no BCR). Clone ids must be
|
|
417
|
+
unique across donors (see :func:`threadfin.define_clones`).
|
|
418
|
+
basis
|
|
419
|
+
``obsm`` key of the embedding. Use a batch-integrated embedding built
|
|
420
|
+
without immunoglobulin genes (:func:`threadfin.pp.prepare_embedding`).
|
|
421
|
+
context_key
|
|
422
|
+
``obs`` column of sampling contexts whose mean is removed before
|
|
423
|
+
profiling (e.g. sample, or donor x timepoint). Choose *nuisance*
|
|
424
|
+
strata only: biology you want to compare between clones (tissue,
|
|
425
|
+
sort gate, infection status) must not be a context.
|
|
426
|
+
donor_key
|
|
427
|
+
Optional donor column, recorded per clone for stratified tests.
|
|
428
|
+
representation
|
|
429
|
+
``"mean"`` (shrunken centroid) or ``"kernel"`` (shrunken kernel mean
|
|
430
|
+
embedding from ``n_features`` random Fourier features, reduced to
|
|
431
|
+
``n_components`` principal components). :func:`threadfin.run` uses
|
|
432
|
+
``"kernel"``; this function keeps ``"mean"`` as its default so that
|
|
433
|
+
code written for earlier versions gives the same result.
|
|
434
|
+
bandwidth
|
|
435
|
+
Kernel bandwidth: ``"median"`` (median distance between cells, the
|
|
436
|
+
standard heuristic), ``"local"`` (distance to the 30th nearest
|
|
437
|
+
neighbour; sharper, but needs larger clones) or a number.
|
|
438
|
+
smooth
|
|
439
|
+
Average each cell with its ``smooth`` nearest cells before profiling
|
|
440
|
+
(denoises small clones by borrowing strength from their local state).
|
|
441
|
+
Default: 15 for ``"kernel"``, 0 for ``"mean"``. The coherence test and
|
|
442
|
+
clonal memory always use unsmoothed cells.
|
|
443
|
+
min_cells
|
|
444
|
+
Clones with fewer cells are not profiled. The default of 2 is **not** a quality filter --
|
|
445
|
+
Threadfin never filters on reliability -- it is what the clone map is *of*. Singleton clones
|
|
446
|
+
are 75 to 98 % of clones in these deposits, so including them turns the clone embedding back
|
|
447
|
+
into the cell UMAP it is supposed to summarise. Set 1 to include them; they are then drawn as
|
|
448
|
+
triangles rather than circles so a reader can see how much of a map rests on a single cell.
|
|
449
|
+
Threadfin does not filter clones. Every clone, and every cell in it, is measured data, and
|
|
450
|
+
the profiles are already **shrunk**: a clone seen in few cells has its profile pulled towards the
|
|
451
|
+
population mean by the variance-component model, so its influence is down-weighted in proportion to
|
|
452
|
+
how little was seen. Cutting on reliability *on top of* that shrinkage discards the same information
|
|
453
|
+
twice, and it discards it silently — on GSE287123 a 0.5 floor removes 75 % (RBD) and 90 % (NP-OVA) of
|
|
454
|
+
profiled clones, and on GSE246382 it removes all of them, which is how that dataset came to look as
|
|
455
|
+
though it had no clone structure when its map separates the sorted gate at p < 0.001.
|
|
456
|
+
|
|
457
|
+
Returns
|
|
458
|
+
-------
|
|
459
|
+
Clone table (one row per profiled clone) with ``n_cells``,
|
|
460
|
+
``reliability`` and, when available, ``donor`` and ``n_contexts``.
|
|
461
|
+
Profiles and model parameters go to ``adata.uns['threadfin']['profiles']``.
|
|
462
|
+
"""
|
|
463
|
+
ensure_actual(adata, verbose)
|
|
464
|
+
require_obs(adata, clone_key, context="clone profiling")
|
|
465
|
+
require_complete_groups(adata, context_key, donor_key, context="clone profiling")
|
|
466
|
+
if representation not in {"mean", "kernel"}:
|
|
467
|
+
raise ValueError("representation must be 'mean' or 'kernel'.")
|
|
468
|
+
require_positive_int(min_cells, "min_cells")
|
|
469
|
+
require_positive_int(n_components, "n_components")
|
|
470
|
+
labels = pd.Index(pd.unique(adata.obs[clone_key].dropna()))
|
|
471
|
+
if not labels.astype(str).is_unique:
|
|
472
|
+
raise ValueError("clone ids must remain unique when converted to strings.")
|
|
473
|
+
if donor_key is not None:
|
|
474
|
+
groups = adata.obs.dropna(subset=[clone_key]).groupby(clone_key, observed=True)[donor_key].nunique()
|
|
475
|
+
if groups.gt(1).any():
|
|
476
|
+
shared = groups.index[groups.gt(1)]
|
|
477
|
+
raise ValueError(
|
|
478
|
+
f"{len(shared):,} clone id(s) occur in more than one donor (e.g. {shared[0]!r}). A clone belongs to one "
|
|
479
|
+
f"individual: call clones within donors, or make the ids donor-private, e.g. "
|
|
480
|
+
f"adata.obs['{clone_key}'] = adata.obs['{donor_key}'].astype(str) + '|' + adata.obs['{clone_key}'].astype(str)")
|
|
481
|
+
if smooth is None:
|
|
482
|
+
smooth = 15 if representation == "kernel" else 0
|
|
483
|
+
model = build_model(
|
|
484
|
+
adata, clone_key=clone_key, basis=basis, context_key=context_key,
|
|
485
|
+
representation=representation, n_features=n_features, bandwidth=bandwidth,
|
|
486
|
+
smooth=smooth, random_state=random_state,
|
|
487
|
+
)
|
|
488
|
+
blup, n_eff, rel = model.group_blups(model.clone_codes, len(model.clone_index))
|
|
489
|
+
keep = n_eff >= min_cells
|
|
490
|
+
if keep.sum() < 3:
|
|
491
|
+
raise ValueError(
|
|
492
|
+
f"Only {int(keep.sum())} clones have >= {min_cells} cells; Threadfin needs at least 3 "
|
|
493
|
+
"expanded clones to estimate clone profiles. Include more BCR-matched cells, lower "
|
|
494
|
+
"min_cells when scientifically appropriate, or use a larger dataset."
|
|
495
|
+
)
|
|
496
|
+
ids = pd.Index(model.clone_index[keep].astype(str), name=clone_key)
|
|
497
|
+
feats = blup[keep]
|
|
498
|
+
names = [f"f{j}" for j in range(feats.shape[1])]
|
|
499
|
+
reduction = None
|
|
500
|
+
if representation == "kernel":
|
|
501
|
+
# PCA of the clone x feature matrix keeps the downstream graph compact
|
|
502
|
+
raw_mean = feats.mean(axis=0)
|
|
503
|
+
u, s, vt = np.linalg.svd(feats - raw_mean, full_matrices=False)
|
|
504
|
+
k = int(min(n_components, int((s > 1e-12).sum())))
|
|
505
|
+
feats = u[:, :k] * s[:k]
|
|
506
|
+
names = [f"kpc{j}" for j in range(k)]
|
|
507
|
+
reduction = {"components": vt[:k], "raw_mean": raw_mean}
|
|
508
|
+
|
|
509
|
+
table = pd.DataFrame({"n_cells": n_eff[keep].astype(int), "reliability": rel[keep]}, index=ids)
|
|
510
|
+
obs = adata.obs
|
|
511
|
+
if donor_key is not None:
|
|
512
|
+
require_obs(adata, donor_key)
|
|
513
|
+
sub = obs[[clone_key, donor_key]].dropna()
|
|
514
|
+
donor = sub.groupby(sub[clone_key].astype(str), observed=True)[donor_key].agg(modal)
|
|
515
|
+
table["donor"] = donor.reindex(table.index).astype(str).values
|
|
516
|
+
if context_key is not None:
|
|
517
|
+
sub = obs[[clone_key, context_key]].dropna()
|
|
518
|
+
nctx = sub.groupby(sub[clone_key].astype(str), observed=True)[context_key].nunique()
|
|
519
|
+
table["n_contexts"] = nctx.reindex(table.index).fillna(0).astype(int).values
|
|
520
|
+
|
|
521
|
+
vc = model.vc
|
|
522
|
+
get_uns(adata)[PROFILE_KEY] = {
|
|
523
|
+
"features": pd.DataFrame(feats, index=table.index, columns=names),
|
|
524
|
+
"clone_table": table,
|
|
525
|
+
"variance_components": vc.as_dict(),
|
|
526
|
+
"params": {**model.params, "donor_key": donor_key, "min_cells": int(min_cells),
|
|
527
|
+
"n_components": int(n_components),
|
|
528
|
+
"input_sha256": _input_fingerprint(adata, basis, clone_key, context_key, donor_key)},
|
|
529
|
+
"reduction": reduction,
|
|
530
|
+
}
|
|
531
|
+
log(
|
|
532
|
+
# the clonal ICC itself is reported by clonal_coherence, on unsmoothed cells; the variance components
|
|
533
|
+
# of the profile features (smoothed for kernel profiles) only set the shrinkage, so they are not quoted
|
|
534
|
+
f"clone profiles: {table.shape[0]} clones (>= {min_cells} cells, {vc.n_cells} cells in all). Profiles are "
|
|
535
|
+
f"shrunk, so a clone reaches reliability 0.5 at {vc.min_cells_for(0.5)} cells; nothing is filtered on that.",
|
|
536
|
+
verbose,
|
|
537
|
+
)
|
|
538
|
+
return table
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def get_profiles(adata) -> dict:
|
|
542
|
+
"""Stored profile results (raises if :func:`clone_profiles` was not run)."""
|
|
543
|
+
prof = get_uns(adata).get(PROFILE_KEY)
|
|
544
|
+
if prof is None:
|
|
545
|
+
raise ValueError("Run threadfin.tl.clone_profiles() (or threadfin.run()) first.")
|
|
546
|
+
return prof
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def _feature_rng_state(adata) -> str:
|
|
550
|
+
"""Recover old serialized models by matching their stored features.
|
|
551
|
+
|
|
552
|
+
Version 4.0 did not store the RNG state after bandwidth estimation.
|
|
553
|
+
Its numeric/median/local paths can be tested against the saved projection.
|
|
554
|
+
Refuse a mismatch instead of silently projecting a different feature map.
|
|
555
|
+
"""
|
|
556
|
+
prof = get_profiles(adata)
|
|
557
|
+
p = prof["params"]
|
|
558
|
+
if p.get("feature_rng_state") is not None:
|
|
559
|
+
return p["feature_rng_state"]
|
|
560
|
+
for bandwidth in (p.get("bandwidth", "median"), "median", "local"):
|
|
561
|
+
model = build_model(
|
|
562
|
+
adata, clone_key=p["clone_key"], basis=p["basis"], context_key=p.get("context_key"),
|
|
563
|
+
representation="kernel", n_features=p["n_features"], bandwidth=bandwidth,
|
|
564
|
+
smooth=p.get("smooth", 0), random_state=p["random_state"],
|
|
565
|
+
)
|
|
566
|
+
if not np.isclose(model.params["bandwidth"], p.get("bandwidth", model.params["bandwidth"])):
|
|
567
|
+
continue
|
|
568
|
+
blup, _, _ = model.group_blups(model.clone_codes, len(model.clone_index))
|
|
569
|
+
index = pd.Index(model.clone_index.astype(str)).get_indexer(prof["features"].index)
|
|
570
|
+
if np.any(index < 0):
|
|
571
|
+
break
|
|
572
|
+
projected = project_features(adata, blup[index])
|
|
573
|
+
if np.allclose(projected, prof["features"].to_numpy(), rtol=1e-7, atol=1e-9):
|
|
574
|
+
p["feature_rng_state"] = model.params["feature_rng_state"]
|
|
575
|
+
return p["feature_rng_state"]
|
|
576
|
+
raise ValueError("Stored kernel features cannot be reproduced from the current cells and settings. "
|
|
577
|
+
"Rerun clone_profiles before bootstrap or snapshot analysis.")
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def model_from_adata(adata, *, unsmoothed: bool = False, representation: str | None = None) -> ProfileModel:
|
|
581
|
+
"""Rebuild the fitted feature map; optionally omit smoothing for statistics."""
|
|
582
|
+
p = get_profiles(adata)["params"]
|
|
583
|
+
if p.get("input_sha256") is not None:
|
|
584
|
+
current = _input_fingerprint(adata, p["basis"], p["clone_key"], p.get("context_key"), p.get("donor_key"))
|
|
585
|
+
if current != p["input_sha256"]:
|
|
586
|
+
raise ValueError("Cells, clone/context labels or embedding changed since clone_profiles; "
|
|
587
|
+
"refit profiles before downstream analysis.")
|
|
588
|
+
rep = representation or p["representation"]
|
|
589
|
+
state = _feature_rng_state(adata) if rep == "kernel" and p["representation"] == "kernel" else None
|
|
590
|
+
return build_model(
|
|
591
|
+
adata, clone_key=p["clone_key"], basis=p["basis"], context_key=p.get("context_key"),
|
|
592
|
+
representation=rep, n_features=p["n_features"],
|
|
593
|
+
bandwidth=p.get("bandwidth", "median"), smooth=0 if unsmoothed else p.get("smooth", 0),
|
|
594
|
+
random_state=p["random_state"], feature_rng_state=state,
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
def project_features(adata, blup: np.ndarray) -> np.ndarray:
|
|
599
|
+
"""Map raw BLUPs (bootstrap replicates, time snapshots) into profile space."""
|
|
600
|
+
red = get_profiles(adata).get("reduction")
|
|
601
|
+
if red is None:
|
|
602
|
+
return blup
|
|
603
|
+
return (blup - np.asarray(red["raw_mean"])) @ np.asarray(red["components"]).T
|