threadfin 4.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- threadfin/__init__.py +86 -0
- threadfin/_utils.py +260 -0
- threadfin/api.py +658 -0
- threadfin/bcrgraph.py +255 -0
- threadfin/clones.py +653 -0
- threadfin/core.py +531 -0
- threadfin/dandelion.py +458 -0
- threadfin/density.py +409 -0
- threadfin/dynamics.py +275 -0
- threadfin/export.py +215 -0
- threadfin/expression.py +121 -0
- threadfin/heritability.py +224 -0
- threadfin/integrate.py +350 -0
- threadfin/io.py +280 -0
- threadfin/maptune.py +743 -0
- threadfin/metrics.py +137 -0
- threadfin/migration.py +165 -0
- threadfin/phylo.py +834 -0
- threadfin/plotting.py +535 -0
- threadfin/pp.py +253 -0
- threadfin/profiles.py +603 -0
- threadfin/programmes.py +1028 -0
- threadfin/programs.py +133 -0
- threadfin/reclustering.py +68 -0
- threadfin/sequence.py +379 -0
- threadfin/simulate.py +206 -0
- threadfin/specificity.py +355 -0
- threadfin/stats.py +476 -0
- threadfin/tl.py +62 -0
- threadfin-4.5.0.dist-info/METADATA +288 -0
- threadfin-4.5.0.dist-info/RECORD +34 -0
- threadfin-4.5.0.dist-info/WHEEL +5 -0
- threadfin-4.5.0.dist-info/licenses/LICENSE +21 -0
- threadfin-4.5.0.dist-info/top_level.txt +1 -0
threadfin/dynamics.py
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""Clonal memory across time points or tissues.
|
|
2
|
+
|
|
3
|
+
Question: when the same clone is sampled twice (two time points, or two
|
|
4
|
+
tissues), is it still in the same state?
|
|
5
|
+
|
|
6
|
+
Why not compare programme labels? Programmes are assigned per clone from
|
|
7
|
+
*all* of its cells, so every snapshot of a clone carries the same label by
|
|
8
|
+
construction and a label "transition matrix" is trivially diagonal.
|
|
9
|
+
:func:`clonal_memory` instead profiles every (clone, time) *snapshot*
|
|
10
|
+
separately and asks how similar a clone's consecutive snapshots are, compared
|
|
11
|
+
with snapshots of unrelated clones.
|
|
12
|
+
|
|
13
|
+
The memory index
|
|
14
|
+
----------------
|
|
15
|
+
For consecutive snapshots ``a`` and ``b`` of the same clone, with ``n_a`` and
|
|
16
|
+
``n_b`` cells and context-centred mean profiles ``m_a`` and ``m_b``::
|
|
17
|
+
|
|
18
|
+
D_same = ||m_a - m_b||^2 - sum_j sigma2_j * (1/n_a + 1/n_b)
|
|
19
|
+
|
|
20
|
+
where ``sigma2`` is the within-snapshot variance, so ``D_same`` estimates how
|
|
21
|
+
much the clone's *true* state changed (sampling noise removed). ``D_random``
|
|
22
|
+
is the same quantity when ``b`` is replaced by a snapshot of a different clone
|
|
23
|
+
from the same donor and time point. The memory index is::
|
|
24
|
+
|
|
25
|
+
M = 1 - mean(D_same) / mean(D_random)
|
|
26
|
+
|
|
27
|
+
``M = 1``: clones keep everything that distinguished them; ``M = 0``: a clone
|
|
28
|
+
is no more similar to its former self than to a random clone. ``M`` behaves
|
|
29
|
+
like a test-retest reliability of clone state. The p-value compares
|
|
30
|
+
``mean(D_same)`` with random re-pairings; the 95% interval bootstraps clones.
|
|
31
|
+
|
|
32
|
+
As a descriptive companion, each snapshot is also assigned to the nearest
|
|
33
|
+
programme centroid and the programme transition table is reported together
|
|
34
|
+
with its expectation under random pairing.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import warnings
|
|
40
|
+
|
|
41
|
+
import numpy as np
|
|
42
|
+
import pandas as pd
|
|
43
|
+
|
|
44
|
+
from ._utils import codes, get_uns, h5ad_safe, log, require_obs, require_positive_int
|
|
45
|
+
from .profiles import get_profiles, model_from_adata, variance_components
|
|
46
|
+
|
|
47
|
+
MEMORY_KEY = "memory"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _ordered_levels(values, order):
|
|
51
|
+
levels = list(pd.unique(pd.Series(values).dropna().astype(str)))
|
|
52
|
+
if order is not None:
|
|
53
|
+
order = [str(o) for o in order]
|
|
54
|
+
missing = [lv for lv in levels if lv not in order]
|
|
55
|
+
if missing:
|
|
56
|
+
raise ValueError(f"order is missing levels: {missing}")
|
|
57
|
+
return [o for o in order if o in levels]
|
|
58
|
+
num = pd.to_numeric(pd.Series(levels).str.extract(r"(-?\d+\.?\d*)")[0], errors="coerce")
|
|
59
|
+
if num.notna().all():
|
|
60
|
+
return [levels[i] for i in np.argsort(num.to_numpy(), kind="stable")]
|
|
61
|
+
return sorted(levels)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _snapshots(adata, model, clone_key, time_key, order, min_cells):
|
|
65
|
+
"""Unshrunk mean profile, size and donor of every (clone, time) snapshot."""
|
|
66
|
+
obs = adata.obs
|
|
67
|
+
clone_str = obs[clone_key].astype("object").map(lambda c: str(c) if pd.notna(c) else None)
|
|
68
|
+
levels = _ordered_levels(obs.loc[clone_str.notna(), time_key], order)
|
|
69
|
+
t_index = obs[time_key].astype(str).map({lv: i for i, lv in enumerate(levels)})
|
|
70
|
+
valid = (clone_str.notna() & t_index.notna()).to_numpy()
|
|
71
|
+
key = clone_str[valid] + "\x1f" + t_index[valid].astype(int).astype(str)
|
|
72
|
+
snap_valid, snap_index = codes(key)
|
|
73
|
+
snap = -np.ones(adata.n_obs, dtype=np.int64)
|
|
74
|
+
snap[np.flatnonzero(valid)] = snap_valid
|
|
75
|
+
n_snap = len(snap_index)
|
|
76
|
+
|
|
77
|
+
import scipy.sparse as sp
|
|
78
|
+
|
|
79
|
+
ok = np.flatnonzero(snap >= 0)
|
|
80
|
+
member = sp.csr_matrix((np.ones(ok.size), (snap[ok], ok)), shape=(n_snap, snap.size))
|
|
81
|
+
n = np.asarray(member.sum(axis=1)).ravel()
|
|
82
|
+
means = np.asarray(member @ model.residuals) / np.maximum(n, 1)[:, None]
|
|
83
|
+
# within-snapshot noise variance (pure sampling noise, no temporal change)
|
|
84
|
+
sigma2 = variance_components(model.residuals, snap).sigma2
|
|
85
|
+
info = pd.DataFrame({
|
|
86
|
+
"clone": [s.split("\x1f")[0] for s in snap_index],
|
|
87
|
+
"t": [int(s.split("\x1f")[1]) for s in snap_index],
|
|
88
|
+
"n_cells": n.astype(int),
|
|
89
|
+
})
|
|
90
|
+
keep = (info["n_cells"] >= min_cells).to_numpy()
|
|
91
|
+
return info[keep].reset_index(drop=True), means[keep], sigma2, levels
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def clonal_memory(
|
|
95
|
+
adata,
|
|
96
|
+
time_key: str,
|
|
97
|
+
*,
|
|
98
|
+
order=None,
|
|
99
|
+
programme_key: str = "clone_programme",
|
|
100
|
+
min_cells: int = 1,
|
|
101
|
+
min_pairs: int = 10,
|
|
102
|
+
n_null: int = 500,
|
|
103
|
+
n_boot: int = 500,
|
|
104
|
+
random_state: int = 0,
|
|
105
|
+
verbose: bool = True,
|
|
106
|
+
) -> dict:
|
|
107
|
+
"""Do clones keep their state between time points (or tissues)?
|
|
108
|
+
|
|
109
|
+
Parameters
|
|
110
|
+
----------
|
|
111
|
+
time_key
|
|
112
|
+
``obs`` column with the time point (or tissue) of each cell.
|
|
113
|
+
order
|
|
114
|
+
Order of the levels of ``time_key``; inferred from numbers in the
|
|
115
|
+
labels (``d7`` < ``d28``) when omitted. Each clone's consecutive
|
|
116
|
+
observed levels are paired.
|
|
117
|
+
programme_key
|
|
118
|
+
Programme column of the clone table (for the descriptive transition
|
|
119
|
+
table); skipped when absent.
|
|
120
|
+
min_cells
|
|
121
|
+
Minimum cells per snapshot.
|
|
122
|
+
min_pairs
|
|
123
|
+
Fewest snapshot pairs for which an index is reported. Below this the
|
|
124
|
+
bootstrap interval collapses and the estimate says nothing, so the
|
|
125
|
+
function raises instead of returning a number.
|
|
126
|
+
n_null, n_boot
|
|
127
|
+
Random re-pairings for the p-value; bootstrap replicates for the CI.
|
|
128
|
+
|
|
129
|
+
Returns
|
|
130
|
+
-------
|
|
131
|
+
dict with ``memory_index`` (see module docstring), its bootstrap
|
|
132
|
+
``memory_index_ci``, ``p_value``, ``n_clones``, ``n_pairs``, the
|
|
133
|
+
per-pair table ``pairs`` and, when programmes exist, programme
|
|
134
|
+
``persistence`` with ``persistence_null`` and the ``transitions`` /
|
|
135
|
+
``transitions_expected`` tables. Stored in
|
|
136
|
+
``adata.uns['threadfin']['memory'][time_key]``.
|
|
137
|
+
"""
|
|
138
|
+
for name, value in [("min_cells", min_cells), ("min_pairs", min_pairs), ("n_null", n_null), ("n_boot", n_boot)]:
|
|
139
|
+
require_positive_int(value, name)
|
|
140
|
+
prof = get_profiles(adata)
|
|
141
|
+
clone_key = prof["params"]["clone_key"]
|
|
142
|
+
require_obs(adata, time_key)
|
|
143
|
+
rng = np.random.default_rng(random_state)
|
|
144
|
+
model = model_from_adata(adata, unsmoothed=True) # smoothing would leak between snapshots
|
|
145
|
+
snaps, means, sigma2, levels = _snapshots(adata, model, clone_key, time_key, order, min_cells)
|
|
146
|
+
table = prof["clone_table"]
|
|
147
|
+
donor = table["donor"].astype(str) if "donor" in table.columns else pd.Series(dtype=str)
|
|
148
|
+
snaps["donor"] = snaps["clone"].map(donor).fillna("all").to_numpy()
|
|
149
|
+
|
|
150
|
+
# consecutive snapshot pairs of the same clone
|
|
151
|
+
pa, pb = [], []
|
|
152
|
+
for _, grp in snaps.groupby("clone", sort=False):
|
|
153
|
+
if len(grp) >= 2:
|
|
154
|
+
rows = grp.sort_values("t").index.to_numpy()
|
|
155
|
+
pa.extend(rows[:-1])
|
|
156
|
+
pb.extend(rows[1:])
|
|
157
|
+
pa, pb = np.asarray(pa, dtype=np.int64), np.asarray(pb, dtype=np.int64)
|
|
158
|
+
if pa.size == 0:
|
|
159
|
+
raise ValueError(f"No clone has >= {min_cells} cells at two levels of '{time_key}'.")
|
|
160
|
+
if pa.size < min_pairs:
|
|
161
|
+
raise ValueError(f"Only {pa.size} clones have >= {min_cells} cells at two levels of '{time_key}' "
|
|
162
|
+
f"(need >= {min_pairs}); no memory index is reported.")
|
|
163
|
+
|
|
164
|
+
n_cells = snaps["n_cells"].to_numpy(dtype=float)
|
|
165
|
+
noise = sigma2.sum()
|
|
166
|
+
|
|
167
|
+
def corrected_sqdist(a, b):
|
|
168
|
+
diff = means[a] - means[b]
|
|
169
|
+
return np.einsum("ij,ij->i", diff, diff) - noise * (1.0 / n_cells[a] + 1.0 / n_cells[b])
|
|
170
|
+
|
|
171
|
+
# candidate substitutes for every pair: other clones, same donor and time point
|
|
172
|
+
t_arr, d_arr, c_arr = snaps["t"].to_numpy(), snaps["donor"].to_numpy(), snaps["clone"].to_numpy()
|
|
173
|
+
pools: dict = {}
|
|
174
|
+
for i in range(len(snaps)):
|
|
175
|
+
pools.setdefault((d_arr[i], t_arr[i]), []).append(i)
|
|
176
|
+
cands = []
|
|
177
|
+
for a, b in zip(pa, pb):
|
|
178
|
+
pool = np.asarray(pools[(d_arr[b], t_arr[b])])
|
|
179
|
+
pool = pool[c_arr[pool] != c_arr[a]]
|
|
180
|
+
cands.append(pool)
|
|
181
|
+
usable = np.array([c.size > 0 for c in cands])
|
|
182
|
+
n_unmatched = int((~usable).sum())
|
|
183
|
+
pa, pb = pa[usable], pb[usable]
|
|
184
|
+
if pa.size < min_pairs:
|
|
185
|
+
raise ValueError(f"Only {pa.size} snapshot pairs have another clone from the same donor "
|
|
186
|
+
f"and target time (need >= {min_pairs}); no memory index is reported.")
|
|
187
|
+
cands = [c for c, u in zip(cands, usable) if u]
|
|
188
|
+
|
|
189
|
+
d_same = corrected_sqdist(pa, pb)
|
|
190
|
+
|
|
191
|
+
def draw():
|
|
192
|
+
return np.array([c[rng.integers(c.size)] for c in cands])
|
|
193
|
+
|
|
194
|
+
null_stats = np.empty(n_null)
|
|
195
|
+
d_random_all = []
|
|
196
|
+
for r in range(n_null):
|
|
197
|
+
dr = corrected_sqdist(pa, draw())
|
|
198
|
+
null_stats[r] = dr.mean()
|
|
199
|
+
if r < 50:
|
|
200
|
+
d_random_all.append(dr)
|
|
201
|
+
d_random = np.mean(d_random_all, axis=0) # per-pair expected distance to a random clone
|
|
202
|
+
if not np.isfinite(d_random).all() or d_random.mean() <= 0:
|
|
203
|
+
raise ValueError("Noise-corrected matched-clone distance is non-positive; "
|
|
204
|
+
"the memory ratio is not identifiable for these samples.")
|
|
205
|
+
mem = 1.0 - d_same.mean() / d_random.mean()
|
|
206
|
+
p_value = (1 + int((null_stats <= d_same.mean()).sum())) / (n_null + 1)
|
|
207
|
+
|
|
208
|
+
# bootstrap over clones
|
|
209
|
+
clones_of_pairs = c_arr[pa]
|
|
210
|
+
uniq, inv = np.unique(clones_of_pairs, return_inverse=True)
|
|
211
|
+
boots = np.empty(n_boot)
|
|
212
|
+
for r in range(n_boot):
|
|
213
|
+
w = np.bincount(rng.integers(0, uniq.size, size=uniq.size), minlength=uniq.size)[inv]
|
|
214
|
+
denom = np.sum(w * d_random)
|
|
215
|
+
boots[r] = 1.0 - np.sum(w * d_same) / denom if denom > 0 else np.nan
|
|
216
|
+
identifiable = np.isfinite(boots)
|
|
217
|
+
# Dropping undefined ratios would produce a falsely reassuring conditional CI.
|
|
218
|
+
ci = (float(np.quantile(boots, 0.025)), float(np.quantile(boots, 0.975))) if identifiable.all() else (np.nan, np.nan)
|
|
219
|
+
if not identifiable.all():
|
|
220
|
+
warnings.warn("The confidence interval of the memory index could not be computed: in some bootstrap "
|
|
221
|
+
"resamples of clones no between-clone difference was left to compare with (too few clones "
|
|
222
|
+
"seen at two levels). The memory index and its P value are unaffected; the interval is "
|
|
223
|
+
"reported as missing.", stacklevel=2)
|
|
224
|
+
|
|
225
|
+
pairs = pd.DataFrame({
|
|
226
|
+
"clone": clones_of_pairs, "earlier": [levels[t] for t in t_arr[pa]],
|
|
227
|
+
"later": [levels[t] for t in t_arr[pb]], "n_cells_earlier": n_cells[pa].astype(int),
|
|
228
|
+
"n_cells_later": n_cells[pb].astype(int), "change": d_same, "change_vs_random": d_random,
|
|
229
|
+
})
|
|
230
|
+
res = {
|
|
231
|
+
"time_key": time_key, "levels": levels, "n_clones": int(uniq.size), "n_pairs": int(pa.size),
|
|
232
|
+
"n_unmatched_pairs": n_unmatched, "null_matching": "same donor and target time; no cross-donor fallback",
|
|
233
|
+
"memory_index": float(mem), "memory_index_ci": ci, "p_value": float(p_value),
|
|
234
|
+
"bootstrap_identifiable_fraction": float(identifiable.mean()),
|
|
235
|
+
"mean_change": float(d_same.mean()), "mean_change_random": float(d_random.mean()),
|
|
236
|
+
"pairs": pairs,
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
# descriptive: programme of each snapshot by nearest programme centroid (needs >= 2 programmes)
|
|
240
|
+
if programme_key in table.columns and table[programme_key].dropna().nunique() >= 2:
|
|
241
|
+
lab = table[programme_key].dropna().astype(str)
|
|
242
|
+
clone_means = pd.DataFrame(index=snaps.index)
|
|
243
|
+
centroids, names = [], sorted(lab.unique(), key=lambda s: (len(s), s))
|
|
244
|
+
member_snaps = snaps["clone"].map(lab)
|
|
245
|
+
for name in names:
|
|
246
|
+
rows = np.flatnonzero((member_snaps == name).to_numpy())
|
|
247
|
+
centroids.append(means[rows].mean(axis=0) if rows.size else np.full(means.shape[1], np.nan))
|
|
248
|
+
centroids = np.vstack(centroids)
|
|
249
|
+
d2 = ((means[:, None, :] - centroids[None, :, :]) ** 2).sum(-1)
|
|
250
|
+
nearest = np.nanargmin(np.where(np.isfinite(d2), d2, np.inf), axis=1)
|
|
251
|
+
same = nearest[pa] == nearest[pb]
|
|
252
|
+
null_same = np.mean([np.mean(nearest[pa] == nearest[draw()]) for _ in range(100)])
|
|
253
|
+
trans = pd.crosstab(pd.Categorical(np.asarray(names)[nearest[pa]], categories=names),
|
|
254
|
+
pd.Categorical(np.asarray(names)[nearest[pb]], categories=names), dropna=False)
|
|
255
|
+
trans.index.name, trans.columns.name = "earlier", "later"
|
|
256
|
+
exp = np.zeros_like(trans.to_numpy(), dtype=float)
|
|
257
|
+
for _ in range(100):
|
|
258
|
+
exp += pd.crosstab(pd.Categorical(np.asarray(names)[nearest[pa]], categories=names),
|
|
259
|
+
pd.Categorical(np.asarray(names)[nearest[draw()]], categories=names),
|
|
260
|
+
dropna=False).to_numpy()
|
|
261
|
+
res.update({
|
|
262
|
+
"persistence": float(same.mean()), "persistence_null": float(null_same),
|
|
263
|
+
"transitions": trans,
|
|
264
|
+
"transitions_expected": pd.DataFrame(exp / 100, index=trans.index, columns=trans.columns),
|
|
265
|
+
})
|
|
266
|
+
pairs["programme_earlier"] = np.asarray(names)[nearest[pa]]
|
|
267
|
+
pairs["programme_later"] = np.asarray(names)[nearest[pb]]
|
|
268
|
+
del clone_means
|
|
269
|
+
|
|
270
|
+
get_uns(adata).setdefault(MEMORY_KEY, {})[time_key] = h5ad_safe(res)
|
|
271
|
+
extra = (f"; same programme {res['persistence']:.2f} vs {res['persistence_null']:.2f} for random clones"
|
|
272
|
+
if "persistence" in res else "")
|
|
273
|
+
log(f"clonal memory over '{time_key}': {uniq.size} clones, {pa.size} snapshot pairs; memory index "
|
|
274
|
+
f"{mem:.2f} [{ci[0]:.2f}, {ci[1]:.2f}], p = {p_value:.3g}{extra}.", verbose)
|
|
275
|
+
return res
|
threadfin/export.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""When did a clone's output cells leave the germinal centre?
|
|
2
|
+
|
|
3
|
+
Question: a germinal centre and the plasma cells it exports are different compartments, but the receptor
|
|
4
|
+
says which of them belong to the same clone. Somatic hypermutation only ever accumulates, so inside one
|
|
5
|
+
clone the mutation load of an output cell cannot exceed the load the germinal centre had **when that cell
|
|
6
|
+
left**. The difference between a clone's germinal-centre cells and its own output cells is therefore a
|
|
7
|
+
clock, and it runs in only one direction.
|
|
8
|
+
|
|
9
|
+
lag(clone) = mean V-region mutation in the clone's output cells
|
|
10
|
+
- mean V-region mutation in the clone's germinal-centre cells
|
|
11
|
+
|
|
12
|
+
A negative lag means the output left while the clone was less mutated than it is now; zero means the export
|
|
13
|
+
is contemporaneous with the germinal centre that is being sampled.
|
|
14
|
+
|
|
15
|
+
Why this needs the clone as the unit
|
|
16
|
+
------------------------------------
|
|
17
|
+
At cell level, germinal-centre and plasma-cell populations differ in mutation for many reasons — different
|
|
18
|
+
donors, different visits, different sorts, and the simple fact that the two compartments are populated at
|
|
19
|
+
different times. None of those apply here, because every comparison is a clone against **itself**: the same
|
|
20
|
+
receptor, the same animal, and (with ``within``) the same visit and tissue. The clone is its own control,
|
|
21
|
+
which is what makes a one-directional reading of the difference defensible.
|
|
22
|
+
|
|
23
|
+
Turning the clock into a time
|
|
24
|
+
-----------------------------
|
|
25
|
+
Given a time column, the germinal centre's own maturation curve converts a mutation gap into days.
|
|
26
|
+
:func:`export_lag` fits mutation against time across the source compartment (Theil-Sen, so a few
|
|
27
|
+
hypermutated outliers cannot set the slope) and reports each clone's lag in days as well as in mutation
|
|
28
|
+
units. The conversion assumes the clone matured at the rate the population did, which is an assumption
|
|
29
|
+
about the average and is reported as such.
|
|
30
|
+
|
|
31
|
+
What it does not establish
|
|
32
|
+
--------------------------
|
|
33
|
+
No parent-offspring relationship between any two cells; no claim that the output cells seen at a visit
|
|
34
|
+
descend from the germinal-centre cells seen at that same visit; and nothing about affinity, which is not
|
|
35
|
+
measured by mutation count.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
from __future__ import annotations
|
|
39
|
+
|
|
40
|
+
import numpy as np
|
|
41
|
+
import pandas as pd
|
|
42
|
+
|
|
43
|
+
from ._utils import h5ad_safe, log, require_obs
|
|
44
|
+
|
|
45
|
+
__all__ = ["export_lag"]
|
|
46
|
+
|
|
47
|
+
SWITCHED = ("IGHG", "IGHA", "IGHE")
|
|
48
|
+
UNSWITCHED = ("IGHM", "IGHD")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _switched(values: pd.Series) -> np.ndarray:
|
|
52
|
+
iso = values.astype(str).str.upper()
|
|
53
|
+
known = iso.str.startswith(SWITCHED + UNSWITCHED)
|
|
54
|
+
return np.where(known, iso.str.startswith(SWITCHED).astype(float), np.nan)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _theil_sen(x: np.ndarray, y: np.ndarray, max_pairs: int = 5_000_000, random_state: int = 0) -> float:
|
|
58
|
+
"""Median pairwise slope; robust to the long right tail of mutation frequencies.
|
|
59
|
+
|
|
60
|
+
All pairs are used up to ``max_pairs``; above that (about 3,200 cells) the median is taken over a
|
|
61
|
+
fixed-seed random sample of ``max_pairs`` pairs, so memory stays bounded (all pairs of 40,000 cells
|
|
62
|
+
would be 8 x 10^8).
|
|
63
|
+
"""
|
|
64
|
+
ok = np.isfinite(x) & np.isfinite(y)
|
|
65
|
+
x, y = x[ok], y[ok]
|
|
66
|
+
if x.size < 3 or np.ptp(x) == 0:
|
|
67
|
+
return float("nan")
|
|
68
|
+
n_pairs = x.size * (x.size - 1) // 2
|
|
69
|
+
if n_pairs <= max_pairs:
|
|
70
|
+
i, j = np.triu_indices(x.size, 1)
|
|
71
|
+
else:
|
|
72
|
+
rng = np.random.default_rng(random_state)
|
|
73
|
+
i = rng.integers(0, x.size, max_pairs)
|
|
74
|
+
j = rng.integers(0, x.size, max_pairs)
|
|
75
|
+
distinct = i != j
|
|
76
|
+
i, j = i[distinct], j[distinct]
|
|
77
|
+
dx = x[j] - x[i]
|
|
78
|
+
keep = dx != 0
|
|
79
|
+
if not keep.any():
|
|
80
|
+
return float("nan")
|
|
81
|
+
return float(np.median((y[j] - y[i])[keep] / dx[keep]))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def export_lag(
|
|
85
|
+
adata,
|
|
86
|
+
*,
|
|
87
|
+
state_key: str,
|
|
88
|
+
source_states,
|
|
89
|
+
output_states,
|
|
90
|
+
clone_key: str = "clone_id",
|
|
91
|
+
mutation_key: str = "mutation_frequency",
|
|
92
|
+
within=(),
|
|
93
|
+
time_key: str | None = None,
|
|
94
|
+
isotype_key: str | None = "isotype",
|
|
95
|
+
switched_only: bool = True,
|
|
96
|
+
min_cells: int = 1,
|
|
97
|
+
) -> dict:
|
|
98
|
+
"""Date each clone's output against its own germinal centre.
|
|
99
|
+
|
|
100
|
+
Parameters
|
|
101
|
+
----------
|
|
102
|
+
state_key
|
|
103
|
+
``obs`` column holding the captured cell state.
|
|
104
|
+
source_states, output_states
|
|
105
|
+
Which levels of ``state_key`` are the germinal centre and which are its output, for example
|
|
106
|
+
``source_states=["GC"]`` and ``output_states=["PB", "LNPC"]``.
|
|
107
|
+
within
|
|
108
|
+
``obs`` columns that must match between the two cells being compared, on top of the clone itself.
|
|
109
|
+
Pass the visit and the tissue here: a clone's germinal-centre cell from one visit should not be
|
|
110
|
+
compared with its plasma cell from another.
|
|
111
|
+
time_key
|
|
112
|
+
Optional numeric (or ``d28``-style) column. When given, the source compartment's own maturation
|
|
113
|
+
slope converts each clone's lag into days.
|
|
114
|
+
switched_only
|
|
115
|
+
Compare only class-switched cells. Class switching is irreversible, so an IgM cell and an IgG cell
|
|
116
|
+
of one clone are separated by an event that mutation count does not describe.
|
|
117
|
+
min_cells
|
|
118
|
+
Cells of the clone required on each side of a comparison.
|
|
119
|
+
|
|
120
|
+
Returns
|
|
121
|
+
-------
|
|
122
|
+
dict with
|
|
123
|
+
|
|
124
|
+
``pairs``
|
|
125
|
+
one row per clone (and stratum): ``source``, ``output``, ``lag``, cell counts, and ``lag_days``
|
|
126
|
+
when ``time_key`` is given.
|
|
127
|
+
``n_clones``, ``n_output_earlier``, ``n_output_later``
|
|
128
|
+
how many clones were compared and how the lag fell.
|
|
129
|
+
``median_lag``
|
|
130
|
+
median lag in mutation-frequency units, with a clone bootstrap interval.
|
|
131
|
+
``maturation_slope``
|
|
132
|
+
mutation per unit time in the source compartment, when ``time_key`` is given.
|
|
133
|
+
|
|
134
|
+
Examples
|
|
135
|
+
--------
|
|
136
|
+
>>> res = export_lag(adata, state_key="cell_state", source_states=["GC"],
|
|
137
|
+
... output_states=["PB"], within=("timepoint",), time_key="timepoint")
|
|
138
|
+
>>> res["median_lag"], res["n_output_earlier"], res["n_clones"]
|
|
139
|
+
"""
|
|
140
|
+
require_obs(adata, clone_key, state_key, mutation_key, context="export_lag")
|
|
141
|
+
source_states, output_states = list(source_states), list(output_states)
|
|
142
|
+
overlap = set(source_states) & set(output_states)
|
|
143
|
+
if overlap:
|
|
144
|
+
raise ValueError(f"source_states and output_states overlap: {sorted(overlap)}")
|
|
145
|
+
|
|
146
|
+
obs = adata.obs
|
|
147
|
+
keep = obs[clone_key].notna() & obs[state_key].isin(source_states + output_states)
|
|
148
|
+
keep &= pd.to_numeric(obs[mutation_key], errors="coerce").notna()
|
|
149
|
+
if switched_only:
|
|
150
|
+
if isotype_key is None or isotype_key not in obs:
|
|
151
|
+
raise ValueError("switched_only=True needs an isotype column; pass isotype_key=None to skip.")
|
|
152
|
+
keep &= _switched(obs[isotype_key]) == 1
|
|
153
|
+
d = obs.loc[keep, [clone_key, state_key, mutation_key, *within]].copy()
|
|
154
|
+
if time_key is not None and time_key not in d.columns:
|
|
155
|
+
d[time_key] = obs.loc[keep, time_key]
|
|
156
|
+
if d.empty:
|
|
157
|
+
raise ValueError("No cells left after filtering; check the state levels and the isotype column.")
|
|
158
|
+
d["_side"] = np.where(d[state_key].isin(source_states), "source", "output")
|
|
159
|
+
d[mutation_key] = pd.to_numeric(d[mutation_key], errors="coerce")
|
|
160
|
+
|
|
161
|
+
keys = [clone_key, *within]
|
|
162
|
+
g = d.groupby(keys + ["_side"])[mutation_key].agg(["mean", "size"])
|
|
163
|
+
wide = g.unstack("_side")
|
|
164
|
+
missing = [s for s in ("source", "output") if ("mean", s) not in wide.columns]
|
|
165
|
+
if missing:
|
|
166
|
+
# one side has no usable cell at all — say which, rather than failing on a missing column
|
|
167
|
+
raise ValueError(
|
|
168
|
+
f"No cell with a mutation frequency on the {' and '.join(missing)} side, so no clone can be "
|
|
169
|
+
"captured on both sides. Check the state levels, the isotype filter and the mutation column.")
|
|
170
|
+
wide = wide.dropna(subset=[("mean", "source"), ("mean", "output")])
|
|
171
|
+
wide = wide[(wide[("size", "source")] >= min_cells) & (wide[("size", "output")] >= min_cells)]
|
|
172
|
+
if wide.empty:
|
|
173
|
+
raise ValueError("No clone was captured on both sides; widen `within` or lower `min_cells`.")
|
|
174
|
+
|
|
175
|
+
pairs = pd.DataFrame({
|
|
176
|
+
"source": wide[("mean", "source")].to_numpy(),
|
|
177
|
+
"output": wide[("mean", "output")].to_numpy(),
|
|
178
|
+
"n_source": wide[("size", "source")].to_numpy().astype(int),
|
|
179
|
+
"n_output": wide[("size", "output")].to_numpy().astype(int),
|
|
180
|
+
}, index=wide.index).reset_index()
|
|
181
|
+
pairs["lag"] = pairs["output"] - pairs["source"]
|
|
182
|
+
|
|
183
|
+
slope = np.nan
|
|
184
|
+
if time_key is not None:
|
|
185
|
+
t_all = pd.to_numeric(
|
|
186
|
+
obs.loc[keep, time_key].astype(str).str.extract(r"(\d+\.?\d*)")[0], errors="coerce")
|
|
187
|
+
src = d["_side"].eq("source").to_numpy()
|
|
188
|
+
slope = _theil_sen(t_all.to_numpy()[src], d[mutation_key].to_numpy()[src])
|
|
189
|
+
if np.isfinite(slope) and slope > 0:
|
|
190
|
+
pairs["lag_days"] = pairs["lag"] / slope
|
|
191
|
+
|
|
192
|
+
rng = np.random.default_rng(0)
|
|
193
|
+
lags = pairs["lag"].to_numpy()
|
|
194
|
+
boot = [np.median(lags[rng.integers(0, lags.size, lags.size)]) for _ in range(1000)]
|
|
195
|
+
out = {
|
|
196
|
+
"pairs": pairs,
|
|
197
|
+
"n_clones": int(len(pairs)),
|
|
198
|
+
"n_output_earlier": int((pairs["lag"] < 0).sum()),
|
|
199
|
+
"n_output_later": int((pairs["lag"] > 0).sum()),
|
|
200
|
+
"n_equal": int((pairs["lag"] == 0).sum()),
|
|
201
|
+
"median_lag": float(np.median(lags)),
|
|
202
|
+
"median_lag_ci": (float(np.percentile(boot, 2.5)), float(np.percentile(boot, 97.5))),
|
|
203
|
+
"maturation_slope": float(slope),
|
|
204
|
+
"median_lag_days": float(np.median(pairs["lag_days"])) if "lag_days" in pairs else float("nan"),
|
|
205
|
+
"settings": {"source_states": source_states, "output_states": output_states,
|
|
206
|
+
"within": list(within), "switched_only": bool(switched_only),
|
|
207
|
+
"min_cells": int(min_cells), "time_key": time_key},
|
|
208
|
+
}
|
|
209
|
+
log(f"export_lag: {out['n_clones']} clones captured on both sides; "
|
|
210
|
+
f"{out['n_output_earlier']} with output earlier than their own source, "
|
|
211
|
+
f"{out['n_output_later']} later; median lag {out['median_lag']:+.4f}"
|
|
212
|
+
+ (f" ({out['median_lag_days']:+.1f} days)" if np.isfinite(out["median_lag_days"]) else ""))
|
|
213
|
+
adata.uns.setdefault("threadfin", {})["export_lag"] = h5ad_safe({
|
|
214
|
+
k: v for k, v in out.items() if k != "pairs"})
|
|
215
|
+
return out
|
threadfin/expression.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""A train-only expression embedding for prospective/held-out validation."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
import scipy.sparse as sp
|
|
8
|
+
from sklearn.decomposition import PCA
|
|
9
|
+
from ._utils import require_positive_int
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class FrozenExpressionModel:
|
|
13
|
+
"""HVG selection, scaling and PCA fitted on reference cells only.
|
|
14
|
+
|
|
15
|
+
Per-cell normalisation is to 10,000 counts, followed by log1p.
|
|
16
|
+
Seurat HVGs exclude immunoglobulin and TCR genes. Mean, standard
|
|
17
|
+
deviation, clipping and PCA are frozen for all query cells. There is no
|
|
18
|
+
Harmony step: a query donor cannot change the fitted coordinates.
|
|
19
|
+
This controls train/query leakage, not biological batch confounding.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
@staticmethod
|
|
23
|
+
def _counts(adata, layer):
|
|
24
|
+
x = adata.X if layer is None else adata.layers[layer]
|
|
25
|
+
values = x.data if sp.issparse(x) else np.asarray(x)
|
|
26
|
+
if not np.isfinite(values).all() or np.any(values < 0):
|
|
27
|
+
raise ValueError("Frozen expression embedding requires finite non-negative counts.")
|
|
28
|
+
if not adata.var_names.is_unique:
|
|
29
|
+
raise ValueError("Gene names must be unique.")
|
|
30
|
+
return x
|
|
31
|
+
|
|
32
|
+
@staticmethod
|
|
33
|
+
def _lognorm(x):
|
|
34
|
+
total = np.asarray(x.sum(axis=1)).ravel()
|
|
35
|
+
if np.any(total <= 0):
|
|
36
|
+
raise ValueError("Every input cell needs a positive library size.")
|
|
37
|
+
factor = 1e4 / total
|
|
38
|
+
if sp.issparse(x):
|
|
39
|
+
out = x.tocsr().multiply(factor[:, None]).tocsr()
|
|
40
|
+
out.data = np.log1p(out.data)
|
|
41
|
+
return out
|
|
42
|
+
return np.log1p(np.asarray(x, dtype=float) * factor[:, None])
|
|
43
|
+
|
|
44
|
+
@classmethod
|
|
45
|
+
def fit(cls, reference, *, counts_layer=None, n_top_genes=2000,
|
|
46
|
+
n_comps=30, clip=10.0, random_state=0, batch_size=2048):
|
|
47
|
+
"""Fit all data-dependent choices on reference cells, without mutating them."""
|
|
48
|
+
import anndata as ad
|
|
49
|
+
import scanpy as sc
|
|
50
|
+
from .pp import ig_gene_mask
|
|
51
|
+
for name, value in [("n_top_genes", n_top_genes), ("n_comps", n_comps), ("batch_size", batch_size)]:
|
|
52
|
+
require_positive_int(value, name)
|
|
53
|
+
if not np.isfinite(clip) or clip <= 0:
|
|
54
|
+
raise ValueError("clip must be positive and finite.")
|
|
55
|
+
if reference.n_obs < 3:
|
|
56
|
+
raise ValueError("At least three reference cells are required.")
|
|
57
|
+
x = cls._lognorm(cls._counts(reference, counts_layer))
|
|
58
|
+
genes = pd.Index(reference.var_names).astype(str)
|
|
59
|
+
keep = ~ig_gene_mask(genes, constant=True, tr_genes=True)
|
|
60
|
+
if keep.sum() < 2:
|
|
61
|
+
raise ValueError("At least two non-receptor genes are required.")
|
|
62
|
+
tmp = ad.AnnData(x[:, keep], var=pd.DataFrame(index=genes[keep]))
|
|
63
|
+
hv = sc.pp.highly_variable_genes(tmp, n_top_genes=min(n_top_genes, tmp.n_vars),
|
|
64
|
+
flavor="seurat", inplace=False)
|
|
65
|
+
selected = np.flatnonzero(keep)[hv.highly_variable.to_numpy()]
|
|
66
|
+
sub = x[:, selected]
|
|
67
|
+
sub = sub.toarray() if sp.issparse(sub) else np.asarray(sub)
|
|
68
|
+
obj = cls()
|
|
69
|
+
obj.gene_universe = genes.to_numpy(dtype=str)
|
|
70
|
+
obj.genes = genes[selected].to_numpy(dtype=str)
|
|
71
|
+
obj.mean = sub.mean(axis=0)
|
|
72
|
+
obj.scale = sub.std(axis=0, ddof=1)
|
|
73
|
+
obj.scale[obj.scale <= 0] = 1.0
|
|
74
|
+
obj.clip, obj.batch_size = float(clip), int(batch_size)
|
|
75
|
+
z = np.clip((sub-obj.mean) / obj.scale, -clip, clip)
|
|
76
|
+
n_comps = min(n_comps, reference.n_obs-1, len(selected))
|
|
77
|
+
pca = PCA(n_components=n_comps, svd_solver="randomized", random_state=random_state)
|
|
78
|
+
pca.fit(z)
|
|
79
|
+
obj.components = pca.components_
|
|
80
|
+
obj.pca_mean = pca.mean_
|
|
81
|
+
obj.explained_variance_ratio = pca.explained_variance_ratio_
|
|
82
|
+
obj.random_state = int(random_state)
|
|
83
|
+
obj.n_reference_cells = int(reference.n_obs)
|
|
84
|
+
return obj
|
|
85
|
+
|
|
86
|
+
def transform(self, query, *, counts_layer=None):
|
|
87
|
+
"""Project query cells in batches; no fit, centering update or integration."""
|
|
88
|
+
counts = self._counts(query, counts_layer)
|
|
89
|
+
genes = pd.Index(query.var_names).astype(str)
|
|
90
|
+
if len(genes) != len(self.gene_universe) or set(genes) != set(self.gene_universe):
|
|
91
|
+
raise ValueError("Query must contain the same gene universe as the reference (order may differ).")
|
|
92
|
+
selected = genes.get_indexer(self.genes)
|
|
93
|
+
out = np.empty((query.n_obs, len(self.components)))
|
|
94
|
+
for start in range(0, query.n_obs, self.batch_size):
|
|
95
|
+
stop = min(start+self.batch_size, query.n_obs)
|
|
96
|
+
sub = self._lognorm(counts[start:stop])[:, selected]
|
|
97
|
+
sub = sub.toarray() if sp.issparse(sub) else np.asarray(sub)
|
|
98
|
+
z = np.clip((sub-self.mean) / self.scale, -self.clip, self.clip)
|
|
99
|
+
out[start:stop] = (z-self.pca_mean) @ self.components.T
|
|
100
|
+
return out
|
|
101
|
+
|
|
102
|
+
def save(self, path):
|
|
103
|
+
"""Save frozen numeric parameters without pickle."""
|
|
104
|
+
parameters = {k: getattr(self, k) for k in ("clip", "batch_size", "random_state", "n_reference_cells")}
|
|
105
|
+
with Path(path).open("wb") as handle:
|
|
106
|
+
np.savez_compressed(handle, parameters=json.dumps(parameters),
|
|
107
|
+
**{k: getattr(self, k) for k in
|
|
108
|
+
("gene_universe", "genes", "mean", "scale", "components",
|
|
109
|
+
"pca_mean", "explained_variance_ratio")})
|
|
110
|
+
|
|
111
|
+
@classmethod
|
|
112
|
+
def load(cls, path):
|
|
113
|
+
"""Read frozen numeric parameters with pickle disabled."""
|
|
114
|
+
obj = cls()
|
|
115
|
+
with np.load(path, allow_pickle=False) as saved:
|
|
116
|
+
for key, value in json.loads(str(saved["parameters"])).items():
|
|
117
|
+
setattr(obj, key, value)
|
|
118
|
+
for key in ("gene_universe", "genes", "mean", "scale", "components", "pca_mean",
|
|
119
|
+
"explained_variance_ratio"):
|
|
120
|
+
setattr(obj, key, saved[key].copy())
|
|
121
|
+
return obj
|