threadfin 4.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- threadfin/__init__.py +86 -0
- threadfin/_utils.py +260 -0
- threadfin/api.py +658 -0
- threadfin/bcrgraph.py +255 -0
- threadfin/clones.py +653 -0
- threadfin/core.py +531 -0
- threadfin/dandelion.py +458 -0
- threadfin/density.py +409 -0
- threadfin/dynamics.py +275 -0
- threadfin/export.py +215 -0
- threadfin/expression.py +121 -0
- threadfin/heritability.py +224 -0
- threadfin/integrate.py +350 -0
- threadfin/io.py +280 -0
- threadfin/maptune.py +743 -0
- threadfin/metrics.py +137 -0
- threadfin/migration.py +165 -0
- threadfin/phylo.py +834 -0
- threadfin/plotting.py +535 -0
- threadfin/pp.py +253 -0
- threadfin/profiles.py +603 -0
- threadfin/programmes.py +1028 -0
- threadfin/programs.py +133 -0
- threadfin/reclustering.py +68 -0
- threadfin/sequence.py +379 -0
- threadfin/simulate.py +206 -0
- threadfin/specificity.py +355 -0
- threadfin/stats.py +476 -0
- threadfin/tl.py +62 -0
- threadfin-4.5.0.dist-info/METADATA +288 -0
- threadfin-4.5.0.dist-info/RECORD +34 -0
- threadfin-4.5.0.dist-info/WHEEL +5 -0
- threadfin-4.5.0.dist-info/licenses/LICENSE +21 -0
- threadfin-4.5.0.dist-info/top_level.txt +1 -0
threadfin/stats.py
ADDED
|
@@ -0,0 +1,476 @@
|
|
|
1
|
+
"""Clone-level statistics with permutation nulls that respect the sampling design.
|
|
2
|
+
|
|
3
|
+
Two principles run through this module:
|
|
4
|
+
|
|
5
|
+
1. **The clone is the unit of replication.** Cells of one clone share their
|
|
6
|
+
ancestry, so testing at cell level counts one expanded clone thousands of
|
|
7
|
+
times (pseudoreplication) and makes p-values meaningless. Every test here
|
|
8
|
+
counts clones.
|
|
9
|
+
2. **Permute within strata.** Clonally related cells are usually sampled
|
|
10
|
+
together (same donor, sample, tissue, timepoint or sort gate). A null that
|
|
11
|
+
ignores this attributes sample composition to clonality. Labels are
|
|
12
|
+
therefore shuffled only among cells (or clones) that share a stratum.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
20
|
+
from ._utils import bh_fdr, codes, get_uns, h5ad_safe, log, perm_pvalue, permute_within, require_obs, require_complete_groups, require_positive_int
|
|
21
|
+
from .profiles import get_profiles, model_from_adata, variance_components
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# --------------------------------------------------------------------------- clonal coherence
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def clonal_coherence(
|
|
28
|
+
adata,
|
|
29
|
+
*,
|
|
30
|
+
strata_key: str | None = None,
|
|
31
|
+
n_perm: int = 200,
|
|
32
|
+
random_state: int = 0,
|
|
33
|
+
verbose: bool = True,
|
|
34
|
+
) -> dict:
|
|
35
|
+
"""Test whether clone identity explains transcriptional state.
|
|
36
|
+
|
|
37
|
+
The statistic is the clonal ICC (fraction of embedding variance explained
|
|
38
|
+
by clone, see :mod:`threadfin.profiles`). The null distribution comes from
|
|
39
|
+
shuffling clone labels among BCR+ cells *within* ``strata_key`` (for
|
|
40
|
+
example sample, or donor x tissue x timepoint), which keeps each
|
|
41
|
+
stratum's cell-state composition and clone-size distribution intact and
|
|
42
|
+
destroys only the link between a cell's clone and its state.
|
|
43
|
+
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
strata_key
|
|
47
|
+
``obs`` column defining permutation strata. Defaults to the
|
|
48
|
+
``context_key`` used for :func:`threadfin.tl.clone_profiles`; with
|
|
49
|
+
neither, labels are shuffled globally (not recommended).
|
|
50
|
+
n_perm
|
|
51
|
+
Number of permutations.
|
|
52
|
+
|
|
53
|
+
Returns
|
|
54
|
+
-------
|
|
55
|
+
dict with the observed ``icc``, null mean/sd/95th percentile,
|
|
56
|
+
``excess`` (observed - null mean), ``p_value`` and bookkeeping; also stored
|
|
57
|
+
in ``adata.uns['threadfin']['coherence']``.
|
|
58
|
+
"""
|
|
59
|
+
require_positive_int(n_perm, "n_perm")
|
|
60
|
+
prof = get_profiles(adata)
|
|
61
|
+
# the ICC is always measured on unsmoothed cells in the embedding itself, so it reads
|
|
62
|
+
# as "share of transcriptional variance explained by clone" whatever the profile type
|
|
63
|
+
model = model_from_adata(adata, unsmoothed=True, representation="mean")
|
|
64
|
+
strata_key = strata_key or prof["params"].get("context_key")
|
|
65
|
+
strata = None
|
|
66
|
+
if strata_key is not None:
|
|
67
|
+
require_complete_groups(adata, strata_key, context="coherence permutations")
|
|
68
|
+
strata, _ = codes(adata.obs[strata_key])
|
|
69
|
+
|
|
70
|
+
observed = model.vc.icc
|
|
71
|
+
rng = np.random.default_rng(random_state)
|
|
72
|
+
null = np.empty(n_perm)
|
|
73
|
+
for b in range(n_perm):
|
|
74
|
+
perm = permute_within(model.clone_codes, strata, rng)
|
|
75
|
+
null[b] = variance_components(model.residuals, perm).icc
|
|
76
|
+
res = {
|
|
77
|
+
"icc": float(observed),
|
|
78
|
+
"null_mean": float(null.mean()),
|
|
79
|
+
"null_sd": float(null.std(ddof=1)) if n_perm > 1 else float("nan"),
|
|
80
|
+
"null_q95": float(np.quantile(null, 0.95)),
|
|
81
|
+
"excess": float(observed - null.mean()),
|
|
82
|
+
"p_value": perm_pvalue(observed, null, "greater"),
|
|
83
|
+
"n_perm": int(n_perm),
|
|
84
|
+
"strata_key": strata_key,
|
|
85
|
+
"n_clones": int(model.vc.n_clones),
|
|
86
|
+
"n_cells": int(model.vc.n_cells),
|
|
87
|
+
"icc_per_feature": model.vc.icc_per_feature,
|
|
88
|
+
"null": null,
|
|
89
|
+
}
|
|
90
|
+
get_uns(adata)["coherence"] = h5ad_safe(res)
|
|
91
|
+
log(
|
|
92
|
+
f"clonal coherence: ICC {observed:.3f} vs {res['null_mean']:.3f} under within-"
|
|
93
|
+
f"'{strata_key}' permutation (p = {res['p_value']:.3g}, {n_perm} permutations).",
|
|
94
|
+
verbose,
|
|
95
|
+
)
|
|
96
|
+
return res
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
# --------------------------------------------------------------------------- clone-level labels
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def clone_labels(
|
|
103
|
+
adata,
|
|
104
|
+
label_key: str,
|
|
105
|
+
*,
|
|
106
|
+
clone_key: str | None = None,
|
|
107
|
+
how: str = "auto",
|
|
108
|
+
min_frac: float = 0.5,
|
|
109
|
+
) -> pd.Series:
|
|
110
|
+
"""Collapse a cell-level ``obs`` column to one value per clone.
|
|
111
|
+
|
|
112
|
+
Parameters
|
|
113
|
+
----------
|
|
114
|
+
how
|
|
115
|
+
``"majority"``: modal value, kept only if it covers at least
|
|
116
|
+
``min_frac`` of the clone's labelled cells (else NaN);
|
|
117
|
+
``"mean"`` / ``"median"``: for numeric columns;
|
|
118
|
+
``"fraction:<level>"``: fraction of the clone's labelled cells equal
|
|
119
|
+
to ``<level>`` (numeric output);
|
|
120
|
+
``"auto"``: ``"mean"`` for numeric columns, else ``"majority"``.
|
|
121
|
+
"""
|
|
122
|
+
clone_key = clone_key or get_profiles(adata)["params"]["clone_key"]
|
|
123
|
+
require_obs(adata, clone_key, label_key)
|
|
124
|
+
df = adata.obs[[clone_key, label_key]].dropna()
|
|
125
|
+
df = df.assign(_clone=df[clone_key].astype(str))
|
|
126
|
+
numeric = pd.api.types.is_numeric_dtype(df[label_key]) and not pd.api.types.is_bool_dtype(df[label_key])
|
|
127
|
+
if how == "auto":
|
|
128
|
+
how = "mean" if numeric else "majority"
|
|
129
|
+
grouped = df.groupby("_clone", observed=True)[label_key]
|
|
130
|
+
if how in ("mean", "median"):
|
|
131
|
+
values = pd.to_numeric(df[label_key], errors="coerce")
|
|
132
|
+
grouped = values.groupby(df["_clone"])
|
|
133
|
+
out = grouped.mean() if how == "mean" else grouped.median()
|
|
134
|
+
elif how.startswith("fraction:"):
|
|
135
|
+
level = how.split(":", 1)[1]
|
|
136
|
+
out = df[label_key].astype(str).eq(level).groupby(df["_clone"]).mean()
|
|
137
|
+
elif how == "majority":
|
|
138
|
+
def _maj(s):
|
|
139
|
+
vc = s.astype(str).value_counts(normalize=True)
|
|
140
|
+
tied = len(vc) > 1 and vc.iloc[1] == vc.iloc[0]
|
|
141
|
+
return None if tied or vc.iloc[0] < min_frac else vc.index[0]
|
|
142
|
+
out = grouped.agg(_maj)
|
|
143
|
+
else:
|
|
144
|
+
raise ValueError(f"unknown how={how!r}")
|
|
145
|
+
out.index.name = clone_key
|
|
146
|
+
return out.rename(label_key)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
# --------------------------------------------------------------------------- association tests
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _mantel_haenszel(a, b, c, d):
|
|
153
|
+
"""MH pooled odds ratio and 95% CI (Robins-Breslow-Greenland) over strata."""
|
|
154
|
+
n = a + b + c + d
|
|
155
|
+
ok = n > 0
|
|
156
|
+
a, b, c, d, n = a[ok], b[ok], c[ok], d[ok], n[ok]
|
|
157
|
+
r = a * d / n
|
|
158
|
+
s = b * c / n
|
|
159
|
+
R, S = r.sum(), s.sum()
|
|
160
|
+
if R == 0 or S == 0:
|
|
161
|
+
or_mh = np.inf if S == 0 and R > 0 else (0.0 if R == 0 and S > 0 else np.nan)
|
|
162
|
+
return or_mh, np.nan, np.nan
|
|
163
|
+
p_ = (a + d) / n
|
|
164
|
+
q_ = (b + c) / n
|
|
165
|
+
var = (np.sum(p_ * r) / (2 * R**2) + np.sum(p_ * s + q_ * r) / (2 * R * S)
|
|
166
|
+
+ np.sum(q_ * s) / (2 * S**2))
|
|
167
|
+
lor = np.log(R / S)
|
|
168
|
+
half = 1.959964 * np.sqrt(var)
|
|
169
|
+
return float(np.exp(lor)), float(np.exp(lor - half)), float(np.exp(lor + half))
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _assoc_table(adata, label_key, programme_key, clone_key, strata_key, how, min_frac):
|
|
173
|
+
prof = get_profiles(adata)
|
|
174
|
+
table = prof["clone_table"]
|
|
175
|
+
if programme_key not in table.columns:
|
|
176
|
+
raise KeyError(f"'{programme_key}' not in the clone table; run find_programmes() first.")
|
|
177
|
+
lab = clone_labels(adata, label_key, clone_key=clone_key, how=how, min_frac=min_frac)
|
|
178
|
+
df = pd.DataFrame({
|
|
179
|
+
"programme": table[programme_key].astype("object"),
|
|
180
|
+
"label": lab.reindex(table.index),
|
|
181
|
+
"n_cells": table["n_cells"],
|
|
182
|
+
})
|
|
183
|
+
if strata_key is not None:
|
|
184
|
+
if strata_key in table.columns:
|
|
185
|
+
df["stratum"] = table[strata_key].astype(str)
|
|
186
|
+
else:
|
|
187
|
+
df["stratum"] = clone_labels(adata, strata_key, clone_key=clone_key, how="majority",
|
|
188
|
+
min_frac=0.0).reindex(table.index).astype(str)
|
|
189
|
+
else:
|
|
190
|
+
df["stratum"] = "all"
|
|
191
|
+
return df.dropna(subset=["programme", "label"])
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def association_test(
|
|
195
|
+
adata,
|
|
196
|
+
label_key: str,
|
|
197
|
+
*,
|
|
198
|
+
programme_key: str = "clone_programme",
|
|
199
|
+
clone_key: str | None = None,
|
|
200
|
+
strata_key: str | None = "auto",
|
|
201
|
+
how: str = "auto",
|
|
202
|
+
min_frac: float = 0.5,
|
|
203
|
+
n_perm: int = 2000,
|
|
204
|
+
random_state: int = 0,
|
|
205
|
+
verbose: bool = True,
|
|
206
|
+
) -> pd.DataFrame:
|
|
207
|
+
"""Clone-level association between programmes and a label.
|
|
208
|
+
|
|
209
|
+
Each clone contributes once. Cell-level labels are first collapsed per
|
|
210
|
+
clone (:func:`clone_labels`). P-values come from permuting clone labels
|
|
211
|
+
within strata (default: donor), which keeps every donor's label
|
|
212
|
+
frequencies fixed; Benjamini-Hochberg FDR is computed across all rows.
|
|
213
|
+
|
|
214
|
+
*Categorical labels* (e.g. antigen specificity, isotype, infection
|
|
215
|
+
status) give one row per (programme, level) with the observed and
|
|
216
|
+
expected number of clones, the Mantel-Haenszel odds ratio across strata
|
|
217
|
+
with a 95% confidence interval, and a two-sided permutation p-value.
|
|
218
|
+
|
|
219
|
+
*Numeric labels* (e.g. mutation frequency, fraction of a clone's cells in
|
|
220
|
+
a sort gate) give one row per programme with the median in vs out of the
|
|
221
|
+
programme, a rank-based effect (difference in mean ranks scaled to
|
|
222
|
+
[-1, 1]) and a two-sided permutation p-value; an omnibus Kruskal-Wallis
|
|
223
|
+
test with the same permutation scheme is stored in ``.attrs``.
|
|
224
|
+
|
|
225
|
+
Parameters
|
|
226
|
+
----------
|
|
227
|
+
label_key
|
|
228
|
+
``obs`` column to test.
|
|
229
|
+
strata_key
|
|
230
|
+
Permutation strata. ``"auto"`` uses the donor recorded by
|
|
231
|
+
:func:`threadfin.tl.clone_profiles` (if any); ``None`` permutes
|
|
232
|
+
globally.
|
|
233
|
+
how, min_frac
|
|
234
|
+
Passed to :func:`clone_labels`.
|
|
235
|
+
"""
|
|
236
|
+
require_positive_int(n_perm, "n_perm")
|
|
237
|
+
prof = get_profiles(adata)
|
|
238
|
+
clone_key = clone_key or prof["params"]["clone_key"]
|
|
239
|
+
if strata_key == "auto":
|
|
240
|
+
strata_key = "donor" if "donor" in prof["clone_table"].columns else None
|
|
241
|
+
df = _assoc_table(adata, label_key, programme_key, clone_key, strata_key, how, min_frac)
|
|
242
|
+
if df.empty:
|
|
243
|
+
raise ValueError(f"No clones with both a programme and a '{label_key}' value.")
|
|
244
|
+
rng = np.random.default_rng(random_state)
|
|
245
|
+
prog_codes, progs = codes(df["programme"])
|
|
246
|
+
if len(progs) < 2:
|
|
247
|
+
raise ValueError("Only one programme: clone-level tests compare programmes, so at least two are needed.")
|
|
248
|
+
strata = codes(df["stratum"])[0]
|
|
249
|
+
numeric = pd.api.types.is_numeric_dtype(df["label"]) and not pd.api.types.is_bool_dtype(df["label"])
|
|
250
|
+
|
|
251
|
+
if numeric:
|
|
252
|
+
out = _numeric_association(df, prog_codes, progs, strata, n_perm, rng)
|
|
253
|
+
else:
|
|
254
|
+
out = _categorical_association(df, prog_codes, progs, strata, n_perm, rng)
|
|
255
|
+
out.insert(0, "label", label_key)
|
|
256
|
+
out["fdr"] = bh_fdr(out["pvalue"].to_numpy())
|
|
257
|
+
out.attrs["strata_key"] = strata_key
|
|
258
|
+
out.attrs["n_clones"] = int(len(df))
|
|
259
|
+
get_uns(adata).setdefault("associations", {})[label_key] = h5ad_safe(out)
|
|
260
|
+
n_sig = int((out["fdr"] < 0.05).sum())
|
|
261
|
+
log(f"association '{label_key}': {len(df)} clones, {n_sig} programme effects at FDR < 0.05 "
|
|
262
|
+
f"(within-'{strata_key}' permutation).", verbose)
|
|
263
|
+
return out
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
# --------------------------------------------------------------------------- label vs clone profiles
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def profile_association(
|
|
270
|
+
adata,
|
|
271
|
+
label_key: str,
|
|
272
|
+
*,
|
|
273
|
+
clone_key: str | None = None,
|
|
274
|
+
strata_key: str | None = "auto",
|
|
275
|
+
how: str = "auto",
|
|
276
|
+
min_frac: float = 0.5,
|
|
277
|
+
min_reliability: float = 0.0,
|
|
278
|
+
n_perm: int = 2000,
|
|
279
|
+
random_state: int = 0,
|
|
280
|
+
verbose: bool = True,
|
|
281
|
+
) -> dict:
|
|
282
|
+
"""Does a clone-level label explain how clones differ? (no programmes needed)
|
|
283
|
+
|
|
284
|
+
Programme tests (:func:`association_test`) need distinct programmes. When
|
|
285
|
+
clones vary along a continuum there are none, but a label (division
|
|
286
|
+
history, antigen binding, isotype) can still track where a clone sits.
|
|
287
|
+
This test asks how much of the variation between clone profiles is
|
|
288
|
+
explained by the label: the share of the profile sum of squares between
|
|
289
|
+
label groups (categorical labels, as in PERMANOVA; Anderson 2001) or
|
|
290
|
+
captured by a linear trend in the label (numeric labels, as in
|
|
291
|
+
distance-based redundancy analysis; McArdle & Anderson 2001). The
|
|
292
|
+
p-value comes from permuting labels among clones within strata (donors).
|
|
293
|
+
|
|
294
|
+
Parameters
|
|
295
|
+
----------
|
|
296
|
+
label_key, how, min_frac
|
|
297
|
+
Cell-level label and how it is collapsed per clone (see
|
|
298
|
+
:func:`clone_labels`).
|
|
299
|
+
strata_key
|
|
300
|
+
Clone-table column whose levels define permutation strata;
|
|
301
|
+
``"auto"`` uses the donor when available.
|
|
302
|
+
min_reliability
|
|
303
|
+
Reliability floor; ``0.0`` (the default) uses every profiled clone.
|
|
304
|
+
Threadfin does not filter clones. Every clone, and every cell in it, is measured data, and
|
|
305
|
+
the profiles are already **shrunk**: a clone seen in few cells has its profile pulled towards the
|
|
306
|
+
population mean by the variance-component model, so its influence is down-weighted in proportion to
|
|
307
|
+
how little was seen. Cutting on reliability *on top of* that shrinkage discards the same information
|
|
308
|
+
twice, and it discards it silently — on GSE287123 a 0.5 floor removes 75 % (RBD) and 90 % (NP-OVA) of
|
|
309
|
+
profiled clones, and on GSE246382 it removes all of them, which is how that dataset came to look as
|
|
310
|
+
though it had no clone structure when its map separates the sorted gate at p < 0.001.
|
|
311
|
+
|
|
312
|
+
Returns
|
|
313
|
+
-------
|
|
314
|
+
dict with ``r2`` (fraction of profile variance explained by the label),
|
|
315
|
+
the permutation ``null_mean``, ``excess_r2`` (``r2 - null_mean``),
|
|
316
|
+
``p_value``, ``n_clones``, ``kind`` and, for every level of a categorical
|
|
317
|
+
label (or for the numeric label), Spearman correlations with the first
|
|
318
|
+
three principal axes of the profiles (``axis_correlation``). Stored in
|
|
319
|
+
``adata.uns['threadfin']['profile_associations'][label_key]``.
|
|
320
|
+
"""
|
|
321
|
+
from scipy.stats import spearmanr
|
|
322
|
+
|
|
323
|
+
require_positive_int(n_perm, "n_perm")
|
|
324
|
+
prof = get_profiles(adata)
|
|
325
|
+
table = prof["clone_table"]
|
|
326
|
+
clone_key = clone_key or prof["params"]["clone_key"]
|
|
327
|
+
if strata_key == "auto":
|
|
328
|
+
strata_key = "donor" if "donor" in table.columns else None
|
|
329
|
+
labels = clone_labels(adata, label_key, clone_key=clone_key, how=how, min_frac=min_frac)
|
|
330
|
+
use = table.index[(table["reliability"] >= min_reliability).to_numpy()]
|
|
331
|
+
use = use[labels.reindex(use).notna().to_numpy()]
|
|
332
|
+
if use.size < 10:
|
|
333
|
+
raise ValueError(f"Only {use.size} reliable clones have a '{label_key}' value (need >= 10).")
|
|
334
|
+
x = prof["features"].loc[use].to_numpy(dtype=float)
|
|
335
|
+
x = x - x.mean(axis=0)
|
|
336
|
+
total = float(np.einsum("ij,ij->", x, x))
|
|
337
|
+
if not np.isfinite(total) or total <= 0:
|
|
338
|
+
raise ValueError("Clone profiles have no finite between-clone variation to explain.")
|
|
339
|
+
y = labels.loc[use]
|
|
340
|
+
strata = (codes(table.loc[use, strata_key].astype(str))[0] if strata_key is not None
|
|
341
|
+
else np.zeros(use.size, dtype=np.int64))
|
|
342
|
+
numeric = pd.api.types.is_numeric_dtype(y) and not pd.api.types.is_bool_dtype(y)
|
|
343
|
+
rng = np.random.default_rng(random_state)
|
|
344
|
+
|
|
345
|
+
if numeric:
|
|
346
|
+
yv = y.to_numpy(dtype=float)
|
|
347
|
+
|
|
348
|
+
def stat(v):
|
|
349
|
+
v = v - v.mean()
|
|
350
|
+
ss = float(v @ v)
|
|
351
|
+
return 0.0 if ss <= 0 else float(np.sum((x.T @ v) ** 2) / ss / total)
|
|
352
|
+
else:
|
|
353
|
+
yv, levels = codes(y.astype(str))
|
|
354
|
+
if len(levels) < 2:
|
|
355
|
+
raise ValueError(f"'{label_key}' has a single level among the reliable clones.")
|
|
356
|
+
|
|
357
|
+
def stat(v):
|
|
358
|
+
sums = np.zeros((len(levels), x.shape[1]))
|
|
359
|
+
np.add.at(sums, v, x)
|
|
360
|
+
n = np.bincount(v, minlength=len(levels)).astype(float)
|
|
361
|
+
ok = n > 0
|
|
362
|
+
return float(np.sum(sums[ok] ** 2 / n[ok, None]) / total)
|
|
363
|
+
|
|
364
|
+
observed = stat(yv)
|
|
365
|
+
null = np.array([stat(permute_within(yv, strata, rng)) for _ in range(n_perm)])
|
|
366
|
+
p_value = perm_pvalue(observed, null)
|
|
367
|
+
|
|
368
|
+
# where along the main axes of clone variation does the label sit?
|
|
369
|
+
_, _, vt = np.linalg.svd(x, full_matrices=False)
|
|
370
|
+
axes = x @ vt[: min(3, vt.shape[0])].T
|
|
371
|
+
if numeric:
|
|
372
|
+
axis_corr = {"label": [float(spearmanr(axes[:, k], yv).statistic) for k in range(axes.shape[1])]}
|
|
373
|
+
else:
|
|
374
|
+
axis_corr = {str(lv): [float(spearmanr(axes[:, k], (yv == i).astype(float)).statistic)
|
|
375
|
+
for k in range(axes.shape[1])] for i, lv in enumerate(levels)}
|
|
376
|
+
res = {
|
|
377
|
+
"label": label_key, "kind": "numeric" if numeric else "categorical", "r2": observed,
|
|
378
|
+
"null_mean": float(null.mean()), "excess_r2": float(observed - null.mean()), "p_value": float(p_value),
|
|
379
|
+
"n_clones": int(use.size), "n_levels": None if numeric else int(len(levels)),
|
|
380
|
+
"strata_key": strata_key, "n_perm": int(n_perm), "axis_correlation": axis_corr,
|
|
381
|
+
}
|
|
382
|
+
get_uns(adata).setdefault("profile_associations", {})[label_key] = h5ad_safe(res)
|
|
383
|
+
log(f"profile association '{label_key}': explains {100 * observed:.1f}% of clone-profile variance vs "
|
|
384
|
+
f"{100 * res['null_mean']:.1f}% for permuted labels (p = {p_value:.3g}, {use.size} clones, "
|
|
385
|
+
f"within-'{strata_key}' permutation).", verbose)
|
|
386
|
+
return res
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _categorical_association(df, prog_codes, progs, strata, n_perm, rng):
|
|
390
|
+
lab_codes, levels = codes(df["label"].astype(str))
|
|
391
|
+
n_p, n_l = len(progs), len(levels)
|
|
392
|
+
n_s = int(strata.max()) + 1
|
|
393
|
+
|
|
394
|
+
def counts(labs):
|
|
395
|
+
return np.bincount(prog_codes * n_l + labs, minlength=n_p * n_l).reshape(n_p, n_l)
|
|
396
|
+
|
|
397
|
+
observed = counts(lab_codes)
|
|
398
|
+
# hypergeometric expectation under within-stratum permutation
|
|
399
|
+
expected = np.zeros((n_p, n_l))
|
|
400
|
+
for s in range(n_s):
|
|
401
|
+
m = strata == s
|
|
402
|
+
if not m.any():
|
|
403
|
+
continue
|
|
404
|
+
cp = np.bincount(prog_codes[m], minlength=n_p).astype(float)
|
|
405
|
+
cl = np.bincount(lab_codes[m], minlength=n_l).astype(float)
|
|
406
|
+
expected += np.outer(cp, cl) / m.sum()
|
|
407
|
+
null_dev = np.zeros((n_p, n_l))
|
|
408
|
+
obs_dev = np.abs(observed - expected)
|
|
409
|
+
for _ in range(n_perm):
|
|
410
|
+
perm = permute_within(lab_codes, strata, rng)
|
|
411
|
+
null_dev += np.abs(counts(perm) - expected) >= obs_dev - 1e-9
|
|
412
|
+
pvals = (1 + null_dev) / (n_perm + 1)
|
|
413
|
+
|
|
414
|
+
rows = []
|
|
415
|
+
for i, prog in enumerate(progs):
|
|
416
|
+
in_p = prog_codes == i
|
|
417
|
+
for j, level in enumerate(levels):
|
|
418
|
+
has = lab_codes == j
|
|
419
|
+
a = np.bincount(strata[in_p & has], minlength=n_s).astype(float)
|
|
420
|
+
b = np.bincount(strata[in_p & ~has], minlength=n_s).astype(float)
|
|
421
|
+
c = np.bincount(strata[~in_p & has], minlength=n_s).astype(float)
|
|
422
|
+
d = np.bincount(strata[~in_p & ~has], minlength=n_s).astype(float)
|
|
423
|
+
or_mh, lo, hi = _mantel_haenszel(a, b, c, d)
|
|
424
|
+
n_in = int(in_p.sum())
|
|
425
|
+
rows.append({
|
|
426
|
+
"programme": prog, "level": level,
|
|
427
|
+
"n_clones_programme": n_in, "n_clones_level": int(has.sum()),
|
|
428
|
+
"observed": int(observed[i, j]), "expected": float(expected[i, j]),
|
|
429
|
+
"frac_in_programme": observed[i, j] / n_in if n_in else np.nan,
|
|
430
|
+
"frac_elsewhere": float(c.sum() / max((~in_p).sum(), 1)),
|
|
431
|
+
"odds_ratio": or_mh, "ci_low": lo, "ci_high": hi,
|
|
432
|
+
"pvalue": float(pvals[i, j]),
|
|
433
|
+
})
|
|
434
|
+
return pd.DataFrame(rows)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _numeric_association(df, prog_codes, progs, strata, n_perm, rng):
|
|
438
|
+
values = df["label"].to_numpy(dtype=float)
|
|
439
|
+
ranks = pd.Series(values).rank().to_numpy()
|
|
440
|
+
n = ranks.size
|
|
441
|
+
n_p = len(progs)
|
|
442
|
+
sizes = np.bincount(prog_codes, minlength=n_p).astype(float)
|
|
443
|
+
|
|
444
|
+
def rank_effects(r):
|
|
445
|
+
sums = np.bincount(prog_codes, weights=r, minlength=n_p)
|
|
446
|
+
mean_in = sums / np.maximum(sizes, 1)
|
|
447
|
+
mean_out = (r.sum() - sums) / np.maximum(n - sizes, 1)
|
|
448
|
+
return (mean_in - mean_out) / (n / 2.0) # scaled to roughly [-1, 1]
|
|
449
|
+
|
|
450
|
+
def kruskal_h(r):
|
|
451
|
+
sums = np.bincount(prog_codes, weights=r, minlength=n_p)
|
|
452
|
+
return 12.0 / (n * (n + 1)) * np.sum(sums**2 / np.maximum(sizes, 1)) - 3 * (n + 1)
|
|
453
|
+
|
|
454
|
+
obs_eff = rank_effects(ranks)
|
|
455
|
+
obs_h = kruskal_h(ranks)
|
|
456
|
+
exceed = np.zeros(n_p)
|
|
457
|
+
exceed_h = 0
|
|
458
|
+
for _ in range(n_perm):
|
|
459
|
+
r = permute_within(np.arange(n), strata, rng)
|
|
460
|
+
pr = ranks[r]
|
|
461
|
+
exceed += np.abs(rank_effects(pr)) >= np.abs(obs_eff) - 1e-12
|
|
462
|
+
exceed_h += kruskal_h(pr) >= obs_h - 1e-12
|
|
463
|
+
rows = []
|
|
464
|
+
for i, prog in enumerate(progs):
|
|
465
|
+
in_p = prog_codes == i
|
|
466
|
+
rows.append({
|
|
467
|
+
"programme": prog, "n_clones_programme": int(in_p.sum()),
|
|
468
|
+
"median_in_programme": float(np.median(values[in_p])) if in_p.any() else np.nan,
|
|
469
|
+
"median_elsewhere": float(np.median(values[~in_p])) if (~in_p).any() else np.nan,
|
|
470
|
+
"rank_effect": float(obs_eff[i]),
|
|
471
|
+
"pvalue": float((1 + exceed[i]) / (n_perm + 1)),
|
|
472
|
+
})
|
|
473
|
+
out = pd.DataFrame(rows)
|
|
474
|
+
out.attrs["kruskal_h"] = float(obs_h)
|
|
475
|
+
out.attrs["kruskal_pvalue"] = float((1 + exceed_h) / (n_perm + 1))
|
|
476
|
+
return out
|
threadfin/tl.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Step-by-step analysis functions.
|
|
2
|
+
|
|
3
|
+
:func:`threadfin.run` chains these in order; call them yourself when you
|
|
4
|
+
need finer control:
|
|
5
|
+
|
|
6
|
+
1. :func:`define_clones_dandelion` - BCR contigs -> clones (within donors); Dandelion's
|
|
7
|
+
paired heavy + light definition is Threadfin's clone definition
|
|
8
|
+
(:func:`define_clones`, the old heavy-chain-only caller, is deprecated)
|
|
9
|
+
2. :func:`clone_profiles` - one shrunken state profile per clone
|
|
10
|
+
3. :func:`clonal_coherence` - does clone identity explain cell state?
|
|
11
|
+
4. :func:`find_programmes` - group clones with similar profiles (significant
|
|
12
|
+
splits only), with bootstrap stability; :func:`embed_clones` - the clone map,
|
|
13
|
+
whose UMAP settings :func:`tune_clone_map` chooses automatically (Adam), and
|
|
14
|
+
:func:`map_quality` - how coherent and faithful a clone map is
|
|
15
|
+
5. :func:`association_test` / :func:`programme_markers` - clone-level tests
|
|
16
|
+
of programmes vs labels and genes; :func:`profile_association` - does a
|
|
17
|
+
label explain how clones differ (works without distinct programmes)
|
|
18
|
+
6. :func:`clonal_memory` - do clones keep their programme over time/tissue?
|
|
19
|
+
:func:`export_lag` - how far behind its own germinal centre is a clone's output?
|
|
20
|
+
7. :func:`gene_heritability` - which genes are clonally inherited?
|
|
21
|
+
8. :func:`lineage_forest` + :func:`lineage_heritability` - inside a clone, do cells that are close
|
|
22
|
+
relatives on the receptor lineage tree share transcriptional state?
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from .clones import define_clones, find_threshold, mutation_frequency
|
|
26
|
+
from .dandelion import define_clones_dandelion, read_contigs
|
|
27
|
+
from .density import clone_densities
|
|
28
|
+
from .dynamics import clonal_memory
|
|
29
|
+
from .export import export_lag
|
|
30
|
+
from .heritability import gene_heritability, geneset_heritability
|
|
31
|
+
from .maptune import layout_clone_map, map_quality, tune_clone_map
|
|
32
|
+
from .phylo import lineage_forest, lineage_heritability
|
|
33
|
+
from .profiles import clone_profiles
|
|
34
|
+
from .programmes import embed_clones, find_programmes, landscape_genes, programme_composition, programme_markers
|
|
35
|
+
from .stats import association_test, clonal_coherence, clone_labels, profile_association
|
|
36
|
+
|
|
37
|
+
__all__ = [
|
|
38
|
+
"association_test",
|
|
39
|
+
"clonal_coherence",
|
|
40
|
+
"clonal_memory",
|
|
41
|
+
"export_lag",
|
|
42
|
+
"clone_labels",
|
|
43
|
+
"clone_profiles",
|
|
44
|
+
"clone_densities",
|
|
45
|
+
"define_clones",
|
|
46
|
+
"define_clones_dandelion",
|
|
47
|
+
"embed_clones",
|
|
48
|
+
"find_programmes",
|
|
49
|
+
"find_threshold",
|
|
50
|
+
"gene_heritability",
|
|
51
|
+
"geneset_heritability",
|
|
52
|
+
"lineage_forest",
|
|
53
|
+
"layout_clone_map",
|
|
54
|
+
"lineage_heritability",
|
|
55
|
+
"map_quality",
|
|
56
|
+
"mutation_frequency",
|
|
57
|
+
"profile_association",
|
|
58
|
+
"programme_composition",
|
|
59
|
+
"programme_markers",
|
|
60
|
+
"landscape_genes",
|
|
61
|
+
"tune_clone_map",
|
|
62
|
+
]
|