threadfin 4.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- threadfin/__init__.py +86 -0
- threadfin/_utils.py +260 -0
- threadfin/api.py +658 -0
- threadfin/bcrgraph.py +255 -0
- threadfin/clones.py +653 -0
- threadfin/core.py +531 -0
- threadfin/dandelion.py +458 -0
- threadfin/density.py +409 -0
- threadfin/dynamics.py +275 -0
- threadfin/export.py +215 -0
- threadfin/expression.py +121 -0
- threadfin/heritability.py +224 -0
- threadfin/integrate.py +350 -0
- threadfin/io.py +280 -0
- threadfin/maptune.py +743 -0
- threadfin/metrics.py +137 -0
- threadfin/migration.py +165 -0
- threadfin/phylo.py +834 -0
- threadfin/plotting.py +535 -0
- threadfin/pp.py +253 -0
- threadfin/profiles.py +603 -0
- threadfin/programmes.py +1028 -0
- threadfin/programs.py +133 -0
- threadfin/reclustering.py +68 -0
- threadfin/sequence.py +379 -0
- threadfin/simulate.py +206 -0
- threadfin/specificity.py +355 -0
- threadfin/stats.py +476 -0
- threadfin/tl.py +62 -0
- threadfin-4.5.0.dist-info/METADATA +288 -0
- threadfin-4.5.0.dist-info/RECORD +34 -0
- threadfin-4.5.0.dist-info/WHEEL +5 -0
- threadfin-4.5.0.dist-info/licenses/LICENSE +21 -0
- threadfin-4.5.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"""Gene-level clonal ICC: which genes vary consistently with clone identity?
|
|
2
|
+
|
|
3
|
+
For every gene, the same one-way random-effects model as
|
|
4
|
+
:mod:`threadfin.profiles` is fitted to its context-centred log expression:
|
|
5
|
+
the gene's clonal ICC is the fraction of its (within-context) variance that
|
|
6
|
+
is explained by clone identity. Genes that define a stable, inherited fate
|
|
7
|
+
(for example a plasma-cell commitment programme) are expected to have high
|
|
8
|
+
ICC; this is a hypothesis, not evidence of inherited fate. Genes that every clone cycles through (for example dark-zone / light-zone
|
|
9
|
+
germinal-centre programmes, or the cell cycle) are expected to have low ICC
|
|
10
|
+
even when they are highly variable. Significance comes from shuffling clone
|
|
11
|
+
labels among cells within strata, exactly as in
|
|
12
|
+
:func:`threadfin.tl.clonal_coherence`.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
import pandas as pd
|
|
19
|
+
import scipy.sparse as sp
|
|
20
|
+
|
|
21
|
+
from ._utils import bh_fdr, codes, get_uns, h5ad_safe, log, permute_within, require_obs, require_complete_groups, require_positive_int
|
|
22
|
+
from .profiles import variance_components
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _dense_chunk(x, cols):
|
|
26
|
+
sub = x[:, cols]
|
|
27
|
+
return sub.toarray() if sp.issparse(sub) else np.asarray(sub)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def gene_heritability(
|
|
31
|
+
adata,
|
|
32
|
+
*,
|
|
33
|
+
clone_key: str | None = None,
|
|
34
|
+
context_key: str | None = "auto",
|
|
35
|
+
strata_key: str | None = None,
|
|
36
|
+
layer: str | None = "log_norm",
|
|
37
|
+
genes=None,
|
|
38
|
+
min_frac_expressed: float = 0.05,
|
|
39
|
+
exclude_receptor_genes: bool = True,
|
|
40
|
+
n_perm: int = 100,
|
|
41
|
+
chunk_size: int = 1000,
|
|
42
|
+
random_state: int = 0,
|
|
43
|
+
verbose: bool = True,
|
|
44
|
+
) -> pd.DataFrame:
|
|
45
|
+
"""Clonal ICC of every gene, with a within-stratum permutation p-value.
|
|
46
|
+
|
|
47
|
+
Parameters
|
|
48
|
+
----------
|
|
49
|
+
clone_key, context_key
|
|
50
|
+
Default to the values used by :func:`threadfin.tl.clone_profiles`
|
|
51
|
+
(``context_key="auto"``); pass ``None`` to disable context centring.
|
|
52
|
+
strata_key
|
|
53
|
+
Permutation strata (defaults to the context).
|
|
54
|
+
layer
|
|
55
|
+
Log-normalised expression layer (``None`` uses ``adata.X``).
|
|
56
|
+
genes
|
|
57
|
+
Genes to test (default: genes expressed in at least
|
|
58
|
+
``min_frac_expressed`` of the analysed cells).
|
|
59
|
+
exclude_receptor_genes
|
|
60
|
+
Leave immunoglobulin/TCR genes out of the scan. They encode the
|
|
61
|
+
receptor itself, so they are clonal by definition; their median ICC is
|
|
62
|
+
still reported (``uns[...]['receptor_gene_icc']``) as a positive
|
|
63
|
+
control for the statistic.
|
|
64
|
+
n_perm
|
|
65
|
+
Permutations (shared across genes, so the cost is ``n_perm`` matrix
|
|
66
|
+
products per chunk of genes).
|
|
67
|
+
|
|
68
|
+
Returns
|
|
69
|
+
-------
|
|
70
|
+
DataFrame indexed by gene: ``icc``, ``tau2``, ``sigma2``, ``mean``,
|
|
71
|
+
``frac_expressed``, ``null_mean``, ``pvalue``, ``fdr`` and ``excess_icc``
|
|
72
|
+
(``icc - null_mean``, the effect size), sorted by ``excess_icc``. In large
|
|
73
|
+
datasets nearly every gene passes FDR < 0.05; interpret ``excess_icc``.
|
|
74
|
+
Also stored in ``adata.uns['threadfin']['gene_heritability']``.
|
|
75
|
+
"""
|
|
76
|
+
require_positive_int(n_perm, "n_perm")
|
|
77
|
+
require_positive_int(chunk_size, "chunk_size")
|
|
78
|
+
if not 0 <= min_frac_expressed <= 1:
|
|
79
|
+
raise ValueError("min_frac_expressed must be between 0 and 1.")
|
|
80
|
+
prof = get_uns(adata).get("profiles", {})
|
|
81
|
+
params = prof.get("params", {})
|
|
82
|
+
clone_key = clone_key or params.get("clone_key", "clone_id")
|
|
83
|
+
if context_key == "auto":
|
|
84
|
+
context_key = params.get("context_key")
|
|
85
|
+
strata_key = strata_key or context_key
|
|
86
|
+
require_obs(adata, clone_key, context_key, strata_key, context="gene heritability")
|
|
87
|
+
|
|
88
|
+
require_complete_groups(adata, context_key, strata_key, context="gene heritability")
|
|
89
|
+
x = adata.layers[layer] if layer is not None else adata.X
|
|
90
|
+
if sp.issparse(x):
|
|
91
|
+
x = x.tocsc()
|
|
92
|
+
clone_codes, _ = codes(adata.obs[clone_key])
|
|
93
|
+
if not (clone_codes >= 0).any():
|
|
94
|
+
raise ValueError("Too few cells in clones with >= 2 cells.")
|
|
95
|
+
sizes = np.bincount(clone_codes[clone_codes >= 0])
|
|
96
|
+
in_multi = (clone_codes >= 0) & (sizes[np.maximum(clone_codes, 0)] >= 2)
|
|
97
|
+
cells = np.flatnonzero(in_multi)
|
|
98
|
+
if cells.size < 20:
|
|
99
|
+
raise ValueError("Too few cells in clones with >= 2 cells.")
|
|
100
|
+
|
|
101
|
+
# context means come from all cells of each context (like the profile model)
|
|
102
|
+
if context_key is not None:
|
|
103
|
+
ctx, ctx_index = codes(adata.obs[context_key])
|
|
104
|
+
else:
|
|
105
|
+
ctx, ctx_index = np.zeros(adata.n_obs, dtype=np.int64), pd.Index(["all"])
|
|
106
|
+
ctx_member = sp.csr_matrix(
|
|
107
|
+
(np.ones((ctx >= 0).sum()), (ctx[ctx >= 0], np.flatnonzero(ctx >= 0))),
|
|
108
|
+
shape=(len(ctx_index), adata.n_obs),
|
|
109
|
+
)
|
|
110
|
+
ctx_n = np.asarray(ctx_member.sum(axis=1)).ravel()
|
|
111
|
+
|
|
112
|
+
var_names = pd.Index(adata.var_names)
|
|
113
|
+
if genes is None:
|
|
114
|
+
sub = x[cells] if not sp.issparse(x) else x.tocsr()[cells]
|
|
115
|
+
nnz = np.asarray((sub > 0).sum(axis=0)).ravel() if sp.issparse(sub) else (sub > 0).sum(axis=0)
|
|
116
|
+
gene_idx = np.flatnonzero(nnz / cells.size >= min_frac_expressed)
|
|
117
|
+
else:
|
|
118
|
+
gene_idx = var_names.get_indexer(pd.Index(genes))
|
|
119
|
+
if (gene_idx < 0).any():
|
|
120
|
+
missing = list(pd.Index(genes)[gene_idx < 0][:5])
|
|
121
|
+
raise KeyError(f"genes not in var_names, e.g. {missing}")
|
|
122
|
+
|
|
123
|
+
from .pp import ig_gene_mask
|
|
124
|
+
|
|
125
|
+
receptor = ig_gene_mask(var_names[gene_idx], constant=True, tr_genes=True)
|
|
126
|
+
control_idx = gene_idx[receptor]
|
|
127
|
+
if exclude_receptor_genes:
|
|
128
|
+
gene_idx = gene_idx[~receptor]
|
|
129
|
+
|
|
130
|
+
strata = codes(adata.obs[strata_key])[0][cells] if strata_key is not None else None
|
|
131
|
+
cl = clone_codes[cells]
|
|
132
|
+
rng = np.random.default_rng(random_state)
|
|
133
|
+
perms = [permute_within(cl, strata, rng) for _ in range(n_perm)]
|
|
134
|
+
|
|
135
|
+
rows = []
|
|
136
|
+
for start in range(0, gene_idx.size, chunk_size):
|
|
137
|
+
cols = gene_idx[start:start + chunk_size]
|
|
138
|
+
full = _dense_chunk(x, cols)
|
|
139
|
+
mu = np.asarray(ctx_member @ full) / np.maximum(ctx_n, 1)[:, None]
|
|
140
|
+
resid = full[cells] - mu[np.maximum(ctx[cells], 0)]
|
|
141
|
+
vc = variance_components(resid, cl)
|
|
142
|
+
obs_icc = vc.icc_per_feature
|
|
143
|
+
exceed = np.zeros(cols.size)
|
|
144
|
+
null_sum = np.zeros(cols.size)
|
|
145
|
+
for perm in perms:
|
|
146
|
+
null_icc = variance_components(resid, perm).icc_per_feature
|
|
147
|
+
exceed += null_icc >= obs_icc - 1e-12
|
|
148
|
+
null_sum += null_icc
|
|
149
|
+
sub = full[cells]
|
|
150
|
+
rows.append(pd.DataFrame({
|
|
151
|
+
"icc": obs_icc, "tau2": vc.tau2, "sigma2": vc.sigma2,
|
|
152
|
+
"mean": sub.mean(axis=0), "frac_expressed": (sub > 0).mean(axis=0),
|
|
153
|
+
"null_mean": null_sum / max(n_perm, 1),
|
|
154
|
+
"pvalue": (1 + exceed) / (n_perm + 1),
|
|
155
|
+
}, index=var_names[cols]))
|
|
156
|
+
if not rows:
|
|
157
|
+
raise ValueError("No genes remain after expression and receptor filtering.")
|
|
158
|
+
out = pd.concat(rows)
|
|
159
|
+
out["fdr"] = bh_fdr(out["pvalue"].to_numpy())
|
|
160
|
+
# effect size: clonal ICC beyond what sample structure alone produces; with tens of
|
|
161
|
+
# thousands of cells almost every gene is "significant", so rank by this instead
|
|
162
|
+
out["excess_icc"] = out["icc"] - out["null_mean"]
|
|
163
|
+
control_icc = float("nan")
|
|
164
|
+
if exclude_receptor_genes and control_idx.size:
|
|
165
|
+
full = _dense_chunk(x, control_idx[:200])
|
|
166
|
+
mu = np.asarray(ctx_member @ full) / np.maximum(ctx_n, 1)[:, None]
|
|
167
|
+
control_icc = float(np.median(variance_components(full[cells] - mu[np.maximum(ctx[cells], 0)], cl)
|
|
168
|
+
.icc_per_feature))
|
|
169
|
+
out = out.sort_values("excess_icc", ascending=False)
|
|
170
|
+
out.index.name = "gene"
|
|
171
|
+
get_uns(adata)["gene_heritability"] = h5ad_safe(out)
|
|
172
|
+
get_uns(adata)["gene_heritability_receptor_icc"] = control_icc
|
|
173
|
+
log(
|
|
174
|
+
f"gene heritability: {out.shape[0]} genes, {cells.size} cells in "
|
|
175
|
+
f"{int((sizes >= 2).sum())} clones; {int((out['fdr'] < 0.05).sum())} genes clonally "
|
|
176
|
+
f"inherited at FDR < 0.05 (median ICC {out['icc'].median():.3f}; receptor-gene positive "
|
|
177
|
+
f"control {control_icc:.2f}).",
|
|
178
|
+
verbose,
|
|
179
|
+
)
|
|
180
|
+
return out
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def geneset_heritability(table: pd.DataFrame, gene_sets: dict[str, list[str]], *, n_bins: int = 10,
|
|
184
|
+
seed: int = 0) -> pd.DataFrame:
|
|
185
|
+
"""Summarise :func:`gene_heritability` over named gene sets.
|
|
186
|
+
|
|
187
|
+
Highly expressed genes are measured with less noise and therefore have
|
|
188
|
+
higher ICCs, so each set is compared with an **expression-matched**
|
|
189
|
+
background: other tested genes drawn from the same mean-expression deciles
|
|
190
|
+
as the set's genes (up to 20 background genes per set gene). Reports the
|
|
191
|
+
set's median ICC, the matched background median, the fraction of set genes
|
|
192
|
+
significant at FDR < 0.05 and a Mann-Whitney p-value (set vs matched
|
|
193
|
+
background).
|
|
194
|
+
"""
|
|
195
|
+
from scipy.stats import mannwhitneyu
|
|
196
|
+
|
|
197
|
+
rng = np.random.default_rng(seed)
|
|
198
|
+
bins = pd.Series(pd.qcut(table["mean"].rank(method="first"), n_bins, labels=False), index=table.index)
|
|
199
|
+
rows = []
|
|
200
|
+
for name, genes in gene_sets.items():
|
|
201
|
+
present = [g for g in genes if g in table.index]
|
|
202
|
+
if len(present) < 3:
|
|
203
|
+
continue
|
|
204
|
+
others = table.index.difference(present)
|
|
205
|
+
background = []
|
|
206
|
+
for b, k in bins.loc[present].value_counts().items():
|
|
207
|
+
pool = others[bins.loc[others].to_numpy() == b]
|
|
208
|
+
if pool.size:
|
|
209
|
+
background.extend(rng.choice(pool, size=min(pool.size, 20 * k), replace=False))
|
|
210
|
+
col = "excess_icc" if "excess_icc" in table.columns else "icc"
|
|
211
|
+
inside = table.loc[present, col].to_numpy()
|
|
212
|
+
matched = table.loc[background, col].to_numpy()
|
|
213
|
+
p = mannwhitneyu(inside, matched, alternative="two-sided").pvalue if matched.size else np.nan
|
|
214
|
+
rows.append({
|
|
215
|
+
"gene_set": name, "n_genes": len(present), "statistic": col,
|
|
216
|
+
"median_icc": float(np.median(inside)),
|
|
217
|
+
"matched_background_median_icc": float(np.median(matched)) if matched.size else np.nan,
|
|
218
|
+
"frac_significant": float((table.loc[present, "fdr"] < 0.05).mean()),
|
|
219
|
+
"pvalue_vs_matched": float(p), "genes": ",".join(present),
|
|
220
|
+
})
|
|
221
|
+
out = pd.DataFrame(rows)
|
|
222
|
+
if not out.empty:
|
|
223
|
+
out["fdr"] = bh_fdr(out["pvalue_vs_matched"].to_numpy())
|
|
224
|
+
return out
|
threadfin/integrate.py
ADDED
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
"""Coupled GEX+BCR graph integration: joint clone embedding and diagnostics.
|
|
2
|
+
|
|
3
|
+
Implements the "Coupled Laplacian" principle of docs/DESIGN_V2.md: instead of
|
|
4
|
+
Benisse's ADMM on dense matrices, the coupled clone graph
|
|
5
|
+
``W = (1 - lam) * W_gex + lam * W_bcr`` (each row-normalized) is embedded
|
|
6
|
+
directly with row-normalized Laplacian eigenvectors. These coordinates are
|
|
7
|
+
not commute-time distances. Graph storage is sparse; neighbour search and
|
|
8
|
+
the small-graph eigensolver fallback have separate computational costs.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import warnings
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pandas as pd
|
|
17
|
+
import scipy.sparse as sp
|
|
18
|
+
|
|
19
|
+
from .core import _CLONE_INFO_KEY, _CLONE_MAP_KEY, clone_centroids
|
|
20
|
+
|
|
21
|
+
_VJ_COLUMNS = ("v_call", "j_call", "cdr3")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _adaptive_gaussian_knn(coords: np.ndarray, n_neighbors: int) -> sp.csr_matrix:
|
|
25
|
+
"""Sparse symmetric kNN graph with the self-scaled Gaussian kernel used by
|
|
26
|
+
``core._leiden_on_distances`` (sigma = distance to the k-th neighbor)."""
|
|
27
|
+
from sklearn.neighbors import NearestNeighbors
|
|
28
|
+
|
|
29
|
+
n = coords.shape[0]
|
|
30
|
+
k = int(min(n_neighbors, n - 1))
|
|
31
|
+
if k < 1:
|
|
32
|
+
raise ValueError("Need at least 2 clones to build the GEX graph.")
|
|
33
|
+
nn = NearestNeighbors(n_neighbors=k + 1).fit(coords)
|
|
34
|
+
d_knn, idx = nn.kneighbors(coords)
|
|
35
|
+
d_knn, idx = d_knn[:, 1:], idx[:, 1:] # drop self
|
|
36
|
+
sigma = np.maximum(d_knn[:, -1:], 1e-10)
|
|
37
|
+
w = np.exp(-(d_knn**2) / (sigma**2))
|
|
38
|
+
rows = np.repeat(np.arange(n), k)
|
|
39
|
+
graph = sp.csr_matrix((w.ravel(), (rows, idx.ravel())), shape=(n, n))
|
|
40
|
+
return graph.maximum(graph.T)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _row_normalize(graph: sp.csr_matrix) -> sp.csr_matrix:
|
|
44
|
+
row_sum = np.asarray(graph.sum(axis=1)).ravel()
|
|
45
|
+
inv = np.divide(1.0, row_sum, out=np.zeros_like(row_sum), where=row_sum > 0)
|
|
46
|
+
return sp.diags(inv) @ graph
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _largest_component(graph: sp.csr_matrix) -> np.ndarray:
|
|
50
|
+
n_comp, labels = sp.csgraph.connected_components(graph, directed=False)
|
|
51
|
+
if n_comp == 1:
|
|
52
|
+
return np.arange(graph.shape[0])
|
|
53
|
+
sizes = np.bincount(labels)
|
|
54
|
+
keep = labels == sizes.argmax()
|
|
55
|
+
dropped = graph.shape[0] - int(keep.sum())
|
|
56
|
+
if dropped / graph.shape[0] > 0.1:
|
|
57
|
+
warnings.warn(
|
|
58
|
+
f"Largest connected component drops {dropped}/{graph.shape[0]} "
|
|
59
|
+
f"clones ({dropped / graph.shape[0]:.0%}); consider a lower lam, "
|
|
60
|
+
"more neighbors, or a denser BCR graph.",
|
|
61
|
+
stacklevel=2,
|
|
62
|
+
)
|
|
63
|
+
return np.flatnonzero(keep)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _spectral_embedding(
|
|
67
|
+
graph: sp.csr_matrix, n_components: int, random_state: int
|
|
68
|
+
) -> np.ndarray:
|
|
69
|
+
"""Normalized-Laplacian eigenmap: bottom nontrivial eigenvectors of
|
|
70
|
+
``L = I - D^-1/2 W D^-1/2``, rows renormalized to unit norm."""
|
|
71
|
+
n = graph.shape[0]
|
|
72
|
+
deg = np.asarray(graph.sum(axis=1)).ravel()
|
|
73
|
+
d_inv_sqrt = np.divide(1.0, np.sqrt(deg), out=np.zeros_like(deg), where=deg > 0)
|
|
74
|
+
norm = sp.diags(d_inv_sqrt)
|
|
75
|
+
lap = sp.identity(n) - norm @ graph @ norm
|
|
76
|
+
|
|
77
|
+
k = int(min(n_components + 1, n - 1))
|
|
78
|
+
if k < 2:
|
|
79
|
+
raise ValueError("Need at least 3 connected clones for an embedding.")
|
|
80
|
+
if n <= k + 2:
|
|
81
|
+
# dense fallback when too few clones for ARPACK
|
|
82
|
+
vals, vecs = np.linalg.eigh(lap.toarray())
|
|
83
|
+
order = np.argsort(vals)
|
|
84
|
+
vals, vecs = vals[order], vecs[:, order]
|
|
85
|
+
vals, vecs = vals[:k], vecs[:, :k]
|
|
86
|
+
else:
|
|
87
|
+
# L is PSD, so "SA" selects the same bottom eigenpairs "SM" would,
|
|
88
|
+
# but ARPACK handles smallest-algebraic far more robustly.
|
|
89
|
+
v0 = np.random.default_rng(random_state).standard_normal(n)
|
|
90
|
+
try:
|
|
91
|
+
vals, vecs = sp.linalg.eigsh(lap, k=k, which="SA", v0=v0, tol=1e-10)
|
|
92
|
+
except Exception:
|
|
93
|
+
vals, vecs = np.linalg.eigh(lap.toarray())
|
|
94
|
+
order = np.argsort(vals)
|
|
95
|
+
vals, vecs = vals[order], vecs[:, order]
|
|
96
|
+
order = np.argsort(vals)
|
|
97
|
+
vals, vecs = vals[order], vecs[:, order]
|
|
98
|
+
vals, vecs = vals[:k], vecs[:, :k]
|
|
99
|
+
|
|
100
|
+
emb = vecs[:, 1:] # drop the trivial constant eigenvector
|
|
101
|
+
norms = np.linalg.norm(emb, axis=1, keepdims=True)
|
|
102
|
+
return np.divide(emb, norms, out=np.zeros_like(emb), where=norms > 0)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _get_bcr_graph(adata, clone_ids, bcr_graph, cdr3_sim_threshold):
|
|
106
|
+
"""Return W_bcr as a CSR matrix aligned to ``clone_ids`` order."""
|
|
107
|
+
clone_info = adata.uns.get(_CLONE_INFO_KEY)
|
|
108
|
+
n = len(clone_ids)
|
|
109
|
+
if bcr_graph is not None:
|
|
110
|
+
graph = sp.csr_matrix(bcr_graph)
|
|
111
|
+
if clone_info is not None and graph.shape[0] == clone_info.shape[0]:
|
|
112
|
+
pos = clone_info.index.get_indexer(clone_ids)
|
|
113
|
+
if (pos < 0).any():
|
|
114
|
+
raise ValueError(
|
|
115
|
+
"adata.uns['threadfin_clones'] does not cover all centroid "
|
|
116
|
+
"clones; cannot align the supplied bcr_graph."
|
|
117
|
+
)
|
|
118
|
+
return graph[pos][:, pos]
|
|
119
|
+
if graph.shape[0] != n:
|
|
120
|
+
raise ValueError(
|
|
121
|
+
f"bcr_graph has {graph.shape[0]} rows but there are {n} "
|
|
122
|
+
"clones; supply a graph aligned with "
|
|
123
|
+
"adata.uns['threadfin_clones'] or with the centroid clones."
|
|
124
|
+
)
|
|
125
|
+
return graph
|
|
126
|
+
|
|
127
|
+
if clone_info is None:
|
|
128
|
+
raise KeyError(
|
|
129
|
+
"bcr_graph=None requires clone-level BCR info in "
|
|
130
|
+
"adata.uns['threadfin_clones']. Run threadfin.attach_bcr() first, "
|
|
131
|
+
"or pass a precomputed bcr_graph."
|
|
132
|
+
)
|
|
133
|
+
missing = [c for c in _VJ_COLUMNS if c not in clone_info.columns]
|
|
134
|
+
if missing:
|
|
135
|
+
raise KeyError(
|
|
136
|
+
f"adata.uns['threadfin_clones'] is missing required columns "
|
|
137
|
+
f"{missing} (need v_call, j_call, cdr3)."
|
|
138
|
+
)
|
|
139
|
+
try:
|
|
140
|
+
from .bcrgraph import bcr_similarity_graph
|
|
141
|
+
except ImportError as e:
|
|
142
|
+
raise ImportError(
|
|
143
|
+
"bcr_graph=None requires the threadfin.bcrgraph module, which is "
|
|
144
|
+
"not available in this install. Pass a precomputed bcr_graph "
|
|
145
|
+
"(scipy sparse matrix aligned with adata.uns['threadfin_clones'])."
|
|
146
|
+
) from e
|
|
147
|
+
shared = clone_ids.intersection(clone_info.index, sort=False)
|
|
148
|
+
if len(shared) == 0:
|
|
149
|
+
raise ValueError(
|
|
150
|
+
"No overlap between centroid clone ids and "
|
|
151
|
+
"adata.uns['threadfin_clones'] index."
|
|
152
|
+
)
|
|
153
|
+
graph = sp.csr_matrix(
|
|
154
|
+
bcr_similarity_graph(
|
|
155
|
+
clone_info.loc[shared], cdr3_sim_threshold=cdr3_sim_threshold
|
|
156
|
+
)
|
|
157
|
+
).tocoo()
|
|
158
|
+
if len(shared) == n:
|
|
159
|
+
return sp.csr_matrix(graph)
|
|
160
|
+
pos = clone_ids.get_indexer(shared)
|
|
161
|
+
full = sp.csr_matrix(
|
|
162
|
+
(graph.data, (pos[graph.row], pos[graph.col])), shape=(n, n)
|
|
163
|
+
)
|
|
164
|
+
return full
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def joint_embedding(
|
|
168
|
+
adata,
|
|
169
|
+
*,
|
|
170
|
+
clone_key: str = "clone_id",
|
|
171
|
+
basis: str = "X_pca",
|
|
172
|
+
min_clone_size: int = 3,
|
|
173
|
+
n_neighbors: int = 20,
|
|
174
|
+
lam: float = 0.5,
|
|
175
|
+
n_components: int = 20,
|
|
176
|
+
bcr_graph=None,
|
|
177
|
+
cdr3_sim_threshold: float = 0.85,
|
|
178
|
+
random_state: int = 0,
|
|
179
|
+
) -> pd.DataFrame:
|
|
180
|
+
"""Coupled-Laplacian eigenmap embedding of clones.
|
|
181
|
+
|
|
182
|
+
Builds an adaptive-Gaussian kNN graph over GEX centroids and a BCR
|
|
183
|
+
sequence-similarity graph, couples them as
|
|
184
|
+
``W = (1 - lam) * W_gex + lam * W_bcr`` (each row-normalized), and embeds
|
|
185
|
+
clones with the bottom nontrivial eigenvectors of the symmetric normalized
|
|
186
|
+
Laplacian.
|
|
187
|
+
|
|
188
|
+
Returns a DataFrame indexed by clone id with columns
|
|
189
|
+
``joint_0..joint_{m-1}`` and ``n_cells``; also stored in
|
|
190
|
+
``adata.uns['threadfin']['joint']`` with the coupled graph alongside.
|
|
191
|
+
"""
|
|
192
|
+
if not 0 <= lam <= 1:
|
|
193
|
+
raise ValueError("lam must be in [0, 1].")
|
|
194
|
+
|
|
195
|
+
centroids = clone_centroids(
|
|
196
|
+
adata, clone_key=clone_key, basis=basis, min_clone_size=min_clone_size
|
|
197
|
+
)
|
|
198
|
+
coord_cols = [c for c in centroids.columns if c != "n_cells"]
|
|
199
|
+
clone_ids = centroids.index
|
|
200
|
+
|
|
201
|
+
w_bcr = sp.csr_matrix(
|
|
202
|
+
_get_bcr_graph(adata, clone_ids, bcr_graph, cdr3_sim_threshold)
|
|
203
|
+
)
|
|
204
|
+
w_gex = _adaptive_gaussian_knn(centroids[coord_cols].to_numpy(), n_neighbors)
|
|
205
|
+
|
|
206
|
+
w_gex = _row_normalize(w_gex)
|
|
207
|
+
w_bcr = _row_normalize(w_bcr)
|
|
208
|
+
coupled = (1.0 - lam) * w_gex + lam * w_bcr
|
|
209
|
+
coupled = coupled.maximum(coupled.T).tocsr()
|
|
210
|
+
|
|
211
|
+
keep = _largest_component(coupled)
|
|
212
|
+
if len(keep) < coupled.shape[0]:
|
|
213
|
+
coupled = coupled[keep][:, keep]
|
|
214
|
+
w_bcr = w_bcr[keep][:, keep]
|
|
215
|
+
centroids = centroids.iloc[keep]
|
|
216
|
+
|
|
217
|
+
emb = _spectral_embedding(coupled, n_components, random_state)
|
|
218
|
+
joint = pd.DataFrame(
|
|
219
|
+
emb,
|
|
220
|
+
index=centroids.index,
|
|
221
|
+
columns=[f"joint_{i}" for i in range(emb.shape[1])],
|
|
222
|
+
)
|
|
223
|
+
joint["n_cells"] = centroids["n_cells"]
|
|
224
|
+
|
|
225
|
+
uns = adata.uns.setdefault(_CLONE_MAP_KEY, {})
|
|
226
|
+
uns["joint"] = joint
|
|
227
|
+
uns["joint_graph"] = coupled
|
|
228
|
+
uns["joint_graph_bcr"] = w_bcr
|
|
229
|
+
uns["joint_params"] = {
|
|
230
|
+
"clone_key": clone_key,
|
|
231
|
+
"basis": basis,
|
|
232
|
+
"min_clone_size": min_clone_size,
|
|
233
|
+
"n_neighbors": n_neighbors,
|
|
234
|
+
"lam": lam,
|
|
235
|
+
"n_components": int(emb.shape[1]),
|
|
236
|
+
"cdr3_sim_threshold": cdr3_sim_threshold,
|
|
237
|
+
"random_state": random_state,
|
|
238
|
+
"n_clones": int(joint.shape[0]),
|
|
239
|
+
}
|
|
240
|
+
print(
|
|
241
|
+
f"[threadfin] joint embedding: {joint.shape[0]} clones x "
|
|
242
|
+
f"{emb.shape[1]} components (lam={lam}) stored in "
|
|
243
|
+
"uns['threadfin']['joint']."
|
|
244
|
+
)
|
|
245
|
+
return joint
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _pairwise_rows(mat: np.ndarray, i: np.ndarray, j: np.ndarray) -> np.ndarray:
|
|
249
|
+
return np.linalg.norm(mat[i] - mat[j], axis=1)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _safe_abs_corr(a: np.ndarray, b: np.ndarray) -> float:
|
|
253
|
+
if a.std() == 0 or b.std() == 0:
|
|
254
|
+
return 0.0
|
|
255
|
+
return float(abs(np.corrcoef(a, b)[0, 1]))
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def integration_diagnostics(
|
|
259
|
+
adata,
|
|
260
|
+
joint: pd.DataFrame | None = None,
|
|
261
|
+
*,
|
|
262
|
+
basis: str = "X_pca",
|
|
263
|
+
clone_key: str = "clone_id",
|
|
264
|
+
) -> dict:
|
|
265
|
+
"""Benisse testCor analogues and a heuristic modality-contribution split.
|
|
266
|
+
|
|
267
|
+
On the edges of the coupled graph: Spearman correlation between latent
|
|
268
|
+
distances (joint embedding) and (i) GEX centroid distances
|
|
269
|
+
(``testcor_gex``), (ii) BCR dissimilarity ``1 - w_bcr`` over BCR edges
|
|
270
|
+
only (``testcor_bcr``). ``modality_contribution`` is a simple heuristic:
|
|
271
|
+
per joint dimension, the max absolute correlation with the top GEX
|
|
272
|
+
centroid PCs versus the absolute correlation with each clone's mean
|
|
273
|
+
BCR-edge-weight profile; the two means are normalized to sum to 1.
|
|
274
|
+
"""
|
|
275
|
+
from scipy.stats import spearmanr
|
|
276
|
+
|
|
277
|
+
uns = adata.uns.get(_CLONE_MAP_KEY, {})
|
|
278
|
+
if joint is None:
|
|
279
|
+
joint = uns.get("joint")
|
|
280
|
+
if joint is None:
|
|
281
|
+
raise ValueError("Run threadfin.integrate.joint_embedding() first.")
|
|
282
|
+
coupled = uns.get("joint_graph")
|
|
283
|
+
w_bcr = uns.get("joint_graph_bcr")
|
|
284
|
+
if coupled is None or coupled.shape[0] != joint.shape[0]:
|
|
285
|
+
raise ValueError(
|
|
286
|
+
"Coupled graph not found in uns['threadfin']; run "
|
|
287
|
+
"threadfin.integrate.joint_embedding() first."
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
triu = sp.triu(coupled, k=1).tocoo()
|
|
291
|
+
i, j = triu.row, triu.col
|
|
292
|
+
n_edges = len(i)
|
|
293
|
+
|
|
294
|
+
joint_cols = [c for c in joint.columns if c != "n_cells"]
|
|
295
|
+
emb = joint[joint_cols].to_numpy()
|
|
296
|
+
latent_d = _pairwise_rows(emb, i, j)
|
|
297
|
+
|
|
298
|
+
centroids = clone_centroids(
|
|
299
|
+
adata, clone_key=clone_key, basis=basis, min_clone_size=1
|
|
300
|
+
).reindex(joint.index)
|
|
301
|
+
coord_cols = [c for c in centroids.columns if c != "n_cells"]
|
|
302
|
+
gex = centroids[coord_cols].to_numpy()
|
|
303
|
+
gex_d = _pairwise_rows(gex, i, j)
|
|
304
|
+
r_gex = float(spearmanr(latent_d, gex_d).statistic)
|
|
305
|
+
|
|
306
|
+
if w_bcr is not None:
|
|
307
|
+
bcr_triu = sp.triu(w_bcr, k=1).tocsr()
|
|
308
|
+
bcr_w = np.asarray(bcr_triu[i, j]).ravel()
|
|
309
|
+
mask = bcr_w > 0
|
|
310
|
+
if mask.sum() >= 3:
|
|
311
|
+
r_bcr = float(
|
|
312
|
+
spearmanr(latent_d[mask], 1.0 - bcr_w[mask]).statistic
|
|
313
|
+
)
|
|
314
|
+
else:
|
|
315
|
+
r_bcr = np.nan
|
|
316
|
+
else:
|
|
317
|
+
bcr_w = np.zeros(n_edges)
|
|
318
|
+
r_bcr = np.nan
|
|
319
|
+
|
|
320
|
+
# modality contribution (heuristic; see docstring)
|
|
321
|
+
n_pcs = int(min(5, gex.shape[0] - 1, gex.shape[1]))
|
|
322
|
+
xc = gex - gex.mean(axis=0)
|
|
323
|
+
vt = np.linalg.svd(xc, full_matrices=False)[2]
|
|
324
|
+
pcs = xc @ vt[:n_pcs].T
|
|
325
|
+
bcr_profile = (
|
|
326
|
+
np.asarray(w_bcr.mean(axis=1)).ravel()
|
|
327
|
+
if w_bcr is not None
|
|
328
|
+
else np.zeros(joint.shape[0])
|
|
329
|
+
)
|
|
330
|
+
gex_scores, bcr_scores = [], []
|
|
331
|
+
for col in range(emb.shape[1]):
|
|
332
|
+
v = emb[:, col]
|
|
333
|
+
gex_scores.append(max(_safe_abs_corr(v, pcs[:, p]) for p in range(n_pcs)))
|
|
334
|
+
bcr_scores.append(_safe_abs_corr(v, bcr_profile))
|
|
335
|
+
gex_total = float(np.mean(gex_scores))
|
|
336
|
+
bcr_total = float(np.mean(bcr_scores))
|
|
337
|
+
total = gex_total + bcr_total
|
|
338
|
+
if total > 0:
|
|
339
|
+
contribution = {"gex": gex_total / total, "bcr": bcr_total / total}
|
|
340
|
+
else:
|
|
341
|
+
contribution = {"gex": 0.5, "bcr": 0.5}
|
|
342
|
+
|
|
343
|
+
diagnostics = {
|
|
344
|
+
"testcor_gex": r_gex,
|
|
345
|
+
"testcor_bcr": r_bcr,
|
|
346
|
+
"n_edges": int(n_edges),
|
|
347
|
+
"modality_contribution": contribution,
|
|
348
|
+
}
|
|
349
|
+
adata.uns[_CLONE_MAP_KEY]["joint_diagnostics"] = diagnostics
|
|
350
|
+
return diagnostics
|