threadfin 4.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- threadfin/__init__.py +86 -0
- threadfin/_utils.py +260 -0
- threadfin/api.py +658 -0
- threadfin/bcrgraph.py +255 -0
- threadfin/clones.py +653 -0
- threadfin/core.py +531 -0
- threadfin/dandelion.py +458 -0
- threadfin/density.py +409 -0
- threadfin/dynamics.py +275 -0
- threadfin/export.py +215 -0
- threadfin/expression.py +121 -0
- threadfin/heritability.py +224 -0
- threadfin/integrate.py +350 -0
- threadfin/io.py +280 -0
- threadfin/maptune.py +743 -0
- threadfin/metrics.py +137 -0
- threadfin/migration.py +165 -0
- threadfin/phylo.py +834 -0
- threadfin/plotting.py +535 -0
- threadfin/pp.py +253 -0
- threadfin/profiles.py +603 -0
- threadfin/programmes.py +1028 -0
- threadfin/programs.py +133 -0
- threadfin/reclustering.py +68 -0
- threadfin/sequence.py +379 -0
- threadfin/simulate.py +206 -0
- threadfin/specificity.py +355 -0
- threadfin/stats.py +476 -0
- threadfin/tl.py +62 -0
- threadfin-4.5.0.dist-info/METADATA +288 -0
- threadfin-4.5.0.dist-info/RECORD +34 -0
- threadfin-4.5.0.dist-info/WHEEL +5 -0
- threadfin-4.5.0.dist-info/licenses/LICENSE +21 -0
- threadfin-4.5.0.dist-info/top_level.txt +1 -0
threadfin/phylo.py
ADDED
|
@@ -0,0 +1,834 @@
|
|
|
1
|
+
"""Lineage heritability: is a cell's state shared with its close relatives inside a B-cell clone?
|
|
2
|
+
|
|
3
|
+
Threadfin's clone profiles describe how clones differ from each other. This module asks the
|
|
4
|
+
complementary question inside clones: after removing everything a clone's cells share (the clone's
|
|
5
|
+
mean state in each sampling block), are cells that sit close together on the clone's somatic-
|
|
6
|
+
hypermutation lineage tree still more alike than distant relatives?
|
|
7
|
+
|
|
8
|
+
Model
|
|
9
|
+
-----
|
|
10
|
+
For one state feature ``y`` (a module score, a principal component, or a recorded cell label coded
|
|
11
|
+
0/1), cell ``i`` of clone ``c`` sampled in block ``b`` (mouse, sort gate, library...) is modelled as
|
|
12
|
+
|
|
13
|
+
y_i = mu_{c,b} + x_i' beta + g_i + e_i,
|
|
14
|
+
g ~ N(0, sigma_g^2 K), e ~ N(0, sigma_e^2 I),
|
|
15
|
+
|
|
16
|
+
where ``mu_{c,b}`` is a free intercept for every clone x block *unit* (so clone-wide and block-wide
|
|
17
|
+
state, however large, cannot contribute), ``x_i`` are cell covariates (sequencing depth, the cell's
|
|
18
|
+
mutation depth on the tree, sample indicators...), and ``K`` is a kinship kernel built from the tree:
|
|
19
|
+
|
|
20
|
+
* ``"exponential"`` (default): ``K_ij = exp(-d_ij / l)``, ``d_ij`` the number of mutations on the
|
|
21
|
+
tree path between the two cells' genotypes;
|
|
22
|
+
* ``"identical"``: ``K_ij = 1`` when the two cells carry the same receptor genotype;
|
|
23
|
+
* ``"identical_mutated"``: as ``"identical"``, but only for genotypes that carry at least one
|
|
24
|
+
mutation (cells still identical to the germline need not be recent relatives);
|
|
25
|
+
* ``"brownian"``: ``K_ij`` = mutations shared from the root to the two cells' common ancestor,
|
|
26
|
+
scaled by the mean root-to-tip depth (Brownian motion along the tree).
|
|
27
|
+
|
|
28
|
+
``h2 = sigma_g^2 / (sigma_g^2 + sigma_e^2)`` is the share of within-unit cell variation that is
|
|
29
|
+
structured by the lineage; for the exponential and identity kernels it is the correlation of two
|
|
30
|
+
cells with the same genotype after the unit mean is removed.
|
|
31
|
+
|
|
32
|
+
Why not a pairwise correlation of tree and state distances
|
|
33
|
+
----------------------------------------------------------
|
|
34
|
+
The distance between two cells on a tree is roughly the sum of their depths, so a pairwise
|
|
35
|
+
statistic mixes two different effects: a state that depends on how mutated a cell is (a fixed effect
|
|
36
|
+
of depth) and genuine resemblance of relatives (the random effect ``g``). Here depth is a covariate
|
|
37
|
+
and only ``g`` is tested.
|
|
38
|
+
|
|
39
|
+
Estimation and inference
|
|
40
|
+
------------------------
|
|
41
|
+
Every unit is projected onto the contrasts orthogonal to its intercept (Helmert basis ``P``) and
|
|
42
|
+
rotated into the eigenbasis of ``P' K P``. In that basis the covariance is diagonal,
|
|
43
|
+
``sigma_e^2 (1 + lambda d)``, so the restricted (REML) likelihood of ``lambda = h2 / (1 - h2)``,
|
|
44
|
+
pooled over thousands of small trees, is a sum over rows and is maximised by a 1-D search.
|
|
45
|
+
``h2`` comes with a profile-likelihood interval and an asymptotic boundary likelihood-ratio test
|
|
46
|
+
(``0.5 chi2_0 + 0.5 chi2_1``). **Both are likelihood-based and both have been measured to fail on real
|
|
47
|
+
repertoire data**: with per-unit heteroscedastic residuals the boundary LRT rejects 32-38% of the time at a
|
|
48
|
+
nominal 5%, and with heavy-tailed residuals 16-23%; the interval's coverage is 0.65-0.87 instead of 0.95.
|
|
49
|
+
Use ``p_lrt`` and ``h2_lo``/``h2_hi`` as rough diagnostics only, and quote the permutation p-value and a
|
|
50
|
+
bootstrap over clones (and over donors) for any reported interval. The primary test is the REML score
|
|
51
|
+
statistic
|
|
52
|
+
``Q = sum_i d_i r_i^2 / sum_i r_i^2`` of the null residuals ``r``, with a Freedman-Lane permutation
|
|
53
|
+
null: residuals of the null model are permuted among the cells of each unit, which keeps every
|
|
54
|
+
unit's tree, its covariates and its set of residual states, and destroys only which cell sits where.
|
|
55
|
+
|
|
56
|
+
:func:`lineage_power` plants a lineage component of known ``h2`` on the real trees, on top of a real
|
|
57
|
+
feature whose own lineage structure has first been destroyed by permutation, and reports bias, type I
|
|
58
|
+
error and power, so that a null result can be read as "``h2`` above x would have been detected".
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
from __future__ import annotations
|
|
62
|
+
|
|
63
|
+
from dataclasses import dataclass, field
|
|
64
|
+
|
|
65
|
+
import numpy as np
|
|
66
|
+
import pandas as pd
|
|
67
|
+
from scipy import stats
|
|
68
|
+
|
|
69
|
+
from ._utils import codes, log, require_positive_int
|
|
70
|
+
|
|
71
|
+
__all__ = [
|
|
72
|
+
"KERNELS",
|
|
73
|
+
"LineageForest",
|
|
74
|
+
"LineageHeritability",
|
|
75
|
+
"lineage_forest",
|
|
76
|
+
"lineage_heritability",
|
|
77
|
+
"lineage_power",
|
|
78
|
+
"simulate_lineage_trait",
|
|
79
|
+
# pairwise diagnostic, superseded by lineage_heritability (kept for reproducibility)
|
|
80
|
+
"LineageSignal",
|
|
81
|
+
"lineage_signal",
|
|
82
|
+
"planted_calibration",
|
|
83
|
+
"pair_table",
|
|
84
|
+
]
|
|
85
|
+
|
|
86
|
+
KERNELS = ("exponential", "identical", "identical_mutated", "brownian")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# =========================================================================== forest
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass
|
|
93
|
+
class LineageForest:
|
|
94
|
+
"""Cells placed on per-clone lineage trees, grouped into clone x block units.
|
|
95
|
+
|
|
96
|
+
Attributes
|
|
97
|
+
----------
|
|
98
|
+
n_cells
|
|
99
|
+
Number of rows of the data the forest indexes into.
|
|
100
|
+
units
|
|
101
|
+
One integer array of row indices per unit (all cells of one clone in one block that are
|
|
102
|
+
placed on the clone's tree); every unit has at least two cells.
|
|
103
|
+
dist
|
|
104
|
+
Per unit, the matrix of tree-path distances (mutations) between its cells.
|
|
105
|
+
shared
|
|
106
|
+
Per unit, the matrix of mutations shared from the root to each pair's common ancestor
|
|
107
|
+
(diagonal = the cell's depth).
|
|
108
|
+
depth
|
|
109
|
+
Per row, the root-to-tip depth of the cell's genotype (NaN for rows not in the forest).
|
|
110
|
+
unit_clone, unit_block
|
|
111
|
+
Labels of each unit.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
n_cells: int
|
|
115
|
+
units: list
|
|
116
|
+
dist: list
|
|
117
|
+
shared: list
|
|
118
|
+
depth: np.ndarray
|
|
119
|
+
unit_clone: list
|
|
120
|
+
unit_block: list
|
|
121
|
+
notes: dict = field(default_factory=dict)
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def n_units(self) -> int:
|
|
125
|
+
return len(self.units)
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def rows(self) -> np.ndarray:
|
|
129
|
+
"""All row indices that belong to a unit."""
|
|
130
|
+
return np.concatenate(self.units) if self.units else np.empty(0, dtype=np.int64)
|
|
131
|
+
|
|
132
|
+
def summary(self) -> dict:
|
|
133
|
+
sizes = np.array([u.size for u in self.units])
|
|
134
|
+
return {"units": int(self.n_units), "cells": int(sizes.sum()) if sizes.size else 0,
|
|
135
|
+
"clones": int(len(set(self.unit_clone))),
|
|
136
|
+
"median_unit_size": float(np.median(sizes)) if sizes.size else float("nan"),
|
|
137
|
+
"max_unit_size": int(sizes.max()) if sizes.size else 0,
|
|
138
|
+
"identical_pairs": int(sum(((d == 0).sum() - d.shape[0]) // 2 for d in self.dist)),
|
|
139
|
+
"pairs": int(sum(d.shape[0] * (d.shape[0] - 1) // 2 for d in self.dist)),
|
|
140
|
+
"mean_depth": float(np.nanmean(self.depth[self.rows])) if self.units else float("nan"),
|
|
141
|
+
**self.notes}
|
|
142
|
+
|
|
143
|
+
def subset(self, keep) -> "LineageForest":
|
|
144
|
+
"""The forest restricted to rows where ``keep`` is True (units falling below 2 cells drop)."""
|
|
145
|
+
keep = np.asarray(keep, dtype=bool)
|
|
146
|
+
units, dist, shared, uc, ub = [], [], [], [], []
|
|
147
|
+
for u, d, s, c, b in zip(self.units, self.dist, self.shared, self.unit_clone, self.unit_block):
|
|
148
|
+
m = keep[u]
|
|
149
|
+
if m.sum() >= 2:
|
|
150
|
+
units.append(u[m])
|
|
151
|
+
dist.append(d[np.ix_(m, m)])
|
|
152
|
+
shared.append(s[np.ix_(m, m)])
|
|
153
|
+
uc.append(c)
|
|
154
|
+
ub.append(b)
|
|
155
|
+
return LineageForest(self.n_cells, units, dist, shared, self.depth, uc, ub, dict(self.notes))
|
|
156
|
+
|
|
157
|
+
def kernel(self, kind: str = "exponential", length_scale: float = 2.0) -> list:
|
|
158
|
+
"""Per-unit kinship matrices for ``kind`` (see module docstring)."""
|
|
159
|
+
if kind not in KERNELS:
|
|
160
|
+
raise ValueError(f"kernel must be one of {KERNELS}, got {kind!r}.")
|
|
161
|
+
out = []
|
|
162
|
+
if kind == "brownian":
|
|
163
|
+
scale = float(np.nanmean(self.depth[self.rows])) if self.units else 1.0
|
|
164
|
+
scale = scale if scale > 0 else 1.0
|
|
165
|
+
for u, d, s in zip(self.units, self.dist, self.shared):
|
|
166
|
+
if kind == "exponential":
|
|
167
|
+
if not length_scale or length_scale <= 0:
|
|
168
|
+
raise ValueError("length_scale must be positive for the exponential kernel.")
|
|
169
|
+
k = np.exp(-d / float(length_scale))
|
|
170
|
+
elif kind == "identical":
|
|
171
|
+
k = (d == 0).astype(float)
|
|
172
|
+
elif kind == "identical_mutated":
|
|
173
|
+
mutated = self.depth[u] > 0
|
|
174
|
+
k = ((d == 0) & mutated[:, None] & mutated[None, :]).astype(float)
|
|
175
|
+
np.fill_diagonal(k, 1.0)
|
|
176
|
+
else:
|
|
177
|
+
k = s / scale
|
|
178
|
+
out.append(0.5 * (k + k.T))
|
|
179
|
+
return out
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def lineage_forest(clones, pairs, *, blocks=None, depth=None, min_cells: int = 2,
|
|
183
|
+
max_depth: float | None = None) -> LineageForest:
|
|
184
|
+
"""Group cells into clone x block units and collect their lineage-tree distances.
|
|
185
|
+
|
|
186
|
+
Parameters
|
|
187
|
+
----------
|
|
188
|
+
clones
|
|
189
|
+
Clone label per data row (length ``n_cells``).
|
|
190
|
+
pairs
|
|
191
|
+
Long table of within-clone cell pairs with integer row indices ``i``, ``j``, the tree-path
|
|
192
|
+
distance ``d`` (mutations between the two cells' genotypes) and, optionally, ``shared``
|
|
193
|
+
(mutations from the root to their common ancestor). Pairs across clones are ignored.
|
|
194
|
+
blocks
|
|
195
|
+
Sampling block per row. Units are clone x block. Default: one block, i.e. units are whole clones.
|
|
196
|
+
|
|
197
|
+
**Use the finest separately processed unit — the library or sort gate, not the animal.** A clone x
|
|
198
|
+
gate interaction (one clone behaving differently in two sorted gates of the same animal, which
|
|
199
|
+
physical sorting all but guarantees) is *not* removed by fitting the gate as a fixed effect. On real
|
|
200
|
+
germinal-centre trees that omitted interaction, with no lineage component present at all, produced
|
|
201
|
+
h2 = 0.09-0.16 and rejected 49-88% of the time. Blocking by animal when gates are nested inside
|
|
202
|
+
animals is therefore not a valid test of lineage heritability.
|
|
203
|
+
depth
|
|
204
|
+
Root-to-tip depth per row. If omitted it is taken from ``pairs`` columns ``depth_i`` and
|
|
205
|
+
``depth_j``.
|
|
206
|
+
min_cells
|
|
207
|
+
Minimum cells for a unit to be kept (at least 2).
|
|
208
|
+
max_depth
|
|
209
|
+
Drop cells whose depth exceeds this (alignment or assignment artefacts).
|
|
210
|
+
"""
|
|
211
|
+
clones = pd.Series(np.asarray(clones, dtype=object)).astype(str).to_numpy()
|
|
212
|
+
n = clones.size
|
|
213
|
+
p = pd.DataFrame({"i": np.asarray(pairs["i"], dtype=np.int64), "j": np.asarray(pairs["j"], dtype=np.int64),
|
|
214
|
+
"d": np.asarray(pairs["d"], dtype=float)})
|
|
215
|
+
if "shared" in pairs:
|
|
216
|
+
p["shared"] = np.asarray(pairs["shared"], dtype=float)
|
|
217
|
+
if (p[["i", "j"]].to_numpy() < 0).any() or (p[["i", "j"]].to_numpy() >= n).any():
|
|
218
|
+
raise ValueError("pair indices must be row indices into the data (0 .. n_cells-1).")
|
|
219
|
+
if depth is None:
|
|
220
|
+
if not {"depth_i", "depth_j"} <= set(pairs.keys() if hasattr(pairs, "keys") else []):
|
|
221
|
+
raise ValueError("give depth, or pairs with depth_i and depth_j columns.")
|
|
222
|
+
depth = np.full(n, np.nan)
|
|
223
|
+
depth[np.asarray(pairs["i"], dtype=np.int64)] = np.asarray(pairs["depth_i"], dtype=float)
|
|
224
|
+
depth[np.asarray(pairs["j"], dtype=np.int64)] = np.asarray(pairs["depth_j"], dtype=float)
|
|
225
|
+
depth = np.asarray(depth, dtype=float).copy()
|
|
226
|
+
if depth.size != n:
|
|
227
|
+
raise ValueError("depth must have one value per data row.")
|
|
228
|
+
block = np.zeros(n, dtype=object) if blocks is None else np.asarray(blocks, dtype=object).astype(str)
|
|
229
|
+
notes = {"pairs_in": int(len(p))}
|
|
230
|
+
p = p[(clones[p["i"].to_numpy()] == clones[p["j"].to_numpy()]) & np.isfinite(p["d"].to_numpy())]
|
|
231
|
+
p = p[block[p["i"].to_numpy()] == block[p["j"].to_numpy()]]
|
|
232
|
+
if max_depth is not None:
|
|
233
|
+
bad = np.isfinite(depth) & (depth > max_depth)
|
|
234
|
+
notes["cells_dropped_max_depth"] = int(bad.sum())
|
|
235
|
+
p = p[~bad[p["i"].to_numpy()] & ~bad[p["j"].to_numpy()]]
|
|
236
|
+
unit_key = pd.Series(clones).astype(str) + "\x1f" + pd.Series(block).astype(str)
|
|
237
|
+
ucode, ulabels = codes(unit_key)
|
|
238
|
+
p = p.assign(u=ucode[p["i"].to_numpy()])
|
|
239
|
+
units, dist, shared, uc, ub = [], [], [], [], []
|
|
240
|
+
incomplete = 0
|
|
241
|
+
for u, grp in p.groupby("u", sort=True):
|
|
242
|
+
cells = np.unique(np.r_[grp["i"].to_numpy(), grp["j"].to_numpy()])
|
|
243
|
+
m = cells.size
|
|
244
|
+
if m < max(2, min_cells):
|
|
245
|
+
continue
|
|
246
|
+
pos = {c: k for k, c in enumerate(cells)}
|
|
247
|
+
dm = np.full((m, m), np.nan)
|
|
248
|
+
sm = np.full((m, m), np.nan)
|
|
249
|
+
a = np.array([pos[c] for c in grp["i"].to_numpy()])
|
|
250
|
+
b = np.array([pos[c] for c in grp["j"].to_numpy()])
|
|
251
|
+
dm[a, b] = dm[b, a] = grp["d"].to_numpy()
|
|
252
|
+
np.fill_diagonal(dm, 0.0)
|
|
253
|
+
if "shared" in grp:
|
|
254
|
+
sm[a, b] = sm[b, a] = grp["shared"].to_numpy()
|
|
255
|
+
# keep a complete sub-matrix: drop the cell with most missing distances until complete
|
|
256
|
+
keep = np.ones(m, dtype=bool)
|
|
257
|
+
while keep.sum() >= 2 and np.isnan(dm[np.ix_(keep, keep)]).any():
|
|
258
|
+
miss = np.isnan(dm[np.ix_(keep, keep)]).sum(axis=1)
|
|
259
|
+
idx = np.flatnonzero(keep)[np.argmax(miss)]
|
|
260
|
+
keep[idx] = False
|
|
261
|
+
incomplete += 1
|
|
262
|
+
if keep.sum() < max(2, min_cells):
|
|
263
|
+
continue
|
|
264
|
+
cells, dm, sm = cells[keep], dm[np.ix_(keep, keep)], sm[np.ix_(keep, keep)]
|
|
265
|
+
dc = depth[cells]
|
|
266
|
+
if not np.isfinite(dc).all():
|
|
267
|
+
continue
|
|
268
|
+
# shared root path: from pairs when given, else (depth_i + depth_j - d_ij) / 2
|
|
269
|
+
derived = 0.5 * (dc[:, None] + dc[None, :] - dm)
|
|
270
|
+
sm = np.where(np.isfinite(sm), sm, derived)
|
|
271
|
+
np.fill_diagonal(sm, dc)
|
|
272
|
+
units.append(cells.astype(np.int64))
|
|
273
|
+
dist.append(dm)
|
|
274
|
+
shared.append(np.clip(sm, 0, None))
|
|
275
|
+
clone_lab, block_lab = ulabels[u].split("\x1f", 1)
|
|
276
|
+
uc.append(clone_lab)
|
|
277
|
+
ub.append(block_lab)
|
|
278
|
+
notes["cells_dropped_incomplete"] = int(incomplete)
|
|
279
|
+
return LineageForest(n, units, dist, shared, depth, uc, ub, notes)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
# =========================================================================== REML engine
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _helmert(n: int) -> np.ndarray:
|
|
286
|
+
"""Orthonormal ``(n, n-1)`` basis of the contrasts orthogonal to the constant vector."""
|
|
287
|
+
h = np.zeros((n, n - 1))
|
|
288
|
+
for k in range(1, n):
|
|
289
|
+
h[:k, k - 1] = 1.0
|
|
290
|
+
h[k, k - 1] = -k
|
|
291
|
+
h[:, k - 1] /= np.sqrt(k * (k + 1))
|
|
292
|
+
return h
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
class _Engine:
|
|
296
|
+
"""Per-unit projections that diagonalise the kinship kernel (shared by fit, test, simulation)."""
|
|
297
|
+
|
|
298
|
+
def __init__(self, forest: LineageForest, kernels: list):
|
|
299
|
+
self.forest = forest
|
|
300
|
+
sizes = np.array([u.size for u in forest.units])
|
|
301
|
+
self.groups = [] # (unit indices of this size, stacked T (m, n-1, n), stacked d (m, n-1))
|
|
302
|
+
order_d = []
|
|
303
|
+
for n in np.unique(sizes):
|
|
304
|
+
idx = np.flatnonzero(sizes == n)
|
|
305
|
+
P = _helmert(int(n))
|
|
306
|
+
T = np.empty((idx.size, n - 1, n))
|
|
307
|
+
dd = np.empty((idx.size, n - 1))
|
|
308
|
+
for k, u in enumerate(idx):
|
|
309
|
+
M = P.T @ kernels[u] @ P
|
|
310
|
+
w, V = np.linalg.eigh(0.5 * (M + M.T))
|
|
311
|
+
dd[k] = np.clip(w, 0, None)
|
|
312
|
+
T[k] = V.T @ P.T
|
|
313
|
+
rows = np.stack([forest.units[u] for u in idx]) # (m, n)
|
|
314
|
+
self.groups.append((idx, rows, T, dd))
|
|
315
|
+
order_d.append(dd.ravel())
|
|
316
|
+
self.d = np.concatenate(order_d) if order_d else np.empty(0)
|
|
317
|
+
self.N = self.d.size
|
|
318
|
+
|
|
319
|
+
def transform(self, Y: np.ndarray) -> np.ndarray:
|
|
320
|
+
"""Stacked projected rows for a ``(n_cells, k)`` matrix (row order of :attr:`d`)."""
|
|
321
|
+
out = []
|
|
322
|
+
for _, rows, T, _ in self.groups:
|
|
323
|
+
out.append(np.einsum("mij,mjk->mik", T, Y[rows]).reshape(-1, Y.shape[1]))
|
|
324
|
+
return np.concatenate(out, axis=0) if out else np.empty((0, Y.shape[1]))
|
|
325
|
+
|
|
326
|
+
def transform_permuted(self, E: np.ndarray, rng: np.random.Generator) -> np.ndarray:
|
|
327
|
+
"""As :meth:`transform`, after permuting the rows of ``E`` within every unit."""
|
|
328
|
+
out = []
|
|
329
|
+
for _, rows, T, _ in self.groups:
|
|
330
|
+
perm = np.argsort(rng.random(rows.shape), axis=1)
|
|
331
|
+
prow = np.take_along_axis(rows, perm, axis=1)
|
|
332
|
+
out.append(np.einsum("mij,mjk->mik", T, E[prow]).reshape(-1, E.shape[1]))
|
|
333
|
+
return np.concatenate(out, axis=0)
|
|
334
|
+
|
|
335
|
+
def unit_centre(self, Y: np.ndarray) -> np.ndarray:
|
|
336
|
+
"""``Y`` with each unit's mean removed (rows outside the forest are left at 0)."""
|
|
337
|
+
out = np.zeros_like(Y, dtype=float)
|
|
338
|
+
for _, rows, _, _ in self.groups:
|
|
339
|
+
blk = Y[rows]
|
|
340
|
+
out[rows] = blk - blk.mean(axis=1, keepdims=True)
|
|
341
|
+
return out
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _clean_design(X: np.ndarray | None, tol: float = 1e-8) -> np.ndarray | None:
|
|
345
|
+
"""Drop numerically redundant columns of the projected design."""
|
|
346
|
+
if X is None or X.size == 0:
|
|
347
|
+
return None
|
|
348
|
+
norms = np.linalg.norm(X, axis=0)
|
|
349
|
+
X = X[:, norms > tol * max(1.0, norms.max(initial=0.0))]
|
|
350
|
+
if X.shape[1] == 0:
|
|
351
|
+
return None
|
|
352
|
+
q, r, piv = _qr_pivot(X)
|
|
353
|
+
diag = np.abs(np.diag(r))
|
|
354
|
+
rank = int((diag > tol * diag.max()).sum()) if diag.size else 0
|
|
355
|
+
return X[:, np.sort(piv[:rank])] if rank else None
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _qr_pivot(X):
|
|
359
|
+
from scipy.linalg import qr
|
|
360
|
+
|
|
361
|
+
return qr(X, mode="economic", pivoting=True)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _reml_profile(z: np.ndarray, X: np.ndarray | None, d: np.ndarray, lam: float) -> float:
|
|
365
|
+
"""Profiled REML log-likelihood (up to a constant) at ``lambda = sigma_g^2 / sigma_e^2``."""
|
|
366
|
+
w = 1.0 / (1.0 + lam * d)
|
|
367
|
+
if X is None:
|
|
368
|
+
q = float(np.dot(w * z, z))
|
|
369
|
+
n_eff, logdet = z.size, 0.0
|
|
370
|
+
else:
|
|
371
|
+
XtW = X.T * w
|
|
372
|
+
A = XtW @ X
|
|
373
|
+
beta = np.linalg.solve(A, XtW @ z)
|
|
374
|
+
r = z - X @ beta
|
|
375
|
+
q = float(np.dot(w * r, r))
|
|
376
|
+
sign, logdet = np.linalg.slogdet(A)
|
|
377
|
+
n_eff = z.size - X.shape[1]
|
|
378
|
+
if q <= 0:
|
|
379
|
+
return -np.inf
|
|
380
|
+
return -0.5 * (np.log1p(lam * d).sum() + logdet + n_eff * np.log(q / n_eff))
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
_H_GRID = np.r_[0.0, 0.0025, 0.005, 0.01, 0.02, 0.03, 0.05, 0.075, np.arange(0.1, 0.951, 0.05)]
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _fit_h2(z, X, d):
|
|
387
|
+
"""REML h2 (golden-section refined), 95% profile interval and boundary LRT."""
|
|
388
|
+
lam = _H_GRID / (1 - _H_GRID)
|
|
389
|
+
ll = np.array([_reml_profile(z, X, d, l) for l in lam])
|
|
390
|
+
k = int(np.argmax(ll))
|
|
391
|
+
lo_h, hi_h = _H_GRID[max(k - 1, 0)], _H_GRID[min(k + 1, _H_GRID.size - 1)]
|
|
392
|
+
f = lambda h: -_reml_profile(z, X, d, h / (1 - h)) # noqa: E731
|
|
393
|
+
gr = (np.sqrt(5) - 1) / 2
|
|
394
|
+
a, b = lo_h, hi_h
|
|
395
|
+
c, e = b - gr * (b - a), a + gr * (b - a)
|
|
396
|
+
fc, fe = f(c), f(e)
|
|
397
|
+
for _ in range(40):
|
|
398
|
+
if fc < fe:
|
|
399
|
+
b, e, fe = e, c, fc
|
|
400
|
+
c = b - gr * (b - a)
|
|
401
|
+
fc = f(c)
|
|
402
|
+
else:
|
|
403
|
+
a, c, fc = c, e, fe
|
|
404
|
+
e = a + gr * (b - a)
|
|
405
|
+
fe = f(e)
|
|
406
|
+
h_hat = 0.5 * (a + b)
|
|
407
|
+
l_hat = -f(h_hat)
|
|
408
|
+
if ll[k] > l_hat:
|
|
409
|
+
h_hat, l_hat = float(_H_GRID[k]), float(ll[k])
|
|
410
|
+
l0 = float(ll[0])
|
|
411
|
+
lrt = max(0.0, 2 * (l_hat - l0))
|
|
412
|
+
p_lrt = 0.5 * stats.chi2.sf(lrt, 1) if lrt > 0 else 1.0
|
|
413
|
+
# Profile interval: the two points where 2 (l_hat - l(h)) crosses the chi2_1 0.95 cut-off, found by
|
|
414
|
+
# linear interpolation *between* grid points. Snapping to the grid instead (as an earlier version did)
|
|
415
|
+
# makes the interval a staircase and destroys its coverage even when the model is correct.
|
|
416
|
+
fine = np.unique(np.r_[_H_GRID, h_hat])
|
|
417
|
+
llf = np.array([_reml_profile(z, X, d, h / (1 - h)) for h in fine])
|
|
418
|
+
drop = 2 * (l_hat - llf)
|
|
419
|
+
cut = stats.chi2.ppf(0.95, 1)
|
|
420
|
+
inside = drop <= cut
|
|
421
|
+
|
|
422
|
+
def cross(i, j):
|
|
423
|
+
"""Linear interpolation of the cut-off crossing between grid points i (inside) and j (outside)."""
|
|
424
|
+
x0, x1, y0, y1 = fine[i], fine[j], drop[i], drop[j]
|
|
425
|
+
return float(x0 + (x1 - x0) * (cut - y0) / (y1 - y0)) if y1 != y0 else float(x0)
|
|
426
|
+
|
|
427
|
+
if not inside.any():
|
|
428
|
+
lo = hi = float("nan")
|
|
429
|
+
else:
|
|
430
|
+
idx = np.flatnonzero(inside)
|
|
431
|
+
first, last = int(idx[0]), int(idx[-1])
|
|
432
|
+
lo = float(fine[first]) if first == 0 else cross(first, first - 1)
|
|
433
|
+
hi = float(fine[last]) if last == fine.size - 1 else cross(last, last + 1)
|
|
434
|
+
return float(h_hat), lo, hi, float(lrt), float(p_lrt)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
@dataclass
|
|
438
|
+
class LineageHeritability:
|
|
439
|
+
"""Result of :func:`lineage_heritability` (one row per feature in :attr:`table`)."""
|
|
440
|
+
|
|
441
|
+
table: pd.DataFrame
|
|
442
|
+
combined_p: float
|
|
443
|
+
settings: dict
|
|
444
|
+
forest: dict
|
|
445
|
+
|
|
446
|
+
def __repr__(self) -> str:
|
|
447
|
+
cols = ["feature", "h2", "h2_lo", "h2_hi", "p_perm"]
|
|
448
|
+
return ("LineageHeritability(" f"{self.forest.get('units')} units, {self.forest.get('cells')} cells, "
|
|
449
|
+
f"kernel={self.settings.get('kernel')}, combined p={self.combined_p:.3g})\n"
|
|
450
|
+
+ self.table[cols].round(4).to_string(index=False))
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def lineage_heritability(
|
|
454
|
+
y,
|
|
455
|
+
forest: LineageForest,
|
|
456
|
+
*,
|
|
457
|
+
covariates=None,
|
|
458
|
+
depth_effect: bool = True,
|
|
459
|
+
kernel: str = "exponential",
|
|
460
|
+
length_scale: float = 2.0,
|
|
461
|
+
n_perm: int = 999,
|
|
462
|
+
feature_names=None,
|
|
463
|
+
random_state: int = 0,
|
|
464
|
+
verbose: bool = True,
|
|
465
|
+
) -> LineageHeritability:
|
|
466
|
+
"""Estimate and test lineage heritability of cell state inside clones.
|
|
467
|
+
|
|
468
|
+
Parameters
|
|
469
|
+
----------
|
|
470
|
+
y
|
|
471
|
+
``(n_cells,)`` or ``(n_cells, k)`` state features, row-aligned with the data the forest
|
|
472
|
+
indexes (module scores, principal components, or 0/1 recorded labels). Rows outside the
|
|
473
|
+
forest are ignored; rows inside must be finite (use :meth:`LineageForest.subset`).
|
|
474
|
+
forest
|
|
475
|
+
Output of :func:`lineage_forest`.
|
|
476
|
+
covariates
|
|
477
|
+
Optional ``(n_cells, p)`` cell-level fixed effects (log sequencing depth, quality metrics,
|
|
478
|
+
one-hot sample indicators...). Clone x block intercepts are always included.
|
|
479
|
+
depth_effect
|
|
480
|
+
Add the cell's root-to-tip mutation depth as a fixed effect (recommended: it separates "more
|
|
481
|
+
mutated cells differ" from "relatives resemble each other").
|
|
482
|
+
kernel, length_scale
|
|
483
|
+
Kinship kernel (``"exponential"``, ``"identical"``, ``"identical_mutated"``, ``"brownian"``)
|
|
484
|
+
and, for the exponential kernel, its length scale in mutations.
|
|
485
|
+
n_perm
|
|
486
|
+
Freedman-Lane permutations for the score test (0 to skip).
|
|
487
|
+
|
|
488
|
+
Returns
|
|
489
|
+
-------
|
|
490
|
+
:class:`LineageHeritability` with, per feature, the REML ``h2`` and its 95% profile interval,
|
|
491
|
+
the boundary likelihood-ratio test, the score statistic and its permutation p-value, and a
|
|
492
|
+
combined permutation p-value over all features (sum of standardised scores).
|
|
493
|
+
"""
|
|
494
|
+
if forest.n_units == 0:
|
|
495
|
+
raise ValueError("the forest has no unit with two or more cells.")
|
|
496
|
+
Y = np.asarray(y, dtype=float)
|
|
497
|
+
Y = Y[:, None] if Y.ndim == 1 else Y
|
|
498
|
+
if Y.shape[0] != forest.n_cells:
|
|
499
|
+
raise ValueError(f"y has {Y.shape[0]} rows; the forest indexes {forest.n_cells}.")
|
|
500
|
+
rows = forest.rows
|
|
501
|
+
if not np.isfinite(Y[rows]).all():
|
|
502
|
+
raise ValueError("y has missing values on cells of the forest; subset the forest first.")
|
|
503
|
+
names = list(feature_names) if feature_names is not None else [f"y{k}" for k in range(Y.shape[1])]
|
|
504
|
+
eng = _Engine(forest, forest.kernel(kernel, length_scale))
|
|
505
|
+
parts = []
|
|
506
|
+
if covariates is not None:
|
|
507
|
+
C = np.asarray(covariates, dtype=float)
|
|
508
|
+
C = C[:, None] if C.ndim == 1 else C
|
|
509
|
+
if C.shape[0] != forest.n_cells or not np.isfinite(C[rows]).all():
|
|
510
|
+
raise ValueError("covariates must be finite with one row per data row.")
|
|
511
|
+
parts.append(C)
|
|
512
|
+
if depth_effect:
|
|
513
|
+
parts.append(np.nan_to_num(forest.depth, nan=0.0)[:, None])
|
|
514
|
+
Xraw = np.column_stack(parts) if parts else None
|
|
515
|
+
Xt = _clean_design(eng.transform(Xraw)) if Xraw is not None else None
|
|
516
|
+
Z = eng.transform(Y)
|
|
517
|
+
d = eng.d
|
|
518
|
+
|
|
519
|
+
# null-model residuals (OLS in the projected space == within-unit OLS)
|
|
520
|
+
if Xt is not None:
|
|
521
|
+
Q, _ = np.linalg.qr(Xt)
|
|
522
|
+
R0 = Z - Q @ (Q.T @ Z)
|
|
523
|
+
p_fix = Xt.shape[1]
|
|
524
|
+
else:
|
|
525
|
+
Q, R0, p_fix = None, Z.copy(), 0
|
|
526
|
+
# cell-level null residuals for Freedman-Lane: unit-centred y minus the fitted covariate part
|
|
527
|
+
Yc = eng.unit_centre(Y)
|
|
528
|
+
if Xraw is not None and Xt is not None:
|
|
529
|
+
Xc = eng.unit_centre(Xraw)
|
|
530
|
+
beta0, *_ = np.linalg.lstsq(eng.transform(Xc), Z, rcond=None)
|
|
531
|
+
E0 = Yc - Xc @ beta0
|
|
532
|
+
else:
|
|
533
|
+
E0 = Yc
|
|
534
|
+
|
|
535
|
+
def score(R):
|
|
536
|
+
return (d[:, None] * R * R).sum(axis=0) / np.maximum((R * R).sum(axis=0), 1e-300)
|
|
537
|
+
|
|
538
|
+
obs = score(R0)
|
|
539
|
+
rows_out = []
|
|
540
|
+
for k, name in enumerate(names):
|
|
541
|
+
h, lo, hi, lrt, plrt = _fit_h2(Z[:, k], Xt, d)
|
|
542
|
+
rows_out.append({"feature": name, "h2": h, "h2_lo": lo, "h2_hi": hi, "lrt": lrt, "p_lrt": plrt,
|
|
543
|
+
"score": float(obs[k])})
|
|
544
|
+
tab = pd.DataFrame(rows_out)
|
|
545
|
+
combined = float("nan")
|
|
546
|
+
if n_perm:
|
|
547
|
+
require_positive_int(n_perm, "n_perm")
|
|
548
|
+
rng = np.random.default_rng(random_state)
|
|
549
|
+
null = np.empty((n_perm, Y.shape[1]))
|
|
550
|
+
for b in range(n_perm):
|
|
551
|
+
Rb = eng.transform_permuted(E0, rng)
|
|
552
|
+
if Q is not None:
|
|
553
|
+
Rb = Rb - Q @ (Q.T @ Rb)
|
|
554
|
+
null[b] = score(Rb)
|
|
555
|
+
mu, sd = null.mean(axis=0), null.std(axis=0, ddof=1)
|
|
556
|
+
tab["score_z"] = (obs - mu) / np.where(sd > 0, sd, np.nan)
|
|
557
|
+
tab["p_perm"] = (1 + (null >= obs[None, :] - 1e-12).sum(axis=0)) / (n_perm + 1)
|
|
558
|
+
zsum_null = ((null - mu) / np.where(sd > 0, sd, np.nan)).sum(axis=1)
|
|
559
|
+
zsum = np.nansum(tab["score_z"].to_numpy())
|
|
560
|
+
combined = float((1 + (zsum_null >= zsum - 1e-12).sum()) / (n_perm + 1))
|
|
561
|
+
fs = forest.summary()
|
|
562
|
+
tab["n_cells"] = fs["cells"]
|
|
563
|
+
tab["n_units"] = fs["units"]
|
|
564
|
+
tab["n_clones"] = fs["clones"]
|
|
565
|
+
settings = {"kernel": kernel, "length_scale": float(length_scale) if kernel == "exponential" else None,
|
|
566
|
+
"depth_effect": bool(depth_effect), "n_fixed_effects": int(p_fix), "n_perm": int(n_perm),
|
|
567
|
+
"random_state": int(random_state), "N_projected_rows": int(eng.N)}
|
|
568
|
+
res = LineageHeritability(tab, combined, settings, fs)
|
|
569
|
+
if verbose:
|
|
570
|
+
log(f"lineage heritability ({kernel}): {fs['units']} units, {fs['cells']} cells; "
|
|
571
|
+
+ "; ".join(f"{r.feature} h2={r.h2:.3f}" for r in tab.itertuples()), verbose)
|
|
572
|
+
return res
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
# =========================================================================== simulation / calibration
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def simulate_lineage_trait(forest: LineageForest, h2: float, *, kernel: str = "exponential",
|
|
579
|
+
length_scale: float = 2.0, base=None, random_state: int = 0) -> np.ndarray:
|
|
580
|
+
"""A feature with lineage heritability ``h2`` on the forest's real trees.
|
|
581
|
+
|
|
582
|
+
``base`` (one value per row) supplies the non-heritable part: it is permuted within every unit
|
|
583
|
+
(destroying any lineage structure it had) and standardised within units; without ``base`` the
|
|
584
|
+
non-heritable part is Gaussian. Rows outside the forest are NaN.
|
|
585
|
+
"""
|
|
586
|
+
if not 0 <= h2 < 1:
|
|
587
|
+
raise ValueError("h2 must be in [0, 1).")
|
|
588
|
+
rng = np.random.default_rng(random_state)
|
|
589
|
+
out = np.full(forest.n_cells, np.nan)
|
|
590
|
+
ks = forest.kernel(kernel, length_scale)
|
|
591
|
+
base = None if base is None else np.asarray(base, dtype=float)
|
|
592
|
+
for u, K in zip(forest.units, ks):
|
|
593
|
+
w, V = np.linalg.eigh(K)
|
|
594
|
+
g = V @ (np.sqrt(np.clip(w, 0, None)) * rng.normal(size=u.size))
|
|
595
|
+
if base is None:
|
|
596
|
+
e = rng.normal(size=u.size)
|
|
597
|
+
else:
|
|
598
|
+
e = base[u][rng.permutation(u.size)]
|
|
599
|
+
e = (e - e.mean()) / (e.std() if e.std() > 0 else 1.0)
|
|
600
|
+
out[u] = np.sqrt(h2) * g + np.sqrt(1 - h2) * e
|
|
601
|
+
return out
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def lineage_power(forest: LineageForest, *, base=None, h2_grid=(0.0, 0.02, 0.05, 0.1, 0.2),
|
|
605
|
+
n_sim: int = 100, kernel: str = "exponential", length_scale: float = 2.0,
|
|
606
|
+
depth_effect: bool = True, n_perm: int = 199, alpha: float = 0.05,
|
|
607
|
+
random_state: int = 0, verbose: bool = True) -> pd.DataFrame:
|
|
608
|
+
"""Bias, type I error (``h2 = 0``) and power of :func:`lineage_heritability` on these trees."""
|
|
609
|
+
rng = np.random.default_rng(random_state)
|
|
610
|
+
rows = []
|
|
611
|
+
for h2 in h2_grid:
|
|
612
|
+
est, p_perm, p_lrt = [], [], []
|
|
613
|
+
for s in range(n_sim):
|
|
614
|
+
seed = int(rng.integers(2**31))
|
|
615
|
+
y = simulate_lineage_trait(forest, h2, kernel=kernel, length_scale=length_scale, base=base,
|
|
616
|
+
random_state=seed)
|
|
617
|
+
r = lineage_heritability(np.nan_to_num(y), forest, kernel=kernel, length_scale=length_scale,
|
|
618
|
+
depth_effect=depth_effect, n_perm=n_perm, random_state=seed + 1,
|
|
619
|
+
verbose=False)
|
|
620
|
+
est.append(r.table["h2"].iloc[0])
|
|
621
|
+
p_perm.append(r.table["p_perm"].iloc[0] if n_perm else np.nan)
|
|
622
|
+
p_lrt.append(r.table["p_lrt"].iloc[0])
|
|
623
|
+
est, p_perm, p_lrt = map(np.asarray, (est, p_perm, p_lrt))
|
|
624
|
+
rows.append({"h2_true": h2, "n_sim": n_sim, "h2_mean": est.mean(), "h2_sd": est.std(ddof=1),
|
|
625
|
+
"rejection_perm": float((p_perm < alpha).mean()) if n_perm else np.nan,
|
|
626
|
+
"rejection_lrt": float((p_lrt < alpha).mean()), "kernel": kernel,
|
|
627
|
+
"length_scale": length_scale if kernel == "exponential" else np.nan})
|
|
628
|
+
log(f" h2={h2:.3f}: mean estimate {est.mean():.3f}, rejection perm "
|
|
629
|
+
f"{rows[-1]['rejection_perm']:.2f}, LRT {rows[-1]['rejection_lrt']:.2f}", verbose)
|
|
630
|
+
return pd.DataFrame(rows)
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
# =========================================================================== pairwise diagnostic
|
|
634
|
+
# The functions below implement the first, pairwise statistic (correlation of within-unit tree
|
|
635
|
+
# distances with state distances). It is superseded by lineage_heritability because a pair's tree
|
|
636
|
+
# distance is roughly the sum of the two cells' depths, which confounds a depth effect with lineage
|
|
637
|
+
# resemblance; it is kept so that earlier outputs remain reproducible.
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def pair_table(clones, blocks=None, distances=None) -> pd.DataFrame:
|
|
641
|
+
"""Within-clone, within-block cell pairs with their receptor distance."""
|
|
642
|
+
if distances is None:
|
|
643
|
+
raise ValueError("distances are required: give within-clone cell-pair receptor distances.")
|
|
644
|
+
d = pd.DataFrame({"i": np.asarray(distances["i"], dtype=np.int64),
|
|
645
|
+
"j": np.asarray(distances["j"], dtype=np.int64),
|
|
646
|
+
"d": np.asarray(distances["d"], dtype=float)})
|
|
647
|
+
clone_codes, _ = codes(pd.Series(clones).astype(str))
|
|
648
|
+
if blocks is None:
|
|
649
|
+
block_codes = clone_codes
|
|
650
|
+
else:
|
|
651
|
+
block_codes, _ = codes(pd.Series(clones).astype(str) + "\x1f" + pd.Series(blocks).astype(str))
|
|
652
|
+
same = (clone_codes[d["i"]] == clone_codes[d["j"]]) & (block_codes[d["i"]] == block_codes[d["j"]])
|
|
653
|
+
out = d[same & np.isfinite(d["d"])].copy()
|
|
654
|
+
out["block"] = block_codes[out["i"].to_numpy()]
|
|
655
|
+
out.attrs["dropped_cross_block"] = int((~same).sum())
|
|
656
|
+
out.attrs["block_codes"] = block_codes
|
|
657
|
+
return out.reset_index(drop=True)
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def _block_center(v: np.ndarray, block: np.ndarray, n_blocks: int) -> np.ndarray:
|
|
661
|
+
cnt = np.bincount(block, minlength=n_blocks)
|
|
662
|
+
tot = np.bincount(block, weights=v, minlength=n_blocks)
|
|
663
|
+
return v - (tot / np.maximum(cnt, 1))[block]
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def _residualise(v: np.ndarray, cov: np.ndarray | None) -> np.ndarray:
|
|
667
|
+
if cov is None or cov.size == 0:
|
|
668
|
+
return v
|
|
669
|
+
coef, *_ = np.linalg.lstsq(cov, v, rcond=None)
|
|
670
|
+
return v - cov @ coef
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def _state_distance(x: np.ndarray, i: np.ndarray, j: np.ndarray) -> np.ndarray:
|
|
674
|
+
diff = x[i] - x[j]
|
|
675
|
+
return np.sqrt(np.einsum("ij,ij->i", diff, diff))
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def _pooled_r(s_c, e, block, n_blocks, cov) -> float:
|
|
679
|
+
e_c = _block_center(e, block, n_blocks)
|
|
680
|
+
if cov is not None:
|
|
681
|
+
e_c = _residualise(e_c, cov)
|
|
682
|
+
num = float(s_c @ e_c)
|
|
683
|
+
den = np.sqrt(float(s_c @ s_c) * float(e_c @ e_c))
|
|
684
|
+
return num / den if den > 0 else np.nan
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def _permute_within_blocks(block_of_cell: np.ndarray, rng: np.random.Generator) -> np.ndarray:
|
|
688
|
+
order = np.argsort(block_of_cell, kind="stable")
|
|
689
|
+
sizes = np.bincount(block_of_cell)
|
|
690
|
+
out = np.empty(order.size, dtype=np.int64)
|
|
691
|
+
start = 0
|
|
692
|
+
for s in sizes:
|
|
693
|
+
if s:
|
|
694
|
+
seg = order[start:start + s]
|
|
695
|
+
out[seg] = seg[rng.permutation(s)]
|
|
696
|
+
start += s
|
|
697
|
+
return out
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
@dataclass
|
|
701
|
+
class LineageSignal:
|
|
702
|
+
"""Result of :func:`lineage_signal` (pairwise diagnostic)."""
|
|
703
|
+
|
|
704
|
+
r: float
|
|
705
|
+
p_value: float
|
|
706
|
+
null_mean: float
|
|
707
|
+
null_sd: float
|
|
708
|
+
n_pairs: int
|
|
709
|
+
n_blocks: int
|
|
710
|
+
n_clones: int
|
|
711
|
+
n_cells: int
|
|
712
|
+
settings: dict = field(default_factory=dict)
|
|
713
|
+
identical_pairs: int = 0
|
|
714
|
+
r_identical: float = float("nan")
|
|
715
|
+
null: np.ndarray | None = field(default=None, repr=False)
|
|
716
|
+
h2_equivalent: float | None = None
|
|
717
|
+
h2_ci: tuple | None = None
|
|
718
|
+
|
|
719
|
+
@property
|
|
720
|
+
def z(self) -> float:
|
|
721
|
+
return (self.r - self.null_mean) / self.null_sd if self.null_sd else float("nan")
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def lineage_signal(x, clones, distances, *, blocks=None, covariates=None, include_identical: bool = True,
|
|
725
|
+
n_perm: int = 2000, random_state: int = 0, verbose: bool = True) -> LineageSignal:
|
|
726
|
+
"""Pooled within-unit correlation of receptor distance and state distance (superseded)."""
|
|
727
|
+
require_positive_int(n_perm, "n_perm")
|
|
728
|
+
x = np.asarray(x, dtype=float)
|
|
729
|
+
pairs = pair_table(clones, blocks, distances)
|
|
730
|
+
if not include_identical:
|
|
731
|
+
pairs = pairs[pairs["d"] > 0].reset_index(drop=True)
|
|
732
|
+
if len(pairs) < 10:
|
|
733
|
+
raise ValueError(f"only {len(pairs)} usable within-clone pairs; need at least 10.")
|
|
734
|
+
block_of_cell = pairs.attrs["block_codes"]
|
|
735
|
+
used, block = np.unique(pairs["block"].to_numpy(), return_inverse=True)
|
|
736
|
+
n_blocks = used.size
|
|
737
|
+
i = pairs["i"].to_numpy()
|
|
738
|
+
j = pairs["j"].to_numpy()
|
|
739
|
+
s_c = _block_center(pairs["d"].to_numpy(), block, n_blocks)
|
|
740
|
+
cov = None
|
|
741
|
+
if covariates is not None:
|
|
742
|
+
c = np.asarray(covariates, dtype=float)
|
|
743
|
+
c = c[:, None] if c.ndim == 1 else c
|
|
744
|
+
parts = []
|
|
745
|
+
for k in range(c.shape[1]):
|
|
746
|
+
parts.append(_block_center(c[i, k] + c[j, k], block, n_blocks))
|
|
747
|
+
parts.append(_block_center(np.abs(c[i, k] - c[j, k]), block, n_blocks))
|
|
748
|
+
cov = np.column_stack(parts)
|
|
749
|
+
s_c = _residualise(s_c, cov)
|
|
750
|
+
obs = _pooled_r(s_c, _state_distance(x, i, j), block, n_blocks, cov)
|
|
751
|
+
rng = np.random.default_rng(random_state)
|
|
752
|
+
null = np.empty(n_perm)
|
|
753
|
+
for b in range(n_perm):
|
|
754
|
+
perm = _permute_within_blocks(block_of_cell, rng)
|
|
755
|
+
null[b] = _pooled_r(s_c, _state_distance(x, perm[i], perm[j]), block, n_blocks, cov)
|
|
756
|
+
finite = null[np.isfinite(null)]
|
|
757
|
+
centre = finite.mean()
|
|
758
|
+
p = (1 + np.sum(np.abs(finite - centre) >= abs(obs - centre) - 1e-12)) / (finite.size + 1)
|
|
759
|
+
ident = pairs["d"].to_numpy() == 0
|
|
760
|
+
r_ident = float("nan")
|
|
761
|
+
if ident.sum() >= 10 and (~ident).sum() >= 10:
|
|
762
|
+
e = _state_distance(x, i, j)
|
|
763
|
+
e_c = _block_center(e, block, n_blocks)
|
|
764
|
+
r_ident = float(e_c[ident].mean() - e_c[~ident].mean())
|
|
765
|
+
res = LineageSignal(
|
|
766
|
+
r=float(obs), p_value=float(p), null_mean=float(centre),
|
|
767
|
+
null_sd=float(finite.std(ddof=1)) if finite.size > 1 else float("nan"),
|
|
768
|
+
n_pairs=int(len(pairs)), n_blocks=int(n_blocks),
|
|
769
|
+
n_clones=int(pd.unique(np.asarray(clones, dtype=object)[np.unique(np.r_[i, j])]).size),
|
|
770
|
+
n_cells=int(np.unique(np.r_[i, j]).size), null=null, identical_pairs=int(ident.sum()),
|
|
771
|
+
r_identical=r_ident,
|
|
772
|
+
settings={"include_identical": bool(include_identical), "n_perm": int(n_perm),
|
|
773
|
+
"random_state": int(random_state), "blocked": blocks is not None,
|
|
774
|
+
"n_covariates": 0 if cov is None else cov.shape[1], "n_features": int(x.shape[1]),
|
|
775
|
+
"dropped_cross_block": int(pairs.attrs.get("dropped_cross_block", 0))},
|
|
776
|
+
)
|
|
777
|
+
log(f"lineage signal: r = {obs:+.4f}, p = {p:.3g}; {res.n_clones} clones, {res.n_pairs} pairs.", verbose)
|
|
778
|
+
return res
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def planted_calibration(x, clones, distances, *, blocks=None, h2_grid=(0.0, 0.02, 0.05, 0.1, 0.2, 0.4),
|
|
782
|
+
n_sim: int = 50, n_perm: int = 299, length_scale: float | str = "median",
|
|
783
|
+
alpha: float = 0.05, random_state: int = 0, verbose: bool = True) -> pd.DataFrame:
|
|
784
|
+
"""Planted-signal calibration of the pairwise diagnostic (plants on the real, un-nullified data)."""
|
|
785
|
+
rng = np.random.default_rng(random_state)
|
|
786
|
+
x = np.asarray(x, dtype=float)
|
|
787
|
+
pairs = pair_table(clones, blocks, distances)
|
|
788
|
+
block_of_cell = pairs.attrs["block_codes"]
|
|
789
|
+
used, block = np.unique(pairs["block"].to_numpy(), return_inverse=True)
|
|
790
|
+
n_blocks = used.size
|
|
791
|
+
i, j = pairs["i"].to_numpy(), pairs["j"].to_numpy()
|
|
792
|
+
d = pairs["d"].to_numpy()
|
|
793
|
+
s_c = _block_center(d, block, n_blocks)
|
|
794
|
+
pos = d[d > 0]
|
|
795
|
+
ls = float(np.median(pos)) if length_scale == "median" and pos.size else (
|
|
796
|
+
1.0 if length_scale == "median" else float(length_scale))
|
|
797
|
+
cells_in_block = {b: np.flatnonzero(block_of_cell == b) for b in np.unique(block_of_cell)}
|
|
798
|
+
dist_lookup = {}
|
|
799
|
+
for b in used:
|
|
800
|
+
idx = cells_in_block[b]
|
|
801
|
+
pos_of = {c: k for k, c in enumerate(idx)}
|
|
802
|
+
m = np.zeros((idx.size, idx.size))
|
|
803
|
+
sel = pairs["block"].to_numpy() == b
|
|
804
|
+
for a_, b_, dd in zip(i[sel], j[sel], d[sel]):
|
|
805
|
+
m[pos_of[a_], pos_of[b_]] = m[pos_of[b_], pos_of[a_]] = dd
|
|
806
|
+
c = np.exp(-m / ls)
|
|
807
|
+
np.fill_diagonal(c, 1.0)
|
|
808
|
+
w, v = np.linalg.eigh(c)
|
|
809
|
+
dist_lookup[b] = (idx, v * np.sqrt(np.clip(w, 0, None)))
|
|
810
|
+
xb = x - pd.DataFrame(x).groupby(block_of_cell).transform("mean").to_numpy()
|
|
811
|
+
base_var = float(np.einsum("ij,ij->", xb, xb) / xb.shape[0])
|
|
812
|
+
rows = []
|
|
813
|
+
for h2 in h2_grid:
|
|
814
|
+
hits, rs = 0, []
|
|
815
|
+
for _ in range(n_sim):
|
|
816
|
+
xs = x.copy()
|
|
817
|
+
if h2 > 0:
|
|
818
|
+
u = rng.normal(size=x.shape[1])
|
|
819
|
+
u /= np.linalg.norm(u)
|
|
820
|
+
amp = np.sqrt(h2 / (1 - h2) * base_var)
|
|
821
|
+
for b, (idx, L) in dist_lookup.items():
|
|
822
|
+
xs[idx] += amp * (L @ rng.normal(size=idx.size))[:, None] * u[None, :]
|
|
823
|
+
obs = _pooled_r(s_c, _state_distance(xs, i, j), block, n_blocks, None)
|
|
824
|
+
null = np.empty(n_perm)
|
|
825
|
+
for k in range(n_perm):
|
|
826
|
+
perm = _permute_within_blocks(block_of_cell, rng)
|
|
827
|
+
null[k] = _pooled_r(s_c, _state_distance(xs, perm[i], perm[j]), block, n_blocks, None)
|
|
828
|
+
centre = null.mean()
|
|
829
|
+
pv = (1 + np.sum(np.abs(null - centre) >= abs(obs - centre) - 1e-12)) / (null.size + 1)
|
|
830
|
+
hits += pv < alpha
|
|
831
|
+
rs.append(obs)
|
|
832
|
+
rows.append({"h2_planted": h2, "n_sim": int(n_sim), "power": hits / n_sim, "mean_r": float(np.mean(rs)),
|
|
833
|
+
"sd_r": float(np.std(rs, ddof=1)) if n_sim > 1 else np.nan, "length_scale": ls})
|
|
834
|
+
return pd.DataFrame(rows)
|