threadfin 4.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
threadfin/phylo.py ADDED
@@ -0,0 +1,834 @@
1
+ """Lineage heritability: is a cell's state shared with its close relatives inside a B-cell clone?
2
+
3
+ Threadfin's clone profiles describe how clones differ from each other. This module asks the
4
+ complementary question inside clones: after removing everything a clone's cells share (the clone's
5
+ mean state in each sampling block), are cells that sit close together on the clone's somatic-
6
+ hypermutation lineage tree still more alike than distant relatives?
7
+
8
+ Model
9
+ -----
10
+ For one state feature ``y`` (a module score, a principal component, or a recorded cell label coded
11
+ 0/1), cell ``i`` of clone ``c`` sampled in block ``b`` (mouse, sort gate, library...) is modelled as
12
+
13
+ y_i = mu_{c,b} + x_i' beta + g_i + e_i,
14
+ g ~ N(0, sigma_g^2 K), e ~ N(0, sigma_e^2 I),
15
+
16
+ where ``mu_{c,b}`` is a free intercept for every clone x block *unit* (so clone-wide and block-wide
17
+ state, however large, cannot contribute), ``x_i`` are cell covariates (sequencing depth, the cell's
18
+ mutation depth on the tree, sample indicators...), and ``K`` is a kinship kernel built from the tree:
19
+
20
+ * ``"exponential"`` (default): ``K_ij = exp(-d_ij / l)``, ``d_ij`` the number of mutations on the
21
+ tree path between the two cells' genotypes;
22
+ * ``"identical"``: ``K_ij = 1`` when the two cells carry the same receptor genotype;
23
+ * ``"identical_mutated"``: as ``"identical"``, but only for genotypes that carry at least one
24
+ mutation (cells still identical to the germline need not be recent relatives);
25
+ * ``"brownian"``: ``K_ij`` = mutations shared from the root to the two cells' common ancestor,
26
+ scaled by the mean root-to-tip depth (Brownian motion along the tree).
27
+
28
+ ``h2 = sigma_g^2 / (sigma_g^2 + sigma_e^2)`` is the share of within-unit cell variation that is
29
+ structured by the lineage; for the exponential and identity kernels it is the correlation of two
30
+ cells with the same genotype after the unit mean is removed.
31
+
32
+ Why not a pairwise correlation of tree and state distances
33
+ ----------------------------------------------------------
34
+ The distance between two cells on a tree is roughly the sum of their depths, so a pairwise
35
+ statistic mixes two different effects: a state that depends on how mutated a cell is (a fixed effect
36
+ of depth) and genuine resemblance of relatives (the random effect ``g``). Here depth is a covariate
37
+ and only ``g`` is tested.
38
+
39
+ Estimation and inference
40
+ ------------------------
41
+ Every unit is projected onto the contrasts orthogonal to its intercept (Helmert basis ``P``) and
42
+ rotated into the eigenbasis of ``P' K P``. In that basis the covariance is diagonal,
43
+ ``sigma_e^2 (1 + lambda d)``, so the restricted (REML) likelihood of ``lambda = h2 / (1 - h2)``,
44
+ pooled over thousands of small trees, is a sum over rows and is maximised by a 1-D search.
45
+ ``h2`` comes with a profile-likelihood interval and an asymptotic boundary likelihood-ratio test
46
+ (``0.5 chi2_0 + 0.5 chi2_1``). **Both are likelihood-based and both have been measured to fail on real
47
+ repertoire data**: with per-unit heteroscedastic residuals the boundary LRT rejects 32-38% of the time at a
48
+ nominal 5%, and with heavy-tailed residuals 16-23%; the interval's coverage is 0.65-0.87 instead of 0.95.
49
+ Use ``p_lrt`` and ``h2_lo``/``h2_hi`` as rough diagnostics only, and quote the permutation p-value and a
50
+ bootstrap over clones (and over donors) for any reported interval. The primary test is the REML score
51
+ statistic
52
+ ``Q = sum_i d_i r_i^2 / sum_i r_i^2`` of the null residuals ``r``, with a Freedman-Lane permutation
53
+ null: residuals of the null model are permuted among the cells of each unit, which keeps every
54
+ unit's tree, its covariates and its set of residual states, and destroys only which cell sits where.
55
+
56
+ :func:`lineage_power` plants a lineage component of known ``h2`` on the real trees, on top of a real
57
+ feature whose own lineage structure has first been destroyed by permutation, and reports bias, type I
58
+ error and power, so that a null result can be read as "``h2`` above x would have been detected".
59
+ """
60
+
61
+ from __future__ import annotations
62
+
63
+ from dataclasses import dataclass, field
64
+
65
+ import numpy as np
66
+ import pandas as pd
67
+ from scipy import stats
68
+
69
+ from ._utils import codes, log, require_positive_int
70
+
71
+ __all__ = [
72
+ "KERNELS",
73
+ "LineageForest",
74
+ "LineageHeritability",
75
+ "lineage_forest",
76
+ "lineage_heritability",
77
+ "lineage_power",
78
+ "simulate_lineage_trait",
79
+ # pairwise diagnostic, superseded by lineage_heritability (kept for reproducibility)
80
+ "LineageSignal",
81
+ "lineage_signal",
82
+ "planted_calibration",
83
+ "pair_table",
84
+ ]
85
+
86
+ KERNELS = ("exponential", "identical", "identical_mutated", "brownian")
87
+
88
+
89
+ # =========================================================================== forest
90
+
91
+
92
+ @dataclass
93
+ class LineageForest:
94
+ """Cells placed on per-clone lineage trees, grouped into clone x block units.
95
+
96
+ Attributes
97
+ ----------
98
+ n_cells
99
+ Number of rows of the data the forest indexes into.
100
+ units
101
+ One integer array of row indices per unit (all cells of one clone in one block that are
102
+ placed on the clone's tree); every unit has at least two cells.
103
+ dist
104
+ Per unit, the matrix of tree-path distances (mutations) between its cells.
105
+ shared
106
+ Per unit, the matrix of mutations shared from the root to each pair's common ancestor
107
+ (diagonal = the cell's depth).
108
+ depth
109
+ Per row, the root-to-tip depth of the cell's genotype (NaN for rows not in the forest).
110
+ unit_clone, unit_block
111
+ Labels of each unit.
112
+ """
113
+
114
+ n_cells: int
115
+ units: list
116
+ dist: list
117
+ shared: list
118
+ depth: np.ndarray
119
+ unit_clone: list
120
+ unit_block: list
121
+ notes: dict = field(default_factory=dict)
122
+
123
+ @property
124
+ def n_units(self) -> int:
125
+ return len(self.units)
126
+
127
+ @property
128
+ def rows(self) -> np.ndarray:
129
+ """All row indices that belong to a unit."""
130
+ return np.concatenate(self.units) if self.units else np.empty(0, dtype=np.int64)
131
+
132
+ def summary(self) -> dict:
133
+ sizes = np.array([u.size for u in self.units])
134
+ return {"units": int(self.n_units), "cells": int(sizes.sum()) if sizes.size else 0,
135
+ "clones": int(len(set(self.unit_clone))),
136
+ "median_unit_size": float(np.median(sizes)) if sizes.size else float("nan"),
137
+ "max_unit_size": int(sizes.max()) if sizes.size else 0,
138
+ "identical_pairs": int(sum(((d == 0).sum() - d.shape[0]) // 2 for d in self.dist)),
139
+ "pairs": int(sum(d.shape[0] * (d.shape[0] - 1) // 2 for d in self.dist)),
140
+ "mean_depth": float(np.nanmean(self.depth[self.rows])) if self.units else float("nan"),
141
+ **self.notes}
142
+
143
+ def subset(self, keep) -> "LineageForest":
144
+ """The forest restricted to rows where ``keep`` is True (units falling below 2 cells drop)."""
145
+ keep = np.asarray(keep, dtype=bool)
146
+ units, dist, shared, uc, ub = [], [], [], [], []
147
+ for u, d, s, c, b in zip(self.units, self.dist, self.shared, self.unit_clone, self.unit_block):
148
+ m = keep[u]
149
+ if m.sum() >= 2:
150
+ units.append(u[m])
151
+ dist.append(d[np.ix_(m, m)])
152
+ shared.append(s[np.ix_(m, m)])
153
+ uc.append(c)
154
+ ub.append(b)
155
+ return LineageForest(self.n_cells, units, dist, shared, self.depth, uc, ub, dict(self.notes))
156
+
157
+ def kernel(self, kind: str = "exponential", length_scale: float = 2.0) -> list:
158
+ """Per-unit kinship matrices for ``kind`` (see module docstring)."""
159
+ if kind not in KERNELS:
160
+ raise ValueError(f"kernel must be one of {KERNELS}, got {kind!r}.")
161
+ out = []
162
+ if kind == "brownian":
163
+ scale = float(np.nanmean(self.depth[self.rows])) if self.units else 1.0
164
+ scale = scale if scale > 0 else 1.0
165
+ for u, d, s in zip(self.units, self.dist, self.shared):
166
+ if kind == "exponential":
167
+ if not length_scale or length_scale <= 0:
168
+ raise ValueError("length_scale must be positive for the exponential kernel.")
169
+ k = np.exp(-d / float(length_scale))
170
+ elif kind == "identical":
171
+ k = (d == 0).astype(float)
172
+ elif kind == "identical_mutated":
173
+ mutated = self.depth[u] > 0
174
+ k = ((d == 0) & mutated[:, None] & mutated[None, :]).astype(float)
175
+ np.fill_diagonal(k, 1.0)
176
+ else:
177
+ k = s / scale
178
+ out.append(0.5 * (k + k.T))
179
+ return out
180
+
181
+
182
+ def lineage_forest(clones, pairs, *, blocks=None, depth=None, min_cells: int = 2,
183
+ max_depth: float | None = None) -> LineageForest:
184
+ """Group cells into clone x block units and collect their lineage-tree distances.
185
+
186
+ Parameters
187
+ ----------
188
+ clones
189
+ Clone label per data row (length ``n_cells``).
190
+ pairs
191
+ Long table of within-clone cell pairs with integer row indices ``i``, ``j``, the tree-path
192
+ distance ``d`` (mutations between the two cells' genotypes) and, optionally, ``shared``
193
+ (mutations from the root to their common ancestor). Pairs across clones are ignored.
194
+ blocks
195
+ Sampling block per row. Units are clone x block. Default: one block, i.e. units are whole clones.
196
+
197
+ **Use the finest separately processed unit — the library or sort gate, not the animal.** A clone x
198
+ gate interaction (one clone behaving differently in two sorted gates of the same animal, which
199
+ physical sorting all but guarantees) is *not* removed by fitting the gate as a fixed effect. On real
200
+ germinal-centre trees that omitted interaction, with no lineage component present at all, produced
201
+ h2 = 0.09-0.16 and rejected 49-88% of the time. Blocking by animal when gates are nested inside
202
+ animals is therefore not a valid test of lineage heritability.
203
+ depth
204
+ Root-to-tip depth per row. If omitted it is taken from ``pairs`` columns ``depth_i`` and
205
+ ``depth_j``.
206
+ min_cells
207
+ Minimum cells for a unit to be kept (at least 2).
208
+ max_depth
209
+ Drop cells whose depth exceeds this (alignment or assignment artefacts).
210
+ """
211
+ clones = pd.Series(np.asarray(clones, dtype=object)).astype(str).to_numpy()
212
+ n = clones.size
213
+ p = pd.DataFrame({"i": np.asarray(pairs["i"], dtype=np.int64), "j": np.asarray(pairs["j"], dtype=np.int64),
214
+ "d": np.asarray(pairs["d"], dtype=float)})
215
+ if "shared" in pairs:
216
+ p["shared"] = np.asarray(pairs["shared"], dtype=float)
217
+ if (p[["i", "j"]].to_numpy() < 0).any() or (p[["i", "j"]].to_numpy() >= n).any():
218
+ raise ValueError("pair indices must be row indices into the data (0 .. n_cells-1).")
219
+ if depth is None:
220
+ if not {"depth_i", "depth_j"} <= set(pairs.keys() if hasattr(pairs, "keys") else []):
221
+ raise ValueError("give depth, or pairs with depth_i and depth_j columns.")
222
+ depth = np.full(n, np.nan)
223
+ depth[np.asarray(pairs["i"], dtype=np.int64)] = np.asarray(pairs["depth_i"], dtype=float)
224
+ depth[np.asarray(pairs["j"], dtype=np.int64)] = np.asarray(pairs["depth_j"], dtype=float)
225
+ depth = np.asarray(depth, dtype=float).copy()
226
+ if depth.size != n:
227
+ raise ValueError("depth must have one value per data row.")
228
+ block = np.zeros(n, dtype=object) if blocks is None else np.asarray(blocks, dtype=object).astype(str)
229
+ notes = {"pairs_in": int(len(p))}
230
+ p = p[(clones[p["i"].to_numpy()] == clones[p["j"].to_numpy()]) & np.isfinite(p["d"].to_numpy())]
231
+ p = p[block[p["i"].to_numpy()] == block[p["j"].to_numpy()]]
232
+ if max_depth is not None:
233
+ bad = np.isfinite(depth) & (depth > max_depth)
234
+ notes["cells_dropped_max_depth"] = int(bad.sum())
235
+ p = p[~bad[p["i"].to_numpy()] & ~bad[p["j"].to_numpy()]]
236
+ unit_key = pd.Series(clones).astype(str) + "\x1f" + pd.Series(block).astype(str)
237
+ ucode, ulabels = codes(unit_key)
238
+ p = p.assign(u=ucode[p["i"].to_numpy()])
239
+ units, dist, shared, uc, ub = [], [], [], [], []
240
+ incomplete = 0
241
+ for u, grp in p.groupby("u", sort=True):
242
+ cells = np.unique(np.r_[grp["i"].to_numpy(), grp["j"].to_numpy()])
243
+ m = cells.size
244
+ if m < max(2, min_cells):
245
+ continue
246
+ pos = {c: k for k, c in enumerate(cells)}
247
+ dm = np.full((m, m), np.nan)
248
+ sm = np.full((m, m), np.nan)
249
+ a = np.array([pos[c] for c in grp["i"].to_numpy()])
250
+ b = np.array([pos[c] for c in grp["j"].to_numpy()])
251
+ dm[a, b] = dm[b, a] = grp["d"].to_numpy()
252
+ np.fill_diagonal(dm, 0.0)
253
+ if "shared" in grp:
254
+ sm[a, b] = sm[b, a] = grp["shared"].to_numpy()
255
+ # keep a complete sub-matrix: drop the cell with most missing distances until complete
256
+ keep = np.ones(m, dtype=bool)
257
+ while keep.sum() >= 2 and np.isnan(dm[np.ix_(keep, keep)]).any():
258
+ miss = np.isnan(dm[np.ix_(keep, keep)]).sum(axis=1)
259
+ idx = np.flatnonzero(keep)[np.argmax(miss)]
260
+ keep[idx] = False
261
+ incomplete += 1
262
+ if keep.sum() < max(2, min_cells):
263
+ continue
264
+ cells, dm, sm = cells[keep], dm[np.ix_(keep, keep)], sm[np.ix_(keep, keep)]
265
+ dc = depth[cells]
266
+ if not np.isfinite(dc).all():
267
+ continue
268
+ # shared root path: from pairs when given, else (depth_i + depth_j - d_ij) / 2
269
+ derived = 0.5 * (dc[:, None] + dc[None, :] - dm)
270
+ sm = np.where(np.isfinite(sm), sm, derived)
271
+ np.fill_diagonal(sm, dc)
272
+ units.append(cells.astype(np.int64))
273
+ dist.append(dm)
274
+ shared.append(np.clip(sm, 0, None))
275
+ clone_lab, block_lab = ulabels[u].split("\x1f", 1)
276
+ uc.append(clone_lab)
277
+ ub.append(block_lab)
278
+ notes["cells_dropped_incomplete"] = int(incomplete)
279
+ return LineageForest(n, units, dist, shared, depth, uc, ub, notes)
280
+
281
+
282
+ # =========================================================================== REML engine
283
+
284
+
285
+ def _helmert(n: int) -> np.ndarray:
286
+ """Orthonormal ``(n, n-1)`` basis of the contrasts orthogonal to the constant vector."""
287
+ h = np.zeros((n, n - 1))
288
+ for k in range(1, n):
289
+ h[:k, k - 1] = 1.0
290
+ h[k, k - 1] = -k
291
+ h[:, k - 1] /= np.sqrt(k * (k + 1))
292
+ return h
293
+
294
+
295
+ class _Engine:
296
+ """Per-unit projections that diagonalise the kinship kernel (shared by fit, test, simulation)."""
297
+
298
+ def __init__(self, forest: LineageForest, kernels: list):
299
+ self.forest = forest
300
+ sizes = np.array([u.size for u in forest.units])
301
+ self.groups = [] # (unit indices of this size, stacked T (m, n-1, n), stacked d (m, n-1))
302
+ order_d = []
303
+ for n in np.unique(sizes):
304
+ idx = np.flatnonzero(sizes == n)
305
+ P = _helmert(int(n))
306
+ T = np.empty((idx.size, n - 1, n))
307
+ dd = np.empty((idx.size, n - 1))
308
+ for k, u in enumerate(idx):
309
+ M = P.T @ kernels[u] @ P
310
+ w, V = np.linalg.eigh(0.5 * (M + M.T))
311
+ dd[k] = np.clip(w, 0, None)
312
+ T[k] = V.T @ P.T
313
+ rows = np.stack([forest.units[u] for u in idx]) # (m, n)
314
+ self.groups.append((idx, rows, T, dd))
315
+ order_d.append(dd.ravel())
316
+ self.d = np.concatenate(order_d) if order_d else np.empty(0)
317
+ self.N = self.d.size
318
+
319
+ def transform(self, Y: np.ndarray) -> np.ndarray:
320
+ """Stacked projected rows for a ``(n_cells, k)`` matrix (row order of :attr:`d`)."""
321
+ out = []
322
+ for _, rows, T, _ in self.groups:
323
+ out.append(np.einsum("mij,mjk->mik", T, Y[rows]).reshape(-1, Y.shape[1]))
324
+ return np.concatenate(out, axis=0) if out else np.empty((0, Y.shape[1]))
325
+
326
+ def transform_permuted(self, E: np.ndarray, rng: np.random.Generator) -> np.ndarray:
327
+ """As :meth:`transform`, after permuting the rows of ``E`` within every unit."""
328
+ out = []
329
+ for _, rows, T, _ in self.groups:
330
+ perm = np.argsort(rng.random(rows.shape), axis=1)
331
+ prow = np.take_along_axis(rows, perm, axis=1)
332
+ out.append(np.einsum("mij,mjk->mik", T, E[prow]).reshape(-1, E.shape[1]))
333
+ return np.concatenate(out, axis=0)
334
+
335
+ def unit_centre(self, Y: np.ndarray) -> np.ndarray:
336
+ """``Y`` with each unit's mean removed (rows outside the forest are left at 0)."""
337
+ out = np.zeros_like(Y, dtype=float)
338
+ for _, rows, _, _ in self.groups:
339
+ blk = Y[rows]
340
+ out[rows] = blk - blk.mean(axis=1, keepdims=True)
341
+ return out
342
+
343
+
344
+ def _clean_design(X: np.ndarray | None, tol: float = 1e-8) -> np.ndarray | None:
345
+ """Drop numerically redundant columns of the projected design."""
346
+ if X is None or X.size == 0:
347
+ return None
348
+ norms = np.linalg.norm(X, axis=0)
349
+ X = X[:, norms > tol * max(1.0, norms.max(initial=0.0))]
350
+ if X.shape[1] == 0:
351
+ return None
352
+ q, r, piv = _qr_pivot(X)
353
+ diag = np.abs(np.diag(r))
354
+ rank = int((diag > tol * diag.max()).sum()) if diag.size else 0
355
+ return X[:, np.sort(piv[:rank])] if rank else None
356
+
357
+
358
+ def _qr_pivot(X):
359
+ from scipy.linalg import qr
360
+
361
+ return qr(X, mode="economic", pivoting=True)
362
+
363
+
364
+ def _reml_profile(z: np.ndarray, X: np.ndarray | None, d: np.ndarray, lam: float) -> float:
365
+ """Profiled REML log-likelihood (up to a constant) at ``lambda = sigma_g^2 / sigma_e^2``."""
366
+ w = 1.0 / (1.0 + lam * d)
367
+ if X is None:
368
+ q = float(np.dot(w * z, z))
369
+ n_eff, logdet = z.size, 0.0
370
+ else:
371
+ XtW = X.T * w
372
+ A = XtW @ X
373
+ beta = np.linalg.solve(A, XtW @ z)
374
+ r = z - X @ beta
375
+ q = float(np.dot(w * r, r))
376
+ sign, logdet = np.linalg.slogdet(A)
377
+ n_eff = z.size - X.shape[1]
378
+ if q <= 0:
379
+ return -np.inf
380
+ return -0.5 * (np.log1p(lam * d).sum() + logdet + n_eff * np.log(q / n_eff))
381
+
382
+
383
+ _H_GRID = np.r_[0.0, 0.0025, 0.005, 0.01, 0.02, 0.03, 0.05, 0.075, np.arange(0.1, 0.951, 0.05)]
384
+
385
+
386
+ def _fit_h2(z, X, d):
387
+ """REML h2 (golden-section refined), 95% profile interval and boundary LRT."""
388
+ lam = _H_GRID / (1 - _H_GRID)
389
+ ll = np.array([_reml_profile(z, X, d, l) for l in lam])
390
+ k = int(np.argmax(ll))
391
+ lo_h, hi_h = _H_GRID[max(k - 1, 0)], _H_GRID[min(k + 1, _H_GRID.size - 1)]
392
+ f = lambda h: -_reml_profile(z, X, d, h / (1 - h)) # noqa: E731
393
+ gr = (np.sqrt(5) - 1) / 2
394
+ a, b = lo_h, hi_h
395
+ c, e = b - gr * (b - a), a + gr * (b - a)
396
+ fc, fe = f(c), f(e)
397
+ for _ in range(40):
398
+ if fc < fe:
399
+ b, e, fe = e, c, fc
400
+ c = b - gr * (b - a)
401
+ fc = f(c)
402
+ else:
403
+ a, c, fc = c, e, fe
404
+ e = a + gr * (b - a)
405
+ fe = f(e)
406
+ h_hat = 0.5 * (a + b)
407
+ l_hat = -f(h_hat)
408
+ if ll[k] > l_hat:
409
+ h_hat, l_hat = float(_H_GRID[k]), float(ll[k])
410
+ l0 = float(ll[0])
411
+ lrt = max(0.0, 2 * (l_hat - l0))
412
+ p_lrt = 0.5 * stats.chi2.sf(lrt, 1) if lrt > 0 else 1.0
413
+ # Profile interval: the two points where 2 (l_hat - l(h)) crosses the chi2_1 0.95 cut-off, found by
414
+ # linear interpolation *between* grid points. Snapping to the grid instead (as an earlier version did)
415
+ # makes the interval a staircase and destroys its coverage even when the model is correct.
416
+ fine = np.unique(np.r_[_H_GRID, h_hat])
417
+ llf = np.array([_reml_profile(z, X, d, h / (1 - h)) for h in fine])
418
+ drop = 2 * (l_hat - llf)
419
+ cut = stats.chi2.ppf(0.95, 1)
420
+ inside = drop <= cut
421
+
422
+ def cross(i, j):
423
+ """Linear interpolation of the cut-off crossing between grid points i (inside) and j (outside)."""
424
+ x0, x1, y0, y1 = fine[i], fine[j], drop[i], drop[j]
425
+ return float(x0 + (x1 - x0) * (cut - y0) / (y1 - y0)) if y1 != y0 else float(x0)
426
+
427
+ if not inside.any():
428
+ lo = hi = float("nan")
429
+ else:
430
+ idx = np.flatnonzero(inside)
431
+ first, last = int(idx[0]), int(idx[-1])
432
+ lo = float(fine[first]) if first == 0 else cross(first, first - 1)
433
+ hi = float(fine[last]) if last == fine.size - 1 else cross(last, last + 1)
434
+ return float(h_hat), lo, hi, float(lrt), float(p_lrt)
435
+
436
+
437
+ @dataclass
438
+ class LineageHeritability:
439
+ """Result of :func:`lineage_heritability` (one row per feature in :attr:`table`)."""
440
+
441
+ table: pd.DataFrame
442
+ combined_p: float
443
+ settings: dict
444
+ forest: dict
445
+
446
+ def __repr__(self) -> str:
447
+ cols = ["feature", "h2", "h2_lo", "h2_hi", "p_perm"]
448
+ return ("LineageHeritability(" f"{self.forest.get('units')} units, {self.forest.get('cells')} cells, "
449
+ f"kernel={self.settings.get('kernel')}, combined p={self.combined_p:.3g})\n"
450
+ + self.table[cols].round(4).to_string(index=False))
451
+
452
+
453
+ def lineage_heritability(
454
+ y,
455
+ forest: LineageForest,
456
+ *,
457
+ covariates=None,
458
+ depth_effect: bool = True,
459
+ kernel: str = "exponential",
460
+ length_scale: float = 2.0,
461
+ n_perm: int = 999,
462
+ feature_names=None,
463
+ random_state: int = 0,
464
+ verbose: bool = True,
465
+ ) -> LineageHeritability:
466
+ """Estimate and test lineage heritability of cell state inside clones.
467
+
468
+ Parameters
469
+ ----------
470
+ y
471
+ ``(n_cells,)`` or ``(n_cells, k)`` state features, row-aligned with the data the forest
472
+ indexes (module scores, principal components, or 0/1 recorded labels). Rows outside the
473
+ forest are ignored; rows inside must be finite (use :meth:`LineageForest.subset`).
474
+ forest
475
+ Output of :func:`lineage_forest`.
476
+ covariates
477
+ Optional ``(n_cells, p)`` cell-level fixed effects (log sequencing depth, quality metrics,
478
+ one-hot sample indicators...). Clone x block intercepts are always included.
479
+ depth_effect
480
+ Add the cell's root-to-tip mutation depth as a fixed effect (recommended: it separates "more
481
+ mutated cells differ" from "relatives resemble each other").
482
+ kernel, length_scale
483
+ Kinship kernel (``"exponential"``, ``"identical"``, ``"identical_mutated"``, ``"brownian"``)
484
+ and, for the exponential kernel, its length scale in mutations.
485
+ n_perm
486
+ Freedman-Lane permutations for the score test (0 to skip).
487
+
488
+ Returns
489
+ -------
490
+ :class:`LineageHeritability` with, per feature, the REML ``h2`` and its 95% profile interval,
491
+ the boundary likelihood-ratio test, the score statistic and its permutation p-value, and a
492
+ combined permutation p-value over all features (sum of standardised scores).
493
+ """
494
+ if forest.n_units == 0:
495
+ raise ValueError("the forest has no unit with two or more cells.")
496
+ Y = np.asarray(y, dtype=float)
497
+ Y = Y[:, None] if Y.ndim == 1 else Y
498
+ if Y.shape[0] != forest.n_cells:
499
+ raise ValueError(f"y has {Y.shape[0]} rows; the forest indexes {forest.n_cells}.")
500
+ rows = forest.rows
501
+ if not np.isfinite(Y[rows]).all():
502
+ raise ValueError("y has missing values on cells of the forest; subset the forest first.")
503
+ names = list(feature_names) if feature_names is not None else [f"y{k}" for k in range(Y.shape[1])]
504
+ eng = _Engine(forest, forest.kernel(kernel, length_scale))
505
+ parts = []
506
+ if covariates is not None:
507
+ C = np.asarray(covariates, dtype=float)
508
+ C = C[:, None] if C.ndim == 1 else C
509
+ if C.shape[0] != forest.n_cells or not np.isfinite(C[rows]).all():
510
+ raise ValueError("covariates must be finite with one row per data row.")
511
+ parts.append(C)
512
+ if depth_effect:
513
+ parts.append(np.nan_to_num(forest.depth, nan=0.0)[:, None])
514
+ Xraw = np.column_stack(parts) if parts else None
515
+ Xt = _clean_design(eng.transform(Xraw)) if Xraw is not None else None
516
+ Z = eng.transform(Y)
517
+ d = eng.d
518
+
519
+ # null-model residuals (OLS in the projected space == within-unit OLS)
520
+ if Xt is not None:
521
+ Q, _ = np.linalg.qr(Xt)
522
+ R0 = Z - Q @ (Q.T @ Z)
523
+ p_fix = Xt.shape[1]
524
+ else:
525
+ Q, R0, p_fix = None, Z.copy(), 0
526
+ # cell-level null residuals for Freedman-Lane: unit-centred y minus the fitted covariate part
527
+ Yc = eng.unit_centre(Y)
528
+ if Xraw is not None and Xt is not None:
529
+ Xc = eng.unit_centre(Xraw)
530
+ beta0, *_ = np.linalg.lstsq(eng.transform(Xc), Z, rcond=None)
531
+ E0 = Yc - Xc @ beta0
532
+ else:
533
+ E0 = Yc
534
+
535
+ def score(R):
536
+ return (d[:, None] * R * R).sum(axis=0) / np.maximum((R * R).sum(axis=0), 1e-300)
537
+
538
+ obs = score(R0)
539
+ rows_out = []
540
+ for k, name in enumerate(names):
541
+ h, lo, hi, lrt, plrt = _fit_h2(Z[:, k], Xt, d)
542
+ rows_out.append({"feature": name, "h2": h, "h2_lo": lo, "h2_hi": hi, "lrt": lrt, "p_lrt": plrt,
543
+ "score": float(obs[k])})
544
+ tab = pd.DataFrame(rows_out)
545
+ combined = float("nan")
546
+ if n_perm:
547
+ require_positive_int(n_perm, "n_perm")
548
+ rng = np.random.default_rng(random_state)
549
+ null = np.empty((n_perm, Y.shape[1]))
550
+ for b in range(n_perm):
551
+ Rb = eng.transform_permuted(E0, rng)
552
+ if Q is not None:
553
+ Rb = Rb - Q @ (Q.T @ Rb)
554
+ null[b] = score(Rb)
555
+ mu, sd = null.mean(axis=0), null.std(axis=0, ddof=1)
556
+ tab["score_z"] = (obs - mu) / np.where(sd > 0, sd, np.nan)
557
+ tab["p_perm"] = (1 + (null >= obs[None, :] - 1e-12).sum(axis=0)) / (n_perm + 1)
558
+ zsum_null = ((null - mu) / np.where(sd > 0, sd, np.nan)).sum(axis=1)
559
+ zsum = np.nansum(tab["score_z"].to_numpy())
560
+ combined = float((1 + (zsum_null >= zsum - 1e-12).sum()) / (n_perm + 1))
561
+ fs = forest.summary()
562
+ tab["n_cells"] = fs["cells"]
563
+ tab["n_units"] = fs["units"]
564
+ tab["n_clones"] = fs["clones"]
565
+ settings = {"kernel": kernel, "length_scale": float(length_scale) if kernel == "exponential" else None,
566
+ "depth_effect": bool(depth_effect), "n_fixed_effects": int(p_fix), "n_perm": int(n_perm),
567
+ "random_state": int(random_state), "N_projected_rows": int(eng.N)}
568
+ res = LineageHeritability(tab, combined, settings, fs)
569
+ if verbose:
570
+ log(f"lineage heritability ({kernel}): {fs['units']} units, {fs['cells']} cells; "
571
+ + "; ".join(f"{r.feature} h2={r.h2:.3f}" for r in tab.itertuples()), verbose)
572
+ return res
573
+
574
+
575
+ # =========================================================================== simulation / calibration
576
+
577
+
578
+ def simulate_lineage_trait(forest: LineageForest, h2: float, *, kernel: str = "exponential",
579
+ length_scale: float = 2.0, base=None, random_state: int = 0) -> np.ndarray:
580
+ """A feature with lineage heritability ``h2`` on the forest's real trees.
581
+
582
+ ``base`` (one value per row) supplies the non-heritable part: it is permuted within every unit
583
+ (destroying any lineage structure it had) and standardised within units; without ``base`` the
584
+ non-heritable part is Gaussian. Rows outside the forest are NaN.
585
+ """
586
+ if not 0 <= h2 < 1:
587
+ raise ValueError("h2 must be in [0, 1).")
588
+ rng = np.random.default_rng(random_state)
589
+ out = np.full(forest.n_cells, np.nan)
590
+ ks = forest.kernel(kernel, length_scale)
591
+ base = None if base is None else np.asarray(base, dtype=float)
592
+ for u, K in zip(forest.units, ks):
593
+ w, V = np.linalg.eigh(K)
594
+ g = V @ (np.sqrt(np.clip(w, 0, None)) * rng.normal(size=u.size))
595
+ if base is None:
596
+ e = rng.normal(size=u.size)
597
+ else:
598
+ e = base[u][rng.permutation(u.size)]
599
+ e = (e - e.mean()) / (e.std() if e.std() > 0 else 1.0)
600
+ out[u] = np.sqrt(h2) * g + np.sqrt(1 - h2) * e
601
+ return out
602
+
603
+
604
+ def lineage_power(forest: LineageForest, *, base=None, h2_grid=(0.0, 0.02, 0.05, 0.1, 0.2),
605
+ n_sim: int = 100, kernel: str = "exponential", length_scale: float = 2.0,
606
+ depth_effect: bool = True, n_perm: int = 199, alpha: float = 0.05,
607
+ random_state: int = 0, verbose: bool = True) -> pd.DataFrame:
608
+ """Bias, type I error (``h2 = 0``) and power of :func:`lineage_heritability` on these trees."""
609
+ rng = np.random.default_rng(random_state)
610
+ rows = []
611
+ for h2 in h2_grid:
612
+ est, p_perm, p_lrt = [], [], []
613
+ for s in range(n_sim):
614
+ seed = int(rng.integers(2**31))
615
+ y = simulate_lineage_trait(forest, h2, kernel=kernel, length_scale=length_scale, base=base,
616
+ random_state=seed)
617
+ r = lineage_heritability(np.nan_to_num(y), forest, kernel=kernel, length_scale=length_scale,
618
+ depth_effect=depth_effect, n_perm=n_perm, random_state=seed + 1,
619
+ verbose=False)
620
+ est.append(r.table["h2"].iloc[0])
621
+ p_perm.append(r.table["p_perm"].iloc[0] if n_perm else np.nan)
622
+ p_lrt.append(r.table["p_lrt"].iloc[0])
623
+ est, p_perm, p_lrt = map(np.asarray, (est, p_perm, p_lrt))
624
+ rows.append({"h2_true": h2, "n_sim": n_sim, "h2_mean": est.mean(), "h2_sd": est.std(ddof=1),
625
+ "rejection_perm": float((p_perm < alpha).mean()) if n_perm else np.nan,
626
+ "rejection_lrt": float((p_lrt < alpha).mean()), "kernel": kernel,
627
+ "length_scale": length_scale if kernel == "exponential" else np.nan})
628
+ log(f" h2={h2:.3f}: mean estimate {est.mean():.3f}, rejection perm "
629
+ f"{rows[-1]['rejection_perm']:.2f}, LRT {rows[-1]['rejection_lrt']:.2f}", verbose)
630
+ return pd.DataFrame(rows)
631
+
632
+
633
+ # =========================================================================== pairwise diagnostic
634
+ # The functions below implement the first, pairwise statistic (correlation of within-unit tree
635
+ # distances with state distances). It is superseded by lineage_heritability because a pair's tree
636
+ # distance is roughly the sum of the two cells' depths, which confounds a depth effect with lineage
637
+ # resemblance; it is kept so that earlier outputs remain reproducible.
638
+
639
+
640
+ def pair_table(clones, blocks=None, distances=None) -> pd.DataFrame:
641
+ """Within-clone, within-block cell pairs with their receptor distance."""
642
+ if distances is None:
643
+ raise ValueError("distances are required: give within-clone cell-pair receptor distances.")
644
+ d = pd.DataFrame({"i": np.asarray(distances["i"], dtype=np.int64),
645
+ "j": np.asarray(distances["j"], dtype=np.int64),
646
+ "d": np.asarray(distances["d"], dtype=float)})
647
+ clone_codes, _ = codes(pd.Series(clones).astype(str))
648
+ if blocks is None:
649
+ block_codes = clone_codes
650
+ else:
651
+ block_codes, _ = codes(pd.Series(clones).astype(str) + "\x1f" + pd.Series(blocks).astype(str))
652
+ same = (clone_codes[d["i"]] == clone_codes[d["j"]]) & (block_codes[d["i"]] == block_codes[d["j"]])
653
+ out = d[same & np.isfinite(d["d"])].copy()
654
+ out["block"] = block_codes[out["i"].to_numpy()]
655
+ out.attrs["dropped_cross_block"] = int((~same).sum())
656
+ out.attrs["block_codes"] = block_codes
657
+ return out.reset_index(drop=True)
658
+
659
+
660
+ def _block_center(v: np.ndarray, block: np.ndarray, n_blocks: int) -> np.ndarray:
661
+ cnt = np.bincount(block, minlength=n_blocks)
662
+ tot = np.bincount(block, weights=v, minlength=n_blocks)
663
+ return v - (tot / np.maximum(cnt, 1))[block]
664
+
665
+
666
+ def _residualise(v: np.ndarray, cov: np.ndarray | None) -> np.ndarray:
667
+ if cov is None or cov.size == 0:
668
+ return v
669
+ coef, *_ = np.linalg.lstsq(cov, v, rcond=None)
670
+ return v - cov @ coef
671
+
672
+
673
+ def _state_distance(x: np.ndarray, i: np.ndarray, j: np.ndarray) -> np.ndarray:
674
+ diff = x[i] - x[j]
675
+ return np.sqrt(np.einsum("ij,ij->i", diff, diff))
676
+
677
+
678
+ def _pooled_r(s_c, e, block, n_blocks, cov) -> float:
679
+ e_c = _block_center(e, block, n_blocks)
680
+ if cov is not None:
681
+ e_c = _residualise(e_c, cov)
682
+ num = float(s_c @ e_c)
683
+ den = np.sqrt(float(s_c @ s_c) * float(e_c @ e_c))
684
+ return num / den if den > 0 else np.nan
685
+
686
+
687
+ def _permute_within_blocks(block_of_cell: np.ndarray, rng: np.random.Generator) -> np.ndarray:
688
+ order = np.argsort(block_of_cell, kind="stable")
689
+ sizes = np.bincount(block_of_cell)
690
+ out = np.empty(order.size, dtype=np.int64)
691
+ start = 0
692
+ for s in sizes:
693
+ if s:
694
+ seg = order[start:start + s]
695
+ out[seg] = seg[rng.permutation(s)]
696
+ start += s
697
+ return out
698
+
699
+
700
+ @dataclass
701
+ class LineageSignal:
702
+ """Result of :func:`lineage_signal` (pairwise diagnostic)."""
703
+
704
+ r: float
705
+ p_value: float
706
+ null_mean: float
707
+ null_sd: float
708
+ n_pairs: int
709
+ n_blocks: int
710
+ n_clones: int
711
+ n_cells: int
712
+ settings: dict = field(default_factory=dict)
713
+ identical_pairs: int = 0
714
+ r_identical: float = float("nan")
715
+ null: np.ndarray | None = field(default=None, repr=False)
716
+ h2_equivalent: float | None = None
717
+ h2_ci: tuple | None = None
718
+
719
+ @property
720
+ def z(self) -> float:
721
+ return (self.r - self.null_mean) / self.null_sd if self.null_sd else float("nan")
722
+
723
+
724
+ def lineage_signal(x, clones, distances, *, blocks=None, covariates=None, include_identical: bool = True,
725
+ n_perm: int = 2000, random_state: int = 0, verbose: bool = True) -> LineageSignal:
726
+ """Pooled within-unit correlation of receptor distance and state distance (superseded)."""
727
+ require_positive_int(n_perm, "n_perm")
728
+ x = np.asarray(x, dtype=float)
729
+ pairs = pair_table(clones, blocks, distances)
730
+ if not include_identical:
731
+ pairs = pairs[pairs["d"] > 0].reset_index(drop=True)
732
+ if len(pairs) < 10:
733
+ raise ValueError(f"only {len(pairs)} usable within-clone pairs; need at least 10.")
734
+ block_of_cell = pairs.attrs["block_codes"]
735
+ used, block = np.unique(pairs["block"].to_numpy(), return_inverse=True)
736
+ n_blocks = used.size
737
+ i = pairs["i"].to_numpy()
738
+ j = pairs["j"].to_numpy()
739
+ s_c = _block_center(pairs["d"].to_numpy(), block, n_blocks)
740
+ cov = None
741
+ if covariates is not None:
742
+ c = np.asarray(covariates, dtype=float)
743
+ c = c[:, None] if c.ndim == 1 else c
744
+ parts = []
745
+ for k in range(c.shape[1]):
746
+ parts.append(_block_center(c[i, k] + c[j, k], block, n_blocks))
747
+ parts.append(_block_center(np.abs(c[i, k] - c[j, k]), block, n_blocks))
748
+ cov = np.column_stack(parts)
749
+ s_c = _residualise(s_c, cov)
750
+ obs = _pooled_r(s_c, _state_distance(x, i, j), block, n_blocks, cov)
751
+ rng = np.random.default_rng(random_state)
752
+ null = np.empty(n_perm)
753
+ for b in range(n_perm):
754
+ perm = _permute_within_blocks(block_of_cell, rng)
755
+ null[b] = _pooled_r(s_c, _state_distance(x, perm[i], perm[j]), block, n_blocks, cov)
756
+ finite = null[np.isfinite(null)]
757
+ centre = finite.mean()
758
+ p = (1 + np.sum(np.abs(finite - centre) >= abs(obs - centre) - 1e-12)) / (finite.size + 1)
759
+ ident = pairs["d"].to_numpy() == 0
760
+ r_ident = float("nan")
761
+ if ident.sum() >= 10 and (~ident).sum() >= 10:
762
+ e = _state_distance(x, i, j)
763
+ e_c = _block_center(e, block, n_blocks)
764
+ r_ident = float(e_c[ident].mean() - e_c[~ident].mean())
765
+ res = LineageSignal(
766
+ r=float(obs), p_value=float(p), null_mean=float(centre),
767
+ null_sd=float(finite.std(ddof=1)) if finite.size > 1 else float("nan"),
768
+ n_pairs=int(len(pairs)), n_blocks=int(n_blocks),
769
+ n_clones=int(pd.unique(np.asarray(clones, dtype=object)[np.unique(np.r_[i, j])]).size),
770
+ n_cells=int(np.unique(np.r_[i, j]).size), null=null, identical_pairs=int(ident.sum()),
771
+ r_identical=r_ident,
772
+ settings={"include_identical": bool(include_identical), "n_perm": int(n_perm),
773
+ "random_state": int(random_state), "blocked": blocks is not None,
774
+ "n_covariates": 0 if cov is None else cov.shape[1], "n_features": int(x.shape[1]),
775
+ "dropped_cross_block": int(pairs.attrs.get("dropped_cross_block", 0))},
776
+ )
777
+ log(f"lineage signal: r = {obs:+.4f}, p = {p:.3g}; {res.n_clones} clones, {res.n_pairs} pairs.", verbose)
778
+ return res
779
+
780
+
781
+ def planted_calibration(x, clones, distances, *, blocks=None, h2_grid=(0.0, 0.02, 0.05, 0.1, 0.2, 0.4),
782
+ n_sim: int = 50, n_perm: int = 299, length_scale: float | str = "median",
783
+ alpha: float = 0.05, random_state: int = 0, verbose: bool = True) -> pd.DataFrame:
784
+ """Planted-signal calibration of the pairwise diagnostic (plants on the real, un-nullified data)."""
785
+ rng = np.random.default_rng(random_state)
786
+ x = np.asarray(x, dtype=float)
787
+ pairs = pair_table(clones, blocks, distances)
788
+ block_of_cell = pairs.attrs["block_codes"]
789
+ used, block = np.unique(pairs["block"].to_numpy(), return_inverse=True)
790
+ n_blocks = used.size
791
+ i, j = pairs["i"].to_numpy(), pairs["j"].to_numpy()
792
+ d = pairs["d"].to_numpy()
793
+ s_c = _block_center(d, block, n_blocks)
794
+ pos = d[d > 0]
795
+ ls = float(np.median(pos)) if length_scale == "median" and pos.size else (
796
+ 1.0 if length_scale == "median" else float(length_scale))
797
+ cells_in_block = {b: np.flatnonzero(block_of_cell == b) for b in np.unique(block_of_cell)}
798
+ dist_lookup = {}
799
+ for b in used:
800
+ idx = cells_in_block[b]
801
+ pos_of = {c: k for k, c in enumerate(idx)}
802
+ m = np.zeros((idx.size, idx.size))
803
+ sel = pairs["block"].to_numpy() == b
804
+ for a_, b_, dd in zip(i[sel], j[sel], d[sel]):
805
+ m[pos_of[a_], pos_of[b_]] = m[pos_of[b_], pos_of[a_]] = dd
806
+ c = np.exp(-m / ls)
807
+ np.fill_diagonal(c, 1.0)
808
+ w, v = np.linalg.eigh(c)
809
+ dist_lookup[b] = (idx, v * np.sqrt(np.clip(w, 0, None)))
810
+ xb = x - pd.DataFrame(x).groupby(block_of_cell).transform("mean").to_numpy()
811
+ base_var = float(np.einsum("ij,ij->", xb, xb) / xb.shape[0])
812
+ rows = []
813
+ for h2 in h2_grid:
814
+ hits, rs = 0, []
815
+ for _ in range(n_sim):
816
+ xs = x.copy()
817
+ if h2 > 0:
818
+ u = rng.normal(size=x.shape[1])
819
+ u /= np.linalg.norm(u)
820
+ amp = np.sqrt(h2 / (1 - h2) * base_var)
821
+ for b, (idx, L) in dist_lookup.items():
822
+ xs[idx] += amp * (L @ rng.normal(size=idx.size))[:, None] * u[None, :]
823
+ obs = _pooled_r(s_c, _state_distance(xs, i, j), block, n_blocks, None)
824
+ null = np.empty(n_perm)
825
+ for k in range(n_perm):
826
+ perm = _permute_within_blocks(block_of_cell, rng)
827
+ null[k] = _pooled_r(s_c, _state_distance(xs, perm[i], perm[j]), block, n_blocks, None)
828
+ centre = null.mean()
829
+ pv = (1 + np.sum(np.abs(null - centre) >= abs(obs - centre) - 1e-12)) / (null.size + 1)
830
+ hits += pv < alpha
831
+ rs.append(obs)
832
+ rows.append({"h2_planted": h2, "n_sim": int(n_sim), "power": hits / n_sim, "mean_r": float(np.mean(rs)),
833
+ "sd_r": float(np.std(rs, ddof=1)) if n_sim > 1 else np.nan, "length_scale": ls})
834
+ return pd.DataFrame(rows)