threadfin 4.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
threadfin/dynamics.py ADDED
@@ -0,0 +1,275 @@
1
+ """Clonal memory across time points or tissues.
2
+
3
+ Question: when the same clone is sampled twice (two time points, or two
4
+ tissues), is it still in the same state?
5
+
6
+ Why not compare programme labels? Programmes are assigned per clone from
7
+ *all* of its cells, so every snapshot of a clone carries the same label by
8
+ construction and a label "transition matrix" is trivially diagonal.
9
+ :func:`clonal_memory` instead profiles every (clone, time) *snapshot*
10
+ separately and asks how similar a clone's consecutive snapshots are, compared
11
+ with snapshots of unrelated clones.
12
+
13
+ The memory index
14
+ ----------------
15
+ For consecutive snapshots ``a`` and ``b`` of the same clone, with ``n_a`` and
16
+ ``n_b`` cells and context-centred mean profiles ``m_a`` and ``m_b``::
17
+
18
+ D_same = ||m_a - m_b||^2 - sum_j sigma2_j * (1/n_a + 1/n_b)
19
+
20
+ where ``sigma2`` is the within-snapshot variance, so ``D_same`` estimates how
21
+ much the clone's *true* state changed (sampling noise removed). ``D_random``
22
+ is the same quantity when ``b`` is replaced by a snapshot of a different clone
23
+ from the same donor and time point. The memory index is::
24
+
25
+ M = 1 - mean(D_same) / mean(D_random)
26
+
27
+ ``M = 1``: clones keep everything that distinguished them; ``M = 0``: a clone
28
+ is no more similar to its former self than to a random clone. ``M`` behaves
29
+ like a test-retest reliability of clone state. The p-value compares
30
+ ``mean(D_same)`` with random re-pairings; the 95% interval bootstraps clones.
31
+
32
+ As a descriptive companion, each snapshot is also assigned to the nearest
33
+ programme centroid and the programme transition table is reported together
34
+ with its expectation under random pairing.
35
+ """
36
+
37
+ from __future__ import annotations
38
+
39
+ import warnings
40
+
41
+ import numpy as np
42
+ import pandas as pd
43
+
44
+ from ._utils import codes, get_uns, h5ad_safe, log, require_obs, require_positive_int
45
+ from .profiles import get_profiles, model_from_adata, variance_components
46
+
47
+ MEMORY_KEY = "memory"
48
+
49
+
50
+ def _ordered_levels(values, order):
51
+ levels = list(pd.unique(pd.Series(values).dropna().astype(str)))
52
+ if order is not None:
53
+ order = [str(o) for o in order]
54
+ missing = [lv for lv in levels if lv not in order]
55
+ if missing:
56
+ raise ValueError(f"order is missing levels: {missing}")
57
+ return [o for o in order if o in levels]
58
+ num = pd.to_numeric(pd.Series(levels).str.extract(r"(-?\d+\.?\d*)")[0], errors="coerce")
59
+ if num.notna().all():
60
+ return [levels[i] for i in np.argsort(num.to_numpy(), kind="stable")]
61
+ return sorted(levels)
62
+
63
+
64
+ def _snapshots(adata, model, clone_key, time_key, order, min_cells):
65
+ """Unshrunk mean profile, size and donor of every (clone, time) snapshot."""
66
+ obs = adata.obs
67
+ clone_str = obs[clone_key].astype("object").map(lambda c: str(c) if pd.notna(c) else None)
68
+ levels = _ordered_levels(obs.loc[clone_str.notna(), time_key], order)
69
+ t_index = obs[time_key].astype(str).map({lv: i for i, lv in enumerate(levels)})
70
+ valid = (clone_str.notna() & t_index.notna()).to_numpy()
71
+ key = clone_str[valid] + "\x1f" + t_index[valid].astype(int).astype(str)
72
+ snap_valid, snap_index = codes(key)
73
+ snap = -np.ones(adata.n_obs, dtype=np.int64)
74
+ snap[np.flatnonzero(valid)] = snap_valid
75
+ n_snap = len(snap_index)
76
+
77
+ import scipy.sparse as sp
78
+
79
+ ok = np.flatnonzero(snap >= 0)
80
+ member = sp.csr_matrix((np.ones(ok.size), (snap[ok], ok)), shape=(n_snap, snap.size))
81
+ n = np.asarray(member.sum(axis=1)).ravel()
82
+ means = np.asarray(member @ model.residuals) / np.maximum(n, 1)[:, None]
83
+ # within-snapshot noise variance (pure sampling noise, no temporal change)
84
+ sigma2 = variance_components(model.residuals, snap).sigma2
85
+ info = pd.DataFrame({
86
+ "clone": [s.split("\x1f")[0] for s in snap_index],
87
+ "t": [int(s.split("\x1f")[1]) for s in snap_index],
88
+ "n_cells": n.astype(int),
89
+ })
90
+ keep = (info["n_cells"] >= min_cells).to_numpy()
91
+ return info[keep].reset_index(drop=True), means[keep], sigma2, levels
92
+
93
+
94
+ def clonal_memory(
95
+ adata,
96
+ time_key: str,
97
+ *,
98
+ order=None,
99
+ programme_key: str = "clone_programme",
100
+ min_cells: int = 1,
101
+ min_pairs: int = 10,
102
+ n_null: int = 500,
103
+ n_boot: int = 500,
104
+ random_state: int = 0,
105
+ verbose: bool = True,
106
+ ) -> dict:
107
+ """Do clones keep their state between time points (or tissues)?
108
+
109
+ Parameters
110
+ ----------
111
+ time_key
112
+ ``obs`` column with the time point (or tissue) of each cell.
113
+ order
114
+ Order of the levels of ``time_key``; inferred from numbers in the
115
+ labels (``d7`` < ``d28``) when omitted. Each clone's consecutive
116
+ observed levels are paired.
117
+ programme_key
118
+ Programme column of the clone table (for the descriptive transition
119
+ table); skipped when absent.
120
+ min_cells
121
+ Minimum cells per snapshot.
122
+ min_pairs
123
+ Fewest snapshot pairs for which an index is reported. Below this the
124
+ bootstrap interval collapses and the estimate says nothing, so the
125
+ function raises instead of returning a number.
126
+ n_null, n_boot
127
+ Random re-pairings for the p-value; bootstrap replicates for the CI.
128
+
129
+ Returns
130
+ -------
131
+ dict with ``memory_index`` (see module docstring), its bootstrap
132
+ ``memory_index_ci``, ``p_value``, ``n_clones``, ``n_pairs``, the
133
+ per-pair table ``pairs`` and, when programmes exist, programme
134
+ ``persistence`` with ``persistence_null`` and the ``transitions`` /
135
+ ``transitions_expected`` tables. Stored in
136
+ ``adata.uns['threadfin']['memory'][time_key]``.
137
+ """
138
+ for name, value in [("min_cells", min_cells), ("min_pairs", min_pairs), ("n_null", n_null), ("n_boot", n_boot)]:
139
+ require_positive_int(value, name)
140
+ prof = get_profiles(adata)
141
+ clone_key = prof["params"]["clone_key"]
142
+ require_obs(adata, time_key)
143
+ rng = np.random.default_rng(random_state)
144
+ model = model_from_adata(adata, unsmoothed=True) # smoothing would leak between snapshots
145
+ snaps, means, sigma2, levels = _snapshots(adata, model, clone_key, time_key, order, min_cells)
146
+ table = prof["clone_table"]
147
+ donor = table["donor"].astype(str) if "donor" in table.columns else pd.Series(dtype=str)
148
+ snaps["donor"] = snaps["clone"].map(donor).fillna("all").to_numpy()
149
+
150
+ # consecutive snapshot pairs of the same clone
151
+ pa, pb = [], []
152
+ for _, grp in snaps.groupby("clone", sort=False):
153
+ if len(grp) >= 2:
154
+ rows = grp.sort_values("t").index.to_numpy()
155
+ pa.extend(rows[:-1])
156
+ pb.extend(rows[1:])
157
+ pa, pb = np.asarray(pa, dtype=np.int64), np.asarray(pb, dtype=np.int64)
158
+ if pa.size == 0:
159
+ raise ValueError(f"No clone has >= {min_cells} cells at two levels of '{time_key}'.")
160
+ if pa.size < min_pairs:
161
+ raise ValueError(f"Only {pa.size} clones have >= {min_cells} cells at two levels of '{time_key}' "
162
+ f"(need >= {min_pairs}); no memory index is reported.")
163
+
164
+ n_cells = snaps["n_cells"].to_numpy(dtype=float)
165
+ noise = sigma2.sum()
166
+
167
+ def corrected_sqdist(a, b):
168
+ diff = means[a] - means[b]
169
+ return np.einsum("ij,ij->i", diff, diff) - noise * (1.0 / n_cells[a] + 1.0 / n_cells[b])
170
+
171
+ # candidate substitutes for every pair: other clones, same donor and time point
172
+ t_arr, d_arr, c_arr = snaps["t"].to_numpy(), snaps["donor"].to_numpy(), snaps["clone"].to_numpy()
173
+ pools: dict = {}
174
+ for i in range(len(snaps)):
175
+ pools.setdefault((d_arr[i], t_arr[i]), []).append(i)
176
+ cands = []
177
+ for a, b in zip(pa, pb):
178
+ pool = np.asarray(pools[(d_arr[b], t_arr[b])])
179
+ pool = pool[c_arr[pool] != c_arr[a]]
180
+ cands.append(pool)
181
+ usable = np.array([c.size > 0 for c in cands])
182
+ n_unmatched = int((~usable).sum())
183
+ pa, pb = pa[usable], pb[usable]
184
+ if pa.size < min_pairs:
185
+ raise ValueError(f"Only {pa.size} snapshot pairs have another clone from the same donor "
186
+ f"and target time (need >= {min_pairs}); no memory index is reported.")
187
+ cands = [c for c, u in zip(cands, usable) if u]
188
+
189
+ d_same = corrected_sqdist(pa, pb)
190
+
191
+ def draw():
192
+ return np.array([c[rng.integers(c.size)] for c in cands])
193
+
194
+ null_stats = np.empty(n_null)
195
+ d_random_all = []
196
+ for r in range(n_null):
197
+ dr = corrected_sqdist(pa, draw())
198
+ null_stats[r] = dr.mean()
199
+ if r < 50:
200
+ d_random_all.append(dr)
201
+ d_random = np.mean(d_random_all, axis=0) # per-pair expected distance to a random clone
202
+ if not np.isfinite(d_random).all() or d_random.mean() <= 0:
203
+ raise ValueError("Noise-corrected matched-clone distance is non-positive; "
204
+ "the memory ratio is not identifiable for these samples.")
205
+ mem = 1.0 - d_same.mean() / d_random.mean()
206
+ p_value = (1 + int((null_stats <= d_same.mean()).sum())) / (n_null + 1)
207
+
208
+ # bootstrap over clones
209
+ clones_of_pairs = c_arr[pa]
210
+ uniq, inv = np.unique(clones_of_pairs, return_inverse=True)
211
+ boots = np.empty(n_boot)
212
+ for r in range(n_boot):
213
+ w = np.bincount(rng.integers(0, uniq.size, size=uniq.size), minlength=uniq.size)[inv]
214
+ denom = np.sum(w * d_random)
215
+ boots[r] = 1.0 - np.sum(w * d_same) / denom if denom > 0 else np.nan
216
+ identifiable = np.isfinite(boots)
217
+ # Dropping undefined ratios would produce a falsely reassuring conditional CI.
218
+ ci = (float(np.quantile(boots, 0.025)), float(np.quantile(boots, 0.975))) if identifiable.all() else (np.nan, np.nan)
219
+ if not identifiable.all():
220
+ warnings.warn("The confidence interval of the memory index could not be computed: in some bootstrap "
221
+ "resamples of clones no between-clone difference was left to compare with (too few clones "
222
+ "seen at two levels). The memory index and its P value are unaffected; the interval is "
223
+ "reported as missing.", stacklevel=2)
224
+
225
+ pairs = pd.DataFrame({
226
+ "clone": clones_of_pairs, "earlier": [levels[t] for t in t_arr[pa]],
227
+ "later": [levels[t] for t in t_arr[pb]], "n_cells_earlier": n_cells[pa].astype(int),
228
+ "n_cells_later": n_cells[pb].astype(int), "change": d_same, "change_vs_random": d_random,
229
+ })
230
+ res = {
231
+ "time_key": time_key, "levels": levels, "n_clones": int(uniq.size), "n_pairs": int(pa.size),
232
+ "n_unmatched_pairs": n_unmatched, "null_matching": "same donor and target time; no cross-donor fallback",
233
+ "memory_index": float(mem), "memory_index_ci": ci, "p_value": float(p_value),
234
+ "bootstrap_identifiable_fraction": float(identifiable.mean()),
235
+ "mean_change": float(d_same.mean()), "mean_change_random": float(d_random.mean()),
236
+ "pairs": pairs,
237
+ }
238
+
239
+ # descriptive: programme of each snapshot by nearest programme centroid (needs >= 2 programmes)
240
+ if programme_key in table.columns and table[programme_key].dropna().nunique() >= 2:
241
+ lab = table[programme_key].dropna().astype(str)
242
+ clone_means = pd.DataFrame(index=snaps.index)
243
+ centroids, names = [], sorted(lab.unique(), key=lambda s: (len(s), s))
244
+ member_snaps = snaps["clone"].map(lab)
245
+ for name in names:
246
+ rows = np.flatnonzero((member_snaps == name).to_numpy())
247
+ centroids.append(means[rows].mean(axis=0) if rows.size else np.full(means.shape[1], np.nan))
248
+ centroids = np.vstack(centroids)
249
+ d2 = ((means[:, None, :] - centroids[None, :, :]) ** 2).sum(-1)
250
+ nearest = np.nanargmin(np.where(np.isfinite(d2), d2, np.inf), axis=1)
251
+ same = nearest[pa] == nearest[pb]
252
+ null_same = np.mean([np.mean(nearest[pa] == nearest[draw()]) for _ in range(100)])
253
+ trans = pd.crosstab(pd.Categorical(np.asarray(names)[nearest[pa]], categories=names),
254
+ pd.Categorical(np.asarray(names)[nearest[pb]], categories=names), dropna=False)
255
+ trans.index.name, trans.columns.name = "earlier", "later"
256
+ exp = np.zeros_like(trans.to_numpy(), dtype=float)
257
+ for _ in range(100):
258
+ exp += pd.crosstab(pd.Categorical(np.asarray(names)[nearest[pa]], categories=names),
259
+ pd.Categorical(np.asarray(names)[nearest[draw()]], categories=names),
260
+ dropna=False).to_numpy()
261
+ res.update({
262
+ "persistence": float(same.mean()), "persistence_null": float(null_same),
263
+ "transitions": trans,
264
+ "transitions_expected": pd.DataFrame(exp / 100, index=trans.index, columns=trans.columns),
265
+ })
266
+ pairs["programme_earlier"] = np.asarray(names)[nearest[pa]]
267
+ pairs["programme_later"] = np.asarray(names)[nearest[pb]]
268
+ del clone_means
269
+
270
+ get_uns(adata).setdefault(MEMORY_KEY, {})[time_key] = h5ad_safe(res)
271
+ extra = (f"; same programme {res['persistence']:.2f} vs {res['persistence_null']:.2f} for random clones"
272
+ if "persistence" in res else "")
273
+ log(f"clonal memory over '{time_key}': {uniq.size} clones, {pa.size} snapshot pairs; memory index "
274
+ f"{mem:.2f} [{ci[0]:.2f}, {ci[1]:.2f}], p = {p_value:.3g}{extra}.", verbose)
275
+ return res
threadfin/export.py ADDED
@@ -0,0 +1,215 @@
1
+ """When did a clone's output cells leave the germinal centre?
2
+
3
+ Question: a germinal centre and the plasma cells it exports are different compartments, but the receptor
4
+ says which of them belong to the same clone. Somatic hypermutation only ever accumulates, so inside one
5
+ clone the mutation load of an output cell cannot exceed the load the germinal centre had **when that cell
6
+ left**. The difference between a clone's germinal-centre cells and its own output cells is therefore a
7
+ clock, and it runs in only one direction.
8
+
9
+ lag(clone) = mean V-region mutation in the clone's output cells
10
+ - mean V-region mutation in the clone's germinal-centre cells
11
+
12
+ A negative lag means the output left while the clone was less mutated than it is now; zero means the export
13
+ is contemporaneous with the germinal centre that is being sampled.
14
+
15
+ Why this needs the clone as the unit
16
+ ------------------------------------
17
+ At cell level, germinal-centre and plasma-cell populations differ in mutation for many reasons — different
18
+ donors, different visits, different sorts, and the simple fact that the two compartments are populated at
19
+ different times. None of those apply here, because every comparison is a clone against **itself**: the same
20
+ receptor, the same animal, and (with ``within``) the same visit and tissue. The clone is its own control,
21
+ which is what makes a one-directional reading of the difference defensible.
22
+
23
+ Turning the clock into a time
24
+ -----------------------------
25
+ Given a time column, the germinal centre's own maturation curve converts a mutation gap into days.
26
+ :func:`export_lag` fits mutation against time across the source compartment (Theil-Sen, so a few
27
+ hypermutated outliers cannot set the slope) and reports each clone's lag in days as well as in mutation
28
+ units. The conversion assumes the clone matured at the rate the population did, which is an assumption
29
+ about the average and is reported as such.
30
+
31
+ What it does not establish
32
+ --------------------------
33
+ No parent-offspring relationship between any two cells; no claim that the output cells seen at a visit
34
+ descend from the germinal-centre cells seen at that same visit; and nothing about affinity, which is not
35
+ measured by mutation count.
36
+ """
37
+
38
+ from __future__ import annotations
39
+
40
+ import numpy as np
41
+ import pandas as pd
42
+
43
+ from ._utils import h5ad_safe, log, require_obs
44
+
45
+ __all__ = ["export_lag"]
46
+
47
+ SWITCHED = ("IGHG", "IGHA", "IGHE")
48
+ UNSWITCHED = ("IGHM", "IGHD")
49
+
50
+
51
+ def _switched(values: pd.Series) -> np.ndarray:
52
+ iso = values.astype(str).str.upper()
53
+ known = iso.str.startswith(SWITCHED + UNSWITCHED)
54
+ return np.where(known, iso.str.startswith(SWITCHED).astype(float), np.nan)
55
+
56
+
57
+ def _theil_sen(x: np.ndarray, y: np.ndarray, max_pairs: int = 5_000_000, random_state: int = 0) -> float:
58
+ """Median pairwise slope; robust to the long right tail of mutation frequencies.
59
+
60
+ All pairs are used up to ``max_pairs``; above that (about 3,200 cells) the median is taken over a
61
+ fixed-seed random sample of ``max_pairs`` pairs, so memory stays bounded (all pairs of 40,000 cells
62
+ would be 8 x 10^8).
63
+ """
64
+ ok = np.isfinite(x) & np.isfinite(y)
65
+ x, y = x[ok], y[ok]
66
+ if x.size < 3 or np.ptp(x) == 0:
67
+ return float("nan")
68
+ n_pairs = x.size * (x.size - 1) // 2
69
+ if n_pairs <= max_pairs:
70
+ i, j = np.triu_indices(x.size, 1)
71
+ else:
72
+ rng = np.random.default_rng(random_state)
73
+ i = rng.integers(0, x.size, max_pairs)
74
+ j = rng.integers(0, x.size, max_pairs)
75
+ distinct = i != j
76
+ i, j = i[distinct], j[distinct]
77
+ dx = x[j] - x[i]
78
+ keep = dx != 0
79
+ if not keep.any():
80
+ return float("nan")
81
+ return float(np.median((y[j] - y[i])[keep] / dx[keep]))
82
+
83
+
84
+ def export_lag(
85
+ adata,
86
+ *,
87
+ state_key: str,
88
+ source_states,
89
+ output_states,
90
+ clone_key: str = "clone_id",
91
+ mutation_key: str = "mutation_frequency",
92
+ within=(),
93
+ time_key: str | None = None,
94
+ isotype_key: str | None = "isotype",
95
+ switched_only: bool = True,
96
+ min_cells: int = 1,
97
+ ) -> dict:
98
+ """Date each clone's output against its own germinal centre.
99
+
100
+ Parameters
101
+ ----------
102
+ state_key
103
+ ``obs`` column holding the captured cell state.
104
+ source_states, output_states
105
+ Which levels of ``state_key`` are the germinal centre and which are its output, for example
106
+ ``source_states=["GC"]`` and ``output_states=["PB", "LNPC"]``.
107
+ within
108
+ ``obs`` columns that must match between the two cells being compared, on top of the clone itself.
109
+ Pass the visit and the tissue here: a clone's germinal-centre cell from one visit should not be
110
+ compared with its plasma cell from another.
111
+ time_key
112
+ Optional numeric (or ``d28``-style) column. When given, the source compartment's own maturation
113
+ slope converts each clone's lag into days.
114
+ switched_only
115
+ Compare only class-switched cells. Class switching is irreversible, so an IgM cell and an IgG cell
116
+ of one clone are separated by an event that mutation count does not describe.
117
+ min_cells
118
+ Cells of the clone required on each side of a comparison.
119
+
120
+ Returns
121
+ -------
122
+ dict with
123
+
124
+ ``pairs``
125
+ one row per clone (and stratum): ``source``, ``output``, ``lag``, cell counts, and ``lag_days``
126
+ when ``time_key`` is given.
127
+ ``n_clones``, ``n_output_earlier``, ``n_output_later``
128
+ how many clones were compared and how the lag fell.
129
+ ``median_lag``
130
+ median lag in mutation-frequency units, with a clone bootstrap interval.
131
+ ``maturation_slope``
132
+ mutation per unit time in the source compartment, when ``time_key`` is given.
133
+
134
+ Examples
135
+ --------
136
+ >>> res = export_lag(adata, state_key="cell_state", source_states=["GC"],
137
+ ... output_states=["PB"], within=("timepoint",), time_key="timepoint")
138
+ >>> res["median_lag"], res["n_output_earlier"], res["n_clones"]
139
+ """
140
+ require_obs(adata, clone_key, state_key, mutation_key, context="export_lag")
141
+ source_states, output_states = list(source_states), list(output_states)
142
+ overlap = set(source_states) & set(output_states)
143
+ if overlap:
144
+ raise ValueError(f"source_states and output_states overlap: {sorted(overlap)}")
145
+
146
+ obs = adata.obs
147
+ keep = obs[clone_key].notna() & obs[state_key].isin(source_states + output_states)
148
+ keep &= pd.to_numeric(obs[mutation_key], errors="coerce").notna()
149
+ if switched_only:
150
+ if isotype_key is None or isotype_key not in obs:
151
+ raise ValueError("switched_only=True needs an isotype column; pass isotype_key=None to skip.")
152
+ keep &= _switched(obs[isotype_key]) == 1
153
+ d = obs.loc[keep, [clone_key, state_key, mutation_key, *within]].copy()
154
+ if time_key is not None and time_key not in d.columns:
155
+ d[time_key] = obs.loc[keep, time_key]
156
+ if d.empty:
157
+ raise ValueError("No cells left after filtering; check the state levels and the isotype column.")
158
+ d["_side"] = np.where(d[state_key].isin(source_states), "source", "output")
159
+ d[mutation_key] = pd.to_numeric(d[mutation_key], errors="coerce")
160
+
161
+ keys = [clone_key, *within]
162
+ g = d.groupby(keys + ["_side"])[mutation_key].agg(["mean", "size"])
163
+ wide = g.unstack("_side")
164
+ missing = [s for s in ("source", "output") if ("mean", s) not in wide.columns]
165
+ if missing:
166
+ # one side has no usable cell at all — say which, rather than failing on a missing column
167
+ raise ValueError(
168
+ f"No cell with a mutation frequency on the {' and '.join(missing)} side, so no clone can be "
169
+ "captured on both sides. Check the state levels, the isotype filter and the mutation column.")
170
+ wide = wide.dropna(subset=[("mean", "source"), ("mean", "output")])
171
+ wide = wide[(wide[("size", "source")] >= min_cells) & (wide[("size", "output")] >= min_cells)]
172
+ if wide.empty:
173
+ raise ValueError("No clone was captured on both sides; widen `within` or lower `min_cells`.")
174
+
175
+ pairs = pd.DataFrame({
176
+ "source": wide[("mean", "source")].to_numpy(),
177
+ "output": wide[("mean", "output")].to_numpy(),
178
+ "n_source": wide[("size", "source")].to_numpy().astype(int),
179
+ "n_output": wide[("size", "output")].to_numpy().astype(int),
180
+ }, index=wide.index).reset_index()
181
+ pairs["lag"] = pairs["output"] - pairs["source"]
182
+
183
+ slope = np.nan
184
+ if time_key is not None:
185
+ t_all = pd.to_numeric(
186
+ obs.loc[keep, time_key].astype(str).str.extract(r"(\d+\.?\d*)")[0], errors="coerce")
187
+ src = d["_side"].eq("source").to_numpy()
188
+ slope = _theil_sen(t_all.to_numpy()[src], d[mutation_key].to_numpy()[src])
189
+ if np.isfinite(slope) and slope > 0:
190
+ pairs["lag_days"] = pairs["lag"] / slope
191
+
192
+ rng = np.random.default_rng(0)
193
+ lags = pairs["lag"].to_numpy()
194
+ boot = [np.median(lags[rng.integers(0, lags.size, lags.size)]) for _ in range(1000)]
195
+ out = {
196
+ "pairs": pairs,
197
+ "n_clones": int(len(pairs)),
198
+ "n_output_earlier": int((pairs["lag"] < 0).sum()),
199
+ "n_output_later": int((pairs["lag"] > 0).sum()),
200
+ "n_equal": int((pairs["lag"] == 0).sum()),
201
+ "median_lag": float(np.median(lags)),
202
+ "median_lag_ci": (float(np.percentile(boot, 2.5)), float(np.percentile(boot, 97.5))),
203
+ "maturation_slope": float(slope),
204
+ "median_lag_days": float(np.median(pairs["lag_days"])) if "lag_days" in pairs else float("nan"),
205
+ "settings": {"source_states": source_states, "output_states": output_states,
206
+ "within": list(within), "switched_only": bool(switched_only),
207
+ "min_cells": int(min_cells), "time_key": time_key},
208
+ }
209
+ log(f"export_lag: {out['n_clones']} clones captured on both sides; "
210
+ f"{out['n_output_earlier']} with output earlier than their own source, "
211
+ f"{out['n_output_later']} later; median lag {out['median_lag']:+.4f}"
212
+ + (f" ({out['median_lag_days']:+.1f} days)" if np.isfinite(out["median_lag_days"]) else ""))
213
+ adata.uns.setdefault("threadfin", {})["export_lag"] = h5ad_safe({
214
+ k: v for k, v in out.items() if k != "pairs"})
215
+ return out
@@ -0,0 +1,121 @@
1
+ """A train-only expression embedding for prospective/held-out validation."""
2
+ from __future__ import annotations
3
+ import json
4
+ from pathlib import Path
5
+ import numpy as np
6
+ import pandas as pd
7
+ import scipy.sparse as sp
8
+ from sklearn.decomposition import PCA
9
+ from ._utils import require_positive_int
10
+
11
+
12
+ class FrozenExpressionModel:
13
+ """HVG selection, scaling and PCA fitted on reference cells only.
14
+
15
+ Per-cell normalisation is to 10,000 counts, followed by log1p.
16
+ Seurat HVGs exclude immunoglobulin and TCR genes. Mean, standard
17
+ deviation, clipping and PCA are frozen for all query cells. There is no
18
+ Harmony step: a query donor cannot change the fitted coordinates.
19
+ This controls train/query leakage, not biological batch confounding.
20
+ """
21
+
22
+ @staticmethod
23
+ def _counts(adata, layer):
24
+ x = adata.X if layer is None else adata.layers[layer]
25
+ values = x.data if sp.issparse(x) else np.asarray(x)
26
+ if not np.isfinite(values).all() or np.any(values < 0):
27
+ raise ValueError("Frozen expression embedding requires finite non-negative counts.")
28
+ if not adata.var_names.is_unique:
29
+ raise ValueError("Gene names must be unique.")
30
+ return x
31
+
32
+ @staticmethod
33
+ def _lognorm(x):
34
+ total = np.asarray(x.sum(axis=1)).ravel()
35
+ if np.any(total <= 0):
36
+ raise ValueError("Every input cell needs a positive library size.")
37
+ factor = 1e4 / total
38
+ if sp.issparse(x):
39
+ out = x.tocsr().multiply(factor[:, None]).tocsr()
40
+ out.data = np.log1p(out.data)
41
+ return out
42
+ return np.log1p(np.asarray(x, dtype=float) * factor[:, None])
43
+
44
+ @classmethod
45
+ def fit(cls, reference, *, counts_layer=None, n_top_genes=2000,
46
+ n_comps=30, clip=10.0, random_state=0, batch_size=2048):
47
+ """Fit all data-dependent choices on reference cells, without mutating them."""
48
+ import anndata as ad
49
+ import scanpy as sc
50
+ from .pp import ig_gene_mask
51
+ for name, value in [("n_top_genes", n_top_genes), ("n_comps", n_comps), ("batch_size", batch_size)]:
52
+ require_positive_int(value, name)
53
+ if not np.isfinite(clip) or clip <= 0:
54
+ raise ValueError("clip must be positive and finite.")
55
+ if reference.n_obs < 3:
56
+ raise ValueError("At least three reference cells are required.")
57
+ x = cls._lognorm(cls._counts(reference, counts_layer))
58
+ genes = pd.Index(reference.var_names).astype(str)
59
+ keep = ~ig_gene_mask(genes, constant=True, tr_genes=True)
60
+ if keep.sum() < 2:
61
+ raise ValueError("At least two non-receptor genes are required.")
62
+ tmp = ad.AnnData(x[:, keep], var=pd.DataFrame(index=genes[keep]))
63
+ hv = sc.pp.highly_variable_genes(tmp, n_top_genes=min(n_top_genes, tmp.n_vars),
64
+ flavor="seurat", inplace=False)
65
+ selected = np.flatnonzero(keep)[hv.highly_variable.to_numpy()]
66
+ sub = x[:, selected]
67
+ sub = sub.toarray() if sp.issparse(sub) else np.asarray(sub)
68
+ obj = cls()
69
+ obj.gene_universe = genes.to_numpy(dtype=str)
70
+ obj.genes = genes[selected].to_numpy(dtype=str)
71
+ obj.mean = sub.mean(axis=0)
72
+ obj.scale = sub.std(axis=0, ddof=1)
73
+ obj.scale[obj.scale <= 0] = 1.0
74
+ obj.clip, obj.batch_size = float(clip), int(batch_size)
75
+ z = np.clip((sub-obj.mean) / obj.scale, -clip, clip)
76
+ n_comps = min(n_comps, reference.n_obs-1, len(selected))
77
+ pca = PCA(n_components=n_comps, svd_solver="randomized", random_state=random_state)
78
+ pca.fit(z)
79
+ obj.components = pca.components_
80
+ obj.pca_mean = pca.mean_
81
+ obj.explained_variance_ratio = pca.explained_variance_ratio_
82
+ obj.random_state = int(random_state)
83
+ obj.n_reference_cells = int(reference.n_obs)
84
+ return obj
85
+
86
+ def transform(self, query, *, counts_layer=None):
87
+ """Project query cells in batches; no fit, centering update or integration."""
88
+ counts = self._counts(query, counts_layer)
89
+ genes = pd.Index(query.var_names).astype(str)
90
+ if len(genes) != len(self.gene_universe) or set(genes) != set(self.gene_universe):
91
+ raise ValueError("Query must contain the same gene universe as the reference (order may differ).")
92
+ selected = genes.get_indexer(self.genes)
93
+ out = np.empty((query.n_obs, len(self.components)))
94
+ for start in range(0, query.n_obs, self.batch_size):
95
+ stop = min(start+self.batch_size, query.n_obs)
96
+ sub = self._lognorm(counts[start:stop])[:, selected]
97
+ sub = sub.toarray() if sp.issparse(sub) else np.asarray(sub)
98
+ z = np.clip((sub-self.mean) / self.scale, -self.clip, self.clip)
99
+ out[start:stop] = (z-self.pca_mean) @ self.components.T
100
+ return out
101
+
102
+ def save(self, path):
103
+ """Save frozen numeric parameters without pickle."""
104
+ parameters = {k: getattr(self, k) for k in ("clip", "batch_size", "random_state", "n_reference_cells")}
105
+ with Path(path).open("wb") as handle:
106
+ np.savez_compressed(handle, parameters=json.dumps(parameters),
107
+ **{k: getattr(self, k) for k in
108
+ ("gene_universe", "genes", "mean", "scale", "components",
109
+ "pca_mean", "explained_variance_ratio")})
110
+
111
+ @classmethod
112
+ def load(cls, path):
113
+ """Read frozen numeric parameters with pickle disabled."""
114
+ obj = cls()
115
+ with np.load(path, allow_pickle=False) as saved:
116
+ for key, value in json.loads(str(saved["parameters"])).items():
117
+ setattr(obj, key, value)
118
+ for key in ("gene_universe", "genes", "mean", "scale", "components", "pca_mean",
119
+ "explained_variance_ratio"):
120
+ setattr(obj, key, saved[key].copy())
121
+ return obj