threadfin 4.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
threadfin/density.py ADDED
@@ -0,0 +1,409 @@
1
+ """Sampling-aware clone state proportions on a frozen reference.
2
+
3
+ Integer counts in fixed Voronoi regions have a multinomial likelihood with a
4
+ Dirichlet prior. Credible intervals condition on the reference and partition;
5
+ they do not include reference uncertainty, donor effects or capture bias.
6
+ A fixed Gaussian mixture provides an optional smooth display, not a fitted
7
+ cell-level mixture model. This module uses no graph diffusion, RFF or PCA.
8
+ """
9
+ from __future__ import annotations
10
+ from dataclasses import dataclass
11
+ import hashlib
12
+ import json
13
+ from pathlib import Path
14
+
15
+ import numpy as np
16
+ import pandas as pd
17
+ from scipy.spatial.distance import cdist, pdist
18
+ from scipy.special import logsumexp
19
+ from scipy.stats import beta
20
+ from sklearn.cluster import KMeans
21
+ from ._utils import get_basis, get_uns, require_obs, require_positive_int
22
+
23
+
24
+ def _matrix(x, n_features=None):
25
+ x = np.asarray(x, dtype=float)
26
+ if x.ndim != 2 or min(x.shape) == 0 or not np.isfinite(x).all():
27
+ raise ValueError("Provide a non-empty, finite cell-by-feature matrix.")
28
+ if n_features is not None and x.shape[1] != n_features:
29
+ raise ValueError("Query and reference must use the same frozen feature coordinates.")
30
+ return x
31
+
32
+
33
+ def _labels(values, n, name):
34
+ s = pd.Series(values)
35
+ if len(s) != n or s.isna().any() or s.astype(str).str.strip().eq("").any():
36
+ raise ValueError(f"{name} needs one non-missing, non-blank label per cell.")
37
+ unique = pd.Index(pd.unique(s))
38
+ if not unique.astype(str).is_unique:
39
+ raise ValueError(f"{name} must remain unique when converted to strings.")
40
+ return s.astype(str).to_numpy()
41
+
42
+
43
+ @dataclass
44
+ class DensityResult:
45
+ """Posterior region weights and conditional marginal intervals, one row per clone."""
46
+ proportions: pd.DataFrame
47
+ raw_proportions: pd.DataFrame
48
+ counts: pd.DataFrame
49
+ background: pd.DataFrame
50
+ posterior_alpha: pd.DataFrame
51
+ lower: pd.DataFrame
52
+ upper: pd.DataFrame
53
+ sampling_lower: pd.DataFrame
54
+ sampling_upper: pd.DataFrame
55
+ clone_table: pd.DataFrame
56
+ settings: dict
57
+
58
+ def to_uns(self):
59
+ """Tables and metadata suitable for AnnData serialization."""
60
+ return {k: getattr(self, k) for k in self.__dataclass_fields__}
61
+
62
+ def contrast(self, left, right, *, n_draws=4000, random_state=0):
63
+ """Posterior differences (left minus right), conditional on the reference.
64
+
65
+ Intervals are marginal, without multiplicity correction. The posterior
66
+ probability of a positive difference is not a p-value. Dependence due
67
+ to a shared, uncertain reference is not modelled.
68
+ """
69
+ require_positive_int(n_draws, "n_draws")
70
+ left, right = str(left), str(right)
71
+ a, b = self.posterior_alpha.loc[left].to_numpy(), self.posterior_alpha.loc[right].to_numpy()
72
+ rng = np.random.default_rng(random_state)
73
+ delta = (rng.dirichlet(a, n_draws) - rng.dirichlet(b, n_draws)
74
+ if left != right else np.zeros((n_draws, len(a))))
75
+ tail = (1 - self.settings["credible_level"]) / 2
76
+ return pd.DataFrame({
77
+ "difference": self.proportions.loc[left] - self.proportions.loc[right],
78
+ "lower": np.quantile(delta, tail, axis=0),
79
+ "upper": np.quantile(delta, 1 - tail, axis=0),
80
+ "probability_positive": (delta > 0).mean(axis=0),
81
+ }, index=self.proportions.columns)
82
+
83
+
84
+ class StateDensityModel:
85
+ """Reusable regions and a clone-balanced reference background.
86
+
87
+ Freeze RNA preprocessing as well as this model for inductive validation.
88
+ Reusing an embedding fitted on query cells is transductive. When a query
89
+ clone occurs in the reference, its entire contribution is excluded from
90
+ the background, including all contexts. Region centres remain fixed; this
91
+ does not constitute a refit without that clone. Use disjoint reference
92
+ clones for a fully independent evaluation.
93
+ """
94
+
95
+ @classmethod
96
+ def fit(cls, x, clone_ids, *, contexts=None, n_states=16,
97
+ prior_strength=2.0, background_floor=0.05,
98
+ scales=(0.5, 1.0, 2.0), random_state=0, batch_size=4096):
99
+ """Fit once, optionally choosing prior strength on reference split cells.
100
+
101
+ The default prior strength 2.0 is fixed. 'auto' chooses from a fixed
102
+ grid using clone-balanced held-out-cell predictive log scores and
103
+ leave-own-clone-out backgrounds. It never uses outcome labels.
104
+ A small uniform background component keeps every region positive.
105
+ """
106
+ x = _matrix(x)
107
+ require_positive_int(n_states, "n_states")
108
+ require_positive_int(batch_size, "batch_size")
109
+ if not 2 <= n_states <= len(x):
110
+ raise ValueError("n_states must be between 2 and the number of reference cells.")
111
+ clones = _labels(clone_ids, len(x), "clone_ids")
112
+ ctx = np.repeat("all", len(x)) if contexts is None else _labels(contexts, len(x), "contexts")
113
+ if np.unique(clones).size < 2:
114
+ raise ValueError("At least two reference clones are required for leave-clone-out backgrounds.")
115
+ if not np.isfinite(background_floor) or not 0 < background_floor <= 1:
116
+ raise ValueError("background_floor must be in (0, 1].")
117
+ scales = np.asarray(scales, dtype=float)
118
+ if scales.ndim != 1 or not scales.size or not np.isfinite(scales).all() or (scales <= 0).any():
119
+ raise ValueError("scales must be a non-empty vector of positive finite numbers.")
120
+ if prior_strength != "auto":
121
+ if not np.isfinite(float(prior_strength)) or float(prior_strength) <= 0:
122
+ raise ValueError("prior_strength must be positive or 'auto'.")
123
+ obj = cls()
124
+ obj.n_states, obj.n_features, obj.batch_size = int(n_states), x.shape[1], int(batch_size)
125
+ obj.background_floor, obj.scales = float(background_floor), scales.copy()
126
+ obj.random_state = int(random_state)
127
+ _, inverse, sizes = np.unique(clones, return_inverse=True, return_counts=True)
128
+ km = KMeans(n_clusters=n_states, n_init=10, random_state=random_state)
129
+ km.fit(x, sample_weight=1.0 / sizes[inverse])
130
+ obj.centres = np.asarray(km.cluster_centers_)
131
+ if np.unique(obj.centres, axis=0).shape[0] < n_states:
132
+ raise ValueError("Reference has fewer distinct regions than n_states; reduce n_states.")
133
+ states, distances = obj._assign(x, return_distances=True)
134
+ positive = distances[distances > 0]
135
+ obj.bandwidth = float(np.median(positive)) if positive.size else float(np.median(pdist(obj.centres)))
136
+ if obj.bandwidth <= 0:
137
+ raise ValueError("Reference has zero spatial spread.")
138
+ obj.reference = obj._aggregate(states, clones, ctx)
139
+ obj._index_reference()
140
+ obj.calibration = pd.DataFrame(columns=["prior_strength", "mean_log_score", "n_profiles"])
141
+ obj.prior_method = "reference_split_cell" if prior_strength == "auto" else "fixed"
142
+ obj.prior_strength = 2.0 if prior_strength == "auto" else float(prior_strength)
143
+ if prior_strength == "auto":
144
+ obj._calibrate(states, clones, ctx)
145
+ return obj
146
+
147
+ @property
148
+ def state_names(self):
149
+ return [f"state_{i}" for i in range(self.n_states)]
150
+
151
+ def _assign(self, x, *, return_distances=False):
152
+ states = np.empty(len(x), dtype=np.int64)
153
+ distances = np.empty(len(x)) if return_distances else None
154
+ for start in range(0, len(x), self.batch_size):
155
+ stop = min(start + self.batch_size, len(x))
156
+ d2 = cdist(x[start:stop], self.centres, metric="sqeuclidean")
157
+ states[start:stop] = d2.argmin(axis=1)
158
+ if return_distances:
159
+ distances[start:stop] = np.sqrt(d2[np.arange(stop-start), states[start:stop]])
160
+ return (states, distances) if return_distances else states
161
+
162
+ def _aggregate(self, states, clones, contexts):
163
+ keys = pd.MultiIndex.from_arrays([clones, contexts], names=["clone", "context"])
164
+ code, levels = pd.factorize(keys, sort=True)
165
+ counts = np.bincount(code * self.n_states + states,
166
+ minlength=len(levels) * self.n_states).reshape(-1, self.n_states)
167
+ out = levels.to_frame(index=False)
168
+ out.columns = ["clone", "context"]
169
+ out[self.state_names] = counts
170
+ return out
171
+
172
+ def _index_reference(self):
173
+ cols = self.state_names
174
+ self._global = self.reference.groupby("clone", sort=True)[cols].sum()
175
+ self._global = self._global.div(self._global.sum(axis=1), axis=0)
176
+ ref = self.reference.copy()
177
+ ref[cols] = ref[cols].div(ref[cols].sum(axis=1), axis=0)
178
+ self._context = {name: group.set_index("clone")[cols]
179
+ for name, group in ref.groupby("context", sort=True)}
180
+ self._global_sum = self._global.sum(axis=0).to_numpy()
181
+ self._context_sum = {key: value.sum(axis=0).to_numpy() for key, value in self._context.items()}
182
+
183
+ def _background(self, clone, context):
184
+ tab = self._context.get(context)
185
+ fallback = tab is None or len(tab) - int(clone in tab.index) < 1
186
+ if fallback:
187
+ tab, total = self._global, self._global_sum.copy()
188
+ else:
189
+ total = self._context_sum[context].copy()
190
+ count, excluded = len(tab), clone in tab.index
191
+ if excluded:
192
+ total -= tab.loc[clone].to_numpy()
193
+ count -= 1
194
+ if count < 1:
195
+ raise ValueError("No independent reference clone remains for this query.")
196
+ q = np.maximum(total / count, 0)
197
+ q = (1 - self.background_floor) * q + self.background_floor / self.n_states
198
+ return q / q.sum(), fallback, excluded, count
199
+
200
+ def _calibrate(self, states, clones, contexts):
201
+ rng = np.random.default_rng(self.random_state + 104729)
202
+ groups = pd.DataFrame({"clone": clones, "context": contexts}).groupby(
203
+ ["clone", "context"], sort=True).indices
204
+ grid = np.array([0.5, 1., 2., 5., 10., 20., 50., 100.])
205
+ scores = []
206
+ for (clone, context), idx in groups.items():
207
+ if len(idx) < 4:
208
+ continue
209
+ idx = rng.permutation(idx)
210
+ cut = len(idx) // 2
211
+ train = np.bincount(states[idx[:cut]], minlength=self.n_states)
212
+ test = np.bincount(states[idx[cut:]], minlength=self.n_states)
213
+ q, _, _, _ = self._background(clone, context)
214
+ pred = (train[None, :] + grid[:, None] * q) / (cut + grid[:, None])
215
+ scores.append((clone, (np.log(pred) * test).sum(axis=1) / test.sum()))
216
+ if len({c for c, _ in scores}) < 6:
217
+ raise ValueError("Automatic prior calibration needs at least six reference clones with >=4 cells.")
218
+ balanced = pd.DataFrame([row for _, row in scores], index=[c for c, _ in scores]).groupby(level=0).mean()
219
+ mean_score = balanced.mean(axis=0).to_numpy()
220
+ self.prior_strength = float(grid[np.argmax(mean_score)])
221
+ self.calibration = pd.DataFrame({"prior_strength": grid, "mean_log_score": mean_score,
222
+ "n_profiles": len(balanced)})
223
+
224
+ def metadata(self):
225
+ digest = hashlib.sha256()
226
+ digest.update(np.ascontiguousarray(self.centres).tobytes())
227
+ digest.update(self.reference.to_csv(index=False).encode())
228
+ params = {"n_states": self.n_states, "n_features": self.n_features,
229
+ "prior_strength": self.prior_strength, "prior_method": self.prior_method,
230
+ "background_floor": self.background_floor,
231
+ "bandwidth": self.bandwidth, "scales": self.scales.tolist(), "random_state": self.random_state}
232
+ digest.update(json.dumps(params, sort_keys=True).encode())
233
+ return {**params, "reference_sha256": digest.hexdigest(),
234
+ "interval_scope": "conditional marginal Dirichlet; fixed partition and reference",
235
+ "smoothing_scope": "fixed Gaussian-mixture display; no graph diffusion",
236
+ "reference_weighting": "one vote per clone; own clone excluded from background"}
237
+
238
+ def transform(self, x, clone_ids, *, contexts=None, credible_level=0.95):
239
+ """Estimate proportions without refitting reference or preprocessing."""
240
+ return self.transform_batches([(x, clone_ids, contexts)], credible_level=credible_level)
241
+
242
+ def transform_batches(self, batches, *, credible_level=0.95):
243
+ """Stream (embedding, clone_ids, contexts_or_None) batches.
244
+
245
+ Clones may span batches and contexts. Counts are accumulated before
246
+ posterior estimation. Auxiliary memory is O(batch_size * n_states +
247
+ n_clone_contexts * n_states), excluding input and retained reference.
248
+ """
249
+ if not np.isfinite(credible_level) or not 0 < credible_level < 1:
250
+ raise ValueError("credible_level must be strictly between zero and one.")
251
+ totals = {}
252
+ for x, clone_ids, contexts in batches:
253
+ x = _matrix(x, self.n_features)
254
+ clones = _labels(clone_ids, len(x), "clone_ids")
255
+ ctx = np.repeat("all", len(x)) if contexts is None else _labels(contexts, len(x), "contexts")
256
+ aggregate = self._aggregate(self._assign(x), clones, ctx)
257
+ for row in aggregate.itertuples(index=False, name=None):
258
+ key, count = (row[0], row[1]), np.asarray(row[2:], dtype=np.int64)
259
+ totals[key] = totals.get(key, np.zeros(self.n_states, dtype=np.int64)) + count
260
+ if not totals:
261
+ raise ValueError("No query cells were supplied.")
262
+ grouped = {}
263
+ for (clone, context), counts in sorted(totals.items()):
264
+ grouped.setdefault(clone, []).append((context, counts))
265
+ ids, counts, backgrounds, rows = [], [], [], []
266
+ for clone, parts in grouped.items():
267
+ n = sum(int(c.sum()) for _, c in parts)
268
+ raw_count = sum((c for _, c in parts), np.zeros(self.n_states, dtype=np.int64))
269
+ bg = np.zeros(self.n_states)
270
+ fallback_n, excluded_n, reference_n = 0, 0, []
271
+ for ctx, c in parts:
272
+ q, fallback, excluded, n_ref = self._background(clone, ctx)
273
+ bg += c.sum() * q / n
274
+ fallback_n += int(c.sum()) * int(fallback)
275
+ excluded_n += int(c.sum()) * int(excluded)
276
+ reference_n.append(n_ref)
277
+ ids.append(clone)
278
+ counts.append(raw_count)
279
+ backgrounds.append(bg)
280
+ rows.append({"n_cells": n, "n_contexts": len(parts),
281
+ "data_weight": n / (n + self.prior_strength),
282
+ "global_background_fraction": fallback_n / n,
283
+ "own_clone_excluded_fraction": excluded_n / n,
284
+ "min_reference_clones": min(reference_n)})
285
+ index = pd.Index(ids, name="clone_id")
286
+ counts, backgrounds = np.asarray(counts), np.asarray(backgrounds)
287
+ n = counts.sum(axis=1, keepdims=True)
288
+ alpha = counts + self.prior_strength * backgrounds
289
+ total = alpha.sum(axis=1, keepdims=True)
290
+ tail = (1 - credible_level) / 2
291
+ # Exact binomial intervals with Bonferroni correction over this clone's
292
+ # fixed regions. Unlike posterior intervals these retain coverage at
293
+ # pure/rare-state boundaries under independent multinomial sampling.
294
+ sampling_tail = tail / self.n_states
295
+ sampling_lower = np.zeros_like(alpha)
296
+ sampling_upper = np.ones_like(alpha)
297
+ positive, incomplete = counts > 0, counts < n
298
+ sampling_lower[positive] = beta.ppf(sampling_tail, counts, n-counts+1)[positive]
299
+ sampling_upper[incomplete] = beta.ppf(1-sampling_tail, counts+1, n-counts)[incomplete]
300
+ make = lambda values: pd.DataFrame(values, index=index, columns=self.state_names)
301
+ return DensityResult(
302
+ make(alpha / total), make(counts / n), make(counts), make(backgrounds), make(alpha),
303
+ make(beta.ppf(tail, alpha, total - alpha)),
304
+ make(beta.ppf(1 - tail, alpha, total - alpha)),
305
+ make(sampling_lower), make(sampling_upper),
306
+ pd.DataFrame(rows, index=index),
307
+ {**self.metadata(), "credible_level": float(credible_level),
308
+ "sampling_interval_scope": "exact Bonferroni simultaneous regions within each clone; iid counts"},
309
+
310
+ )
311
+
312
+ def annotate_regions(self, x, values, clone_ids):
313
+ """Describe fixed regions with numeric gene/module/annotation values.
314
+
315
+ Rows of values must correspond to rows of x. Cells are first averaged
316
+ within each (clone, region), then clones are weighted equally within
317
+ the region. This is a descriptive annotation, with no p-value or
318
+ independent validation claim; it cannot change the fitted partition.
319
+ """
320
+ x = _matrix(x, self.n_features)
321
+ clones = _labels(clone_ids, len(x), "clone_ids")
322
+ values = pd.DataFrame(values).reset_index(drop=True)
323
+ if len(values) != len(x) or not values.columns.is_unique:
324
+ raise ValueError("values must have one aligned row per cell and unique columns.")
325
+ numeric = values.to_numpy(dtype=float)
326
+ if numeric.ndim != 2 or not numeric.shape[1] or not np.isfinite(numeric).all():
327
+ raise ValueError("Region annotations must contain finite numeric values.")
328
+ states = self._assign(x)
329
+ # Grouping keys are separate from feature columns, avoiding name collisions.
330
+ keys = pd.MultiIndex.from_arrays([clones, states], names=["clone", "region"])
331
+ frame = pd.DataFrame(numeric, index=keys, columns=values.columns)
332
+ per_clone = frame.groupby(level=["clone", "region"], sort=True).mean()
333
+ means = per_clone.groupby(level="region").mean().reindex(range(self.n_states))
334
+ coverage = pd.DataFrame({
335
+ "n_cells": np.bincount(states, minlength=self.n_states),
336
+ "n_clones": per_clone.groupby(level="region").size().reindex(range(self.n_states), fill_value=0),
337
+ })
338
+ means.index = coverage.index = pd.Index(self.state_names, name="region")
339
+ return {"values": means, "coverage": coverage}
340
+
341
+ def log_density(self, points, proportions):
342
+ """Log density of one clone's normalised Gaussian-mixture display.
343
+
344
+ Scales have equal weights. Values depend on embedding coordinates;
345
+ they cannot be compared across independently fitted embeddings.
346
+ """
347
+ x = _matrix(points, self.n_features)
348
+ p = np.asarray(proportions, dtype=float)
349
+ if p.shape != (self.n_states,) or not np.isfinite(p).all() or (p < 0).any() or not np.isclose(p.sum(), 1):
350
+ raise ValueError("proportions must be a nonnegative probability vector in state order.")
351
+ logp = np.full(self.n_states, -np.inf)
352
+ np.log(p, out=logp, where=p > 0)
353
+ out = np.empty(len(x))
354
+ for start in range(0, len(x), self.batch_size):
355
+ d2 = cdist(x[start:start+self.batch_size], self.centres, metric="sqeuclidean")
356
+ by_scale = []
357
+ for scale in self.scales:
358
+ variance = (self.bandwidth * scale) ** 2
359
+ logkernel = -0.5 * (d2 / variance + self.n_features * np.log(2 * np.pi * variance))
360
+ by_scale.append(logsumexp(logkernel + logp, axis=1))
361
+ out[start:start+len(d2)] = logsumexp(by_scale, axis=0) - np.log(len(self.scales))
362
+ return out
363
+
364
+ def save(self, path):
365
+ """Portable NPZ model without pickle or executable estimator state."""
366
+ params = {**self.metadata(), "batch_size": self.batch_size}
367
+ with Path(path).open("wb") as handle:
368
+ np.savez_compressed(handle, centres=self.centres, parameters=json.dumps(params),
369
+ reference_clones=self.reference["clone"].to_numpy(dtype=str),
370
+ reference_contexts=self.reference["context"].to_numpy(dtype=str),
371
+ reference_counts=self.reference[self.state_names].to_numpy(),
372
+ calibration=self.calibration.to_numpy(dtype=float))
373
+
374
+ @classmethod
375
+ def load(cls, path):
376
+ """Read a saved model with pickle disabled and verify its checksum."""
377
+ with np.load(path, allow_pickle=False) as saved:
378
+ params = json.loads(str(saved["parameters"]))
379
+ obj = cls()
380
+ for key in ("n_states", "n_features", "prior_strength", "prior_method", "background_floor",
381
+ "bandwidth", "random_state", "batch_size"):
382
+ setattr(obj, key, params[key])
383
+ obj.scales, obj.centres = np.asarray(params["scales"]), saved["centres"].copy()
384
+ obj.reference = pd.DataFrame({"clone": saved["reference_clones"],
385
+ "context": saved["reference_contexts"]})
386
+ obj.reference[obj.state_names] = saved["reference_counts"]
387
+ obj.calibration = pd.DataFrame(saved["calibration"],
388
+ columns=["prior_strength", "mean_log_score", "n_profiles"])
389
+ obj._index_reference()
390
+ if obj.metadata()["reference_sha256"] != params["reference_sha256"]:
391
+ raise ValueError("Saved reference checksum does not match its data.")
392
+ return obj
393
+
394
+
395
+ def clone_densities(adata, *, reference_model, clone_key="clone_id", basis="X_pca",
396
+ context_key=None, credible_level=0.95, key_added="densities"):
397
+ """AnnData wrapper requiring an explicitly fitted StateDensityModel.
398
+
399
+ Cells without a clone are omitted. Reference and query must share feature
400
+ coordinates. Results are separate from existing mean/kernel profiles.
401
+ """
402
+ require_obs(adata, clone_key, context_key, context="clone densities")
403
+ valid = adata.obs[clone_key].notna().to_numpy()
404
+ x = get_basis(adata, basis)[valid]
405
+ contexts = adata.obs.loc[valid, context_key] if context_key is not None else None
406
+ result = reference_model.transform(x, adata.obs.loc[valid, clone_key],
407
+ contexts=contexts, credible_level=credible_level)
408
+ get_uns(adata)[key_added] = result.to_uns()
409
+ return result