diffsvg 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
diffsvg/tl/model.py ADDED
@@ -0,0 +1,1857 @@
1
+ """Per-gene GAM (P-spline) models for differential spatially/pseudotemporally variable genes.
2
+
3
+ `setup_model` reads the output of `diffsvg.pp.bin_obs` from ``adata.uns["binned"]``; `fit_model` fits one
4
+ model per gene in parallel, predicts both condition curves on a fixed grid, tests them against each other
5
+ and writes the result to ``adata.varm``.
6
+
7
+ Parallelization: workers receive only plain arrays (the shared design matrices plus the gene's `Y`) and
8
+ build their own `pm.Model` with `_build_gene_model`. A built model cannot be shipped instead (PyTensor
9
+ graph nodes do not pickle), and every worker is a separate process, so each compiles its own.
10
+ """
11
+
12
+ import contextlib
13
+ import logging
14
+ import os
15
+ import warnings
16
+ from dataclasses import dataclass
17
+
18
+ import numpy as np
19
+ import pandas as pd
20
+ from joblib import Parallel, delayed
21
+ from scipy import optimize
22
+ from scipy.stats import chi2, f, gamma, ncf, ncx2
23
+ from tqdm.auto import tqdm
24
+
25
+ from diffsvg.pp.utils import BinnedData, _bspline_basis_01, _expression, spline_basis_at
26
+ from diffsvg.tl import _exact
27
+
28
+ try: # the `pymc` extra: only needed for inference="nuts"
29
+ import arviz as az
30
+ import pymc as pm
31
+ except ImportError:
32
+ az = pm = None
33
+
34
+
35
+ def _require_pymc(inference):
36
+ if pm is None:
37
+ raise ImportError(
38
+ f"inference={inference!r} needs PyMC, which is an optional dependency: pip install 'diffsvg[pymc]'. "
39
+ "The default inference='auto' uses the exact posterior wherever the model allows it."
40
+ )
41
+
42
+
43
+ # ============================================================
44
+ # 1) Priors and likelihoods
45
+ # ============================================================
46
+ # Registries, so alternatives can be added without touching the model builders. A prior is a tuple
47
+ # ``(distribution, *params)``, parameters in the order listed below.
48
+
49
+ _PRIOR_DISTRIBUTIONS = {
50
+ "normal": ("Normal", ("mu", "sigma")),
51
+ "laplace": ("Laplace", ("mu", "b")),
52
+ "studentt": ("StudentT", ("nu", "mu", "sigma")),
53
+ "halfnormal": ("HalfNormal", ("sigma",)),
54
+ "halfcauchy": ("HalfCauchy", ("beta",)),
55
+ "exponential": ("Exponential", ("lam",)),
56
+ }
57
+
58
+ #: Default priors; ``setup_model(priors=...)`` overrides any of them.
59
+ DEFAULT_PRIORS = {
60
+ "intercept": ("normal", 0.0, 1.0),
61
+ "beta": ("normal", 0.0, 1.0),
62
+ "beta_1": ("normal", 0.0, 1.0),
63
+ "categorical_beta": ("normal", 0.0, 1.0),
64
+ "sigma": ("halfnormal", 1.0),
65
+ # Only used by fits with `sample` (see `bin_obs`): how far a slide's level / curve strays from its condition's.
66
+ "sample_level_sd": ("halfnormal", 1.0),
67
+ "sample_curve_sd": ("halfnormal", 1.0),
68
+ }
69
+
70
+
71
+ def _resolve_priors(priors):
72
+ """Merge a partial prior spec (e.g. ``{"beta": ("normal", 0.0, 2.0)}``) over `DEFAULT_PRIORS` and validate it."""
73
+ merged = dict(DEFAULT_PRIORS)
74
+ for name, spec in (priors or {}).items():
75
+ if name not in DEFAULT_PRIORS:
76
+ raise ValueError(f"Unknown prior {name!r}. Available: {sorted(DEFAULT_PRIORS)}")
77
+ merged[name] = tuple(spec)
78
+
79
+ for name, spec in merged.items():
80
+ distribution, *params = spec
81
+ if distribution not in _PRIOR_DISTRIBUTIONS:
82
+ raise ValueError(
83
+ f"Unknown distribution {distribution!r} for prior {name!r}. Available: {sorted(_PRIOR_DISTRIBUTIONS)}"
84
+ )
85
+ expected = _PRIOR_DISTRIBUTIONS[distribution][1]
86
+ if len(params) != len(expected):
87
+ raise ValueError(
88
+ f"Prior {name!r}: {distribution!r} takes {len(expected)} parameter(s) {expected}, got {len(params)}."
89
+ )
90
+ return merged
91
+
92
+
93
+ def _draw_prior(name, spec, shape=None, initval=None):
94
+ """Instantiate one prior from its ``(distribution, *params)`` tuple."""
95
+ distribution, *params = spec
96
+ dist_name, param_names = _PRIOR_DISTRIBUTIONS[distribution]
97
+ dist = getattr(pm, dist_name)
98
+ kwargs = dict(zip(param_names, params, strict=True))
99
+ if shape is not None:
100
+ kwargs["shape"] = shape
101
+ if initval is not None:
102
+ kwargs["initval"] = initval
103
+ return dist(name, **kwargs)
104
+
105
+
106
+ def _normal_likelihood(mu, observed, total_size, priors, weights=None, ss_within=None):
107
+ sigma = _draw_prior("sigma", priors["sigma"], initval=0.5)
108
+ if weights is None:
109
+ return pm.Normal("D", mu=mu, sigma=sigma, observed=observed, total_size=total_size)
110
+ # A row that averages n cells has variance sigma^2 / n, sigma being the cell-level noise.
111
+ D = pm.Normal("D", mu=mu, sigma=sigma / pm.math.sqrt(weights), observed=observed, total_size=total_size)
112
+ if ss_within is not None:
113
+ # What the per-cell likelihood has on top of the row means: the spread of
114
+ # the cells around their row's mean, `ss` over `df` = n_cells - n_rows.
115
+ ss, df = ss_within
116
+ pm.Potential("within_rows", -0.5 * (2.0 * df * pm.math.log(sigma) + ss / sigma**2))
117
+ return D
118
+
119
+
120
+ _LIKELIHOODS = {
121
+ "normal": _normal_likelihood,
122
+ }
123
+
124
+
125
+ # ============================================================
126
+ # 2) Model setup
127
+ # ============================================================
128
+
129
+
130
+ @dataclass
131
+ class DiffSVGModelSetup:
132
+ """Config bundle returned by `setup_model` and consumed by `fit_model`.
133
+
134
+ Holds no `pm.Model`: `fit_model` builds each gene's model inside its worker. To inspect one gene's
135
+ model, call `_build_gene_model` directly.
136
+
137
+ Attributes
138
+ ----------
139
+ binned
140
+ The `diffsvg.pp.utils.BinnedData` this setup was built from.
141
+ likelihood
142
+ Observation model every gene's model is built with.
143
+ priors
144
+ Prior spec passed to `setup_model`, merged over `DEFAULT_PRIORS`.
145
+ """
146
+
147
+ binned: BinnedData
148
+ likelihood: str
149
+ priors: dict | None = None
150
+
151
+ def __post_init__(self):
152
+ self.priors = _resolve_priors(self.priors)
153
+
154
+
155
+ def _build_gene_model(
156
+ Y,
157
+ B,
158
+ B_masked,
159
+ condition_arr,
160
+ n_bsplines,
161
+ intercept_initval,
162
+ likelihood="normal",
163
+ priors=None,
164
+ weights=None,
165
+ ss_within=None,
166
+ record_mu=True,
167
+ sample_index=None,
168
+ n_samples=None,
169
+ sample_scales=None,
170
+ ):
171
+ """Build the P-spline GAM for a single gene.
172
+
173
+ `B`/`B_masked`/`condition_arr` are shared across genes; only `Y` and `intercept_initval` change per
174
+ gene. Building is cheap (graph construction only), so every worker builds its own.
175
+
176
+ - `weights` (cells per row) and `ss_within` (``(within-row sum of squares, n_cells - n_rows)``) come
177
+ from contiguous bins: rows are weighted by their cell count and the noise is the cell-level one.
178
+ `weights` lives in a ``"w_data"`` container so predictions can set it per grid point.
179
+ - `record_mu` keeps the row-wise mean as the ``"mu"`` deterministic (one value per row and draw; off
180
+ in `fit_model`, nothing reads it).
181
+ - `sample_index` (the slide of every row) adds a non-centred random level and random curve per slide,
182
+ ``sample_level_sd * z_level[slide] + B @ (sample_curve_sd * z_curve[slide])``, multiplied by a
183
+ ``"sample_on"`` container that a prediction grid sets to 0 for the population-level curves.
184
+ `sample_scales` ``(level_sd, curve_sd)`` fixes the two scales instead (empirical Bayes).
185
+ """
186
+ if likelihood not in _LIKELIHOODS:
187
+ raise ValueError(f"Unknown likelihood {likelihood!r}. Available: {sorted(_LIKELIHOODS)}")
188
+ likelihood_fn = _LIKELIHOODS[likelihood]
189
+ priors = _resolve_priors(priors)
190
+
191
+ with pm.Model() as gene_model:
192
+ Y_data = pm.Data("Y_data", np.asarray(Y, dtype=float))
193
+ B_data = pm.Data("B_data", B)
194
+ condition_data = pm.Data("condition_data", np.asarray(condition_arr, dtype=float))
195
+ B_masked_data = pm.Data("B_masked_data", B_masked)
196
+
197
+ intercept = _draw_prior("intercept", priors["intercept"], initval=intercept_initval)
198
+ beta = _draw_prior("beta", priors["beta"], shape=n_bsplines, initval=np.zeros(n_bsplines))
199
+ beta_1 = _draw_prior("beta_1", priors["beta_1"], shape=n_bsplines, initval=np.zeros(n_bsplines))
200
+ categorical_beta = _draw_prior("categorical_beta", priors["categorical_beta"], initval=0.0)
201
+
202
+ mu = (
203
+ intercept
204
+ + pm.math.dot(B_data, beta)
205
+ + categorical_beta * condition_data
206
+ + pm.math.dot(B_masked_data, beta_1)
207
+ )
208
+ if sample_index is not None:
209
+ if sample_scales is None:
210
+ level_sd = _draw_prior("sample_level_sd", priors["sample_level_sd"], initval=0.1)
211
+ curve_sd = _draw_prior("sample_curve_sd", priors["sample_curve_sd"], initval=0.1)
212
+ else:
213
+ level_sd, curve_sd = (float(v) for v in sample_scales)
214
+ level_z = pm.Normal("sample_level_z", 0.0, 1.0, shape=n_samples)
215
+ curve_z = pm.Normal("sample_curve_z", 0.0, 1.0, shape=(n_samples, n_bsplines))
216
+ slide = pm.Data("sample_data", np.asarray(sample_index, dtype="int64"))
217
+ on = pm.Data("sample_on", np.ones(len(sample_index)))
218
+ mu = mu + on * (level_sd * level_z[slide] + pm.math.sum(B_data * (curve_sd * curve_z)[slide], axis=1))
219
+ if record_mu:
220
+ mu = pm.Deterministic("mu", mu)
221
+
222
+ w_data = None if weights is None else pm.Data("w_data", np.asarray(weights, dtype=float))
223
+ likelihood_fn(mu, Y_data, len(Y), priors, weights=w_data, ss_within=ss_within)
224
+
225
+ return gene_model
226
+
227
+
228
+ def _build_profile_model(
229
+ Y, B, n_bsplines, intercept_initval, likelihood="normal", priors=None, weights=None, ss_within=None
230
+ ):
231
+ """`_build_gene_model` without the `condition` covariate and its spline interaction (for `fit_profiles`)."""
232
+ if likelihood not in _LIKELIHOODS:
233
+ raise ValueError(f"Unknown likelihood {likelihood!r}. Available: {sorted(_LIKELIHOODS)}")
234
+ likelihood_fn = _LIKELIHOODS[likelihood]
235
+ priors = _resolve_priors(priors)
236
+
237
+ with pm.Model() as gene_model:
238
+ Y_data = pm.Data("Y_data", np.asarray(Y, dtype=float))
239
+ B_data = pm.Data("B_data", B)
240
+
241
+ intercept = _draw_prior("intercept", priors["intercept"], initval=intercept_initval)
242
+ beta = _draw_prior("beta", priors["beta"], shape=n_bsplines, initval=np.zeros(n_bsplines))
243
+
244
+ mu = pm.Deterministic("mu", intercept + pm.math.dot(B_data, beta))
245
+
246
+ w_data = None if weights is None else pm.Data("w_data", np.asarray(weights, dtype=float))
247
+ likelihood_fn(mu, Y_data, len(Y), priors, weights=w_data, ss_within=ss_within)
248
+
249
+ return gene_model
250
+
251
+
252
+ def setup_model(adata, likelihood="normal", priors=None):
253
+ """Configure the per-gene GAM, given `adata` pre-processed by `diffsvg.pp.bin_obs`.
254
+
255
+ This validates `adata`/`likelihood` and bundles them into a
256
+ `DiffSVGModelSetup` - it does **not** build any `pm.Model`. Actual model
257
+ construction happens lazily, per gene, once `fit_model` starts fitting
258
+ (see `DiffSVGModelSetup`), so this is cheap regardless of gene count.
259
+
260
+ Parameters
261
+ ----------
262
+ adata
263
+ Annotated data matrix that `diffsvg.pp.bin_obs` was already run on
264
+ (i.e. ``"binned" in adata.uns``).
265
+ likelihood
266
+ Observation model. Only ``"normal"`` (the default) is implemented.
267
+ priors
268
+ Overrides for any of `DEFAULT_PRIORS`, as ``{name: (distribution,
269
+ *params)}`` - e.g. ``{"beta": ("normal", 0.0, 2.0), "sigma":
270
+ ("exponential", 1.0)}``. Anything not named keeps its default, so
271
+ the models are unchanged unless asked. Validated here rather than
272
+ inside a worker, so a typo fails immediately.
273
+
274
+ Returns
275
+ -------
276
+ DiffSVGModelSetup
277
+ """
278
+ if likelihood not in _LIKELIHOODS:
279
+ raise ValueError(f"Unknown likelihood {likelihood!r}. Available: {sorted(_LIKELIHOODS)}")
280
+ if "binned" not in adata.uns:
281
+ raise KeyError("adata.uns['binned'] not found - run `diffsvg.pp.bin_obs(adata, ...)` first.")
282
+
283
+ binned = BinnedData(**adata.uns["binned"])
284
+ return DiffSVGModelSetup(binned=binned, likelihood=likelihood, priors=priors)
285
+
286
+
287
+ # ============================================================
288
+ # 3) Fitting (parallel across genes) + posterior predictive
289
+ # ============================================================
290
+
291
+
292
+ @dataclass
293
+ class _FitShared:
294
+ """The plain-array payload shared by every gene (no PyMC objects, see the module docstring)."""
295
+
296
+ B: np.ndarray
297
+ B_masked: np.ndarray
298
+ condition_fit: np.ndarray
299
+ n_bsplines: int
300
+ likelihood: str
301
+ priors: dict
302
+ bin_centers: np.ndarray
303
+ B_pred: np.ndarray
304
+ B_pred_masked_0: np.ndarray
305
+ B_pred_masked_1: np.ndarray
306
+ condition_pred_0: np.ndarray
307
+ condition_pred_1: np.ndarray
308
+ # Which condition test to run, see `_TESTS`. `shape_metric`/`slope_weights`
309
+ # are only filled for the parallel-curves test.
310
+ test: str = "wald"
311
+ tau: float = 0.1
312
+ shape_metric: np.ndarray | None = None
313
+ slope_weights: np.ndarray | None = None
314
+ # Cells per row, and per grid point for predictions, when fitting weighted
315
+ # contiguous bins (see `_weighting`); `None` otherwise.
316
+ weights: np.ndarray | None = None
317
+ # How the posterior is obtained, see `_INFERENCE`.
318
+ inference: str = "nuts"
319
+ # The slide of every fitted row and the denominator degrees of freedom of the test, for fits with `sample`.
320
+ sample_index: np.ndarray | None = None
321
+ n_samples: int | None = None
322
+ df_denominator: int | None = None
323
+
324
+
325
+ @dataclass
326
+ class _ProfileShared:
327
+ """`_FitShared`'s counterpart for the condition-free model - same contract, fewer arrays."""
328
+
329
+ B: np.ndarray
330
+ n_bsplines: int
331
+ likelihood: str
332
+ priors: dict
333
+ B_pred: np.ndarray
334
+ weights: np.ndarray | None = None
335
+
336
+
337
+ def _fitted_cells(design, binned):
338
+ """Which cells a fit on single cells uses: those with an anchor value, a condition and (if any) a sample."""
339
+ keep = np.isfinite(design[binned.anchor].to_numpy(dtype=float))
340
+ if binned.has_condition:
341
+ keep &= design[binned.condition].notna().to_numpy()
342
+ if binned.sample is not None:
343
+ keep &= design[binned.sample].notna().to_numpy()
344
+ return keep
345
+
346
+
347
+ def _sample_inputs(adata, binned, use_binned):
348
+ """The slide of every fitted row and the number of slides, or ``(None, None)`` for a fit without `sample`."""
349
+ if binned.sample is None:
350
+ return None, None
351
+ if use_binned:
352
+ return np.asarray(binned.sample_index, dtype=int), binned.n_samples
353
+ design = adata.obsm[binned.design_matrix]
354
+ position = {label: i for i, label in enumerate(binned.sample_labels)}
355
+ index = design[binned.sample].astype(str).map(position)[_fitted_cells(design, binned)]
356
+ if index.isna().any():
357
+ raise ValueError(
358
+ f"`{binned.sample}` holds samples that were not seen when binning (expected {binned.sample_labels}) - "
359
+ "re-run `diffsvg.pp.bin_obs`."
360
+ )
361
+ return index.to_numpy(dtype=int), binned.n_samples
362
+
363
+
364
+ def _fit_inputs(adata, binned, use_binned):
365
+ """The targets and design matrices to fit on: the binned aggregates, or one row per cell.
366
+
367
+ For cells, the basis is the binned one evaluated per cell (`diffsvg.pp.spline_basis_at`), so the
368
+ coefficients stay valid on the prediction grid, and `Y` is read from `adata`. Cells without an anchor
369
+ value or a condition are left out, as in the binning.
370
+
371
+ Returns ``(B, B_masked, condition_fit, Y, intercept_initvals)``.
372
+ """
373
+ if use_binned:
374
+ return binned.B, binned.B_masked, binned.condition_fit, binned.Y, binned.intercept_initvals
375
+
376
+ design = adata.obsm[binned.design_matrix]
377
+ anchor_vals = design[binned.anchor].to_numpy(dtype=float)
378
+ keep = _fitted_cells(design, binned)
379
+ # The binned basis evaluated per cell. A basis rebuilt from the cells would
380
+ # be centred on the cells rather than the bins, and predicting with
381
+ # `binned.B_pred` would then shift every curve by a constant.
382
+ B = spline_basis_at(binned, anchor_vals[keep])
383
+
384
+ condition_fit, B_masked = None, None
385
+ if binned.has_condition:
386
+ code_map = {label: float(code) for code, label in binned.condition_categories.items()}
387
+ condition_fit = design[binned.condition].map(code_map).to_numpy(dtype=float)[keep]
388
+ if np.isnan(condition_fit).any():
389
+ raise ValueError(
390
+ f"`condition` column {binned.condition!r} holds values that were not seen when binning "
391
+ f"(expected {sorted(code_map)}) - re-run `diffsvg.pp.bin_obs`."
392
+ )
393
+ B_masked = B * condition_fit[:, None]
394
+
395
+ X = _expression(adata, binned.genes, binned.layer)[keep]
396
+ Y = {gene: X[:, i] for i, gene in enumerate(binned.genes)}
397
+ # As for bins (mean expression at anchor 0), but no cell sits exactly at 0, so the first bin stands in.
398
+ first_bin = np.asarray(binned.bin_assignment)[keep] == 0
399
+ intercept_initvals = {
400
+ gene: float(y[first_bin].mean()) if first_bin.any() else float(y.mean()) for gene, y in Y.items()
401
+ }
402
+ return B, B_masked, condition_fit, Y, intercept_initvals
403
+
404
+
405
+ def _weighting(binned, use_binned):
406
+ """What a fit on weighted contiguous bins needs on top of `_fit_inputs`, or `None`s otherwise.
407
+
408
+ Returns ``(weights, ss_within, n_obs_grid)``: the cells behind each row,
409
+ per gene the within-row sum of squares with its degrees of freedom
410
+ (``n_cells - n_rows``), and the cells per grid point. Predictions use the
411
+ latter, so a predicted value is a bin mean just like a fitted row. Fits on
412
+ the cells are unweighted. Overlapping bins are weighted by their cell
413
+ counts too (a window with 2 cells must not count as much as one with
414
+ 500, which otherwise bends the curve at the sparse ends of the range),
415
+ but have no within-row term: their rows share cells.
416
+ """
417
+ if not use_binned:
418
+ return None, None, None
419
+ if not binned.weighted:
420
+ if binned.n_obs is None: # binned before the counts were recorded for overlapping bins
421
+ return None, None, None
422
+ return np.asarray(binned.n_obs, dtype=float), None, None
423
+ weights = np.asarray(binned.n_obs, dtype=float)
424
+ df = float(weights.sum() - len(weights))
425
+ ss_within = {gene: (float(binned.ss_within[gene]), df) for gene in binned.genes}
426
+ return weights, ss_within, np.asarray(binned.n_obs_grid, dtype=float)
427
+
428
+
429
+ @contextlib.contextmanager
430
+ def _quiet_fds():
431
+ """Redirect the OS-level stdout/stderr file descriptors to `os.devnull`.
432
+
433
+ Unlike `contextlib.redirect_stdout`, this also catches compiled extensions (nutpie's Rust sampler)
434
+ writing straight to the fds. Exceptions still propagate; the fds are restored before anything prints them.
435
+ """
436
+ saved = [os.dup(fd) for fd in (1, 2)]
437
+ devnull = os.open(os.devnull, os.O_WRONLY)
438
+ try:
439
+ os.dup2(devnull, 1)
440
+ os.dup2(devnull, 2)
441
+ yield
442
+ finally:
443
+ for fd, saved_fd in zip((1, 2), saved, strict=True):
444
+ os.dup2(saved_fd, fd)
445
+ os.close(saved_fd)
446
+ os.close(devnull)
447
+
448
+
449
+ @contextlib.contextmanager
450
+ def _quiet_pymc():
451
+ """Silence PyMC/PyTensor/nutpie's logging, warnings and raw fd output in the process that samples.
452
+
453
+ Runs inside each worker; the outer `tqdm` bar lives in the parent process and is unaffected.
454
+ """
455
+ loggers = [logging.getLogger(name) for name in ("pymc", "pytensor", "nutpie")]
456
+ previous_levels = [logger.level for logger in loggers]
457
+ for logger in loggers:
458
+ logger.setLevel(logging.ERROR)
459
+ try:
460
+ with warnings.catch_warnings(), _quiet_fds():
461
+ warnings.simplefilter("ignore")
462
+ yield
463
+ finally:
464
+ for logger, level in zip(loggers, previous_levels, strict=True):
465
+ logger.setLevel(level)
466
+
467
+
468
+ # ------------------------------------------------------------
469
+ # Posterior inference by sampling (the exact path lives in `diffsvg.tl._exact`)
470
+ # ------------------------------------------------------------
471
+ # `_INFERENCE` maps a sampler name to a function that runs inside ``with model`` and returns the posterior as
472
+ # `InferenceData` plus a dict of diagnostic columns. NUTS is only needed for non-normal coefficient priors.
473
+
474
+
475
+ def _sample_nuts(model, sample_kwargs, diag_vars):
476
+ """Sample the posterior with NUTS. `diag_vars` are the parameters the convergence diagnostics are reported for."""
477
+ idata = pm.sample(**sample_kwargs)
478
+ n_divergent = int(idata.sample_stats["diverging"].sum()) if "diverging" in idata.sample_stats else 0
479
+ rhat_ds = az.rhat(idata, var_names=diag_vars)
480
+ ess_ds = az.ess(idata, var_names=diag_vars, method="bulk")
481
+ columns = {
482
+ "n_divergent": n_divergent,
483
+ "rhat_max": float(max(np.asarray(rhat_ds[v]).max() for v in rhat_ds.data_vars)),
484
+ "ess_bulk_min": float(min(np.asarray(ess_ds[v]).min() for v in ess_ds.data_vars)),
485
+ }
486
+ return idata, columns
487
+
488
+
489
+ _INFERENCE = {
490
+ "nuts": _sample_nuts,
491
+ }
492
+
493
+
494
+ def _posterior_draws(idata, name):
495
+ """Posterior draws of one variable, flattened over chains: ``(n_draws,)`` or ``(n_draws, dim)``.
496
+
497
+ `idata` is an `arviz.InferenceData` or, for exact inference, a dict ``{name: draws}``.
498
+ """
499
+ if isinstance(idata, dict):
500
+ return idata[name]
501
+ values = idata.posterior[name].stack(sample=("chain", "draw")).to_numpy()
502
+ return values.T if values.ndim == 2 else values
503
+
504
+
505
+ def _mean_curve_draws(idata, shared):
506
+ """Posterior draws of both condition curves on the grid, ``(n_draws, n_grid)`` each.
507
+
508
+ The mean function ``intercept + B beta + condition * categorical_beta + B_masked beta_1`` at every
509
+ draw, without observation noise and without slide effects.
510
+ """
511
+ intercept = _posterior_draws(idata, "intercept")
512
+ beta = _posterior_draws(idata, "beta")
513
+ beta_1 = _posterior_draws(idata, "beta_1")
514
+ cb = _posterior_draws(idata, "categorical_beta")
515
+ base = intercept[:, None] + beta @ shared.B_pred.T
516
+ draws0 = base + cb[:, None] * shared.condition_pred_0[None, :] + beta_1 @ shared.B_pred_masked_0.T
517
+ draws1 = base + cb[:, None] * shared.condition_pred_1[None, :] + beta_1 @ shared.B_pred_masked_1.T
518
+ return draws0, draws1
519
+
520
+
521
+ def _fit_and_predict_gene(gene, Y, intercept_initval, shared, sample_kwargs, ss_within=None):
522
+ """Build and fit one gene's model in this worker, then predict both condition curves on the grid.
523
+
524
+ Returns ``(gene, stats_row, curve_row)``: `stats_row` holds the test's columns and the shared ones;
525
+ `curve_row` holds the mean difference curve ``"delta"``, per condition the mean curve and its 94% band
526
+ (``"cond{0,1}_{mean,ci_low,ci_high}"``) and the posterior means of ``"beta"`` and ``"beta_1"``.
527
+ `ss_within` is the gene's within-row term (see `_weighting`).
528
+ """
529
+
530
+ def build(sample_scales=None):
531
+ return _build_gene_model(
532
+ Y,
533
+ shared.B,
534
+ shared.B_masked,
535
+ shared.condition_fit,
536
+ shared.n_bsplines,
537
+ intercept_initval,
538
+ shared.likelihood,
539
+ shared.priors,
540
+ weights=shared.weights,
541
+ ss_within=ss_within,
542
+ record_mu=False,
543
+ sample_index=shared.sample_index,
544
+ n_samples=shared.n_samples,
545
+ sample_scales=sample_scales,
546
+ )
547
+
548
+ model = build()
549
+ slides = shared.sample_index is not None
550
+
551
+ with _quiet_pymc():
552
+ with model:
553
+ diag_vars = ["intercept", "beta", "beta_1", "categorical_beta", "sigma"]
554
+ idata, fit_columns = _INFERENCE[shared.inference](model, sample_kwargs, diag_vars)
555
+ if slides:
556
+ fit_columns = {
557
+ **fit_columns,
558
+ "sample_level_sd": float(idata.posterior["sample_level_sd"].mean()),
559
+ "sample_curve_sd": float(idata.posterior["sample_curve_sd"].mean()),
560
+ }
561
+ return _summarise_gene(gene, idata, shared, fit_columns)
562
+
563
+
564
+ def _condition_maps(shared):
565
+ """Linear maps from ``theta = (intercept, beta, categorical_beta, beta_1)`` to the two grid curves."""
566
+ n_grid, K = shared.B_pred.shape
567
+ ones = np.ones((n_grid, 1))
568
+ A0 = np.hstack([ones, shared.B_pred, shared.condition_pred_0[:, None], shared.B_pred_masked_0])
569
+ A1 = np.hstack([ones, shared.B_pred, shared.condition_pred_1[:, None], shared.B_pred_masked_1])
570
+ return A0, A1
571
+
572
+
573
+ def _summarise_gene(gene, idata, shared, fit_columns, moments=None):
574
+ """Curves, test and summary columns of one gene from its posterior; see `_fit_and_predict_gene`.
575
+
576
+ `idata` holds the posterior draws (InferenceData or a dict). `moments` ``(mean, cov)`` of ``theta``
577
+ (exact inference) replace the draw estimates wherever only moments are needed: the means, the tests.
578
+ """
579
+ # The condition curves are the posterior of the mean function on the grid (slide effects off, i.e.
580
+ # population level). No observation noise: the bands are credible bands of the curve, and the Wald
581
+ # test sees the covariance of the coefficients rather than that of new observations.
582
+ draws0, draws1 = _mean_curve_draws(idata, shared)
583
+ test_row, delta_mean = _TESTS[shared.test][0](draws0, draws1, idata, shared, moments)
584
+ if moments is None:
585
+ cond0_mean, cond1_mean = draws0.mean(axis=0), draws1.mean(axis=0)
586
+ else:
587
+ A0, A1 = _condition_maps(shared)
588
+ cond0_mean, cond1_mean = A0 @ moments[0], A1 @ moments[0]
589
+ cond0_ci_low, cond0_ci_high = np.quantile(draws0, [0.03, 0.97], axis=0)
590
+ cond1_ci_low, cond1_ci_high = np.quantile(draws1, [0.03, 0.97], axis=0)
591
+ integral = float(np.trapezoid(delta_mean, x=shared.bin_centers))
592
+
593
+ post_cb = _posterior_draws(idata, "categorical_beta")
594
+ ci_low, ci_high = (float(v) for v in np.quantile(post_cb, [0.03, 0.97]))
595
+ K = shared.n_bsplines
596
+ if moments is None:
597
+ cb_mean = float(post_cb.mean())
598
+ beta_mean = _posterior_draws(idata, "beta").mean(axis=0)
599
+ beta_1_mean = _posterior_draws(idata, "beta_1").mean(axis=0)
600
+ else:
601
+ cb_mean = float(moments[0][K + 1])
602
+ beta_mean, beta_1_mean = moments[0][1 : K + 1], moments[0][K + 2 :]
603
+
604
+ stats_row = {
605
+ "gene": gene,
606
+ **test_row,
607
+ "integral": integral,
608
+ "categorical_beta_mean": cb_mean,
609
+ "categorical_beta_ci_low": ci_low,
610
+ "categorical_beta_ci_high": ci_high,
611
+ **fit_columns,
612
+ }
613
+ curve_row = {
614
+ "delta": delta_mean,
615
+ "cond0_mean": cond0_mean,
616
+ "cond0_ci_low": cond0_ci_low,
617
+ "cond0_ci_high": cond0_ci_high,
618
+ "cond1_mean": cond1_mean,
619
+ "cond1_ci_low": cond1_ci_low,
620
+ "cond1_ci_high": cond1_ci_high,
621
+ "beta": np.asarray(beta_mean),
622
+ "beta_1": np.asarray(beta_1_mean),
623
+ }
624
+ return gene, stats_row, curve_row
625
+
626
+
627
+ def _exact_settings(sample_kwargs):
628
+ settings = {"draws": 1000, "random_seed": 42, "n_grid": 81, "n_hyper": 72}
629
+ given = dict(sample_kwargs or {})
630
+ ignored = sorted(set(given) - set(settings))
631
+ if ignored:
632
+ warnings.warn(
633
+ f"inference='exact' ignores the sampler settings {ignored} (it uses only draws, random_seed, n_grid, n_hyper).",
634
+ UserWarning,
635
+ stacklevel=3,
636
+ )
637
+ settings.update({k: v for k, v in given.items() if k in settings})
638
+ return settings
639
+
640
+
641
+ def _exact_fit(design, Y, genes, weights, ss_within, priors, prior_names, n_bsplines, settings):
642
+ """`_exact.posterior` for all `genes` on the design `design` (columns ordered as `prior_names`)."""
643
+ Ymat = np.column_stack([np.asarray(Y[g], dtype=float) for g in genes])
644
+ XtWX, XtWY, yWy = _exact.sufficient_statistics(design, Ymat, weights)
645
+ sizes = {"intercept": 1, "categorical_beta": 1, "beta": n_bsplines, "beta_1": n_bsplines}
646
+ m0 = np.concatenate([[priors[n][1]] * sizes[n] for n in prior_names])
647
+ s0 = np.concatenate([[priors[n][2]] * sizes[n] for n in prior_names])
648
+ extra_df = extra_ss = None
649
+ if ss_within is not None:
650
+ extra_df = ss_within[genes[0]][1]
651
+ extra_ss = np.array([ss_within[g][0] for g in genes])
652
+ return _exact.posterior(
653
+ XtWX, XtWY, yWy, design.shape[0], m0, s0, priors["sigma"], extra_df, extra_ss, n_grid=settings["n_grid"]
654
+ )
655
+
656
+
657
+ def _exact_slide_results(shared, Y, genes, ss_within, settings):
658
+ """Exact inference for the two-condition model with slide effects."""
659
+ priors = _resolve_priors(shared.priors)
660
+ K = shared.n_bsplines
661
+ fixed = np.column_stack([np.ones(shared.B.shape[0]), shared.B, shared.condition_fit, shared.B_masked])
662
+ X = _exact.slide_design(fixed, shared.B, np.asarray(shared.sample_index, dtype=int), shared.n_samples)
663
+ sizes = {"intercept": 1, "categorical_beta": 1, "beta": K, "beta_1": K}
664
+ names = ("intercept", "beta", "categorical_beta", "beta_1")
665
+ m0 = np.concatenate([[priors[n][1]] * sizes[n] for n in names])
666
+ s0 = np.concatenate([[priors[n][2]] * sizes[n] for n in names])
667
+ Ymat = np.column_stack([np.asarray(Y[g], dtype=float) for g in genes])
668
+ extra_df = extra_ss = None
669
+ if ss_within is not None:
670
+ extra_df = ss_within[genes[0]][1]
671
+ extra_ss = np.array([ss_within[g][0] for g in genes])
672
+ res = _exact.slide_posterior(
673
+ X, Ymat, shared.weights, fixed.shape[1], shared.n_samples, K, m0, s0, priors["sigma"],
674
+ priors["sample_level_sd"], priors["sample_curve_sd"], extra_df, extra_ss, n_draws=settings["draws"],
675
+ seeds=np.random.SeedSequence(settings["random_seed"]).spawn(len(genes)), n_hyper=settings.get("n_hyper", 72),
676
+ ) # fmt: skip
677
+ results = []
678
+ for i, gene in enumerate(genes):
679
+ theta = res["draws"][i]
680
+ draws = {
681
+ "intercept": theta[:, 0],
682
+ "beta": theta[:, 1 : K + 1],
683
+ "categorical_beta": theta[:, K + 1],
684
+ "beta_1": theta[:, K + 2 :],
685
+ }
686
+ fit_columns = {
687
+ "sigma_mean": float(np.sqrt(res["sigma2_mean"][i])),
688
+ "sample_level_sd": float(res["level_sd_mean"][i]),
689
+ "sample_curve_sd": float(res["curve_sd_mean"][i]),
690
+ "exact_edge_mass": float(res["edge_mass"][i]),
691
+ }
692
+ results.append(_summarise_gene(gene, draws, shared, fit_columns, moments=(res["mean"][i], res["cov"][i])))
693
+ return results
694
+
695
+
696
+ def _exact_gene_results(shared, Y, genes, ss_within, settings):
697
+ """Exact inference for the two-condition model: ``[(gene, stats_row, curve_row), ...]``."""
698
+ if shared.sample_index is not None:
699
+ return _exact_slide_results(shared, Y, genes, ss_within, settings)
700
+ design = np.column_stack([np.ones(shared.B.shape[0]), shared.B, shared.condition_fit, shared.B_masked])
701
+ names = ("intercept", "beta", "categorical_beta", "beta_1")
702
+ post = _exact_fit(
703
+ design, Y, genes, shared.weights, ss_within, _resolve_priors(shared.priors), names, shared.n_bsplines, settings
704
+ )
705
+ K = shared.n_bsplines
706
+ seeds = np.random.SeedSequence(settings["random_seed"]).spawn(len(genes))
707
+ results = []
708
+ for i, gene in enumerate(genes):
709
+ theta = _exact.draws(post, i, settings["draws"], np.random.default_rng(seeds[i]))
710
+ draws = {
711
+ "intercept": theta[:, 0],
712
+ "beta": theta[:, 1 : K + 1],
713
+ "categorical_beta": theta[:, K + 1],
714
+ "beta_1": theta[:, K + 2 :],
715
+ }
716
+ fit_columns = {"sigma_mean": float(np.sqrt(post.sigma2_mean[i])), "exact_edge_mass": float(post.edge_mass[i])}
717
+ results.append(_summarise_gene(gene, draws, shared, fit_columns, moments=(post.mean[i], post.cov[i])))
718
+ return results
719
+
720
+
721
+ def _exact_profile_results(shared, Y, genes, ss_within, settings):
722
+ """Exact inference for the condition-free model: ``[(gene, diag_row, curve_row), ...]``."""
723
+ design = np.column_stack([np.ones(shared.B.shape[0]), shared.B])
724
+ post = _exact_fit(design, Y, genes, shared.weights, ss_within, _resolve_priors(shared.priors),
725
+ ("intercept", "beta"), shared.n_bsplines, settings) # fmt: skip
726
+ A = np.column_stack([np.ones(shared.B_pred.shape[0]), shared.B_pred])
727
+ seeds = np.random.SeedSequence(settings["random_seed"]).spawn(len(genes))
728
+ results = []
729
+ for i, gene in enumerate(genes):
730
+ curves = _exact.draws(post, i, settings["draws"], np.random.default_rng(seeds[i])) @ A.T
731
+ ci_low, ci_high = np.quantile(curves, [0.03, 0.97], axis=0)
732
+ diag_row = {
733
+ "gene": gene,
734
+ "sigma_mean": float(np.sqrt(post.sigma2_mean[i])),
735
+ "exact_edge_mass": float(post.edge_mass[i]),
736
+ }
737
+ curve_row = {"mean": A @ post.mean[i], "ci_low": ci_low, "ci_high": ci_high, "beta": post.mean[i][1:],
738
+ "beta_cov": post.cov[i][1:, 1:]} # fmt: skip
739
+ results.append((gene, diag_row, curve_row))
740
+ return results
741
+
742
+
743
+ def _fit_and_predict_profile(gene, Y, intercept_initval, shared, sample_kwargs, ss_within=None):
744
+ """`_fit_and_predict_gene` for the condition-free model: one curve, no test. Returns ``(gene, diag_row, curve_row)``."""
745
+ model = _build_profile_model(
746
+ Y,
747
+ shared.B,
748
+ shared.n_bsplines,
749
+ intercept_initval,
750
+ shared.likelihood,
751
+ shared.priors,
752
+ weights=shared.weights,
753
+ ss_within=ss_within,
754
+ )
755
+ with model, _quiet_pymc():
756
+ idata, diag_columns = _sample_nuts(model, sample_kwargs, ["intercept", "beta", "sigma"])
757
+
758
+ # Posterior of the mean function on the grid: credible bands, no observation noise (see `_mean_curve_draws`).
759
+ draws = _posterior_draws(idata, "intercept")[:, None] + _posterior_draws(idata, "beta") @ shared.B_pred.T
760
+ mean = draws.mean(axis=0)
761
+ ci_low, ci_high = np.quantile(draws, [0.03, 0.97], axis=0)
762
+ beta_mean = idata.posterior["beta"].mean(("chain", "draw")).to_numpy()
763
+
764
+ diag_row = {"gene": gene, **diag_columns}
765
+ beta_cov = np.cov(_posterior_draws(idata, "beta"), rowvar=False)
766
+ curve_row = {"mean": mean, "ci_low": ci_low, "ci_high": ci_high, "beta": beta_mean, "beta_cov": beta_cov}
767
+ return gene, diag_row, curve_row
768
+
769
+
770
+ def _gamma_null_pvalues(T_values, null_quantile=0.9):
771
+ """Gamma-null p-values for `T_values`, the null fitted to the lower `null_quantile` of them.
772
+
773
+ Assumes that at least a `null_quantile` fraction of the values come from the null. The Gamma is fitted
774
+ by maximum likelihood to the values below the cutoff *as a truncated sample* (each term divided by the
775
+ Gamma's mass below the cutoff); fitting it as if untruncated gives too light a tail and p-values that
776
+ are too small. Values that are missing or not finite get a missing p-value.
777
+ """
778
+ T_values = np.asarray(T_values, dtype=float)
779
+ finite = np.isfinite(T_values)
780
+ pvals = np.full(T_values.shape, np.nan)
781
+ if finite.sum() < 3:
782
+ return pvals
783
+ cutoff = np.quantile(T_values[finite], null_quantile)
784
+ pool = T_values[finite & (T_values <= cutoff) & (T_values > 0)]
785
+ if len(pool) < 3:
786
+ return pvals
787
+ shape0, _, scale0 = gamma.fit(pool, floc=0)
788
+
789
+ def neg_log_lik(log_params):
790
+ a, scale = np.exp(log_params)
791
+ return -(gamma.logpdf(pool, a, scale=scale).sum() - len(pool) * gamma.logcdf(cutoff, a, scale=scale))
792
+
793
+ fit = optimize.minimize(neg_log_lik, np.log([shape0, scale0]), method="Nelder-Mead")
794
+ a, scale = np.exp(fit.x)
795
+ pvals[finite] = gamma.sf(T_values[finite], a, scale=scale)
796
+ return pvals
797
+
798
+
799
+ def _benjamini_hochberg(pvals):
800
+ """Benjamini-Hochberg FDR-adjusted p-values over the non-missing entries; missing ones stay missing."""
801
+ pvals = np.asarray(pvals, dtype=float)
802
+ out = np.full(pvals.shape, np.nan)
803
+ present = np.isfinite(pvals)
804
+ p = pvals[present]
805
+ n = len(p)
806
+ if n == 0:
807
+ return out
808
+ order = np.argsort(p)
809
+ ranked = p[order] * n / np.arange(1, n + 1)
810
+ adjusted = np.clip(np.minimum.accumulate(ranked[::-1])[::-1], 0, 1)
811
+ filled = np.empty(n)
812
+ filled[order] = adjusted
813
+ out[present] = filled
814
+ return out
815
+
816
+
817
+ # Condition tests, a registry like the priors/likelihoods. Each has a per-gene
818
+ # part, run in the worker, which returns the test's own columns plus the mean
819
+ # difference curve on the grid, and a panel part for what needs every gene.
820
+
821
+
822
+ def _wald_gene_stats(draws0, draws1, idata, shared, moments=None):
823
+ """Wald test that the two condition curves are equal (level or shape), plus their `T` distance.
824
+
825
+ `draws0`/`draws1` are posterior draws of the mean curves, so the covariance is that of the
826
+ coefficients behind ``d(x) = f1(x) - f0(x)`` (rank ``n_bsplines``: the level plus the shape directions).
827
+ """
828
+ if moments is None:
829
+ delta = draws1 - draws0
830
+ delta_mean = delta.mean(axis=0)
831
+ cov = np.cov(delta, rowvar=False)
832
+ else:
833
+ A0, A1 = _condition_maps(shared)
834
+ A = A1 - A0
835
+ delta_mean, cov = A @ moments[0], A @ moments[1] @ A.T
836
+ wald_stat, wald_pval, wald_df = _wald_from_moments(delta_mean, cov, df_denominator=shared.df_denominator)
837
+ T = float(np.mean(delta_mean**2))
838
+ row = {"wald_stat": wald_stat, "wald_pval": wald_pval, "wald_df": wald_df, "T": T}
839
+ if shared.df_denominator is not None:
840
+ row["wald_df_denominator"] = shared.df_denominator
841
+ return row, delta_mean
842
+
843
+
844
+ def _level_weights(bin_centers):
845
+ """Trapezoid weights that turn a curve on the grid into its mean over ``[0, 1]``."""
846
+ x = np.asarray(bin_centers, dtype=float)
847
+ w = np.zeros(len(x))
848
+ dx = np.diff(x)
849
+ w[:-1] += dx / 2
850
+ w[1:] += dx / 2
851
+ return w / (x[-1] - x[0])
852
+
853
+
854
+ def _level_gene_stats(draws0, draws1, idata, shared, moments=None):
855
+ """Test of the level difference ``mean_x (f1 - f0)``, ``H0: |level| <= tau``, plus its credible interval."""
856
+ a = _level_weights(shared.bin_centers)
857
+ level_draws = (draws1 - draws0) @ a
858
+ if moments is None:
859
+ delta_mean = (draws1 - draws0).mean(axis=0)
860
+ level, var = float(level_draws.mean()), float(level_draws.var(ddof=1))
861
+ else:
862
+ A0, A1 = _condition_maps(shared)
863
+ A = A1 - A0
864
+ delta_mean = A @ moments[0]
865
+ level, var = float(a @ delta_mean), float(a @ A @ moments[1] @ A.T @ a)
866
+ stat = level**2 / var
867
+ pval = _one_df_pval(stat, var, shared.tau, shared.df_denominator)
868
+ low, high = (float(v) for v in np.quantile(level_draws, [0.03, 0.97]))
869
+ row = {
870
+ "level_diff": level,
871
+ "level_diff_ci_low": low,
872
+ "level_diff_ci_high": high,
873
+ "level_stat": stat,
874
+ "level_pval": pval,
875
+ }
876
+ if shared.df_denominator is not None:
877
+ row["level_df_denominator"] = shared.df_denominator
878
+ return row, delta_mean
879
+
880
+
881
+ def _level_panel_stats(stats_df, null_quantile):
882
+ stats_df["level_pval_adj"] = _benjamini_hochberg(stats_df["level_pval"].to_numpy())
883
+
884
+
885
+ def _one_df_pval(stat, var, tau, df_denominator):
886
+ """p-value of ``stat = estimate^2 / var`` for ``H0: |effect| <= tau`` (1 df; F with slides)."""
887
+ noncentrality = tau**2 / var
888
+ if df_denominator is None:
889
+ return float(ncx2.sf(stat, 1, noncentrality)) if tau > 0 else float(chi2.sf(stat, 1))
890
+ if tau > 0:
891
+ return float(ncf.sf(stat, 1, df_denominator, noncentrality))
892
+ return float(f.sf(stat, 1, df_denominator))
893
+
894
+
895
+ def _slope_gene_stats(draws0, draws1, idata, shared, moments=None):
896
+ """Test of the slope difference: the least-squares slope of ``f1 - f0`` over ``[0, 1]``, ``H0: |slope| <= tau``.
897
+
898
+ One degree of freedom, so with few replicates it has far more power than the ``K - 1`` df parallel test
899
+ for a gradient that steepens or flattens.
900
+ """
901
+ weights = shared.slope_weights
902
+ beta_1 = _posterior_draws(idata, "beta_1")
903
+ slope_draws = beta_1 @ weights
904
+ if moments is None:
905
+ slope, var = float(slope_draws.mean()), float(slope_draws.var(ddof=1))
906
+ delta_mean = (draws1 - draws0).mean(axis=0)
907
+ else:
908
+ b1 = slice(shared.n_bsplines + 2, 2 * shared.n_bsplines + 2)
909
+ slope, var = float(moments[0][b1] @ weights), float(weights @ moments[1][b1, b1] @ weights)
910
+ A0, A1 = _condition_maps(shared)
911
+ delta_mean = (A1 - A0) @ moments[0]
912
+ stat = slope**2 / var
913
+ low, high = (float(v) for v in np.quantile(slope_draws, [0.03, 0.97]))
914
+ row = {
915
+ "slope_diff": slope,
916
+ "slope_diff_ci_low": low,
917
+ "slope_diff_ci_high": high,
918
+ "slope_stat": stat,
919
+ "slope_pval": _one_df_pval(stat, var, shared.tau, shared.df_denominator),
920
+ }
921
+ if shared.df_denominator is not None:
922
+ row["slope_df_denominator"] = shared.df_denominator
923
+ return row, delta_mean
924
+
925
+
926
+ def _slope_panel_stats(stats_df, null_quantile):
927
+ stats_df["slope_pval_adj"] = _benjamini_hochberg(stats_df["slope_pval"].to_numpy())
928
+
929
+
930
+ def _wald_panel_stats(stats_df, null_quantile):
931
+ # Per-gene p-values; only the FDR adjustment needs the panel. (The panel nulls on `T` that used to be
932
+ # here were circular: they always flagged the top `1 - null_quantile` of the genes.)
933
+ stats_df["wald_pval_adj"] = _benjamini_hochberg(stats_df["wald_pval"].to_numpy())
934
+
935
+
936
+ def _parallel_gene_stats(draws0, draws1, idata, shared, moments=None):
937
+ """`parallel_curves_test` on the posterior of `beta_1`, plus the linear slope of the difference curve."""
938
+ beta_1 = _posterior_draws(idata, "beta_1") # (n_draws, K)
939
+ if moments is None:
940
+ shape_diff, stat, df, pval = parallel_curves_test(
941
+ beta_1, shared.shape_metric, tau=shared.tau, df_denominator=shared.df_denominator
942
+ )
943
+ delta_mean = (draws1 - draws0).mean(axis=0)
944
+ else:
945
+ K = shared.n_bsplines
946
+ b1 = slice(K + 2, 2 * K + 2)
947
+ shape_diff, stat, df, pval = parallel_curves_test(
948
+ moments[0][b1], shared.shape_metric, tau=shared.tau, cov=moments[1][b1, b1],
949
+ df_denominator=shared.df_denominator,
950
+ ) # fmt: skip
951
+ A0, A1 = _condition_maps(shared)
952
+ delta_mean = (A1 - A0) @ moments[0]
953
+ slope = beta_1 @ shared.slope_weights
954
+ slope_low, slope_high = (float(v) for v in np.quantile(slope, [0.03, 0.97]))
955
+ row = {
956
+ "shape_diff": shape_diff,
957
+ "slope_diff": float(slope.mean()),
958
+ "slope_diff_ci_low": slope_low,
959
+ "slope_diff_ci_high": slope_high,
960
+ "parallel_stat": stat,
961
+ "parallel_df": df,
962
+ "parallel_pval": pval,
963
+ }
964
+ if moments is not None:
965
+ row["slope_diff"] = float(moments[0][slice(shared.n_bsplines + 2, None)] @ shared.slope_weights)
966
+ if shared.df_denominator is not None:
967
+ row["parallel_df_denominator"] = shared.df_denominator
968
+ return row, delta_mean
969
+
970
+
971
+ def _parallel_panel_stats(stats_df, null_quantile):
972
+ # The p-values are per gene; only the FDR adjustment needs the panel.
973
+ stats_df["parallel_pval_adj"] = _benjamini_hochberg(stats_df["parallel_pval"].to_numpy())
974
+
975
+
976
+ #: ``{name: (per-gene stats, panel stats)}`` - the condition tests `fit_model` can run.
977
+ _TESTS = {
978
+ "wald": (_wald_gene_stats, _wald_panel_stats),
979
+ "parallel": (_parallel_gene_stats, _parallel_panel_stats),
980
+ "level": (_level_gene_stats, _level_panel_stats),
981
+ "slope": (_slope_gene_stats, _slope_panel_stats),
982
+ }
983
+
984
+
985
+ _CURVE_SUFFIXES = ("cond0_mean", "cond0_ci_low", "cond0_ci_high", "cond1_mean", "cond1_ci_low", "cond1_ci_high")
986
+ _COEF_NAMES = ("beta", "beta_1")
987
+
988
+
989
+ def _coef_columns(n_bsplines):
990
+ """Column labels of a coefficient table, one per spline basis function (strings, so they round-trip through h5ad)."""
991
+ return [f"B{k}" for k in range(n_bsplines)]
992
+
993
+
994
+ def _resolve_inference(inference, model_setup):
995
+ """``"auto"`` picks exact inference where the model allows it (see `_exact.supported`), NUTS otherwise."""
996
+ allowed = ("auto", "exact", *_INFERENCE)
997
+ if inference not in allowed:
998
+ raise ValueError(f"Unknown inference {inference!r}. Available: {sorted(allowed)}")
999
+ if model_setup is None:
1000
+ return "nuts" if inference == "auto" else inference
1001
+ ok = _exact.supported(
1002
+ _resolve_priors(model_setup.priors), model_setup.likelihood, model_setup.binned.sample is not None
1003
+ )
1004
+ if inference == "exact" and not ok:
1005
+ raise ValueError(
1006
+ "inference='exact' needs the normal likelihood and normal priors on the coefficients; use 'nuts'."
1007
+ )
1008
+ resolved = ("exact" if ok else "nuts") if inference == "auto" else inference
1009
+ if resolved != "exact":
1010
+ _require_pymc(resolved)
1011
+ return resolved
1012
+
1013
+
1014
+ def _warn_predictive_seed(predictive_seed):
1015
+ if predictive_seed is not None:
1016
+ warnings.warn(
1017
+ "`predictive_seed` no longer has an effect: the curves are now the posterior of the mean function, "
1018
+ "computed from the posterior draws without new random draws.",
1019
+ DeprecationWarning,
1020
+ stacklevel=3,
1021
+ )
1022
+
1023
+
1024
+ def _fittable_genes(genes, Y):
1025
+ """The genes worth fitting.
1026
+
1027
+ Genes whose fitted rows are constant (e.g. all zero) are left out with a warning: the model reproduces
1028
+ them exactly, so the noise scale has no lower bound and the posterior is improper (the sampler diverges
1029
+ and the test would be meaningless). Their stats stay missing.
1030
+ """
1031
+ degenerate = [g for g in genes if not np.isfinite(Y[g]).all() or np.ptp(np.asarray(Y[g], dtype=float)) == 0]
1032
+ if degenerate:
1033
+ shown = ", ".join(degenerate[:10]) + (" ..." if len(degenerate) > 10 else "")
1034
+ warnings.warn(
1035
+ f"{len(degenerate)} gene(s) are constant or non-finite over the fitted rows and were not fitted "
1036
+ f"(their stats are missing): {shown}",
1037
+ UserWarning,
1038
+ stacklevel=3,
1039
+ )
1040
+ return [g for g in genes if g not in set(degenerate)]
1041
+
1042
+
1043
+ def _check_slides_per_condition(binned, sample_index, condition_fit):
1044
+ """The F reference uses ``n_slides - 2`` degrees of freedom, which assumes at least two slides per condition."""
1045
+ codes = np.asarray(condition_fit, dtype=int)
1046
+ per_condition = [len(np.unique(np.asarray(sample_index)[codes == c])) for c in (0, 1)]
1047
+ if min(per_condition) < 2:
1048
+ labels = binned.condition_categories
1049
+ raise ValueError(
1050
+ "Comparing conditions across slides needs at least 2 slides per condition, found "
1051
+ f"{per_condition[0]} for {labels['0']!r} and {per_condition[1]} for {labels['1']!r}."
1052
+ )
1053
+
1054
+
1055
+ def _warn_unsupported_range(binned):
1056
+ """Warn when one condition has no cells over part of ``[0, 1]``: the curve there is extrapolated."""
1057
+ empty = (np.asarray(binned.n_obs_grid) == 0).mean(axis=1)
1058
+ if empty.max() > 0.1:
1059
+ labels = binned.condition_categories
1060
+ detail = ", ".join(f"{labels[str(c)]!r}: {100 * empty[c]:.0f}%" for c in (0, 1))
1061
+ warnings.warn(
1062
+ f"Part of the anchor range has no cells in one condition ({detail} of the grid empty). The curves "
1063
+ "there are extrapolated from the spline and the prior, which inflates `shape_diff`/`integral`.",
1064
+ UserWarning,
1065
+ stacklevel=3,
1066
+ )
1067
+
1068
+
1069
+ def fit_model(
1070
+ adata,
1071
+ model_setup,
1072
+ n_workers=1,
1073
+ progressbar=True,
1074
+ inplace=True,
1075
+ use_binned=True,
1076
+ stats_key="dsvg_stats",
1077
+ preds_key="dsvg_preds",
1078
+ curves_key="dsvg_curves",
1079
+ null_quantile=0.9,
1080
+ sample_kwargs=None,
1081
+ test="parallel",
1082
+ tau=None,
1083
+ predictive_seed=None,
1084
+ inference="auto",
1085
+ coefs_key="dsvg_coefs",
1086
+ **pymc_kwargs,
1087
+ ):
1088
+ """Fit every per-gene model from `setup_model`, in parallel, and summarize the result.
1089
+
1090
+ For each gene: (re-)builds its own private model and obtains the
1091
+ posterior (see `inference`), then samples the posterior predictive on the
1092
+ fixed `anchor` grid under both levels of `condition`, and tests the two
1093
+ curves against each other (see `test`).
1094
+
1095
+ Parameters
1096
+ ----------
1097
+ adata
1098
+ The same `AnnData` object `model_setup.binned` was built from; only
1099
+ used as the write target when `inplace` is `True`.
1100
+ model_setup
1101
+ Output of `setup_model`.
1102
+ n_workers
1103
+ Number of worker processes (`joblib`, `loky` backend). Each gene's
1104
+ model lives entirely within one worker for its whole lifetime -
1105
+ genes are never split across workers or re-synced, so there is
1106
+ nothing to race. `n_workers=1` runs sequentially in-process (no
1107
+ extra pickling at all).
1108
+ use_binned
1109
+ Fit on the per-bin averages from `diffsvg.pp.bin_obs` (default), or
1110
+ on the raw per-cell values when `False`. Raw keeps every
1111
+ observation instead of collapsing each window to its mean, at the
1112
+ cost of a far larger design matrix - with spatial data that is
1113
+ mostly zero counts, which is what the binning is there to avoid,
1114
+ this is typically much slower per gene. Knots, degree and the
1115
+ condition encoding are identical either way, so the two are
1116
+ directly comparable. Bins built with ``window_radius=None`` are
1117
+ weighted by their cell counts, which makes the binned fit carry the
1118
+ same information as the per-cell one at the cost of the binned fit.
1119
+ progressbar
1120
+ Show a `tqdm` progress bar over completed genes. Per-worker MCMC
1121
+ progress bars are always disabled (they would interleave garbage
1122
+ across processes); pass e.g. ``sample_kwargs={"progressbar": True}``
1123
+ to override that only when `n_workers=1`.
1124
+ inplace
1125
+ If `True` (default), write results to `adata.varm`/`adata.uns`
1126
+ (aligned to ``adata.var_names``, with genes not in `model_setup`
1127
+ left as `NaN`) and return `None`. If `False`, leave `adata` untouched
1128
+ and return ``(stats_df, preds_df, curve_dfs)``; `curve_dfs` then also
1129
+ holds the coefficient tables under ``"beta"`` and ``"beta_1"``.
1130
+ stats_key
1131
+ `adata.varm` key for the per-gene summary statistics (`n_genes x n`;
1132
+ see `diffsvg.pl.plot_volcano`/`diffsvg.pl.plot_pvalue_histogram`).
1133
+ Its columns depend on `test`; which test ran (and its `tau`) is
1134
+ recorded in ``adata.uns[f"{stats_key}_params"]``.
1135
+ preds_key
1136
+ `adata.varm` key for the per-gene, per-bin predicted
1137
+ ``condition==1`` minus ``condition==0`` difference curve
1138
+ (`n_genes x (n_bins + 1)`).
1139
+ curves_key
1140
+ Prefix for six `adata.varm` entries (`n_genes x (n_bins + 1)` each),
1141
+ one per ``{"cond0", "cond1"} x {"mean", "ci_low", "ci_high"}``: the
1142
+ per-condition predicted curves with a credible band, consumed by
1143
+ `diffsvg.pl.plot_prediction_curve`. The condition labels are also
1144
+ written to ``adata.uns[f"{curves_key}_condition_categories"]``.
1145
+ null_quantile
1146
+ No longer used by `fit_model` (kept for backwards compatibility).
1147
+ The panel-based empirical/Gamma nulls it configured were circular;
1148
+ ``test="wald"`` now has a per-gene p-value.
1149
+ sample_kwargs
1150
+ Extra keyword arguments merged into the `pymc.sample` call (applied
1151
+ to every gene). Any extra keyword arguments to `fit_model` itself are
1152
+ forwarded the same way, so ``fit_model(..., chains=2)`` also works.
1153
+ With ``inference="exact"`` only ``draws``, ``random_seed``,
1154
+ ``n_grid`` and ``n_hyper`` apply.
1155
+ test
1156
+ Which test to run between the two condition curves:
1157
+
1158
+ - ``"parallel"`` (default): whether the curves are parallel, i.e. differ only
1159
+ in level, via `parallel_curves_test` on the posterior of the
1160
+ interaction coefficients. The p-value is per gene, so it does not
1161
+ depend on the panel and works for a single gene. Columns
1162
+ ``shape_diff`` (how far from parallel), ``slope_diff`` (the linear
1163
+ trend of ``f1 - f0`` over the anchor range, with a 94% credible
1164
+ interval ``slope_diff_ci_low``/``slope_diff_ci_high``),
1165
+ ``parallel_stat``, ``parallel_df``, ``parallel_pval`` and
1166
+ ``parallel_pval_adj`` (Benjamini-Hochberg).
1167
+ - ``"wald"``: whether the curves are equal at all (level or shape),
1168
+ a tradeSeq-style functional Wald test on the posterior of the
1169
+ difference curve, chi-square with ``n_bsplines`` degrees of
1170
+ freedom (columns ``wald_stat``, ``wald_pval``, ``wald_df``,
1171
+ ``wald_pval_adj`` and ``T``, the mean squared difference). It has
1172
+ no `tau`, so with many cells nearly every gene with any difference
1173
+ becomes significant; rank by ``wald_stat`` rather than thresholding.
1174
+ - ``"level"``: whether the curves differ in level, i.e. in the mean
1175
+ of ``f1 - f0`` over the anchor range, beyond `tau` (``H0: |level|
1176
+ <= tau``; noncentral chi-square with 1 degree of freedom). The
1177
+ complement of ``"parallel"``, which ignores the level. Columns
1178
+ ``level_diff`` (with a 94% credible interval), ``level_stat``,
1179
+ ``level_pval`` and ``level_pval_adj``.
1180
+ - ``"slope"``: whether the gap between the curves changes linearly
1181
+ along the anchor, i.e. whether one gradient is steeper than the
1182
+ other: the least-squares slope of ``f1 - f0`` over ``[0, 1]``,
1183
+ ``H0: |slope| <= tau``, 1 degree of freedom. With few replicates it
1184
+ has far more power than ``"parallel"`` (``n_bsplines - 1`` df) for
1185
+ gradients that steepen or flatten, and it misses non-monotone
1186
+ changes, which ``"parallel"`` covers. Columns ``slope_diff`` (with
1187
+ a 94% credible interval), ``slope_stat``, ``slope_pval`` and
1188
+ ``slope_pval_adj``.
1189
+
1190
+ With data binned with `sample` (several slides per condition) the model has a random level and a
1191
+ random curve per slide, so the covariance of the interaction
1192
+ coefficients holds the variation between slides, and the p-value
1193
+ uses an F reference with ``n_slides - 2`` denominator degrees of
1194
+ freedom for every test (column ``*_df_denominator``). The fitted curves are
1195
+ the population-level ones. Fit on bins (NUTS is fine) or on cells
1196
+ (``use_binned=False``).
1197
+
1198
+ Both tests also write ``integral`` (the area between the curves),
1199
+ the ``categorical_beta`` summary and the sampler diagnostics.
1200
+ Binned fits with overlapping windows (`diffsvg.pp.bin_obs`'s default
1201
+ `window_radius`) treat neighbouring bins as independent although they
1202
+ share cells, which makes the model over-confident and the
1203
+ ``"parallel"`` p-values too small (a warning says so); contiguous
1204
+ bins (``bin_obs(..., window_radius=None)``) and fits on the cells
1205
+ (``use_binned=False``) do not have that problem.
1206
+ tau
1207
+ For the ``"parallel"``, ``"level"`` and ``"slope"`` tests: the
1208
+ smallest difference that counts (``shape_diff``, ``level_diff`` or
1209
+ ``slope_diff``). The p-value tests ``|difference| <= tau``, so smaller
1210
+ differences do not become significant however many cells there are.
1211
+ Defaults to ``0.1`` for pooled data and to ``0`` for data binned with
1212
+ `sample`, where the variation between slides already calibrates the
1213
+ test and a positive `tau` only costs power.
1214
+ predictive_seed
1215
+ Deprecated, has no effect. The curves, their bands, ``integral`` and
1216
+ the Wald test are now computed from the posterior draws of the mean
1217
+ function (no observation noise), so they need no extra random draws.
1218
+ inference
1219
+ How each gene's posterior is obtained:
1220
+
1221
+ - ``"nuts"``: MCMC with `pymc.sample` (needs the ``pymc`` extra), configured by
1222
+ `sample_kwargs`. Reports ``n_divergent``, ``rhat_max`` and
1223
+ ``ess_bulk_min``.
1224
+ - ``"exact"``: the exact posterior (see `diffsvg.tl._exact`): the
1225
+ coefficients are conjugate given the noise scale, which is
1226
+ integrated out numerically, as are the two slide scales. No Monte
1227
+ Carlo error and no sampler; needs the normal likelihood and normal
1228
+ coefficient priors. Reports ``exact_edge_mass``.
1229
+ - ``"auto"`` (default): ``"exact"`` where the model allows it,
1230
+ ``"nuts"`` otherwise.
1231
+ coefs_key
1232
+ Prefix for two `adata.varm` tables (`n_genes x n_bsplines`) with the
1233
+ posterior means of the spline coefficients, consumed by
1234
+ `diffsvg.tl.cluster_curves`: ``f"{coefs_key}_beta"`` (the main spline
1235
+ term; with the 0/1 condition coding this is the curve of condition 0,
1236
+ not a pooled one) and ``f"{coefs_key}_beta_1"`` (the interaction, how
1237
+ the curve of condition 1 differs from it).
1238
+
1239
+ Returns
1240
+ -------
1241
+ None or tuple[pandas.DataFrame, pandas.DataFrame, dict[str, pandas.DataFrame]]
1242
+ """
1243
+ if test not in _TESTS:
1244
+ raise ValueError(f"Unknown test {test!r}. Available: {sorted(_TESTS)}")
1245
+ if tau is not None and tau < 0:
1246
+ raise ValueError(f"`tau` must be >= 0, got {tau}.")
1247
+ inference = _resolve_inference(inference, model_setup)
1248
+ if inference == "exact":
1249
+ sample_defaults = _exact_settings({**(sample_kwargs or {}), **pymc_kwargs})
1250
+ elif inference == "nuts":
1251
+ sample_defaults = {
1252
+ "nuts_sampler": "nutpie",
1253
+ "draws": 1000,
1254
+ "tune": 1000,
1255
+ "chains": 4,
1256
+ "random_seed": 42,
1257
+ "compute_convergence_checks": False,
1258
+ "init": "adapt_diag",
1259
+ "progressbar": False,
1260
+ }
1261
+ if inference != "exact":
1262
+ sample_defaults.update(sample_kwargs or {})
1263
+ sample_defaults.update(pymc_kwargs)
1264
+ binned = model_setup.binned
1265
+ if not binned.has_condition:
1266
+ raise ValueError(
1267
+ "adata.uns['binned'] was built without a `condition`, so there are no two curves to compare. "
1268
+ "Re-run `diffsvg.pp.bin_obs` with one, or use `diffsvg.tl.fit_profiles` + "
1269
+ "`diffsvg.tl.one_vs_rest` for the condition-free one-vs-rest test."
1270
+ )
1271
+ B, B_masked, condition_fit, Y, intercept_initvals = _fit_inputs(adata, binned, use_binned)
1272
+ weights, ss_within, n_obs_grid = _weighting(binned, use_binned)
1273
+ sample_index, n_samples = _sample_inputs(adata, binned, use_binned)
1274
+ if tau is None:
1275
+ tau = 0.0 if n_samples is not None else 0.1
1276
+ if n_samples is not None:
1277
+ if n_samples < 3:
1278
+ raise ValueError(f"Comparing conditions across slides needs at least 3 slides, found {n_samples}.")
1279
+ if test == "parallel" and use_binned and not binned.weighted:
1280
+ warnings.warn(
1281
+ "test='parallel' on overlapping bins: neighbouring bins share cells, so the p-values come out too "
1282
+ "small. Re-run `diffsvg.pp.bin_obs(..., window_radius=None)`, or fit with `use_binned=False`.",
1283
+ UserWarning,
1284
+ stacklevel=2,
1285
+ )
1286
+ _warn_predictive_seed(predictive_seed)
1287
+ if n_samples is not None:
1288
+ _check_slides_per_condition(binned, sample_index, condition_fit)
1289
+ if use_binned and binned.condition is not None and binned.n_obs_grid is not None:
1290
+ _warn_unsupported_range(binned)
1291
+ fit_genes = _fittable_genes(binned.genes, Y)
1292
+ shared = _FitShared(
1293
+ B=B,
1294
+ B_masked=B_masked,
1295
+ condition_fit=condition_fit,
1296
+ n_bsplines=binned.n_bsplines,
1297
+ likelihood=model_setup.likelihood,
1298
+ priors=model_setup.priors,
1299
+ bin_centers=binned.bin_centers,
1300
+ B_pred=binned.B_pred,
1301
+ B_pred_masked_0=binned.B_pred_masked_0,
1302
+ B_pred_masked_1=binned.B_pred_masked_1,
1303
+ condition_pred_0=binned.condition_pred_0,
1304
+ condition_pred_1=binned.condition_pred_1,
1305
+ test=test,
1306
+ tau=float(tau),
1307
+ shape_metric=shape_metric(binned.n_bsplines, binned.degree) if test == "parallel" else None,
1308
+ slope_weights=_slope_weights(binned.n_bsplines, binned.degree) if test in ("parallel", "slope") else None,
1309
+ weights=weights,
1310
+ inference=inference,
1311
+ sample_index=sample_index,
1312
+ n_samples=n_samples,
1313
+ df_denominator=None if n_samples is None else n_samples - 2,
1314
+ )
1315
+
1316
+ n_jobs = 1 if n_workers is None else n_workers
1317
+ if inference == "exact":
1318
+ results = _exact_gene_results(shared, Y, fit_genes, ss_within, sample_defaults)
1319
+ else:
1320
+ results = Parallel(n_jobs=n_jobs, backend="loky", return_as="generator_unordered")(
1321
+ delayed(_fit_and_predict_gene)(
1322
+ gene,
1323
+ Y[gene],
1324
+ intercept_initvals[gene],
1325
+ shared,
1326
+ sample_defaults,
1327
+ ss_within=None if ss_within is None else ss_within[gene],
1328
+ )
1329
+ for gene in fit_genes
1330
+ )
1331
+ if progressbar:
1332
+ results = tqdm(results, total=len(fit_genes), desc="Fitting DiffSVG models")
1333
+
1334
+ stats_rows = []
1335
+ curve_rows = {}
1336
+ for gene, stats_row, curve_row in results:
1337
+ stats_rows.append(stats_row)
1338
+ curve_rows[gene] = curve_row
1339
+
1340
+ stats_df = pd.DataFrame(stats_rows).set_index("gene").reindex(binned.genes)
1341
+ _TESTS[test][1](stats_df, null_quantile)
1342
+ test_params = {"test": test, "tau": float(tau)} if test in ("parallel", "level", "slope") else {"test": test}
1343
+
1344
+ # Column labels must be strings (not e.g. plain floats) so that
1345
+ # `adata.varm[...]` round-trips through h5ad, whose DataFrame writer
1346
+ # requires string column names.
1347
+ bin_cols = [str(round(float(c), 4)) for c in binned.bin_centers]
1348
+ preds_df = pd.DataFrame.from_dict(
1349
+ {gene: row["delta"] for gene, row in curve_rows.items()}, orient="index", columns=bin_cols
1350
+ ).reindex(binned.genes)
1351
+ curve_dfs = {
1352
+ suffix: pd.DataFrame.from_dict(
1353
+ {gene: row[suffix] for gene, row in curve_rows.items()}, orient="index", columns=bin_cols
1354
+ ).reindex(binned.genes)
1355
+ for suffix in _CURVE_SUFFIXES
1356
+ }
1357
+ coef_dfs = {
1358
+ name: pd.DataFrame.from_dict(
1359
+ {gene: row[name] for gene, row in curve_rows.items()},
1360
+ orient="index",
1361
+ columns=_coef_columns(binned.n_bsplines),
1362
+ ).reindex(binned.genes)
1363
+ for name in _COEF_NAMES
1364
+ }
1365
+
1366
+ if not inplace:
1367
+ stats_df.attrs.update(test_params)
1368
+ return stats_df, preds_df, {**curve_dfs, **coef_dfs}
1369
+
1370
+ adata.varm[stats_key] = stats_df.reindex(adata.var_names)
1371
+ adata.varm[preds_key] = preds_df.reindex(adata.var_names)
1372
+ for suffix, df in curve_dfs.items():
1373
+ adata.varm[f"{curves_key}_{suffix}"] = df.reindex(adata.var_names)
1374
+ for name, df in coef_dfs.items():
1375
+ adata.varm[f"{coefs_key}_{name}"] = df.reindex(adata.var_names)
1376
+ adata.uns[f"{curves_key}_condition_categories"] = binned.condition_categories
1377
+ adata.uns[f"{stats_key}_params"] = test_params
1378
+ return None
1379
+
1380
+
1381
+ # ============================================================
1382
+ # 3b) Condition-free profiles + the one-vs-rest test
1383
+ # ============================================================
1384
+
1385
+
1386
+ _PROFILE_SUFFIXES = ("mean", "ci_low", "ci_high")
1387
+
1388
+
1389
+ def fit_profiles(
1390
+ adata,
1391
+ model_setup,
1392
+ n_workers=1,
1393
+ progressbar=True,
1394
+ inplace=True,
1395
+ use_binned=True,
1396
+ profiles_key="dsvg_profiles",
1397
+ sample_kwargs=None,
1398
+ predictive_seed=None,
1399
+ inference="auto",
1400
+ **pymc_kwargs,
1401
+ ):
1402
+ """Fit the condition-free model for every gene, giving one expression profile per gene.
1403
+
1404
+ The counterpart to `fit_model` for data binned without a `condition`
1405
+ (`diffsvg.pp.bin_obs(..., condition=None)`): it drops the condition
1406
+ covariate and its spline interaction, so each gene gets a single curve
1407
+ rather than one per condition. No differential statistic is computed
1408
+ here - `one_vs_rest` produces that by comparing genes against each
1409
+ other.
1410
+
1411
+ Also usable on data that *does* have conditions, in which case both
1412
+ conditions' bins feed one pooled curve per gene. That is an alternative
1413
+ to what `one_vs_rest` does by default there (averaging the two fitted
1414
+ condition curves), and the two need not agree exactly.
1415
+
1416
+ Parameters
1417
+ ----------
1418
+ adata
1419
+ The `AnnData` `model_setup.binned` was built from; the write target
1420
+ when `inplace` is `True`.
1421
+ model_setup
1422
+ Output of `setup_model`.
1423
+ n_workers, progressbar, use_binned, sample_kwargs, predictive_seed, **pymc_kwargs
1424
+ As in `fit_model`.
1425
+ inplace
1426
+ If `True` (default), write the results to `adata.varm` and return
1427
+ `None`; otherwise return ``(profile_dfs, diagnostics_df)``.
1428
+ profiles_key
1429
+ Prefix for three `adata.varm` tables (`n_genes x (n_bins + 1)`), one
1430
+ per ``{"mean", "ci_low", "ci_high"}``, plus a
1431
+ ``f"{profiles_key}_diagnostics"`` table of per-gene sampler
1432
+ diagnostics and ``f"{profiles_key}_beta"`` (`n_genes x n_bsplines`,
1433
+ the posterior means of the spline coefficients, for
1434
+ `diffsvg.tl.cluster_curves` with ``by="profile"``). With
1435
+ ``inplace=False`` that table is ``profile_dfs["beta"]``.
1436
+
1437
+ Returns
1438
+ -------
1439
+ None or tuple[dict[str, pandas.DataFrame], pandas.DataFrame]
1440
+ """
1441
+ if inference not in ("auto", "exact", "nuts"):
1442
+ raise ValueError(f"Unknown inference {inference!r} for fit_profiles. Available: ['auto', 'exact', 'nuts']")
1443
+ priors = _resolve_priors(model_setup.priors)
1444
+ exact_ok = (
1445
+ model_setup.likelihood == "normal"
1446
+ and model_setup.binned.sample is None
1447
+ and all(priors[n][0] == "normal" for n in ("intercept", "beta"))
1448
+ and priors["sigma"][0] in _exact._SIGMA_LOGPDF
1449
+ )
1450
+ if inference == "exact" and not exact_ok:
1451
+ raise ValueError("inference='exact' needs the normal likelihood and normal priors on the coefficients.")
1452
+ inference = ("exact" if exact_ok else "nuts") if inference == "auto" else inference
1453
+ if inference != "exact":
1454
+ _require_pymc(inference)
1455
+ sample_defaults = {
1456
+ "nuts_sampler": "nutpie",
1457
+ "draws": 1000,
1458
+ "tune": 1000,
1459
+ "chains": 4,
1460
+ "random_seed": 42,
1461
+ "compute_convergence_checks": False,
1462
+ "init": "adapt_diag",
1463
+ "progressbar": False,
1464
+ }
1465
+ if inference == "exact":
1466
+ sample_defaults = _exact_settings({**(sample_kwargs or {}), **pymc_kwargs})
1467
+ else:
1468
+ sample_defaults.update(sample_kwargs or {})
1469
+ sample_defaults.update(pymc_kwargs)
1470
+
1471
+ binned = model_setup.binned
1472
+ if binned.sample is not None:
1473
+ raise ValueError("`fit_profiles` does not model slides; bin without `sample`.")
1474
+ B, _, _, Y, intercept_initvals = _fit_inputs(adata, binned, use_binned)
1475
+ weights, ss_within, n_obs_grid = _weighting(binned, use_binned)
1476
+ _warn_predictive_seed(predictive_seed)
1477
+ fit_genes = _fittable_genes(binned.genes, Y)
1478
+ shared = _ProfileShared(
1479
+ B=B,
1480
+ n_bsplines=binned.n_bsplines,
1481
+ likelihood=model_setup.likelihood,
1482
+ priors=model_setup.priors,
1483
+ B_pred=binned.B_pred,
1484
+ weights=weights,
1485
+ # One pooled curve: with conditions binned separately, a grid point holds both levels' cells.
1486
+ )
1487
+
1488
+ n_jobs = 1 if n_workers is None else n_workers
1489
+ if inference == "exact":
1490
+ results = _exact_profile_results(shared, Y, fit_genes, ss_within, sample_defaults)
1491
+ else:
1492
+ results = Parallel(n_jobs=n_jobs, backend="loky", return_as="generator_unordered")(
1493
+ delayed(_fit_and_predict_profile)(
1494
+ gene,
1495
+ Y[gene],
1496
+ intercept_initvals[gene],
1497
+ shared,
1498
+ sample_defaults,
1499
+ ss_within=None if ss_within is None else ss_within[gene],
1500
+ )
1501
+ for gene in fit_genes
1502
+ )
1503
+ if progressbar:
1504
+ results = tqdm(results, total=len(binned.genes), desc="Fitting DiffSVG profiles")
1505
+
1506
+ diag_rows = []
1507
+ curve_rows = {}
1508
+ for gene, diag_row, curve_row in results:
1509
+ diag_rows.append(diag_row)
1510
+ curve_rows[gene] = curve_row
1511
+
1512
+ diagnostics_df = pd.DataFrame(diag_rows).set_index("gene").reindex(binned.genes)
1513
+ bin_cols = [str(round(float(c), 4)) for c in binned.bin_centers]
1514
+ profile_dfs = {
1515
+ suffix: pd.DataFrame.from_dict(
1516
+ {gene: row[suffix] for gene, row in curve_rows.items()}, orient="index", columns=bin_cols
1517
+ ).reindex(binned.genes)
1518
+ for suffix in _PROFILE_SUFFIXES
1519
+ }
1520
+ profile_dfs["beta"] = pd.DataFrame.from_dict(
1521
+ {gene: row["beta"] for gene, row in curve_rows.items()},
1522
+ orient="index",
1523
+ columns=_coef_columns(binned.n_bsplines),
1524
+ ).reindex(binned.genes)
1525
+
1526
+ if not inplace:
1527
+ return profile_dfs, diagnostics_df
1528
+
1529
+ for suffix, df in profile_dfs.items():
1530
+ adata.varm[f"{profiles_key}_{suffix}"] = df.reindex(adata.var_names)
1531
+ adata.varm[f"{profiles_key}_diagnostics"] = diagnostics_df.reindex(adata.var_names)
1532
+ fitted = [g for g in binned.genes if g in curve_rows]
1533
+ adata.uns[f"{profiles_key}_beta_cov"] = {
1534
+ "genes": np.asarray(fitted, dtype=str),
1535
+ "cov": np.stack([curve_rows[g]["beta_cov"] for g in fitted]),
1536
+ "degree": int(binned.degree),
1537
+ }
1538
+ return None
1539
+
1540
+
1541
+ def _gene_profiles(adata, curves_key, profiles_key):
1542
+ """One curve per gene, as a (genes x bins) frame, plus a note on where it came from.
1543
+
1544
+ Prefers an explicit `fit_profiles` result; otherwise averages the two
1545
+ per-condition curves from `fit_model`, which is what collapses a
1546
+ two-condition fit into a single profile per gene.
1547
+ """
1548
+ if f"{profiles_key}_mean" in adata.varm:
1549
+ return adata.varm[f"{profiles_key}_mean"], "profiles"
1550
+ if f"{curves_key}_cond0_mean" in adata.varm:
1551
+ cond0 = adata.varm[f"{curves_key}_cond0_mean"]
1552
+ cond1 = adata.varm[f"{curves_key}_cond1_mean"]
1553
+ return (cond0 + cond1) / 2.0, "condition-average"
1554
+ raise KeyError(
1555
+ f"neither adata.varm[{profiles_key + '_mean'!r}] nor adata.varm[{curves_key + '_cond0_mean'!r}] found - "
1556
+ "run `diffsvg.tl.fit_profiles` (no conditions) or `diffsvg.tl.fit_model` (with conditions) first."
1557
+ )
1558
+
1559
+
1560
+ def one_vs_rest(
1561
+ adata,
1562
+ gene,
1563
+ curves_key="dsvg_curves",
1564
+ profiles_key="dsvg_profiles",
1565
+ null_quantile=0.9,
1566
+ inplace=False,
1567
+ key_added=None,
1568
+ tau=0.1,
1569
+ ):
1570
+ """Compare one gene's expression profile against every other gene's.
1571
+
1572
+ Each gene is reduced to a single curve along `anchor` (see
1573
+ `_gene_profiles`), and the probe gene is then tested against each of the
1574
+ remaining genes individually.
1575
+
1576
+ With a `fit_profiles` result (which stores each gene's posterior
1577
+ covariance of its spline coefficients) every pair gets the parallel-curves
1578
+ test of `parallel_curves_test`: do the two genes' curves differ in shape,
1579
+ beyond `tau`, rather than only in level? The covariance of the difference
1580
+ is the sum of the two genes' posterior covariances (their fits are
1581
+ independent). Without it (curves averaged from `fit_model`), the pairs
1582
+ are ranked by ``T`` against a Gamma null fitted to the "rest" genes.
1583
+
1584
+ Parameters
1585
+ ----------
1586
+ adata
1587
+ Annotated data matrix that `fit_profiles` or `fit_model` wrote into.
1588
+ gene
1589
+ The probe gene, i.e. the "one" in one-vs-rest.
1590
+ curves_key, profiles_key
1591
+ `adata.varm` prefixes written by `fit_model` / `fit_profiles`. An
1592
+ explicit `fit_profiles` result wins when both are present.
1593
+ null_quantile
1594
+ Only without a `fit_profiles` result: fraction of the "rest" genes
1595
+ assumed to follow the null; the Gamma null is fitted to the lower
1596
+ `null_quantile` of their ``T`` values, as a truncated sample (see
1597
+ `_gamma_null_pvalues`).
1598
+ inplace
1599
+ If `True`, also write the table to ``adata.varm[key_added]``
1600
+ (reindexed to ``adata.var_names``, so the probe gene's own row is
1601
+ `NaN`). The table is returned either way.
1602
+ key_added
1603
+ `adata.varm` key used when `inplace`; defaults to
1604
+ ``f"dsvg_ovr_{gene}"``.
1605
+ tau
1606
+ Smallest shape difference between two genes' curves that counts, as in
1607
+ `parallel_curves_test`.
1608
+
1609
+ Returns
1610
+ -------
1611
+ pandas.DataFrame
1612
+ Indexed by the "rest" genes, with ``delta`` (the signed area between
1613
+ the probe curve and that gene's, negative when the probe sits below
1614
+ it) and ``T``; with a `fit_profiles` result ``shape_diff``,
1615
+ ``parallel_stat``, ``parallel_pval`` and ``parallel_pval_adj``,
1616
+ otherwise the Gamma-null ``gamma_pval`` and ``gamma_pval_adj``.
1617
+ """
1618
+ profiles, source = _gene_profiles(adata, curves_key, profiles_key)
1619
+ if gene not in profiles.index:
1620
+ raise KeyError(f"{gene!r} not found in adata.var_names")
1621
+
1622
+ profiles = profiles.dropna(how="all")
1623
+ if gene not in profiles.index:
1624
+ raise KeyError(f"{gene!r} has no fitted curve - it was not part of the fitted gene set.")
1625
+ rest = profiles.drop(index=gene)
1626
+ if rest.empty:
1627
+ raise ValueError("need at least two genes with fitted curves for a one-vs-rest comparison.")
1628
+
1629
+ bin_centers = profiles.columns.to_numpy(dtype=float)
1630
+ # probe minus rest, so a downregulated probe gives a negative delta.
1631
+ delta = profiles.loc[gene].to_numpy()[None, :] - rest.to_numpy()
1632
+
1633
+ result = pd.DataFrame(
1634
+ {
1635
+ "delta": np.trapezoid(delta, x=bin_centers, axis=1),
1636
+ "T": (delta**2).mean(axis=1),
1637
+ },
1638
+ index=rest.index,
1639
+ )
1640
+ cov_key = f"{profiles_key}_beta_cov"
1641
+ if source == "profiles" and cov_key in adata.uns:
1642
+ stored = adata.uns[cov_key]
1643
+ position = {str(g): i for i, g in enumerate(stored["genes"])}
1644
+ covs = np.asarray(stored["cov"])
1645
+ betas = adata.varm[f"{profiles_key}_beta"]
1646
+ metric = shape_metric(covs.shape[1], int(stored["degree"]))
1647
+ rows = []
1648
+ for other in rest.index:
1649
+ d_beta = betas.loc[gene].to_numpy(dtype=float) - betas.loc[other].to_numpy(dtype=float)
1650
+ cov = covs[position[gene]] + covs[position[other]]
1651
+ rows.append(parallel_curves_test(d_beta, metric, tau=tau, cov=cov))
1652
+ shape_diff, stat, _, pval = (np.asarray(v) for v in zip(*rows, strict=True))
1653
+ result["shape_diff"], result["parallel_stat"], result["parallel_pval"] = shape_diff, stat, pval
1654
+ result["parallel_pval_adj"] = _benjamini_hochberg(pval)
1655
+ else:
1656
+ result["gamma_pval"] = _gamma_null_pvalues(result["T"].to_numpy(), null_quantile=null_quantile)
1657
+ result["gamma_pval_adj"] = _benjamini_hochberg(result["gamma_pval"].to_numpy())
1658
+ result.attrs["probe_gene"] = gene
1659
+ result.attrs["profile_source"] = source
1660
+
1661
+ if inplace:
1662
+ adata.varm[key_added or f"dsvg_ovr_{gene}"] = result.reindex(adata.var_names)
1663
+ return result
1664
+
1665
+
1666
+ # ============================================================
1667
+ # 4) Statistics helpers
1668
+ # ============================================================
1669
+
1670
+
1671
+ def pattern_wald_test(pred0, pred1, ridge=0.0):
1672
+ """tradeSeq-like functional Wald test that two curves are equal, from posterior draws of the curves.
1673
+
1674
+ Pass draws of the *mean* curves (as `fit_model` does): posterior-predictive draws add observation
1675
+ noise to the covariance, which makes the test depend on how the data were binned.
1676
+
1677
+ Parameters
1678
+ ----------
1679
+ pred0, pred1 : array, shape (n_draws, M)
1680
+ Posterior draws of the two condition curves.
1681
+ ridge : float
1682
+ Diagonal regularization of the covariance (relative to its largest eigenvalue).
1683
+
1684
+ Returns
1685
+ -------
1686
+ stat : float
1687
+ Wald statistic.
1688
+ pval : float
1689
+ Chi-square p-value.
1690
+ df : int
1691
+ Effective degrees of freedom (rank of the covariance).
1692
+ delta_mean : array
1693
+ Mean difference curve ``pred1 - pred0``.
1694
+ """
1695
+ delta = pred1 - pred0 # (draws, M)
1696
+ mu = delta.mean(axis=0)
1697
+
1698
+ W, pval, df = _wald_from_moments(mu, np.cov(delta, rowvar=False), ridge=ridge)
1699
+ return W, pval, df, mu
1700
+
1701
+
1702
+ def _wald_from_moments(mu, Sigma, ridge=0.0, df_denominator=None):
1703
+ """Wald statistic of ``mu`` against covariance `Sigma`, pseudoinverse over its non-null directions.
1704
+
1705
+ The cutoff is relative, so the degrees of freedom are the rank of the difference curve, not the
1706
+ number of grid points. With `df_denominator` (slides) the p-value is from ``F(df, df_denominator)``
1707
+ on ``stat / df``, as in `parallel_curves_test`. Returns ``(stat, pval, df)``.
1708
+ """
1709
+ eigvals, eigvecs = np.linalg.eigh(Sigma)
1710
+ if ridge:
1711
+ eigvals = eigvals + ridge * eigvals.max()
1712
+ keep = eigvals > eigvals.max() * 1e-10
1713
+ z = eigvecs[:, keep].T @ mu
1714
+ W = float((z**2 / eigvals[keep]).sum())
1715
+ df = int(keep.sum())
1716
+ pval = float(chi2.sf(W, df)) if df_denominator is None else float(f.sf(W / df, df, df_denominator))
1717
+ return W, pval, df
1718
+
1719
+
1720
+ def _shape_grid(n_bsplines, degree, n_grid):
1721
+ """Fine grid on ``[0, 1]``, its trapezoid weights (summing to 1) and the uncentred basis on it."""
1722
+ x = np.linspace(0.0, 1.0, n_grid)
1723
+ w = np.full(n_grid, 1.0 / (n_grid - 1))
1724
+ w[[0, -1]] /= 2
1725
+ return x, w, _bspline_basis_01(x, n_bsplines, degree)
1726
+
1727
+
1728
+ def shape_metric(n_bsplines=10, degree=3, n_grid=2001):
1729
+ """Quadratic form `Q` that measures how far two condition curves are from parallel.
1730
+
1731
+ For interaction coefficients `b` (``beta_1`` of the per-gene model),
1732
+ ``sqrt(b @ Q @ b)`` is the shape difference of the difference curve
1733
+ ``d(x) = f1(x) - f0(x)`` on ``[0, 1]``, in slope-equivalent units:
1734
+ ``sqrt(12) * RMS(d - mean(d))``. When the gap between the curves changes
1735
+ linearly, ``d(x) = a + s * x``, this is exactly ``|s|``, the slope
1736
+ difference; for any other shape it is the linear change with the same RMS
1737
+ departure from parallel. Level differences contribute nothing
1738
+ (``Q @ ones = 0``). Centring the basis does not change `Q`, so it depends
1739
+ only on the knots, i.e. on `n_bsplines` and `degree` of
1740
+ `diffsvg.pp.bin_obs`.
1741
+
1742
+ Parameters
1743
+ ----------
1744
+ n_bsplines, degree
1745
+ As in `diffsvg.pp.bin_obs`.
1746
+ n_grid
1747
+ Grid points for the numerical integral.
1748
+
1749
+ Returns
1750
+ -------
1751
+ numpy.ndarray
1752
+ `Q`, shape ``(n_bsplines, n_bsplines)``, positive semi-definite with
1753
+ rank ``n_bsplines - 1``.
1754
+ """
1755
+ _, w, B = _shape_grid(n_bsplines, degree, n_grid)
1756
+ centred = B - w @ B
1757
+ return 12.0 * (centred * w[:, None]).T @ centred
1758
+
1759
+
1760
+ def _slope_weights(n_bsplines, degree, n_grid=2001):
1761
+ """`g` with ``g @ b`` = the least-squares slope over ``[0, 1]`` of the difference curve for coefficients `b`."""
1762
+ x, w, B = _shape_grid(n_bsplines, degree, n_grid)
1763
+ return 12.0 * ((x - 0.5) * w) @ B
1764
+
1765
+
1766
+ def parallel_curves_test(beta_1, metric, tau=0.1, cov=None, df_denominator=None):
1767
+ """Test whether two condition curves are parallel, up to a shape difference `tau`.
1768
+
1769
+ The curves are parallel when they differ only in level, i.e. when their
1770
+ difference ``d(x) = f1(x) - f0(x)`` is constant. How far they are from
1771
+ that is ``shape_diff = sqrt(beta_1 @ metric @ beta_1)`` (see
1772
+ `shape_metric`; for a gap that changes linearly it is the slope
1773
+ difference), and the test is ``H0: shape_diff <= tau`` against
1774
+ ``H1: shape_diff > tau``.
1775
+
1776
+ The statistic is the Wald statistic of the ``K - 1`` shape directions of
1777
+ `beta_1` (the constant direction is the level and is left out). With
1778
+ ``tau = 0`` it is the classic test of parallel curves, chi-square with
1779
+ ``K - 1`` degrees of freedom. With ``tau > 0`` it is compared against the
1780
+ largest distribution it can have when the true shape difference is
1781
+ `tau`: noncentral chi-square with noncentrality ``tau**2`` over the
1782
+ variance of the best-measured shape direction. Shape differences below
1783
+ `tau` therefore do not become significant however precisely they are
1784
+ measured, while the p-value stays valid for every difference up to `tau`.
1785
+
1786
+ With `df_denominator` (replicates: the posterior covariance then holds
1787
+ the variation between slides, which was estimated from only a few) the
1788
+ statistic over its ``K - 1`` degrees of freedom is compared against an F
1789
+ distribution instead (noncentral for ``tau > 0``, with the same
1790
+ noncentrality), which is wider than the chi-square and so keeps the test
1791
+ honest when there are few slides.
1792
+
1793
+ Parameters
1794
+ ----------
1795
+ beta_1 : array, shape (n_draws, K) or (K,)
1796
+ Posterior draws of the interaction coefficients, or their posterior
1797
+ mean (then `cov` is required).
1798
+ metric : array, shape (K, K)
1799
+ `shape_metric` for the basis `beta_1` belongs to.
1800
+ tau : float
1801
+ Smallest shape difference that counts, in units of ``shape_diff``.
1802
+ cov : array, shape (K, K), optional
1803
+ Posterior covariance of `beta_1`, when `beta_1` is a mean.
1804
+ df_denominator : int, optional
1805
+ Denominator degrees of freedom of the F reference, e.g. the number
1806
+ of slides minus 2 when comparing two groups of slides.
1807
+
1808
+ Returns
1809
+ -------
1810
+ shape_diff : float
1811
+ Shape difference of the posterior-mean curves. Noise in the fitted
1812
+ curves adds to it, so poorly measured genes show a sizeable
1813
+ ``shape_diff`` even when the true curves are parallel; the p-value
1814
+ accounts for that, the raw number does not.
1815
+ stat : float
1816
+ Wald statistic.
1817
+ df : int
1818
+ Degrees of freedom, ``K - 1``.
1819
+ pval : float
1820
+ """
1821
+ beta_1 = np.asarray(beta_1, dtype=float)
1822
+ if beta_1.ndim == 2:
1823
+ mean, cov = beta_1.mean(axis=0), np.cov(beta_1, rowvar=False)
1824
+ elif cov is None:
1825
+ raise ValueError("pass `cov` along with a posterior mean, or pass the posterior draws.")
1826
+ else:
1827
+ mean, cov = beta_1, np.asarray(cov, dtype=float)
1828
+ if tau < 0:
1829
+ raise ValueError(f"`tau` must be >= 0, got {tau}.")
1830
+
1831
+ # Q = L L' over its K - 1 non-zero directions, so z = L' beta_1 has |z| = shape_diff.
1832
+ evals, evecs = np.linalg.eigh(np.asarray(metric, dtype=float))
1833
+ keep = evals > evals.max() * 1e-10
1834
+ L = evecs[:, keep] * np.sqrt(evals[keep])
1835
+ z = L.T @ mean
1836
+ omega = L.T @ cov @ L
1837
+ omega_min = np.linalg.eigvalsh(omega)[0]
1838
+ if omega_min <= 0:
1839
+ raise ValueError("the covariance of `beta_1` is singular in the shape directions.")
1840
+
1841
+ stat = float(z @ np.linalg.solve(omega, z))
1842
+ df = int(keep.sum())
1843
+ noncentrality = tau**2 / omega_min
1844
+ if df_denominator is None:
1845
+ pval = float(ncx2.sf(stat, df, noncentrality)) if tau > 0 else float(chi2.sf(stat, df))
1846
+ elif df_denominator < 1:
1847
+ raise ValueError(f"`df_denominator` must be >= 1, got {df_denominator}.")
1848
+ elif tau > 0:
1849
+ pval = float(ncf.sf(stat / df, df, df_denominator, noncentrality))
1850
+ else:
1851
+ pval = float(f.sf(stat / df, df, df_denominator))
1852
+ return float(np.sqrt(z @ z)), stat, df, pval
1853
+
1854
+
1855
+ def compute_T(curve0, curve1):
1856
+ """Functional L2 distance between two curves."""
1857
+ return np.mean((curve1 - curve0) ** 2)