diffsvg 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffsvg/__init__.py +3 -0
- diffsvg/pl/__init__.py +5 -0
- diffsvg/pl/plot_clusters.py +216 -0
- diffsvg/pl/plot_results.py +484 -0
- diffsvg/pl/plot_stability.py +184 -0
- diffsvg/pp/__init__.py +10 -0
- diffsvg/pp/utils.py +620 -0
- diffsvg/py.typed +0 -0
- diffsvg/tl/__init__.py +24 -0
- diffsvg/tl/_exact.py +332 -0
- diffsvg/tl/clustering.py +154 -0
- diffsvg/tl/model.py +1857 -0
- diffsvg/tl/stability.py +335 -0
- diffsvg-0.1.0.dist-info/METADATA +132 -0
- diffsvg-0.1.0.dist-info/RECORD +17 -0
- diffsvg-0.1.0.dist-info/WHEEL +4 -0
- diffsvg-0.1.0.dist-info/licenses/LICENSE +29 -0
diffsvg/tl/model.py
ADDED
|
@@ -0,0 +1,1857 @@
|
|
|
1
|
+
"""Per-gene GAM (P-spline) models for differential spatially/pseudotemporally variable genes.
|
|
2
|
+
|
|
3
|
+
`setup_model` reads the output of `diffsvg.pp.bin_obs` from ``adata.uns["binned"]``; `fit_model` fits one
|
|
4
|
+
model per gene in parallel, predicts both condition curves on a fixed grid, tests them against each other
|
|
5
|
+
and writes the result to ``adata.varm``.
|
|
6
|
+
|
|
7
|
+
Parallelization: workers receive only plain arrays (the shared design matrices plus the gene's `Y`) and
|
|
8
|
+
build their own `pm.Model` with `_build_gene_model`. A built model cannot be shipped instead (PyTensor
|
|
9
|
+
graph nodes do not pickle), and every worker is a separate process, so each compiles its own.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import contextlib
|
|
13
|
+
import logging
|
|
14
|
+
import os
|
|
15
|
+
import warnings
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
|
|
18
|
+
import numpy as np
|
|
19
|
+
import pandas as pd
|
|
20
|
+
from joblib import Parallel, delayed
|
|
21
|
+
from scipy import optimize
|
|
22
|
+
from scipy.stats import chi2, f, gamma, ncf, ncx2
|
|
23
|
+
from tqdm.auto import tqdm
|
|
24
|
+
|
|
25
|
+
from diffsvg.pp.utils import BinnedData, _bspline_basis_01, _expression, spline_basis_at
|
|
26
|
+
from diffsvg.tl import _exact
|
|
27
|
+
|
|
28
|
+
try: # the `pymc` extra: only needed for inference="nuts"
|
|
29
|
+
import arviz as az
|
|
30
|
+
import pymc as pm
|
|
31
|
+
except ImportError:
|
|
32
|
+
az = pm = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _require_pymc(inference):
|
|
36
|
+
if pm is None:
|
|
37
|
+
raise ImportError(
|
|
38
|
+
f"inference={inference!r} needs PyMC, which is an optional dependency: pip install 'diffsvg[pymc]'. "
|
|
39
|
+
"The default inference='auto' uses the exact posterior wherever the model allows it."
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
# ============================================================
|
|
44
|
+
# 1) Priors and likelihoods
|
|
45
|
+
# ============================================================
|
|
46
|
+
# Registries, so alternatives can be added without touching the model builders. A prior is a tuple
|
|
47
|
+
# ``(distribution, *params)``, parameters in the order listed below.
|
|
48
|
+
|
|
49
|
+
_PRIOR_DISTRIBUTIONS = {
|
|
50
|
+
"normal": ("Normal", ("mu", "sigma")),
|
|
51
|
+
"laplace": ("Laplace", ("mu", "b")),
|
|
52
|
+
"studentt": ("StudentT", ("nu", "mu", "sigma")),
|
|
53
|
+
"halfnormal": ("HalfNormal", ("sigma",)),
|
|
54
|
+
"halfcauchy": ("HalfCauchy", ("beta",)),
|
|
55
|
+
"exponential": ("Exponential", ("lam",)),
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
#: Default priors; ``setup_model(priors=...)`` overrides any of them.
|
|
59
|
+
DEFAULT_PRIORS = {
|
|
60
|
+
"intercept": ("normal", 0.0, 1.0),
|
|
61
|
+
"beta": ("normal", 0.0, 1.0),
|
|
62
|
+
"beta_1": ("normal", 0.0, 1.0),
|
|
63
|
+
"categorical_beta": ("normal", 0.0, 1.0),
|
|
64
|
+
"sigma": ("halfnormal", 1.0),
|
|
65
|
+
# Only used by fits with `sample` (see `bin_obs`): how far a slide's level / curve strays from its condition's.
|
|
66
|
+
"sample_level_sd": ("halfnormal", 1.0),
|
|
67
|
+
"sample_curve_sd": ("halfnormal", 1.0),
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _resolve_priors(priors):
|
|
72
|
+
"""Merge a partial prior spec (e.g. ``{"beta": ("normal", 0.0, 2.0)}``) over `DEFAULT_PRIORS` and validate it."""
|
|
73
|
+
merged = dict(DEFAULT_PRIORS)
|
|
74
|
+
for name, spec in (priors or {}).items():
|
|
75
|
+
if name not in DEFAULT_PRIORS:
|
|
76
|
+
raise ValueError(f"Unknown prior {name!r}. Available: {sorted(DEFAULT_PRIORS)}")
|
|
77
|
+
merged[name] = tuple(spec)
|
|
78
|
+
|
|
79
|
+
for name, spec in merged.items():
|
|
80
|
+
distribution, *params = spec
|
|
81
|
+
if distribution not in _PRIOR_DISTRIBUTIONS:
|
|
82
|
+
raise ValueError(
|
|
83
|
+
f"Unknown distribution {distribution!r} for prior {name!r}. Available: {sorted(_PRIOR_DISTRIBUTIONS)}"
|
|
84
|
+
)
|
|
85
|
+
expected = _PRIOR_DISTRIBUTIONS[distribution][1]
|
|
86
|
+
if len(params) != len(expected):
|
|
87
|
+
raise ValueError(
|
|
88
|
+
f"Prior {name!r}: {distribution!r} takes {len(expected)} parameter(s) {expected}, got {len(params)}."
|
|
89
|
+
)
|
|
90
|
+
return merged
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _draw_prior(name, spec, shape=None, initval=None):
|
|
94
|
+
"""Instantiate one prior from its ``(distribution, *params)`` tuple."""
|
|
95
|
+
distribution, *params = spec
|
|
96
|
+
dist_name, param_names = _PRIOR_DISTRIBUTIONS[distribution]
|
|
97
|
+
dist = getattr(pm, dist_name)
|
|
98
|
+
kwargs = dict(zip(param_names, params, strict=True))
|
|
99
|
+
if shape is not None:
|
|
100
|
+
kwargs["shape"] = shape
|
|
101
|
+
if initval is not None:
|
|
102
|
+
kwargs["initval"] = initval
|
|
103
|
+
return dist(name, **kwargs)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _normal_likelihood(mu, observed, total_size, priors, weights=None, ss_within=None):
|
|
107
|
+
sigma = _draw_prior("sigma", priors["sigma"], initval=0.5)
|
|
108
|
+
if weights is None:
|
|
109
|
+
return pm.Normal("D", mu=mu, sigma=sigma, observed=observed, total_size=total_size)
|
|
110
|
+
# A row that averages n cells has variance sigma^2 / n, sigma being the cell-level noise.
|
|
111
|
+
D = pm.Normal("D", mu=mu, sigma=sigma / pm.math.sqrt(weights), observed=observed, total_size=total_size)
|
|
112
|
+
if ss_within is not None:
|
|
113
|
+
# What the per-cell likelihood has on top of the row means: the spread of
|
|
114
|
+
# the cells around their row's mean, `ss` over `df` = n_cells - n_rows.
|
|
115
|
+
ss, df = ss_within
|
|
116
|
+
pm.Potential("within_rows", -0.5 * (2.0 * df * pm.math.log(sigma) + ss / sigma**2))
|
|
117
|
+
return D
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
_LIKELIHOODS = {
|
|
121
|
+
"normal": _normal_likelihood,
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# ============================================================
|
|
126
|
+
# 2) Model setup
|
|
127
|
+
# ============================================================
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass
|
|
131
|
+
class DiffSVGModelSetup:
|
|
132
|
+
"""Config bundle returned by `setup_model` and consumed by `fit_model`.
|
|
133
|
+
|
|
134
|
+
Holds no `pm.Model`: `fit_model` builds each gene's model inside its worker. To inspect one gene's
|
|
135
|
+
model, call `_build_gene_model` directly.
|
|
136
|
+
|
|
137
|
+
Attributes
|
|
138
|
+
----------
|
|
139
|
+
binned
|
|
140
|
+
The `diffsvg.pp.utils.BinnedData` this setup was built from.
|
|
141
|
+
likelihood
|
|
142
|
+
Observation model every gene's model is built with.
|
|
143
|
+
priors
|
|
144
|
+
Prior spec passed to `setup_model`, merged over `DEFAULT_PRIORS`.
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
binned: BinnedData
|
|
148
|
+
likelihood: str
|
|
149
|
+
priors: dict | None = None
|
|
150
|
+
|
|
151
|
+
def __post_init__(self):
|
|
152
|
+
self.priors = _resolve_priors(self.priors)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _build_gene_model(
|
|
156
|
+
Y,
|
|
157
|
+
B,
|
|
158
|
+
B_masked,
|
|
159
|
+
condition_arr,
|
|
160
|
+
n_bsplines,
|
|
161
|
+
intercept_initval,
|
|
162
|
+
likelihood="normal",
|
|
163
|
+
priors=None,
|
|
164
|
+
weights=None,
|
|
165
|
+
ss_within=None,
|
|
166
|
+
record_mu=True,
|
|
167
|
+
sample_index=None,
|
|
168
|
+
n_samples=None,
|
|
169
|
+
sample_scales=None,
|
|
170
|
+
):
|
|
171
|
+
"""Build the P-spline GAM for a single gene.
|
|
172
|
+
|
|
173
|
+
`B`/`B_masked`/`condition_arr` are shared across genes; only `Y` and `intercept_initval` change per
|
|
174
|
+
gene. Building is cheap (graph construction only), so every worker builds its own.
|
|
175
|
+
|
|
176
|
+
- `weights` (cells per row) and `ss_within` (``(within-row sum of squares, n_cells - n_rows)``) come
|
|
177
|
+
from contiguous bins: rows are weighted by their cell count and the noise is the cell-level one.
|
|
178
|
+
`weights` lives in a ``"w_data"`` container so predictions can set it per grid point.
|
|
179
|
+
- `record_mu` keeps the row-wise mean as the ``"mu"`` deterministic (one value per row and draw; off
|
|
180
|
+
in `fit_model`, nothing reads it).
|
|
181
|
+
- `sample_index` (the slide of every row) adds a non-centred random level and random curve per slide,
|
|
182
|
+
``sample_level_sd * z_level[slide] + B @ (sample_curve_sd * z_curve[slide])``, multiplied by a
|
|
183
|
+
``"sample_on"`` container that a prediction grid sets to 0 for the population-level curves.
|
|
184
|
+
`sample_scales` ``(level_sd, curve_sd)`` fixes the two scales instead (empirical Bayes).
|
|
185
|
+
"""
|
|
186
|
+
if likelihood not in _LIKELIHOODS:
|
|
187
|
+
raise ValueError(f"Unknown likelihood {likelihood!r}. Available: {sorted(_LIKELIHOODS)}")
|
|
188
|
+
likelihood_fn = _LIKELIHOODS[likelihood]
|
|
189
|
+
priors = _resolve_priors(priors)
|
|
190
|
+
|
|
191
|
+
with pm.Model() as gene_model:
|
|
192
|
+
Y_data = pm.Data("Y_data", np.asarray(Y, dtype=float))
|
|
193
|
+
B_data = pm.Data("B_data", B)
|
|
194
|
+
condition_data = pm.Data("condition_data", np.asarray(condition_arr, dtype=float))
|
|
195
|
+
B_masked_data = pm.Data("B_masked_data", B_masked)
|
|
196
|
+
|
|
197
|
+
intercept = _draw_prior("intercept", priors["intercept"], initval=intercept_initval)
|
|
198
|
+
beta = _draw_prior("beta", priors["beta"], shape=n_bsplines, initval=np.zeros(n_bsplines))
|
|
199
|
+
beta_1 = _draw_prior("beta_1", priors["beta_1"], shape=n_bsplines, initval=np.zeros(n_bsplines))
|
|
200
|
+
categorical_beta = _draw_prior("categorical_beta", priors["categorical_beta"], initval=0.0)
|
|
201
|
+
|
|
202
|
+
mu = (
|
|
203
|
+
intercept
|
|
204
|
+
+ pm.math.dot(B_data, beta)
|
|
205
|
+
+ categorical_beta * condition_data
|
|
206
|
+
+ pm.math.dot(B_masked_data, beta_1)
|
|
207
|
+
)
|
|
208
|
+
if sample_index is not None:
|
|
209
|
+
if sample_scales is None:
|
|
210
|
+
level_sd = _draw_prior("sample_level_sd", priors["sample_level_sd"], initval=0.1)
|
|
211
|
+
curve_sd = _draw_prior("sample_curve_sd", priors["sample_curve_sd"], initval=0.1)
|
|
212
|
+
else:
|
|
213
|
+
level_sd, curve_sd = (float(v) for v in sample_scales)
|
|
214
|
+
level_z = pm.Normal("sample_level_z", 0.0, 1.0, shape=n_samples)
|
|
215
|
+
curve_z = pm.Normal("sample_curve_z", 0.0, 1.0, shape=(n_samples, n_bsplines))
|
|
216
|
+
slide = pm.Data("sample_data", np.asarray(sample_index, dtype="int64"))
|
|
217
|
+
on = pm.Data("sample_on", np.ones(len(sample_index)))
|
|
218
|
+
mu = mu + on * (level_sd * level_z[slide] + pm.math.sum(B_data * (curve_sd * curve_z)[slide], axis=1))
|
|
219
|
+
if record_mu:
|
|
220
|
+
mu = pm.Deterministic("mu", mu)
|
|
221
|
+
|
|
222
|
+
w_data = None if weights is None else pm.Data("w_data", np.asarray(weights, dtype=float))
|
|
223
|
+
likelihood_fn(mu, Y_data, len(Y), priors, weights=w_data, ss_within=ss_within)
|
|
224
|
+
|
|
225
|
+
return gene_model
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _build_profile_model(
|
|
229
|
+
Y, B, n_bsplines, intercept_initval, likelihood="normal", priors=None, weights=None, ss_within=None
|
|
230
|
+
):
|
|
231
|
+
"""`_build_gene_model` without the `condition` covariate and its spline interaction (for `fit_profiles`)."""
|
|
232
|
+
if likelihood not in _LIKELIHOODS:
|
|
233
|
+
raise ValueError(f"Unknown likelihood {likelihood!r}. Available: {sorted(_LIKELIHOODS)}")
|
|
234
|
+
likelihood_fn = _LIKELIHOODS[likelihood]
|
|
235
|
+
priors = _resolve_priors(priors)
|
|
236
|
+
|
|
237
|
+
with pm.Model() as gene_model:
|
|
238
|
+
Y_data = pm.Data("Y_data", np.asarray(Y, dtype=float))
|
|
239
|
+
B_data = pm.Data("B_data", B)
|
|
240
|
+
|
|
241
|
+
intercept = _draw_prior("intercept", priors["intercept"], initval=intercept_initval)
|
|
242
|
+
beta = _draw_prior("beta", priors["beta"], shape=n_bsplines, initval=np.zeros(n_bsplines))
|
|
243
|
+
|
|
244
|
+
mu = pm.Deterministic("mu", intercept + pm.math.dot(B_data, beta))
|
|
245
|
+
|
|
246
|
+
w_data = None if weights is None else pm.Data("w_data", np.asarray(weights, dtype=float))
|
|
247
|
+
likelihood_fn(mu, Y_data, len(Y), priors, weights=w_data, ss_within=ss_within)
|
|
248
|
+
|
|
249
|
+
return gene_model
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def setup_model(adata, likelihood="normal", priors=None):
|
|
253
|
+
"""Configure the per-gene GAM, given `adata` pre-processed by `diffsvg.pp.bin_obs`.
|
|
254
|
+
|
|
255
|
+
This validates `adata`/`likelihood` and bundles them into a
|
|
256
|
+
`DiffSVGModelSetup` - it does **not** build any `pm.Model`. Actual model
|
|
257
|
+
construction happens lazily, per gene, once `fit_model` starts fitting
|
|
258
|
+
(see `DiffSVGModelSetup`), so this is cheap regardless of gene count.
|
|
259
|
+
|
|
260
|
+
Parameters
|
|
261
|
+
----------
|
|
262
|
+
adata
|
|
263
|
+
Annotated data matrix that `diffsvg.pp.bin_obs` was already run on
|
|
264
|
+
(i.e. ``"binned" in adata.uns``).
|
|
265
|
+
likelihood
|
|
266
|
+
Observation model. Only ``"normal"`` (the default) is implemented.
|
|
267
|
+
priors
|
|
268
|
+
Overrides for any of `DEFAULT_PRIORS`, as ``{name: (distribution,
|
|
269
|
+
*params)}`` - e.g. ``{"beta": ("normal", 0.0, 2.0), "sigma":
|
|
270
|
+
("exponential", 1.0)}``. Anything not named keeps its default, so
|
|
271
|
+
the models are unchanged unless asked. Validated here rather than
|
|
272
|
+
inside a worker, so a typo fails immediately.
|
|
273
|
+
|
|
274
|
+
Returns
|
|
275
|
+
-------
|
|
276
|
+
DiffSVGModelSetup
|
|
277
|
+
"""
|
|
278
|
+
if likelihood not in _LIKELIHOODS:
|
|
279
|
+
raise ValueError(f"Unknown likelihood {likelihood!r}. Available: {sorted(_LIKELIHOODS)}")
|
|
280
|
+
if "binned" not in adata.uns:
|
|
281
|
+
raise KeyError("adata.uns['binned'] not found - run `diffsvg.pp.bin_obs(adata, ...)` first.")
|
|
282
|
+
|
|
283
|
+
binned = BinnedData(**adata.uns["binned"])
|
|
284
|
+
return DiffSVGModelSetup(binned=binned, likelihood=likelihood, priors=priors)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
# ============================================================
|
|
288
|
+
# 3) Fitting (parallel across genes) + posterior predictive
|
|
289
|
+
# ============================================================
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
@dataclass
|
|
293
|
+
class _FitShared:
|
|
294
|
+
"""The plain-array payload shared by every gene (no PyMC objects, see the module docstring)."""
|
|
295
|
+
|
|
296
|
+
B: np.ndarray
|
|
297
|
+
B_masked: np.ndarray
|
|
298
|
+
condition_fit: np.ndarray
|
|
299
|
+
n_bsplines: int
|
|
300
|
+
likelihood: str
|
|
301
|
+
priors: dict
|
|
302
|
+
bin_centers: np.ndarray
|
|
303
|
+
B_pred: np.ndarray
|
|
304
|
+
B_pred_masked_0: np.ndarray
|
|
305
|
+
B_pred_masked_1: np.ndarray
|
|
306
|
+
condition_pred_0: np.ndarray
|
|
307
|
+
condition_pred_1: np.ndarray
|
|
308
|
+
# Which condition test to run, see `_TESTS`. `shape_metric`/`slope_weights`
|
|
309
|
+
# are only filled for the parallel-curves test.
|
|
310
|
+
test: str = "wald"
|
|
311
|
+
tau: float = 0.1
|
|
312
|
+
shape_metric: np.ndarray | None = None
|
|
313
|
+
slope_weights: np.ndarray | None = None
|
|
314
|
+
# Cells per row, and per grid point for predictions, when fitting weighted
|
|
315
|
+
# contiguous bins (see `_weighting`); `None` otherwise.
|
|
316
|
+
weights: np.ndarray | None = None
|
|
317
|
+
# How the posterior is obtained, see `_INFERENCE`.
|
|
318
|
+
inference: str = "nuts"
|
|
319
|
+
# The slide of every fitted row and the denominator degrees of freedom of the test, for fits with `sample`.
|
|
320
|
+
sample_index: np.ndarray | None = None
|
|
321
|
+
n_samples: int | None = None
|
|
322
|
+
df_denominator: int | None = None
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
@dataclass
|
|
326
|
+
class _ProfileShared:
|
|
327
|
+
"""`_FitShared`'s counterpart for the condition-free model - same contract, fewer arrays."""
|
|
328
|
+
|
|
329
|
+
B: np.ndarray
|
|
330
|
+
n_bsplines: int
|
|
331
|
+
likelihood: str
|
|
332
|
+
priors: dict
|
|
333
|
+
B_pred: np.ndarray
|
|
334
|
+
weights: np.ndarray | None = None
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _fitted_cells(design, binned):
|
|
338
|
+
"""Which cells a fit on single cells uses: those with an anchor value, a condition and (if any) a sample."""
|
|
339
|
+
keep = np.isfinite(design[binned.anchor].to_numpy(dtype=float))
|
|
340
|
+
if binned.has_condition:
|
|
341
|
+
keep &= design[binned.condition].notna().to_numpy()
|
|
342
|
+
if binned.sample is not None:
|
|
343
|
+
keep &= design[binned.sample].notna().to_numpy()
|
|
344
|
+
return keep
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _sample_inputs(adata, binned, use_binned):
|
|
348
|
+
"""The slide of every fitted row and the number of slides, or ``(None, None)`` for a fit without `sample`."""
|
|
349
|
+
if binned.sample is None:
|
|
350
|
+
return None, None
|
|
351
|
+
if use_binned:
|
|
352
|
+
return np.asarray(binned.sample_index, dtype=int), binned.n_samples
|
|
353
|
+
design = adata.obsm[binned.design_matrix]
|
|
354
|
+
position = {label: i for i, label in enumerate(binned.sample_labels)}
|
|
355
|
+
index = design[binned.sample].astype(str).map(position)[_fitted_cells(design, binned)]
|
|
356
|
+
if index.isna().any():
|
|
357
|
+
raise ValueError(
|
|
358
|
+
f"`{binned.sample}` holds samples that were not seen when binning (expected {binned.sample_labels}) - "
|
|
359
|
+
"re-run `diffsvg.pp.bin_obs`."
|
|
360
|
+
)
|
|
361
|
+
return index.to_numpy(dtype=int), binned.n_samples
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _fit_inputs(adata, binned, use_binned):
|
|
365
|
+
"""The targets and design matrices to fit on: the binned aggregates, or one row per cell.
|
|
366
|
+
|
|
367
|
+
For cells, the basis is the binned one evaluated per cell (`diffsvg.pp.spline_basis_at`), so the
|
|
368
|
+
coefficients stay valid on the prediction grid, and `Y` is read from `adata`. Cells without an anchor
|
|
369
|
+
value or a condition are left out, as in the binning.
|
|
370
|
+
|
|
371
|
+
Returns ``(B, B_masked, condition_fit, Y, intercept_initvals)``.
|
|
372
|
+
"""
|
|
373
|
+
if use_binned:
|
|
374
|
+
return binned.B, binned.B_masked, binned.condition_fit, binned.Y, binned.intercept_initvals
|
|
375
|
+
|
|
376
|
+
design = adata.obsm[binned.design_matrix]
|
|
377
|
+
anchor_vals = design[binned.anchor].to_numpy(dtype=float)
|
|
378
|
+
keep = _fitted_cells(design, binned)
|
|
379
|
+
# The binned basis evaluated per cell. A basis rebuilt from the cells would
|
|
380
|
+
# be centred on the cells rather than the bins, and predicting with
|
|
381
|
+
# `binned.B_pred` would then shift every curve by a constant.
|
|
382
|
+
B = spline_basis_at(binned, anchor_vals[keep])
|
|
383
|
+
|
|
384
|
+
condition_fit, B_masked = None, None
|
|
385
|
+
if binned.has_condition:
|
|
386
|
+
code_map = {label: float(code) for code, label in binned.condition_categories.items()}
|
|
387
|
+
condition_fit = design[binned.condition].map(code_map).to_numpy(dtype=float)[keep]
|
|
388
|
+
if np.isnan(condition_fit).any():
|
|
389
|
+
raise ValueError(
|
|
390
|
+
f"`condition` column {binned.condition!r} holds values that were not seen when binning "
|
|
391
|
+
f"(expected {sorted(code_map)}) - re-run `diffsvg.pp.bin_obs`."
|
|
392
|
+
)
|
|
393
|
+
B_masked = B * condition_fit[:, None]
|
|
394
|
+
|
|
395
|
+
X = _expression(adata, binned.genes, binned.layer)[keep]
|
|
396
|
+
Y = {gene: X[:, i] for i, gene in enumerate(binned.genes)}
|
|
397
|
+
# As for bins (mean expression at anchor 0), but no cell sits exactly at 0, so the first bin stands in.
|
|
398
|
+
first_bin = np.asarray(binned.bin_assignment)[keep] == 0
|
|
399
|
+
intercept_initvals = {
|
|
400
|
+
gene: float(y[first_bin].mean()) if first_bin.any() else float(y.mean()) for gene, y in Y.items()
|
|
401
|
+
}
|
|
402
|
+
return B, B_masked, condition_fit, Y, intercept_initvals
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _weighting(binned, use_binned):
|
|
406
|
+
"""What a fit on weighted contiguous bins needs on top of `_fit_inputs`, or `None`s otherwise.
|
|
407
|
+
|
|
408
|
+
Returns ``(weights, ss_within, n_obs_grid)``: the cells behind each row,
|
|
409
|
+
per gene the within-row sum of squares with its degrees of freedom
|
|
410
|
+
(``n_cells - n_rows``), and the cells per grid point. Predictions use the
|
|
411
|
+
latter, so a predicted value is a bin mean just like a fitted row. Fits on
|
|
412
|
+
the cells are unweighted. Overlapping bins are weighted by their cell
|
|
413
|
+
counts too (a window with 2 cells must not count as much as one with
|
|
414
|
+
500, which otherwise bends the curve at the sparse ends of the range),
|
|
415
|
+
but have no within-row term: their rows share cells.
|
|
416
|
+
"""
|
|
417
|
+
if not use_binned:
|
|
418
|
+
return None, None, None
|
|
419
|
+
if not binned.weighted:
|
|
420
|
+
if binned.n_obs is None: # binned before the counts were recorded for overlapping bins
|
|
421
|
+
return None, None, None
|
|
422
|
+
return np.asarray(binned.n_obs, dtype=float), None, None
|
|
423
|
+
weights = np.asarray(binned.n_obs, dtype=float)
|
|
424
|
+
df = float(weights.sum() - len(weights))
|
|
425
|
+
ss_within = {gene: (float(binned.ss_within[gene]), df) for gene in binned.genes}
|
|
426
|
+
return weights, ss_within, np.asarray(binned.n_obs_grid, dtype=float)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
@contextlib.contextmanager
|
|
430
|
+
def _quiet_fds():
|
|
431
|
+
"""Redirect the OS-level stdout/stderr file descriptors to `os.devnull`.
|
|
432
|
+
|
|
433
|
+
Unlike `contextlib.redirect_stdout`, this also catches compiled extensions (nutpie's Rust sampler)
|
|
434
|
+
writing straight to the fds. Exceptions still propagate; the fds are restored before anything prints them.
|
|
435
|
+
"""
|
|
436
|
+
saved = [os.dup(fd) for fd in (1, 2)]
|
|
437
|
+
devnull = os.open(os.devnull, os.O_WRONLY)
|
|
438
|
+
try:
|
|
439
|
+
os.dup2(devnull, 1)
|
|
440
|
+
os.dup2(devnull, 2)
|
|
441
|
+
yield
|
|
442
|
+
finally:
|
|
443
|
+
for fd, saved_fd in zip((1, 2), saved, strict=True):
|
|
444
|
+
os.dup2(saved_fd, fd)
|
|
445
|
+
os.close(saved_fd)
|
|
446
|
+
os.close(devnull)
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
@contextlib.contextmanager
|
|
450
|
+
def _quiet_pymc():
|
|
451
|
+
"""Silence PyMC/PyTensor/nutpie's logging, warnings and raw fd output in the process that samples.
|
|
452
|
+
|
|
453
|
+
Runs inside each worker; the outer `tqdm` bar lives in the parent process and is unaffected.
|
|
454
|
+
"""
|
|
455
|
+
loggers = [logging.getLogger(name) for name in ("pymc", "pytensor", "nutpie")]
|
|
456
|
+
previous_levels = [logger.level for logger in loggers]
|
|
457
|
+
for logger in loggers:
|
|
458
|
+
logger.setLevel(logging.ERROR)
|
|
459
|
+
try:
|
|
460
|
+
with warnings.catch_warnings(), _quiet_fds():
|
|
461
|
+
warnings.simplefilter("ignore")
|
|
462
|
+
yield
|
|
463
|
+
finally:
|
|
464
|
+
for logger, level in zip(loggers, previous_levels, strict=True):
|
|
465
|
+
logger.setLevel(level)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
# ------------------------------------------------------------
|
|
469
|
+
# Posterior inference by sampling (the exact path lives in `diffsvg.tl._exact`)
|
|
470
|
+
# ------------------------------------------------------------
|
|
471
|
+
# `_INFERENCE` maps a sampler name to a function that runs inside ``with model`` and returns the posterior as
|
|
472
|
+
# `InferenceData` plus a dict of diagnostic columns. NUTS is only needed for non-normal coefficient priors.
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def _sample_nuts(model, sample_kwargs, diag_vars):
|
|
476
|
+
"""Sample the posterior with NUTS. `diag_vars` are the parameters the convergence diagnostics are reported for."""
|
|
477
|
+
idata = pm.sample(**sample_kwargs)
|
|
478
|
+
n_divergent = int(idata.sample_stats["diverging"].sum()) if "diverging" in idata.sample_stats else 0
|
|
479
|
+
rhat_ds = az.rhat(idata, var_names=diag_vars)
|
|
480
|
+
ess_ds = az.ess(idata, var_names=diag_vars, method="bulk")
|
|
481
|
+
columns = {
|
|
482
|
+
"n_divergent": n_divergent,
|
|
483
|
+
"rhat_max": float(max(np.asarray(rhat_ds[v]).max() for v in rhat_ds.data_vars)),
|
|
484
|
+
"ess_bulk_min": float(min(np.asarray(ess_ds[v]).min() for v in ess_ds.data_vars)),
|
|
485
|
+
}
|
|
486
|
+
return idata, columns
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
_INFERENCE = {
|
|
490
|
+
"nuts": _sample_nuts,
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def _posterior_draws(idata, name):
|
|
495
|
+
"""Posterior draws of one variable, flattened over chains: ``(n_draws,)`` or ``(n_draws, dim)``.
|
|
496
|
+
|
|
497
|
+
`idata` is an `arviz.InferenceData` or, for exact inference, a dict ``{name: draws}``.
|
|
498
|
+
"""
|
|
499
|
+
if isinstance(idata, dict):
|
|
500
|
+
return idata[name]
|
|
501
|
+
values = idata.posterior[name].stack(sample=("chain", "draw")).to_numpy()
|
|
502
|
+
return values.T if values.ndim == 2 else values
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _mean_curve_draws(idata, shared):
|
|
506
|
+
"""Posterior draws of both condition curves on the grid, ``(n_draws, n_grid)`` each.
|
|
507
|
+
|
|
508
|
+
The mean function ``intercept + B beta + condition * categorical_beta + B_masked beta_1`` at every
|
|
509
|
+
draw, without observation noise and without slide effects.
|
|
510
|
+
"""
|
|
511
|
+
intercept = _posterior_draws(idata, "intercept")
|
|
512
|
+
beta = _posterior_draws(idata, "beta")
|
|
513
|
+
beta_1 = _posterior_draws(idata, "beta_1")
|
|
514
|
+
cb = _posterior_draws(idata, "categorical_beta")
|
|
515
|
+
base = intercept[:, None] + beta @ shared.B_pred.T
|
|
516
|
+
draws0 = base + cb[:, None] * shared.condition_pred_0[None, :] + beta_1 @ shared.B_pred_masked_0.T
|
|
517
|
+
draws1 = base + cb[:, None] * shared.condition_pred_1[None, :] + beta_1 @ shared.B_pred_masked_1.T
|
|
518
|
+
return draws0, draws1
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def _fit_and_predict_gene(gene, Y, intercept_initval, shared, sample_kwargs, ss_within=None):
|
|
522
|
+
"""Build and fit one gene's model in this worker, then predict both condition curves on the grid.
|
|
523
|
+
|
|
524
|
+
Returns ``(gene, stats_row, curve_row)``: `stats_row` holds the test's columns and the shared ones;
|
|
525
|
+
`curve_row` holds the mean difference curve ``"delta"``, per condition the mean curve and its 94% band
|
|
526
|
+
(``"cond{0,1}_{mean,ci_low,ci_high}"``) and the posterior means of ``"beta"`` and ``"beta_1"``.
|
|
527
|
+
`ss_within` is the gene's within-row term (see `_weighting`).
|
|
528
|
+
"""
|
|
529
|
+
|
|
530
|
+
def build(sample_scales=None):
|
|
531
|
+
return _build_gene_model(
|
|
532
|
+
Y,
|
|
533
|
+
shared.B,
|
|
534
|
+
shared.B_masked,
|
|
535
|
+
shared.condition_fit,
|
|
536
|
+
shared.n_bsplines,
|
|
537
|
+
intercept_initval,
|
|
538
|
+
shared.likelihood,
|
|
539
|
+
shared.priors,
|
|
540
|
+
weights=shared.weights,
|
|
541
|
+
ss_within=ss_within,
|
|
542
|
+
record_mu=False,
|
|
543
|
+
sample_index=shared.sample_index,
|
|
544
|
+
n_samples=shared.n_samples,
|
|
545
|
+
sample_scales=sample_scales,
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
model = build()
|
|
549
|
+
slides = shared.sample_index is not None
|
|
550
|
+
|
|
551
|
+
with _quiet_pymc():
|
|
552
|
+
with model:
|
|
553
|
+
diag_vars = ["intercept", "beta", "beta_1", "categorical_beta", "sigma"]
|
|
554
|
+
idata, fit_columns = _INFERENCE[shared.inference](model, sample_kwargs, diag_vars)
|
|
555
|
+
if slides:
|
|
556
|
+
fit_columns = {
|
|
557
|
+
**fit_columns,
|
|
558
|
+
"sample_level_sd": float(idata.posterior["sample_level_sd"].mean()),
|
|
559
|
+
"sample_curve_sd": float(idata.posterior["sample_curve_sd"].mean()),
|
|
560
|
+
}
|
|
561
|
+
return _summarise_gene(gene, idata, shared, fit_columns)
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _condition_maps(shared):
|
|
565
|
+
"""Linear maps from ``theta = (intercept, beta, categorical_beta, beta_1)`` to the two grid curves."""
|
|
566
|
+
n_grid, K = shared.B_pred.shape
|
|
567
|
+
ones = np.ones((n_grid, 1))
|
|
568
|
+
A0 = np.hstack([ones, shared.B_pred, shared.condition_pred_0[:, None], shared.B_pred_masked_0])
|
|
569
|
+
A1 = np.hstack([ones, shared.B_pred, shared.condition_pred_1[:, None], shared.B_pred_masked_1])
|
|
570
|
+
return A0, A1
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def _summarise_gene(gene, idata, shared, fit_columns, moments=None):
|
|
574
|
+
"""Curves, test and summary columns of one gene from its posterior; see `_fit_and_predict_gene`.
|
|
575
|
+
|
|
576
|
+
`idata` holds the posterior draws (InferenceData or a dict). `moments` ``(mean, cov)`` of ``theta``
|
|
577
|
+
(exact inference) replace the draw estimates wherever only moments are needed: the means, the tests.
|
|
578
|
+
"""
|
|
579
|
+
# The condition curves are the posterior of the mean function on the grid (slide effects off, i.e.
|
|
580
|
+
# population level). No observation noise: the bands are credible bands of the curve, and the Wald
|
|
581
|
+
# test sees the covariance of the coefficients rather than that of new observations.
|
|
582
|
+
draws0, draws1 = _mean_curve_draws(idata, shared)
|
|
583
|
+
test_row, delta_mean = _TESTS[shared.test][0](draws0, draws1, idata, shared, moments)
|
|
584
|
+
if moments is None:
|
|
585
|
+
cond0_mean, cond1_mean = draws0.mean(axis=0), draws1.mean(axis=0)
|
|
586
|
+
else:
|
|
587
|
+
A0, A1 = _condition_maps(shared)
|
|
588
|
+
cond0_mean, cond1_mean = A0 @ moments[0], A1 @ moments[0]
|
|
589
|
+
cond0_ci_low, cond0_ci_high = np.quantile(draws0, [0.03, 0.97], axis=0)
|
|
590
|
+
cond1_ci_low, cond1_ci_high = np.quantile(draws1, [0.03, 0.97], axis=0)
|
|
591
|
+
integral = float(np.trapezoid(delta_mean, x=shared.bin_centers))
|
|
592
|
+
|
|
593
|
+
post_cb = _posterior_draws(idata, "categorical_beta")
|
|
594
|
+
ci_low, ci_high = (float(v) for v in np.quantile(post_cb, [0.03, 0.97]))
|
|
595
|
+
K = shared.n_bsplines
|
|
596
|
+
if moments is None:
|
|
597
|
+
cb_mean = float(post_cb.mean())
|
|
598
|
+
beta_mean = _posterior_draws(idata, "beta").mean(axis=0)
|
|
599
|
+
beta_1_mean = _posterior_draws(idata, "beta_1").mean(axis=0)
|
|
600
|
+
else:
|
|
601
|
+
cb_mean = float(moments[0][K + 1])
|
|
602
|
+
beta_mean, beta_1_mean = moments[0][1 : K + 1], moments[0][K + 2 :]
|
|
603
|
+
|
|
604
|
+
stats_row = {
|
|
605
|
+
"gene": gene,
|
|
606
|
+
**test_row,
|
|
607
|
+
"integral": integral,
|
|
608
|
+
"categorical_beta_mean": cb_mean,
|
|
609
|
+
"categorical_beta_ci_low": ci_low,
|
|
610
|
+
"categorical_beta_ci_high": ci_high,
|
|
611
|
+
**fit_columns,
|
|
612
|
+
}
|
|
613
|
+
curve_row = {
|
|
614
|
+
"delta": delta_mean,
|
|
615
|
+
"cond0_mean": cond0_mean,
|
|
616
|
+
"cond0_ci_low": cond0_ci_low,
|
|
617
|
+
"cond0_ci_high": cond0_ci_high,
|
|
618
|
+
"cond1_mean": cond1_mean,
|
|
619
|
+
"cond1_ci_low": cond1_ci_low,
|
|
620
|
+
"cond1_ci_high": cond1_ci_high,
|
|
621
|
+
"beta": np.asarray(beta_mean),
|
|
622
|
+
"beta_1": np.asarray(beta_1_mean),
|
|
623
|
+
}
|
|
624
|
+
return gene, stats_row, curve_row
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def _exact_settings(sample_kwargs):
|
|
628
|
+
settings = {"draws": 1000, "random_seed": 42, "n_grid": 81, "n_hyper": 72}
|
|
629
|
+
given = dict(sample_kwargs or {})
|
|
630
|
+
ignored = sorted(set(given) - set(settings))
|
|
631
|
+
if ignored:
|
|
632
|
+
warnings.warn(
|
|
633
|
+
f"inference='exact' ignores the sampler settings {ignored} (it uses only draws, random_seed, n_grid, n_hyper).",
|
|
634
|
+
UserWarning,
|
|
635
|
+
stacklevel=3,
|
|
636
|
+
)
|
|
637
|
+
settings.update({k: v for k, v in given.items() if k in settings})
|
|
638
|
+
return settings
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def _exact_fit(design, Y, genes, weights, ss_within, priors, prior_names, n_bsplines, settings):
|
|
642
|
+
"""`_exact.posterior` for all `genes` on the design `design` (columns ordered as `prior_names`)."""
|
|
643
|
+
Ymat = np.column_stack([np.asarray(Y[g], dtype=float) for g in genes])
|
|
644
|
+
XtWX, XtWY, yWy = _exact.sufficient_statistics(design, Ymat, weights)
|
|
645
|
+
sizes = {"intercept": 1, "categorical_beta": 1, "beta": n_bsplines, "beta_1": n_bsplines}
|
|
646
|
+
m0 = np.concatenate([[priors[n][1]] * sizes[n] for n in prior_names])
|
|
647
|
+
s0 = np.concatenate([[priors[n][2]] * sizes[n] for n in prior_names])
|
|
648
|
+
extra_df = extra_ss = None
|
|
649
|
+
if ss_within is not None:
|
|
650
|
+
extra_df = ss_within[genes[0]][1]
|
|
651
|
+
extra_ss = np.array([ss_within[g][0] for g in genes])
|
|
652
|
+
return _exact.posterior(
|
|
653
|
+
XtWX, XtWY, yWy, design.shape[0], m0, s0, priors["sigma"], extra_df, extra_ss, n_grid=settings["n_grid"]
|
|
654
|
+
)
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def _exact_slide_results(shared, Y, genes, ss_within, settings):
|
|
658
|
+
"""Exact inference for the two-condition model with slide effects."""
|
|
659
|
+
priors = _resolve_priors(shared.priors)
|
|
660
|
+
K = shared.n_bsplines
|
|
661
|
+
fixed = np.column_stack([np.ones(shared.B.shape[0]), shared.B, shared.condition_fit, shared.B_masked])
|
|
662
|
+
X = _exact.slide_design(fixed, shared.B, np.asarray(shared.sample_index, dtype=int), shared.n_samples)
|
|
663
|
+
sizes = {"intercept": 1, "categorical_beta": 1, "beta": K, "beta_1": K}
|
|
664
|
+
names = ("intercept", "beta", "categorical_beta", "beta_1")
|
|
665
|
+
m0 = np.concatenate([[priors[n][1]] * sizes[n] for n in names])
|
|
666
|
+
s0 = np.concatenate([[priors[n][2]] * sizes[n] for n in names])
|
|
667
|
+
Ymat = np.column_stack([np.asarray(Y[g], dtype=float) for g in genes])
|
|
668
|
+
extra_df = extra_ss = None
|
|
669
|
+
if ss_within is not None:
|
|
670
|
+
extra_df = ss_within[genes[0]][1]
|
|
671
|
+
extra_ss = np.array([ss_within[g][0] for g in genes])
|
|
672
|
+
res = _exact.slide_posterior(
|
|
673
|
+
X, Ymat, shared.weights, fixed.shape[1], shared.n_samples, K, m0, s0, priors["sigma"],
|
|
674
|
+
priors["sample_level_sd"], priors["sample_curve_sd"], extra_df, extra_ss, n_draws=settings["draws"],
|
|
675
|
+
seeds=np.random.SeedSequence(settings["random_seed"]).spawn(len(genes)), n_hyper=settings.get("n_hyper", 72),
|
|
676
|
+
) # fmt: skip
|
|
677
|
+
results = []
|
|
678
|
+
for i, gene in enumerate(genes):
|
|
679
|
+
theta = res["draws"][i]
|
|
680
|
+
draws = {
|
|
681
|
+
"intercept": theta[:, 0],
|
|
682
|
+
"beta": theta[:, 1 : K + 1],
|
|
683
|
+
"categorical_beta": theta[:, K + 1],
|
|
684
|
+
"beta_1": theta[:, K + 2 :],
|
|
685
|
+
}
|
|
686
|
+
fit_columns = {
|
|
687
|
+
"sigma_mean": float(np.sqrt(res["sigma2_mean"][i])),
|
|
688
|
+
"sample_level_sd": float(res["level_sd_mean"][i]),
|
|
689
|
+
"sample_curve_sd": float(res["curve_sd_mean"][i]),
|
|
690
|
+
"exact_edge_mass": float(res["edge_mass"][i]),
|
|
691
|
+
}
|
|
692
|
+
results.append(_summarise_gene(gene, draws, shared, fit_columns, moments=(res["mean"][i], res["cov"][i])))
|
|
693
|
+
return results
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def _exact_gene_results(shared, Y, genes, ss_within, settings):
|
|
697
|
+
"""Exact inference for the two-condition model: ``[(gene, stats_row, curve_row), ...]``."""
|
|
698
|
+
if shared.sample_index is not None:
|
|
699
|
+
return _exact_slide_results(shared, Y, genes, ss_within, settings)
|
|
700
|
+
design = np.column_stack([np.ones(shared.B.shape[0]), shared.B, shared.condition_fit, shared.B_masked])
|
|
701
|
+
names = ("intercept", "beta", "categorical_beta", "beta_1")
|
|
702
|
+
post = _exact_fit(
|
|
703
|
+
design, Y, genes, shared.weights, ss_within, _resolve_priors(shared.priors), names, shared.n_bsplines, settings
|
|
704
|
+
)
|
|
705
|
+
K = shared.n_bsplines
|
|
706
|
+
seeds = np.random.SeedSequence(settings["random_seed"]).spawn(len(genes))
|
|
707
|
+
results = []
|
|
708
|
+
for i, gene in enumerate(genes):
|
|
709
|
+
theta = _exact.draws(post, i, settings["draws"], np.random.default_rng(seeds[i]))
|
|
710
|
+
draws = {
|
|
711
|
+
"intercept": theta[:, 0],
|
|
712
|
+
"beta": theta[:, 1 : K + 1],
|
|
713
|
+
"categorical_beta": theta[:, K + 1],
|
|
714
|
+
"beta_1": theta[:, K + 2 :],
|
|
715
|
+
}
|
|
716
|
+
fit_columns = {"sigma_mean": float(np.sqrt(post.sigma2_mean[i])), "exact_edge_mass": float(post.edge_mass[i])}
|
|
717
|
+
results.append(_summarise_gene(gene, draws, shared, fit_columns, moments=(post.mean[i], post.cov[i])))
|
|
718
|
+
return results
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
def _exact_profile_results(shared, Y, genes, ss_within, settings):
|
|
722
|
+
"""Exact inference for the condition-free model: ``[(gene, diag_row, curve_row), ...]``."""
|
|
723
|
+
design = np.column_stack([np.ones(shared.B.shape[0]), shared.B])
|
|
724
|
+
post = _exact_fit(design, Y, genes, shared.weights, ss_within, _resolve_priors(shared.priors),
|
|
725
|
+
("intercept", "beta"), shared.n_bsplines, settings) # fmt: skip
|
|
726
|
+
A = np.column_stack([np.ones(shared.B_pred.shape[0]), shared.B_pred])
|
|
727
|
+
seeds = np.random.SeedSequence(settings["random_seed"]).spawn(len(genes))
|
|
728
|
+
results = []
|
|
729
|
+
for i, gene in enumerate(genes):
|
|
730
|
+
curves = _exact.draws(post, i, settings["draws"], np.random.default_rng(seeds[i])) @ A.T
|
|
731
|
+
ci_low, ci_high = np.quantile(curves, [0.03, 0.97], axis=0)
|
|
732
|
+
diag_row = {
|
|
733
|
+
"gene": gene,
|
|
734
|
+
"sigma_mean": float(np.sqrt(post.sigma2_mean[i])),
|
|
735
|
+
"exact_edge_mass": float(post.edge_mass[i]),
|
|
736
|
+
}
|
|
737
|
+
curve_row = {"mean": A @ post.mean[i], "ci_low": ci_low, "ci_high": ci_high, "beta": post.mean[i][1:],
|
|
738
|
+
"beta_cov": post.cov[i][1:, 1:]} # fmt: skip
|
|
739
|
+
results.append((gene, diag_row, curve_row))
|
|
740
|
+
return results
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def _fit_and_predict_profile(gene, Y, intercept_initval, shared, sample_kwargs, ss_within=None):
|
|
744
|
+
"""`_fit_and_predict_gene` for the condition-free model: one curve, no test. Returns ``(gene, diag_row, curve_row)``."""
|
|
745
|
+
model = _build_profile_model(
|
|
746
|
+
Y,
|
|
747
|
+
shared.B,
|
|
748
|
+
shared.n_bsplines,
|
|
749
|
+
intercept_initval,
|
|
750
|
+
shared.likelihood,
|
|
751
|
+
shared.priors,
|
|
752
|
+
weights=shared.weights,
|
|
753
|
+
ss_within=ss_within,
|
|
754
|
+
)
|
|
755
|
+
with model, _quiet_pymc():
|
|
756
|
+
idata, diag_columns = _sample_nuts(model, sample_kwargs, ["intercept", "beta", "sigma"])
|
|
757
|
+
|
|
758
|
+
# Posterior of the mean function on the grid: credible bands, no observation noise (see `_mean_curve_draws`).
|
|
759
|
+
draws = _posterior_draws(idata, "intercept")[:, None] + _posterior_draws(idata, "beta") @ shared.B_pred.T
|
|
760
|
+
mean = draws.mean(axis=0)
|
|
761
|
+
ci_low, ci_high = np.quantile(draws, [0.03, 0.97], axis=0)
|
|
762
|
+
beta_mean = idata.posterior["beta"].mean(("chain", "draw")).to_numpy()
|
|
763
|
+
|
|
764
|
+
diag_row = {"gene": gene, **diag_columns}
|
|
765
|
+
beta_cov = np.cov(_posterior_draws(idata, "beta"), rowvar=False)
|
|
766
|
+
curve_row = {"mean": mean, "ci_low": ci_low, "ci_high": ci_high, "beta": beta_mean, "beta_cov": beta_cov}
|
|
767
|
+
return gene, diag_row, curve_row
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def _gamma_null_pvalues(T_values, null_quantile=0.9):
|
|
771
|
+
"""Gamma-null p-values for `T_values`, the null fitted to the lower `null_quantile` of them.
|
|
772
|
+
|
|
773
|
+
Assumes that at least a `null_quantile` fraction of the values come from the null. The Gamma is fitted
|
|
774
|
+
by maximum likelihood to the values below the cutoff *as a truncated sample* (each term divided by the
|
|
775
|
+
Gamma's mass below the cutoff); fitting it as if untruncated gives too light a tail and p-values that
|
|
776
|
+
are too small. Values that are missing or not finite get a missing p-value.
|
|
777
|
+
"""
|
|
778
|
+
T_values = np.asarray(T_values, dtype=float)
|
|
779
|
+
finite = np.isfinite(T_values)
|
|
780
|
+
pvals = np.full(T_values.shape, np.nan)
|
|
781
|
+
if finite.sum() < 3:
|
|
782
|
+
return pvals
|
|
783
|
+
cutoff = np.quantile(T_values[finite], null_quantile)
|
|
784
|
+
pool = T_values[finite & (T_values <= cutoff) & (T_values > 0)]
|
|
785
|
+
if len(pool) < 3:
|
|
786
|
+
return pvals
|
|
787
|
+
shape0, _, scale0 = gamma.fit(pool, floc=0)
|
|
788
|
+
|
|
789
|
+
def neg_log_lik(log_params):
|
|
790
|
+
a, scale = np.exp(log_params)
|
|
791
|
+
return -(gamma.logpdf(pool, a, scale=scale).sum() - len(pool) * gamma.logcdf(cutoff, a, scale=scale))
|
|
792
|
+
|
|
793
|
+
fit = optimize.minimize(neg_log_lik, np.log([shape0, scale0]), method="Nelder-Mead")
|
|
794
|
+
a, scale = np.exp(fit.x)
|
|
795
|
+
pvals[finite] = gamma.sf(T_values[finite], a, scale=scale)
|
|
796
|
+
return pvals
|
|
797
|
+
|
|
798
|
+
|
|
799
|
+
def _benjamini_hochberg(pvals):
|
|
800
|
+
"""Benjamini-Hochberg FDR-adjusted p-values over the non-missing entries; missing ones stay missing."""
|
|
801
|
+
pvals = np.asarray(pvals, dtype=float)
|
|
802
|
+
out = np.full(pvals.shape, np.nan)
|
|
803
|
+
present = np.isfinite(pvals)
|
|
804
|
+
p = pvals[present]
|
|
805
|
+
n = len(p)
|
|
806
|
+
if n == 0:
|
|
807
|
+
return out
|
|
808
|
+
order = np.argsort(p)
|
|
809
|
+
ranked = p[order] * n / np.arange(1, n + 1)
|
|
810
|
+
adjusted = np.clip(np.minimum.accumulate(ranked[::-1])[::-1], 0, 1)
|
|
811
|
+
filled = np.empty(n)
|
|
812
|
+
filled[order] = adjusted
|
|
813
|
+
out[present] = filled
|
|
814
|
+
return out
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
# Condition tests, a registry like the priors/likelihoods. Each has a per-gene
|
|
818
|
+
# part, run in the worker, which returns the test's own columns plus the mean
|
|
819
|
+
# difference curve on the grid, and a panel part for what needs every gene.
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
def _wald_gene_stats(draws0, draws1, idata, shared, moments=None):
|
|
823
|
+
"""Wald test that the two condition curves are equal (level or shape), plus their `T` distance.
|
|
824
|
+
|
|
825
|
+
`draws0`/`draws1` are posterior draws of the mean curves, so the covariance is that of the
|
|
826
|
+
coefficients behind ``d(x) = f1(x) - f0(x)`` (rank ``n_bsplines``: the level plus the shape directions).
|
|
827
|
+
"""
|
|
828
|
+
if moments is None:
|
|
829
|
+
delta = draws1 - draws0
|
|
830
|
+
delta_mean = delta.mean(axis=0)
|
|
831
|
+
cov = np.cov(delta, rowvar=False)
|
|
832
|
+
else:
|
|
833
|
+
A0, A1 = _condition_maps(shared)
|
|
834
|
+
A = A1 - A0
|
|
835
|
+
delta_mean, cov = A @ moments[0], A @ moments[1] @ A.T
|
|
836
|
+
wald_stat, wald_pval, wald_df = _wald_from_moments(delta_mean, cov, df_denominator=shared.df_denominator)
|
|
837
|
+
T = float(np.mean(delta_mean**2))
|
|
838
|
+
row = {"wald_stat": wald_stat, "wald_pval": wald_pval, "wald_df": wald_df, "T": T}
|
|
839
|
+
if shared.df_denominator is not None:
|
|
840
|
+
row["wald_df_denominator"] = shared.df_denominator
|
|
841
|
+
return row, delta_mean
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def _level_weights(bin_centers):
|
|
845
|
+
"""Trapezoid weights that turn a curve on the grid into its mean over ``[0, 1]``."""
|
|
846
|
+
x = np.asarray(bin_centers, dtype=float)
|
|
847
|
+
w = np.zeros(len(x))
|
|
848
|
+
dx = np.diff(x)
|
|
849
|
+
w[:-1] += dx / 2
|
|
850
|
+
w[1:] += dx / 2
|
|
851
|
+
return w / (x[-1] - x[0])
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
def _level_gene_stats(draws0, draws1, idata, shared, moments=None):
|
|
855
|
+
"""Test of the level difference ``mean_x (f1 - f0)``, ``H0: |level| <= tau``, plus its credible interval."""
|
|
856
|
+
a = _level_weights(shared.bin_centers)
|
|
857
|
+
level_draws = (draws1 - draws0) @ a
|
|
858
|
+
if moments is None:
|
|
859
|
+
delta_mean = (draws1 - draws0).mean(axis=0)
|
|
860
|
+
level, var = float(level_draws.mean()), float(level_draws.var(ddof=1))
|
|
861
|
+
else:
|
|
862
|
+
A0, A1 = _condition_maps(shared)
|
|
863
|
+
A = A1 - A0
|
|
864
|
+
delta_mean = A @ moments[0]
|
|
865
|
+
level, var = float(a @ delta_mean), float(a @ A @ moments[1] @ A.T @ a)
|
|
866
|
+
stat = level**2 / var
|
|
867
|
+
pval = _one_df_pval(stat, var, shared.tau, shared.df_denominator)
|
|
868
|
+
low, high = (float(v) for v in np.quantile(level_draws, [0.03, 0.97]))
|
|
869
|
+
row = {
|
|
870
|
+
"level_diff": level,
|
|
871
|
+
"level_diff_ci_low": low,
|
|
872
|
+
"level_diff_ci_high": high,
|
|
873
|
+
"level_stat": stat,
|
|
874
|
+
"level_pval": pval,
|
|
875
|
+
}
|
|
876
|
+
if shared.df_denominator is not None:
|
|
877
|
+
row["level_df_denominator"] = shared.df_denominator
|
|
878
|
+
return row, delta_mean
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def _level_panel_stats(stats_df, null_quantile):
|
|
882
|
+
stats_df["level_pval_adj"] = _benjamini_hochberg(stats_df["level_pval"].to_numpy())
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
def _one_df_pval(stat, var, tau, df_denominator):
|
|
886
|
+
"""p-value of ``stat = estimate^2 / var`` for ``H0: |effect| <= tau`` (1 df; F with slides)."""
|
|
887
|
+
noncentrality = tau**2 / var
|
|
888
|
+
if df_denominator is None:
|
|
889
|
+
return float(ncx2.sf(stat, 1, noncentrality)) if tau > 0 else float(chi2.sf(stat, 1))
|
|
890
|
+
if tau > 0:
|
|
891
|
+
return float(ncf.sf(stat, 1, df_denominator, noncentrality))
|
|
892
|
+
return float(f.sf(stat, 1, df_denominator))
|
|
893
|
+
|
|
894
|
+
|
|
895
|
+
def _slope_gene_stats(draws0, draws1, idata, shared, moments=None):
|
|
896
|
+
"""Test of the slope difference: the least-squares slope of ``f1 - f0`` over ``[0, 1]``, ``H0: |slope| <= tau``.
|
|
897
|
+
|
|
898
|
+
One degree of freedom, so with few replicates it has far more power than the ``K - 1`` df parallel test
|
|
899
|
+
for a gradient that steepens or flattens.
|
|
900
|
+
"""
|
|
901
|
+
weights = shared.slope_weights
|
|
902
|
+
beta_1 = _posterior_draws(idata, "beta_1")
|
|
903
|
+
slope_draws = beta_1 @ weights
|
|
904
|
+
if moments is None:
|
|
905
|
+
slope, var = float(slope_draws.mean()), float(slope_draws.var(ddof=1))
|
|
906
|
+
delta_mean = (draws1 - draws0).mean(axis=0)
|
|
907
|
+
else:
|
|
908
|
+
b1 = slice(shared.n_bsplines + 2, 2 * shared.n_bsplines + 2)
|
|
909
|
+
slope, var = float(moments[0][b1] @ weights), float(weights @ moments[1][b1, b1] @ weights)
|
|
910
|
+
A0, A1 = _condition_maps(shared)
|
|
911
|
+
delta_mean = (A1 - A0) @ moments[0]
|
|
912
|
+
stat = slope**2 / var
|
|
913
|
+
low, high = (float(v) for v in np.quantile(slope_draws, [0.03, 0.97]))
|
|
914
|
+
row = {
|
|
915
|
+
"slope_diff": slope,
|
|
916
|
+
"slope_diff_ci_low": low,
|
|
917
|
+
"slope_diff_ci_high": high,
|
|
918
|
+
"slope_stat": stat,
|
|
919
|
+
"slope_pval": _one_df_pval(stat, var, shared.tau, shared.df_denominator),
|
|
920
|
+
}
|
|
921
|
+
if shared.df_denominator is not None:
|
|
922
|
+
row["slope_df_denominator"] = shared.df_denominator
|
|
923
|
+
return row, delta_mean
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
def _slope_panel_stats(stats_df, null_quantile):
|
|
927
|
+
stats_df["slope_pval_adj"] = _benjamini_hochberg(stats_df["slope_pval"].to_numpy())
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
def _wald_panel_stats(stats_df, null_quantile):
|
|
931
|
+
# Per-gene p-values; only the FDR adjustment needs the panel. (The panel nulls on `T` that used to be
|
|
932
|
+
# here were circular: they always flagged the top `1 - null_quantile` of the genes.)
|
|
933
|
+
stats_df["wald_pval_adj"] = _benjamini_hochberg(stats_df["wald_pval"].to_numpy())
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def _parallel_gene_stats(draws0, draws1, idata, shared, moments=None):
|
|
937
|
+
"""`parallel_curves_test` on the posterior of `beta_1`, plus the linear slope of the difference curve."""
|
|
938
|
+
beta_1 = _posterior_draws(idata, "beta_1") # (n_draws, K)
|
|
939
|
+
if moments is None:
|
|
940
|
+
shape_diff, stat, df, pval = parallel_curves_test(
|
|
941
|
+
beta_1, shared.shape_metric, tau=shared.tau, df_denominator=shared.df_denominator
|
|
942
|
+
)
|
|
943
|
+
delta_mean = (draws1 - draws0).mean(axis=0)
|
|
944
|
+
else:
|
|
945
|
+
K = shared.n_bsplines
|
|
946
|
+
b1 = slice(K + 2, 2 * K + 2)
|
|
947
|
+
shape_diff, stat, df, pval = parallel_curves_test(
|
|
948
|
+
moments[0][b1], shared.shape_metric, tau=shared.tau, cov=moments[1][b1, b1],
|
|
949
|
+
df_denominator=shared.df_denominator,
|
|
950
|
+
) # fmt: skip
|
|
951
|
+
A0, A1 = _condition_maps(shared)
|
|
952
|
+
delta_mean = (A1 - A0) @ moments[0]
|
|
953
|
+
slope = beta_1 @ shared.slope_weights
|
|
954
|
+
slope_low, slope_high = (float(v) for v in np.quantile(slope, [0.03, 0.97]))
|
|
955
|
+
row = {
|
|
956
|
+
"shape_diff": shape_diff,
|
|
957
|
+
"slope_diff": float(slope.mean()),
|
|
958
|
+
"slope_diff_ci_low": slope_low,
|
|
959
|
+
"slope_diff_ci_high": slope_high,
|
|
960
|
+
"parallel_stat": stat,
|
|
961
|
+
"parallel_df": df,
|
|
962
|
+
"parallel_pval": pval,
|
|
963
|
+
}
|
|
964
|
+
if moments is not None:
|
|
965
|
+
row["slope_diff"] = float(moments[0][slice(shared.n_bsplines + 2, None)] @ shared.slope_weights)
|
|
966
|
+
if shared.df_denominator is not None:
|
|
967
|
+
row["parallel_df_denominator"] = shared.df_denominator
|
|
968
|
+
return row, delta_mean
|
|
969
|
+
|
|
970
|
+
|
|
971
|
+
def _parallel_panel_stats(stats_df, null_quantile):
|
|
972
|
+
# The p-values are per gene; only the FDR adjustment needs the panel.
|
|
973
|
+
stats_df["parallel_pval_adj"] = _benjamini_hochberg(stats_df["parallel_pval"].to_numpy())
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
#: ``{name: (per-gene stats, panel stats)}`` - the condition tests `fit_model` can run.
|
|
977
|
+
_TESTS = {
|
|
978
|
+
"wald": (_wald_gene_stats, _wald_panel_stats),
|
|
979
|
+
"parallel": (_parallel_gene_stats, _parallel_panel_stats),
|
|
980
|
+
"level": (_level_gene_stats, _level_panel_stats),
|
|
981
|
+
"slope": (_slope_gene_stats, _slope_panel_stats),
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
_CURVE_SUFFIXES = ("cond0_mean", "cond0_ci_low", "cond0_ci_high", "cond1_mean", "cond1_ci_low", "cond1_ci_high")
|
|
986
|
+
_COEF_NAMES = ("beta", "beta_1")
|
|
987
|
+
|
|
988
|
+
|
|
989
|
+
def _coef_columns(n_bsplines):
|
|
990
|
+
"""Column labels of a coefficient table, one per spline basis function (strings, so they round-trip through h5ad)."""
|
|
991
|
+
return [f"B{k}" for k in range(n_bsplines)]
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
def _resolve_inference(inference, model_setup):
|
|
995
|
+
"""``"auto"`` picks exact inference where the model allows it (see `_exact.supported`), NUTS otherwise."""
|
|
996
|
+
allowed = ("auto", "exact", *_INFERENCE)
|
|
997
|
+
if inference not in allowed:
|
|
998
|
+
raise ValueError(f"Unknown inference {inference!r}. Available: {sorted(allowed)}")
|
|
999
|
+
if model_setup is None:
|
|
1000
|
+
return "nuts" if inference == "auto" else inference
|
|
1001
|
+
ok = _exact.supported(
|
|
1002
|
+
_resolve_priors(model_setup.priors), model_setup.likelihood, model_setup.binned.sample is not None
|
|
1003
|
+
)
|
|
1004
|
+
if inference == "exact" and not ok:
|
|
1005
|
+
raise ValueError(
|
|
1006
|
+
"inference='exact' needs the normal likelihood and normal priors on the coefficients; use 'nuts'."
|
|
1007
|
+
)
|
|
1008
|
+
resolved = ("exact" if ok else "nuts") if inference == "auto" else inference
|
|
1009
|
+
if resolved != "exact":
|
|
1010
|
+
_require_pymc(resolved)
|
|
1011
|
+
return resolved
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
def _warn_predictive_seed(predictive_seed):
|
|
1015
|
+
if predictive_seed is not None:
|
|
1016
|
+
warnings.warn(
|
|
1017
|
+
"`predictive_seed` no longer has an effect: the curves are now the posterior of the mean function, "
|
|
1018
|
+
"computed from the posterior draws without new random draws.",
|
|
1019
|
+
DeprecationWarning,
|
|
1020
|
+
stacklevel=3,
|
|
1021
|
+
)
|
|
1022
|
+
|
|
1023
|
+
|
|
1024
|
+
def _fittable_genes(genes, Y):
|
|
1025
|
+
"""The genes worth fitting.
|
|
1026
|
+
|
|
1027
|
+
Genes whose fitted rows are constant (e.g. all zero) are left out with a warning: the model reproduces
|
|
1028
|
+
them exactly, so the noise scale has no lower bound and the posterior is improper (the sampler diverges
|
|
1029
|
+
and the test would be meaningless). Their stats stay missing.
|
|
1030
|
+
"""
|
|
1031
|
+
degenerate = [g for g in genes if not np.isfinite(Y[g]).all() or np.ptp(np.asarray(Y[g], dtype=float)) == 0]
|
|
1032
|
+
if degenerate:
|
|
1033
|
+
shown = ", ".join(degenerate[:10]) + (" ..." if len(degenerate) > 10 else "")
|
|
1034
|
+
warnings.warn(
|
|
1035
|
+
f"{len(degenerate)} gene(s) are constant or non-finite over the fitted rows and were not fitted "
|
|
1036
|
+
f"(their stats are missing): {shown}",
|
|
1037
|
+
UserWarning,
|
|
1038
|
+
stacklevel=3,
|
|
1039
|
+
)
|
|
1040
|
+
return [g for g in genes if g not in set(degenerate)]
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
def _check_slides_per_condition(binned, sample_index, condition_fit):
|
|
1044
|
+
"""The F reference uses ``n_slides - 2`` degrees of freedom, which assumes at least two slides per condition."""
|
|
1045
|
+
codes = np.asarray(condition_fit, dtype=int)
|
|
1046
|
+
per_condition = [len(np.unique(np.asarray(sample_index)[codes == c])) for c in (0, 1)]
|
|
1047
|
+
if min(per_condition) < 2:
|
|
1048
|
+
labels = binned.condition_categories
|
|
1049
|
+
raise ValueError(
|
|
1050
|
+
"Comparing conditions across slides needs at least 2 slides per condition, found "
|
|
1051
|
+
f"{per_condition[0]} for {labels['0']!r} and {per_condition[1]} for {labels['1']!r}."
|
|
1052
|
+
)
|
|
1053
|
+
|
|
1054
|
+
|
|
1055
|
+
def _warn_unsupported_range(binned):
|
|
1056
|
+
"""Warn when one condition has no cells over part of ``[0, 1]``: the curve there is extrapolated."""
|
|
1057
|
+
empty = (np.asarray(binned.n_obs_grid) == 0).mean(axis=1)
|
|
1058
|
+
if empty.max() > 0.1:
|
|
1059
|
+
labels = binned.condition_categories
|
|
1060
|
+
detail = ", ".join(f"{labels[str(c)]!r}: {100 * empty[c]:.0f}%" for c in (0, 1))
|
|
1061
|
+
warnings.warn(
|
|
1062
|
+
f"Part of the anchor range has no cells in one condition ({detail} of the grid empty). The curves "
|
|
1063
|
+
"there are extrapolated from the spline and the prior, which inflates `shape_diff`/`integral`.",
|
|
1064
|
+
UserWarning,
|
|
1065
|
+
stacklevel=3,
|
|
1066
|
+
)
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def fit_model(
|
|
1070
|
+
adata,
|
|
1071
|
+
model_setup,
|
|
1072
|
+
n_workers=1,
|
|
1073
|
+
progressbar=True,
|
|
1074
|
+
inplace=True,
|
|
1075
|
+
use_binned=True,
|
|
1076
|
+
stats_key="dsvg_stats",
|
|
1077
|
+
preds_key="dsvg_preds",
|
|
1078
|
+
curves_key="dsvg_curves",
|
|
1079
|
+
null_quantile=0.9,
|
|
1080
|
+
sample_kwargs=None,
|
|
1081
|
+
test="parallel",
|
|
1082
|
+
tau=None,
|
|
1083
|
+
predictive_seed=None,
|
|
1084
|
+
inference="auto",
|
|
1085
|
+
coefs_key="dsvg_coefs",
|
|
1086
|
+
**pymc_kwargs,
|
|
1087
|
+
):
|
|
1088
|
+
"""Fit every per-gene model from `setup_model`, in parallel, and summarize the result.
|
|
1089
|
+
|
|
1090
|
+
For each gene: (re-)builds its own private model and obtains the
|
|
1091
|
+
posterior (see `inference`), then samples the posterior predictive on the
|
|
1092
|
+
fixed `anchor` grid under both levels of `condition`, and tests the two
|
|
1093
|
+
curves against each other (see `test`).
|
|
1094
|
+
|
|
1095
|
+
Parameters
|
|
1096
|
+
----------
|
|
1097
|
+
adata
|
|
1098
|
+
The same `AnnData` object `model_setup.binned` was built from; only
|
|
1099
|
+
used as the write target when `inplace` is `True`.
|
|
1100
|
+
model_setup
|
|
1101
|
+
Output of `setup_model`.
|
|
1102
|
+
n_workers
|
|
1103
|
+
Number of worker processes (`joblib`, `loky` backend). Each gene's
|
|
1104
|
+
model lives entirely within one worker for its whole lifetime -
|
|
1105
|
+
genes are never split across workers or re-synced, so there is
|
|
1106
|
+
nothing to race. `n_workers=1` runs sequentially in-process (no
|
|
1107
|
+
extra pickling at all).
|
|
1108
|
+
use_binned
|
|
1109
|
+
Fit on the per-bin averages from `diffsvg.pp.bin_obs` (default), or
|
|
1110
|
+
on the raw per-cell values when `False`. Raw keeps every
|
|
1111
|
+
observation instead of collapsing each window to its mean, at the
|
|
1112
|
+
cost of a far larger design matrix - with spatial data that is
|
|
1113
|
+
mostly zero counts, which is what the binning is there to avoid,
|
|
1114
|
+
this is typically much slower per gene. Knots, degree and the
|
|
1115
|
+
condition encoding are identical either way, so the two are
|
|
1116
|
+
directly comparable. Bins built with ``window_radius=None`` are
|
|
1117
|
+
weighted by their cell counts, which makes the binned fit carry the
|
|
1118
|
+
same information as the per-cell one at the cost of the binned fit.
|
|
1119
|
+
progressbar
|
|
1120
|
+
Show a `tqdm` progress bar over completed genes. Per-worker MCMC
|
|
1121
|
+
progress bars are always disabled (they would interleave garbage
|
|
1122
|
+
across processes); pass e.g. ``sample_kwargs={"progressbar": True}``
|
|
1123
|
+
to override that only when `n_workers=1`.
|
|
1124
|
+
inplace
|
|
1125
|
+
If `True` (default), write results to `adata.varm`/`adata.uns`
|
|
1126
|
+
(aligned to ``adata.var_names``, with genes not in `model_setup`
|
|
1127
|
+
left as `NaN`) and return `None`. If `False`, leave `adata` untouched
|
|
1128
|
+
and return ``(stats_df, preds_df, curve_dfs)``; `curve_dfs` then also
|
|
1129
|
+
holds the coefficient tables under ``"beta"`` and ``"beta_1"``.
|
|
1130
|
+
stats_key
|
|
1131
|
+
`adata.varm` key for the per-gene summary statistics (`n_genes x n`;
|
|
1132
|
+
see `diffsvg.pl.plot_volcano`/`diffsvg.pl.plot_pvalue_histogram`).
|
|
1133
|
+
Its columns depend on `test`; which test ran (and its `tau`) is
|
|
1134
|
+
recorded in ``adata.uns[f"{stats_key}_params"]``.
|
|
1135
|
+
preds_key
|
|
1136
|
+
`adata.varm` key for the per-gene, per-bin predicted
|
|
1137
|
+
``condition==1`` minus ``condition==0`` difference curve
|
|
1138
|
+
(`n_genes x (n_bins + 1)`).
|
|
1139
|
+
curves_key
|
|
1140
|
+
Prefix for six `adata.varm` entries (`n_genes x (n_bins + 1)` each),
|
|
1141
|
+
one per ``{"cond0", "cond1"} x {"mean", "ci_low", "ci_high"}``: the
|
|
1142
|
+
per-condition predicted curves with a credible band, consumed by
|
|
1143
|
+
`diffsvg.pl.plot_prediction_curve`. The condition labels are also
|
|
1144
|
+
written to ``adata.uns[f"{curves_key}_condition_categories"]``.
|
|
1145
|
+
null_quantile
|
|
1146
|
+
No longer used by `fit_model` (kept for backwards compatibility).
|
|
1147
|
+
The panel-based empirical/Gamma nulls it configured were circular;
|
|
1148
|
+
``test="wald"`` now has a per-gene p-value.
|
|
1149
|
+
sample_kwargs
|
|
1150
|
+
Extra keyword arguments merged into the `pymc.sample` call (applied
|
|
1151
|
+
to every gene). Any extra keyword arguments to `fit_model` itself are
|
|
1152
|
+
forwarded the same way, so ``fit_model(..., chains=2)`` also works.
|
|
1153
|
+
With ``inference="exact"`` only ``draws``, ``random_seed``,
|
|
1154
|
+
``n_grid`` and ``n_hyper`` apply.
|
|
1155
|
+
test
|
|
1156
|
+
Which test to run between the two condition curves:
|
|
1157
|
+
|
|
1158
|
+
- ``"parallel"`` (default): whether the curves are parallel, i.e. differ only
|
|
1159
|
+
in level, via `parallel_curves_test` on the posterior of the
|
|
1160
|
+
interaction coefficients. The p-value is per gene, so it does not
|
|
1161
|
+
depend on the panel and works for a single gene. Columns
|
|
1162
|
+
``shape_diff`` (how far from parallel), ``slope_diff`` (the linear
|
|
1163
|
+
trend of ``f1 - f0`` over the anchor range, with a 94% credible
|
|
1164
|
+
interval ``slope_diff_ci_low``/``slope_diff_ci_high``),
|
|
1165
|
+
``parallel_stat``, ``parallel_df``, ``parallel_pval`` and
|
|
1166
|
+
``parallel_pval_adj`` (Benjamini-Hochberg).
|
|
1167
|
+
- ``"wald"``: whether the curves are equal at all (level or shape),
|
|
1168
|
+
a tradeSeq-style functional Wald test on the posterior of the
|
|
1169
|
+
difference curve, chi-square with ``n_bsplines`` degrees of
|
|
1170
|
+
freedom (columns ``wald_stat``, ``wald_pval``, ``wald_df``,
|
|
1171
|
+
``wald_pval_adj`` and ``T``, the mean squared difference). It has
|
|
1172
|
+
no `tau`, so with many cells nearly every gene with any difference
|
|
1173
|
+
becomes significant; rank by ``wald_stat`` rather than thresholding.
|
|
1174
|
+
- ``"level"``: whether the curves differ in level, i.e. in the mean
|
|
1175
|
+
of ``f1 - f0`` over the anchor range, beyond `tau` (``H0: |level|
|
|
1176
|
+
<= tau``; noncentral chi-square with 1 degree of freedom). The
|
|
1177
|
+
complement of ``"parallel"``, which ignores the level. Columns
|
|
1178
|
+
``level_diff`` (with a 94% credible interval), ``level_stat``,
|
|
1179
|
+
``level_pval`` and ``level_pval_adj``.
|
|
1180
|
+
- ``"slope"``: whether the gap between the curves changes linearly
|
|
1181
|
+
along the anchor, i.e. whether one gradient is steeper than the
|
|
1182
|
+
other: the least-squares slope of ``f1 - f0`` over ``[0, 1]``,
|
|
1183
|
+
``H0: |slope| <= tau``, 1 degree of freedom. With few replicates it
|
|
1184
|
+
has far more power than ``"parallel"`` (``n_bsplines - 1`` df) for
|
|
1185
|
+
gradients that steepen or flatten, and it misses non-monotone
|
|
1186
|
+
changes, which ``"parallel"`` covers. Columns ``slope_diff`` (with
|
|
1187
|
+
a 94% credible interval), ``slope_stat``, ``slope_pval`` and
|
|
1188
|
+
``slope_pval_adj``.
|
|
1189
|
+
|
|
1190
|
+
With data binned with `sample` (several slides per condition) the model has a random level and a
|
|
1191
|
+
random curve per slide, so the covariance of the interaction
|
|
1192
|
+
coefficients holds the variation between slides, and the p-value
|
|
1193
|
+
uses an F reference with ``n_slides - 2`` denominator degrees of
|
|
1194
|
+
freedom for every test (column ``*_df_denominator``). The fitted curves are
|
|
1195
|
+
the population-level ones. Fit on bins (NUTS is fine) or on cells
|
|
1196
|
+
(``use_binned=False``).
|
|
1197
|
+
|
|
1198
|
+
Both tests also write ``integral`` (the area between the curves),
|
|
1199
|
+
the ``categorical_beta`` summary and the sampler diagnostics.
|
|
1200
|
+
Binned fits with overlapping windows (`diffsvg.pp.bin_obs`'s default
|
|
1201
|
+
`window_radius`) treat neighbouring bins as independent although they
|
|
1202
|
+
share cells, which makes the model over-confident and the
|
|
1203
|
+
``"parallel"`` p-values too small (a warning says so); contiguous
|
|
1204
|
+
bins (``bin_obs(..., window_radius=None)``) and fits on the cells
|
|
1205
|
+
(``use_binned=False``) do not have that problem.
|
|
1206
|
+
tau
|
|
1207
|
+
For the ``"parallel"``, ``"level"`` and ``"slope"`` tests: the
|
|
1208
|
+
smallest difference that counts (``shape_diff``, ``level_diff`` or
|
|
1209
|
+
``slope_diff``). The p-value tests ``|difference| <= tau``, so smaller
|
|
1210
|
+
differences do not become significant however many cells there are.
|
|
1211
|
+
Defaults to ``0.1`` for pooled data and to ``0`` for data binned with
|
|
1212
|
+
`sample`, where the variation between slides already calibrates the
|
|
1213
|
+
test and a positive `tau` only costs power.
|
|
1214
|
+
predictive_seed
|
|
1215
|
+
Deprecated, has no effect. The curves, their bands, ``integral`` and
|
|
1216
|
+
the Wald test are now computed from the posterior draws of the mean
|
|
1217
|
+
function (no observation noise), so they need no extra random draws.
|
|
1218
|
+
inference
|
|
1219
|
+
How each gene's posterior is obtained:
|
|
1220
|
+
|
|
1221
|
+
- ``"nuts"``: MCMC with `pymc.sample` (needs the ``pymc`` extra), configured by
|
|
1222
|
+
`sample_kwargs`. Reports ``n_divergent``, ``rhat_max`` and
|
|
1223
|
+
``ess_bulk_min``.
|
|
1224
|
+
- ``"exact"``: the exact posterior (see `diffsvg.tl._exact`): the
|
|
1225
|
+
coefficients are conjugate given the noise scale, which is
|
|
1226
|
+
integrated out numerically, as are the two slide scales. No Monte
|
|
1227
|
+
Carlo error and no sampler; needs the normal likelihood and normal
|
|
1228
|
+
coefficient priors. Reports ``exact_edge_mass``.
|
|
1229
|
+
- ``"auto"`` (default): ``"exact"`` where the model allows it,
|
|
1230
|
+
``"nuts"`` otherwise.
|
|
1231
|
+
coefs_key
|
|
1232
|
+
Prefix for two `adata.varm` tables (`n_genes x n_bsplines`) with the
|
|
1233
|
+
posterior means of the spline coefficients, consumed by
|
|
1234
|
+
`diffsvg.tl.cluster_curves`: ``f"{coefs_key}_beta"`` (the main spline
|
|
1235
|
+
term; with the 0/1 condition coding this is the curve of condition 0,
|
|
1236
|
+
not a pooled one) and ``f"{coefs_key}_beta_1"`` (the interaction, how
|
|
1237
|
+
the curve of condition 1 differs from it).
|
|
1238
|
+
|
|
1239
|
+
Returns
|
|
1240
|
+
-------
|
|
1241
|
+
None or tuple[pandas.DataFrame, pandas.DataFrame, dict[str, pandas.DataFrame]]
|
|
1242
|
+
"""
|
|
1243
|
+
if test not in _TESTS:
|
|
1244
|
+
raise ValueError(f"Unknown test {test!r}. Available: {sorted(_TESTS)}")
|
|
1245
|
+
if tau is not None and tau < 0:
|
|
1246
|
+
raise ValueError(f"`tau` must be >= 0, got {tau}.")
|
|
1247
|
+
inference = _resolve_inference(inference, model_setup)
|
|
1248
|
+
if inference == "exact":
|
|
1249
|
+
sample_defaults = _exact_settings({**(sample_kwargs or {}), **pymc_kwargs})
|
|
1250
|
+
elif inference == "nuts":
|
|
1251
|
+
sample_defaults = {
|
|
1252
|
+
"nuts_sampler": "nutpie",
|
|
1253
|
+
"draws": 1000,
|
|
1254
|
+
"tune": 1000,
|
|
1255
|
+
"chains": 4,
|
|
1256
|
+
"random_seed": 42,
|
|
1257
|
+
"compute_convergence_checks": False,
|
|
1258
|
+
"init": "adapt_diag",
|
|
1259
|
+
"progressbar": False,
|
|
1260
|
+
}
|
|
1261
|
+
if inference != "exact":
|
|
1262
|
+
sample_defaults.update(sample_kwargs or {})
|
|
1263
|
+
sample_defaults.update(pymc_kwargs)
|
|
1264
|
+
binned = model_setup.binned
|
|
1265
|
+
if not binned.has_condition:
|
|
1266
|
+
raise ValueError(
|
|
1267
|
+
"adata.uns['binned'] was built without a `condition`, so there are no two curves to compare. "
|
|
1268
|
+
"Re-run `diffsvg.pp.bin_obs` with one, or use `diffsvg.tl.fit_profiles` + "
|
|
1269
|
+
"`diffsvg.tl.one_vs_rest` for the condition-free one-vs-rest test."
|
|
1270
|
+
)
|
|
1271
|
+
B, B_masked, condition_fit, Y, intercept_initvals = _fit_inputs(adata, binned, use_binned)
|
|
1272
|
+
weights, ss_within, n_obs_grid = _weighting(binned, use_binned)
|
|
1273
|
+
sample_index, n_samples = _sample_inputs(adata, binned, use_binned)
|
|
1274
|
+
if tau is None:
|
|
1275
|
+
tau = 0.0 if n_samples is not None else 0.1
|
|
1276
|
+
if n_samples is not None:
|
|
1277
|
+
if n_samples < 3:
|
|
1278
|
+
raise ValueError(f"Comparing conditions across slides needs at least 3 slides, found {n_samples}.")
|
|
1279
|
+
if test == "parallel" and use_binned and not binned.weighted:
|
|
1280
|
+
warnings.warn(
|
|
1281
|
+
"test='parallel' on overlapping bins: neighbouring bins share cells, so the p-values come out too "
|
|
1282
|
+
"small. Re-run `diffsvg.pp.bin_obs(..., window_radius=None)`, or fit with `use_binned=False`.",
|
|
1283
|
+
UserWarning,
|
|
1284
|
+
stacklevel=2,
|
|
1285
|
+
)
|
|
1286
|
+
_warn_predictive_seed(predictive_seed)
|
|
1287
|
+
if n_samples is not None:
|
|
1288
|
+
_check_slides_per_condition(binned, sample_index, condition_fit)
|
|
1289
|
+
if use_binned and binned.condition is not None and binned.n_obs_grid is not None:
|
|
1290
|
+
_warn_unsupported_range(binned)
|
|
1291
|
+
fit_genes = _fittable_genes(binned.genes, Y)
|
|
1292
|
+
shared = _FitShared(
|
|
1293
|
+
B=B,
|
|
1294
|
+
B_masked=B_masked,
|
|
1295
|
+
condition_fit=condition_fit,
|
|
1296
|
+
n_bsplines=binned.n_bsplines,
|
|
1297
|
+
likelihood=model_setup.likelihood,
|
|
1298
|
+
priors=model_setup.priors,
|
|
1299
|
+
bin_centers=binned.bin_centers,
|
|
1300
|
+
B_pred=binned.B_pred,
|
|
1301
|
+
B_pred_masked_0=binned.B_pred_masked_0,
|
|
1302
|
+
B_pred_masked_1=binned.B_pred_masked_1,
|
|
1303
|
+
condition_pred_0=binned.condition_pred_0,
|
|
1304
|
+
condition_pred_1=binned.condition_pred_1,
|
|
1305
|
+
test=test,
|
|
1306
|
+
tau=float(tau),
|
|
1307
|
+
shape_metric=shape_metric(binned.n_bsplines, binned.degree) if test == "parallel" else None,
|
|
1308
|
+
slope_weights=_slope_weights(binned.n_bsplines, binned.degree) if test in ("parallel", "slope") else None,
|
|
1309
|
+
weights=weights,
|
|
1310
|
+
inference=inference,
|
|
1311
|
+
sample_index=sample_index,
|
|
1312
|
+
n_samples=n_samples,
|
|
1313
|
+
df_denominator=None if n_samples is None else n_samples - 2,
|
|
1314
|
+
)
|
|
1315
|
+
|
|
1316
|
+
n_jobs = 1 if n_workers is None else n_workers
|
|
1317
|
+
if inference == "exact":
|
|
1318
|
+
results = _exact_gene_results(shared, Y, fit_genes, ss_within, sample_defaults)
|
|
1319
|
+
else:
|
|
1320
|
+
results = Parallel(n_jobs=n_jobs, backend="loky", return_as="generator_unordered")(
|
|
1321
|
+
delayed(_fit_and_predict_gene)(
|
|
1322
|
+
gene,
|
|
1323
|
+
Y[gene],
|
|
1324
|
+
intercept_initvals[gene],
|
|
1325
|
+
shared,
|
|
1326
|
+
sample_defaults,
|
|
1327
|
+
ss_within=None if ss_within is None else ss_within[gene],
|
|
1328
|
+
)
|
|
1329
|
+
for gene in fit_genes
|
|
1330
|
+
)
|
|
1331
|
+
if progressbar:
|
|
1332
|
+
results = tqdm(results, total=len(fit_genes), desc="Fitting DiffSVG models")
|
|
1333
|
+
|
|
1334
|
+
stats_rows = []
|
|
1335
|
+
curve_rows = {}
|
|
1336
|
+
for gene, stats_row, curve_row in results:
|
|
1337
|
+
stats_rows.append(stats_row)
|
|
1338
|
+
curve_rows[gene] = curve_row
|
|
1339
|
+
|
|
1340
|
+
stats_df = pd.DataFrame(stats_rows).set_index("gene").reindex(binned.genes)
|
|
1341
|
+
_TESTS[test][1](stats_df, null_quantile)
|
|
1342
|
+
test_params = {"test": test, "tau": float(tau)} if test in ("parallel", "level", "slope") else {"test": test}
|
|
1343
|
+
|
|
1344
|
+
# Column labels must be strings (not e.g. plain floats) so that
|
|
1345
|
+
# `adata.varm[...]` round-trips through h5ad, whose DataFrame writer
|
|
1346
|
+
# requires string column names.
|
|
1347
|
+
bin_cols = [str(round(float(c), 4)) for c in binned.bin_centers]
|
|
1348
|
+
preds_df = pd.DataFrame.from_dict(
|
|
1349
|
+
{gene: row["delta"] for gene, row in curve_rows.items()}, orient="index", columns=bin_cols
|
|
1350
|
+
).reindex(binned.genes)
|
|
1351
|
+
curve_dfs = {
|
|
1352
|
+
suffix: pd.DataFrame.from_dict(
|
|
1353
|
+
{gene: row[suffix] for gene, row in curve_rows.items()}, orient="index", columns=bin_cols
|
|
1354
|
+
).reindex(binned.genes)
|
|
1355
|
+
for suffix in _CURVE_SUFFIXES
|
|
1356
|
+
}
|
|
1357
|
+
coef_dfs = {
|
|
1358
|
+
name: pd.DataFrame.from_dict(
|
|
1359
|
+
{gene: row[name] for gene, row in curve_rows.items()},
|
|
1360
|
+
orient="index",
|
|
1361
|
+
columns=_coef_columns(binned.n_bsplines),
|
|
1362
|
+
).reindex(binned.genes)
|
|
1363
|
+
for name in _COEF_NAMES
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
if not inplace:
|
|
1367
|
+
stats_df.attrs.update(test_params)
|
|
1368
|
+
return stats_df, preds_df, {**curve_dfs, **coef_dfs}
|
|
1369
|
+
|
|
1370
|
+
adata.varm[stats_key] = stats_df.reindex(adata.var_names)
|
|
1371
|
+
adata.varm[preds_key] = preds_df.reindex(adata.var_names)
|
|
1372
|
+
for suffix, df in curve_dfs.items():
|
|
1373
|
+
adata.varm[f"{curves_key}_{suffix}"] = df.reindex(adata.var_names)
|
|
1374
|
+
for name, df in coef_dfs.items():
|
|
1375
|
+
adata.varm[f"{coefs_key}_{name}"] = df.reindex(adata.var_names)
|
|
1376
|
+
adata.uns[f"{curves_key}_condition_categories"] = binned.condition_categories
|
|
1377
|
+
adata.uns[f"{stats_key}_params"] = test_params
|
|
1378
|
+
return None
|
|
1379
|
+
|
|
1380
|
+
|
|
1381
|
+
# ============================================================
|
|
1382
|
+
# 3b) Condition-free profiles + the one-vs-rest test
|
|
1383
|
+
# ============================================================
|
|
1384
|
+
|
|
1385
|
+
|
|
1386
|
+
_PROFILE_SUFFIXES = ("mean", "ci_low", "ci_high")
|
|
1387
|
+
|
|
1388
|
+
|
|
1389
|
+
def fit_profiles(
|
|
1390
|
+
adata,
|
|
1391
|
+
model_setup,
|
|
1392
|
+
n_workers=1,
|
|
1393
|
+
progressbar=True,
|
|
1394
|
+
inplace=True,
|
|
1395
|
+
use_binned=True,
|
|
1396
|
+
profiles_key="dsvg_profiles",
|
|
1397
|
+
sample_kwargs=None,
|
|
1398
|
+
predictive_seed=None,
|
|
1399
|
+
inference="auto",
|
|
1400
|
+
**pymc_kwargs,
|
|
1401
|
+
):
|
|
1402
|
+
"""Fit the condition-free model for every gene, giving one expression profile per gene.
|
|
1403
|
+
|
|
1404
|
+
The counterpart to `fit_model` for data binned without a `condition`
|
|
1405
|
+
(`diffsvg.pp.bin_obs(..., condition=None)`): it drops the condition
|
|
1406
|
+
covariate and its spline interaction, so each gene gets a single curve
|
|
1407
|
+
rather than one per condition. No differential statistic is computed
|
|
1408
|
+
here - `one_vs_rest` produces that by comparing genes against each
|
|
1409
|
+
other.
|
|
1410
|
+
|
|
1411
|
+
Also usable on data that *does* have conditions, in which case both
|
|
1412
|
+
conditions' bins feed one pooled curve per gene. That is an alternative
|
|
1413
|
+
to what `one_vs_rest` does by default there (averaging the two fitted
|
|
1414
|
+
condition curves), and the two need not agree exactly.
|
|
1415
|
+
|
|
1416
|
+
Parameters
|
|
1417
|
+
----------
|
|
1418
|
+
adata
|
|
1419
|
+
The `AnnData` `model_setup.binned` was built from; the write target
|
|
1420
|
+
when `inplace` is `True`.
|
|
1421
|
+
model_setup
|
|
1422
|
+
Output of `setup_model`.
|
|
1423
|
+
n_workers, progressbar, use_binned, sample_kwargs, predictive_seed, **pymc_kwargs
|
|
1424
|
+
As in `fit_model`.
|
|
1425
|
+
inplace
|
|
1426
|
+
If `True` (default), write the results to `adata.varm` and return
|
|
1427
|
+
`None`; otherwise return ``(profile_dfs, diagnostics_df)``.
|
|
1428
|
+
profiles_key
|
|
1429
|
+
Prefix for three `adata.varm` tables (`n_genes x (n_bins + 1)`), one
|
|
1430
|
+
per ``{"mean", "ci_low", "ci_high"}``, plus a
|
|
1431
|
+
``f"{profiles_key}_diagnostics"`` table of per-gene sampler
|
|
1432
|
+
diagnostics and ``f"{profiles_key}_beta"`` (`n_genes x n_bsplines`,
|
|
1433
|
+
the posterior means of the spline coefficients, for
|
|
1434
|
+
`diffsvg.tl.cluster_curves` with ``by="profile"``). With
|
|
1435
|
+
``inplace=False`` that table is ``profile_dfs["beta"]``.
|
|
1436
|
+
|
|
1437
|
+
Returns
|
|
1438
|
+
-------
|
|
1439
|
+
None or tuple[dict[str, pandas.DataFrame], pandas.DataFrame]
|
|
1440
|
+
"""
|
|
1441
|
+
if inference not in ("auto", "exact", "nuts"):
|
|
1442
|
+
raise ValueError(f"Unknown inference {inference!r} for fit_profiles. Available: ['auto', 'exact', 'nuts']")
|
|
1443
|
+
priors = _resolve_priors(model_setup.priors)
|
|
1444
|
+
exact_ok = (
|
|
1445
|
+
model_setup.likelihood == "normal"
|
|
1446
|
+
and model_setup.binned.sample is None
|
|
1447
|
+
and all(priors[n][0] == "normal" for n in ("intercept", "beta"))
|
|
1448
|
+
and priors["sigma"][0] in _exact._SIGMA_LOGPDF
|
|
1449
|
+
)
|
|
1450
|
+
if inference == "exact" and not exact_ok:
|
|
1451
|
+
raise ValueError("inference='exact' needs the normal likelihood and normal priors on the coefficients.")
|
|
1452
|
+
inference = ("exact" if exact_ok else "nuts") if inference == "auto" else inference
|
|
1453
|
+
if inference != "exact":
|
|
1454
|
+
_require_pymc(inference)
|
|
1455
|
+
sample_defaults = {
|
|
1456
|
+
"nuts_sampler": "nutpie",
|
|
1457
|
+
"draws": 1000,
|
|
1458
|
+
"tune": 1000,
|
|
1459
|
+
"chains": 4,
|
|
1460
|
+
"random_seed": 42,
|
|
1461
|
+
"compute_convergence_checks": False,
|
|
1462
|
+
"init": "adapt_diag",
|
|
1463
|
+
"progressbar": False,
|
|
1464
|
+
}
|
|
1465
|
+
if inference == "exact":
|
|
1466
|
+
sample_defaults = _exact_settings({**(sample_kwargs or {}), **pymc_kwargs})
|
|
1467
|
+
else:
|
|
1468
|
+
sample_defaults.update(sample_kwargs or {})
|
|
1469
|
+
sample_defaults.update(pymc_kwargs)
|
|
1470
|
+
|
|
1471
|
+
binned = model_setup.binned
|
|
1472
|
+
if binned.sample is not None:
|
|
1473
|
+
raise ValueError("`fit_profiles` does not model slides; bin without `sample`.")
|
|
1474
|
+
B, _, _, Y, intercept_initvals = _fit_inputs(adata, binned, use_binned)
|
|
1475
|
+
weights, ss_within, n_obs_grid = _weighting(binned, use_binned)
|
|
1476
|
+
_warn_predictive_seed(predictive_seed)
|
|
1477
|
+
fit_genes = _fittable_genes(binned.genes, Y)
|
|
1478
|
+
shared = _ProfileShared(
|
|
1479
|
+
B=B,
|
|
1480
|
+
n_bsplines=binned.n_bsplines,
|
|
1481
|
+
likelihood=model_setup.likelihood,
|
|
1482
|
+
priors=model_setup.priors,
|
|
1483
|
+
B_pred=binned.B_pred,
|
|
1484
|
+
weights=weights,
|
|
1485
|
+
# One pooled curve: with conditions binned separately, a grid point holds both levels' cells.
|
|
1486
|
+
)
|
|
1487
|
+
|
|
1488
|
+
n_jobs = 1 if n_workers is None else n_workers
|
|
1489
|
+
if inference == "exact":
|
|
1490
|
+
results = _exact_profile_results(shared, Y, fit_genes, ss_within, sample_defaults)
|
|
1491
|
+
else:
|
|
1492
|
+
results = Parallel(n_jobs=n_jobs, backend="loky", return_as="generator_unordered")(
|
|
1493
|
+
delayed(_fit_and_predict_profile)(
|
|
1494
|
+
gene,
|
|
1495
|
+
Y[gene],
|
|
1496
|
+
intercept_initvals[gene],
|
|
1497
|
+
shared,
|
|
1498
|
+
sample_defaults,
|
|
1499
|
+
ss_within=None if ss_within is None else ss_within[gene],
|
|
1500
|
+
)
|
|
1501
|
+
for gene in fit_genes
|
|
1502
|
+
)
|
|
1503
|
+
if progressbar:
|
|
1504
|
+
results = tqdm(results, total=len(binned.genes), desc="Fitting DiffSVG profiles")
|
|
1505
|
+
|
|
1506
|
+
diag_rows = []
|
|
1507
|
+
curve_rows = {}
|
|
1508
|
+
for gene, diag_row, curve_row in results:
|
|
1509
|
+
diag_rows.append(diag_row)
|
|
1510
|
+
curve_rows[gene] = curve_row
|
|
1511
|
+
|
|
1512
|
+
diagnostics_df = pd.DataFrame(diag_rows).set_index("gene").reindex(binned.genes)
|
|
1513
|
+
bin_cols = [str(round(float(c), 4)) for c in binned.bin_centers]
|
|
1514
|
+
profile_dfs = {
|
|
1515
|
+
suffix: pd.DataFrame.from_dict(
|
|
1516
|
+
{gene: row[suffix] for gene, row in curve_rows.items()}, orient="index", columns=bin_cols
|
|
1517
|
+
).reindex(binned.genes)
|
|
1518
|
+
for suffix in _PROFILE_SUFFIXES
|
|
1519
|
+
}
|
|
1520
|
+
profile_dfs["beta"] = pd.DataFrame.from_dict(
|
|
1521
|
+
{gene: row["beta"] for gene, row in curve_rows.items()},
|
|
1522
|
+
orient="index",
|
|
1523
|
+
columns=_coef_columns(binned.n_bsplines),
|
|
1524
|
+
).reindex(binned.genes)
|
|
1525
|
+
|
|
1526
|
+
if not inplace:
|
|
1527
|
+
return profile_dfs, diagnostics_df
|
|
1528
|
+
|
|
1529
|
+
for suffix, df in profile_dfs.items():
|
|
1530
|
+
adata.varm[f"{profiles_key}_{suffix}"] = df.reindex(adata.var_names)
|
|
1531
|
+
adata.varm[f"{profiles_key}_diagnostics"] = diagnostics_df.reindex(adata.var_names)
|
|
1532
|
+
fitted = [g for g in binned.genes if g in curve_rows]
|
|
1533
|
+
adata.uns[f"{profiles_key}_beta_cov"] = {
|
|
1534
|
+
"genes": np.asarray(fitted, dtype=str),
|
|
1535
|
+
"cov": np.stack([curve_rows[g]["beta_cov"] for g in fitted]),
|
|
1536
|
+
"degree": int(binned.degree),
|
|
1537
|
+
}
|
|
1538
|
+
return None
|
|
1539
|
+
|
|
1540
|
+
|
|
1541
|
+
def _gene_profiles(adata, curves_key, profiles_key):
|
|
1542
|
+
"""One curve per gene, as a (genes x bins) frame, plus a note on where it came from.
|
|
1543
|
+
|
|
1544
|
+
Prefers an explicit `fit_profiles` result; otherwise averages the two
|
|
1545
|
+
per-condition curves from `fit_model`, which is what collapses a
|
|
1546
|
+
two-condition fit into a single profile per gene.
|
|
1547
|
+
"""
|
|
1548
|
+
if f"{profiles_key}_mean" in adata.varm:
|
|
1549
|
+
return adata.varm[f"{profiles_key}_mean"], "profiles"
|
|
1550
|
+
if f"{curves_key}_cond0_mean" in adata.varm:
|
|
1551
|
+
cond0 = adata.varm[f"{curves_key}_cond0_mean"]
|
|
1552
|
+
cond1 = adata.varm[f"{curves_key}_cond1_mean"]
|
|
1553
|
+
return (cond0 + cond1) / 2.0, "condition-average"
|
|
1554
|
+
raise KeyError(
|
|
1555
|
+
f"neither adata.varm[{profiles_key + '_mean'!r}] nor adata.varm[{curves_key + '_cond0_mean'!r}] found - "
|
|
1556
|
+
"run `diffsvg.tl.fit_profiles` (no conditions) or `diffsvg.tl.fit_model` (with conditions) first."
|
|
1557
|
+
)
|
|
1558
|
+
|
|
1559
|
+
|
|
1560
|
+
def one_vs_rest(
|
|
1561
|
+
adata,
|
|
1562
|
+
gene,
|
|
1563
|
+
curves_key="dsvg_curves",
|
|
1564
|
+
profiles_key="dsvg_profiles",
|
|
1565
|
+
null_quantile=0.9,
|
|
1566
|
+
inplace=False,
|
|
1567
|
+
key_added=None,
|
|
1568
|
+
tau=0.1,
|
|
1569
|
+
):
|
|
1570
|
+
"""Compare one gene's expression profile against every other gene's.
|
|
1571
|
+
|
|
1572
|
+
Each gene is reduced to a single curve along `anchor` (see
|
|
1573
|
+
`_gene_profiles`), and the probe gene is then tested against each of the
|
|
1574
|
+
remaining genes individually.
|
|
1575
|
+
|
|
1576
|
+
With a `fit_profiles` result (which stores each gene's posterior
|
|
1577
|
+
covariance of its spline coefficients) every pair gets the parallel-curves
|
|
1578
|
+
test of `parallel_curves_test`: do the two genes' curves differ in shape,
|
|
1579
|
+
beyond `tau`, rather than only in level? The covariance of the difference
|
|
1580
|
+
is the sum of the two genes' posterior covariances (their fits are
|
|
1581
|
+
independent). Without it (curves averaged from `fit_model`), the pairs
|
|
1582
|
+
are ranked by ``T`` against a Gamma null fitted to the "rest" genes.
|
|
1583
|
+
|
|
1584
|
+
Parameters
|
|
1585
|
+
----------
|
|
1586
|
+
adata
|
|
1587
|
+
Annotated data matrix that `fit_profiles` or `fit_model` wrote into.
|
|
1588
|
+
gene
|
|
1589
|
+
The probe gene, i.e. the "one" in one-vs-rest.
|
|
1590
|
+
curves_key, profiles_key
|
|
1591
|
+
`adata.varm` prefixes written by `fit_model` / `fit_profiles`. An
|
|
1592
|
+
explicit `fit_profiles` result wins when both are present.
|
|
1593
|
+
null_quantile
|
|
1594
|
+
Only without a `fit_profiles` result: fraction of the "rest" genes
|
|
1595
|
+
assumed to follow the null; the Gamma null is fitted to the lower
|
|
1596
|
+
`null_quantile` of their ``T`` values, as a truncated sample (see
|
|
1597
|
+
`_gamma_null_pvalues`).
|
|
1598
|
+
inplace
|
|
1599
|
+
If `True`, also write the table to ``adata.varm[key_added]``
|
|
1600
|
+
(reindexed to ``adata.var_names``, so the probe gene's own row is
|
|
1601
|
+
`NaN`). The table is returned either way.
|
|
1602
|
+
key_added
|
|
1603
|
+
`adata.varm` key used when `inplace`; defaults to
|
|
1604
|
+
``f"dsvg_ovr_{gene}"``.
|
|
1605
|
+
tau
|
|
1606
|
+
Smallest shape difference between two genes' curves that counts, as in
|
|
1607
|
+
`parallel_curves_test`.
|
|
1608
|
+
|
|
1609
|
+
Returns
|
|
1610
|
+
-------
|
|
1611
|
+
pandas.DataFrame
|
|
1612
|
+
Indexed by the "rest" genes, with ``delta`` (the signed area between
|
|
1613
|
+
the probe curve and that gene's, negative when the probe sits below
|
|
1614
|
+
it) and ``T``; with a `fit_profiles` result ``shape_diff``,
|
|
1615
|
+
``parallel_stat``, ``parallel_pval`` and ``parallel_pval_adj``,
|
|
1616
|
+
otherwise the Gamma-null ``gamma_pval`` and ``gamma_pval_adj``.
|
|
1617
|
+
"""
|
|
1618
|
+
profiles, source = _gene_profiles(adata, curves_key, profiles_key)
|
|
1619
|
+
if gene not in profiles.index:
|
|
1620
|
+
raise KeyError(f"{gene!r} not found in adata.var_names")
|
|
1621
|
+
|
|
1622
|
+
profiles = profiles.dropna(how="all")
|
|
1623
|
+
if gene not in profiles.index:
|
|
1624
|
+
raise KeyError(f"{gene!r} has no fitted curve - it was not part of the fitted gene set.")
|
|
1625
|
+
rest = profiles.drop(index=gene)
|
|
1626
|
+
if rest.empty:
|
|
1627
|
+
raise ValueError("need at least two genes with fitted curves for a one-vs-rest comparison.")
|
|
1628
|
+
|
|
1629
|
+
bin_centers = profiles.columns.to_numpy(dtype=float)
|
|
1630
|
+
# probe minus rest, so a downregulated probe gives a negative delta.
|
|
1631
|
+
delta = profiles.loc[gene].to_numpy()[None, :] - rest.to_numpy()
|
|
1632
|
+
|
|
1633
|
+
result = pd.DataFrame(
|
|
1634
|
+
{
|
|
1635
|
+
"delta": np.trapezoid(delta, x=bin_centers, axis=1),
|
|
1636
|
+
"T": (delta**2).mean(axis=1),
|
|
1637
|
+
},
|
|
1638
|
+
index=rest.index,
|
|
1639
|
+
)
|
|
1640
|
+
cov_key = f"{profiles_key}_beta_cov"
|
|
1641
|
+
if source == "profiles" and cov_key in adata.uns:
|
|
1642
|
+
stored = adata.uns[cov_key]
|
|
1643
|
+
position = {str(g): i for i, g in enumerate(stored["genes"])}
|
|
1644
|
+
covs = np.asarray(stored["cov"])
|
|
1645
|
+
betas = adata.varm[f"{profiles_key}_beta"]
|
|
1646
|
+
metric = shape_metric(covs.shape[1], int(stored["degree"]))
|
|
1647
|
+
rows = []
|
|
1648
|
+
for other in rest.index:
|
|
1649
|
+
d_beta = betas.loc[gene].to_numpy(dtype=float) - betas.loc[other].to_numpy(dtype=float)
|
|
1650
|
+
cov = covs[position[gene]] + covs[position[other]]
|
|
1651
|
+
rows.append(parallel_curves_test(d_beta, metric, tau=tau, cov=cov))
|
|
1652
|
+
shape_diff, stat, _, pval = (np.asarray(v) for v in zip(*rows, strict=True))
|
|
1653
|
+
result["shape_diff"], result["parallel_stat"], result["parallel_pval"] = shape_diff, stat, pval
|
|
1654
|
+
result["parallel_pval_adj"] = _benjamini_hochberg(pval)
|
|
1655
|
+
else:
|
|
1656
|
+
result["gamma_pval"] = _gamma_null_pvalues(result["T"].to_numpy(), null_quantile=null_quantile)
|
|
1657
|
+
result["gamma_pval_adj"] = _benjamini_hochberg(result["gamma_pval"].to_numpy())
|
|
1658
|
+
result.attrs["probe_gene"] = gene
|
|
1659
|
+
result.attrs["profile_source"] = source
|
|
1660
|
+
|
|
1661
|
+
if inplace:
|
|
1662
|
+
adata.varm[key_added or f"dsvg_ovr_{gene}"] = result.reindex(adata.var_names)
|
|
1663
|
+
return result
|
|
1664
|
+
|
|
1665
|
+
|
|
1666
|
+
# ============================================================
|
|
1667
|
+
# 4) Statistics helpers
|
|
1668
|
+
# ============================================================
|
|
1669
|
+
|
|
1670
|
+
|
|
1671
|
+
def pattern_wald_test(pred0, pred1, ridge=0.0):
|
|
1672
|
+
"""tradeSeq-like functional Wald test that two curves are equal, from posterior draws of the curves.
|
|
1673
|
+
|
|
1674
|
+
Pass draws of the *mean* curves (as `fit_model` does): posterior-predictive draws add observation
|
|
1675
|
+
noise to the covariance, which makes the test depend on how the data were binned.
|
|
1676
|
+
|
|
1677
|
+
Parameters
|
|
1678
|
+
----------
|
|
1679
|
+
pred0, pred1 : array, shape (n_draws, M)
|
|
1680
|
+
Posterior draws of the two condition curves.
|
|
1681
|
+
ridge : float
|
|
1682
|
+
Diagonal regularization of the covariance (relative to its largest eigenvalue).
|
|
1683
|
+
|
|
1684
|
+
Returns
|
|
1685
|
+
-------
|
|
1686
|
+
stat : float
|
|
1687
|
+
Wald statistic.
|
|
1688
|
+
pval : float
|
|
1689
|
+
Chi-square p-value.
|
|
1690
|
+
df : int
|
|
1691
|
+
Effective degrees of freedom (rank of the covariance).
|
|
1692
|
+
delta_mean : array
|
|
1693
|
+
Mean difference curve ``pred1 - pred0``.
|
|
1694
|
+
"""
|
|
1695
|
+
delta = pred1 - pred0 # (draws, M)
|
|
1696
|
+
mu = delta.mean(axis=0)
|
|
1697
|
+
|
|
1698
|
+
W, pval, df = _wald_from_moments(mu, np.cov(delta, rowvar=False), ridge=ridge)
|
|
1699
|
+
return W, pval, df, mu
|
|
1700
|
+
|
|
1701
|
+
|
|
1702
|
+
def _wald_from_moments(mu, Sigma, ridge=0.0, df_denominator=None):
|
|
1703
|
+
"""Wald statistic of ``mu`` against covariance `Sigma`, pseudoinverse over its non-null directions.
|
|
1704
|
+
|
|
1705
|
+
The cutoff is relative, so the degrees of freedom are the rank of the difference curve, not the
|
|
1706
|
+
number of grid points. With `df_denominator` (slides) the p-value is from ``F(df, df_denominator)``
|
|
1707
|
+
on ``stat / df``, as in `parallel_curves_test`. Returns ``(stat, pval, df)``.
|
|
1708
|
+
"""
|
|
1709
|
+
eigvals, eigvecs = np.linalg.eigh(Sigma)
|
|
1710
|
+
if ridge:
|
|
1711
|
+
eigvals = eigvals + ridge * eigvals.max()
|
|
1712
|
+
keep = eigvals > eigvals.max() * 1e-10
|
|
1713
|
+
z = eigvecs[:, keep].T @ mu
|
|
1714
|
+
W = float((z**2 / eigvals[keep]).sum())
|
|
1715
|
+
df = int(keep.sum())
|
|
1716
|
+
pval = float(chi2.sf(W, df)) if df_denominator is None else float(f.sf(W / df, df, df_denominator))
|
|
1717
|
+
return W, pval, df
|
|
1718
|
+
|
|
1719
|
+
|
|
1720
|
+
def _shape_grid(n_bsplines, degree, n_grid):
|
|
1721
|
+
"""Fine grid on ``[0, 1]``, its trapezoid weights (summing to 1) and the uncentred basis on it."""
|
|
1722
|
+
x = np.linspace(0.0, 1.0, n_grid)
|
|
1723
|
+
w = np.full(n_grid, 1.0 / (n_grid - 1))
|
|
1724
|
+
w[[0, -1]] /= 2
|
|
1725
|
+
return x, w, _bspline_basis_01(x, n_bsplines, degree)
|
|
1726
|
+
|
|
1727
|
+
|
|
1728
|
+
def shape_metric(n_bsplines=10, degree=3, n_grid=2001):
|
|
1729
|
+
"""Quadratic form `Q` that measures how far two condition curves are from parallel.
|
|
1730
|
+
|
|
1731
|
+
For interaction coefficients `b` (``beta_1`` of the per-gene model),
|
|
1732
|
+
``sqrt(b @ Q @ b)`` is the shape difference of the difference curve
|
|
1733
|
+
``d(x) = f1(x) - f0(x)`` on ``[0, 1]``, in slope-equivalent units:
|
|
1734
|
+
``sqrt(12) * RMS(d - mean(d))``. When the gap between the curves changes
|
|
1735
|
+
linearly, ``d(x) = a + s * x``, this is exactly ``|s|``, the slope
|
|
1736
|
+
difference; for any other shape it is the linear change with the same RMS
|
|
1737
|
+
departure from parallel. Level differences contribute nothing
|
|
1738
|
+
(``Q @ ones = 0``). Centring the basis does not change `Q`, so it depends
|
|
1739
|
+
only on the knots, i.e. on `n_bsplines` and `degree` of
|
|
1740
|
+
`diffsvg.pp.bin_obs`.
|
|
1741
|
+
|
|
1742
|
+
Parameters
|
|
1743
|
+
----------
|
|
1744
|
+
n_bsplines, degree
|
|
1745
|
+
As in `diffsvg.pp.bin_obs`.
|
|
1746
|
+
n_grid
|
|
1747
|
+
Grid points for the numerical integral.
|
|
1748
|
+
|
|
1749
|
+
Returns
|
|
1750
|
+
-------
|
|
1751
|
+
numpy.ndarray
|
|
1752
|
+
`Q`, shape ``(n_bsplines, n_bsplines)``, positive semi-definite with
|
|
1753
|
+
rank ``n_bsplines - 1``.
|
|
1754
|
+
"""
|
|
1755
|
+
_, w, B = _shape_grid(n_bsplines, degree, n_grid)
|
|
1756
|
+
centred = B - w @ B
|
|
1757
|
+
return 12.0 * (centred * w[:, None]).T @ centred
|
|
1758
|
+
|
|
1759
|
+
|
|
1760
|
+
def _slope_weights(n_bsplines, degree, n_grid=2001):
|
|
1761
|
+
"""`g` with ``g @ b`` = the least-squares slope over ``[0, 1]`` of the difference curve for coefficients `b`."""
|
|
1762
|
+
x, w, B = _shape_grid(n_bsplines, degree, n_grid)
|
|
1763
|
+
return 12.0 * ((x - 0.5) * w) @ B
|
|
1764
|
+
|
|
1765
|
+
|
|
1766
|
+
def parallel_curves_test(beta_1, metric, tau=0.1, cov=None, df_denominator=None):
|
|
1767
|
+
"""Test whether two condition curves are parallel, up to a shape difference `tau`.
|
|
1768
|
+
|
|
1769
|
+
The curves are parallel when they differ only in level, i.e. when their
|
|
1770
|
+
difference ``d(x) = f1(x) - f0(x)`` is constant. How far they are from
|
|
1771
|
+
that is ``shape_diff = sqrt(beta_1 @ metric @ beta_1)`` (see
|
|
1772
|
+
`shape_metric`; for a gap that changes linearly it is the slope
|
|
1773
|
+
difference), and the test is ``H0: shape_diff <= tau`` against
|
|
1774
|
+
``H1: shape_diff > tau``.
|
|
1775
|
+
|
|
1776
|
+
The statistic is the Wald statistic of the ``K - 1`` shape directions of
|
|
1777
|
+
`beta_1` (the constant direction is the level and is left out). With
|
|
1778
|
+
``tau = 0`` it is the classic test of parallel curves, chi-square with
|
|
1779
|
+
``K - 1`` degrees of freedom. With ``tau > 0`` it is compared against the
|
|
1780
|
+
largest distribution it can have when the true shape difference is
|
|
1781
|
+
`tau`: noncentral chi-square with noncentrality ``tau**2`` over the
|
|
1782
|
+
variance of the best-measured shape direction. Shape differences below
|
|
1783
|
+
`tau` therefore do not become significant however precisely they are
|
|
1784
|
+
measured, while the p-value stays valid for every difference up to `tau`.
|
|
1785
|
+
|
|
1786
|
+
With `df_denominator` (replicates: the posterior covariance then holds
|
|
1787
|
+
the variation between slides, which was estimated from only a few) the
|
|
1788
|
+
statistic over its ``K - 1`` degrees of freedom is compared against an F
|
|
1789
|
+
distribution instead (noncentral for ``tau > 0``, with the same
|
|
1790
|
+
noncentrality), which is wider than the chi-square and so keeps the test
|
|
1791
|
+
honest when there are few slides.
|
|
1792
|
+
|
|
1793
|
+
Parameters
|
|
1794
|
+
----------
|
|
1795
|
+
beta_1 : array, shape (n_draws, K) or (K,)
|
|
1796
|
+
Posterior draws of the interaction coefficients, or their posterior
|
|
1797
|
+
mean (then `cov` is required).
|
|
1798
|
+
metric : array, shape (K, K)
|
|
1799
|
+
`shape_metric` for the basis `beta_1` belongs to.
|
|
1800
|
+
tau : float
|
|
1801
|
+
Smallest shape difference that counts, in units of ``shape_diff``.
|
|
1802
|
+
cov : array, shape (K, K), optional
|
|
1803
|
+
Posterior covariance of `beta_1`, when `beta_1` is a mean.
|
|
1804
|
+
df_denominator : int, optional
|
|
1805
|
+
Denominator degrees of freedom of the F reference, e.g. the number
|
|
1806
|
+
of slides minus 2 when comparing two groups of slides.
|
|
1807
|
+
|
|
1808
|
+
Returns
|
|
1809
|
+
-------
|
|
1810
|
+
shape_diff : float
|
|
1811
|
+
Shape difference of the posterior-mean curves. Noise in the fitted
|
|
1812
|
+
curves adds to it, so poorly measured genes show a sizeable
|
|
1813
|
+
``shape_diff`` even when the true curves are parallel; the p-value
|
|
1814
|
+
accounts for that, the raw number does not.
|
|
1815
|
+
stat : float
|
|
1816
|
+
Wald statistic.
|
|
1817
|
+
df : int
|
|
1818
|
+
Degrees of freedom, ``K - 1``.
|
|
1819
|
+
pval : float
|
|
1820
|
+
"""
|
|
1821
|
+
beta_1 = np.asarray(beta_1, dtype=float)
|
|
1822
|
+
if beta_1.ndim == 2:
|
|
1823
|
+
mean, cov = beta_1.mean(axis=0), np.cov(beta_1, rowvar=False)
|
|
1824
|
+
elif cov is None:
|
|
1825
|
+
raise ValueError("pass `cov` along with a posterior mean, or pass the posterior draws.")
|
|
1826
|
+
else:
|
|
1827
|
+
mean, cov = beta_1, np.asarray(cov, dtype=float)
|
|
1828
|
+
if tau < 0:
|
|
1829
|
+
raise ValueError(f"`tau` must be >= 0, got {tau}.")
|
|
1830
|
+
|
|
1831
|
+
# Q = L L' over its K - 1 non-zero directions, so z = L' beta_1 has |z| = shape_diff.
|
|
1832
|
+
evals, evecs = np.linalg.eigh(np.asarray(metric, dtype=float))
|
|
1833
|
+
keep = evals > evals.max() * 1e-10
|
|
1834
|
+
L = evecs[:, keep] * np.sqrt(evals[keep])
|
|
1835
|
+
z = L.T @ mean
|
|
1836
|
+
omega = L.T @ cov @ L
|
|
1837
|
+
omega_min = np.linalg.eigvalsh(omega)[0]
|
|
1838
|
+
if omega_min <= 0:
|
|
1839
|
+
raise ValueError("the covariance of `beta_1` is singular in the shape directions.")
|
|
1840
|
+
|
|
1841
|
+
stat = float(z @ np.linalg.solve(omega, z))
|
|
1842
|
+
df = int(keep.sum())
|
|
1843
|
+
noncentrality = tau**2 / omega_min
|
|
1844
|
+
if df_denominator is None:
|
|
1845
|
+
pval = float(ncx2.sf(stat, df, noncentrality)) if tau > 0 else float(chi2.sf(stat, df))
|
|
1846
|
+
elif df_denominator < 1:
|
|
1847
|
+
raise ValueError(f"`df_denominator` must be >= 1, got {df_denominator}.")
|
|
1848
|
+
elif tau > 0:
|
|
1849
|
+
pval = float(ncf.sf(stat / df, df, df_denominator, noncentrality))
|
|
1850
|
+
else:
|
|
1851
|
+
pval = float(f.sf(stat / df, df, df_denominator))
|
|
1852
|
+
return float(np.sqrt(z @ z)), stat, df, pval
|
|
1853
|
+
|
|
1854
|
+
|
|
1855
|
+
def compute_T(curve0, curve1):
|
|
1856
|
+
"""Functional L2 distance between two curves."""
|
|
1857
|
+
return np.mean((curve1 - curve0) ** 2)
|