diffsvg 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffsvg/__init__.py +3 -0
- diffsvg/pl/__init__.py +5 -0
- diffsvg/pl/plot_clusters.py +216 -0
- diffsvg/pl/plot_results.py +484 -0
- diffsvg/pl/plot_stability.py +184 -0
- diffsvg/pp/__init__.py +10 -0
- diffsvg/pp/utils.py +620 -0
- diffsvg/py.typed +0 -0
- diffsvg/tl/__init__.py +24 -0
- diffsvg/tl/_exact.py +332 -0
- diffsvg/tl/clustering.py +154 -0
- diffsvg/tl/model.py +1857 -0
- diffsvg/tl/stability.py +335 -0
- diffsvg-0.1.0.dist-info/METADATA +132 -0
- diffsvg-0.1.0.dist-info/RECORD +17 -0
- diffsvg-0.1.0.dist-info/WHEEL +4 -0
- diffsvg-0.1.0.dist-info/licenses/LICENSE +29 -0
diffsvg/pp/utils.py
ADDED
|
@@ -0,0 +1,620 @@
|
|
|
1
|
+
"""Helpers for turning per-cell expression into per-bin summaries used by the GAMs in ``diffsvg.tl.model``."""
|
|
2
|
+
|
|
3
|
+
import warnings
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
import pandas as pd
|
|
8
|
+
import patsy
|
|
9
|
+
from scipy.sparse import csr_matrix, issparse
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _expression(adata, genes, layer):
|
|
13
|
+
"""Dense float ``(n_obs, n_genes)`` expression of `genes`, from ``adata.layers[layer]`` or ``adata.X``."""
|
|
14
|
+
sub = adata[:, genes]
|
|
15
|
+
X = sub.layers[layer] if layer is not None else sub.X
|
|
16
|
+
X = X.toarray() if issparse(X) else np.asarray(X)
|
|
17
|
+
return np.asarray(X, dtype=float)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _nearest_bin(values, n_bins):
|
|
21
|
+
"""Index of the nearest of the ``n_bins + 1`` grid centres for every value; ``-1`` where it is missing."""
|
|
22
|
+
values = np.asarray(values, dtype=float)
|
|
23
|
+
index = np.full(values.shape, -1, dtype=int)
|
|
24
|
+
present = np.isfinite(values)
|
|
25
|
+
index[present] = np.clip(np.rint(values[present] * n_bins).astype(int), 0, n_bins)
|
|
26
|
+
return index
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def bin_counts(
|
|
30
|
+
adata,
|
|
31
|
+
genes,
|
|
32
|
+
anchor,
|
|
33
|
+
category,
|
|
34
|
+
design_matrix="design_matrix",
|
|
35
|
+
n_bins=50,
|
|
36
|
+
window_radius=None,
|
|
37
|
+
layer=None,
|
|
38
|
+
drop_empty_bins=True,
|
|
39
|
+
):
|
|
40
|
+
"""Average expression of one or more genes in windows along `anchor`, per `category` level.
|
|
41
|
+
|
|
42
|
+
For a grid of `n_bins` + 1 centers on ``[0, 1]``, all cells within `window_radius` of a center are
|
|
43
|
+
averaged, separately for every level of `category`. The grid does not depend on the gene, so all
|
|
44
|
+
`genes` are binned at once.
|
|
45
|
+
|
|
46
|
+
Parameters
|
|
47
|
+
----------
|
|
48
|
+
adata
|
|
49
|
+
Annotated data matrix.
|
|
50
|
+
genes
|
|
51
|
+
Gene name(s) to bin.
|
|
52
|
+
anchor
|
|
53
|
+
Column in `design_matrix` holding the ordering coordinate, in ``[0, 1]``.
|
|
54
|
+
category
|
|
55
|
+
Column in `design_matrix` to stratify by (e.g. the condition), also the column name in the output.
|
|
56
|
+
`None` bins along `anchor` only, as a single group.
|
|
57
|
+
design_matrix
|
|
58
|
+
``adata.obsm`` key of a data frame holding `anchor` and `category`.
|
|
59
|
+
n_bins
|
|
60
|
+
Number of bins spanning ``[0, 1]``; the grid has `n_bins` + 1 centers.
|
|
61
|
+
window_radius
|
|
62
|
+
Half-width of the averaging window. Windows wider than ``0.5 / n_bins`` overlap. `None` (default) makes
|
|
63
|
+
them contiguous: each cell counts once, in the bin of its nearest center.
|
|
64
|
+
layer
|
|
65
|
+
If given, bin ``adata.layers[layer]`` instead of ``adata.X``.
|
|
66
|
+
drop_empty_bins
|
|
67
|
+
Drop ``(category, bin)`` rows without cells (the same rows for every gene).
|
|
68
|
+
|
|
69
|
+
Returns
|
|
70
|
+
-------
|
|
71
|
+
dict[str, pandas.DataFrame]
|
|
72
|
+
One data frame per gene with columns ``[category, anchor,
|
|
73
|
+
"window_start", "window_end", "n_obs", "avg_var"]``, plus ``"ss_var"``
|
|
74
|
+
(the within-bin sum of squared deviations from ``avg_var``) when
|
|
75
|
+
``window_radius=None``.
|
|
76
|
+
"""
|
|
77
|
+
design_matrix = adata.obsm[design_matrix]
|
|
78
|
+
|
|
79
|
+
genes = [genes] if isinstance(genes, str) else list(genes)
|
|
80
|
+
sub = adata[:, genes]
|
|
81
|
+
X = sub.layers[layer] if layer is not None else sub.X
|
|
82
|
+
X = csr_matrix(X, dtype=float) if issparse(X) else np.asarray(X, dtype=float)
|
|
83
|
+
|
|
84
|
+
value = np.asarray(design_matrix[anchor].values, dtype=float)
|
|
85
|
+
# `category=None` bins along `anchor` alone, as one group.
|
|
86
|
+
cat = np.zeros(len(value)) if category is None else np.asarray(design_matrix[category].values)
|
|
87
|
+
|
|
88
|
+
bin_centers = np.linspace(0, 1, n_bins + 1)
|
|
89
|
+
categories = np.unique(cat[~pd.isna(cat)]) # cells without a category (e.g. a slide without anchors) are left out
|
|
90
|
+
contiguous = window_radius is None
|
|
91
|
+
n_grid = n_bins + 1
|
|
92
|
+
|
|
93
|
+
# One sparse indicator matrix (rows = (category, bin), columns = cells) replaces a pass over all cells per
|
|
94
|
+
# row: row means are M @ X / n and within-row sums of squares M @ X^2 - n * mean^2.
|
|
95
|
+
cat_code = np.full(len(value), -1)
|
|
96
|
+
for k, c in enumerate(categories):
|
|
97
|
+
cat_code[cat == c] = k
|
|
98
|
+
present = (cat_code >= 0) & np.isfinite(value)
|
|
99
|
+
if contiguous:
|
|
100
|
+
nearest = _nearest_bin(value, n_bins)
|
|
101
|
+
cells = np.flatnonzero(present)
|
|
102
|
+
rows = cat_code[cells] * n_grid + nearest[cells]
|
|
103
|
+
lower = np.maximum(0.0, bin_centers - 0.5 / n_bins)
|
|
104
|
+
upper = np.minimum(1.0, bin_centers + 0.5 / n_bins)
|
|
105
|
+
else:
|
|
106
|
+
lower = np.maximum(0.0, bin_centers - window_radius)
|
|
107
|
+
upper = np.minimum(1.0, bin_centers + window_radius)
|
|
108
|
+
row_parts, cell_parts = [], []
|
|
109
|
+
for k in range(len(categories)):
|
|
110
|
+
idx = np.flatnonzero(present & (cat_code == k))
|
|
111
|
+
order = idx[np.argsort(value[idx])]
|
|
112
|
+
sorted_vals = value[order]
|
|
113
|
+
starts = np.searchsorted(sorted_vals, lower, side="left")
|
|
114
|
+
ends = np.searchsorted(sorted_vals, upper, side="right")
|
|
115
|
+
for i in range(n_grid):
|
|
116
|
+
members = order[starts[i] : ends[i]]
|
|
117
|
+
row_parts.append(np.full(len(members), k * n_grid + i))
|
|
118
|
+
cell_parts.append(members)
|
|
119
|
+
rows = np.concatenate(row_parts) if row_parts else np.zeros(0, dtype=int)
|
|
120
|
+
cells = np.concatenate(cell_parts) if cell_parts else np.zeros(0, dtype=int)
|
|
121
|
+
M = csr_matrix((np.ones(len(rows)), (rows, cells)), shape=(len(categories) * n_grid, len(value)))
|
|
122
|
+
n_obs = np.asarray(M.sum(axis=1)).ravel()
|
|
123
|
+
sums = M @ X
|
|
124
|
+
sums = sums.toarray() if issparse(sums) else np.asarray(sums)
|
|
125
|
+
with np.errstate(invalid="ignore", divide="ignore"):
|
|
126
|
+
avg = sums / n_obs[:, None]
|
|
127
|
+
avg[n_obs == 0] = np.nan
|
|
128
|
+
ss = None
|
|
129
|
+
if contiguous:
|
|
130
|
+
sq = M @ (X.multiply(X) if issparse(X) else X**2)
|
|
131
|
+
sq = sq.toarray() if issparse(sq) else np.asarray(sq)
|
|
132
|
+
ss = np.clip(sq - n_obs[:, None] * np.nan_to_num(avg) ** 2, 0.0, None)
|
|
133
|
+
ss[n_obs == 0] = np.nan
|
|
134
|
+
|
|
135
|
+
meta_cols = ["_group" if category is None else category, anchor, "window_start", "window_end", "n_obs"]
|
|
136
|
+
meta = pd.DataFrame(
|
|
137
|
+
{
|
|
138
|
+
meta_cols[0]: np.repeat(categories, n_grid),
|
|
139
|
+
anchor: np.tile(bin_centers, len(categories)),
|
|
140
|
+
"window_start": np.tile(lower, len(categories)),
|
|
141
|
+
"window_end": np.tile(upper, len(categories)),
|
|
142
|
+
"n_obs": n_obs.astype(int),
|
|
143
|
+
}
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
if drop_empty_bins:
|
|
147
|
+
keep = meta["n_obs"].to_numpy() > 0
|
|
148
|
+
meta = meta.loc[keep].reset_index(drop=True)
|
|
149
|
+
avg = avg[keep]
|
|
150
|
+
ss = ss[keep] if contiguous else None
|
|
151
|
+
|
|
152
|
+
if contiguous:
|
|
153
|
+
return {gene: meta.assign(avg_var=avg[:, i], ss_var=ss[:, i]) for i, gene in enumerate(genes)}
|
|
154
|
+
return {gene: meta.assign(avg_var=avg[:, i]) for i, gene in enumerate(genes)}
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def compute_interior_knots(n_bsplines, degree=3):
|
|
158
|
+
"""Interior knot positions (plus the ``0``/``1`` endpoints) for a B-spline basis on ``[0, 1]``.
|
|
159
|
+
|
|
160
|
+
For `patsy`'s ``bs(x, knots=interior_knots, degree=degree, include_intercept=True)``
|
|
161
|
+
the number of resulting basis columns is ``len(interior_knots) + degree +
|
|
162
|
+
1``, so `n_bsplines` interior knots are chosen accordingly.
|
|
163
|
+
|
|
164
|
+
Parameters
|
|
165
|
+
----------
|
|
166
|
+
n_bsplines
|
|
167
|
+
Desired number of B-spline basis functions.
|
|
168
|
+
degree
|
|
169
|
+
Degree of the B-spline (3 = cubic).
|
|
170
|
+
|
|
171
|
+
Returns
|
|
172
|
+
-------
|
|
173
|
+
numpy.ndarray
|
|
174
|
+
Knot grid including the ``0`` and ``1`` endpoints, i.e. the interior
|
|
175
|
+
knots to pass to `patsy` are ``result[1:-1]``.
|
|
176
|
+
"""
|
|
177
|
+
n_interior = n_bsplines - degree - 1
|
|
178
|
+
if n_interior < 0:
|
|
179
|
+
raise ValueError(f"n_bsplines must be >= degree + 1 ({degree + 1}); got {n_bsplines}")
|
|
180
|
+
return np.linspace(0, 1, n_interior + 2)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def build_spline_basis(x, interior_knots, degree=3):
|
|
184
|
+
"""Build a (centered, intercept-free) B-spline design matrix for `x`.
|
|
185
|
+
|
|
186
|
+
Parameters
|
|
187
|
+
----------
|
|
188
|
+
x
|
|
189
|
+
1D array of coordinates in ``[0, 1]``.
|
|
190
|
+
interior_knots
|
|
191
|
+
Interior knots, e.g. ``compute_interior_knots(n_bsplines, degree)[1:-1]``.
|
|
192
|
+
degree
|
|
193
|
+
Degree of the B-spline (3 = cubic).
|
|
194
|
+
|
|
195
|
+
Returns
|
|
196
|
+
-------
|
|
197
|
+
B : numpy.ndarray
|
|
198
|
+
Design matrix of shape ``(len(x), len(interior_knots) + degree + 1)``.
|
|
199
|
+
design_info : patsy.design_info.DesignInfo
|
|
200
|
+
Design info, needed to evaluate the same basis at new `x` values via
|
|
201
|
+
``patsy.build_design_matrices([design_info], {"x": new_x})``.
|
|
202
|
+
"""
|
|
203
|
+
dm = patsy.dmatrix(
|
|
204
|
+
f"center(bs(x, knots=interior_knots, degree={degree}, include_intercept=True, lower_bound=0, upper_bound=1)) - 1",
|
|
205
|
+
{"x": np.asarray(x), "interior_knots": np.asarray(interior_knots)},
|
|
206
|
+
)
|
|
207
|
+
return np.asarray(dm), dm.design_info
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _bspline_basis_01(x, n_bsplines, degree):
|
|
211
|
+
"""Uncentred B-spline basis with `bin_obs`'s knots, boundary knots pinned at ``0``/``1``.
|
|
212
|
+
|
|
213
|
+
`build_spline_basis` pins the boundary knots at ``0`` and ``1`` too, so this reproduces
|
|
214
|
+
`bin_obs`'s basis for any `x`.
|
|
215
|
+
"""
|
|
216
|
+
interior_knots = compute_interior_knots(n_bsplines, degree=degree)[1:-1]
|
|
217
|
+
dm = patsy.dmatrix(
|
|
218
|
+
f"bs(x, knots=interior_knots, degree={degree}, include_intercept=True, lower_bound=0, upper_bound=1) - 1",
|
|
219
|
+
{"x": np.asarray(x, dtype=float), "interior_knots": interior_knots},
|
|
220
|
+
)
|
|
221
|
+
return np.asarray(dm)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def spline_basis_at(binned, x):
|
|
225
|
+
"""Evaluate the spline basis of `binned` at new `anchor` values.
|
|
226
|
+
|
|
227
|
+
Same knots, boundary knots and centring as ``binned.B``/``binned.B_pred``,
|
|
228
|
+
so coefficients fitted on this basis are valid on the prediction grid and
|
|
229
|
+
vice versa. Used to fit on single cells (``use_binned=False``).
|
|
230
|
+
|
|
231
|
+
Parameters
|
|
232
|
+
----------
|
|
233
|
+
binned
|
|
234
|
+
A `BinnedData`, i.e. the output of `bin_obs`.
|
|
235
|
+
x
|
|
236
|
+
Anchor values in ``[0, 1]``, without missing values.
|
|
237
|
+
|
|
238
|
+
Returns
|
|
239
|
+
-------
|
|
240
|
+
numpy.ndarray
|
|
241
|
+
Design matrix of shape ``(len(x), binned.n_bsplines)``.
|
|
242
|
+
"""
|
|
243
|
+
x = np.asarray(x, dtype=float)
|
|
244
|
+
if not np.isfinite(x).all():
|
|
245
|
+
raise ValueError("`x` holds missing or infinite anchor values - drop those cells first.")
|
|
246
|
+
if ((x < 0) | (x > 1)).any():
|
|
247
|
+
raise ValueError(f"anchor values must lie in [0, 1], got range [{x.min()}, {x.max()}].")
|
|
248
|
+
|
|
249
|
+
# `binned.B_pred` is the centred basis on the grid, so the centring `bin_obs`
|
|
250
|
+
# applied is the (row-constant) gap between the uncentred and centred grid basis.
|
|
251
|
+
offsets = _bspline_basis_01(binned.bin_centers, binned.n_bsplines, binned.degree) - np.asarray(binned.B_pred)
|
|
252
|
+
centre = offsets.mean(axis=0)
|
|
253
|
+
if not np.allclose(offsets, centre, atol=1e-8):
|
|
254
|
+
raise ValueError("could not reproduce the spline basis of `binned` - was it built by `bin_obs`?")
|
|
255
|
+
return _bspline_basis_01(x, binned.n_bsplines, binned.degree) - centre
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
@dataclass
|
|
259
|
+
class BinnedData:
|
|
260
|
+
"""Output of `bin_obs`, consumed by `diffsvg.tl.setup_model`.
|
|
261
|
+
|
|
262
|
+
Everything a per-gene GAM needs that does not depend on the gene, plus the binned targets `Y`.
|
|
263
|
+
|
|
264
|
+
Attributes
|
|
265
|
+
----------
|
|
266
|
+
genes
|
|
267
|
+
Gene order matching `Y`/`intercept_initvals`.
|
|
268
|
+
Y
|
|
269
|
+
Per-gene binned expression.
|
|
270
|
+
intercept_initvals
|
|
271
|
+
Per-gene initial value of `intercept` (mean binned expression at ``anchor == 0``).
|
|
272
|
+
anchor, condition
|
|
273
|
+
Columns of the design matrix the data was binned on. `condition` is `None` when binned without
|
|
274
|
+
one; every condition-dependent field is then `None` too.
|
|
275
|
+
condition_categories
|
|
276
|
+
``{"0": <label>, "1": <label>}``, how `condition` was encoded (string keys for h5ad).
|
|
277
|
+
n_bsplines, degree, n_bins
|
|
278
|
+
Binning/basis hyperparameters.
|
|
279
|
+
B, B_masked, condition_fit
|
|
280
|
+
Spline design matrices and encoded condition of the fitted rows.
|
|
281
|
+
bin_centers
|
|
282
|
+
Grid (`n_bins` + 1 points on ``[0, 1]``), also the prediction grid.
|
|
283
|
+
bin_assignment
|
|
284
|
+
Per cell, the index of its nearest bin center, ``-1`` without an anchor value.
|
|
285
|
+
design_matrix, layer
|
|
286
|
+
Where the unbinned data lives, for ``use_binned=False``.
|
|
287
|
+
B_pred, B_pred_masked_0, B_pred_masked_1, condition_pred_0, condition_pred_1
|
|
288
|
+
Design matrices for the prediction grid.
|
|
289
|
+
sample, sample_labels, sample_index
|
|
290
|
+
Only when binned with `sample`: the column, the samples that have rows, and per row its sample's
|
|
291
|
+
index. Rows are then one per (sample, bin).
|
|
292
|
+
n_obs
|
|
293
|
+
Cells per row. The fitters weight rows by it.
|
|
294
|
+
n_obs_grid, ss_within
|
|
295
|
+
Only for contiguous bins (``window_radius=None``): cells per grid point
|
|
296
|
+
(``n_categories x (n_bins + 1)``) and per gene the within-bin sum of squares, so a binned fit
|
|
297
|
+
matches a fit on the cells.
|
|
298
|
+
"""
|
|
299
|
+
|
|
300
|
+
genes: list
|
|
301
|
+
Y: dict
|
|
302
|
+
intercept_initvals: dict
|
|
303
|
+
anchor: str
|
|
304
|
+
condition: str | None
|
|
305
|
+
condition_categories: dict | None
|
|
306
|
+
n_bsplines: int
|
|
307
|
+
degree: int
|
|
308
|
+
n_bins: int
|
|
309
|
+
B: np.ndarray
|
|
310
|
+
B_masked: np.ndarray | None
|
|
311
|
+
condition_fit: np.ndarray | None
|
|
312
|
+
bin_centers: np.ndarray
|
|
313
|
+
bin_assignment: np.ndarray
|
|
314
|
+
B_pred: np.ndarray
|
|
315
|
+
B_pred_masked_0: np.ndarray | None
|
|
316
|
+
B_pred_masked_1: np.ndarray | None
|
|
317
|
+
condition_pred_0: np.ndarray | None
|
|
318
|
+
condition_pred_1: np.ndarray | None
|
|
319
|
+
# Defaulted so that a `binned` dict written before these existed still reconstructs.
|
|
320
|
+
design_matrix: str = "design_matrix"
|
|
321
|
+
layer: str | None = None
|
|
322
|
+
n_obs: np.ndarray | None = None
|
|
323
|
+
n_obs_grid: np.ndarray | None = None
|
|
324
|
+
ss_within: dict | None = None
|
|
325
|
+
sample: str | None = None
|
|
326
|
+
sample_labels: list | None = None
|
|
327
|
+
sample_index: np.ndarray | None = None
|
|
328
|
+
|
|
329
|
+
@property
|
|
330
|
+
def n_samples(self):
|
|
331
|
+
"""Number of samples that have rows, or `None` when binned without `sample`."""
|
|
332
|
+
return None if self.sample_labels is None else len(self.sample_labels)
|
|
333
|
+
|
|
334
|
+
@property
|
|
335
|
+
def has_condition(self):
|
|
336
|
+
"""Whether this was binned with a `condition`, i.e. whether the two-condition models can be fit."""
|
|
337
|
+
return self.condition is not None
|
|
338
|
+
|
|
339
|
+
@property
|
|
340
|
+
def weighted(self):
|
|
341
|
+
"""Whether the bins are contiguous and record the within-bin spread.
|
|
342
|
+
|
|
343
|
+
Then a binned fit carries the same information as a fit on the cells. (Overlapping bins record
|
|
344
|
+
`n_obs` too, and fits weight rows by it, but their rows share cells.)
|
|
345
|
+
"""
|
|
346
|
+
return self.ss_within is not None
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _warn_possible_replicates(design_matrix_df, anchor, condition):
|
|
350
|
+
"""Warn when a design-matrix column looks like slides (nested in `condition`, >= 2 per level) but is not used.
|
|
351
|
+
|
|
352
|
+
Without `sample`, every cell counts as an independent replicate. Differences between slides then look like
|
|
353
|
+
differences between conditions; on real multi-slide data this produces many false positives.
|
|
354
|
+
"""
|
|
355
|
+
cond = design_matrix_df[condition]
|
|
356
|
+
for col in design_matrix_df.columns:
|
|
357
|
+
if col in (anchor, condition):
|
|
358
|
+
continue
|
|
359
|
+
values = design_matrix_df[col]
|
|
360
|
+
if pd.api.types.is_float_dtype(values) or values.nunique(dropna=True) > 1000:
|
|
361
|
+
continue
|
|
362
|
+
pairs = pd.DataFrame({"s": values, "c": cond}).dropna().drop_duplicates()
|
|
363
|
+
nested = not pairs["s"].duplicated().any()
|
|
364
|
+
per_condition = pairs.groupby("c", observed=True)["s"].nunique()
|
|
365
|
+
if nested and len(per_condition) == 2 and per_condition.min() >= 2:
|
|
366
|
+
warnings.warn(
|
|
367
|
+
f"Column {col!r} looks like replicates (slides/samples) nested in {condition!r} "
|
|
368
|
+
f"({per_condition.min()}+ per condition), but `sample` was not given, so every cell is treated as "
|
|
369
|
+
f"an independent replicate and slide-to-slide variation counts as a condition effect. Pass "
|
|
370
|
+
f"`sample={col!r}` to model it.",
|
|
371
|
+
UserWarning,
|
|
372
|
+
stacklevel=3,
|
|
373
|
+
)
|
|
374
|
+
return
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _condition_levels(values):
|
|
378
|
+
"""The levels of a condition column, in the order that fixes the 0/1 coding.
|
|
379
|
+
|
|
380
|
+
A categorical column keeps its category order (only categories that occur), anything else is sorted
|
|
381
|
+
as strings. Missing values are left out, as are the cells that hold them.
|
|
382
|
+
"""
|
|
383
|
+
values = pd.Series(values)
|
|
384
|
+
if isinstance(values.dtype, pd.CategoricalDtype):
|
|
385
|
+
observed = set(values.dropna().unique())
|
|
386
|
+
return [c for c in values.cat.categories if c in observed]
|
|
387
|
+
return sorted(pd.unique(values.dropna()), key=str)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def bin_obs(
|
|
391
|
+
adata,
|
|
392
|
+
anchor,
|
|
393
|
+
condition,
|
|
394
|
+
genes=None,
|
|
395
|
+
n_bsplines=10,
|
|
396
|
+
degree=3,
|
|
397
|
+
n_bins=50,
|
|
398
|
+
window_radius=None,
|
|
399
|
+
layer=None,
|
|
400
|
+
design_matrix="design_matrix",
|
|
401
|
+
inplace=True,
|
|
402
|
+
sample=None,
|
|
403
|
+
):
|
|
404
|
+
"""Assign each cell a bin along `anchor`, bin every gene's expression, and precompute the spline design matrices `diffsvg.tl.setup_model` needs.
|
|
405
|
+
|
|
406
|
+
This is the preprocessing step: it only bins/reshapes data (via
|
|
407
|
+
`bin_counts`) and builds the (gene-independent) B-spline design
|
|
408
|
+
matrices. It does not touch PyMC at all - `diffsvg.tl.setup_model` reads
|
|
409
|
+
its output back off `adata` and defines the actual model.
|
|
410
|
+
|
|
411
|
+
Parameters
|
|
412
|
+
----------
|
|
413
|
+
adata
|
|
414
|
+
Annotated data matrix. ``adata.obsm["design_matrix"]`` must contain
|
|
415
|
+
`anchor` (assumed to lie in ``[0, 1]``) and `condition` (must have
|
|
416
|
+
exactly two levels).
|
|
417
|
+
anchor
|
|
418
|
+
Column in ``adata.obsm["design_matrix"]`` giving the ordering
|
|
419
|
+
coordinate (e.g. pseudotime, spatial distance) that the spline basis
|
|
420
|
+
is built on.
|
|
421
|
+
condition
|
|
422
|
+
Column in ``adata.obsm["design_matrix"]`` giving the two groups to
|
|
423
|
+
compare (e.g. control vs. treatment). Pass `None` for data without
|
|
424
|
+
conditions: binning then runs along `anchor` only, and the result
|
|
425
|
+
supports the condition-free models used by
|
|
426
|
+
`diffsvg.tl.fit_profiles`/`diffsvg.tl.one_vs_rest`.
|
|
427
|
+
genes
|
|
428
|
+
Genes to model. Defaults to all of ``adata.var_names``.
|
|
429
|
+
n_bsplines
|
|
430
|
+
Number of B-spline basis functions used for both the `anchor` effect
|
|
431
|
+
and its `condition`-interaction.
|
|
432
|
+
degree
|
|
433
|
+
Degree of the B-spline basis (3 = cubic).
|
|
434
|
+
n_bins
|
|
435
|
+
Number of bins spanning `anchor` in ``[0, 1]``; the fitting/prediction
|
|
436
|
+
grid has `n_bins` + 1 points. Every cell is also assigned to its
|
|
437
|
+
nearest of these `n_bins` + 1 bin centers (see `bin_assignment`
|
|
438
|
+
below).
|
|
439
|
+
window_radius
|
|
440
|
+
Half-width of the averaging window used by `bin_counts` (independent
|
|
441
|
+
of, and typically wider than, half a bin - a cell can contribute to
|
|
442
|
+
several neighboring bins' averages even though it is only ever
|
|
443
|
+
*assigned* to one). Overlapping windows make neighbouring bins share
|
|
444
|
+
cells, which the models treat as independent, so their uncertainty
|
|
445
|
+
comes out too small. `None` bins every cell once, in its assigned
|
|
446
|
+
bin, and records the cell counts and within-bin spread; the fitters
|
|
447
|
+
then weight each bin by its cell count, so a binned fit matches a fit
|
|
448
|
+
on the cells (needed for calibrated ``test="parallel"`` p-values).
|
|
449
|
+
`None` is the default. With overlapping windows the rows are still
|
|
450
|
+
weighted by how many cells they average (so a sparse edge window does
|
|
451
|
+
not count as much as a full one), but neighbouring rows stay
|
|
452
|
+
correlated and the p-values too small.
|
|
453
|
+
layer
|
|
454
|
+
If given, use ``adata.layers[layer]`` instead of ``adata.X``.
|
|
455
|
+
design_matrix
|
|
456
|
+
``adata.obsm`` key of the design matrix. Defaults to
|
|
457
|
+
``"design_matrix"``.
|
|
458
|
+
sample
|
|
459
|
+
Column in ``adata.obsm["design_matrix"]`` naming the slide (replicate) of every cell, for data with
|
|
460
|
+
several slides per `condition`. Each slide must belong to one `condition`. Bins are then built per slide
|
|
461
|
+
(one row per slide and bin, contiguous: needs ``window_radius=None``), and the fitters add a random
|
|
462
|
+
level and a random curve per slide, so that the condition curves are compared against the variation
|
|
463
|
+
between slides rather than between cells (see ``diffsvg.tl.fit_model``).
|
|
464
|
+
inplace
|
|
465
|
+
If `True` (default), write ``adata.obs[f"{anchor}_bin"]`` (each
|
|
466
|
+
cell's assigned bin index) and ``adata.uns["binned"]`` (everything
|
|
467
|
+
`diffsvg.tl.setup_model` needs) and return `None`. If `False`, leave
|
|
468
|
+
`adata` untouched and return the `BinnedData` instead.
|
|
469
|
+
|
|
470
|
+
Returns
|
|
471
|
+
-------
|
|
472
|
+
None or BinnedData
|
|
473
|
+
"""
|
|
474
|
+
design_matrix_df = adata.obsm[design_matrix]
|
|
475
|
+
for col in (anchor, condition, sample):
|
|
476
|
+
if col is not None and col not in design_matrix_df.columns:
|
|
477
|
+
raise KeyError(f"{col!r} not found in adata.obsm[{design_matrix!r}]")
|
|
478
|
+
if sample is not None:
|
|
479
|
+
if condition is None:
|
|
480
|
+
raise ValueError("`sample` needs a `condition`: it exists to compare conditions across replicates.")
|
|
481
|
+
if window_radius is not None:
|
|
482
|
+
raise ValueError("`sample` needs contiguous bins - pass `window_radius=None`.")
|
|
483
|
+
|
|
484
|
+
genes = list(adata.var_names) if genes is None else list(genes)
|
|
485
|
+
missing = set(genes) - set(adata.var_names)
|
|
486
|
+
if missing:
|
|
487
|
+
raise KeyError(f"Genes not found in adata.var_names: {sorted(missing)}")
|
|
488
|
+
|
|
489
|
+
anchor_values = design_matrix_df[anchor].to_numpy(dtype=float)
|
|
490
|
+
present = anchor_values[np.isfinite(anchor_values)]
|
|
491
|
+
if present.size and (present.min() < 0 or present.max() > 1):
|
|
492
|
+
raise ValueError(
|
|
493
|
+
f"`anchor` values must lie in [0, 1] (missing values are allowed and left out); "
|
|
494
|
+
f"got range [{present.min()}, {present.max()}]. Normalise the distances first, e.g. with "
|
|
495
|
+
"`squidpy.tl.var_by_distance`."
|
|
496
|
+
)
|
|
497
|
+
|
|
498
|
+
levels = None
|
|
499
|
+
if condition is not None:
|
|
500
|
+
levels = _condition_levels(design_matrix_df[condition])
|
|
501
|
+
if len(levels) != 2:
|
|
502
|
+
raise ValueError(f"`condition` column {condition!r} must have exactly two levels, found: {levels}")
|
|
503
|
+
|
|
504
|
+
if condition is not None and sample is None:
|
|
505
|
+
_warn_possible_replicates(design_matrix_df, anchor, condition)
|
|
506
|
+
|
|
507
|
+
binned = bin_counts(
|
|
508
|
+
adata,
|
|
509
|
+
genes,
|
|
510
|
+
anchor,
|
|
511
|
+
condition if sample is None else sample,
|
|
512
|
+
design_matrix=design_matrix,
|
|
513
|
+
n_bins=n_bins,
|
|
514
|
+
window_radius=window_radius,
|
|
515
|
+
layer=layer,
|
|
516
|
+
)
|
|
517
|
+
|
|
518
|
+
sample_labels = sample_index = None
|
|
519
|
+
if sample is not None:
|
|
520
|
+
# Rows are (sample, bin); each sample's rows get its condition.
|
|
521
|
+
pairs = design_matrix_df[[sample, condition]].dropna().drop_duplicates()
|
|
522
|
+
pairs[sample] = pairs[sample].astype(str)
|
|
523
|
+
mixed = pairs[sample][pairs[sample].duplicated()].unique()
|
|
524
|
+
if len(mixed):
|
|
525
|
+
raise ValueError(f"`sample` must be nested in `condition`; these samples have several: {sorted(mixed)}")
|
|
526
|
+
sample_condition = dict(zip(pairs[sample], pairs[condition], strict=True))
|
|
527
|
+
sample_labels = sorted({str(v) for v in binned[genes[0]][sample]})
|
|
528
|
+
position = {label: i for i, label in enumerate(sample_labels)}
|
|
529
|
+
sample_index = binned[genes[0]][sample].astype(str).map(position).to_numpy(dtype=int)
|
|
530
|
+
for gene in genes:
|
|
531
|
+
frame = binned[gene]
|
|
532
|
+
frame[condition] = frame[sample].astype(str).map(sample_condition)
|
|
533
|
+
|
|
534
|
+
# Encode `condition` to 0.0/1.0 so it can be used directly as a numeric
|
|
535
|
+
# covariate; keep the original labels around for interpretability.
|
|
536
|
+
if condition is not None:
|
|
537
|
+
code_map = {levels[0]: 0.0, levels[1]: 1.0}
|
|
538
|
+
for gene in genes:
|
|
539
|
+
binned[gene][condition] = binned[gene][condition].map(code_map).astype(float)
|
|
540
|
+
|
|
541
|
+
knots_full = compute_interior_knots(n_bsplines, degree=degree)
|
|
542
|
+
interior_knots = knots_full[1:-1]
|
|
543
|
+
|
|
544
|
+
# `anchor`/`condition` are identical across genes (see bin_counts), so
|
|
545
|
+
# the spline basis is built once and shared by every gene's model; only
|
|
546
|
+
# `Y` (binned expression) differs per gene.
|
|
547
|
+
template = binned[genes[0]]
|
|
548
|
+
anchor_vals = template[anchor].to_numpy()
|
|
549
|
+
condition_fit = None if condition is None else template[condition].to_numpy(dtype=float)
|
|
550
|
+
B, design_info = build_spline_basis(anchor_vals, interior_knots, degree=degree)
|
|
551
|
+
B_masked = None if condition is None else B * condition_fit[:, None]
|
|
552
|
+
zero_mask = anchor_vals == 0.0
|
|
553
|
+
|
|
554
|
+
Y = {}
|
|
555
|
+
intercept_initvals = {}
|
|
556
|
+
for gene in genes:
|
|
557
|
+
y = binned[gene]["avg_var"].to_numpy()
|
|
558
|
+
Y[gene] = y
|
|
559
|
+
baseline = y[zero_mask]
|
|
560
|
+
intercept_initvals[gene] = float(baseline.mean()) if baseline.size else float(y.mean())
|
|
561
|
+
|
|
562
|
+
# Fixed grid used for posterior-predictive prediction, evaluated at both
|
|
563
|
+
# condition levels.
|
|
564
|
+
bin_centers = np.linspace(0, 1, n_bins + 1)
|
|
565
|
+
B_pred = np.asarray(patsy.build_design_matrices([design_info], {"x": bin_centers})[0])
|
|
566
|
+
condition_pred_0 = None if condition is None else np.zeros(n_bins + 1)
|
|
567
|
+
condition_pred_1 = None if condition is None else np.ones(n_bins + 1)
|
|
568
|
+
B_pred_masked_0 = None if condition is None else B_pred * condition_pred_0[:, None]
|
|
569
|
+
B_pred_masked_1 = None if condition is None else B_pred * condition_pred_1[:, None]
|
|
570
|
+
|
|
571
|
+
# Every cell (not just those in `genes`) gets assigned to its nearest
|
|
572
|
+
# bin center - `bin_centers` is an evenly spaced grid, so this is just
|
|
573
|
+
# rounding to the nearest multiple of 1/n_bins. Cells without an anchor
|
|
574
|
+
# value get -1.
|
|
575
|
+
bin_assignment = _nearest_bin(design_matrix_df[anchor].to_numpy(dtype=float), n_bins)
|
|
576
|
+
|
|
577
|
+
n_obs = template["n_obs"].to_numpy(dtype=float)
|
|
578
|
+
n_obs_grid = ss_within = None
|
|
579
|
+
if window_radius is None:
|
|
580
|
+
codes = np.zeros(len(template), dtype=int) if condition is None else condition_fit.astype(int)
|
|
581
|
+
n_obs_grid = np.zeros((1 if condition is None else 2, n_bins + 1))
|
|
582
|
+
np.add.at(n_obs_grid, (codes, np.rint(anchor_vals * n_bins).astype(int)), n_obs)
|
|
583
|
+
ss_within = {gene: float(binned[gene]["ss_var"].sum()) for gene in genes}
|
|
584
|
+
|
|
585
|
+
binned_data = BinnedData(
|
|
586
|
+
genes=genes,
|
|
587
|
+
Y=Y,
|
|
588
|
+
intercept_initvals=intercept_initvals,
|
|
589
|
+
anchor=anchor,
|
|
590
|
+
condition=condition,
|
|
591
|
+
condition_categories=None if condition is None else {"0": levels[0], "1": levels[1]},
|
|
592
|
+
n_bsplines=n_bsplines,
|
|
593
|
+
degree=degree,
|
|
594
|
+
n_bins=n_bins,
|
|
595
|
+
B=B,
|
|
596
|
+
B_masked=B_masked,
|
|
597
|
+
condition_fit=condition_fit,
|
|
598
|
+
bin_centers=bin_centers,
|
|
599
|
+
bin_assignment=bin_assignment,
|
|
600
|
+
design_matrix=design_matrix,
|
|
601
|
+
layer=layer,
|
|
602
|
+
B_pred=B_pred,
|
|
603
|
+
B_pred_masked_0=B_pred_masked_0,
|
|
604
|
+
B_pred_masked_1=B_pred_masked_1,
|
|
605
|
+
condition_pred_0=condition_pred_0,
|
|
606
|
+
condition_pred_1=condition_pred_1,
|
|
607
|
+
n_obs=n_obs,
|
|
608
|
+
n_obs_grid=n_obs_grid,
|
|
609
|
+
ss_within=ss_within,
|
|
610
|
+
sample=sample,
|
|
611
|
+
sample_labels=sample_labels,
|
|
612
|
+
sample_index=sample_index,
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
if not inplace:
|
|
616
|
+
return binned_data
|
|
617
|
+
|
|
618
|
+
adata.obs[f"{anchor}_bin"] = bin_assignment
|
|
619
|
+
adata.uns["binned"] = vars(binned_data).copy()
|
|
620
|
+
return None
|
diffsvg/py.typed
ADDED
|
File without changes
|
diffsvg/tl/__init__.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from .clustering import cluster_curves
|
|
2
|
+
from .model import (
|
|
3
|
+
DiffSVGModelSetup,
|
|
4
|
+
fit_model,
|
|
5
|
+
fit_profiles,
|
|
6
|
+
one_vs_rest,
|
|
7
|
+
parallel_curves_test,
|
|
8
|
+
setup_model,
|
|
9
|
+
shape_metric,
|
|
10
|
+
)
|
|
11
|
+
from .stability import StabilityResult, check_stability
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"DiffSVGModelSetup",
|
|
15
|
+
"StabilityResult",
|
|
16
|
+
"check_stability",
|
|
17
|
+
"cluster_curves",
|
|
18
|
+
"fit_model",
|
|
19
|
+
"fit_profiles",
|
|
20
|
+
"one_vs_rest",
|
|
21
|
+
"parallel_curves_test",
|
|
22
|
+
"setup_model",
|
|
23
|
+
"shape_metric",
|
|
24
|
+
]
|