strsc 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strsc/__init__.py +15 -0
- strsc/__main__.py +30 -0
- strsc/api.py +1232 -0
- strsc/cell_biology.py +241 -0
- strsc/cell_core.py +420 -0
- strsc/cell_prepare.py +223 -0
- strsc/cell_rna.py +274 -0
- strsc/cell_score.py +361 -0
- strsc/evaluation_export.py +288 -0
- strsc/example.py +69 -0
- strsc/export.py +119 -0
- strsc/files.py +757 -0
- strsc/fourier.py +271 -0
- strsc/her2.py +303 -0
- strsc/her2_core.py +958 -0
- strsc/hest.py +246 -0
- strsc/hest_core.py +289 -0
- strsc/rsc_cli.py +101 -0
- strsc/training/__init__.py +5 -0
- strsc/training/adapters/__init__.py +0 -0
- strsc/training/adapters/deepspot2cell/__init__.py +40 -0
- strsc/training/adapters/deepspot2cell/engineering_gate.py +239 -0
- strsc/training/adapters/deepspot2cell/formal_runner.py +254 -0
- strsc/training/adapters/deepspot2cell/graph.py +228 -0
- strsc/training/adapters/deepspot2cell/initialization.py +36 -0
- strsc/training/adapters/deepspot2cell/objective.py +212 -0
- strsc/training/adapters/deepspot2cell/overlay.py +79 -0
- strsc/training/adapters/deepspot2cell/paired_data.py +229 -0
- strsc/training/adapters/hest_deephis2exp/__init__.py +48 -0
- strsc/training/adapters/hest_deephis2exp/batching.py +730 -0
- strsc/training/adapters/hest_deephis2exp/data.py +448 -0
- strsc/training/adapters/hest_deephis2exp/engineering_gate.py +312 -0
- strsc/training/adapters/hest_deephis2exp/export.py +173 -0
- strsc/training/adapters/hest_deephis2exp/formal_runner.py +480 -0
- strsc/training/adapters/hest_deephis2exp/models.py +95 -0
- strsc/training/adapters/hest_deephis2exp/objective.py +492 -0
- strsc/training/adapters/hest_deephis2exp/overlay.py +141 -0
- strsc/training/adapters/hest_deephis2exp/split.py +80 -0
- strsc/training/adapters/hest_deepspot_hist/__init__.py +62 -0
- strsc/training/adapters/hest_deepspot_hist/contract.py +136 -0
- strsc/training/adapters/hest_deepspot_hist/data.py +746 -0
- strsc/training/adapters/hest_deepspot_hist/deepspot.py +162 -0
- strsc/training/adapters/hest_deepspot_hist/engineering_gate.py +498 -0
- strsc/training/adapters/hest_deepspot_hist/formal_runner.py +1003 -0
- strsc/training/adapters/hest_deepspot_hist/hist.py +164 -0
- strsc/training/adapters/hest_deepspot_hist/manifest.py +108 -0
- strsc/training/adapters/hest_deepspot_hist/objective.py +94 -0
- strsc/training/adapters/hest_deepspot_hist/outputs.py +154 -0
- strsc/training/adapters/hest_deepspot_hist/paired.py +54 -0
- strsc/training/adapters/hest_egn_tcgn/__init__.py +27 -0
- strsc/training/adapters/hest_egn_tcgn/contracts.py +124 -0
- strsc/training/adapters/hest_egn_tcgn/engineering_gate.py +729 -0
- strsc/training/adapters/hest_egn_tcgn/exemplars.py +241 -0
- strsc/training/adapters/hest_egn_tcgn/hest_data.py +448 -0
- strsc/training/adapters/hest_egn_tcgn/model_inputs.py +139 -0
- strsc/training/adapters/hest_egn_tcgn/outputs.py +137 -0
- strsc/training/adapters/hest_egn_tcgn/overlay.py +388 -0
- strsc/training/adapters/hest_egn_tcgn/pairing.py +66 -0
- strsc/training/adapters/hest_egn_tcgn/train_pair.py +458 -0
- strsc/training/adapters/hest_egn_tcgn/training.py +750 -0
- strsc/training/adapters/hest_egn_tcgn/upstream.py +345 -0
- strsc/training/adapters/hest_thitogene/__init__.py +15 -0
- strsc/training/adapters/hest_thitogene/engineering_gate.py +270 -0
- strsc/training/adapters/hest_thitogene/hest_data.py +396 -0
- strsc/training/adapters/hest_thitogene/overlay.py +260 -0
- strsc/training/adapters/hest_thitogene/train_pair.py +584 -0
- strsc/training/adapters/hest_thitogene/upstream.py +336 -0
- strsc/training/adapters/ridge_mlp/__init__.py +6 -0
- strsc/training/adapters/ridge_mlp/hest_contract.py +135 -0
- strsc/training/adapters/ridge_mlp/model.py +23 -0
- strsc/training/adapters/ridge_mlp/ridge.py +55 -0
- strsc/training/adapters/ridge_mlp/train_pair.py +621 -0
- strsc/training/adapters/scellst/__init__.py +42 -0
- strsc/training/adapters/scellst/data.py +279 -0
- strsc/training/adapters/scellst/engineering_gate.py +148 -0
- strsc/training/adapters/scellst/formal_runner.py +311 -0
- strsc/training/adapters/scellst/lightning_overlay.py +148 -0
- strsc/training/adapters/scellst/objective.py +87 -0
- strsc/training/adapters/scellst/runtime.py +416 -0
- strsc/training/adapters/scellst/split.py +232 -0
- strsc/training/hest_data.py +413 -0
- strsc/training/non_neighbours.py +256 -0
- strsc/training/shared/__init__.py +25 -0
- strsc/training/shared/adjacency.py +169 -0
- strsc/training/shared/losses.py +92 -0
- strsc/training/shared/noop.py +102 -0
- strsc-0.4.0.dist-info/METADATA +173 -0
- strsc-0.4.0.dist-info/RECORD +93 -0
- strsc-0.4.0.dist-info/WHEEL +5 -0
- strsc-0.4.0.dist-info/entry_points.txt +2 -0
- strsc-0.4.0.dist-info/licenses/LICENSE +21 -0
- strsc-0.4.0.dist-info/licenses/NOTICE.md +13 -0
- strsc-0.4.0.dist-info/top_level.txt +1 -0
strsc/api.py
ADDED
|
@@ -0,0 +1,1232 @@
|
|
|
1
|
+
"""Reproducible-signal correlation (RSC) in one call.
|
|
2
|
+
|
|
3
|
+
import strsc
|
|
4
|
+
counts = strsc.read_counts("sample.h5ad")
|
|
5
|
+
result = strsc.rsc(counts, {"my_model": "predictions.csv"}, layout="spots")
|
|
6
|
+
print(result.scales)
|
|
7
|
+
|
|
8
|
+
Each layout is scored as in the paper (Methods, "Reproducible-signal correlation" and
|
|
9
|
+
"Scales and count transforms"), with the toolkit's own functions:
|
|
10
|
+
|
|
11
|
+
``"grid"``
|
|
12
|
+
Bins on a regular grid, such as Visium HD. A two-dimensional Fourier basis divides the
|
|
13
|
+
variation into seven physical scales, from below 128 µm to 4,096 µm and above. Half-counts
|
|
14
|
+
are transformed as sqrt(N_half / (0.5 a)) with a the bin area; five count splits. These are the
|
|
15
|
+
steps of :func:`strsc.fourier.score`, which gives the same RSC.
|
|
16
|
+
``"spots"``
|
|
17
|
+
Spots on a neighbor graph, such as Visium or HEST. The eigenvectors of the graph Laplacian
|
|
18
|
+
give five relative scales, Broadest to Finest. Half-counts are transformed as log(1 + 2 N_half);
|
|
19
|
+
twenty count splits. With two or more sections, each gene is weighted by the agreement of its
|
|
20
|
+
two halves in the other sections, as for the HEST samples of the paper. A single section has
|
|
21
|
+
no other sections, so each gene's two halves are divided by a common factor, as for the
|
|
22
|
+
colorectal bins and the breast cancer cells of the paper; the values are then not directly
|
|
23
|
+
comparable with those of the HEST weighting.
|
|
24
|
+
``"cells"``
|
|
25
|
+
Single cells, such as Xenium. Lazy diffusion operators on the graph of the eight nearest
|
|
26
|
+
neighbors within 40 µm give five relative scales. Half-counts are transformed as
|
|
27
|
+
log(1 + 2 N_half); five count splits. ``within`` scores the cells of one class on their own
|
|
28
|
+
graph, as for the malignant cells of the paper.
|
|
29
|
+
|
|
30
|
+
:func:`regions` scores agreement between and within annotated regions, as for the TLS sections,
|
|
31
|
+
and :func:`ideal_check` repeats the paper's test with the expected expression as the prediction.
|
|
32
|
+
|
|
33
|
+
The workflows of the paper (``strsc hest``, ``strsc fourier``, ``strsc.cell_score``) are unchanged.
|
|
34
|
+
"""
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import json
|
|
38
|
+
import math
|
|
39
|
+
import sys
|
|
40
|
+
from dataclasses import dataclass, field
|
|
41
|
+
from pathlib import Path
|
|
42
|
+
from typing import Callable, Iterable, Mapping, Sequence
|
|
43
|
+
|
|
44
|
+
import numpy as np
|
|
45
|
+
import pandas as pd
|
|
46
|
+
from scipy import sparse
|
|
47
|
+
|
|
48
|
+
if sys.platform == "emscripten": # pragma: no cover - only in the browser
|
|
49
|
+
# WebAssembly cannot start threads, and fourier.fft_fields asks scipy for two FFT workers.
|
|
50
|
+
# The number of workers changes how the work is shared, not the result.
|
|
51
|
+
import scipy.fft
|
|
52
|
+
|
|
53
|
+
if not getattr(scipy.fft.fft2, "_single_worker", False):
|
|
54
|
+
_fft2 = scipy.fft.fft2
|
|
55
|
+
|
|
56
|
+
def _fft2_single(*args, **kwargs):
|
|
57
|
+
return _fft2(*args, **{**kwargs, "workers": 1})
|
|
58
|
+
|
|
59
|
+
_fft2_single._single_worker = True
|
|
60
|
+
scipy.fft.fft2 = _fft2_single
|
|
61
|
+
# numpy in the browser multiplies matrices without BLAS; SciPy's BLAS does the spot projections
|
|
62
|
+
# about 15 times faster, in the same single precision.
|
|
63
|
+
from scipy.linalg import blas as _browser_blas
|
|
64
|
+
|
|
65
|
+
from . import __version__
|
|
66
|
+
from . import cell_core, evaluation_export, fourier
|
|
67
|
+
from . import hest_core as cal
|
|
68
|
+
from .files import Counts, InputError, Prediction, read_counts, read_predictions
|
|
69
|
+
|
|
70
|
+
LAYOUTS = ("spots", "grid", "cells")
|
|
71
|
+
RELATIVE = ("Broadest", "Broad", "Mid", "Fine", "Finest")
|
|
72
|
+
SPOT_SEEDS = tuple(cal.THINNING_SEEDS) # 20 count splits, as for the HEST spots
|
|
73
|
+
GRID_SEEDS = tuple(fourier.SPLIT_SEEDS) # 5, as for the colorectal bins
|
|
74
|
+
CELL_SEEDS = (20260819, 20260820, 20260821, 20260822, 20260823) # 5, as for the breast cancer cells
|
|
75
|
+
REGION_SPLITS = 20 # as for the TLS sections
|
|
76
|
+
CELL_GRAPH = dict(neighbors=8, max_edge_um=40.0, minimum_component_cells=20)
|
|
77
|
+
CELL_GENE_BLOCK = 64
|
|
78
|
+
GRID_LABELS = {"LT128": "<128 µm", "128_256": "128–256 µm", "256_512": "256–512 µm", "512_1024": "512–1,024 µm",
|
|
79
|
+
"1024_2048": "1,024–2,048 µm", "2048_4096": "2,048–4,096 µm", "GE4096": "≥4,096 µm"}
|
|
80
|
+
TOL = 1e-12
|
|
81
|
+
|
|
82
|
+
Progress = Callable[[dict], None] | None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# ---------------------------------------------------------------- results
|
|
86
|
+
|
|
87
|
+
@dataclass
|
|
88
|
+
class Result:
|
|
89
|
+
"""Scores of one call. ``scales`` has one row per model, section and scale (or region term);
|
|
90
|
+
``summary`` one row per model and section; ``genes`` the same scores gene by gene."""
|
|
91
|
+
|
|
92
|
+
layout: str
|
|
93
|
+
scales: pd.DataFrame
|
|
94
|
+
summary: pd.DataFrame
|
|
95
|
+
genes: pd.DataFrame
|
|
96
|
+
settings: dict
|
|
97
|
+
notes: list[str] = field(default_factory=list)
|
|
98
|
+
|
|
99
|
+
def __repr__(self) -> str:
|
|
100
|
+
shown = self.summary.drop(columns=[c for c in self.summary.columns if c.startswith("n_")], errors="ignore")
|
|
101
|
+
return f"RSC, {self.layout}: {len(self.scales)} rows\n" + shown.to_string(index=False, float_format=lambda v: f"{v:.3f}")
|
|
102
|
+
|
|
103
|
+
def to_bundle(self, title: str = "RSC computed with strsc", synthetic: bool = False) -> dict:
|
|
104
|
+
"""The scores as an evaluation bundle, which the Computed results tab of strsc.org opens."""
|
|
105
|
+
return _bundle(self, title, synthetic)
|
|
106
|
+
|
|
107
|
+
def save(self, folder: str | Path, title: str = "RSC computed with strsc") -> list[Path]:
|
|
108
|
+
folder = Path(folder)
|
|
109
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
110
|
+
written = []
|
|
111
|
+
for name, frame in (("scales.tsv", self.scales), ("summary.tsv", self.summary), ("genes.tsv", self.genes)):
|
|
112
|
+
frame.to_csv(folder / name, sep="\t", index=False)
|
|
113
|
+
written.append(folder / name)
|
|
114
|
+
(folder / "settings.json").write_text(json.dumps(evaluation_export.finite_json(self.settings), indent=2, ensure_ascii=False))
|
|
115
|
+
(folder / "strsc-evaluation.json").write_text(json.dumps(self.to_bundle(title)))
|
|
116
|
+
return written + [folder / "settings.json", folder / "strsc-evaluation.json"]
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _ratio(c: float, vrna: float, vmodel: float) -> tuple[float, str]:
|
|
120
|
+
if not (vmodel > TOL):
|
|
121
|
+
return math.nan, "NO_PREDICTED_VARIANCE"
|
|
122
|
+
if not (vrna > 0):
|
|
123
|
+
return math.nan, "RNA_UNRESOLVED"
|
|
124
|
+
return float(c / math.sqrt(vrna * vmodel)), "ESTIMABLE"
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _record(model, section, scale, order, c, vmodel, vrna, va, vb, seeds_c=None, seeds_vrna=None, **extra) -> dict:
|
|
128
|
+
"""One row: RSC, the Pearson r of the prediction with the mean of the two halves, and the split-half reliability."""
|
|
129
|
+
rsc, status = _ratio(c, vrna, vmodel)
|
|
130
|
+
mean_var = 0.25 * (va + vb + 2.0 * vrna)
|
|
131
|
+
pearson = float(c / math.sqrt(vmodel * mean_var)) if vmodel > TOL and mean_var > TOL else math.nan
|
|
132
|
+
reliability = float(vrna / math.sqrt(va * vb)) if va > TOL and vb > TOL else math.nan
|
|
133
|
+
lo = hi = math.nan
|
|
134
|
+
if seeds_c is not None and seeds_vrna is not None and vmodel > TOL:
|
|
135
|
+
per = [c_ / math.sqrt(v_ * vmodel) for c_, v_ in zip(seeds_c, seeds_vrna) if v_ > 0]
|
|
136
|
+
if per:
|
|
137
|
+
lo, hi = float(min(per)), float(max(per))
|
|
138
|
+
return dict(model=model, section=section, scale=scale, scale_order=order, rsc=rsc, pearson_r=pearson,
|
|
139
|
+
split_half_reliability=reliability, C=float(c), V_model=float(vmodel), V_RNA=float(vrna),
|
|
140
|
+
V_A=float(va), V_B=float(vb), rsc_split_min=lo, rsc_split_max=hi, status=status, **extra)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _gene_rows(model, section, genes, labels, c, vmodel, vrna, va, vb, eligible=None) -> list[dict]:
|
|
144
|
+
"""Per-gene scores: c, vmodel, ... are (scales, genes) arrays."""
|
|
145
|
+
rows = []
|
|
146
|
+
for g, gene in enumerate(genes):
|
|
147
|
+
if eligible is not None and not eligible[g]:
|
|
148
|
+
continue
|
|
149
|
+
for k, label in enumerate(labels):
|
|
150
|
+
rsc, _ = _ratio(c[k, g], vrna[k, g], vmodel[k, g])
|
|
151
|
+
rel = vrna[k, g] / math.sqrt(va[k, g] * vb[k, g]) if va[k, g] > TOL and vb[k, g] > TOL else math.nan
|
|
152
|
+
rows.append(dict(model=model, section=section, gene=gene, scale=label, scale_order=k + 1, rsc=rsc,
|
|
153
|
+
split_half_reliability=float(rel), C=float(c[k, g]), V_model=float(vmodel[k, g]), V_RNA=float(vrna[k, g])))
|
|
154
|
+
return rows
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
# ---------------------------------------------------------------- inputs
|
|
158
|
+
|
|
159
|
+
def _models(predictions, counts: Counts) -> list[Prediction]:
|
|
160
|
+
if isinstance(predictions, Prediction):
|
|
161
|
+
return [predictions]
|
|
162
|
+
if isinstance(predictions, (str, Path)):
|
|
163
|
+
return [read_predictions(predictions, counts)]
|
|
164
|
+
if isinstance(predictions, Mapping):
|
|
165
|
+
out = []
|
|
166
|
+
for name, value in predictions.items():
|
|
167
|
+
if isinstance(value, Prediction):
|
|
168
|
+
out.append(Prediction(str(name), value.values, value.genes, value.source))
|
|
169
|
+
elif isinstance(value, (str, Path)):
|
|
170
|
+
out.append(read_predictions(value, counts, str(name)))
|
|
171
|
+
else:
|
|
172
|
+
array = np.asarray(value, dtype=np.float64)
|
|
173
|
+
if array.shape != (len(counts), len(counts.genes)):
|
|
174
|
+
raise InputError(f"{name}: an array prediction needs the shape of the counts, {len(counts):,} × {len(counts.genes):,}.")
|
|
175
|
+
out.append(Prediction(str(name), array, list(counts.genes), "array"))
|
|
176
|
+
return out
|
|
177
|
+
if isinstance(predictions, Sequence):
|
|
178
|
+
return [p if isinstance(p, Prediction) else read_predictions(p, counts) for p in predictions]
|
|
179
|
+
raise InputError("Give the predictions as a file, a Prediction, or a dict of model name to file or array.")
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _common(counts: Counts, models: list[Prediction], genes: Iterable[str] | None):
|
|
183
|
+
"""Genes predicted by every model, and the locations that have a value for each of them."""
|
|
184
|
+
names = [m.name for m in models]
|
|
185
|
+
if not models:
|
|
186
|
+
raise InputError("Give at least one prediction.")
|
|
187
|
+
if len(set(names)) != len(names):
|
|
188
|
+
raise InputError("Two predictions have the same name. Name each model once.")
|
|
189
|
+
wanted = list(counts.genes) if genes is None else [g for g in genes]
|
|
190
|
+
if genes is not None:
|
|
191
|
+
missing = [g for g in wanted if g not in set(counts.genes)]
|
|
192
|
+
if missing:
|
|
193
|
+
raise InputError(f"{len(missing)} of the chosen genes are not in the counts, for example {missing[0]}.")
|
|
194
|
+
shared = [g for g in wanted if all(g in set(m.genes) for m in models)]
|
|
195
|
+
if len(shared) < 1:
|
|
196
|
+
raise InputError("No gene is in the counts and in every prediction. Use the same gene names.")
|
|
197
|
+
columns = np.asarray([counts.genes.index(g) for g in shared])
|
|
198
|
+
complete = np.ones(len(counts), dtype=bool)
|
|
199
|
+
for m in models:
|
|
200
|
+
complete &= np.isfinite(m.values[:, columns]).all(axis=1)
|
|
201
|
+
notes = []
|
|
202
|
+
dropped_genes = len(wanted) - len(shared)
|
|
203
|
+
if dropped_genes:
|
|
204
|
+
notes.append(f"{dropped_genes:,} gene{'s' if dropped_genes != 1 else ''} of the counts {'are' if dropped_genes != 1 else 'is'} not predicted by every model and {'were' if dropped_genes != 1 else 'was'} left out.")
|
|
205
|
+
outside = int((~complete).sum())
|
|
206
|
+
if outside:
|
|
207
|
+
notes.append(f"{outside:,} location{'s' if outside != 1 else ''} without a prediction for every gene from every model {'were' if outside != 1 else 'was'} left out.")
|
|
208
|
+
if not complete.any():
|
|
209
|
+
raise InputError("No spot, bin or cell has a prediction for every gene from every model.")
|
|
210
|
+
return shared, columns, complete, notes
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _settings(layout, counts, genes, models, seeds, **extra) -> dict:
|
|
214
|
+
return dict(toolkit=f"strsc {__version__}", layout=layout, sections=counts.section_names(), genes=len(genes),
|
|
215
|
+
models=[m.name for m in models], count_splits=len(seeds), split_seeds=list(map(int, seeds)), **extra)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
# ---------------------------------------------------------------- spots
|
|
219
|
+
|
|
220
|
+
def _molecule_moments(counts: np.ndarray, specimen: str, task: str, seeds) -> dict:
|
|
221
|
+
"""As in strsc hest: per gene, the mean variance of a half-count, the mean covariance of the halves
|
|
222
|
+
and their inner product, averaged over count splits."""
|
|
223
|
+
d2 = np.zeros(counts.shape[1])
|
|
224
|
+
covariance = np.zeros_like(d2)
|
|
225
|
+
ell2 = np.zeros_like(d2)
|
|
226
|
+
for seed in seeds:
|
|
227
|
+
a, b = cal._halves(counts, task, specimen, seed)
|
|
228
|
+
d2 += 0.5 * (np.mean(a * a, axis=0) + np.mean(b * b, axis=0))
|
|
229
|
+
covariance += np.mean(a * b, axis=0)
|
|
230
|
+
ell2 += np.sum(a * b, axis=0)
|
|
231
|
+
scale = 1.0 / len(seeds)
|
|
232
|
+
return {"d2": d2 * scale, "covariance": covariance * scale, "ell2": ell2 * scale}
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def spot_bands(xy: np.ndarray, progress=None):
|
|
236
|
+
"""The five equal-rank groups of nonconstant Laplacian eigenmodes of each connected component,
|
|
237
|
+
as in ``hest_core.build_basis`` (combinatorial Laplacian, constant mode dropped, stable order),
|
|
238
|
+
with numpy's dense eigensolver."""
|
|
239
|
+
adjacency, keep, labels, d0, radius = cal._graph(xy)
|
|
240
|
+
kept_n = int(keep.sum())
|
|
241
|
+
components = sorted(np.unique(labels).tolist())
|
|
242
|
+
chunks_by_component, totals = {}, np.zeros(cal.BANDS, dtype=np.int64)
|
|
243
|
+
for component in components:
|
|
244
|
+
size = int(np.sum(labels == component))
|
|
245
|
+
chunks = np.array_split(np.arange(max(size - 1, 0)), cal.BANDS)
|
|
246
|
+
chunks_by_component[component] = chunks
|
|
247
|
+
totals += np.asarray([len(chunk) for chunk in chunks])
|
|
248
|
+
bands = [np.zeros((kept_n, int(totals[k])), dtype=np.float64) for k in range(cal.BANDS)]
|
|
249
|
+
offsets = np.zeros(cal.BANDS, dtype=np.int64)
|
|
250
|
+
for component in components:
|
|
251
|
+
local = np.flatnonzero(labels == component)
|
|
252
|
+
block = adjacency[local][:, local]
|
|
253
|
+
laplacian = -block.toarray().astype(np.float64)
|
|
254
|
+
degree = np.asarray(block.sum(axis=1)).reshape(-1).astype(np.float64)
|
|
255
|
+
laplacian[np.diag_indices_from(laplacian)] = degree
|
|
256
|
+
values, vectors = np.linalg.eigh(laplacian)
|
|
257
|
+
order = np.argsort(values, kind="stable")
|
|
258
|
+
vectors = vectors[:, order][:, 1:]
|
|
259
|
+
for k, indices in enumerate(chunks_by_component[component]):
|
|
260
|
+
start, end = int(offsets[k]), int(offsets[k]) + len(indices)
|
|
261
|
+
if len(indices):
|
|
262
|
+
bands[k][local, start:end] = vectors[:, indices]
|
|
263
|
+
offsets[k] = end
|
|
264
|
+
if progress:
|
|
265
|
+
progress()
|
|
266
|
+
return keep, bands, dict(spots_retained=kept_n, components=len(components), d0=d0, edge_radius=radius, band_modes=totals.tolist())
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _coefficients(bands, x):
|
|
270
|
+
"""Band projections in single precision, as ``hest_core._coefficients``."""
|
|
271
|
+
x32 = np.asarray(x, dtype=np.float32)
|
|
272
|
+
if sys.platform == "emscripten": # pragma: no cover - only in the browser
|
|
273
|
+
return [np.asarray(_browser_blas.sgemm(1.0, np.asarray(u, dtype=np.float32), x32, trans_a=True), dtype=np.float64, order="C")
|
|
274
|
+
for u in bands]
|
|
275
|
+
return [(np.asarray(u, dtype=np.float32).T @ x32).astype(np.float64) for u in bands]
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _task_label(names, default: str = "Sections") -> str:
|
|
279
|
+
"""The HEST task when the sections are exactly the samples of one, so that the count splits are
|
|
280
|
+
those of ``strsc hest``; otherwise a plain label. The label seeds every count split."""
|
|
281
|
+
return next((task for task, samples in cal.TASKS.items() if set(samples) == set(names)), default)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def score_spot_sections(sections: list[dict], genes: list[str], seeds=SPOT_SEEDS, step=None, max_spots: int | None = None,
|
|
285
|
+
task: str | None = None) -> list[dict]:
|
|
286
|
+
"""Relative scales on spots. ``sections``: dicts with name, counts (int, spots x genes), xy and
|
|
287
|
+
predictions {model: spots x genes}. With two or more sections the gene weights come from the
|
|
288
|
+
other sections, as ``strsc hest``; one section uses the common factor of each gene's halves."""
|
|
289
|
+
task = task or _task_label([s["name"] for s in sections])
|
|
290
|
+
weighted = len(sections) >= 2
|
|
291
|
+
retained, moments = {}, {}
|
|
292
|
+
for s in sections:
|
|
293
|
+
_, keep, _, _, _ = cal._graph(s["xy"])
|
|
294
|
+
if not keep.any():
|
|
295
|
+
raise InputError(f"Section {s['name']}: no connected group of at least 20 spots.")
|
|
296
|
+
if max_spots and int(keep.sum()) > max_spots:
|
|
297
|
+
raise InputError(f"Section {s['name']} has {int(keep.sum()):,} connected spots. Here the limit is {max_spots:,}; run the toolkit on your own machine for larger sections, without cropping them.")
|
|
298
|
+
retained[s["name"]] = keep
|
|
299
|
+
if weighted:
|
|
300
|
+
moments[s["name"]] = _molecule_moments(s["counts"][keep], s["name"], task, seeds)
|
|
301
|
+
records = []
|
|
302
|
+
for s in sections:
|
|
303
|
+
name, keep = s["name"], retained[s["name"]]
|
|
304
|
+
counts = s["counts"][keep]
|
|
305
|
+
if weighted:
|
|
306
|
+
others = [o["name"] for o in sections if o["name"] != name]
|
|
307
|
+
d2 = np.mean(np.stack([moments[o]["d2"] for o in others]), axis=0)
|
|
308
|
+
cov = np.mean(np.stack([moments[o]["covariance"] for o in others]), axis=0)
|
|
309
|
+
h = np.divide(cov, d2, out=np.zeros_like(d2), where=d2 > cal.NUMERIC_TOL)
|
|
310
|
+
w, ell2 = np.maximum(h, 0.0), moments[name]["ell2"]
|
|
311
|
+
eligible = (d2 > cal.NUMERIC_TOL) & (w > 0) & (ell2 > 0)
|
|
312
|
+
d = np.sqrt(d2)
|
|
313
|
+
rna_scale = np.divide(np.sqrt(w), d, out=np.zeros_like(w), where=d > cal.NUMERIC_TOL)
|
|
314
|
+
rna_scale[~eligible] = 0.0
|
|
315
|
+
p_scale = np.divide(np.sqrt(w) * np.sqrt(np.maximum(ell2, 0)), d, out=np.zeros_like(w), where=d > cal.NUMERIC_TOL)
|
|
316
|
+
p_scale[~eligible] = 0.0
|
|
317
|
+
else:
|
|
318
|
+
others = []
|
|
319
|
+
eligible = np.asarray(counts.sum(axis=0)).reshape(-1) > 0
|
|
320
|
+
if step:
|
|
321
|
+
step(name, "graph")
|
|
322
|
+
keep2, bands, graph = spot_bands(s["xy"])
|
|
323
|
+
assert np.array_equal(keep2, keep)
|
|
324
|
+
prepared = {}
|
|
325
|
+
for model, prediction_full in s["predictions"].items():
|
|
326
|
+
prediction = prediction_full[keep]
|
|
327
|
+
centered = prediction - prediction.mean(axis=0, keepdims=True)
|
|
328
|
+
pnorm = np.sqrt(np.sum(centered * centered, axis=0))
|
|
329
|
+
p = np.divide(centered, pnorm[None, :], out=np.zeros_like(centered), where=pnorm[None, :] > cal.NUMERIC_TOL)
|
|
330
|
+
if weighted:
|
|
331
|
+
p = p * p_scale[None, :]
|
|
332
|
+
else:
|
|
333
|
+
p[:, ~eligible] = 0.0
|
|
334
|
+
prepared[model] = _coefficients(bands, p)
|
|
335
|
+
n_genes = counts.shape[1]
|
|
336
|
+
vrna, va, vb = np.zeros(5), np.zeros(5), np.zeros(5)
|
|
337
|
+
vrna_g, va_g, vb_g = np.zeros((5, n_genes)), np.zeros((5, n_genes)), np.zeros((5, n_genes))
|
|
338
|
+
mean_observed = [None] * 5
|
|
339
|
+
seed_vrna = np.zeros((len(seeds), 5))
|
|
340
|
+
seed_c = {model: np.zeros((len(seeds), 5)) for model in prepared}
|
|
341
|
+
for i, seed in enumerate(seeds):
|
|
342
|
+
a, b = cal._halves(counts, task, name, seed)
|
|
343
|
+
if weighted:
|
|
344
|
+
a *= rna_scale[None, :]
|
|
345
|
+
b *= rna_scale[None, :]
|
|
346
|
+
else:
|
|
347
|
+
factor = np.sqrt(0.5 * (np.sum(a * a, axis=0) + np.sum(b * b, axis=0)))
|
|
348
|
+
ok = (factor > cal.NUMERIC_TOL) & eligible
|
|
349
|
+
a[:, ok] /= factor[ok]
|
|
350
|
+
b[:, ok] /= factor[ok]
|
|
351
|
+
a[:, ~ok] = 0.0
|
|
352
|
+
b[:, ~ok] = 0.0
|
|
353
|
+
ac, bc = _coefficients(bands, a), _coefficients(bands, b)
|
|
354
|
+
for k in range(5):
|
|
355
|
+
vrna[k] += np.sum(ac[k] * bc[k])
|
|
356
|
+
current = 0.5 * (ac[k] + bc[k])
|
|
357
|
+
mean_observed[k] = current if mean_observed[k] is None else mean_observed[k] + current
|
|
358
|
+
va[k] += np.sum(ac[k] * ac[k])
|
|
359
|
+
vb[k] += np.sum(bc[k] * bc[k])
|
|
360
|
+
vrna_g[k] += np.sum(ac[k] * bc[k], axis=0)
|
|
361
|
+
va_g[k] += np.sum(ac[k] * ac[k], axis=0)
|
|
362
|
+
vb_g[k] += np.sum(bc[k] * bc[k], axis=0)
|
|
363
|
+
seed_vrna[i, k] = np.sum(ac[k] * bc[k])
|
|
364
|
+
for model, pc in prepared.items():
|
|
365
|
+
seed_c[model][i, k] = np.sum(pc[k] * current)
|
|
366
|
+
if step:
|
|
367
|
+
step(name, "split")
|
|
368
|
+
n = len(seeds)
|
|
369
|
+
vrna /= n
|
|
370
|
+
va /= n
|
|
371
|
+
vb /= n
|
|
372
|
+
vrna_g /= n
|
|
373
|
+
va_g /= n
|
|
374
|
+
vb_g /= n
|
|
375
|
+
mean_observed = [x / n for x in mean_observed]
|
|
376
|
+
truth = np.log1p(s["counts"])
|
|
377
|
+
models = []
|
|
378
|
+
for model, pc in prepared.items():
|
|
379
|
+
prediction_full = s["predictions"][model]
|
|
380
|
+
vpred = np.asarray([np.sum(x * x) for x in pc])
|
|
381
|
+
covariance = np.asarray([np.sum(pc[k] * mean_observed[k]) for k in range(5)])
|
|
382
|
+
c_g = np.stack([np.sum(pc[k] * mean_observed[k], axis=0) for k in range(5)])
|
|
383
|
+
vm_g = np.stack([np.sum(pc[k] * pc[k], axis=0) for k in range(5)])
|
|
384
|
+
gene_pcc = cal._corr_cols(prediction_full[keep], truth[keep])
|
|
385
|
+
full_pcc = cal._corr_cols(prediction_full, truth)
|
|
386
|
+
models.append(dict(model=model, vpred=vpred, covariance=covariance, c_g=c_g, vm_g=vm_g, seed_c=seed_c[model],
|
|
387
|
+
gene_pcc=gene_pcc, full_pcc=full_pcc))
|
|
388
|
+
records.append(dict(section=name, task=task, weighted=weighted, reference_sections=others, graph=graph,
|
|
389
|
+
spots=len(keep), retained=int(keep.sum()), eligible=eligible, vrna=vrna, va=va, vb=vb,
|
|
390
|
+
vrna_g=vrna_g, va_g=va_g, vb_g=vb_g, seed_vrna=seed_vrna, models=models))
|
|
391
|
+
return records
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _spot_result(records, genes, settings, notes) -> Result:
|
|
395
|
+
scale_rows, gene_rows, summaries = [], [], []
|
|
396
|
+
for r in records:
|
|
397
|
+
for m in r["models"]:
|
|
398
|
+
rows = []
|
|
399
|
+
for k, label in enumerate(RELATIVE):
|
|
400
|
+
rows.append(_record(m["model"], r["section"], label, k + 1, m["covariance"][k], m["vpred"][k], r["vrna"][k],
|
|
401
|
+
r["va"][k], r["vb"][k], m["seed_c"][:, k], r["seed_vrna"][:, k]))
|
|
402
|
+
scale_rows.extend(rows)
|
|
403
|
+
rsc = [row["rsc"] for row in rows]
|
|
404
|
+
gene_rows.extend(_gene_rows(m["model"], r["section"], genes, RELATIVE, m["c_g"], m["vm_g"], r["vrna_g"], r["va_g"], r["vb_g"], r["eligible"]))
|
|
405
|
+
summaries.append(dict(model=m["model"], section=r["section"],
|
|
406
|
+
fine_rsc=_mean2(rsc[3], rsc[4]), broad_rsc=_mean2(rsc[0], rsc[1]),
|
|
407
|
+
broadest_minus_finest=rsc[0] - rsc[4] if all(map(math.isfinite, (rsc[0], rsc[4]))) else math.nan,
|
|
408
|
+
fine_split_half_reliability=_mean2(rows[3]["split_half_reliability"], rows[4]["split_half_reliability"]),
|
|
409
|
+
whole_section_pearson_r=_nanmean(m["gene_pcc"]),
|
|
410
|
+
n_locations=r["retained"], n_genes=int(np.sum(r["eligible"]))))
|
|
411
|
+
scales, summary = pd.DataFrame(scale_rows), pd.DataFrame(summaries)
|
|
412
|
+
if len(records) > 1:
|
|
413
|
+
scales, summary = _all_sections_mean(scales, summary)
|
|
414
|
+
return Result("spots", scales, summary, pd.DataFrame(gene_rows), settings, notes)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _mean2(x, y) -> float:
|
|
418
|
+
return 0.5 * (x + y) if math.isfinite(x) and math.isfinite(y) else math.nan
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _nanmean(values) -> float:
|
|
422
|
+
values = np.asarray(values, dtype=np.float64)
|
|
423
|
+
return float(np.nanmean(values)) if np.isfinite(values).any() else math.nan
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _all_sections_mean(scales: pd.DataFrame, summary: pd.DataFrame):
|
|
427
|
+
"""'All sections': the mean over sections of each section's scores, as the means over samples in the paper."""
|
|
428
|
+
numeric = ["rsc", "pearson_r", "split_half_reliability"]
|
|
429
|
+
pooled = (scales.groupby(["model", "scale", "scale_order"], as_index=False, sort=False)[numeric].mean())
|
|
430
|
+
pooled.insert(1, "section", "All sections")
|
|
431
|
+
pooled["status"] = "MEAN_OVER_SECTIONS"
|
|
432
|
+
keep = [c for c in summary.columns if c not in ("model", "section") and not c.startswith("n_")]
|
|
433
|
+
overall = summary.groupby("model", as_index=False, sort=False)[keep].mean()
|
|
434
|
+
overall.insert(1, "section", "All sections")
|
|
435
|
+
return pd.concat([scales, pooled], ignore_index=True), pd.concat([summary, overall], ignore_index=True)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
# ---------------------------------------------------------------- grid
|
|
439
|
+
|
|
440
|
+
def score_grid_section(counts: sparse.csr_matrix, prediction: np.ndarray, coords: np.ndarray, area: np.ndarray, *,
|
|
441
|
+
spacing_um: float, models: Sequence[str], seeds=GRID_SEEDS, step=None) -> dict:
|
|
442
|
+
"""The steps of :func:`strsc.fourier.score`, unchanged, which also keep the variance of each half,
|
|
443
|
+
the terms of each gene and of each count split. Its RSC equals that of ``fourier.score``."""
|
|
444
|
+
counts = sparse.csr_matrix(counts)
|
|
445
|
+
counts.sum_duplicates()
|
|
446
|
+
counts.eliminate_zeros()
|
|
447
|
+
counts.sort_indices()
|
|
448
|
+
n, genes = counts.shape
|
|
449
|
+
windows = pd.DataFrame({"center_x_um": coords[:, 0], "center_y_um": coords[:, 1]})
|
|
450
|
+
scaled = (coords - coords.min(axis=0)) / spacing_um
|
|
451
|
+
if not np.allclose(scaled, np.rint(scaled), rtol=0, atol=1e-6):
|
|
452
|
+
raise InputError("The coordinates do not lie on a regular grid with this spacing.")
|
|
453
|
+
iy, ix, ny, nx, masks = fourier.grid_contract(windows, spacing_um)
|
|
454
|
+
names = list(masks)
|
|
455
|
+
zeros = lambda: {band: np.zeros(genes) for band in names}
|
|
456
|
+
vrna_g, va_g, vb_g = zeros(), zeros(), zeros()
|
|
457
|
+
c_g = {model: zeros() for model in models}
|
|
458
|
+
vm_g = {model: zeros() for model in models}
|
|
459
|
+
terms = {(model, band): {"v_rna": 0.0, "c_model": 0.0, "v_model": 0.0} for model in models for band in names}
|
|
460
|
+
halves = {band: {"v_a": 0.0, "v_b": 0.0} for band in names}
|
|
461
|
+
seed_vrna = {band: np.zeros(len(seeds)) for band in names}
|
|
462
|
+
seed_c = {(model, band): np.zeros(len(seeds)) for model in models for band in names}
|
|
463
|
+
for offset in range(0, genes, fourier.GENE_BLOCK):
|
|
464
|
+
stop = min(offset + fourier.GENE_BLOCK, genes)
|
|
465
|
+
block = counts[:, offset:stop].tocsr()
|
|
466
|
+
fp = {model: fourier.fft_fields(fourier.normalize_prediction(prediction[:, offset:stop, j]), iy, ix, ny, nx)
|
|
467
|
+
for j, model in enumerate(models)}
|
|
468
|
+
for model in models:
|
|
469
|
+
for band, mask in masks.items():
|
|
470
|
+
terms[model, band]["v_model"] += float(np.square(np.abs(fp[model][mask, :])).sum())
|
|
471
|
+
vm_g[model][band][offset:stop] += np.square(np.abs(fp[model][mask, :])).sum(axis=0)
|
|
472
|
+
block_vrna = {band: 0.0 for band in names}
|
|
473
|
+
block_c = {(model, band): 0.0 for model in models for band in names}
|
|
474
|
+
for i, seed in enumerate(seeds):
|
|
475
|
+
aa, bb = fourier.split_transform(block, area, int(seed) + offset * 1009)
|
|
476
|
+
fa, fb = (fourier.fft_fields(values, iy, ix, ny, nx) for values in (aa, bb))
|
|
477
|
+
for band, mask in masks.items():
|
|
478
|
+
cross = np.real(np.conj(fa[mask, :]) * fb[mask, :])
|
|
479
|
+
value = float(cross.sum())
|
|
480
|
+
block_vrna[band] += value
|
|
481
|
+
seed_vrna[band][i] += value
|
|
482
|
+
vrna_g[band][offset:stop] += cross.sum(axis=0) / len(seeds)
|
|
483
|
+
ea, eb = np.square(np.abs(fa[mask, :])), np.square(np.abs(fb[mask, :]))
|
|
484
|
+
halves[band]["v_a"] += float(ea.sum()) / len(seeds)
|
|
485
|
+
halves[band]["v_b"] += float(eb.sum()) / len(seeds)
|
|
486
|
+
va_g[band][offset:stop] += ea.sum(axis=0) / len(seeds)
|
|
487
|
+
vb_g[band][offset:stop] += eb.sum(axis=0) / len(seeds)
|
|
488
|
+
judge_sum = fa[mask, :] + fb[mask, :]
|
|
489
|
+
for model in models:
|
|
490
|
+
product = 0.5 * np.real(np.conj(fp[model][mask, :]) * judge_sum)
|
|
491
|
+
value = float(product.sum())
|
|
492
|
+
block_c[model, band] += value
|
|
493
|
+
seed_c[model, band][i] += value
|
|
494
|
+
c_g[model][band][offset:stop] += product.sum(axis=0) / len(seeds)
|
|
495
|
+
for band in names:
|
|
496
|
+
for model in models:
|
|
497
|
+
terms[model, band]["v_rna"] += block_vrna[band] / len(seeds)
|
|
498
|
+
terms[model, band]["c_model"] += block_c[model, band] / len(seeds)
|
|
499
|
+
if step:
|
|
500
|
+
step(stop, genes)
|
|
501
|
+
modes = {band: int(mask.sum()) for band, mask in masks.items()}
|
|
502
|
+
return dict(terms=terms, halves=halves, modes=modes, seed_vrna=seed_vrna, seed_c=seed_c, vrna_g=vrna_g, va_g=va_g,
|
|
503
|
+
vb_g=vb_g, c_g=c_g, vm_g=vm_g, grid=(ny, nx), sites=n)
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _grid_result(sections, genes, models, spacing_um, settings, notes) -> Result:
|
|
507
|
+
order = [band for band, *_ in reversed(fourier.WAVELENGTH_BANDS)] # broadest first
|
|
508
|
+
lows = {band: low for band, low, _ in fourier.WAVELENGTH_BANDS}
|
|
509
|
+
scale_rows, gene_rows, summaries = [], [], []
|
|
510
|
+
for s in sections:
|
|
511
|
+
out = s["out"]
|
|
512
|
+
present = [band for band in order if out["modes"][band] > 0]
|
|
513
|
+
for model in models:
|
|
514
|
+
rows = []
|
|
515
|
+
for k, band in enumerate(present):
|
|
516
|
+
t, h = out["terms"][model, band], out["halves"][band]
|
|
517
|
+
rows.append(_record(model, s["name"], GRID_LABELS[band], k + 1, t["c_model"], t["v_model"], t["v_rna"], h["v_a"], h["v_b"],
|
|
518
|
+
out["seed_c"][model, band] / 1.0, out["seed_vrna"][band],
|
|
519
|
+
band=band, fourier_modes=out["modes"][band],
|
|
520
|
+
partial_window=bool(lows[band] < 2.0 * spacing_um)))
|
|
521
|
+
# per-seed values are sums over gene blocks of that seed's terms
|
|
522
|
+
scale_rows.extend(rows)
|
|
523
|
+
stack = lambda d: np.stack([d[band] for band in present])
|
|
524
|
+
gene_rows.extend(_gene_rows(model, s["name"], genes, [GRID_LABELS[b] for b in present], stack(out["c_g"][model]),
|
|
525
|
+
stack(out["vm_g"][model]), stack(out["vrna_g"]), stack(out["va_g"]), stack(out["vb_g"]),
|
|
526
|
+
s["eligible"]))
|
|
527
|
+
rsc = [row["rsc"] for row in rows]
|
|
528
|
+
finest = next((row for row in reversed(rows) if math.isfinite(row["rsc"])), None)
|
|
529
|
+
summaries.append(dict(model=model, section=s["name"], broadest_rsc=rsc[0] if rsc else math.nan,
|
|
530
|
+
finest_rsc=finest["rsc"] if finest else math.nan, finest_scale=finest["scale"] if finest else "",
|
|
531
|
+
broadest_minus_finest=(rsc[0] - finest["rsc"]) if finest and rsc and math.isfinite(rsc[0]) else math.nan,
|
|
532
|
+
whole_section_pearson_r=s["pearson"][model], n_locations=s["sites"], n_genes=int(np.sum(s["eligible"]))))
|
|
533
|
+
scales, summary = pd.DataFrame(scale_rows), pd.DataFrame(summaries)
|
|
534
|
+
if len(sections) > 1:
|
|
535
|
+
scales, summary = _all_sections_mean(scales, summary)
|
|
536
|
+
return Result("grid", scales, summary, pd.DataFrame(gene_rows), settings, notes)
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
# ---------------------------------------------------------------- cells
|
|
540
|
+
|
|
541
|
+
PAIRS = (("A", "A"), ("A", "B"), ("A", "P"), ("B", "B"), ("B", "P"), ("P", "P"))
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def _band_grams_by_gene(diffusion: sparse.csr_matrix, matrices: dict[str, np.ndarray]) -> dict[str, dict[tuple[str, str], np.ndarray]]:
|
|
545
|
+
"""``cell_core.lazy_band_grams`` for each gene: the same products, summed over cells only."""
|
|
546
|
+
names = list(matrices)
|
|
547
|
+
original = {name: np.asarray(matrices[name], np.float32) for name in names}
|
|
548
|
+
state = {name: value.copy() for name, value in original.items()}
|
|
549
|
+
milestones: dict[int, dict[tuple[str, str], np.ndarray]] = {}
|
|
550
|
+
for step in range(17):
|
|
551
|
+
if step in cell_core.MILESTONES:
|
|
552
|
+
gram = {}
|
|
553
|
+
for i, left in enumerate(names):
|
|
554
|
+
for right in names[i:]:
|
|
555
|
+
gram[(left, right)] = 0.5 * (np.sum(original[left] * state[right], axis=0, dtype=np.float64)
|
|
556
|
+
+ np.sum(original[right] * state[left], axis=0, dtype=np.float64))
|
|
557
|
+
milestones[step] = gram
|
|
558
|
+
if step < 16:
|
|
559
|
+
for name in names:
|
|
560
|
+
state[name] = diffusion @ state[name]
|
|
561
|
+
out = {}
|
|
562
|
+
for band, high, low in (("B1", 16, None), ("B2", 8, 16), ("B3", 4, 8), ("B4", 2, 4), ("B5", 0, 2)):
|
|
563
|
+
out[band] = {pair: value if low is None else value - milestones[low][pair] for pair, value in milestones[high].items()}
|
|
564
|
+
return out
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def score_cell_view(counts: sparse.csr_matrix, prediction: np.ndarray, coords: np.ndarray, membership: sparse.csr_matrix,
|
|
568
|
+
view_coords: np.ndarray, seeds=CELL_SEEDS, specimen: str = "section", step=None) -> dict:
|
|
569
|
+
"""One graph view of a section, as ``strsc.cell_score``: the graph is built on the view's cells,
|
|
570
|
+
the counts are split on all cells, and the halves and the prediction are transformed and
|
|
571
|
+
filtered on the view's graph."""
|
|
572
|
+
diffusion, retained, labels, receipt = cell_core.build_lazy_graph(view_coords, CELL_GRAPH["neighbors"], CELL_GRAPH["max_edge_um"],
|
|
573
|
+
CELL_GRAPH["minimum_component_cells"])
|
|
574
|
+
exposure = np.ones(membership.shape[0])
|
|
575
|
+
pred_view = np.asarray(membership @ prediction, np.float32)
|
|
576
|
+
pred_view /= np.asarray(exposure, np.float32)[:, None]
|
|
577
|
+
pred_view = pred_view[retained]
|
|
578
|
+
genes = counts.shape[1]
|
|
579
|
+
per_seed = {seed: {band: {pair: np.zeros(genes) for pair in PAIRS} for band in cell_core.BANDS} for seed in seeds}
|
|
580
|
+
eligible_any = np.zeros(genes, dtype=bool)
|
|
581
|
+
pcc = np.full(genes, np.nan)
|
|
582
|
+
blocks = [slice(start, min(start + CELL_GENE_BLOCK, genes)) for start in range(0, genes, CELL_GENE_BLOCK)]
|
|
583
|
+
view_counts = (membership @ counts).tocsr()[retained]
|
|
584
|
+
for block in blocks:
|
|
585
|
+
truth = np.log1p(np.asarray(view_counts[:, block].todense(), dtype=np.float64))
|
|
586
|
+
pcc[block] = cal._corr_cols(pred_view[:, block].astype(np.float64), truth)
|
|
587
|
+
del view_counts
|
|
588
|
+
# One count split at a time, so that memory does not grow with the number of splits. Every split
|
|
589
|
+
# and every gene block is computed on its own, so the order changes none of the numbers.
|
|
590
|
+
for k, seed in enumerate(seeds):
|
|
591
|
+
a, b = cell_core.split_counts(counts, specimen, seed)
|
|
592
|
+
a_view, b_view = (membership @ a).tocsr()[retained], (membership @ b).tocsr()[retained]
|
|
593
|
+
del a, b
|
|
594
|
+
for i, block in enumerate(blocks):
|
|
595
|
+
pp, pred_eligible = cell_core.transform_prediction(pred_view[:, block], labels)
|
|
596
|
+
aa, bb, rna_eligible = cell_core.transform_rna_pair(a_view[:, block], b_view[:, block], exposure[retained], labels)
|
|
597
|
+
eligible = pred_eligible & rna_eligible
|
|
598
|
+
aa[:, ~eligible] = 0
|
|
599
|
+
bb[:, ~eligible] = 0
|
|
600
|
+
local_p = pp.copy()
|
|
601
|
+
local_p[:, ~eligible] = 0
|
|
602
|
+
eligible_any[block] |= eligible
|
|
603
|
+
grams = _band_grams_by_gene(diffusion, {"A": aa, "B": bb, "P": local_p})
|
|
604
|
+
for band in cell_core.BANDS:
|
|
605
|
+
for pair in PAIRS:
|
|
606
|
+
per_seed[seed][band][pair][block] += grams[band][pair]
|
|
607
|
+
if step:
|
|
608
|
+
step(k * len(blocks) + i + 1, len(seeds) * len(blocks))
|
|
609
|
+
del a_view, b_view
|
|
610
|
+
return dict(per_seed=per_seed, eligible=eligible_any, receipt=receipt, pearson=_nanmean(pcc), cells=int(retained.sum()))
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def _cell_terms(view, seeds) -> dict:
|
|
614
|
+
"""Totals over genes, averaged over seeds; per-gene terms averaged over seeds; and per-seed totals."""
|
|
615
|
+
mean_g = {band: {pair: np.mean([view["per_seed"][s][band][pair] for s in seeds], axis=0) for pair in PAIRS} for band in cell_core.BANDS}
|
|
616
|
+
total = {band: {pair: float(np.sum(mean_g[band][pair])) for pair in PAIRS} for band in cell_core.BANDS}
|
|
617
|
+
seed_total = {band: {pair: np.asarray([float(np.sum(view["per_seed"][s][band][pair])) for s in seeds]) for pair in PAIRS} for band in cell_core.BANDS}
|
|
618
|
+
return dict(mean_g=mean_g, total=total, seed_total=seed_total)
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def _cell_result(views, genes, settings, notes) -> Result:
|
|
622
|
+
scale_rows, gene_rows, summaries = [], [], []
|
|
623
|
+
bands = list(cell_core.BANDS)
|
|
624
|
+
for v in views:
|
|
625
|
+
terms = _cell_terms(v["out"], v["seeds"])
|
|
626
|
+
t, st, g = terms["total"], terms["seed_total"], terms["mean_g"]
|
|
627
|
+
rows = []
|
|
628
|
+
for k, band in enumerate(bands):
|
|
629
|
+
c = 0.5 * (t[band][("A", "P")] + t[band][("B", "P")])
|
|
630
|
+
seeds_c = 0.5 * (st[band][("A", "P")] + st[band][("B", "P")])
|
|
631
|
+
rows.append(_record(v["model"], v["section"], RELATIVE[k], k + 1, c, t[band][("P", "P")], t[band][("A", "B")],
|
|
632
|
+
t[band][("A", "A")], t[band][("B", "B")], seeds_c, st[band][("A", "B")], population=v["population"]))
|
|
633
|
+
scale_rows.extend(rows)
|
|
634
|
+
combined = {}
|
|
635
|
+
for label, members in (("fine", ("B4", "B5")), ("broad", ("B1", "B2"))):
|
|
636
|
+
c = sum(0.5 * (t[b][("A", "P")] + t[b][("B", "P")]) for b in members)
|
|
637
|
+
vm = sum(t[b][("P", "P")] for b in members)
|
|
638
|
+
vr = sum(t[b][("A", "B")] for b in members)
|
|
639
|
+
va_ = sum(t[b][("A", "A")] for b in members)
|
|
640
|
+
vb_ = sum(t[b][("B", "B")] for b in members)
|
|
641
|
+
combined[label] = _record(v["model"], v["section"], label, 0, c, vm, vr, va_, vb_)
|
|
642
|
+
stack = lambda pair: np.stack([g[b][pair] for b in bands])
|
|
643
|
+
gene_rows.extend({**row, "population": v["population"]} for row in _gene_rows(
|
|
644
|
+
v["model"], v["section"], genes, RELATIVE, 0.5 * (stack(("A", "P")) + stack(("B", "P"))), stack(("P", "P")),
|
|
645
|
+
stack(("A", "B")), stack(("A", "A")), stack(("B", "B")), v["out"]["eligible"]))
|
|
646
|
+
rsc = [row["rsc"] for row in rows]
|
|
647
|
+
summaries.append(dict(model=v["model"], section=v["section"], population=v["population"],
|
|
648
|
+
fine_rsc=combined["fine"]["rsc"], broad_rsc=combined["broad"]["rsc"],
|
|
649
|
+
broadest_minus_finest=rsc[0] - rsc[4] if all(map(math.isfinite, (rsc[0], rsc[4]))) else math.nan,
|
|
650
|
+
fine_split_half_reliability=combined["fine"]["split_half_reliability"],
|
|
651
|
+
whole_section_pearson_r=v["out"]["pearson"], n_locations=v["out"]["cells"],
|
|
652
|
+
n_genes=int(np.sum(v["out"]["eligible"]))))
|
|
653
|
+
scales, summary = pd.DataFrame(scale_rows), pd.DataFrame(summaries)
|
|
654
|
+
sections = summary["section"].unique()
|
|
655
|
+
if len(sections) > 1:
|
|
656
|
+
# As for the patients of the paper: C, V_model and V_RNA averaged over sections before the ratio.
|
|
657
|
+
pooled_rows, pooled_summary = [], []
|
|
658
|
+
for (model, population), frame in scales.groupby(["model", "population"], sort=False):
|
|
659
|
+
for (scale, order), part in frame.groupby(["scale", "scale_order"], sort=False):
|
|
660
|
+
m = part[["C", "V_model", "V_RNA", "V_A", "V_B"]].mean()
|
|
661
|
+
pooled_rows.append(_record(model, "All sections", scale, order, m.C, m.V_model, m.V_RNA, m.V_A, m.V_B, population=population))
|
|
662
|
+
fine = frame[frame.scale.isin(["Fine", "Finest"])].groupby("section")[["C", "V_model", "V_RNA", "V_A", "V_B"]].sum().mean()
|
|
663
|
+
broad = frame[frame.scale.isin(["Broadest", "Broad"])].groupby("section")[["C", "V_model", "V_RNA", "V_A", "V_B"]].sum().mean()
|
|
664
|
+
f = _record(model, "All sections", "fine", 0, fine.C, fine.V_model, fine.V_RNA, fine.V_A, fine.V_B)
|
|
665
|
+
b = _record(model, "All sections", "broad", 0, broad.C, broad.V_model, broad.V_RNA, broad.V_A, broad.V_B)
|
|
666
|
+
ordered = [r for r in pooled_rows if r["model"] == model and r["population"] == population]
|
|
667
|
+
pooled_summary.append(dict(model=model, section="All sections", population=population, fine_rsc=f["rsc"], broad_rsc=b["rsc"],
|
|
668
|
+
broadest_minus_finest=ordered[0]["rsc"] - ordered[4]["rsc"],
|
|
669
|
+
fine_split_half_reliability=f["split_half_reliability"],
|
|
670
|
+
whole_section_pearson_r=float(summary[(summary.model == model) & (summary.population == population)].whole_section_pearson_r.mean())))
|
|
671
|
+
scales = pd.concat([scales, pd.DataFrame(pooled_rows)], ignore_index=True)
|
|
672
|
+
summary = pd.concat([summary, pd.DataFrame(pooled_summary)], ignore_index=True)
|
|
673
|
+
return Result("cells", scales, summary, pd.DataFrame(gene_rows), settings, notes)
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
# ---------------------------------------------------------------- the call
|
|
677
|
+
|
|
678
|
+
def rsc(counts: Counts | str | Path, predictions, layout: str = "spots", *, spacing_um: float | None = None,
|
|
679
|
+
cell_class: str = "cell_class", within: str | Sequence[str] | None = None, all_cells: bool = True,
|
|
680
|
+
genes: Iterable[str] | None = None, splits: int | None = None, max_spots: int | None = None,
|
|
681
|
+
progress: Progress = None) -> Result:
|
|
682
|
+
"""Reproducible-signal correlation of one or more predictions, at each spatial scale.
|
|
683
|
+
|
|
684
|
+
``counts``: a :class:`Counts` from :func:`read_counts`, or a path. ``predictions``: a file, a
|
|
685
|
+
:class:`Prediction`, or a dict of model name to file or array (locations × genes of the counts).
|
|
686
|
+
``layout``: ``"spots"``, ``"grid"`` or ``"cells"`` (see the module docstring). For ``"grid"``,
|
|
687
|
+
``spacing_um`` is the bin spacing (read from Visium HD files when not given). For ``"cells"``,
|
|
688
|
+
``within`` names one or more values of the ``cell_class`` column to score on their own graphs,
|
|
689
|
+
and ``all_cells`` whether to score all cells as well. ``genes`` restricts the genes, ``splits``
|
|
690
|
+
uses only the first count splits of the paper (fewer splits give noisier values).
|
|
691
|
+
"""
|
|
692
|
+
if layout not in LAYOUTS:
|
|
693
|
+
raise InputError(f"layout must be one of {', '.join(LAYOUTS)}.")
|
|
694
|
+
if not isinstance(counts, Counts):
|
|
695
|
+
counts = read_counts(counts)
|
|
696
|
+
models = _models(predictions, counts)
|
|
697
|
+
shared, columns, complete, notes = _common(counts, models, genes)
|
|
698
|
+
say = progress or (lambda _: None)
|
|
699
|
+
if layout == "spots":
|
|
700
|
+
seeds = SPOT_SEEDS[: splits or len(SPOT_SEEDS)]
|
|
701
|
+
sections = []
|
|
702
|
+
for name in counts.section_names():
|
|
703
|
+
rows = np.flatnonzero((counts.sections == name) & complete)
|
|
704
|
+
if len(rows) < 20:
|
|
705
|
+
raise InputError(f"Section {name} has fewer than 20 spots with every prediction.")
|
|
706
|
+
values = np.asarray(counts.counts[rows][:, columns].todense(), dtype=np.int64)
|
|
707
|
+
sections.append(dict(name=name, counts=values, xy=counts.xy[rows],
|
|
708
|
+
predictions={m.name: m.values[np.ix_(rows, columns)] for m in models}))
|
|
709
|
+
weighted = len(sections) >= 2
|
|
710
|
+
if not weighted:
|
|
711
|
+
notes.append("One section: each gene's two halves are divided by a common factor, as for the colorectal bins and breast cancer cells of the paper. "
|
|
712
|
+
"With two or more sections the genes are weighted by the agreement of their halves in the other sections, as for the HEST samples; the two give different values.")
|
|
713
|
+
total = len(sections) * (1 + len(seeds))
|
|
714
|
+
done = [0]
|
|
715
|
+
|
|
716
|
+
def step(name, stage):
|
|
717
|
+
done[0] += 1
|
|
718
|
+
say(dict(stage=stage, section=name, done=done[0], total=total))
|
|
719
|
+
records = score_spot_sections(sections, shared, seeds, step, max_spots)
|
|
720
|
+
settings = _settings("spots", counts, shared, models, seeds, transform="log(1 + 2 N_half), centered by gene",
|
|
721
|
+
scales="five equal-rank groups of the nonconstant eigenvectors of the neighbor-graph Laplacian (edges within 1.25 × the median nearest-neighbor distance, components of at least 20 spots)",
|
|
722
|
+
gene_weighting="agreement of the halves in the other sections (HEST)" if weighted else "common factor of each gene's halves",
|
|
723
|
+
fine_scale="mean RSC at the Fine and Finest scales", broad_scale="mean RSC at the Broadest and Broad scales",
|
|
724
|
+
task_label=records[0]["task"])
|
|
725
|
+
return _spot_result(records, shared, settings, notes)
|
|
726
|
+
if layout == "grid":
|
|
727
|
+
spacing = float(spacing_um or counts.spacing_um or 0)
|
|
728
|
+
if not math.isfinite(spacing) or spacing <= 0:
|
|
729
|
+
raise InputError("Give the grid spacing in micrometers.")
|
|
730
|
+
if counts.unit != "um" and spacing_um is None:
|
|
731
|
+
raise InputError("The grid needs coordinates in micrometers: give um_per_unit when reading the counts, and the spacing.")
|
|
732
|
+
seeds = GRID_SEEDS[: splits or len(GRID_SEEDS)]
|
|
733
|
+
area_all = counts.column("area")
|
|
734
|
+
if area_all is None:
|
|
735
|
+
notes.append(f"No area column, so every bin counts as a full {spacing:g} × {spacing:g} µm square of tissue.")
|
|
736
|
+
sections = []
|
|
737
|
+
names = counts.section_names()
|
|
738
|
+
parts = max(1, math.ceil(len(columns) / fourier.GENE_BLOCK))
|
|
739
|
+
for index, name in enumerate(names):
|
|
740
|
+
rows = np.flatnonzero((counts.sections == name) & complete)
|
|
741
|
+
block = counts.counts[rows][:, columns].tocsr()
|
|
742
|
+
eligible = np.asarray(block.sum(axis=0)).reshape(-1) > 0
|
|
743
|
+
area = np.asarray(area_all[rows], dtype=float) if area_all is not None else np.full(len(rows), spacing * spacing)
|
|
744
|
+
prediction = np.stack([m.values[np.ix_(rows, columns)] for m in models], axis=2)
|
|
745
|
+
keep_genes = np.flatnonzero(eligible)
|
|
746
|
+
if len(keep_genes) < len(columns):
|
|
747
|
+
notes.append(f"Section {name}: {len(columns) - len(keep_genes):,} gene(s) without a single molecule were left out.")
|
|
748
|
+
say(dict(stage="section", section=name, done=index * parts, total=len(names) * parts))
|
|
749
|
+
|
|
750
|
+
def grid_step(stop, _genes, name=name, index=index):
|
|
751
|
+
say(dict(stage="section", section=name, done=index * parts + math.ceil(stop / fourier.GENE_BLOCK), total=len(names) * parts))
|
|
752
|
+
out = score_grid_section(block[:, keep_genes], prediction[:, keep_genes, :], counts.xy[rows], area, spacing_um=spacing,
|
|
753
|
+
models=[m.name for m in models], seeds=seeds, step=grid_step)
|
|
754
|
+
truth = np.sqrt(np.asarray(block[:, keep_genes].todense(), dtype=np.float64) / area[:, None])
|
|
755
|
+
pearson = {m.name: _nanmean(cal._corr_cols(prediction[:, keep_genes, j], truth)) for j, m in enumerate(models)}
|
|
756
|
+
sections.append(dict(name=name, out=out, eligible=np.ones(len(keep_genes), bool), sites=len(rows), pearson=pearson,
|
|
757
|
+
genes=[shared[g] for g in keep_genes]))
|
|
758
|
+
genes_used = sections[0]["genes"] if len(sections) == 1 else shared
|
|
759
|
+
if len(sections) > 1:
|
|
760
|
+
for s in sections: # per-gene tables need a common gene axis
|
|
761
|
+
if s["genes"] != shared:
|
|
762
|
+
s["eligible"] = np.asarray([g in set(s["genes"]) for g in shared])
|
|
763
|
+
_expand_grid_genes(s, shared)
|
|
764
|
+
settings = _settings("grid", counts, genes_used, models, seeds, spacing_um=spacing,
|
|
765
|
+
transform="sqrt(N_half / (0.5 a)), centered, each gene's halves divided by a common factor",
|
|
766
|
+
scales="seven physical scales of a two-dimensional Fourier basis: <128, 128–256, 256–512, 512–1,024, 1,024–2,048, 2,048–4,096 and ≥4,096 µm (full spatial cycles)",
|
|
767
|
+
gene_weighting="common factor of each gene's halves")
|
|
768
|
+
return _grid_result(sections, genes_used, [m.name for m in models], spacing, settings, notes)
|
|
769
|
+
# cells
|
|
770
|
+
if counts.unit != "um":
|
|
771
|
+
notes.append("The coordinates are taken to be micrometers: the cell graph links neighbors within 40 µm.")
|
|
772
|
+
seeds = CELL_SEEDS[: splits or len(CELL_SEEDS)]
|
|
773
|
+
populations = [within] if isinstance(within, str) else list(within or [])
|
|
774
|
+
labels_all = counts.column(cell_class)
|
|
775
|
+
if populations and labels_all is None:
|
|
776
|
+
raise InputError(f"No {cell_class} column, so the cells cannot be scored within a class.")
|
|
777
|
+
views = []
|
|
778
|
+
names = counts.section_names()
|
|
779
|
+
plan = [(name, pop) for name in names for pop in ((["All cells"] if all_cells or not populations else []) + populations)]
|
|
780
|
+
parts = len(seeds) * max(1, math.ceil(len(columns) / CELL_GENE_BLOCK))
|
|
781
|
+
total = len(plan) * len(models) * parts
|
|
782
|
+
for index, (name, population) in enumerate(plan):
|
|
783
|
+
rows = np.flatnonzero((counts.sections == name) & complete)
|
|
784
|
+
block = counts.counts[rows][:, columns].tocsr()
|
|
785
|
+
prediction = {m.name: m.values[np.ix_(rows, columns)].astype(np.float32) for m in models}
|
|
786
|
+
coords = counts.xy[rows]
|
|
787
|
+
if population == "All cells":
|
|
788
|
+
membership, view_coords = sparse.eye(len(rows), format="csr", dtype=np.float32), coords
|
|
789
|
+
else:
|
|
790
|
+
source = np.flatnonzero(np.asarray(labels_all[rows]).astype(str) == str(population))
|
|
791
|
+
if len(source) < CELL_GRAPH["minimum_component_cells"]:
|
|
792
|
+
raise InputError(f"Section {name}: fewer than {CELL_GRAPH['minimum_component_cells']} cells of class {population}.")
|
|
793
|
+
membership = sparse.coo_matrix((np.ones(len(source), np.float32), (np.arange(len(source)), source)),
|
|
794
|
+
shape=(len(source), len(rows))).tocsr()
|
|
795
|
+
view_coords = coords[source]
|
|
796
|
+
for k, (model, values) in enumerate(prediction.items()):
|
|
797
|
+
first = (index * len(models) + k) * parts
|
|
798
|
+
say(dict(stage="view", section=name, population=population, model=model, done=first, total=total))
|
|
799
|
+
|
|
800
|
+
def cell_step(done, _parts, name=name, population=population, model=model, first=first):
|
|
801
|
+
say(dict(stage="view", section=name, population=population, model=model, done=first + done, total=total))
|
|
802
|
+
try:
|
|
803
|
+
out = score_cell_view(block, values, coords, membership, view_coords, seeds, name, step=cell_step)
|
|
804
|
+
except cell_core.ContractError as error:
|
|
805
|
+
raise InputError(f"Section {name}, {population}: the cell graph could not be built ({error}).") from None
|
|
806
|
+
views.append(dict(model=model, section=name, population=population, out=out, seeds=seeds))
|
|
807
|
+
settings = _settings("cells", counts, shared, models, seeds, transform="log(1 + 2 N_half), centered within graph components, each gene's halves divided by a common factor",
|
|
808
|
+
scales="lazy symmetric diffusion S on the graph of the 8 nearest neighbors within 40 µm (Gaussian edge weights, components of at least 20 cells): S^16, S^8 − S^16, S^4 − S^8, S^2 − S^4, I − S^2",
|
|
809
|
+
gene_weighting="common factor of each gene's halves", fine_scale="C, V_model and V_RNA summed over the Fine and Finest scales",
|
|
810
|
+
populations=["All cells"] * bool(all_cells or not populations) + populations, cell_class_column=cell_class if populations else None)
|
|
811
|
+
return _cell_result(views, shared, settings, notes)
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
def _expand_grid_genes(section: dict, shared: list[str]) -> None:
|
|
815
|
+
"""Put a section's per-gene arrays on the common gene axis, with zeros for genes it left out."""
|
|
816
|
+
index = [shared.index(g) for g in section["genes"]]
|
|
817
|
+
out = section["out"]
|
|
818
|
+
for key in ("vrna_g", "va_g", "vb_g"):
|
|
819
|
+
for band, values in out[key].items():
|
|
820
|
+
full = np.zeros(len(shared))
|
|
821
|
+
full[index] = values
|
|
822
|
+
out[key][band] = full
|
|
823
|
+
for key in ("c_g", "vm_g"):
|
|
824
|
+
for model in out[key]:
|
|
825
|
+
for band, values in out[key][model].items():
|
|
826
|
+
full = np.zeros(len(shared))
|
|
827
|
+
full[index] = values
|
|
828
|
+
out[key][model][band] = full
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
# ---------------------------------------------------------------- regions
|
|
832
|
+
|
|
833
|
+
def _expression(counts: np.ndarray, totals: np.ndarray) -> np.ndarray:
|
|
834
|
+
"""log(1 + 10,000 N / total), as in the TLS analysis of the paper; a spot with no molecules stays at 0."""
|
|
835
|
+
return np.log1p(1e4 * counts / np.maximum(totals, 1)[:, None])
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
def _group_means(values: np.ndarray, codes: np.ndarray, sizes: np.ndarray) -> np.ndarray:
|
|
839
|
+
"""The mean of ``values`` over each group, at every location of the group."""
|
|
840
|
+
return (np.bincount(codes, weights=values, minlength=len(sizes)) / sizes)[codes]
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
def _focus_units(xy: np.ndarray, focus_rows: np.ndarray) -> np.ndarray:
|
|
844
|
+
"""Connected groups of focus spots, such as single TLS: focus spots are joined when they are
|
|
845
|
+
neighbors, closer than 1.25 times the median nearest-neighbor distance of the section's spots."""
|
|
846
|
+
from scipy.sparse.csgraph import connected_components
|
|
847
|
+
from scipy.spatial import cKDTree
|
|
848
|
+
|
|
849
|
+
near = cKDTree(xy).query(xy, k=2)[0][:, 1]
|
|
850
|
+
radius = cal.EDGE_MULTIPLIER * float(np.median(near[near > 0]))
|
|
851
|
+
pts = xy[focus_rows]
|
|
852
|
+
pairs = cKDTree(pts).query_pairs(radius, output_type="ndarray")
|
|
853
|
+
graph = sparse.coo_matrix((np.ones(len(pairs)), (pairs[:, 0], pairs[:, 1])), shape=(len(pts), len(pts)))
|
|
854
|
+
return connected_components(graph, directed=False)[1]
|
|
855
|
+
|
|
856
|
+
|
|
857
|
+
REGION_TERMS = ("whole", "between", "within", "focus", "focus_between", "focus_within")
|
|
858
|
+
REGION_RESAMPLES = 2000 # resamples of focus structures, and of blocks of 10 x 10 spots, as in the paper
|
|
859
|
+
REGION_BLOCK = 10
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
def _region_blocks(xy: np.ndarray, array_row, array_col) -> np.ndarray:
|
|
863
|
+
"""Blocks of 10 x 10 spots, the resampling units of the terms that are not within the focus region.
|
|
864
|
+
With the spot grid of Space Ranger these are 10 array rows by 10 spots of a row (20 array columns
|
|
865
|
+
on Visium's staggered grid); otherwise blocks of 10 spot spacings each way, from the coordinates."""
|
|
866
|
+
if array_row is not None and array_col is not None:
|
|
867
|
+
r = np.asarray(array_row, dtype=np.float64)
|
|
868
|
+
q = np.asarray(array_col, dtype=np.float64)
|
|
869
|
+
if np.isfinite(r).all() and np.isfinite(q).all():
|
|
870
|
+
order = np.lexsort((q, r))
|
|
871
|
+
same_row = np.diff(r[order]) == 0
|
|
872
|
+
steps = np.diff(q[order])[same_row]
|
|
873
|
+
step = float(np.min(steps[steps > 0])) if (steps > 0).any() else 1.0
|
|
874
|
+
keys = np.column_stack([np.floor(r / REGION_BLOCK), np.floor(q / (REGION_BLOCK * step))])
|
|
875
|
+
return np.unique(keys, axis=0, return_inverse=True)[1].reshape(-1)
|
|
876
|
+
from scipy.spatial import cKDTree
|
|
877
|
+
|
|
878
|
+
dist, nearest = cKDTree(xy).query(xy, k=2)
|
|
879
|
+
d0 = float(np.median(dist[:, 1][dist[:, 1] > 0]))
|
|
880
|
+
dy = np.abs(xy[nearest[:, 1], 1] - xy[:, 1])
|
|
881
|
+
r0 = float(np.median(dy[dy > 1e-9 * d0])) if (dy > 1e-9 * d0).any() else d0
|
|
882
|
+
keys = np.column_stack([np.floor((xy[:, 1] - xy[:, 1].min()) / (REGION_BLOCK * r0)),
|
|
883
|
+
np.floor((xy[:, 0] - xy[:, 0].min()) / (REGION_BLOCK * d0))])
|
|
884
|
+
return np.unique(keys, axis=0, return_inverse=True)[1].reshape(-1)
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
def _resample(per_spot: dict, codes: np.ndarray, rng, draws: int) -> dict:
|
|
888
|
+
"""Means over spots for resamples of whole units (focus structures or blocks): each draw takes as
|
|
889
|
+
many units as there are, with replacement."""
|
|
890
|
+
n = int(codes.max()) + 1
|
|
891
|
+
sums = {key: np.bincount(codes, weights=value, minlength=n) for key, value in per_spot.items()}
|
|
892
|
+
count = np.bincount(codes, minlength=n).astype(np.float64)
|
|
893
|
+
weights = np.stack([np.bincount(rng.integers(0, n, n), minlength=n) for _ in range(draws)]).astype(np.float64)
|
|
894
|
+
denominator = weights @ count
|
|
895
|
+
return {key: (weights @ value) / denominator for key, value in sums.items()}
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
def _region_parts(v: np.ndarray, region: tuple, units: tuple | None, focus_rows: np.ndarray | None, sd_all: float, sd_focus: float) -> dict:
|
|
899
|
+
"""One gene's terms: the whole section, between and within regions, and within the focus region,
|
|
900
|
+
between its structures and within one structure. Each is divided by the standard deviation of the
|
|
901
|
+
gene's unsplit expression over the spots of the term."""
|
|
902
|
+
rm = _group_means(v, *region)
|
|
903
|
+
res = v - rm
|
|
904
|
+
sa = sd_all if sd_all > 0 else np.inf
|
|
905
|
+
out = dict(whole=(v - v.mean()) / sa, between=(rm - v.mean()) / sa, within=res / sa)
|
|
906
|
+
if units is not None:
|
|
907
|
+
st = sd_focus if sd_focus > 0 else np.inf
|
|
908
|
+
rt = res[focus_rows]
|
|
909
|
+
cm = _group_means(rt, *units)
|
|
910
|
+
out.update(focus=rt / st, focus_between=cm / st, focus_within=(rt - cm) / st)
|
|
911
|
+
return out
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
def _tested(point: np.ndarray, boot: np.ndarray) -> tuple[bool, float, float]:
|
|
915
|
+
"""The paper's rule: a term is evaluated only when the lower limit of the 95% interval of its
|
|
916
|
+
split-half reliability is above zero. ``point`` and the rows of ``boot`` are (C, V_model, V_RNA)."""
|
|
917
|
+
testable = bool(np.percentile(boot[:, 2], 2.5) > 0)
|
|
918
|
+
ok = (boot[:, 1] > TOL) & (boot[:, 2] > 0)
|
|
919
|
+
ratios = boot[ok, 0] / np.sqrt(boot[ok, 1] * boot[ok, 2])
|
|
920
|
+
if testable and ok.mean() > 0.9 and point[1] > TOL and point[2] > 0:
|
|
921
|
+
return True, float(np.percentile(ratios, 2.5)), float(np.percentile(ratios, 97.5))
|
|
922
|
+
return testable, math.nan, math.nan
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
def regions(counts: Counts | str | Path, predictions, *, region: str = "region", focus: str | None = None,
|
|
926
|
+
genes: Iterable[str] | None = None, splits: int = REGION_SPLITS, resamples: int = REGION_RESAMPLES,
|
|
927
|
+
progress: Progress = None) -> Result:
|
|
928
|
+
"""Agreement between and within annotated regions, as for the TLS sections of the paper
|
|
929
|
+
(Methods, "Decomposition by region" and "TLS-associated genes"). Each prediction and each RNA half
|
|
930
|
+
is split into the mean of its region and the residual from that mean. RSC is computed for the
|
|
931
|
+
whole section, between regions (the region means) and within regions (the residuals); when
|
|
932
|
+
``focus`` names a region such as TLS, also within it (the residuals of its spots), between its
|
|
933
|
+
structures (connected groups of its spots) and within one structure. There is no spatial
|
|
934
|
+
decomposition.
|
|
935
|
+
|
|
936
|
+
As in the paper, measured expression is log(1 + 10,000 N / total), the total over all genes, each
|
|
937
|
+
half with its own total; each count split halves the scored genes' counts gene by gene and the
|
|
938
|
+
remaining molecules of each spot together. Every gene of the RNA halves and of the predictions is
|
|
939
|
+
divided by the standard deviation of the gene's unsplit expression over the spots of the term (all
|
|
940
|
+
spots with a region; the focus spots for the focus terms), so predictions should be on the same
|
|
941
|
+
log scale. C, V_model and V_RNA are summed over genes before the ratio. Intervals are 95%
|
|
942
|
+
percentile intervals from ``resamples`` resamples of focus structures (terms within the focus
|
|
943
|
+
region) and of blocks of 10 x 10 spots (the other terms), and a term is scored in a section only
|
|
944
|
+
when the lower limit of the interval of its split-half reliability is above zero. Over several
|
|
945
|
+
sections, the sums of the sections where a term is scored are averaged, in every resample too."""
|
|
946
|
+
if not isinstance(counts, Counts):
|
|
947
|
+
counts = read_counts(counts)
|
|
948
|
+
labels_all = counts.column(region)
|
|
949
|
+
if labels_all is None:
|
|
950
|
+
raise InputError(f"No {region} column: give a region for every spot.")
|
|
951
|
+
models = _models(predictions, counts)
|
|
952
|
+
shared, columns, complete, notes = _common(counts, models, genes)
|
|
953
|
+
say = progress or (lambda _: None)
|
|
954
|
+
labels_text = np.asarray(["" if (isinstance(v, float) and math.isnan(v)) or v is None else str(v) for v in labels_all], dtype=object)
|
|
955
|
+
labelled = labels_text != ""
|
|
956
|
+
if (~labelled & complete).any():
|
|
957
|
+
notes.append(f"{int((~labelled & complete).sum()):,} spots without a region were left out.")
|
|
958
|
+
ceiling = math.log1p(1e4) # the largest value log(1 + 10,000 N / total) can take
|
|
959
|
+
for m in models:
|
|
960
|
+
top = np.nanmax(m.values[np.ix_(np.flatnonzero(complete & labelled), columns)]) if (complete & labelled).any() else np.nan
|
|
961
|
+
if np.isfinite(top) and top > ceiling + 1e-9:
|
|
962
|
+
notes.append(f"{m.name}: some values are above {ceiling:.2f}, the largest that log(1 + 10,000 N/total) can take, so the "
|
|
963
|
+
"predictions are probably on another scale. Between and within regions each gene is divided by the "
|
|
964
|
+
"standard deviation of the measured expression, as in the paper, so predictions should be on that log scale.")
|
|
965
|
+
labels = {"whole": "Whole section", "between": "Between regions", "within": "Within regions"}
|
|
966
|
+
if focus:
|
|
967
|
+
labels.update(focus=f"Within {focus}", focus_between=f"Between {focus} structures", focus_within=f"Within one {focus} structure")
|
|
968
|
+
kinds = [(k, labels[k]) for k in REGION_TERMS if k in labels]
|
|
969
|
+
order = {k: i + 1 for i, (k, _) in enumerate(kinds)}
|
|
970
|
+
array_row_all, array_col_all = counts.column("array_row"), counts.column("array_col")
|
|
971
|
+
term_rows, gene_rows, pooled_in = [], [], {}
|
|
972
|
+
names = counts.section_names()
|
|
973
|
+
for index, name in enumerate(names):
|
|
974
|
+
rows = np.flatnonzero((counts.sections == name) & complete & labelled)
|
|
975
|
+
if len(rows) < 3:
|
|
976
|
+
raise InputError(f"Section {name}: fewer than three spots with a region and every prediction.")
|
|
977
|
+
say(dict(stage="section", section=name, done=index * splits, total=len(names) * splits))
|
|
978
|
+
totals = np.asarray(counts.counts[rows].sum(axis=1)).reshape(-1).astype(np.int64) # all genes
|
|
979
|
+
c = np.asarray(counts.counts[rows][:, columns].todense(), dtype=np.int32)
|
|
980
|
+
full = _expression(c, totals)
|
|
981
|
+
region_here = labels_text[rows]
|
|
982
|
+
_, codes = np.unique(region_here, return_inverse=True)
|
|
983
|
+
region_groups = (codes, np.bincount(codes).astype(np.float64))
|
|
984
|
+
focus_rows = (region_here == focus) if focus else None
|
|
985
|
+
units = None
|
|
986
|
+
if focus and int(focus_rows.sum()) >= 2:
|
|
987
|
+
unit_codes = _focus_units(counts.xy[rows], focus_rows)
|
|
988
|
+
units = (unit_codes, np.bincount(unit_codes).astype(np.float64))
|
|
989
|
+
notes.append(f"Section {name}: {int(focus_rows.sum()):,} {focus} spots in {len(units[1]):,} structure{'s' if len(units[1]) != 1 else ''}."
|
|
990
|
+
+ (f" With one structure the intervals of the terms within {focus} say nothing, and there is no part between structures." if len(units[1]) == 1 else ""))
|
|
991
|
+
elif focus:
|
|
992
|
+
notes.append(f"Section {name}: fewer than two spots in {focus}, so the terms within {focus} are missing there.")
|
|
993
|
+
terms_here = [(k, label) for k, label in kinds if units is not None or not k.startswith("focus")]
|
|
994
|
+
sizes = {k: (int(focus_rows.sum()) if k.startswith("focus") else len(rows)) for k, _ in terms_here}
|
|
995
|
+
blocks = _region_blocks(counts.xy[rows], None if array_row_all is None else array_row_all[rows],
|
|
996
|
+
None if array_col_all is None else array_col_all[rows])
|
|
997
|
+
sd_all = full.std(axis=0)
|
|
998
|
+
sd_focus = full[focus_rows].std(axis=0) if units is not None else np.zeros(len(columns))
|
|
999
|
+
del full
|
|
1000
|
+
values = {m.name: m.values[np.ix_(rows, columns)] for m in models}
|
|
1001
|
+
n_genes = len(columns)
|
|
1002
|
+
acc = {k: {q: np.zeros(n_genes) for q in ("VR", "VA", "VB")} for k, _ in terms_here}
|
|
1003
|
+
acc_c = {(k, m): np.zeros(n_genes) for k, _ in terms_here for m in values}
|
|
1004
|
+
spot = {k: {q: np.zeros(sizes[k]) for q in ("VR", "VA", "VB")} for k, _ in terms_here} # summed over genes
|
|
1005
|
+
spot_c = {(k, m): np.zeros(sizes[k]) for k, _ in terms_here for m in values}
|
|
1006
|
+
seeds_c = {(k, m): np.zeros(splits) for k, _ in terms_here for m in values}
|
|
1007
|
+
seeds_vr = {k: np.zeros(splits) for k, _ in terms_here}
|
|
1008
|
+
rng = np.random.default_rng(cell_core.stable_seed("strsc.regions", name))
|
|
1009
|
+
rest = totals - c.sum(axis=1, dtype=np.int64)
|
|
1010
|
+
ha = np.empty_like(c)
|
|
1011
|
+
for i in range(splits):
|
|
1012
|
+
for j in range(n_genes): # gene by gene, then the other molecules: the order of the draws
|
|
1013
|
+
ha[:, j] = rng.binomial(c[:, j], 0.5)
|
|
1014
|
+
ta = rng.binomial(rest, 0.5) + ha.sum(axis=1, dtype=np.int64)
|
|
1015
|
+
per_a, per_b = np.maximum(ta, 1)[:, None], np.maximum(totals - ta, 1)[:, None]
|
|
1016
|
+
for g in range(n_genes): # each half one gene at a time: the same values as for all genes at once
|
|
1017
|
+
a_g = np.log1p(1e4 * ha[:, g:g + 1] / per_a)[:, 0]
|
|
1018
|
+
b_g = np.log1p(1e4 * (c[:, g:g + 1] - ha[:, g:g + 1]) / per_b)[:, 0]
|
|
1019
|
+
pa = _region_parts(a_g, region_groups, units, focus_rows, sd_all[g], sd_focus[g])
|
|
1020
|
+
pb = _region_parts(b_g, region_groups, units, focus_rows, sd_all[g], sd_focus[g])
|
|
1021
|
+
pp = {m: _region_parts(values[m][:, g], region_groups, units, focus_rows, sd_all[g], sd_focus[g]) for m in values}
|
|
1022
|
+
for k, _ in terms_here:
|
|
1023
|
+
A, B = pa[k], pb[k]
|
|
1024
|
+
ab = A * B
|
|
1025
|
+
vr = float(np.mean(ab))
|
|
1026
|
+
acc[k]["VR"][g] += vr
|
|
1027
|
+
acc[k]["VA"][g] += float(np.mean(A * A))
|
|
1028
|
+
acc[k]["VB"][g] += float(np.mean(B * B))
|
|
1029
|
+
spot[k]["VR"] += ab
|
|
1030
|
+
spot[k]["VA"] += A * A
|
|
1031
|
+
spot[k]["VB"] += B * B
|
|
1032
|
+
seeds_vr[k][i] += vr
|
|
1033
|
+
for m in values:
|
|
1034
|
+
both = 0.5 * (pp[m][k] * A + pp[m][k] * B)
|
|
1035
|
+
cc = float(np.mean(both))
|
|
1036
|
+
acc_c[k, m][g] += cc
|
|
1037
|
+
spot_c[k, m] += both
|
|
1038
|
+
seeds_c[k, m][i] += cc
|
|
1039
|
+
say(dict(stage="section", section=name, done=index * splits + i + 1, total=len(names) * splits))
|
|
1040
|
+
vm = {(k, m): np.zeros(n_genes) for k, _ in terms_here for m in values}
|
|
1041
|
+
spot_vm = {(k, m): np.zeros(sizes[k]) for k, _ in terms_here for m in values}
|
|
1042
|
+
for g in range(n_genes):
|
|
1043
|
+
for m in values:
|
|
1044
|
+
pp = _region_parts(values[m][:, g], region_groups, units, focus_rows, sd_all[g], sd_focus[g])
|
|
1045
|
+
for k, _ in terms_here:
|
|
1046
|
+
vm[k, m][g] = float(np.mean(pp[k] * pp[k]))
|
|
1047
|
+
spot_vm[k, m] += pp[k] * pp[k]
|
|
1048
|
+
boot_rng = np.random.default_rng(cell_core.stable_seed("strsc.regions.resamples", name))
|
|
1049
|
+
for k, label in terms_here:
|
|
1050
|
+
vr_g, va_g, vb_g = (acc[k][q] / splits for q in ("VR", "VA", "VB"))
|
|
1051
|
+
per_spot = {q: spot[k][q] / splits for q in ("VR", "VA", "VB")}
|
|
1052
|
+
for m in values:
|
|
1053
|
+
per_spot[("C", m)] = spot_c[k, m] / splits
|
|
1054
|
+
per_spot[("VM", m)] = spot_vm[k, m]
|
|
1055
|
+
boot = _resample(per_spot, units[0] if k.startswith("focus") else blocks, boot_rng, resamples)
|
|
1056
|
+
use = (sd_focus if k.startswith("focus") else sd_all) > 0
|
|
1057
|
+
for m in values:
|
|
1058
|
+
c_g, vm_g = acc_c[k, m] / splits, vm[k, m]
|
|
1059
|
+
point = np.array([c_g.sum(), vm_g.sum(), vr_g.sum(), va_g.sum(), vb_g.sum()])
|
|
1060
|
+
bt = np.column_stack([boot[("C", m)], boot[("VM", m)], boot["VR"], boot["VA"], boot["VB"]])
|
|
1061
|
+
testable, lo, hi = _tested(point, bt)
|
|
1062
|
+
row = _record(m, name, label, order[k], *point, seeds_c[k, m], seeds_vr[k], term=k, locations=sizes[k])
|
|
1063
|
+
row.update(testable=testable, rsc_ci_low=lo, rsc_ci_high=hi, V_RNA_ci_low=float(np.percentile(bt[:, 2], 2.5)),
|
|
1064
|
+
V_RNA_ci_high=float(np.percentile(bt[:, 2], 97.5)))
|
|
1065
|
+
if not testable:
|
|
1066
|
+
row.update(rsc=math.nan, status="NOT_TESTABLE")
|
|
1067
|
+
term_rows.append(row)
|
|
1068
|
+
gene_rows.extend({**r, "term": k} for r in _gene_rows(m, name, shared, [label], c_g[None, :], vm_g[None, :],
|
|
1069
|
+
vr_g[None, :], va_g[None, :], vb_g[None, :], use))
|
|
1070
|
+
pooled_in.setdefault((m, k, label), []).append((testable, point, bt))
|
|
1071
|
+
scales = pd.DataFrame(term_rows)
|
|
1072
|
+
if len(names) > 1:
|
|
1073
|
+
pooled = []
|
|
1074
|
+
for (model, kind, label), entries in pooled_in.items():
|
|
1075
|
+
have = [(p_, b_) for t_, p_, b_ in entries if t_]
|
|
1076
|
+
if not have:
|
|
1077
|
+
continue
|
|
1078
|
+
point = np.mean([p_ for p_, _ in have], axis=0)
|
|
1079
|
+
bt = np.mean([b_ for _, b_ in have], axis=0)
|
|
1080
|
+
testable, lo, hi = _tested(point, bt)
|
|
1081
|
+
row = _record(model, "All sections", label, order[kind], *point, term=kind, sections=len(have))
|
|
1082
|
+
row.update(testable=testable, rsc_ci_low=lo, rsc_ci_high=hi, V_RNA_ci_low=float(np.percentile(bt[:, 2], 2.5)),
|
|
1083
|
+
V_RNA_ci_high=float(np.percentile(bt[:, 2], 97.5)))
|
|
1084
|
+
if not testable:
|
|
1085
|
+
row.update(rsc=math.nan, status="NOT_TESTABLE")
|
|
1086
|
+
pooled.append(row)
|
|
1087
|
+
scales = pd.concat([scales, pd.DataFrame(pooled)], ignore_index=True)
|
|
1088
|
+
# every term gets a column, also where it was scored in no section
|
|
1089
|
+
summary_rows = {}
|
|
1090
|
+
for _, r in scales.iterrows():
|
|
1091
|
+
summary_rows.setdefault((r["model"], r["section"]), {"model": r["model"], "section": r["section"]})[r["term"]] = r["rsc"]
|
|
1092
|
+
summary = pd.DataFrame(list(summary_rows.values())).reindex(columns=["model", "section"] + [k for k, _ in kinds])
|
|
1093
|
+
settings = _settings("regions", counts, shared, models, tuple(range(splits)), region_column=region, focus=focus,
|
|
1094
|
+
transform="log(1 + 10,000 N / total over all genes), each half with its own total; each split halves the scored genes' counts gene by gene and the other molecules of each spot together",
|
|
1095
|
+
terms=[label for _, label in kinds],
|
|
1096
|
+
structures=(f"connected groups of {focus} spots: neighbors closer than 1.25 times the median nearest-neighbor distance" if focus else None),
|
|
1097
|
+
gene_weighting="every gene of the RNA halves and of the predictions divided by the standard deviation of its unsplit expression over the spots of the term",
|
|
1098
|
+
intervals=f"95% percentile intervals from {resamples:,} resamples of {focus or 'focus'} structures (terms within {focus or 'the focus region'}) and of blocks of 10 x 10 spots (other terms); a term is scored only when the lower limit for its V_RNA is above zero",
|
|
1099
|
+
pooling="mean over the sections where a term is scored, in every resample too")
|
|
1100
|
+
return Result("regions", scales, summary, pd.DataFrame(gene_rows), settings, notes)
|
|
1101
|
+
|
|
1102
|
+
|
|
1103
|
+
# ---------------------------------------------------------------- the paper's test with the expected expression
|
|
1104
|
+
|
|
1105
|
+
def _expected(values: np.ndarray, transform: Callable[[np.ndarray], np.ndarray]) -> np.ndarray:
|
|
1106
|
+
"""E[transform(X)] for X ~ Poisson(N / 2), for every integer N in ``values``: the expected
|
|
1107
|
+
transformed half-count when the measured counts N are taken as Poisson rates."""
|
|
1108
|
+
from scipy import stats
|
|
1109
|
+
|
|
1110
|
+
unique, inverse = np.unique(values, return_inverse=True)
|
|
1111
|
+
out = np.zeros(len(unique))
|
|
1112
|
+
for i, n in enumerate(unique):
|
|
1113
|
+
mu = n / 2.0
|
|
1114
|
+
if mu == 0:
|
|
1115
|
+
out[i] = float(transform(np.zeros(1))[0])
|
|
1116
|
+
continue
|
|
1117
|
+
k = np.arange(0, int(math.ceil(mu + 12 * math.sqrt(mu) + 30)) + 1)
|
|
1118
|
+
out[i] = float(np.sum(stats.poisson.pmf(k, mu) * transform(k.astype(np.float64))))
|
|
1119
|
+
return out[inverse].reshape(values.shape)
|
|
1120
|
+
|
|
1121
|
+
|
|
1122
|
+
def ideal_check(counts: Counts | str | Path, layout: str = "spots", *, draws: int = 3, spacing_um: float | None = None,
|
|
1123
|
+
cell_class: str = "cell_class", within: str | Sequence[str] | None = None, genes: Iterable[str] | None = None,
|
|
1124
|
+
splits: int | None = None, seed: int = 20261106, progress: Progress = None) -> pd.DataFrame:
|
|
1125
|
+
"""The paper's test of RSC (Methods, "Expected expression as a prediction"): the measured counts are
|
|
1126
|
+
taken as Poisson rates, count matrices are drawn from them, and on each the expected transformed
|
|
1127
|
+
half-count is scored as the prediction. Its RSC should be near 1 at every scale; in the paper it was
|
|
1128
|
+
0.956 to 1.000. Returns the mean, lowest and highest RSC over the draws at each scale."""
|
|
1129
|
+
if not isinstance(counts, Counts):
|
|
1130
|
+
counts = read_counts(counts)
|
|
1131
|
+
columns = np.arange(len(counts.genes)) if genes is None else np.asarray([counts.genes.index(g) for g in genes])
|
|
1132
|
+
gene_names = [counts.genes[i] for i in columns]
|
|
1133
|
+
dense = np.asarray(counts.counts[:, columns].todense(), dtype=np.int64)
|
|
1134
|
+
if layout == "grid":
|
|
1135
|
+
spacing = float(spacing_um or counts.spacing_um or 0)
|
|
1136
|
+
area = counts.column("area")
|
|
1137
|
+
area = np.asarray(area, dtype=float) if area is not None else np.full(len(counts), spacing * spacing)
|
|
1138
|
+
root = _expected(dense, np.sqrt)
|
|
1139
|
+
expected = root / np.sqrt(0.5 * area)[:, None]
|
|
1140
|
+
else:
|
|
1141
|
+
expected = _expected(dense, lambda k: np.log1p(2.0 * k))
|
|
1142
|
+
rng = np.random.default_rng(seed)
|
|
1143
|
+
frames = []
|
|
1144
|
+
for d in range(draws):
|
|
1145
|
+
if progress:
|
|
1146
|
+
progress(dict(stage="draw", done=d, total=draws))
|
|
1147
|
+
drawn = rng.poisson(dense).astype(np.int64)
|
|
1148
|
+
simulated = Counts(sparse.csr_matrix(drawn), list(gene_names), counts.ids, counts.sections, counts.xy, counts.unit,
|
|
1149
|
+
counts.columns, counts.spacing_um, "drawn counts")
|
|
1150
|
+
result = rsc(simulated, {"Expected expression": expected}, layout, spacing_um=spacing_um, cell_class=cell_class,
|
|
1151
|
+
within=within, splits=splits)
|
|
1152
|
+
frame = result.scales[["section", "scale", "scale_order", "rsc"] + (["population"] if "population" in result.scales else [])].copy()
|
|
1153
|
+
frame["draw"] = d + 1
|
|
1154
|
+
frames.append(frame)
|
|
1155
|
+
joined = pd.concat(frames, ignore_index=True)
|
|
1156
|
+
keys = ["section"] + (["population"] if "population" in joined else []) + ["scale", "scale_order"]
|
|
1157
|
+
out = joined.groupby(keys, as_index=False, sort=False)["rsc"].agg(rsc_mean="mean", rsc_min="min", rsc_max="max")
|
|
1158
|
+
out["draws"] = draws
|
|
1159
|
+
return out
|
|
1160
|
+
|
|
1161
|
+
|
|
1162
|
+
# ---------------------------------------------------------------- bundle
|
|
1163
|
+
|
|
1164
|
+
def _bundle(result: Result, title: str, synthetic: bool) -> dict:
|
|
1165
|
+
layout = result.layout
|
|
1166
|
+
workflow = {"spots": "hest", "grid": "fourier", "cells": "cell", "regions": "regions"}[layout]
|
|
1167
|
+
bundle = evaluation_export.Bundle(workflow, title, synthetic)
|
|
1168
|
+
details_common = {k: v for k, v in result.settings.items() if k in ("toolkit", "count_splits", "gene_weighting", "transform")}
|
|
1169
|
+
if layout == "spots":
|
|
1170
|
+
bundle.metric("pearson_graph_support", "Whole-section Pearson r", "Mean gene-wise Pearson r with log(1 + count) on the graph-retained spots.", "correlation")
|
|
1171
|
+
bundle.metric("rsc_broad", "Broad-scale RSC", "Mean RSC at the Broadest and Broad scales, as for the spot-level models of the paper.")
|
|
1172
|
+
bundle.metric("rsc_fine", "Fine-scale RSC", "Mean RSC at the Fine and Finest scales, as for the spot-level models of the paper.")
|
|
1173
|
+
for row in result.summary.to_dict("records"):
|
|
1174
|
+
if row["section"] == "All sections":
|
|
1175
|
+
continue
|
|
1176
|
+
for field_, metric in (("whole_section_pearson_r", "pearson_graph_support"), ("broad_rsc", "rsc_broad"), ("fine_rsc", "rsc_fine")):
|
|
1177
|
+
bundle.row(row["model"], "Sections", row["section"], "specimen", metric, row[field_], "graph-retained spots", None, details_common)
|
|
1178
|
+
for k, label in enumerate(RELATIVE, start=1):
|
|
1179
|
+
bundle.metric(f"rsc_B{k}", f"RSC, {label}", "Relative graph scale; B1 broadest, B5 finest. These are not micrometer intervals.")
|
|
1180
|
+
bundle.metric("rna_split_agreement", "Split-half reliability", "Correlation between the two RNA halves at this scale.", "correlation", "split_half")
|
|
1181
|
+
for row in result.scales.to_dict("records"):
|
|
1182
|
+
if row["section"] == "All sections":
|
|
1183
|
+
continue
|
|
1184
|
+
details = {**details_common, **{k: row[k] for k in ("C", "V_model", "V_RNA", "V_A", "V_B", "pearson_r", "rsc_split_min", "rsc_split_max")}}
|
|
1185
|
+
bundle.row(row["model"], "Sections", row["section"], "specimen", f"rsc_B{row['scale_order']}", row["rsc"], "graph-retained spots", row["status"], details)
|
|
1186
|
+
key = ("Split-half reliability", f"Sections | B{row['scale_order']}", row["section"], "rna_split_agreement")
|
|
1187
|
+
if key not in bundle.identities:
|
|
1188
|
+
bundle.row("Split-half reliability", f"Sections | B{row['scale_order']}", row["section"], "specimen", "rna_split_agreement",
|
|
1189
|
+
row["split_half_reliability"], "graph-retained spots", None, details_common, role="split_half")
|
|
1190
|
+
elif layout == "grid":
|
|
1191
|
+
bundle.metric("physical_rsc", "RSC by physical scale", "RSC in this window of physical wavelength (a full spatial cycle).")
|
|
1192
|
+
bundle.metric("rna_split_agreement", "Split-half reliability", "Correlation between the two RNA halves in this window.", "correlation", "split_half")
|
|
1193
|
+
spacing = result.settings.get("spacing_um")
|
|
1194
|
+
for row in result.scales.to_dict("records"):
|
|
1195
|
+
if row["section"] == "All sections":
|
|
1196
|
+
continue
|
|
1197
|
+
task = f"{row['band']} | grid {spacing:g} um"
|
|
1198
|
+
details = {**details_common, **{k: row[k] for k in ("C", "V_model", "V_RNA", "V_A", "V_B", "pearson_r", "fourier_modes", "partial_window")}}
|
|
1199
|
+
bundle.row(row["model"], task, row["section"], "specimen", "physical_rsc", row["rsc"],
|
|
1200
|
+
"occupied grid bins" + ("; partial window" if row.get("partial_window") else ""), row["status"], details)
|
|
1201
|
+
key = ("Split-half reliability", task, row["section"], "rna_split_agreement")
|
|
1202
|
+
if key not in bundle.identities:
|
|
1203
|
+
bundle.row("Split-half reliability", task, row["section"], "specimen", "rna_split_agreement", row["split_half_reliability"],
|
|
1204
|
+
"occupied grid bins", None, details_common, role="split_half")
|
|
1205
|
+
elif layout == "cells":
|
|
1206
|
+
bundle.metric("cell_rsc", "Cell-level RSC", "RSC on the cell graph of this view and diffusion scale.")
|
|
1207
|
+
bundle.metric("rna_split_agreement", "Split-half reliability", "Correlation between the two RNA halves on this cell graph view.", "correlation", "split_half")
|
|
1208
|
+
for row in result.scales.to_dict("records"):
|
|
1209
|
+
if row["section"] == "All sections":
|
|
1210
|
+
continue
|
|
1211
|
+
task = f"{row['population']} | B{row['scale_order']}"
|
|
1212
|
+
bundle.row(row["model"], task, row["section"], "specimen", "cell_rsc", row["rsc"], "graph-retained cells", row["status"],
|
|
1213
|
+
{**details_common, **{k: row[k] for k in ("C", "V_model", "V_RNA", "V_A", "V_B", "pearson_r")}})
|
|
1214
|
+
key = ("Split-half reliability", task, row["section"], "rna_split_agreement")
|
|
1215
|
+
if key not in bundle.identities:
|
|
1216
|
+
bundle.row("Split-half reliability", task, row["section"], "specimen", "rna_split_agreement", row["split_half_reliability"],
|
|
1217
|
+
"graph-retained cells", None, details_common, role="split_half")
|
|
1218
|
+
else:
|
|
1219
|
+
bundle.metric("region_rsc", "RSC by region term", "RSC of the whole section, the region means (between), the residuals (within), and within the focus region, between its structures and within one structure; missing where the term was not testable.")
|
|
1220
|
+
bundle.metric("rna_split_agreement", "Split-half reliability", "Correlation between the two RNA halves in this term.", "correlation", "split_half")
|
|
1221
|
+
for row in result.scales.to_dict("records"):
|
|
1222
|
+
if row["section"] == "All sections":
|
|
1223
|
+
continue
|
|
1224
|
+
bundle.row(row["model"], row["scale"], row["section"], "specimen", "region_rsc", row["rsc"], "annotated spots", row["status"],
|
|
1225
|
+
{**details_common, **{k: row[k] for k in ("C", "V_model", "V_RNA", "V_A", "V_B", "pearson_r", "rsc_ci_low", "rsc_ci_high", "testable") if k in row}})
|
|
1226
|
+
key = ("Split-half reliability", row["scale"], row["section"], "rna_split_agreement")
|
|
1227
|
+
if key not in bundle.identities:
|
|
1228
|
+
bundle.row("Split-half reliability", row["scale"], row["section"], "specimen", "rna_split_agreement", row["split_half_reliability"],
|
|
1229
|
+
"annotated spots", None, details_common, role="split_half")
|
|
1230
|
+
bundle.data["notes"] = [f"Reproducible-signal correlation (RSC) computed with {result.settings.get('toolkit')}, layout {layout}."] + list(result.notes)
|
|
1231
|
+
bundle.data["settings"] = evaluation_export.finite_json(result.settings)
|
|
1232
|
+
return bundle.data
|