strsc 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. strsc/__init__.py +15 -0
  2. strsc/__main__.py +30 -0
  3. strsc/api.py +1232 -0
  4. strsc/cell_biology.py +241 -0
  5. strsc/cell_core.py +420 -0
  6. strsc/cell_prepare.py +223 -0
  7. strsc/cell_rna.py +274 -0
  8. strsc/cell_score.py +361 -0
  9. strsc/evaluation_export.py +288 -0
  10. strsc/example.py +69 -0
  11. strsc/export.py +119 -0
  12. strsc/files.py +757 -0
  13. strsc/fourier.py +271 -0
  14. strsc/her2.py +303 -0
  15. strsc/her2_core.py +958 -0
  16. strsc/hest.py +246 -0
  17. strsc/hest_core.py +289 -0
  18. strsc/rsc_cli.py +101 -0
  19. strsc/training/__init__.py +5 -0
  20. strsc/training/adapters/__init__.py +0 -0
  21. strsc/training/adapters/deepspot2cell/__init__.py +40 -0
  22. strsc/training/adapters/deepspot2cell/engineering_gate.py +239 -0
  23. strsc/training/adapters/deepspot2cell/formal_runner.py +254 -0
  24. strsc/training/adapters/deepspot2cell/graph.py +228 -0
  25. strsc/training/adapters/deepspot2cell/initialization.py +36 -0
  26. strsc/training/adapters/deepspot2cell/objective.py +212 -0
  27. strsc/training/adapters/deepspot2cell/overlay.py +79 -0
  28. strsc/training/adapters/deepspot2cell/paired_data.py +229 -0
  29. strsc/training/adapters/hest_deephis2exp/__init__.py +48 -0
  30. strsc/training/adapters/hest_deephis2exp/batching.py +730 -0
  31. strsc/training/adapters/hest_deephis2exp/data.py +448 -0
  32. strsc/training/adapters/hest_deephis2exp/engineering_gate.py +312 -0
  33. strsc/training/adapters/hest_deephis2exp/export.py +173 -0
  34. strsc/training/adapters/hest_deephis2exp/formal_runner.py +480 -0
  35. strsc/training/adapters/hest_deephis2exp/models.py +95 -0
  36. strsc/training/adapters/hest_deephis2exp/objective.py +492 -0
  37. strsc/training/adapters/hest_deephis2exp/overlay.py +141 -0
  38. strsc/training/adapters/hest_deephis2exp/split.py +80 -0
  39. strsc/training/adapters/hest_deepspot_hist/__init__.py +62 -0
  40. strsc/training/adapters/hest_deepspot_hist/contract.py +136 -0
  41. strsc/training/adapters/hest_deepspot_hist/data.py +746 -0
  42. strsc/training/adapters/hest_deepspot_hist/deepspot.py +162 -0
  43. strsc/training/adapters/hest_deepspot_hist/engineering_gate.py +498 -0
  44. strsc/training/adapters/hest_deepspot_hist/formal_runner.py +1003 -0
  45. strsc/training/adapters/hest_deepspot_hist/hist.py +164 -0
  46. strsc/training/adapters/hest_deepspot_hist/manifest.py +108 -0
  47. strsc/training/adapters/hest_deepspot_hist/objective.py +94 -0
  48. strsc/training/adapters/hest_deepspot_hist/outputs.py +154 -0
  49. strsc/training/adapters/hest_deepspot_hist/paired.py +54 -0
  50. strsc/training/adapters/hest_egn_tcgn/__init__.py +27 -0
  51. strsc/training/adapters/hest_egn_tcgn/contracts.py +124 -0
  52. strsc/training/adapters/hest_egn_tcgn/engineering_gate.py +729 -0
  53. strsc/training/adapters/hest_egn_tcgn/exemplars.py +241 -0
  54. strsc/training/adapters/hest_egn_tcgn/hest_data.py +448 -0
  55. strsc/training/adapters/hest_egn_tcgn/model_inputs.py +139 -0
  56. strsc/training/adapters/hest_egn_tcgn/outputs.py +137 -0
  57. strsc/training/adapters/hest_egn_tcgn/overlay.py +388 -0
  58. strsc/training/adapters/hest_egn_tcgn/pairing.py +66 -0
  59. strsc/training/adapters/hest_egn_tcgn/train_pair.py +458 -0
  60. strsc/training/adapters/hest_egn_tcgn/training.py +750 -0
  61. strsc/training/adapters/hest_egn_tcgn/upstream.py +345 -0
  62. strsc/training/adapters/hest_thitogene/__init__.py +15 -0
  63. strsc/training/adapters/hest_thitogene/engineering_gate.py +270 -0
  64. strsc/training/adapters/hest_thitogene/hest_data.py +396 -0
  65. strsc/training/adapters/hest_thitogene/overlay.py +260 -0
  66. strsc/training/adapters/hest_thitogene/train_pair.py +584 -0
  67. strsc/training/adapters/hest_thitogene/upstream.py +336 -0
  68. strsc/training/adapters/ridge_mlp/__init__.py +6 -0
  69. strsc/training/adapters/ridge_mlp/hest_contract.py +135 -0
  70. strsc/training/adapters/ridge_mlp/model.py +23 -0
  71. strsc/training/adapters/ridge_mlp/ridge.py +55 -0
  72. strsc/training/adapters/ridge_mlp/train_pair.py +621 -0
  73. strsc/training/adapters/scellst/__init__.py +42 -0
  74. strsc/training/adapters/scellst/data.py +279 -0
  75. strsc/training/adapters/scellst/engineering_gate.py +148 -0
  76. strsc/training/adapters/scellst/formal_runner.py +311 -0
  77. strsc/training/adapters/scellst/lightning_overlay.py +148 -0
  78. strsc/training/adapters/scellst/objective.py +87 -0
  79. strsc/training/adapters/scellst/runtime.py +416 -0
  80. strsc/training/adapters/scellst/split.py +232 -0
  81. strsc/training/hest_data.py +413 -0
  82. strsc/training/non_neighbours.py +256 -0
  83. strsc/training/shared/__init__.py +25 -0
  84. strsc/training/shared/adjacency.py +169 -0
  85. strsc/training/shared/losses.py +92 -0
  86. strsc/training/shared/noop.py +102 -0
  87. strsc-0.4.0.dist-info/METADATA +173 -0
  88. strsc-0.4.0.dist-info/RECORD +93 -0
  89. strsc-0.4.0.dist-info/WHEEL +5 -0
  90. strsc-0.4.0.dist-info/entry_points.txt +2 -0
  91. strsc-0.4.0.dist-info/licenses/LICENSE +21 -0
  92. strsc-0.4.0.dist-info/licenses/NOTICE.md +13 -0
  93. strsc-0.4.0.dist-info/top_level.txt +1 -0
strsc/api.py ADDED
@@ -0,0 +1,1232 @@
1
+ """Reproducible-signal correlation (RSC) in one call.
2
+
3
+ import strsc
4
+ counts = strsc.read_counts("sample.h5ad")
5
+ result = strsc.rsc(counts, {"my_model": "predictions.csv"}, layout="spots")
6
+ print(result.scales)
7
+
8
+ Each layout is scored as in the paper (Methods, "Reproducible-signal correlation" and
9
+ "Scales and count transforms"), with the toolkit's own functions:
10
+
11
+ ``"grid"``
12
+ Bins on a regular grid, such as Visium HD. A two-dimensional Fourier basis divides the
13
+ variation into seven physical scales, from below 128 µm to 4,096 µm and above. Half-counts
14
+ are transformed as sqrt(N_half / (0.5 a)) with a the bin area; five count splits. These are the
15
+ steps of :func:`strsc.fourier.score`, which gives the same RSC.
16
+ ``"spots"``
17
+ Spots on a neighbor graph, such as Visium or HEST. The eigenvectors of the graph Laplacian
18
+ give five relative scales, Broadest to Finest. Half-counts are transformed as log(1 + 2 N_half);
19
+ twenty count splits. With two or more sections, each gene is weighted by the agreement of its
20
+ two halves in the other sections, as for the HEST samples of the paper. A single section has
21
+ no other sections, so each gene's two halves are divided by a common factor, as for the
22
+ colorectal bins and the breast cancer cells of the paper; the values are then not directly
23
+ comparable with those of the HEST weighting.
24
+ ``"cells"``
25
+ Single cells, such as Xenium. Lazy diffusion operators on the graph of the eight nearest
26
+ neighbors within 40 µm give five relative scales. Half-counts are transformed as
27
+ log(1 + 2 N_half); five count splits. ``within`` scores the cells of one class on their own
28
+ graph, as for the malignant cells of the paper.
29
+
30
+ :func:`regions` scores agreement between and within annotated regions, as for the TLS sections,
31
+ and :func:`ideal_check` repeats the paper's test with the expected expression as the prediction.
32
+
33
+ The workflows of the paper (``strsc hest``, ``strsc fourier``, ``strsc.cell_score``) are unchanged.
34
+ """
35
+ from __future__ import annotations
36
+
37
+ import json
38
+ import math
39
+ import sys
40
+ from dataclasses import dataclass, field
41
+ from pathlib import Path
42
+ from typing import Callable, Iterable, Mapping, Sequence
43
+
44
+ import numpy as np
45
+ import pandas as pd
46
+ from scipy import sparse
47
+
48
+ if sys.platform == "emscripten": # pragma: no cover - only in the browser
49
+ # WebAssembly cannot start threads, and fourier.fft_fields asks scipy for two FFT workers.
50
+ # The number of workers changes how the work is shared, not the result.
51
+ import scipy.fft
52
+
53
+ if not getattr(scipy.fft.fft2, "_single_worker", False):
54
+ _fft2 = scipy.fft.fft2
55
+
56
+ def _fft2_single(*args, **kwargs):
57
+ return _fft2(*args, **{**kwargs, "workers": 1})
58
+
59
+ _fft2_single._single_worker = True
60
+ scipy.fft.fft2 = _fft2_single
61
+ # numpy in the browser multiplies matrices without BLAS; SciPy's BLAS does the spot projections
62
+ # about 15 times faster, in the same single precision.
63
+ from scipy.linalg import blas as _browser_blas
64
+
65
+ from . import __version__
66
+ from . import cell_core, evaluation_export, fourier
67
+ from . import hest_core as cal
68
+ from .files import Counts, InputError, Prediction, read_counts, read_predictions
69
+
70
+ LAYOUTS = ("spots", "grid", "cells")
71
+ RELATIVE = ("Broadest", "Broad", "Mid", "Fine", "Finest")
72
+ SPOT_SEEDS = tuple(cal.THINNING_SEEDS) # 20 count splits, as for the HEST spots
73
+ GRID_SEEDS = tuple(fourier.SPLIT_SEEDS) # 5, as for the colorectal bins
74
+ CELL_SEEDS = (20260819, 20260820, 20260821, 20260822, 20260823) # 5, as for the breast cancer cells
75
+ REGION_SPLITS = 20 # as for the TLS sections
76
+ CELL_GRAPH = dict(neighbors=8, max_edge_um=40.0, minimum_component_cells=20)
77
+ CELL_GENE_BLOCK = 64
78
+ GRID_LABELS = {"LT128": "<128 µm", "128_256": "128–256 µm", "256_512": "256–512 µm", "512_1024": "512–1,024 µm",
79
+ "1024_2048": "1,024–2,048 µm", "2048_4096": "2,048–4,096 µm", "GE4096": "≥4,096 µm"}
80
+ TOL = 1e-12
81
+
82
+ Progress = Callable[[dict], None] | None
83
+
84
+
85
+ # ---------------------------------------------------------------- results
86
+
87
+ @dataclass
88
+ class Result:
89
+ """Scores of one call. ``scales`` has one row per model, section and scale (or region term);
90
+ ``summary`` one row per model and section; ``genes`` the same scores gene by gene."""
91
+
92
+ layout: str
93
+ scales: pd.DataFrame
94
+ summary: pd.DataFrame
95
+ genes: pd.DataFrame
96
+ settings: dict
97
+ notes: list[str] = field(default_factory=list)
98
+
99
+ def __repr__(self) -> str:
100
+ shown = self.summary.drop(columns=[c for c in self.summary.columns if c.startswith("n_")], errors="ignore")
101
+ return f"RSC, {self.layout}: {len(self.scales)} rows\n" + shown.to_string(index=False, float_format=lambda v: f"{v:.3f}")
102
+
103
+ def to_bundle(self, title: str = "RSC computed with strsc", synthetic: bool = False) -> dict:
104
+ """The scores as an evaluation bundle, which the Computed results tab of strsc.org opens."""
105
+ return _bundle(self, title, synthetic)
106
+
107
+ def save(self, folder: str | Path, title: str = "RSC computed with strsc") -> list[Path]:
108
+ folder = Path(folder)
109
+ folder.mkdir(parents=True, exist_ok=True)
110
+ written = []
111
+ for name, frame in (("scales.tsv", self.scales), ("summary.tsv", self.summary), ("genes.tsv", self.genes)):
112
+ frame.to_csv(folder / name, sep="\t", index=False)
113
+ written.append(folder / name)
114
+ (folder / "settings.json").write_text(json.dumps(evaluation_export.finite_json(self.settings), indent=2, ensure_ascii=False))
115
+ (folder / "strsc-evaluation.json").write_text(json.dumps(self.to_bundle(title)))
116
+ return written + [folder / "settings.json", folder / "strsc-evaluation.json"]
117
+
118
+
119
+ def _ratio(c: float, vrna: float, vmodel: float) -> tuple[float, str]:
120
+ if not (vmodel > TOL):
121
+ return math.nan, "NO_PREDICTED_VARIANCE"
122
+ if not (vrna > 0):
123
+ return math.nan, "RNA_UNRESOLVED"
124
+ return float(c / math.sqrt(vrna * vmodel)), "ESTIMABLE"
125
+
126
+
127
+ def _record(model, section, scale, order, c, vmodel, vrna, va, vb, seeds_c=None, seeds_vrna=None, **extra) -> dict:
128
+ """One row: RSC, the Pearson r of the prediction with the mean of the two halves, and the split-half reliability."""
129
+ rsc, status = _ratio(c, vrna, vmodel)
130
+ mean_var = 0.25 * (va + vb + 2.0 * vrna)
131
+ pearson = float(c / math.sqrt(vmodel * mean_var)) if vmodel > TOL and mean_var > TOL else math.nan
132
+ reliability = float(vrna / math.sqrt(va * vb)) if va > TOL and vb > TOL else math.nan
133
+ lo = hi = math.nan
134
+ if seeds_c is not None and seeds_vrna is not None and vmodel > TOL:
135
+ per = [c_ / math.sqrt(v_ * vmodel) for c_, v_ in zip(seeds_c, seeds_vrna) if v_ > 0]
136
+ if per:
137
+ lo, hi = float(min(per)), float(max(per))
138
+ return dict(model=model, section=section, scale=scale, scale_order=order, rsc=rsc, pearson_r=pearson,
139
+ split_half_reliability=reliability, C=float(c), V_model=float(vmodel), V_RNA=float(vrna),
140
+ V_A=float(va), V_B=float(vb), rsc_split_min=lo, rsc_split_max=hi, status=status, **extra)
141
+
142
+
143
+ def _gene_rows(model, section, genes, labels, c, vmodel, vrna, va, vb, eligible=None) -> list[dict]:
144
+ """Per-gene scores: c, vmodel, ... are (scales, genes) arrays."""
145
+ rows = []
146
+ for g, gene in enumerate(genes):
147
+ if eligible is not None and not eligible[g]:
148
+ continue
149
+ for k, label in enumerate(labels):
150
+ rsc, _ = _ratio(c[k, g], vrna[k, g], vmodel[k, g])
151
+ rel = vrna[k, g] / math.sqrt(va[k, g] * vb[k, g]) if va[k, g] > TOL and vb[k, g] > TOL else math.nan
152
+ rows.append(dict(model=model, section=section, gene=gene, scale=label, scale_order=k + 1, rsc=rsc,
153
+ split_half_reliability=float(rel), C=float(c[k, g]), V_model=float(vmodel[k, g]), V_RNA=float(vrna[k, g])))
154
+ return rows
155
+
156
+
157
+ # ---------------------------------------------------------------- inputs
158
+
159
+ def _models(predictions, counts: Counts) -> list[Prediction]:
160
+ if isinstance(predictions, Prediction):
161
+ return [predictions]
162
+ if isinstance(predictions, (str, Path)):
163
+ return [read_predictions(predictions, counts)]
164
+ if isinstance(predictions, Mapping):
165
+ out = []
166
+ for name, value in predictions.items():
167
+ if isinstance(value, Prediction):
168
+ out.append(Prediction(str(name), value.values, value.genes, value.source))
169
+ elif isinstance(value, (str, Path)):
170
+ out.append(read_predictions(value, counts, str(name)))
171
+ else:
172
+ array = np.asarray(value, dtype=np.float64)
173
+ if array.shape != (len(counts), len(counts.genes)):
174
+ raise InputError(f"{name}: an array prediction needs the shape of the counts, {len(counts):,} × {len(counts.genes):,}.")
175
+ out.append(Prediction(str(name), array, list(counts.genes), "array"))
176
+ return out
177
+ if isinstance(predictions, Sequence):
178
+ return [p if isinstance(p, Prediction) else read_predictions(p, counts) for p in predictions]
179
+ raise InputError("Give the predictions as a file, a Prediction, or a dict of model name to file or array.")
180
+
181
+
182
+ def _common(counts: Counts, models: list[Prediction], genes: Iterable[str] | None):
183
+ """Genes predicted by every model, and the locations that have a value for each of them."""
184
+ names = [m.name for m in models]
185
+ if not models:
186
+ raise InputError("Give at least one prediction.")
187
+ if len(set(names)) != len(names):
188
+ raise InputError("Two predictions have the same name. Name each model once.")
189
+ wanted = list(counts.genes) if genes is None else [g for g in genes]
190
+ if genes is not None:
191
+ missing = [g for g in wanted if g not in set(counts.genes)]
192
+ if missing:
193
+ raise InputError(f"{len(missing)} of the chosen genes are not in the counts, for example {missing[0]}.")
194
+ shared = [g for g in wanted if all(g in set(m.genes) for m in models)]
195
+ if len(shared) < 1:
196
+ raise InputError("No gene is in the counts and in every prediction. Use the same gene names.")
197
+ columns = np.asarray([counts.genes.index(g) for g in shared])
198
+ complete = np.ones(len(counts), dtype=bool)
199
+ for m in models:
200
+ complete &= np.isfinite(m.values[:, columns]).all(axis=1)
201
+ notes = []
202
+ dropped_genes = len(wanted) - len(shared)
203
+ if dropped_genes:
204
+ notes.append(f"{dropped_genes:,} gene{'s' if dropped_genes != 1 else ''} of the counts {'are' if dropped_genes != 1 else 'is'} not predicted by every model and {'were' if dropped_genes != 1 else 'was'} left out.")
205
+ outside = int((~complete).sum())
206
+ if outside:
207
+ notes.append(f"{outside:,} location{'s' if outside != 1 else ''} without a prediction for every gene from every model {'were' if outside != 1 else 'was'} left out.")
208
+ if not complete.any():
209
+ raise InputError("No spot, bin or cell has a prediction for every gene from every model.")
210
+ return shared, columns, complete, notes
211
+
212
+
213
+ def _settings(layout, counts, genes, models, seeds, **extra) -> dict:
214
+ return dict(toolkit=f"strsc {__version__}", layout=layout, sections=counts.section_names(), genes=len(genes),
215
+ models=[m.name for m in models], count_splits=len(seeds), split_seeds=list(map(int, seeds)), **extra)
216
+
217
+
218
+ # ---------------------------------------------------------------- spots
219
+
220
+ def _molecule_moments(counts: np.ndarray, specimen: str, task: str, seeds) -> dict:
221
+ """As in strsc hest: per gene, the mean variance of a half-count, the mean covariance of the halves
222
+ and their inner product, averaged over count splits."""
223
+ d2 = np.zeros(counts.shape[1])
224
+ covariance = np.zeros_like(d2)
225
+ ell2 = np.zeros_like(d2)
226
+ for seed in seeds:
227
+ a, b = cal._halves(counts, task, specimen, seed)
228
+ d2 += 0.5 * (np.mean(a * a, axis=0) + np.mean(b * b, axis=0))
229
+ covariance += np.mean(a * b, axis=0)
230
+ ell2 += np.sum(a * b, axis=0)
231
+ scale = 1.0 / len(seeds)
232
+ return {"d2": d2 * scale, "covariance": covariance * scale, "ell2": ell2 * scale}
233
+
234
+
235
+ def spot_bands(xy: np.ndarray, progress=None):
236
+ """The five equal-rank groups of nonconstant Laplacian eigenmodes of each connected component,
237
+ as in ``hest_core.build_basis`` (combinatorial Laplacian, constant mode dropped, stable order),
238
+ with numpy's dense eigensolver."""
239
+ adjacency, keep, labels, d0, radius = cal._graph(xy)
240
+ kept_n = int(keep.sum())
241
+ components = sorted(np.unique(labels).tolist())
242
+ chunks_by_component, totals = {}, np.zeros(cal.BANDS, dtype=np.int64)
243
+ for component in components:
244
+ size = int(np.sum(labels == component))
245
+ chunks = np.array_split(np.arange(max(size - 1, 0)), cal.BANDS)
246
+ chunks_by_component[component] = chunks
247
+ totals += np.asarray([len(chunk) for chunk in chunks])
248
+ bands = [np.zeros((kept_n, int(totals[k])), dtype=np.float64) for k in range(cal.BANDS)]
249
+ offsets = np.zeros(cal.BANDS, dtype=np.int64)
250
+ for component in components:
251
+ local = np.flatnonzero(labels == component)
252
+ block = adjacency[local][:, local]
253
+ laplacian = -block.toarray().astype(np.float64)
254
+ degree = np.asarray(block.sum(axis=1)).reshape(-1).astype(np.float64)
255
+ laplacian[np.diag_indices_from(laplacian)] = degree
256
+ values, vectors = np.linalg.eigh(laplacian)
257
+ order = np.argsort(values, kind="stable")
258
+ vectors = vectors[:, order][:, 1:]
259
+ for k, indices in enumerate(chunks_by_component[component]):
260
+ start, end = int(offsets[k]), int(offsets[k]) + len(indices)
261
+ if len(indices):
262
+ bands[k][local, start:end] = vectors[:, indices]
263
+ offsets[k] = end
264
+ if progress:
265
+ progress()
266
+ return keep, bands, dict(spots_retained=kept_n, components=len(components), d0=d0, edge_radius=radius, band_modes=totals.tolist())
267
+
268
+
269
+ def _coefficients(bands, x):
270
+ """Band projections in single precision, as ``hest_core._coefficients``."""
271
+ x32 = np.asarray(x, dtype=np.float32)
272
+ if sys.platform == "emscripten": # pragma: no cover - only in the browser
273
+ return [np.asarray(_browser_blas.sgemm(1.0, np.asarray(u, dtype=np.float32), x32, trans_a=True), dtype=np.float64, order="C")
274
+ for u in bands]
275
+ return [(np.asarray(u, dtype=np.float32).T @ x32).astype(np.float64) for u in bands]
276
+
277
+
278
+ def _task_label(names, default: str = "Sections") -> str:
279
+ """The HEST task when the sections are exactly the samples of one, so that the count splits are
280
+ those of ``strsc hest``; otherwise a plain label. The label seeds every count split."""
281
+ return next((task for task, samples in cal.TASKS.items() if set(samples) == set(names)), default)
282
+
283
+
284
+ def score_spot_sections(sections: list[dict], genes: list[str], seeds=SPOT_SEEDS, step=None, max_spots: int | None = None,
285
+ task: str | None = None) -> list[dict]:
286
+ """Relative scales on spots. ``sections``: dicts with name, counts (int, spots x genes), xy and
287
+ predictions {model: spots x genes}. With two or more sections the gene weights come from the
288
+ other sections, as ``strsc hest``; one section uses the common factor of each gene's halves."""
289
+ task = task or _task_label([s["name"] for s in sections])
290
+ weighted = len(sections) >= 2
291
+ retained, moments = {}, {}
292
+ for s in sections:
293
+ _, keep, _, _, _ = cal._graph(s["xy"])
294
+ if not keep.any():
295
+ raise InputError(f"Section {s['name']}: no connected group of at least 20 spots.")
296
+ if max_spots and int(keep.sum()) > max_spots:
297
+ raise InputError(f"Section {s['name']} has {int(keep.sum()):,} connected spots. Here the limit is {max_spots:,}; run the toolkit on your own machine for larger sections, without cropping them.")
298
+ retained[s["name"]] = keep
299
+ if weighted:
300
+ moments[s["name"]] = _molecule_moments(s["counts"][keep], s["name"], task, seeds)
301
+ records = []
302
+ for s in sections:
303
+ name, keep = s["name"], retained[s["name"]]
304
+ counts = s["counts"][keep]
305
+ if weighted:
306
+ others = [o["name"] for o in sections if o["name"] != name]
307
+ d2 = np.mean(np.stack([moments[o]["d2"] for o in others]), axis=0)
308
+ cov = np.mean(np.stack([moments[o]["covariance"] for o in others]), axis=0)
309
+ h = np.divide(cov, d2, out=np.zeros_like(d2), where=d2 > cal.NUMERIC_TOL)
310
+ w, ell2 = np.maximum(h, 0.0), moments[name]["ell2"]
311
+ eligible = (d2 > cal.NUMERIC_TOL) & (w > 0) & (ell2 > 0)
312
+ d = np.sqrt(d2)
313
+ rna_scale = np.divide(np.sqrt(w), d, out=np.zeros_like(w), where=d > cal.NUMERIC_TOL)
314
+ rna_scale[~eligible] = 0.0
315
+ p_scale = np.divide(np.sqrt(w) * np.sqrt(np.maximum(ell2, 0)), d, out=np.zeros_like(w), where=d > cal.NUMERIC_TOL)
316
+ p_scale[~eligible] = 0.0
317
+ else:
318
+ others = []
319
+ eligible = np.asarray(counts.sum(axis=0)).reshape(-1) > 0
320
+ if step:
321
+ step(name, "graph")
322
+ keep2, bands, graph = spot_bands(s["xy"])
323
+ assert np.array_equal(keep2, keep)
324
+ prepared = {}
325
+ for model, prediction_full in s["predictions"].items():
326
+ prediction = prediction_full[keep]
327
+ centered = prediction - prediction.mean(axis=0, keepdims=True)
328
+ pnorm = np.sqrt(np.sum(centered * centered, axis=0))
329
+ p = np.divide(centered, pnorm[None, :], out=np.zeros_like(centered), where=pnorm[None, :] > cal.NUMERIC_TOL)
330
+ if weighted:
331
+ p = p * p_scale[None, :]
332
+ else:
333
+ p[:, ~eligible] = 0.0
334
+ prepared[model] = _coefficients(bands, p)
335
+ n_genes = counts.shape[1]
336
+ vrna, va, vb = np.zeros(5), np.zeros(5), np.zeros(5)
337
+ vrna_g, va_g, vb_g = np.zeros((5, n_genes)), np.zeros((5, n_genes)), np.zeros((5, n_genes))
338
+ mean_observed = [None] * 5
339
+ seed_vrna = np.zeros((len(seeds), 5))
340
+ seed_c = {model: np.zeros((len(seeds), 5)) for model in prepared}
341
+ for i, seed in enumerate(seeds):
342
+ a, b = cal._halves(counts, task, name, seed)
343
+ if weighted:
344
+ a *= rna_scale[None, :]
345
+ b *= rna_scale[None, :]
346
+ else:
347
+ factor = np.sqrt(0.5 * (np.sum(a * a, axis=0) + np.sum(b * b, axis=0)))
348
+ ok = (factor > cal.NUMERIC_TOL) & eligible
349
+ a[:, ok] /= factor[ok]
350
+ b[:, ok] /= factor[ok]
351
+ a[:, ~ok] = 0.0
352
+ b[:, ~ok] = 0.0
353
+ ac, bc = _coefficients(bands, a), _coefficients(bands, b)
354
+ for k in range(5):
355
+ vrna[k] += np.sum(ac[k] * bc[k])
356
+ current = 0.5 * (ac[k] + bc[k])
357
+ mean_observed[k] = current if mean_observed[k] is None else mean_observed[k] + current
358
+ va[k] += np.sum(ac[k] * ac[k])
359
+ vb[k] += np.sum(bc[k] * bc[k])
360
+ vrna_g[k] += np.sum(ac[k] * bc[k], axis=0)
361
+ va_g[k] += np.sum(ac[k] * ac[k], axis=0)
362
+ vb_g[k] += np.sum(bc[k] * bc[k], axis=0)
363
+ seed_vrna[i, k] = np.sum(ac[k] * bc[k])
364
+ for model, pc in prepared.items():
365
+ seed_c[model][i, k] = np.sum(pc[k] * current)
366
+ if step:
367
+ step(name, "split")
368
+ n = len(seeds)
369
+ vrna /= n
370
+ va /= n
371
+ vb /= n
372
+ vrna_g /= n
373
+ va_g /= n
374
+ vb_g /= n
375
+ mean_observed = [x / n for x in mean_observed]
376
+ truth = np.log1p(s["counts"])
377
+ models = []
378
+ for model, pc in prepared.items():
379
+ prediction_full = s["predictions"][model]
380
+ vpred = np.asarray([np.sum(x * x) for x in pc])
381
+ covariance = np.asarray([np.sum(pc[k] * mean_observed[k]) for k in range(5)])
382
+ c_g = np.stack([np.sum(pc[k] * mean_observed[k], axis=0) for k in range(5)])
383
+ vm_g = np.stack([np.sum(pc[k] * pc[k], axis=0) for k in range(5)])
384
+ gene_pcc = cal._corr_cols(prediction_full[keep], truth[keep])
385
+ full_pcc = cal._corr_cols(prediction_full, truth)
386
+ models.append(dict(model=model, vpred=vpred, covariance=covariance, c_g=c_g, vm_g=vm_g, seed_c=seed_c[model],
387
+ gene_pcc=gene_pcc, full_pcc=full_pcc))
388
+ records.append(dict(section=name, task=task, weighted=weighted, reference_sections=others, graph=graph,
389
+ spots=len(keep), retained=int(keep.sum()), eligible=eligible, vrna=vrna, va=va, vb=vb,
390
+ vrna_g=vrna_g, va_g=va_g, vb_g=vb_g, seed_vrna=seed_vrna, models=models))
391
+ return records
392
+
393
+
394
+ def _spot_result(records, genes, settings, notes) -> Result:
395
+ scale_rows, gene_rows, summaries = [], [], []
396
+ for r in records:
397
+ for m in r["models"]:
398
+ rows = []
399
+ for k, label in enumerate(RELATIVE):
400
+ rows.append(_record(m["model"], r["section"], label, k + 1, m["covariance"][k], m["vpred"][k], r["vrna"][k],
401
+ r["va"][k], r["vb"][k], m["seed_c"][:, k], r["seed_vrna"][:, k]))
402
+ scale_rows.extend(rows)
403
+ rsc = [row["rsc"] for row in rows]
404
+ gene_rows.extend(_gene_rows(m["model"], r["section"], genes, RELATIVE, m["c_g"], m["vm_g"], r["vrna_g"], r["va_g"], r["vb_g"], r["eligible"]))
405
+ summaries.append(dict(model=m["model"], section=r["section"],
406
+ fine_rsc=_mean2(rsc[3], rsc[4]), broad_rsc=_mean2(rsc[0], rsc[1]),
407
+ broadest_minus_finest=rsc[0] - rsc[4] if all(map(math.isfinite, (rsc[0], rsc[4]))) else math.nan,
408
+ fine_split_half_reliability=_mean2(rows[3]["split_half_reliability"], rows[4]["split_half_reliability"]),
409
+ whole_section_pearson_r=_nanmean(m["gene_pcc"]),
410
+ n_locations=r["retained"], n_genes=int(np.sum(r["eligible"]))))
411
+ scales, summary = pd.DataFrame(scale_rows), pd.DataFrame(summaries)
412
+ if len(records) > 1:
413
+ scales, summary = _all_sections_mean(scales, summary)
414
+ return Result("spots", scales, summary, pd.DataFrame(gene_rows), settings, notes)
415
+
416
+
417
+ def _mean2(x, y) -> float:
418
+ return 0.5 * (x + y) if math.isfinite(x) and math.isfinite(y) else math.nan
419
+
420
+
421
+ def _nanmean(values) -> float:
422
+ values = np.asarray(values, dtype=np.float64)
423
+ return float(np.nanmean(values)) if np.isfinite(values).any() else math.nan
424
+
425
+
426
+ def _all_sections_mean(scales: pd.DataFrame, summary: pd.DataFrame):
427
+ """'All sections': the mean over sections of each section's scores, as the means over samples in the paper."""
428
+ numeric = ["rsc", "pearson_r", "split_half_reliability"]
429
+ pooled = (scales.groupby(["model", "scale", "scale_order"], as_index=False, sort=False)[numeric].mean())
430
+ pooled.insert(1, "section", "All sections")
431
+ pooled["status"] = "MEAN_OVER_SECTIONS"
432
+ keep = [c for c in summary.columns if c not in ("model", "section") and not c.startswith("n_")]
433
+ overall = summary.groupby("model", as_index=False, sort=False)[keep].mean()
434
+ overall.insert(1, "section", "All sections")
435
+ return pd.concat([scales, pooled], ignore_index=True), pd.concat([summary, overall], ignore_index=True)
436
+
437
+
438
+ # ---------------------------------------------------------------- grid
439
+
440
+ def score_grid_section(counts: sparse.csr_matrix, prediction: np.ndarray, coords: np.ndarray, area: np.ndarray, *,
441
+ spacing_um: float, models: Sequence[str], seeds=GRID_SEEDS, step=None) -> dict:
442
+ """The steps of :func:`strsc.fourier.score`, unchanged, which also keep the variance of each half,
443
+ the terms of each gene and of each count split. Its RSC equals that of ``fourier.score``."""
444
+ counts = sparse.csr_matrix(counts)
445
+ counts.sum_duplicates()
446
+ counts.eliminate_zeros()
447
+ counts.sort_indices()
448
+ n, genes = counts.shape
449
+ windows = pd.DataFrame({"center_x_um": coords[:, 0], "center_y_um": coords[:, 1]})
450
+ scaled = (coords - coords.min(axis=0)) / spacing_um
451
+ if not np.allclose(scaled, np.rint(scaled), rtol=0, atol=1e-6):
452
+ raise InputError("The coordinates do not lie on a regular grid with this spacing.")
453
+ iy, ix, ny, nx, masks = fourier.grid_contract(windows, spacing_um)
454
+ names = list(masks)
455
+ zeros = lambda: {band: np.zeros(genes) for band in names}
456
+ vrna_g, va_g, vb_g = zeros(), zeros(), zeros()
457
+ c_g = {model: zeros() for model in models}
458
+ vm_g = {model: zeros() for model in models}
459
+ terms = {(model, band): {"v_rna": 0.0, "c_model": 0.0, "v_model": 0.0} for model in models for band in names}
460
+ halves = {band: {"v_a": 0.0, "v_b": 0.0} for band in names}
461
+ seed_vrna = {band: np.zeros(len(seeds)) for band in names}
462
+ seed_c = {(model, band): np.zeros(len(seeds)) for model in models for band in names}
463
+ for offset in range(0, genes, fourier.GENE_BLOCK):
464
+ stop = min(offset + fourier.GENE_BLOCK, genes)
465
+ block = counts[:, offset:stop].tocsr()
466
+ fp = {model: fourier.fft_fields(fourier.normalize_prediction(prediction[:, offset:stop, j]), iy, ix, ny, nx)
467
+ for j, model in enumerate(models)}
468
+ for model in models:
469
+ for band, mask in masks.items():
470
+ terms[model, band]["v_model"] += float(np.square(np.abs(fp[model][mask, :])).sum())
471
+ vm_g[model][band][offset:stop] += np.square(np.abs(fp[model][mask, :])).sum(axis=0)
472
+ block_vrna = {band: 0.0 for band in names}
473
+ block_c = {(model, band): 0.0 for model in models for band in names}
474
+ for i, seed in enumerate(seeds):
475
+ aa, bb = fourier.split_transform(block, area, int(seed) + offset * 1009)
476
+ fa, fb = (fourier.fft_fields(values, iy, ix, ny, nx) for values in (aa, bb))
477
+ for band, mask in masks.items():
478
+ cross = np.real(np.conj(fa[mask, :]) * fb[mask, :])
479
+ value = float(cross.sum())
480
+ block_vrna[band] += value
481
+ seed_vrna[band][i] += value
482
+ vrna_g[band][offset:stop] += cross.sum(axis=0) / len(seeds)
483
+ ea, eb = np.square(np.abs(fa[mask, :])), np.square(np.abs(fb[mask, :]))
484
+ halves[band]["v_a"] += float(ea.sum()) / len(seeds)
485
+ halves[band]["v_b"] += float(eb.sum()) / len(seeds)
486
+ va_g[band][offset:stop] += ea.sum(axis=0) / len(seeds)
487
+ vb_g[band][offset:stop] += eb.sum(axis=0) / len(seeds)
488
+ judge_sum = fa[mask, :] + fb[mask, :]
489
+ for model in models:
490
+ product = 0.5 * np.real(np.conj(fp[model][mask, :]) * judge_sum)
491
+ value = float(product.sum())
492
+ block_c[model, band] += value
493
+ seed_c[model, band][i] += value
494
+ c_g[model][band][offset:stop] += product.sum(axis=0) / len(seeds)
495
+ for band in names:
496
+ for model in models:
497
+ terms[model, band]["v_rna"] += block_vrna[band] / len(seeds)
498
+ terms[model, band]["c_model"] += block_c[model, band] / len(seeds)
499
+ if step:
500
+ step(stop, genes)
501
+ modes = {band: int(mask.sum()) for band, mask in masks.items()}
502
+ return dict(terms=terms, halves=halves, modes=modes, seed_vrna=seed_vrna, seed_c=seed_c, vrna_g=vrna_g, va_g=va_g,
503
+ vb_g=vb_g, c_g=c_g, vm_g=vm_g, grid=(ny, nx), sites=n)
504
+
505
+
506
+ def _grid_result(sections, genes, models, spacing_um, settings, notes) -> Result:
507
+ order = [band for band, *_ in reversed(fourier.WAVELENGTH_BANDS)] # broadest first
508
+ lows = {band: low for band, low, _ in fourier.WAVELENGTH_BANDS}
509
+ scale_rows, gene_rows, summaries = [], [], []
510
+ for s in sections:
511
+ out = s["out"]
512
+ present = [band for band in order if out["modes"][band] > 0]
513
+ for model in models:
514
+ rows = []
515
+ for k, band in enumerate(present):
516
+ t, h = out["terms"][model, band], out["halves"][band]
517
+ rows.append(_record(model, s["name"], GRID_LABELS[band], k + 1, t["c_model"], t["v_model"], t["v_rna"], h["v_a"], h["v_b"],
518
+ out["seed_c"][model, band] / 1.0, out["seed_vrna"][band],
519
+ band=band, fourier_modes=out["modes"][band],
520
+ partial_window=bool(lows[band] < 2.0 * spacing_um)))
521
+ # per-seed values are sums over gene blocks of that seed's terms
522
+ scale_rows.extend(rows)
523
+ stack = lambda d: np.stack([d[band] for band in present])
524
+ gene_rows.extend(_gene_rows(model, s["name"], genes, [GRID_LABELS[b] for b in present], stack(out["c_g"][model]),
525
+ stack(out["vm_g"][model]), stack(out["vrna_g"]), stack(out["va_g"]), stack(out["vb_g"]),
526
+ s["eligible"]))
527
+ rsc = [row["rsc"] for row in rows]
528
+ finest = next((row for row in reversed(rows) if math.isfinite(row["rsc"])), None)
529
+ summaries.append(dict(model=model, section=s["name"], broadest_rsc=rsc[0] if rsc else math.nan,
530
+ finest_rsc=finest["rsc"] if finest else math.nan, finest_scale=finest["scale"] if finest else "",
531
+ broadest_minus_finest=(rsc[0] - finest["rsc"]) if finest and rsc and math.isfinite(rsc[0]) else math.nan,
532
+ whole_section_pearson_r=s["pearson"][model], n_locations=s["sites"], n_genes=int(np.sum(s["eligible"]))))
533
+ scales, summary = pd.DataFrame(scale_rows), pd.DataFrame(summaries)
534
+ if len(sections) > 1:
535
+ scales, summary = _all_sections_mean(scales, summary)
536
+ return Result("grid", scales, summary, pd.DataFrame(gene_rows), settings, notes)
537
+
538
+
539
+ # ---------------------------------------------------------------- cells
540
+
541
+ PAIRS = (("A", "A"), ("A", "B"), ("A", "P"), ("B", "B"), ("B", "P"), ("P", "P"))
542
+
543
+
544
+ def _band_grams_by_gene(diffusion: sparse.csr_matrix, matrices: dict[str, np.ndarray]) -> dict[str, dict[tuple[str, str], np.ndarray]]:
545
+ """``cell_core.lazy_band_grams`` for each gene: the same products, summed over cells only."""
546
+ names = list(matrices)
547
+ original = {name: np.asarray(matrices[name], np.float32) for name in names}
548
+ state = {name: value.copy() for name, value in original.items()}
549
+ milestones: dict[int, dict[tuple[str, str], np.ndarray]] = {}
550
+ for step in range(17):
551
+ if step in cell_core.MILESTONES:
552
+ gram = {}
553
+ for i, left in enumerate(names):
554
+ for right in names[i:]:
555
+ gram[(left, right)] = 0.5 * (np.sum(original[left] * state[right], axis=0, dtype=np.float64)
556
+ + np.sum(original[right] * state[left], axis=0, dtype=np.float64))
557
+ milestones[step] = gram
558
+ if step < 16:
559
+ for name in names:
560
+ state[name] = diffusion @ state[name]
561
+ out = {}
562
+ for band, high, low in (("B1", 16, None), ("B2", 8, 16), ("B3", 4, 8), ("B4", 2, 4), ("B5", 0, 2)):
563
+ out[band] = {pair: value if low is None else value - milestones[low][pair] for pair, value in milestones[high].items()}
564
+ return out
565
+
566
+
567
+ def score_cell_view(counts: sparse.csr_matrix, prediction: np.ndarray, coords: np.ndarray, membership: sparse.csr_matrix,
568
+ view_coords: np.ndarray, seeds=CELL_SEEDS, specimen: str = "section", step=None) -> dict:
569
+ """One graph view of a section, as ``strsc.cell_score``: the graph is built on the view's cells,
570
+ the counts are split on all cells, and the halves and the prediction are transformed and
571
+ filtered on the view's graph."""
572
+ diffusion, retained, labels, receipt = cell_core.build_lazy_graph(view_coords, CELL_GRAPH["neighbors"], CELL_GRAPH["max_edge_um"],
573
+ CELL_GRAPH["minimum_component_cells"])
574
+ exposure = np.ones(membership.shape[0])
575
+ pred_view = np.asarray(membership @ prediction, np.float32)
576
+ pred_view /= np.asarray(exposure, np.float32)[:, None]
577
+ pred_view = pred_view[retained]
578
+ genes = counts.shape[1]
579
+ per_seed = {seed: {band: {pair: np.zeros(genes) for pair in PAIRS} for band in cell_core.BANDS} for seed in seeds}
580
+ eligible_any = np.zeros(genes, dtype=bool)
581
+ pcc = np.full(genes, np.nan)
582
+ blocks = [slice(start, min(start + CELL_GENE_BLOCK, genes)) for start in range(0, genes, CELL_GENE_BLOCK)]
583
+ view_counts = (membership @ counts).tocsr()[retained]
584
+ for block in blocks:
585
+ truth = np.log1p(np.asarray(view_counts[:, block].todense(), dtype=np.float64))
586
+ pcc[block] = cal._corr_cols(pred_view[:, block].astype(np.float64), truth)
587
+ del view_counts
588
+ # One count split at a time, so that memory does not grow with the number of splits. Every split
589
+ # and every gene block is computed on its own, so the order changes none of the numbers.
590
+ for k, seed in enumerate(seeds):
591
+ a, b = cell_core.split_counts(counts, specimen, seed)
592
+ a_view, b_view = (membership @ a).tocsr()[retained], (membership @ b).tocsr()[retained]
593
+ del a, b
594
+ for i, block in enumerate(blocks):
595
+ pp, pred_eligible = cell_core.transform_prediction(pred_view[:, block], labels)
596
+ aa, bb, rna_eligible = cell_core.transform_rna_pair(a_view[:, block], b_view[:, block], exposure[retained], labels)
597
+ eligible = pred_eligible & rna_eligible
598
+ aa[:, ~eligible] = 0
599
+ bb[:, ~eligible] = 0
600
+ local_p = pp.copy()
601
+ local_p[:, ~eligible] = 0
602
+ eligible_any[block] |= eligible
603
+ grams = _band_grams_by_gene(diffusion, {"A": aa, "B": bb, "P": local_p})
604
+ for band in cell_core.BANDS:
605
+ for pair in PAIRS:
606
+ per_seed[seed][band][pair][block] += grams[band][pair]
607
+ if step:
608
+ step(k * len(blocks) + i + 1, len(seeds) * len(blocks))
609
+ del a_view, b_view
610
+ return dict(per_seed=per_seed, eligible=eligible_any, receipt=receipt, pearson=_nanmean(pcc), cells=int(retained.sum()))
611
+
612
+
613
+ def _cell_terms(view, seeds) -> dict:
614
+ """Totals over genes, averaged over seeds; per-gene terms averaged over seeds; and per-seed totals."""
615
+ mean_g = {band: {pair: np.mean([view["per_seed"][s][band][pair] for s in seeds], axis=0) for pair in PAIRS} for band in cell_core.BANDS}
616
+ total = {band: {pair: float(np.sum(mean_g[band][pair])) for pair in PAIRS} for band in cell_core.BANDS}
617
+ seed_total = {band: {pair: np.asarray([float(np.sum(view["per_seed"][s][band][pair])) for s in seeds]) for pair in PAIRS} for band in cell_core.BANDS}
618
+ return dict(mean_g=mean_g, total=total, seed_total=seed_total)
619
+
620
+
621
+ def _cell_result(views, genes, settings, notes) -> Result:
622
+ scale_rows, gene_rows, summaries = [], [], []
623
+ bands = list(cell_core.BANDS)
624
+ for v in views:
625
+ terms = _cell_terms(v["out"], v["seeds"])
626
+ t, st, g = terms["total"], terms["seed_total"], terms["mean_g"]
627
+ rows = []
628
+ for k, band in enumerate(bands):
629
+ c = 0.5 * (t[band][("A", "P")] + t[band][("B", "P")])
630
+ seeds_c = 0.5 * (st[band][("A", "P")] + st[band][("B", "P")])
631
+ rows.append(_record(v["model"], v["section"], RELATIVE[k], k + 1, c, t[band][("P", "P")], t[band][("A", "B")],
632
+ t[band][("A", "A")], t[band][("B", "B")], seeds_c, st[band][("A", "B")], population=v["population"]))
633
+ scale_rows.extend(rows)
634
+ combined = {}
635
+ for label, members in (("fine", ("B4", "B5")), ("broad", ("B1", "B2"))):
636
+ c = sum(0.5 * (t[b][("A", "P")] + t[b][("B", "P")]) for b in members)
637
+ vm = sum(t[b][("P", "P")] for b in members)
638
+ vr = sum(t[b][("A", "B")] for b in members)
639
+ va_ = sum(t[b][("A", "A")] for b in members)
640
+ vb_ = sum(t[b][("B", "B")] for b in members)
641
+ combined[label] = _record(v["model"], v["section"], label, 0, c, vm, vr, va_, vb_)
642
+ stack = lambda pair: np.stack([g[b][pair] for b in bands])
643
+ gene_rows.extend({**row, "population": v["population"]} for row in _gene_rows(
644
+ v["model"], v["section"], genes, RELATIVE, 0.5 * (stack(("A", "P")) + stack(("B", "P"))), stack(("P", "P")),
645
+ stack(("A", "B")), stack(("A", "A")), stack(("B", "B")), v["out"]["eligible"]))
646
+ rsc = [row["rsc"] for row in rows]
647
+ summaries.append(dict(model=v["model"], section=v["section"], population=v["population"],
648
+ fine_rsc=combined["fine"]["rsc"], broad_rsc=combined["broad"]["rsc"],
649
+ broadest_minus_finest=rsc[0] - rsc[4] if all(map(math.isfinite, (rsc[0], rsc[4]))) else math.nan,
650
+ fine_split_half_reliability=combined["fine"]["split_half_reliability"],
651
+ whole_section_pearson_r=v["out"]["pearson"], n_locations=v["out"]["cells"],
652
+ n_genes=int(np.sum(v["out"]["eligible"]))))
653
+ scales, summary = pd.DataFrame(scale_rows), pd.DataFrame(summaries)
654
+ sections = summary["section"].unique()
655
+ if len(sections) > 1:
656
+ # As for the patients of the paper: C, V_model and V_RNA averaged over sections before the ratio.
657
+ pooled_rows, pooled_summary = [], []
658
+ for (model, population), frame in scales.groupby(["model", "population"], sort=False):
659
+ for (scale, order), part in frame.groupby(["scale", "scale_order"], sort=False):
660
+ m = part[["C", "V_model", "V_RNA", "V_A", "V_B"]].mean()
661
+ pooled_rows.append(_record(model, "All sections", scale, order, m.C, m.V_model, m.V_RNA, m.V_A, m.V_B, population=population))
662
+ fine = frame[frame.scale.isin(["Fine", "Finest"])].groupby("section")[["C", "V_model", "V_RNA", "V_A", "V_B"]].sum().mean()
663
+ broad = frame[frame.scale.isin(["Broadest", "Broad"])].groupby("section")[["C", "V_model", "V_RNA", "V_A", "V_B"]].sum().mean()
664
+ f = _record(model, "All sections", "fine", 0, fine.C, fine.V_model, fine.V_RNA, fine.V_A, fine.V_B)
665
+ b = _record(model, "All sections", "broad", 0, broad.C, broad.V_model, broad.V_RNA, broad.V_A, broad.V_B)
666
+ ordered = [r for r in pooled_rows if r["model"] == model and r["population"] == population]
667
+ pooled_summary.append(dict(model=model, section="All sections", population=population, fine_rsc=f["rsc"], broad_rsc=b["rsc"],
668
+ broadest_minus_finest=ordered[0]["rsc"] - ordered[4]["rsc"],
669
+ fine_split_half_reliability=f["split_half_reliability"],
670
+ whole_section_pearson_r=float(summary[(summary.model == model) & (summary.population == population)].whole_section_pearson_r.mean())))
671
+ scales = pd.concat([scales, pd.DataFrame(pooled_rows)], ignore_index=True)
672
+ summary = pd.concat([summary, pd.DataFrame(pooled_summary)], ignore_index=True)
673
+ return Result("cells", scales, summary, pd.DataFrame(gene_rows), settings, notes)
674
+
675
+
676
+ # ---------------------------------------------------------------- the call
677
+
678
+ def rsc(counts: Counts | str | Path, predictions, layout: str = "spots", *, spacing_um: float | None = None,
679
+ cell_class: str = "cell_class", within: str | Sequence[str] | None = None, all_cells: bool = True,
680
+ genes: Iterable[str] | None = None, splits: int | None = None, max_spots: int | None = None,
681
+ progress: Progress = None) -> Result:
682
+ """Reproducible-signal correlation of one or more predictions, at each spatial scale.
683
+
684
+ ``counts``: a :class:`Counts` from :func:`read_counts`, or a path. ``predictions``: a file, a
685
+ :class:`Prediction`, or a dict of model name to file or array (locations × genes of the counts).
686
+ ``layout``: ``"spots"``, ``"grid"`` or ``"cells"`` (see the module docstring). For ``"grid"``,
687
+ ``spacing_um`` is the bin spacing (read from Visium HD files when not given). For ``"cells"``,
688
+ ``within`` names one or more values of the ``cell_class`` column to score on their own graphs,
689
+ and ``all_cells`` whether to score all cells as well. ``genes`` restricts the genes, ``splits``
690
+ uses only the first count splits of the paper (fewer splits give noisier values).
691
+ """
692
+ if layout not in LAYOUTS:
693
+ raise InputError(f"layout must be one of {', '.join(LAYOUTS)}.")
694
+ if not isinstance(counts, Counts):
695
+ counts = read_counts(counts)
696
+ models = _models(predictions, counts)
697
+ shared, columns, complete, notes = _common(counts, models, genes)
698
+ say = progress or (lambda _: None)
699
+ if layout == "spots":
700
+ seeds = SPOT_SEEDS[: splits or len(SPOT_SEEDS)]
701
+ sections = []
702
+ for name in counts.section_names():
703
+ rows = np.flatnonzero((counts.sections == name) & complete)
704
+ if len(rows) < 20:
705
+ raise InputError(f"Section {name} has fewer than 20 spots with every prediction.")
706
+ values = np.asarray(counts.counts[rows][:, columns].todense(), dtype=np.int64)
707
+ sections.append(dict(name=name, counts=values, xy=counts.xy[rows],
708
+ predictions={m.name: m.values[np.ix_(rows, columns)] for m in models}))
709
+ weighted = len(sections) >= 2
710
+ if not weighted:
711
+ notes.append("One section: each gene's two halves are divided by a common factor, as for the colorectal bins and breast cancer cells of the paper. "
712
+ "With two or more sections the genes are weighted by the agreement of their halves in the other sections, as for the HEST samples; the two give different values.")
713
+ total = len(sections) * (1 + len(seeds))
714
+ done = [0]
715
+
716
+ def step(name, stage):
717
+ done[0] += 1
718
+ say(dict(stage=stage, section=name, done=done[0], total=total))
719
+ records = score_spot_sections(sections, shared, seeds, step, max_spots)
720
+ settings = _settings("spots", counts, shared, models, seeds, transform="log(1 + 2 N_half), centered by gene",
721
+ scales="five equal-rank groups of the nonconstant eigenvectors of the neighbor-graph Laplacian (edges within 1.25 × the median nearest-neighbor distance, components of at least 20 spots)",
722
+ gene_weighting="agreement of the halves in the other sections (HEST)" if weighted else "common factor of each gene's halves",
723
+ fine_scale="mean RSC at the Fine and Finest scales", broad_scale="mean RSC at the Broadest and Broad scales",
724
+ task_label=records[0]["task"])
725
+ return _spot_result(records, shared, settings, notes)
726
+ if layout == "grid":
727
+ spacing = float(spacing_um or counts.spacing_um or 0)
728
+ if not math.isfinite(spacing) or spacing <= 0:
729
+ raise InputError("Give the grid spacing in micrometers.")
730
+ if counts.unit != "um" and spacing_um is None:
731
+ raise InputError("The grid needs coordinates in micrometers: give um_per_unit when reading the counts, and the spacing.")
732
+ seeds = GRID_SEEDS[: splits or len(GRID_SEEDS)]
733
+ area_all = counts.column("area")
734
+ if area_all is None:
735
+ notes.append(f"No area column, so every bin counts as a full {spacing:g} × {spacing:g} µm square of tissue.")
736
+ sections = []
737
+ names = counts.section_names()
738
+ parts = max(1, math.ceil(len(columns) / fourier.GENE_BLOCK))
739
+ for index, name in enumerate(names):
740
+ rows = np.flatnonzero((counts.sections == name) & complete)
741
+ block = counts.counts[rows][:, columns].tocsr()
742
+ eligible = np.asarray(block.sum(axis=0)).reshape(-1) > 0
743
+ area = np.asarray(area_all[rows], dtype=float) if area_all is not None else np.full(len(rows), spacing * spacing)
744
+ prediction = np.stack([m.values[np.ix_(rows, columns)] for m in models], axis=2)
745
+ keep_genes = np.flatnonzero(eligible)
746
+ if len(keep_genes) < len(columns):
747
+ notes.append(f"Section {name}: {len(columns) - len(keep_genes):,} gene(s) without a single molecule were left out.")
748
+ say(dict(stage="section", section=name, done=index * parts, total=len(names) * parts))
749
+
750
+ def grid_step(stop, _genes, name=name, index=index):
751
+ say(dict(stage="section", section=name, done=index * parts + math.ceil(stop / fourier.GENE_BLOCK), total=len(names) * parts))
752
+ out = score_grid_section(block[:, keep_genes], prediction[:, keep_genes, :], counts.xy[rows], area, spacing_um=spacing,
753
+ models=[m.name for m in models], seeds=seeds, step=grid_step)
754
+ truth = np.sqrt(np.asarray(block[:, keep_genes].todense(), dtype=np.float64) / area[:, None])
755
+ pearson = {m.name: _nanmean(cal._corr_cols(prediction[:, keep_genes, j], truth)) for j, m in enumerate(models)}
756
+ sections.append(dict(name=name, out=out, eligible=np.ones(len(keep_genes), bool), sites=len(rows), pearson=pearson,
757
+ genes=[shared[g] for g in keep_genes]))
758
+ genes_used = sections[0]["genes"] if len(sections) == 1 else shared
759
+ if len(sections) > 1:
760
+ for s in sections: # per-gene tables need a common gene axis
761
+ if s["genes"] != shared:
762
+ s["eligible"] = np.asarray([g in set(s["genes"]) for g in shared])
763
+ _expand_grid_genes(s, shared)
764
+ settings = _settings("grid", counts, genes_used, models, seeds, spacing_um=spacing,
765
+ transform="sqrt(N_half / (0.5 a)), centered, each gene's halves divided by a common factor",
766
+ scales="seven physical scales of a two-dimensional Fourier basis: <128, 128–256, 256–512, 512–1,024, 1,024–2,048, 2,048–4,096 and ≥4,096 µm (full spatial cycles)",
767
+ gene_weighting="common factor of each gene's halves")
768
+ return _grid_result(sections, genes_used, [m.name for m in models], spacing, settings, notes)
769
+ # cells
770
+ if counts.unit != "um":
771
+ notes.append("The coordinates are taken to be micrometers: the cell graph links neighbors within 40 µm.")
772
+ seeds = CELL_SEEDS[: splits or len(CELL_SEEDS)]
773
+ populations = [within] if isinstance(within, str) else list(within or [])
774
+ labels_all = counts.column(cell_class)
775
+ if populations and labels_all is None:
776
+ raise InputError(f"No {cell_class} column, so the cells cannot be scored within a class.")
777
+ views = []
778
+ names = counts.section_names()
779
+ plan = [(name, pop) for name in names for pop in ((["All cells"] if all_cells or not populations else []) + populations)]
780
+ parts = len(seeds) * max(1, math.ceil(len(columns) / CELL_GENE_BLOCK))
781
+ total = len(plan) * len(models) * parts
782
+ for index, (name, population) in enumerate(plan):
783
+ rows = np.flatnonzero((counts.sections == name) & complete)
784
+ block = counts.counts[rows][:, columns].tocsr()
785
+ prediction = {m.name: m.values[np.ix_(rows, columns)].astype(np.float32) for m in models}
786
+ coords = counts.xy[rows]
787
+ if population == "All cells":
788
+ membership, view_coords = sparse.eye(len(rows), format="csr", dtype=np.float32), coords
789
+ else:
790
+ source = np.flatnonzero(np.asarray(labels_all[rows]).astype(str) == str(population))
791
+ if len(source) < CELL_GRAPH["minimum_component_cells"]:
792
+ raise InputError(f"Section {name}: fewer than {CELL_GRAPH['minimum_component_cells']} cells of class {population}.")
793
+ membership = sparse.coo_matrix((np.ones(len(source), np.float32), (np.arange(len(source)), source)),
794
+ shape=(len(source), len(rows))).tocsr()
795
+ view_coords = coords[source]
796
+ for k, (model, values) in enumerate(prediction.items()):
797
+ first = (index * len(models) + k) * parts
798
+ say(dict(stage="view", section=name, population=population, model=model, done=first, total=total))
799
+
800
+ def cell_step(done, _parts, name=name, population=population, model=model, first=first):
801
+ say(dict(stage="view", section=name, population=population, model=model, done=first + done, total=total))
802
+ try:
803
+ out = score_cell_view(block, values, coords, membership, view_coords, seeds, name, step=cell_step)
804
+ except cell_core.ContractError as error:
805
+ raise InputError(f"Section {name}, {population}: the cell graph could not be built ({error}).") from None
806
+ views.append(dict(model=model, section=name, population=population, out=out, seeds=seeds))
807
+ settings = _settings("cells", counts, shared, models, seeds, transform="log(1 + 2 N_half), centered within graph components, each gene's halves divided by a common factor",
808
+ scales="lazy symmetric diffusion S on the graph of the 8 nearest neighbors within 40 µm (Gaussian edge weights, components of at least 20 cells): S^16, S^8 − S^16, S^4 − S^8, S^2 − S^4, I − S^2",
809
+ gene_weighting="common factor of each gene's halves", fine_scale="C, V_model and V_RNA summed over the Fine and Finest scales",
810
+ populations=["All cells"] * bool(all_cells or not populations) + populations, cell_class_column=cell_class if populations else None)
811
+ return _cell_result(views, shared, settings, notes)
812
+
813
+
814
+ def _expand_grid_genes(section: dict, shared: list[str]) -> None:
815
+ """Put a section's per-gene arrays on the common gene axis, with zeros for genes it left out."""
816
+ index = [shared.index(g) for g in section["genes"]]
817
+ out = section["out"]
818
+ for key in ("vrna_g", "va_g", "vb_g"):
819
+ for band, values in out[key].items():
820
+ full = np.zeros(len(shared))
821
+ full[index] = values
822
+ out[key][band] = full
823
+ for key in ("c_g", "vm_g"):
824
+ for model in out[key]:
825
+ for band, values in out[key][model].items():
826
+ full = np.zeros(len(shared))
827
+ full[index] = values
828
+ out[key][model][band] = full
829
+
830
+
831
+ # ---------------------------------------------------------------- regions
832
+
833
+ def _expression(counts: np.ndarray, totals: np.ndarray) -> np.ndarray:
834
+ """log(1 + 10,000 N / total), as in the TLS analysis of the paper; a spot with no molecules stays at 0."""
835
+ return np.log1p(1e4 * counts / np.maximum(totals, 1)[:, None])
836
+
837
+
838
+ def _group_means(values: np.ndarray, codes: np.ndarray, sizes: np.ndarray) -> np.ndarray:
839
+ """The mean of ``values`` over each group, at every location of the group."""
840
+ return (np.bincount(codes, weights=values, minlength=len(sizes)) / sizes)[codes]
841
+
842
+
843
+ def _focus_units(xy: np.ndarray, focus_rows: np.ndarray) -> np.ndarray:
844
+ """Connected groups of focus spots, such as single TLS: focus spots are joined when they are
845
+ neighbors, closer than 1.25 times the median nearest-neighbor distance of the section's spots."""
846
+ from scipy.sparse.csgraph import connected_components
847
+ from scipy.spatial import cKDTree
848
+
849
+ near = cKDTree(xy).query(xy, k=2)[0][:, 1]
850
+ radius = cal.EDGE_MULTIPLIER * float(np.median(near[near > 0]))
851
+ pts = xy[focus_rows]
852
+ pairs = cKDTree(pts).query_pairs(radius, output_type="ndarray")
853
+ graph = sparse.coo_matrix((np.ones(len(pairs)), (pairs[:, 0], pairs[:, 1])), shape=(len(pts), len(pts)))
854
+ return connected_components(graph, directed=False)[1]
855
+
856
+
857
+ REGION_TERMS = ("whole", "between", "within", "focus", "focus_between", "focus_within")
858
+ REGION_RESAMPLES = 2000 # resamples of focus structures, and of blocks of 10 x 10 spots, as in the paper
859
+ REGION_BLOCK = 10
860
+
861
+
862
+ def _region_blocks(xy: np.ndarray, array_row, array_col) -> np.ndarray:
863
+ """Blocks of 10 x 10 spots, the resampling units of the terms that are not within the focus region.
864
+ With the spot grid of Space Ranger these are 10 array rows by 10 spots of a row (20 array columns
865
+ on Visium's staggered grid); otherwise blocks of 10 spot spacings each way, from the coordinates."""
866
+ if array_row is not None and array_col is not None:
867
+ r = np.asarray(array_row, dtype=np.float64)
868
+ q = np.asarray(array_col, dtype=np.float64)
869
+ if np.isfinite(r).all() and np.isfinite(q).all():
870
+ order = np.lexsort((q, r))
871
+ same_row = np.diff(r[order]) == 0
872
+ steps = np.diff(q[order])[same_row]
873
+ step = float(np.min(steps[steps > 0])) if (steps > 0).any() else 1.0
874
+ keys = np.column_stack([np.floor(r / REGION_BLOCK), np.floor(q / (REGION_BLOCK * step))])
875
+ return np.unique(keys, axis=0, return_inverse=True)[1].reshape(-1)
876
+ from scipy.spatial import cKDTree
877
+
878
+ dist, nearest = cKDTree(xy).query(xy, k=2)
879
+ d0 = float(np.median(dist[:, 1][dist[:, 1] > 0]))
880
+ dy = np.abs(xy[nearest[:, 1], 1] - xy[:, 1])
881
+ r0 = float(np.median(dy[dy > 1e-9 * d0])) if (dy > 1e-9 * d0).any() else d0
882
+ keys = np.column_stack([np.floor((xy[:, 1] - xy[:, 1].min()) / (REGION_BLOCK * r0)),
883
+ np.floor((xy[:, 0] - xy[:, 0].min()) / (REGION_BLOCK * d0))])
884
+ return np.unique(keys, axis=0, return_inverse=True)[1].reshape(-1)
885
+
886
+
887
+ def _resample(per_spot: dict, codes: np.ndarray, rng, draws: int) -> dict:
888
+ """Means over spots for resamples of whole units (focus structures or blocks): each draw takes as
889
+ many units as there are, with replacement."""
890
+ n = int(codes.max()) + 1
891
+ sums = {key: np.bincount(codes, weights=value, minlength=n) for key, value in per_spot.items()}
892
+ count = np.bincount(codes, minlength=n).astype(np.float64)
893
+ weights = np.stack([np.bincount(rng.integers(0, n, n), minlength=n) for _ in range(draws)]).astype(np.float64)
894
+ denominator = weights @ count
895
+ return {key: (weights @ value) / denominator for key, value in sums.items()}
896
+
897
+
898
+ def _region_parts(v: np.ndarray, region: tuple, units: tuple | None, focus_rows: np.ndarray | None, sd_all: float, sd_focus: float) -> dict:
899
+ """One gene's terms: the whole section, between and within regions, and within the focus region,
900
+ between its structures and within one structure. Each is divided by the standard deviation of the
901
+ gene's unsplit expression over the spots of the term."""
902
+ rm = _group_means(v, *region)
903
+ res = v - rm
904
+ sa = sd_all if sd_all > 0 else np.inf
905
+ out = dict(whole=(v - v.mean()) / sa, between=(rm - v.mean()) / sa, within=res / sa)
906
+ if units is not None:
907
+ st = sd_focus if sd_focus > 0 else np.inf
908
+ rt = res[focus_rows]
909
+ cm = _group_means(rt, *units)
910
+ out.update(focus=rt / st, focus_between=cm / st, focus_within=(rt - cm) / st)
911
+ return out
912
+
913
+
914
+ def _tested(point: np.ndarray, boot: np.ndarray) -> tuple[bool, float, float]:
915
+ """The paper's rule: a term is evaluated only when the lower limit of the 95% interval of its
916
+ split-half reliability is above zero. ``point`` and the rows of ``boot`` are (C, V_model, V_RNA)."""
917
+ testable = bool(np.percentile(boot[:, 2], 2.5) > 0)
918
+ ok = (boot[:, 1] > TOL) & (boot[:, 2] > 0)
919
+ ratios = boot[ok, 0] / np.sqrt(boot[ok, 1] * boot[ok, 2])
920
+ if testable and ok.mean() > 0.9 and point[1] > TOL and point[2] > 0:
921
+ return True, float(np.percentile(ratios, 2.5)), float(np.percentile(ratios, 97.5))
922
+ return testable, math.nan, math.nan
923
+
924
+
925
+ def regions(counts: Counts | str | Path, predictions, *, region: str = "region", focus: str | None = None,
926
+ genes: Iterable[str] | None = None, splits: int = REGION_SPLITS, resamples: int = REGION_RESAMPLES,
927
+ progress: Progress = None) -> Result:
928
+ """Agreement between and within annotated regions, as for the TLS sections of the paper
929
+ (Methods, "Decomposition by region" and "TLS-associated genes"). Each prediction and each RNA half
930
+ is split into the mean of its region and the residual from that mean. RSC is computed for the
931
+ whole section, between regions (the region means) and within regions (the residuals); when
932
+ ``focus`` names a region such as TLS, also within it (the residuals of its spots), between its
933
+ structures (connected groups of its spots) and within one structure. There is no spatial
934
+ decomposition.
935
+
936
+ As in the paper, measured expression is log(1 + 10,000 N / total), the total over all genes, each
937
+ half with its own total; each count split halves the scored genes' counts gene by gene and the
938
+ remaining molecules of each spot together. Every gene of the RNA halves and of the predictions is
939
+ divided by the standard deviation of the gene's unsplit expression over the spots of the term (all
940
+ spots with a region; the focus spots for the focus terms), so predictions should be on the same
941
+ log scale. C, V_model and V_RNA are summed over genes before the ratio. Intervals are 95%
942
+ percentile intervals from ``resamples`` resamples of focus structures (terms within the focus
943
+ region) and of blocks of 10 x 10 spots (the other terms), and a term is scored in a section only
944
+ when the lower limit of the interval of its split-half reliability is above zero. Over several
945
+ sections, the sums of the sections where a term is scored are averaged, in every resample too."""
946
+ if not isinstance(counts, Counts):
947
+ counts = read_counts(counts)
948
+ labels_all = counts.column(region)
949
+ if labels_all is None:
950
+ raise InputError(f"No {region} column: give a region for every spot.")
951
+ models = _models(predictions, counts)
952
+ shared, columns, complete, notes = _common(counts, models, genes)
953
+ say = progress or (lambda _: None)
954
+ labels_text = np.asarray(["" if (isinstance(v, float) and math.isnan(v)) or v is None else str(v) for v in labels_all], dtype=object)
955
+ labelled = labels_text != ""
956
+ if (~labelled & complete).any():
957
+ notes.append(f"{int((~labelled & complete).sum()):,} spots without a region were left out.")
958
+ ceiling = math.log1p(1e4) # the largest value log(1 + 10,000 N / total) can take
959
+ for m in models:
960
+ top = np.nanmax(m.values[np.ix_(np.flatnonzero(complete & labelled), columns)]) if (complete & labelled).any() else np.nan
961
+ if np.isfinite(top) and top > ceiling + 1e-9:
962
+ notes.append(f"{m.name}: some values are above {ceiling:.2f}, the largest that log(1 + 10,000 N/total) can take, so the "
963
+ "predictions are probably on another scale. Between and within regions each gene is divided by the "
964
+ "standard deviation of the measured expression, as in the paper, so predictions should be on that log scale.")
965
+ labels = {"whole": "Whole section", "between": "Between regions", "within": "Within regions"}
966
+ if focus:
967
+ labels.update(focus=f"Within {focus}", focus_between=f"Between {focus} structures", focus_within=f"Within one {focus} structure")
968
+ kinds = [(k, labels[k]) for k in REGION_TERMS if k in labels]
969
+ order = {k: i + 1 for i, (k, _) in enumerate(kinds)}
970
+ array_row_all, array_col_all = counts.column("array_row"), counts.column("array_col")
971
+ term_rows, gene_rows, pooled_in = [], [], {}
972
+ names = counts.section_names()
973
+ for index, name in enumerate(names):
974
+ rows = np.flatnonzero((counts.sections == name) & complete & labelled)
975
+ if len(rows) < 3:
976
+ raise InputError(f"Section {name}: fewer than three spots with a region and every prediction.")
977
+ say(dict(stage="section", section=name, done=index * splits, total=len(names) * splits))
978
+ totals = np.asarray(counts.counts[rows].sum(axis=1)).reshape(-1).astype(np.int64) # all genes
979
+ c = np.asarray(counts.counts[rows][:, columns].todense(), dtype=np.int32)
980
+ full = _expression(c, totals)
981
+ region_here = labels_text[rows]
982
+ _, codes = np.unique(region_here, return_inverse=True)
983
+ region_groups = (codes, np.bincount(codes).astype(np.float64))
984
+ focus_rows = (region_here == focus) if focus else None
985
+ units = None
986
+ if focus and int(focus_rows.sum()) >= 2:
987
+ unit_codes = _focus_units(counts.xy[rows], focus_rows)
988
+ units = (unit_codes, np.bincount(unit_codes).astype(np.float64))
989
+ notes.append(f"Section {name}: {int(focus_rows.sum()):,} {focus} spots in {len(units[1]):,} structure{'s' if len(units[1]) != 1 else ''}."
990
+ + (f" With one structure the intervals of the terms within {focus} say nothing, and there is no part between structures." if len(units[1]) == 1 else ""))
991
+ elif focus:
992
+ notes.append(f"Section {name}: fewer than two spots in {focus}, so the terms within {focus} are missing there.")
993
+ terms_here = [(k, label) for k, label in kinds if units is not None or not k.startswith("focus")]
994
+ sizes = {k: (int(focus_rows.sum()) if k.startswith("focus") else len(rows)) for k, _ in terms_here}
995
+ blocks = _region_blocks(counts.xy[rows], None if array_row_all is None else array_row_all[rows],
996
+ None if array_col_all is None else array_col_all[rows])
997
+ sd_all = full.std(axis=0)
998
+ sd_focus = full[focus_rows].std(axis=0) if units is not None else np.zeros(len(columns))
999
+ del full
1000
+ values = {m.name: m.values[np.ix_(rows, columns)] for m in models}
1001
+ n_genes = len(columns)
1002
+ acc = {k: {q: np.zeros(n_genes) for q in ("VR", "VA", "VB")} for k, _ in terms_here}
1003
+ acc_c = {(k, m): np.zeros(n_genes) for k, _ in terms_here for m in values}
1004
+ spot = {k: {q: np.zeros(sizes[k]) for q in ("VR", "VA", "VB")} for k, _ in terms_here} # summed over genes
1005
+ spot_c = {(k, m): np.zeros(sizes[k]) for k, _ in terms_here for m in values}
1006
+ seeds_c = {(k, m): np.zeros(splits) for k, _ in terms_here for m in values}
1007
+ seeds_vr = {k: np.zeros(splits) for k, _ in terms_here}
1008
+ rng = np.random.default_rng(cell_core.stable_seed("strsc.regions", name))
1009
+ rest = totals - c.sum(axis=1, dtype=np.int64)
1010
+ ha = np.empty_like(c)
1011
+ for i in range(splits):
1012
+ for j in range(n_genes): # gene by gene, then the other molecules: the order of the draws
1013
+ ha[:, j] = rng.binomial(c[:, j], 0.5)
1014
+ ta = rng.binomial(rest, 0.5) + ha.sum(axis=1, dtype=np.int64)
1015
+ per_a, per_b = np.maximum(ta, 1)[:, None], np.maximum(totals - ta, 1)[:, None]
1016
+ for g in range(n_genes): # each half one gene at a time: the same values as for all genes at once
1017
+ a_g = np.log1p(1e4 * ha[:, g:g + 1] / per_a)[:, 0]
1018
+ b_g = np.log1p(1e4 * (c[:, g:g + 1] - ha[:, g:g + 1]) / per_b)[:, 0]
1019
+ pa = _region_parts(a_g, region_groups, units, focus_rows, sd_all[g], sd_focus[g])
1020
+ pb = _region_parts(b_g, region_groups, units, focus_rows, sd_all[g], sd_focus[g])
1021
+ pp = {m: _region_parts(values[m][:, g], region_groups, units, focus_rows, sd_all[g], sd_focus[g]) for m in values}
1022
+ for k, _ in terms_here:
1023
+ A, B = pa[k], pb[k]
1024
+ ab = A * B
1025
+ vr = float(np.mean(ab))
1026
+ acc[k]["VR"][g] += vr
1027
+ acc[k]["VA"][g] += float(np.mean(A * A))
1028
+ acc[k]["VB"][g] += float(np.mean(B * B))
1029
+ spot[k]["VR"] += ab
1030
+ spot[k]["VA"] += A * A
1031
+ spot[k]["VB"] += B * B
1032
+ seeds_vr[k][i] += vr
1033
+ for m in values:
1034
+ both = 0.5 * (pp[m][k] * A + pp[m][k] * B)
1035
+ cc = float(np.mean(both))
1036
+ acc_c[k, m][g] += cc
1037
+ spot_c[k, m] += both
1038
+ seeds_c[k, m][i] += cc
1039
+ say(dict(stage="section", section=name, done=index * splits + i + 1, total=len(names) * splits))
1040
+ vm = {(k, m): np.zeros(n_genes) for k, _ in terms_here for m in values}
1041
+ spot_vm = {(k, m): np.zeros(sizes[k]) for k, _ in terms_here for m in values}
1042
+ for g in range(n_genes):
1043
+ for m in values:
1044
+ pp = _region_parts(values[m][:, g], region_groups, units, focus_rows, sd_all[g], sd_focus[g])
1045
+ for k, _ in terms_here:
1046
+ vm[k, m][g] = float(np.mean(pp[k] * pp[k]))
1047
+ spot_vm[k, m] += pp[k] * pp[k]
1048
+ boot_rng = np.random.default_rng(cell_core.stable_seed("strsc.regions.resamples", name))
1049
+ for k, label in terms_here:
1050
+ vr_g, va_g, vb_g = (acc[k][q] / splits for q in ("VR", "VA", "VB"))
1051
+ per_spot = {q: spot[k][q] / splits for q in ("VR", "VA", "VB")}
1052
+ for m in values:
1053
+ per_spot[("C", m)] = spot_c[k, m] / splits
1054
+ per_spot[("VM", m)] = spot_vm[k, m]
1055
+ boot = _resample(per_spot, units[0] if k.startswith("focus") else blocks, boot_rng, resamples)
1056
+ use = (sd_focus if k.startswith("focus") else sd_all) > 0
1057
+ for m in values:
1058
+ c_g, vm_g = acc_c[k, m] / splits, vm[k, m]
1059
+ point = np.array([c_g.sum(), vm_g.sum(), vr_g.sum(), va_g.sum(), vb_g.sum()])
1060
+ bt = np.column_stack([boot[("C", m)], boot[("VM", m)], boot["VR"], boot["VA"], boot["VB"]])
1061
+ testable, lo, hi = _tested(point, bt)
1062
+ row = _record(m, name, label, order[k], *point, seeds_c[k, m], seeds_vr[k], term=k, locations=sizes[k])
1063
+ row.update(testable=testable, rsc_ci_low=lo, rsc_ci_high=hi, V_RNA_ci_low=float(np.percentile(bt[:, 2], 2.5)),
1064
+ V_RNA_ci_high=float(np.percentile(bt[:, 2], 97.5)))
1065
+ if not testable:
1066
+ row.update(rsc=math.nan, status="NOT_TESTABLE")
1067
+ term_rows.append(row)
1068
+ gene_rows.extend({**r, "term": k} for r in _gene_rows(m, name, shared, [label], c_g[None, :], vm_g[None, :],
1069
+ vr_g[None, :], va_g[None, :], vb_g[None, :], use))
1070
+ pooled_in.setdefault((m, k, label), []).append((testable, point, bt))
1071
+ scales = pd.DataFrame(term_rows)
1072
+ if len(names) > 1:
1073
+ pooled = []
1074
+ for (model, kind, label), entries in pooled_in.items():
1075
+ have = [(p_, b_) for t_, p_, b_ in entries if t_]
1076
+ if not have:
1077
+ continue
1078
+ point = np.mean([p_ for p_, _ in have], axis=0)
1079
+ bt = np.mean([b_ for _, b_ in have], axis=0)
1080
+ testable, lo, hi = _tested(point, bt)
1081
+ row = _record(model, "All sections", label, order[kind], *point, term=kind, sections=len(have))
1082
+ row.update(testable=testable, rsc_ci_low=lo, rsc_ci_high=hi, V_RNA_ci_low=float(np.percentile(bt[:, 2], 2.5)),
1083
+ V_RNA_ci_high=float(np.percentile(bt[:, 2], 97.5)))
1084
+ if not testable:
1085
+ row.update(rsc=math.nan, status="NOT_TESTABLE")
1086
+ pooled.append(row)
1087
+ scales = pd.concat([scales, pd.DataFrame(pooled)], ignore_index=True)
1088
+ # every term gets a column, also where it was scored in no section
1089
+ summary_rows = {}
1090
+ for _, r in scales.iterrows():
1091
+ summary_rows.setdefault((r["model"], r["section"]), {"model": r["model"], "section": r["section"]})[r["term"]] = r["rsc"]
1092
+ summary = pd.DataFrame(list(summary_rows.values())).reindex(columns=["model", "section"] + [k for k, _ in kinds])
1093
+ settings = _settings("regions", counts, shared, models, tuple(range(splits)), region_column=region, focus=focus,
1094
+ transform="log(1 + 10,000 N / total over all genes), each half with its own total; each split halves the scored genes' counts gene by gene and the other molecules of each spot together",
1095
+ terms=[label for _, label in kinds],
1096
+ structures=(f"connected groups of {focus} spots: neighbors closer than 1.25 times the median nearest-neighbor distance" if focus else None),
1097
+ gene_weighting="every gene of the RNA halves and of the predictions divided by the standard deviation of its unsplit expression over the spots of the term",
1098
+ intervals=f"95% percentile intervals from {resamples:,} resamples of {focus or 'focus'} structures (terms within {focus or 'the focus region'}) and of blocks of 10 x 10 spots (other terms); a term is scored only when the lower limit for its V_RNA is above zero",
1099
+ pooling="mean over the sections where a term is scored, in every resample too")
1100
+ return Result("regions", scales, summary, pd.DataFrame(gene_rows), settings, notes)
1101
+
1102
+
1103
+ # ---------------------------------------------------------------- the paper's test with the expected expression
1104
+
1105
+ def _expected(values: np.ndarray, transform: Callable[[np.ndarray], np.ndarray]) -> np.ndarray:
1106
+ """E[transform(X)] for X ~ Poisson(N / 2), for every integer N in ``values``: the expected
1107
+ transformed half-count when the measured counts N are taken as Poisson rates."""
1108
+ from scipy import stats
1109
+
1110
+ unique, inverse = np.unique(values, return_inverse=True)
1111
+ out = np.zeros(len(unique))
1112
+ for i, n in enumerate(unique):
1113
+ mu = n / 2.0
1114
+ if mu == 0:
1115
+ out[i] = float(transform(np.zeros(1))[0])
1116
+ continue
1117
+ k = np.arange(0, int(math.ceil(mu + 12 * math.sqrt(mu) + 30)) + 1)
1118
+ out[i] = float(np.sum(stats.poisson.pmf(k, mu) * transform(k.astype(np.float64))))
1119
+ return out[inverse].reshape(values.shape)
1120
+
1121
+
1122
+ def ideal_check(counts: Counts | str | Path, layout: str = "spots", *, draws: int = 3, spacing_um: float | None = None,
1123
+ cell_class: str = "cell_class", within: str | Sequence[str] | None = None, genes: Iterable[str] | None = None,
1124
+ splits: int | None = None, seed: int = 20261106, progress: Progress = None) -> pd.DataFrame:
1125
+ """The paper's test of RSC (Methods, "Expected expression as a prediction"): the measured counts are
1126
+ taken as Poisson rates, count matrices are drawn from them, and on each the expected transformed
1127
+ half-count is scored as the prediction. Its RSC should be near 1 at every scale; in the paper it was
1128
+ 0.956 to 1.000. Returns the mean, lowest and highest RSC over the draws at each scale."""
1129
+ if not isinstance(counts, Counts):
1130
+ counts = read_counts(counts)
1131
+ columns = np.arange(len(counts.genes)) if genes is None else np.asarray([counts.genes.index(g) for g in genes])
1132
+ gene_names = [counts.genes[i] for i in columns]
1133
+ dense = np.asarray(counts.counts[:, columns].todense(), dtype=np.int64)
1134
+ if layout == "grid":
1135
+ spacing = float(spacing_um or counts.spacing_um or 0)
1136
+ area = counts.column("area")
1137
+ area = np.asarray(area, dtype=float) if area is not None else np.full(len(counts), spacing * spacing)
1138
+ root = _expected(dense, np.sqrt)
1139
+ expected = root / np.sqrt(0.5 * area)[:, None]
1140
+ else:
1141
+ expected = _expected(dense, lambda k: np.log1p(2.0 * k))
1142
+ rng = np.random.default_rng(seed)
1143
+ frames = []
1144
+ for d in range(draws):
1145
+ if progress:
1146
+ progress(dict(stage="draw", done=d, total=draws))
1147
+ drawn = rng.poisson(dense).astype(np.int64)
1148
+ simulated = Counts(sparse.csr_matrix(drawn), list(gene_names), counts.ids, counts.sections, counts.xy, counts.unit,
1149
+ counts.columns, counts.spacing_um, "drawn counts")
1150
+ result = rsc(simulated, {"Expected expression": expected}, layout, spacing_um=spacing_um, cell_class=cell_class,
1151
+ within=within, splits=splits)
1152
+ frame = result.scales[["section", "scale", "scale_order", "rsc"] + (["population"] if "population" in result.scales else [])].copy()
1153
+ frame["draw"] = d + 1
1154
+ frames.append(frame)
1155
+ joined = pd.concat(frames, ignore_index=True)
1156
+ keys = ["section"] + (["population"] if "population" in joined else []) + ["scale", "scale_order"]
1157
+ out = joined.groupby(keys, as_index=False, sort=False)["rsc"].agg(rsc_mean="mean", rsc_min="min", rsc_max="max")
1158
+ out["draws"] = draws
1159
+ return out
1160
+
1161
+
1162
+ # ---------------------------------------------------------------- bundle
1163
+
1164
+ def _bundle(result: Result, title: str, synthetic: bool) -> dict:
1165
+ layout = result.layout
1166
+ workflow = {"spots": "hest", "grid": "fourier", "cells": "cell", "regions": "regions"}[layout]
1167
+ bundle = evaluation_export.Bundle(workflow, title, synthetic)
1168
+ details_common = {k: v for k, v in result.settings.items() if k in ("toolkit", "count_splits", "gene_weighting", "transform")}
1169
+ if layout == "spots":
1170
+ bundle.metric("pearson_graph_support", "Whole-section Pearson r", "Mean gene-wise Pearson r with log(1 + count) on the graph-retained spots.", "correlation")
1171
+ bundle.metric("rsc_broad", "Broad-scale RSC", "Mean RSC at the Broadest and Broad scales, as for the spot-level models of the paper.")
1172
+ bundle.metric("rsc_fine", "Fine-scale RSC", "Mean RSC at the Fine and Finest scales, as for the spot-level models of the paper.")
1173
+ for row in result.summary.to_dict("records"):
1174
+ if row["section"] == "All sections":
1175
+ continue
1176
+ for field_, metric in (("whole_section_pearson_r", "pearson_graph_support"), ("broad_rsc", "rsc_broad"), ("fine_rsc", "rsc_fine")):
1177
+ bundle.row(row["model"], "Sections", row["section"], "specimen", metric, row[field_], "graph-retained spots", None, details_common)
1178
+ for k, label in enumerate(RELATIVE, start=1):
1179
+ bundle.metric(f"rsc_B{k}", f"RSC, {label}", "Relative graph scale; B1 broadest, B5 finest. These are not micrometer intervals.")
1180
+ bundle.metric("rna_split_agreement", "Split-half reliability", "Correlation between the two RNA halves at this scale.", "correlation", "split_half")
1181
+ for row in result.scales.to_dict("records"):
1182
+ if row["section"] == "All sections":
1183
+ continue
1184
+ details = {**details_common, **{k: row[k] for k in ("C", "V_model", "V_RNA", "V_A", "V_B", "pearson_r", "rsc_split_min", "rsc_split_max")}}
1185
+ bundle.row(row["model"], "Sections", row["section"], "specimen", f"rsc_B{row['scale_order']}", row["rsc"], "graph-retained spots", row["status"], details)
1186
+ key = ("Split-half reliability", f"Sections | B{row['scale_order']}", row["section"], "rna_split_agreement")
1187
+ if key not in bundle.identities:
1188
+ bundle.row("Split-half reliability", f"Sections | B{row['scale_order']}", row["section"], "specimen", "rna_split_agreement",
1189
+ row["split_half_reliability"], "graph-retained spots", None, details_common, role="split_half")
1190
+ elif layout == "grid":
1191
+ bundle.metric("physical_rsc", "RSC by physical scale", "RSC in this window of physical wavelength (a full spatial cycle).")
1192
+ bundle.metric("rna_split_agreement", "Split-half reliability", "Correlation between the two RNA halves in this window.", "correlation", "split_half")
1193
+ spacing = result.settings.get("spacing_um")
1194
+ for row in result.scales.to_dict("records"):
1195
+ if row["section"] == "All sections":
1196
+ continue
1197
+ task = f"{row['band']} | grid {spacing:g} um"
1198
+ details = {**details_common, **{k: row[k] for k in ("C", "V_model", "V_RNA", "V_A", "V_B", "pearson_r", "fourier_modes", "partial_window")}}
1199
+ bundle.row(row["model"], task, row["section"], "specimen", "physical_rsc", row["rsc"],
1200
+ "occupied grid bins" + ("; partial window" if row.get("partial_window") else ""), row["status"], details)
1201
+ key = ("Split-half reliability", task, row["section"], "rna_split_agreement")
1202
+ if key not in bundle.identities:
1203
+ bundle.row("Split-half reliability", task, row["section"], "specimen", "rna_split_agreement", row["split_half_reliability"],
1204
+ "occupied grid bins", None, details_common, role="split_half")
1205
+ elif layout == "cells":
1206
+ bundle.metric("cell_rsc", "Cell-level RSC", "RSC on the cell graph of this view and diffusion scale.")
1207
+ bundle.metric("rna_split_agreement", "Split-half reliability", "Correlation between the two RNA halves on this cell graph view.", "correlation", "split_half")
1208
+ for row in result.scales.to_dict("records"):
1209
+ if row["section"] == "All sections":
1210
+ continue
1211
+ task = f"{row['population']} | B{row['scale_order']}"
1212
+ bundle.row(row["model"], task, row["section"], "specimen", "cell_rsc", row["rsc"], "graph-retained cells", row["status"],
1213
+ {**details_common, **{k: row[k] for k in ("C", "V_model", "V_RNA", "V_A", "V_B", "pearson_r")}})
1214
+ key = ("Split-half reliability", task, row["section"], "rna_split_agreement")
1215
+ if key not in bundle.identities:
1216
+ bundle.row("Split-half reliability", task, row["section"], "specimen", "rna_split_agreement", row["split_half_reliability"],
1217
+ "graph-retained cells", None, details_common, role="split_half")
1218
+ else:
1219
+ bundle.metric("region_rsc", "RSC by region term", "RSC of the whole section, the region means (between), the residuals (within), and within the focus region, between its structures and within one structure; missing where the term was not testable.")
1220
+ bundle.metric("rna_split_agreement", "Split-half reliability", "Correlation between the two RNA halves in this term.", "correlation", "split_half")
1221
+ for row in result.scales.to_dict("records"):
1222
+ if row["section"] == "All sections":
1223
+ continue
1224
+ bundle.row(row["model"], row["scale"], row["section"], "specimen", "region_rsc", row["rsc"], "annotated spots", row["status"],
1225
+ {**details_common, **{k: row[k] for k in ("C", "V_model", "V_RNA", "V_A", "V_B", "pearson_r", "rsc_ci_low", "rsc_ci_high", "testable") if k in row}})
1226
+ key = ("Split-half reliability", row["scale"], row["section"], "rna_split_agreement")
1227
+ if key not in bundle.identities:
1228
+ bundle.row("Split-half reliability", row["scale"], row["section"], "specimen", "rna_split_agreement", row["split_half_reliability"],
1229
+ "annotated spots", None, details_common, role="split_half")
1230
+ bundle.data["notes"] = [f"Reproducible-signal correlation (RSC) computed with {result.settings.get('toolkit')}, layout {layout}."] + list(result.notes)
1231
+ bundle.data["settings"] = evaluation_export.finite_json(result.settings)
1232
+ return bundle.data