spatialbatch 0.0.1rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spatialbatch/__init__.py +9 -0
- spatialbatch/abc/__init__.py +3 -0
- spatialbatch/abc/sampler.py +7 -0
- spatialbatch/compat.py +7 -0
- spatialbatch/io.py +255 -0
- spatialbatch/loader.py +529 -0
- spatialbatch/py.typed +0 -0
- spatialbatch/samplers/__init__.py +5 -0
- spatialbatch/samplers/_random_sampler.py +68 -0
- spatialbatch/types.py +55 -0
- spatialbatch/utils.py +280 -0
- spatialbatch-0.0.1rc1.dist-info/METADATA +169 -0
- spatialbatch-0.0.1rc1.dist-info/RECORD +15 -0
- spatialbatch-0.0.1rc1.dist-info/WHEEL +4 -0
- spatialbatch-0.0.1rc1.dist-info/licenses/LICENSE +21 -0
spatialbatch/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
from importlib.metadata import version
|
|
2
|
+
|
|
3
|
+
from . import abc, samplers, types
|
|
4
|
+
from .io import SpatialDatasetCollection, write_sharded_sdata
|
|
5
|
+
from .loader import Loader
|
|
6
|
+
|
|
7
|
+
__version__ = version("spatialbatch")
|
|
8
|
+
|
|
9
|
+
__all__ = ["Loader", "SpatialDatasetCollection", "abc", "samplers", "types", "write_sharded_sdata"]
|
spatialbatch/compat.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Optional-dependency shims (torch, cupy, jax), shared with annbatch."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from annbatch.compat import CupyArray, CupyCSRMatrix, IterableDataset, JaxArray, JAXCSRMatrix, Tensor
|
|
6
|
+
|
|
7
|
+
__all__ = ["CupyArray", "CupyCSRMatrix", "IterableDataset", "JAXCSRMatrix", "JaxArray", "Tensor"]
|
spatialbatch/io.py
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any, Literal, cast
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import zarr
|
|
7
|
+
from annbatch.io import DATASET_PREFIX, V1_ENCODING, DatasetCollection, write_sharded
|
|
8
|
+
from annbatch.utils import load_all_aligned
|
|
9
|
+
from tqdm.auto import tqdm
|
|
10
|
+
from zarr.codecs import ZstdCodec
|
|
11
|
+
|
|
12
|
+
from .utils import (
|
|
13
|
+
TileGeometry,
|
|
14
|
+
aligned_tiles,
|
|
15
|
+
element_to_cs_matrix,
|
|
16
|
+
is_identity_xy,
|
|
17
|
+
morton_order,
|
|
18
|
+
scale0,
|
|
19
|
+
tile_starts,
|
|
20
|
+
validate_inputs,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from collections.abc import Iterable, Sequence
|
|
25
|
+
from os import PathLike
|
|
26
|
+
|
|
27
|
+
import pandas as pd
|
|
28
|
+
from spatialdata import SpatialData
|
|
29
|
+
from zarr.abc.codec import BytesBytesCodec
|
|
30
|
+
|
|
31
|
+
from .loader import _Item
|
|
32
|
+
|
|
33
|
+
__all__ = ["SpatialDatasetCollection", "write_sharded_sdata"]
|
|
34
|
+
|
|
35
|
+
#: Key of the group attribute that describes how a dataset of a collection was written.
|
|
36
|
+
ENCODING_KEY = "spatialbatch"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _copy_sharded(
|
|
40
|
+
src: zarr.Array,
|
|
41
|
+
group: zarr.Group,
|
|
42
|
+
name: str,
|
|
43
|
+
*,
|
|
44
|
+
chunk: int,
|
|
45
|
+
shard: int,
|
|
46
|
+
compressors: Iterable[BytesBytesCodec],
|
|
47
|
+
) -> None:
|
|
48
|
+
"""Copy `src` to a sharded array, one shard-sized block at a time so that memory use stays at ~one shard."""
|
|
49
|
+
lead = src.shape[:-2]
|
|
50
|
+
height, width = src.shape[-2:]
|
|
51
|
+
dst = group.create_array(
|
|
52
|
+
name,
|
|
53
|
+
shape=src.shape,
|
|
54
|
+
dtype=src.dtype,
|
|
55
|
+
chunks=(*lead, min(chunk, height), min(chunk, width)),
|
|
56
|
+
shards=(*lead, min(shard, height), min(shard, width)),
|
|
57
|
+
compressors=tuple(compressors),
|
|
58
|
+
fill_value=0,
|
|
59
|
+
overwrite=True,
|
|
60
|
+
)
|
|
61
|
+
blocks = [(y, x) for y in range(0, height, shard) for x in range(0, width, shard)]
|
|
62
|
+
for y, x in tqdm(blocks, desc=f"Writing {name}", leave=False):
|
|
63
|
+
dst[..., y : y + shard, x : x + shard] = src[..., y : y + shard, x : x + shard]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def write_sharded_sdata(
|
|
67
|
+
group: zarr.Group,
|
|
68
|
+
sdata: SpatialData,
|
|
69
|
+
*,
|
|
70
|
+
key: str,
|
|
71
|
+
region: str = "cell_circles",
|
|
72
|
+
images: Sequence[str] = ("morphology_focus",),
|
|
73
|
+
labels: Sequence[str] = (),
|
|
74
|
+
table_name: str = "table",
|
|
75
|
+
cs: str = "global",
|
|
76
|
+
tile_dim_in_units: float | None = None,
|
|
77
|
+
tile_scale: float = 1.0,
|
|
78
|
+
block_px: int = 1024,
|
|
79
|
+
n_obs_per_chunk: int = 64,
|
|
80
|
+
shard_size: int | str = "1GB",
|
|
81
|
+
image_chunk: int = 256,
|
|
82
|
+
image_shard: int = 4096,
|
|
83
|
+
compressors: Iterable[BytesBytesCodec] = (ZstdCodec(level=3),),
|
|
84
|
+
) -> None:
|
|
85
|
+
"""Write one backed :class:`spatialdata.SpatialData` as a dataset of a spatialbatch collection.
|
|
86
|
+
|
|
87
|
+
The counterpart of :func:`annbatch.write_sharded`. What it writes at ``group[key]``:
|
|
88
|
+
|
|
89
|
+
- the table, with its rows sorted along a Morton (Z-order) curve of the tile centers so that consecutive rows
|
|
90
|
+
are spatially close, written with :func:`annbatch.write_sharded`, plus ``obsm["tile_origin"]``, the first
|
|
91
|
+
pixel ``(x, y)`` of each tile;
|
|
92
|
+
- ``images/<name>`` and ``labels/<name>``: the full-resolution arrays as sharded zarr v3 arrays, chunked so that
|
|
93
|
+
a tile touches few chunks and sharded so that a request is a few large reads.
|
|
94
|
+
|
|
95
|
+
Only elements whose pixel grid is the coordinate system `cs` (no rotation or scaling) can be written, because a
|
|
96
|
+
tile is then a plain slice. Elements such as a rotated H&E image are tiled in place instead, see
|
|
97
|
+
:meth:`spatialbatch.Loader.add_sdata`.
|
|
98
|
+
|
|
99
|
+
Parameters
|
|
100
|
+
----------
|
|
101
|
+
group
|
|
102
|
+
The destination group, must be zarr v3.
|
|
103
|
+
sdata
|
|
104
|
+
A SpatialData read from zarr, whose table annotates only `region`.
|
|
105
|
+
key
|
|
106
|
+
The name of the dataset in `group`, ``dataset_<i>`` for it to be found by :class:`SpatialDatasetCollection`.
|
|
107
|
+
region, table_name, cs, tile_dim_in_units, tile_scale
|
|
108
|
+
See :meth:`spatialbatch.Loader.add_sdata` and :class:`spatialbatch.Loader`.
|
|
109
|
+
images, labels
|
|
110
|
+
Names of the elements to write.
|
|
111
|
+
block_px
|
|
112
|
+
Side in pixels of the grid used to order cells. Match it to the window the loader reads (1024 by default).
|
|
113
|
+
n_obs_per_chunk, shard_size
|
|
114
|
+
Chunking of the table, see :func:`annbatch.write_sharded`.
|
|
115
|
+
image_chunk, image_shard
|
|
116
|
+
Side in pixels of the chunks and of the shards of the images and labels. `image_shard` must be divisible by
|
|
117
|
+
`image_chunk`.
|
|
118
|
+
compressors
|
|
119
|
+
Compressors for the images and labels. The table uses the :func:`annbatch.write_sharded` default.
|
|
120
|
+
"""
|
|
121
|
+
if not sdata.is_backed():
|
|
122
|
+
raise ValueError("`sdata` must be backed by a zarr store (use `spatialdata.read_zarr`).")
|
|
123
|
+
if image_shard % image_chunk != 0:
|
|
124
|
+
raise ValueError(f"image_shard ({image_shard}) must be divisible by image_chunk ({image_chunk}).")
|
|
125
|
+
validate_inputs(sdata, region, images, labels, cs, table_name)
|
|
126
|
+
for name in (*images, *labels):
|
|
127
|
+
if not is_identity_xy(element_to_cs_matrix(sdata[name], cs)):
|
|
128
|
+
raise NotImplementedError(
|
|
129
|
+
f"`{name}` is not in the pixel grid of `{cs}`, so its tiles are not slices and it cannot be written to a "
|
|
130
|
+
"collection. Tile it in place with `Loader.add_sdata` instead."
|
|
131
|
+
)
|
|
132
|
+
table, tiles = aligned_tiles(
|
|
133
|
+
sdata, region, table_name, cs, tile_scale=tile_scale, tile_dim_in_units=tile_dim_in_units
|
|
134
|
+
)
|
|
135
|
+
geometry = tile_starts(tiles)
|
|
136
|
+
order = morton_order(tiles["x"].to_numpy(), tiles["y"].to_numpy(), block_px)
|
|
137
|
+
|
|
138
|
+
adata = table[order].copy()
|
|
139
|
+
adata.uns.pop("spatialdata_attrs", None) # refers to the element names of the source, not to the collection
|
|
140
|
+
adata.obsm["tile_origin"] = np.stack([geometry.x0, geometry.y0], axis=1)[order].astype(np.int32)
|
|
141
|
+
write_sharded(group, adata, n_obs_per_chunk=min(n_obs_per_chunk, adata.shape[0]), shard_size=shard_size, key=key)
|
|
142
|
+
|
|
143
|
+
out = cast("zarr.Group", group[key])
|
|
144
|
+
kinds: tuple[tuple[Literal["images", "labels"], Sequence[str]], ...] = (("images", images), ("labels", labels))
|
|
145
|
+
for kind, names in kinds:
|
|
146
|
+
for name in names:
|
|
147
|
+
_copy_sharded(
|
|
148
|
+
scale0(str(sdata.path), kind, name),
|
|
149
|
+
out.require_group(kind),
|
|
150
|
+
name,
|
|
151
|
+
chunk=image_chunk,
|
|
152
|
+
shard=image_shard,
|
|
153
|
+
compressors=compressors,
|
|
154
|
+
)
|
|
155
|
+
out.update_attributes(
|
|
156
|
+
{
|
|
157
|
+
ENCODING_KEY: {
|
|
158
|
+
"encoding-version": "0.1.0",
|
|
159
|
+
"tile_hw": [geometry.height, geometry.width],
|
|
160
|
+
"cs": cs,
|
|
161
|
+
"region": region,
|
|
162
|
+
"images": list(images),
|
|
163
|
+
"labels": list(labels),
|
|
164
|
+
"block_px": block_px,
|
|
165
|
+
"source": str(sdata.path),
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def open_dataset_group(
|
|
172
|
+
group: zarr.Group,
|
|
173
|
+
images: Sequence[str],
|
|
174
|
+
labels: Sequence[str],
|
|
175
|
+
annotations: Sequence[str] | None,
|
|
176
|
+
) -> tuple[Any, pd.DataFrame | None, pd.DataFrame, _Item]:
|
|
177
|
+
"""Open a dataset written by :func:`write_sharded_sdata` for :class:`spatialbatch.Loader`."""
|
|
178
|
+
from .loader import _Item, _Source
|
|
179
|
+
|
|
180
|
+
meta = cast("dict[str, Any] | None", group.attrs.get(ENCODING_KEY))
|
|
181
|
+
if meta is None:
|
|
182
|
+
raise ValueError(f"{group.name} was not written by `write_sharded_sdata`.")
|
|
183
|
+
missing = [n for n in images if n not in meta["images"]] + [n for n in labels if n not in meta["labels"]]
|
|
184
|
+
if missing:
|
|
185
|
+
raise ValueError(
|
|
186
|
+
f"{missing} are not in the collection, which has images {meta['images']} and labels {meta['labels']}."
|
|
187
|
+
)
|
|
188
|
+
adata = load_all_aligned(group)
|
|
189
|
+
origin = np.asarray(adata.obsm["tile_origin"][...])
|
|
190
|
+
height, width = meta["tile_hw"]
|
|
191
|
+
sources: dict[str, _Source] = {}
|
|
192
|
+
kinds: tuple[tuple[Literal["images", "labels"], Sequence[str]], ...] = (("images", images), ("labels", labels))
|
|
193
|
+
for kind, names in kinds:
|
|
194
|
+
for name in names:
|
|
195
|
+
array = cast("zarr.Array", cast("zarr.Group", group[kind])[name])
|
|
196
|
+
sources[name] = _Source(kind, array, array.shape[0] if kind == "images" else None, array.dtype)
|
|
197
|
+
obs = adata.obs if annotations is None else adata.obs[list(annotations)]
|
|
198
|
+
coords = np.asarray(adata.obsm["spatial"][...]) if "spatial" in adata.obsm else None
|
|
199
|
+
item = _Item(
|
|
200
|
+
TileGeometry(origin[:, 0].astype(np.int64), origin[:, 1].astype(np.int64), height, width), sources, coords
|
|
201
|
+
)
|
|
202
|
+
return adata.X, None if len(obs.columns) == 0 else obs, adata.var, item
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class SpatialDatasetCollection(DatasetCollection):
|
|
206
|
+
"""A collection of spatially sorted, sharded SpatialData datasets, for :meth:`spatialbatch.Loader.use_collection`.
|
|
207
|
+
|
|
208
|
+
This is :class:`annbatch.DatasetCollection` with a different writer. Iteration over the datasets,
|
|
209
|
+
:attr:`is_empty` and :meth:`obs` are inherited. A dataset is one SpatialData, written by
|
|
210
|
+
:func:`write_sharded_sdata`: unlike annbatch there is no mixing of rows across files on disk, since pixels are not
|
|
211
|
+
rows that can be moved around. Slides are mixed when sampling, as the chunks drawn by one request come from any
|
|
212
|
+
dataset.
|
|
213
|
+
|
|
214
|
+
Parameters
|
|
215
|
+
----------
|
|
216
|
+
group
|
|
217
|
+
The base location of the collection, a :class:`zarr.Group` or a path ending in ``.zarr``.
|
|
218
|
+
mode
|
|
219
|
+
The mode the store is opened with if `group` is a path.
|
|
220
|
+
|
|
221
|
+
Examples
|
|
222
|
+
--------
|
|
223
|
+
>>> collection = SpatialDatasetCollection("collection.zarr")
|
|
224
|
+
>>> collection.add_sdatas(["a.zarr", "b.zarr"], images=["morphology_focus"], tile_dim_in_units=224)
|
|
225
|
+
>>> loader = Loader(images=["morphology_focus"], batch_size=32, shuffle=True).use_collection(collection)
|
|
226
|
+
"""
|
|
227
|
+
|
|
228
|
+
def __init__(self, group: zarr.Group | str | PathLike[str], *, mode: str = "a"):
|
|
229
|
+
super().__init__(group, mode=mode) # type: ignore[arg-type]
|
|
230
|
+
if not isinstance(self._group, zarr.Group):
|
|
231
|
+
raise TypeError("A SpatialDatasetCollection is always zarr.")
|
|
232
|
+
|
|
233
|
+
def add_adatas(self, *args: Any, **kwargs: Any) -> None: # type: ignore[override]
|
|
234
|
+
"""Not supported: a dataset of this collection holds pixels, written by :meth:`add_sdatas`."""
|
|
235
|
+
raise TypeError("A SpatialDatasetCollection holds pixels too: use `add_sdatas`.")
|
|
236
|
+
|
|
237
|
+
def add_sdatas(self, sdata_paths: Iterable[str | PathLike[str]], **kwargs: Any) -> None:
|
|
238
|
+
"""Write SpatialData stores to the collection, each as a new dataset.
|
|
239
|
+
|
|
240
|
+
Parameters
|
|
241
|
+
----------
|
|
242
|
+
sdata_paths
|
|
243
|
+
Paths to SpatialData zarr stores.
|
|
244
|
+
**kwargs
|
|
245
|
+
Passed to :func:`write_sharded_sdata`.
|
|
246
|
+
"""
|
|
247
|
+
import spatialdata as sd
|
|
248
|
+
|
|
249
|
+
paths = list(sdata_paths)
|
|
250
|
+
first = len(self._dataset_keys)
|
|
251
|
+
for i, path in enumerate(tqdm(paths, desc="Adding to collection")):
|
|
252
|
+
write_sharded_sdata(
|
|
253
|
+
self._group, sd.read_zarr(cast("str", path)), key=f"{DATASET_PREFIX}_{first + i}", **kwargs
|
|
254
|
+
)
|
|
255
|
+
self._group.update_attributes(V1_ENCODING)
|