spatialbatch 0.0.1rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,9 @@
1
+ from importlib.metadata import version
2
+
3
+ from . import abc, samplers, types
4
+ from .io import SpatialDatasetCollection, write_sharded_sdata
5
+ from .loader import Loader
6
+
7
+ __version__ = version("spatialbatch")
8
+
9
+ __all__ = ["Loader", "SpatialDatasetCollection", "abc", "samplers", "types", "write_sharded_sdata"]
@@ -0,0 +1,3 @@
1
+ from .sampler import Sampler
2
+
3
+ __all__ = ["Sampler"]
@@ -0,0 +1,7 @@
1
+ """The abstract sampler is shared with annbatch: a sampler written for one loader works in the other."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from annbatch.abc.sampler import Sampler
6
+
7
+ __all__ = ["Sampler"]
spatialbatch/compat.py ADDED
@@ -0,0 +1,7 @@
1
+ """Optional-dependency shims (torch, cupy, jax), shared with annbatch."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from annbatch.compat import CupyArray, CupyCSRMatrix, IterableDataset, JaxArray, JAXCSRMatrix, Tensor
6
+
7
+ __all__ = ["CupyArray", "CupyCSRMatrix", "IterableDataset", "JAXCSRMatrix", "JaxArray", "Tensor"]
spatialbatch/io.py ADDED
@@ -0,0 +1,255 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING, Any, Literal, cast
4
+
5
+ import numpy as np
6
+ import zarr
7
+ from annbatch.io import DATASET_PREFIX, V1_ENCODING, DatasetCollection, write_sharded
8
+ from annbatch.utils import load_all_aligned
9
+ from tqdm.auto import tqdm
10
+ from zarr.codecs import ZstdCodec
11
+
12
+ from .utils import (
13
+ TileGeometry,
14
+ aligned_tiles,
15
+ element_to_cs_matrix,
16
+ is_identity_xy,
17
+ morton_order,
18
+ scale0,
19
+ tile_starts,
20
+ validate_inputs,
21
+ )
22
+
23
+ if TYPE_CHECKING:
24
+ from collections.abc import Iterable, Sequence
25
+ from os import PathLike
26
+
27
+ import pandas as pd
28
+ from spatialdata import SpatialData
29
+ from zarr.abc.codec import BytesBytesCodec
30
+
31
+ from .loader import _Item
32
+
33
+ __all__ = ["SpatialDatasetCollection", "write_sharded_sdata"]
34
+
35
+ #: Key of the group attribute that describes how a dataset of a collection was written.
36
+ ENCODING_KEY = "spatialbatch"
37
+
38
+
39
+ def _copy_sharded(
40
+ src: zarr.Array,
41
+ group: zarr.Group,
42
+ name: str,
43
+ *,
44
+ chunk: int,
45
+ shard: int,
46
+ compressors: Iterable[BytesBytesCodec],
47
+ ) -> None:
48
+ """Copy `src` to a sharded array, one shard-sized block at a time so that memory use stays at ~one shard."""
49
+ lead = src.shape[:-2]
50
+ height, width = src.shape[-2:]
51
+ dst = group.create_array(
52
+ name,
53
+ shape=src.shape,
54
+ dtype=src.dtype,
55
+ chunks=(*lead, min(chunk, height), min(chunk, width)),
56
+ shards=(*lead, min(shard, height), min(shard, width)),
57
+ compressors=tuple(compressors),
58
+ fill_value=0,
59
+ overwrite=True,
60
+ )
61
+ blocks = [(y, x) for y in range(0, height, shard) for x in range(0, width, shard)]
62
+ for y, x in tqdm(blocks, desc=f"Writing {name}", leave=False):
63
+ dst[..., y : y + shard, x : x + shard] = src[..., y : y + shard, x : x + shard]
64
+
65
+
66
+ def write_sharded_sdata(
67
+ group: zarr.Group,
68
+ sdata: SpatialData,
69
+ *,
70
+ key: str,
71
+ region: str = "cell_circles",
72
+ images: Sequence[str] = ("morphology_focus",),
73
+ labels: Sequence[str] = (),
74
+ table_name: str = "table",
75
+ cs: str = "global",
76
+ tile_dim_in_units: float | None = None,
77
+ tile_scale: float = 1.0,
78
+ block_px: int = 1024,
79
+ n_obs_per_chunk: int = 64,
80
+ shard_size: int | str = "1GB",
81
+ image_chunk: int = 256,
82
+ image_shard: int = 4096,
83
+ compressors: Iterable[BytesBytesCodec] = (ZstdCodec(level=3),),
84
+ ) -> None:
85
+ """Write one backed :class:`spatialdata.SpatialData` as a dataset of a spatialbatch collection.
86
+
87
+ The counterpart of :func:`annbatch.write_sharded`. What it writes at ``group[key]``:
88
+
89
+ - the table, with its rows sorted along a Morton (Z-order) curve of the tile centers so that consecutive rows
90
+ are spatially close, written with :func:`annbatch.write_sharded`, plus ``obsm["tile_origin"]``, the first
91
+ pixel ``(x, y)`` of each tile;
92
+ - ``images/<name>`` and ``labels/<name>``: the full-resolution arrays as sharded zarr v3 arrays, chunked so that
93
+ a tile touches few chunks and sharded so that a request is a few large reads.
94
+
95
+ Only elements whose pixel grid is the coordinate system `cs` (no rotation or scaling) can be written, because a
96
+ tile is then a plain slice. Elements such as a rotated H&E image are tiled in place instead, see
97
+ :meth:`spatialbatch.Loader.add_sdata`.
98
+
99
+ Parameters
100
+ ----------
101
+ group
102
+ The destination group, must be zarr v3.
103
+ sdata
104
+ A SpatialData read from zarr, whose table annotates only `region`.
105
+ key
106
+ The name of the dataset in `group`, ``dataset_<i>`` for it to be found by :class:`SpatialDatasetCollection`.
107
+ region, table_name, cs, tile_dim_in_units, tile_scale
108
+ See :meth:`spatialbatch.Loader.add_sdata` and :class:`spatialbatch.Loader`.
109
+ images, labels
110
+ Names of the elements to write.
111
+ block_px
112
+ Side in pixels of the grid used to order cells. Match it to the window the loader reads (1024 by default).
113
+ n_obs_per_chunk, shard_size
114
+ Chunking of the table, see :func:`annbatch.write_sharded`.
115
+ image_chunk, image_shard
116
+ Side in pixels of the chunks and of the shards of the images and labels. `image_shard` must be divisible by
117
+ `image_chunk`.
118
+ compressors
119
+ Compressors for the images and labels. The table uses the :func:`annbatch.write_sharded` default.
120
+ """
121
+ if not sdata.is_backed():
122
+ raise ValueError("`sdata` must be backed by a zarr store (use `spatialdata.read_zarr`).")
123
+ if image_shard % image_chunk != 0:
124
+ raise ValueError(f"image_shard ({image_shard}) must be divisible by image_chunk ({image_chunk}).")
125
+ validate_inputs(sdata, region, images, labels, cs, table_name)
126
+ for name in (*images, *labels):
127
+ if not is_identity_xy(element_to_cs_matrix(sdata[name], cs)):
128
+ raise NotImplementedError(
129
+ f"`{name}` is not in the pixel grid of `{cs}`, so its tiles are not slices and it cannot be written to a "
130
+ "collection. Tile it in place with `Loader.add_sdata` instead."
131
+ )
132
+ table, tiles = aligned_tiles(
133
+ sdata, region, table_name, cs, tile_scale=tile_scale, tile_dim_in_units=tile_dim_in_units
134
+ )
135
+ geometry = tile_starts(tiles)
136
+ order = morton_order(tiles["x"].to_numpy(), tiles["y"].to_numpy(), block_px)
137
+
138
+ adata = table[order].copy()
139
+ adata.uns.pop("spatialdata_attrs", None) # refers to the element names of the source, not to the collection
140
+ adata.obsm["tile_origin"] = np.stack([geometry.x0, geometry.y0], axis=1)[order].astype(np.int32)
141
+ write_sharded(group, adata, n_obs_per_chunk=min(n_obs_per_chunk, adata.shape[0]), shard_size=shard_size, key=key)
142
+
143
+ out = cast("zarr.Group", group[key])
144
+ kinds: tuple[tuple[Literal["images", "labels"], Sequence[str]], ...] = (("images", images), ("labels", labels))
145
+ for kind, names in kinds:
146
+ for name in names:
147
+ _copy_sharded(
148
+ scale0(str(sdata.path), kind, name),
149
+ out.require_group(kind),
150
+ name,
151
+ chunk=image_chunk,
152
+ shard=image_shard,
153
+ compressors=compressors,
154
+ )
155
+ out.update_attributes(
156
+ {
157
+ ENCODING_KEY: {
158
+ "encoding-version": "0.1.0",
159
+ "tile_hw": [geometry.height, geometry.width],
160
+ "cs": cs,
161
+ "region": region,
162
+ "images": list(images),
163
+ "labels": list(labels),
164
+ "block_px": block_px,
165
+ "source": str(sdata.path),
166
+ }
167
+ }
168
+ )
169
+
170
+
171
+ def open_dataset_group(
172
+ group: zarr.Group,
173
+ images: Sequence[str],
174
+ labels: Sequence[str],
175
+ annotations: Sequence[str] | None,
176
+ ) -> tuple[Any, pd.DataFrame | None, pd.DataFrame, _Item]:
177
+ """Open a dataset written by :func:`write_sharded_sdata` for :class:`spatialbatch.Loader`."""
178
+ from .loader import _Item, _Source
179
+
180
+ meta = cast("dict[str, Any] | None", group.attrs.get(ENCODING_KEY))
181
+ if meta is None:
182
+ raise ValueError(f"{group.name} was not written by `write_sharded_sdata`.")
183
+ missing = [n for n in images if n not in meta["images"]] + [n for n in labels if n not in meta["labels"]]
184
+ if missing:
185
+ raise ValueError(
186
+ f"{missing} are not in the collection, which has images {meta['images']} and labels {meta['labels']}."
187
+ )
188
+ adata = load_all_aligned(group)
189
+ origin = np.asarray(adata.obsm["tile_origin"][...])
190
+ height, width = meta["tile_hw"]
191
+ sources: dict[str, _Source] = {}
192
+ kinds: tuple[tuple[Literal["images", "labels"], Sequence[str]], ...] = (("images", images), ("labels", labels))
193
+ for kind, names in kinds:
194
+ for name in names:
195
+ array = cast("zarr.Array", cast("zarr.Group", group[kind])[name])
196
+ sources[name] = _Source(kind, array, array.shape[0] if kind == "images" else None, array.dtype)
197
+ obs = adata.obs if annotations is None else adata.obs[list(annotations)]
198
+ coords = np.asarray(adata.obsm["spatial"][...]) if "spatial" in adata.obsm else None
199
+ item = _Item(
200
+ TileGeometry(origin[:, 0].astype(np.int64), origin[:, 1].astype(np.int64), height, width), sources, coords
201
+ )
202
+ return adata.X, None if len(obs.columns) == 0 else obs, adata.var, item
203
+
204
+
205
+ class SpatialDatasetCollection(DatasetCollection):
206
+ """A collection of spatially sorted, sharded SpatialData datasets, for :meth:`spatialbatch.Loader.use_collection`.
207
+
208
+ This is :class:`annbatch.DatasetCollection` with a different writer. Iteration over the datasets,
209
+ :attr:`is_empty` and :meth:`obs` are inherited. A dataset is one SpatialData, written by
210
+ :func:`write_sharded_sdata`: unlike annbatch there is no mixing of rows across files on disk, since pixels are not
211
+ rows that can be moved around. Slides are mixed when sampling, as the chunks drawn by one request come from any
212
+ dataset.
213
+
214
+ Parameters
215
+ ----------
216
+ group
217
+ The base location of the collection, a :class:`zarr.Group` or a path ending in ``.zarr``.
218
+ mode
219
+ The mode the store is opened with if `group` is a path.
220
+
221
+ Examples
222
+ --------
223
+ >>> collection = SpatialDatasetCollection("collection.zarr")
224
+ >>> collection.add_sdatas(["a.zarr", "b.zarr"], images=["morphology_focus"], tile_dim_in_units=224)
225
+ >>> loader = Loader(images=["morphology_focus"], batch_size=32, shuffle=True).use_collection(collection)
226
+ """
227
+
228
+ def __init__(self, group: zarr.Group | str | PathLike[str], *, mode: str = "a"):
229
+ super().__init__(group, mode=mode) # type: ignore[arg-type]
230
+ if not isinstance(self._group, zarr.Group):
231
+ raise TypeError("A SpatialDatasetCollection is always zarr.")
232
+
233
+ def add_adatas(self, *args: Any, **kwargs: Any) -> None: # type: ignore[override]
234
+ """Not supported: a dataset of this collection holds pixels, written by :meth:`add_sdatas`."""
235
+ raise TypeError("A SpatialDatasetCollection holds pixels too: use `add_sdatas`.")
236
+
237
+ def add_sdatas(self, sdata_paths: Iterable[str | PathLike[str]], **kwargs: Any) -> None:
238
+ """Write SpatialData stores to the collection, each as a new dataset.
239
+
240
+ Parameters
241
+ ----------
242
+ sdata_paths
243
+ Paths to SpatialData zarr stores.
244
+ **kwargs
245
+ Passed to :func:`write_sharded_sdata`.
246
+ """
247
+ import spatialdata as sd
248
+
249
+ paths = list(sdata_paths)
250
+ first = len(self._dataset_keys)
251
+ for i, path in enumerate(tqdm(paths, desc="Adding to collection")):
252
+ write_sharded_sdata(
253
+ self._group, sd.read_zarr(cast("str", path)), key=f"{DATASET_PREFIX}_{first + i}", **kwargs
254
+ )
255
+ self._group.update_attributes(V1_ENCODING)