cellcanvas-core 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cellcanvas_core/__init__.py +10 -0
- cellcanvas_core/data/_geometry.py +84 -0
- cellcanvas_core/data/_image.py +71 -0
- cellcanvas_core/data/_tests/__init__.py +0 -0
- cellcanvas_core/data/_tests/test_annotation_alignment.py +116 -0
- cellcanvas_core/data/_tests/test_data_manager.py +62 -0
- cellcanvas_core/data/_tests/test_feature_chunks.py +41 -0
- cellcanvas_core/data/_tests/test_feature_contracts.py +83 -0
- cellcanvas_core/data/_tests/test_ngff_images.py +132 -0
- cellcanvas_core/data/_tests/test_seeding.py +188 -0
- cellcanvas_core/data/_tests/test_spatial_coordinates.py +126 -0
- cellcanvas_core/data/_tests/test_storage.py +87 -0
- cellcanvas_core/data/data_manager.py +65 -0
- cellcanvas_core/data/data_set.py +202 -0
- cellcanvas_core/data/seeding.py +173 -0
- cellcanvas_core/py.typed +5 -0
- cellcanvas_core/semantic/__init__.py +0 -0
- cellcanvas_core/semantic/_tests/__init__.py +0 -0
- cellcanvas_core/semantic/_tests/mocks.py +27 -0
- cellcanvas_core/semantic/_tests/test_fewshot.py +316 -0
- cellcanvas_core/semantic/_tests/test_picking.py +129 -0
- cellcanvas_core/semantic/_tests/test_prediction_storage.py +189 -0
- cellcanvas_core/semantic/_tests/test_routing.py +473 -0
- cellcanvas_core/semantic/_tests/test_segmentation_manager.py +72 -0
- cellcanvas_core/semantic/fewshot.py +495 -0
- cellcanvas_core/semantic/picking.py +122 -0
- cellcanvas_core/semantic/routing.py +389 -0
- cellcanvas_core/semantic/segmentation_manager.py +241 -0
- cellcanvas_core-0.1.0.dist-info/METADATA +56 -0
- cellcanvas_core-0.1.0.dist-info/RECORD +32 -0
- cellcanvas_core-0.1.0.dist-info/WHEEL +4 -0
- cellcanvas_core-0.1.0.dist-info/licenses/LICENSE +22 -0
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Core components for CellCanvas"""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("cellcanvas-core")
|
|
7
|
+
except PackageNotFoundError:
|
|
8
|
+
__version__ = "uninstalled"
|
|
9
|
+
__author__ = "CellCanvas team"
|
|
10
|
+
__email__ = "kevin.yamauchi@gmail.com"
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Spatial coordinates for the selected NGFF image level."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class SpatialTransform:
|
|
10
|
+
"""ZYX coordinates: world = voxel * scale + translation.
|
|
11
|
+
|
|
12
|
+
Missing metadata means pixel coordinates with unspecified units.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
scale: tuple[float, float, float] = (1.0, 1.0, 1.0)
|
|
16
|
+
translation: tuple[float, float, float] = (0.0, 0.0, 0.0)
|
|
17
|
+
units: tuple[str | None, str | None, str | None] = (None, None, None)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def spatial_transform(metadata, level=0, axes=("c", "z", "y", "x")):
|
|
21
|
+
"""Compose level transforms followed by multiscale transforms.
|
|
22
|
+
|
|
23
|
+
Supports inline scale and translation vectors from NGFF 0.4/0.5.
|
|
24
|
+
Unsupported transforms raise an error rather than assuming unit spacing.
|
|
25
|
+
"""
|
|
26
|
+
image_metadata = metadata.get("ome", metadata)
|
|
27
|
+
if "multiscales" not in image_metadata:
|
|
28
|
+
return SpatialTransform()
|
|
29
|
+
multiscales = image_metadata["multiscales"]
|
|
30
|
+
if not isinstance(multiscales, list) or len(multiscales) != 1:
|
|
31
|
+
raise ValueError("Select a group with exactly one multiscale image")
|
|
32
|
+
multiscale = multiscales[0]
|
|
33
|
+
declared_axes = multiscale.get("axes", [])
|
|
34
|
+
if axes not in (("c", "z", "y", "x"), ("z", "y", "x")):
|
|
35
|
+
raise ValueError("Spatial coordinates require CZYX or ZYX axes")
|
|
36
|
+
if [axis.get("name") for axis in declared_axes] != list(axes):
|
|
37
|
+
raise ValueError("Spatial coordinates must match the declared axes")
|
|
38
|
+
datasets = multiscale.get("datasets", [])
|
|
39
|
+
if not isinstance(level, int) or isinstance(level, bool) or not 0 <= level < len(datasets):
|
|
40
|
+
raise ValueError("image_level is not listed in NGFF datasets")
|
|
41
|
+
scale = np.ones(len(axes))
|
|
42
|
+
translation = np.zeros(len(axes))
|
|
43
|
+
transform_lists = [datasets[level].get("coordinateTransformations")]
|
|
44
|
+
if "coordinateTransformations" in multiscale:
|
|
45
|
+
transform_lists.append(multiscale["coordinateTransformations"])
|
|
46
|
+
for transforms in transform_lists:
|
|
47
|
+
if not isinstance(transforms, list) or not all(
|
|
48
|
+
isinstance(transform, dict) for transform in transforms
|
|
49
|
+
) or [
|
|
50
|
+
transform.get("type") for transform in transforms
|
|
51
|
+
] not in (["scale"], ["scale", "translation"]):
|
|
52
|
+
raise ValueError("NGFF transforms require one scale followed by an optional translation")
|
|
53
|
+
for transform in transforms:
|
|
54
|
+
kind = transform["type"]
|
|
55
|
+
try:
|
|
56
|
+
vector = np.asarray(transform[kind], dtype=float)
|
|
57
|
+
except (KeyError, TypeError, ValueError):
|
|
58
|
+
raise ValueError("NGFF transforms require inline numeric vectors") from None
|
|
59
|
+
if vector.shape != (len(axes),) or not np.all(np.isfinite(vector)):
|
|
60
|
+
raise ValueError("NGFF transform vectors must match the axes and contain finite values")
|
|
61
|
+
if kind == "scale":
|
|
62
|
+
if np.any(vector <= 0):
|
|
63
|
+
raise ValueError("NGFF scale values must be positive")
|
|
64
|
+
scale *= vector
|
|
65
|
+
translation *= vector
|
|
66
|
+
else:
|
|
67
|
+
translation += vector
|
|
68
|
+
units = tuple(axis.get("unit") for axis in declared_axes[-3:])
|
|
69
|
+
return SpatialTransform(tuple(scale[-3:]), tuple(translation[-3:]), units)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def validate_alignment(metadata, level, expected):
|
|
73
|
+
"""Require annotation coordinates to agree within a billionth of a voxel.
|
|
74
|
+
|
|
75
|
+
Units must match exactly; no implicit unit conversion is performed.
|
|
76
|
+
"""
|
|
77
|
+
actual = spatial_transform(metadata, level, axes=("z", "y", "x"))
|
|
78
|
+
if actual.units != expected.units:
|
|
79
|
+
raise ValueError("Annotation spatial units do not match the image")
|
|
80
|
+
if not np.allclose(actual.scale, expected.scale, rtol=1e-9, atol=0):
|
|
81
|
+
raise ValueError("Annotation spatial scale does not match the image")
|
|
82
|
+
if np.any(np.abs(np.asarray(actual.translation) - expected.translation) >
|
|
83
|
+
np.asarray(expected.scale) * 1e-9):
|
|
84
|
+
raise ValueError("Annotation spatial origin does not match the image")
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Read image arrays without guessing NGFF axes or resolution paths."""
|
|
2
|
+
|
|
3
|
+
from copy import deepcopy
|
|
4
|
+
|
|
5
|
+
import zarr
|
|
6
|
+
|
|
7
|
+
from cellcanvas_core.data._geometry import spatial_transform
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def open_image(source, level=0, axes=("c", "z", "y", "x"), mode="r"):
|
|
11
|
+
"""Return a CZYX array and a copy of its source metadata.
|
|
12
|
+
|
|
13
|
+
NGFF 0.5 and legacy 0.4 groups use ordered dataset paths from metadata.
|
|
14
|
+
Bare arrays and legacy groups containing ``0`` remain supported.
|
|
15
|
+
Coordinate transformations are preserved, not applied to pixel values.
|
|
16
|
+
"""
|
|
17
|
+
if not isinstance(level, int) or isinstance(level, bool) or level < 0:
|
|
18
|
+
raise ValueError("image_level must be a nonnegative integer")
|
|
19
|
+
root = zarr.open(source, mode=mode)
|
|
20
|
+
metadata = deepcopy(root.attrs.asdict())
|
|
21
|
+
require_dimension_names = False
|
|
22
|
+
if isinstance(root, zarr.Array):
|
|
23
|
+
if level != 0:
|
|
24
|
+
raise ValueError("Bare image arrays only support image_level=0")
|
|
25
|
+
image = root
|
|
26
|
+
elif "ome" in metadata or "multiscales" in metadata:
|
|
27
|
+
ome = metadata.get("ome")
|
|
28
|
+
if ome is not None:
|
|
29
|
+
if not isinstance(ome, dict) or ome.get("version") != "0.5":
|
|
30
|
+
raise ValueError("Only NGFF version 0.5 is supported in the ome namespace")
|
|
31
|
+
if root.metadata.zarr_format != 3:
|
|
32
|
+
raise ValueError("NGFF 0.5 requires Zarr format 3")
|
|
33
|
+
require_dimension_names = True
|
|
34
|
+
multiscales = ome.get("multiscales")
|
|
35
|
+
else:
|
|
36
|
+
multiscales = metadata["multiscales"]
|
|
37
|
+
if not isinstance(multiscales, list) or len(multiscales) != 1:
|
|
38
|
+
raise ValueError("Select a group with exactly one multiscale image")
|
|
39
|
+
multiscale = multiscales[0]
|
|
40
|
+
if ome is None and multiscale.get("version") != "0.4":
|
|
41
|
+
raise ValueError("Only NGFF version 0.4 image metadata is supported")
|
|
42
|
+
declared_axes = multiscale.get("axes", [])
|
|
43
|
+
if not all(isinstance(axis, dict) for axis in declared_axes) or [
|
|
44
|
+
axis.get("name") for axis in declared_axes
|
|
45
|
+
] != list(axes):
|
|
46
|
+
raise ValueError(f"NGFF image axes must be ordered {', '.join(axes)}")
|
|
47
|
+
datasets = multiscale.get("datasets", [])
|
|
48
|
+
if level >= len(datasets):
|
|
49
|
+
raise ValueError("image_level is not listed in NGFF datasets")
|
|
50
|
+
path = datasets[level].get("path")
|
|
51
|
+
if not isinstance(path, str) or not path or any(
|
|
52
|
+
part in ("", ".", "..") for part in path.split("/")
|
|
53
|
+
):
|
|
54
|
+
raise ValueError("NGFF dataset path must be relative to the image group")
|
|
55
|
+
try:
|
|
56
|
+
image = root[path]
|
|
57
|
+
except KeyError:
|
|
58
|
+
raise ValueError(f"NGFF image dataset is missing: {path}") from None
|
|
59
|
+
else:
|
|
60
|
+
if level != 0 or "0" not in root:
|
|
61
|
+
raise ValueError("Image group needs NGFF metadata or a legacy array named 0")
|
|
62
|
+
image = root["0"]
|
|
63
|
+
if not isinstance(image, zarr.Array) or image.ndim != len(axes) or any(
|
|
64
|
+
size == 0 for size in image.shape
|
|
65
|
+
):
|
|
66
|
+
raise ValueError(f"Arrays must have nonempty {tuple(axis.upper() for axis in axes)} axes")
|
|
67
|
+
dimension_names = getattr(image.metadata, "dimension_names", None)
|
|
68
|
+
if (require_dimension_names or dimension_names is not None) and dimension_names != tuple(axes):
|
|
69
|
+
raise ValueError("Array dimension_names must match the declared axes")
|
|
70
|
+
spatial_transform(metadata, level, axes=axes)
|
|
71
|
+
return image, metadata
|
|
File without changes
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
from copy import deepcopy
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
import zarr
|
|
6
|
+
from sklearn.tree import DecisionTreeClassifier
|
|
7
|
+
|
|
8
|
+
from cellcanvas_core.data.data_manager import DataManager
|
|
9
|
+
from cellcanvas_core.data.data_set import DataSet
|
|
10
|
+
from cellcanvas_core.semantic.segmentation_manager import SemanticSegmentationManager
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def make_ngff(path, axes, zarr_format, image=False):
|
|
14
|
+
group = zarr.open_group(str(path), mode="w", zarr_format=zarr_format)
|
|
15
|
+
shapes = [(1, 3, 4), (1, 2, 2)]
|
|
16
|
+
for level, shape in enumerate(shapes):
|
|
17
|
+
if image:
|
|
18
|
+
shape = (2,) + shape
|
|
19
|
+
options = {"dimension_names": tuple(axes)} if zarr_format == 3 else {}
|
|
20
|
+
group.create_array(f"resolution/{level}", data=np.zeros(shape, dtype="i4"), **options)
|
|
21
|
+
prefix = [1] if image else []
|
|
22
|
+
zero = [0] if image else []
|
|
23
|
+
multiscale = {
|
|
24
|
+
"axes": [{"name": name, "type": "channel" if name == "c" else "space",
|
|
25
|
+
**({} if name == "c" else {"unit": "micrometer"})} for name in axes],
|
|
26
|
+
"datasets": [{"path": f"resolution/{level}", "coordinateTransformations": [
|
|
27
|
+
{"type": "scale", "scale": prefix + [2, 3 * (level + 1), 5 * (level + 1)]},
|
|
28
|
+
{"type": "translation", "translation": zero + [7, 11, 13]},
|
|
29
|
+
]} for level in range(2)],
|
|
30
|
+
"coordinateTransformations": [
|
|
31
|
+
{"type": "scale", "scale": prefix + [10, 20, 30]},
|
|
32
|
+
{"type": "translation", "translation": zero + [100, 200, 300]},
|
|
33
|
+
],
|
|
34
|
+
}
|
|
35
|
+
if zarr_format == 3:
|
|
36
|
+
group.attrs["ome"] = {"version": "0.5", "multiscales": [multiscale]}
|
|
37
|
+
else:
|
|
38
|
+
multiscale["version"] = "0.4"
|
|
39
|
+
group.attrs["multiscales"] = [multiscale]
|
|
40
|
+
return group
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def setup_project(tmp_path, zarr_format=3, level=0):
|
|
44
|
+
image = make_ngff(tmp_path / "image", "czyx", zarr_format, image=True)
|
|
45
|
+
labels = make_ngff(tmp_path / "labels", "zyx", zarr_format)
|
|
46
|
+
output = make_ngff(tmp_path / "output", "zyx", zarr_format)
|
|
47
|
+
shape = labels[f"resolution/{level}"].shape
|
|
48
|
+
indices = np.arange(np.prod(shape)).reshape(shape)
|
|
49
|
+
zarr.save_array(str(tmp_path / "features"), np.stack([indices, indices % 2]))
|
|
50
|
+
return image, labels, output, indices
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def load_project(tmp_path, backend, level=0):
|
|
54
|
+
paths = [str(tmp_path / name) for name in ("image", "features", "labels", "output")]
|
|
55
|
+
if backend == "paths":
|
|
56
|
+
return DataSet.from_paths(*paths, image_level=level)
|
|
57
|
+
return DataSet.from_stores(*(zarr.storage.LocalStore(path) for path in paths), image_level=level)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@pytest.mark.parametrize("backend", ["paths", "stores"])
|
|
61
|
+
@pytest.mark.parametrize("zarr_format", [2, 3])
|
|
62
|
+
@pytest.mark.parametrize("level", [0, 1])
|
|
63
|
+
def test_aligned_ngff_annotations_survive_prediction(tmp_path, backend, zarr_format, level):
|
|
64
|
+
image, labels, output, indices = setup_project(tmp_path, zarr_format, level)
|
|
65
|
+
before = [deepcopy(group.attrs.asdict()) for group in (image, labels, output)]
|
|
66
|
+
dataset = load_project(tmp_path, backend, level)
|
|
67
|
+
dataset.labels[:] = indices % 2 + 1
|
|
68
|
+
manager = SemanticSegmentationManager(DataManager([dataset]), DecisionTreeClassifier(random_state=0))
|
|
69
|
+
manager.fit()
|
|
70
|
+
manager.predict_to_segmentation(dataset, batch_size=3)
|
|
71
|
+
reopened = load_project(tmp_path, backend, level)
|
|
72
|
+
np.testing.assert_array_equal(reopened.segmentation[:], indices % 2 + 1)
|
|
73
|
+
assert reopened.spatial_transform == dataset.spatial_transform
|
|
74
|
+
for group, metadata in zip((image, labels, output), before):
|
|
75
|
+
assert group.attrs.asdict() == metadata
|
|
76
|
+
assert group.metadata.zarr_format == zarr_format
|
|
77
|
+
# No label arrays are invented alongside the declared NGFF datasets.
|
|
78
|
+
assert "labels" not in labels and "labels" not in output
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@pytest.mark.parametrize("backend", ["paths", "stores"])
|
|
82
|
+
@pytest.mark.parametrize("target", ["labels", "output"])
|
|
83
|
+
@pytest.mark.parametrize("mismatch", ["scale", "origin", "units"])
|
|
84
|
+
def test_mismatched_annotations_are_rejected_without_writes(tmp_path, backend, target, mismatch):
|
|
85
|
+
image, labels, output, _ = setup_project(tmp_path)
|
|
86
|
+
group = labels if target == "labels" else output
|
|
87
|
+
metadata = group.attrs["ome"]
|
|
88
|
+
multiscale = metadata["multiscales"][0]
|
|
89
|
+
if mismatch == "units":
|
|
90
|
+
multiscale["axes"][1]["unit"] = "nanometer"
|
|
91
|
+
elif mismatch == "scale":
|
|
92
|
+
multiscale["datasets"][0]["coordinateTransformations"][0]["scale"][0] *= 2
|
|
93
|
+
else:
|
|
94
|
+
multiscale["datasets"][0]["coordinateTransformations"][1]["translation"][0] += 1
|
|
95
|
+
group.attrs["ome"] = metadata
|
|
96
|
+
before = [deepcopy(node.attrs.asdict()) for node in (image, labels, output)]
|
|
97
|
+
with pytest.raises(ValueError, match=f"Annotation spatial {mismatch}"):
|
|
98
|
+
load_project(tmp_path, backend)
|
|
99
|
+
for node, metadata in zip((image, labels, output), before):
|
|
100
|
+
assert node.attrs.asdict() == metadata
|
|
101
|
+
assert "labels" not in node
|
|
102
|
+
np.testing.assert_array_equal(node["resolution/0"][:], 0)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@pytest.mark.parametrize("backend", ["paths", "stores"])
|
|
106
|
+
def test_bare_array_axis_names_are_checked(tmp_path, backend):
|
|
107
|
+
for name, values, axes in [
|
|
108
|
+
("image", np.zeros((2, 1, 3, 4)), "czyx"),
|
|
109
|
+
("features", np.zeros((5, 1, 3, 4)), None),
|
|
110
|
+
("labels", np.zeros((1, 3, 4), dtype="i4"), "zxy"),
|
|
111
|
+
("output", np.zeros((1, 3, 4), dtype="i4"), "zyx"),
|
|
112
|
+
]:
|
|
113
|
+
zarr.create_array(str(tmp_path / name), data=values,
|
|
114
|
+
dimension_names=tuple(axes) if axes else None)
|
|
115
|
+
with pytest.raises(ValueError, match="Annotation dimension_names"):
|
|
116
|
+
load_project(tmp_path, backend)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from unittest.mock import create_autospec
|
|
3
|
+
|
|
4
|
+
import dask.array as da
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
from cellcanvas_core.data.data_manager import DataManager
|
|
8
|
+
from cellcanvas_core.data.data_set import DataSet
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TestDataManager(unittest.TestCase):
|
|
12
|
+
def setUp(self):
|
|
13
|
+
# Create a mock DataSet
|
|
14
|
+
self.dataset_mock = create_autospec(DataSet)
|
|
15
|
+
|
|
16
|
+
# Create fake data for the dataset
|
|
17
|
+
self.concatenated_features = np.random.rand(3, 10, 10, 2) # (c, h, w, d)
|
|
18
|
+
self.labels = np.random.randint(0, 3, (10, 10, 2)) # (h, w, d), with some zeros
|
|
19
|
+
|
|
20
|
+
# Assign the fake data to the mock
|
|
21
|
+
self.dataset_mock.concatenated_features = da.from_array(
|
|
22
|
+
self.concatenated_features
|
|
23
|
+
)
|
|
24
|
+
self.dataset_mock.labels = da.from_array(self.labels)
|
|
25
|
+
|
|
26
|
+
def test_initialization_no_datasets(self):
|
|
27
|
+
manager = DataManager()
|
|
28
|
+
# TODO this list used to be a napari SelectableEventedList
|
|
29
|
+
self.assertIsInstance(manager.datasets, list)
|
|
30
|
+
self.assertEqual(len(manager.datasets), 0)
|
|
31
|
+
|
|
32
|
+
def test_initialization_with_datasets(self):
|
|
33
|
+
manager = DataManager([self.dataset_mock])
|
|
34
|
+
self.assertEqual(len(manager.datasets), 1)
|
|
35
|
+
|
|
36
|
+
def test_initialization_with_single_dataset(self):
|
|
37
|
+
# Wrap mock in a list to simulate single dataset initialization
|
|
38
|
+
manager = DataManager([self.dataset_mock])
|
|
39
|
+
self.assertEqual(len(manager.datasets), 1)
|
|
40
|
+
|
|
41
|
+
def test_get_training_data(self):
|
|
42
|
+
manager = DataManager([self.dataset_mock])
|
|
43
|
+
features, labels = manager.get_training_data()
|
|
44
|
+
|
|
45
|
+
# Compute the results
|
|
46
|
+
features = features.compute()
|
|
47
|
+
labels = labels.compute()
|
|
48
|
+
|
|
49
|
+
# Check shapes and content
|
|
50
|
+
valid_indices = np.where(self.labels.flatten() > 0)[0]
|
|
51
|
+
reshaped_features = self.concatenated_features.reshape(3, -1)
|
|
52
|
+
expected_features = np.stack(
|
|
53
|
+
[reshaped_features[i, valid_indices] for i in range(3)], axis=1
|
|
54
|
+
)
|
|
55
|
+
expected_labels = self.labels.flatten()[valid_indices] - 1
|
|
56
|
+
|
|
57
|
+
np.testing.assert_array_almost_equal(features, expected_features)
|
|
58
|
+
np.testing.assert_array_equal(labels, expected_labels)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
if __name__ == "__main__":
|
|
62
|
+
unittest.main()
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Lazy features keep the storage chunks of Zarr arrays."""
|
|
2
|
+
|
|
3
|
+
import dask
|
|
4
|
+
import numpy as np
|
|
5
|
+
import zarr
|
|
6
|
+
|
|
7
|
+
from cellcanvas_core.data.data_set import DataSet
|
|
8
|
+
|
|
9
|
+
SHAPE = (4, 12, 12)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_zarr_features_keep_their_storage_chunks(tmp_path):
|
|
13
|
+
"""Storage chunks above dask's chunk-size limit are not split across blocks."""
|
|
14
|
+
features = zarr.create_array(
|
|
15
|
+
str(tmp_path / "features.zarr"),
|
|
16
|
+
shape=(3, *SHAPE),
|
|
17
|
+
chunks=(3, 2, 12, 12),
|
|
18
|
+
dtype="f4",
|
|
19
|
+
)
|
|
20
|
+
features[:] = np.random.default_rng(0).normal(size=(3, *SHAPE))
|
|
21
|
+
dataset = DataSet(
|
|
22
|
+
np.zeros((1, *SHAPE)),
|
|
23
|
+
{"features": features},
|
|
24
|
+
np.zeros(SHAPE, dtype="i4"),
|
|
25
|
+
np.zeros(SHAPE, dtype="i4"),
|
|
26
|
+
)
|
|
27
|
+
with dask.config.set({"array.chunk-size": "1KiB"}): # smaller than a chunk
|
|
28
|
+
lazy = dataset.concatenated_features
|
|
29
|
+
assert lazy.chunksize == (3, 2, 12, 12)
|
|
30
|
+
np.testing.assert_array_equal(np.asarray(lazy), features[:])
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_other_arrays_are_unchanged():
|
|
34
|
+
features = np.random.default_rng(0).normal(size=(3, *SHAPE)).astype("f4")
|
|
35
|
+
dataset = DataSet(
|
|
36
|
+
np.zeros((1, *SHAPE)),
|
|
37
|
+
{"features": features},
|
|
38
|
+
np.zeros(SHAPE, dtype="i4"),
|
|
39
|
+
np.zeros(SHAPE, dtype="i4"),
|
|
40
|
+
)
|
|
41
|
+
np.testing.assert_array_equal(np.asarray(dataset.concatenated_features), features)
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import dask.array as da
|
|
2
|
+
import numpy as np
|
|
3
|
+
import pytest
|
|
4
|
+
import zarr
|
|
5
|
+
from sklearn.tree import DecisionTreeClassifier
|
|
6
|
+
|
|
7
|
+
from cellcanvas_core.data.data_manager import DataManager
|
|
8
|
+
from cellcanvas_core.data.data_set import DataSet
|
|
9
|
+
from cellcanvas_core.semantic.segmentation_manager import SemanticSegmentationManager
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def make_dataset(axis=0):
|
|
13
|
+
image = np.arange(24).reshape(2, 3, 4)
|
|
14
|
+
features = {"a": np.stack([image, image + 100]),
|
|
15
|
+
"b": np.stack([image + 200, image + 300, image + 400])}
|
|
16
|
+
if axis == -1:
|
|
17
|
+
features = {key: np.moveaxis(value, 0, -1) for key, value in features.items()}
|
|
18
|
+
labels = (image % 2 + 1).astype("i4")
|
|
19
|
+
labels.flat[0] = 0
|
|
20
|
+
return DataSet(np.stack([image, image + 1000]), features,
|
|
21
|
+
labels, np.zeros_like(labels), feature_axis=axis)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@pytest.mark.parametrize("axis", [0, -1])
|
|
25
|
+
def test_feature_rows_keep_their_voxel_and_channel_order(axis):
|
|
26
|
+
dataset = make_dataset(axis)
|
|
27
|
+
assert isinstance(dataset.concatenated_features, da.Array)
|
|
28
|
+
features, labels = DataManager([dataset]).get_training_data()
|
|
29
|
+
expected = np.arange(1, 24)[:, None] + np.arange(0, 500, 100)[None, :]
|
|
30
|
+
np.testing.assert_array_equal(features.compute(), expected)
|
|
31
|
+
np.testing.assert_array_equal(labels.compute(), np.arange(1, 24) % 2)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_reject_mismatched_shapes():
|
|
35
|
+
dataset = make_dataset()
|
|
36
|
+
with pytest.raises(ValueError, match="does not match"):
|
|
37
|
+
DataSet(dataset.image, {"bad": np.zeros((2, 3, 4, 5))},
|
|
38
|
+
dataset.labels, dataset.segmentation)
|
|
39
|
+
with pytest.raises(ValueError, match="must match"):
|
|
40
|
+
DataSet(dataset.image, dataset.features, np.zeros((2, 3)), dataset.segmentation)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_no_painted_labels_has_a_useful_error():
|
|
44
|
+
dataset = make_dataset()
|
|
45
|
+
dataset.labels[:] = 0
|
|
46
|
+
with pytest.raises(ValueError, match="Paint"):
|
|
47
|
+
DataManager([dataset]).get_training_data()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_image_axes_and_single_slice_2d():
|
|
51
|
+
image = np.zeros((2, 1, 3, 4))
|
|
52
|
+
labels = np.zeros((1, 3, 4), dtype="i4")
|
|
53
|
+
dataset = DataSet(image, {"features": np.ones((5, 1, 3, 4))}, labels, labels.copy())
|
|
54
|
+
assert dataset.image.shape == (2, 1, 3, 4)
|
|
55
|
+
assert dataset.labels.shape == (1, 3, 4)
|
|
56
|
+
assert dataset.image[np.newaxis].shape == (1, 2, 1, 3, 4)
|
|
57
|
+
with pytest.raises(ValueError, match="C, Z, Y, X"):
|
|
58
|
+
DataSet(image[0], dataset.features, labels, labels.copy())
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_train_predict_save_reopen_with_bounded_batches(tmp_path):
|
|
62
|
+
dataset = make_dataset()
|
|
63
|
+
rows_seen = []
|
|
64
|
+
|
|
65
|
+
class RecordingTree(DecisionTreeClassifier):
|
|
66
|
+
def predict(self, features):
|
|
67
|
+
rows_seen.append(len(features))
|
|
68
|
+
assert isinstance(features, np.ndarray)
|
|
69
|
+
return super().predict(features)
|
|
70
|
+
|
|
71
|
+
manager = SemanticSegmentationManager(DataManager([dataset]), RecordingTree(random_state=0))
|
|
72
|
+
manager.fit()
|
|
73
|
+
path = str(tmp_path / "features")
|
|
74
|
+
zarr.save_array(path, dataset.concatenated_features.compute())
|
|
75
|
+
predictions = manager.predict(zarr.open(path, mode="r"), batch_size=5)
|
|
76
|
+
assert max(rows_seen) <= 5
|
|
77
|
+
assert sum(rows_seen) == 24
|
|
78
|
+
np.testing.assert_array_equal(predictions.ravel()[1:], dataset.labels.ravel()[1:] - 1)
|
|
79
|
+
stored = predictions + 1
|
|
80
|
+
output = str(tmp_path / "predictions")
|
|
81
|
+
zarr.save_array(output, stored)
|
|
82
|
+
np.testing.assert_array_equal(zarr.open(output, mode="r")[:], stored)
|
|
83
|
+
assert set(np.unique(stored)) == {1, 2}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
from copy import deepcopy
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import pytest
|
|
5
|
+
import zarr
|
|
6
|
+
from sklearn.tree import DecisionTreeClassifier
|
|
7
|
+
|
|
8
|
+
from cellcanvas_core.data._image import open_image
|
|
9
|
+
from cellcanvas_core.data.data_manager import DataManager
|
|
10
|
+
from cellcanvas_core.data.data_set import DataSet
|
|
11
|
+
from cellcanvas_core.semantic.segmentation_manager import SemanticSegmentationManager
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def make_image(path, zarr_format=3):
|
|
15
|
+
group = zarr.open_group(str(path), mode="w", zarr_format=zarr_format)
|
|
16
|
+
image = np.arange(48).reshape(2, 1, 4, 6)
|
|
17
|
+
options = {"dimension_names": ("c", "z", "y", "x")} if zarr_format == 3 else {}
|
|
18
|
+
group.create_array("fine/image", data=image, **options)
|
|
19
|
+
group.create_array("coarse/image", data=image[:, :, ::2, ::2], **options)
|
|
20
|
+
multiscales = [{
|
|
21
|
+
"axes": [{"name": "c", "type": "channel"}] + [
|
|
22
|
+
{"name": axis, "type": "space", "unit": "micrometer"}
|
|
23
|
+
for axis in ("z", "y", "x")
|
|
24
|
+
],
|
|
25
|
+
"datasets": [
|
|
26
|
+
{"path": name, "coordinateTransformations": [
|
|
27
|
+
{"type": "scale", "scale": scale},
|
|
28
|
+
{"type": "translation", "translation": [0, 7, 11, 13]},
|
|
29
|
+
]}
|
|
30
|
+
for name, scale in [("fine/image", [1, 3, 0.5, 0.5]),
|
|
31
|
+
("coarse/image", [1, 3, 1, 1])]
|
|
32
|
+
],
|
|
33
|
+
}]
|
|
34
|
+
if zarr_format == 3:
|
|
35
|
+
group.attrs["ome"] = {"version": "0.5", "multiscales": multiscales}
|
|
36
|
+
else:
|
|
37
|
+
multiscales[0]["version"] = "0.4"
|
|
38
|
+
group.attrs["multiscales"] = multiscales
|
|
39
|
+
return group
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@pytest.mark.parametrize("backend", ["paths", "stores"])
|
|
43
|
+
@pytest.mark.parametrize("level", [0, 1])
|
|
44
|
+
@pytest.mark.parametrize("zarr_format", [2, 3])
|
|
45
|
+
def test_ngff_train_predict_save_reopen(tmp_path, backend, level, zarr_format):
|
|
46
|
+
group = make_image(tmp_path / "image", zarr_format=zarr_format)
|
|
47
|
+
original_metadata = deepcopy(group.attrs.asdict())
|
|
48
|
+
image_path = ["fine/image", "coarse/image"][level]
|
|
49
|
+
expected = group[image_path][:]
|
|
50
|
+
shape = expected.shape[1:]
|
|
51
|
+
indices = np.arange(np.prod(shape)).reshape(shape)
|
|
52
|
+
features = np.stack([indices, indices % 2])
|
|
53
|
+
feature_path = str(tmp_path / "features")
|
|
54
|
+
zarr.save_array(feature_path, features, zarr_format=zarr_format)
|
|
55
|
+
labels_path = str(tmp_path / "labels")
|
|
56
|
+
predictions_path = str(tmp_path / "predictions")
|
|
57
|
+
|
|
58
|
+
def load():
|
|
59
|
+
if backend == "paths":
|
|
60
|
+
return DataSet.from_paths(
|
|
61
|
+
str(tmp_path / "image"), feature_path, labels_path,
|
|
62
|
+
predictions_path, make_missing_datasets=True, image_level=level,
|
|
63
|
+
)
|
|
64
|
+
return DataSet.from_stores(
|
|
65
|
+
group.store, zarr.storage.LocalStore(feature_path),
|
|
66
|
+
zarr.storage.LocalStore(labels_path), zarr.storage.LocalStore(predictions_path),
|
|
67
|
+
image_level=level,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
dataset = load()
|
|
71
|
+
np.testing.assert_array_equal(dataset.image[:], expected)
|
|
72
|
+
assert dataset.labels.shape == shape
|
|
73
|
+
assert dataset.image_metadata == original_metadata
|
|
74
|
+
assert dataset.labels.metadata.zarr_format == 3
|
|
75
|
+
assert dataset.segmentation.metadata.zarr_format == 3
|
|
76
|
+
dataset.labels[:] = indices % 2 + 1
|
|
77
|
+
manager = SemanticSegmentationManager(
|
|
78
|
+
DataManager([dataset]), DecisionTreeClassifier(random_state=0)
|
|
79
|
+
)
|
|
80
|
+
manager.fit()
|
|
81
|
+
dataset.segmentation[:] = manager.predict(dataset.concatenated_features, batch_size=5) + 1
|
|
82
|
+
reopened = load()
|
|
83
|
+
np.testing.assert_array_equal(reopened.segmentation[:], indices % 2 + 1)
|
|
84
|
+
np.testing.assert_array_equal(reopened.labels[:], indices % 2 + 1)
|
|
85
|
+
assert reopened.image_metadata == original_metadata
|
|
86
|
+
assert group.attrs.asdict() == original_metadata
|
|
87
|
+
np.testing.assert_array_equal(group[image_path][:], expected)
|
|
88
|
+
metadata = dataset.image_metadata.get("ome", dataset.image_metadata)
|
|
89
|
+
metadata["multiscales"][0]["axes"][1]["unit"] = "nanometer"
|
|
90
|
+
assert group.attrs.asdict() == original_metadata
|
|
91
|
+
assert group.metadata.zarr_format == zarr_format
|
|
92
|
+
assert group[image_path].metadata.zarr_format == zarr_format
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@pytest.mark.parametrize("axes", [
|
|
96
|
+
["z", "y", "x", "c"], ["t", "c", "z", "y", "x"],
|
|
97
|
+
])
|
|
98
|
+
def test_wrong_axes_fail_before_creating_annotations(tmp_path, axes):
|
|
99
|
+
group = make_image(tmp_path / "image")
|
|
100
|
+
metadata = group.attrs["ome"]
|
|
101
|
+
metadata["multiscales"][0]["axes"] = [{"name": name} for name in axes]
|
|
102
|
+
group.attrs["ome"] = metadata
|
|
103
|
+
with pytest.raises(ValueError, match="axes must be ordered"):
|
|
104
|
+
DataSet.from_paths(str(tmp_path / "image"), "unused", str(tmp_path / "labels"),
|
|
105
|
+
str(tmp_path / "predictions"), make_missing_datasets=True)
|
|
106
|
+
assert not (tmp_path / "labels").exists()
|
|
107
|
+
assert not (tmp_path / "predictions").exists()
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@pytest.mark.parametrize("level", [-1, 2, True])
|
|
111
|
+
def test_invalid_level(tmp_path, level):
|
|
112
|
+
group = make_image(tmp_path / "image")
|
|
113
|
+
with pytest.raises(ValueError, match="image_level"):
|
|
114
|
+
open_image(group.store, level=level)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_missing_ngff_array_does_not_fall_back_to_zero(tmp_path):
|
|
118
|
+
group = make_image(tmp_path / "image")
|
|
119
|
+
group.create_array("0", data=np.zeros((2, 1, 4, 6)))
|
|
120
|
+
del group["fine/image"]
|
|
121
|
+
with pytest.raises(ValueError, match="dataset is missing"):
|
|
122
|
+
open_image(group.store)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@pytest.mark.parametrize("dimension_names", [None, ("c", "z", "x", "y")])
|
|
126
|
+
def test_ngff_dimension_names_must_match_axes(tmp_path, dimension_names):
|
|
127
|
+
group = make_image(tmp_path / "image")
|
|
128
|
+
values = group["fine/image"][:]
|
|
129
|
+
del group["fine/image"]
|
|
130
|
+
group.create_array("fine/image", data=values, dimension_names=dimension_names)
|
|
131
|
+
with pytest.raises(ValueError, match="dimension_names"):
|
|
132
|
+
open_image(group.store)
|