webknossos 4.2.2__tar.gz → 4.2.4__tar.gz

{webknossos-4.2.2 → webknossos-4.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webknossos
-Version: 4.2.2
+Version: 4.2.4
 Summary: Python API for working with WEBKNOSSOS datasets, annotations, and for WEBKNOSSOS server interaction.
 Author-email: scalable minds <hello@scalableminds.com>
 License: AGPL-3.0
@@ -37,7 +37,7 @@ Requires-Dist: boltons>=21.0.0
 Requires-Dist: cattrs>=22.0.0
 Requires-Dist: certifi>=2023
 Requires-Dist: click<8.2.0
-Requires-Dist: cluster-tools==4.2.2
+Requires-Dist: cluster-tools==4.2.4
 Requires-Dist: httpx~=0.27.0
 Requires-Dist: imageio>=2.37.0
 Requires-Dist: lz4>=4.0.0

{webknossos-4.2.2 → webknossos-4.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "webknossos"
-version = "4.2.2"                                                                                                    # filled by dunamai
+version = "4.2.4"                                                                                                    # filled by dunamai
 description = "Python API for working with WEBKNOSSOS datasets, annotations, and for WEBKNOSSOS server interaction."
 authors = [{ name = "scalable minds", email = "hello@scalableminds.com" }]
 readme = "README.md"
@@ -39,7 +39,7 @@ dependencies = [
     "cattrs >=22.0.0",
     "certifi >=2023",
     "click <8.2.0", # pinned because of incompatibility with typer
-    "cluster-tools==4.2.2",
+    "cluster-tools==4.2.4",
     "httpx ~=0.27.0",
     "imageio >=2.37.0",
     "lz4 >=4.0.0",

{webknossos-4.2.2 → webknossos-4.2.4}/tests/test_cli.py

@@ -25,7 +25,7 @@ from tests.constants import (
     TESTDATA_DIR,
 )
 from tests.data_fixtures import download_wklibs_sample_archive
-from webknossos import BoundingBox, DataFormat, Dataset, Mag
+from webknossos import BoundingBox, DataFormat, Dataset, Mag, NDBoundingBox
 from webknossos.cli.export_as_tiff import _apply_mapping, _make_tiff_name
 from webknossos.cli.main import app
 from webknossos.dataset._utils.tensorstore_helpers import (
@@ -629,6 +629,46 @@ def test_export_tiff_stack(tmp_upath: UPath) -> None:
         )
 
 
+def test_export_tiff_stack_nd_layer(tmp_upath: UPath) -> None:
+    """Tests export of a layer whose array is stored as (c, t, x, y, z)."""
+
+    source_path = tmp_upath / "nd_dataset"
+    destination_path = tmp_upath / "nd_dataset_tiff"
+    nd_bbox = NDBoundingBox(
+        (0, 0, 0, 0, 0),
+        (1, 1, 16, 16, 8),
+        axes=("c", "t", "x", "y", "z"),
+        index=(0, 1, 2, 3, 4),
+    )
+    data = np.random.default_rng(0).integers(0, 255, (1, 1, 16, 16, 8), dtype="uint8")
+    layer = Dataset(source_path, voxel_size=(11, 11, 25)).add_layer(
+        "color", "color", dtype="uint8", data_format="zarr3", bounding_box=nd_bbox
+    )
+    layer.add_mag(1).write(data=data, absolute_bounding_box=nd_bbox)
+    assert layer.get_mag(1).info.bounding_box.axes == ("c", "t", "x", "y", "z")
+
+    result = runner.invoke(
+        app,
+        [
+            "export-as-tiff",
+            "--layer-name",
+            "color",
+            "--bbox",
+            "2,4,1,8,10,5",
+            str(source_path),
+            str(destination_path),
+        ],
+    )
+
+    assert result.exit_code == 0, result.stdout
+    for slice_index in range(5):
+        with (destination_path / _make_tiff_name("", slice_index)).open("rb") as f:
+            test_image = np.array(Image.open(f)).T
+        np.testing.assert_array_equal(
+            test_image, data[0, 0, 2:10, 4:14, 1 + slice_index]
+        )
+
+
 def test_export_tiff_stack_tile_size(tmp_upath: UPath) -> None:
     """Tests the tile size support of exporting a tiff stack."""
 

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos/annotation/annotation_info.py

@@ -16,6 +16,7 @@ class AnnotationInfo:
     state: AnnotationState
     duration_in_seconds: float | None
     modified: int | None
+    created: int | None = None
 
     def download_annotation(self) -> Annotation:
         """Downloads and returns the annotation that is described by this AnnotationInfo object"""
@@ -35,6 +36,7 @@ class AnnotationInfo:
             if api_annotation.tracing_time is not None
             else None,
             modified=api_annotation.modified,
+            created=api_annotation.created,
         )
 
     @classmethod

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos/cli/export_as_tiff.py

@@ -109,7 +109,7 @@ def export_tiff_slice_batch(
         mapping_array = open_zarr3_array(mapping_path, context=ts_context)
 
     if tiling_size is None:
-        tiff_data = view.read()
+        tiff_data = view.read_cxyz()
         if mapping_array is not None:
             tiff_data = _apply_mapping(tiff_data, mapping_array)
     else:
@@ -118,7 +118,7 @@ def export_tiff_slice_batch(
             tiling_size[1] * ceil(tiff_bbox.size.y / tiling_size[1]),
             tiff_bbox.size.z,
         )
-        tiff_data = view.read()
+        tiff_data = view.read_cxyz()
         if mapping_array is not None:
             tiff_data = _apply_mapping(tiff_data, mapping_array)
         padded_tiff_data = np.zeros(

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos/client/api_client/models.py

@@ -451,10 +451,11 @@ class ApiAnnotation:
     name: str
     description: str
     state: str
-    modified: int
     annotation_layers: list[ApiAnnotationLayer] | None
+    modified: int
     data_store: ApiDataStore | None = None
     tracing_time: int | None = None  # millis
+    created: int | None = None
 
 
 @attr.s(auto_attribs=True)

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos/dataset/layer/export.py

@@ -64,7 +64,7 @@ def _resolve_export_bbox(
     The layer's bounding box can't be used for that: for remote layers it may
     lack the channel axis that the streamed array has (or vice versa). The
     request is clipped to the layer per axis name, axes the array doesn't
-    have are ignored and the channel axis always spans all channels.
+    have are ignored and the channel axis always spans all channels of the layer.
     """
     layer_bbox = layer.normalized_bounding_box
     if bounding_box is None:
@@ -73,7 +73,14 @@ def _resolve_export_bbox(
     topleft, size = [], []
     for axis in array_axes:
         if axis == C_AXIS:
+            mag_bbox = mag_view.normalized_bounding_box
+            array_start, array_end = mag_view.info.bounding_box.get_bounds(C_AXIS)
             start, end = 0, layer.num_channels
+            if C_AXIS in mag_bbox.axes:
+                mag_start, mag_end = mag_bbox.get_bounds(C_AXIS)
+                # Streamed arrays might only contain the layer's channels.
+                if array_start <= mag_start and mag_end <= array_end:
+                    start, end = mag_start, mag_end
         elif axis in layer_bbox.axes:
             start, end = layer_bbox.get_bounds(axis)
             if axis in bounding_box.axes:

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos/dataset/layer/layer.py

@@ -868,6 +868,12 @@ class Layer(AbstractLayer):
                 mag=mag,
                 path=dump_path(foreign_mag_view.path, self.dataset.resolved_path),
                 cube_length=foreign_mag_view._properties.cube_length,
+                # Not set for mags whose arrays are re-encoded by zarr streaming.
+                axis_order=(
+                    foreign_mag_view._properties.axis_order
+                    if foreign_mag_view._get_dimension_names() is not None
+                    else None
+                ),
             )
         )
         self._save_layer_properties()

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos/dataset/layer/view/_array.py

@@ -436,12 +436,35 @@ class WKWArray(BaseArray):
 class TensorStoreArray(BaseArray):
     _cached_array: tensorstore.TensorStore | None
     _tensorstore_data_format: str
+    _dimension_names: tuple[str, ...] | None
 
     def __init__(
-        self, path: UPath, _cached_array: tensorstore.TensorStore | None = None
+        self,
+        path: UPath,
+        _cached_array: tensorstore.TensorStore | None = None,
+        dimension_names: tuple[str, ...] | None = None,
     ):
+        """`dimension_names` (e.g. from the `axisOrder` of the layer properties)
+        take precedence over the dimension names in the array's metadata, if their
+        number matches the array's rank."""
         super().__init__(path)
-        self._cached_array = _cached_array
+        self._dimension_names = dimension_names
+        self._cached_array = (
+            None
+            if _cached_array is None
+            else self._apply_dimension_names(_cached_array)
+        )
+
+    def _apply_dimension_names(
+        self, array: tensorstore.TensorStore
+    ) -> tensorstore.TensorStore:
+        if (
+            self._dimension_names is not None
+            and len(self._dimension_names) == array.rank
+            and array.domain.labels != self._dimension_names
+        ):
+            return array[tensorstore.d[:].label[self._dimension_names]]
+        return array
 
     @staticmethod
     def _get_array_dimensions(
@@ -560,7 +583,7 @@ class TensorStoreArray(BaseArray):
         }
 
     @classmethod
-    def _open(cls, path: UPath) -> Self:
+    def _open(cls, path: UPath, dimension_names: tuple[str, ...] | None = None) -> Self:
         try:
             _array = call_with_retries(
                 lambda: tensorstore.open(
@@ -572,7 +595,7 @@ class TensorStoreArray(BaseArray):
                 ).result(),
                 description="Opening tensorstore array",
             )  # check that everything exists
-            return cls(path, _array)
+            return cls(path, _array, dimension_names=dimension_names)
         except Exception as exc:
             raise ArrayException(f"Could not open array at {path}.") from exc
 
@@ -620,16 +643,18 @@ class TensorStoreArray(BaseArray):
 
         if new_domain != array.domain:
             # Check on-disk for changes to shape
-            current_array = call_with_retries(
-                lambda: tensorstore.open(
-                    {
-                        "driver": self._tensorstore_data_format,
-                        "kvstore": _make_kvstore(self._path),
-                    },
-                    context=TS_CONTEXT,
-                    recheck_cached="open",
-                ).result(),
-                description="Opening tensorstore array for resizing",
+            current_array = self._apply_dimension_names(
+                call_with_retries(
+                    lambda: tensorstore.open(
+                        {
+                            "driver": self._tensorstore_data_format,
+                            "kvstore": _make_kvstore(self._path),
+                        },
+                        context=TS_CONTEXT,
+                        recheck_cached="open",
+                    ).result(),
+                    description="Opening tensorstore array for resizing",
+                )
             )
             if array.domain != current_array.domain:
                 raise RuntimeError(
@@ -739,6 +764,7 @@ class TensorStoreArray(BaseArray):
                     ).result(),
                     description="Creating tensorstore array",
                 )
+                self._cached_array = self._apply_dimension_names(self._cached_array)
             except Exception as e:
                 raise ArrayException(
                     f"Exception while opening array for {_make_kvstore(self._path)}"
@@ -767,8 +793,10 @@ class Zarr3Array(TensorStoreArray):
     _tensorstore_data_format = "zarr3"
 
     @classmethod
-    def open(cls, path: UPath) -> "Zarr3Array":
-        return cls._open(path)
+    def open(
+        cls, path: UPath, dimension_names: tuple[str, ...] | None = None
+    ) -> "Zarr3Array":
+        return cls._open(path, dimension_names)
 
     @property
     def info(self) -> Zarr3ArrayInfo:
@@ -887,8 +915,10 @@ class Zarr2Array(TensorStoreArray):
     _tensorstore_data_format = "zarr"
 
     @classmethod
-    def open(cls, path: UPath) -> "Zarr2Array":
-        return cls._open(path)
+    def open(
+        cls, path: UPath, dimension_names: tuple[str, ...] | None = None
+    ) -> "Zarr2Array":
+        return cls._open(path, dimension_names)
 
     @property
     def info(self) -> ArrayInfo:

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos/dataset/layer/view/mag_view.py

@@ -17,7 +17,7 @@ from webknossos.utils import (
     warn_deprecated,
 )
 
-from ....dataset_properties import DataFormat, MagViewProperties
+from ....dataset_properties import DataFormat, LayerProperties, MagViewProperties
 from ....geometry import (
     Mag,
     NDBoundingBox,
@@ -202,6 +202,7 @@ class MagView(View, Generic[LayerTypeT]):
             read_only=read_only,
         )
         self._layer = layer
+        self._cached_dimension_names: tuple[str, ...] | None = None
 
     def _ensure_writable(self) -> None:
         if self._read_only:
@@ -241,6 +242,29 @@ class MagView(View, Generic[LayerTypeT]):
         """
         return self.layer.num_channels
 
+    def _get_dimension_names(self) -> tuple[str, ...] | None:
+        # An explicit `axisOrder` takes precedence over the array's metadata.
+        # The bounding box's axes are sorted by their index in the array.
+        if self._cached_dimension_names is None:
+            mag_properties = self._find_properties()
+            if mag_properties is None or mag_properties.axis_order is None:
+                return None
+            self._cached_dimension_names = self.normalized_bounding_box.axes
+        return self._cached_dimension_names
+
+    def _get_layer_properties(self) -> LayerProperties:
+        return self.layer._properties
+
+    def _find_properties(self) -> MagViewProperties | None:
+        return next(
+            (
+                mag_property
+                for mag_property in self._get_layer_properties().mags
+                if mag_property.mag == self._mag
+            ),
+            None,
+        )
+
     # Own methods:
 
     @property
@@ -395,6 +419,7 @@ class MagView(View, Generic[LayerTypeT]):
             rel_mag1_bbox=relative_bounding_box,
             current_mag_size=data_shape,
         )
+        self._check_write_shape(data, mag1_bbox)
 
         # Only update the layer's bbox if we are actually larger
         # than the mag-aligned, rounded up bbox (self.bounding_box):
@@ -872,11 +897,10 @@ class MagView(View, Generic[LayerTypeT]):
             MagViewProperties: Properties object for this magnification level.
 
         """
-        return next(
-            mag_property
-            for mag_property in self.layer._properties.mags
-            if mag_property.mag == self._mag
-        )
+        mag_properties = self._find_properties()
+        if mag_properties is None:
+            raise KeyError(f"No properties found for mag {self._mag}.")
+        return mag_properties
 
     def __repr__(self) -> str:
         return f"MagView(name={repr(self.name)}, bounding_box={self.bounding_box})"

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos/dataset/layer/view/remote_mag_view.py

@@ -93,6 +93,16 @@ class RemoteMagView(MagView["RemoteLayer"]):
     def num_channels(self) -> int:
         return self.normalized_bounding_box.size.get("c", 1)
 
+    def _get_layer_properties(self) -> "LayerProperties":
+        return self._layer_properties
+
+    def _get_dimension_names(self) -> tuple[str, ...] | None:
+        # Zarr streaming re-encodes the arrays with their own axis order, while
+        # the other access modes serve the original arrays.
+        if self._access_mode == RemoteAccessMode.ZARR_STREAMING:
+            return None
+        return super()._get_dimension_names()
+
     @property
     def paths_by_access_mode(self) -> Mapping[RemoteAccessMode, UPath]:
         """All paths at which this mag's data can be reached, keyed by access mode.

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos/dataset/layer/view/view.py

@@ -28,7 +28,7 @@ from ....utils import (
     wait_and_ensure_success,
     warn_deprecated,
 )
-from ._array import ArrayInfo, BaseArray
+from ._array import ArrayInfo, BaseArray, Zarr2Array, Zarr3Array
 
 if TYPE_CHECKING:
     from ..._utils.buffered_slice_reader import BufferedSliceReader
@@ -81,6 +81,7 @@ class View:
         data_format: DataFormat,
         read_only: bool = False,
         cached_array: BaseArray | None = None,
+        dimension_names: tuple[str, ...] | None = None,
     ):
         """Initialize a View instance for accessing and manipulating dataset regions.
 
@@ -113,6 +114,7 @@ class View:
         self._normalized_bounding_box = bounding_box
         self._read_only = read_only
         self._cached_array = cached_array
+        self._dimension_names = dimension_names
         self._mag = mag
 
     @property
@@ -225,6 +227,22 @@ class View:
         """
         return self._read_only
 
+    def _normalize_axes_with_channel_start(
+        self, bbox: NDBoundingBox
+    ) -> NormalizedBoundingBox:
+        normalized_bbox = bbox.normalize_axes(self.num_channels)
+        if (
+            C_AXIS not in bbox.axes
+            and C_AXIS in normalized_bbox.axes
+            and C_AXIS in self.normalized_bounding_box.axes
+        ):
+            # Start at the view's first channel, which is non-zero for
+            # layers with a `channelIndex`.
+            normalized_bbox = normalized_bbox.with_bounds(
+                C_AXIS, self.normalized_bounding_box.topleft.c, self.num_channels
+            )
+        return normalized_bbox
+
     def _get_mag1_bbox(
         self,
         *,
@@ -255,10 +273,10 @@ class View:
             )
 
         if abs_mag1_bbox is not None:
-            return abs_mag1_bbox.normalize_axes(self.num_channels)
+            return self._normalize_axes_with_channel_start(abs_mag1_bbox)
 
         if rel_mag1_bbox is not None:
-            return rel_mag1_bbox.normalize_axes(self.num_channels).offset(
+            return self._normalize_axes_with_channel_start(rel_mag1_bbox).offset(
                 self.bounding_box.topleft
             )
 
@@ -335,6 +353,7 @@ class View:
                 - Number of channels doesn't match the dataset
                 - Write region is outside the view's bounding box
                 - Multiple positioning parameters are provided
+            ValueError: If the data shape doesn't match the size of the target region.
 
         Examples:
             ```python
@@ -434,6 +453,8 @@ class View:
 
         current_mag_bbox = mag1_bbox.in_mag(self._mag)
 
+        self._check_write_shape(data, mag1_bbox)
+
         if not allow_unaligned:
             if self._data_format == DataFormat.WKW and not self._is_compressed():
                 try:
@@ -456,6 +477,28 @@ class View:
         else:
             self._array.write(current_mag_bbox, data)
 
+    def _check_write_shape(
+        self, data: np.ndarray, mag1_bbox: NormalizedBoundingBox
+    ) -> None:
+        current_mag_bbox = mag1_bbox.in_mag(self._mag)
+        c_index = current_mag_bbox.index.c if C_AXIS in current_mag_bbox.axes else None
+        expected_shape = tuple(current_mag_bbox.size)
+        data_shape = tuple(data.shape)
+        if c_index is not None and len(data_shape) + 1 == len(expected_shape):
+            # data without channel axis
+            data_shape = data_shape[:c_index] + (1,) + data_shape[c_index:]
+        # the number of channels is validated separately
+        if len(data_shape) != len(expected_shape) or any(
+            actual != expected
+            for i, (actual, expected) in enumerate(zip(data_shape, expected_shape))
+            if i != c_index
+        ):
+            raise ValueError(
+                f"The shape of the passed data {data.shape} does not match the size "
+                f"of the bounding box to write {expected_shape} "
+                f"(axes {current_mag_bbox.axes}, in {self._mag})."
+            )
+
     def _resolve_cxyz_write(
         self,
         data: np.ndarray,
@@ -1094,6 +1137,7 @@ class View:
             data_format=self._data_format,
             read_only=read_only,
             cached_array=self._cached_array,
+            dimension_names=self._get_dimension_names(),
         )
 
     def get_buffered_slice_writer(
@@ -1669,10 +1713,20 @@ class View:
     def _get_file_dimensions_mag1(self) -> Vec3Int:
         return Vec3Int(self._get_file_dimensions() * self.mag.to_vec3_int())
 
+    def _get_dimension_names(self) -> tuple[str, ...] | None:
+        return self._dimension_names
+
     @property
     def _array(self) -> BaseArray:
         if self._cached_array is None:
-            self._cached_array = BaseArray.get_class(self._data_format).open(self._path)
+            array_class = BaseArray.get_class(self._data_format)
+            dimension_names = self._get_dimension_names()
+            if dimension_names is not None and issubclass(
+                array_class, (Zarr3Array, Zarr2Array)
+            ):
+                self._cached_array = array_class.open(self._path, dimension_names)
+            else:
+                self._cached_array = array_class.open(self._path)
         return self._cached_array
 
     @_array.deleter

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos/dataset_properties/dataset_properties.py

@@ -79,6 +79,11 @@ class MagViewProperties:
     Could be None for older datasource-proterties.json files.
     """
     cube_length: int | None = None
+    axis_order: dict[str, int] | None = attr.field(default=None, eq=False)
+    """
+    Index of each axis in the array, as read from the datasource-properties.json.
+    None if not set there. On save, it is derived from the layer's bounding box.
+    """
 
 
 @attr.define

webknossos-4.2.4/webknossos/version.py

@@ -0,0 +1 @@
+__version__ = '4.2.4'

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webknossos
-Version: 4.2.2
+Version: 4.2.4
 Summary: Python API for working with WEBKNOSSOS datasets, annotations, and for WEBKNOSSOS server interaction.
 Author-email: scalable minds <hello@scalableminds.com>
 License: AGPL-3.0
@@ -37,7 +37,7 @@ Requires-Dist: boltons>=21.0.0
 Requires-Dist: cattrs>=22.0.0
 Requires-Dist: certifi>=2023
 Requires-Dist: click<8.2.0
-Requires-Dist: cluster-tools==4.2.2
+Requires-Dist: cluster-tools==4.2.4
 Requires-Dist: httpx~=0.27.0
 Requires-Dist: imageio>=2.37.0
 Requires-Dist: lz4>=4.0.0

{webknossos-4.2.2 → webknossos-4.2.4}/webknossos.egg-info/requires.txt

@@ -4,7 +4,7 @@ boltons>=21.0.0
 cattrs>=22.0.0
 certifi>=2023
 click<8.2.0
-cluster-tools==4.2.2
+cluster-tools==4.2.4
 httpx~=0.27.0
 imageio>=2.37.0
 lz4>=4.0.0

webknossos-4.2.2/webknossos/version.py

@@ -1 +0,0 @@
-__version__ = '4.2.2'