careamics 0.0.19__py3-none-any.whl

careamics/dataset_ng/image_stack_loader/image_stack_loader_protocol.py

@@ -0,0 +1,70 @@
+from collections.abc import Sequence
+from typing import Any, Protocol
+
+from typing_extensions import ParamSpec
+
+from careamics.utils import BaseEnum
+
+from ..image_stack import GenericImageStack
+
+P = ParamSpec("P")
+
+
+class SupportedDataDev(str, BaseEnum):
+    ZARR = "zarr"
+
+
+class ImageStackLoader(Protocol[P, GenericImageStack]):
+    """
+    Protocol to define how `ImageStacks` should be loaded.
+
+    An `ImageStackLoader` is a callable that must take the `source` of the data as the
+    first argument, and the data `axes` as the second argument.
+
+    Additional `*args` and `**kwargs` are allowed, but they should only be used to
+    determine _how_ the data is loaded, not _what_ data is loaded. The `source`
+    argument has to wholly determine _what_ data is loaded, this is because,
+    downstream, both an input-source and a target-source have to be specified but they
+    will share `*args` and `**kwargs`.
+
+    An `ImageStackLoader` must return a sequence of the `ImageStack` class. This could
+    be a sequence of one of the existing concrete implementations, such as
+    `ZarrImageStack`, or a custom user defined `ImageStack`.
+
+    Example
+    -------
+    The following example demonstrates how an `ImageStackLoader` could be defined
+    for loading non-OME Zarr images. Returning a list of `ZarrImageStack` instances.
+
+    >>> from typing import TypedDict
+
+    >>> from zarr.storage import FsspecStore
+
+    >>> from careamics.config import DataConfig
+    >>> from careamics.dataset_ng.image_stack import ZarrImageStack
+
+    >>> # Define a zarr source
+    >>> # It encompasses multiple arguments that determine what data will be loaded
+    >>> class ZarrSource(TypedDict):
+    ...     store: FsspecStore
+    ...     data_paths: Sequence[str]
+
+    >>> def custom_image_stack_loader(
+    ...     source: ZarrSource, axes: str, *args, **kwargs
+    ... ) -> list[ZarrImageStack]:
+    ...     image_stacks = [
+    ...         ZarrImageStack(store=source["store"], data_path=data_path, axes=axes)
+    ...         for data_path in source["data_paths"]
+    ...     ]
+    ...     return image_stacks
+
+    TODO: show example use in the `CAREamicsDataset`
+
+    The example above defines a `ZarrSource` dict because to determine _which_ ZARR
+    images will be loaded both a ZARR store and the internal data paths need to be
+    specified.
+    """
+
+    def __call__(
+        self, source: Any, axes: str, *args: P.args, **kwargs: P.kwargs
+    ) -> Sequence[GenericImageStack]: ...

careamics/dataset_ng/image_stack_loader/image_stack_loaders.py

@@ -0,0 +1,273 @@
+from collections.abc import Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal
+
+import zarr
+from numpy.typing import NDArray
+from zarr.storage import StorePath
+
+from careamics.config.validators import check_czi_axes_validity
+from careamics.file_io import ReadFunc
+
+from ..image_stack import (
+    FileImageStack,
+    InMemoryImageStack,
+    ZarrImageStack,
+)
+from ..image_stack.czi_image_stack import CziImageStack
+from .zarr_utils import collect_arrays, decipher_zarr_uri, is_ome_zarr, is_valid_uri
+
+if TYPE_CHECKING:
+    from careamics.file_io.read import ReadFunc
+
+
+def load_arrays(source: Sequence[NDArray[Any]], axes: str) -> list[InMemoryImageStack]:
+    """
+    Load image stacks from a sequence of numpy arrays.
+
+    Parameters
+    ----------
+    source: sequence of numpy.ndarray
+        The source arrays of the data.
+    axes: str
+        The original axes of the data, must be a subset of "STCZYX".
+
+    Returns
+    -------
+    list[InMemoryImageStack]
+    """
+    return [InMemoryImageStack.from_array(data=array, axes=axes) for array in source]
+
+
+# TIFF case
+def load_tiffs(source: Sequence[Path], axes: str) -> list[InMemoryImageStack]:
+    """
+    Load image stacks from a sequence of TIFF files.
+
+    Parameters
+    ----------
+    source: sequence of Path
+        The source files for the data.
+    axes: str
+        The original axes of the data, must be a subset of "STCZYX".
+
+    Returns
+    -------
+    list[InMemoryImageStack]
+    """
+    return [InMemoryImageStack.from_tiff(path=path, axes=axes) for path in source]
+
+
+# TODO: better name
+# iter Tiff
+def load_iter_tiff(source: Sequence[Path], axes: str) -> list[FileImageStack]:
+    # TODO: better docs
+    """
+    Load image stacks from a sequence of TIFF files.
+
+    Parameters
+    ----------
+    source: sequence of Path
+        The source files for the data.
+    axes: str
+        The original axes of the data, must be a subset of "STCZYX".
+
+    Returns
+    -------
+    list[FileImageStack]
+    """
+    return [FileImageStack.from_tiff(path=path, axes=axes) for path in source]
+
+
+# Custom file type case (loaded into memory)
+def load_custom_file(
+    source: Sequence[Path],
+    axes: str,
+    *,
+    read_func: ReadFunc,
+    read_kwargs: dict[str, Any],
+) -> list[InMemoryImageStack]:
+    """
+    Load image stacks from a sequence of files of a custom type.
+
+    Parameters
+    ----------
+    source: sequence of Path
+        The source files for the data.
+    axes: str
+        The original axes of the data, must be a subset of "STCZYX".
+    read_func : ReadFunc
+        A function to read the custom file type, see the `ReadFunc` protocol.
+    read_kwargs : dict of {str: Any}
+        Kwargs that will be passed to the custom `read_func`.
+
+    Returns
+    -------
+    list[InMemoryImageStack]
+    """
+    # TODO: lazy loading custom files
+    return [
+        InMemoryImageStack.from_custom_file_type(
+            path=path,
+            axes=axes,
+            read_func=read_func,
+            **read_kwargs,
+        )
+        for path in source
+    ]
+
+
+def load_zarrs(
+    source: Sequence[str | Path | StorePath],
+    axes: str,
+) -> list[ZarrImageStack]:
+    """Create a list of ZarrImageStack from a sequence of zarr file paths or URIs.
+
+    File paths must point to a zarr store (ending with .zarr) and URIs must be in the
+    format "file://path/to/zarr_store.zarr/group/path/array_name".
+
+    If the zarr file is an OME-Zarr, the specified multiscale level will be used. Note
+    that OME-Zarrs are only supported when providing a path to the zarr store, not when
+    using a file URI. One can, however, provide a file URI to the specific resolution
+    array within the OME-Zarr.
+
+    Parameters
+    ----------
+    source : sequence of str or Path
+        The source zarr file paths or URIs.
+    axes : str
+        The original axes of the data, must be a subset of "STCZYX".
+
+    Returns
+    -------
+    list of ZarrImageStack
+        A list of ZarrImageStack created from the sources.
+    """
+
+    image_stacks: list[ZarrImageStack] = []
+
+    for data_source in source:
+        data_str = str(data_source)
+
+        # either a path to a zarr file or a uri "file://path/to/zarr/array_path"
+        if data_str.endswith(".zarr"):
+            zarr_group = zarr.open_group(data_str, mode="r")
+
+            # test if ome-zarr (minimum assumption of multiscales)
+            if is_ome_zarr(zarr_group):
+                # TODO placeholder for handling OME-Zarr
+                # - Need to potentially select multiscale level
+                # - Extract axes and compare with provided ones
+                raise NotImplementedError(
+                    "OME-Zarr support is not yet implemented when providing a "
+                    "path to the zarr store. Please provide a file URI to the "
+                    "specific array within the OME-Zarr."
+                )
+            else:
+                # collect all arrays
+                array_paths = collect_arrays(zarr_group)
+
+                # sort names
+                array_paths.sort()
+
+            # instantiate image stacks
+            for array_path in array_paths:
+                image_stacks.append(
+                    ZarrImageStack(group=zarr_group, data_path=array_path, axes=axes)
+                )
+
+        elif is_valid_uri(data_str):
+            # decipher the uri and open the group
+            store_path, parent_path, name = decipher_zarr_uri(data_str)
+
+            zarr_group = zarr.open_group(store_path, path=parent_path, mode="r")
+            content = zarr_group[name]
+
+            # assert if group or array
+            if isinstance(content, zarr.Group):
+                array_paths = collect_arrays(content)
+
+                # sort the names
+                array_paths.sort()
+
+                for array_path in array_paths:
+                    image_stacks.append(
+                        ZarrImageStack(group=content, data_path=array_path, axes=axes)
+                    )
+            else:
+                if not isinstance(content, zarr.Array):
+                    raise TypeError(
+                        f"Content at '{data_str}' is neither a zarr.Group nor "
+                        f"a zarr.Array."
+                    )
+
+                # create image stack from a single array
+                image_stacks.append(
+                    ZarrImageStack(
+                        group=zarr_group,
+                        data_path=name,
+                        axes=axes,
+                    )
+                )
+
+        else:
+            raise ValueError(
+                f"Source '{data_source}' is neither a zarr file nor a file URI."
+            )
+
+    return image_stacks
+
+
+def load_czis(
+    source: Sequence[Path],
+    axes: str,
+) -> list[CziImageStack]:
+    """
+    Load CZI image stacks from a sequence of CZI files paths.
+
+    If the CZI files contain multiple scenes, one image stack will be created for
+    each scene.
+
+    Axes should be in the format "SC(Z/T)YX", where Z or T are optional, and S and C
+    can be singleton dimensions, but must be provided.
+
+    Parameters
+    ----------
+    source: sequence of Path
+        The source files for the data.
+    axes: str
+        Specifies which axes of the data to use and how.
+        If this string ends with `"ZYX"` or `"TYX"`, the data will consist of 3-D
+        patches, using `Z` or `T` as third dimension, respectively.
+        If the string does not end with "ZYX", the data will consist of 2-D patches.
+
+    Returns
+    -------
+    list[CziImageStack]
+
+    Raises
+    ------
+    ValueError
+        If the provided axes are not valid.
+    """
+    if check_czi_axes_validity(axes) is False:
+        raise ValueError(
+            f"Provided axes '{axes}' are not valid. Axes must be in the `SC(Z/T)YX` "
+            f"format, where Z or T are optional, and S and C can be singleton "
+            f"dimensions, but must be provided."
+        )
+
+    depth_axis: Literal["none", "Z", "T"] = "none"
+    if axes.endswith("TYX"):
+        depth_axis = "T"
+    elif axes.endswith("ZYX"):
+        depth_axis = "Z"
+
+    image_stacks: list[CziImageStack] = []
+    for path in source:
+        scene_rectangles = CziImageStack.get_bounding_rectangles(path)
+        image_stacks.extend(
+            CziImageStack(path, scene=scene, depth_axis=depth_axis)
+            for scene in scene_rectangles.keys()
+        )
+    return image_stacks

careamics/dataset_ng/image_stack_loader/zarr_utils.py

@@ -0,0 +1,130 @@
+import warnings
+from pathlib import Path
+from urllib.parse import urlparse
+
+import zarr
+
+INPUT = str | Path
+
+
+def is_valid_uri(path: str | Path) -> bool:
+    """
+    Check if a path is a valid URI.
+
+    Parameters
+    ----------
+    path : str | Path
+        The path to check.
+
+    Returns
+    -------
+    bool
+        True if the path is valid URI, False otherwise.
+    """
+    parsed = urlparse(str(path))
+
+    valid_schemes = {"file", "s3", "gs", "az", "https", "http", "zip"}
+
+    if parsed.scheme and parsed.scheme.lower() in valid_schemes:
+        return True
+
+    return False
+
+
+def collect_arrays(zarr_group: zarr.Group) -> list[str]:
+    """
+    Collect all arrays in a Zarr group into a list.
+
+    Only run on the first level of the group.
+
+    Parameters
+    ----------
+    zarr_group : zarr.Group
+        The Zarr group to collect arrays from.
+
+    Returns
+    -------
+    listof str
+        A list of Zarr arrays contained in the group as relative path to the group.
+    """
+    arrays: list[str] = []
+
+    for name in zarr_group.array_keys():
+        if isinstance(zarr_group[name], zarr.Array):
+            arrays.append(name)
+
+    if arrays == []:
+        warnings.warn(
+            f"No arrays found in zarr group at '{zarr_group.path}'.",
+            UserWarning,
+            stacklevel=2,
+        )
+
+    return arrays
+
+
+def decipher_zarr_uri(source: str) -> tuple[str, str, str]:
+    """Extract the zarr store path, group path and array path from a zarr source string.
+
+    The input string is expected to be in the format:
+    \"file://path/to/zarr_store.zarr/group/path/array_name\"
+
+    Note that the root folder of the zarr store must end with ".zarr".
+
+    Parameters
+    ----------
+    source : str
+        The zarr source string.
+
+    Returns
+    -------
+    str
+        The path to the zarr store.
+    str
+        The parent group within the zarr store, if it is not the root, else "".
+    str
+        The group or array name the source is pointing to.
+
+    Raises
+    ------
+    ValueError
+        If the source string does not start with "file://".
+    ValueError
+        If the source string does not contain a ".zarr" file extension.
+    """
+    key = "file://"
+
+    if source[: len(key)] != key:
+        raise ValueError(f"Remote file not supported: {source}")
+
+    if ".zarr" not in source:
+        raise ValueError(f"No .zarr file extension found in source: {source}")
+
+    _source = source[len(key) :]
+    groups = _source.split("/")
+
+    # find .zarr entry
+    zarr_index = next((i for i, p in enumerate(groups) if p.endswith(".zarr")))
+
+    path_to_zarr = groups[: zarr_index + 1]
+    parent_path = groups[zarr_index + 1 : -1]
+    content_path = groups[-1]
+
+    return "/".join(path_to_zarr), "/".join(parent_path), content_path
+
+
+# TODO use yaozarrs models to validate OME-Zarr structure
+def is_ome_zarr(zarr_group: zarr.Group) -> bool:
+    """Check if a Zarr group is an OME-Zarr.
+
+    Parameters
+    ----------
+    zarr_group : zarr.Group
+        The Zarr group to check.
+
+    Returns
+    -------
+    bool
+        True if the Zarr group is an OME-Zarr, False otherwise.
+    """
+    return False

careamics/dataset_ng/legacy_interoperability.py

@@ -0,0 +1,175 @@
+"""
+A module for utility functions that adapts the new dataset outputs to work with previous
+code until it is updated.
+"""
+
+from collections.abc import Sequence
+from typing import cast
+
+import numpy as np
+from numpy.typing import NDArray
+
+from careamics.config.data.tile_information import TileInformation
+from careamics.utils.deprecation import deprecated
+
+from .dataset import ImageRegionData
+from .patching_strategies import TileSpecs
+
+
+@deprecated(
+    "use careamics.lightning.dataset_ng.prediction functions instead for stitching and"
+    " combining predictions."
+)
+def imageregions_to_tileinfos(
+    image_regions: Sequence[ImageRegionData],
+) -> list[tuple[NDArray, list[TileInformation]]]:
+    """
+    Converts a series of `TileSpecs` dictionaries to `TileInformation` pydantic class.
+
+    Parameters
+    ----------
+    image_regions : sequence of ImageRegionData
+        A list of ImageRegionData, it must have an instance of `TileSpecs` as it's
+        `region_data` field.
+
+    Returns
+    -------
+    list of TileInformation
+        The converted tile information.
+    """
+
+    tile_infos: list[TileInformation] = []
+
+    data = [image_region.data for image_region in image_regions]
+    tile_specs = [image_region.region_spec for image_region in image_regions]
+
+    data_indices: NDArray[np.int_] = np.array(
+        [tile_spec["data_idx"] for tile_spec in tile_specs], dtype=int
+    )
+    unique_data_indices = np.unique(data_indices)
+    # data_idx denotes which image stack a patch belongs to
+    # separate TileSpecs by image_stack
+    for data_idx in unique_data_indices:
+        # collect all ImageRegions
+        data_image_regions: list[ImageRegionData] = [
+            image_region
+            for image_region in image_regions
+            if image_region.region_spec["data_idx"] == data_idx
+        ]
+
+        # --- find last indices
+        # make sure tiles belonging to the same sample are together
+        data_image_regions.sort(
+            key=lambda image_region: image_region.region_spec["sample_idx"]
+        )
+        sample_indices = np.array(
+            [
+                image_region.region_spec["sample_idx"]
+                for image_region in data_image_regions
+            ]
+        )
+        # reverse array so indices returned are at far edge
+        _, unique_indices = np.unique(sample_indices[::-1], return_index=True)
+        # un reverse indices
+        last_indices = len(sample_indices) - 1 - unique_indices
+
+        # convert each ImageRegionData to tile_info
+        for i, image_region in enumerate(data_image_regions):
+            last_tile = i in last_indices
+            tile_info = _imageregion_to_tileinfo(image_region, last_tile)
+            tile_infos.append(tile_info)
+
+    return [
+        (data, [tile_info]) for data, tile_info in zip(data, tile_infos, strict=False)
+    ]
+
+
+def _imageregion_to_tileinfo(
+    image_region: ImageRegionData, last_tile: bool
+) -> TileInformation:
+    """
+    Convert a single `ImageRegionData` instance to a `TileInformation` instance. Whether
+    it is the last tile in a sequence needs to be supplied.
+
+    Parameters
+    ----------
+    image_region : ImageRegionData
+        An instance of `ImageRegionData`, it must have an instance of `TileSpecs` as
+        it's `region_data` field.
+    last_tile : bool
+        Whether a tile is the last tile in a sequence, for stitching.
+
+    Returns
+    -------
+    TileInformation
+        A tile information object.
+
+    Raises
+    ------
+    KeyError
+        If `image_region.region_spec` does not contain the keys: {'crop_coords',
+        'crop_size', 'stitch_coords'}.
+    """
+    patch_spec = image_region.region_spec
+    data_shape = image_region.data_shape
+
+    # TODO: In python 3.11 and greater, NamedTuples can inherit from Generic
+    #   so we could do image_region: ImageRegionData[TileSpecs]
+    #   and not have to do this check here + cast
+    # make sure image_region.region_spec is TileSpec
+    if (
+        ("crop_coords" not in patch_spec)
+        or ("crop_size" not in patch_spec)
+        or ("stitch_coords" not in patch_spec)
+    ):
+        raise KeyError(
+            "Could not find all keys: {'crop_coords', 'crop_size', 'stitch_coords'} in "
+            "`image_region.region_spec`."
+        )
+    tile_spec = cast(TileSpecs, patch_spec)  # ugly cast for mypy
+    return _tilespec_to_tileinfo(tile_spec, data_shape, last_tile)
+
+
+def _tilespec_to_tileinfo(
+    tile_spec: TileSpecs, data_shape: Sequence[int], last_tile: bool
+) -> TileInformation:
+    """
+    Convert a single `TileSpec` to a `TileInformation`. Whether it is the last tile
+    needs to be supplied.
+
+    Parameters
+    ----------
+    tile_spec : TileSpecs
+        A tile spec dictionary.
+    data_shape : sequence of int
+        The original shape of the data the tile came from, labeling the dimensions of
+        axes SC(Z)YX.
+    last_tile : bool
+        Whether a tile is the last tile in a sequence, for stitching.
+
+    Returns
+    -------
+    TileInformation
+        A tile information object.
+    """
+    overlap_crop_coords = tuple(
+        (
+            tile_spec["crop_coords"][i],
+            tile_spec["crop_coords"][i] + tile_spec["crop_size"][i],
+        )
+        for i in range(len(tile_spec["crop_coords"]))
+    )
+    stitch_coords = tuple(
+        (
+            tile_spec["stitch_coords"][i],
+            tile_spec["stitch_coords"][i] + tile_spec["crop_size"][i],
+        )
+        for i in range(len(tile_spec["crop_coords"]))
+    )
+    return TileInformation(
+        array_shape=tuple(data_shape[1:]),  # remove sample dimension
+        last_tile=last_tile,
+        overlap_crop_coords=overlap_crop_coords,
+        stitch_coords=stitch_coords,
+        sample_id=tile_spec["sample_idx"],
+    )