careamics 0.1.0rc5__py3-none-any.whl → 0.1.0rc7__py3-none-any.whl

careamics/config/support/supported_transforms.py

@@ -1,23 +1,11 @@
+"""Transforms supported by CAREamics."""
+
 from careamics.utils import BaseEnum
 
 
 class SupportedTransform(str, BaseEnum):
-    """Transforms officially supported by CAREamics.
-
-    - Flip: from Albumentations, randomly flip the input horizontally, vertically or
-        both, parameter `p` can be used to set the probability to apply the transform.
-    - XYRandomRotate90: #TODO
-    - Normalize # TODO add details, in particular about the parameters
-    - ManipulateN2V # TODO add details, in particular about the parameters
-    - NDFlip
-
-    Note that while any Albumentations (see https://albumentations.ai/) transform can be
-    used in CAREamics, no check are implemented to verify the compatibility of any other
-    transforms than the ones officially supported.
-    """
+    """Transforms officially supported by CAREamics."""
 
-    NDFLIP = "NDFlip"
+    XY_FLIP = "XYFlip"
     XY_RANDOM_ROTATE90 = "XYRandomRotate90"
-    NORMALIZE = "Normalize"
     N2V_MANIPULATE = "N2VManipulate"
-    # CUSTOM = "Custom"

careamics/config/tile_information.py

@@ -1,8 +1,8 @@
-from __future__ import annotations
+"""Pydantic model representing the metadata of a prediction tile."""
 
-from typing import Optional, Tuple
+from __future__ import annotations
 
-from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
+from pydantic import BaseModel, ConfigDict, field_validator
 
 
 class TileInformation(BaseModel):
@@ -11,30 +11,33 @@ class TileInformation(BaseModel):
 
     This model is used to represent the information required to stitch back a tile into
     a larger image. It is used throughout the prediction pipeline of CAREamics.
+
+    Array shape should be (C)(Z)YX, where C and Z are optional dimensions, and must not
+    contain singleton dimensions.
     """
 
     model_config = ConfigDict(validate_default=True)
 
-    array_shape: Tuple[int, ...]
-    tiled: bool = False
+    array_shape: tuple[int, ...]
     last_tile: bool = False
-    overlap_crop_coords: Optional[Tuple[Tuple[int, ...], ...]] = Field(default=None)
-    stitch_coords: Optional[Tuple[Tuple[int, ...], ...]] = Field(default=None)
+    overlap_crop_coords: tuple[tuple[int, ...], ...]
+    stitch_coords: tuple[tuple[int, ...], ...]
+    sample_id: int
 
     @field_validator("array_shape")
     @classmethod
-    def no_singleton_dimensions(cls, v: Tuple[int, ...]):
+    def no_singleton_dimensions(cls, v: tuple[int, ...]):
         """
         Check that the array shape does not have any singleton dimensions.
 
         Parameters
         ----------
-        v : Tuple[int, ...]
+        v : tuple of int
             Array shape to check.
 
         Returns
         -------
-        Tuple[int, ...]
+        tuple of int
             The array shape if it does not contain singleton dimensions.
 
         Raises
@@ -46,59 +49,26 @@ class TileInformation(BaseModel):
             raise ValueError("Array shape must not contain singleton dimensions.")
         return v
 
-    @field_validator("last_tile")
-    @classmethod
-    def only_if_tiled(cls, v: bool, values: ValidationInfo):
-        """
-        Check that the last tile flag is only set if tiling is enabled.
+    def __eq__(self, other_tile: object):
+        """Check if two tile information objects are equal.
 
         Parameters
         ----------
-        v : bool
-            Last tile flag.
-        values : ValidationInfo
-            Validation information.
+        other_tile : object
+            Tile information object to compare with.
 
         Returns
         -------
         bool
-            The last tile flag.
-        """
-        if not values.data["tiled"]:
-            return False
-        return v
-
-    @field_validator("overlap_crop_coords", "stitch_coords")
-    @classmethod
-    def mandatory_if_tiled(
-        cls, v: Optional[Tuple[int, ...]], values: ValidationInfo
-    ) -> Optional[Tuple[int, ...]]:
-        """
-        Check that the coordinates are not `None` if tiling is enabled.
-
-        The method also return `None` if tiling is not enabled.
-
-        Parameters
-        ----------
-        v : Optional[Tuple[int, ...]]
-            Coordinates to check.
-        values : ValidationInfo
-            Validation information.
-
-        Returns
-        -------
-        Optional[Tuple[int, ...]]
-            The coordinates if tiling is enabled, otherwise `None`.
-
-        Raises
-        ------
-        ValueError
-            If the coordinates are `None` and tiling is enabled.
+            Whether the two tile information objects are equal.
         """
-        if values.data["tiled"]:
-            if v is None:
-                raise ValueError("Value must be specified if tiling is enabled.")
-
-            return v
-        else:
-            return None
+        if not isinstance(other_tile, TileInformation):
+            return NotImplemented
+
+        return (
+            self.array_shape == other_tile.array_shape
+            and self.last_tile == other_tile.last_tile
+            and self.overlap_crop_coords == other_tile.overlap_crop_coords
+            and self.stitch_coords == other_tile.stitch_coords
+            and self.sample_id == other_tile.sample_id
+        )

careamics/config/transformations/__init__.py

@@ -2,13 +2,14 @@
 
 __all__ = [
     "N2VManipulateModel",
-    "NDFlipModel",
+    "XYFlipModel",
     "NormalizeModel",
     "XYRandomRotate90Model",
+    "XorYFlipModel",
 ]
 
 
 from .n2v_manipulate_model import N2VManipulateModel
-from .nd_flip_model import NDFlipModel
 from .normalize_model import NormalizeModel
+from .xy_flip_model import XYFlipModel
 from .xy_random_rotate90_model import XYRandomRotate90Model

careamics/config/transformations/normalize_model.py

@@ -1,8 +1,9 @@
 """Pydantic model for the Normalize transform."""
 
-from typing import Literal
+from typing import Literal, Optional
 
-from pydantic import ConfigDict, Field
+from pydantic import ConfigDict, Field, model_validator
+from typing_extensions import Self
 
 from .transform_model import TransformModel
 
@@ -28,5 +29,32 @@ class NormalizeModel(TransformModel):
     )
 
     name: Literal["Normalize"] = "Normalize"
-    mean: float = Field(default=0.485)  # albumentations defaults
-    std: float = Field(default=0.229)
+    image_means: list = Field(..., min_length=0, max_length=32)
+    image_stds: list = Field(..., min_length=0, max_length=32)
+    target_means: Optional[list] = Field(default=None, min_length=0, max_length=32)
+    target_stds: Optional[list] = Field(default=None, min_length=0, max_length=32)
+
+    @model_validator(mode="after")
+    def validate_means_stds(self: Self) -> Self:
+        """Validate that the means and stds have the same length.
+
+        Returns
+        -------
+        Self
+            The instance of the model.
+        """
+        if len(self.image_means) != len(self.image_stds):
+            raise ValueError("The number of image means and stds must be the same.")
+
+        if (self.target_means is None) != (self.target_stds is None):
+            raise ValueError(
+                "Both target means and stds must be provided together, or bot None."
+            )
+
+        if self.target_means is not None and self.target_stds is not None:
+            if len(self.target_means) != len(self.target_stds):
+                raise ValueError(
+                    "The number of target means and stds must be the same."
+                )
+
+        return self

careamics/config/transformations/xy_flip_model.py

@@ -0,0 +1,43 @@
+"""Pydantic model for the XYFlip transform."""
+
+from typing import Literal, Optional
+
+from pydantic import ConfigDict, Field
+
+from .transform_model import TransformModel
+
+
+class XYFlipModel(TransformModel):
+    """
+    Pydantic model used to represent XYFlip transformation.
+
+    Attributes
+    ----------
+    name : Literal["XYFlip"]
+        Name of the transformation.
+    p : float
+        Probability of applying the transform, by default 0.5.
+    seed : Optional[int]
+        Seed for the random number generator,  by default None.
+    """
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+    )
+
+    name: Literal["XYFlip"] = "XYFlip"
+    flip_x: bool = Field(
+        True,
+        description="Whether to flip along the X axis.",
+    )
+    flip_y: bool = Field(
+        True,
+        description="Whether to flip along the Y axis.",
+    )
+    p: float = Field(
+        0.5,
+        description="Probability of applying the transform.",
+        ge=0,
+        le=1,
+    )
+    seed: Optional[int] = None

careamics/config/transformations/xy_random_rotate90_model.py

@@ -2,21 +2,23 @@
 
 from typing import Literal, Optional
 
-from pydantic import ConfigDict
+from pydantic import ConfigDict, Field
 
 from .transform_model import TransformModel
 
 
 class XYRandomRotate90Model(TransformModel):
     """
-    Pydantic model used to represent NDFlip transformation.
+    Pydantic model used to represent the XY random 90 degree rotation transformation.
 
     Attributes
     ----------
     name : Literal["XYRandomRotate90"]
         Name of the transformation.
+    p : float
+        Probability of applying the transform, by default 0.5.
     seed : Optional[int]
-        Seed for the random number generator.
+        Seed for the random number generator, by default None.
     """
 
     model_config = ConfigDict(
@@ -24,4 +26,10 @@ class XYRandomRotate90Model(TransformModel):
     )
 
     name: Literal["XYRandomRotate90"] = "XYRandomRotate90"
+    p: float = Field(
+        0.5,
+        description="Probability of applying the transform.",
+        ge=0,
+        le=1,
+    )
     seed: Optional[int] = None

careamics/config/validators/validator_utils.py

@@ -72,7 +72,7 @@ def value_ge_than_8_power_of_2(
         If the value is not a power of 2.
     """
     if value < 8:
-        raise ValueError(f"Value must be non-zero positive (got {value}).")
+        raise ValueError(f"Value must be greater than 8 (got {value}).")
 
     if (value & (value - 1)) != 0:
         raise ValueError(f"Value must be a power of 2 (got {value}).")

careamics/conftest.py

@@ -14,6 +14,18 @@ from sybil.parsers.doctest import DocTestParser
 
 @pytest.fixture(scope="module")
 def my_path(tmpdir_factory: TempPathFactory) -> Path:
+    """Fixture used in doctest to create a temporary directory.
+
+    Parameters
+    ----------
+    tmpdir_factory : TempPathFactory
+        Temporary path factory from pytest.
+
+    Returns
+    -------
+    Path
+        Temporary directory path.
+    """
     return tmpdir_factory.mktemp("my_path")
 
 

careamics/dataset/__init__.py

@@ -1,6 +1,17 @@
 """Dataset module."""
 
-__all__ = ["InMemoryDataset", "PathIterableDataset"]
+__all__ = [
+    "InMemoryDataset",
+    "InMemoryPredDataset",
+    "InMemoryTiledPredDataset",
+    "PathIterableDataset",
+    "IterableTiledPredDataset",
+    "IterablePredDataset",
+]
 
 from .in_memory_dataset import InMemoryDataset
+from .in_memory_pred_dataset import InMemoryPredDataset
+from .in_memory_tiled_pred_dataset import InMemoryTiledPredDataset
 from .iterable_dataset import PathIterableDataset
+from .iterable_pred_dataset import IterablePredDataset
+from .iterable_tiled_pred_dataset import IterableTiledPredDataset

careamics/dataset/dataset_utils/__init__.py

@@ -2,17 +2,24 @@
 
 __all__ = [
     "reshape_array",
+    "compute_normalization_stats",
     "get_files_size",
     "list_files",
     "validate_source_target_files",
     "read_tiff",
     "get_read_func",
     "read_zarr",
+    "iterate_over_files",
+    "WelfordStatistics",
 ]
 
 
-from .dataset_utils import reshape_array
+from .dataset_utils import (
+    reshape_array,
+)
 from .file_utils import get_files_size, list_files, validate_source_target_files
+from .iterate_over_files import iterate_over_files
 from .read_tiff import read_tiff
 from .read_utils import get_read_func
 from .read_zarr import read_zarr
+from .running_stats import WelfordStatistics, compute_normalization_stats

careamics/dataset/dataset_utils/dataset_utils.py

@@ -1,4 +1,4 @@
-"""Convenience methods for datasets."""
+"""Dataset utilities."""
 
 from typing import List, Tuple
 
@@ -17,12 +17,12 @@ def _get_shape_order(
 
     Parameters
     ----------
-    shape_in : Tuple
+    shape_in : Tuple[int, ...]
         Input shape.
-    ref_axes : str
-        Reference axes.
     axes_in : str
         Input axes.
+    ref_axes : str
+        Reference axes.
 
     Returns
     -------

careamics/dataset/dataset_utils/file_utils.py

@@ -1,3 +1,5 @@
+"""File utilities."""
+
 from fnmatch import fnmatch
 from pathlib import Path
 from typing import List, Union
@@ -11,8 +13,7 @@ logger = get_logger(__name__)
 
 
 def get_files_size(files: List[Path]) -> float:
-    """
-    Get files size in MB.
+    """Get files size in MB.
 
     Parameters
     ----------
@@ -32,7 +33,7 @@ def list_files(
     data_type: Union[str, SupportedData],
     extension_filter: str = "",
 ) -> List[Path]:
-    """Creates a recursive list of files in `data_path`.
+    """List recursively files in `data_path` and return a sorted list.
 
     If `data_path` is a file, its name is validated against the `data_type` using
     `fnmatch`, and the method returns `data_path` itself.

careamics/dataset/dataset_utils/iterate_over_files.py

@@ -0,0 +1,83 @@
+"""Function to iterate over files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Callable, Generator, Optional, Union
+
+from numpy.typing import NDArray
+from torch.utils.data import get_worker_info
+
+from careamics.config import DataConfig, InferenceConfig
+from careamics.utils.logging import get_logger
+
+from .dataset_utils import reshape_array
+from .read_tiff import read_tiff
+
+logger = get_logger(__name__)
+
+
+def iterate_over_files(
+    data_config: Union[DataConfig, InferenceConfig],
+    data_files: list[Path],
+    target_files: Optional[list[Path]] = None,
+    read_source_func: Callable = read_tiff,
+) -> Generator[tuple[NDArray, Optional[NDArray]], None, None]:
+    """Iterate over data source and yield whole reshaped images.
+
+    Parameters
+    ----------
+    data_config : CAREamics DataConfig or InferenceConfig
+        Configuration.
+    data_files : list of pathlib.Path
+        List of data files.
+    target_files : list of pathlib.Path, optional
+        List of target files, by default None.
+    read_source_func : Callable, optional
+        Function to read the source, by default read_tiff.
+
+    Yields
+    ------
+    NDArray
+        Image.
+    """
+    # When num_workers > 0, each worker process will have a different copy of the
+    # dataset object
+    # Configuring each copy independently to avoid having duplicate data returned
+    # from the workers
+    worker_info = get_worker_info()
+    worker_id = worker_info.id if worker_info is not None else 0
+    num_workers = worker_info.num_workers if worker_info is not None else 1
+
+    # iterate over the files
+    for i, filename in enumerate(data_files):
+        # retrieve file corresponding to the worker id
+        if i % num_workers == worker_id:
+            try:
+                # read data
+                sample = read_source_func(filename, data_config.axes)
+
+                # reshape array
+                reshaped_sample = reshape_array(sample, data_config.axes)
+
+                # read target, if available
+                if target_files is not None:
+                    if filename.name != target_files[i].name:
+                        raise ValueError(
+                            f"File {filename} does not match target file "
+                            f"{target_files[i]}. Have you passed sorted "
+                            f"arrays?"
+                        )
+
+                    # read target
+                    target = read_source_func(target_files[i], data_config.axes)
+
+                    # reshape target
+                    reshaped_target = reshape_array(target, data_config.axes)
+
+                    yield reshaped_sample, reshaped_target
+                else:
+                    yield reshaped_sample, None
+
+            except Exception as e:
+                logger.error(f"Error reading file {filename}: {e}")

careamics/dataset/dataset_utils/read_tiff.py

@@ -1,3 +1,5 @@
+"""Funtions to read tiff images."""
+
 import logging
 from fnmatch import fnmatch
 from pathlib import Path
@@ -19,8 +21,10 @@ def read_tiff(file_path: Path, *args: list, **kwargs: dict) -> np.ndarray:
     ----------
     file_path : Path
         Path to a file.
-    axes : str
-        Description of axes in format STCZYX.
+    *args : list
+        Additional arguments.
+    **kwargs : dict
+        Additional keyword arguments.
 
     Returns
     -------
@@ -49,13 +53,4 @@ def read_tiff(file_path: Path, *args: list, **kwargs: dict) -> np.ndarray:
     else:
         raise ValueError(f"File {file_path} is not a valid tiff.")
 
-    # check dimensions
-    # TODO or should this really be done here? probably in the LightningDataModule
-    # TODO this should also be centralized somewhere else (validate_dimensions)
-    if len(array.shape) < 2 or len(array.shape) > 6:
-        raise ValueError(
-            f"Incorrect data dimensions. Must be 2, 3 or 4 (got {array.shape} for"
-            f"file {file_path})."
-        )
-
     return array

careamics/dataset/dataset_utils/read_utils.py

@@ -1,3 +1,5 @@
+"""Read function utilities."""
+
 from typing import Callable, Union
 
 from careamics.config.support import SupportedData

careamics/dataset/dataset_utils/read_zarr.py

@@ -1,3 +1,5 @@
+"""Function to read zarr images."""
+
 from typing import Union
 
 from zarr import Group, core, hierarchy, storage
@@ -6,26 +8,28 @@ from zarr import Group, core, hierarchy, storage
 def read_zarr(
     zarr_source: Group, axes: str
 ) -> Union[core.Array, storage.DirectoryStore, hierarchy.Group]:
-    """Reads a file and returns a pointer.
+    """Read a file and returns a pointer.
 
     Parameters
     ----------
-    file_path : Path
-        pathlib.Path object containing a path to a file
+    zarr_source : Group
+        Zarr storage.
+    axes : str
+        Axes of the data.
 
     Returns
     -------
     np.ndarray
-        Pointer to zarr storage
+        Pointer to zarr storage.
 
     Raises
     ------
     ValueError, OSError
-        if a file is not a valid tiff or damaged
+        if a file is not a valid tiff or damaged.
     ValueError
-        if data dimensions are not 2, 3 or 4
+        if data dimensions are not 2, 3 or 4.
     ValueError
-        if axes parameter from config is not consistent with data dimensions
+        if axes parameter from config is not consistent with data dimensions.
     """
     if isinstance(zarr_source, hierarchy.Group):
         array = zarr_source[0]