careamics 0.1.0rc2__py3-none-any.whl → 0.1.0rc3__py3-none-any.whl

careamics/config/training_model.py

@@ -0,0 +1,65 @@
+"""Training configuration."""
+from __future__ import annotations
+
+from pprint import pformat
+from typing import Literal, Optional
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+)
+
+from .callback_model import CheckpointModel, EarlyStoppingModel
+
+
+class TrainingModel(BaseModel):
+    """
+    Parameters related to the training.
+
+    Mandatory parameters are:
+        - num_epochs: number of epochs, greater than 0.
+        - batch_size: batch size, greater than 0.
+        - augmentation: whether to use data augmentation or not (True or False).
+
+    Attributes
+    ----------
+    num_epochs : int
+        Number of epochs, greater than 0.
+    """
+
+    # Pydantic class configuration
+    model_config = ConfigDict(
+        validate_assignment=True,
+    )
+
+    num_epochs: int = Field(default=20, ge=1)
+
+    logger: Optional[Literal["wandb", "tensorboard"]] = None
+
+    checkpoint_callback: CheckpointModel = CheckpointModel()
+
+    early_stopping_callback: Optional[EarlyStoppingModel] = Field(
+        default=None, validate_default=True
+    )
+    # precision: Literal["64", "32", "16", "bf16"] = 32
+
+    def __str__(self) -> str:
+        """Pretty string reprensenting the configuration.
+
+        Returns
+        -------
+        str
+            Pretty string.
+        """
+        return pformat(self.model_dump())
+
+    def has_logger(self) -> bool:
+        """Check if the logger is defined.
+
+        Returns
+        -------
+        bool
+            Whether the logger is defined or not.
+        """
+        return self.logger is not None

careamics/config/transformations/__init__.py

@@ -0,0 +1,14 @@
+"""CAREamics transformation Pydantic models."""
+
+__all__ = [
+    "N2VManipulateModel",
+    "NDFlipModel",
+    "NormalizeModel",
+    "XYRandomRotate90Model",
+]
+
+
+from .n2v_manipulate_model import N2VManipulateModel
+from .nd_flip_model import NDFlipModel
+from .normalize_model import NormalizeModel
+from .xy_random_rotate90_model import XYRandomRotate90Model

careamics/config/transformations/n2v_manipulate_model.py

@@ -0,0 +1,63 @@
+"""Pydantic model for the N2VManipulate transform."""
+from typing import Literal
+
+from pydantic import ConfigDict, Field, field_validator
+
+from .transform_model import TransformModel
+
+
+class N2VManipulateModel(TransformModel):
+    """
+    Pydantic model used to represent N2V manipulation.
+
+    Attributes
+    ----------
+    name : Literal["N2VManipulate"]
+        Name of the transformation.
+    roi_size : int
+        Size of the masking region, by default 11.
+    masked_pixel_percentage : float
+        Percentage of masked pixels, by default 0.2.
+    strategy : Literal["uniform", "median"]
+        Strategy pixel value replacement, by default "uniform".
+    struct_mask_axis : Literal["horizontal", "vertical", "none"]
+        Axis of the structN2V mask, by default "none".
+    struct_mask_span : int
+        Span of the structN2V mask, by default 5.
+    """
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+    )
+
+    name: Literal["N2VManipulate"]
+    roi_size: int = Field(default=11, ge=3, le=21)
+    masked_pixel_percentage: float = Field(default=0.2, ge=0.05, le=1.0)
+    strategy: Literal["uniform", "median"] = Field(default="uniform")
+    struct_mask_axis: Literal["horizontal", "vertical", "none"] = Field(default="none")
+    struct_mask_span: int = Field(default=5, ge=3, le=15)
+
+    @field_validator("roi_size", "struct_mask_span")
+    @classmethod
+    def odd_value(cls, v: int) -> int:
+        """
+        Validate that the value is odd.
+
+        Parameters
+        ----------
+        v : int
+            Value to validate.
+
+        Returns
+        -------
+        int
+            The validated value.
+
+        Raises
+        ------
+        ValueError
+            If the value is even.
+        """
+        if v % 2 == 0:
+            raise ValueError("Size must be an odd number.")
+        return v

careamics/config/transformations/nd_flip_model.py

@@ -0,0 +1,32 @@
+"""Pydantic model for the NDFlip transform."""
+from typing import Literal
+
+from pydantic import ConfigDict, Field
+
+from .transform_model import TransformModel
+
+
+class NDFlipModel(TransformModel):
+    """
+    Pydantic model used to represent NDFlip transformation.
+
+    Attributes
+    ----------
+    name : Literal["NDFlip"]
+        Name of the transformation.
+    p : float
+        Probability of applying the transformation, by default 0.5.
+    is_3D : bool
+        Whether the transformation should be applied in 3D, by default False.
+    flip_z : bool
+        Whether to flip the z axis, by default True.
+    """
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+    )
+
+    name: Literal["NDFlip"]
+    p: float = Field(default=0.5, ge=0.0, le=1.0)
+    is_3D: bool = Field(default=False)
+    flip_z: bool = Field(default=True)

careamics/config/transformations/normalize_model.py

@@ -0,0 +1,31 @@
+"""Pydantic model for the Normalize transform."""
+from typing import Literal
+
+from pydantic import ConfigDict, Field
+
+from .transform_model import TransformModel
+
+
+class NormalizeModel(TransformModel):
+    """
+    Pydantic model used to represent Normalize transformation.
+
+    The Normalize transform is a zero mean and unit variance transformation.
+
+    Attributes
+    ----------
+    name : Literal["Normalize"]
+        Name of the transformation.
+    mean : float
+        Mean value for normalization.
+    std : float
+        Standard deviation value for normalization.
+    """
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+    )
+
+    name: Literal["Normalize"]
+    mean: float = Field(default=0.485)  # albumentations defaults
+    std: float = Field(default=0.229)

careamics/config/transformations/transform_model.py

@@ -0,0 +1,44 @@
+"""Parent model for the transforms."""
+from typing import Any, Dict
+
+from pydantic import BaseModel, ConfigDict
+
+
+class TransformModel(BaseModel):
+    """
+    Pydantic model used to represent a transformation.
+
+    The `model_dump` method is overwritten to exclude the name field.
+
+    Attributes
+    ----------
+    name : str
+        Name of the transformation.
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",  # throw errors if the parameters are not properly passed
+    )
+
+    name: str
+
+    def model_dump(self, **kwargs) -> Dict[str, Any]:
+        """
+        Return the model as a dictionary.
+
+        Parameters
+        ----------
+        **kwargs
+            Pydantic BaseMode model_dump method keyword arguments.
+
+        Returns
+        -------
+        Dict[str, Any]
+            Dictionary representation of the model.
+        """
+        model_dict = super().model_dump(**kwargs)
+
+        # remove the name field
+        model_dict.pop("name")
+
+        return model_dict

careamics/config/transformations/xy_random_rotate90_model.py

@@ -0,0 +1,29 @@
+"""Pydantic model for the XYRandomRotate90 transform."""
+from typing import Literal
+
+from pydantic import ConfigDict, Field
+
+from .transform_model import TransformModel
+
+
+class XYRandomRotate90Model(TransformModel):
+    """
+    Pydantic model used to represent NDFlip transformation.
+
+    Attributes
+    ----------
+    name : Literal["XYRandomRotate90"]
+        Name of the transformation.
+    p : float
+        Probability of applying the transformation, by default 0.5.
+    is_3D : bool
+        Whether the transformation should be applied in 3D, by default False.
+    """
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+    )
+
+    name: Literal["XYRandomRotate90"]
+    p: float = Field(default=0.5, ge=0.0, le=1.0)
+    is_3D: bool = Field(default=False)

careamics/config/validators/__init__.py

@@ -0,0 +1,5 @@
+"""Validator utilities."""
+
+__all__ = ["check_axes_validity", "patch_size_ge_than_8_power_of_2"]
+
+from .validator_utils import check_axes_validity, patch_size_ge_than_8_power_of_2

careamics/config/validators/validator_utils.py

@@ -0,0 +1,100 @@
+"""
+Validator functions.
+
+These functions are used to validate dimensions and axes of inputs.
+"""
+from typing import List, Optional, Tuple, Union
+
+_AXES = "STCZYX"
+
+
+def check_axes_validity(axes: str) -> None:
+    """
+    Sanity check on axes.
+
+    The constraints on the axes are the following:
+    - must be a combination of 'STCZYX'
+    - must not contain duplicates
+    - must contain at least 2 contiguous axes: X and Y
+    - must contain at most 4 axes
+    - cannot contain both S and T axes
+
+    Axes do not need to be in the order 'STCZYX', as this depends on the user data.
+
+    Parameters
+    ----------
+    axes : str
+        Axes to validate.
+    """
+    _axes = axes.upper()
+
+    # Minimum is 2 (XY) and maximum is 4 (TZYX)
+    if len(_axes) < 2 or len(_axes) > 6:
+        raise ValueError(
+            f"Invalid axes {axes}. Must contain at least 2 and at most 6 axes."
+        )
+
+    if "YX" not in _axes and "XY" not in _axes:
+        raise ValueError(
+            f"Invalid axes {axes}. Must contain at least X and Y axes consecutively."
+        )
+
+    # all characters must be in REF_AXES = 'STCZYX'
+    if not all(s in _AXES for s in _axes):
+        raise ValueError(f"Invalid axes {axes}. Must be a combination of {_AXES}.")
+
+    # check for repeating characters
+    for i, s in enumerate(_axes):
+        if i != _axes.rfind(s):
+            raise ValueError(
+                f"Invalid axes {axes}. Cannot contain duplicate axes"
+                f" (got multiple {axes[i]})."
+            )
+
+
+def value_ge_than_8_power_of_2(
+    value: int,
+) -> None:
+    """
+    Validate that the value is greater or equal than 8 and a power of 2.
+
+    Parameters
+    ----------
+    value : int
+        Value to validate.
+
+    Raises
+    ------
+    ValueError
+        If the value is smaller than 8.
+    ValueError
+        If the value is not a power of 2.
+    """
+    if value < 8:
+        raise ValueError(f"Value must be non-zero positive (got {value}).")
+
+    if (value & (value - 1)) != 0:
+        raise ValueError(f"Value must be a power of 2 (got {value}).")
+
+
+def patch_size_ge_than_8_power_of_2(
+    patch_list: Optional[Union[List[int], Union[Tuple[int, ...]]]],
+) -> None:
+    """
+    Validate that each entry is greater or equal than 8 and a power of 2.
+
+    Parameters
+    ----------
+    patch_list : Optional[Union[List[int]]]
+        Patch size.
+
+    Raises
+    ------
+    ValueError
+        If the patch size if smaller than 8.
+    ValueError
+        If the patch size is not a power of 2.
+    """
+    if patch_list is not None:
+        for dim in patch_list:
+            value_ge_than_8_power_of_2(dim)

careamics/conftest.py

@@ -0,0 +1,26 @@
+"""File used to discover python modules and run doctest.
+
+See https://sybil.readthedocs.io/en/latest/use.html#pytest
+"""
+from pathlib import Path
+
+import pytest
+from pytest import TempPathFactory
+from sybil import Sybil
+from sybil.parsers.codeblock import PythonCodeBlockParser
+from sybil.parsers.doctest import DocTestParser
+
+
+@pytest.fixture(scope="module")
+def my_path(tmpdir_factory: TempPathFactory) -> Path:
+    return tmpdir_factory.mktemp("my_path")
+
+
+pytest_collect_file = Sybil(
+    parsers=[
+        DocTestParser(),
+        PythonCodeBlockParser(future_imports=["print_function"]),
+    ],
+    pattern="*.py",
+    fixtures=["my_path"],
+).pytest()

careamics/dataset/__init__.py

@@ -1 +1,6 @@
 """Dataset module."""
+
+__all__ = ["InMemoryDataset", "PathIterableDataset"]
+
+from .in_memory_dataset import InMemoryDataset
+from .iterable_dataset import PathIterableDataset

careamics/dataset/dataset_utils/__init__.py

@@ -0,0 +1,19 @@
+"""Files and arrays utils used in the datasets."""
+
+
+__all__ = [
+    "reshape_array",
+    "get_files_size",
+    "list_files",
+    "validate_source_target_files",
+    "read_tiff",
+    "get_read_func",
+    "read_zarr",
+]
+
+
+from .dataset_utils import reshape_array
+from .file_utils import get_files_size, list_files, validate_source_target_files
+from .read_tiff import read_tiff
+from .read_utils import get_read_func
+from .read_zarr import read_zarr

careamics/dataset/dataset_utils/dataset_utils.py

@@ -0,0 +1,100 @@
+"""Convenience methods for datasets."""
+from typing import List, Tuple
+
+import numpy as np
+
+from careamics.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def _get_shape_order(
+    shape_in: Tuple[int, ...], axes_in: str, ref_axes: str = "STCZYX"
+) -> Tuple[Tuple[int, ...], str, List[int]]:
+    """
+    Compute a new shape for the array based on the reference axes.
+
+    Parameters
+    ----------
+    shape_in : Tuple
+        Input shape.
+    ref_axes : str
+        Reference axes.
+    axes_in : str
+        Input axes.
+
+    Returns
+    -------
+    Tuple[Tuple[int, ...], str, List[int]]
+        New shape, new axes, indices of axes in the new axes order.
+    """
+    indices = [axes_in.find(k) for k in ref_axes]
+
+    # remove all non-existing axes (index == -1)
+    new_indices = list(filter(lambda k: k != -1, indices))
+
+    # find axes order and get new shape
+    new_axes = [axes_in[ind] for ind in new_indices]
+    new_shape = tuple([shape_in[ind] for ind in new_indices])
+
+    return new_shape, "".join(new_axes), new_indices
+
+
+def reshape_array(x: np.ndarray, axes: str) -> np.ndarray:
+    """Reshape the data to (S, C, (Z), Y, X) by moving axes.
+
+    If the data has both S and T axes, the two axes will be merged. A singleton
+    dimension is added if there are no C axis.
+
+    Parameters
+    ----------
+    x : np.ndarray
+        Input array.
+    axes : str
+        Description of axes in format `STCZYX`.
+
+    Returns
+    -------
+    np.ndarray
+        Reshaped array with shape (S, C, (Z), Y, X).
+    """
+    _x = x
+    _axes = axes
+
+    # sanity checks
+    if len(_axes) != len(_x.shape):
+        raise ValueError(
+            f"Incompatible data shape ({_x.shape}) and axes ({_axes}). Are the axes "
+            f"correct?"
+        )
+
+    # get new x shape
+    new_x_shape, new_axes, indices = _get_shape_order(_x.shape, _axes)
+
+    # if S is not in the list of axes, then add a singleton S
+    if "S" not in new_axes:
+        new_axes = "S" + new_axes
+        _x = _x[np.newaxis, ...]
+        new_x_shape = (1,) + new_x_shape
+
+        # need to change the array of indices
+        indices = [0] + [1 + i for i in indices]
+
+    # reshape by moving axes
+    destination = list(range(len(indices)))
+    _x = np.moveaxis(_x, indices, destination)
+
+    # remove T if necessary
+    if "T" in new_axes:
+        new_x_shape = (-1,) + new_x_shape[2:]  # remove T and S
+        new_axes = new_axes.replace("T", "")
+
+        # reshape S and T together
+        _x = _x.reshape(new_x_shape)
+
+    # add channel
+    if "C" not in new_axes:
+        # Add channel axis after S
+        _x = np.expand_dims(_x, new_axes.index("S") + 1)
+
+    return _x

careamics/dataset/dataset_utils/file_utils.py

@@ -0,0 +1,140 @@
+from fnmatch import fnmatch
+from pathlib import Path
+from typing import List, Union
+
+import numpy as np
+
+from careamics.config.support import SupportedData
+from careamics.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def get_files_size(files: List[Path]) -> float:
+    """
+    Get files size in MB.
+
+    Parameters
+    ----------
+    files : List[Path]
+        List of files.
+
+    Returns
+    -------
+    float
+        Total size of the files in MB.
+    """
+    return np.sum([f.stat().st_size / 1024**2 for f in files])
+
+
+def list_files(
+    data_path: Union[str, Path],
+    data_type: Union[str, SupportedData],
+    extension_filter: str = "",
+) -> List[Path]:
+    """Creates a recursive list of files in `data_path`.
+
+    If `data_path` is a file, its name is validated against the `data_type` using
+    `fnmatch`, and the method returns `data_path` itself.
+
+    By default, if `data_type` is equal to `custom`, all files will be listed. To
+    further filter the files, use `extension_filter`.
+
+    `extension_filter` must be compatible with `fnmatch` and `Path.rglob`, e.g. "*.npy"
+    or "*.czi".
+
+    Parameters
+    ----------
+    data_path : Union[str, Path]
+        Path to the folder containing the data.
+    data_type : Union[str, SupportedData]
+        One of the supported data type (e.g. tif, custom).
+    extension_filter : str, optional
+        Extension filter, by default "".
+
+    Returns
+    -------
+    List[Path]
+        List of pathlib.Path objects.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the data path does not exist.
+    ValueError
+        If the data path is empty or no files with the extension were found.
+    ValueError
+        If the file does not match the requested extension.
+    """
+    # convert to Path
+    data_path = Path(data_path)
+
+    # raise error if does not exists
+    if not data_path.exists():
+        raise FileNotFoundError(f"Data path {data_path} does not exist.")
+
+    # get extension compatible with fnmatch and rglob search
+    extension = SupportedData.get_extension(data_type)
+
+    if data_type == SupportedData.CUSTOM and extension_filter != "":
+        extension = extension_filter
+
+    # search recurively
+    if data_path.is_dir():
+        # search recursively the path for files with the extension
+        files = sorted(data_path.rglob(extension))
+    else:
+        # raise error if it has the wrong extension
+        if not fnmatch(str(data_path.absolute()), extension):
+            raise ValueError(
+                f"File {data_path} does not match the requested extension "
+                f'"{extension}".'
+            )
+
+        # save in list
+        files = [data_path]
+
+    # raise error if no files were found
+    if len(files) == 0:
+        raise ValueError(
+            f'Data path {data_path} is empty or files with extension "{extension}" '
+            f"were not found."
+        )
+
+    return files
+
+
+def validate_source_target_files(src_files: List[Path], tar_files: List[Path]) -> None:
+    """
+    Validate source and target path lists.
+
+    The two lists should have the same number of files, and the filenames should match.
+
+    Parameters
+    ----------
+    src_files : List[Path]
+        List of source files.
+    tar_files : List[Path]
+        List of target files.
+
+    Raises
+    ------
+    ValueError
+        If the number of files in source and target folders is not the same.
+    ValueError
+        If some filenames in Train and target folders are not the same.
+    """
+    # check equal length
+    if len(src_files) != len(tar_files):
+        raise ValueError(
+            f"The number of source files ({len(src_files)}) is not equal to the number "
+            f"of target files ({len(tar_files)})."
+        )
+
+    # check identical names
+    src_names = {f.name for f in src_files}
+    tar_names = {f.name for f in tar_files}
+    difference = src_names.symmetric_difference(tar_names)
+
+    if len(difference) > 0:
+        raise ValueError(f"Source and target files have different names: {difference}.")