careamics 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl

careamics/config/transformations/xy_random_rotate90_model.py

@@ -0,0 +1,35 @@
+"""Pydantic model for the XYRandomRotate90 transform."""
+
+from typing import Literal, Optional
+
+from pydantic import ConfigDict, Field
+
+from .transform_model import TransformModel
+
+
+class XYRandomRotate90Model(TransformModel):
+    """
+    Pydantic model used to represent the XY random 90 degree rotation transformation.
+
+    Attributes
+    ----------
+    name : Literal["XYRandomRotate90"]
+        Name of the transformation.
+    p : float
+        Probability of applying the transform, by default 0.5.
+    seed : Optional[int]
+        Seed for the random number generator, by default None.
+    """
+
+    model_config = ConfigDict(
+        validate_assignment=True,
+    )
+
+    name: Literal["XYRandomRotate90"] = "XYRandomRotate90"
+    p: float = Field(
+        0.5,
+        description="Probability of applying the transform.",
+        ge=0,
+        le=1,
+    )
+    seed: Optional[int] = None

careamics/config/validators/__init__.py

@@ -0,0 +1,5 @@
+"""Validator utilities."""
+
+__all__ = ["check_axes_validity", "patch_size_ge_than_8_power_of_2"]
+
+from .validator_utils import check_axes_validity, patch_size_ge_than_8_power_of_2

careamics/config/validators/validator_utils.py

@@ -0,0 +1,101 @@
+"""
+Validator functions.
+
+These functions are used to validate dimensions and axes of inputs.
+"""
+
+from typing import List, Optional, Tuple, Union
+
+_AXES = "STCZYX"
+
+
+def check_axes_validity(axes: str) -> None:
+    """
+    Sanity check on axes.
+
+    The constraints on the axes are the following:
+    - must be a combination of 'STCZYX'
+    - must not contain duplicates
+    - must contain at least 2 contiguous axes: X and Y
+    - must contain at most 4 axes
+    - cannot contain both S and T axes
+
+    Axes do not need to be in the order 'STCZYX', as this depends on the user data.
+
+    Parameters
+    ----------
+    axes : str
+        Axes to validate.
+    """
+    _axes = axes.upper()
+
+    # Minimum is 2 (XY) and maximum is 4 (TZYX)
+    if len(_axes) < 2 or len(_axes) > 6:
+        raise ValueError(
+            f"Invalid axes {axes}. Must contain at least 2 and at most 6 axes."
+        )
+
+    if "YX" not in _axes and "XY" not in _axes:
+        raise ValueError(
+            f"Invalid axes {axes}. Must contain at least X and Y axes consecutively."
+        )
+
+    # all characters must be in REF_AXES = 'STCZYX'
+    if not all(s in _AXES for s in _axes):
+        raise ValueError(f"Invalid axes {axes}. Must be a combination of {_AXES}.")
+
+    # check for repeating characters
+    for i, s in enumerate(_axes):
+        if i != _axes.rfind(s):
+            raise ValueError(
+                f"Invalid axes {axes}. Cannot contain duplicate axes"
+                f" (got multiple {axes[i]})."
+            )
+
+
+def value_ge_than_8_power_of_2(
+    value: int,
+) -> None:
+    """
+    Validate that the value is greater or equal than 8 and a power of 2.
+
+    Parameters
+    ----------
+    value : int
+        Value to validate.
+
+    Raises
+    ------
+    ValueError
+        If the value is smaller than 8.
+    ValueError
+        If the value is not a power of 2.
+    """
+    if value < 8:
+        raise ValueError(f"Value must be greater than 8 (got {value}).")
+
+    if (value & (value - 1)) != 0:
+        raise ValueError(f"Value must be a power of 2 (got {value}).")
+
+
+def patch_size_ge_than_8_power_of_2(
+    patch_list: Optional[Union[List[int], Union[Tuple[int, ...]]]],
+) -> None:
+    """
+    Validate that each entry is greater or equal than 8 and a power of 2.
+
+    Parameters
+    ----------
+    patch_list : Optional[Union[List[int]]]
+        Patch size.
+
+    Raises
+    ------
+    ValueError
+        If the patch size if smaller than 8.
+    ValueError
+        If the patch size is not a power of 2.
+    """
+    if patch_list is not None:
+        for dim in patch_list:
+            value_ge_than_8_power_of_2(dim)

careamics/conftest.py

@@ -0,0 +1,39 @@
+"""File used to discover python modules and run doctest.
+
+See https://sybil.readthedocs.io/en/latest/use.html#pytest
+"""
+
+from pathlib import Path
+
+import pytest
+from pytest import TempPathFactory
+from sybil import Sybil
+from sybil.parsers.codeblock import PythonCodeBlockParser
+from sybil.parsers.doctest import DocTestParser
+
+
+@pytest.fixture(scope="module")
+def my_path(tmpdir_factory: TempPathFactory) -> Path:
+    """Fixture used in doctest to create a temporary directory.
+
+    Parameters
+    ----------
+    tmpdir_factory : TempPathFactory
+        Temporary path factory from pytest.
+
+    Returns
+    -------
+    Path
+        Temporary directory path.
+    """
+    return tmpdir_factory.mktemp("my_path")
+
+
+pytest_collect_file = Sybil(
+    parsers=[
+        DocTestParser(),
+        PythonCodeBlockParser(future_imports=["print_function"]),
+    ],
+    pattern="*.py",
+    fixtures=["my_path"],
+).pytest()

careamics/dataset/__init__.py

@@ -0,0 +1,17 @@
+"""Dataset module."""
+
+__all__ = [
+    "InMemoryDataset",
+    "InMemoryPredDataset",
+    "InMemoryTiledPredDataset",
+    "PathIterableDataset",
+    "IterableTiledPredDataset",
+    "IterablePredDataset",
+]
+
+from .in_memory_dataset import InMemoryDataset
+from .in_memory_pred_dataset import InMemoryPredDataset
+from .in_memory_tiled_pred_dataset import InMemoryTiledPredDataset
+from .iterable_dataset import PathIterableDataset
+from .iterable_pred_dataset import IterablePredDataset
+from .iterable_tiled_pred_dataset import IterableTiledPredDataset

careamics/dataset/dataset_utils/__init__.py

@@ -0,0 +1,19 @@
+"""Files and arrays utils used in the datasets."""
+
+__all__ = [
+    "reshape_array",
+    "compute_normalization_stats",
+    "get_files_size",
+    "list_files",
+    "validate_source_target_files",
+    "iterate_over_files",
+    "WelfordStatistics",
+]
+
+
+from .dataset_utils import (
+    reshape_array,
+)
+from .file_utils import get_files_size, list_files, validate_source_target_files
+from .iterate_over_files import iterate_over_files
+from .running_stats import WelfordStatistics, compute_normalization_stats

careamics/dataset/dataset_utils/dataset_utils.py

@@ -0,0 +1,101 @@
+"""Dataset utilities."""
+
+from typing import List, Tuple
+
+import numpy as np
+
+from careamics.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def _get_shape_order(
+    shape_in: Tuple[int, ...], axes_in: str, ref_axes: str = "STCZYX"
+) -> Tuple[Tuple[int, ...], str, List[int]]:
+    """
+    Compute a new shape for the array based on the reference axes.
+
+    Parameters
+    ----------
+    shape_in : Tuple[int, ...]
+        Input shape.
+    axes_in : str
+        Input axes.
+    ref_axes : str
+        Reference axes.
+
+    Returns
+    -------
+    Tuple[Tuple[int, ...], str, List[int]]
+        New shape, new axes, indices of axes in the new axes order.
+    """
+    indices = [axes_in.find(k) for k in ref_axes]
+
+    # remove all non-existing axes (index == -1)
+    new_indices = list(filter(lambda k: k != -1, indices))
+
+    # find axes order and get new shape
+    new_axes = [axes_in[ind] for ind in new_indices]
+    new_shape = tuple([shape_in[ind] for ind in new_indices])
+
+    return new_shape, "".join(new_axes), new_indices
+
+
+def reshape_array(x: np.ndarray, axes: str) -> np.ndarray:
+    """Reshape the data to (S, C, (Z), Y, X) by moving axes.
+
+    If the data has both S and T axes, the two axes will be merged. A singleton
+    dimension is added if there are no C axis.
+
+    Parameters
+    ----------
+    x : np.ndarray
+        Input array.
+    axes : str
+        Description of axes in format `STCZYX`.
+
+    Returns
+    -------
+    np.ndarray
+        Reshaped array with shape (S, C, (Z), Y, X).
+    """
+    _x = x
+    _axes = axes
+
+    # sanity checks
+    if len(_axes) != len(_x.shape):
+        raise ValueError(
+            f"Incompatible data shape ({_x.shape}) and axes ({_axes}). Are the axes "
+            f"correct?"
+        )
+
+    # get new x shape
+    new_x_shape, new_axes, indices = _get_shape_order(_x.shape, _axes)
+
+    # if S is not in the list of axes, then add a singleton S
+    if "S" not in new_axes:
+        new_axes = "S" + new_axes
+        _x = _x[np.newaxis, ...]
+        new_x_shape = (1,) + new_x_shape
+
+        # need to change the array of indices
+        indices = [0] + [1 + i for i in indices]
+
+    # reshape by moving axes
+    destination = list(range(len(indices)))
+    _x = np.moveaxis(_x, indices, destination)
+
+    # remove T if necessary
+    if "T" in new_axes:
+        new_x_shape = (-1,) + new_x_shape[2:]  # remove T and S
+        new_axes = new_axes.replace("T", "")
+
+        # reshape S and T together
+        _x = _x.reshape(new_x_shape)
+
+    # add channel
+    if "C" not in new_axes:
+        # Add channel axis after S
+        _x = np.expand_dims(_x, new_axes.index("S") + 1)
+
+    return _x

careamics/dataset/dataset_utils/file_utils.py

@@ -0,0 +1,141 @@
+"""File utilities."""
+
+from fnmatch import fnmatch
+from pathlib import Path
+from typing import List, Union
+
+import numpy as np
+
+from careamics.config.support import SupportedData
+from careamics.utils.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def get_files_size(files: List[Path]) -> float:
+    """Get files size in MB.
+
+    Parameters
+    ----------
+    files : List[Path]
+        List of files.
+
+    Returns
+    -------
+    float
+        Total size of the files in MB.
+    """
+    return np.sum([f.stat().st_size / 1024**2 for f in files])
+
+
+def list_files(
+    data_path: Union[str, Path],
+    data_type: Union[str, SupportedData],
+    extension_filter: str = "",
+) -> List[Path]:
+    """List recursively files in `data_path` and return a sorted list.
+
+    If `data_path` is a file, its name is validated against the `data_type` using
+    `fnmatch`, and the method returns `data_path` itself.
+
+    By default, if `data_type` is equal to `custom`, all files will be listed. To
+    further filter the files, use `extension_filter`.
+
+    `extension_filter` must be compatible with `fnmatch` and `Path.rglob`, e.g. "*.npy"
+    or "*.czi".
+
+    Parameters
+    ----------
+    data_path : Union[str, Path]
+        Path to the folder containing the data.
+    data_type : Union[str, SupportedData]
+        One of the supported data type (e.g. tif, custom).
+    extension_filter : str, optional
+        Extension filter, by default "".
+
+    Returns
+    -------
+    List[Path]
+        List of pathlib.Path objects.
+
+    Raises
+    ------
+    FileNotFoundError
+        If the data path does not exist.
+    ValueError
+        If the data path is empty or no files with the extension were found.
+    ValueError
+        If the file does not match the requested extension.
+    """
+    # convert to Path
+    data_path = Path(data_path)
+
+    # raise error if does not exists
+    if not data_path.exists():
+        raise FileNotFoundError(f"Data path {data_path} does not exist.")
+
+    # get extension compatible with fnmatch and rglob search
+    extension = SupportedData.get_extension_pattern(data_type)
+
+    if data_type == SupportedData.CUSTOM and extension_filter != "":
+        extension = extension_filter
+
+    # search recurively
+    if data_path.is_dir():
+        # search recursively the path for files with the extension
+        files = sorted(data_path.rglob(extension))
+    else:
+        # raise error if it has the wrong extension
+        if not fnmatch(str(data_path.absolute()), extension):
+            raise ValueError(
+                f"File {data_path} does not match the requested extension "
+                f'"{extension}".'
+            )
+
+        # save in list
+        files = [data_path]
+
+    # raise error if no files were found
+    if len(files) == 0:
+        raise ValueError(
+            f'Data path {data_path} is empty or files with extension "{extension}" '
+            f"were not found."
+        )
+
+    return files
+
+
+def validate_source_target_files(src_files: List[Path], tar_files: List[Path]) -> None:
+    """
+    Validate source and target path lists.
+
+    The two lists should have the same number of files, and the filenames should match.
+
+    Parameters
+    ----------
+    src_files : List[Path]
+        List of source files.
+    tar_files : List[Path]
+        List of target files.
+
+    Raises
+    ------
+    ValueError
+        If the number of files in source and target folders is not the same.
+    ValueError
+        If some filenames in Train and target folders are not the same.
+    """
+    # check equal length
+    if len(src_files) != len(tar_files):
+        raise ValueError(
+            f"The number of source files ({len(src_files)}) is not equal to the number "
+            f"of target files ({len(tar_files)})."
+        )
+
+    # check identical names
+    src_names = {f.name for f in src_files}
+    tar_names = {f.name for f in tar_files}
+    difference = src_names.symmetric_difference(tar_names)
+
+    if len(difference) > 0:
+        raise ValueError(f"Source and target files have different names: {difference}.")

careamics/dataset/dataset_utils/iterate_over_files.py

@@ -0,0 +1,83 @@
+"""Function to iterate over files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Callable, Generator, Optional, Union
+
+from numpy.typing import NDArray
+from torch.utils.data import get_worker_info
+
+from careamics.config import DataConfig, InferenceConfig
+from careamics.file_io.read import read_tiff
+from careamics.utils.logging import get_logger
+
+from .dataset_utils import reshape_array
+
+logger = get_logger(__name__)
+
+
+def iterate_over_files(
+    data_config: Union[DataConfig, InferenceConfig],
+    data_files: list[Path],
+    target_files: Optional[list[Path]] = None,
+    read_source_func: Callable = read_tiff,
+) -> Generator[tuple[NDArray, Optional[NDArray]], None, None]:
+    """Iterate over data source and yield whole reshaped images.
+
+    Parameters
+    ----------
+    data_config : CAREamics DataConfig or InferenceConfig
+        Configuration.
+    data_files : list of pathlib.Path
+        List of data files.
+    target_files : list of pathlib.Path, optional
+        List of target files, by default None.
+    read_source_func : Callable, optional
+        Function to read the source, by default read_tiff.
+
+    Yields
+    ------
+    NDArray
+        Image.
+    """
+    # When num_workers > 0, each worker process will have a different copy of the
+    # dataset object
+    # Configuring each copy independently to avoid having duplicate data returned
+    # from the workers
+    worker_info = get_worker_info()
+    worker_id = worker_info.id if worker_info is not None else 0
+    num_workers = worker_info.num_workers if worker_info is not None else 1
+
+    # iterate over the files
+    for i, filename in enumerate(data_files):
+        # retrieve file corresponding to the worker id
+        if i % num_workers == worker_id:
+            try:
+                # read data
+                sample = read_source_func(filename, data_config.axes)
+
+                # reshape array
+                reshaped_sample = reshape_array(sample, data_config.axes)
+
+                # read target, if available
+                if target_files is not None:
+                    if filename.name != target_files[i].name:
+                        raise ValueError(
+                            f"File {filename} does not match target file "
+                            f"{target_files[i]}. Have you passed sorted "
+                            f"arrays?"
+                        )
+
+                    # read target
+                    target = read_source_func(target_files[i], data_config.axes)
+
+                    # reshape target
+                    reshaped_target = reshape_array(target, data_config.axes)
+
+                    yield reshaped_sample, reshaped_target
+                else:
+                    yield reshaped_sample, None
+
+            except Exception as e:
+                logger.error(f"Error reading file {filename}: {e}")