careamics 0.1.0rc5__py3-none-any.whl → 0.1.0rc6__py3-none-any.whl

careamics/dataset/in_memory_dataset.py

@@ -13,6 +13,7 @@ from careamics.transforms import Compose
 
 from ..config import DataConfig, InferenceConfig
 from ..config.tile_information import TileInformation
+from ..config.transformations import NormalizeModel
 from ..utils.logging import get_logger
 from .dataset_utils import read_tiff, reshape_array
 from .patching.patching import (
@@ -27,24 +28,49 @@ logger = get_logger(__name__)
 
 
 class InMemoryDataset(Dataset):
-    """Dataset storing data in memory and allowing generating patches from it."""
+    """Dataset storing data in memory and allowing generating patches from it.
+
+    Parameters
+    ----------
+    data_config : DataConfig
+        Data configuration.
+    inputs : Union[np.ndarray, List[Path]]
+        Input data.
+    input_target : Optional[Union[np.ndarray, List[Path]]], optional
+        Target data, by default None.
+    read_source_func : Callable, optional
+        Read source function for custom types, by default read_tiff.
+    **kwargs : Any
+        Additional keyword arguments, unused.
+    """
 
     def __init__(
         self,
         data_config: DataConfig,
         inputs: Union[np.ndarray, List[Path]],
-        data_target: Optional[Union[np.ndarray, List[Path]]] = None,
+        input_target: Optional[Union[np.ndarray, List[Path]]] = None,
         read_source_func: Callable = read_tiff,
         **kwargs: Any,
     ) -> None:
         """
         Constructor.
 
-        # TODO
+        Parameters
+        ----------
+        data_config : DataConfig
+            Data configuration.
+        inputs : Union[np.ndarray, List[Path]]
+            Input data.
+        input_target : Optional[Union[np.ndarray, List[Path]]], optional
+            Target data, by default None.
+        read_source_func : Callable, optional
+            Read source function for custom types, by default read_tiff.
+        **kwargs : Any
+            Additional keyword arguments, unused.
         """
         self.data_config = data_config
         self.inputs = inputs
-        self.data_target = data_target
+        self.input_targets = input_target
         self.axes = self.data_config.axes
         self.patch_size = self.data_config.patch_size
 
@@ -52,11 +78,11 @@ class InMemoryDataset(Dataset):
         self.read_source_func = read_source_func
 
         # Generate patches
-        supervised = self.data_target is not None
-        patches = self._prepare_patches(supervised)
+        supervised = self.input_targets is not None
+        patch_data = self._prepare_patches(supervised)
 
         # Add results to members
-        self.data, self.data_targets, computed_mean, computed_std = patches
+        self.patches, self.patch_targets, computed_mean, computed_std = patch_data
 
         if not self.data_config.mean or not self.data_config.std:
             self.mean, self.std = computed_mean, computed_std
@@ -91,18 +117,18 @@ class InMemoryDataset(Dataset):
         """
         if supervised:
             if isinstance(self.inputs, np.ndarray) and isinstance(
-                self.data_target, np.ndarray
+                self.input_targets, np.ndarray
             ):
                 return prepare_patches_supervised_array(
                     self.inputs,
                     self.axes,
-                    self.data_target,
+                    self.input_targets,
                     self.patch_size,
                 )
-            elif isinstance(self.inputs, list) and isinstance(self.data_target, list):
+            elif isinstance(self.inputs, list) and isinstance(self.input_targets, list):
                 return prepare_patches_supervised(
                     self.inputs,
-                    self.data_target,
+                    self.input_targets,
                     self.axes,
                     self.patch_size,
                     self.read_source_func,
@@ -111,7 +137,7 @@ class InMemoryDataset(Dataset):
                 raise ValueError(
                     f"Data and target must be of the same type, either both numpy "
                     f"arrays or both lists of paths, got {type(self.inputs)} (data) "
-                    f"and {type(self.data_target)} (target)."
+                    f"and {type(self.input_targets)} (target)."
                 )
         else:
             if isinstance(self.inputs, np.ndarray):
@@ -137,9 +163,9 @@ class InMemoryDataset(Dataset):
         int
             Length of the dataset.
         """
-        return len(self.data)
+        return len(self.patches)
 
-    def __getitem__(self, index: int) -> Tuple[np.ndarray]:
+    def __getitem__(self, index: int) -> Tuple[np.ndarray, ...]:
         """
         Return the patch corresponding to the provided index.
 
@@ -158,12 +184,12 @@ class InMemoryDataset(Dataset):
         ValueError
             If dataset mean and std are not set.
         """
-        patch = self.data[index]
+        patch = self.patches[index]
 
         # if there is a target
-        if self.data_target is not None:
+        if self.patch_targets is not None:
             # get target
-            target = self.data_targets[index]
+            target = self.patch_targets[index]
 
             return self.patch_transform(patch=patch, target=target)
 
@@ -223,25 +249,25 @@ class InMemoryDataset(Dataset):
         indices = np.random.choice(total_patches, n_patches, replace=False)
 
         # extract patches
-        val_patches = self.data[indices]
+        val_patches = self.patches[indices]
 
         # remove patches from self.patch
-        self.data = np.delete(self.data, indices, axis=0)
+        self.patches = np.delete(self.patches, indices, axis=0)
 
         # same for targets
-        if self.data_targets is not None:
-            val_targets = self.data_targets[indices]
-            self.data_targets = np.delete(self.data_targets, indices, axis=0)
+        if self.patch_targets is not None:
+            val_targets = self.patch_targets[indices]
+            self.patch_targets = np.delete(self.patch_targets, indices, axis=0)
 
         # clone the dataset
         dataset = copy.deepcopy(self)
 
         # reassign patches
-        dataset.data = val_patches
+        dataset.patches = val_patches
 
         # reassign targets
-        if self.data_targets is not None:
-            dataset.data_targets = val_targets
+        if self.patch_targets is not None:
+            dataset.patch_targets = val_targets
 
         return dataset
 
@@ -250,7 +276,16 @@ class InMemoryPredictionDataset(Dataset):
     """
     Dataset storing data in memory and allowing generating patches from it.
 
-    # TODO
+    Parameters
+    ----------
+    prediction_config : InferenceConfig
+        Prediction configuration.
+    inputs : np.ndarray
+        Input data.
+    data_target : Optional[np.ndarray], optional
+        Target data, by default None.
+    read_source_func : Optional[Callable], optional
+        Read source function for custom types, by default read_tiff.
     """
 
     def __init__(
@@ -264,10 +299,14 @@ class InMemoryPredictionDataset(Dataset):
 
         Parameters
         ----------
-        array : np.ndarray
-            Array containing the data.
-        axes : str
-            Description of axes in format STCZYX.
+        prediction_config : InferenceConfig
+            Prediction configuration.
+        inputs : np.ndarray
+            Input data.
+        data_target : Optional[np.ndarray], optional
+            Target data, by default None.
+        read_source_func : Optional[Callable], optional
+            Read source function for custom types, by default read_tiff.
 
         Raises
         ------
@@ -295,7 +334,7 @@ class InMemoryPredictionDataset(Dataset):
 
         # get transforms
         self.patch_transform = Compose(
-            transform_list=self.pred_config.transforms,
+            transform_list=[NormalizeModel(mean=self.mean, std=self.std)],
         )
 
     def _prepare_tiles(self) -> List[Tuple[np.ndarray, TileInformation]]:
@@ -310,7 +349,7 @@ class InMemoryPredictionDataset(Dataset):
         # reshape array
         reshaped_sample = reshape_array(self.input_array, self.axes)
 
-        if self.tiling:
+        if self.tiling and self.tile_size is not None and self.tile_overlap is not None:
             # generate patches, which returns a generator
             patch_generator = extract_tiles(
                 arr=reshaped_sample,

careamics/dataset/iterable_dataset.py

@@ -1,3 +1,5 @@
+"""Iterable dataset used to load data file by file."""
+
 from __future__ import annotations
 
 import copy
@@ -11,6 +13,7 @@ from careamics.transforms import Compose
 
 from ..config import DataConfig, InferenceConfig
 from ..config.tile_information import TileInformation
+from ..config.transformations import NormalizeModel
 from ..utils.logging import get_logger
 from .dataset_utils import read_tiff, reshape_array
 from .patching.random_patching import extract_patches_random
@@ -19,13 +22,85 @@ from .patching.tiled_patching import extract_tiles
 logger = get_logger(__name__)
 
 
+def _iterate_over_files(
+    data_config: Union[DataConfig, InferenceConfig],
+    data_files: List[Path],
+    target_files: Optional[List[Path]] = None,
+    read_source_func: Callable = read_tiff,
+) -> Generator[Tuple[np.ndarray, Optional[np.ndarray]], None, None]:
+    """
+    Iterate over data source and yield whole image.
+
+    Parameters
+    ----------
+    data_config : Union[DataConfig, InferenceConfig]
+        Data configuration.
+    data_files : List[Path]
+        List of data files.
+    target_files : Optional[List[Path]]
+        List of target files, by default None.
+    read_source_func : Optional[Callable]
+        Function to read the source, by default read_tiff.
+
+    Yields
+    ------
+    np.ndarray
+        Image.
+    """
+    # When num_workers > 0, each worker process will have a different copy of the
+    # dataset object
+    # Configuring each copy independently to avoid having duplicate data returned
+    # from the workers
+    worker_info = get_worker_info()
+    worker_id = worker_info.id if worker_info is not None else 0
+    num_workers = worker_info.num_workers if worker_info is not None else 1
+
+    # iterate over the files
+    for i, filename in enumerate(data_files):
+        # retrieve file corresponding to the worker id
+        if i % num_workers == worker_id:
+            try:
+                # read data
+                sample = read_source_func(filename, data_config.axes)
+
+                # read target, if available
+                if target_files is not None:
+                    if filename.name != target_files[i].name:
+                        raise ValueError(
+                            f"File {filename} does not match target file "
+                            f"{target_files[i]}. Have you passed sorted "
+                            f"arrays?"
+                        )
+
+                    # read target
+                    target = read_source_func(target_files[i], data_config.axes)
+
+                    yield sample, target
+                else:
+                    yield sample, None
+
+            except Exception as e:
+                logger.error(f"Error reading file {filename}: {e}")
+
+
 class PathIterableDataset(IterableDataset):
     """
     Dataset allowing extracting patches w/o loading whole data into memory.
 
     Parameters
     ----------
-    data_path : Union[str, Path]
+    data_config : DataConfig
+        Data configuration.
+    src_files : List[Path]
+        List of data files.
+    target_files : Optional[List[Path]], optional
+        Optional list of target files, by default None.
+    read_source_func : Callable, optional
+        Read source function for custom types, by default read_tiff.
+
+    Attributes
+    ----------
+    data_path : List[Path]
         Path to the data, must be a directory.
     axes : str
         Description of axes in format STCZYX.
@@ -45,11 +120,24 @@ class PathIterableDataset(IterableDataset):
 
     def __init__(
         self,
-        data_config: Union[DataConfig, InferenceConfig],
+        data_config: DataConfig,
         src_files: List[Path],
         target_files: Optional[List[Path]] = None,
         read_source_func: Callable = read_tiff,
     ) -> None:
+        """Constructors.
+
+        Parameters
+        ----------
+        data_config : DataConfig
+            Data configuration.
+        src_files : List[Path]
+            List of data files.
+        target_files : Optional[List[Path]], optional
+            Optional list of target files, by default None.
+        read_source_func : Callable, optional
+            Read source function for custom types, by default read_tiff.
+        """
         self.data_config = data_config
         self.data_files = src_files
         self.target_files = target_files
@@ -82,7 +170,9 @@ class PathIterableDataset(IterableDataset):
         means, stds = 0, 0
         num_samples = 0
 
-        for sample, _ in self._iterate_over_files():
+        for sample, _ in _iterate_over_files(
+            self.data_config, self.data_files, self.target_files, self.read_source_func
+        ):
             means += sample.mean()
             stds += sample.std()
             num_samples += 1
@@ -97,57 +187,9 @@ class PathIterableDataset(IterableDataset):
         logger.info(f"Mean: {result_mean}, std: {result_std}")
         return result_mean, result_std
 
-    def _iterate_over_files(
-        self,
-    ) -> Generator[Tuple[np.ndarray, Optional[np.ndarray]], None, None]:
-        """
-        Iterate over data source and yield whole image.
-
-        Yields
-        ------
-        np.ndarray
-            Image.
-        """
-        # When num_workers > 0, each worker process will have a different copy of the
-        # dataset object
-        # Configuring each copy independently to avoid having duplicate data returned
-        # from the workers
-        worker_info = get_worker_info()
-        worker_id = worker_info.id if worker_info is not None else 0
-        num_workers = worker_info.num_workers if worker_info is not None else 1
-
-        # iterate over the files
-        for i, filename in enumerate(self.data_files):
-            # retrieve file corresponding to the worker id
-            if i % num_workers == worker_id:
-                try:
-                    # read data
-                    sample = self.read_source_func(filename, self.data_config.axes)
-
-                    # read target, if available
-                    if self.target_files is not None:
-                        if filename.name != self.target_files[i].name:
-                            raise ValueError(
-                                f"File {filename} does not match target file "
-                                f"{self.target_files[i]}. Have you passed sorted "
-                                f"arrays?"
-                            )
-
-                        # read target
-                        target = self.read_source_func(
-                            self.target_files[i], self.data_config.axes
-                        )
-
-                        yield sample, target
-                    else:
-                        yield sample, None
-
-                except Exception as e:
-                    logger.error(f"Error reading file {filename}: {e}")
-
     def __iter__(
         self,
-    ) -> Generator[Tuple[np.ndarray, np.ndarray, Optional[np.ndarray]], None, None]:
+    ) -> Generator[Tuple[np.ndarray, ...], None, None]:
         """
         Iterate over data source and yield single patch.
 
@@ -161,7 +203,9 @@ class PathIterableDataset(IterableDataset):
         ), "Mean and std must be provided"
 
         # iterate over files
-        for sample_input, sample_target in self._iterate_over_files():
+        for sample_input, sample_target in _iterate_over_files(
+            self.data_config, self.data_files, self.target_files, self.read_source_func
+        ):
             reshaped_sample = reshape_array(sample_input, self.data_config.axes)
             reshaped_target = (
                 None
@@ -209,9 +253,9 @@ class PathIterableDataset(IterableDataset):
         Parameters
         ----------
         percentage : float, optional
-            Percentage of files to split up, by default 0.1
+            Percentage of files to split up, by default 0.1.
         minimum_number : int, optional
-            Minimum number of files to split up, by default 5
+            Minimum number of files to split up, by default 5.
 
         Returns
         -------
@@ -275,12 +319,23 @@ class PathIterableDataset(IterableDataset):
         return dataset
 
 
-class IterablePredictionDataset(PathIterableDataset):
+class IterablePredictionDataset(IterableDataset):
     """
-    Dataset allowing extracting patches w/o loading whole data into memory.
+    Prediction dataset.
 
     Parameters
     ----------
+    prediction_config : InferenceConfig
+        Inference configuration.
+    src_files : List[Path]
+        List of data files.
+    read_source_func : Callable, optional
+        Read source function for custom types, by default read_tiff.
+    **kwargs : Any
+        Additional keyword arguments, unused.
+
+    Attributes
+    ----------
     data_path : Union[str, Path]
         Path to the data, must be a directory.
     axes : str
@@ -300,13 +355,26 @@ class IterablePredictionDataset(PathIterableDataset):
         read_source_func: Callable = read_tiff,
         **kwargs: Any,
     ) -> None:
-        super().__init__(
-            data_config=prediction_config,
-            src_files=src_files,
-            read_source_func=read_source_func,
-        )
+        """Constructor.
 
+        Parameters
+        ----------
+        prediction_config : InferenceConfig
+            Inference configuration.
+        src_files : List[Path]
+            List of data files.
+        read_source_func : Callable, optional
+            Read source function for custom types, by default read_tiff.
+        **kwargs : Any
+            Additional keyword arguments, unused.
+
+        Raises
+        ------
+        ValueError
+            If mean and std are not provided in the inference configuration.
+        """
         self.prediction_config = prediction_config
+        self.data_files = src_files
         self.axes = prediction_config.axes
         self.tile_size = self.prediction_config.tile_size
         self.tile_overlap = self.prediction_config.tile_overlap
@@ -315,10 +383,21 @@ class IterablePredictionDataset(PathIterableDataset):
         # tile only if both tile size and overlaps are provided
         self.tile = self.tile_size is not None and self.tile_overlap is not None
 
-        # get tta transforms
-        self.patch_transform = Compose(
-            transform_list=prediction_config.transforms,
-        )
+        # check mean and std and create normalize transform
+        if self.prediction_config.mean is None or self.prediction_config.std is None:
+            raise ValueError("Mean and std must be provided for prediction.")
+        else:
+            self.mean = self.prediction_config.mean
+            self.std = self.prediction_config.std
+
+            # instantiate normalize transform
+            self.patch_transform = Compose(
+                transform_list=[
+                    NormalizeModel(
+                        mean=prediction_config.mean, std=prediction_config.std
+                    )
+                ],
+            )
 
     def __iter__(
         self,
@@ -335,11 +414,19 @@ class IterablePredictionDataset(PathIterableDataset):
             self.mean is not None and self.std is not None
         ), "Mean and std must be provided"
 
-        for sample, _ in self._iterate_over_files():
+        for sample, _ in _iterate_over_files(
+            self.prediction_config,
+            self.data_files,
+            read_source_func=self.read_source_func,
+        ):
             # reshape array
             reshaped_sample = reshape_array(sample, self.axes)
 
-            if self.tile:
+            if (
+                self.tile
+                and self.tile_size is not None
+                and self.tile_overlap is not None
+            ):
                 # generate patches, return a generator
                 patch_gen = extract_tiles(
                     arr=reshaped_sample,

careamics/dataset/patching/patching.py

@@ -1,8 +1,4 @@
-"""
-Tiling submodule.
-
-These functions are used to tile images into patches or tiles.
-"""
+"""Patching functions."""
 
 from pathlib import Path
 from typing import Callable, List, Tuple, Union
@@ -21,12 +17,25 @@ def prepare_patches_supervised(
     train_files: List[Path],
     target_files: List[Path],
     axes: str,
-    patch_size: Union[List[int], Tuple[int]],
+    patch_size: Union[List[int], Tuple[int, ...]],
     read_source_func: Callable,
 ) -> Tuple[np.ndarray, np.ndarray, float, float]:
     """
     Iterate over data source and create an array of patches and corresponding targets.
 
+    Parameters
+    ----------
+    train_files : List[Path]
+        List of paths to training data.
+    target_files : List[Path]
+        List of paths to target data.
+    axes : str
+        Axes of the data.
+    patch_size : Union[List[int], Tuple[int]]
+        Size of the patches.
+    read_source_func : Callable
+        Function to read the data.
+
     Returns
     -------
     np.ndarray
@@ -95,13 +104,25 @@ def prepare_patches_unsupervised(
     patch_size: Union[List[int], Tuple[int]],
     read_source_func: Callable,
 ) -> Tuple[np.ndarray, None, float, float]:
-    """
-    Iterate over data source and create an array of patches.
+    """Iterate over data source and create an array of patches.
+
+    This method returns the mean and standard deviation of the image.
+
+    Parameters
+    ----------
+    train_files : List[Path]
+        List of paths to training data.
+    axes : str
+        Axes of the data.
+    patch_size : Union[List[int], Tuple[int]]
+        Size of the patches.
+    read_source_func : Callable
+        Function to read the data.
 
     Returns
     -------
-    np.ndarray
-        Array of patches.
+    Tuple[np.ndarray, None, float, float]
+        Source and target patches, mean and standard deviation.
     """
     means, stds, num_samples = 0, 0, 0
     all_patches = []
@@ -150,10 +171,21 @@ def prepare_patches_supervised_array(
 
     Patches returned are of shape SC(Z)YX, where S is now the patches dimension.
 
+    Parameters
+    ----------
+    data : np.ndarray
+        Input data array.
+    axes : str
+        Axes of the data.
+    data_target : np.ndarray
+        Target data array.
+    patch_size : Union[List[int], Tuple[int]]
+        Size of the patches.
+
     Returns
     -------
-    np.ndarray
-        Array of patches.
+    Tuple[np.ndarray, np.ndarray, float, float]
+        Source and target patches, mean and standard deviation.
     """
     # compute statistics
     mean = data.mean()
@@ -195,10 +227,19 @@ def prepare_patches_unsupervised_array(
 
     Patches returned are of shape SC(Z)YX, where S is now the patches dimension.
 
+    Parameters
+    ----------
+    data : np.ndarray
+        Input data array.
+    axes : str
+        Axes of the data.
+    patch_size : Union[List[int], Tuple[int]]
+        Size of the patches.
+
     Returns
     -------
-    np.ndarray
-        Array of patches.
+    Tuple[np.ndarray, None, float, float]
+        Source patches, mean and standard deviation.
     """
     # calculate mean and std
     mean = data.mean()
@@ -210,4 +251,4 @@ def prepare_patches_unsupervised_array(
     # generate patches, return a generator
     patches, _ = extract_patches_sequential(reshaped_sample, patch_size=patch_size)
 
-    return patches, _, mean, std  # TODO inelegant, replace  by dataclass?
+    return patches, _, mean, std  # TODO inelegant, replace by dataclass?

careamics/dataset/patching/random_patching.py

@@ -1,3 +1,5 @@
+"""Random patching utilities."""
+
 from typing import Generator, List, Optional, Tuple, Union
 
 import numpy as np
@@ -30,6 +32,8 @@ def extract_patches_random(
         Input image array.
     patch_size : Tuple[int]
         Patch sizes in each dimension.
+    target : Optional[np.ndarray], optional
+        Target array, by default None.
 
     Yields
     ------
@@ -120,10 +124,12 @@ def extract_patches_random_from_chunks(
     ----------
     arr : np.ndarray
         Input image array.
-    patch_size : Tuple[int]
+    patch_size : Union[List[int], Tuple[int, ...]]
         Patch sizes in each dimension.
-    chunk_size : Tuple[int]
+    chunk_size : Union[List[int], Tuple[int, ...]]
         Chunk sizes to load from the.
+    chunk_limit : Optional[int], optional
+        Number of chunks to load, by default None.
 
     Yields
     ------