junifer 0.0.7.dev105__py3-none-any.whl → 0.0.7.dev121__py3-none-any.whl

junifer/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.7.dev105'
-__version_tuple__ = version_tuple = (0, 0, 7, 'dev105')
+__version__ = version = '0.0.7.dev121'
+__version_tuple__ = version_tuple = (0, 0, 7, 'dev121')

junifer/api/decorators.py

@@ -6,8 +6,13 @@
 # License: AGPL
 
 from ..data import DataDispatcher
-from ..pipeline import PipelineComponentRegistry
+from ..pipeline import (
+    AssetDumperDispatcher,
+    AssetLoaderDispatcher,
+    PipelineComponentRegistry,
+)
 from ..typing import (
+    DataDumpAssetLike,
     DataGrabberLike,
     DataRegistryLike,
     MarkerLike,
@@ -17,6 +22,7 @@ from ..typing import (
 
 
 __all__ = [
+    "register_data_dump_asset",
     "register_data_registry",
     "register_datagrabber",
     "register_datareader",
@@ -184,3 +190,49 @@ def register_data_registry(name: str) -> DataRegistryLike:
         return klass
 
     return decorator
+
+
+def register_data_dump_asset(
+    types: list[type], exts: list[str]
+) -> DataDumpAssetLike:
+    """Asset registration decorator.
+
+    Registers the data dump asset for ``types`` with ``exts``.
+
+    Parameters
+    ----------
+    types : list of class
+        The classes to dump.
+    exts : list of str
+        The extensions to load.
+
+    Returns
+    -------
+    class
+        The unmodified input class.
+
+    """
+
+    def decorator(klass: DataDumpAssetLike) -> DataDumpAssetLike:
+        """Actual decorator.
+
+        Parameters
+        ----------
+        klass : class
+            The class of the data dump asset to register.
+
+        Returns
+        -------
+        class
+            The unmodified input class.
+
+        """
+        # Add asset dumper
+        for t in types:
+            AssetDumperDispatcher()[t] = klass
+        # Add asset loader
+        for e in exts:
+            AssetLoaderDispatcher()[e] = klass
+        return klass
+
+    return decorator

junifer/api/tests/test_decorators.py

@@ -3,8 +3,18 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from junifer.api.decorators import register_data_registry
+import pickle
+
+from junifer.api.decorators import (
+    register_data_dump_asset,
+    register_data_registry,
+)
 from junifer.data import BasePipelineDataRegistry, DataDispatcher
+from junifer.pipeline import (
+    AssetDumperDispatcher,
+    AssetLoaderDispatcher,
+    BaseDataDumpAsset,
+)
 
 
 def test_register_data_registry() -> None:
@@ -30,3 +40,39 @@ def test_register_data_registry() -> None:
     assert "dumb" in DataDispatcher()
     _ = DataDispatcher().pop("dumb")
     assert "dumb" not in DataDispatcher()
+
+
+def test_register_data_dump_asset() -> None:
+    """Test data dump asset registration."""
+
+    class Int(int): ...
+
+    class Float(float): ...
+
+    @register_data_dump_asset([Int, Float], [".int", ".float"])
+    class DumAsset(BaseDataDumpAsset):
+        def dump(self):
+            suffix = ""
+            if isinstance(self.data, Int):
+                suffix = ".int"
+            else:
+                suffix = ".float"
+            pickle.dump(self.data, self.path_without_ext.with_suffix(suffix))
+
+        @classmethod
+        def load(cls, path):
+            return pickle.load(path)
+
+    assert Int in AssetDumperDispatcher()
+    assert Float in AssetDumperDispatcher()
+    _ = AssetDumperDispatcher().pop(Int)
+    _ = AssetDumperDispatcher().pop(Float)
+    assert Int not in AssetDumperDispatcher()
+    assert Float not in AssetDumperDispatcher()
+
+    assert ".int" in AssetLoaderDispatcher()
+    assert ".float" in AssetLoaderDispatcher()
+    _ = AssetLoaderDispatcher().pop(".int")
+    _ = AssetLoaderDispatcher().pop(".float")
+    assert ".int" not in AssetLoaderDispatcher()
+    assert ".float" not in AssetLoaderDispatcher()

junifer/onthefly/read_transform.py

@@ -5,6 +5,7 @@
 
 from typing import Optional
 
+import numpy as np
 import pandas as pd
 
 from ..typing import StorageLike
@@ -19,6 +20,7 @@ def read_transform(
     transform: str,
     feature_name: Optional[str] = None,
     feature_md5: Optional[str] = None,
+    nan_policy: Optional[str] = "bypass",
     transform_args: Optional[tuple] = None,
     transform_kw_args: Optional[dict] = None,
 ) -> pd.DataFrame:
@@ -35,6 +37,16 @@ def read_transform(
         Name of the feature to read (default None).
     feature_md5 : str, optional
         MD5 hash of the feature to read (default None).
+    nan_policy : str, optional
+        The policy to handle NaN values (default "ignore").
+        Options are:
+
+        * "bypass": Do nothing and pass NaN values to the transform function.
+        * "drop_element": Drop (skip) elements with NaN values.
+        * "drop_rows": Drop (skip) rows with NaN values.
+        * "drop_columns": Drop (skip) columns with NaN values.
+        * "drop_symmetric": Drop (skip) symmetric pairs with NaN values.
+
     transform_args : tuple, optional
         The positional arguments for the callable of ``transform``
         (default None).
@@ -47,6 +59,18 @@ def read_transform(
     pandas.DataFrame
         The transformed feature as a dataframe.
 
+    Raises
+    ------
+    ValueError
+        If ``nan_policy`` is invalid or
+        if *package* is invalid.
+    RuntimeError
+        If *package* is ``bctpy`` and stored data kind is not ``matrix``.
+    ImportError
+        If ``bctpy`` cannot be imported.
+    AttributeError
+        If *function* to be invoked in invalid.
+
     Notes
     -----
     This function has been only tested for:
@@ -63,6 +87,18 @@ def read_transform(
     transform_args = transform_args or ()
     transform_kw_args = transform_kw_args or {}
 
+    if nan_policy not in [
+        "bypass",
+        "drop_element",
+        "drop_rows",
+        "drop_columns",
+        "drop_symmetric",
+    ]:
+        raise_error(
+            f"Unknown nan_policy: {nan_policy}",
+            klass=ValueError,
+        )
+
     # Read storage
     stored_data = storage.read(
         feature_name=feature_name, feature_md5=feature_md5
@@ -106,22 +142,52 @@ def read_transform(
         except AttributeError as err:
             raise_error(msg=str(err), klass=AttributeError)
 
-        # Apply function and store subject-wise
+        # Apply function and store element-wise
         output_list = []
+        element_list = []
         logger.debug(
             f"Computing '{package}.{func_str}' for feature "
             f"{feature_name or feature_md5} ..."
         )
-        for subject in range(stored_data["data"].shape[2]):
+        for i_element, element in enumerate(stored_data["element"]):
+            t_data = stored_data["data"][:, :, i_element]
+            has_nan = np.isnan(np.min(t_data))
+            if nan_policy == "drop_element" and has_nan:
+                logger.debug(
+                    f"Skipping element {element} due to NaN values ..."
+                )
+                continue
+            elif nan_policy == "drop_rows" and has_nan:
+                logger.debug(
+                    f"Skipping rows with NaN values in element {element} ..."
+                )
+                t_data = t_data[~np.isnan(t_data).any(axis=1)]
+            elif nan_policy == "drop_columns" and has_nan:
+                logger.debug(
+                    f"Skipping columns with NaN values in element {element} "
+                    "..."
+                )
+                t_data = t_data[:, ~np.isnan(t_data).any(axis=0)]
+            elif nan_policy == "drop_symmetric":
+                logger.debug(
+                    f"Skipping pairs of rows/columns with NaN values in "
+                    f"element {element}..."
+                )
+                good_rows = ~np.isnan(t_data).any(axis=1)
+                good_columns = ~np.isnan(t_data).any(axis=0)
+                good_idx = np.logical_and(good_rows, good_columns)
+                t_data = t_data[good_idx][:, good_idx]
+
             output = func(
-                stored_data["data"][:, :, subject],
+                t_data,
                 *transform_args,
                 **transform_kw_args,
             )
             output_list.append(output)
+            element_list.append(element)
 
         # Create dataframe for index
-        idx_df = pd.DataFrame(data=stored_data["element"])
+        idx_df = pd.DataFrame(data=element_list)
         # Create multiindex from dataframe
         logger.debug(
             "Generating pandas.MultiIndex for feature "

junifer/onthefly/tests/test_read_transform.py

@@ -64,6 +64,36 @@ def matrix_storage(tmp_path: Path) -> HDF5FeatureStorage:
     return storage
 
 
+@pytest.fixture
+def matrix_storage_with_nan(tmp_path: Path) -> HDF5FeatureStorage:
+    """Return a HDF5FeatureStorage with matrix data.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    storage = HDF5FeatureStorage(tmp_path / "matrix_store_nan.hdf5")
+    data = np.arange(36).reshape(3, 3, 4).astype(float)
+    data[1, 1, 2] = np.nan
+    data[1, 2, 2] = np.nan
+    for i in range(4):
+        storage.store(
+            kind="matrix",
+            meta={
+                "element": {"subject": f"test{i + 1}"},
+                "dependencies": [],
+                "marker": {"name": "matrix"},
+                "type": "BOLD",
+            },
+            data=data[:, :, i],
+            col_names=["f1", "f2", "f3"],
+            row_names=["g1", "g2", "g3"],
+        )
+    return storage
+
+
 def test_incorrect_package(matrix_storage: HDF5FeatureStorage) -> None:
     """Test error check for incorrect package name.
 
@@ -176,3 +206,57 @@ def test_bctpy_function(
         )
         assert "Computing" in caplog.text
         assert "Generating" in caplog.text
+
+
+@pytest.mark.parametrize(
+    "nan_policy, error_msg",
+    [
+        ("drop_element", None),
+        ("drop_rows", "square"),
+        ("drop_columns", "square"),
+        ("drop_symmetric", None),
+        ("bypass", "NaNs"),
+        ("wrong", "Unknown"),
+    ],
+)
+def test_bctpy_nans(
+    matrix_storage_with_nan: HDF5FeatureStorage,
+    caplog: pytest.LogCaptureFixture,
+    nan_policy: str,
+    error_msg: str,
+) -> None:
+    """Test working function of bctpy.
+
+    Parameters
+    ----------
+    matrix_storage_with_nan : HDF5FeatureStorage
+        The HDF5FeatureStorage with matrix data, as fixture.
+    caplog : pytest.LogCaptureFixture
+        The pytest.LogCaptureFixture object.
+    nan_policy : str
+        The NAN policy to test.
+    error_msg : str
+        The expected error message snippet. If None, no error should be raised.
+
+    """
+    # Skip test if import fails
+    pytest.importorskip("bct")
+
+    with caplog.at_level(logging.DEBUG):
+        if error_msg is None:
+            read_transform(
+                storage=matrix_storage_with_nan,  # type: ignore
+                feature_name="BOLD_matrix",
+                transform="bctpy_eigenvector_centrality_und",
+                nan_policy=nan_policy,
+            )
+            assert "Computing" in caplog.text
+            assert "Generating" in caplog.text
+        else:
+            with pytest.raises(ValueError, match=error_msg):
+                read_transform(
+                    storage=matrix_storage_with_nan,  # type: ignore
+                    feature_name="BOLD_matrix",
+                    transform="bctpy_eigenvector_centrality_und",
+                    nan_policy=nan_policy,
+                )

junifer/pipeline/__init__.pyi

@@ -1,4 +1,8 @@
 __all__ = [
+    "AssetDumperDispatcher",
+    "AssetLoaderDispatcher",
+    "BaseDataDumpAsset",
+    "DataObjectDumper",
     "PipelineComponentRegistry",
     "PipelineStepMixin",
     "UpdateMetaMixin",
@@ -6,6 +10,12 @@ __all__ = [
     "MarkerCollection",
 ]
 
+from ._data_object_dumper import (
+    AssetDumperDispatcher,
+    AssetLoaderDispatcher,
+    BaseDataDumpAsset,
+    DataObjectDumper,
+)
 from .pipeline_component_registry import PipelineComponentRegistry
 from .pipeline_step_mixin import PipelineStepMixin
 from .update_meta_mixin import UpdateMetaMixin

junifer/pipeline/_data_object_dumper.py

@@ -0,0 +1,347 @@
+"""Provide pipeline data object dumper and data dump asset classes."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterator, MutableMapping
+from copy import deepcopy
+from pathlib import Path
+from typing import Any
+
+import nibabel
+import pandas
+
+from ..utils import raise_error, yaml
+
+
+__all__ = [
+    "AssetDumperDispatcher",
+    "AssetLoaderDispatcher",
+    "BaseDataDumpAsset",
+    "DataObjectDumper",
+]
+
+
+class BaseDataDumpAsset(ABC):
+    """Abstract base class for a data dump asset.
+
+    Parameters
+    ----------
+    data : Any
+        Data to save.
+    path_without_ext : pathlib.Path
+        Path to the asset without extension.
+        The subclass should add the extension when saving.
+
+    """
+
+    def __init__(self, data: Any, path_without_ext: Path) -> None:
+        """Initialize the class."""
+        self.data = data
+        self.path_without_ext = path_without_ext
+
+    @abstractmethod
+    def dump(self) -> None:
+        """Dump asset."""
+        raise_error(
+            msg="Concrete classes need to implement dump().",
+            klass=NotImplementedError,
+        )
+
+    @classmethod
+    @abstractmethod
+    def load(cls: type["BaseDataDumpAsset"], path: Path) -> Any:
+        """Load asset from path."""
+        raise_error(
+            msg="Concrete classes need to implement load().",
+            klass=NotImplementedError,
+        )
+
+
+class Nifti1ImageAsset(BaseDataDumpAsset):
+    """Class for ``nibabel.Nifti1Image`` dumper."""
+
+    def dump(self) -> None:
+        nibabel.save(self.data, self.path_without_ext.with_suffix(".nii.gz"))
+
+    @classmethod
+    def load(cls: "Nifti1ImageAsset", path: Path) -> nibabel.Nifti1Image:
+        return nibabel.load(path)
+
+
+class PandasDataFrameAsset(BaseDataDumpAsset):
+    """Class for ``pandas.DataFrame`` dumper."""
+
+    def dump(self) -> None:
+        self.data.to_csv(self.path_without_ext.with_suffix(".csv"))
+
+    @classmethod
+    def load(cls: "PandasDataFrameAsset", path: Path) -> pandas.DataFrame:
+        return pandas.read_csv(path, index_col=0)
+
+
+class AssetDumperDispatcher(MutableMapping):
+    """Class for helping dynamic asset dumper dispatch."""
+
+    _instance = None
+
+    def __new__(cls):
+        # Make class singleton
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            # Set dumpers
+            cls._dumpers: dict[type, type[BaseDataDumpAsset]] = {}
+            cls._builtin: dict[type, type[BaseDataDumpAsset]] = {}
+            cls._external: dict[type, type[BaseDataDumpAsset]] = {}
+            cls._builtin.update(
+                {
+                    nibabel.Nifti1Image: Nifti1ImageAsset,
+                    pandas.DataFrame: PandasDataFrameAsset,
+                }
+            )
+            cls._dumpers.update(cls._builtin)
+        return cls._instance
+
+    def __getitem__(self, key: type) -> type[BaseDataDumpAsset]:
+        return self._dumpers[key]
+
+    def __iter__(self) -> Iterator[type]:
+        return iter(self._dumpers)
+
+    def __len__(self) -> int:
+        return len(self._dumpers)
+
+    def __delitem__(self, key: type) -> None:
+        # Internal check
+        if key in self._builtin:
+            raise_error(f"Cannot delete in-built key: {key}")
+        # Non-existing key
+        if key not in self._external:
+            raise_error(klass=KeyError, msg=str(key))
+        # Update external
+        _ = self._external.pop(key)
+        # Update global
+        _ = self._dumpers.pop(key)
+
+    def __setitem__(self, key: type, value: type[BaseDataDumpAsset]) -> None:
+        # Internal check
+        if key in self._builtin:
+            raise_error(f"Cannot set value for in-built key: {key}")
+        # Value type check
+        if not issubclass(value, BaseDataDumpAsset):
+            raise_error(f"Invalid value type: {type(value)}")
+        # Update external
+        self._external[key] = value
+        # Update global
+        self._dumpers[key] = value
+
+    def popitem():
+        """Not implemented."""
+        pass
+
+    def clear(self):
+        """Not implemented."""
+        pass
+
+    def setdefault(self, key: type, value=None):
+        """Not implemented."""
+        pass
+
+
+class AssetLoaderDispatcher(MutableMapping):
+    """Class for helping dynamic asset loader dispatch."""
+
+    _instance = None
+
+    def __new__(cls):
+        # Make class singleton
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            # Set loaders
+            cls._loaders: dict[str, type[BaseDataDumpAsset]] = {}
+            cls._builtin: dict[str, type[BaseDataDumpAsset]] = {}
+            cls._external: dict[str, type[BaseDataDumpAsset]] = {}
+            cls._builtin.update(
+                {
+                    ".nii.gz": Nifti1ImageAsset,
+                    ".nii": Nifti1ImageAsset,
+                    ".csv": PandasDataFrameAsset,
+                }
+            )
+            cls._loaders.update(cls._builtin)
+        return cls._instance
+
+    def __getitem__(self, key: str) -> type[BaseDataDumpAsset]:
+        return self._loaders[key]
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._loaders)
+
+    def __len__(self) -> int:
+        return len(self._loaders)
+
+    def __delitem__(self, key: str) -> None:
+        # Internal check
+        if key in self._builtin:
+            raise_error(f"Cannot delete in-built key: {key}")
+        # Non-existing key
+        if key not in self._external:
+            raise_error(klass=KeyError, msg=key)
+        # Update external
+        _ = self._external.pop(key)
+        # Update global
+        _ = self._loaders.pop(key)
+
+    def __setitem__(self, key: str, value: type[BaseDataDumpAsset]) -> None:
+        # Internal check
+        if key in self._builtin:
+            raise_error(f"Cannot set value for in-built key: {key}")
+        # Value type check
+        if not issubclass(value, BaseDataDumpAsset):
+            raise_error(f"Invalid value type: {type(value)}")
+        # Update external
+        self._external[key] = value
+        # Update global
+        self._loaders[key] = value
+
+    def popitem():
+        """Not implemented."""
+        pass
+
+    def clear(self):
+        """Not implemented."""
+        pass
+
+    def setdefault(self, key: str, value=None):
+        """Not implemented."""
+        pass
+
+
+class DataObjectDumper:
+    """Class for pipeline data object dumping."""
+
+    _instance = None
+
+    def __new__(cls):
+        """Overridden to make the class singleton."""
+        # Make class singleton
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def dump(self, data: dict, path: Path, step: str) -> None:
+        """Dump data object at path.
+
+        Parameters
+        ----------
+        data : dict
+            The data object state to dump.
+        path : pathlib.Path
+            The path to dump the data object.
+        step : str
+            The step name. Also sets the dump directory.
+
+        """
+        # Make a deep copy of data
+        data_copy = deepcopy(data)
+        # Initialize list for storing assets to save
+        assets = []
+
+        dump_file_root = path / step
+
+        for k, v in data_copy.items():
+            # Conditional for Warp type; kept separate for low cognitive load
+            if isinstance(v, list):
+                for idx, _ in enumerate(v):
+                    data_copy[k][idx]["path"] = str(data_copy[k][idx]["path"])
+                continue
+
+            # Transform Path to str
+            data_copy[k]["path"] = str(data_copy[k]["path"])
+            # Pop out first level assets; some data types might not have
+            if "data" in v:
+                dumper = AssetDumperDispatcher()[type(v["data"])]
+                assets.append(
+                    dumper(
+                        data=v.pop("data"),
+                        path_without_ext=dump_file_root / k,
+                    )
+                )
+            for kk, vv in v.items():
+                if isinstance(vv, dict) and kk != "meta":
+                    # Transform Path to str
+                    data_copy[k][kk]["path"] = str(data_copy[k][kk]["path"])
+                    # Pop out second level assets
+                    if "data" in vv:
+                        dumper = AssetDumperDispatcher()[type(vv["data"])]
+                        assets.append(
+                            dumper(
+                                data=vv.pop("data"),
+                                path_without_ext=dump_file_root / f"{k}_{kk}",
+                            )
+                        )
+
+        # Save yaml
+        dump_file_path = dump_file_root / "data.yaml"
+        dump_file_path.parent.mkdir(parents=True, exist_ok=True)
+        yaml.dump(data_copy, stream=dump_file_path)
+
+        # Save assets
+        for x in assets:
+            x.dump()
+
+    def load(self, path: Path) -> dict:
+        """Load data object from path.
+
+        Parameters
+        ----------
+        path : pathlib.Path
+            The path to the dumped data object.
+
+        Returns
+        -------
+        dict
+            The restored data object dump.
+
+        """
+        data = yaml.load(path)
+        # Load assets; stem => path mapping
+        assets = {
+            child.stem.split(".")[0]: child
+            for child in path.parent.iterdir()
+            if "".join(child.suffixes) in AssetLoaderDispatcher()
+        }
+
+        for k, v in data.items():
+            # Conditional for Warp type; kept separate for low cognitive load
+            if isinstance(v, list):
+                for idx, _ in enumerate(v):
+                    data[k][idx]["path"] = Path(data[k][idx]["path"])
+                continue
+
+            # Transform str to Path
+            data[k]["path"] = Path(data[k]["path"])
+            # Insert first level assets if matching asset is found
+            if k in assets:
+                # Get path
+                p = assets[k]
+                data[k]["path"] = p
+                # Get correct loader using extension
+                loader = AssetLoaderDispatcher()["".join(p.suffixes)]
+                data[k]["data"] = loader.load(p)
+            for kk, vv in v.items():
+                if isinstance(vv, dict) and kk != "meta":
+                    # Transform str to Path
+                    data[k][kk]["path"] = Path(data[k][kk]["path"])
+                    # Insert second level assets
+                    key = f"{k}_{kk}"
+                    if key in assets:
+                        # Get path
+                        pp = assets[key]
+                        data[k][kk]["path"] = pp
+                        # Get correct loader using extension
+                        loader = AssetLoaderDispatcher()["".join(pp.suffixes)]
+                        data[k][kk]["data"] = loader.load(pp)
+
+        return data

junifer/pipeline/marker_collection.py

@@ -5,12 +5,13 @@
 # License: AGPL
 
 from collections import Counter
+from pathlib import Path
 from typing import Optional
 
 from ..datareader import DefaultDataReader
-from ..pipeline import PipelineStepMixin, WorkDirManager
+from ..pipeline import DataObjectDumper, PipelineStepMixin, WorkDirManager
 from ..typing import DataGrabberLike, MarkerLike, PreprocessorLike, StorageLike
-from ..utils import logger, raise_error
+from ..utils import config, logger, raise_error
 
 
 __all__ = ["MarkerCollection"]
@@ -80,16 +81,53 @@ class MarkerCollection:
 
         # Fetch actual data using datareader
         data = self._datareader.fit_transform(input)
+        # Conditional data dump
+        if (
+            config.get("preprocessing.dump.location") is not None
+            and config.get("preprocessing.dump.granularity") == "full"
+        ):
+            DataObjectDumper().dump(
+                data=data,
+                path=Path(config.get("preprocessing.dump.location")),
+                step=f"0_datareader_{self._datareader.__class__.__name__}",
+            )
 
         # Apply preprocessing steps
         if self._preprocessors is not None:
-            for preprocessor in self._preprocessors:
+            for idx, preprocessor in enumerate(self._preprocessors):
                 logger.info(
                     "Preprocessing data with "
                     f"{preprocessor.__class__.__name__}"
                 )
                 # Mutate data after every iteration
                 data = preprocessor.fit_transform(data)
+                # Conditional data dump
+                if (
+                    config.get("preprocessing.dump.location") is not None
+                    and config.get("preprocessing.dump.granularity") == "full"
+                ):
+                    DataObjectDumper().dump(
+                        data=data,
+                        path=Path(config.get("preprocessing.dump.location")),
+                        step=(
+                            f"{idx + 1}_preprocessor_"
+                            f"{preprocessor.__class__.__name__}"
+                        ),
+                    )
+
+            # Conditional data dump
+            if (
+                config.get("preprocessing.dump.location") is not None
+                and config.get("preprocessing.dump.granularity") == "final"
+            ):
+                DataObjectDumper().dump(
+                    data=data,
+                    path=Path(config.get("preprocessing.dump.location")),
+                    step=(
+                        f"final_preprocessor_"
+                        f"{self._preprocessors[-1].__class__.__name__}"
+                    ),
+                )
 
         # Compute markers
         out = {}

junifer/pipeline/tests/test_data_object_dumper.py

@@ -0,0 +1,225 @@
+"""Provide tests for data object dumping."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+import pickle
+from pathlib import Path
+from typing import Union
+
+import nibabel
+import pytest
+
+from junifer.markers import FunctionalConnectivitySpheres
+from junifer.pipeline import (
+    AssetDumperDispatcher,
+    AssetLoaderDispatcher,
+    BaseDataDumpAsset,
+    DataObjectDumper,
+    MarkerCollection,
+)
+from junifer.preprocess import fMRIPrepConfoundRemover
+from junifer.testing.datagrabbers import PartlyCloudyTestingDataGrabber
+from junifer.utils import config
+
+
+@pytest.mark.parametrize(
+    "dispatcher, inbuilt_key, ext_key, val",
+    [
+        (
+            AssetDumperDispatcher,
+            nibabel.Nifti1Image,
+            nibabel.Nifti2Image,
+            dict,
+        ),
+        (AssetLoaderDispatcher, ".nii", ".tsv", dict),
+    ],
+)
+def test_dispatcher_addition_errors(
+    dispatcher: Union[AssetDumperDispatcher, AssetLoaderDispatcher],
+    inbuilt_key: Union[str, type],
+    ext_key: Union[str, type],
+    val: type,
+) -> None:
+    """Test asset dumper / loader addition errors.
+
+    Parameters
+    ----------
+    dispatcher : AssetDumperDispatcher or AssetLoaderDispatcher,
+        The parametrized dispatcher.
+    inbuilt_key : str or type
+        The parametrized in-built key.
+    ext_key : str or type
+        The parametrized external key.
+    val : type
+        The parametrized value.
+
+    """
+    with pytest.raises(ValueError, match="Cannot set"):
+        dispatcher()[inbuilt_key] = val
+
+    with pytest.raises(ValueError, match="Invalid"):
+        dispatcher()[ext_key] = val
+
+
+@pytest.mark.parametrize(
+    "dispatcher, inbuilt_key, ext_key",
+    [
+        (AssetDumperDispatcher, nibabel.Nifti1Image, nibabel.Nifti2Image),
+        (AssetLoaderDispatcher, ".nii", ".tsv"),
+    ],
+)
+def test_dispatcher_removal_errors(
+    dispatcher: Union[AssetDumperDispatcher, AssetLoaderDispatcher],
+    inbuilt_key: Union[str, type],
+    ext_key: Union[str, type],
+) -> None:
+    """Test asset dumper / loader removal errors.
+
+    Parameters
+    ----------
+    dispatcher : AssetDumperDispatcher or AssetLoaderDispatcher,
+        The parametrized dispatcher.
+    inbuilt_key : str or type
+        The parametrized in-built key.
+    ext_key : str or type
+        The parametrized external key.
+
+    """
+    with pytest.raises(ValueError, match="Cannot delete"):
+        _ = dispatcher().pop(inbuilt_key)
+
+    with pytest.raises(KeyError, match=f"{ext_key}"):
+        del dispatcher()[ext_key]
+
+
+def test_dispatcher() -> None:
+    """Test asset dumper / loader addition and removal."""
+
+    class Int(int): ...
+
+    class Float(float): ...
+
+    class DumAsset(BaseDataDumpAsset):
+        def dump(self):
+            suffix = ""
+            if isinstance(self.data, Int):
+                suffix = ".int"
+            else:
+                suffix = ".float"
+            pickle.dump(self.data, self.path_without_ext.with_suffix(suffix))
+
+        @classmethod
+        def load(cls, path):
+            return pickle.load(path)
+
+    AssetDumperDispatcher().update({nibabel.Nifti2Image: DumAsset})
+    assert nibabel.Nifti2Image in AssetDumperDispatcher()
+    _ = AssetDumperDispatcher().pop(nibabel.Nifti2Image)
+    assert nibabel.Nifti2Image not in AssetDumperDispatcher()
+
+    AssetLoaderDispatcher().update({".n+2": DumAsset})
+    assert ".n+2" in AssetLoaderDispatcher()
+    _ = AssetLoaderDispatcher().pop(".n+2")
+    assert ".n+2" not in AssetLoaderDispatcher()
+
+
+@pytest.mark.parametrize(
+    "granularity, expected_dir_count",
+    [
+        ("full", 2),
+        ("final", 1),
+    ],
+)
+def test_data_object_dumper(
+    tmp_path: Path, granularity: str, expected_dir_count: int
+) -> None:
+    """Test data object dumper.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    granularity : str
+        The parametrized granularity.
+    expected_dir_count : int
+        The parametrized expected directory count.
+
+    """
+    config.set(key="preprocessing.dump.location", val=tmp_path)
+    config.set(key="preprocessing.dump.granularity", val=granularity)
+
+    mc = MarkerCollection(
+        preprocessors=[
+            fMRIPrepConfoundRemover(
+                strategy={
+                    "motion": "full",
+                    "wm_csf": "full",
+                },
+                detrend=True,
+                standardize=True,
+                low_pass=0.08,
+                high_pass=0.01,
+            ),
+        ],
+        markers=[
+            FunctionalConnectivitySpheres(
+                name="dmnbuckner_5mm_fc_spheres",
+                coords="DMNBuckner",
+                radius=5.0,
+                conn_method="correlation",
+            ),
+        ],
+    )
+    dg = PartlyCloudyTestingDataGrabber()
+
+    with dg:
+        mc.fit(dg["sub-01"])
+
+    dirs = list(tmp_path.iterdir())
+    assert len(dirs) == expected_dir_count
+
+    dump_load = DataObjectDumper().load(dirs[-1] / "data.yaml")
+    assert "BOLD" in dump_load
+
+    config.delete("preprocessing.dump.location")
+    config.delete("preprocessing.dump.granularity")
+
+
+def test_data_object_dumper_with_warp(tmp_path: Path) -> None:
+    """Test data object dumper with Warp data type.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    DataObjectDumper().dump(
+        data={
+            "Warp": [
+                {
+                    "path": (
+                        tmp_path / "from-MNI152NLin2009cAsym_to-T1w_"
+                        "mode-image_xfm.h5"
+                    ),
+                    "src": "MNI152NLin2009cAsym",
+                    "dst": "native",
+                    "warper": "ants",
+                },
+                {
+                    "path": (
+                        tmp_path / "from-T1w_to-MNI152NLin2009cAsym_"
+                        "mode-image_xfm.h5"
+                    ),
+                    "src": "native",
+                    "dst": "MNI152NLin2009cAsym",
+                    "warper": "ants",
+                },
+            ],
+        },
+        path=tmp_path,
+        step="warp_test",
+    )
+    dump_load = DataObjectDumper().load(tmp_path / "warp_test" / "data.yaml")
+    assert "Warp" in dump_load

junifer/typing/__init__.pyi

@@ -1,4 +1,5 @@
 __all__ = [
+    "DataDumpAssetLike",
     "DataGrabberLike",
     "DataRegistryLike",
     "PreprocessorLike",
@@ -16,6 +17,7 @@ __all__ = [
 ]
 
 from ._typing import (
+    DataDumpAssetLike,
     DataGrabberLike,
     DataRegistryLike,
     PreprocessorLike,

junifer/typing/_typing.py

@@ -22,6 +22,7 @@ if TYPE_CHECKING:
 __all__ = [
     "ConditionalDependencies",
     "ConfigVal",
+    "DataDumpAssetLike",
     "DataGrabberLike",
     "DataGrabberPatterns",
     "DataRegistryLike",
@@ -37,6 +38,7 @@ __all__ = [
 ]
 
 
+DataDumpAssetLike = type["DataDumpAssetLike"]
 DataRegistryLike = type["BasePipelineDataRegistry"]
 DataGrabberLike = type["BaseDataGrabber"]
 PreprocessorLike = type["BasePreprocessor"]

{junifer-0.0.7.dev105.dist-info → junifer-0.0.7.dev121.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.7.dev105
+Version: 0.0.7.dev121
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

{junifer-0.0.7.dev105.dist-info → junifer-0.0.7.dev121.dist-info}/RECORD

@@ -1,12 +1,12 @@
 junifer/__init__.py,sha256=2McgH1yNue6Z1V26-uN_mfMjbTcx4CLhym-DMBl5xA4,266
 junifer/__init__.pyi,sha256=SsTvgq2Dod6UqJN96GH1lCphH6hJQQurEJHGNhHjGUI,508
-junifer/_version.py,sha256=jFH3nrzs5AspealBCIydRQVFvENTPE2XjxPHECcRoME,528
+junifer/_version.py,sha256=EcmNCxfnmwXc_dYkB9704nW6N01vTPj7Wl2RQTVG5-c,528
 junifer/conftest.py,sha256=PWYkkRDU8ly2lYwv7VBKMHje4et6HX7Yey3Md_I2KbA,613
 junifer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 junifer/stats.py,sha256=e9aaagMGtgpRfW3Wdpz9ocpnYld1IWylCDcjFUgX9Mk,6225
 junifer/api/__init__.py,sha256=aAXW_KAEGQ8aAP5Eni2G1R4MWBF7UgjKOgM6akLuJco,252
 junifer/api/__init__.pyi,sha256=UJu55ApMFd43N0xlQyNKrYpCdzqhAxA3Jjaj0ETwCXU,169
-junifer/api/decorators.py,sha256=hBmJbJIedXvCxzvxWBQond3Nu9oSTd2e7dvJ_QZ9zF0,3635
+junifer/api/decorators.py,sha256=7yFhb63zRuyd9uyV_8e7gr4oD0yvZAC4So6CYlwm4tY,4723
 junifer/api/functions.py,sha256=LXKPqsfWINq1iSUShdryGB8hPOyyydc1ldHfr68bP20,14226
 junifer/api/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 junifer/api/queue_context/__init__.py,sha256=glr8x4aMm4EvVrHywDIlugdNlwD1RzqV2FTDNPqYQZ4,204
@@ -40,7 +40,7 @@ junifer/api/res/fsl/flirt,sha256=tSjiUco8ui8AbHD7mTzChEwbR0Rf_4iJTgzYTPF_WuQ,42
 junifer/api/res/fsl/img2imgcoord,sha256=Zmaw3oJYrEltcXiPyEubXry9ppAq3SND52tdDWGgeZk,49
 junifer/api/res/fsl/run_fsl_docker.sh,sha256=pq-fcNdLuvHzVIQePN4GebZGlcE2UF-xj5rBIqAMz4g,1122
 junifer/api/res/fsl/std2imgcoord,sha256=-X5wRH6XMl0yqnTACJX6MFhO8DFOEWg42MHRxGvimXg,49
-junifer/api/tests/test_decorators.py,sha256=GN4wSRDJncqa1Jz9krNY_Ls9e6ge9xaGyOpaiSSXibk,759
+junifer/api/tests/test_decorators.py,sha256=2-ajsMWqXvGdHDnNU4ueSd5GcJ8ROIYOX8tFrJINbaI,2062
 junifer/api/tests/test_functions.py,sha256=HcJIBCtcgL1xJlDwtGHrGOWBMjXgCoAFoVxQW6n2Tds,20676
 junifer/cli/__init__.py,sha256=LRmpmMe0DdZKYZTV61onUiLLxYZ_ZYSfmRbH55bBJMg,500
 junifer/cli/__init__.pyi,sha256=PiV4znUnzSeuSSJGz-RT8N21PiMqoSMwYcypi7nt2Js,40
@@ -237,17 +237,19 @@ junifer/markers/tests/test_parcel_aggregation.py,sha256=04OqtY_Z-KW4W1jU5K6GeWnL
 junifer/markers/tests/test_sphere_aggregation.py,sha256=HPaLD6xKdewTt0iANz3nYOD7ZI-g7BqMTiRdV-4sM8M,10669
 junifer/onthefly/__init__.py,sha256=TA6tPuw54ynDlumb9Ii-2p59hw2rGoCMe1-vQ89JzZ8,238
 junifer/onthefly/_brainprint.py,sha256=-BswaAV9SLHU8mmWJ2KbPL7FgERJzIQIbSdV-NYiiYI,3802
-junifer/onthefly/read_transform.py,sha256=JfTJIiZnautkJ6DzsjeWQ7AEDRHV4omfinvDkow3FFM,4272
-junifer/onthefly/tests/test_read_transform.py,sha256=Ed6gtj8bsD11fe0Y1AxG2JndtITDSBje4g3hx3wkbAo,4718
+junifer/onthefly/read_transform.py,sha256=pUwwsO4oBwq6u4ybRpnQ5s6MujtwD_1AOMv-RdavAFg,6690
+junifer/onthefly/tests/test_read_transform.py,sha256=U8BwImmgH9e2eA_WXVWyKgGzFQNEoD0teCNv2Udlhok,7246
 junifer/pipeline/__init__.py,sha256=rxKQGRwc6_sts1KhVIcVVpuXeiFABf11mQQ2h5jgA3U,194
-junifer/pipeline/__init__.pyi,sha256=hhcvNcABhtLaUQiZdTjo5sMWC3rtDkwVshL0sxD5JAE,399
-junifer/pipeline/marker_collection.py,sha256=1Kmf5f0E2MFhDpO9OBui046b_6h1u9U64AdEqrxso-o,5377
+junifer/pipeline/__init__.pyi,sha256=T2SzqOHE8bD7j3s2HUZrcVB_To1Sv8HDhXTG9YFsWtM,642
+junifer/pipeline/_data_object_dumper.py,sha256=UV0h6onoVOIgD2q80XB8OU9Xe8NZYygoaYEoI2mmHmE,10870
+junifer/pipeline/marker_collection.py,sha256=bVEcrc8Gf3Bm96Ez3FJa6U-NUTVjg4x10x74egbKMQk,7000
 junifer/pipeline/pipeline_component_registry.py,sha256=N80XfOZB33tscuqUlrri0r8sMUGVkPL6Li01Of70qrA,9517
 junifer/pipeline/pipeline_step_mixin.py,sha256=oXfJh27yifHs1V3V_tMPCanRiHX1ggOVIbHTvMzq3cY,7853
 junifer/pipeline/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 junifer/pipeline/update_meta_mixin.py,sha256=yzGCx8AUbc9mMnWKRu4qaIXTBBSIxtNlGH5zIQIUvzM,1812
 junifer/pipeline/utils.py,sha256=qS0Xg_43ri-xtLeMJR838Axj9FkQ6s2H4r8CmSD58X8,10285
 junifer/pipeline/workdir_manager.py,sha256=L9_roiBW1IgCSPVuFhCouGcH2PCLFetk4OKhHIu6bNA,8605
+junifer/pipeline/tests/test_data_object_dumper.py,sha256=ONn3J21Un5SrJqZWXDHpf-3jeHUxdcSCUbY6I2ABMCk,6252
 junifer/pipeline/tests/test_marker_collection.py,sha256=FwxJvjYQ3mh_e3uFZSlOnuGu0EIx4L-Niqt4UOKU6YM,6968
 junifer/pipeline/tests/test_pipeline_component_registry.py,sha256=mrbz285K_TzSILRn9X-AyzcNXuPRHGBZY6dQiq5_9So,5776
 junifer/pipeline/tests/test_pipeline_step_mixin.py,sha256=KCdhFdThm9TGkUvhGzQF3zR9SoZ9ont1z8yZELB2TtQ,7752
@@ -309,8 +311,8 @@ junifer/testing/tests/test_testing_registry.py,sha256=MK4a_q4MHieCvYhnhuPm_dH76l
 junifer/tests/test_main.py,sha256=GMff7jlisGM9_FsiUwWDte43j-KQJGFRYZpwRRqTkd8,373
 junifer/tests/test_stats.py,sha256=NljoGFu2JOPADbi9W0WeUHwpf8nZSdOkcCgCv-Z1fY4,4149
 junifer/typing/__init__.py,sha256=e0UbuxozXUIxz8h8pLokMOxZV629Q1lnA7vvgm95WF0,215
-junifer/typing/__init__.pyi,sha256=5jzVAkras38Eou5abUvdP1AXhbpCSnPAllLx88YuPB8,640
-junifer/typing/_typing.py,sha256=JogiI9wCZWHuqgTaZarjk89aA5pR0yTvFx2JfheLT_Y,1783
+junifer/typing/__init__.pyi,sha256=l_AHfe7LkM6lhaUxnlZ5frBxtZeKbblVUFY3yyWLg70,688
+junifer/typing/_typing.py,sha256=kzlXa-mv2fZytwrmTFGBeod3qipm0zJJTh09iNsoAoA,1854
 junifer/utils/__init__.py,sha256=I3tYaePAD_ZEU-36-TJ_OYeqW_aMmi5MZ3jmqie6RfU,260
 junifer/utils/__init__.pyi,sha256=CMb4rq1VcQ00IRuiBFfAWu07Vb-vA4qtVLAoY0ll-bA,422
 junifer/utils/_config.py,sha256=cfxyv1bfklID2atQseu6y3J7mZrCXPwnGEfBSImG9CM,3054
@@ -324,10 +326,10 @@ junifer/utils/tests/test_config.py,sha256=7ltIXuwb_W4Mv_1dxQWyiyM10XgUAfsWKV6D_i
 junifer/utils/tests/test_fs.py,sha256=WQS7cKlKEZ742CIuiOYYpueeAhY9PqlastfDVpVVtvE,923
 junifer/utils/tests/test_helpers.py,sha256=k5qqfxK8dFyuewTJyR1Qn6-nFaYNuVr0ysc18bfPjyU,929
 junifer/utils/tests/test_logging.py,sha256=W4tFKmaf8_CxnWZ-o_-XxM7DQbhGG18RsLZJk8bZelI,8163
-junifer-0.0.7.dev105.dist-info/licenses/AUTHORS.rst,sha256=rmULKpchpSol4ExWFdm-qu4fkpSZPYqIESVJBZtGb6E,163
-junifer-0.0.7.dev105.dist-info/licenses/LICENSE.md,sha256=MqCnOBu8uXsEOzRZWh9EBVfVz-kE9NkXcLCrtGXo2yU,34354
-junifer-0.0.7.dev105.dist-info/METADATA,sha256=y9qwZWakS8Oc_XurJ3dN1hQ-ndxPyy5BdikbsZsaIG8,8388
-junifer-0.0.7.dev105.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-junifer-0.0.7.dev105.dist-info/entry_points.txt,sha256=6O8ru0BP-SP7YMUZiizFNoaZ2HvJpadO2G7nKk4PwjI,48
-junifer-0.0.7.dev105.dist-info/top_level.txt,sha256=4bAq1R2QFQ4b3hohjys2JBvxrl0GKk5LNFzYvz9VGcA,8
-junifer-0.0.7.dev105.dist-info/RECORD,,
+junifer-0.0.7.dev121.dist-info/licenses/AUTHORS.rst,sha256=rmULKpchpSol4ExWFdm-qu4fkpSZPYqIESVJBZtGb6E,163
+junifer-0.0.7.dev121.dist-info/licenses/LICENSE.md,sha256=MqCnOBu8uXsEOzRZWh9EBVfVz-kE9NkXcLCrtGXo2yU,34354
+junifer-0.0.7.dev121.dist-info/METADATA,sha256=T-IK6ff8Ahg-PIyefj_5qmUGlkzQs9N0weQFGMMlj48,8388
+junifer-0.0.7.dev121.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+junifer-0.0.7.dev121.dist-info/entry_points.txt,sha256=6O8ru0BP-SP7YMUZiizFNoaZ2HvJpadO2G7nKk4PwjI,48
+junifer-0.0.7.dev121.dist-info/top_level.txt,sha256=4bAq1R2QFQ4b3hohjys2JBvxrl0GKk5LNFzYvz9VGcA,8
+junifer-0.0.7.dev121.dist-info/RECORD,,