junifer 0.0.6.dev154__py3-none-any.whl → 0.0.6.dev175__py3-none-any.whl

junifer/__init__.pyi

@@ -12,6 +12,7 @@ __all__ = [
     "utils",
     "external",
     "onthefly",
+    "testing",
     "__version__",
 ]
 
@@ -28,6 +29,7 @@ from . import (
     storage,
     utils,
     external,
+    testing,
     onthefly,
 )
 from ._version import __version__

junifer/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.6.dev154'
-__version_tuple__ = version_tuple = (0, 0, 6, 'dev154')
+__version__ = version = '0.0.6.dev175'
+__version_tuple__ = version_tuple = (0, 0, 6, 'dev175')

junifer/api/decorators.py

@@ -7,7 +7,7 @@
 
 from typing import Type
 
-from ..pipeline.registry import register
+from ..pipeline import PipelineComponentRegistry
 
 
 __all__ = [
@@ -39,9 +39,8 @@ def register_datagrabber(klass: Type) -> Type:
     It should only be used as a decorator.
 
     """
-    register(
+    PipelineComponentRegistry().register(
         step="datagrabber",
-        name=klass.__name__,
         klass=klass,
     )
     return klass
@@ -67,9 +66,8 @@ def register_datareader(klass: Type) -> Type:
     It should only be used as a decorator.
 
     """
-    register(
+    PipelineComponentRegistry().register(
         step="datareader",
-        name=klass.__name__,
         klass=klass,
     )
     return klass
@@ -91,9 +89,8 @@ def register_preprocessor(klass: Type) -> Type:
         The unmodified input class.
 
     """
-    register(
+    PipelineComponentRegistry().register(
         step="preprocessing",
-        name=klass.__name__,
         klass=klass,
     )
     return klass
@@ -115,9 +112,8 @@ def register_marker(klass: Type) -> Type:
         The unmodified input class.
 
     """
-    register(
+    PipelineComponentRegistry().register(
         step="marker",
-        name=klass.__name__,
         klass=klass,
     )
     return klass
@@ -139,9 +135,8 @@ def register_storage(klass: Type) -> Type:
         The unmodified input class.
 
     """
-    register(
+    PipelineComponentRegistry().register(
         step="storage",
-        name=klass.__name__,
         klass=klass,
     )
     return klass

junifer/api/functions.py

@@ -7,15 +7,17 @@
 
 import os
 import shutil
-import typing
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
 from ..api.queue_context import GnuParallelLocalAdapter, HTCondorAdapter
 from ..datagrabber.base import BaseDataGrabber
 from ..markers.base import BaseMarker
-from ..pipeline import MarkerCollection, WorkDirManager
-from ..pipeline.registry import build
+from ..pipeline import (
+    MarkerCollection,
+    PipelineComponentRegistry,
+    WorkDirManager,
+)
 from ..preprocess.base import BasePreprocessor
 from ..storage.base import BaseFeatureStorage
 from ..utils import logger, raise_error, yaml
@@ -38,42 +40,78 @@ def _get_datagrabber(datagrabber_config: Dict) -> BaseDataGrabber:
         The DataGrabber.
 
     """
-    datagrabber_params = datagrabber_config.copy()
-    datagrabber_kind = datagrabber_params.pop("kind")
-    datagrabber = build(
+    return PipelineComponentRegistry().build_component_instance(
         step="datagrabber",
-        name=datagrabber_kind,
+        name=datagrabber_config.pop("kind"),
         baseclass=BaseDataGrabber,
-        init_params=datagrabber_params,
+        init_params=datagrabber_config,
     )
-    datagrabber = typing.cast(BaseDataGrabber, datagrabber)
-    return datagrabber
 
 
 def _get_preprocessor(preprocessing_config: Dict) -> BasePreprocessor:
-    """Get preprocessor.
+    """Get Preprocessor.
 
     Parameters
     ----------
     preprocessing_config : dict
-        The config to get the preprocessor using.
+        The config to get the Preprocessor using.
 
     Returns
     -------
     dict
-        The preprocessor.
+        The Preprocessor.
 
     """
-    preprocessor_params = preprocessing_config.copy()
-    preprocessor_kind = preprocessor_params.pop("kind")
-    preprocessor = build(
+    return PipelineComponentRegistry().build_component_instance(
         step="preprocessing",
-        name=preprocessor_kind,
+        name=preprocessing_config.pop("kind"),
         baseclass=BasePreprocessor,
-        init_params=preprocessor_params,
+        init_params=preprocessing_config,
+    )
+
+
+def _get_marker(marker_config: Dict) -> BaseMarker:
+    """Get Marker.
+
+    Parameters
+    ----------
+    marker_config : dict
+        The config to get the Marker using.
+
+    Returns
+    -------
+    object
+        The Marker.
+
+    """
+    return PipelineComponentRegistry().build_component_instance(
+        step="marker",
+        name=marker_config.pop("kind"),
+        baseclass=BaseMarker,
+        init_params=marker_config,
+    )
+
+
+def _get_storage(storage_config: Dict) -> BaseFeatureStorage:
+    """Get Storage.
+
+    Parameters
+    ----------
+    storage_config : dict
+        The config to get the Storage using.
+
+    Returns
+    -------
+    dict
+        The Storage.
+
+    """
+    return PipelineComponentRegistry().build_component_instance(
+        step="storage",
+        name=storage_config.pop("kind"),
+        baseclass=BaseFeatureStorage,
+        init_params=storage_config,
     )
-    preprocessor = typing.cast(BasePreprocessor, preprocessor)
-    return preprocessor
 
 
 def run(
@@ -123,41 +161,22 @@ def run(
         elements = [elements]
 
     # Get datagrabber to use
-    datagrabber_object = _get_datagrabber(datagrabber)
+    datagrabber_object = _get_datagrabber(datagrabber.copy())
 
-    # Copy to avoid changing the original dict
-    _markers = [x.copy() for x in markers]
-    built_markers = []
-    for t_marker in _markers:
-        kind = t_marker.pop("kind")
-        t_m = build(
-            step="marker",
-            name=kind,
-            baseclass=BaseMarker,
-            init_params=t_marker,
-        )
-        built_markers.append(t_m)
+    # Get markers to use
+    built_markers = [_get_marker(marker) for marker in markers.copy()]
 
     # Get storage engine to use
-    storage_params = storage.copy()
-    storage_kind = storage_params.pop("kind")
-    if "single_output" not in storage_params:
-        storage_params["single_output"] = False
-    storage_object = build(
-        step="storage",
-        name=storage_kind,
-        baseclass=BaseFeatureStorage,
-        init_params=storage_params,
-    )
-    storage_object = typing.cast(BaseFeatureStorage, storage_object)
+    if "single_output" not in storage:
+        storage["single_output"] = False
+    storage_object = _get_storage(storage.copy())
 
     # Get preprocessor to use (if provided)
     if preprocessors is not None:
-        _preprocessors = [x.copy() for x in preprocessors]
-        built_preprocessors = []
-        for preprocessor in _preprocessors:
-            preprocessor_object = _get_preprocessor(preprocessor)
-            built_preprocessors.append(preprocessor_object)
+        built_preprocessors = [
+            _get_preprocessor(preprocessor)
+            for preprocessor in preprocessors.copy()
+        ]
     else:
         built_preprocessors = None
 
@@ -167,6 +186,7 @@ def run(
         preprocessors=built_preprocessors,
         storage=storage_object,
     )
+    # Validate the marker collection for the datagrabber
     mc.validate(datagrabber_object)
 
     # Fit elements
@@ -192,19 +212,11 @@ def collect(storage: Dict) -> None:
         constructor.
 
     """
-    storage_params = storage.copy()
-    storage_kind = storage_params.pop("kind")
-    logger.info(f"Collecting data using {storage_kind}")
-    logger.debug(f"\tStorage params: {storage_params}")
-    if "single_output" not in storage_params:
-        storage_params["single_output"] = False
-    storage_object = build(
-        step="storage",
-        name=storage_kind,
-        baseclass=BaseFeatureStorage,
-        init_params=storage_params,
-    )
-    storage_object = typing.cast(BaseFeatureStorage, storage_object)
+    logger.info(f"Collecting data using {storage['kind']}")
+    logger.debug(f"\tStorage params: {storage}")
+    if "single_output" not in storage:
+        storage["single_output"] = False
+    storage_object = _get_storage(storage.copy())
     logger.debug("Running storage.collect()")
     storage_object.collect()
     logger.info("Collect done")

junifer/api/tests/test_functions.py

@@ -15,7 +15,7 @@ from ruamel.yaml import YAML
 import junifer.testing.registry  # noqa: F401
 from junifer.api import collect, list_elements, queue, reset, run
 from junifer.datagrabber.base import BaseDataGrabber
-from junifer.pipeline.registry import build
+from junifer.pipeline import PipelineComponentRegistry
 
 
 # Configure YAML class
@@ -240,7 +240,7 @@ def test_run_and_collect(
         storage=storage,
     )
     # Get datagrabber
-    dg = build(
+    dg = PipelineComponentRegistry().build_component_instance(
         step="datagrabber", name=datagrabber["kind"], baseclass=BaseDataGrabber
     )
     elements = dg.get_elements()  # type: ignore

junifer/datagrabber/aomic/piop2.py

@@ -59,7 +59,7 @@ class DataladAOMICPIOP2(PatternDataladDataGrabber):
             "restingstate",
             "stopsignal",
             "workingmemory",
-            "emomatching"
+            "emomatching",
         ]
         # Set default tasks
         if tasks is None:

junifer/pipeline/__init__.pyi

@@ -1,12 +1,12 @@
 __all__ = [
-    "registry",
+    "PipelineComponentRegistry",
     "PipelineStepMixin",
     "UpdateMetaMixin",
     "WorkDirManager",
     "MarkerCollection",
 ]
 
-from . import registry
+from .pipeline_component_registry import PipelineComponentRegistry
 from .pipeline_step_mixin import PipelineStepMixin
 from .update_meta_mixin import UpdateMetaMixin
 from .workdir_manager import WorkDirManager

junifer/pipeline/pipeline_component_registry.py

@@ -0,0 +1,299 @@
+"""Provide a class for centralized pipeline component registry."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+#          Leonard Sasse <l.sasse@fz-juelich.de>
+#          Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+import importlib
+from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Union
+
+from ..utils import logger, raise_error
+from .singleton import singleton
+
+
+if TYPE_CHECKING:
+    from ..datagrabber import BaseDataGrabber
+    from ..storage import BaseFeatureStorage
+    from .pipeline_step_mixin import PipelineStepMixin
+
+
+__all__ = ["PipelineComponentRegistry"]
+
+
+@singleton
+class PipelineComponentRegistry:
+    """Class for pipeline component registry.
+
+    This class is a singleton and is used for managing pipeline components.
+    It serves as a centralized registry for built-in and third-party pipeline
+    components like datagrabbers, datareaders, preprocessors, markers and
+    storage.
+
+    Attributes
+    ----------
+    steps : list of str
+        Valid pipeline steps.
+    components : dict
+        Registered components for valid pipeline steps.
+
+    """
+
+    def __init__(self) -> None:
+        """Initialize the class."""
+        # Valid steps for operation
+        self._steps = [
+            "datagrabber",
+            "datareader",
+            "preprocessing",
+            "marker",
+            "storage",
+        ]
+        # Step to sub-package mapping
+        self._step_to_subpkg_mappings = {
+            "datagrabber": "datagrabber",
+            "datareader": "datareader",
+            "preprocessing": "preprocess",
+            "marker": "markers",
+            "storage": "storage",
+        }
+        # Component registry for valid steps
+        self._components = {
+            "datagrabber": {
+                "HCP1200": "HCP1200",
+                "BaseDataGrabber": "BaseDataGrabber",
+                "DataladAOMICID1000": "DataladAOMICID1000",
+                "DataladAOMICPIOP1": "DataladAOMICPIOP1",
+                "DataladAOMICPIOP2": "DataladAOMICPIOP2",
+                "DataladDataGrabber": "DataladDataGrabber",
+                "DataladHCP1200": "DataladHCP1200",
+                "DMCC13Benchmark": "DMCC13Benchmark",
+                "MultipleDataGrabber": "MultipleDataGrabber",
+                "PatternDataGrabber": "PatternDataGrabber",
+                "PatternDataladDataGrabber": "PatternDataladDataGrabber",
+            },
+            "datareader": {
+                "DefaultDataReader": "DefaultDataReader",
+            },
+            "preprocessing": {
+                "BasePreprocessor": "BasePreprocessor",
+                "Smoothing": "Smoothing",
+                "SpaceWarper": "SpaceWarper",
+                "fMRIPrepConfoundRemover": "fMRIPrepConfoundRemover",
+            },
+            "marker": {
+                "ALFFParcels": "ALFFParcels",
+                "ALFFSpheres": "ALFFSpheres",
+                "BaseMarker": "BaseMarker",
+                "BrainPrint": "BrainPrint",
+                "CrossParcellationFC": "CrossParcellationFC",
+                "EdgeCentricFCParcels": "EdgeCentricFCParcels",
+                "EdgeCentricFCSpheres": "EdgeCentricFCSpheres",
+                "FunctionalConnectivityParcels": (
+                    "FunctionalConnectivityParcels"
+                ),
+                "FunctionalConnectivitySpheres": (
+                    "FunctionalConnectivitySpheres"
+                ),
+                "ParcelAggregation": "ParcelAggregation",
+                "ReHoParcels": "ReHoParcels",
+                "ReHoSpheres": "ReHoSpheres",
+                "RSSETSMarker": "RSSETSMarker",
+                "SphereAggregation": "SphereAggregation",
+                "TemporalSNRParcels": "TemporalSNRParcels",
+                "TemporalSNRSpheres": "TemporalSNRSpheres",
+            },
+            "storage": {
+                "BaseFeatureStorage": "BaseFeatureStorage",
+                "HDF5FeatureStorage": "HDF5FeatureStorage",
+                "PandasBaseFeatureStorage": "PandasBaseFeatureStorage",
+                "SQLiteFeatureStorage": "SQLiteFeatureStorage",
+            },
+        }
+
+    def _check_valid_step(self, step: str) -> None:
+        """Check if ``step`` is valid."""
+        if step not in self._steps:
+            raise_error(msg=f"Invalid step: {step}", klass=ValueError)
+
+    @property
+    def steps(self) -> List[str]:
+        """Get valid pipeline steps."""
+        return self._steps
+
+    @property
+    def components(self) -> Mapping[str, Mapping[str, Union[str, type]]]:
+        """Get registered components for valid pipeline steps."""
+        return self._components
+
+    def register(self, step: str, klass: type) -> None:
+        """Register ``klass`` under ``step``.
+
+        Parameters
+        ----------
+        step : str
+            Name of the pipeline step. For valid steps, check :meth:`.steps`.
+        klass : class
+            Class to be registered.
+
+        Raises
+        ------
+        ValueError
+            If the ``step`` is invalid.
+
+        """
+        # Verify step
+        self._check_valid_step(step)
+        # Log and register
+        name = klass.__name__
+        logger.info(f"Registering {name} in {step}")
+        self._components[step][name] = klass
+
+    def deregister(self, step: str, klass: type) -> None:
+        """De-register ``klass`` under ``step``.
+
+        Parameters
+        ----------
+        step : str
+            Name of the pipeline step. For valid steps, check :meth:`.steps`.
+        klass : class
+            Class to be de-registered.
+
+        Raises
+        ------
+        ValueError
+            If the ``step`` is invalid.
+
+        """
+        # Verify step
+        self._check_valid_step(step)
+        # Log and register
+        name = klass.__name__
+        logger.info(f"De-registering {name} in {step}")
+        _ = self._components[step].pop(name)
+
+    def step_components(self, step: str) -> List[str]:
+        """Get registered components for ``step``.
+
+        Parameters
+        ----------
+        step : str
+            Name of the pipeline step.
+
+        Returns
+        -------
+        list of str
+            List of registered component classes.
+
+        Raises
+        ------
+        ValueError
+            If the ``step`` is invalid.
+
+        """
+        # Verify step
+        self._check_valid_step(step)
+
+        return list(self._components[step].keys())
+
+    def get_class(self, step: str, name: str) -> type:
+        """Get the class registered under ``name`` for ``step``.
+
+        Parameters
+        ----------
+        step : str
+            Name of the pipeline step.
+        name : str
+            Name of the component.
+
+        Returns
+        -------
+        class
+            Registered class.
+
+        Raises
+        ------
+        ValueError
+            If the ``step`` or ``name`` is invalid.
+
+        """
+        # Verify step
+        self._check_valid_step(step)
+        # Verify step name
+        if name not in self._components[step]:
+            raise_error(msg=f"Invalid name: {name}", klass=ValueError)
+
+        # Check if first-time import, then import it
+        if isinstance(self._components[step][name], str):
+            klass = getattr(
+                importlib.import_module(
+                    f"junifer.{self._step_to_subpkg_mappings[step]}"
+                ),
+                name,
+            )
+        else:
+            klass = self._components[step][name]
+
+        return klass
+
+    def build_component_instance(
+        self,
+        step: str,
+        name: str,
+        baseclass: type,
+        init_params: Optional[Dict] = None,
+    ) -> Union["BaseDataGrabber", "PipelineStepMixin", "BaseFeatureStorage"]:
+        """Build an instance of class registered as ``name``.
+
+        Parameters
+        ----------
+        step : str
+            Name of the pipeline step.
+        name : str
+            Name of the component.
+        baseclass : class
+            Base class to be checked against.
+        init_params : dict or None, optional
+            Parameters to pass to the class constructor (default None).
+
+        Returns
+        -------
+        object
+            An instance of the class registered as ``name`` under ``step``.
+
+        Raises
+        ------
+        RuntimeError
+            If there is a problem creating the instance.
+        ValueError
+            If the created object with the given name is not a subclass of the
+            base class ``baseclass``.
+
+        """
+        # Set default init parameters
+        if init_params is None:
+            init_params = {}
+        # Get registered class
+        logger.debug(f"Building {step}/{name}")
+        klass = self.get_class(step=step, name=name)
+        logger.debug(f"\tClass: {klass.__name__}")
+        logger.debug(f"\tInit params: {init_params}")
+        try:
+            # Create instance of the class
+            object_ = klass(**init_params)
+        except (ValueError, TypeError) as e:
+            raise_error(
+                msg=f"Failed to create {step} ({name}). Error: {e}",
+                klass=RuntimeError,
+                exception=e,
+            )
+        # Verify created instance belongs to the base class
+        if not isinstance(object_, baseclass):
+            raise_error(
+                msg=(
+                    f"Invalid {step} ({object_.__class__.__name__}). "
+                    f"Must inherit from {baseclass.__name__}"
+                ),
+                klass=ValueError,
+            )
+        return object_