junifer 0.0.5.dev242__py3-none-any.whl → 0.0.6__py3-none-any.whl

junifer/onthefly/_brainprint.py

@@ -0,0 +1,138 @@
+"""Provide onthefly functions for BrainPrint post-analysis."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+
+from ..typing import StorageLike
+from ..utils import raise_error
+
+
+__all__ = ["normalize", "reweight"]
+
+
+def normalize(
+    storage: StorageLike,
+    features: dict[str, dict[str, Optional[str]]],
+    kind: str,
+) -> pd.DataFrame:
+    """Read stored brainprint data and normalize either surfaces or volumes.
+
+    Parameters
+    ----------
+    storage : storage-like
+        The storage class, for example, :class:`.HDF5FeatureStorage`.
+    features : dict, optional
+        The feature names or MD5 hashes to read as dict.
+        The dict should have the keys:
+
+        * ``"areas"`` (if ``kind="surface"``)
+        * ``"volumes"`` (if ``kind="volume"``)
+        * ``"eigenvalues"``
+
+        and the corresponding value for each of the keys is again
+        a dict with the keys:
+
+        * ``"feature_name"`` : str or None
+        * ``"feature_md5"`` : str or None
+
+        Either one of ``"feature_name"`` or ``"feature_md5"`` needs to be
+        not None for each first-level key, but both keys are mandatory.
+
+    kind : {"surface", "volume"}
+        The kind of normalization.
+
+    Returns
+    -------
+    pandas.DataFrame
+        The transformed feature as a ``pandas.DataFrame``.
+
+    Raises
+    ------
+    ValueError
+        If ``kind`` is invalid.
+
+    """
+    # Read storage
+    data_dict = {}
+    for k, v in features.items():
+        data_dict[k] = storage.read_df(**v)  # type: ignore
+
+    # Check and normalize
+    valid_kind = ["surface", "volume"]
+    normalized_df = None
+    if kind == "surface":
+        eigenvalues_df = data_dict["eigenvalues"]
+        areas_df = data_dict["areas"]
+        normalized_df = eigenvalues_df.combine(
+            areas_df, lambda left, right: left * right
+        )
+    elif kind == "volume":
+        eigenvalues_df = data_dict["eigenvalues"]
+        volumes_df = data_dict["volumes"]
+        normalized_df = eigenvalues_df.combine(
+            volumes_df, lambda left, right: left * right ** np.divide(2.0, 3.0)
+        )
+    else:
+        raise_error(
+            "Invalid value for `kind`, should be one of: " f"{valid_kind}"
+        )
+
+    return normalized_df
+
+
+def reweight(
+    storage: StorageLike,
+    feature_name: Optional[str] = None,
+    feature_md5: Optional[str] = None,
+) -> pd.DataFrame:
+    """Read stored brainprint data and reweight eigenvalues.
+
+    Parameters
+    ----------
+    storage : storage-like
+        The storage class, for example, :class:`.HDF5FeatureStorage`.
+    feature_name : str, optional
+        Name of the feature to read (default None).
+    feature_md5 : str, optional
+        MD5 hash of the feature to read (default None).
+
+    Returns
+    -------
+    pandas.DataFrame
+        The transformed feature as a ``pandas.DataFrame``.
+
+    """
+    # Read storage
+    eigenvalues_df = storage.read_df(
+        feature_name=feature_name, feature_md5=feature_md5
+    )  # type: ignore
+
+    # Create data for operation
+    exploded_count_idx_df = (
+        eigenvalues_df.reset_index("eigenvalue")
+        .index.value_counts()
+        .apply(lambda x: np.arange(1, x + 1).astype(float))
+        .to_frame()
+        .explode("count")
+    )
+    idx_data = (
+        pd.concat(
+            [exploded_count_idx_df] * len(eigenvalues_df.columns), axis=1
+        )
+        .reset_index()
+        .drop("subject", axis=1, inplace=False)
+        .to_numpy()
+    )
+    idx_df = pd.DataFrame(
+        data=idx_data,
+        index=eigenvalues_df.index,
+        columns=eigenvalues_df.columns,
+    )
+
+    # Combine
+    return eigenvalues_df.combine(idx_df, lambda left, right: left / right)

junifer/onthefly/read_transform.py

@@ -4,27 +4,24 @@
 # License: AGPL
 
 
-from typing import TYPE_CHECKING, Dict, Optional, Tuple, Type
+from typing import Optional
 
 import pandas as pd
 
+from ..typing import StorageLike
 from ..utils import logger, raise_error, warn_with_log
 
 
-if TYPE_CHECKING:
-    from junifer.storage import BaseFeatureStorage
-
-
 __all__ = ["read_transform"]
 
 
 def read_transform(
-    storage: Type["BaseFeatureStorage"],
+    storage: StorageLike,
     transform: str,
     feature_name: Optional[str] = None,
     feature_md5: Optional[str] = None,
-    transform_args: Optional[Tuple] = None,
-    transform_kw_args: Optional[Dict] = None,
+    transform_args: Optional[tuple] = None,
+    transform_kw_args: Optional[dict] = None,
 ) -> pd.DataFrame:
     """Read stored feature and transform to specific statistical output.
 

junifer/pipeline/__init__.py

@@ -3,15 +3,7 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from . import registry
-from .pipeline_step_mixin import PipelineStepMixin
-from .update_meta_mixin import UpdateMetaMixin
-from .workdir_manager import WorkDirManager
+import lazy_loader as lazy
 
 
-__all__ = [
-    "registry",
-    "PipelineStepMixin",
-    "UpdateMetaMixin",
-    "WorkDirManager",
-]
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)

junifer/pipeline/__init__.pyi

@@ -0,0 +1,13 @@
+__all__ = [
+    "PipelineComponentRegistry",
+    "PipelineStepMixin",
+    "UpdateMetaMixin",
+    "WorkDirManager",
+    "MarkerCollection",
+]
+
+from .pipeline_component_registry import PipelineComponentRegistry
+from .pipeline_step_mixin import PipelineStepMixin
+from .update_meta_mixin import UpdateMetaMixin
+from .workdir_manager import WorkDirManager
+from .marker_collection import MarkerCollection

junifer/{markers/collection.py → pipeline/marker_collection.py}

@@ -5,20 +5,14 @@
 # License: AGPL
 
 from collections import Counter
-from typing import TYPE_CHECKING, Dict, List, Optional
+from typing import Optional
 
-from ..datareader.default import DefaultDataReader
-from ..markers.base import BaseMarker
+from ..datareader import DefaultDataReader
 from ..pipeline import PipelineStepMixin, WorkDirManager
-from ..preprocess.base import BasePreprocessor
-from ..storage.base import BaseFeatureStorage
+from ..typing import DataGrabberLike, MarkerLike, PreprocessorLike, StorageLike
 from ..utils import logger, raise_error
 
 
-if TYPE_CHECKING:
-    from junifer.datagrabber import BaseDataGrabber
-
-
 __all__ = ["MarkerCollection"]
 
 
@@ -45,10 +39,10 @@ class MarkerCollection:
 
     def __init__(
         self,
-        markers: List[BaseMarker],
+        markers: list[MarkerLike],
         datareader: Optional[PipelineStepMixin] = None,
-        preprocessors: Optional[List[BasePreprocessor]] = None,
-        storage: Optional[BaseFeatureStorage] = None,
+        preprocessors: Optional[list[PreprocessorLike]] = None,
+        storage: Optional[StorageLike] = None,
     ):
         # Check that the markers have different names
         marker_names = [m.name for m in markers]
@@ -65,7 +59,7 @@ class MarkerCollection:
         self._preprocessors = preprocessors
         self._storage = storage
 
-    def fit(self, input: Dict[str, Dict]) -> Optional[Dict]:
+    def fit(self, input: dict[str, dict]) -> Optional[dict]:
         """Fit the pipeline.
 
         Parameters
@@ -111,7 +105,7 @@ class MarkerCollection:
 
         return None if self._storage else out
 
-    def validate(self, datagrabber: "BaseDataGrabber") -> None:
+    def validate(self, datagrabber: DataGrabberLike) -> None:
         """Validate the pipeline.
 
         Without doing any computation, check if the marker collection can

junifer/pipeline/pipeline_component_registry.py

@@ -0,0 +1,294 @@
+"""Provide a class for centralized pipeline component registry."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+#          Leonard Sasse <l.sasse@fz-juelich.de>
+#          Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+import importlib
+from collections.abc import Mapping
+from typing import Optional, Union
+
+from ..typing import DataGrabberLike, MarkerLike, PreprocessorLike, StorageLike
+from ..utils import logger, raise_error
+from ..utils.singleton import Singleton
+
+
+__all__ = ["PipelineComponentRegistry"]
+
+
+class PipelineComponentRegistry(metaclass=Singleton):
+    """Class for pipeline component registry.
+
+    This class is a singleton and is used for managing pipeline components.
+    It serves as a centralized registry for built-in and third-party pipeline
+    components like datagrabbers, datareaders, preprocessors, markers and
+    storage.
+
+    Attributes
+    ----------
+    steps : list of str
+        Valid pipeline steps.
+    components : dict
+        Registered components for valid pipeline steps.
+
+    """
+
+    def __init__(self) -> None:
+        """Initialize the class."""
+        # Valid steps for operation
+        self._steps = [
+            "datagrabber",
+            "datareader",
+            "preprocessing",
+            "marker",
+            "storage",
+        ]
+        # Step to sub-package mapping
+        self._step_to_subpkg_mappings = {
+            "datagrabber": "datagrabber",
+            "datareader": "datareader",
+            "preprocessing": "preprocess",
+            "marker": "markers",
+            "storage": "storage",
+        }
+        # Component registry for valid steps
+        self._components = {
+            "datagrabber": {
+                "HCP1200": "HCP1200",
+                "BaseDataGrabber": "BaseDataGrabber",
+                "DataladAOMICID1000": "DataladAOMICID1000",
+                "DataladAOMICPIOP1": "DataladAOMICPIOP1",
+                "DataladAOMICPIOP2": "DataladAOMICPIOP2",
+                "DataladDataGrabber": "DataladDataGrabber",
+                "DataladHCP1200": "DataladHCP1200",
+                "DMCC13Benchmark": "DMCC13Benchmark",
+                "MultipleDataGrabber": "MultipleDataGrabber",
+                "PatternDataGrabber": "PatternDataGrabber",
+                "PatternDataladDataGrabber": "PatternDataladDataGrabber",
+            },
+            "datareader": {
+                "DefaultDataReader": "DefaultDataReader",
+            },
+            "preprocessing": {
+                "BasePreprocessor": "BasePreprocessor",
+                "Smoothing": "Smoothing",
+                "SpaceWarper": "SpaceWarper",
+                "fMRIPrepConfoundRemover": "fMRIPrepConfoundRemover",
+            },
+            "marker": {
+                "ALFFParcels": "ALFFParcels",
+                "ALFFSpheres": "ALFFSpheres",
+                "BaseMarker": "BaseMarker",
+                "BrainPrint": "BrainPrint",
+                "CrossParcellationFC": "CrossParcellationFC",
+                "EdgeCentricFCParcels": "EdgeCentricFCParcels",
+                "EdgeCentricFCSpheres": "EdgeCentricFCSpheres",
+                "FunctionalConnectivityParcels": (
+                    "FunctionalConnectivityParcels"
+                ),
+                "FunctionalConnectivitySpheres": (
+                    "FunctionalConnectivitySpheres"
+                ),
+                "ParcelAggregation": "ParcelAggregation",
+                "ReHoParcels": "ReHoParcels",
+                "ReHoSpheres": "ReHoSpheres",
+                "RSSETSMarker": "RSSETSMarker",
+                "SphereAggregation": "SphereAggregation",
+                "TemporalSNRParcels": "TemporalSNRParcels",
+                "TemporalSNRSpheres": "TemporalSNRSpheres",
+            },
+            "storage": {
+                "BaseFeatureStorage": "BaseFeatureStorage",
+                "HDF5FeatureStorage": "HDF5FeatureStorage",
+                "PandasBaseFeatureStorage": "PandasBaseFeatureStorage",
+                "SQLiteFeatureStorage": "SQLiteFeatureStorage",
+            },
+        }
+
+    def _check_valid_step(self, step: str) -> None:
+        """Check if ``step`` is valid."""
+        if step not in self._steps:
+            raise_error(msg=f"Invalid step: {step}", klass=ValueError)
+
+    @property
+    def steps(self) -> list[str]:
+        """Get valid pipeline steps."""
+        return self._steps
+
+    @property
+    def components(self) -> Mapping[str, Mapping[str, Union[str, type]]]:
+        """Get registered components for valid pipeline steps."""
+        return self._components
+
+    def register(self, step: str, klass: type) -> None:
+        """Register ``klass`` under ``step``.
+
+        Parameters
+        ----------
+        step : str
+            Name of the pipeline step. For valid steps, check :meth:`.steps`.
+        klass : class
+            Class to be registered.
+
+        Raises
+        ------
+        ValueError
+            If the ``step`` is invalid.
+
+        """
+        # Verify step
+        self._check_valid_step(step)
+        # Log and register
+        name = klass.__name__
+        logger.info(f"Registering {name} in {step}")
+        self._components[step][name] = klass
+
+    def deregister(self, step: str, klass: type) -> None:
+        """De-register ``klass`` under ``step``.
+
+        Parameters
+        ----------
+        step : str
+            Name of the pipeline step. For valid steps, check :meth:`.steps`.
+        klass : class
+            Class to be de-registered.
+
+        Raises
+        ------
+        ValueError
+            If the ``step`` is invalid.
+
+        """
+        # Verify step
+        self._check_valid_step(step)
+        # Log and de-register
+        name = klass.__name__
+        logger.info(f"De-registering {name} in {step}")
+        _ = self._components[step].pop(name)
+
+    def step_components(self, step: str) -> list[str]:
+        """Get registered components for ``step``.
+
+        Parameters
+        ----------
+        step : str
+            Name of the pipeline step.
+
+        Returns
+        -------
+        list of str
+            List of registered component classes.
+
+        Raises
+        ------
+        ValueError
+            If the ``step`` is invalid.
+
+        """
+        # Verify step
+        self._check_valid_step(step)
+
+        return list(self._components[step].keys())
+
+    def get_class(self, step: str, name: str) -> type:
+        """Get the class registered under ``name`` for ``step``.
+
+        Parameters
+        ----------
+        step : str
+            Name of the pipeline step.
+        name : str
+            Name of the component.
+
+        Returns
+        -------
+        class
+            Registered class.
+
+        Raises
+        ------
+        ValueError
+            If the ``step`` or ``name`` is invalid.
+
+        """
+        # Verify step
+        self._check_valid_step(step)
+        # Verify step name
+        if name not in self._components[step]:
+            raise_error(msg=f"Invalid name: {name}", klass=ValueError)
+
+        # Check if first-time import, then import it
+        if isinstance(self._components[step][name], str):
+            klass = getattr(
+                importlib.import_module(
+                    f"junifer.{self._step_to_subpkg_mappings[step]}"
+                ),
+                name,
+            )
+        else:
+            klass = self._components[step][name]
+
+        return klass
+
+    def build_component_instance(
+        self,
+        step: str,
+        name: str,
+        baseclass: type,
+        init_params: Optional[dict] = None,
+    ) -> Union[DataGrabberLike, PreprocessorLike, MarkerLike, StorageLike]:
+        """Build an instance of class registered as ``name``.
+
+        Parameters
+        ----------
+        step : str
+            Name of the pipeline step.
+        name : str
+            Name of the component.
+        baseclass : class
+            Base class to be checked against.
+        init_params : dict or None, optional
+            Parameters to pass to the class constructor (default None).
+
+        Returns
+        -------
+        object
+            An instance of the class registered as ``name`` under ``step``.
+
+        Raises
+        ------
+        RuntimeError
+            If there is a problem creating the instance.
+        ValueError
+            If the created object with the given name is not a subclass of the
+            base class ``baseclass``.
+
+        """
+        # Set default init parameters
+        if init_params is None:
+            init_params = {}
+        # Get registered class
+        logger.debug(f"Building {step}/{name}")
+        klass = self.get_class(step=step, name=name)
+        logger.debug(f"\tClass: {klass.__name__}")
+        logger.debug(f"\tInit params: {init_params}")
+        try:
+            # Create instance of the class
+            object_ = klass(**init_params)
+        except (ValueError, TypeError) as e:
+            raise_error(
+                msg=f"Failed to create {step} ({name}). Error: {e}",
+                klass=RuntimeError,
+                exception=e,
+            )
+        # Verify created instance belongs to the base class
+        if not isinstance(object_, baseclass):
+            raise_error(
+                msg=(
+                    f"Invalid {step} ({object_.__class__.__name__}). "
+                    f"Must inherit from {baseclass.__name__}"
+                ),
+                klass=ValueError,
+            )
+        return object_

junifer/pipeline/pipeline_step_mixin.py

@@ -15,7 +15,7 @@ else:
 
 from importlib.util import find_spec
 from itertools import chain
-from typing import Any, Dict, List
+from typing import Any
 
 from ..utils import raise_error
 from .utils import check_ext_dependencies
@@ -27,7 +27,7 @@ __all__ = ["PipelineStepMixin"]
 class PipelineStepMixin:
     """Mixin class for a pipeline step."""
 
-    def validate_input(self, input: List[str]) -> List[str]:
+    def validate_input(self, input: list[str]) -> list[str]:
         """Validate the input to the pipeline step.
 
         Parameters
@@ -74,8 +74,8 @@ class PipelineStepMixin:
 
     def _fit_transform(
         self,
-        input: Dict[str, Dict],
-    ) -> Dict[str, Dict]:
+        input: dict[str, dict],
+    ) -> dict[str, dict]:
         """Fit and transform.
 
         Parameters
@@ -94,7 +94,7 @@ class PipelineStepMixin:
             klass=NotImplementedError,
         )  # pragma: no cover
 
-    def validate(self, input: List[str]) -> List[str]:
+    def validate(self, input: list[str]) -> list[str]:
         """Validate the the pipeline step.
 
         Parameters
@@ -196,10 +196,14 @@ class PipelineStepMixin:
                     for dependency in obj._CONDITIONAL_DEPENDENCIES:
                         if dependency["using"] == obj.using:
                             depends_on = dependency["depends_on"]
-                            # Check dependencies
-                            _check_dependencies(depends_on)
-                            # Check external dependencies
-                            _check_ext_dependencies(depends_on)
+                            # Conditional to make `using="auto"` work
+                            if not isinstance(depends_on, list):
+                                depends_on = [depends_on]
+                            for entry in depends_on:
+                                # Check dependencies
+                                _check_dependencies(entry)
+                                # Check external dependencies
+                                _check_ext_dependencies(entry)
 
         # Check dependencies
         _check_dependencies(self)
@@ -224,8 +228,8 @@ class PipelineStepMixin:
         return outputs
 
     def fit_transform(
-        self, input: Dict[str, Dict], **kwargs: Any
-    ) -> Dict[str, Dict]:
+        self, input: dict[str, dict], **kwargs: Any
+    ) -> dict[str, dict]:
         """Fit and transform.
 
         Parameters

junifer/{markers/tests/test_collection.py → pipeline/tests/test_marker_collection.py}

@@ -1,4 +1,4 @@
-"""Provide tests for marker collection."""
+"""Provide tests for MarkerCollection."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Synchon Mandal <s.mandal@fz-juelich.de>
@@ -12,10 +12,9 @@ from numpy.testing import assert_array_equal
 from junifer.datareader.default import DefaultDataReader
 from junifer.markers import (
     FunctionalConnectivityParcels,
-    MarkerCollection,
     ParcelAggregation,
 )
-from junifer.pipeline import PipelineStepMixin
+from junifer.pipeline import MarkerCollection, PipelineStepMixin
 from junifer.preprocess import fMRIPrepConfoundRemover
 from junifer.storage import SQLiteFeatureStorage
 from junifer.testing.datagrabbers import (