junifer 0.0.7.dev332__py3-none-any.whl → 0.0.8__py3-none-any.whl

junifer/pipeline/__init__.pyi

@@ -3,11 +3,12 @@ __all__ = [
     "AssetLoaderDispatcher",
     "BaseDataDumpAsset",
     "DataObjectDumper",
+    "ExtDep",
+    "MarkerCollection",
     "PipelineComponentRegistry",
     "PipelineStepMixin",
     "UpdateMetaMixin",
     "WorkDirManager",
-    "MarkerCollection",
 ]
 
 from ._data_object_dumper import (
@@ -16,8 +17,9 @@ from ._data_object_dumper import (
     BaseDataDumpAsset,
     DataObjectDumper,
 )
+from .marker_collection import MarkerCollection
 from .pipeline_component_registry import PipelineComponentRegistry
 from .pipeline_step_mixin import PipelineStepMixin
 from .update_meta_mixin import UpdateMetaMixin
+from .utils import ExtDep
 from .workdir_manager import WorkDirManager
-from .marker_collection import MarkerCollection

junifer/pipeline/marker_collection.py

@@ -6,16 +6,20 @@
 
 from collections import Counter
 from pathlib import Path
-from typing import Optional
+
+import structlog
 
 from ..datareader import DefaultDataReader
 from ..pipeline import DataObjectDumper, PipelineStepMixin, WorkDirManager
 from ..typing import DataGrabberLike, MarkerLike, PreprocessorLike, StorageLike
-from ..utils import config, logger, raise_error
+from ..utils import config, raise_error
 
 
 __all__ = ["MarkerCollection"]
 
+_log = structlog.get_logger("junifer")
+logger = _log.bind(pkg="pipeline")
+
 
 class MarkerCollection:
     """Class for marker collection.
@@ -41,9 +45,9 @@ class MarkerCollection:
     def __init__(
         self,
         markers: list[MarkerLike],
-        datareader: Optional[PipelineStepMixin] = None,
-        preprocessors: Optional[list[PreprocessorLike]] = None,
-        storage: Optional[StorageLike] = None,
+        datareader: PipelineStepMixin | None = None,
+        preprocessors: list[PreprocessorLike] | None = None,
+        storage: StorageLike | None = None,
     ):
         # Check that the markers have different names
         marker_names = [m.name for m in markers]
@@ -60,7 +64,7 @@ class MarkerCollection:
         self._preprocessors = preprocessors
         self._storage = storage
 
-    def fit(self, input: dict[str, dict]) -> Optional[dict]:
+    def fit(self, input: dict[str, dict]) -> dict | None:
         """Fit the pipeline.
 
         Parameters
@@ -162,7 +166,7 @@ class MarkerCollection:
         logger.info(f"DataGrabber output type: {t_data}")
 
         logger.info("Validating Data Reader:")
-        t_data = self._datareader.validate(t_data)
+        t_data = self._datareader.validate_component(t_data)
         logger.info(f"Data Reader output type: {t_data}")
 
         if self._preprocessors is not None:
@@ -175,7 +179,7 @@ class MarkerCollection:
                 old_t_data = t_data.copy()
                 logger.info(f"Preprocessor input type: {t_data}")
                 # Validate preprocessor
-                new_t_data = preprocessor.validate(old_t_data)
+                new_t_data = preprocessor.validate_component(old_t_data)
                 # Set new data types
                 t_data = list(set(old_t_data) | set(new_t_data))
                 logger.info(f"Preprocessor output type: {t_data}")
@@ -183,10 +187,10 @@ class MarkerCollection:
         for marker in self._markers:
             logger.info(f"Validating Marker: {marker.name}")
             # Validate marker
-            m_data = marker.validate(input=t_data)
+            m_data = marker.validate_component(input=t_data)
             logger.info(f"Marker output type: {m_data}")
             # Check storage for the marker
             if self._storage is not None:
                 logger.info(f"Validating storage for {marker.name}")
                 # Validate storage
-                self._storage.validate(input_=m_data)
+                self._storage.validate_input(input_=m_data)

junifer/pipeline/pipeline_component_registry.py

@@ -7,15 +7,19 @@
 
 import importlib
 from collections.abc import Mapping
-from typing import Optional, Union
+
+import structlog
 
 from ..typing import DataGrabberLike, MarkerLike, PreprocessorLike, StorageLike
-from ..utils import logger, raise_error
+from ..utils import raise_error
 from ..utils.singleton import Singleton
 
 
 __all__ = ["PipelineComponentRegistry"]
 
+_log = structlog.get_logger("junifer")
+logger = _log.bind(pkg="pipeline")
+
 
 class PipelineComponentRegistry(metaclass=Singleton):
     """Class for pipeline component registry.
@@ -79,24 +83,30 @@ class PipelineComponentRegistry(metaclass=Singleton):
                 "TemporalFilter": "TemporalFilter",
             },
             "marker": {
+                "ALFFMaps": "ALFFMaps",
                 "ALFFParcels": "ALFFParcels",
                 "ALFFSpheres": "ALFFSpheres",
                 "BaseMarker": "BaseMarker",
                 "BrainPrint": "BrainPrint",
                 "CrossParcellationFC": "CrossParcellationFC",
+                "EdgeCentricFCMaps": "EdgeCentricFCMaps",
                 "EdgeCentricFCParcels": "EdgeCentricFCParcels",
                 "EdgeCentricFCSpheres": "EdgeCentricFCSpheres",
+                "FunctionalConnectivityMaps": "FunctionalConnectivityMaps",
                 "FunctionalConnectivityParcels": (
                     "FunctionalConnectivityParcels"
                 ),
                 "FunctionalConnectivitySpheres": (
                     "FunctionalConnectivitySpheres"
                 ),
+                "MapsAggregation": "MapsAggregation",
                 "ParcelAggregation": "ParcelAggregation",
+                "ReHoMaps": "ReHoMaps",
                 "ReHoParcels": "ReHoParcels",
                 "ReHoSpheres": "ReHoSpheres",
                 "RSSETSMarker": "RSSETSMarker",
                 "SphereAggregation": "SphereAggregation",
+                "TemporalSNRMaps": "TemporalSNRMaps",
                 "TemporalSNRParcels": "TemporalSNRParcels",
                 "TemporalSNRSpheres": "TemporalSNRSpheres",
             },
@@ -119,7 +129,7 @@ class PipelineComponentRegistry(metaclass=Singleton):
         return self._steps
 
     @property
-    def components(self) -> Mapping[str, Mapping[str, Union[str, type]]]:
+    def components(self) -> Mapping[str, Mapping[str, str | type]]:
         """Get registered components for valid pipeline steps."""
         return self._components
 
@@ -238,8 +248,8 @@ class PipelineComponentRegistry(metaclass=Singleton):
         step: str,
         name: str,
         baseclass: type,
-        init_params: Optional[dict] = None,
-    ) -> Union[DataGrabberLike, PreprocessorLike, MarkerLike, StorageLike]:
+        init_params: dict | None = None,
+    ) -> DataGrabberLike | PreprocessorLike | MarkerLike | StorageLike:
         """Build an instance of class registered as ``name``.
 
         Parameters
@@ -277,7 +287,16 @@ class PipelineComponentRegistry(metaclass=Singleton):
         logger.debug(f"\tInit params: {init_params}")
         try:
             # Create instance of the class
-            object_ = klass(**init_params)
+            if step == "datagrabber" and name == "MultipleDataGrabber":
+                dgs = []
+                for d in init_params["datagrabbers"]:
+                    k = d.pop("kind")
+                    dg_klass = self.get_class(step=step, name=k)
+                    dg_obj = dg_klass(**d)
+                    dgs.append(dg_obj)
+                object_ = klass(datagrabbers=dgs)
+            else:
+                object_ = klass(**init_params)
         except (ValueError, TypeError) as e:
             raise_error(
                 msg=f"Failed to create {step} ({name}). Error: {e}",

junifer/pipeline/pipeline_step_mixin.py

@@ -4,15 +4,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-import sys
-
-
-if sys.version_info < (3, 11):  # pragma: no cover
-    from importlib_metadata import packages_distributions
-else:
-    from importlib.metadata import packages_distributions
-
-
+from importlib.metadata import packages_distributions
 from importlib.util import find_spec
 from itertools import chain
 from typing import Any
@@ -75,8 +67,8 @@ class PipelineStepMixin:
             klass=NotImplementedError,
         )  # pragma: no cover
 
-    def validate(self, input: list[str]) -> list[str]:
-        """Validate the the pipeline step.
+    def validate_component(self, input: list[str]) -> list[str]:
+        """Validate the pipeline component.
 
         Parameters
         ----------
@@ -228,5 +220,6 @@ class PipelineStepMixin:
             The processed output of the pipeline step.
 
         """
-        self.validate(input=list(input.keys()))
+        # Needs to be validated if called directly via API
+        self.validate_component(input=list(input.keys()))
         return self._fit_transform(input=input, **kwargs)

junifer/pipeline/tests/test_data_object_dumper.py

@@ -5,7 +5,6 @@
 
 import pickle
 from pathlib import Path
-from typing import Union
 
 import nibabel
 import pytest
@@ -23,6 +22,9 @@ from junifer.testing.datagrabbers import PartlyCloudyTestingDataGrabber
 from junifer.utils import config
 
 
+pytestmark = pytest.mark.external
+
+
 @pytest.mark.parametrize(
     "dispatcher, inbuilt_key, ext_key, val",
     [
@@ -36,9 +38,9 @@ from junifer.utils import config
     ],
 )
 def test_dispatcher_addition_errors(
-    dispatcher: Union[AssetDumperDispatcher, AssetLoaderDispatcher],
-    inbuilt_key: Union[str, type],
-    ext_key: Union[str, type],
+    dispatcher: AssetDumperDispatcher | AssetLoaderDispatcher,
+    inbuilt_key: str | type,
+    ext_key: str | type,
     val: type,
 ) -> None:
     """Test asset dumper / loader addition errors.
@@ -70,9 +72,9 @@ def test_dispatcher_addition_errors(
     ],
 )
 def test_dispatcher_removal_errors(
-    dispatcher: Union[AssetDumperDispatcher, AssetLoaderDispatcher],
-    inbuilt_key: Union[str, type],
-    ext_key: Union[str, type],
+    dispatcher: AssetDumperDispatcher | AssetLoaderDispatcher,
+    inbuilt_key: str | type,
+    ext_key: str | type,
 ) -> None:
     """Test asset dumper / loader removal errors.
 
@@ -157,9 +159,7 @@ def test_data_object_dumper(
                     "wm_csf": "full",
                 },
                 detrend=True,
-                standardize=True,
-                low_pass=0.08,
-                high_pass=0.01,
+                standardize="zscore_sample",
             ),
         ],
         markers=[

junifer/pipeline/tests/test_marker_collection.py

@@ -6,141 +6,145 @@
 
 from pathlib import Path
 
+import numpy as np
 import pytest
-from numpy.testing import assert_array_equal
+from numpy.testing import assert_allclose, assert_array_equal
 
-from junifer.datareader.default import DefaultDataReader
-from junifer.markers import (
-    FunctionalConnectivityParcels,
-    ParcelAggregation,
-)
+from junifer.datagrabber import DataType
+from junifer.datareader import DefaultDataReader
+from junifer.markers import FunctionalConnectivityParcels, ParcelAggregation
+from junifer.markers.base import BaseMarker
 from junifer.pipeline import MarkerCollection, PipelineStepMixin
 from junifer.preprocess import fMRIPrepConfoundRemover
-from junifer.storage import SQLiteFeatureStorage
-from junifer.testing.datagrabbers import (
-    PartlyCloudyTestingDataGrabber,
-)
+from junifer.storage import HDF5FeatureStorage, StorageType
+from junifer.testing.datagrabbers import PartlyCloudyTestingDataGrabber
 
 
-def test_marker_collection_incorrect_markers() -> None:
-    """Test incorrect markers for MarkerCollection."""
-    wrong_markers = [
-        ParcelAggregation(
-            parcellation="Schaefer100x7",
-            method="mean",
-            name="gmd_schaefer100x7_mean",
-        ),
-        ParcelAggregation(
-            parcellation="Schaefer100x7",
-            method="mean",
-            name="gmd_schaefer100x7_mean",
-        ),
-    ]
-    with pytest.raises(ValueError, match=r"must have different names"):
-        MarkerCollection(wrong_markers)  # type: ignore
+pytestmark = pytest.mark.external
 
+PARCELLATION = "TianxS2x3TxMNInonlinear2009cAsym"
 
-def test_marker_collection() -> None:
-    """Test MarkerCollection."""
-    markers = [
+
+def _markers() -> list[ParcelAggregation]:
+    """Get markers aggregating the BOLD data in different ways.
+
+    Returns
+    -------
+    list of ParcelAggregation
+        The markers, named ``tian_mean``, ``tian_std`` and
+        ``tian_trim_mean90``.
+
+    """
+    return [
         ParcelAggregation(
-            parcellation="TianxS2x3TxMNInonlinear2009cAsym",
-            method="mean",
-            name="tian_mean",
+            parcellation=PARCELLATION, method="mean", name="tian_mean"
         ),
         ParcelAggregation(
-            parcellation="TianxS2x3TxMNInonlinear2009cAsym",
-            method="std",
-            name="tian_std",
+            parcellation=PARCELLATION, method="std", name="tian_std"
         ),
         ParcelAggregation(
-            parcellation="TianxS2x3TxMNInonlinear2009cAsym",
+            parcellation=PARCELLATION,
             method="trim_mean",
             method_params={"proportiontocut": 0.1},
             name="tian_trim_mean90",
         ),
     ]
+
+
+def _fit(mc: MarkerCollection) -> dict | None:
+    """Validate and fit a marker collection on one subject.
+
+    Parameters
+    ----------
+    mc : MarkerCollection
+        The marker collection.
+
+    Returns
+    -------
+    dict or None
+        The output of the marker collection.
+
+    """
+    dg = PartlyCloudyTestingDataGrabber(n_timepoints=50)
+    mc.validate(dg)
+    with dg:
+        return mc.fit(dg["sub-01"])
+
+
+def test_marker_collection_same_names() -> None:
+    """Test markers must have different names."""
+    markers = [
+        ParcelAggregation(parcellation=PARCELLATION, method="mean", name="a"),
+        ParcelAggregation(parcellation=PARCELLATION, method="std", name="a"),
+    ]
+    with pytest.raises(ValueError, match=r"must have different names"):
+        MarkerCollection(markers=markers)  # type: ignore
+
+
+def test_marker_collection_defaults() -> None:
+    """Test the defaults of MarkerCollection."""
+    markers = _markers()
     mc = MarkerCollection(markers=markers)  # type: ignore
     assert mc._markers == markers
     assert mc._preprocessors is None
     assert mc._storage is None
     assert isinstance(mc._datareader, DefaultDataReader)
 
-    # Create testing datagrabber
-    dg = PartlyCloudyTestingDataGrabber()
-    mc.validate(dg)
 
-    with dg:
-        input = dg["sub-01"]
-        out = mc.fit(input)
-        assert out is not None
-        assert isinstance(out, dict)
-        assert len(out) == 3
-        assert "tian_mean" in out
-        assert "tian_std" in out
-        assert "tian_trim_mean90" in out
-
-        for t_marker in markers:
-            t_name = t_marker.name
-            assert "BOLD" in out[t_name]
-            t_bold = out[t_name]["BOLD"]["aggregation"]
-            assert "data" in t_bold
-            assert "col_names" in t_bold
-            assert "meta" in t_bold
-
-    # Test preprocessing
-    class BypassPreprocessing(PipelineStepMixin):
+def test_marker_collection_fit() -> None:
+    """Test MarkerCollection returns the output of each marker."""
+    out = _fit(MarkerCollection(markers=_markers()))  # type: ignore
+    assert out is not None
+    assert set(out) == {"tian_mean", "tian_std", "tian_trim_mean90"}
+    for name in out:
+        aggregation = out[name]["BOLD"]["aggregation"]
+        assert {"data", "col_names", "meta"} <= set(aggregation)
+
+
+def test_marker_collection_preprocessing() -> None:
+    """Test MarkerCollection computes the markers on the preprocessed data."""
+
+    class Unchanged(PipelineStepMixin):
+        """Preprocessing that returns the data unchanged."""
+
+        def validate_input(self, input):
+            return input
+
         def fit_transform(self, input):
             return input
 
-    mc2 = MarkerCollection(
-        markers=markers,  # type: ignore
-        preprocessors=[BypassPreprocessing()],  # type: ignore
-        datareader=DefaultDataReader(),
+    out = _fit(MarkerCollection(markers=_markers()))  # type: ignore
+    out_preprocessed = _fit(
+        MarkerCollection(
+            markers=_markers(),  # type: ignore
+            preprocessors=[Unchanged()],  # type: ignore
+        )
     )
-    assert isinstance(mc2._datareader, DefaultDataReader)
-    with dg:
-        input = dg["sub-01"]
-        out2 = mc2.fit(input)
-        assert out2 is not None
-        for t_marker in markers:
-            t_name = t_marker.name
-            assert_array_equal(
-                out[t_name]["BOLD"]["aggregation"]["data"],
-                out2[t_name]["BOLD"]["aggregation"]["data"],
-            )
-
-
-def test_marker_collection_with_preprocessing() -> None:
-    """Test MarkerCollection with preprocessing."""
-    markers = [
-        FunctionalConnectivityParcels(
-            parcellation="Schaefer100x17",
-            agg_method="mean",
-            name="Schaefer100x17_mean_FC",
-        ),
-        FunctionalConnectivityParcels(
-            parcellation="TianxS2x3TxMNInonlinear2009cAsym",
-            agg_method="mean",
-            name="TianxS2x3TxMNInonlinear2009cAsym_mean_FC",
-        ),
-    ]
+    assert out is not None
+    assert out_preprocessed is not None
+    for name in out:
+        assert_array_equal(
+            out[name]["BOLD"]["aggregation"]["data"],
+            out_preprocessed[name]["BOLD"]["aggregation"]["data"],
+        )
+
+
+def test_marker_collection_validate_preprocessing() -> None:
+    """Test MarkerCollection validates with a confound removal step."""
     mc = MarkerCollection(
-        markers=markers,  # type: ignore
+        markers=[  # type: ignore
+            FunctionalConnectivityParcels(
+                parcellation=PARCELLATION, agg_method="mean", name="tian_fc"
+            ),
+        ],
         preprocessors=[fMRIPrepConfoundRemover()],
     )
-    assert mc._markers == markers
     assert mc._preprocessors is not None
-    assert mc._storage is None
-    assert isinstance(mc._datareader, DefaultDataReader)
-
-    # Create testing datagrabber
-    dg = PartlyCloudyTestingDataGrabber(reduce_confounds=False)
-    mc.validate(dg)
+    mc.validate(PartlyCloudyTestingDataGrabber(reduce_confounds=False))
 
 
 def test_marker_collection_storage(tmp_path: Path) -> None:
-    """Test marker collection with storage.
+    """Test MarkerCollection stores the output of each marker.
 
     Parameters
     ----------
@@ -148,74 +152,47 @@ def test_marker_collection_storage(tmp_path: Path) -> None:
         The path to the test directory.
 
     """
-    markers = [
-        ParcelAggregation(
-            parcellation="TianxS2x3TxMNInonlinear2009cAsym",
-            method="mean",
-            name="tian_mean",
-        ),
-        ParcelAggregation(
-            parcellation="TianxS2x3TxMNInonlinear2009cAsym",
-            method="std",
-            name="tian_std",
-        ),
-        ParcelAggregation(
-            parcellation="TianxS2x3TxMNInonlinear2009cAsym",
-            method="trim_mean",
-            method_params={"proportiontocut": 0.1},
-            name="tian_trim_mean90",
-        ),
-    ]
-    # Setup datagrabber
-    dg = PartlyCloudyTestingDataGrabber()
-    # Setup storage
-    storage = SQLiteFeatureStorage(
-        tmp_path / "test_marker_collection_storage.sqlite"
-    )
-    mc = MarkerCollection(
-        markers=markers,  # type: ignore
-        storage=storage,
-        datareader=DefaultDataReader(),
-    )
-    mc.validate(dg)
-    assert mc._storage is not None
-    assert mc._storage.uri == storage.uri
-    with dg:
-        input = dg["sub-01"]
-        out = mc.fit(input)
-        assert out is None
+    storage = HDF5FeatureStorage(uri=tmp_path / "features.hdf5")
+    mc = MarkerCollection(markers=_markers(), storage=storage)  # type: ignore
+    assert mc._storage is storage
+    # Nothing is returned when storing
+    assert _fit(mc) is None
+
+    # The stored features are the same as the ones returned without storage
+    out = _fit(MarkerCollection(markers=_markers()))  # type: ignore
+    assert out is not None
+    features = storage.list_features()
+    assert len(features) == len(out)
+    for md5, feature in features.items():
+        # Features are named "<type>_<marker>_<feature>"
+        name = feature["name"].removeprefix("BOLD_")
+        name = name.removesuffix("_aggregation")
+        aggregation = out[name]["BOLD"]["aggregation"]
+        stored = storage.read_df(feature_md5=md5)
+        # The storage casts to float32
+        assert_allclose(
+            stored[aggregation["col_names"]].to_numpy(),
+            aggregation["data"],
+            rtol=1e-6,
+        )
+
+
+def test_marker_collection_marker_changes_data() -> None:
+    """Test markers cannot change the data shared with the other markers."""
+
+    class ChangingMarker(BaseMarker):
+        """Marker that replaces the BOLD data."""
+
+        _MARKER_INOUT_MAPPINGS = {  # noqa: RUF012
+            DataType.BOLD: {"feat": StorageType.Vector},
+        }
+
+        def compute(self, input, extra_input=None):
+            input["data"] = "changed"
+            return {"feat": {"data": np.zeros((1, 1)), "col_names": ["x"]}}
 
-    mc2 = MarkerCollection(
-        markers=markers,
-        datareader=DefaultDataReader(),  # type: ignore
+    mc = MarkerCollection(
+        markers=[ChangingMarker(name="changing"), *_markers()]  # type: ignore
     )
-    mc2.validate(dg)
-    assert mc2._storage is None
-
-    with dg:
-        input = dg["sub-01"]
-        out = mc2.fit(input)
-
-    features = storage.list_features()
-    assert len(features) == 3
-
-    feature_md5 = next(iter(features.keys()))
-    t_feature = storage.read_df(feature_md5=feature_md5)
-    fname = "tian_mean"
-    t_data = out[fname]["BOLD"]["aggregation"]["data"]  # type: ignore
-    cols = out[fname]["BOLD"]["aggregation"]["col_names"]  # type: ignore
-    assert_array_equal(t_feature[cols].values, t_data)  # type: ignore
-
-    feature_md5 = list(features.keys())[1]
-    t_feature = storage.read_df(feature_md5=feature_md5)
-    fname = "tian_std"
-    t_data = out[fname]["BOLD"]["aggregation"]["data"]  # type: ignore
-    cols = out[fname]["BOLD"]["aggregation"]["col_names"]  # type: ignore
-    assert_array_equal(t_feature[cols].values, t_data)  # type: ignore
-
-    feature_md5 = list(features.keys())[2]
-    t_feature = storage.read_df(feature_md5=feature_md5)
-    fname = "tian_trim_mean90"
-    t_data = out[fname]["BOLD"]["aggregation"]["data"]  # type: ignore
-    cols = out[fname]["BOLD"]["aggregation"]["col_names"]  # type: ignore
-    assert_array_equal(t_feature[cols].values, t_data)  # type: ignore
+    with pytest.raises(RuntimeError, match=r"changing changed .*BOLD.data"):
+        _fit(mc)