junifer 0.0.7.dev332__py3-none-any.whl → 0.0.8__py3-none-any.whl

junifer/external/nilearn/tests/test_junifer_nifti_spheres_masker.py

@@ -3,7 +3,7 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-import warnings
+from pathlib import Path
 
 import nibabel
 import numpy as np
@@ -14,6 +14,12 @@ from nilearn.maskers import NiftiSpheresMasker
 from numpy.testing import assert_array_equal
 
 from junifer.external.nilearn import JuniferNiftiSpheresMasker
+from junifer.external.nilearn.junifer_nifti_spheres_masker import (
+    _apply_mask_and_get_affinity,
+    _get_sphere_voxels,
+    _iter_signals_from_spheres,
+)
+from junifer.testing import config_override
 
 
 # New BSD License
@@ -262,13 +268,16 @@ def test_standardization() -> None:
     data = np.random.RandomState(42).random_sample((3, 3, 3, 5))
     img = nibabel.Nifti1Image(data, np.eye(4))
 
-    # test zscore
-    masker = JuniferNiftiSpheresMasker(seeds=[(1, 1, 1)], standardize="zscore")
+    # test zscore_sample
+    masker = JuniferNiftiSpheresMasker(
+        seeds=[(1, 1, 1)], standardize="zscore_sample"
+    )
     # Test the fit
     s = masker.fit_transform(img)
 
     np.testing.assert_almost_equal(s.mean(), 0)
-    np.testing.assert_almost_equal(s.std(), 1)
+    # Uses the sample standard deviation
+    np.testing.assert_almost_equal(s.std(ddof=1), 1)
 
     # test psc
     masker = JuniferNiftiSpheresMasker(seeds=[(1, 1, 1)], standardize="psc")
@@ -294,10 +303,10 @@ def test_nifti_spheres_masker_inverse_transform() -> None:
 
 
 def test_nifti_spheres_masker_io_shapes() -> None:
-    """Ensure that masker handles 1D/2D/3D/4D data appropriately.
+    """Ensure that masker handles 3D/4D data appropriately.
 
-    transform(4D image) --> 2D output, no warning
-    transform(3D image) --> 2D output, DeprecationWarning
+    transform(4D image) --> 2D output
+    transform(3D image) --> 1D output
 
     """
     n_regions, n_volumes = 2, 5
@@ -318,20 +327,11 @@ def test_nifti_spheres_masker_io_shapes() -> None:
     )
     masker.fit()
 
-    # DeprecationWarning *should* be raised for 3D inputs
-    with pytest.warns(DeprecationWarning, match=r"Starting in version 0.12"):
-        test_data = masker.transform(img_3d)
-        assert test_data.shape == (1, n_regions)
-
-    # DeprecationWarning should *not* be raised for 4D inputs
-    with warnings.catch_warnings():
-        warnings.filterwarnings(
-            "error",
-            message="Starting in version 0.12",
-            category=DeprecationWarning,
-        )
-        test_data = masker.transform(img_4d)
-        assert test_data.shape == (n_volumes, n_regions)
+    test_data = masker.transform(img_3d)
+    assert test_data.shape == (n_regions,)
+
+    test_data = masker.transform(img_4d)
+    assert test_data.shape == (n_volumes, n_regions)
 
 
 @pytest.mark.parametrize(
@@ -405,3 +405,213 @@ def test_junifer_and_nilearn_mean_agg_are_equal(
     # Checks
     assert junifer_output.shape == nilearn_output.shape
     np.testing.assert_almost_equal(junifer_output, nilearn_output)
+
+
+def _spheres_data(
+    tmp_path: Path, dtype: str = "float32", with_nan: bool = True
+) -> tuple[nibabel.Nifti1Image, nibabel.Nifti1Image, list]:
+    """Create a file-backed 4D image, a mask and seeds for the spheres."""
+    rng = np.random.default_rng(42)
+    affine = np.diag([3.0, 3.0, 3.0, 1.0])
+    affine[:3, 3] = -30
+    data = rng.normal(100, 10, size=(21, 21, 21, 23))
+    if with_nan:
+        data[10, 10, 10, 3] = np.nan
+    nibabel.save(
+        nibabel.Nifti1Image(data.astype(dtype), affine), tmp_path / "data.nii"
+    )
+    mask = np.zeros((21, 21, 21), dtype="int8")
+    mask[4:17, 3:18, 5:16] = 1
+    mask_img = nibabel.Nifti1Image(mask, affine)
+    # The last seed is outside the image, so its sphere is empty
+    seeds = [(0, 0, 0), (12, -9, 6), (-24, 24, 24), (100, 100, 100)]
+    return nibabel.load(tmp_path / "data.nii"), mask_img, seeds
+
+
+@pytest.mark.parametrize("chunk_size", [1, 5, 23, 100])
+@pytest.mark.parametrize("masked", [True, False])
+@pytest.mark.parametrize(
+    "dtype, with_nan",
+    [
+        ("float32", True),
+        ("float32", False),
+        ("float64", True),
+        ("int16", False),
+    ],
+)
+def test_iter_signals_from_spheres_chunks(
+    tmp_path: Path, chunk_size: int, masked: bool, dtype: str, with_nan: bool
+) -> None:
+    """Test the signals of the spheres read in chunks of volumes.
+
+    They are the same as the ones extracted from the whole data.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    chunk_size : int
+        The parametrized number of volumes to read at once.
+    masked : bool
+        The parametrized use of a mask.
+    dtype : str
+        The parametrized data type.
+    with_nan : bool
+        The parametrized presence of a NaN value in the data.
+
+    """
+    img, mask_img, seeds = _spheres_data(tmp_path, dtype, with_nan)
+    mask_img = mask_img if masked else None
+    X, A = _apply_mask_and_get_affinity(
+        seeds, img, radius=4.0, allow_overlap=True, mask_img=mask_img
+    )
+    with config_override("markers.spheres.chunksize", chunk_size):
+        signals = list(
+            _iter_signals_from_spheres(
+                seeds, img, radius=4.0, allow_overlap=True, mask_img=mask_img
+            )
+        )
+    assert len(signals) == len(seeds)
+    assert signals[-1].shape == (23, 0)
+    for row, t_signals in zip(A.rows, signals, strict=True):
+        assert t_signals.dtype == X.dtype
+        assert_array_equal(t_signals, X[:, row])
+
+
+def test_JuniferNiftiSpheresMasker_input_unchanged(tmp_path: Path) -> None:
+    """Test the masker does not change the input data.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    img, _, seeds = _spheres_data(tmp_path)
+    data = np.asarray(img.dataobj).copy()
+    in_memory = nibabel.Nifti1Image(data.copy(), img.affine)
+    with pytest.warns(RuntimeWarning, match="NaN values"):
+        JuniferNiftiSpheresMasker(
+            seeds=seeds, radius=4.0, allow_overlap=True
+        ).fit_transform(in_memory)
+    # The NaN is still there
+    assert_array_equal(np.asarray(in_memory.dataobj), data)
+
+
+@pytest.mark.parametrize("chunk_size", [0, -1, 1.5, True, "10"])
+def test_JuniferNiftiSpheresMasker_chunk_size_error(
+    tmp_path: Path, chunk_size: object
+) -> None:
+    """Test the masker with an invalid number of volumes to read at once.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    chunk_size : object
+        The parametrized invalid number of volumes.
+
+    """
+    img, _, seeds = _spheres_data(tmp_path)
+    with config_override("markers.spheres.chunksize", chunk_size):
+        with pytest.raises(ValueError, match="must be a positive integer"):
+            JuniferNiftiSpheresMasker(seeds=seeds, radius=4.0).fit_transform(
+                img
+            )
+
+
+def test_JuniferNiftiSpheresMasker_reports() -> None:
+    """Test the reports are disabled by default, as they load the data."""
+    assert JuniferNiftiSpheresMasker(seeds=[(1, 1, 1)]).reports is False
+    masker = JuniferNiftiSpheresMasker(seeds=[(1, 1, 1)], reports=True)
+    assert masker.reports is True
+
+
+def test_get_sphere_voxels_mask_only(tmp_path: Path) -> None:
+    """Test the voxels of the spheres with only a mask, without an image.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    img, mask_img, seeds = _spheres_data(tmp_path)
+    voxels, A = _get_sphere_voxels(
+        seeds, img, radius=4.0, allow_overlap=True, mask_img=mask_img
+    )
+    mask_voxels, mask_A = _get_sphere_voxels(
+        seeds, None, radius=4.0, allow_overlap=True, mask_img=mask_img
+    )
+    assert_array_equal(mask_voxels, voxels)
+    assert mask_A.rows.tolist() == A.rows.tolist()
+    # No signals without an image
+    X, _ = _apply_mask_and_get_affinity(
+        seeds, None, radius=4.0, allow_overlap=True, mask_img=mask_img
+    )
+    assert X is None
+    with pytest.raises(ValueError, match="Either a niimg or a mask_img"):
+        _get_sphere_voxels(seeds, None, radius=4.0, allow_overlap=True)
+
+
+@pytest.mark.parametrize("dtype", ["auto", "float64"])
+def test_JuniferNiftiSpheresMasker_dtype(tmp_path: Path, dtype: str) -> None:
+    """Test the data type of the signals is the same as with nilearn.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    dtype : str
+        The parametrized data type of the extraction.
+
+    """
+    img, _, seeds = _spheres_data(tmp_path, "float32", with_nan=False)
+    junifer_signals = JuniferNiftiSpheresMasker(
+        seeds=seeds[:3], radius=4.0, allow_overlap=True, dtype=dtype
+    ).fit_transform(img)
+    nilearn_signals = NiftiSpheresMasker(
+        seeds=seeds[:3], radius=4.0, allow_overlap=True, dtype=dtype
+    ).fit_transform(img)
+    assert junifer_signals.dtype == nilearn_signals.dtype
+    np.testing.assert_allclose(junifer_signals, nilearn_signals, rtol=1e-6)
+
+
+@pytest.mark.parametrize("chunk_size", [5, 100])
+@pytest.mark.parametrize("masked", [True, False])
+@pytest.mark.parametrize("with_nan", [True, False])
+def test_junifer_and_nilearn_spheres_are_equal(
+    tmp_path: Path, chunk_size: int, masked: bool, with_nan: bool
+) -> None:
+    """Test the signals of the spheres are the same as with nilearn.
+
+    The signals are read in chunks of volumes from a file-backed image.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    chunk_size : int
+        The parametrized number of volumes to read at once.
+    masked : bool
+        The parametrized use of a mask.
+    with_nan : bool
+        The parametrized presence of a NaN value in the data.
+
+    """
+    img, mask_img, seeds = _spheres_data(tmp_path, with_nan=with_nan)
+    # Only the seeds with voxels in the mask, as nilearn fails for empty
+    # spheres
+    params = {
+        "seeds": seeds[:2],
+        "radius": 4.0,
+        "allow_overlap": True,
+        "mask_img": mask_img if masked else None,
+    }
+    with config_override("markers.spheres.chunksize", chunk_size):
+        junifer_signals = JuniferNiftiSpheresMasker(**params).fit_transform(
+            img
+        )
+    nilearn_signals = NiftiSpheresMasker(**params).fit_transform(img)
+    assert junifer_signals.dtype == nilearn_signals.dtype
+    np.testing.assert_allclose(junifer_signals, nilearn_signals, rtol=1e-6)

junifer/markers/__init__.pyi

@@ -11,9 +11,11 @@ __all__ = [
     "EdgeCentricFCMaps",
     "EdgeCentricFCParcels",
     "EdgeCentricFCSpheres",
+    "ReHoImpl",
     "ReHoMaps",
     "ReHoParcels",
     "ReHoSpheres",
+    "ALFFImpl",
     "ALFFMaps",
     "ALFFParcels",
     "ALFFSpheres",
@@ -21,9 +23,10 @@ __all__ = [
     "TemporalSNRParcels",
     "TemporalSNRSpheres",
     "BrainPrint",
+    "logger",
 ]
 
-from .base import BaseMarker
+from .base import BaseMarker, logger
 from .ets_rss import RSSETSMarker
 from .maps_aggregation import MapsAggregation
 from .parcel_aggregation import ParcelAggregation
@@ -37,8 +40,8 @@ from .functional_connectivity import (
     EdgeCentricFCParcels,
     EdgeCentricFCSpheres,
 )
-from .reho import ReHoMaps, ReHoParcels, ReHoSpheres
-from .falff import ALFFMaps, ALFFParcels, ALFFSpheres
+from .reho import ReHoImpl, ReHoMaps, ReHoParcels, ReHoSpheres
+from .falff import ALFFImpl, ALFFMaps, ALFFParcels, ALFFSpheres
 from .temporal_snr import (
     TemporalSNRMaps,
     TemporalSNRParcels,

junifer/markers/base.py

@@ -6,17 +6,27 @@
 
 from abc import ABC, abstractmethod
 from copy import deepcopy
-from typing import Any, ClassVar, Optional, Union
+from typing import Annotated, Any, ClassVar
 
+import structlog
+from pydantic import BaseModel, BeforeValidator, ConfigDict
+
+from ..datagrabber import DataType
 from ..pipeline import PipelineStepMixin, UpdateMetaMixin
+from ..storage import StorageType
 from ..typing import MarkerInOutMappings, StorageLike
-from ..utils import logger, raise_error
+from ..utils import ensure_list_or_none, raise_error
+from .utils import _changed_paths, _fingerprint
 
 
 __all__ = ["BaseMarker"]
 
 
-class BaseMarker(ABC, PipelineStepMixin, UpdateMetaMixin):
+_log = structlog.get_logger("junifer")
+logger = _log.bind(pkg="markers", step="marker")
+
+
+class BaseMarker(BaseModel, ABC, PipelineStepMixin, UpdateMetaMixin):
     """Abstract base class for marker.
 
     For every marker, one needs to provide a concrete
@@ -24,12 +34,17 @@ class BaseMarker(ABC, PipelineStepMixin, UpdateMetaMixin):
 
     Parameters
     ----------
-    on : str or list of str or None, optional
-        The data type to apply the marker on. If None,
-        will work on all available data types (default None).
-    name : str, optional
-        The name of the marker. If None, will use the class name as the
-        name of the marker (default None).
+    on : :enum:`.DataType` or list of variants or None, optional
+        The data type(s) to apply the marker on.
+        If None, will work on all available data types.
+        Check :enum:`.DataType` for valid values (default None).
+    name : str or None, optional
+        The name of the marker.
+        If None, will use the class name (default None).
+
+    Attributes
+    ----------
+    valid_inputs
 
     Raises
     ------
@@ -42,11 +57,15 @@ class BaseMarker(ABC, PipelineStepMixin, UpdateMetaMixin):
 
     _MARKER_INOUT_MAPPINGS: ClassVar[MarkerInOutMappings]
 
-    def __init__(
-        self,
-        on: Optional[Union[list[str], str]] = None,
-        name: Optional[str] = None,
-    ) -> None:
+    model_config = ConfigDict(extra="forbid", use_enum_values=True)
+
+    on: Annotated[
+        DataType | list[DataType] | None,
+        BeforeValidator(ensure_list_or_none),
+    ] = None
+    name: str | None = None
+
+    def model_post_init(self, context: Any):  # noqa: D102
         # Check for missing mapping attribute
         if not hasattr(self, "_MARKER_INOUT_MAPPINGS"):
             raise_error(
@@ -54,18 +73,51 @@ class BaseMarker(ABC, PipelineStepMixin, UpdateMetaMixin):
                 klass=AttributeError,
             )
         # Use all data types if not provided
-        if on is None:
-            on = self.get_valid_inputs()
-        # Convert data types to list
-        if not isinstance(on, list):
-            on = [on]
+        if self.on is None:
+            self.on = self.valid_inputs
+        else:
+            # Convert to correct data type
+            self.on = [
+                DataType(t) if isinstance(t, str) else t for t in self.on
+            ]
+            # Check if required input data types are provided
+            if any(x not in self.valid_inputs for x in self.on):
+                wrong_on = [
+                    x.value for x in self.on if x not in self.valid_inputs
+                ]
+                raise_error(
+                    f"{self.__class__.__name__} cannot be computed on "
+                    f"{wrong_on}"
+                )
+        # Run extra validation for markers and fail early if needed
+        self.validate_marker_params()
         # Set default name if not provided
-        self.name = self.__class__.__name__ if name is None else name
-        # Check if required inputs are found
-        if any(x not in self.get_valid_inputs() for x in on):
-            wrong_on = [x for x in on if x not in self.get_valid_inputs()]
-            raise_error(f"{self.name} cannot be computed on {wrong_on}")
-        self._on = on
+        self.name = self.__class__.__name__ if self.name is None else self.name
+        logger.info(
+            f"Parameters: {self.model_dump(mode='json')}", component=self.name
+        )
+
+    @property
+    def valid_inputs(self) -> list[DataType]:
+        """Valid data types to operate on.
+
+        Returns
+        -------
+        list of :enum:`.DataType`
+            The list of data types that can be used as input for this marker.
+
+        """
+        return [
+            DataType(x) if isinstance(x, str) else x
+            for x in self._MARKER_INOUT_MAPPINGS.keys()
+        ]
+
+    def validate_marker_params(self) -> None:
+        """Run extra logical validation for marker.
+
+        Subclasses can override to provide validation.
+        """
+        pass
 
     def validate_input(self, input: list[str]) -> list[str]:
         """Validate input.
@@ -88,45 +140,36 @@ class BaseMarker(ABC, PipelineStepMixin, UpdateMetaMixin):
             If the input does not have the required data.
 
         """
-        if not any(x in input for x in self._on):
+        if not any(x in input for x in self.on):
             raise_error(
-                "Input does not have the required data."
-                f"\t Input: {input}"
-                f"\t Required (any of): {self._on}"
+                "Input does not have the required data.\n"
+                f"\t Input: {input}\n"
+                f"\t Required (any of): {[t.value for t in self.on]}"
             )
-        return [x for x in self._on if x in input]
+        return [x.value for x in self.on if x in input]
 
-    def get_valid_inputs(self) -> list[str]:
-        """Get valid data types for input.
-
-        Returns
-        -------
-        list of str
-            The list of data types that can be used as input for this marker.
-
-        """
-        return list(self._MARKER_INOUT_MAPPINGS.keys())
-
-    def storage_type(self, input_type: str, output_feature: str) -> str:
-        """Get storage type for a feature.
+    def storage_type(
+        self, input_type: DataType, output_feature: str
+    ) -> StorageType:
+        """Get :enum:`.StorageType` for a feature.
 
         Parameters
         ----------
-        input_type : str
+        input_type : :enum:`.DataType`
             The data type input to the marker.
         output_feature : str
             The feature output of the marker.
 
         Returns
         -------
-        str
+        :enum:`.StorageType`
             The storage type output of the marker.
 
         """
         return self._MARKER_INOUT_MAPPINGS[input_type][output_feature]
 
     @abstractmethod
-    def compute(self, input: dict, extra_input: Optional[dict] = None) -> dict:
+    def compute(self, input: dict, extra_input: dict | None = None) -> dict:
         """Compute.
 
         Parameters
@@ -155,33 +198,33 @@ class BaseMarker(ABC, PipelineStepMixin, UpdateMetaMixin):
 
     def store(
         self,
-        type_: str,
+        data_type: DataType,
         feature: str,
-        out: dict[str, Any],
+        output: dict[str, Any],
         storage: StorageLike,
     ) -> None:
         """Store.
 
         Parameters
         ----------
-        type_ : str
+        data_type : :enum:`.DataType`
             The data type to store.
         feature : str
             The feature to store.
-        out : dict
+        output : dict
             The computed result as a dictionary to store.
         storage : storage-like
-            The storage class, for example, SQLiteFeatureStorage.
+            The storage class, for example, HDF5FeatureStorage.
 
         """
-        output_type_ = self.storage_type(type_, feature)
-        logger.debug(f"Storing {output_type_} in {storage}")
-        storage.store(kind=output_type_, **out)
+        s_type = self.storage_type(data_type, feature)
+        logger.debug(f"Storing {s_type} in {storage}")
+        storage.store(kind=s_type, **output)
 
     def _fit_transform(
         self,
         input: dict[str, dict],
-        storage: Optional[StorageLike] = None,
+        storage: StorageLike | None = None,
     ) -> dict:
         """Fit and transform.
 
@@ -190,63 +233,70 @@ class BaseMarker(ABC, PipelineStepMixin, UpdateMetaMixin):
         input : dict
             The Junifer Data object.
         storage : storage-like, optional
-            The storage class, for example, SQLiteFeatureStorage.
+            The storage class, for example, HDF5FeatureStorage.
 
         Returns
         -------
         dict
-            The processed output as a dictionary. If `storage` is provided,
+            The processed output as a dictionary. If ``storage`` is provided,
             empty dictionary is returned.
 
         """
         out = {}
-        for type_ in self._on:
-            if type_ in input.keys():
-                logger.info(f"Computing {type_}")
+        for t in self.on:
+            if t in input.keys():
+                logger.info(f"Computing {t}")
                 # Get data dict for data type
-                t_input = input[type_]
+                t_input = input[t]
                 # Pass the other data types as extra input, removing
                 # the current type
                 extra_input = input.copy()
-                extra_input.pop(type_)
+                extra_input.pop(t)
                 logger.debug(
                     f"Extra data type for feature extraction: "
                     f"{extra_input.keys()}"
                 )
                 # Copy metadata
                 t_meta = t_input["meta"].copy()
-                t_meta["type"] = type_
-                # Compute marker
+                t_meta["type"] = t.value
+                # Compute marker, checking it does not change the data, as
+                # it is shared with the markers computed afterwards
+                fingerprint = _fingerprint(input)
                 t_out = self.compute(input=t_input, extra_input=extra_input)
+                changed = _changed_paths(fingerprint, _fingerprint(input))
+                if changed:
+                    raise_error(
+                        f"Marker {self.name} changed its input data, which is "
+                        f"shared with the other markers: {', '.join(changed)}",
+                        klass=RuntimeError,
+                    )
                 # Initialize empty dictionary if no storage object is provided
                 if storage is None:
-                    out[type_] = {}
+                    out[t] = {}
                 # Store individual features
-                for feature_name, feature_data in t_out.items():
+                for f_name, f_data in t_out.items():
                     # Make deep copy of the feature data for manipulation
-                    feature_data_copy = deepcopy(feature_data)
+                    f_data_copy = deepcopy(f_data)
                     # Make deep copy of metadata and add to feature data
-                    feature_data_copy["meta"] = deepcopy(t_meta)
+                    f_data_copy["meta"] = deepcopy(t_meta)
                     # Update metadata for the feature,
                     # feature data is not manipulated, only meta
-                    self.update_meta(feature_data_copy, "marker")
+                    self.update_meta(f_data_copy, "marker")
                     # Update marker feature's metadata name
-                    feature_data_copy["meta"]["marker"]["name"] += (
-                        f"_{feature_name}"
-                    )
+                    f_data_copy["meta"]["marker"]["name"] += f"_{f_name}"
 
                     if storage is not None:
                         logger.info(f"Storing in {storage}")
                         self.store(
-                            type_=type_,
-                            feature=feature_name,
-                            out=feature_data_copy,
+                            data_type=t,
+                            feature=f_name,
+                            output=f_data_copy,
                             storage=storage,
                         )
                     else:
                         logger.info(
                             "No storage specified, returning dictionary"
                         )
-                        out[type_][feature_name] = feature_data_copy
+                        out[t][f_name] = f_data_copy
 
         return out