junifer 0.0.7.dev332__py3-none-any.whl → 0.0.8__py3-none-any.whl

junifer/external/nilearn/junifer_nifti_spheres_masker.py

@@ -3,22 +3,19 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import TYPE_CHECKING, Callable, Optional, Union
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Union
 
 import numpy as np
 from nilearn import image, masking
-from nilearn._utils.class_inspect import get_params
-from nilearn._utils.niimg import img_data_dtype
-from nilearn._utils.niimg_conversions import (
-    check_niimg_3d,
-    check_niimg_4d,
-    safe_get_data,
-)
+from nilearn._utils.niimg import img_data_dtype, safe_get_data
+from nilearn.image import check_niimg_3d, check_niimg_4d, load_img
 from nilearn.maskers import NiftiSpheresMasker
-from nilearn.maskers.base_masker import _filter_and_extract
+from nilearn.maskers.base_masker import filter_and_extract
 from sklearn import neighbors
+from sklearn.utils.validation import check_is_fitted
 
-from ...utils import raise_error, warn_with_log
+from ...utils import config, raise_error, warn_with_log
 
 
 if TYPE_CHECKING:
@@ -27,6 +24,7 @@ if TYPE_CHECKING:
     from nibabel import Nifti1Image, Nifti2Image
     from numpy.typing import ArrayLike, DTypeLike
     from pandas import DataFrame
+    from scipy.sparse import lil_matrix
 
 
 __all__ = ["JuniferNiftiSpheresMasker"]
@@ -65,14 +63,40 @@ __all__ = ["JuniferNiftiSpheresMasker"]
 # DAMAGE.
 
 
-def _apply_mask_and_get_affinity(
+def _get_chunk_size() -> int:
+    """Get the number of volumes to read at once.
+
+    Returns
+    -------
+    int
+        The value of ``markers.spheres.chunksize`` (default 100).
+
+    Raises
+    ------
+    ValueError
+        If ``markers.spheres.chunksize`` is not a positive integer.
+
+    """
+    chunk_size = config.get("markers.spheres.chunksize", 100)
+    if (
+        isinstance(chunk_size, bool)
+        or not isinstance(chunk_size, int)
+        or chunk_size < 1
+    ):
+        raise_error(
+            "`markers.spheres.chunksize` must be a positive integer, "
+            f"but is {chunk_size}"
+        )
+    return chunk_size
+
+
+def _get_sphere_voxels(
     seeds, niimg, radius, allow_overlap, mask_img=None
-):
-    """Apply mask and get affinity matrix for given seeds.
+) -> tuple[np.ndarray, "lil_matrix"]:
+    """Get the voxels of the spheres at given seed locations.
 
-    Utility function to get only the rows which are occupied by sphere at
-    given seed locations and the provided radius. Rows are in target_affine and
-    target_shape space.
+    The voxels only depend on the geometry (and the mask), so the data of
+    ``niimg`` is not read.
 
     Parameters
     ----------
@@ -82,8 +106,6 @@ def _apply_mask_and_get_affinity(
     niimg : 3D/4D Niimg-like object
         See :ref:`extracting_data`.
         Images to process.
-        If a 3D niimg is provided, a singleton dimension will be added to
-        the output to represent the single scan in the niimg.
     radius : float
         Indicates, in millimeters, the radius for the sphere around the seed.
     allow_overlap : boolean
@@ -95,9 +117,9 @@ def _apply_mask_and_get_affinity(
 
     Returns
     -------
-    X : 2D numpy.ndarray
-        Signal for each brain voxel in the (masked) niimgs.
-        shape: (number of scans, number of voxels)
+    voxels : 1D numpy.ndarray
+        The flat (C order) index in ``niimg`` of each voxel (column of ``A``):
+        the voxels in the mask or all the voxels if there is no mask.
     A : scipy.sparse.lil_matrix
         Contains the boolean indices for each sphere.
         shape: (number of seeds, number of voxels)
@@ -105,54 +127,38 @@ def _apply_mask_and_get_affinity(
     Raises
     ------
     ValueError
-        If ``niimg`` and ``mask_img`` are both provided or
+        If ``niimg`` and ``mask_img`` are both not provided or
         if overlap is detected between spheres.
 
-    Warns
-    -----
-    RuntimeWarning
-        If the provided images contain NaN, they will be converted to zeroes.
-
     """
     seeds = list(seeds)
-
     # Compute world coordinates of all in-mask voxels.
     if niimg is None:
+        if mask_img is None:
+            raise_error("Either a niimg or a mask_img must be provided.")
         mask, affine = masking.load_mask_img(mask_img)
+        shape = mask.shape
         # Get coordinate for all voxels inside of mask
         mask_coords = np.asarray(np.nonzero(mask)).T.tolist()
-        X = None
-
     elif mask_img is not None:
         affine = niimg.affine
+        shape = niimg.shape[:3]
         mask_img = check_niimg_3d(mask_img)
         mask_img = image.resample_img(
             mask_img,
             target_affine=affine,
-            target_shape=niimg.shape[:3],
+            target_shape=shape,
             interpolation="nearest",
         )
         mask, _ = masking.load_mask_img(mask_img)
-        mask_coords = list(zip(*np.where(mask != 0)))
-
-        X = masking.apply_mask_fmri(niimg, mask_img)
-
-    elif niimg is not None:
-        affine = niimg.affine
-        if np.isnan(np.sum(safe_get_data(niimg))):
-            warn_with_log(
-                "The imgs you have fed into fit_transform() contains NaN "
-                "values which will be converted to zeroes."
-            )
-            X = safe_get_data(niimg, True).reshape([-1, niimg.shape[3]]).T
-        else:
-            X = safe_get_data(niimg).reshape([-1, niimg.shape[3]]).T
-
-        mask_coords = list(np.ndindex(niimg.shape[:3]))
-
+        mask_coords = list(zip(*np.where(mask != 0), strict=False))
     else:
-        raise_error("Either a niimg or a mask_img must be provided.")
-
+        affine = niimg.affine
+        shape = niimg.shape[:3]
+        mask_coords = list(np.ndindex(shape))
+    voxels = np.ravel_multi_index(
+        tuple(np.asarray(mask_coords, dtype=int).reshape(-1, 3).T), shape
+    )
     # For each seed, get coordinates of nearest voxel
     nearests = []
     for sx, sy, sz in seeds:
@@ -166,7 +172,7 @@ def _apply_mask_and_get_affinity(
         except ValueError:
             nearests.append(None)
 
-    mask_coords = np.asarray(list(zip(*mask_coords)))
+    mask_coords = np.asarray(list(zip(*mask_coords, strict=False)))
     mask_coords = image.resampling.coord_transform(
         mask_coords[0], mask_coords[1], mask_coords[2], affine
     )
@@ -193,6 +199,132 @@ def _apply_mask_and_get_affinity(
     if (not allow_overlap) and np.any(A.sum(axis=0) >= 2):
         raise_error("Overlap detected between spheres")
 
+    return voxels, A
+
+
+def _read_signals(niimg, voxels, masked) -> np.ndarray:
+    """Read the signals of some voxels, a few volumes at a time.
+
+    The data is read in chunks of ``markers.spheres.chunksize`` volumes and
+    only the signals of ``voxels`` are kept, so the whole data is never in
+    memory. The values are the same as extracting the signals from the whole
+    data with :func:`nilearn.masking.apply_mask_fmri` (``masked``) or
+    :func:`nilearn._utils.niimg.safe_get_data` (not ``masked``).
+
+    Parameters
+    ----------
+    niimg : 4D Niimg-like object
+        The image to read.
+    voxels : 1D numpy.ndarray
+        The flat (C order) indices of the voxels to read.
+    masked : bool
+        Whether the voxels are the ones of a mask: as
+        :func:`nilearn.masking.apply_mask_fmri` does, the signals are then
+        float32 if the data is not floating point, and the non-finite values
+        are set to 0. Otherwise, the signals keep the data type of the image
+        and the non-finite values are set to 0 only if the image has NaN
+        values, with a warning.
+
+    Returns
+    -------
+    numpy.ndarray of shape (number of scans, number of voxels)
+        The signals of the voxels.
+
+    """
+    n_scans = niimg.shape[3]
+    chunk_size = _get_chunk_size()
+    signals = None
+    has_nan = False
+    for start in range(0, n_scans, chunk_size):
+        chunk = np.asanyarray(niimg.dataobj[..., start : start + chunk_size])
+        if not masked and not has_nan:
+            has_nan = bool(np.isnan(np.sum(chunk)))
+        chunk = chunk.reshape(-1, chunk.shape[-1])[voxels].T
+        if masked and chunk.dtype.kind != "f":
+            chunk = chunk.astype(np.float32)
+        if signals is None:
+            signals = np.empty((n_scans, len(voxels)), dtype=chunk.dtype)
+        signals[start : start + chunk.shape[0]] = chunk
+    if masked or has_nan:
+        if has_nan:
+            warn_with_log(
+                "The imgs you have fed into fit_transform() contains NaN "
+                "values which will be converted to zeroes."
+            )
+        signals[~np.isfinite(signals)] = 0
+    return signals
+
+
+def _apply_mask_and_get_affinity(
+    seeds, niimg, radius, allow_overlap, mask_img=None
+):
+    """Apply mask and get affinity matrix for given seeds.
+
+    Utility function to get only the rows which are occupied by sphere at
+    given seed locations and the provided radius. Rows are in target_affine and
+    target_shape space.
+
+    Parameters
+    ----------
+    seeds : List of triplets of coordinates in native space
+        Seed definitions. List of coordinates of the seeds in the same space
+        as target_affine.
+    niimg : 3D/4D Niimg-like object
+        See :ref:`extracting_data`.
+        Images to process.
+        If a 3D niimg is provided, a singleton dimension will be added to
+        the output to represent the single scan in the niimg.
+    radius : float
+        Indicates, in millimeters, the radius for the sphere around the seed.
+    allow_overlap : boolean
+        If False, a ValueError is raised if VOIs overlap
+    mask_img : Niimg-like object, optional
+        Mask to apply to regions before extracting signals. If niimg is None,
+        mask_img is used as a reference space in which the spheres 'indices are
+        placed.
+
+    Returns
+    -------
+    X : 2D numpy.ndarray
+        Signal for each brain voxel in the (masked) niimgs.
+        shape: (number of scans, number of voxels)
+    A : scipy.sparse.lil_matrix
+        Contains the boolean indices for each sphere.
+        shape: (number of seeds, number of voxels)
+
+    Raises
+    ------
+    ValueError
+        If ``niimg`` and ``mask_img`` are both provided or
+        if overlap is detected between spheres.
+
+    Warns
+    -----
+    RuntimeWarning
+        If the provided images contain NaN, they will be converted to zeroes.
+
+    """
+    _, A = _get_sphere_voxels(
+        seeds, niimg, radius, allow_overlap, mask_img=mask_img
+    )
+    X = None
+    if niimg is not None and mask_img is not None:
+        mask_img = image.resample_img(
+            check_niimg_3d(mask_img),
+            target_affine=niimg.affine,
+            target_shape=niimg.shape[:3],
+            interpolation="nearest",
+        )
+        X = masking.apply_mask_fmri(niimg, mask_img)
+    elif niimg is not None:
+        if np.isnan(np.sum(safe_get_data(niimg))):
+            warn_with_log(
+                "The imgs you have fed into fit_transform() contains NaN "
+                "values which will be converted to zeroes."
+            )
+            X = safe_get_data(niimg, True).reshape([-1, niimg.shape[3]]).T
+        else:
+            X = safe_get_data(niimg).reshape([-1, niimg.shape[3]]).T
     return X, A
 
 
@@ -201,16 +333,17 @@ def _iter_signals_from_spheres(
 ):
     """Iterate over spheres.
 
+    Only the signals of the voxels of the spheres are read, a few volumes at
+    a time (see :func:`_read_signals`), so the whole data is never in memory.
+
     Parameters
     ----------
     seeds : :obj:`list` of triplets of coordinates in native space
         Seed definitions. List of coordinates of the seeds in the same space
         as the images (typically MNI or TAL).
-    niimg : 3D/4D Niimg-like object
+    niimg : 4D Niimg-like object
         See :ref:`extracting_data`.
         Images to process.
-        If a 3D niimg is provided, a singleton dimension will be added to
-        the output to represent the single scan in the niimg.
     radius: float
         Indicates, in millimeters, the radius for the sphere around the seed.
     allow_overlap: boolean
@@ -221,11 +354,17 @@ def _iter_signals_from_spheres(
         Mask to apply to regions before extracting signals.
 
     """
-    X, A = _apply_mask_and_get_affinity(
+    voxels, A = _get_sphere_voxels(
         seeds, niimg, radius, allow_overlap, mask_img=mask_img
     )
-    for row in A.rows:
-        yield X[:, row]
+    rows = [np.asarray(row, dtype=int) for row in A.rows]
+    # The voxels of any sphere, and their position in the signals read
+    used = np.unique(np.concatenate([np.zeros(0, dtype=int), *rows]))
+    position = np.full(len(voxels), -1, dtype=int)
+    position[used] = np.arange(len(used))
+    signals = _read_signals(niimg, voxels[used], masked=mask_img is not None)
+    for row in rows:
+        yield signals[:, position[row]]
 
 
 class _JuniferExtractionFunctor:
@@ -255,7 +394,7 @@ class _JuniferExtractionFunctor:
     def __init__(
         self,
         seeds_: "ArrayLike",
-        radius: Optional[float],
+        radius: float | None,
         mask_img: Union["Nifti1Image", "Nifti2Image", None],
         agg_func: Callable,
         allow_overlap: bool,
@@ -324,6 +463,11 @@ class JuniferNiftiSpheresMasker(NiftiSpheresMasker):
     * it allows to pass any callable as the ``agg_func`` parameter.
     * empty spheres do not create an error. Instead, ``agg_func`` is applied to
       an empty array and the result is passed.
+    * only the signals of the voxels of the spheres are read, a few volumes at
+      a time (``markers.spheres.chunksize``), so the whole data is never in
+      memory.
+    * ``reports`` is False by default, as the data for the reports (e.g., the
+      middle volume of the resampled images) needs the whole data in memory.
 
     Parameters
     ----------
@@ -352,7 +496,7 @@ class JuniferNiftiSpheresMasker(NiftiSpheresMasker):
     def __init__(
         self,
         seeds: "ArrayLike",
-        radius: Optional[float] = None,
+        radius: float | None = None,
         mask_img: Union["Nifti1Image", "Nifti2Image", None] = None,
         agg_func: Callable = np.mean,
         allow_overlap: bool = False,
@@ -360,11 +504,17 @@ class JuniferNiftiSpheresMasker(NiftiSpheresMasker):
         **kwargs,  # TODO: to keep or not?
     ) -> None:
         self.agg_func = agg_func
+        # nilearn's default (False) is deprecated, use the new equivalent
+        kwargs.setdefault("standardize", None)
+        # The reports are not used, and computing their data at fit time
+        # loads (and resamples) the whole data
+        kwargs.setdefault("reports", False)
         super().__init__(
             seeds=seeds,
             radius=radius,
             mask_img=mask_img,
             allow_overlap=allow_overlap,
+            dtype=dtype,
             **kwargs,
         )
 
@@ -409,16 +559,19 @@ class JuniferNiftiSpheresMasker(NiftiSpheresMasker):
             inputs.
 
         """
-        self._check_fitted()
+        check_is_fitted(self)
 
-        params = get_params(NiftiSpheresMasker, self)
-
-        # New in nilearn 0.10.1
-        if hasattr(self, "clean_kwargs"):
-            params["clean_kwargs"] = self.clean_kwargs
+        # The parameters are taken from NiftiSpheresMasker, as the ones not
+        # in this class' signature are passed as keyword arguments
+        params = {
+            k: getattr(self, k)
+            for k in NiftiSpheresMasker._get_param_names()
+            if k not in {"memory", "memory_level", "verbose", "copy", "n_jobs"}
+        }
+        params["clean_kwargs"] = self.clean_args_
 
         signals, _ = self._cache(
-            _filter_and_extract, ignore=["verbose", "memory", "memory_level"]
+            filter_and_extract, ignore=["verbose", "memory", "memory_level"]
         )(
             imgs,
             _JuniferExtractionFunctor(
@@ -434,12 +587,18 @@ class JuniferNiftiSpheresMasker(NiftiSpheresMasker):
             confounds=confounds,
             sample_mask=sample_mask,
             dtype=self.dtype,
+            sklearn_output_config=getattr(
+                self, "_sklearn_output_config", None
+            ),
             # Caching
-            memory=self.memory,
+            memory=self.memory_,
             memory_level=self.memory_level,
             # kwargs
             verbose=self.verbose,
         )
+        target_dtype = self._get_target_dtype(load_img(imgs))
+        if target_dtype is not None:
+            signals = signals.astype(target_dtype)
         return signals
 
     def inverse_transform(self, region_signals: "ArrayLike") -> "Nifti1Image":

junifer/external/nilearn/tests/test_junifer_connectivity_measure.py

@@ -7,11 +7,15 @@ import copy
 import sys
 import warnings
 from math import cosh, exp, log, sinh, sqrt
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Optional
 
 import numpy as np
 import pytest
 from nilearn.connectome.connectivity_matrices import sym_matrix_to_vec
+
+# Not part of nilearn's public API, so it might move in future versions.
+# Importing it evaluates the markers of nilearn's test module, which need to
+# be registered in the pytest markers (see pyproject.toml).
 from nilearn.tests.test_signal import generate_signals
 from numpy.testing import (
     assert_allclose,
@@ -21,6 +25,7 @@ from numpy.testing import (
 from pandas import DataFrame
 from scipy import linalg
 from sklearn.covariance import EmpiricalCovariance, LedoitWolf
+from sklearn.exceptions import NotFittedError
 
 from junifer.external.nilearn import JuniferConnectivityMeasure
 from junifer.external.nilearn.junifer_connectivity_measure import (
@@ -88,7 +93,7 @@ def random_diagonal(
     p: int,
     v_min: float = 1.0,
     v_max: float = 2.0,
-    random_state: Union[int, np.random.RandomState] = 0,
+    random_state: int | np.random.RandomState = 0,
 ) -> np.ndarray:
     """Generate a random diagonal matrix.
 
@@ -120,7 +125,7 @@ def random_spd(
     p: int,
     eig_min: float,
     cond: float,
-    random_state: Union[int, np.random.RandomState] = 0,
+    random_state: int | np.random.RandomState = 0,
 ) -> np.ndarray:
     """Generate a random symmetric positive definite matrix.
 
@@ -190,7 +195,7 @@ def signals() -> list[np.ndarray]:
 
 @pytest.fixture
 def signals_and_covariances(
-    cov_estimator: Union[LedoitWolf, EmpiricalCovariance],
+    cov_estimator: LedoitWolf | EmpiricalCovariance,
 ) -> tuple[list[np.ndarray], list[float]]:
     """Return signals and covariances for a covariance estimator.
 
@@ -343,7 +348,7 @@ def test_geometric_mean_properties() -> None:
 
     # Generic
     assert isinstance(spds, list)
-    for spd, input_spd in zip(spds, input_spds):
+    for spd, input_spd in zip(spds, input_spds, strict=False):
         assert_array_equal(spd, input_spd)
     assert is_spd(gmean, decimal=7)
 
@@ -352,7 +357,7 @@ def random_non_singular(
     p: int,
     sing_min: float = 1.0,
     sing_max: float = 2.0,
-    random_state: Union[int, np.random.RandomState] = 0,
+    random_state: int | np.random.RandomState = 0,
 ) -> np.ndarray:
     """Generate a random nonsingular matrix.
 
@@ -587,9 +592,7 @@ def test_connectivity_measure_errors():
     # Raising error for input subjects not iterable
     conn_measure = JuniferConnectivityMeasure()
 
-    with pytest.raises(
-        ValueError, match=r"'subjects' input argument must be an iterable"
-    ):
+    with pytest.raises(TypeError, match=r"Input must be an iterable"):
         conn_measure.fit(1.0)
 
     # input subjects not 2D numpy.ndarrays
@@ -605,6 +608,15 @@ def test_connectivity_measure_errors():
     ):
         conn_measure.fit([np.ones((100, 40)), np.ones((100, 41))])
 
+    # invalid kind, listing all the allowed kinds
+    conn_measure = JuniferConnectivityMeasure(kind="foo")
+    with pytest.raises(
+        ValueError,
+        match=r"Allowed connectivity kinds .*'spearman correlation'.*"
+        r"'xi correlation'.* Got kind foo",
+    ):
+        conn_measure.fit([np.ones((100, 40))])
+
     # fit_transform with a single subject and kind=tangent
     conn_measure = JuniferConnectivityMeasure(kind="tangent")
 
@@ -669,7 +681,7 @@ def _assert_connectivity_tangent(connectivities, conn_measure, covs) -> None:
 
     """
     for true_covariance_matrix, estimated_covariance_matrix in zip(
-        covs, connectivities
+        covs, connectivities, strict=False
     ):
         assert_array_almost_equal(
             estimated_covariance_matrix, estimated_covariance_matrix.T
@@ -701,7 +713,7 @@ def _assert_connectivity_precision(connectivities, covs) -> None:
 
     """
     for true_covariance_matrix, estimated_covariance_matrix in zip(
-        covs, connectivities
+        covs, connectivities, strict=False
     ):
         assert is_spd(estimated_covariance_matrix, decimal=7)
         assert_array_almost_equal(
@@ -729,7 +741,7 @@ def _assert_connectivity_correlation(
 
     """
     for true_covariance_matrix, estimated_covariance_matrix in zip(
-        covs, connectivities
+        covs, connectivities, strict=False
     ):
         assert is_spd(estimated_covariance_matrix, decimal=7)
 
@@ -750,7 +762,7 @@ def _assert_connectivity_correlation(
 def _assert_connectivity_partial_correlation(connectivities, covs) -> None:
     """Assert partial correlation connectivity matrix."""
     for true_covariance_matrix, estimated_covariance_matrix in zip(
-        covs, connectivities
+        covs, connectivities, strict=False
     ):
         precision_matrix = linalg.inv(true_covariance_matrix)
 
@@ -874,7 +886,7 @@ def test_connectivity_measure_check_vectorization_option(
     )
 
     # Check not fitted error
-    with pytest.raises(ValueError, match=r"has not been fitted. "):
+    with pytest.raises(NotFittedError):
         JuniferConnectivityMeasure().inverse_transform(
             vectorized_connectivities
         )
@@ -1053,15 +1065,15 @@ def test_confounds_connectivity_measure_errors() -> None:
     conn_measure = JuniferConnectivityMeasure(vectorize=True)
     msg = r"'confounds' input argument must be an iterable"
 
-    with pytest.raises(ValueError, match=msg):
+    with pytest.raises(TypeError, match=msg):
         conn_measure._check_input(X=signals, confounds=1.0)
 
-    with pytest.raises(ValueError, match=msg):
+    with pytest.raises(TypeError, match=msg):
         conn_measure._fit_transform(
             X=signals, do_fit=True, do_transform=True, confounds=1.0
         )
 
-    with pytest.raises(ValueError, match=msg):
+    with pytest.raises(TypeError, match=msg):
         conn_measure.fit_transform(X=signals, y=None, confounds=1.0)
 
     # Raising error for input confounds are given but not vectorize=True
@@ -1075,7 +1087,7 @@ def test_confounds_connectivity_measure_errors() -> None:
 def test_connectivity_measure_standardize(
     signals: list[np.ndarray],
 ) -> None:
-    """Check warning is raised and then suppressed with setting standardize.
+    """Check standardize strategies and the error for booleans.
 
     Parameters
     ----------
@@ -1083,17 +1095,17 @@ def test_connectivity_measure_standardize(
         The input signals.
 
     """
-    match = r"default strategy for standardize"
-
-    with pytest.warns(DeprecationWarning, match=match):
+    # No warnings about standardize with the default strategy
+    with warnings.catch_warnings(record=True) as record:
+        warnings.simplefilter("always")
         JuniferConnectivityMeasure(kind="correlation").fit_transform(signals)
+    assert not [m for m in record if "standardize" in str(m.message)]
 
-    with warnings.catch_warnings(record=True) as record:
-        JuniferConnectivityMeasure(
-            kind="correlation", standardize="zscore_sample"
-        ).fit_transform(signals)
-        for m in record:
-            assert match not in m.message
+    for standardize in (True, False):
+        with pytest.raises(ValueError, match="does not accept booleans"):
+            JuniferConnectivityMeasure(
+                kind="correlation", standardize=standardize
+            ).fit_transform(signals)
 
 
 @pytest.mark.skipif(