junifer 0.0.7.dev332__py3-none-any.whl → 0.0.8__py3-none-any.whl

junifer/markers/tests/test_sphere_aggregation.py

@@ -6,20 +6,25 @@
 
 from pathlib import Path
 
+import numpy as np
 import pytest
 from nilearn.maskers import NiftiSpheresMasker
 from numpy.testing import assert_array_equal
 
 from junifer.data import CoordinatesRegistry, MaskRegistry
+from junifer.datagrabber import DataType
 from junifer.datareader import DefaultDataReader
 from junifer.markers.sphere_aggregation import SphereAggregation
-from junifer.storage import SQLiteFeatureStorage
+from junifer.storage import HDF5FeatureStorage, StorageType
 from junifer.testing.datagrabbers import (
-    OasisVBMTestingDataGrabber,
     SPMAuditoryTestingDataGrabber,
+    UCLACNPVBMTestingDataGrabber,
 )
 
 
+pytestmark = pytest.mark.external
+
+
 # Define common variables
 COORDS = "DMNBuckner"
 RADIUS = 8
@@ -29,70 +34,70 @@ RADIUS = 8
     "input_type, storage_type",
     [
         (
-            "T1w",
-            "vector",
+            DataType.T1w,
+            StorageType.Vector,
         ),
         (
-            "T2w",
-            "vector",
+            DataType.T2w,
+            StorageType.Vector,
         ),
         (
-            "BOLD",
-            "timeseries",
+            DataType.BOLD,
+            StorageType.Timeseries,
         ),
         (
-            "VBM_GM",
-            "vector",
+            DataType.VBM_GM,
+            StorageType.Vector,
         ),
         (
-            "VBM_WM",
-            "vector",
+            DataType.VBM_WM,
+            StorageType.Vector,
         ),
         (
-            "VBM_CSF",
-            "vector",
+            DataType.VBM_CSF,
+            StorageType.Vector,
         ),
         (
-            "fALFF",
-            "vector",
+            DataType.FALFF,
+            StorageType.Vector,
         ),
         (
-            "GCOR",
-            "vector",
+            DataType.GCOR,
+            StorageType.Vector,
         ),
         (
-            "LCOR",
-            "vector",
+            DataType.LCOR,
+            StorageType.Vector,
         ),
     ],
 )
 def test_SphereAggregation_input_output(
-    input_type: str, storage_type: str
+    input_type: DataType, storage_type: StorageType
 ) -> None:
     """Test SphereAggregation input and output types.
 
     Parameters
     ----------
-    input_type : str
+    input_type : DataType
         The parametrized input type.
-    storage_type : str
+    storage_type : StorageType
         The parametrized storage type.
 
     """
     assert storage_type == SphereAggregation(
         coords="DMNBuckner",
         method="mean",
-        on=input_type,
+        on=[input_type],
     ).storage_type(input_type=input_type, output_feature="aggregation")
 
 
 def test_SphereAggregation_3D() -> None:
     """Test SphereAggregation object on 3D images."""
-    with OasisVBMTestingDataGrabber() as dg:
-        element_data = DefaultDataReader().fit_transform(dg["sub-01"])
+    with UCLACNPVBMTestingDataGrabber() as dg:
+        element_data = DefaultDataReader().fit_transform(dg["sub-10206"])
         # Create SphereAggregation object
         marker = SphereAggregation(
-            coords=COORDS, method="mean", radius=RADIUS, on="VBM_GM"
+            coords=COORDS, method="mean", radius=RADIUS, on=[DataType.VBM_GM]
         )
         sphere_agg_vbm_gm_data = marker.fit_transform(element_data)["VBM_GM"][
             "aggregation"
@@ -107,8 +112,9 @@ def test_SphereAggregation_3D() -> None:
         nifti_spheres_masker = NiftiSpheresMasker(
             seeds=testing_coords, radius=RADIUS
         )
-        nifti_spheres_masked_vbm_gm = nifti_spheres_masker.fit_transform(
-            element_data["VBM_GM"]["data"]
+        # nilearn returns a 1D array for 3D images
+        nifti_spheres_masked_vbm_gm = np.atleast_2d(
+            nifti_spheres_masker.fit_transform(element_data["VBM_GM"]["data"])
         )
 
         assert sphere_agg_vbm_gm_data.ndim == 2
@@ -124,7 +130,7 @@ def test_SphereAggregation_4D() -> None:
         element_data = DefaultDataReader().fit_transform(dg["sub001"])
         # Create SphereAggregation object
         marker = SphereAggregation(
-            coords=COORDS, method="mean", radius=RADIUS, on="BOLD"
+            coords=COORDS, method="mean", radius=RADIUS, on=DataType.BOLD
         )
         sphere_agg_bold_data = marker.fit_transform(element_data)["BOLD"][
             "aggregation"
@@ -160,13 +166,13 @@ def test_SphereAggregation_storage(tmp_path: Path) -> None:
 
     """
     # Store 3D
-    with OasisVBMTestingDataGrabber() as dg:
-        element_data = DefaultDataReader().fit_transform(dg["sub-01"])
-        storage = SQLiteFeatureStorage(
-            uri=tmp_path / "test_sphere_storage_3D.sqlite", upsert="ignore"
+    with UCLACNPVBMTestingDataGrabber() as dg:
+        element_data = DefaultDataReader().fit_transform(dg["sub-10206"])
+        storage = HDF5FeatureStorage(
+            uri=tmp_path / "test_sphere_storage_3D.hdf5",
         )
         marker = SphereAggregation(
-            coords=COORDS, method="mean", radius=RADIUS, on="VBM_GM"
+            coords=COORDS, method="mean", radius=RADIUS, on=[DataType.VBM_GM]
         )
         marker.fit_transform(input=element_data, storage=storage)
         features = storage.list_features()
@@ -178,11 +184,11 @@ def test_SphereAggregation_storage(tmp_path: Path) -> None:
     # Store 4D
     with SPMAuditoryTestingDataGrabber() as dg:
         element_data = DefaultDataReader().fit_transform(dg["sub001"])
-        storage = SQLiteFeatureStorage(
-            uri=tmp_path / "test_sphere_storage_4D.sqlite", upsert="ignore"
+        storage = HDF5FeatureStorage(
+            uri=tmp_path / "test_sphere_storage_4D.hdf5",
         )
         marker = SphereAggregation(
-            coords=COORDS, method="mean", radius=RADIUS, on="BOLD"
+            coords=COORDS, method="mean", radius=RADIUS, on=DataType.BOLD
         )
         marker.fit_transform(input=element_data, storage=storage)
         features = storage.list_features()
@@ -194,15 +200,15 @@ def test_SphereAggregation_storage(tmp_path: Path) -> None:
 
 def test_SphereAggregation_3D_mask() -> None:
     """Test SphereAggregation object on 3D images using mask."""
-    with OasisVBMTestingDataGrabber() as dg:
-        element_data = DefaultDataReader().fit_transform(dg["sub-01"])
+    with UCLACNPVBMTestingDataGrabber() as dg:
+        element_data = DefaultDataReader().fit_transform(dg["sub-10206"])
         # Create SphereAggregation object
         marker = SphereAggregation(
             coords=COORDS,
             method="mean",
             radius=RADIUS,
-            on="VBM_GM",
-            masks="compute_brain_mask",
+            on=[DataType.VBM_GM],
+            masks=["compute_brain_mask"],
         )
         sphere_agg_vbm_gm_data = marker.fit_transform(element_data)["VBM_GM"][
             "aggregation"
@@ -245,7 +251,7 @@ def test_SphereAggregation_4D_agg_time() -> None:
             method="mean",
             radius=RADIUS,
             time_method="mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
         sphere_agg_bold_data = marker.fit_transform(element_data)["BOLD"][
             "aggregation"
@@ -281,7 +287,7 @@ def test_SphereAggregation_4D_agg_time() -> None:
             radius=RADIUS,
             time_method="select",
             time_method_params={"pick": [0]},
-            on="BOLD",
+            on=DataType.BOLD,
         )
         sphere_agg_bold_data = marker.fit_transform(element_data)["BOLD"][
             "aggregation"
@@ -305,7 +311,7 @@ def test_SphereAggregation_errors() -> None:
             radius=RADIUS,
             time_method="pick",
             time_method_params={"pick": [0]},
-            on="VBM_GM",
+            on=[DataType.VBM_GM],
         )
 
     with pytest.raises(
@@ -316,7 +322,7 @@ def test_SphereAggregation_errors() -> None:
             method="mean",
             radius=RADIUS,
             time_method_params={"pick": [0]},
-            on="VBM_GM",
+            on=[DataType.VBM_GM],
         )
 
 
@@ -333,7 +339,7 @@ def test_SphereAggregation_warning() -> None:
                 radius=RADIUS,
                 time_method="select",
                 time_method_params={"pick": [0]},
-                on="BOLD",
+                on=DataType.BOLD,
             )
             element_data["BOLD"]["data"] = element_data["BOLD"]["data"].slicer[
                 ..., 0:1

junifer/markers/utils.py

@@ -8,19 +8,86 @@
 #          Amir Omidvarnia <a.omidvarnia@fz-juelich.de>
 # License: AGPL
 
-from typing import Callable, Optional, Union
+import hashlib
+import zlib
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
 
+import nibabel as nib
 import numpy as np
 import pandas as pd
+from nibabel.spatialimages import SpatialImage
 from scipy.stats import zscore
 
+from ..pipeline import WorkDirManager
 from ..utils import raise_error
 
 
+def _save_mask(mask_img: SpatialImage, prefix: str) -> Path:
+    """Save mask to the element directory, named by its content.
+
+    The same mask always gets the same path within an element, so that the
+    cached computations (e.g., ReHo or ALFF) can be reused by markers using
+    the same mask. The element directory is deleted after the element is
+    processed.
+
+    Parameters
+    ----------
+    mask_img : Niimg-like object
+        The mask to save.
+    prefix : str
+        The prefix of the file name.
+
+    Returns
+    -------
+    pathlib.Path
+        The path to the saved mask.
+
+    """
+    mask_data = np.squeeze(np.asarray(mask_img.dataobj)) != 0
+    digest = hashlib.sha256()
+    digest.update(str(mask_data.shape).encode())
+    digest.update(np.packbits(mask_data).tobytes())
+    digest.update(mask_img.affine.tobytes())
+    mask_path = (
+        WorkDirManager().get_elementdir()
+        / f"{prefix}_{digest.hexdigest()[:16]}.nii"
+    )
+    if not mask_path.exists():
+        nib.save(
+            nib.Nifti1Image(
+                mask_data.astype(np.uint8), mask_img.affine, mask_img.header
+            ),
+            mask_path,
+        )
+    return mask_path
+
+
+def _has_data(data: np.ndarray) -> np.ndarray:
+    """Get the columns with data.
+
+    Regions that are lost when resampling or warping the parcellation to the
+    target image have no voxels, so their time series are all NaN.
+
+    Parameters
+    ----------
+    data : np.ndarray
+        The data (time x columns).
+
+    Returns
+    -------
+    np.ndarray
+        Boolean mask of the columns that are not all NaN.
+
+    """
+    return ~np.all(np.isnan(data), axis=0)
+
+
 def _ets(
     bold_ts: np.ndarray,
-    roi_names: Union[None, list[str]] = None,
-) -> tuple[np.ndarray, Optional[list[str]]]:
+    roi_names: list[str] | None = None,
+) -> tuple[np.ndarray, list[str] | None]:
     """Compute the edge-wise time series based on BOLD time series.
 
     Take a timeseries of brain areas, and calculate timeseries for each
@@ -73,7 +140,8 @@ def _ets(
             )
         _roi_names = np.array(roi_names)
         edge_names = [
-            "~".join([x, y]) for x, y in zip(_roi_names[u], _roi_names[v])
+            "~".join([x, y])
+            for x, y in zip(_roi_names[u], _roi_names[v], strict=False)
         ]
         return ets, edge_names
 
@@ -81,7 +149,7 @@ def _ets(
 def _correlate_dataframes(
     df1: pd.DataFrame,
     df2: pd.DataFrame,
-    method: Union[str, Callable] = "pearson",
+    method: str | Callable = "pearson",
 ) -> pd.DataFrame:
     """Column-wise correlations between two dataframes.
 
@@ -118,3 +186,197 @@ def _correlate_dataframes(
         .corr(method=method)  # type: ignore
         .loc["df2", "df1"]
     )
+
+
+def _checksum(array: np.ndarray) -> tuple:
+    """Get the checksum of an array.
+
+    Parameters
+    ----------
+    array : numpy.ndarray
+        The array.
+
+    Returns
+    -------
+    tuple
+        The shape, the dtype and the CRC-32 of the bytes of the array (None
+        for arrays of objects).
+
+    """
+    crc = None
+    if not array.dtype.hasobject:
+        # The transpose of an array in Fortran order (e.g., the data of a
+        # NIfTI image) is in C order, so it is not copied
+        if array.flags.f_contiguous and not array.flags.c_contiguous:
+            array = array.T
+        crc = zlib.crc32(memoryview(np.ascontiguousarray(array)).cast("B"))
+    return (array.shape, array.dtype.str, crc)
+
+
+def _image_data(img: SpatialImage) -> tuple:
+    """Get the data of an image in memory.
+
+    The data read from the file of an image is not loaded, as it cannot be
+    changed (unless it is cached) and it would be kept in memory.
+
+    Parameters
+    ----------
+    img : nibabel.spatialimages.SpatialImage
+        The image.
+
+    Returns
+    -------
+    tuple
+        The data of the image if it is in memory (the data of an image
+        created from an array, or the data cached by nilearn, e.g., with
+        ``nilearn.image.get_data``) and the data cached by nibabel's
+        ``get_fdata``, or None.
+
+    """
+    data = img.dataobj
+    if not isinstance(data, np.ndarray):
+        data = getattr(img, "_data_cache", None)
+    return data, getattr(img, "_fdata_cache", None)
+
+
+def _checksums(value: Any) -> tuple:
+    """Get the checksums of an array, data frame, image or set.
+
+    Parameters
+    ----------
+    value : Any
+        The value.
+
+    Returns
+    -------
+    tuple
+        The checksums, empty for other values. For data frames and series:
+        the values and index, and the column labels or name. For images: the
+        affine and the data in memory (see :func:`_image_data`, None if not in
+        memory). For sets: their contents.
+
+    """
+    if isinstance(value, np.ndarray):
+        return (_checksum(value),)
+    if isinstance(value, pd.DataFrame | pd.Series):
+        labels = (
+            tuple(value.columns)
+            if isinstance(value, pd.DataFrame)
+            else value.name
+        )
+        return (
+            _checksum(pd.util.hash_pandas_object(value).to_numpy()),
+            labels,
+        )
+    if isinstance(value, SpatialImage):
+        return (
+            _checksum(value.affine),
+            *(
+                None if data is None else _checksum(data)
+                for data in _image_data(value)
+            ),
+        )
+    if isinstance(value, set | frozenset):
+        return (frozenset(value),)
+    return ()
+
+
+def _fingerprint(data: Any, path: tuple = ()) -> dict[tuple, tuple]:
+    """Get a fingerprint of the data, to check whether it is changed.
+
+    Parameters
+    ----------
+    data : Any
+        The data.
+    path : tuple, optional
+        The keys (or indices) leading to ``data`` (default ()).
+
+    Returns
+    -------
+    dict
+        For each path, the value (to check it is not replaced) and its
+        checksums (to check it is not changed in place).
+
+    """
+    out = {path: (data, _checksums(data))}
+    if isinstance(data, dict):
+        items = data.items()
+    elif isinstance(data, list | tuple):
+        items = enumerate(data)
+    else:
+        items = ()
+    for key, value in items:
+        out.update(_fingerprint(value, (*path, key)))
+    return out
+
+
+def _is_changed(old: tuple, new: tuple) -> bool:
+    """Check whether a value is changed.
+
+    Parameters
+    ----------
+    old : tuple
+        The value and its checksums before.
+    new : tuple
+        The value and its checksums after.
+
+    Returns
+    -------
+    bool
+        Whether the value was replaced or changed in place.
+
+    """
+    (old_value, old_sums), (new_value, new_sums) = old, new
+    if new_value is not old_value:
+        return True
+    if isinstance(new_value, SpatialImage):
+        # The affine must be the same
+        if old_sums[0] != new_sums[0]:
+            return True
+        for old_data, new_data, data in zip(
+            old_sums[1:], new_sums[1:], _image_data(new_value), strict=True
+        ):
+            if new_data is None:
+                # Not in memory (or no longer cached)
+                continue
+            if old_data is not None and old_data[1] == new_data[1]:
+                # Same dtype: the data in memory must be the same
+                if old_data != new_data:
+                    return True
+            else:
+                # The data was cached during the computation, possibly with
+                # another dtype: compare it with the data read from the file
+                expected = np.asanyarray(new_value.dataobj, dtype=data.dtype)
+                if not np.array_equal(data, expected, equal_nan=True):
+                    return True
+        return False
+    return old_sums != new_sums
+
+
+def _changed_paths(
+    before: dict[tuple, tuple], after: dict[tuple, tuple]
+) -> list[str]:
+    """Get the paths of the data that are changed.
+
+    Parameters
+    ----------
+    before : dict
+        The fingerprint of the data before (see :func:`_fingerprint`).
+    after : dict
+        The fingerprint of the data after.
+
+    Returns
+    -------
+    list of str
+        The paths (keys joined by ".") that were added, removed, replaced or
+        changed in place.
+
+    """
+    changed = [
+        path
+        for path in sorted(before.keys() | after.keys(), key=str)
+        if path not in before
+        or path not in after
+        or _is_changed(before[path], after[path])
+    ]
+    return [".".join(str(key) for key in path) for path in changed]

junifer/onthefly/__init__.py

@@ -3,8 +3,7 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from .read_transform import read_transform
-from . import _brainprint as brainprint
+import lazy_loader as lazy
 
 
-__all__ = ["read_transform", "brainprint"]
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)

junifer/onthefly/__init__.pyi

@@ -0,0 +1,4 @@
+__all__ = ["read_transform", "brainprint"]
+
+from .read_transform import read_transform
+from . import _brainprint as brainprint

junifer/onthefly/_brainprint.py

@@ -3,8 +3,6 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Optional
-
 import numpy as np
 import pandas as pd
 
@@ -17,7 +15,7 @@ __all__ = ["normalize", "reweight"]
 
 def normalize(
     storage: StorageLike,
-    features: dict[str, dict[str, Optional[str]]],
+    features: dict[str, dict[str, str | None]],
     kind: str,
 ) -> pd.DataFrame:  # pragma: no cover
     """Read stored brainprint data and normalize either surfaces or volumes.
@@ -87,8 +85,8 @@ def normalize(
 
 def reweight(
     storage: StorageLike,
-    feature_name: Optional[str] = None,
-    feature_md5: Optional[str] = None,
+    feature_name: str | None = None,
+    feature_md5: str | None = None,
 ) -> pd.DataFrame:  # pragma: no cover
     """Read stored brainprint data and reweight eigenvalues.
 

junifer/onthefly/read_transform.py

@@ -3,33 +3,35 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Optional
-
 import numpy as np
 import pandas as pd
+import structlog
 
 from ..typing import StorageLike
-from ..utils import logger, raise_error, warn_with_log
+from ..utils import raise_error, warn_with_log
 
 
 __all__ = ["read_transform"]
 
+_log = structlog.get_logger("junifer")
+logger = _log.bind(pkg="onthefly")
+
 
 def read_transform(
     storage: StorageLike,
     transform: str,
-    feature_name: Optional[str] = None,
-    feature_md5: Optional[str] = None,
-    nan_policy: Optional[str] = "bypass",
-    transform_args: Optional[tuple] = None,
-    transform_kw_args: Optional[dict] = None,
+    feature_name: str | None = None,
+    feature_md5: str | None = None,
+    nan_policy: str | None = "bypass",
+    transform_args: tuple | None = None,
+    transform_kw_args: dict | None = None,
 ) -> pd.DataFrame:
     """Read stored feature and transform to specific statistical output.
 
     Parameters
     ----------
     storage : storage-like
-        The storage class, for example, SQLiteFeatureStorage.
+        The storage class, for example, HDF5FeatureStorage.
     transform : str
         The kind of transform formatted as ``<package>_<function>``,
         for example, ``bctpy_degrees_und``.

junifer/onthefly/tests/test_read_transform.py

@@ -10,7 +10,7 @@ import numpy as np
 import pytest
 
 from junifer.onthefly import read_transform
-from junifer.storage.hdf5 import HDF5FeatureStorage
+from junifer.storage import HDF5FeatureStorage, StorageType
 
 
 @pytest.fixture
@@ -23,9 +23,9 @@ def vector_storage(tmp_path: Path) -> HDF5FeatureStorage:
         The path to the test directory.
 
     """
-    storage = HDF5FeatureStorage(tmp_path / "vector_store.hdf5")
+    storage = HDF5FeatureStorage(uri=tmp_path / "vector_store.hdf5")
     storage.store(
-        kind="vector",
+        kind=StorageType.Vector,
         meta={
             "element": {"subject": "test"},
             "dependencies": [],
@@ -48,9 +48,9 @@ def matrix_storage(tmp_path: Path) -> HDF5FeatureStorage:
         The path to the test directory.
 
     """
-    storage = HDF5FeatureStorage(tmp_path / "matrix_store.hdf5")
+    storage = HDF5FeatureStorage(uri=tmp_path / "matrix_store.hdf5")
     storage.store(
-        kind="matrix",
+        kind=StorageType.Matrix,
         meta={
             "element": {"subject": "test"},
             "dependencies": [],
@@ -74,13 +74,13 @@ def matrix_storage_with_nan(tmp_path: Path) -> HDF5FeatureStorage:
         The path to the test directory.
 
     """
-    storage = HDF5FeatureStorage(tmp_path / "matrix_store_nan.hdf5")
+    storage = HDF5FeatureStorage(uri=tmp_path / "matrix_store_nan.hdf5")
     data = np.arange(36).reshape(3, 3, 4).astype(float)
     data[1, 1, 2] = np.nan
     data[1, 2, 2] = np.nan
     for i in range(4):
         storage.store(
-            kind="matrix",
+            kind=StorageType.Matrix,
             meta={
                 "element": {"subject": f"test{i + 1}"},
                 "dependencies": [],