junifer 0.0.7.dev332__py3-none-any.whl → 0.0.8__py3-none-any.whl

junifer/markers/tests/test_markers_base.py

@@ -4,9 +4,18 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+from collections.abc import Callable
+from pathlib import Path
+
+import nibabel as nib
+import numpy as np
+import pandas as pd
 import pytest
+from nilearn.image import get_data
 
-from junifer.markers.base import BaseMarker
+from junifer.datagrabber import DataType
+from junifer.markers import BaseMarker
+from junifer.storage import StorageType
 
 
 def test_base_marker_abstractness() -> None:
@@ -21,14 +30,12 @@ def test_base_marker_subclassing() -> None:
     # Create concrete class
     class MyBaseMarker(BaseMarker):
         _MARKER_INOUT_MAPPINGS = {  # noqa: RUF012
-            "BOLD": {
-                "feat_1": "timeseries",
+            DataType.BOLD: {
+                "feat_1": StorageType.Timeseries,
             },
         }
 
-        def __init__(self, on, name=None) -> None:
-            self.parameter = 1
-            super().__init__(on, name)
+        parameter: int = 1
 
         def compute(self, input, extra_input):
             return {
@@ -105,3 +112,265 @@ def test_base_marker_subclassing() -> None:
     # Check output
     assert "BOLD" in output
     assert "T2" not in output
+
+
+def _fit(change: Callable, tmp_path: Path) -> dict:
+    """Fit a marker that applies a function to its input.
+
+    Parameters
+    ----------
+    change : callable
+        The function, called with the input and the extra input.
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    Returns
+    -------
+    dict
+        The output of the marker.
+
+    """
+
+    class Marker(BaseMarker):
+        _MARKER_INOUT_MAPPINGS = {  # noqa: RUF012
+            DataType.BOLD: {"feat": StorageType.Vector},
+        }
+
+        def compute(self, input, extra_input=None):
+            change(input, extra_input)
+            return {"feat": {"data": np.zeros((1, 1)), "col_names": ["x"]}}
+
+    marker = Marker(on=["BOLD"], name="marker")
+    return marker.fit_transform(input=_input(tmp_path))
+
+
+def _input(tmp_path: Path) -> dict:
+    """Get input data with the types of values the markers check.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    Returns
+    -------
+    dict
+        The input data.
+
+    """
+    # Non-integer values, which change when read as float32
+    values = np.full((2, 2, 2), 0.1)
+    # File-backed image, with its data cached
+    t1w_path = tmp_path / "t1w.nii.gz"
+    nib.save(nib.Nifti1Image(values, np.eye(4)), t1w_path)
+    t1w = nib.load(t1w_path)
+    t1w.get_fdata()
+    # File-backed image, without its data loaded
+    mask_path = tmp_path / "mask.nii.gz"
+    nib.save(nib.Nifti1Image(values, np.eye(4)), mask_path)
+    meta = {
+        "datagrabber": "dg",
+        "element": {"subject": "sub-01"},
+        "dependencies": {"numpy"},
+    }
+    return {
+        "BOLD": {
+            "path": tmp_path / "bold.nii.gz",
+            # In-memory image
+            "data": nib.Nifti1Image(np.full((2, 2, 2, 3), 0.1), np.eye(4)),
+            "array": np.arange(4.0),
+            "mask": nib.load(mask_path),
+            "confounds": {"data": pd.DataFrame({"a": [1.0, 2.0, 3.0]})},
+            "meta": meta,
+        },
+        "T1w": {"path": t1w_path, "data": t1w, "meta": meta.copy()},
+        "Warp": [{"src": "MNI", "dst": "native"}],
+    }
+
+
+def replace_value(input: dict, extra_input: dict) -> None:
+    """Replace a value."""
+    input["data"] = "new"
+
+
+def add_nested_value(input: dict, extra_input: dict) -> None:
+    """Add a value to a nested dictionary."""
+    input["meta"]["new"] = 1
+
+
+def remove_nested_value(input: dict, extra_input: dict) -> None:
+    """Remove the values of a nested dictionary."""
+    input["meta"]["element"].clear()
+
+
+def remove_value(input: dict, extra_input: dict) -> None:
+    """Remove a value."""
+    del input["path"]
+
+
+def change_array(input: dict, extra_input: dict) -> None:
+    """Change an array in place."""
+    input["array"][0] = 9.0
+
+
+def reshape_array(input: dict, extra_input: dict) -> None:
+    """Change the shape of an array in place."""
+    input["array"].shape = (2, 2)
+
+
+def change_array_dtype(input: dict, extra_input: dict) -> None:
+    """Change the dtype of an array in place."""
+    input["array"].dtype = np.int64
+
+
+def change_image_data(input: dict, extra_input: dict) -> None:
+    """Change the data of an in-memory image in place."""
+    np.asarray(input["data"].dataobj)[0, 0, 0, 0] = 9.0
+
+
+def change_image_affine(input: dict, extra_input: dict) -> None:
+    """Change the affine of an image in place."""
+    input["data"].affine[0, 3] = 9.0
+
+
+def change_dataframe(input: dict, extra_input: dict) -> None:
+    """Change a column of a data frame in place."""
+    input["confounds"]["data"]["a"] = 0.0
+
+
+def rename_dataframe_column(input: dict, extra_input: dict) -> None:
+    """Rename a column of a data frame in place."""
+    input["confounds"]["data"].rename(columns={"a": "b"}, inplace=True)
+
+
+def change_list_item(input: dict, extra_input: dict) -> None:
+    """Change an item of a list."""
+    extra_input["Warp"][0]["dst"] = "MNI"
+
+
+def append_to_list(input: dict, extra_input: dict) -> None:
+    """Append an item to a list."""
+    extra_input["Warp"].append({})
+
+
+def clear_set(input: dict, extra_input: dict) -> None:
+    """Remove the items of a set."""
+    input["meta"]["dependencies"].clear()
+
+
+def change_cached_image_data(input: dict, extra_input: dict) -> None:
+    """Change the cached data of an image of the extra input."""
+    extra_input["T1w"]["data"].get_fdata()[0, 0, 0] = 9.0
+
+
+def change_image_data_read_with_nilearn(
+    input: dict, extra_input: dict
+) -> None:
+    """Change the data of an image, after loading it with nilearn."""
+    get_data(input["mask"])[0, 0, 0] = 9.0
+
+
+def change_image_data_read_with_nibabel(
+    input: dict, extra_input: dict
+) -> None:
+    """Change the data of an image, after loading it with nibabel."""
+    input["mask"].get_fdata()[0, 0, 0] = 9.0
+
+
+# The changes of the input and the paths of the data they change
+_CHANGES = {
+    replace_value: "BOLD.data",
+    add_nested_value: "BOLD.meta.new",
+    remove_nested_value: "BOLD.meta.element.subject",
+    remove_value: "BOLD.path",
+    change_array: "BOLD.array",
+    reshape_array: "BOLD.array",
+    change_array_dtype: "BOLD.array",
+    change_image_data: "BOLD.data",
+    change_image_affine: "BOLD.data",
+    change_dataframe: "BOLD.confounds.data",
+    rename_dataframe_column: "BOLD.confounds.data",
+    change_list_item: "Warp.0.dst",
+    append_to_list: "Warp.1",
+    clear_set: "BOLD.meta.dependencies",
+    change_cached_image_data: "T1w.data",
+    change_image_data_read_with_nilearn: "BOLD.mask",
+    change_image_data_read_with_nibabel: "BOLD.mask",
+}
+
+
+@pytest.mark.parametrize(
+    "change, changed",
+    _CHANGES.items(),
+    ids=[change.__name__ for change in _CHANGES],
+)
+def test_base_marker_input_changed(
+    tmp_path: Path, change: Callable, changed: str
+) -> None:
+    """Test markers cannot change their input data.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    change : callable
+        The parametrized change of the input.
+    changed : str
+        The parametrized path of the changed data.
+
+    """
+    with pytest.raises(RuntimeError, match=rf"marker changed .*{changed}"):
+        _fit(change, tmp_path)
+
+
+def test_base_marker_input_not_changed(tmp_path: Path) -> None:
+    """Test markers that only read their input data.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+
+    def read(input: dict, extra_input: dict) -> None:
+        input["data"].get_fdata()
+        float(np.sum(input["array"]))
+        float(input["confounds"]["data"]["a"].sum())
+        # Replace the cached float64 data with float32
+        extra_input["T1w"]["data"].get_fdata(dtype=np.float32)
+        # Cache float32 data
+        input["mask"].get_fdata(dtype=np.float32)
+        get_data(input["mask"])
+        set(input["meta"]["dependencies"])
+        [warp["dst"] for warp in extra_input["Warp"]]
+
+    out = _fit(read, tmp_path)
+    assert "feat" in out["BOLD"]
+
+
+def test_base_marker_input_not_loaded(tmp_path: Path) -> None:
+    """Test checking the input does not load the data of the images.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    input = _input(tmp_path)
+
+    class Marker(BaseMarker):
+        _MARKER_INOUT_MAPPINGS = {  # noqa: RUF012
+            DataType.BOLD: {"feat": StorageType.Vector},
+        }
+
+        def compute(self, input, extra_input=None):
+            return {"feat": {"data": np.zeros((1, 1)), "col_names": ["x"]}}
+
+    Marker(on=["BOLD"], name="marker").fit_transform(input=input)
+    # The file-backed image without its data loaded is still not loaded
+    mask = input["BOLD"]["mask"]
+    assert not mask.in_memory
+    assert getattr(mask, "_data_cache", None) is None
+    assert getattr(mask, "_fdata_cache", None) is None

junifer/markers/tests/test_parcel_aggregation.py

@@ -17,68 +17,75 @@ from numpy.testing import assert_array_almost_equal, assert_array_equal
 from scipy.stats import trim_mean
 
 from junifer.data import MaskRegistry, ParcellationRegistry
+from junifer.datagrabber import DataType
 from junifer.datareader import DefaultDataReader
 from junifer.markers.parcel_aggregation import ParcelAggregation
-from junifer.storage import SQLiteFeatureStorage
-from junifer.testing.datagrabbers import PartlyCloudyTestingDataGrabber
+from junifer.storage import HDF5FeatureStorage, StorageType
+from junifer.testing.datagrabbers import (
+    PartlyCloudyTestingDataGrabber,
+    UCLACNPVBMTestingDataGrabber,
+)
+
+
+pytestmark = pytest.mark.external
 
 
 @pytest.mark.parametrize(
     "input_type, storage_type",
     [
         (
-            "T1w",
-            "vector",
+            DataType.T1w,
+            StorageType.Vector,
         ),
         (
-            "T2w",
-            "vector",
+            DataType.T2w,
+            StorageType.Vector,
         ),
         (
-            "BOLD",
-            "timeseries",
+            DataType.BOLD,
+            StorageType.Timeseries,
         ),
         (
-            "VBM_GM",
-            "vector",
+            DataType.VBM_GM,
+            StorageType.Vector,
         ),
         (
-            "VBM_WM",
-            "vector",
+            DataType.VBM_WM,
+            StorageType.Vector,
         ),
         (
-            "VBM_CSF",
-            "vector",
+            DataType.VBM_CSF,
+            StorageType.Vector,
         ),
         (
-            "fALFF",
-            "vector",
+            DataType.FALFF,
+            StorageType.Vector,
         ),
         (
-            "GCOR",
-            "vector",
+            DataType.GCOR,
+            StorageType.Vector,
         ),
         (
-            "LCOR",
-            "vector",
+            DataType.LCOR,
+            StorageType.Vector,
         ),
     ],
 )
 def test_ParcelAggregation_input_output(
-    input_type: str, storage_type: str
+    input_type: DataType, storage_type: StorageType
 ) -> None:
     """Test ParcelAggregation input and output types.
 
     Parameters
     ----------
-    input_type : str
+    input_type : DataType
         The parametrized input type.
-    storage_type : str
+    storage_type : StorageType
         The parametrized storage type.
 
     """
     assert storage_type == ParcelAggregation(
-        parcellation="Schaefer100x7", method="mean", on=input_type
+        parcellation=["Schaefer100x7"], method="mean", on=[input_type]
     ).storage_type(input_type=input_type, output_feature="aggregation")
 
 
@@ -90,7 +97,7 @@ def test_ParcelAggregation_3D() -> None:
         marker = ParcelAggregation(
             parcellation="TianxS1x3TxMNInonlinear2009cAsym",
             method="mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
         element_data["BOLD"]["data"] = element_data["BOLD"]["data"].slicer[
             ..., 0:1
@@ -108,10 +115,7 @@ def test_ParcelAggregation_3D() -> None:
             img=testing_parcellation,
         )
         # Create NiftiMasker
-        masker = NiftiMasker(
-            testing_parcellation_bin,
-            target_affine=element_data["BOLD"]["data"].affine,
-        )
+        masker = NiftiMasker(testing_parcellation_bin)
         data = masker.fit_transform(element_data["BOLD"]["data"])
         parcellation_values = np.squeeze(
             masker.transform(testing_parcellation)
@@ -157,7 +161,7 @@ def test_ParcelAggregation_3D() -> None:
         marker = ParcelAggregation(
             parcellation="TianxS1x3TxMNInonlinear2009cAsym",
             method="std",
-            on="BOLD",
+            on=DataType.BOLD,
         )
         parcel_agg_std_bold_data = marker.fit_transform(element_data)["BOLD"][
             "aggregation"
@@ -182,7 +186,7 @@ def test_ParcelAggregation_3D() -> None:
             parcellation="TianxS1x3TxMNInonlinear2009cAsym",
             method="trim_mean",
             method_params={"proportiontocut": 0.1},
-            on="BOLD",
+            on=DataType.BOLD,
         )
         parcel_agg_trim_mean_bold_data = marker.fit_transform(element_data)[
             "BOLD"
@@ -192,6 +196,40 @@ def test_ParcelAggregation_3D() -> None:
         assert_array_equal(parcel_agg_trim_mean_bold_data, manual)
 
 
+def test_ParcelAggregation_VBM() -> None:
+    """Test ParcelAggregation on VBM data in the parcellation space."""
+    parcellation = "TianxS1x3TxMNInonlinear2009cAsym"
+    with UCLACNPVBMTestingDataGrabber() as dg:
+        element_data = DefaultDataReader().fit_transform(dg["sub-10206"])
+        vbm = element_data["VBM_GM"]
+        marker = ParcelAggregation(
+            parcellation=parcellation, method="mean", on=DataType.VBM_GM
+        )
+        parcel_agg_vbm_data = marker.fit_transform(element_data)["VBM_GM"][
+            "aggregation"
+        ]["data"]
+
+        # Same space and grid, so the parcellation is used as is
+        tailored, _ = ParcellationRegistry().get(
+            parcellations=[parcellation], target_data=vbm
+        )
+        raw, _, _, _ = ParcellationRegistry().load(
+            name=parcellation,
+            target_space=vbm["space"],
+            resolution=2,
+        )
+        assert_array_equal(tailored.affine, raw.affine)
+        assert_array_equal(tailored.get_fdata(), raw.get_fdata())
+
+        # Compare with nilearn
+        # nilearn returns a 1D array for 3D images
+        nifti_labels_masked_vbm = np.atleast_2d(
+            NiftiLabelsMasker(labels_img=raw).fit_transform(vbm["data"])
+        )
+        assert parcel_agg_vbm_data.shape == (1, 16)
+        assert_array_almost_equal(nifti_labels_masked_vbm, parcel_agg_vbm_data)
+
+
 def test_ParcelAggregation_4D():
     """Test ParcelAggregation object on 4D images."""
     with PartlyCloudyTestingDataGrabber() as dg:
@@ -237,13 +275,13 @@ def test_ParcelAggregation_storage(tmp_path: Path) -> None:
     # Store 3D
     with PartlyCloudyTestingDataGrabber() as dg:
         element_data = DefaultDataReader().fit_transform(dg["sub-01"])
-        storage = SQLiteFeatureStorage(
-            uri=tmp_path / "test_parcel_storage_3D.sqlite", upsert="ignore"
+        storage = HDF5FeatureStorage(
+            uri=tmp_path / "test_parcel_storage_3D.hdf5",
         )
         marker = ParcelAggregation(
             parcellation="TianxS1x3TxMNInonlinear2009cAsym",
             method="mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
         element_data["BOLD"]["data"] = element_data["BOLD"]["data"].slicer[
             ..., 0:1
@@ -258,13 +296,13 @@ def test_ParcelAggregation_storage(tmp_path: Path) -> None:
     # Store 4D
     with PartlyCloudyTestingDataGrabber() as dg:
         element_data = DefaultDataReader().fit_transform(dg["sub-01"])
-        storage = SQLiteFeatureStorage(
-            uri=tmp_path / "test_parcel_storage_4D.sqlite", upsert="ignore"
+        storage = HDF5FeatureStorage(
+            uri=tmp_path / "test_parcel_storage_4D.hdf5",
         )
         marker = ParcelAggregation(
             parcellation="TianxS1x3TxMNInonlinear2009cAsym",
             method="mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
         marker.fit_transform(input=element_data, storage=storage)
         features = storage.list_features()
@@ -283,8 +321,8 @@ def test_ParcelAggregation_3D_mask() -> None:
             parcellation="TianxS1x3TxMNInonlinear2009cAsym",
             method="mean",
             name="tian_mean",
-            on="BOLD",
-            masks="compute_brain_mask",
+            on=DataType.BOLD,
+            masks=["compute_brain_mask"],
         )
         element_data["BOLD"]["data"] = element_data["BOLD"]["data"].slicer[
             ..., 0:1
@@ -360,9 +398,9 @@ def test_ParcelAggregation_3D_mask_computed() -> None:
         marker = ParcelAggregation(
             parcellation="TianxS1x3TxMNInonlinear2009cAsym",
             method="mean",
-            masks={"compute_brain_mask": {"threshold": 0.2}},
+            masks=[{"compute_brain_mask": {"threshold": 0.2}}],
             name="tian_mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
         parcel_agg_mean_bold_data = marker.fit_transform(element_data)["BOLD"][
             "aggregation"
@@ -444,7 +482,7 @@ def test_ParcelAggregation_3D_multiple_non_overlapping(tmp_path: Path) -> None:
             parcellation="TianxS1x3TxMNInonlinear2009cAsym",
             method="mean",
             name="tian_mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
         orig_mean = marker_original.fit_transform(element_data)["BOLD"][
             "aggregation"
@@ -462,7 +500,7 @@ def test_ParcelAggregation_3D_multiple_non_overlapping(tmp_path: Path) -> None:
             ],
             method="mean",
             name="tian_mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
 
         # No warnings should be raised
@@ -549,7 +587,7 @@ def test_ParcelAggregation_3D_multiple_overlapping(tmp_path: Path) -> None:
             parcellation="TianxS1x3TxMNInonlinear2009cAsym",
             method="mean",
             name="tian_mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
         orig_mean = marker_original.fit_transform(element_data)["BOLD"][
             "aggregation"
@@ -567,7 +605,7 @@ def test_ParcelAggregation_3D_multiple_overlapping(tmp_path: Path) -> None:
             ],
             method="mean",
             name="tian_mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
         # Warning should be raised
         with pytest.warns(RuntimeWarning, match="overlapping voxels"):
@@ -658,7 +696,7 @@ def test_ParcelAggregation_3D_multiple_duplicated_labels(
             parcellation="TianxS1x3TxMNInonlinear2009cAsym",
             method="mean",
             name="tian_mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
         orig_mean = marker_original.fit_transform(element_data)["BOLD"][
             "aggregation"
@@ -676,7 +714,7 @@ def test_ParcelAggregation_3D_multiple_duplicated_labels(
             ],
             method="mean",
             name="tian_mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
 
         # Warning should be raised
@@ -712,7 +750,7 @@ def test_ParcelAggregation_4D_agg_time():
             parcellation="TianxS1x3TxMNInonlinear2009cAsym",
             method="mean",
             time_method="mean",
-            on="BOLD",
+            on=DataType.BOLD,
         )
         parcel_agg_bold_data = marker.fit_transform(element_data)["BOLD"][
             "aggregation"
@@ -744,11 +782,11 @@ def test_ParcelAggregation_4D_agg_time():
         # Test picking first time point
         nifti_labels_masked_bold_pick_0 = nifti_labels_masked_bold[:1, :]
         marker = ParcelAggregation(
-            parcellation="TianxS1x3TxMNInonlinear2009cAsym",
+            parcellation=["TianxS1x3TxMNInonlinear2009cAsym"],
             method="mean",
             time_method="select",
             time_method_params={"pick": [0]},
-            on="BOLD",
+            on=DataType.BOLD,
         )
         parcel_agg_bold_data = marker.fit_transform(element_data)["BOLD"][
             "aggregation"
@@ -771,7 +809,7 @@ def test_ParcelAggregation_errors() -> None:
             method="mean",
             time_method="select",
             time_method_params={"pick": [0]},
-            on="VBM_GM",
+            on=[DataType.VBM_GM],
         )
 
     with pytest.raises(
@@ -781,7 +819,7 @@ def test_ParcelAggregation_errors() -> None:
             parcellation="Schaefer100x7",
             method="mean",
             time_method_params={"pick": [0]},
-            on="VBM_GM",
+            on=[DataType.VBM_GM],
         )
 
 
@@ -797,7 +835,7 @@ def test_ParcelAggregation_warning() -> None:
                 method="mean",
                 time_method="select",
                 time_method_params={"pick": [0]},
-                on="BOLD",
+                on=DataType.BOLD,
             )
             element_data["BOLD"]["data"] = element_data["BOLD"]["data"].slicer[
                 ..., 0:1