junifer 0.0.5.dev68__py3-none-any.whl → 0.0.5.dev93__py3-none-any.whl

junifer/preprocess/confounds/tests/test_fmriprep_confound_remover.py

@@ -20,6 +20,7 @@ from junifer.testing import get_testing_data
 from junifer.testing.datagrabbers import (
     OasisVBMTestingDataGrabber,
     PartlyCloudyTestingDataGrabber,
+    SPMAuditoryTestingDataGrabber,
 )
 
 
@@ -42,35 +43,10 @@ def test_fMRIPrepConfoundRemover_init() -> None:
 @pytest.mark.parametrize(
     "input_",
     [
-        ["T1w"],
         ["BOLD"],
         ["T1w", "BOLD"],
     ],
 )
-def test_fMRIPrepConfoundRemover_validate_input_errors(
-    input_: List[str],
-) -> None:
-    """Test errors for fMRIPrepConfoundRemover validate_input.
-
-    Parameters
-    ----------
-    input_ : list of str
-        The input data types.
-
-    """
-    confound_remover = fMRIPrepConfoundRemover()
-
-    with pytest.raises(ValueError, match="not have the required data"):
-        confound_remover.validate_input(input_)
-
-
-@pytest.mark.parametrize(
-    "input_",
-    [
-        ["BOLD", "BOLD_confounds"],
-        ["T1w", "BOLD", "BOLD_confounds"],
-    ],
-)
 def test_fMRIPrepConfoundRemover_validate_input(input_: List[str]) -> None:
     """Test fMRIPrepConfoundRemover validate_input.
 
@@ -302,13 +278,13 @@ def test_fMRIPRepConfoundRemover__pick_confounds_fmriprep() -> None:
     with PartlyCloudyTestingDataGrabber() as dg:
         input = dg["sub-01"]
         input = reader.fit_transform(input)
-        out1 = confound_remover._pick_confounds(input["BOLD_confounds"])
+        out1 = confound_remover._pick_confounds(input["BOLD"]["confounds"])
         assert set(out1.columns) == {*fmriprep_all_vars, "spike"}
 
     with PartlyCloudyTestingDataGrabber(reduce_confounds=False) as dg:
         input = dg["sub-01"]
         input = reader.fit_transform(input)
-        out2 = confound_remover._pick_confounds(input["BOLD_confounds"])
+        out2 = confound_remover._pick_confounds(input["BOLD"]["confounds"])
         assert set(out2.columns) == {*fmriprep_all_vars, "spike"}
 
     assert_frame_equal(out1, out2)
@@ -348,123 +324,106 @@ def test_fMRIPRepConfoundRemover__pick_confounds_fmriprep_compute() -> None:
 def test_fMRIPrepConfoundRemover__validate_data() -> None:
     """Test fMRIPrepConfoundRemover validate data."""
     confound_remover = fMRIPrepConfoundRemover(strategy={"wm_csf": "full"})
-
+    # Check correct data type
     with OasisVBMTestingDataGrabber() as dg:
         element_data = DefaultDataReader().fit_transform(dg["sub-01"])
         vbm = element_data["VBM_GM"]
         with pytest.raises(
             DimensionError, match="incompatible dimensionality"
         ):
-            confound_remover._validate_data(vbm, None)
-
-    with PartlyCloudyTestingDataGrabber(reduce_confounds=False) as dg:
+            confound_remover._validate_data(vbm)
+    # Check missing nested type in correct data type
+    with SPMAuditoryTestingDataGrabber() as dg:
         element_data = DefaultDataReader().fit_transform(dg["sub-01"])
         bold = element_data["BOLD"]
-
-        with pytest.raises(ValueError, match="No extra input"):
-            confound_remover._validate_data(bold, None)
+        # Test confound type
         with pytest.raises(
-            ValueError, match="`BOLD_confounds` data type not provided"
+            ValueError, match="`BOLD.confounds` data type not provided"
         ):
-            confound_remover._validate_data(bold, {})
+            confound_remover._validate_data(bold)
+        # Test confound data
+        bold["confounds"] = {}
         with pytest.raises(
-            ValueError, match="`BOLD_confounds.data` not provided"
+            ValueError, match="`BOLD.confounds.data` not provided"
         ):
-            confound_remover._validate_data(bold, {"BOLD_confounds": {}})
-
-        extra_input = {
-            "BOLD_confounds": {"data": "wrong"},
-        }
-        msg = "must be a `pandas.DataFrame`"
-        with pytest.raises(ValueError, match=msg):
-            confound_remover._validate_data(bold, extra_input)
-
-        extra_input = {"BOLD_confounds": {"data": pd.DataFrame()}}
+            confound_remover._validate_data(bold)
+        # Test confound data is valid type
+        bold["confounds"] = {"data": None}
+        with pytest.raises(ValueError, match="must be a `pandas.DataFrame`"):
+            confound_remover._validate_data(bold)
+        # Test confound data dimension mismatch with BOLD
+        bold["confounds"] = {"data": pd.DataFrame()}
         with pytest.raises(ValueError, match="Image time series and"):
-            confound_remover._validate_data(bold, extra_input)
-
-        extra_input = {
-            "BOLD_confounds": {"data": element_data["BOLD_confounds"]["data"]}
+            confound_remover._validate_data(bold)
+    # Check nested type variations
+    with PartlyCloudyTestingDataGrabber(reduce_confounds=False) as dg:
+        element_data = DefaultDataReader().fit_transform(dg["sub-01"])
+        # Test format
+        modified_bold = {
+            "data": element_data["BOLD"]["data"],
+            "confounds": {
+                "data": element_data["BOLD"]["confounds"]["data"],
+                "format": "adhoc",
+            },
         }
+        # Test incorrect format
+        modified_bold["confounds"].update({"format": "wrong"})
+        with pytest.raises(ValueError, match="Invalid confounds format"):
+            confound_remover._validate_data(modified_bold)
+        # Test missing mappings for adhoc
+        modified_bold["confounds"].update({"format": "adhoc"})
         with pytest.raises(
-            ValueError, match="`BOLD_confounds.format` not provided"
+            ValueError, match="`BOLD.confounds.mappings` need to be set"
         ):
-            confound_remover._validate_data(bold, extra_input)
-
-        extra_input = {
-            "BOLD_confounds": {
-                "data": element_data["BOLD_confounds"]["data"],
-                "format": "wrong",
-            }
-        }
-        with pytest.raises(ValueError, match="Invalid confounds format"):
-            confound_remover._validate_data(bold, extra_input)
-
-        extra_input = {
-            "BOLD_confounds": {
-                "data": element_data["BOLD_confounds"]["data"],
-                "format": "adhoc",
-            }
-        }
-        with pytest.raises(ValueError, match="need to be set"):
-            confound_remover._validate_data(bold, extra_input)
-
-        extra_input = {
-            "BOLD_confounds": {
-                "data": element_data["BOLD_confounds"]["data"],
-                "format": "adhoc",
-                "mappings": {},
-            }
-        }
-        with pytest.raises(ValueError, match="need to be set"):
-            confound_remover._validate_data(bold, extra_input)
-
-        extra_input = {
-            "BOLD_confounds": {
-                "data": element_data["BOLD_confounds"]["data"],
-                "format": "adhoc",
+            confound_remover._validate_data(modified_bold)
+        # Test missing fmriprep mappings for adhoc
+        modified_bold["confounds"].update({"mappings": {}})
+        with pytest.raises(
+            ValueError,
+            match="`BOLD.confounds.mappings.fmriprep` need to be set",
+        ):
+            confound_remover._validate_data(modified_bold)
+        # Test incorrect fmriprep mappings for adhoc
+        modified_bold["confounds"].update(
+            {
                 "mappings": {
                     "fmriprep": {
                         "rot_x": "wrong",
                         "rot_y": "rot_z",
                         "rot_z": "rot_y",
-                    }
-                },
+                    },
+                }
             }
-        }
+        )
         with pytest.raises(ValueError, match=r"names: \['wrong'\]"):
-            confound_remover._validate_data(bold, extra_input)
-
-        extra_input = {
-            "BOLD_confounds": {
-                "data": element_data["BOLD_confounds"]["data"],
-                "format": "adhoc",
+            confound_remover._validate_data(modified_bold)
+        # Test missing fmriprep mappings for adhoc
+        modified_bold["confounds"].update(
+            {
                 "mappings": {
                     "fmriprep": {
                         "wrong": "rot_x",
                         "rot_y": "rot_z",
                         "rot_z": "rot_y",
-                    }
-                },
+                    },
+                }
             }
-        }
+        )
         with pytest.raises(ValueError, match=r"Missing columns: \['wrong'\]"):
-            confound_remover._validate_data(bold, extra_input)
-
-        extra_input = {
-            "BOLD_confounds": {
-                "data": element_data["BOLD_confounds"]["data"],
-                "format": "adhoc",
+            confound_remover._validate_data(modified_bold)
+        # Test correct adhoc format
+        modified_bold["confounds"].update(
+            {
                 "mappings": {
                     "fmriprep": {
                         "rot_x": "rot_x",
                         "rot_y": "rot_z",
                         "rot_z": "rot_y",
-                    }
-                },
+                    },
+                }
             }
-        }
-        confound_remover._validate_data(bold, extra_input)
+        )
+        confound_remover._validate_data(modified_bold)
 
 
 def test_fMRIPrepConfoundRemover_preprocess() -> None:
@@ -476,7 +435,9 @@ def test_fMRIPrepConfoundRemover_preprocess() -> None:
         element_data = DefaultDataReader().fit_transform(dg["sub-01"])
         orig_bold = element_data["BOLD"]["data"].get_fdata().copy()
         pre_input = element_data["BOLD"]
-        pre_extra_input = {"BOLD_confounds": element_data["BOLD_confounds"]}
+        pre_extra_input = {
+            "BOLD": {"confounds": element_data["BOLD"]["confounds"]}
+        }
         output, _ = confound_remover.preprocess(pre_input, pre_extra_input)
         trans_bold = output["data"].get_fdata()
         # Transformation is in place
@@ -530,7 +491,7 @@ def test_fMRIPrepConfoundRemover_fit_transform() -> None:
         assert t_meta["t_r"] is None
         assert t_meta["masks"] is None
 
-        assert "BOLD_mask" not in output
+        assert "mask" not in output["BOLD"]
 
         assert "dependencies" in output["BOLD"]["meta"]
         dependencies = output["BOLD"]["meta"]["dependencies"]
@@ -582,9 +543,7 @@ def test_fMRIPrepConfoundRemover_fit_transform_masks() -> None:
         assert "threshold" in t_meta["masks"]["compute_brain_mask"]
         assert t_meta["masks"]["compute_brain_mask"]["threshold"] == 0.2
 
-        assert "BOLD_mask" in output
-        assert "mask_item" in output["BOLD"]
-        assert output["BOLD"]["mask_item"] == "BOLD_mask"
+        assert "mask" in output["BOLD"]
 
         assert "dependencies" in output["BOLD"]["meta"]
         dependencies = output["BOLD"]["meta"]["dependencies"]

junifer/storage/base.py

@@ -189,7 +189,7 @@ class BaseFeatureStorage(ABC):
 
         Parameters
         ----------
-        kind : {"matrix", "timeseries", "vector"}
+        kind : {"matrix", "timeseries", "vector", "scalar_table"}
             The storage kind.
         **kwargs
             The keyword arguments.
@@ -218,6 +218,10 @@ class BaseFeatureStorage(ABC):
             )
         elif kind == "vector":
             self.store_vector(meta_md5=meta_md5, element=t_element, **kwargs)
+        elif kind == "scalar_table":
+            self.store_scalar_table(
+                meta_md5=meta_md5, element=t_element, **kwargs
+            )
 
     def store_matrix(
         self,
@@ -313,6 +317,38 @@ class BaseFeatureStorage(ABC):
             klass=NotImplementedError,
         )
 
+    def store_scalar_table(
+        self,
+        meta_md5: str,
+        element: Dict,
+        data: np.ndarray,
+        col_names: Optional[Iterable[str]] = None,
+        row_names: Optional[Iterable[str]] = None,
+        row_header_col_name: Optional[str] = "feature",
+    ) -> None:
+        """Store table with scalar values.
+
+        Parameters
+        ----------
+        meta_md5 : str
+            The metadata MD5 hash.
+        element : dict
+            The element as a dictionary.
+        data : numpy.ndarray
+            The timeseries data to store.
+        col_names : list or tuple of str, optional
+            The column labels (default None).
+        row_names : str, optional
+            The row labels (default None).
+        row_header_col_name : str, optional
+            The column name for the row header column (default "feature").
+
+        """
+        raise_error(
+            msg="Concrete classes need to implement store_scalar_table().",
+            klass=NotImplementedError,
+        )
+
     @abstractmethod
     def collect(self) -> None:
         """Collect data."""

junifer/storage/hdf5.py

@@ -56,7 +56,8 @@ def _create_chunk(
     Raises
     ------
     ValueError
-        If `kind` is not one of ['vector', 'matrix', 'timeseries'].
+        If `kind` is not one of ['vector', 'matrix', 'timeseries',
+        'scalar_table'].
 
     """
     if kind in ["vector", "matrix"]:
@@ -77,7 +78,7 @@ def _create_chunk(
             chunk_size=tuple(array_chunk_size),
             n_chunk=i_chunk,
         )
-    elif kind == "timeseries":
+    elif kind in ["timeseries", "scalar_table"]:
         out = ChunkedList(
             data=chunk_data,
             size=element_count,
@@ -86,7 +87,8 @@ def _create_chunk(
     else:
         raise_error(
             f"Invalid kind: {kind}. "
-            "Must be one of ['vector', 'matrix', 'timeseries']."
+            "Must be one of ['vector', 'matrix', 'timeseries',"
+            "'scalar_table']."
         )
     return out
 
@@ -146,7 +148,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
             uri.parent.mkdir(parents=True, exist_ok=True)
 
         # Available storage kinds
-        storage_types = ["vector", "timeseries", "matrix"]
+        storage_types = ["vector", "timeseries", "matrix", "scalar_table"]
 
         super().__init__(
             uri=uri,
@@ -169,7 +171,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
             storage.
 
         """
-        return ["matrix", "vector", "timeseries"]
+        return ["matrix", "vector", "timeseries", "scalar_table"]
 
     def _fetch_correct_uri_for_io(self, element: Optional[Dict]) -> str:
         """Return proper URI for I/O based on `element`.
@@ -508,6 +510,26 @@ class HDF5FeatureStorage(BaseFeatureStorage):
             columns = hdf_data["column_headers"]
             # Convert data from 3D to 2D
             reshaped_data = np.concatenate(all_data, axis=0)
+        elif hdf_data["kind"] == "scalar_table":
+            # Create dictionary for aggregating index data
+            element_idx = defaultdict(list)
+            all_data = []
+            for idx, element in enumerate(hdf_data["element"]):
+                # Get row count for the element
+                t_data = hdf_data["data"][idx]
+                all_data.append(t_data)
+                n_rows = len(hdf_data["row_headers"])
+                # Set rows for the index
+                for key, val in element.items():
+                    element_idx[key].extend([val] * n_rows)
+                # Add extra column for row header column name
+                element_idx[hdf_data["row_header_column_name"]].extend(
+                    hdf_data["row_headers"]
+                )
+            # Set column headers for dataframe
+            columns = hdf_data["column_headers"]
+            # Convert data from 3D to 2D
+            reshaped_data = np.concatenate(all_data, axis=0)
 
         # Create dataframe for index
         idx_df = pd.DataFrame(data=element_idx)  # type: ignore
@@ -643,7 +665,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
 
         Parameters
         ----------
-        kind : {"matrix", "vector", "timeseries"}
+        kind : {"matrix", "vector", "timeseries", "scalar_table"}
             The storage kind.
         meta_md5 : str
             The metadata MD5 hash.
@@ -739,8 +761,8 @@ class HDF5FeatureStorage(BaseFeatureStorage):
             )
 
             t_data = stored_data["data"]
-            if kind == "timeseries":
-                t_data.append(data)
+            if kind in ["timeseries", "scalar_table"]:
+                t_data += data
             else:
                 t_data = np.concatenate((t_data, data), axis=-1)
             # Existing entry; append to existing
@@ -921,6 +943,43 @@ class HDF5FeatureStorage(BaseFeatureStorage):
             row_header_column_name="timepoint",
         )
 
+    def store_scalar_table(
+        self,
+        meta_md5: str,
+        element: Dict,
+        data: np.ndarray,
+        col_names: Optional[Iterable[str]] = None,
+        row_names: Optional[Iterable[str]] = None,
+        row_header_col_name: Optional[str] = "feature",
+    ) -> None:
+        """Store table with scalar values.
+
+        Parameters
+        ----------
+        meta_md5 : str
+            The metadata MD5 hash.
+        element : dict
+            The element as a dictionary.
+        data : numpy.ndarray
+            The scalar table data to store.
+        col_names : list or tuple of str, optional
+            The column labels (default None).
+        row_names : str, optional
+            The row labels (default None).
+        row_header_col_name : str, optional
+            The column name for the row header column (default "feature").
+
+        """
+        self._store_data(
+            kind="scalar_table",
+            meta_md5=meta_md5,
+            element=[element],  # convert to list
+            data=[data],  # convert to list
+            column_headers=col_names,
+            row_headers=row_names,
+            row_header_column_name=row_header_col_name,
+        )
+
     def collect(self) -> None:
         """Implement data collection.
 
@@ -1029,7 +1088,7 @@ class HDF5FeatureStorage(BaseFeatureStorage):
                     kind = static_data["kind"]
 
                 # Append the "dynamic" data
-                if kind == "timeseries":
+                if kind in ["timeseries", "scalar_table"]:
                     chunk_data.extend(t_data["data"])
                 else:
                     chunk_data.append(t_data["data"])

junifer/storage/tests/test_hdf5.py

@@ -25,7 +25,12 @@ from junifer.storage.utils import (
 def test_get_valid_inputs() -> None:
     """Test valid inputs."""
     storage = HDF5FeatureStorage(uri="/tmp")
-    assert storage.get_valid_inputs() == ["matrix", "vector", "timeseries"]
+    assert storage.get_valid_inputs() == [
+        "matrix",
+        "vector",
+        "timeseries",
+        "scalar_table",
+    ]
 
 
 def test_single_output(tmp_path: Path) -> None:
@@ -808,7 +813,7 @@ def test_store_timeseries(tmp_path: Path) -> None:
     data = np.array([[10], [20], [30], [40], [50]])
     col_names = ["signal"]
 
-    # Store vector
+    # Store timeseries
     storage.store_timeseries(
         meta_md5=meta_md5,
         element=element_to_store,
@@ -822,6 +827,53 @@ def test_store_timeseries(tmp_path: Path) -> None:
     assert_array_equal(read_df.values, data)
 
 
+def test_store_scalar_table(tmp_path: Path) -> None:
+    """Test scalar table store.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    uri = tmp_path / "test_store_scalar_table.hdf5"
+    storage = HDF5FeatureStorage(uri=uri)
+    # Metadata to store
+    element = {"subject": "test"}
+    meta = {
+        "element": element,
+        "dependencies": ["numpy"],
+        "marker": {"name": "brainprint"},
+        "type": "FreeSurfer",
+    }
+    # Process the metadata
+    meta_md5, meta_to_store, element_to_store = process_meta(meta)
+    # Store metadata
+    storage.store_metadata(
+        meta_md5=meta_md5, element=element_to_store, meta=meta_to_store
+    )
+
+    # Data to store
+    data = np.array([[10, 20], [30, 40], [50, 60]])
+    col_names = ["roi1", "roi2"]
+    row_names = ["ev1", "ev2", "ev3"]
+
+    # Store timeseries
+    storage.store_scalar_table(
+        meta_md5=meta_md5,
+        element=element_to_store,
+        data=data,
+        col_names=col_names,
+        row_names=row_names,
+        row_header_col_name="eigenvalue",
+    )
+
+    # Read into dataframe
+    read_df = storage.read_df(feature_md5=meta_md5)
+    # Check if data are equal
+    assert_array_equal(read_df.values, data)
+
+
 def _create_data_to_store(n_elements: int, kind: str) -> Tuple[str, Dict]:
     """Create data to store.
 
@@ -854,13 +906,19 @@ def _create_data_to_store(n_elements: int, kind: str) -> Tuple[str, Dict]:
             "col_names": [f"col-{i}" for i in range(10)],
             "matrix_kind": "full",
         }
-    elif kind == "timeseries":
+    elif kind in "timeseries":
         data_to_store = {
             "data": np.arange(20).reshape(2, 10),
             "col_names": [f"col-{i}" for i in range(10)],
         }
-    else:
-        raise ValueError(f"Unknown kind {kind}.")
+    elif kind in "scalar_table":
+        data_to_store = {
+            "data": np.arange(50).reshape(5, 10),
+            "row_names": [f"row-{i}" for i in range(5)],
+            "col_names": [f"col-{i}" for i in range(10)],
+            "row_header_col_name": "row",
+        }
+
     for i in range(n_elements):
         element = {"subject": f"sub-{i // 2}", "session": f"ses-{i % 2}"}
         meta = {
@@ -903,6 +961,7 @@ def _create_data_to_store(n_elements: int, kind: str) -> Tuple[str, Dict]:
         (10, 3, "matrix"),
         (10, 5, "matrix"),
         (10, 5, "timeseries"),
+        (10, 5, "scalar_table"),
     ],
 )
 def test_multi_output_store_and_collect(
@@ -930,21 +989,20 @@ def test_multi_output_store_and_collect(
     meta_md5, all_data = _create_data_to_store(n_elements, kind)
 
     for t_data in all_data:
-        # Store metadata for tables
+        # Store metadata
         storage.store_metadata(
             meta_md5=meta_md5,
             element=t_data["element"],
             meta=t_data["meta"],
         )
+        # Store data
         if kind == "vector":
-            # Store tables
             storage.store_vector(
                 meta_md5=meta_md5,
                 element=t_data["element"],
                 **t_data["data"],
             )
         elif kind == "matrix":
-            # Store tables
             storage.store_matrix(
                 meta_md5=meta_md5,
                 element=t_data["element"],
@@ -956,11 +1014,17 @@ def test_multi_output_store_and_collect(
                 element=t_data["element"],
                 **t_data["data"],
             )
+        elif kind == "scalar_table":
+            storage.store_scalar_table(
+                meta_md5=meta_md5,
+                element=t_data["element"],
+                **t_data["data"],
+            )
     # Check that base URI does not exist yet
     assert not uri.exists()
 
     for t_data in all_data:
-        # Convert element to preifx
+        # Convert element to prefix
         prefix = element_to_prefix(t_data["element"])
         # URIs for data storage
         elem_uri = uri.parent / f"{prefix}{uri.name}"
@@ -977,7 +1041,7 @@ def test_multi_output_store_and_collect(
     # Check that base URI exists now
     assert uri.exists()
 
-    # # Read unified metadata
+    # Read unified metadata
     read_unified_meta = storage.list_features()
     assert meta_md5 in read_unified_meta
 
@@ -989,6 +1053,10 @@ def test_multi_output_store_and_collect(
         data_size = np.sum([x["data"]["data"].shape[0] for x in all_data])
         assert len(all_df) == data_size
         idx_names = [x for x in all_df.index.names if x != "timepoint"]
+    elif kind == "scalar_table":
+        data_size = np.sum([x["data"]["data"].shape[0] for x in all_data])
+        assert len(all_df) == data_size
+        idx_names = [x for x in all_df.index.names if x != "row"]
     else:
         assert len(all_df) == len(all_data)
         idx_names = all_df.index.names
@@ -1013,6 +1081,10 @@ def test_multi_output_store_and_collect(
             assert_array_equal(t_series.values, t_data["data"]["data"])
             series_names = t_series.columns.values.tolist()
             assert series_names == t_data["data"]["col_names"]
+        elif kind == "scalar_table":
+            assert_array_equal(t_series.values, t_data["data"]["data"])
+            series_names = t_series.columns.values.tolist()
+            assert series_names == t_data["data"]["col_names"]
 
 
 def test_collect_error_single_output() -> None: