junifer 0.0.5.dev240__py3-none-any.whl → 0.0.6__py3-none-any.whl

junifer/storage/tests/test_hdf5.py

@@ -6,7 +6,6 @@
 
 from copy import deepcopy
 from pathlib import Path
-from typing import Dict, Tuple
 
 import h5py
 import numpy as np
@@ -827,6 +826,70 @@ def test_store_timeseries(tmp_path: Path) -> None:
     assert_array_equal(read_df.values, data)
 
 
+def test_store_timeseries2d(tmp_path: Path) -> None:
+    """Test 2D timeseries store.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    uri = tmp_path / "test_store_timeseries_2d.hdf5"
+    storage = HDF5FeatureStorage(uri=uri)
+    # Metadata to store
+    element = {"subject": "test"}
+    meta = {
+        "element": element,
+        "dependencies": ["numpy"],
+        "marker": {"name": "fc"},
+        "type": "BOLD",
+    }
+    # Process the metadata
+    meta_md5, meta_to_store, element_to_store = process_meta(meta)
+    # Store metadata
+    storage.store_metadata(
+        meta_md5=meta_md5, element=element_to_store, meta=meta_to_store
+    )
+
+    # Data to store
+    data = np.array(
+        [[10, 11, 12], [20, 21, 22], [30, 31, 32], [40, 41, 42], [50, 51, 52]]
+    )
+    data = np.c_[[data + (i * 100) for i in range(4)]]  # Generate timeseries
+
+    col_names = ["roi1", "roi2", "roi3"]
+    row_names = ["ev1", "ev2", "ev3", "ev4", "ev5"]
+
+    # Store 2D timeseries
+    storage.store_timeseries_2d(
+        meta_md5=meta_md5,
+        element=element_to_store,
+        data=data,
+        col_names=col_names,
+        row_names=row_names,
+    )
+
+    # Read into dataframe
+    read_data = storage.read(feature_md5=meta_md5)
+    # Check if data are equal
+    assert_array_equal(read_data["data"][0], data)
+    assert read_data["column_headers"] == col_names
+    assert read_data["row_headers"], row_names
+
+    read_df = storage.read_df(feature_md5=meta_md5)
+    flatted_names = [f"{row}~{col}" for row in row_names for col in col_names]
+
+    expected_flat_data = np.array(
+        [10, 11, 12, 20, 21, 22, 30, 31, 32, 40, 41, 42, 50, 51, 52]
+    )
+    expected_flat_data = np.c_[
+        [expected_flat_data + (i * 100) for i in range(4)]
+    ]  # Generate timeseries
+    assert_array_equal(read_df.values, expected_flat_data)
+    assert read_df.columns.to_list() == flatted_names
+
+
 def test_store_scalar_table(tmp_path: Path) -> None:
     """Test scalar table store.
 
@@ -858,7 +921,7 @@ def test_store_scalar_table(tmp_path: Path) -> None:
     col_names = ["roi1", "roi2"]
     row_names = ["ev1", "ev2", "ev3"]
 
-    # Store timeseries
+    # Store scalar table
     storage.store_scalar_table(
         meta_md5=meta_md5,
         element=element_to_store,
@@ -874,7 +937,7 @@ def test_store_scalar_table(tmp_path: Path) -> None:
     assert_array_equal(read_df.values, data)
 
 
-def _create_data_to_store(n_elements: int, kind: str) -> Tuple[str, Dict]:
+def _create_data_to_store(n_elements: int, kind: str) -> tuple[str, dict]:
     """Create data to store.
 
     Parameters
@@ -911,6 +974,12 @@ def _create_data_to_store(n_elements: int, kind: str) -> Tuple[str, Dict]:
             "data": np.arange(20).reshape(2, 10),
             "col_names": [f"col-{i}" for i in range(10)],
         }
+    elif kind in "timeseries_2d":
+        data_to_store = {
+            "data": np.arange(120).reshape(6, 5, 4),
+            "row_names": [f"row-{i}" for i in range(5)],
+            "col_names": [f"col-{i}" for i in range(4)],
+        }
     elif kind in "scalar_table":
         data_to_store = {
             "data": np.arange(50).reshape(5, 10),
@@ -962,6 +1031,7 @@ def _create_data_to_store(n_elements: int, kind: str) -> Tuple[str, Dict]:
         (10, 5, "matrix"),
         (10, 5, "timeseries"),
         (10, 5, "scalar_table"),
+        (10, 5, "timeseries_2d"),
     ],
 )
 def test_multi_output_store_and_collect(
@@ -983,7 +1053,9 @@ def test_multi_output_store_and_collect(
     """
     uri = tmp_path / "test_multi_output_store_and_collect.hdf5"
     storage = HDF5FeatureStorage(
-        uri=uri, single_output=False, chunk_size=chunk_size
+        uri=uri,
+        single_output=False,
+        chunk_size=chunk_size,
     )
 
     meta_md5, all_data = _create_data_to_store(n_elements, kind)
@@ -1014,6 +1086,12 @@ def test_multi_output_store_and_collect(
                 element=t_data["element"],
                 **t_data["data"],
             )
+        elif kind == "timeseries_2d":
+            storage.store_timeseries_2d(
+                meta_md5=meta_md5,
+                element=t_data["element"],
+                **t_data["data"],
+            )
         elif kind == "scalar_table":
             storage.store_scalar_table(
                 meta_md5=meta_md5,
@@ -1053,6 +1131,10 @@ def test_multi_output_store_and_collect(
         data_size = np.sum([x["data"]["data"].shape[0] for x in all_data])
         assert len(all_df) == data_size
         idx_names = [x for x in all_df.index.names if x != "timepoint"]
+    elif kind == "timeseries_2d":
+        data_size = np.sum([x["data"]["data"].shape[0] for x in all_data])
+        assert len(all_df) == data_size
+        idx_names = [x for x in all_df.index.names if x != "timepoint"]
     elif kind == "scalar_table":
         data_size = np.sum([x["data"]["data"].shape[0] for x in all_data])
         assert len(all_df) == data_size

junifer/storage/tests/test_sqlite.py

@@ -5,7 +5,7 @@
 # License: AGPL
 
 from pathlib import Path
-from typing import List, Union
+from typing import Union
 
 import numpy as np
 import pandas as pd
@@ -56,7 +56,7 @@ df_ignore = pd.DataFrame(
 
 
 def _read_sql(
-    table_name: str, uri: str, index_col: Union[str, List[str]]
+    table_name: str, uri: str, index_col: Union[str, list[str]]
 ) -> pd.DataFrame:
     """Read database table into a pandas DataFrame.
 

junifer/storage/tests/test_storage_base.py

@@ -25,7 +25,7 @@ def test_BaseFeatureStorage() -> None:
         """Implement concrete class."""
 
         def __init__(self, uri, single_output=True):
-            storage_types = ["matrix", "vector", "timeseries"]
+            storage_types = ["matrix", "vector", "timeseries", "timeseries_2d"]
             super().__init__(
                 uri=uri,
                 storage_types=storage_types,
@@ -33,7 +33,7 @@ def test_BaseFeatureStorage() -> None:
             )
 
         def get_valid_inputs(self):
-            return ["matrix", "vector", "timeseries"]
+            return ["matrix", "vector", "timeseries", "timeseries_2d"]
 
         def list_features(self):
             super().list_features()
@@ -97,6 +97,9 @@ def test_BaseFeatureStorage() -> None:
     with pytest.raises(NotImplementedError):
         st.store(kind="timeseries", meta=meta)
 
+    with pytest.raises(NotImplementedError):
+        st.store(kind="timeseries_2d", meta=meta)
+
     with pytest.raises(NotImplementedError):
         st.store(kind="vector", meta=meta)
 

junifer/storage/tests/test_utils.py

@@ -4,7 +4,8 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Dict, Iterable, List, Tuple, Union
+from collections.abc import Iterable
+from typing import Union
 
 import numpy as np
 import pytest
@@ -16,6 +17,7 @@ from junifer.storage.utils import (
     matrix_to_vector,
     process_meta,
     store_matrix_checks,
+    timeseries2d_to_vector,
 )
 
 
@@ -198,7 +200,7 @@ def test_process_meta_invalid_metadata_key() -> None:
         ),
     ],
 )
-def test_process_meta_element(meta: Dict, elements: List[str]) -> None:
+def test_process_meta_element(meta: dict, elements: list[str]) -> None:
     """Test metadata element after processing.
 
     Parameters
@@ -215,7 +217,7 @@ def test_process_meta_element(meta: Dict, elements: List[str]) -> None:
     assert "A" in processed_meta
     assert "B" in processed_meta
     assert "element" not in processed_meta
-    assert isinstance(processed_meta["dependencies"], Dict)
+    assert isinstance(processed_meta["dependencies"], dict)
     assert all(
         x in processed_meta["dependencies"] for x in meta["dependencies"]
     )
@@ -232,7 +234,7 @@ def test_process_meta_element(meta: Dict, elements: List[str]) -> None:
         ({"subject": 1, "session": 2}, "element_1_2_"),
     ],
 )
-def test_element_to_prefix(element: Dict, prefix: str) -> None:
+def test_element_to_prefix(element: dict, prefix: str) -> None:
     """Test converting element to prefix (for file naming).
 
     Parameters
@@ -320,7 +322,7 @@ def test_element_to_prefix_invalid_type() -> None:
     ],
 )
 def test_store_matrix_checks(
-    params: Dict[str, Union[str, bool, Tuple[int, int], int]], err_msg: str
+    params: dict[str, Union[str, bool, tuple[int, int], int]], err_msg: str
 ) -> None:
     """Test matrix storing parameter checks.
 
@@ -401,9 +403,9 @@ def test_store_matrix_checks(
     ],
 )
 def test_matrix_to_vector(
-    params: Dict[str, Union[np.ndarray, Iterable[str], str, bool]],
+    params: dict[str, Union[np.ndarray, Iterable[str], str, bool]],
     expected_data: np.ndarray,
-    expected_columns: List[str],
+    expected_columns: list[str],
 ) -> None:
     """Test matrix to vector.
 
@@ -420,3 +422,27 @@ def test_matrix_to_vector(
     data, columns = matrix_to_vector(**params)  # type: ignore
     assert_array_equal(data, expected_data)
     assert columns == expected_columns
+
+
+def test_timeseries2d_to_vector() -> None:
+    """Test timeseries2d to vector."""
+    data = np.array(
+        [[10, 11, 12], [20, 21, 22], [30, 31, 32], [40, 41, 42], [50, 51, 52]]
+    )
+    data = np.c_[[data + (i * 100) for i in range(4)]]  # Generate timeseries
+    col_names = ["c0", "c1", "c2"]
+    row_names = ["r0", "r1", "r2", "r3", "r4"]
+    flat_data, columns = timeseries2d_to_vector(
+        data=data,
+        col_names=col_names,
+        row_names=row_names,
+    )
+
+    expected_flat_data = np.array(
+        [10, 11, 12, 20, 21, 22, 30, 31, 32, 40, 41, 42, 50, 51, 52]
+    )
+    expected_flat_data = np.c_[
+        [expected_flat_data + (i * 100) for i in range(4)]
+    ]  # Generate timeseries
+    assert_array_equal(flat_data, expected_flat_data)
+    assert columns == [f"{r}~{c}" for r in row_names for c in col_names]

junifer/storage/utils.py

@@ -6,8 +6,8 @@
 
 import hashlib
 import json
+from collections.abc import Iterable
 from importlib.metadata import PackageNotFoundError, version
-from typing import Dict, Iterable, List, Tuple
 
 import numpy as np
 
@@ -15,11 +15,11 @@ from ..utils.logging import logger, raise_error
 
 
 __all__ = [
+    "element_to_prefix",
     "get_dependency_version",
+    "matrix_to_vector",
     "process_meta",
-    "element_to_prefix",
     "store_matrix_checks",
-    "matrix_to_vector",
 ]
 
 
@@ -50,7 +50,7 @@ def get_dependency_version(dependency: str) -> str:
     return dep_version
 
 
-def _meta_hash(meta: Dict) -> str:
+def _meta_hash(meta: dict) -> str:
     """Compute the MD5 hash of the metadata.
 
     Parameters
@@ -78,7 +78,7 @@ def _meta_hash(meta: Dict) -> str:
     return meta_md5
 
 
-def process_meta(meta: Dict) -> Tuple[str, Dict, Dict]:
+def process_meta(meta: dict) -> tuple[str, dict, dict]:
     """Process the metadata for storage.
 
     It removes the key "element" and adds the "_element_keys" with the keys
@@ -109,7 +109,7 @@ def process_meta(meta: Dict) -> Tuple[str, Dict, Dict]:
     # Copy the metadata
     t_meta = meta.copy()
     # Remove key "element"
-    element: Dict = t_meta.pop("element", None)
+    element: dict = t_meta.pop("element", None)
     if element is None:
         raise_error(msg="`meta` must contain the key 'element'")
     if "marker" not in t_meta:
@@ -128,7 +128,7 @@ def process_meta(meta: Dict) -> Tuple[str, Dict, Dict]:
     return md5_hash, t_meta, element
 
 
-def element_to_prefix(element: Dict) -> str:
+def element_to_prefix(element: dict) -> str:
     """Convert the element metadata to prefix.
 
     Parameters
@@ -156,7 +156,7 @@ def element_to_prefix(element: Dict) -> str:
 def store_matrix_checks(
     matrix_kind: str,
     diagonal: bool,
-    data_shape: Tuple[int, int],
+    data_shape: tuple[int, int],
     row_names_len: int,
     col_names_len: int,
 ) -> None:
@@ -181,6 +181,15 @@ def store_matrix_checks(
     col_names_len : int
         The length of column labels.
 
+    Raises
+    ------
+    ValueError
+        If the matrix kind is invalid
+        If the diagonal is False and the matrix kind is "full"
+        If the matrix kind is "triu" or "tril" and the matrix is not square
+        If the number of row names does not match the number of rows
+        If the number of column names does not match the number of columns
+
     """
     # Matrix kind validation
     if matrix_kind not in ("triu", "tril", "full"):
@@ -212,13 +221,58 @@ def store_matrix_checks(
         )
 
 
+def store_timeseries_2d_checks(
+    data_shape: tuple[int, int, int],
+    row_names_len: int,
+    col_names_len: int,
+) -> None:
+    """Run parameter checks for store_timeseries_2d() methods.
+
+    Parameters
+    ----------
+    data_shape : tuple of int and int
+        The shape of the matrix data to store.
+    row_names_len : int
+        The length of row labels.
+    col_names_len : int
+        The length of column labels.
+
+    Raises
+    ------
+    ValueError
+        If the data is not a 3D array (timepoints, rows, columns)
+        If the number of row names does not match the number of rows
+        If the number of column names does not match the number of columns
+
+    """
+    # data validation
+    if len(data_shape) != 3:
+        raise_error(
+            msg="Data must be a 3D array",
+            klass=ValueError,
+        )
+
+    # Row label validation
+    if row_names_len != data_shape[1]:  # type: ignore
+        raise_error(
+            msg="Number of row names does not match number of rows",
+            klass=ValueError,
+        )
+    # Column label validation
+    if col_names_len != data_shape[2]:  # type: ignore
+        raise_error(
+            msg="Number of column names does not match number of columns",
+            klass=ValueError,
+        )
+
+
 def matrix_to_vector(
     data: np.ndarray,
     col_names: Iterable[str],
     row_names: Iterable[str],
     matrix_kind: str,
     diagonal: bool,
-) -> Tuple[np.ndarray, List[str]]:
+) -> tuple[np.ndarray, list[str]]:
     """Convert matrix to vector based on parameters.
 
     Parameters
@@ -268,3 +322,35 @@ def matrix_to_vector(
     ]
 
     return flat_data, columns
+
+
+def timeseries2d_to_vector(
+    data: np.ndarray,
+    col_names: Iterable[str],
+    row_names: Iterable[str],
+) -> tuple[np.ndarray, list[str]]:
+    """Convert matrix to vector based on parameters.
+
+    Parameters
+    ----------
+    data : 2D / 3D numpy.ndarray
+        The matrix / tensor data to store / read.
+    col_names : list or tuple of str
+        The column labels.
+    row_names : list or tuple of str
+        The row labels.
+
+    Returns
+    -------
+    2D numpy.ndarray
+        The vector / matrix data.
+    list of str
+        The column labels.
+
+    """
+    # Reshape data to 2D
+    flat_data = data.reshape(data.shape[0], -1)
+    # Generate flat 1D row X column names
+    columns = [f"{r}~{c}" for r in row_names for c in col_names]
+
+    return flat_data, columns

junifer/testing/__init__.py

@@ -4,8 +4,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from . import datagrabbers
-from .utils import get_testing_data
+import lazy_loader as lazy
 
 
-__all__ = ["datagrabbers", "get_testing_data"]
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)

junifer/testing/__init__.pyi

@@ -0,0 +1,4 @@
+__all__ = ["datagrabbers", "registry", "get_testing_data"]
+
+from . import datagrabbers, registry
+from .utils import get_testing_data

junifer/testing/datagrabbers.py

@@ -6,7 +6,6 @@
 
 import tempfile
 from pathlib import Path
-from typing import Dict, List
 
 import nibabel as nib
 from nilearn import datasets, image
@@ -16,8 +15,8 @@ from ..datagrabber.base import BaseDataGrabber
 
 __all__ = [
     "OasisVBMTestingDataGrabber",
-    "SPMAuditoryTestingDataGrabber",
     "PartlyCloudyTestingDataGrabber",
+    "SPMAuditoryTestingDataGrabber",
 ]
 
 
@@ -35,7 +34,7 @@ class OasisVBMTestingDataGrabber(BaseDataGrabber):
         types = ["VBM_GM"]
         super().__init__(types=types, datadir=datadir)
 
-    def get_element_keys(self) -> List[str]:
+    def get_element_keys(self) -> list[str]:
         """Get element keys.
 
         Returns
@@ -46,7 +45,7 @@ class OasisVBMTestingDataGrabber(BaseDataGrabber):
         """
         return ["subject"]
 
-    def get_item(self, subject: str) -> Dict[str, Dict]:
+    def get_item(self, subject: str) -> dict[str, dict]:
         """Implement indexing support.
 
         Parameters
@@ -80,7 +79,7 @@ class OasisVBMTestingDataGrabber(BaseDataGrabber):
         self._dataset = datasets.fetch_oasis_vbm(n_subjects=10)
         return self
 
-    def get_elements(self) -> List[str]:
+    def get_elements(self) -> list[str]:
         """Get elements.
 
         Returns
@@ -106,7 +105,7 @@ class SPMAuditoryTestingDataGrabber(BaseDataGrabber):
         types = ["BOLD", "T1w"]  # TODO: Check that they are T1w
         super().__init__(types=types, datadir=datadir)
 
-    def get_element_keys(self) -> List[str]:
+    def get_element_keys(self) -> list[str]:
         """Get element keys.
 
         Returns
@@ -117,7 +116,7 @@ class SPMAuditoryTestingDataGrabber(BaseDataGrabber):
         """
         return ["subject"]
 
-    def get_elements(self) -> List[str]:
+    def get_elements(self) -> list[str]:
         """Get elements.
 
         Returns
@@ -128,7 +127,7 @@ class SPMAuditoryTestingDataGrabber(BaseDataGrabber):
         """
         return [f"sub{x:03d}" for x in list(range(1, 11))]
 
-    def get_item(self, subject: str) -> Dict[str, Dict]:
+    def get_item(self, subject: str) -> dict[str, dict]:
         """Implement indexing support.
 
         Parameters
@@ -208,7 +207,7 @@ class PartlyCloudyTestingDataGrabber(BaseDataGrabber):
         )
         return self
 
-    def get_element_keys(self) -> List[str]:
+    def get_element_keys(self) -> list[str]:
         """Get element keys.
 
         Returns
@@ -219,7 +218,7 @@ class PartlyCloudyTestingDataGrabber(BaseDataGrabber):
         """
         return ["subject"]
 
-    def get_elements(self) -> List[str]:
+    def get_elements(self) -> list[str]:
         """Get elements.
 
         Returns
@@ -230,7 +229,7 @@ class PartlyCloudyTestingDataGrabber(BaseDataGrabber):
         """
         return [f"sub-{x:02d}" for x in list(range(1, 11))]
 
-    def get_item(self, subject: str) -> Dict[str, Dict]:
+    def get_item(self, subject: str) -> dict[str, dict]:
         """Implement indexing support.
 
         Parameters

junifer/testing/registry.py

@@ -4,7 +4,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from ..pipeline.registry import register
+from ..pipeline import PipelineComponentRegistry
 from .datagrabbers import (
     OasisVBMTestingDataGrabber,
     PartlyCloudyTestingDataGrabber,
@@ -13,20 +13,17 @@ from .datagrabbers import (
 
 
 # Register testing DataGrabbers
-register(
+PipelineComponentRegistry().register(
     step="datagrabber",
-    name="OasisVBMTestingDataGrabber",
     klass=OasisVBMTestingDataGrabber,
 )
 
-register(
+PipelineComponentRegistry().register(
     step="datagrabber",
-    name="SPMAuditoryTestingDataGrabber",
     klass=SPMAuditoryTestingDataGrabber,
 )
 
-register(
+PipelineComponentRegistry().register(
     step="datagrabber",
-    name="PartlyCloudyTestingDataGrabber",
     klass=PartlyCloudyTestingDataGrabber,
 )

junifer/testing/tests/test_testing_registry.py

@@ -1,24 +1,16 @@
 """Provide tests for testing registry."""
 
-import importlib
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+#          Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
 
-from junifer.pipeline.registry import get_step_names
+from junifer.pipeline import PipelineComponentRegistry
 
 
 def test_testing_registry() -> None:
     """Test testing registry."""
-    import junifer
-
-    importlib.reload(junifer.pipeline.registry)
-    importlib.reload(junifer)
-
-    assert "OasisVBMTestingDataGrabber" not in get_step_names("datagrabber")
-    assert "SPMAuditoryTestingDataGrabber" not in get_step_names("datagrabber")
-    assert "PartlyCloudyTestingDataGrabber" not in get_step_names(
-        "datagrabber"
-    )
-    importlib.reload(junifer.testing.registry)  # type: ignore
-
-    assert "OasisVBMTestingDataGrabber" in get_step_names("datagrabber")
-    assert "SPMAuditoryTestingDataGrabber" in get_step_names("datagrabber")
-    assert "PartlyCloudyTestingDataGrabber" in get_step_names("datagrabber")
+    assert {
+        "OasisVBMTestingDataGrabber",
+        "SPMAuditoryTestingDataGrabber",
+        "PartlyCloudyTestingDataGrabber",
+    }.issubset(set(PipelineComponentRegistry().step_components("datagrabber")))

junifer/tests/test_stats.py

@@ -3,7 +3,7 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Dict, Optional
+from typing import Optional
 
 import numpy as np
 import pytest
@@ -25,7 +25,7 @@ from junifer.stats import count, get_aggfunc_by_name, select, winsorized_mean
         ("mode", {"keepdims": True}),
     ],
 )
-def test_get_aggfunc_by_name(name: str, params: Optional[Dict]) -> None:
+def test_get_aggfunc_by_name(name: str, params: Optional[dict]) -> None:
     """Test aggregation function retrieval by name.
 
     Parameters

junifer/typing/__init__.py

@@ -0,0 +1,9 @@
+"""Type hints for internal and external use."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+import lazy_loader as lazy
+
+
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)