junifer 0.0.7.dev332__py3-none-any.whl → 0.0.8__py3-none-any.whl

junifer/storage/tests/test_repeated_storage.py

@@ -0,0 +1,293 @@
+"""Provide tests for storing the same data more than once."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+# License: AGPL
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from junifer.storage import HDF5FeatureStorage, SQLiteFeatureStorage
+from junifer.storage.base import BaseFeatureStorage
+
+
+# The storage types supported by each storage
+_CASES = [
+    (klass, kind, single_output)
+    for klass, kinds in (
+        (
+            HDF5FeatureStorage,
+            [
+                "vector",
+                "matrix",
+                "timeseries",
+                "scalar_table",
+                "timeseries_2d",
+            ],
+        ),
+        (SQLiteFeatureStorage, ["vector", "matrix", "timeseries"]),
+    )
+    for kind in kinds
+    for single_output in (True, False)
+]
+
+
+def _data(kind: str, value: float) -> dict:
+    """Get data of a storage type with a value.
+
+    Parameters
+    ----------
+    kind : str
+        The storage type.
+    value : float
+        The value of the data.
+
+    Returns
+    -------
+    dict
+        The data and the other parameters to store it.
+
+    """
+    names = {"col_names": ["a", "b"]}
+    if kind == "vector":
+        return {"data": np.full((1, 2), value), **names}
+    if kind == "matrix":
+        return {
+            "data": np.full((2, 2), value),
+            "row_names": ["r1", "r2"],
+            "matrix_kind": "full",
+            **names,
+        }
+    if kind == "timeseries":
+        return {"data": np.full((3, 2), value), **names}
+    if kind == "scalar_table":
+        return {
+            "data": np.full((2, 2), value),
+            "row_names": ["r1", "r2"],
+            "row_header_col_name": "feature",
+            **names,
+        }
+    return {
+        "data": np.full((3, 2, 2), value),
+        "row_names": ["r1", "r2"],
+        **names,
+    }
+
+
+def _store(
+    storage: BaseFeatureStorage,
+    kind: str,
+    element: dict,
+    value: float,
+    element_keys: list[str] | None = None,
+) -> None:
+    """Store data of an element being processed, as a marker does.
+
+    Parameters
+    ----------
+    storage : BaseFeatureStorage
+        The storage.
+    kind : str
+        The storage type.
+    element : dict
+        The element being processed.
+    value : float
+        The value of the data.
+    element_keys : list of str or None, optional
+        The keys of the element that the data depends on. If None, all the
+        keys (default None).
+
+    """
+    meta = {
+        "element": element,
+        "dependencies": set(),
+        "marker": {"name": "test"},
+        "type": "BOLD",
+    }
+    if element_keys is not None:
+        meta["_element_keys"] = element_keys
+    storage.store(kind=kind, meta=meta, **_data(kind, value))
+
+
+def _read(
+    storage: BaseFeatureStorage, collect: bool = True
+) -> tuple[pd.DataFrame, list[tuple]]:
+    """Read the stored feature, collecting it first if needed.
+
+    Parameters
+    ----------
+    storage : BaseFeatureStorage
+        The storage.
+    collect : bool, optional
+        Whether to collect the files of the elements first (default True).
+
+    Returns
+    -------
+    pandas.DataFrame
+        The feature.
+    list of tuple
+        The elements of the feature, without duplicates.
+
+    """
+    if not storage.single_output and collect:
+        storage.collect()
+    reader = type(storage)(uri=storage.uri)
+    (md5,) = reader.list_features()
+    df = reader.read_df(feature_md5=md5)
+    # No repeated rows (the index has the element and, for some storage
+    # types, the rows of the data of an element)
+    assert not df.index.duplicated().any()
+    keys = [k for k in ("subject", "task") if k in df.index.names]
+    levels = [df.index.get_level_values(k) for k in keys]
+    elements = sorted(set(zip(*levels, strict=True)))
+    return df, elements
+
+
+@pytest.mark.parametrize("klass, kind, single_output", _CASES)
+def test_store_again(
+    tmp_path: Path, klass: type, kind: str, single_output: bool
+) -> None:
+    """Test storing the same element again does not change the storage.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    klass : type
+        The parametrized storage class.
+    kind : str
+        The parametrized storage type.
+    single_output : bool
+        The parametrized storage in a single file.
+
+    """
+    storage = klass(uri=tmp_path / "out.db", single_output=single_output)
+    element = {"subject": "sub-01", "task": "rest"}
+    _store(storage, kind, element, 1)
+    # Already stored: skipped, even with other values
+    _store(storage, kind, element, 2)
+    df, elements = _read(storage)
+    assert elements == [("sub-01", "rest")]
+    assert (df.to_numpy() == 1).all()
+
+
+@pytest.mark.parametrize("klass, kind, single_output", _CASES)
+def test_store_element_of_data(
+    tmp_path: Path, klass: type, kind: str, single_output: bool
+) -> None:
+    """Test storing data that does not depend on all the element keys.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    klass : type
+        The parametrized storage class.
+    kind : str
+        The parametrized storage type.
+    single_output : bool
+        The parametrized storage in a single file.
+
+    """
+    storage = klass(uri=tmp_path / "out.db", single_output=single_output)
+    # The data of a subject, the same when processing each of its tasks
+    for task in ("rest", "movie"):
+        element = {"subject": "sub-01", "task": task}
+        _store(storage, kind, element, 1, element_keys=["subject"])
+    if not single_output:
+        # One file for each element being processed
+        files = sorted(x.name for x in tmp_path.glob("*_out.db"))
+        assert files == [
+            "element_sub-01_movie_out.db",
+            "element_sub-01_rest_out.db",
+        ]
+    df, elements = _read(storage)
+    assert elements == [("sub-01",)]
+    assert (df.to_numpy() == 1).all()
+
+
+@pytest.mark.parametrize(
+    "klass, kind, single_output",
+    [case for case in _CASES if not case[2]],
+)
+def test_collect_again(
+    tmp_path: Path, klass: type, kind: str, single_output: bool
+) -> None:
+    """Test collecting again does not change the storage.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    klass : type
+        The parametrized storage class.
+    kind : str
+        The parametrized storage type.
+    single_output : bool
+        The parametrized storage in a single file.
+
+    """
+    storage = klass(uri=tmp_path / "out.db", single_output=single_output)
+    for subject in ("sub-01", "sub-02"):
+        _store(storage, kind, {"subject": subject, "task": "rest"}, 1)
+    storage.collect()
+    _, elements = _read(storage)
+    assert elements == [("sub-01", "rest"), ("sub-02", "rest")]
+
+
+@pytest.mark.parametrize("klass, kind, single_output", _CASES)
+def test_store_trial_then_all(
+    tmp_path: Path, klass: type, kind: str, single_output: bool
+) -> None:
+    """Test storing some elements and then all the elements.
+
+    For example, running a few elements to test a configuration, and then
+    queueing all the elements with the same configuration.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    klass : type
+        The parametrized storage class.
+    kind : str
+        The parametrized storage type.
+    single_output : bool
+        The parametrized storage in a single file.
+
+    """
+    storage = klass(uri=tmp_path / "out.db", single_output=single_output)
+    # Trial
+    _store(storage, kind, {"subject": "sub-01", "task": "rest"}, 1)
+    _read(storage)
+    # All the elements
+    for subject in ("sub-01", "sub-02"):
+        _store(storage, kind, {"subject": subject, "task": "rest"}, 2)
+    df, elements = _read(storage)
+    assert elements == [("sub-01", "rest"), ("sub-02", "rest")]
+    # The trial element keeps its values
+    sub01 = df.xs("sub-01", level="subject")
+    assert (sub01.to_numpy() == 1).all()
+
+
+@pytest.mark.parametrize("klass", [HDF5FeatureStorage, SQLiteFeatureStorage])
+def test_collect_several_elements(tmp_path: Path, klass: type) -> None:
+    """Test collecting a file with more than one element raises an error.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    klass : type
+        The parametrized storage class.
+
+    """
+    # A file with two elements, named as the file of an element
+    several = klass(uri=tmp_path / "element_all_out.db")
+    for subject in ("sub-01", "sub-02"):
+        _store(several, "timeseries", {"subject": subject}, 1)
+    storage = klass(uri=tmp_path / "out.db", single_output=False)
+    with pytest.raises(RuntimeError, match="has 2 elements for the feature"):
+        storage.collect()

junifer/storage/tests/test_sqlite.py

@@ -5,7 +5,6 @@
 # License: AGPL
 
 from pathlib import Path
-from typing import Union
 
 import numpy as np
 import pandas as pd
@@ -14,7 +13,7 @@ from numpy.testing import assert_array_equal
 from pandas.testing import assert_frame_equal
 from sqlalchemy import create_engine
 
-from junifer.storage.sqlite import SQLiteFeatureStorage
+from junifer.storage import MatrixKind, SQLiteFeatureStorage, Upsert
 from junifer.storage.utils import element_to_prefix, process_meta
 
 
@@ -56,7 +55,7 @@ df_ignore = pd.DataFrame(
 
 
 def _read_sql(
-    table_name: str, uri: str, index_col: Union[str, list[str]]
+    table_name: str, uri: str, index_col: str | list[str]
 ) -> pd.DataFrame:
     """Read database table into a pandas DataFrame.
 
@@ -215,7 +214,7 @@ def test_upsert_update(tmp_path: Path) -> None:
 
     """
     uri = tmp_path / "test_upsert_delete.sqlite"
-    storage = SQLiteFeatureStorage(uri=uri)
+    storage = SQLiteFeatureStorage(uri=uri, upsert=Upsert.Update)
     # Save to database
     storage.store_df(
         meta_md5="table_name", element={"subject": "test"}, df=df1
@@ -503,16 +502,6 @@ def test_store_matrix(tmp_path: Path) -> None:
     read_df = storage.read_df(feature_md5=feature_md5)
     assert list(read_df.columns) == stored_names
 
-    with pytest.raises(ValueError, match="Invalid kind"):
-        storage.store_matrix(
-            meta_md5=meta_md5,
-            element=element_to_store,
-            data=data,
-            row_names=row_names,
-            col_names=col_names,
-            matrix_kind="wrong",
-        )
-
     with pytest.raises(ValueError, match="non-square"):
         storage.store_matrix(
             meta_md5=meta_md5,
@@ -520,7 +509,7 @@ def test_store_matrix(tmp_path: Path) -> None:
             data=data,
             row_names=row_names,
             col_names=col_names,
-            matrix_kind="triu",
+            matrix_kind=MatrixKind.UpperTriangle,
         )
 
     with pytest.raises(ValueError, match="cannot be False"):
@@ -530,7 +519,7 @@ def test_store_matrix(tmp_path: Path) -> None:
             data=data,
             row_names=row_names,
             col_names=col_names,
-            matrix_kind="full",
+            matrix_kind=MatrixKind.Full,
             diagonal=False,
         )
 
@@ -550,7 +539,7 @@ def test_store_matrix(tmp_path: Path) -> None:
         data=data,
         row_names=row_names,
         col_names=col_names,
-        matrix_kind="triu",
+        matrix_kind=MatrixKind.UpperTriangle,
     )
 
     stored_names = [
@@ -584,7 +573,7 @@ def test_store_matrix(tmp_path: Path) -> None:
         data=data,
         row_names=row_names,
         col_names=col_names,
-        matrix_kind="triu",
+        matrix_kind=MatrixKind.UpperTriangle,
         diagonal=False,
     )
 
@@ -619,7 +608,7 @@ def test_store_matrix(tmp_path: Path) -> None:
         data=data,
         row_names=row_names,
         col_names=col_names,
-        matrix_kind="tril",
+        matrix_kind=MatrixKind.LowerTriangle,
     )
 
     stored_names = [
@@ -653,7 +642,7 @@ def test_store_matrix(tmp_path: Path) -> None:
         data=data,
         row_names=row_names,
         col_names=col_names,
-        matrix_kind="tril",
+        matrix_kind=MatrixKind.LowerTriangle,
         diagonal=False,
     )
     stored_names = [

junifer/storage/tests/test_storage_base.py

@@ -5,17 +5,18 @@
 # License: AGPL
 
 from collections.abc import Sequence
+from pathlib import Path
 from typing import ClassVar
 
 import pytest
 
-from junifer.storage.base import BaseFeatureStorage
+from junifer.storage import BaseFeatureStorage, StorageType
 
 
 def test_BaseFeatureStorage_abstractness() -> None:
     """Test BaseFeatureStorage is abstract base class."""
     with pytest.raises(TypeError, match=r"abstract"):
-        BaseFeatureStorage(uri="/tmp")
+        BaseFeatureStorage(uri=Path("."))
 
 
 def test_BaseFeatureStorage() -> None:
@@ -25,19 +26,13 @@ def test_BaseFeatureStorage() -> None:
     class MyFeatureStorage(BaseFeatureStorage):
         """Implement concrete class."""
 
-        _STORAGE_TYPES: ClassVar[Sequence[str]] = [
-            "matrix",
-            "vector",
-            "timeseries",
-            "timeseries_2d",
+        _STORAGE_TYPES: ClassVar[Sequence[StorageType]] = [
+            StorageType.Matrix,
+            StorageType.Vector,
+            StorageType.Timeseries,
+            StorageType.Timeseries2D,
         ]
 
-        def __init__(self, uri, single_output=True):
-            super().__init__(
-                uri=uri,
-                single_output=single_output,
-            )
-
         def list_features(self):
             super().list_features()
 
@@ -53,24 +48,26 @@ def test_BaseFeatureStorage() -> None:
                 feature_md5=feature_md5,
             )
 
-        def store_metadata(self, meta_md5, meta, element):
-            super().store_metadata(meta_md5, meta, element)
+        def store_metadata(
+            self, meta_md5, element, meta, processed_element=None
+        ):
+            super().store_metadata(meta_md5, meta, element, processed_element)
 
         def collect(self):
-            return super().collect()
+            super().collect()
 
     # Check single_output is False
-    st = MyFeatureStorage(uri="/tmp", single_output=False)
+    st = MyFeatureStorage(uri=Path("."), single_output=False)
     assert st.single_output is False
     # Check single_output is True
-    st = MyFeatureStorage(uri="/tmp", single_output=True)
+    st = MyFeatureStorage(uri=Path("."), single_output=True)
     assert st.single_output is True
 
     # Check validate with valid argument
-    st.validate(input_=["matrix"])
+    st.validate_input(input_=["matrix"])
     # Check validate with invalid argument
     with pytest.raises(ValueError):
-        st.validate(input_=["duck"])
+        st.validate_input(input_=["duck"])
 
     with pytest.raises(NotImplementedError):
         st.list_features()
@@ -109,4 +106,4 @@ def test_BaseFeatureStorage() -> None:
     with pytest.raises(ValueError):
         st.store(kind="lego", meta=meta)
 
-    assert str(st.uri) == "/tmp"
+    assert str(st.uri) == "."

junifer/storage/tests/test_utils.py

@@ -5,7 +5,6 @@
 # License: AGPL
 
 from collections.abc import Iterable
-from typing import Union
 
 import numpy as np
 import pytest
@@ -24,14 +23,19 @@ from junifer.storage.utils import (
 @pytest.mark.parametrize(
     "dependency, max_version",
     [
-        ("click", "8.2"),
-        ("numpy", "1.27"),
-        ("datalad", "0.20"),
-        ("pandas", "2.2"),
-        ("nibabel", "5.11"),
-        ("nilearn", "0.11.0"),
-        ("sqlalchemy", "2.1.0"),
-        ("ruamel.yaml", "0.18.0"),
+        ("click", "8.6"),
+        ("numpy", "2.5.0"),
+        ("scipy", "1.18.0"),
+        ("datalad", "1.8.0"),
+        ("pandas", "3.1.0"),
+        ("nibabel", "5.5.0"),
+        ("nilearn", "0.15.0"),
+        ("sqlalchemy", "2.2.0"),
+        ("ruamel.yaml", "0.20.0"),
+        ("tqdm", "4.71.0"),
+        ("templateflow", "26.0.0"),
+        ("lapy", "2.0.0"),
+        ("structlog", "27.0.0"),
     ],
 )
 def test_get_dependency_version(dependency: str, max_version: str) -> None:
@@ -47,7 +51,7 @@ def test_get_dependency_version(dependency: str, max_version: str) -> None:
     """
     version = get_dependency_version(dependency)
     if len(version.split(".")) == 3:  # semver
-        assert int(version.split(".")[1]) <= int(max_version.split(".")[1])
+        assert int(version.split(".")[0]) <= int(max_version.split(".")[0])
     else:
         assert version < max_version
 
@@ -150,6 +154,73 @@ def test_process_meta_hash() -> None:
     assert hash4 != hash6
 
 
+def test_process_meta_element_keys() -> None:
+    """Test metadata processing with the element keys of the data."""
+    meta = {
+        "element": {"subject": "sub-01", "task": "rest"},
+        "dependencies": ["numpy"],
+        "marker": {"name": "fc"},
+        "type": "VBM_GM",
+    }
+    # All the keys of the element by default
+    hash_all, meta_all, element_all = process_meta(meta)
+    assert element_all == {"subject": "sub-01", "task": "rest"}
+    assert meta_all["_element_keys"] == ["subject", "task"]
+    # Only the keys of the element that the data depends on, which are the
+    # same for all the tasks
+    hashes = set()
+    for task in ("rest", "movie"):
+        t_meta = {
+            **meta,
+            "element": {"subject": "sub-01", "task": task},
+            "_element_keys": ["subject"],
+        }
+        t_hash, t_processed, t_element = process_meta(t_meta)
+        assert t_element == {"subject": "sub-01"}
+        assert t_processed["_element_keys"] == ["subject"]
+        hashes.add(t_hash)
+    assert len(hashes) == 1
+    assert hash_all not in hashes
+    # Keys that are not in the element
+    with pytest.raises(ValueError, match=r"keys \['session'\]"):
+        process_meta({**meta, "_element_keys": ["subject", "session"]})
+
+
+def test_process_meta_preprocessors() -> None:
+    """Test the metadata hash depends on all the preprocessors."""
+    hashes = []
+    for first in (1, 2):
+        meta = {
+            "element": {"subject": "sub-01"},
+            "dependencies": ["numpy"],
+            "preprocess": [
+                {"class": "A", "parameter": first},
+                {"class": "B", "parameter": 1},
+            ],
+            "marker": {"name": "fc"},
+            "type": "BOLD",
+        }
+        t_hash, _, _ = process_meta(meta)
+        hashes.append(t_hash)
+    # Only the first preprocessor is different
+    assert hashes[0] != hashes[1]
+
+
+def test_process_meta_junifer_version() -> None:
+    """Test the metadata hash depends on the version of junifer."""
+    meta = {
+        "element": {"subject": "sub-01"},
+        "dependencies": ["numpy"],
+        "marker": {"name": "fc"},
+        "type": "BOLD",
+    }
+    _, processed, _ = process_meta(meta)
+    assert processed["dependencies"] == {
+        "numpy": get_dependency_version("numpy"),
+        "junifer": get_dependency_version("junifer"),
+    }
+
+
 def test_process_meta_invalid_metadata_key() -> None:
     """Test invalid metadata key check for metadata hash processing."""
     meta = {}
@@ -259,16 +330,6 @@ def test_element_to_prefix_invalid_type() -> None:
 @pytest.mark.parametrize(
     "params, err_msg",
     [
-        (
-            {
-                "matrix_kind": "half",
-                "diagonal": True,
-                "data_shape": (1, 1),
-                "row_names_len": 1,
-                "col_names_len": 1,
-            },
-            "Invalid kind",
-        ),
         (
             {
                 "matrix_kind": "full",
@@ -322,7 +383,7 @@ def test_element_to_prefix_invalid_type() -> None:
     ],
 )
 def test_store_matrix_checks(
-    params: dict[str, Union[str, bool, tuple[int, int], int]], err_msg: str
+    params: dict[str, str | bool | tuple[int, int] | int], err_msg: str
 ) -> None:
     """Test matrix storing parameter checks.
 
@@ -403,7 +464,7 @@ def test_store_matrix_checks(
     ],
 )
 def test_matrix_to_vector(
-    params: dict[str, Union[np.ndarray, Iterable[str], str, bool]],
+    params: dict[str, np.ndarray | Iterable[str] | str | bool],
     expected_data: np.ndarray,
     expected_columns: list[str],
 ) -> None: