junifer 0.0.5.dev219__py3-none-any.whl → 0.0.5.dev242__py3-none-any.whl

junifer/datagrabber/tests/test_multiple.py

@@ -25,28 +25,26 @@ def test_MultipleDataGrabber() -> None:
     repo_uri = _testing_dataset["example_bids_ses"]["uri"]
     rootdir = "example_bids_ses"
     replacements = ["subject", "session"]
-    pattern1 = {
-        "T1w": {
-            "pattern": (
-                "{subject}/{session}/anat/{subject}_{session}_T1w.nii.gz"
-            ),
-            "space": "native",
-        },
-    }
-    pattern2 = {
-        "BOLD": {
-            "pattern": (
-                "{subject}/{session}/func/"
-                "{subject}_{session}_task-rest_bold.nii.gz"
-            ),
-            "space": "MNI152NLin6Asym",
-        },
-    }
+
     dg1 = PatternDataladDataGrabber(
         rootdir=rootdir,
         uri=repo_uri,
         types=["T1w"],
-        patterns=pattern1,
+        patterns={
+            "T1w": {
+                "pattern": (
+                    "{subject}/{session}/anat/{subject}_{session}_T1w.nii.gz"
+                ),
+                "space": "native",
+                "mask": {
+                    "pattern": (
+                        "{subject}/{session}/anat/{subject}_{session}_"
+                        "brain_mask.nii.gz"
+                    ),
+                    "space": "native",
+                },
+            },
+        },
         replacements=replacements,
     )
 
@@ -54,7 +52,22 @@ def test_MultipleDataGrabber() -> None:
         rootdir=rootdir,
         uri=repo_uri,
         types=["BOLD"],
-        patterns=pattern2,
+        patterns={
+            "BOLD": {
+                "pattern": (
+                    "{subject}/{session}/func/"
+                    "{subject}_{session}_task-rest_bold.nii.gz"
+                ),
+                "space": "MNI152NLin6Asym",
+                "mask": {
+                    "pattern": (
+                        "{subject}/{session}/func/"
+                        "{subject}_{session}_task-rest_brain_mask.nii.gz"
+                    ),
+                    "space": "MNI152NLin6Asym",
+                },
+            },
+        },
         replacements=replacements,
     )
 
@@ -73,14 +86,17 @@ def test_MultipleDataGrabber() -> None:
     with dg:
         subs = list(dg)
         assert set(subs) == set(expected_subs)
-
+        # Check data type
         elem = dg[("sub-01", "ses-01")]
+        # Check data types
         assert "T1w" in elem
         assert "BOLD" in elem
+        # Check meta
         assert "meta" in elem["BOLD"]
         meta = elem["BOLD"]["meta"]["datagrabber"]
         assert "class" in meta
         assert meta["class"] == "MultipleDataGrabber"
+        # Check datagrabbers
         assert "datagrabbers" in meta
         assert len(meta["datagrabbers"]) == 2
         assert meta["datagrabbers"][0]["class"] == "PatternDataladDataGrabber"
@@ -89,40 +105,37 @@ def test_MultipleDataGrabber() -> None:
 
 def test_MultipleDataGrabber_no_intersection() -> None:
     """Test MultipleDataGrabber without intersection (0 elements)."""
-    repo_uri1 = _testing_dataset["example_bids"]["uri"]
-    repo_uri2 = _testing_dataset["example_bids_ses"]["uri"]
     rootdir = "example_bids_ses"
     replacements = ["subject", "session"]
-    pattern1 = {
-        "T1w": {
-            "pattern": (
-                "{subject}/{session}/anat/{subject}_{session}_T1w.nii.gz"
-            ),
-            "space": "native",
-        },
-    }
-    pattern2 = {
-        "BOLD": {
-            "pattern": (
-                "{subject}/{session}/func/"
-                "{subject}_{session}_task-rest_bold.nii.gz"
-            ),
-            "space": "MNI152NLin6Asym",
-        },
-    }
+
     dg1 = PatternDataladDataGrabber(
         rootdir=rootdir,
-        uri=repo_uri1,
+        uri=_testing_dataset["example_bids"]["uri"],
         types=["T1w"],
-        patterns=pattern1,
+        patterns={
+            "T1w": {
+                "pattern": (
+                    "{subject}/{session}/anat/{subject}_{session}_T1w.nii.gz"
+                ),
+                "space": "native",
+            },
+        },
         replacements=replacements,
     )
 
     dg2 = PatternDataladDataGrabber(
         rootdir=rootdir,
-        uri=repo_uri2,
+        uri=_testing_dataset["example_bids_ses"]["uri"],
         types=["BOLD"],
-        patterns=pattern2,
+        patterns={
+            "BOLD": {
+                "pattern": (
+                    "{subject}/{session}/func/"
+                    "{subject}_{session}_task-rest_bold.nii.gz"
+                ),
+                "space": "MNI152NLin6Asym",
+            },
+        },
         replacements=replacements,
     )
 
@@ -135,23 +148,19 @@ def test_MultipleDataGrabber_no_intersection() -> None:
 
 def test_MultipleDataGrabber_get_item() -> None:
     """Test MultipleDataGrabber get_item() error."""
-    repo_uri1 = _testing_dataset["example_bids"]["uri"]
-    rootdir = "example_bids_ses"
-    replacements = ["subject", "session"]
-    pattern1 = {
-        "T1w": {
-            "pattern": (
-                "{subject}/{session}/anat/{subject}_{session}_T1w.nii.gz"
-            ),
-            "space": "native",
-        },
-    }
     dg1 = PatternDataladDataGrabber(
-        rootdir=rootdir,
-        uri=repo_uri1,
+        rootdir="example_bids_ses",
+        uri=_testing_dataset["example_bids"]["uri"],
         types=["T1w"],
-        patterns=pattern1,
-        replacements=replacements,
+        patterns={
+            "T1w": {
+                "pattern": (
+                    "{subject}/{session}/anat/{subject}_{session}_T1w.nii.gz"
+                ),
+                "space": "native",
+            },
+        },
+        replacements=["subject", "session"],
     )
 
     dg = MultipleDataGrabber([dg1])
@@ -161,43 +170,111 @@ def test_MultipleDataGrabber_get_item() -> None:
 
 def test_MultipleDataGrabber_validation() -> None:
     """Test MultipleDataGrabber init validation."""
-    repo_uri1 = _testing_dataset["example_bids"]["uri"]
-    repo_uri2 = _testing_dataset["example_bids_ses"]["uri"]
     rootdir = "example_bids_ses"
-    replacement1 = ["subject", "session"]
-    replacement2 = ["subject"]
-    pattern1 = {
-        "T1w": {
-            "pattern": (
-                "{subject}/{session}/anat/{subject}_{session}_T1w.nii.gz"
-            ),
-            "space": "native",
-        },
-    }
-    pattern2 = {
-        "BOLD": {
-            "pattern": "{subject}/func/{subject}_task-rest_bold.nii.gz",
-            "space": "MNI152NLin6Asym",
-        },
-    }
+
     dg1 = PatternDataladDataGrabber(
         rootdir=rootdir,
-        uri=repo_uri1,
+        uri=_testing_dataset["example_bids"]["uri"],
         types=["T1w"],
-        patterns=pattern1,
-        replacements=replacement1,
+        patterns={
+            "T1w": {
+                "pattern": (
+                    "{subject}/{session}/anat/{subject}_{session}_T1w.nii.gz"
+                ),
+                "space": "native",
+            },
+        },
+        replacements=["subject", "session"],
     )
 
     dg2 = PatternDataladDataGrabber(
         rootdir=rootdir,
-        uri=repo_uri2,
+        uri=_testing_dataset["example_bids_ses"]["uri"],
         types=["BOLD"],
-        patterns=pattern2,
-        replacements=replacement2,
+        patterns={
+            "BOLD": {
+                "pattern": "{subject}/func/{subject}_task-rest_bold.nii.gz",
+                "space": "MNI152NLin6Asym",
+            },
+        },
+        replacements=["subject"],
     )
 
-    with pytest.raises(ValueError, match="different element key"):
+    with pytest.raises(RuntimeError, match="have different element keys"):
         MultipleDataGrabber([dg1, dg2])
 
-    with pytest.raises(ValueError, match="overlapping types"):
+    with pytest.raises(RuntimeError, match="have overlapping mandatory"):
         MultipleDataGrabber([dg1, dg1])
+
+
+def test_MultipleDataGrabber_partial_pattern() -> None:
+    """Test MultipleDataGrabber partial pattern."""
+    repo_uri = _testing_dataset["example_bids_ses"]["uri"]
+    rootdir = "example_bids_ses"
+    replacements = ["subject", "session"]
+
+    dg1 = PatternDataladDataGrabber(
+        rootdir=rootdir,
+        uri=repo_uri,
+        types=["BOLD"],
+        patterns={
+            "BOLD": {
+                "pattern": (
+                    "{subject}/{session}/func/"
+                    "{subject}_{session}_task-rest_bold.nii.gz"
+                ),
+                "space": "MNI152NLin6Asym",
+            },
+        },
+        replacements=replacements,
+    )
+
+    dg2 = PatternDataladDataGrabber(
+        rootdir=rootdir,
+        uri=repo_uri,
+        types=["BOLD"],
+        patterns={
+            "BOLD": {
+                "confounds": {
+                    "pattern": (
+                        "{subject}/{session}/func/"
+                        "{subject}_{session}_task-rest_"
+                        "confounds_regressors.tsv"
+                    ),
+                    "format": "fmriprep",
+                },
+            },
+        },
+        replacements=["subject", "session"],
+        partial_pattern_ok=True,
+    )
+
+    dg = MultipleDataGrabber([dg1, dg2])
+
+    types = dg.get_types()
+    assert "BOLD" in types
+
+    expected_subs = [
+        (f"sub-{i:02d}", f"ses-{j:02d}")
+        for j in range(1, 3)
+        for i in range(1, 10)
+    ]
+
+    with dg:
+        subs = list(dg)
+        assert set(subs) == set(expected_subs)
+        # Fetch element
+        elem = dg[("sub-01", "ses-01")]
+        # Check data type and nested data type
+        assert "BOLD" in elem
+        assert "confounds" in elem["BOLD"]
+        # Check meta
+        assert "meta" in elem["BOLD"]
+        meta = elem["BOLD"]["meta"]["datagrabber"]
+        assert "class" in meta
+        assert meta["class"] == "MultipleDataGrabber"
+        # Check datagrabbers
+        assert "datagrabbers" in meta
+        assert len(meta["datagrabbers"]) == 2
+        assert meta["datagrabbers"][0]["class"] == "PatternDataladDataGrabber"
+        assert meta["datagrabbers"][1]["class"] == "PatternDataladDataGrabber"

junifer/datagrabber/tests/{test_datagrabber_utils.py → test_pattern_validation_mixin.py}

@@ -1,6 +1,7 @@
-"""Provide tests for utils."""
+"""Provide tests for PatternValidationMixin."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+#          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
 from contextlib import nullcontext
@@ -8,136 +9,57 @@ from typing import ContextManager, Dict, List, Union
 
 import pytest
 
-from junifer.datagrabber.utils import (
-    validate_patterns,
-    validate_replacements,
-    validate_types,
-)
+from junifer.datagrabber.pattern_validation_mixin import PatternValidationMixin
 
 
 @pytest.mark.parametrize(
-    "types, expect",
-    [
-        ("wrong", pytest.raises(TypeError, match="must be a list")),
-        ([1], pytest.raises(TypeError, match="must be a list of strings")),
-        (["T1w", "BOLD"], nullcontext()),
-    ],
-)
-def test_validate_types(
-    types: Union[str, List[str], List[int]],
-    expect: ContextManager,
-) -> None:
-    """Test validation of types.
-
-    Parameters
-    ----------
-    types : str, list of int or str
-        The parametrized data types to validate.
-    expect : typing.ContextManager
-        The parametrized ContextManager object.
-
-    """
-    with expect:
-        validate_types(types)  # type: ignore
-
-
-@pytest.mark.parametrize(
-    "replacements, patterns, expect",
+    "types, replacements, patterns, expect",
     [
         (
             "wrong",
-            "also wrong",
-            pytest.raises(TypeError, match="must be a list"),
+            [],
+            {},
+            pytest.raises(TypeError, match="`types` must be a list"),
         ),
         (
             [1],
-            {
-                "T1w": {"pattern": "{subject}/anat/{subject}_T1w.nii.gz"},
-                "BOLD": {
-                    "pattern": "{subject}/func/{subject}_task-rest_bold.nii.gz"
-                },
-            },
-            pytest.raises(TypeError, match="must be a list of strings"),
-        ),
-        (
-            ["session"],
-            {
-                "T1w": {"pattern": "{subject}/anat/{subject}_T1w.nii.gz"},
-                "BOLD": {
-                    "pattern": "{subject}/func/{subject}_task-rest_bold.nii.gz"
-                },
-            },
-            pytest.raises(ValueError, match="is not part of"),
-        ),
-        (
-            ["subject", "session"],
-            {
-                "T1w": {"pattern": "{subject}/anat/_T1w.nii.gz"},
-                "BOLD": {"pattern": "{session}/func/_task-rest_bold.nii.gz"},
-            },
-            pytest.raises(ValueError, match="At least one pattern"),
+            [],
+            {},
+            pytest.raises(
+                TypeError, match="`types` must be a list of strings"
+            ),
         ),
         (
-            ["subject"],
-            {
-                "T1w": {"pattern": "{subject}/anat/{subject}_T1w.nii.gz"},
-                "BOLD": {
-                    "pattern": "{subject}/func/{subject}_task-rest_bold.nii.gz"
-                },
-            },
-            nullcontext(),
-        ),
-    ],
-)
-def test_validate_replacements(
-    replacements: Union[str, List[str], List[int]],
-    patterns: Union[str, Dict[str, Dict[str, str]]],
-    expect: ContextManager,
-) -> None:
-    """Test validation of replacements.
-
-    Parameters
-    ----------
-    replacements : str, list of str or int
-        The parametrized pattern replacements to validate.
-    patterns : str, dict
-        The parametrized patterns to validate against.
-    expect : typing.ContextManager
-        The parametrized ContextManager object.
-
-    """
-    with expect:
-        validate_replacements(replacements=replacements, patterns=patterns)  # type: ignore
-
-
-@pytest.mark.parametrize(
-    "types, patterns, expect",
-    [
-        (
-            ["T1w", "BOLD"],
+            ["BOLD"],
+            [],
             "wrong",
-            pytest.raises(TypeError, match="must be a dict"),
+            pytest.raises(TypeError, match="`patterns` must be a dict"),
         ),
         (
             ["T1w", "BOLD"],
+            "",
             {
                 "T1w": {"pattern": "{subject}/anat/{subject}_T1w.nii.gz"},
             },
             pytest.raises(
                 ValueError,
-                match="Length of `types` more than that of `patterns`.",
+                match="Length of `types` more than that of `patterns`",
             ),
         ),
         (
             ["T1w", "BOLD"],
+            "",
             {
                 "T1w": {"pattern": "{subject}/anat/{subject}_T1w.nii.gz"},
                 "T2w": {"pattern": "{subject}/anat/{subject}_T2w.nii.gz"},
             },
-            pytest.raises(ValueError, match="contain all"),
+            pytest.raises(
+                ValueError, match="`patterns` must contain all `types`"
+            ),
         ),
         (
             ["T3w"],
+            "",
             {
                 "T3w": {"pattern": "{subject}/anat/{subject}_T3w.nii.gz"},
             },
@@ -145,6 +67,7 @@ def test_validate_replacements(
         ),
         (
             ["BOLD"],
+            "",
             {
                 "BOLD": {"patterns": "{subject}/func/{subject}_BOLD.nii.gz"},
             },
@@ -152,6 +75,7 @@ def test_validate_replacements(
         ),
         (
             ["BOLD"],
+            "",
             {
                 "BOLD": {
                     "pattern": (
@@ -169,6 +93,7 @@ def test_validate_replacements(
         ),
         (
             ["T1w"],
+            "",
             {
                 "T1w": {
                     "pattern": "{subject}/anat/{subject}*.nii",
@@ -177,8 +102,65 @@ def test_validate_replacements(
             },
             pytest.raises(ValueError, match="following a replacement"),
         ),
+        (
+            ["T1w"],
+            "wrong",
+            {
+                "T1w": {
+                    "pattern": "{subject}/anat/{subject}_T1w.nii",
+                    "space": "native",
+                },
+            },
+            pytest.raises(TypeError, match="`replacements` must be a list"),
+        ),
+        (
+            ["T1w"],
+            [1],
+            {
+                "T1w": {
+                    "pattern": "{subject}/anat/{subject}_T1w.nii",
+                    "space": "native",
+                },
+            },
+            pytest.raises(
+                TypeError, match="`replacements` must be a list of strings"
+            ),
+        ),
+        (
+            ["T1w", "BOLD"],
+            ["subject", "session"],
+            {
+                "T1w": {
+                    "pattern": "{subject}/anat/{subject}_T1w.nii.gz",
+                    "space": "native",
+                },
+                "BOLD": {
+                    "pattern": (
+                        "{subject}/func/{subject}_task-rest_bold.nii.gz"
+                    ),
+                    "space": "MNI152NLin6Asym",
+                },
+            },
+            pytest.raises(ValueError, match="is not part of any pattern"),
+        ),
+        (
+            ["BOLD"],
+            ["subject", "session"],
+            {
+                "T1w": {
+                    "pattern": "{subject}/anat/_T1w.nii.gz",
+                    "space": "native",
+                },
+                "BOLD": {
+                    "pattern": "{session}/func/_task-rest_bold.nii.gz",
+                    "space": "MNI152NLin6Asym",
+                },
+            },
+            pytest.raises(ValueError, match="At least one pattern"),
+        ),
         (
             ["T1w", "T2w", "BOLD"],
+            ["subject"],
             {
                 "T1w": {
                     "pattern": "{subject}/anat/{subject}_T1w.nii.gz",
@@ -190,7 +172,7 @@ def test_validate_replacements(
                 },
                 "BOLD": {
                     "pattern": (
-                        "{subject}/func/{subject}_task-rest_bold.nii.gz"
+                        "{subject}/func/{session}/{subject}_task-rest_bold.nii.gz"
                     ),
                     "space": "MNI152NLin6Asym",
                     "confounds": {
@@ -203,22 +185,65 @@ def test_validate_replacements(
         ),
     ],
 )
-def test_validate_patterns(
-    types: List[str],
+def test_PatternValidationMixin(
+    types: Union[str, List[str], List[int]],
+    replacements: Union[str, List[str], List[int]],
     patterns: Union[str, Dict[str, Dict[str, str]]],
     expect: ContextManager,
 ) -> None:
-    """Test validation of patterns.
+    """Test validation.
 
     Parameters
     ----------
-    types : list of str
-        The parametrized data types.
+    types : str, list of int or str
+        The parametrized data types to validate.
+    replacements : str, list of str or int
+        The parametrized pattern replacements to validate.
     patterns : str, dict
-        The patterns to validate.
+        The parametrized patterns to validate against.
     expect : typing.ContextManager
         The parametrized ContextManager object.
 
     """
+
+    class MockDataGrabber(PatternValidationMixin):
+        def __init__(
+            self,
+            types,
+            replacements,
+            patterns,
+        ) -> None:
+            self.types = types
+            self.replacements = replacements
+            self.patterns = patterns
+
+        def validate(self) -> None:
+            self.validate_patterns(
+                types=self.types,
+                replacements=self.replacements,
+                patterns=self.patterns,
+            )
+
+    dg = MockDataGrabber(types, replacements, patterns)
     with expect:
-        validate_patterns(types=types, patterns=patterns)  # type: ignore
+        dg.validate()
+
+
+# This test is kept separate as bool doesn't support context manager protocol,
+# used in the earlier test
+def test_PatternValidationMixin_partial_pattern_check() -> None:
+    """Test validation for partial patterns."""
+    with pytest.warns(RuntimeWarning, match="might not work as expected"):
+        PatternValidationMixin().validate_patterns(
+            types=["BOLD"],
+            replacements=["subject"],
+            patterns={
+                "BOLD": {
+                    "mask": {
+                        "pattern": "{subject}/func/{subject}_BOLD.nii.gz",
+                        "space": "MNI152NLin6Asym",
+                    },
+                },
+            },  # type: ignore
+            partial_pattern_ok=True,
+        )

junifer/utils/__init__.py

@@ -6,7 +6,7 @@
 
 from .fs import make_executable
 from .logging import configure_logging, logger, raise_error, warn_with_log
-from .helpers import run_ext_cmd
+from .helpers import run_ext_cmd, deep_update
 
 
 __all__ = [
@@ -16,4 +16,5 @@ __all__ = [
     "raise_error",
     "warn_with_log",
     "run_ext_cmd",
+    "deep_update",
 ]