junifer 0.0.4.dev782__py3-none-any.whl → 0.0.4.dev810__py3-none-any.whl

junifer/datagrabber/pattern.py

@@ -32,11 +32,98 @@ class PatternDataGrabber(BaseDataGrabber):
     types : list of str
         The types of data to be grabbed.
     patterns : dict
-        Patterns for each type of data as a dictionary. The keys are the types
-        and the values are the patterns. Each occurrence of the string
-        ``{subject}`` in the pattern will be replaced by the indexed element.
+        Data type patterns as a dictionary. It has the following schema:
+
+        * ``"T1w"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"T2w"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"BOLD"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": ["mask_item"]
+            }
+
+        * ``"Warp"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "src", "dst"],
+              "optional": []
+            }
+
+        * ``"BOLD_confounds"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "format"],
+              "optional": []
+            }
+
+        * ``"VBM_GM"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"VBM_WM"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        Basically, for each data type, one needs to provide ``mandatory`` keys
+        and can choose to also provide ``optional`` keys. The value for each
+        key is a string. So, one needs to provide necessary data types as a
+        dictionary, for example:
+
+        .. code-block:: none
+
+          {
+              "BOLD": {
+                "pattern": "...",
+                "space": "...",
+              },
+              "T1w": {
+                "pattern": "...",
+                "space": "...",
+              },
+              "Warp": {
+                "pattern": "...",
+                "src": "...",
+                "dst": "...",
+              }
+          }
+
+        taken from :class:`.HCP1200`.
     replacements : str or list of str
-        Replacements in the patterns for each item in the "element" tuple.
+        Replacements in the ``pattern`` key of each data type. The value needs
+        to be a list of all possible replacements.
     datadir : str or pathlib.Path
         The directory where the data is / will be stored.
     confounds_format : {"fmriprep", "adhoc"} or None, optional
@@ -52,7 +139,7 @@ class PatternDataGrabber(BaseDataGrabber):
     def __init__(
         self,
         types: List[str],
-        patterns: Dict[str, str],
+        patterns: Dict[str, Dict[str, str]],
         replacements: Union[List[str], str],
         datadir: Union[str, Path],
         confounds_format: Optional[str] = None,
@@ -69,7 +156,10 @@ class PatternDataGrabber(BaseDataGrabber):
         self.replacements = replacements
 
         # Validate confounds format
-        if confounds_format and confounds_format not in _CONFOUNDS_FORMATS:
+        if (
+            confounds_format is not None
+            and confounds_format not in _CONFOUNDS_FORMATS
+        ):
             raise_error(
                 "Invalid value for `confounds_format`, should be one of "
                 f"{_CONFOUNDS_FORMATS}."
@@ -143,6 +233,11 @@ class PatternDataGrabber(BaseDataGrabber):
         str
             The pattern with the element replaced.
 
+        Raises
+        ------
+        ValueError
+            If element keys do not match with replacements.
+
         """
         if list(element.keys()) != self.replacements:
             raise_error(
@@ -167,7 +262,7 @@ class PatternDataGrabber(BaseDataGrabber):
         return self.replacements
 
     def get_item(self, **element: str) -> Dict[str, Dict]:
-        """Implement single element indexing in the database.
+        """Implement single element indexing for the datagrabber.
 
         This method constructs a real path to the requested item's data, by
         replacing the ``patterns`` with actual values passed via ``**element``.
@@ -184,20 +279,33 @@ class PatternDataGrabber(BaseDataGrabber):
             Dictionary of dictionaries for each type of data required for the
             specified element.
 
+        Raises
+        ------
+        RuntimeError
+            If more than one file matches for a data type's pattern or
+            if no file matches for a data type's pattern or
+            if file cannot be accessed for an element.
+
         """
         out = {}
         for t_type in self.types:
             t_pattern = self.patterns[t_type]
-            t_replace = self._replace_patterns_glob(element, t_pattern)
+            t_replace = self._replace_patterns_glob(
+                element, t_pattern["pattern"]
+            )
             if "*" in t_replace:
                 t_matches = list(self.datadir.absolute().glob(t_replace))
                 if len(t_matches) > 1:
                     raise_error(
                         f"More than one file matches for {element} / {t_type}:"
-                        f" {t_matches}"
+                        f" {t_matches}",
+                        klass=RuntimeError,
                     )
                 elif len(t_matches) == 0:
-                    raise_error(f"No file matches for {element} / {t_type}")
+                    raise_error(
+                        f"No file matches for {element} / {t_type}",
+                        klass=RuntimeError,
+                    )
                 t_out = t_matches[0]
             else:
                 t_out = self.datadir / t_replace
@@ -205,22 +313,13 @@ class PatternDataGrabber(BaseDataGrabber):
                     if not t_out.exists() and not t_out.is_symlink():
                         raise_error(
                             f"Cannot access {t_type} for {element}: "
-                            f"File {t_out} does not exist"
+                            f"File {t_out} does not exist",
+                            klass=RuntimeError,
                         )
             # Update path for the element
-            out[t_type] = {"path": t_out}
-            # Update confounds format for BOLD_confounds
-            # (if found in the datagrabber)
-            if t_type == "BOLD_confounds":
-                if not self.confounds_format:
-                    raise_error(
-                        "`confounds_format` needs to be one of "
-                        f"{_CONFOUNDS_FORMATS}, None provided. "
-                        "As the DataGrabber used specifies "
-                        "'BOLD_confounds', None is invalid."
-                    )
-                # Set the format
-                out[t_type].update({"format": self.confounds_format})
+            out[t_type] = t_pattern.copy()  # copy data type dictionary
+            out[t_type].pop("pattern")  # remove pattern key
+            out[t_type].update({"path": t_out})  # add path key
 
         return out
 
@@ -259,7 +358,7 @@ class PatternDataGrabber(BaseDataGrabber):
                 re_pattern,
                 glob_pattern,
                 t_replacements,
-            ) = self._replace_patterns_regex(t_pattern)
+            ) = self._replace_patterns_regex(t_pattern["pattern"])
             for fname in self.datadir.glob(glob_pattern):
                 suffix = fname.relative_to(self.datadir).as_posix()
                 m = re.match(re_pattern, suffix)

junifer/datagrabber/pattern_datalad.py

@@ -5,12 +5,11 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Dict, List
 
 from ..api.decorators import register_datagrabber
+from ..utils import logger
 from .datalad_base import DataladDataGrabber
 from .pattern import PatternDataGrabber
-from .utils import validate_patterns
 
 
 @register_datagrabber
@@ -25,11 +24,109 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
     types : list of str
         The types of data to be grabbed.
     patterns : dict
-        Patterns for each type of data as a dictionary. The keys are the types
-        and the values are the patterns. Each occurrence of the string
-        ``{subject}`` in the pattern will be replaced by the indexed element.
-    **kwargs
-        Keyword arguments passed to superclass.
+        Data type patterns as a dictionary. It has the following schema:
+
+        * ``"T1w"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"T2w"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"BOLD"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": ["mask_item"]
+            }
+
+        * ``"Warp"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "src", "dst"],
+              "optional": []
+            }
+
+        * ``"BOLD_confounds"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "format"],
+              "optional": []
+            }
+
+        * ``"VBM_GM"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"VBM_WM"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        Basically, for each data type, one needs to provide ``mandatory`` keys
+        and can choose to also provide ``optional`` keys. The value for each
+        key is a string. So, one needs to provide necessary data types as a
+        dictionary, for example:
+
+        .. code-block:: none
+
+          {
+              "BOLD": {
+                "pattern": "...",
+                "space": "...",
+              },
+              "T1w": {
+                "pattern": "...",
+                "space": "...",
+              },
+              "Warp": {
+                "pattern": "...",
+                "src": "...",
+                "dst": "...",
+              }
+          }
+
+        taken from :class:`.HCP1200`.
+    replacements : str or list of str
+        Replacements in the ``pattern`` key of each data type. The value needs
+        to be a list of all possible replacements.
+    confounds_format : {"fmriprep", "adhoc"} or None, optional
+        The format of the confounds for the dataset (default None).
+    datadir : str or pathlib.Path or None, optional
+        That directory where the datalad dataset will be cloned. If None,
+        the datalad dataset will be cloned into a temporary directory
+        (default None).
+    rootdir : str or pathlib.Path, optional
+        The path within the datalad dataset to the root directory
+        (default ".").
+    uri : str or None, optional
+        URI of the datalad sibling (default None).
 
     See Also
     --------
@@ -42,12 +139,13 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
 
     def __init__(
         self,
-        types: List[str],
-        patterns: Dict[str, str],
         **kwargs,
     ) -> None:
-        # Validate patterns
-        validate_patterns(types=types, patterns=patterns)
+        # TODO(synchon): needs to be reworked, DataladDataGrabber needs to be
+        # a mixin to avoid multiple inheritance wherever possible.
+
+        logger.debug("Initializing PatternDataladDataGrabber")
+        for key, val in kwargs.items():
+            logger.debug(f"\t{key} = {val}")
 
-        super().__init__(types=types, patterns=patterns, **kwargs)
-        self.patterns = patterns
+        super().__init__(**kwargs)

junifer/datagrabber/tests/test_base.py

@@ -12,12 +12,6 @@ import pytest
 from junifer.datagrabber import BaseDataGrabber
 
 
-def test_BaseDataGrabber_abstractness() -> None:
-    """Test BaseDataGrabber is abstract base class."""
-    with pytest.raises(TypeError, match=r"abstract"):
-        BaseDataGrabber(datadir="/tmp", types=["func"])  # type: ignore
-
-
 def test_BaseDataGrabber() -> None:
     """Test BaseDataGrabber."""
 

junifer/datagrabber/tests/test_datagrabber_utils.py

@@ -3,6 +3,9 @@
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 # License: AGPL
 
+from contextlib import nullcontext
+from typing import ContextManager, Dict, List, Union
+
 import pytest
 
 from junifer.datagrabber.utils import (
@@ -12,79 +15,204 @@ from junifer.datagrabber.utils import (
 )
 
 
-def test_validate_types() -> None:
-    """Test validation of types."""
-    with pytest.raises(TypeError, match="must be a list"):
-        validate_types("wrong")  # type: ignore
-    with pytest.raises(TypeError, match="must be a list of strings"):
-        validate_types([1])  # type: ignore
-
-    validate_types(["T1w", "BOLD"])
-
-
-def test_validate_replacements() -> None:
-    """Test validation of replacements."""
-    with pytest.raises(TypeError, match="must be a list"):
-        validate_replacements("wrong", "also wrong")  # type: ignore
-    with pytest.raises(TypeError, match="must be a dict"):
-        validate_replacements(["correct"], "wrong")  # type: ignore
-
-    patterns = {
-        "T1w": "{subject}/anat/{subject}_T1w.nii.gz",
-        "BOLD": "{subject}/func/{subject}_task-rest_bold.nii.gz",
-    }
-
-    with pytest.raises(TypeError, match="must be a list of strings"):
-        validate_replacements([1], patterns)  # type: ignore
-
-    with pytest.raises(ValueError, match="is not part of"):
-        validate_replacements(["session"], patterns)
-
-    wrong_patterns = {
-        "T1w": "{subject}/anat/_T1w.nii.gz",
-        "BOLD": "{session}/func/_task-rest_bold.nii.gz",
-    }
-
-    with pytest.raises(ValueError, match="At least one pattern"):
-        validate_replacements(["subject", "session"], wrong_patterns)
-
-    validate_replacements(["subject"], patterns)
-
-
-def test_validate_patterns() -> None:
-    """Test validation of patterns."""
-    types = ["T1w", "BOLD"]
-    with pytest.raises(TypeError, match="must be a dict"):
-        validate_patterns(types, "wrong")  # type: ignore
-
-    wrongpatterns = {
-        "T1w": "{subject}/anat/{subject}_T1w.nii.gz",
-    }
-
-    with pytest.raises(
-        ValueError, match="Length of `types` more than that of `patterns`."
-    ):
-        validate_patterns(types, wrongpatterns)  # type: ignore
-
-    wrongpatterns = {
-        "T1w": "{subject}/anat/{subject}_T1w.nii.gz",
-        "T2": "{subject}/anat/{subject}_T2.nii.gz",
-    }
-
-    with pytest.raises(ValueError, match="contain all"):
-        validate_patterns(types, wrongpatterns)  # type: ignore
-
-    patterns = {
-        "T1w": "{subject}/anat/{subject}_T1w.nii.gz",
-        "BOLD": "{subject}/func/{subject}_task-rest_bold.nii.gz",
-    }
-
-    wrongpatterns = {
-        "T1w": "{subject}/anat/{subject}*.nii",
-        "BOLD": "{subject}/func/{subject}_task-rest_bold.nii.gz",
-    }
-
-    with pytest.raises(ValueError, match="following a replacement"):
-        validate_patterns(types, wrongpatterns)
-
-    validate_patterns(types, patterns)
+@pytest.mark.parametrize(
+    "types, expect",
+    [
+        ("wrong", pytest.raises(TypeError, match="must be a list")),
+        ([1], pytest.raises(TypeError, match="must be a list of strings")),
+        (["T1w", "BOLD"], nullcontext()),
+    ],
+)
+def test_validate_types(
+    types: Union[str, List[str], List[int]],
+    expect: ContextManager,
+) -> None:
+    """Test validation of types.
+
+    Parameters
+    ----------
+    types : str, list of int or str
+        The parametrized data types to validate.
+    expect : typing.ContextManager
+        The parametrized ContextManager object.
+
+    """
+    with expect:
+        validate_types(types)  # type: ignore
+
+
+@pytest.mark.parametrize(
+    "replacements, patterns, expect",
+    [
+        (
+            "wrong",
+            "also wrong",
+            pytest.raises(TypeError, match="must be a list"),
+        ),
+        (
+            [1],
+            {
+                "T1w": {"pattern": "{subject}/anat/{subject}_T1w.nii.gz"},
+                "BOLD": {
+                    "pattern": "{subject}/func/{subject}_task-rest_bold.nii.gz"
+                },
+            },
+            pytest.raises(TypeError, match="must be a list of strings"),
+        ),
+        (
+            ["session"],
+            {
+                "T1w": {"pattern": "{subject}/anat/{subject}_T1w.nii.gz"},
+                "BOLD": {
+                    "pattern": "{subject}/func/{subject}_task-rest_bold.nii.gz"
+                },
+            },
+            pytest.raises(ValueError, match="is not part of"),
+        ),
+        (
+            ["subject", "session"],
+            {
+                "T1w": {"pattern": "{subject}/anat/_T1w.nii.gz"},
+                "BOLD": {"pattern": "{session}/func/_task-rest_bold.nii.gz"},
+            },
+            pytest.raises(ValueError, match="At least one pattern"),
+        ),
+        (
+            ["subject"],
+            {
+                "T1w": {"pattern": "{subject}/anat/{subject}_T1w.nii.gz"},
+                "BOLD": {
+                    "pattern": "{subject}/func/{subject}_task-rest_bold.nii.gz"
+                },
+            },
+            nullcontext(),
+        ),
+    ],
+)
+def test_validate_replacements(
+    replacements: Union[str, List[str], List[int]],
+    patterns: Union[str, Dict[str, Dict[str, str]]],
+    expect: ContextManager,
+) -> None:
+    """Test validation of replacements.
+
+    Parameters
+    ----------
+    replacements : str, list of str or int
+        The parametrized pattern replacements to validate.
+    patterns : str, dict
+        The parametrized patterns to validate against.
+    expect : typing.ContextManager
+        The parametrized ContextManager object.
+
+    """
+    with expect:
+        validate_replacements(replacements=replacements, patterns=patterns)  # type: ignore
+
+
+@pytest.mark.parametrize(
+    "types, patterns, expect",
+    [
+        (
+            ["T1w", "BOLD"],
+            "wrong",
+            pytest.raises(TypeError, match="must be a dict"),
+        ),
+        (
+            ["T1w", "BOLD"],
+            {
+                "T1w": {"pattern": "{subject}/anat/{subject}_T1w.nii.gz"},
+            },
+            pytest.raises(
+                ValueError,
+                match="Length of `types` more than that of `patterns`.",
+            ),
+        ),
+        (
+            ["T1w", "BOLD"],
+            {
+                "T1w": {"pattern": "{subject}/anat/{subject}_T1w.nii.gz"},
+                "T2w": {"pattern": "{subject}/anat/{subject}_T2w.nii.gz"},
+            },
+            pytest.raises(ValueError, match="contain all"),
+        ),
+        (
+            ["T3w"],
+            {
+                "T3w": {"pattern": "{subject}/anat/{subject}_T3w.nii.gz"},
+            },
+            pytest.raises(ValueError, match="Unknown data type"),
+        ),
+        (
+            ["BOLD"],
+            {
+                "BOLD": {"patterns": "{subject}/func/{subject}_BOLD.nii.gz"},
+            },
+            pytest.raises(KeyError, match="Mandatory key"),
+        ),
+        (
+            ["BOLD_confounds"],
+            {
+                "BOLD_confounds": {
+                    "pattern": "{subject}/func/{subject}_confounds.tsv",
+                    "format": "fmriprep",
+                    "space": "MNINLin6Asym",
+                },
+            },
+            pytest.raises(RuntimeError, match="not accepted"),
+        ),
+        (
+            ["T1w"],
+            {
+                "T1w": {
+                    "pattern": "{subject}/anat/{subject}*.nii",
+                    "space": "native",
+                },
+            },
+            pytest.raises(ValueError, match="following a replacement"),
+        ),
+        (
+            ["T1w", "T2w", "BOLD", "BOLD_confounds"],
+            {
+                "T1w": {
+                    "pattern": "{subject}/anat/{subject}_T1w.nii.gz",
+                    "space": "native",
+                },
+                "T2w": {
+                    "pattern": "{subject}/anat/{subject}_T2w.nii.gz",
+                    "space": "native",
+                },
+                "BOLD": {
+                    "pattern": (
+                        "{subject}/func/{subject}_task-rest_bold.nii.gz"
+                    ),
+                    "space": "MNI152NLin6Asym",
+                },
+                "BOLD_confounds": {
+                    "pattern": "{subject}/func/{subject}_confounds.tsv",
+                    "format": "fmriprep",
+                },
+            },
+            nullcontext(),
+        ),
+    ],
+)
+def test_validate_patterns(
+    types: List[str],
+    patterns: Union[str, Dict[str, Dict[str, str]]],
+    expect: ContextManager,
+) -> None:
+    """Test validation of patterns.
+
+    Parameters
+    ----------
+    types : list of str
+        The parametrized data types.
+    patterns : str, dict
+        The patterns to validate.
+    expect : typing.ContextManager
+        The parametrized ContextManager object.
+
+    """
+    with expect:
+        validate_patterns(types=types, patterns=patterns)  # type: ignore

junifer/datagrabber/tests/test_datalad_base.py

@@ -26,12 +26,6 @@ _testing_dataset = {
 }
 
 
-def test_DataladDataGrabber_abstractness() -> None:
-    """Test DataladDataGrabber is abstract base class."""
-    with pytest.raises(TypeError, match=r"abstract"):
-        DataladDataGrabber()  # type: ignore
-
-
 @pytest.fixture
 def concrete_datagrabber() -> Type[DataladDataGrabber]:
     """Return a concrete datalad-based DataGrabber.