junifer 0.0.3.dev186__py3-none-any.whl → 0.0.4__py3-none-any.whl

junifer/datagrabber/pattern.py

@@ -32,36 +32,134 @@ class PatternDataGrabber(BaseDataGrabber):
     types : list of str
         The types of data to be grabbed.
     patterns : dict
-        Patterns for each type of data as a dictionary. The keys are the types
-        and the values are the patterns. Each occurrence of the string
-        ``{subject}`` in the pattern will be replaced by the indexed element.
+        Data type patterns as a dictionary. It has the following schema:
+
+        * ``"T1w"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"T2w"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"BOLD"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": ["mask_item"]
+            }
+
+        * ``"Warp"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "src", "dst"],
+              "optional": []
+            }
+
+        * ``"BOLD_confounds"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "format"],
+              "optional": []
+            }
+
+        * ``"VBM_GM"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"VBM_WM"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        Basically, for each data type, one needs to provide ``mandatory`` keys
+        and can choose to also provide ``optional`` keys. The value for each
+        key is a string. So, one needs to provide necessary data types as a
+        dictionary, for example:
+
+        .. code-block:: none
+
+          {
+              "BOLD": {
+                "pattern": "...",
+                "space": "...",
+              },
+              "T1w": {
+                "pattern": "...",
+                "space": "...",
+              },
+              "Warp": {
+                "pattern": "...",
+                "src": "...",
+                "dst": "...",
+              }
+          }
+
+        taken from :class:`.HCP1200`.
     replacements : str or list of str
-        Replacements in the patterns for each item in the "element" tuple.
+        Replacements in the ``pattern`` key of each data type. The value needs
+        to be a list of all possible replacements.
     datadir : str or pathlib.Path
         The directory where the data is / will be stored.
     confounds_format : {"fmriprep", "adhoc"} or None, optional
         The format of the confounds for the dataset (default None).
 
+    Raises
+    ------
+    ValueError
+        If ``confounds_format`` is invalid.
+
     """
 
     def __init__(
         self,
         types: List[str],
-        patterns: Dict[str, str],
+        patterns: Dict[str, Dict[str, str]],
         replacements: Union[List[str], str],
         datadir: Union[str, Path],
         confounds_format: Optional[str] = None,
     ) -> None:
         # Validate patterns
         validate_patterns(types=types, patterns=patterns)
+        self.patterns = patterns
 
+        # Convert replacements to list if not already
         if not isinstance(replacements, list):
             replacements = [replacements]
         # Validate replacements
         validate_replacements(replacements=replacements, patterns=patterns)
+        self.replacements = replacements
 
         # Validate confounds format
-        if confounds_format and confounds_format not in _CONFOUNDS_FORMATS:
+        if (
+            confounds_format is not None
+            and confounds_format not in _CONFOUNDS_FORMATS
+        ):
             raise_error(
                 "Invalid value for `confounds_format`, should be one of "
                 f"{_CONFOUNDS_FORMATS}."
@@ -72,8 +170,7 @@ class PatternDataGrabber(BaseDataGrabber):
         logger.debug("Initializing PatternDataGrabber")
         logger.debug(f"\tpatterns = {patterns}")
         logger.debug(f"\treplacements = {replacements}")
-        self.patterns = patterns
-        self.replacements = replacements
+        logger.debug(f"\tconfounds_format = {confounds_format}")
 
     @property
     def skip_file_check(self) -> bool:
@@ -136,6 +233,11 @@ class PatternDataGrabber(BaseDataGrabber):
         str
             The pattern with the element replaced.
 
+        Raises
+        ------
+        ValueError
+            If element keys do not match with replacements.
+
         """
         if list(element.keys()) != self.replacements:
             raise_error(
@@ -160,7 +262,7 @@ class PatternDataGrabber(BaseDataGrabber):
         return self.replacements
 
     def get_item(self, **element: str) -> Dict[str, Dict]:
-        """Implement single element indexing in the database.
+        """Implement single element indexing for the datagrabber.
 
         This method constructs a real path to the requested item's data, by
         replacing the ``patterns`` with actual values passed via ``**element``.
@@ -177,20 +279,33 @@ class PatternDataGrabber(BaseDataGrabber):
             Dictionary of dictionaries for each type of data required for the
             specified element.
 
+        Raises
+        ------
+        RuntimeError
+            If more than one file matches for a data type's pattern or
+            if no file matches for a data type's pattern or
+            if file cannot be accessed for an element.
+
         """
         out = {}
         for t_type in self.types:
             t_pattern = self.patterns[t_type]
-            t_replace = self._replace_patterns_glob(element, t_pattern)
+            t_replace = self._replace_patterns_glob(
+                element, t_pattern["pattern"]
+            )
             if "*" in t_replace:
                 t_matches = list(self.datadir.absolute().glob(t_replace))
                 if len(t_matches) > 1:
                     raise_error(
                         f"More than one file matches for {element} / {t_type}:"
-                        f" {t_matches}"
+                        f" {t_matches}",
+                        klass=RuntimeError,
                     )
                 elif len(t_matches) == 0:
-                    raise_error(f"No file matches for {element} / {t_type}")
+                    raise_error(
+                        f"No file matches for {element} / {t_type}",
+                        klass=RuntimeError,
+                    )
                 t_out = t_matches[0]
             else:
                 t_out = self.datadir / t_replace
@@ -198,22 +313,13 @@ class PatternDataGrabber(BaseDataGrabber):
                     if not t_out.exists() and not t_out.is_symlink():
                         raise_error(
                             f"Cannot access {t_type} for {element}: "
-                            f"File {t_out} does not exist"
+                            f"File {t_out} does not exist",
+                            klass=RuntimeError,
                         )
             # Update path for the element
-            out[t_type] = {"path": t_out}
-            # Update confounds format for BOLD_confounds
-            # (if found in the datagrabber)
-            if t_type == "BOLD_confounds":
-                if not self.confounds_format:
-                    raise_error(
-                        "`confounds_format` needs to be one of "
-                        f"{_CONFOUNDS_FORMATS}, None provided. "
-                        "As the DataGrabber used specifies "
-                        "'BOLD_confounds', None is invalid."
-                    )
-                # Set the format
-                out[t_type].update({"format": self.confounds_format})
+            out[t_type] = t_pattern.copy()  # copy data type dictionary
+            out[t_type].pop("pattern")  # remove pattern key
+            out[t_type].update({"path": t_out})  # add path key
 
         return out
 
@@ -252,7 +358,7 @@ class PatternDataGrabber(BaseDataGrabber):
                 re_pattern,
                 glob_pattern,
                 t_replacements,
-            ) = self._replace_patterns_regex(t_pattern)
+            ) = self._replace_patterns_regex(t_pattern["pattern"])
             for fname in self.datadir.glob(glob_pattern):
                 suffix = fname.relative_to(self.datadir).as_posix()
                 m = re.match(re_pattern, suffix)

junifer/datagrabber/pattern_datalad.py

@@ -5,12 +5,11 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Dict, List
 
 from ..api.decorators import register_datagrabber
+from ..utils import logger
 from .datalad_base import DataladDataGrabber
 from .pattern import PatternDataGrabber
-from .utils import validate_patterns
 
 
 @register_datagrabber
@@ -25,11 +24,109 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
     types : list of str
         The types of data to be grabbed.
     patterns : dict
-        Patterns for each type of data as a dictionary. The keys are the types
-        and the values are the patterns. Each occurrence of the string
-        ``{subject}`` in the pattern will be replaced by the indexed element.
-    **kwargs
-        Keyword arguments passed to superclass.
+        Data type patterns as a dictionary. It has the following schema:
+
+        * ``"T1w"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"T2w"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"BOLD"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": ["mask_item"]
+            }
+
+        * ``"Warp"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "src", "dst"],
+              "optional": []
+            }
+
+        * ``"BOLD_confounds"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "format"],
+              "optional": []
+            }
+
+        * ``"VBM_GM"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        * ``"VBM_WM"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "space"],
+              "optional": []
+            }
+
+        Basically, for each data type, one needs to provide ``mandatory`` keys
+        and can choose to also provide ``optional`` keys. The value for each
+        key is a string. So, one needs to provide necessary data types as a
+        dictionary, for example:
+
+        .. code-block:: none
+
+          {
+              "BOLD": {
+                "pattern": "...",
+                "space": "...",
+              },
+              "T1w": {
+                "pattern": "...",
+                "space": "...",
+              },
+              "Warp": {
+                "pattern": "...",
+                "src": "...",
+                "dst": "...",
+              }
+          }
+
+        taken from :class:`.HCP1200`.
+    replacements : str or list of str
+        Replacements in the ``pattern`` key of each data type. The value needs
+        to be a list of all possible replacements.
+    confounds_format : {"fmriprep", "adhoc"} or None, optional
+        The format of the confounds for the dataset (default None).
+    datadir : str or pathlib.Path or None, optional
+        That directory where the datalad dataset will be cloned. If None,
+        the datalad dataset will be cloned into a temporary directory
+        (default None).
+    rootdir : str or pathlib.Path, optional
+        The path within the datalad dataset to the root directory
+        (default ".").
+    uri : str or None, optional
+        URI of the datalad sibling (default None).
 
     See Also
     --------
@@ -42,12 +139,13 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
 
     def __init__(
         self,
-        types: List[str],
-        patterns: Dict[str, str],
         **kwargs,
     ) -> None:
-        # Validate patterns
-        validate_patterns(types=types, patterns=patterns)
+        # TODO(synchon): needs to be reworked, DataladDataGrabber needs to be
+        # a mixin to avoid multiple inheritance wherever possible.
+
+        logger.debug("Initializing PatternDataladDataGrabber")
+        for key, val in kwargs.items():
+            logger.debug(f"\t{key} = {val}")
 
-        super().__init__(types=types, patterns=patterns, **kwargs)
-        self.patterns = patterns
+        super().__init__(**kwargs)

junifer/datagrabber/tests/test_base.py

@@ -12,12 +12,6 @@ import pytest
 from junifer.datagrabber import BaseDataGrabber
 
 
-def test_BaseDataGrabber_abstractness() -> None:
-    """Test BaseDataGrabber is abstract base class."""
-    with pytest.raises(TypeError, match=r"abstract"):
-        BaseDataGrabber(datadir="/tmp", types=["func"])  # type: ignore
-
-
 def test_BaseDataGrabber() -> None:
     """Test BaseDataGrabber."""
 
@@ -67,3 +61,60 @@ def test_BaseDataGrabber() -> None:
 
     with pytest.raises(NotImplementedError):
         dg.get_item(subject=1)  # type: ignore
+
+
+def test_BaseDataGrabber_filter_single() -> None:
+    """Test single-keyed element filter for BaseDataGrabber."""
+
+    # Create concrete class
+    class FilterDataGrabber(BaseDataGrabber):
+        def get_item(self, subject):
+            return {"BOLD": {}}
+
+        def get_elements(self):
+            return ["sub01", "sub02", "sub03"]
+
+        def get_element_keys(self):
+            return ["subject"]
+
+    dg = FilterDataGrabber(datadir="/tmp", types=["BOLD"])
+    with dg:
+        assert "sub01" in list(dg.filter(["sub01"]))
+        assert "sub02" not in list(dg.filter(["sub01"]))
+
+
+def test_BaseDataGrabber_filter_multi() -> None:
+    """Test multi-keyed element filter for BaseDataGrabber."""
+
+    # Create concrete class
+    class FilterDataGrabber(BaseDataGrabber):
+        def get_item(self, subject):
+            return {"BOLD": {}}
+
+        def get_elements(self):
+            return [
+                ("sub01", "rest"),
+                ("sub01", "movie"),
+                ("sub02", "rest"),
+                ("sub02", "movie"),
+                ("sub03", "rest"),
+                ("sub03", "movie"),
+            ]
+
+        def get_element_keys(self):
+            return ["subject", "task"]
+
+    dg = FilterDataGrabber(datadir="/tmp", types=["BOLD"])
+    with dg:
+        assert ("sub01", "rest") in list(
+            dg.filter([("sub01", "rest")])  # type: ignore
+        )
+        assert ("sub01", "movie") not in list(
+            dg.filter([("sub01", "rest")])  # type: ignore
+        )
+        assert ("sub02", "rest") not in list(
+            dg.filter([("sub01", "rest")])  # type: ignore
+        )
+        assert ("sub02", "movie") not in list(
+            dg.filter([("sub01", "rest")])  # type: ignore
+        )