junifer 0.0.4.dev829__py3-none-any.whl → 0.0.5__py3-none-any.whl

junifer/datagrabber/multiple.py

@@ -7,10 +7,15 @@
 
 from typing import Dict, List, Tuple, Union
 
-from ..utils import raise_error
+from ..api.decorators import register_datagrabber
+from ..utils import deep_update, raise_error
 from .base import BaseDataGrabber
 
 
+__all__ = ["MultipleDataGrabber"]
+
+
+@register_datagrabber
 class MultipleDataGrabber(BaseDataGrabber):
     """Concrete implementation for multi sourced data fetching.
 
@@ -24,19 +29,53 @@ class MultipleDataGrabber(BaseDataGrabber):
     **kwargs
         Keyword arguments passed to superclass.
 
+    Raises
+    ------
+    RuntimeError
+        If ``datagrabbers`` have different element keys or
+        overlapping data types or nested data types.
+
     """
 
     def __init__(self, datagrabbers: List[BaseDataGrabber], **kwargs) -> None:
         # Check datagrabbers consistency
-        # 1) same element keys
+        # Check for same element keys
         first_keys = datagrabbers[0].get_element_keys()
         for dg in datagrabbers[1:]:
             if dg.get_element_keys() != first_keys:
-                raise_error("DataGrabbers have different element keys.")
-        # 2) no overlapping types
+                raise_error(
+                    msg="DataGrabbers have different element keys",
+                    klass=RuntimeError,
+                )
+        # Check for no overlapping types (and nested data types)
         types = [x for dg in datagrabbers for x in dg.get_types()]
         if len(types) != len(set(types)):
-            raise_error("DataGrabbers have overlapping types.")
+            if all(hasattr(dg, "patterns") for dg in datagrabbers):
+                first_patterns = datagrabbers[0].patterns
+                for dg in datagrabbers[1:]:
+                    for data_type in set(types):
+                        dtype_pattern = dg.patterns.get(data_type)
+                        if dtype_pattern is None:
+                            continue
+                        # Check if first-level keys of data type are same
+                        if (
+                            dtype_pattern.keys()
+                            == first_patterns[data_type].keys()
+                        ):
+                            raise_error(
+                                msg=(
+                                    "DataGrabbers have overlapping mandatory "
+                                    "and / or optional key(s) for data type: "
+                                    f"`{data_type}`"
+                                ),
+                                klass=RuntimeError,
+                            )
+            else:
+                # Can't check further
+                raise_error(
+                    msg="DataGrabbers have overlapping types",
+                    klass=RuntimeError,
+                )
         self._datagrabbers = datagrabbers
 
     def __getitem__(self, element: Union[str, Tuple]) -> Dict:
@@ -62,7 +101,7 @@ class MultipleDataGrabber(BaseDataGrabber):
         metas = []
         for dg in self._datagrabbers:
             t_out = dg[element]
-            out.update(t_out)
+            deep_update(out, t_out)
             # Now get the meta for this datagrabber
             t_meta = {}
             dg.update_meta(t_meta, "datagrabber")

junifer/datagrabber/pattern.py

@@ -6,6 +6,7 @@
 # License: AGPL
 
 import re
+from copy import deepcopy
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
@@ -14,7 +15,10 @@ import numpy as np
 from ..api.decorators import register_datagrabber
 from ..utils import logger, raise_error
 from .base import BaseDataGrabber
-from .utils import validate_patterns, validate_replacements
+from .pattern_validation_mixin import PatternValidationMixin
+
+
+__all__ = ["PatternDataGrabber"]
 
 
 # Accepted formats for confounds specification
@@ -22,7 +26,7 @@ _CONFOUNDS_FORMATS = ("fmriprep", "adhoc")
 
 
 @register_datagrabber
-class PatternDataGrabber(BaseDataGrabber):
+class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
     """Concrete implementation for pattern-based data fetching.
 
     Implements a DataGrabber that understands patterns to grab data.
@@ -40,7 +44,12 @@ class PatternDataGrabber(BaseDataGrabber):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": []
+              "optional": {
+                  "mask": {
+                      "mandatory": ["pattern", "space"],
+                      "optional": []
+                  }
+              }
             }
 
         * ``"T2w"`` :
@@ -49,7 +58,12 @@ class PatternDataGrabber(BaseDataGrabber):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": []
+              "optional": {
+                  "mask": {
+                      "mandatory": ["pattern", "space"],
+                      "optional": []
+                  }
+              }
             }
 
         * ``"BOLD"`` :
@@ -58,7 +72,16 @@ class PatternDataGrabber(BaseDataGrabber):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": ["mask_item"]
+              "optional": {
+                  "mask": {
+                      "mandatory": ["pattern", "space"],
+                      "optional": []
+                  }
+                  "confounds": {
+                      "mandatory": ["pattern", "format"],
+                      "optional": []
+                  }
+              }
             }
 
         * ``"Warp"`` :
@@ -70,15 +93,6 @@ class PatternDataGrabber(BaseDataGrabber):
               "optional": []
             }
 
-        * ``"BOLD_confounds"`` :
-
-          .. code-block:: none
-
-            {
-              "mandatory": ["pattern", "format"],
-              "optional": []
-            }
-
         * ``"VBM_GM"`` :
 
           .. code-block:: none
@@ -128,6 +142,13 @@ class PatternDataGrabber(BaseDataGrabber):
         The directory where the data is / will be stored.
     confounds_format : {"fmriprep", "adhoc"} or None, optional
         The format of the confounds for the dataset (default None).
+    partial_pattern_ok : bool, optional
+        Whether to raise error if partial pattern for a data type is found.
+        This allows to bypass mandatory key check and issue a warning
+        instead of raising error. This allows one to have a DataGrabber
+        with data types without the corresponding mandatory keys and is
+        powerful when used with :class:`.MultipleDataGrabber`
+        (default True).
 
     Raises
     ------
@@ -143,17 +164,21 @@ class PatternDataGrabber(BaseDataGrabber):
         replacements: Union[List[str], str],
         datadir: Union[str, Path],
         confounds_format: Optional[str] = None,
+        partial_pattern_ok: bool = False,
     ) -> None:
-        # Validate patterns
-        validate_patterns(types=types, patterns=patterns)
-        self.patterns = patterns
-
         # Convert replacements to list if not already
         if not isinstance(replacements, list):
             replacements = [replacements]
-        # Validate replacements
-        validate_replacements(replacements=replacements, patterns=patterns)
+        # Validate patterns
+        self.validate_patterns(
+            types=types,
+            replacements=replacements,
+            patterns=patterns,
+            partial_pattern_ok=partial_pattern_ok,
+        )
         self.replacements = replacements
+        self.patterns = patterns
+        self.partial_pattern_ok = partial_pattern_ok
 
         # Validate confounds format
         if (
@@ -204,18 +229,25 @@ class PatternDataGrabber(BaseDataGrabber):
         t_replacements = [
             x for x in self.replacements if f"{{{x}}}" in pattern
         ]
-
+        # Ops on re_pattern
+        # Remove negated unix glob pattern i.e., [!...] for re_pattern
+        re_pattern = re.sub(r"\[!.?\]", "", re_pattern)
+        # Remove enclosing square brackets from unix glob pattern i.e., [...]
+        # for re_pattern
+        re_pattern = re.sub(r"\[|\]", "", re_pattern)
+        # Iteratively replace the first of each with a named group definition
         for t_r in t_replacements:
-            # Replace the first of each with a named group definition
             re_pattern = re_pattern.replace(f"{{{t_r}}}", f"(?P<{t_r}>.*)", 1)
-
+        # Iteratively replace the second appearance of each with the named
+        # group back reference
         for t_r in t_replacements:
-            # Replace the second appearance of each with the named group
-            # back reference
             re_pattern = re_pattern.replace(f"{{{t_r}}}", f"(?P={t_r})")
-
+        # Ops on glob_pattern
+        # Iteratively replace replacements with wildcard i.e., *
+        # for glob_pattern
         for t_r in t_replacements:
             glob_pattern = glob_pattern.replace(f"{{{t_r}}}", "*")
+
         return re_pattern, glob_pattern, t_replacements
 
     def _replace_patterns_glob(self, element: Dict, pattern: str) -> str:
@@ -244,8 +276,70 @@ class PatternDataGrabber(BaseDataGrabber):
                 f"The element keys must be {self.replacements}, "
                 f"element has {list(element.keys())}."
             )
+        # Remove negated unix glob pattern i.e., [!...]
+        pattern = re.sub(r"\[!.?\]", "", pattern)
+        # Remove enclosing square brackets from unix glob pattern i.e., [...]
+        pattern = re.sub(r"\[|\]", "", pattern)
         return pattern.format(**element)
 
+    def _get_path_from_patterns(
+        self, element: Dict, pattern: str, data_type: str
+    ) -> Path:
+        """Get path from resolved patterns.
+
+        Parameters
+        ----------
+        element : dict
+            The element to be used in the replacement.
+        pattern : str
+            The pattern to be replaced.
+        data_type : str
+            The data type of the pattern.
+
+        Returns
+        -------
+        pathlib.Path
+            The path for the resolved pattern.
+
+        Raises
+        ------
+        RuntimeError
+            If more than one file matches for a data type's pattern or
+            if no file matches for a data type's pattern or
+            if file cannot be accessed for an element.
+
+        """
+        # Replace element in the pattern for globbing
+        resolved_pattern = self._replace_patterns_glob(element, pattern)
+        # Resolve path for wildcard
+        if "*" in resolved_pattern:
+            t_matches = list(self.datadir.absolute().glob(resolved_pattern))
+            # Multiple matches
+            if len(t_matches) > 1:
+                raise_error(
+                    f"More than one file matches for {element} / {data_type}:"
+                    f" {t_matches}",
+                    klass=RuntimeError,
+                )
+            # No matches
+            elif len(t_matches) == 0:
+                raise_error(
+                    f"No file matches for {element} / {data_type}",
+                    klass=RuntimeError,
+                )
+            path = t_matches[0]
+        else:
+            path = self.datadir / resolved_pattern
+            if not self.skip_file_check:
+                if not path.exists() and not path.is_symlink():
+                    raise_error(
+                        f"Cannot access {data_type} for {element}: "
+                        f"File {path} does not exist",
+                        klass=RuntimeError,
+                    )
+
+        return path
+
     def get_element_keys(self) -> List[str]:
         """Get element keys.
 
@@ -279,47 +373,49 @@ class PatternDataGrabber(BaseDataGrabber):
             Dictionary of dictionaries for each type of data required for the
             specified element.
 
-        Raises
-        ------
-        RuntimeError
-            If more than one file matches for a data type's pattern or
-            if no file matches for a data type's pattern or
-            if file cannot be accessed for an element.
-
         """
         out = {}
         for t_type in self.types:
+            # Data type dictionary
             t_pattern = self.patterns[t_type]
-            t_replace = self._replace_patterns_glob(
-                element, t_pattern["pattern"]
-            )
-            if "*" in t_replace:
-                t_matches = list(self.datadir.absolute().glob(t_replace))
-                if len(t_matches) > 1:
-                    raise_error(
-                        f"More than one file matches for {element} / {t_type}:"
-                        f" {t_matches}",
-                        klass=RuntimeError,
+            # Copy data type dictionary in output
+            out[t_type] = deepcopy(t_pattern)
+            # Iterate to check for nested "types" like mask
+            for k, v in t_pattern.items():
+                # Resolve pattern for base data type
+                if k == "pattern":
+                    logger.info(f"Resolving path from pattern for {t_type}")
+                    # Resolve pattern
+                    base_data_type_pattern_path = self._get_path_from_patterns(
+                        element=element,
+                        pattern=v,
+                        data_type=t_type,
                     )
-                elif len(t_matches) == 0:
-                    raise_error(
-                        f"No file matches for {element} / {t_type}",
-                        klass=RuntimeError,
+                    # Remove pattern key
+                    out[t_type].pop("pattern")
+                    # Add path key
+                    out[t_type].update({"path": base_data_type_pattern_path})
+                # Resolve pattern for nested data type
+                if isinstance(v, dict) and "pattern" in v:
+                    # Set nested type key for easier access
+                    t_nested_type = f"{t_type}.{k}"
+                    logger.info(
+                        f"Resolving path from pattern for {t_nested_type}"
                     )
-                t_out = t_matches[0]
-            else:
-                t_out = self.datadir / t_replace
-                if not self.skip_file_check:
-                    if not t_out.exists() and not t_out.is_symlink():
-                        raise_error(
-                            f"Cannot access {t_type} for {element}: "
-                            f"File {t_out} does not exist",
-                            klass=RuntimeError,
+                    # Resolve pattern
+                    nested_data_type_pattern_path = (
+                        self._get_path_from_patterns(
+                            element=element,
+                            pattern=v["pattern"],
+                            data_type=t_nested_type,
                         )
-            # Update path for the element
-            out[t_type] = t_pattern.copy()  # copy data type dictionary
-            out[t_type].pop("pattern")  # remove pattern key
-            out[t_type].update({"path": t_out})  # add path key
+                    )
+                    # Remove pattern key
+                    out[t_type][k].pop("pattern")
+                    # Add path key
+                    out[t_type][k].update(
+                        {"path": nested_data_type_pattern_path}
+                    )
 
         return out
 
@@ -351,14 +447,26 @@ class PatternDataGrabber(BaseDataGrabber):
         for t_idx in reversed(order):
             t_type = self.types[t_idx]
             types_element = set()
-            # Get the pattern
+
+            # Get the pattern dict
             t_pattern = self.patterns[t_type]
+            # Conditional fetch of base pattern for getting elements
+            pattern = None
+            # Try for data type pattern
+            pattern = t_pattern.get("pattern")
+            # Try for nested data type pattern
+            if pattern is None and self.partial_pattern_ok:
+                for v in t_pattern.values():
+                    if isinstance(v, dict) and "pattern" in v:
+                        pattern = v["pattern"]
+                        break
+
             # Replace the pattern
             (
                 re_pattern,
                 glob_pattern,
                 t_replacements,
-            ) = self._replace_patterns_regex(t_pattern["pattern"])
+            ) = self._replace_patterns_regex(pattern)
             for fname in self.datadir.glob(glob_pattern):
                 suffix = fname.relative_to(self.datadir).as_posix()
                 m = re.match(re_pattern, suffix)

junifer/datagrabber/pattern_datalad.py

@@ -12,6 +12,9 @@ from .datalad_base import DataladDataGrabber
 from .pattern import PatternDataGrabber
 
 
+__all__ = ["PatternDataladDataGrabber"]
+
+
 @register_datagrabber
 class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
     """Concrete implementation for pattern and datalad based data fetching.
@@ -32,7 +35,12 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": []
+              "optional": {
+                  "mask": {
+                      "mandatory": ["pattern", "space"],
+                      "optional": []
+                  }
+              }
             }
 
         * ``"T2w"`` :
@@ -41,7 +49,12 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": []
+              "optional": {
+                  "mask": {
+                      "mandatory": ["pattern", "space"],
+                      "optional": []
+                  }
+              }
             }
 
         * ``"BOLD"`` :
@@ -50,7 +63,16 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": ["mask_item"]
+              "optional": {
+                  "mask": {
+                      "mandatory": ["pattern", "space"],
+                      "optional": []
+                  }
+                  "confounds": {
+                      "mandatory": ["pattern", "format"],
+                      "optional": []
+                  }
+              }
             }
 
         * ``"Warp"`` :
@@ -62,15 +84,6 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
               "optional": []
             }
 
-        * ``"BOLD_confounds"`` :
-
-          .. code-block:: none
-
-            {
-              "mandatory": ["pattern", "format"],
-              "optional": []
-            }
-
         * ``"VBM_GM"`` :
 
           .. code-block:: none