junifer 0.0.5__py3-none-any.whl → 0.0.5.dev24__py3-none-any.whl

junifer/datagrabber/multiple.py

@@ -7,15 +7,10 @@
 
 from typing import Dict, List, Tuple, Union
 
-from ..api.decorators import register_datagrabber
-from ..utils import deep_update, raise_error
+from ..utils import raise_error
 from .base import BaseDataGrabber
 
 
-__all__ = ["MultipleDataGrabber"]
-
-
-@register_datagrabber
 class MultipleDataGrabber(BaseDataGrabber):
     """Concrete implementation for multi sourced data fetching.
 
@@ -29,53 +24,19 @@ class MultipleDataGrabber(BaseDataGrabber):
     **kwargs
         Keyword arguments passed to superclass.
 
-    Raises
-    ------
-    RuntimeError
-        If ``datagrabbers`` have different element keys or
-        overlapping data types or nested data types.
-
     """
 
     def __init__(self, datagrabbers: List[BaseDataGrabber], **kwargs) -> None:
         # Check datagrabbers consistency
-        # Check for same element keys
+        # 1) same element keys
         first_keys = datagrabbers[0].get_element_keys()
         for dg in datagrabbers[1:]:
             if dg.get_element_keys() != first_keys:
-                raise_error(
-                    msg="DataGrabbers have different element keys",
-                    klass=RuntimeError,
-                )
-        # Check for no overlapping types (and nested data types)
+                raise_error("DataGrabbers have different element keys.")
+        # 2) no overlapping types
         types = [x for dg in datagrabbers for x in dg.get_types()]
         if len(types) != len(set(types)):
-            if all(hasattr(dg, "patterns") for dg in datagrabbers):
-                first_patterns = datagrabbers[0].patterns
-                for dg in datagrabbers[1:]:
-                    for data_type in set(types):
-                        dtype_pattern = dg.patterns.get(data_type)
-                        if dtype_pattern is None:
-                            continue
-                        # Check if first-level keys of data type are same
-                        if (
-                            dtype_pattern.keys()
-                            == first_patterns[data_type].keys()
-                        ):
-                            raise_error(
-                                msg=(
-                                    "DataGrabbers have overlapping mandatory "
-                                    "and / or optional key(s) for data type: "
-                                    f"`{data_type}`"
-                                ),
-                                klass=RuntimeError,
-                            )
-            else:
-                # Can't check further
-                raise_error(
-                    msg="DataGrabbers have overlapping types",
-                    klass=RuntimeError,
-                )
+            raise_error("DataGrabbers have overlapping types.")
         self._datagrabbers = datagrabbers
 
     def __getitem__(self, element: Union[str, Tuple]) -> Dict:
@@ -101,7 +62,7 @@ class MultipleDataGrabber(BaseDataGrabber):
         metas = []
         for dg in self._datagrabbers:
             t_out = dg[element]
-            deep_update(out, t_out)
+            out.update(t_out)
             # Now get the meta for this datagrabber
             t_meta = {}
             dg.update_meta(t_meta, "datagrabber")

junifer/datagrabber/pattern.py

@@ -6,7 +6,6 @@
 # License: AGPL
 
 import re
-from copy import deepcopy
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
@@ -15,10 +14,7 @@ import numpy as np
 from ..api.decorators import register_datagrabber
 from ..utils import logger, raise_error
 from .base import BaseDataGrabber
-from .pattern_validation_mixin import PatternValidationMixin
-
-
-__all__ = ["PatternDataGrabber"]
+from .utils import validate_patterns, validate_replacements
 
 
 # Accepted formats for confounds specification
@@ -26,7 +22,7 @@ _CONFOUNDS_FORMATS = ("fmriprep", "adhoc")
 
 
 @register_datagrabber
-class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
+class PatternDataGrabber(BaseDataGrabber):
     """Concrete implementation for pattern-based data fetching.
 
     Implements a DataGrabber that understands patterns to grab data.
@@ -44,12 +40,7 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": {
-                  "mask": {
-                      "mandatory": ["pattern", "space"],
-                      "optional": []
-                  }
-              }
+              "optional": []
             }
 
         * ``"T2w"`` :
@@ -58,12 +49,7 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": {
-                  "mask": {
-                      "mandatory": ["pattern", "space"],
-                      "optional": []
-                  }
-              }
+              "optional": []
             }
 
         * ``"BOLD"`` :
@@ -72,16 +58,7 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": {
-                  "mask": {
-                      "mandatory": ["pattern", "space"],
-                      "optional": []
-                  }
-                  "confounds": {
-                      "mandatory": ["pattern", "format"],
-                      "optional": []
-                  }
-              }
+              "optional": ["mask_item"]
             }
 
         * ``"Warp"`` :
@@ -93,6 +70,15 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
               "optional": []
             }
 
+        * ``"BOLD_confounds"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "format"],
+              "optional": []
+            }
+
         * ``"VBM_GM"`` :
 
           .. code-block:: none
@@ -142,13 +128,6 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
         The directory where the data is / will be stored.
     confounds_format : {"fmriprep", "adhoc"} or None, optional
         The format of the confounds for the dataset (default None).
-    partial_pattern_ok : bool, optional
-        Whether to raise error if partial pattern for a data type is found.
-        This allows to bypass mandatory key check and issue a warning
-        instead of raising error. This allows one to have a DataGrabber
-        with data types without the corresponding mandatory keys and is
-        powerful when used with :class:`.MultipleDataGrabber`
-        (default True).
 
     Raises
     ------
@@ -164,21 +143,17 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
         replacements: Union[List[str], str],
         datadir: Union[str, Path],
         confounds_format: Optional[str] = None,
-        partial_pattern_ok: bool = False,
     ) -> None:
+        # Validate patterns
+        validate_patterns(types=types, patterns=patterns)
+        self.patterns = patterns
+
         # Convert replacements to list if not already
         if not isinstance(replacements, list):
             replacements = [replacements]
-        # Validate patterns
-        self.validate_patterns(
-            types=types,
-            replacements=replacements,
-            patterns=patterns,
-            partial_pattern_ok=partial_pattern_ok,
-        )
+        # Validate replacements
+        validate_replacements(replacements=replacements, patterns=patterns)
         self.replacements = replacements
-        self.patterns = patterns
-        self.partial_pattern_ok = partial_pattern_ok
 
         # Validate confounds format
         if (
@@ -229,25 +204,18 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
         t_replacements = [
             x for x in self.replacements if f"{{{x}}}" in pattern
         ]
-        # Ops on re_pattern
-        # Remove negated unix glob pattern i.e., [!...] for re_pattern
-        re_pattern = re.sub(r"\[!.?\]", "", re_pattern)
-        # Remove enclosing square brackets from unix glob pattern i.e., [...]
-        # for re_pattern
-        re_pattern = re.sub(r"\[|\]", "", re_pattern)
-        # Iteratively replace the first of each with a named group definition
+
         for t_r in t_replacements:
+            # Replace the first of each with a named group definition
             re_pattern = re_pattern.replace(f"{{{t_r}}}", f"(?P<{t_r}>.*)", 1)
-        # Iteratively replace the second appearance of each with the named
-        # group back reference
+
         for t_r in t_replacements:
+            # Replace the second appearance of each with the named group
+            # back reference
             re_pattern = re_pattern.replace(f"{{{t_r}}}", f"(?P={t_r})")
-        # Ops on glob_pattern
-        # Iteratively replace replacements with wildcard i.e., *
-        # for glob_pattern
+
         for t_r in t_replacements:
             glob_pattern = glob_pattern.replace(f"{{{t_r}}}", "*")
-
         return re_pattern, glob_pattern, t_replacements
 
     def _replace_patterns_glob(self, element: Dict, pattern: str) -> str:
@@ -276,70 +244,8 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
                 f"The element keys must be {self.replacements}, "
                 f"element has {list(element.keys())}."
             )
-        # Remove negated unix glob pattern i.e., [!...]
-        pattern = re.sub(r"\[!.?\]", "", pattern)
-        # Remove enclosing square brackets from unix glob pattern i.e., [...]
-        pattern = re.sub(r"\[|\]", "", pattern)
         return pattern.format(**element)
 
-    def _get_path_from_patterns(
-        self, element: Dict, pattern: str, data_type: str
-    ) -> Path:
-        """Get path from resolved patterns.
-
-        Parameters
-        ----------
-        element : dict
-            The element to be used in the replacement.
-        pattern : str
-            The pattern to be replaced.
-        data_type : str
-            The data type of the pattern.
-
-        Returns
-        -------
-        pathlib.Path
-            The path for the resolved pattern.
-
-        Raises
-        ------
-        RuntimeError
-            If more than one file matches for a data type's pattern or
-            if no file matches for a data type's pattern or
-            if file cannot be accessed for an element.
-
-        """
-        # Replace element in the pattern for globbing
-        resolved_pattern = self._replace_patterns_glob(element, pattern)
-        # Resolve path for wildcard
-        if "*" in resolved_pattern:
-            t_matches = list(self.datadir.absolute().glob(resolved_pattern))
-            # Multiple matches
-            if len(t_matches) > 1:
-                raise_error(
-                    f"More than one file matches for {element} / {data_type}:"
-                    f" {t_matches}",
-                    klass=RuntimeError,
-                )
-            # No matches
-            elif len(t_matches) == 0:
-                raise_error(
-                    f"No file matches for {element} / {data_type}",
-                    klass=RuntimeError,
-                )
-            path = t_matches[0]
-        else:
-            path = self.datadir / resolved_pattern
-            if not self.skip_file_check:
-                if not path.exists() and not path.is_symlink():
-                    raise_error(
-                        f"Cannot access {data_type} for {element}: "
-                        f"File {path} does not exist",
-                        klass=RuntimeError,
-                    )
-
-        return path
-
     def get_element_keys(self) -> List[str]:
         """Get element keys.
 
@@ -373,49 +279,47 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
             Dictionary of dictionaries for each type of data required for the
             specified element.
 
+        Raises
+        ------
+        RuntimeError
+            If more than one file matches for a data type's pattern or
+            if no file matches for a data type's pattern or
+            if file cannot be accessed for an element.
+
         """
         out = {}
         for t_type in self.types:
-            # Data type dictionary
             t_pattern = self.patterns[t_type]
-            # Copy data type dictionary in output
-            out[t_type] = deepcopy(t_pattern)
-            # Iterate to check for nested "types" like mask
-            for k, v in t_pattern.items():
-                # Resolve pattern for base data type
-                if k == "pattern":
-                    logger.info(f"Resolving path from pattern for {t_type}")
-                    # Resolve pattern
-                    base_data_type_pattern_path = self._get_path_from_patterns(
-                        element=element,
-                        pattern=v,
-                        data_type=t_type,
+            t_replace = self._replace_patterns_glob(
+                element, t_pattern["pattern"]
+            )
+            if "*" in t_replace:
+                t_matches = list(self.datadir.absolute().glob(t_replace))
+                if len(t_matches) > 1:
+                    raise_error(
+                        f"More than one file matches for {element} / {t_type}:"
+                        f" {t_matches}",
+                        klass=RuntimeError,
                     )
-                    # Remove pattern key
-                    out[t_type].pop("pattern")
-                    # Add path key
-                    out[t_type].update({"path": base_data_type_pattern_path})
-                # Resolve pattern for nested data type
-                if isinstance(v, dict) and "pattern" in v:
-                    # Set nested type key for easier access
-                    t_nested_type = f"{t_type}.{k}"
-                    logger.info(
-                        f"Resolving path from pattern for {t_nested_type}"
+                elif len(t_matches) == 0:
+                    raise_error(
+                        f"No file matches for {element} / {t_type}",
+                        klass=RuntimeError,
                     )
-                    # Resolve pattern
-                    nested_data_type_pattern_path = (
-                        self._get_path_from_patterns(
-                            element=element,
-                            pattern=v["pattern"],
-                            data_type=t_nested_type,
+                t_out = t_matches[0]
+            else:
+                t_out = self.datadir / t_replace
+                if not self.skip_file_check:
+                    if not t_out.exists() and not t_out.is_symlink():
+                        raise_error(
+                            f"Cannot access {t_type} for {element}: "
+                            f"File {t_out} does not exist",
+                            klass=RuntimeError,
                         )
-                    )
-                    # Remove pattern key
-                    out[t_type][k].pop("pattern")
-                    # Add path key
-                    out[t_type][k].update(
-                        {"path": nested_data_type_pattern_path}
-                    )
+            # Update path for the element
+            out[t_type] = t_pattern.copy()  # copy data type dictionary
+            out[t_type].pop("pattern")  # remove pattern key
+            out[t_type].update({"path": t_out})  # add path key
 
         return out
 
@@ -447,26 +351,14 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
         for t_idx in reversed(order):
             t_type = self.types[t_idx]
             types_element = set()
-
-            # Get the pattern dict
+            # Get the pattern
             t_pattern = self.patterns[t_type]
-            # Conditional fetch of base pattern for getting elements
-            pattern = None
-            # Try for data type pattern
-            pattern = t_pattern.get("pattern")
-            # Try for nested data type pattern
-            if pattern is None and self.partial_pattern_ok:
-                for v in t_pattern.values():
-                    if isinstance(v, dict) and "pattern" in v:
-                        pattern = v["pattern"]
-                        break
-
             # Replace the pattern
             (
                 re_pattern,
                 glob_pattern,
                 t_replacements,
-            ) = self._replace_patterns_regex(pattern)
+            ) = self._replace_patterns_regex(t_pattern["pattern"])
             for fname in self.datadir.glob(glob_pattern):
                 suffix = fname.relative_to(self.datadir).as_posix()
                 m = re.match(re_pattern, suffix)

junifer/datagrabber/pattern_datalad.py

@@ -12,9 +12,6 @@ from .datalad_base import DataladDataGrabber
 from .pattern import PatternDataGrabber
 
 
-__all__ = ["PatternDataladDataGrabber"]
-
-
 @register_datagrabber
 class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
     """Concrete implementation for pattern and datalad based data fetching.
@@ -35,12 +32,7 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": {
-                  "mask": {
-                      "mandatory": ["pattern", "space"],
-                      "optional": []
-                  }
-              }
+              "optional": []
             }
 
         * ``"T2w"`` :
@@ -49,12 +41,7 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": {
-                  "mask": {
-                      "mandatory": ["pattern", "space"],
-                      "optional": []
-                  }
-              }
+              "optional": []
             }
 
         * ``"BOLD"`` :
@@ -63,16 +50,7 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
 
             {
               "mandatory": ["pattern", "space"],
-              "optional": {
-                  "mask": {
-                      "mandatory": ["pattern", "space"],
-                      "optional": []
-                  }
-                  "confounds": {
-                      "mandatory": ["pattern", "format"],
-                      "optional": []
-                  }
-              }
+              "optional": ["mask_item"]
             }
 
         * ``"Warp"`` :
@@ -84,6 +62,15 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
               "optional": []
             }
 
+        * ``"BOLD_confounds"`` :
+
+          .. code-block:: none
+
+            {
+              "mandatory": ["pattern", "format"],
+              "optional": []
+            }
+
         * ``"VBM_GM"`` :
 
           .. code-block:: none