junifer 0.0.5.dev219__py3-none-any.whl → 0.0.5.dev242__py3-none-any.whl

junifer/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.5.dev219'
-__version_tuple__ = version_tuple = (0, 0, 5, 'dev219')
+__version__ = version = '0.0.5.dev242'
+__version_tuple__ = version_tuple = (0, 0, 5, 'dev242')

junifer/datagrabber/__init__.py

@@ -17,6 +17,7 @@ from .hcp1200 import HCP1200, DataladHCP1200
 from .multiple import MultipleDataGrabber
 from .dmcc13_benchmark import DMCC13Benchmark
 
+from .pattern_validation_mixin import PatternValidationMixin
 
 __all__ = [
     "BaseDataGrabber",
@@ -30,4 +31,5 @@ __all__ = [
     "DataladHCP1200",
     "MultipleDataGrabber",
     "DMCC13Benchmark",
+    "PatternValidationMixin",
 ]

junifer/datagrabber/base.py

@@ -11,7 +11,6 @@ from typing import Dict, Iterator, List, Tuple, Union
 
 from ..pipeline import UpdateMetaMixin
 from ..utils import logger, raise_error
-from .utils import validate_types
 
 
 __all__ = ["BaseDataGrabber"]
@@ -30,16 +29,21 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
     datadir : str or pathlib.Path
         The directory where the data is / will be stored.
 
-    Attributes
-    ----------
-    datadir : pathlib.Path
-        The directory where the data is / will be stored.
+    Raises
+    ------
+    TypeError
+        If ``types`` is not a list or if the values are not string.
 
     """
 
     def __init__(self, types: List[str], datadir: Union[str, Path]) -> None:
         # Validate types
-        validate_types(types)
+        if not isinstance(types, list):
+            raise_error(msg="`types` must be a list", klass=TypeError)
+        if any(not isinstance(x, str) for x in types):
+            raise_error(
+                msg="`types` must be a list of strings", klass=TypeError
+            )
         self.types = types
 
         # Convert str to Path

junifer/datagrabber/hcp1200/hcp1200.py

@@ -10,8 +10,8 @@ from pathlib import Path
 from typing import Dict, List, Union
 
 from ...api.decorators import register_datagrabber
+from ...utils import raise_error
 from ..pattern import PatternDataGrabber
-from ..utils import raise_error
 
 
 __all__ = ["HCP1200"]

junifer/datagrabber/multiple.py

@@ -7,13 +7,15 @@
 
 from typing import Dict, List, Tuple, Union
 
-from ..utils import raise_error
+from ..api.decorators import register_datagrabber
+from ..utils import deep_update, raise_error
 from .base import BaseDataGrabber
 
 
 __all__ = ["MultipleDataGrabber"]
 
 
+@register_datagrabber
 class MultipleDataGrabber(BaseDataGrabber):
     """Concrete implementation for multi sourced data fetching.
 
@@ -27,19 +29,53 @@ class MultipleDataGrabber(BaseDataGrabber):
     **kwargs
         Keyword arguments passed to superclass.
 
+    Raises
+    ------
+    RuntimeError
+        If ``datagrabbers`` have different element keys or
+        overlapping data types or nested data types.
+
     """
 
     def __init__(self, datagrabbers: List[BaseDataGrabber], **kwargs) -> None:
         # Check datagrabbers consistency
-        # 1) same element keys
+        # Check for same element keys
         first_keys = datagrabbers[0].get_element_keys()
         for dg in datagrabbers[1:]:
             if dg.get_element_keys() != first_keys:
-                raise_error("DataGrabbers have different element keys.")
-        # 2) no overlapping types
+                raise_error(
+                    msg="DataGrabbers have different element keys",
+                    klass=RuntimeError,
+                )
+        # Check for no overlapping types (and nested data types)
         types = [x for dg in datagrabbers for x in dg.get_types()]
         if len(types) != len(set(types)):
-            raise_error("DataGrabbers have overlapping types.")
+            if all(hasattr(dg, "patterns") for dg in datagrabbers):
+                first_patterns = datagrabbers[0].patterns
+                for dg in datagrabbers[1:]:
+                    for data_type in set(types):
+                        dtype_pattern = dg.patterns.get(data_type)
+                        if dtype_pattern is None:
+                            continue
+                        # Check if first-level keys of data type are same
+                        if (
+                            dtype_pattern.keys()
+                            == first_patterns[data_type].keys()
+                        ):
+                            raise_error(
+                                msg=(
+                                    "DataGrabbers have overlapping mandatory "
+                                    "and / or optional key(s) for data type: "
+                                    f"`{data_type}`"
+                                ),
+                                klass=RuntimeError,
+                            )
+            else:
+                # Can't check further
+                raise_error(
+                    msg="DataGrabbers have overlapping types",
+                    klass=RuntimeError,
+                )
         self._datagrabbers = datagrabbers
 
     def __getitem__(self, element: Union[str, Tuple]) -> Dict:
@@ -65,7 +101,7 @@ class MultipleDataGrabber(BaseDataGrabber):
         metas = []
         for dg in self._datagrabbers:
             t_out = dg[element]
-            out.update(t_out)
+            deep_update(out, t_out)
             # Now get the meta for this datagrabber
             t_meta = {}
             dg.update_meta(t_meta, "datagrabber")

junifer/datagrabber/pattern.py

@@ -15,7 +15,7 @@ import numpy as np
 from ..api.decorators import register_datagrabber
 from ..utils import logger, raise_error
 from .base import BaseDataGrabber
-from .utils import validate_patterns, validate_replacements
+from .pattern_validation_mixin import PatternValidationMixin
 
 
 __all__ = ["PatternDataGrabber"]
@@ -26,7 +26,7 @@ _CONFOUNDS_FORMATS = ("fmriprep", "adhoc")
 
 
 @register_datagrabber
-class PatternDataGrabber(BaseDataGrabber):
+class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
     """Concrete implementation for pattern-based data fetching.
 
     Implements a DataGrabber that understands patterns to grab data.
@@ -142,6 +142,13 @@ class PatternDataGrabber(BaseDataGrabber):
         The directory where the data is / will be stored.
     confounds_format : {"fmriprep", "adhoc"} or None, optional
         The format of the confounds for the dataset (default None).
+    partial_pattern_ok : bool, optional
+        Whether to raise error if partial pattern for a data type is found.
+        This allows to bypass mandatory key check and issue a warning
+        instead of raising error. This allows one to have a DataGrabber
+        with data types without the corresponding mandatory keys and is
+        powerful when used with :class:`.MultipleDataGrabber`
+        (default True).
 
     Raises
     ------
@@ -157,17 +164,21 @@ class PatternDataGrabber(BaseDataGrabber):
         replacements: Union[List[str], str],
         datadir: Union[str, Path],
         confounds_format: Optional[str] = None,
+        partial_pattern_ok: bool = False,
     ) -> None:
-        # Validate patterns
-        validate_patterns(types=types, patterns=patterns)
-        self.patterns = patterns
-
         # Convert replacements to list if not already
         if not isinstance(replacements, list):
             replacements = [replacements]
-        # Validate replacements
-        validate_replacements(replacements=replacements, patterns=patterns)
+        # Validate patterns
+        self.validate_patterns(
+            types=types,
+            replacements=replacements,
+            patterns=patterns,
+            partial_pattern_ok=partial_pattern_ok,
+        )
         self.replacements = replacements
+        self.patterns = patterns
+        self.partial_pattern_ok = partial_pattern_ok
 
         # Validate confounds format
         if (
@@ -436,14 +447,26 @@ class PatternDataGrabber(BaseDataGrabber):
         for t_idx in reversed(order):
             t_type = self.types[t_idx]
             types_element = set()
-            # Get the pattern
+
+            # Get the pattern dict
             t_pattern = self.patterns[t_type]
+            # Conditional fetch of base pattern for getting elements
+            pattern = None
+            # Try for data type pattern
+            pattern = t_pattern.get("pattern")
+            # Try for nested data type pattern
+            if pattern is None and self.partial_pattern_ok:
+                for v in t_pattern.values():
+                    if isinstance(v, dict) and "pattern" in v:
+                        pattern = v["pattern"]
+                        break
+
             # Replace the pattern
             (
                 re_pattern,
                 glob_pattern,
                 t_replacements,
-            ) = self._replace_patterns_regex(t_pattern["pattern"])
+            ) = self._replace_patterns_regex(pattern)
             for fname in self.datadir.glob(glob_pattern):
                 suffix = fname.relative_to(self.datadir).as_posix()
                 m = re.match(re_pattern, suffix)

junifer/datagrabber/pattern_validation_mixin.py

@@ -0,0 +1,388 @@
+"""Provide mixin validation class for pattern-based DataGrabber."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+from typing import Dict, List
+
+from ..utils import logger, raise_error, warn_with_log
+
+
+__all__ = ["PatternValidationMixin"]
+
+
+# Define schema for pattern-based datagrabber's patterns
+PATTERNS_SCHEMA = {
+    "T1w": {
+        "mandatory": ["pattern", "space"],
+        "optional": {
+            "mask": {"mandatory": ["pattern", "space"], "optional": []},
+        },
+    },
+    "T2w": {
+        "mandatory": ["pattern", "space"],
+        "optional": {
+            "mask": {"mandatory": ["pattern", "space"], "optional": []},
+        },
+    },
+    "BOLD": {
+        "mandatory": ["pattern", "space"],
+        "optional": {
+            "mask": {"mandatory": ["pattern", "space"], "optional": []},
+            "confounds": {
+                "mandatory": ["pattern", "format"],
+                "optional": ["mappings"],
+            },
+        },
+    },
+    "Warp": {
+        "mandatory": ["pattern", "src", "dst"],
+        "optional": {},
+    },
+    "VBM_GM": {
+        "mandatory": ["pattern", "space"],
+        "optional": {},
+    },
+    "VBM_WM": {
+        "mandatory": ["pattern", "space"],
+        "optional": {},
+    },
+    "VBM_CSF": {
+        "mandatory": ["pattern", "space"],
+        "optional": {},
+    },
+    "DWI": {
+        "mandatory": ["pattern"],
+        "optional": {},
+    },
+    "FreeSurfer": {
+        "mandatory": ["pattern"],
+        "optional": {
+            "aseg": {"mandatory": ["pattern"], "optional": []},
+            "norm": {"mandatory": ["pattern"], "optional": []},
+            "lh_white": {"mandatory": ["pattern"], "optional": []},
+            "rh_white": {"mandatory": ["pattern"], "optional": []},
+            "lh_pial": {"mandatory": ["pattern"], "optional": []},
+            "rh_pial": {"mandatory": ["pattern"], "optional": []},
+        },
+    },
+}
+
+
+class PatternValidationMixin:
+    """Mixin class for pattern validation."""
+
+    def _validate_types(self, types: List[str]) -> None:
+        """Validate the types.
+
+        Parameters
+        ----------
+        types : list of str
+            The data types to validate.
+
+        Raises
+        ------
+        TypeError
+            If ``types`` is not a list or if the values are not string.
+
+        """
+        if not isinstance(types, list):
+            raise_error(msg="`types` must be a list", klass=TypeError)
+        if any(not isinstance(x, str) for x in types):
+            raise_error(
+                msg="`types` must be a list of strings", klass=TypeError
+            )
+
+    def _validate_replacements(
+        self,
+        replacements: List[str],
+        patterns: Dict[str, Dict[str, str]],
+        partial_pattern_ok: bool,
+    ) -> None:
+        """Validate the replacements.
+
+        Parameters
+        ----------
+        replacements : list of str
+            The replacements to validate.
+        patterns : dict
+            The patterns to validate replacements against.
+        partial_pattern_ok : bool
+            Whether to raise error if partial pattern for a data type is found.
+
+        Raises
+        ------
+        TypeError
+            If ``replacements`` is not a list or if the values are not string.
+        ValueError
+            If a value in ``replacements`` is not part of a data type pattern
+            and ``partial_pattern_ok=False`` or
+            if no data type patterns contain all values in ``replacements`` and
+            ``partial_pattern_ok=False``.
+
+        Warns
+        -----
+        RuntimeWarning
+            If a value in ``replacements`` is not part of the data type pattern
+            and ``partial_pattern_ok=True``.
+
+        """
+        if not isinstance(replacements, list):
+            raise_error(msg="`replacements` must be a list.", klass=TypeError)
+
+        if any(not isinstance(x, str) for x in replacements):
+            raise_error(
+                msg="`replacements` must be a list of strings.",
+                klass=TypeError,
+            )
+
+        for x in replacements:
+            if all(
+                x not in y
+                for y in [
+                    data_type_val.get("pattern", "")
+                    for data_type_val in patterns.values()
+                ]
+            ):
+                if partial_pattern_ok:
+                    warn_with_log(
+                        f"Replacement: `{x}` is not part of any pattern, "
+                        "things might not work as expected if you are unsure "
+                        "of what you are doing"
+                    )
+                else:
+                    raise_error(
+                        msg=f"Replacement: {x} is not part of any pattern."
+                    )
+
+        # Check that at least one pattern has all the replacements
+        at_least_one = False
+        for data_type_val in patterns.values():
+            if all(
+                x in data_type_val.get("pattern", "") for x in replacements
+            ):
+                at_least_one = True
+        if not at_least_one and not partial_pattern_ok:
+            raise_error(
+                msg="At least one pattern must contain all replacements."
+            )
+
+    def _validate_mandatory_keys(
+        self,
+        keys: List[str],
+        schema: List[str],
+        data_type: str,
+        partial_pattern_ok: bool = False,
+    ) -> None:
+        """Validate mandatory keys.
+
+        Parameters
+        ----------
+        keys : list of str
+            The keys to validate.
+        schema : list of str
+            The schema to validate against.
+        data_type : str
+            The data type being validated.
+        partial_pattern_ok : bool, optional
+            Whether to raise error if partial pattern for a data type is found
+            (default True).
+
+        Raises
+        ------
+        KeyError
+            If any mandatory key is missing for a data type and
+            ``partial_pattern_ok=False``.
+
+        Warns
+        -----
+        RuntimeWarning
+            If any mandatory key is missing for a data type and
+            ``partial_pattern_ok=True``.
+
+        """
+        for key in schema:
+            if key not in keys:
+                if partial_pattern_ok:
+                    warn_with_log(
+                        f"Mandatory key: `{key}` not found for {data_type}, "
+                        "things might not work as expected if you are unsure "
+                        "of what you are doing"
+                    )
+                else:
+                    raise_error(
+                        msg=f"Mandatory key: `{key}` missing for {data_type}",
+                        klass=KeyError,
+                    )
+            else:
+                logger.debug(f"Mandatory key: `{key}` found for {data_type}")
+
+    def _identify_stray_keys(
+        self, keys: List[str], schema: List[str], data_type: str
+    ) -> None:
+        """Identify stray keys.
+
+        Parameters
+        ----------
+        keys : list of str
+            The keys to check.
+        schema : list of str
+            The schema to check against.
+        data_type : str
+            The data type being checked.
+
+        Raises
+        ------
+        RuntimeError
+            If an unknown key is found for a data type.
+
+        """
+        for key in keys:
+            if key not in schema:
+                raise_error(
+                    msg=(
+                        f"Key: {key} not accepted for {data_type} "
+                        "pattern, remove it to proceed"
+                    ),
+                    klass=RuntimeError,
+                )
+
+    def validate_patterns(
+        self,
+        types: List[str],
+        replacements: List[str],
+        patterns: Dict[str, Dict[str, str]],
+        partial_pattern_ok: bool = False,
+    ) -> None:
+        """Validate the patterns.
+
+        Parameters
+        ----------
+        types : list of str
+            The data types to check patterns of.
+        replacements : list of str
+            The replacements to be replaced in the patterns.
+        patterns : dict
+            The patterns to validate.
+        partial_pattern_ok : bool, optional
+            Whether to raise error if partial pattern for a data type is found.
+            If False, a warning is issued instead of raising an error
+            (default False).
+
+        Raises
+        ------
+        TypeError
+            If ``patterns`` is not a dictionary.
+        ValueError
+            If length of ``types`` and ``patterns`` are different or
+            if ``patterns`` is missing entries from ``types`` or
+            if unknown data type is found in ``patterns`` or
+            if data type pattern key contains '*' as value.
+
+        """
+        # Validate types
+        self._validate_types(types=types)
+
+        # Validate patterns
+        if not isinstance(patterns, dict):
+            raise_error(msg="`patterns` must be a dict", klass=TypeError)
+        # Unequal length of objects
+        if len(types) > len(patterns):
+            raise_error(
+                msg="Length of `types` more than that of `patterns`",
+                klass=ValueError,
+            )
+        # Missing type in patterns
+        if any(x not in patterns for x in types):
+            raise_error(
+                msg="`patterns` must contain all `types`", klass=ValueError
+            )
+        # Check against schema
+        for data_type_key, data_type_val in patterns.items():
+            # Check if valid data type is provided
+            if data_type_key not in PATTERNS_SCHEMA:
+                raise_error(
+                    f"Unknown data type: {data_type_key}, "
+                    f"should be one of: {list(PATTERNS_SCHEMA.keys())}"
+                )
+            # Check mandatory keys for data type
+            self._validate_mandatory_keys(
+                keys=list(data_type_val),
+                schema=PATTERNS_SCHEMA[data_type_key]["mandatory"],
+                data_type=data_type_key,
+                partial_pattern_ok=partial_pattern_ok,
+            )
+            # Check optional keys for data type
+            for optional_key, optional_val in PATTERNS_SCHEMA[data_type_key][
+                "optional"
+            ].items():
+                if optional_key not in data_type_val:
+                    logger.debug(
+                        f"Optional key: `{optional_key}` missing for "
+                        f"{data_type_key}"
+                    )
+                else:
+                    logger.debug(
+                        f"Optional key: `{optional_key}` found for "
+                        f"{data_type_key}"
+                    )
+                    # Set nested type name for easier access
+                    nested_data_type = f"{data_type_key}.{optional_key}"
+                    nested_mandatory_keys_schema = PATTERNS_SCHEMA[
+                        data_type_key
+                    ]["optional"][optional_key]["mandatory"]
+                    nested_optional_keys_schema = PATTERNS_SCHEMA[
+                        data_type_key
+                    ]["optional"][optional_key]["optional"]
+                    # Check mandatory keys for nested type
+                    self._validate_mandatory_keys(
+                        keys=list(optional_val["mandatory"]),
+                        schema=nested_mandatory_keys_schema,
+                        data_type=nested_data_type,
+                        partial_pattern_ok=partial_pattern_ok,
+                    )
+                    # Check optional keys for nested type
+                    for nested_optional_key in nested_optional_keys_schema:
+                        if nested_optional_key not in optional_val["optional"]:
+                            logger.debug(
+                                f"Optional key: `{nested_optional_key}` "
+                                f"missing for {nested_data_type}"
+                            )
+                        else:
+                            logger.debug(
+                                f"Optional key: `{nested_optional_key}` found "
+                                f"for {nested_data_type}"
+                            )
+                    # Check stray key for nested data type
+                    self._identify_stray_keys(
+                        keys=optional_val["mandatory"]
+                        + optional_val["optional"],
+                        schema=nested_mandatory_keys_schema
+                        + nested_optional_keys_schema,
+                        data_type=nested_data_type,
+                    )
+            # Check stray key for data type
+            self._identify_stray_keys(
+                keys=list(data_type_val.keys()),
+                schema=(
+                    PATTERNS_SCHEMA[data_type_key]["mandatory"]
+                    + list(PATTERNS_SCHEMA[data_type_key]["optional"].keys())
+                ),
+                data_type=data_type_key,
+            )
+            # Wildcard check in patterns
+            if "}*" in data_type_val.get("pattern", ""):
+                raise_error(
+                    msg=(
+                        f"`{data_type_key}.pattern` must not contain `*` "
+                        "following a replacement"
+                    ),
+                    klass=ValueError,
+                )
+
+        # Validate replacements
+        self._validate_replacements(
+            replacements=replacements,
+            patterns=patterns,
+            partial_pattern_ok=partial_pattern_ok,
+        )