junifer 0.0.5.dev68__py3-none-any.whl → 0.0.5.dev93__py3-none-any.whl

junifer/datagrabber/utils.py

@@ -13,51 +13,45 @@ from ..utils import logger, raise_error
 PATTERNS_SCHEMA = {
     "T1w": {
         "mandatory": ["pattern", "space"],
-        "optional": ["mask_item"],
-    },
-    "T1w_mask": {
-        "mandatory": ["pattern", "space"],
-        "optional": [],
+        "optional": {
+            "mask": {"mandatory": ["pattern", "space"], "optional": []},
+        },
     },
     "T2w": {
         "mandatory": ["pattern", "space"],
-        "optional": ["mask_item"],
-    },
-    "T2w_mask": {
-        "mandatory": ["pattern", "space"],
-        "optional": [],
+        "optional": {
+            "mask": {"mandatory": ["pattern", "space"], "optional": []},
+        },
     },
     "BOLD": {
         "mandatory": ["pattern", "space"],
-        "optional": ["mask_item"],
-    },
-    "BOLD_confounds": {
-        "mandatory": ["pattern", "format"],
-        "optional": [],
-    },
-    "BOLD_mask": {
-        "mandatory": ["pattern", "space"],
-        "optional": [],
+        "optional": {
+            "mask": {"mandatory": ["pattern", "space"], "optional": []},
+            "confounds": {
+                "mandatory": ["pattern", "format"],
+                "optional": ["mappings"],
+            },
+        },
     },
     "Warp": {
         "mandatory": ["pattern", "src", "dst"],
-        "optional": [],
+        "optional": {},
     },
     "VBM_GM": {
         "mandatory": ["pattern", "space"],
-        "optional": [],
+        "optional": {},
     },
     "VBM_WM": {
         "mandatory": ["pattern", "space"],
-        "optional": [],
+        "optional": {},
     },
     "VBM_CSF": {
         "mandatory": ["pattern", "space"],
-        "optional": [],
+        "optional": {},
     },
     "DWI": {
         "mandatory": ["pattern"],
-        "optional": [],
+        "optional": {},
     },
 }
 
@@ -129,6 +123,67 @@ def validate_replacements(
         raise_error(msg="At least one pattern must contain all replacements.")
 
 
+def _validate_mandatory_keys(
+    keys: List[str], schema: List[str], data_type: str
+) -> None:
+    """Validate mandatory keys.
+
+    Parameters
+    ----------
+    keys : list of str
+        The keys to validate.
+    schema : list of str
+        The schema to validate against.
+    data_type : str
+        The data type being validated.
+
+    Raises
+    ------
+    KeyError
+        If any mandatory key is missing for a data type.
+
+    """
+    for key in schema:
+        if key not in keys:
+            raise_error(
+                msg=f"Mandatory key: `{key}` missing for {data_type}",
+                klass=KeyError,
+            )
+        else:
+            logger.debug(f"Mandatory key: `{key}` found for {data_type}")
+
+
+def _identify_stray_keys(
+    keys: List[str], schema: List[str], data_type: str
+) -> None:
+    """Identify stray keys.
+
+    Parameters
+    ----------
+    keys : list of str
+        The keys to check.
+    schema : list of str
+        The schema to check against.
+    data_type : str
+        The data type being checked.
+
+    Raises
+    ------
+    RuntimeError
+        If an unknown key is found for a data type.
+
+    """
+    for key in keys:
+        if key not in schema:
+            raise_error(
+                msg=(
+                    f"Key: {key} not accepted for {data_type} "
+                    "pattern, remove it to proceed"
+                ),
+                klass=RuntimeError,
+            )
+
+
 def validate_patterns(
     types: List[str], patterns: Dict[str, Dict[str, str]]
 ) -> None:
@@ -143,10 +198,6 @@ def validate_patterns(
 
     Raises
     ------
-    KeyError
-        If any mandatory key is missing for a data type.
-    RuntimeError
-        If an unknown key is found for a data type.
     TypeError
         If ``patterns`` is not a dictionary.
     ValueError
@@ -180,22 +231,15 @@ def validate_patterns(
                 f"should be one of: {list(PATTERNS_SCHEMA.keys())}"
             )
         # Check mandatory keys for data type
-        for mandatory_key in PATTERNS_SCHEMA[data_type_key]["mandatory"]:
-            if mandatory_key not in data_type_val:
-                raise_error(
-                    msg=(
-                        f"Mandatory key: `{mandatory_key}` missing for "
-                        f"{data_type_key}"
-                    ),
-                    klass=KeyError,
-                )
-            else:
-                logger.debug(
-                    f"Mandatory key: `{mandatory_key}` found for "
-                    f"{data_type_key}"
-                )
+        _validate_mandatory_keys(
+            keys=list(data_type_val),
+            schema=PATTERNS_SCHEMA[data_type_key]["mandatory"],
+            data_type=data_type_key,
+        )
         # Check optional keys for data type
-        for optional_key in PATTERNS_SCHEMA[data_type_key]["optional"]:
+        for optional_key, optional_val in PATTERNS_SCHEMA[data_type_key][
+            "optional"
+        ].items():
             if optional_key not in data_type_val:
                 logger.debug(
                     f"Optional key: `{optional_key}` missing for "
@@ -206,19 +250,48 @@ def validate_patterns(
                     f"Optional key: `{optional_key}` found for "
                     f"{data_type_key}"
                 )
+                # Set nested type name for easier access
+                nested_data_type = f"{data_type_key}.{optional_key}"
+                nested_mandatory_keys_schema = PATTERNS_SCHEMA[data_type_key][
+                    "optional"
+                ][optional_key]["mandatory"]
+                nested_optional_keys_schema = PATTERNS_SCHEMA[data_type_key][
+                    "optional"
+                ][optional_key]["optional"]
+                # Check mandatory keys for nested type
+                _validate_mandatory_keys(
+                    keys=list(optional_val["mandatory"]),
+                    schema=nested_mandatory_keys_schema,
+                    data_type=nested_data_type,
+                )
+                # Check optional keys for nested type
+                for nested_optional_key in nested_optional_keys_schema:
+                    if nested_optional_key not in optional_val["optional"]:
+                        logger.debug(
+                            f"Optional key: `{nested_optional_key}` missing "
+                            f"for {nested_data_type}"
+                        )
+                    else:
+                        logger.debug(
+                            f"Optional key: `{nested_optional_key}` found for "
+                            f"{nested_data_type}"
+                        )
+                # Check stray key for nested data type
+                _identify_stray_keys(
+                    keys=optional_val["mandatory"] + optional_val["optional"],
+                    schema=nested_mandatory_keys_schema
+                    + nested_optional_keys_schema,
+                    data_type=nested_data_type,
+                )
         # Check stray key for data type
-        for key in data_type_val.keys():
-            if key not in (
+        _identify_stray_keys(
+            keys=list(data_type_val.keys()),
+            schema=(
                 PATTERNS_SCHEMA[data_type_key]["mandatory"]
-                + PATTERNS_SCHEMA[data_type_key]["optional"]
-            ):
-                raise_error(
-                    msg=(
-                        f"Key: {key} not accepted for {data_type_key} "
-                        "pattern, remove it to proceed"
-                    ),
-                    klass=RuntimeError,
-                )
+                + list(PATTERNS_SCHEMA[data_type_key]["optional"].keys())
+            ),
+            data_type=data_type_key,
+        )
         # Wildcard check in patterns
         if "}*" in data_type_val["pattern"]:
             raise_error(

junifer/datareader/default.py

@@ -5,7 +5,7 @@
 # License: AGPL
 
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union
 
 import nibabel as nib
 import pandas as pd
@@ -102,57 +102,106 @@ class DefaultDataReader(PipelineStepMixin, UpdateMetaMixin):
         if params is None:
             params = {}
         # For each type of data, try to read it
-        for type_ in input.keys():
+        for type_key, type_val in input.items():
             # Skip Warp data type
-            if type_ == "Warp":
+            if type_key == "Warp":
                 continue
 
             # Check for malformed datagrabber specification
-            if "path" not in input[type_]:
+            if "path" not in type_val:
                 warn_with_log(
-                    f"Input type {type_} does not provide a path. Skipping."
+                    f"Input type {type_key} does not provide a path. Skipping."
                 )
                 continue
 
-            # Retrieve actual path
-            t_path = input[type_]["path"]
-            # Retrieve loading params for the data type
-            t_params = params.get(type_, {})
-
-            # Convert str to Path
-            if not isinstance(t_path, Path):
-                t_path = Path(t_path)
-                out[type_]["path"] = t_path
-
-            logger.info(f"Reading {type_} from {t_path.as_posix()}")
-            # Initialize variable for file data
-            fread = None
-            # Lowercase path
-            fname = t_path.name.lower()
-            # Loop through extensions to find the correct one
-            for ext, ftype in _extensions.items():
-                if fname.endswith(ext):
-                    logger.info(f"{type_} is type {ftype}")
-                    # Retrieve reader function
-                    reader_func = _readers[ftype]["func"]
-                    # Retrieve reader function params
-                    reader_params = _readers[ftype]["params"]
-                    # Update reader function params
-                    if reader_params is not None:
-                        t_params.update(reader_params)
-                    logger.debug(f"Calling {reader_func} with {t_params}")
+            # Iterate to check for nested "types" like mask;
+            # need to copy to avoid runtime error for changing dict size
+            for k, v in type_val.copy().items():
+                # Read data for base data type
+                if k == "path":
+                    # Convert str to Path
+                    if not isinstance(v, Path):
+                        v = Path(v)
+                    # Update path
+                    out[type_key]["path"] = v
+                    logger.info(f"Reading {type_key} from {v.absolute()!s}")
+                    # Retrieve loading params for the data type
+                    t_params = params.get(type_key, {})
                     # Read data
-                    fread = reader_func(t_path, **t_params)
-                    break
-            # If no file data is found due to unknown extension
-            if fread is None:
-                logger.info(
-                    f"Unknown file type {t_path.as_posix()}, skipping reading"
-                )
+                    out[type_key]["data"] = _read_data(
+                        data_type=type_key, path=v, read_params=t_params
+                    )
+                # Read data for nested data type
+                if isinstance(v, dict) and "path" in v:
+                    # Set path
+                    nested_path = v["path"]
+                    # Convert str to Path
+                    if not isinstance(nested_path, Path):
+                        nested_path = Path(nested_path)
+                    # Update path
+                    out[type_key][k]["path"] = nested_path
+                    # Set nested type key for easier access
+                    nested_type = f"{type_key}.{k}"
+                    logger.info(
+                        f"Reading {nested_type} from "
+                        f"{nested_path.absolute()!s}"
+                    )
+                    # Retrieve loading params for the nested data type
+                    nested_params = params.get(nested_type, {})
+                    # Read data
+                    out[type_key][k]["data"] = _read_data(
+                        data_type=nested_type,
+                        path=nested_path,
+                        read_params=nested_params,
+                    )
 
-            # Set file data for output
-            out[type_]["data"] = fread
             # Update metadata for step
-            self.update_meta(out[type_], "datareader")
+            self.update_meta(out[type_key], "datareader")
 
         return out
+
+
+def _read_data(
+    data_type: str, path: Path, read_params: Dict
+) -> Union[nib.Nifti1Image, pd.DataFrame, None]:
+    """Read data for data type.
+
+    Parameters
+    ----------
+    data_type : str
+        The data type being read.
+    path : pathlib.Path
+        The path to read data from.
+    read_params : dict
+        Parameters for reader function.
+
+    Returns
+    -------
+    nibabel.Nifti1Image or pandas.DataFrame or pandas.TextFileReader or None
+        The data loaded in memory if file type is known else None.
+
+    """
+    # Initialize variable for file data
+    fread = None
+    # Lowercase path
+    fname = path.name.lower()
+    # Loop through extensions to find the correct one
+    for ext, ftype in _extensions.items():
+        if fname.endswith(ext):
+            logger.info(f"{data_type} is of type {ftype}")
+            # Retrieve reader function
+            reader_func = _readers[ftype]["func"]
+            # Retrieve reader function params
+            reader_params = _readers[ftype]["params"]
+            # Update reader function params
+            if reader_params is not None:
+                read_params.update(reader_params)
+            logger.debug(f"Calling {reader_func!s} with {read_params}")
+            # Read data
+            fread = reader_func(path, **read_params)
+            break
+    # If no file data is found due to unknown extension
+    if fread is None:
+        logger.info(f"Unknown file type {path.absolute()!s}, skipping reading")
+
+    return fread

junifer/preprocess/base.py

@@ -146,8 +146,8 @@ class BasePreprocessor(ABC, PipelineStepMixin, UpdateMetaMixin):
             The computed result as dictionary.
         dict or None
             Extra "helper" data types as dictionary to add to the Junifer Data
-            object. For example, computed BOLD mask can be passed via this.
-            If no new "helper" data types is created, None is to be passed.
+            object. If no new "helper" data type(s) is(are) created, None is to
+            be passed.
 
         """
         raise_error(

junifer/preprocess/confounds/fmriprep_confound_remover.py

@@ -203,9 +203,7 @@ class fMRIPrepConfoundRemover(BasePreprocessor):
                 "include it in the future",
                 klass=ValueError,
             )
-        super().__init__(
-            on="BOLD", required_data_types=["BOLD", "BOLD_confounds"]
-        )
+        super().__init__(on="BOLD", required_data_types=["BOLD"])
 
     def get_valid_inputs(self) -> List[str]:
         """Get valid data types for input.
@@ -361,7 +359,7 @@ class fMRIPrepConfoundRemover(BasePreprocessor):
         Parameters
         ----------
         input : dict
-            Dictionary containing the ``BOLD_confounds`` value from the
+            Dictionary containing the ``BOLD.confounds`` value from the
             Junifer Data object.
 
         Returns
@@ -370,7 +368,6 @@ class fMRIPrepConfoundRemover(BasePreprocessor):
             Dataframe containing the relevant confounds.
 
         """
-
         confounds_format = input["format"]
         if confounds_format == "adhoc":
             self._map_adhoc_to_fmriprep(input)
@@ -416,50 +413,42 @@ class fMRIPrepConfoundRemover(BasePreprocessor):
     def _validate_data(
         self,
         input: Dict[str, Any],
-        extra_input: Optional[Dict[str, Any]] = None,
     ) -> None:
         """Validate input data.
 
         Parameters
         ----------
         input : dict
-            Dictionary containing the ``BOLD`` value from the
+            Dictionary containing the ``BOLD`` data from the
             Junifer Data object.
-        extra_input : dict, optional
-            Dictionary containing the rest of the Junifer Data object. Must
-            include the ``BOLD_confounds`` key.
 
         Raises
         ------
         ValueError
-            If ``extra_input`` is None or
-            if ``"BOLD_confounds"`` is not found in ``extra_input`` or
-            if ``"data"`` key is not found in ``"BOLD_confounds"`` or
-            if ``"data"`` is not pandas.DataFrame or
+            If ``"confounds"`` is not found in ``input`` or
+            if ``"data"`` key is not found in ``"input.confounds"`` or
+            if ``"input.confounds.data"`` is not pandas.DataFrame or
             if image time series and confounds have different lengths or
-            if ``"format"`` is not found in ``"BOLD_confounds"`` or
-            if ``format = "adhoc"`` and ``"mappings"`` key or ``"fmriprep"``
-            key or correct fMRIPrep mappings or required fMRIPrep mappings are
-            not found or if invalid confounds format is found.
+            if ``format = "adhoc"`` and ``"mappings"`` key is not found or
+            ``"fmriprep"`` key is not found in ``"mappings"`` or
+            ``"fmriprep"`` has incorrect fMRIPrep mappings or required
+            fMRIPrep mappings are not found or
+            if invalid confounds format is found.
 
         """
         # BOLD must be 4D niimg
         check_niimg_4d(input["data"])
-        # Check for extra inputs
-        if extra_input is None:
-            raise_error(
-                "No extra input provided, requires `BOLD_confounds` data type "
-                "in particular"
-            )
-        if "BOLD_confounds" not in extra_input:
-            raise_error("`BOLD_confounds` data type not provided")
-        if "data" not in extra_input["BOLD_confounds"]:
-            raise_error("`BOLD_confounds.data` not provided")
-        # Confounds must be a pandas.DataFrame
-        if not isinstance(extra_input["BOLD_confounds"]["data"], pd.DataFrame):
-            raise_error("`BOLD_confounds.data` must be a `pandas.DataFrame`")
-
-        confound_df = extra_input["BOLD_confounds"]["data"]
+        # Check for confound data
+        if "confounds" not in input:
+            raise_error("`BOLD.confounds` data type not provided")
+        if "data" not in input["confounds"]:
+            raise_error("`BOLD.confounds.data` not provided")
+        # Confounds must be a pandas.DataFrame;
+        # if extension is unknown, will not be read, which will give None
+        if not isinstance(input["confounds"]["data"], pd.DataFrame):
+            raise_error("`BOLD.confounds.data` must be a `pandas.DataFrame`")
+
+        confound_df = input["confounds"]["data"]
         bold_img = input["data"]
         if bold_img.get_fdata().shape[3] != len(confound_df):
             raise_error(
@@ -469,23 +458,19 @@ class fMRIPrepConfoundRemover(BasePreprocessor):
             )
 
         # Check format
-        if "format" not in extra_input["BOLD_confounds"]:
-            raise_error("`BOLD_confounds.format` not provided")
-        t_format = extra_input["BOLD_confounds"]["format"]
+        t_format = input["confounds"]["format"]
         if t_format == "adhoc":
-            if "mappings" not in extra_input["BOLD_confounds"]:
+            if "mappings" not in input["confounds"]:
                 raise_error(
-                    "`BOLD_confounds.mappings` need to be set when "
-                    "`BOLD_confounds.format == 'adhoc'`"
+                    "`BOLD.confounds.mappings` need to be set when "
+                    "`BOLD.confounds.format == 'adhoc'`"
                 )
-            if "fmriprep" not in extra_input["BOLD_confounds"]["mappings"]:
+            if "fmriprep" not in input["confounds"]["mappings"]:
                 raise_error(
-                    "`BOLD_confounds.mappings.fmriprep` need to be set when "
-                    "`BOLD_confounds.format == 'adhoc'`"
+                    "`BOLD.confounds.mappings.fmriprep` need to be set when "
+                    "`BOLD.confounds.format == 'adhoc'`"
                 )
-            fmriprep_mappings = extra_input["BOLD_confounds"]["mappings"][
-                "fmriprep"
-            ]
+            fmriprep_mappings = input["confounds"]["mappings"]["fmriprep"]
             wrong_names = [
                 x
                 for x in fmriprep_mappings.values()
@@ -525,22 +510,22 @@ class fMRIPrepConfoundRemover(BasePreprocessor):
         input : dict
             A single input from the Junifer Data object to preprocess.
         extra_input : dict, optional
-            The other fields in the Junifer Data object. Must include the
-            ``BOLD_confounds`` key.
+            The other fields in the Junifer Data object.
 
         Returns
         -------
         dict
-            The computed result as dictionary.
-        dict or None
-            If `self.masks` is not None, then the target data computed mask is
-            returned else None.
+            The computed result as dictionary. If `self.masks` is not None,
+            then the target data computed mask is updated for further steps.
+        None
+            Extra "helper" data types as dictionary to add to the Junifer Data
+            object.
 
         """
         # Validate data
-        self._validate_data(input, extra_input)
+        self._validate_data(input)
         # Pick confounds
-        confounds_df = self._pick_confounds(extra_input["BOLD_confounds"])  # type: ignore
+        confounds_df = self._pick_confounds(input["confounds"])  # type: ignore
         # Get BOLD data
         bold_img = input["data"]
         # Set t_r
@@ -553,7 +538,6 @@ class fMRIPrepConfoundRemover(BasePreprocessor):
             )
         # Set mask data
         mask_img = None
-        bold_mask_dict = None
         if self.masks is not None:
             logger.debug(f"Masking with {self.masks}")
             mask_img = get_mask(
@@ -561,15 +545,15 @@ class fMRIPrepConfoundRemover(BasePreprocessor):
             )
             # Return the BOLD mask and link it to the BOLD data type dict;
             # this allows to use "inherit" down the pipeline
-            if extra_input is not None:
-                logger.debug("Setting `BOLD.mask_item`")
-                input["mask_item"] = "BOLD_mask"
-                bold_mask_dict = {
-                    "BOLD_mask": {
+            logger.debug("Setting `BOLD.mask`")
+            input.update(
+                {
+                    "mask": {
                         "data": mask_img,
                         "space": input["space"],
                     }
                 }
+            )
         # Clean image
         logger.info("Cleaning image using nilearn")
         logger.debug(f"\tdetrend: {self.detrend}")
@@ -587,4 +571,4 @@ class fMRIPrepConfoundRemover(BasePreprocessor):
             mask_img=mask_img,
         )
 
-        return input, bold_mask_dict
+        return input, None