junifer 0.0.3.dev188__py3-none-any.whl → 0.0.4__py3-none-any.whl

junifer/datagrabber/base.py

@@ -37,15 +37,16 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
     def __init__(self, types: List[str], datadir: Union[str, Path]) -> None:
         # Validate types
         validate_types(types)
+        self.types = types
+
         # Convert str to Path
         if not isinstance(datadir, Path):
             datadir = Path(datadir)
+        self._datadir = datadir
 
         logger.debug("Initializing BaseDataGrabber")
         logger.debug(f"\t_datadir = {datadir}")
         logger.debug(f"\ttypes = {types}")
-        self._datadir = datadir
-        self.types = types
 
     def __iter__(self) -> Iterator:
         """Enable iterable support.
@@ -58,12 +59,14 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         """
         yield from self.get_elements()
 
-    def __getitem__(self, element: Union[str, Tuple]) -> Dict[str, Dict]:
+    def __getitem__(
+        self, element: Union[str, Tuple[str, ...]]
+    ) -> Dict[str, Dict]:
         """Enable indexing support.
 
         Parameters
         ----------
-        element : str or tuple
+        element : str or tuple of str
             The element to be indexed.
 
         Returns
@@ -74,12 +77,16 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
 
         """
         logger.info(f"Getting element {element}")
+        # Convert element to tuple if not already
         if not isinstance(element, tuple):
             element = (element,)
-        named_element = dict(zip(self.get_element_keys(), element))
+        # Zip through element keys and actual values to construct element
+        # access dictionary
+        named_element: Dict = dict(zip(self.get_element_keys(), element))
         logger.debug(f"Named element: {named_element}")
+        # Fetch element
         out = self.get_item(**named_element)
-
+        # Update metadata
         for _, t_val in out.items():
             self.update_meta(t_val, "datagrabber")
             t_val["meta"]["element"] = named_element
@@ -117,6 +124,51 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         """
         return self._datadir
 
+    def filter(self, selection: List[Union[str, Tuple[str]]]) -> Iterator:
+        """Filter elements to be grabbed.
+
+        Parameters
+        ----------
+        selection : list of str or tuple
+            The list of partial element key values to filter using.
+
+        Yields
+        ------
+        object
+            An element that can be indexed by the DataGrabber.
+
+        """
+
+        def filter_func(element: Union[str, Tuple[str]]) -> bool:
+            """Filter element based on selection.
+
+            Parameters
+            ----------
+            element : str or tuple of str
+                The element to be filtered.
+
+            Returns
+            -------
+            bool
+                If the element passes the filter or not.
+
+            """
+            # Convert element to tuple
+            if not isinstance(element, tuple):
+                element = (element,)
+            # Filter based on selection kind
+            if isinstance(selection[0], str):
+                for opt in selection:
+                    if opt in element:
+                        return True
+            elif isinstance(selection[0], tuple):
+                for opt in selection:
+                    if set(opt).issubset(element):
+                        return True
+            return False
+
+        yield from filter(filter_func, self.get_elements())
+
     @abstractmethod
     def get_element_keys(self) -> List[str]:
         """Get element keys.
@@ -133,10 +185,10 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         raise_error(
             msg="Concrete classes need to implement get_element_keys().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     @abstractmethod
-    def get_elements(self) -> List:
+    def get_elements(self) -> List[Union[str, Tuple[str]]]:
         """Get elements.
 
         Returns
@@ -150,7 +202,7 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         raise_error(
             msg="Concrete classes need to implement get_elements().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover
 
     @abstractmethod
     def get_item(self, **element: Dict) -> Dict[str, Dict]:
@@ -171,4 +223,4 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         raise_error(
             msg="Concrete classes need to implement get_item().",
             klass=NotImplementedError,
-        )
+        )  # pragma: no cover

junifer/datagrabber/datalad_base.py

@@ -17,12 +17,10 @@ import datalad.api as dl
 from datalad.support.exceptions import IncompleteResultsError
 from datalad.support.gitrepo import GitRepo
 
-from ..api.decorators import register_datagrabber
 from ..utils import logger, raise_error, warn_with_log
 from .base import BaseDataGrabber
 
 
-@register_datagrabber
 class DataladDataGrabber(BaseDataGrabber):
     """Abstract base class for datalad-based data fetching.
 

junifer/datagrabber/dmcc13_benchmark.py

@@ -0,0 +1,372 @@
+"""Provide concrete implementation for DMCC13Benchmark DataGrabber."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+from itertools import product
+from pathlib import Path
+from typing import Dict, List, Union
+
+from ..api.decorators import register_datagrabber
+from ..utils import raise_error
+from .pattern_datalad import PatternDataladDataGrabber
+
+
+__all__ = ["DMCC13Benchmark"]
+
+
+@register_datagrabber
+class DMCC13Benchmark(PatternDataladDataGrabber):
+    """Concrete implementation for datalad-based data fetching of DMCC13.
+
+    Parameters
+    ----------
+    datadir : str or Path or None, optional
+        The directory where the datalad dataset will be cloned. If None,
+        the datalad dataset will be cloned into a temporary directory
+        (default None).
+    types: {"BOLD", "BOLD_confounds", "T1w", "VBM_CSF", "VBM_GM", \
+           "VBM_WM"} or a list of the options, optional
+        DMCC data types. If None, all available data types are selected.
+        (default None).
+    sessions: {"ses-wave1bas", "ses-wave1pro", "ses-wave1rea"} or list of \
+            the options, optional
+        DMCC sessions. If None, all available sessions are selected
+        (default None).
+    tasks: {"Rest", "Axcpt", "Cuedts", "Stern", "Stroop"} or \
+            list of the options, optional
+        DMCC task sessions. If None, all available task sessions are selected
+        (default None).
+    phase_encodings : {"AP", "PA"} or list of the options, optional
+        DMCC phase encoding directions. If None, all available phase encodings
+        are selected (default None).
+    runs : {"1", "2"} or list of the options, optional
+        DMCC runs. If None, all available runs are selected (default None).
+    native_t1w : bool, optional
+        Whether to use T1w in native space (default False).
+
+    Raises
+    ------
+    ValueError
+        If invalid value is passed for:
+         * ``sessions``
+         * ``tasks``
+         * ``phase_encodings``
+         * ``runs``
+
+    """
+
+    def __init__(
+        self,
+        datadir: Union[str, Path, None] = None,
+        types: Union[str, List[str], None] = None,
+        sessions: Union[str, List[str], None] = None,
+        tasks: Union[str, List[str], None] = None,
+        phase_encodings: Union[str, List[str], None] = None,
+        runs: Union[str, List[str], None] = None,
+        native_t1w: bool = False,
+    ) -> None:
+        # Declare all sessions
+        all_sessions = [
+            "ses-wave1bas",
+            "ses-wave1pro",
+            "ses-wave1rea",
+        ]
+        # Set default sessions
+        if sessions is None:
+            sessions = all_sessions
+        else:
+            # Convert single session into list
+            if isinstance(sessions, str):
+                sessions = [sessions]
+            # Verify valid sessions
+            for s in sessions:
+                if s not in all_sessions:
+                    raise_error(
+                        f"{s} is not a valid session in the DMCC dataset"
+                    )
+        self.sessions = sessions
+        # Declare all tasks
+        all_tasks = [
+            "Rest",
+            "Axcpt",
+            "Cuedts",
+            "Stern",
+            "Stroop",
+        ]
+        # Set default tasks
+        if tasks is None:
+            tasks = all_tasks
+        else:
+            # Convert single task into list
+            if isinstance(tasks, str):
+                tasks = [tasks]
+            # Verify valid tasks
+            for t in tasks:
+                if t not in all_tasks:
+                    raise_error(f"{t} is not a valid task in the DMCC dataset")
+        self.tasks = tasks
+        # Declare all phase encodings
+        all_phase_encodings = ["AP", "PA"]
+        # Set default phase encodings
+        if phase_encodings is None:
+            phase_encodings = all_phase_encodings
+        else:
+            # Convert single phase encoding into list
+            if isinstance(phase_encodings, str):
+                phase_encodings = [phase_encodings]
+            # Verify valid phase encodings
+            for p in phase_encodings:
+                if p not in all_phase_encodings:
+                    raise_error(
+                        f"{p} is not a valid phase encoding in the DMCC "
+                        "dataset"
+                    )
+        self.phase_encodings = phase_encodings
+        # Declare all runs
+        all_runs = ["1", "2"]
+        # Set default runs
+        if runs is None:
+            runs = all_runs
+        else:
+            # Convert single run into list
+            if isinstance(runs, str):
+                runs = [runs]
+            # Verify valid runs
+            for r in runs:
+                if r not in all_runs:
+                    raise_error(f"{r} is not a valid run in the DMCC dataset")
+        self.runs = runs
+        # The patterns
+        patterns = {
+            "BOLD": {
+                "pattern": (
+                    "derivatives/fmriprep-1.3.2/{subject}/{session}/"
+                    "func/{subject}_{session}_task-{task}_acq-mb4"
+                    "{phase_encoding}_run-{run}_"
+                    "space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz"
+                ),
+                "space": "MNI152NLin2009cAsym",
+                "mask_item": "BOLD_mask",
+            },
+            "BOLD_confounds": {
+                "pattern": (
+                    "derivatives/fmriprep-1.3.2/{subject}/{session}/"
+                    "func/{subject}_{session}_task-{task}_acq-mb4"
+                    "{phase_encoding}_run-{run}_desc-confounds_regressors.tsv"
+                ),
+                "format": "fmriprep",
+            },
+            "BOLD_mask": {
+                "pattern": (
+                    "derivatives/fmriprep-1.3.2/{subject}/{session}/"
+                    "/func/{subject}_{session}_task-{task}_acq-mb4"
+                    "{phase_encoding}_run-{run}_"
+                    "space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz"
+                ),
+                "space": "MNI152NLin2009cAsym",
+            },
+            "T1w": {
+                "pattern": (
+                    "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                    "{subject}_space-MNI152NLin2009cAsym_desc-preproc_T1w.nii.gz"
+                ),
+                "space": "MNI152NLin2009cAsym",
+                "mask_item": "T1w_mask",
+            },
+            "T1w_mask": {
+                "pattern": (
+                    "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                    "{subject}_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz"
+                ),
+                "space": "MNI152NLin2009cAsym",
+            },
+            "VBM_CSF": {
+                "pattern": (
+                    "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                    "{subject}_space-MNI152NLin2009cAsym_label-CSF_probseg.nii.gz"
+                ),
+                "space": "MNI152NLin2009cAsym",
+            },
+            "VBM_GM": {
+                "pattern": (
+                    "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                    "{subject}_space-MNI152NLin2009cAsym_label-GM_probseg.nii.gz"
+                ),
+                "space": "MNI152NLin2009cAsym",
+            },
+            "VBM_WM": {
+                "pattern": (
+                    "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                    "{subject}_space-MNI152NLin2009cAsym_label-WM_probseg.nii.gz"
+                ),
+                "space": "MNI152NLin2009cAsym",
+            },
+        }
+        # Use native T1w assets
+        self.native_t1w = False
+        if native_t1w:
+            self.native_t1w = True
+            patterns.update(
+                {
+                    "T1w": {
+                        "pattern": (
+                            "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                            "{subject}_desc-preproc_T1w.nii.gz"
+                        ),
+                        "space": "native",
+                        "mask_item": "T1w_mask",
+                    },
+                    "T1w_mask": {
+                        "pattern": (
+                            "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                            "{subject}_desc-brain_mask.nii.gz"
+                        ),
+                        "space": "native",
+                    },
+                    "Warp": {
+                        "pattern": (
+                            "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                            "{subject}_from-MNI152NLin2009cAsym_to-T1w_"
+                            "mode-image_xfm.h5"
+                        ),
+                        "src": "MNI152NLin2009cAsym",
+                        "dst": "native",
+                    },
+                }
+            )
+        # Set default types
+        if types is None:
+            types = list(patterns.keys())
+        # Convert single type into list
+        else:
+            if not isinstance(types, list):
+                types = [types]
+        # The replacements
+        replacements = ["subject", "session", "task", "phase_encoding", "run"]
+        uri = "https://github.com/OpenNeuroDatasets/ds003452.git"
+        super().__init__(
+            types=types,
+            datadir=datadir,
+            uri=uri,
+            patterns=patterns,
+            replacements=replacements,
+            confounds_format="fmriprep",
+        )
+
+    def get_item(
+        self,
+        subject: str,
+        session: str,
+        task: str,
+        phase_encoding: str,
+        run: str,
+    ) -> Dict:
+        """Index one element in the dataset.
+
+        Parameters
+        ----------
+        subject : str
+            The subject ID.
+        session : {"ses-wave1bas", "ses-wave1pro", "ses-wave1rea"}
+            The session to get.
+        task : {"Rest", "Axcpt", "Cuedts", "Stern", "Stroop"}
+            The task to get.
+        phase_encoding : {"AP", "PA"}
+            The phase encoding to get.
+        run : {"1", "2"}
+            The run to get.
+
+        Returns
+        -------
+        out : dict
+            Dictionary of paths for each type of data required for the
+            specified element.
+
+        """
+        # Format run
+        if phase_encoding == "AP":
+            run = "1"
+        else:
+            run = "2"
+        # Fetch item
+        out = super().get_item(
+            subject=subject,
+            session=session,
+            task=task,
+            phase_encoding=phase_encoding,
+            run=run,
+        )
+        if out.get("BOLD"):
+            out["BOLD"]["mask_item"] = "BOLD_mask"
+            # Add space information
+            out["BOLD"].update({"space": "MNI152NLin2009cAsym"})
+        if out.get("T1w"):
+            out["T1w"]["mask_item"] = "T1w_mask"
+            # Add space information
+            if self.native_t1w:
+                out["T1w"].update({"space": "native"})
+            else:
+                out["T1w"].update({"space": "MNI152NLin2009cAsym"})
+        if out.get("Warp"):
+            # Add source space information
+            out["Warp"].update({"src": "MNI152NLin2009cAsym"})
+        return out
+
+    def get_elements(self) -> List:
+        """Implement fetching list of subjects in the dataset.
+
+        Returns
+        -------
+        list of str
+            The list of subjects in the dataset.
+
+        """
+        subjects = [
+            "sub-f1031ax",
+            "sub-f1552xo",
+            "sub-f1659oa",
+            "sub-f1670rz",
+            "sub-f1951tt",
+            "sub-f3300jh",
+            "sub-f3720ca",
+            "sub-f5004cr",
+            "sub-f5407sl",
+            "sub-f5416zj",
+            "sub-f8113do",
+            "sub-f8570ui",
+            "sub-f9057kp",
+        ]
+        elems = []
+        # For wave1bas session
+        for subject, session, task, phase_encoding in product(
+            subjects,
+            ["ses-wave1bas"],
+            self.tasks,
+            self.phase_encodings,
+        ):
+            if phase_encoding == "AP":
+                run = "1"
+            else:
+                run = "2"
+            # Bypass for f1951tt not having run 2 for Rest
+            if subject == "sub-f1951tt" and task == "Rest" and run == "2":
+                continue
+            elems.append((subject, session, task, phase_encoding, run))
+        # For other sessions
+        for subject, session, task, phase_encoding in product(
+            subjects,
+            ["ses-wave1pro", "ses-wave1rea"],
+            ["Rest"],
+            self.phase_encodings,
+        ):
+            if phase_encoding == "AP":
+                run = "1"
+            else:
+                run = "2"
+            # Bypass for f5416zj for not having wave1rea session
+            if subject == "sub-f5416zj" and session == "ses-wave1rea":
+                continue
+            elems.append((subject, session, task, phase_encoding, run))
+
+        return elems

junifer/datagrabber/hcp1200/datalad_hcp1200.py

@@ -37,6 +37,11 @@ class DataladHCP1200(DataladDataGrabber, HCP1200):
         Only "REST1" and "REST2" tasks are available with ICA+FIX (default
         False).
 
+    Raises
+    ------
+    ValueError
+        If invalid value is passed for ``tasks`` or ``phase_encodings``.
+
     """
 
     def __init__(

junifer/datagrabber/hcp1200/hcp1200.py

@@ -34,8 +34,11 @@ class HCP1200(PatternDataGrabber):
         Whether to retrieve data that was processed with ICA+FIX.
         Only "REST1" and "REST2" tasks are available with ICA+FIX (default
         False).
-    **kwargs
-        Keyword arguments passed to superclass.
+
+    Raises
+    ------
+    ValueError
+        If invalid value is passed for ``tasks`` or ``phase_encodings``.
 
     """
 
@@ -45,7 +48,6 @@ class HCP1200(PatternDataGrabber):
         tasks: Union[str, List[str], None] = None,
         phase_encodings: Union[str, List[str], None] = None,
         ica_fix: bool = False,
-        **kwargs,
     ) -> None:
         # All tasks
         all_tasks = [
@@ -74,6 +76,7 @@ class HCP1200(PatternDataGrabber):
                         f"Valid task values can be any or all of {all_tasks}."
                     )
             self.tasks: List[str] = tasks
+
         # All phase encodings
         all_phase_encodings = ["LR", "RL"]
         # Set phase encodings
@@ -90,6 +93,7 @@ class HCP1200(PatternDataGrabber):
                     "Valid phase encoding can be any or all of "
                     f"{all_phase_encodings}."
                 )
+        self.phase_encodings = phase_encodings
 
         if ica_fix:
             if not all(task in ["REST1", "REST2"] for task in self.tasks):
@@ -97,16 +101,31 @@ class HCP1200(PatternDataGrabber):
                     "ICA+FIX is only available for 'REST1' and 'REST2' tasks."
                 )
         suffix = "_hp2000_clean" if ica_fix else ""
+
         # The types of data
-        types = ["BOLD"]
+        types = ["BOLD", "T1w", "Warp"]
         # The patterns
         patterns = {
-            "BOLD": (
-                "{subject}/MNINonLinear/Results/"
-                "{task}_{phase_encoding}/"
-                "{task}_{phase_encoding}"
-                f"{suffix}.nii.gz"
-            )
+            "BOLD": {
+                "pattern": (
+                    "{subject}/MNINonLinear/Results/"
+                    "{task}_{phase_encoding}/"
+                    "{task}_{phase_encoding}"
+                    f"{suffix}.nii.gz"
+                ),
+                "space": "MNI152NLin6Asym",
+            },
+            "T1w": {
+                "pattern": "{subject}/T1w/T1w_acpc_dc_restore.nii.gz",
+                "space": "native",
+            },
+            "Warp": {
+                "pattern": (
+                    "{subject}/MNINonLinear/xfms/standard2acpc_dc.nii.gz"
+                ),
+                "src": "MNI152NLin6Asym",
+                "dst": "native",
+            },
         }
         # The replacements
         replacements = ["subject", "task", "phase_encoding"]
@@ -116,7 +135,6 @@ class HCP1200(PatternDataGrabber):
             patterns=patterns,
             replacements=replacements,
         )
-        self.phase_encodings = phase_encodings
 
     def get_item(self, subject: str, task: str, phase_encoding: str) -> Dict:
         """Implement single element indexing in the database.
@@ -144,10 +162,9 @@ class HCP1200(PatternDataGrabber):
         else:
             new_task = f"tfMRI_{task}"
 
-        out = super().get_item(
+        return super().get_item(
             subject=subject, task=new_task, phase_encoding=phase_encoding
         )
-        return out
 
     def get_elements(self) -> List:
         """Implement fetching list of elements in the dataset.