junifer 0.0.6.dev227__py3-none-any.whl → 0.0.6.dev252__py3-none-any.whl

junifer/datagrabber/aomic/tests/test_piop2.py

@@ -7,7 +7,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 import pytest
 
@@ -35,8 +35,8 @@ URI = "https://gin.g-node.org/juaml/datalad-example-aomicpiop2"
 )
 def test_DataladAOMICPIOP2(
     type_: str,
-    nested_types: Optional[List[str]],
-    tasks: Optional[List[str]],
+    nested_types: Optional[list[str]],
+    tasks: Optional[list[str]],
 ) -> None:
     """Test DataladAOMICPIOP2 DataGrabber.
 
@@ -97,7 +97,7 @@ def test_DataladAOMICPIOP2(
     ],
 )
 def test_DataladAOMICPIOP2_partial_data_access(
-    types: Union[str, List[str]],
+    types: Union[str, list[str]],
 ) -> None:
     """Test DataladAOMICPIOP2 DataGrabber partial data access.
 

junifer/datagrabber/base.py

@@ -6,8 +6,9 @@
 # License: AGPL
 
 from abc import ABC, abstractmethod
+from collections.abc import Iterator
 from pathlib import Path
-from typing import Dict, Iterator, List, Tuple, Union
+from typing import Union
 
 from ..pipeline import UpdateMetaMixin
 from ..utils import logger, raise_error
@@ -36,7 +37,7 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
 
     """
 
-    def __init__(self, types: List[str], datadir: Union[str, Path]) -> None:
+    def __init__(self, types: list[str], datadir: Union[str, Path]) -> None:
         # Validate types
         if not isinstance(types, list):
             raise_error(msg="`types` must be a list", klass=TypeError)
@@ -67,8 +68,8 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         yield from self.get_elements()
 
     def __getitem__(
-        self, element: Union[str, Tuple[str, ...]]
-    ) -> Dict[str, Dict]:
+        self, element: Union[str, tuple[str, ...]]
+    ) -> dict[str, dict]:
         """Enable indexing support.
 
         Parameters
@@ -89,14 +90,19 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
             element = (element,)
         # Zip through element keys and actual values to construct element
         # access dictionary
-        named_element: Dict = dict(zip(self.get_element_keys(), element))
+        named_element: dict = dict(zip(self.get_element_keys(), element))
         logger.debug(f"Named element: {named_element}")
         # Fetch element
         out = self.get_item(**named_element)
         # Update metadata
         for _, t_val in out.items():
             self.update_meta(t_val, "datagrabber")
-            t_val["meta"]["element"] = named_element
+            # Conditional for list dtype vals like Warp
+            if isinstance(t_val, list):
+                for entry in t_val:
+                    entry["meta"]["element"] = named_element
+            else:
+                t_val["meta"]["element"] = named_element
 
         return out
 
@@ -108,7 +114,7 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         """Context exit."""
         return None
 
-    def get_types(self) -> List[str]:
+    def get_types(self) -> list[str]:
         """Get types.
 
         Returns
@@ -131,7 +137,7 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         """
         return self._datadir
 
-    def filter(self, selection: List[Union[str, Tuple[str]]]) -> Iterator:
+    def filter(self, selection: list[Union[str, tuple[str]]]) -> Iterator:
         """Filter elements to be grabbed.
 
         Parameters
@@ -146,7 +152,7 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
 
         """
 
-        def filter_func(element: Union[str, Tuple[str]]) -> bool:
+        def filter_func(element: Union[str, tuple[str]]) -> bool:
             """Filter element based on selection.
 
             Parameters
@@ -177,7 +183,7 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         yield from filter(filter_func, self.get_elements())
 
     @abstractmethod
-    def get_element_keys(self) -> List[str]:
+    def get_element_keys(self) -> list[str]:
         """Get element keys.
 
         For each item in the ``element`` tuple passed to ``__getitem__()``,
@@ -195,7 +201,7 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         )  # pragma: no cover
 
     @abstractmethod
-    def get_elements(self) -> List[Union[str, Tuple[str]]]:
+    def get_elements(self) -> list[Union[str, tuple[str]]]:
         """Get elements.
 
         Returns
@@ -212,7 +218,7 @@ class BaseDataGrabber(ABC, UpdateMetaMixin):
         )  # pragma: no cover
 
     @abstractmethod
-    def get_item(self, **element: Dict) -> Dict[str, Dict]:
+    def get_item(self, **element: dict) -> dict[str, dict]:
         """Get the specified item from the dataset.
 
         Parameters

junifer/datagrabber/datalad_base.py

@@ -9,7 +9,7 @@ import atexit
 import os
 import tempfile
 from pathlib import Path
-from typing import Dict, Optional, Tuple, Union
+from typing import Optional, Union
 
 import datalad
 import datalad.api as dl
@@ -165,7 +165,7 @@ class DataladDataGrabber(BaseDataGrabber):
             raise_error("Could not get dataset ID from remote")
         return remote_id
 
-    def _dataset_get(self, out: Dict) -> Dict:
+    def _dataset_get(self, out: dict) -> dict:
         """Get the dataset found from the path in ``out``.
 
         Parameters
@@ -181,14 +181,21 @@ class DataladDataGrabber(BaseDataGrabber):
         """
         to_get = []
         for type_val in out.values():
-            # Iterate to check for nested "types" like mask
-            for k, v in type_val.items():
-                # Add base data type path
-                if k == "path":
-                    to_get.append(v)
-                # Add nested data type path
-                if isinstance(v, dict) and "path" in v:
-                    to_get.append(v["path"])
+            # Conditional for list dtype vals like Warp
+            if isinstance(type_val, list):
+                for entry in type_val:
+                    for k, v in entry.items():
+                        if k == "path":
+                            to_get.append(v)
+            else:
+                # Iterate to check for nested "types" like mask
+                for k, v in type_val.items():
+                    # Add base data type path
+                    if k == "path":
+                        to_get.append(v)
+                    # Add nested data type path
+                    if isinstance(v, dict) and "path" in v:
+                        to_get.append(v["path"])
 
         if len(to_get) > 0:
             logger.debug(f"Getting {len(to_get)} files using datalad:")
@@ -276,7 +283,7 @@ class DataladDataGrabber(BaseDataGrabber):
                 logger.debug(f"Dropping {f}")
                 self._dataset.drop(f, result_renderer="disabled")
 
-    def __getitem__(self, element: Union[str, Tuple]) -> Dict:
+    def __getitem__(self, element: Union[str, tuple]) -> dict:
         """Implement single element indexing in the Datalad database.
 
         It will first obtain the paths from the parent class and then

junifer/datagrabber/dmcc13_benchmark.py

@@ -5,7 +5,7 @@
 
 from itertools import product
 from pathlib import Path
-from typing import Dict, List, Union
+from typing import Union
 
 from ..api.decorators import register_datagrabber
 from ..utils import raise_error
@@ -59,11 +59,11 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
     def __init__(
         self,
         datadir: Union[str, Path, None] = None,
-        types: Union[str, List[str], None] = None,
-        sessions: Union[str, List[str], None] = None,
-        tasks: Union[str, List[str], None] = None,
-        phase_encodings: Union[str, List[str], None] = None,
-        runs: Union[str, List[str], None] = None,
+        types: Union[str, list[str], None] = None,
+        sessions: Union[str, list[str], None] = None,
+        tasks: Union[str, list[str], None] = None,
+        phase_encodings: Union[str, list[str], None] = None,
+        runs: Union[str, list[str], None] = None,
         native_t1w: bool = False,
     ) -> None:
         # Declare all sessions
@@ -150,7 +150,7 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
                 "mask": {
                     "pattern": (
                         "derivatives/fmriprep-1.3.2/{subject}/{session}/"
-                        "/func/{subject}_{session}_task-{task}_acq-mb4"
+                        "func/{subject}_{session}_task-{task}_acq-mb4"
                         "{phase_encoding}_run-{run}_"
                         "space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz"
                     ),
@@ -221,15 +221,28 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
                             "space": "native",
                         },
                     },
-                    "Warp": {
-                        "pattern": (
-                            "derivatives/fmriprep-1.3.2/{subject}/anat/"
-                            "{subject}_from-MNI152NLin2009cAsym_to-T1w_"
-                            "mode-image_xfm.h5"
-                        ),
-                        "src": "MNI152NLin2009cAsym",
-                        "dst": "native",
-                    },
+                    "Warp": [
+                        {
+                            "pattern": (
+                                "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                                "{subject}_from-MNI152NLin2009cAsym_to-T1w_"
+                                "mode-image_xfm.h5"
+                            ),
+                            "src": "MNI152NLin2009cAsym",
+                            "dst": "native",
+                            "warper": "ants",
+                        },
+                        {
+                            "pattern": (
+                                "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                                "{subject}_from-T1w_to-MNI152NLin2009cAsym_"
+                                "mode-image_xfm.h5"
+                            ),
+                            "src": "native",
+                            "dst": "MNI152NLin2009cAsym",
+                            "warper": "ants",
+                        },
+                    ],
                 }
             )
         # Set default types
@@ -258,7 +271,7 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
         task: str,
         phase_encoding: str,
         run: str,
-    ) -> Dict:
+    ) -> dict:
         """Index one element in the dataset.
 
         Parameters
@@ -296,7 +309,7 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
         )
         return out
 
-    def get_elements(self) -> List:
+    def get_elements(self) -> list:
         """Implement fetching list of subjects in the dataset.
 
         Returns

junifer/datagrabber/hcp1200/datalad_hcp1200.py

@@ -6,7 +6,7 @@
 # License: AGPL
 
 from pathlib import Path
-from typing import List, Union
+from typing import Union
 
 from junifer.datagrabber.datalad_base import DataladDataGrabber
 
@@ -50,8 +50,8 @@ class DataladHCP1200(DataladDataGrabber, HCP1200):
     def __init__(
         self,
         datadir: Union[str, Path, None] = None,
-        tasks: Union[str, List[str], None] = None,
-        phase_encodings: Union[str, List[str], None] = None,
+        tasks: Union[str, list[str], None] = None,
+        phase_encodings: Union[str, list[str], None] = None,
         ica_fix: bool = False,
     ) -> None:
         uri = (

junifer/datagrabber/hcp1200/hcp1200.py

@@ -7,7 +7,7 @@
 
 from itertools import product
 from pathlib import Path
-from typing import Dict, List, Union
+from typing import Union
 
 from ...api.decorators import register_datagrabber
 from ...utils import raise_error
@@ -48,8 +48,8 @@ class HCP1200(PatternDataGrabber):
     def __init__(
         self,
         datadir: Union[str, Path],
-        tasks: Union[str, List[str], None] = None,
-        phase_encodings: Union[str, List[str], None] = None,
+        tasks: Union[str, list[str], None] = None,
+        phase_encodings: Union[str, list[str], None] = None,
         ica_fix: bool = False,
     ) -> None:
         # All tasks
@@ -66,10 +66,10 @@ class HCP1200(PatternDataGrabber):
         ]
         # Set default tasks
         if tasks is None:
-            self.tasks: List[str] = all_tasks
+            self.tasks: list[str] = all_tasks
         # Convert single task into list
         else:
-            if not isinstance(tasks, List):
+            if not isinstance(tasks, list):
                 tasks = [tasks]
             # Check for invalid task(s)
             for task in tasks:
@@ -78,7 +78,7 @@ class HCP1200(PatternDataGrabber):
                         f"'{task}' is not a valid HCP-YA fMRI task input. "
                         f"Valid task values can be any or all of {all_tasks}."
                     )
-            self.tasks: List[str] = tasks
+            self.tasks: list[str] = tasks
 
         # All phase encodings
         all_phase_encodings = ["LR", "RL"]
@@ -122,13 +122,24 @@ class HCP1200(PatternDataGrabber):
                 "pattern": "{subject}/T1w/T1w_acpc_dc_restore.nii.gz",
                 "space": "native",
             },
-            "Warp": {
-                "pattern": (
-                    "{subject}/MNINonLinear/xfms/standard2acpc_dc.nii.gz"
-                ),
-                "src": "MNI152NLin6Asym",
-                "dst": "native",
-            },
+            "Warp": [
+                {
+                    "pattern": (
+                        "{subject}/MNINonLinear/xfms/standard2acpc_dc.nii.gz"
+                    ),
+                    "src": "MNI152NLin6Asym",
+                    "dst": "native",
+                    "warper": "fsl",
+                },
+                {
+                    "pattern": (
+                        "{subject}/MNINonLinear/xfms/acpc_dc2standard.nii.gz"
+                    ),
+                    "src": "native",
+                    "dst": "MNI152NLin6Asym",
+                    "warper": "fsl",
+                },
+            ],
         }
         # The replacements
         replacements = ["subject", "task", "phase_encoding"]
@@ -139,7 +150,7 @@ class HCP1200(PatternDataGrabber):
             replacements=replacements,
         )
 
-    def get_item(self, subject: str, task: str, phase_encoding: str) -> Dict:
+    def get_item(self, subject: str, task: str, phase_encoding: str) -> dict:
         """Implement single element indexing in the database.
 
         Parameters
@@ -169,7 +180,7 @@ class HCP1200(PatternDataGrabber):
             subject=subject, task=new_task, phase_encoding=phase_encoding
         )
 
-    def get_elements(self) -> List:
+    def get_elements(self) -> list:
         """Implement fetching list of elements in the dataset.
 
         Returns

junifer/datagrabber/hcp1200/tests/test_hcp1200.py

@@ -5,8 +5,9 @@
 
 import shutil
 import tempfile
+from collections.abc import Iterable
 from pathlib import Path
-from typing import Iterable, Optional
+from typing import Optional
 
 import pytest
 

junifer/datagrabber/multiple.py

@@ -5,7 +5,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Dict, List, Tuple, Union
+from typing import Union
 
 from ..api.decorators import register_datagrabber
 from ..typing import DataGrabberLike
@@ -38,7 +38,7 @@ class MultipleDataGrabber(BaseDataGrabber):
 
     """
 
-    def __init__(self, datagrabbers: List[DataGrabberLike], **kwargs) -> None:
+    def __init__(self, datagrabbers: list[DataGrabberLike], **kwargs) -> None:
         # Check datagrabbers consistency
         # Check for same element keys
         first_keys = datagrabbers[0].get_element_keys()
@@ -79,7 +79,7 @@ class MultipleDataGrabber(BaseDataGrabber):
                 )
         self._datagrabbers = datagrabbers
 
-    def __getitem__(self, element: Union[str, Tuple]) -> Dict:
+    def __getitem__(self, element: Union[str, tuple]) -> dict:
         """Implement indexing.
 
         Parameters
@@ -127,7 +127,7 @@ class MultipleDataGrabber(BaseDataGrabber):
             dg.__exit__(exc_type, exc_value, exc_traceback)
 
     # TODO: return type should be List[List[str]], but base type is List[str]
-    def get_types(self) -> List[str]:
+    def get_types(self) -> list[str]:
         """Get types.
 
         Returns
@@ -139,7 +139,7 @@ class MultipleDataGrabber(BaseDataGrabber):
         types = [x for dg in self._datagrabbers for x in dg.get_types()]
         return types
 
-    def get_element_keys(self) -> List[str]:
+    def get_element_keys(self) -> list[str]:
         """Get element keys.
 
         For each item in the ``element`` tuple passed to ``__getitem__()``,
@@ -153,7 +153,7 @@ class MultipleDataGrabber(BaseDataGrabber):
         """
         return self._datagrabbers[0].get_element_keys()
 
-    def get_elements(self) -> List:
+    def get_elements(self) -> list:
         """Get elements.
 
         Returns
@@ -171,7 +171,7 @@ class MultipleDataGrabber(BaseDataGrabber):
             elements.intersection_update(s)
         return list(elements)
 
-    def get_item(self, **_: Dict) -> Dict[str, Dict]:
+    def get_item(self, **_: dict) -> dict[str, dict]:
         """Get the specified item from the dataset.
 
         Parameters

junifer/datagrabber/pattern.py

@@ -8,7 +8,7 @@
 import re
 from copy import deepcopy
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Optional, Union
 
 import numpy as np
 
@@ -89,7 +89,7 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
           .. code-block:: none
 
             {
-              "mandatory": ["pattern", "src", "dst"],
+              "mandatory": ["pattern", "src", "dst", "warper"],
               "optional": []
             }
 
@@ -127,11 +127,22 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
                 "pattern": "...",
                 "space": "...",
               },
-              "Warp": {
-                "pattern": "...",
-                "src": "...",
-                "dst": "...",
-              }
+          }
+
+        except ``Warp``, which needs to be a list of dictionaries as there can
+        be multiple spaces to warp (for example, with fMRIPrep):
+
+        .. code-block:: none
+
+          {
+              "Warp": [
+                {
+                  "pattern": "...",
+                  "src": "...",
+                  "dst": "...",
+                  "warper": "...",
+                },
+              ],
           }
 
         taken from :class:`.HCP1200`.
@@ -159,9 +170,9 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
 
     def __init__(
         self,
-        types: List[str],
-        patterns: Dict[str, Dict[str, str]],
-        replacements: Union[List[str], str],
+        types: list[str],
+        patterns: dict[str, dict[str, str]],
+        replacements: Union[list[str], str],
         datadir: Union[str, Path],
         confounds_format: Optional[str] = None,
         partial_pattern_ok: bool = False,
@@ -204,7 +215,7 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
 
     def _replace_patterns_regex(
         self, pattern: str
-    ) -> Tuple[str, str, List[str]]:
+    ) -> tuple[str, str, list[str]]:
         """Replace the patterns in ``pattern`` with the named groups.
 
         It allows elements to be obtained from the filesystem.
@@ -250,7 +261,7 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
 
         return re_pattern, glob_pattern, t_replacements
 
-    def _replace_patterns_glob(self, element: Dict, pattern: str) -> str:
+    def _replace_patterns_glob(self, element: dict, pattern: str) -> str:
         """Replace ``pattern`` with the ``element`` so it can be globbed.
 
         Parameters
@@ -283,7 +294,7 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
         return pattern.format(**element)
 
     def _get_path_from_patterns(
-        self, element: Dict, pattern: str, data_type: str
+        self, element: dict, pattern: str, data_type: str
     ) -> Path:
         """Get path from resolved patterns.
 
@@ -340,7 +351,7 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
 
         return path
 
-    def get_element_keys(self) -> List[str]:
+    def get_element_keys(self) -> list[str]:
         """Get element keys.
 
         For each item in the "element" tuple, this functions returns the
@@ -355,7 +366,7 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
         """
         return self.replacements
 
-    def get_item(self, **element: str) -> Dict[str, Dict]:
+    def get_item(self, **element: str) -> dict[str, dict]:
         """Implement single element indexing for the datagrabber.
 
         This method constructs a real path to the requested item's data, by
@@ -380,46 +391,65 @@ class PatternDataGrabber(BaseDataGrabber, PatternValidationMixin):
             t_pattern = self.patterns[t_type]
             # Copy data type dictionary in output
             out[t_type] = deepcopy(t_pattern)
-            # Iterate to check for nested "types" like mask
-            for k, v in t_pattern.items():
-                # Resolve pattern for base data type
-                if k == "pattern":
-                    logger.info(f"Resolving path from pattern for {t_type}")
+            # Conditional for list dtype vals like Warp
+            if isinstance(t_pattern, list):
+                for idx, entry in enumerate(t_pattern):
+                    logger.info(
+                        f"Resolving path from pattern for {t_type}.{idx}"
+                    )
                     # Resolve pattern
-                    base_data_type_pattern_path = self._get_path_from_patterns(
+                    dtype_pattern_path = self._get_path_from_patterns(
                         element=element,
-                        pattern=v,
-                        data_type=t_type,
+                        pattern=entry["pattern"],
+                        data_type=f"{t_type}.{idx}",
                     )
                     # Remove pattern key
-                    out[t_type].pop("pattern")
+                    out[t_type][idx].pop("pattern")
                     # Add path key
-                    out[t_type].update({"path": base_data_type_pattern_path})
-                # Resolve pattern for nested data type
-                if isinstance(v, dict) and "pattern" in v:
-                    # Set nested type key for easier access
-                    t_nested_type = f"{t_type}.{k}"
-                    logger.info(
-                        f"Resolving path from pattern for {t_nested_type}"
-                    )
-                    # Resolve pattern
-                    nested_data_type_pattern_path = (
-                        self._get_path_from_patterns(
+                    out[t_type][idx].update({"path": dtype_pattern_path})
+            else:
+                # Iterate to check for nested "types" like mask
+                for k, v in t_pattern.items():
+                    # Resolve pattern for base data type
+                    if k == "pattern":
+                        logger.info(
+                            f"Resolving path from pattern for {t_type}"
+                        )
+                        # Resolve pattern
+                        base_dtype_pattern_path = self._get_path_from_patterns(
                             element=element,
-                            pattern=v["pattern"],
-                            data_type=t_nested_type,
+                            pattern=v,
+                            data_type=t_type,
+                        )
+                        # Remove pattern key
+                        out[t_type].pop("pattern")
+                        # Add path key
+                        out[t_type].update({"path": base_dtype_pattern_path})
+                    # Resolve pattern for nested data type
+                    if isinstance(v, dict) and "pattern" in v:
+                        # Set nested type key for easier access
+                        t_nested_type = f"{t_type}.{k}"
+                        logger.info(
+                            f"Resolving path from pattern for {t_nested_type}"
+                        )
+                        # Resolve pattern
+                        nested_dtype_pattern_path = (
+                            self._get_path_from_patterns(
+                                element=element,
+                                pattern=v["pattern"],
+                                data_type=t_nested_type,
+                            )
+                        )
+                        # Remove pattern key
+                        out[t_type][k].pop("pattern")
+                        # Add path key
+                        out[t_type][k].update(
+                            {"path": nested_dtype_pattern_path}
                         )
-                    )
-                    # Remove pattern key
-                    out[t_type][k].pop("pattern")
-                    # Add path key
-                    out[t_type][k].update(
-                        {"path": nested_data_type_pattern_path}
-                    )
 
         return out
 
-    def get_elements(self) -> List:
+    def get_elements(self) -> list:
         """Implement fetching list of elements in the dataset.
 
         It will use regex to search for "replacements" in the "patterns" and