junifer 0.0.5.dev240__py3-none-any.whl → 0.0.6__py3-none-any.whl

junifer/datagrabber/datalad_base.py

@@ -7,17 +7,18 @@
 
 import atexit
 import os
-import shutil
 import tempfile
 from pathlib import Path
-from typing import Dict, Optional, Tuple, Union
+from typing import Optional, Union
 
 import datalad
 import datalad.api as dl
 from datalad.support.exceptions import IncompleteResultsError
 from datalad.support.gitrepo import GitRepo
 
-from ..utils import logger, raise_error, warn_with_log
+from ..pipeline import WorkDirManager
+from ..typing import Element
+from ..utils import config, logger, raise_error, warn_with_log
 from .base import BaseDataGrabber
 
 
@@ -78,7 +79,8 @@ class DataladDataGrabber(BaseDataGrabber):
         if datadir is None:
             logger.info("`datadir` is None, creating a temporary directory")
             # Create temporary directory
-            tmpdir = Path(tempfile.mkdtemp())
+            tmpdir = WorkDirManager().get_tempdir(prefix="datalad")
+            self._tmpdir = tmpdir
             datadir = tmpdir / "datadir"
             datadir.mkdir(parents=True, exist_ok=False)
             logger.info(f"`datadir` set to {datadir}")
@@ -104,7 +106,6 @@ class DataladDataGrabber(BaseDataGrabber):
                 "Datalad locks set to "
                 f"{datalad.cfg.get('datalad.locations.locks')}"
             )
-            self._tmpdir = tmpdir
             atexit.register(self._rmtmpdir)
         # TODO: uri can be converted to a positional argument
         if uri is None:
@@ -129,7 +130,7 @@ class DataladDataGrabber(BaseDataGrabber):
         """Remove temporary directory if it exists."""
         if self._tmpdir.exists():
             logger.debug("Removing temporary directory")
-            shutil.rmtree(self._tmpdir)
+            WorkDirManager().delete_tempdir(self._tmpdir)
 
     @property
     def datadir(self) -> Path:
@@ -143,29 +144,51 @@ class DataladDataGrabber(BaseDataGrabber):
         """
         return super().datadir / self._rootdir
 
-    def _get_dataset_id_remote(self) -> str:
+    def _get_dataset_id_remote(self) -> tuple[str, bool]:
         """Get the dataset ID from the remote.
 
         Returns
         -------
         str
             The dataset ID.
+        bool
+            Whether the dataset is dirty.
+
+        Raises
+        ------
+        ValueError
+            If the dataset ID cannot be obtained from the remote.
 
         """
         remote_id = None
+        is_dirty = False
         with tempfile.TemporaryDirectory() as tmpdir:
-            logger.debug(f"Querying {self.uri} for dataset ID")
-            repo = GitRepo.clone(
-                self.uri, path=tmpdir, clone_options=["-n", "--depth=1"]
+            if not config.get("datagrabber.skipidcheck", False):
+                logger.debug(f"Querying {self.uri} for dataset ID")
+                repo = GitRepo.clone(
+                    self.uri, path=tmpdir, clone_options=["-n", "--depth=1"]
+                )
+                repo.checkout(name=".datalad/config", options=["HEAD"])
+                remote_id = repo.config.get("datalad.dataset.id", None)
+                logger.debug(f"Got remote dataset ID = {remote_id}")
+
+                if not config.get("datagrabber.skipdirtycheck", False):
+                    is_dirty = repo.dirty
+                else:
+                    logger.debug("Skipping dirty check")
+                    is_dirty = False
+            else:
+                logger.debug("Skipping dataset ID check")
+                remote_id = self._dataset.id
+                is_dirty = False
+            logger.debug(
+                f"Remote dataset is {'' if is_dirty else 'not'} dirty"
             )
-            repo.checkout(name=".datalad/config", options=["HEAD"])
-            remote_id = repo.config.get("datalad.dataset.id", None)
-            logger.debug(f"Got remote dataset ID = {remote_id}")
         if remote_id is None:
             raise_error("Could not get dataset ID from remote")
-        return remote_id
+        return remote_id, is_dirty
 
-    def _dataset_get(self, out: Dict) -> Dict:
+    def _dataset_get(self, out: dict) -> dict:
         """Get the dataset found from the path in ``out``.
 
         Parameters
@@ -178,17 +201,29 @@ class DataladDataGrabber(BaseDataGrabber):
         dict
             The unmodified input dictionary.
 
+        Raises
+        ------
+        datalad.support.exceptions.IncompleteResultsError
+            If there is a datalad-related problem while fetching data.
+
         """
         to_get = []
         for type_val in out.values():
-            # Iterate to check for nested "types" like mask
-            for k, v in type_val.items():
-                # Add base data type path
-                if k == "path":
-                    to_get.append(v)
-                # Add nested data type path
-                if isinstance(v, dict) and "path" in v:
-                    to_get.append(v["path"])
+            # Conditional for list dtype vals like Warp
+            if isinstance(type_val, list):
+                for entry in type_val:
+                    for k, v in entry.items():
+                        if k == "path":
+                            to_get.append(v)
+            else:
+                # Iterate to check for nested "types" like mask
+                for k, v in type_val.items():
+                    # Add base data type path
+                    if k == "path":
+                        to_get.append(v)
+                    # Add nested data type path
+                    if isinstance(v, dict) and "path" in v:
+                        to_get.append(v["path"])
 
         if len(to_get) > 0:
             logger.debug(f"Getting {len(to_get)} files using datalad:")
@@ -223,6 +258,8 @@ class DataladDataGrabber(BaseDataGrabber):
         ------
         ValueError
             If the dataset is already installed but with a different ID.
+        datalad.support.exceptions.IncompleteResultsError
+            If there is a datalad-related problem while cloning dataset.
 
         """
         isinstalled = dl.Dataset(self._datadir).is_installed()
@@ -231,23 +268,23 @@ class DataladDataGrabber(BaseDataGrabber):
             self._got_files = []
             self._dataset: dl.Dataset = dl.Dataset(self._datadir)
 
-            remote_id = self._get_dataset_id_remote()
+            # Check if dataset is already installed with a different ID
+            remote_id, is_dirty = self._get_dataset_id_remote()
             if remote_id != self._dataset.id:
                 raise_error(
                     "Dataset already installed but with a different "
                     f"ID: {self._dataset.id} (local) != {remote_id} (remote)"
                 )
 
-            # Check for dirty datasets:
-            status = self._dataset.status()
-            if any(x["state"] != "clean" for x in status):
-                self.datalad_dirty = True
+            # Conditional reporting on dataset dirtiness
+            self.datalad_dirty = is_dirty
+            if self.datalad_dirty:
                 warn_with_log(
-                    "At least one file is not clean, Junifer will "
-                    "consider this dataset as dirty."
+                    "At least one file is not clean, "
+                    f"marking dataset (id: {self._dataset.id}) as dirty."
                 )
             else:
-                logger.debug("Dataset is clean")
+                logger.debug(f"Dataset (id: {self._dataset.id}) is clean")
 
         else:
             logger.debug(f"Installing dataset {self.uri} to {self._datadir}")
@@ -276,7 +313,7 @@ class DataladDataGrabber(BaseDataGrabber):
                 logger.debug(f"Dropping {f}")
                 self._dataset.drop(f, result_renderer="disabled")
 
-    def __getitem__(self, element: Union[str, Tuple]) -> Dict:
+    def __getitem__(self, element: Element) -> dict:
         """Implement single element indexing in the Datalad database.
 
         It will first obtain the paths from the parent class and then
@@ -284,11 +321,11 @@ class DataladDataGrabber(BaseDataGrabber):
 
         Parameters
         ----------
-        element : str or tuple
+        element : str or tuple of str
             The element to be indexed. If one string is provided, it is
             assumed to be a tuple with only one item. If a tuple is provided,
             each item in the tuple is the value for the replacement string
-            specified in "replacements".
+            specified in ``"replacements"``.
 
         Returns
         -------

junifer/datagrabber/dmcc13_benchmark.py

@@ -5,7 +5,7 @@
 
 from itertools import product
 from pathlib import Path
-from typing import Dict, List, Union
+from typing import Union
 
 from ..api.decorators import register_datagrabber
 from ..utils import raise_error
@@ -59,11 +59,11 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
     def __init__(
         self,
         datadir: Union[str, Path, None] = None,
-        types: Union[str, List[str], None] = None,
-        sessions: Union[str, List[str], None] = None,
-        tasks: Union[str, List[str], None] = None,
-        phase_encodings: Union[str, List[str], None] = None,
-        runs: Union[str, List[str], None] = None,
+        types: Union[str, list[str], None] = None,
+        sessions: Union[str, list[str], None] = None,
+        tasks: Union[str, list[str], None] = None,
+        phase_encodings: Union[str, list[str], None] = None,
+        runs: Union[str, list[str], None] = None,
         native_t1w: bool = False,
     ) -> None:
         # Declare all sessions
@@ -150,7 +150,7 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
                 "mask": {
                     "pattern": (
                         "derivatives/fmriprep-1.3.2/{subject}/{session}/"
-                        "/func/{subject}_{session}_task-{task}_acq-mb4"
+                        "func/{subject}_{session}_task-{task}_acq-mb4"
                         "{phase_encoding}_run-{run}_"
                         "space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz"
                     ),
@@ -221,15 +221,28 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
                             "space": "native",
                         },
                     },
-                    "Warp": {
-                        "pattern": (
-                            "derivatives/fmriprep-1.3.2/{subject}/anat/"
-                            "{subject}_from-MNI152NLin2009cAsym_to-T1w_"
-                            "mode-image_xfm.h5"
-                        ),
-                        "src": "MNI152NLin2009cAsym",
-                        "dst": "native",
-                    },
+                    "Warp": [
+                        {
+                            "pattern": (
+                                "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                                "{subject}_from-MNI152NLin2009cAsym_to-T1w_"
+                                "mode-image_xfm.h5"
+                            ),
+                            "src": "MNI152NLin2009cAsym",
+                            "dst": "native",
+                            "warper": "ants",
+                        },
+                        {
+                            "pattern": (
+                                "derivatives/fmriprep-1.3.2/{subject}/anat/"
+                                "{subject}_from-T1w_to-MNI152NLin2009cAsym_"
+                                "mode-image_xfm.h5"
+                            ),
+                            "src": "native",
+                            "dst": "MNI152NLin2009cAsym",
+                            "warper": "ants",
+                        },
+                    ],
                 }
             )
         # Set default types
@@ -258,7 +271,7 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
         task: str,
         phase_encoding: str,
         run: str,
-    ) -> Dict:
+    ) -> dict:
         """Index one element in the dataset.
 
         Parameters
@@ -296,7 +309,7 @@ class DMCC13Benchmark(PatternDataladDataGrabber):
         )
         return out
 
-    def get_elements(self) -> List:
+    def get_elements(self) -> list:
         """Implement fetching list of subjects in the dataset.
 
         Returns

junifer/datagrabber/hcp1200/__init__.py

@@ -3,8 +3,7 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from .hcp1200 import HCP1200
-from .datalad_hcp1200 import DataladHCP1200
+import lazy_loader as lazy
 
 
-__all__ = ["HCP1200", "DataladHCP1200"]
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)

junifer/datagrabber/hcp1200/__init__.pyi

@@ -0,0 +1,4 @@
+__all__ = ["HCP1200", "DataladHCP1200"]
+
+from .hcp1200 import HCP1200
+from .datalad_hcp1200 import DataladHCP1200

junifer/datagrabber/hcp1200/datalad_hcp1200.py

@@ -6,7 +6,7 @@
 # License: AGPL
 
 from pathlib import Path
-from typing import List, Union
+from typing import Union
 
 from junifer.datagrabber.datalad_base import DataladDataGrabber
 
@@ -50,8 +50,8 @@ class DataladHCP1200(DataladDataGrabber, HCP1200):
     def __init__(
         self,
         datadir: Union[str, Path, None] = None,
-        tasks: Union[str, List[str], None] = None,
-        phase_encodings: Union[str, List[str], None] = None,
+        tasks: Union[str, list[str], None] = None,
+        phase_encodings: Union[str, list[str], None] = None,
         ica_fix: bool = False,
     ) -> None:
         uri = (

junifer/datagrabber/hcp1200/hcp1200.py

@@ -7,7 +7,7 @@
 
 from itertools import product
 from pathlib import Path
-from typing import Dict, List, Union
+from typing import Union
 
 from ...api.decorators import register_datagrabber
 from ...utils import raise_error
@@ -48,8 +48,8 @@ class HCP1200(PatternDataGrabber):
     def __init__(
         self,
         datadir: Union[str, Path],
-        tasks: Union[str, List[str], None] = None,
-        phase_encodings: Union[str, List[str], None] = None,
+        tasks: Union[str, list[str], None] = None,
+        phase_encodings: Union[str, list[str], None] = None,
         ica_fix: bool = False,
     ) -> None:
         # All tasks
@@ -66,10 +66,10 @@ class HCP1200(PatternDataGrabber):
         ]
         # Set default tasks
         if tasks is None:
-            self.tasks: List[str] = all_tasks
+            self.tasks: list[str] = all_tasks
         # Convert single task into list
         else:
-            if not isinstance(tasks, List):
+            if not isinstance(tasks, list):
                 tasks = [tasks]
             # Check for invalid task(s)
             for task in tasks:
@@ -78,7 +78,7 @@ class HCP1200(PatternDataGrabber):
                         f"'{task}' is not a valid HCP-YA fMRI task input. "
                         f"Valid task values can be any or all of {all_tasks}."
                     )
-            self.tasks: List[str] = tasks
+            self.tasks: list[str] = tasks
 
         # All phase encodings
         all_phase_encodings = ["LR", "RL"]
@@ -122,13 +122,24 @@ class HCP1200(PatternDataGrabber):
                 "pattern": "{subject}/T1w/T1w_acpc_dc_restore.nii.gz",
                 "space": "native",
             },
-            "Warp": {
-                "pattern": (
-                    "{subject}/MNINonLinear/xfms/standard2acpc_dc.nii.gz"
-                ),
-                "src": "MNI152NLin6Asym",
-                "dst": "native",
-            },
+            "Warp": [
+                {
+                    "pattern": (
+                        "{subject}/MNINonLinear/xfms/standard2acpc_dc.nii.gz"
+                    ),
+                    "src": "MNI152NLin6Asym",
+                    "dst": "native",
+                    "warper": "fsl",
+                },
+                {
+                    "pattern": (
+                        "{subject}/MNINonLinear/xfms/acpc_dc2standard.nii.gz"
+                    ),
+                    "src": "native",
+                    "dst": "MNI152NLin6Asym",
+                    "warper": "fsl",
+                },
+            ],
         }
         # The replacements
         replacements = ["subject", "task", "phase_encoding"]
@@ -139,7 +150,7 @@ class HCP1200(PatternDataGrabber):
             replacements=replacements,
         )
 
-    def get_item(self, subject: str, task: str, phase_encoding: str) -> Dict:
+    def get_item(self, subject: str, task: str, phase_encoding: str) -> dict:
         """Implement single element indexing in the database.
 
         Parameters
@@ -169,7 +180,7 @@ class HCP1200(PatternDataGrabber):
             subject=subject, task=new_task, phase_encoding=phase_encoding
         )
 
-    def get_elements(self) -> List:
+    def get_elements(self) -> list:
         """Implement fetching list of elements in the dataset.
 
         Returns

junifer/datagrabber/hcp1200/tests/test_hcp1200.py

@@ -3,7 +3,11 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Iterable, Optional
+import shutil
+import tempfile
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Optional
 
 import pytest
 
@@ -17,7 +21,8 @@ URI = "https://gin.g-node.org/juaml/datalad-example-hcp1200"
 @pytest.fixture(scope="module")
 def hcpdg() -> Iterable[DataladHCP1200]:
     """Return a HCP1200 DataGrabber."""
-    dg = DataladHCP1200()
+    tmpdir = Path(tempfile.gettempdir())
+    dg = DataladHCP1200(datadir=tmpdir / "datadir")
     # Set URI to Gin
     dg.uri = URI
     # Set correct root directory
@@ -26,6 +31,7 @@ def hcpdg() -> Iterable[DataladHCP1200]:
         for t_elem in dg.get_elements():
             dg[t_elem]
         yield dg
+    shutil.rmtree(tmpdir / "datadir", ignore_errors=True)
 
 
 @pytest.mark.parametrize(

junifer/datagrabber/multiple.py

@@ -5,9 +5,10 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Dict, List, Tuple, Union
+from typing import Union
 
 from ..api.decorators import register_datagrabber
+from ..typing import DataGrabberLike
 from ..utils import deep_update, raise_error
 from .base import BaseDataGrabber
 
@@ -37,7 +38,7 @@ class MultipleDataGrabber(BaseDataGrabber):
 
     """
 
-    def __init__(self, datagrabbers: List[BaseDataGrabber], **kwargs) -> None:
+    def __init__(self, datagrabbers: list[DataGrabberLike], **kwargs) -> None:
         # Check datagrabbers consistency
         # Check for same element keys
         first_keys = datagrabbers[0].get_element_keys()
@@ -78,7 +79,7 @@ class MultipleDataGrabber(BaseDataGrabber):
                 )
         self._datagrabbers = datagrabbers
 
-    def __getitem__(self, element: Union[str, Tuple]) -> Dict:
+    def __getitem__(self, element: Union[str, tuple]) -> dict:
         """Implement indexing.
 
         Parameters
@@ -110,8 +111,12 @@ class MultipleDataGrabber(BaseDataGrabber):
 
         # Update all the metas again
         for kind in out:
-            self.update_meta(out[kind], "datagrabber")
-            out[kind]["meta"]["datagrabber"]["datagrabbers"] = metas
+            to_update = out[kind]
+            if not isinstance(to_update, list):
+                to_update = [to_update]
+            for t_kind in to_update:
+                self.update_meta(t_kind, "datagrabber")
+                t_kind["meta"]["datagrabber"]["datagrabbers"] = metas
         return out
 
     def __enter__(self) -> "MultipleDataGrabber":
@@ -126,7 +131,7 @@ class MultipleDataGrabber(BaseDataGrabber):
             dg.__exit__(exc_type, exc_value, exc_traceback)
 
     # TODO: return type should be List[List[str]], but base type is List[str]
-    def get_types(self) -> List[str]:
+    def get_types(self) -> list[str]:
         """Get types.
 
         Returns
@@ -138,7 +143,7 @@ class MultipleDataGrabber(BaseDataGrabber):
         types = [x for dg in self._datagrabbers for x in dg.get_types()]
         return types
 
-    def get_element_keys(self) -> List[str]:
+    def get_element_keys(self) -> list[str]:
         """Get element keys.
 
         For each item in the ``element`` tuple passed to ``__getitem__()``,
@@ -152,7 +157,7 @@ class MultipleDataGrabber(BaseDataGrabber):
         """
         return self._datagrabbers[0].get_element_keys()
 
-    def get_elements(self) -> List:
+    def get_elements(self) -> list:
         """Get elements.
 
         Returns
@@ -170,7 +175,7 @@ class MultipleDataGrabber(BaseDataGrabber):
             elements.intersection_update(s)
         return list(elements)
 
-    def get_item(self, **_: Dict) -> Dict[str, Dict]:
+    def get_item(self, **_: dict) -> dict[str, dict]:
         """Get the specified item from the dataset.
 
         Parameters