junifer 0.0.7.dev332__py3-none-any.whl → 0.0.8__py3-none-any.whl

junifer/datagrabber/pattern_datalad.py

@@ -5,8 +5,14 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+from fnmatch import fnmatchcase
+from pathlib import Path
+
+from pydantic import ConfigDict
+
 from ..api.decorators import register_datagrabber
-from ..utils import logger
+from ..utils import raise_error
+from .base import logger
 from .datalad_base import DataladDataGrabber
 from .pattern import PatternDataGrabber
 
@@ -23,122 +29,24 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
 
     Parameters
     ----------
-    types : list of str
-        The types of data to be grabbed.
-    patterns : dict
-        Data type patterns as a dictionary. It has the following schema:
-
-        * ``"T1w"`` :
-
-          .. code-block:: none
-
-            {
-              "mandatory": ["pattern", "space"],
-              "optional": {
-                  "mask": {
-                      "mandatory": ["pattern", "space"],
-                      "optional": []
-                  }
-              }
-            }
-
-        * ``"T2w"`` :
-
-          .. code-block:: none
-
-            {
-              "mandatory": ["pattern", "space"],
-              "optional": {
-                  "mask": {
-                      "mandatory": ["pattern", "space"],
-                      "optional": []
-                  }
-              }
-            }
-
-        * ``"BOLD"`` :
-
-          .. code-block:: none
-
-            {
-              "mandatory": ["pattern", "space"],
-              "optional": {
-                  "mask": {
-                      "mandatory": ["pattern", "space"],
-                      "optional": []
-                  }
-                  "confounds": {
-                      "mandatory": ["pattern", "format"],
-                      "optional": []
-                  }
-              }
-            }
-
-        * ``"Warp"`` :
-
-          .. code-block:: none
-
-            {
-              "mandatory": ["pattern", "src", "dst"],
-              "optional": []
-            }
-
-        * ``"VBM_GM"`` :
-
-          .. code-block:: none
-
-            {
-              "mandatory": ["pattern", "space"],
-              "optional": []
-            }
-
-        * ``"VBM_WM"`` :
-
-          .. code-block:: none
-
-            {
-              "mandatory": ["pattern", "space"],
-              "optional": []
-            }
-
-        Basically, for each data type, one needs to provide ``mandatory`` keys
-        and can choose to also provide ``optional`` keys. The value for each
-        key is a string. So, one needs to provide necessary data types as a
-        dictionary, for example:
-
-        .. code-block:: none
-
-          {
-              "BOLD": {
-                "pattern": "...",
-                "space": "...",
-              },
-              "T1w": {
-                "pattern": "...",
-                "space": "...",
-              },
-              "Warp": {
-                "pattern": "...",
-                "src": "...",
-                "dst": "...",
-              }
-          }
-
-        taken from :class:`.HCP1200`.
-    replacements : str or list of str
-        Replacements in the ``pattern`` key of each data type. The value needs
-        to be a list of all possible replacements.
-    confounds_format : {"fmriprep", "adhoc"} or None, optional
-        The format of the confounds for the dataset (default None).
-    datadir : str or pathlib.Path or None, optional
-        That directory where the datalad dataset will be cloned. If None,
-        the datalad dataset will be cloned into a temporary directory
-        (default None).
-    rootdir : str or pathlib.Path, optional
+    uri : pydantic.AnyUrl
+        URI of the datalad sibling.
+    types : enum:`.DataType` or list of variants
+        The data type(s) to grab.
+    patterns : ``DataGrabberPatterns``
+        The datagrabber patterns. Check :class:`DataTypeSchema` for the schema.
+    replacements : list of str
+        All possible replacements in ``patterns.<data_type>.pattern``.
+    rootdir : pathlib.Path, optional
         The path within the datalad dataset to the root directory
-        (default ".").
-    uri : str or None, optional
-        URI of the datalad sibling (default None).
+        (default Path(".")).
+    confounds_format : :enum:`.ConfoundsFormat` or None, optional
+        The format of the confounds for the dataset (default None).
+    datadir : pathlib.Path, optional
+        That path where the datalad dataset will be cloned.
+        If not specified, the datalad dataset will be cloned into a temporary
+        directory.
+
 
     See Also
     --------
@@ -149,15 +57,67 @@ class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
 
     """
 
-    def __init__(
-        self,
-        **kwargs,
-    ) -> None:
-        # TODO(synchon): needs to be reworked, DataladDataGrabber needs to be
-        # a mixin to avoid multiple inheritance wherever possible.
+    model_config = ConfigDict(extra="allow")
 
+    def validate_datagrabber_params(self) -> None:
+        """Run extra logical validation for datagrabber."""
+        super().validate_datagrabber_params()
         logger.debug("Initializing PatternDataladDataGrabber")
-        for key, val in kwargs.items():
+        for key, val in self.__pydantic_extra__.items():
             logger.debug(f"\t{key} = {val}")
 
-        super().__init__(**kwargs)
+    def install(self) -> None:
+        """Install the datalad dataset and check the patterns.
+
+        The elements are found by searching the files of the patterns, but the
+        files of a sub-dataset are only there once the sub-dataset is
+        installed, so the patterns cannot search the files of the
+        sub-datasets (e.g., with one sub-dataset per subject).
+
+        Raises
+        ------
+        RuntimeError
+            If a pattern searches the files of a sub-dataset.
+
+        """
+        super().install()
+        subdatasets = [
+            Path(x)
+            for x in self._dataset.subdatasets(
+                result_xfm="relpaths",
+                result_renderer="disabled",
+                return_type="list",
+            )
+        ]
+        if len(subdatasets) == 0:
+            return
+        for data_type in self.types:
+            for pattern in self._get_type_patterns(data_type):
+                _, glob_pattern, _ = self._replace_patterns_regex(pattern)
+                if not any(x in glob_pattern for x in "*?["):
+                    continue
+                pattern_parts = (self.rootdir / glob_pattern).parts
+                for subdataset in subdatasets:
+                    n_parts = len(subdataset.parts)
+                    if len(pattern_parts) > n_parts and all(
+                        fnmatchcase(x, y)
+                        for x, y in zip(
+                            subdataset.parts, pattern_parts, strict=False
+                        )
+                    ):
+                        # Remove the dataset if cloned, as the context is
+                        # not entered
+                        self.cleanup()
+                        raise_error(
+                            msg=(
+                                f"The pattern {pattern} of {data_type} "
+                                "searches the files of the sub-dataset "
+                                f"{subdataset}, which are only there if it "
+                                "is installed. A DataGrabber with files in "
+                                "sub-datasets must list its elements and "
+                                "build the paths of the files without "
+                                "searching them (e.g., without wildcards), "
+                                "as DataladHCP1200."
+                            ),
+                            klass=RuntimeError,
+                        )

junifer/datagrabber/pattern_validation_mixin.py

@@ -3,11 +3,22 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+import sys
+
+
+if sys.version_info < (3, 12):  # pragma: no cover
+    from typing_extensions import TypedDict
+else:
+    from typing import TypedDict
+
+
 from collections.abc import Iterator, MutableMapping
-from typing import TypedDict
+
+from aenum import extend_enum
 
 from ..typing import DataGrabberPatterns
-from ..utils import logger, raise_error, warn_with_log
+from ..utils import raise_error, warn_with_log
+from .base import DataType, logger
 
 
 __all__ = [
@@ -191,37 +202,17 @@ def register_data_type(name: str, schema: DataTypeSchema) -> None:
     ----------
     name : str
         The data type name.
-    schema : DataTypeSchema
+    schema : ``DataTypeSchema``
         The data type schema.
 
     """
     DataTypeManager()[name] = schema
+    extend_enum(DataType, name, name)
 
 
 class PatternValidationMixin:
     """Mixin class for pattern validation."""
 
-    def _validate_types(self, types: list[str]) -> None:
-        """Validate the types.
-
-        Parameters
-        ----------
-        types : list of str
-            The data types to validate.
-
-        Raises
-        ------
-        TypeError
-            If ``types`` is not a list or if the values are not string.
-
-        """
-        if not isinstance(types, list):
-            raise_error(msg="`types` must be a list", klass=TypeError)
-        if any(not isinstance(x, str) for x in types):
-            raise_error(
-                msg="`types` must be a list of strings", klass=TypeError
-            )
-
     def _validate_replacements(
         self,
         replacements: list[str],
@@ -234,15 +225,13 @@ class PatternValidationMixin:
         ----------
         replacements : list of str
             The replacements to validate.
-        patterns : dict
-            The patterns to validate replacements against.
+        patterns : ``DataGrabberPatterns``
+            The patterns to validate ``replacements`` against.
         partial_pattern_ok : bool
             Whether to raise error if partial pattern for a data type is found.
 
         Raises
         ------
-        TypeError
-            If ``replacements`` is not a list or if the values are not string.
         ValueError
             If a value in ``replacements`` is not part of a data type pattern
             and ``partial_pattern_ok=False`` or
@@ -256,15 +245,6 @@ class PatternValidationMixin:
             and ``partial_pattern_ok=True``.
 
         """
-        if not isinstance(replacements, list):
-            raise_error(msg="`replacements` must be a list.", klass=TypeError)
-
-        if any(not isinstance(x, str) for x in replacements):
-            raise_error(
-                msg="`replacements` must be a list of strings",
-                klass=TypeError,
-            )
-
         # Make a list of all patterns recursively
         all_patterns = []
         for dtype_val in patterns.values():
@@ -390,7 +370,7 @@ class PatternValidationMixin:
 
     def validate_patterns(
         self,
-        types: list[str],
+        types: list[DataType],
         replacements: list[str],
         patterns: DataGrabberPatterns,
         partial_pattern_ok: bool = False,
@@ -399,11 +379,11 @@ class PatternValidationMixin:
 
         Parameters
         ----------
-        types : list of str
-            The data types to check patterns of.
+        types : list of :enum:`.DataType`
+            The data type(s) to check patterns of.
         replacements : list of str
-            The replacements to be replaced in the patterns.
-        patterns : dict
+            The replacements to be replaced in the ``patterns``.
+        patterns : ``DataGrabberPatterns``
             The patterns to validate.
         partial_pattern_ok : bool, optional
             Whether to raise error if partial pattern for a data type is found.
@@ -412,8 +392,6 @@ class PatternValidationMixin:
 
         Raises
         ------
-        TypeError
-            If ``patterns`` is not a dictionary.
         ValueError
             If length of ``types`` and ``patterns`` are different or
             if ``patterns`` is missing entries from ``types`` or
@@ -421,12 +399,6 @@ class PatternValidationMixin:
             if data type pattern key contains '*' as value.
 
         """
-        # Validate types
-        self._validate_types(types=types)
-
-        # Validate patterns
-        if not isinstance(patterns, dict):
-            raise_error(msg="`patterns` must be a dict", klass=TypeError)
         # Unequal length of objects
         if len(types) > len(patterns):
             raise_error(

junifer/datagrabber/tests/test_base.py

@@ -26,7 +26,7 @@ def test_BaseDataGrabber() -> None:
         def get_element_keys(self):
             return ["subject"]
 
-    dg = MyDataGrabber(datadir="/tmp", types=["BOLD"])
+    dg = MyDataGrabber(datadir="/tmp", types="BOLD")
     elem = dg["sub01"]
     assert "BOLD" in elem
     assert "meta" in elem["BOLD"]
@@ -55,7 +55,7 @@ def test_BaseDataGrabber() -> None:
         def get_element_keys(self):
             return super().get_element_keys()
 
-    dg = MyDataGrabber2(datadir="/tmp", types=["BOLD"])
+    dg = MyDataGrabber2(datadir="/tmp", types="BOLD")
     with pytest.raises(NotImplementedError):
         dg.get_element_keys()
 
@@ -77,7 +77,7 @@ def test_BaseDataGrabber_filter_single() -> None:
         def get_element_keys(self):
             return ["subject"]
 
-    dg = FilterDataGrabber(datadir="/tmp", types=["BOLD"])
+    dg = FilterDataGrabber(datadir="/tmp", types="BOLD")
     with dg:
         assert "sub01" in list(dg.filter(["sub01"]))
         assert "sub02" not in list(dg.filter(["sub01"]))
@@ -104,7 +104,7 @@ def test_BaseDataGrabber_filter_multi() -> None:
         def get_element_keys(self):
             return ["subject", "task"]
 
-    dg = FilterDataGrabber(datadir="/tmp", types=["BOLD"])
+    dg = FilterDataGrabber(datadir="/tmp", types="BOLD")
     with dg:
         assert ("sub01", "rest") in list(
             dg.filter([("sub01", "rest")])  # type: ignore
@@ -118,3 +118,16 @@ def test_BaseDataGrabber_filter_multi() -> None:
         assert ("sub02", "movie") not in list(
             dg.filter([("sub01", "rest")])  # type: ignore
         )
+        # Partial selectors as values or tuples
+        assert list(dg.filter(["sub01"])) == [
+            ("sub01", "rest"),
+            ("sub01", "movie"),
+        ]
+        assert list(dg.filter([("rest",)])) == [
+            ("sub01", "rest"),
+            ("sub02", "rest"),
+            ("sub03", "rest"),
+        ]
+        # The selectors must have the same number of values
+        with pytest.raises(ValueError, match="same number of values"):
+            list(dg.filter(["sub01", ("sub02", "rest")]))  # type: ignore

junifer/datagrabber/tests/test_datalad_base.py

@@ -9,18 +9,22 @@ from pathlib import Path
 import datalad.api as dl
 import pytest
 
-from junifer.datagrabber import DataladDataGrabber
+from junifer.datagrabber import DataladDataGrabber, DataType
 from junifer.utils import config
 
 
 _testing_dataset = {
     "example_bids": {
-        "uri": "https://gin.g-node.org/juaml/datalad-example-bids",
+        "uri": (
+            "https://cerebra.fz-juelich.de/junifer/datalad-example-bids.git"
+        ),
         "commit": "3f288c8725207ae0c9b3616e093e78cda192b570",
         "id": "582b9696-f13f-42e4-9587-b4e62aa2a8e7",
     },
     "example_bids_ses": {
-        "uri": "https://gin.g-node.org/juaml/datalad-example-bids-ses",
+        "uri": (
+            "https://cerebra.fz-juelich.de/junifer/datalad-example-bids-ses.git"
+        ),
         "commit": "6b163aa98af76a9eac0272273c27e14127850181",
         "id": "715c17cf-a1b9-42d6-9af8-9f74c1a4a724",
     },
@@ -38,23 +42,18 @@ def concrete_datagrabber() -> type[DataladDataGrabber]:
 
     """
 
-    class MyDataGrabber(DataladDataGrabber):  # type: ignore
-        def __init__(self, datadir, uri):
-            super().__init__(
-                datadir=datadir,
-                rootdir="example_bids",
-                uri=uri,
-                types=["T1w", "BOLD"],
-            )
+    class MyDataGrabber(DataladDataGrabber):
+        types: list[DataType] = [DataType.T1w, DataType.BOLD]  # noqa: RUF012
+        rootdir: Path = Path("example_bids")
 
         def get_item(self, subject):
             out = {
                 "T1w": {
-                    "path": self.datadir
+                    "path": self.fulldir
                     / f"{subject}/anat/{subject}_T1w.nii.gz"
                 },
                 "BOLD": {
-                    "path": self.datadir
+                    "path": self.fulldir
                     / f"{subject}/func/{subject}_task-rest_bold.nii.gz"
                 },
             }
@@ -91,7 +90,7 @@ def test_DataladDataGrabber_install_errors(
     # Files are not there
     assert datadir.exists() is False
     # Clone dataset
-    dl.clone(uri, datadir)  # type: ignore
+    dl.clone(uri, datadir)
     dg = concrete_datagrabber(datadir=datadir, uri=uri2)
     with pytest.raises(ValueError, match=r"different ID"):
         with dg:
@@ -160,7 +159,6 @@ def test_DataladDataGrabber_clone_cleanup(
         assert "datagrabber" in meta
         assert "datalad_dirty" in meta["datagrabber"]
         assert meta["datagrabber"]["datalad_dirty"] is False
-        assert hasattr(dg, "_got_files") is False
         assert datadir.exists() is True
         assert elem1_bold.is_file() is True
         assert elem1_bold.is_symlink() is True
@@ -185,8 +183,8 @@ def test_DataladDataGrabber_clone_create_cleanup(
 
     # Clone whole dataset
     uri = _testing_dataset["example_bids"]["uri"]
-    with concrete_datagrabber(datadir=None, uri=uri) as dg:
-        datadir = dg._tmpdir / "datadir"
+    with concrete_datagrabber(uri=uri) as dg:
+        datadir = dg._repodir
         elem1_bold = (
             datadir / "example_bids/sub-01/func/sub-01_task-rest_bold.nii.gz"
         )
@@ -206,7 +204,6 @@ def test_DataladDataGrabber_clone_create_cleanup(
         assert "datagrabber" in meta
         assert "datalad_dirty" in meta["datagrabber"]
         assert meta["datagrabber"]["datalad_dirty"] is False
-        assert hasattr(dg, "_got_files") is False
         assert datadir.exists() is True
         assert elem1_bold.is_file() is True
         assert elem1_bold.is_symlink() is True
@@ -246,7 +243,7 @@ def test_DataladDataGrabber_previously_cloned(
     assert elem1_t1w.exists() is False
 
     # Clone dataset
-    dl.clone(uri, datadir, result_renderer="disabled")  # type: ignore
+    dl.clone(uri, datadir, result_renderer="disabled")
 
     # Files are there, but are empty symbolic links
     assert datadir.exists() is True
@@ -316,7 +313,7 @@ def test_DataladDataGrabber_previously_cloned_and_get(
     assert elem1_t1w.exists() is False
 
     # Clone dataset
-    dl.clone(uri, datadir, result_renderer="disabled")  # type: ignore
+    dl.clone(uri, datadir, result_renderer="disabled")
 
     # Files are there, but are empty symbolic links
     assert datadir.exists() is True
@@ -325,9 +322,7 @@ def test_DataladDataGrabber_previously_cloned_and_get(
     assert elem1_t1w.is_symlink() is True
     assert elem1_t1w.is_file() is False
 
-    dl.get(  # type: ignore
-        elem1_t1w, dataset=datadir, result_renderer="disabled"
-    )
+    dl.get(elem1_t1w, dataset=datadir, result_renderer="disabled")
 
     assert elem1_bold.is_symlink() is True
     assert elem1_bold.is_file() is False
@@ -399,7 +394,7 @@ def test_DataladDataGrabber_previously_cloned_and_get_dirty(
     assert elem1_t1w.exists() is False
 
     # Clone dataset
-    dl.clone(uri, datadir, result_renderer="disabled")  # type: ignore
+    dl.clone(uri, datadir, result_renderer="disabled")
 
     # Files are there, but are empty symbolic links
     assert datadir.exists() is True
@@ -408,9 +403,7 @@ def test_DataladDataGrabber_previously_cloned_and_get_dirty(
     assert elem1_t1w.is_symlink() is True
     assert elem1_t1w.is_file() is False
 
-    dl.get(  # type: ignore
-        elem1_t1w, dataset=datadir, result_renderer="disabled"
-    )
+    dl.get(elem1_t1w, dataset=datadir, result_renderer="disabled")
 
     assert elem1_bold.is_symlink() is True
     assert elem1_bold.is_file() is False