junifer 0.0.8.dev314__tar.gz → 0.0.8.dev318__tar.gz

{junifer-0.0.8.dev314/junifer.egg-info → junifer-0.0.8.dev318}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.8.dev314
+Version: 0.0.8.dev318
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

junifer-0.0.8.dev318/docs/changes/newsfragments/528.bugfix

@@ -0,0 +1 @@
+Fix :class:`.DataladHCP1200` listing only the subjects already installed (the subjects are sub-datasets) and not finding the BOLD data of the subjects not installed, by listing all the subjects without searching the files and building the paths of the BOLD data without wildcards by `Fede Raimondo`_

junifer-0.0.8.dev318/docs/changes/newsfragments/528.change

@@ -0,0 +1 @@
+Raise an error when installing the dataset of a :class:`.PatternDataladDataGrabber` if a pattern searches the files of a sub-dataset, which are only there once the sub-dataset is installed, instead of not finding the elements by `Fede Raimondo`_

junifer-0.0.8.dev318/docs/changes/newsfragments/528.doc

@@ -0,0 +1 @@
+Document that the DataGrabbers with files in datalad sub-datasets must list their elements and build the paths without searching the files by `Fede Raimondo`_

{junifer-0.0.8.dev314 → junifer-0.0.8.dev318}/docs/extending/datagrabber.rst

@@ -399,6 +399,17 @@ This approach can be used directly from the YAML, like so:
      uri: "https://cerebra.fz-juelich.de/junifer/datalad-example-bids-ses.git"
      rootdir: example_bids_ses
 
+.. important::
+
+   The elements are found by searching the files of the dataset, but the files
+   of a datalad *sub-dataset* are only there once the sub-dataset is installed.
+   If the patterns search the files of a sub-dataset (e.g., a dataset with a
+   sub-dataset for each subject), :class:`.PatternDataladDataGrabber` raises
+   an error when installing the dataset. These DataGrabbers must implement
+   ``get_elements`` without searching the files (e.g., from the names of the
+   sub-datasets) and build the paths in the sub-datasets without wildcards,
+   as :class:`.DataladHCP1200` does.
+
 .. _extending_datagrabbers_path_expansion:
 
 Advanced: Using Unix-like path expansion directives

{junifer-0.0.8.dev314 → junifer-0.0.8.dev318}/junifer/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.0.8.dev314'
-__version_tuple__ = version_tuple = (0, 0, 8, 'dev314')
+__version__ = version = '0.0.8.dev318'
+__version_tuple__ = version_tuple = (0, 0, 8, 'dev318')
 
-__commit_id__ = commit_id = 'g968a03e78'
+__commit_id__ = commit_id = 'gd4a9dcd3a'

{junifer-0.0.8.dev314 → junifer-0.0.8.dev318}/junifer/datagrabber/hcp1200/datalad_hcp1200.py

@@ -5,12 +5,14 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+from itertools import product
 from pathlib import Path
 from typing import Annotated, Literal
 
 from pydantic import AnyUrl, BeforeValidator
 
 from ...api.decorators import register_datagrabber
+from ...typing import Elements
 from ...utils import ensure_list
 from ..base import DataType
 from ..datalad_base import DataladDataGrabber
@@ -67,3 +69,29 @@ class DataladHCP1200(DataladDataGrabber, HCP1200):
     def skip_file_check(self) -> bool:
         """Skip file check existence."""
         return True
+
+    def get_elements(self) -> Elements:
+        """Implement fetching list of elements in the dataset.
+
+        The subjects are sub-datasets, whose files are only there once they
+        are installed, so the files are not searched: the elements are all
+        the subjects (the directories of the dataset) with the tasks and
+        phase encodings to grab, even if some of them have no data.
+
+        Returns
+        -------
+        list
+            The list of elements that can be grabbed in the dataset.
+
+        """
+        subjects = sorted(
+            x.name
+            for x in self.fulldir.iterdir()
+            if x.is_dir() and not x.name.startswith(".")
+        )
+        values = {"subject": subjects, **self.get_replacement_values()}
+        element_keys = self.get_element_keys()
+        elements = list(product(*(values[k] for k in element_keys)))
+        if len(element_keys) == 1:
+            return [element[0] for element in elements]
+        return elements

{junifer-0.0.8.dev314 → junifer-0.0.8.dev318}/junifer/datagrabber/hcp1200/hcp1200.py

@@ -162,3 +162,29 @@ class HCP1200(PatternDataGrabber):
             "pattern"
         ].replace("{suffix}", suffix)
         super().validate_datagrabber_params()
+
+    def _replace_patterns_glob(self, element: dict, pattern: str) -> str:
+        """Replace ``pattern`` with the ``element`` so it can be globbed.
+
+        The BOLD data of the resting-state tasks is in ``rfMRI_*`` and of the
+        other tasks in ``tfMRI_*``, so the path of an element is built
+        without searching the files, which are in sub-datasets in
+        :class:`.DataladHCP1200`.
+
+        Parameters
+        ----------
+        element : dict
+            The element to be used in the replacement.
+        pattern : str
+            The pattern to be replaced.
+
+        Returns
+        -------
+        str
+            The pattern with the element replaced.
+
+        """
+        if "task" in element:
+            prefix = "rfMRI" if "REST" in element["task"] else "tfMRI"
+            pattern = pattern.replace("?fMRI_", f"{prefix}_")
+        return super()._replace_patterns_glob(element, pattern)

{junifer-0.0.8.dev314 → junifer-0.0.8.dev318}/junifer/datagrabber/hcp1200/tests/test_hcp1200.py

@@ -8,6 +8,7 @@ import tempfile
 from collections.abc import Iterable
 from pathlib import Path
 
+import datalad.api as dl
 import pytest
 from pydantic import AnyUrl
 
@@ -320,3 +321,63 @@ def test_HCP1200_incorrect_access_icafix(
             tasks=tasks,
             ica_fix=ica_fix,
         )
+
+
+def test_DataladHCP1200_subdatasets(tmp_path: Path) -> None:
+    """Test DataladHCP1200 with a sub-dataset for each subject, as HCP.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    dataset = dl.create(tmp_path / "origin", result_renderer="disabled")
+    for subject in ("100206", "100307"):
+        subdataset = dataset.create(subject, result_renderer="disabled")
+        for folder in ("rfMRI_REST1_LR", "tfMRI_WM_RL"):
+            path = (
+                subdataset.pathobj
+                / "MNINonLinear"
+                / "Results"
+                / folder
+                / f"{folder}.nii.gz"
+            )
+            path.parent.mkdir(parents=True)
+            path.write_text(f"{subject} {folder}")
+        path = subdataset.pathobj / "T1w" / "T1w_acpc_dc_restore.nii.gz"
+        path.parent.mkdir(parents=True)
+        path.write_text(f"{subject} T1w")
+    dataset.save(recursive=True, result_renderer="disabled")
+    dg = DataladHCP1200(
+        uri=AnyUrl(f"file://{dataset.path}"),
+        datadir=tmp_path / "clone",
+        rootdir=Path("."),
+        types=["BOLD", "T1w"],
+        tasks=["REST1", "WM"],
+        phase_encodings=["LR", "RL"],
+    )
+    with config_override("datagrabber.skipidcheck", True), dg:
+        # All the subjects with the tasks and phase encodings, without
+        # installing the sub-datasets
+        assert dg.get_elements() == [
+            (subject, task, phase_encoding)
+            for subject in ("100206", "100307")
+            for task in ("REST1", "WM")
+            for phase_encoding in ("LR", "RL")
+        ]
+        assert not (dg.fulldir / "100206" / "T1w").exists()
+        out = dg[("100307", "WM", "RL")]
+        assert out["BOLD"]["path"].read_text() == "100307 tfMRI_WM_RL"
+        assert out["T1w"]["path"].read_text() == "100307 T1w"
+        out = dg[("100206", "REST1", "LR")]
+        assert out["BOLD"]["path"].read_text() == "100206 rfMRI_REST1_LR"
+    dg = DataladHCP1200(
+        uri=AnyUrl(f"file://{dataset.path}"),
+        datadir=tmp_path / "clone",
+        rootdir=Path("."),
+        types=["T1w"],
+    )
+    with config_override("datagrabber.skipidcheck", True), dg:
+        assert dg.get_elements() == ["100206", "100307"]
+        assert dg["100206"]["T1w"]["path"].read_text() == "100206 T1w"

junifer-0.0.8.dev318/junifer/datagrabber/pattern_datalad.py

@@ -0,0 +1,123 @@
+"""Provide concrete implementation for pattern + datalad based DataGrabber."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+#          Leonard Sasse <l.sasse@fz-juelich.de>
+#          Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+from fnmatch import fnmatchcase
+from pathlib import Path
+
+from pydantic import ConfigDict
+
+from ..api.decorators import register_datagrabber
+from ..utils import raise_error
+from .base import logger
+from .datalad_base import DataladDataGrabber
+from .pattern import PatternDataGrabber
+
+
+__all__ = ["PatternDataladDataGrabber"]
+
+
+@register_datagrabber
+class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
+    """Concrete implementation for pattern and datalad based data fetching.
+
+    Implements a DataGrabber that gets data from a datalad sibling,
+    interpreting patterns.
+
+    Parameters
+    ----------
+    uri : pydantic.AnyUrl
+        URI of the datalad sibling.
+    types : enum:`.DataType` or list of variants
+        The data type(s) to grab.
+    patterns : ``DataGrabberPatterns``
+        The datagrabber patterns. Check :class:`DataTypeSchema` for the schema.
+    replacements : list of str
+        All possible replacements in ``patterns.<data_type>.pattern``.
+    rootdir : pathlib.Path, optional
+        The path within the datalad dataset to the root directory
+        (default Path(".")).
+    confounds_format : :enum:`.ConfoundsFormat` or None, optional
+        The format of the confounds for the dataset (default None).
+    datadir : pathlib.Path, optional
+        That path where the datalad dataset will be cloned.
+        If not specified, the datalad dataset will be cloned into a temporary
+        directory.
+
+
+    See Also
+    --------
+    DataladDataGrabber:
+        Abstract base class for datalad-based data fetching.
+    PatternDataGrabber:
+        Concrete implementation for pattern-based data fetching.
+
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    def validate_datagrabber_params(self) -> None:
+        """Run extra logical validation for datagrabber."""
+        super().validate_datagrabber_params()
+        logger.debug("Initializing PatternDataladDataGrabber")
+        for key, val in self.__pydantic_extra__.items():
+            logger.debug(f"\t{key} = {val}")
+
+    def install(self) -> None:
+        """Install the datalad dataset and check the patterns.
+
+        The elements are found by searching the files of the patterns, but the
+        files of a sub-dataset are only there once the sub-dataset is
+        installed, so the patterns cannot search the files of the
+        sub-datasets (e.g., with one sub-dataset per subject).
+
+        Raises
+        ------
+        RuntimeError
+            If a pattern searches the files of a sub-dataset.
+
+        """
+        super().install()
+        subdatasets = [
+            Path(x)
+            for x in self._dataset.subdatasets(
+                result_xfm="relpaths",
+                result_renderer="disabled",
+                return_type="list",
+            )
+        ]
+        if len(subdatasets) == 0:
+            return
+        for data_type in self.types:
+            for pattern in self._get_type_patterns(data_type):
+                _, glob_pattern, _ = self._replace_patterns_regex(pattern)
+                if not any(x in glob_pattern for x in "*?["):
+                    continue
+                pattern_parts = (self.rootdir / glob_pattern).parts
+                for subdataset in subdatasets:
+                    n_parts = len(subdataset.parts)
+                    if len(pattern_parts) > n_parts and all(
+                        fnmatchcase(x, y)
+                        for x, y in zip(
+                            subdataset.parts, pattern_parts, strict=False
+                        )
+                    ):
+                        # Remove the dataset if cloned, as the context is
+                        # not entered
+                        self.cleanup()
+                        raise_error(
+                            msg=(
+                                f"The pattern {pattern} of {data_type} "
+                                "searches the files of the sub-dataset "
+                                f"{subdataset}, which are only there if it "
+                                "is installed. A DataGrabber with files in "
+                                "sub-datasets must list its elements and "
+                                "build the paths of the files without "
+                                "searching them (e.g., without wildcards), "
+                                "as DataladHCP1200."
+                            ),
+                            klass=RuntimeError,
+                        )

{junifer-0.0.8.dev314 → junifer-0.0.8.dev318}/junifer/datagrabber/tests/test_pattern_datalad.py

@@ -7,6 +7,8 @@
 
 from pathlib import Path
 
+import datalad.api as dl
+import pytest
 from pydantic import AnyUrl
 
 from junifer.datagrabber import DataType, PatternDataladDataGrabber
@@ -174,3 +176,102 @@ def test_bids_PatternDataladDataGrabber_session():
             for i in range(1, 10)
         ]
         assert set(subs) == set(expected_subs)
+
+
+@pytest.fixture
+def subdatasets_uri(tmp_path: Path) -> AnyUrl:
+    """Return the URI of a dataset with a sub-dataset for each subject.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    dataset = dl.create(tmp_path / "origin", result_renderer="disabled")
+    for subject in ("sub-01", "sub-02"):
+        subdataset = dataset.create(subject, result_renderer="disabled")
+        (subdataset.pathobj / "T1w.nii").write_text(subject)
+        (dataset.pathobj / f"{subject}_T1w.nii").write_text(subject)
+    dataset.save(recursive=True, result_renderer="disabled")
+    return AnyUrl(f"file://{dataset.path}")
+
+
+@pytest.mark.parametrize(
+    "pattern",
+    ["{subject}/T1w.nii", "{subject}/T1*.nii"],
+)
+def test_PatternDataladDataGrabber_subdatasets_search(
+    tmp_path: Path, subdatasets_uri: AnyUrl, pattern: str
+) -> None:
+    """Test searching the files of the sub-datasets raises an error.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    subdatasets_uri : pydantic.AnyUrl
+        The URI of the dataset with a sub-dataset for each subject.
+    pattern : str
+        The parametrized pattern, which searches the files of the
+        sub-datasets.
+
+    """
+    dg = PatternDataladDataGrabber(
+        uri=subdatasets_uri,
+        datadir=tmp_path / "clone",
+        types=["T1w"],
+        patterns={"T1w": {"pattern": pattern, "space": "native"}},
+        replacements=["subject"],
+    )
+    with pytest.raises(RuntimeError, match="searches the files of the"):
+        with dg:
+            pass
+    # The cloned dataset is removed
+    assert not (tmp_path / "clone").exists() or not any(
+        (tmp_path / "clone").iterdir()
+    )
+
+
+def test_PatternDataladDataGrabber_subdatasets_files(
+    tmp_path: Path, subdatasets_uri: AnyUrl
+) -> None:
+    """Test the patterns of a dataset with sub-datasets.
+
+    The files of the dataset can be searched, and the patterns without
+    wildcards can have files in the sub-datasets.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    subdatasets_uri : pydantic.AnyUrl
+        The URI of the dataset with a sub-dataset for each subject.
+
+    """
+    dg = PatternDataladDataGrabber(
+        uri=subdatasets_uri,
+        datadir=tmp_path / "clone",
+        types=["T1w"],
+        patterns={"T1w": {"pattern": "{subject}_T1w.nii", "space": "native"}},
+        replacements=["subject"],
+    )
+    with dg:
+        assert dg.get_elements() == ["sub-01", "sub-02"]
+        assert dg["sub-02"]["T1w"]["path"].read_text() == "sub-02"
+    # A pattern without wildcards in a sub-dataset
+    dg = PatternDataladDataGrabber(
+        uri=subdatasets_uri,
+        datadir=tmp_path / "clone",
+        types=["T1w"],
+        patterns={
+            "T1w": {
+                "pattern": "{subject}_T1w.nii",
+                "space": "native",
+                "mask": {"pattern": "sub-01/T1w.nii", "space": "native"},
+            }
+        },
+        replacements=["subject"],
+    )
+    with dg:
+        assert dg.get_elements() == ["sub-01", "sub-02"]

{junifer-0.0.8.dev314 → junifer-0.0.8.dev318/junifer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.8.dev314
+Version: 0.0.8.dev318
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

{junifer-0.0.8.dev314 → junifer-0.0.8.dev318}/junifer.egg-info/SOURCES.txt

@@ -142,6 +142,9 @@ docs/changes/newsfragments/525.change.4
 docs/changes/newsfragments/525.doc
 docs/changes/newsfragments/525.enh
 docs/changes/newsfragments/525.feature
+docs/changes/newsfragments/528.bugfix
+docs/changes/newsfragments/528.change
+docs/changes/newsfragments/528.doc
 docs/extending/confounds_format.rst
 docs/extending/coordinates.rst
 docs/extending/data_dump_asset.rst

{junifer-0.0.8.dev314 → junifer-0.0.8.dev318}/junifer.egg-info/scm_file_list.json

@@ -126,6 +126,9 @@
     "docs/changes/newsfragments/525.doc",
     "docs/changes/newsfragments/525.enh",
     "docs/changes/newsfragments/525.feature",
+    "docs/changes/newsfragments/528.bugfix",
+    "docs/changes/newsfragments/528.change",
+    "docs/changes/newsfragments/528.doc",
     "docs/conf.py",
     "docs/contribution.rst",
     "docs/extending/confounds_format.rst",

junifer-0.0.8.dev318/junifer.egg-info/scm_version.json

@@ -0,0 +1,8 @@
+{
+  "tag": "0.0.7",
+  "distance": 318,
+  "node": "gd4a9dcd3af71183d3e209a5a12a76166af95fc98",
+  "dirty": false,
+  "branch": "main",
+  "node_date": "2026-10-14"
+}

junifer-0.0.8.dev314/junifer/datagrabber/pattern_datalad.py

@@ -1,63 +0,0 @@
-"""Provide concrete implementation for pattern + datalad based DataGrabber."""
-
-# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
-#          Leonard Sasse <l.sasse@fz-juelich.de>
-#          Synchon Mandal <s.mandal@fz-juelich.de>
-# License: AGPL
-
-from pydantic import ConfigDict
-
-from ..api.decorators import register_datagrabber
-from .base import logger
-from .datalad_base import DataladDataGrabber
-from .pattern import PatternDataGrabber
-
-
-__all__ = ["PatternDataladDataGrabber"]
-
-
-@register_datagrabber
-class PatternDataladDataGrabber(DataladDataGrabber, PatternDataGrabber):
-    """Concrete implementation for pattern and datalad based data fetching.
-
-    Implements a DataGrabber that gets data from a datalad sibling,
-    interpreting patterns.
-
-    Parameters
-    ----------
-    uri : pydantic.AnyUrl
-        URI of the datalad sibling.
-    types : enum:`.DataType` or list of variants
-        The data type(s) to grab.
-    patterns : ``DataGrabberPatterns``
-        The datagrabber patterns. Check :class:`DataTypeSchema` for the schema.
-    replacements : list of str
-        All possible replacements in ``patterns.<data_type>.pattern``.
-    rootdir : pathlib.Path, optional
-        The path within the datalad dataset to the root directory
-        (default Path(".")).
-    confounds_format : :enum:`.ConfoundsFormat` or None, optional
-        The format of the confounds for the dataset (default None).
-    datadir : pathlib.Path, optional
-        That path where the datalad dataset will be cloned.
-        If not specified, the datalad dataset will be cloned into a temporary
-        directory.
-
-
-    See Also
-    --------
-    DataladDataGrabber:
-        Abstract base class for datalad-based data fetching.
-    PatternDataGrabber:
-        Concrete implementation for pattern-based data fetching.
-
-    """
-
-    model_config = ConfigDict(extra="allow")
-
-    def validate_datagrabber_params(self) -> None:
-        """Run extra logical validation for datagrabber."""
-        super().validate_datagrabber_params()
-        logger.debug("Initializing PatternDataladDataGrabber")
-        for key, val in self.__pydantic_extra__.items():
-            logger.debug(f"\t{key} = {val}")

junifer-0.0.8.dev314/junifer.egg-info/scm_version.json

@@ -1,8 +0,0 @@
-{
-  "tag": "0.0.7",
-  "distance": 314,
-  "node": "g968a03e78f4e60e1f40e75421a76b8b528206db0",
-  "dirty": false,
-  "branch": "main",
-  "node_date": "2026-10-10"
-}