junifer 0.0.8.dev314__tar.gz → 0.0.8.dev323__tar.gz

{junifer-0.0.8.dev314/junifer.egg-info → junifer-0.0.8.dev323}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.8.dev314
+Version: 0.0.8.dev323
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

{junifer-0.0.8.dev314 → junifer-0.0.8.dev323}/docs/_static/css/custom.css

@@ -44,3 +44,26 @@ body:not([data-theme="light"]) .mermaid svg .marker{
   stroke: white !important;
   fill: white !important;
 }
+
+/* Use the full width of the page for the content (e.g., for wide tables):
+   Furo centers the content with a fixed width */
+.sidebar-drawer {
+  width: 15em;
+}
+
+/* The content takes the width left (not the width of its tables, which do not
+   wrap, and scroll instead), so the table of contents on the right is not
+   narrowed or moved out of the page */
+.main {
+  min-width: 0;
+}
+
+.content {
+  flex: 1 1 0;
+  min-width: 0;
+  width: auto;
+}
+
+.toc-drawer {
+  flex-shrink: 0;
+}

{junifer-0.0.8.dev314 → junifer-0.0.8.dev323}/docs/builtin.rst

@@ -197,6 +197,14 @@ Marker
 Available
 ~~~~~~~~~
 
+The markers with a module in the *With* column need optional dependencies
+(e.g., ``neurokit2`` for ``junifer.markers.complexity``), so they are only
+available in the YAML file if their module is added to ``with``:
+
+.. code-block:: yaml
+
+  with: junifer.markers.complexity
+
 .. list-table::
    :widths: auto
    :header-rows: 1
@@ -205,123 +213,156 @@ Available
      - Description
      - State
      - Version Added
+     - With
    * - :class:`.ParcelAggregation`
      - Apply parcellation and perform aggregation function
      - Done
      - 0.0.1
+     -
    * - :class:`.SphereAggregation`
      - Spherical aggregation using mean
      - Done
      - 0.0.1
+     -
    * - :class:`.MapsAggregation`
      - Apply maps (probabilistic atlas) and extract mean time series per region
      - Done
      - 0.0.7
+     -
    * - :class:`.FunctionalConnectivityParcels`
      - Compute functional connectivity over parcellation
      - Done
      - 0.0.1
+     -
    * - :class:`.FunctionalConnectivitySpheres`
      - Compute functional connectivity over spheres placed on coordinates
      - Done
      - 0.0.1
+     -
    * - :class:`.FunctionalConnectivityMaps`
      - Compute functional connectivity over maps (probabilistic atlas)
      - Done
      - 0.0.7
+     -
    * - :class:`.CrossParcellationFC`
      - Compute functional connectivity across two parcellations
      - Done
      - 0.0.1
+     -
    * - :class:`.RSSETSMarker`
      - Compute root sum of squares of edgewise timeseries
      - Done
      - 0.0.1
+     -
    * - :class:`.ReHoParcels`
      - Calculate regional homogeneity over parcellation
      - Done
      - 0.0.1
+     -
    * - :class:`.ReHoSpheres`
      - Calculate regional homogeneity over spheres placed on coordinates
      - Done
      - 0.0.1
+     -
    * - :class:`.ReHoMaps`
      - Calculate regional homogeneity over maps (probabilistic atlas)
      - Done
      - 0.0.7
+     -
    * - :class:`.ALFFParcels`
      - Calculate (f)ALFF and aggregate using parcellations
      - Done
      - 0.0.1
+     -
    * - :class:`.ALFFSpheres`
      - Calculate (f)ALFF and aggregate using spheres placed on coordinates
      - Done
      - 0.0.1
+     -
    * - :class:`.ALFFMaps`
      - Calculate (f)ALFF and aggregate using maps (probabilistic atlas)
      - Done
      - 0.0.7
+     -
    * - :class:`.EdgeCentricFCParcels`
      - | Calculate edge-centric functional connectivity over parcellation, as
        | found in
        | `Jo et al. (2021) <https://doi.org/10.1016/j.neuroimage.2021.118204>`_
      - Done
      - 0.0.2
+     -
    * - :class:`.EdgeCentricFCSpheres`
      - | Calculate edge-centric functional connectivity over spheres placed on
        | coordinates, as found in
        | `Jo et al. (2021) <https://doi.org/10.1016/j.neuroimage.2021.118204>`_
      - Done
      - 0.0.2
+     -
    * - :class:`.EdgeCentricFCMaps`
      - | Calculate edge-centric functional connectivity over maps (probabilistic atlas),
        | as found in
        | `Jo et al. (2021) <https://doi.org/10.1016/j.neuroimage.2021.118204>`_
      - Done
      - 0.0.7
+     -
    * - :class:`.TemporalSNRParcels`
      - Calculate temporal signal-to-noise ratio using parcellations
      - Done
      - 0.0.2
+     -
    * - :class:`.TemporalSNRSpheres`
      - | Calculate temporal signal-to-noise ratio using spheres placed on
        | coordinates
      - Done
      - 0.0.2
+     -
    * - :class:`.TemporalSNRMaps`
      - Calculate temporal signal-to-noise ratio using maps (probabilistic atlas)
      - Done
      - 0.0.7
+     -
    * - :class:`.HurstExponent`
      - | Calculate Hurst exponent of a time series as found in
        | `Peng et al. (1995) <https://doi.org/10.1063/1.166141>`_
      - Done
      - 0.0.4
+     - ``junifer.markers.complexity``
    * - :class:`.MultiscaleEntropyAUC`
      - | Calculate AUC of multiscale entropy of a time series as found in
        | `Costa et al. (2002) <https://doi.org/10.1103/PhysRevLett.89.068102>`_
      - Done
      - 0.0.4
+     - ``junifer.markers.complexity``
    * - :class:`.PermEntropy`
      - | Calculate permutation entropy of a time series as found in
        | `Bandt at al. (2002) <https://doi.org/10.1103/PhysRevLett.88.174102>`_
      - Done
      - 0.0.4
+     - ``junifer.markers.complexity``
    * - :class:`.RangeEntropy`
      - | Calculate range entropy of a time series as found in
        | `Omidvarnia et al. (2018) <https://doi.org/10.3390/e20120962>`_
      - Done
      - 0.0.4
+     - ``junifer.markers.complexity``
    * - :class:`.RangeEntropyAUC`
      - | Calculate AUC of range entropy of a time series as found in
        | `Omidvarnia et al. (2018) <https://doi.org/10.3390/e20120962>`_
      - Done
      - 0.0.4
+     - ``junifer.markers.complexity``
    * - :class:`.SampleEntropy`
      - | Calculate sample entropy of a time series as found in
        | `Richman et al. (2000) <https://doi.org/10.1152/ajpheart.2000.278.6.H2039>`_
      - Done
      - 0.0.4
+     - ``junifer.markers.complexity``
+   * - :class:`.WeightedPermEntropy`
+     - | Calculate weighted permutation entropy of a time series as found in
+       | `Fadlallah et al. (2013) <https://doi.org/10.1103/PhysRevE.87.022911>`_
+     - Done
+     - 0.0.4
+     - ``junifer.markers.complexity``
 
 Planned
 ~~~~~~~

junifer-0.0.8.dev323/docs/changes/newsfragments/528.bugfix

@@ -0,0 +1 @@
+Fix :class:`.DataladHCP1200` listing only the subjects already installed (the subjects are sub-datasets) and not finding the BOLD data of the subjects not installed, by listing all the subjects without searching the files and building the paths of the BOLD data without wildcards by `Fede Raimondo`_

junifer-0.0.8.dev323/docs/changes/newsfragments/528.change

@@ -0,0 +1 @@
+Raise an error when installing the dataset of a :class:`.PatternDataladDataGrabber` if a pattern searches the files of a sub-dataset, which are only there once the sub-dataset is installed, instead of not finding the elements by `Fede Raimondo`_

junifer-0.0.8.dev323/docs/changes/newsfragments/528.doc

@@ -0,0 +1 @@
+Document that the DataGrabbers with files in datalad sub-datasets must list their elements and build the paths without searching the files by `Fede Raimondo`_

junifer-0.0.8.dev323/docs/changes/newsfragments/529.bugfix

@@ -0,0 +1 @@
+Fix the Maps markers (e.g., :class:`.MapsAggregation`) not being available in the YAML file without importing their modules by `Fede Raimondo`_

junifer-0.0.8.dev323/docs/changes/newsfragments/529.bugfix.1

@@ -0,0 +1 @@
+Fix the complexity markers (e.g., :class:`.HurstExponent`) not being available in the YAML file with ``with: junifer.markers.complexity`` by `Fede Raimondo`_

junifer-0.0.8.dev323/docs/changes/newsfragments/529.bugfix.2

@@ -0,0 +1 @@
+Fix ``junifer run`` and ``junifer list-elements`` ignoring the ``elements`` of the YAML file when ``--element`` is not given by `Fede Raimondo`_

junifer-0.0.8.dev323/docs/changes/newsfragments/529.bugfix.3

@@ -0,0 +1 @@
+Fix the AFNI smoothing of :class:`.Smoothing` leaving ``3dFWHMx.1D`` and ``3dFWHMx.1D.png`` in the current directory by `Fede Raimondo`_

junifer-0.0.8.dev323/docs/changes/newsfragments/529.doc

@@ -0,0 +1 @@
+Fix the ``collect_task`` option of the HTCondor queue being documented as ``collect``, document the ``elements`` key of the YAML file and the ``with`` needed by the markers, and use the full width of the page for the content by `Fede Raimondo`_

junifer-0.0.8.dev323/docs/changes/newsfragments/529.enh

@@ -0,0 +1 @@
+Add the ``cwd`` parameter to :func:`.run_ext_cmd` to run external commands in a directory by `Fede Raimondo`_

{junifer-0.0.8.dev314 → junifer-0.0.8.dev323}/docs/extending/datagrabber.rst

@@ -399,6 +399,17 @@ This approach can be used directly from the YAML, like so:
      uri: "https://cerebra.fz-juelich.de/junifer/datalad-example-bids-ses.git"
      rootdir: example_bids_ses
 
+.. important::
+
+   The elements are found by searching the files of the dataset, but the files
+   of a datalad *sub-dataset* are only there once the sub-dataset is installed.
+   If the patterns search the files of a sub-dataset (e.g., a dataset with a
+   sub-dataset for each subject), :class:`.PatternDataladDataGrabber` raises
+   an error when installing the dataset. These DataGrabbers must implement
+   ``get_elements`` without searching the files (e.g., from the names of the
+   sub-datasets) and build the paths in the sub-datasets without wildcards,
+   as :class:`.DataladHCP1200` does.
+
 .. _extending_datagrabbers_path_expansion:
 
 Advanced: Using Unix-like path expansion directives

{junifer-0.0.8.dev314 → junifer-0.0.8.dev323}/docs/using/queueing.rst

@@ -76,7 +76,7 @@ The following parameters are available for HTCondor:
   temporary directory.
 * ``extra_preamble``: Extra lines to be added to the HTCondor submit file. This
   can be used to add extra parameters to the job, such as ``requirements``.
-* ``collect``: This parameter allows to include a collect to the DAG to collect
+* ``collect_task``: This parameter allows to include a collect to the DAG to collect
   the results once all of the individual element jobs are finished. This is
   useful if you want to run a ``junifer collect`` job only once all of the
   individual element jobs are finished. Valid options are:
@@ -101,7 +101,7 @@ Example in YAML:
       shell: zsh
     mem: 8G
     disk: 2G
-    collect: "yes"  # wrap it in string to avoid boolean
+    collect_task: "yes"  # wrap it in string to avoid boolean
 
 Alternatively, if you use ``venv``, the YAML would look like so:
 
@@ -116,7 +116,7 @@ Alternatively, if you use ``venv``, the YAML would look like so:
       shell: zsh
     mem: 8G
     disk: 2G
-    collect: "yes"  # wrap it in string to avoid boolean
+    collect_task: "yes"  # wrap it in string to avoid boolean
 
 .. _queueing_local:
 

{junifer-0.0.8.dev314 → junifer-0.0.8.dev323}/docs/using/running.rst

@@ -91,6 +91,16 @@ and then the ``elements.txt`` would be like so:
 
     sub-01
 
+The elements can also be set in the YAML file, with the ``elements`` key. They
+are used by ``run``, ``queue`` and ``list-elements`` when ``--element`` is not
+given (``--element`` has precedence over the YAML file):
+
+.. code-block:: yaml
+
+    elements:
+      - [sub-01, ses-01]
+      - [sub-02, ses-01]
+
 .. _running_elements:
 
 Complete and partial elements

{junifer-0.0.8.dev314 → junifer-0.0.8.dev323}/junifer/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.0.8.dev314'
-__version_tuple__ = version_tuple = (0, 0, 8, 'dev314')
+__version__ = version = '0.0.8.dev323'
+__version_tuple__ = version_tuple = (0, 0, 8, 'dev323')
 
-__commit_id__ = commit_id = 'g968a03e78'
+__commit_id__ = commit_id = 'ga9cb942ea'

{junifer-0.0.8.dev314 → junifer-0.0.8.dev323}/junifer/cli/parser.py

@@ -16,7 +16,7 @@ __all__ = ["parse_elements"]
 
 
 def parse_elements(element: tuple[str, ...], config: dict) -> Elements | None:
-    """Parse elements from cli.
+    """Parse elements from cli, or from the configuration if not in the cli.
 
     Parameters
     ----------
@@ -28,13 +28,13 @@ def parse_elements(element: tuple[str, ...], config: dict) -> Elements | None:
     Returns
     -------
     list or None
-        The element(s) as list or None.
+        The element(s) as list, or None (all the elements) if they are not in
+        the command-line options nor in the configuration.
 
     Raises
     ------
     ValueError
-        If no element is found either in the command-line options or
-        the configuration file.
+        If the ``elements`` key of the configuration has no value.
 
     Warns
     -----
@@ -44,9 +44,22 @@ def parse_elements(element: tuple[str, ...], config: dict) -> Elements | None:
 
     """
     logger.debug(f"Parsing elements: {element}")
-    # Early return None to continue with all elements
+    # Use the elements of the configuration if not in the command line
     if not element:
-        return None
+        if "elements" not in config:
+            # None to continue with all elements
+            return None
+        elements = config["elements"]
+        if elements is None:
+            raise_error(
+                "The `elements` key is set in the configuration, but its value"
+                " is `None`. It is likely that there is an empty `elements` "
+                "section in the yaml configuration file."
+            )
+        # A single element, and the elements with multiple values as tuples
+        if not isinstance(elements, list):
+            elements = [elements]
+        return [tuple(x) if isinstance(x, list) else x for x in elements]
     # Check if the element is a file for single element;
     # if yes, then parse elements from it
     if len(element) == 1 and Path(element[0]).resolve().is_file():
@@ -55,7 +68,7 @@ def parse_elements(element: tuple[str, ...], config: dict) -> Elements | None:
         # Process multi-keyed elements
         elements = [tuple(x.split(",")) if "," in x else x for x in element]
     logger.debug(f"Parsed elements: {elements}")
-    if elements is not None and "elements" in config:
+    if "elements" in config:
         warn_with_log(
             "One or more elements have been specified in both the command "
             "line and in the config file. The command line has precedence "
@@ -64,15 +77,6 @@ def parse_elements(element: tuple[str, ...], config: dict) -> Elements | None:
             "the configuration file will be ignored. To remove this warning, "
             "please remove the `elements` item from the configuration file."
         )
-    elif elements is None:
-        # Check in config
-        elements = config.get("elements", None)
-        if elements is None:
-            raise_error(
-                "The `elements` key is set in the configuration, but its value"
-                " is `None`. It is likely that there is an empty `elements` "
-                "section in the yaml configuration file."
-            )
     return elements
 
 

{junifer-0.0.8.dev314 → junifer-0.0.8.dev323}/junifer/cli/tests/test_cli.py

@@ -22,7 +22,7 @@ from junifer.cli.cli import (
     selftest,
     wtf,
 )
-from junifer.cli.parser import _parse_elements_file
+from junifer.cli.parser import _parse_elements_file, parse_elements
 from junifer.pipeline import WorkDirManager
 
 
@@ -205,6 +205,47 @@ def test_multi_element_access(
     assert read_elements == expected_list
 
 
+@pytest.mark.parametrize(
+    "element, config, expected",
+    [
+        ((), {}, None),
+        ((), {"elements": ["sub-01", "sub-02"]}, ["sub-01", "sub-02"]),
+        ((), {"elements": "sub-01"}, ["sub-01"]),
+        ((), {"elements": [["sub-01", "ses-01"]]}, [("sub-01", "ses-01")]),
+        (("sub-01", "sub-02,ses-01"), {}, ["sub-01", ("sub-02", "ses-01")]),
+    ],
+)
+def test_parse_elements(
+    element: tuple[str, ...], config: dict, expected: list | None
+) -> None:
+    """Test parsing the elements from the command line or the configuration.
+
+    Parameters
+    ----------
+    element : tuple of str
+        The parametrized elements of the command line.
+    config : dict
+        The parametrized configuration.
+    expected : list or None
+        The parametrized elements to expect.
+
+    """
+    assert parse_elements(element, config) == expected
+
+
+def test_parse_elements_command_line_precedence() -> None:
+    """Test the elements of the command line have precedence."""
+    with pytest.warns(RuntimeWarning, match="command line has precedence"):
+        elements = parse_elements(("sub-01",), {"elements": ["sub-02"]})
+    assert elements == ["sub-01"]
+
+
+def test_parse_elements_empty_config() -> None:
+    """Test an empty elements key in the configuration."""
+    with pytest.raises(ValueError, match="its value is `None`"):
+        parse_elements((), {"elements": None})
+
+
 def test_queue(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     """Test queue command.
 

{junifer-0.0.8.dev314 → junifer-0.0.8.dev323}/junifer/datagrabber/hcp1200/datalad_hcp1200.py

@@ -5,12 +5,14 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+from itertools import product
 from pathlib import Path
 from typing import Annotated, Literal
 
 from pydantic import AnyUrl, BeforeValidator
 
 from ...api.decorators import register_datagrabber
+from ...typing import Elements
 from ...utils import ensure_list
 from ..base import DataType
 from ..datalad_base import DataladDataGrabber
@@ -67,3 +69,29 @@ class DataladHCP1200(DataladDataGrabber, HCP1200):
     def skip_file_check(self) -> bool:
         """Skip file check existence."""
         return True
+
+    def get_elements(self) -> Elements:
+        """Implement fetching list of elements in the dataset.
+
+        The subjects are sub-datasets, whose files are only there once they
+        are installed, so the files are not searched: the elements are all
+        the subjects (the directories of the dataset) with the tasks and
+        phase encodings to grab, even if some of them have no data.
+
+        Returns
+        -------
+        list
+            The list of elements that can be grabbed in the dataset.
+
+        """
+        subjects = sorted(
+            x.name
+            for x in self.fulldir.iterdir()
+            if x.is_dir() and not x.name.startswith(".")
+        )
+        values = {"subject": subjects, **self.get_replacement_values()}
+        element_keys = self.get_element_keys()
+        elements = list(product(*(values[k] for k in element_keys)))
+        if len(element_keys) == 1:
+            return [element[0] for element in elements]
+        return elements

{junifer-0.0.8.dev314 → junifer-0.0.8.dev323}/junifer/datagrabber/hcp1200/hcp1200.py

@@ -162,3 +162,29 @@ class HCP1200(PatternDataGrabber):
             "pattern"
         ].replace("{suffix}", suffix)
         super().validate_datagrabber_params()
+
+    def _replace_patterns_glob(self, element: dict, pattern: str) -> str:
+        """Replace ``pattern`` with the ``element`` so it can be globbed.
+
+        The BOLD data of the resting-state tasks is in ``rfMRI_*`` and of the
+        other tasks in ``tfMRI_*``, so the path of an element is built
+        without searching the files, which are in sub-datasets in
+        :class:`.DataladHCP1200`.
+
+        Parameters
+        ----------
+        element : dict
+            The element to be used in the replacement.
+        pattern : str
+            The pattern to be replaced.
+
+        Returns
+        -------
+        str
+            The pattern with the element replaced.
+
+        """
+        if "task" in element:
+            prefix = "rfMRI" if "REST" in element["task"] else "tfMRI"
+            pattern = pattern.replace("?fMRI_", f"{prefix}_")
+        return super()._replace_patterns_glob(element, pattern)

{junifer-0.0.8.dev314 → junifer-0.0.8.dev323}/junifer/datagrabber/hcp1200/tests/test_hcp1200.py

@@ -8,6 +8,7 @@ import tempfile
 from collections.abc import Iterable
 from pathlib import Path
 
+import datalad.api as dl
 import pytest
 from pydantic import AnyUrl
 
@@ -320,3 +321,63 @@ def test_HCP1200_incorrect_access_icafix(
             tasks=tasks,
             ica_fix=ica_fix,
         )
+
+
+def test_DataladHCP1200_subdatasets(tmp_path: Path) -> None:
+    """Test DataladHCP1200 with a sub-dataset for each subject, as HCP.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    dataset = dl.create(tmp_path / "origin", result_renderer="disabled")
+    for subject in ("100206", "100307"):
+        subdataset = dataset.create(subject, result_renderer="disabled")
+        for folder in ("rfMRI_REST1_LR", "tfMRI_WM_RL"):
+            path = (
+                subdataset.pathobj
+                / "MNINonLinear"
+                / "Results"
+                / folder
+                / f"{folder}.nii.gz"
+            )
+            path.parent.mkdir(parents=True)
+            path.write_text(f"{subject} {folder}")
+        path = subdataset.pathobj / "T1w" / "T1w_acpc_dc_restore.nii.gz"
+        path.parent.mkdir(parents=True)
+        path.write_text(f"{subject} T1w")
+    dataset.save(recursive=True, result_renderer="disabled")
+    dg = DataladHCP1200(
+        uri=AnyUrl(f"file://{dataset.path}"),
+        datadir=tmp_path / "clone",
+        rootdir=Path("."),
+        types=["BOLD", "T1w"],
+        tasks=["REST1", "WM"],
+        phase_encodings=["LR", "RL"],
+    )
+    with config_override("datagrabber.skipidcheck", True), dg:
+        # All the subjects with the tasks and phase encodings, without
+        # installing the sub-datasets
+        assert dg.get_elements() == [
+            (subject, task, phase_encoding)
+            for subject in ("100206", "100307")
+            for task in ("REST1", "WM")
+            for phase_encoding in ("LR", "RL")
+        ]
+        assert not (dg.fulldir / "100206" / "T1w").exists()
+        out = dg[("100307", "WM", "RL")]
+        assert out["BOLD"]["path"].read_text() == "100307 tfMRI_WM_RL"
+        assert out["T1w"]["path"].read_text() == "100307 T1w"
+        out = dg[("100206", "REST1", "LR")]
+        assert out["BOLD"]["path"].read_text() == "100206 rfMRI_REST1_LR"
+    dg = DataladHCP1200(
+        uri=AnyUrl(f"file://{dataset.path}"),
+        datadir=tmp_path / "clone",
+        rootdir=Path("."),
+        types=["T1w"],
+    )
+    with config_override("datagrabber.skipidcheck", True), dg:
+        assert dg.get_elements() == ["100206", "100307"]
+        assert dg["100206"]["T1w"]["path"].read_text() == "100206 T1w"