junifer 0.0.7.dev332__py3-none-any.whl → 0.0.8__py3-none-any.whl

junifer/testing/tests/test_partlycloudytesting_datagrabber.py

@@ -3,9 +3,19 @@
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 # License: AGPL
 
+import nibabel as nib
+import numpy as np
+import pandas as pd
+import pytest
+from numpy.testing import assert_array_equal
+from pandas.testing import assert_frame_equal
+
 from junifer.testing.datagrabbers import PartlyCloudyTestingDataGrabber
 
 
+pytestmark = pytest.mark.external
+
+
 def test_PartlyCloudyTestingDataGrabber() -> None:
     """Test PartlyCloudyTestingDataGrabber."""
     expected_elements = [
@@ -38,3 +48,45 @@ def test_PartlyCloudyTestingDataGrabber() -> None:
         out = dg["sub-01"]
         assert "format" in out["BOLD"]["confounds"]
         assert "fmriprep" == out["BOLD"]["confounds"]["format"]
+
+
+@pytest.mark.parametrize("reduce_confounds", [True, False])
+def test_PartlyCloudyTestingDataGrabber_n_timepoints(
+    reduce_confounds: bool,
+) -> None:
+    """Test PartlyCloudyTestingDataGrabber with fewer timepoints.
+
+    Parameters
+    ----------
+    reduce_confounds : bool
+        The parametrized confounds reduction.
+
+    """
+    with PartlyCloudyTestingDataGrabber(
+        reduce_confounds=reduce_confounds
+    ) as dg:
+        full = dg["sub-01"]["BOLD"]
+    with PartlyCloudyTestingDataGrabber(
+        reduce_confounds=reduce_confounds, n_timepoints=50
+    ) as dg:
+        out = dg["sub-01"]["BOLD"]
+
+    full_img = nib.load(full["path"])
+    out_img = nib.load(out["path"])
+    assert out_img.shape == (*full_img.shape[:3], 50)
+    assert_array_equal(
+        np.asanyarray(out_img.dataobj),
+        np.asanyarray(full_img.dataobj[..., :50]),
+    )
+
+    full_confounds = pd.read_csv(full["confounds"]["path"], sep="\t")
+    out_confounds = pd.read_csv(out["confounds"]["path"], sep="\t")
+    assert out_confounds.shape == (50, full_confounds.shape[1])
+    assert_frame_equal(out_confounds, full_confounds.iloc[:50])
+
+
+def test_PartlyCloudyTestingDataGrabber_n_subjects() -> None:
+    """Test PartlyCloudyTestingDataGrabber with fewer subjects."""
+    with PartlyCloudyTestingDataGrabber(n_subjects=3) as dg:
+        assert dg.get_elements() == ["sub-01", "sub-02", "sub-03"]
+        assert dg["sub-03"]["BOLD"]["path"].exists()

junifer/testing/tests/test_spmauditory_datagrabber.py

@@ -3,9 +3,15 @@
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 # License: AGPL
 
+import nibabel as nib
+import pytest
+
 from junifer.testing.datagrabbers import SPMAuditoryTestingDataGrabber
 
 
+pytestmark = pytest.mark.external
+
+
 def test_SPMAuditoryTestingDataGrabber() -> None:
     """Test SPMAuditoryTestingDataGrabber."""
     expected_elements = [
@@ -31,3 +37,13 @@ def test_SPMAuditoryTestingDataGrabber() -> None:
         assert "T1w" in out
         assert out["T1w"]["path"].exists()
         assert out["T1w"]["path"].is_file()
+
+        assert nib.load(out["BOLD"]["path"]).shape == (64, 64, 64, 84)
+        assert nib.load(out["T1w"]["path"]).shape == (256, 256, 54)
+
+
+def test_SPMAuditoryTestingDataGrabber_n_timepoints() -> None:
+    """Test SPMAuditoryTestingDataGrabber with fewer timepoints."""
+    with SPMAuditoryTestingDataGrabber(n_timepoints=10) as dg:
+        out = dg["sub001"]
+        assert nib.load(out["BOLD"]["path"]).shape == (64, 64, 64, 10)

junifer/testing/tests/test_testing_registry.py

@@ -5,12 +5,43 @@
 # License: AGPL
 
 from junifer.pipeline import PipelineComponentRegistry
+from junifer.testing.datagrabbers import (
+    ADHDTestingDataGrabber,
+    OasisVBMTestingDataGrabber,
+    PartlyCloudyTestingDataGrabber,
+    SPMAuditoryTestingDataGrabber,
+    UCLACNPVBMTestingDataGrabber,
+)
 
 
 def test_testing_registry() -> None:
     """Test testing registry."""
+    for dg in [
+        ADHDTestingDataGrabber,
+        OasisVBMTestingDataGrabber,
+        SPMAuditoryTestingDataGrabber,
+        PartlyCloudyTestingDataGrabber,
+        UCLACNPVBMTestingDataGrabber,
+    ]:
+        PipelineComponentRegistry().register(
+            step="datagrabber",
+            klass=dg,
+        )
     assert {
+        "ADHDTestingDataGrabber",
         "OasisVBMTestingDataGrabber",
         "SPMAuditoryTestingDataGrabber",
         "PartlyCloudyTestingDataGrabber",
+        "UCLACNPVBMTestingDataGrabber",
     }.issubset(set(PipelineComponentRegistry().step_components("datagrabber")))
+    for dg in [
+        ADHDTestingDataGrabber,
+        OasisVBMTestingDataGrabber,
+        SPMAuditoryTestingDataGrabber,
+        PartlyCloudyTestingDataGrabber,
+        UCLACNPVBMTestingDataGrabber,
+    ]:
+        PipelineComponentRegistry().deregister(
+            step="datagrabber",
+            klass=dg,
+        )

junifer/testing/tests/test_testing_utils.py

@@ -0,0 +1,35 @@
+"""Provide tests for testing utils."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+# License: AGPL
+
+import pytest
+
+from junifer.testing import config_override
+from junifer.utils import config
+
+
+def test_config_override_unset() -> None:
+    """Test config_override deletes a previously unset key."""
+    assert config.get("testing.override") is None
+    with config_override("testing.override", True):
+        assert config.get("testing.override") is True
+    assert "testing.override" not in config._config
+
+
+def test_config_override_restore() -> None:
+    """Test config_override restores a previously set value."""
+    config.set("testing.override", 1)
+    try:
+        with config_override("testing.override", 2):
+            assert config.get("testing.override") == 2
+        assert config.get("testing.override") == 1
+    finally:
+        config.delete("testing.override")
+
+
+def test_config_override_error() -> None:
+    """Test config_override restores the state on error."""
+    with pytest.raises(RuntimeError), config_override("testing.override", 3):
+        raise RuntimeError()
+    assert "testing.override" not in config._config

junifer/testing/tests/test_uclacnpvbmtesting_datagrabber.py

@@ -0,0 +1,81 @@
+"""Provide tests for UCLACNPVBMTestingDataGrabber."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+# License: AGPL
+
+import nibabel as nib
+import numpy as np
+import pytest
+from numpy.testing import assert_array_almost_equal
+from templateflow import api as tflow
+
+from junifer.testing.datagrabbers import UCLACNPVBMTestingDataGrabber
+
+
+pytestmark = pytest.mark.external
+
+
+def test_UCLACNPVBMTestingDataGrabber() -> None:
+    """Test UCLACNPVBMTestingDataGrabber."""
+    with UCLACNPVBMTestingDataGrabber() as dg:
+        elements = dg.get_elements()
+        assert len(elements) == 10
+        out = dg[elements[0]]
+        assert "VBM_GM" in out
+        assert out["VBM_GM"]["path"].is_file()
+        assert out["VBM_GM"]["space"] == "MNI152NLin2009cAsym"
+
+        # Resampled to the templateflow grid, so templates and masks do not
+        # need resampling
+        img = nib.load(out["VBM_GM"]["path"])
+        template = nib.load(
+            tflow.get(
+                "MNI152NLin2009cAsym",
+                resolution=2,
+                suffix="T1w",
+                desc=None,
+                extension="nii.gz",
+            )
+        )
+        assert img.shape == template.shape[:3]
+        assert_array_almost_equal(img.affine, template.affine)
+
+        participants = dg.get_participants()
+        assert list(participants.index) == elements
+        assert {"age", "gender"} <= set(participants.columns)
+
+
+def test_UCLACNPVBMTestingDataGrabber_native() -> None:
+    """Test UCLACNPVBMTestingDataGrabber at the native resolution."""
+    with UCLACNPVBMTestingDataGrabber(resolution=1) as dg:
+        out = dg[dg.get_elements()[0]]
+        # No resampling at the native resolution
+        assert out["VBM_GM"]["path"].name.endswith("_probtissue.nii.gz")
+        assert nib.load(out["VBM_GM"]["path"]).shape == (193, 229, 193)
+
+
+def test_UCLACNPVBMTestingDataGrabber_resample() -> None:
+    """Test UCLACNPVBMTestingDataGrabber resampling to a new resolution."""
+    with UCLACNPVBMTestingDataGrabber(resolution=4) as dg:
+        element = dg.get_elements()[0]
+        # Remove the cached map so it is resampled again (the CI image only
+        # has the default resolution)
+        native = dg._paths[element]
+        cached = native.with_name(
+            native.name.replace(".nii.gz", "_res-4mm.nii")
+        )
+        cached.unlink(missing_ok=True)
+
+        out = dg[element]
+        assert out["VBM_GM"]["path"] == cached
+        img = nib.load(cached)
+        assert img.shape == (49, 58, 49)
+        assert img.get_data_dtype() == np.float32
+        expected_affine = np.diag([4.0, 4.0, 4.0, 1.0])
+        expected_affine[:3, 3] = [-97.5, -133.5, -79.5]
+        assert_array_almost_equal(img.affine, expected_affine)
+
+        # Cached, so it is not resampled again
+        mtime = cached.stat().st_mtime_ns
+        assert dg[element]["VBM_GM"]["path"] == cached
+        assert cached.stat().st_mtime_ns == mtime

junifer/testing/utils.py

@@ -3,10 +3,15 @@
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 # License: AGPL
 
+from collections.abc import Generator
+from contextlib import contextmanager
 from pathlib import Path
 
+from ..utils import config
+from ..utils._config import ConfigVal
 
-__all__ = ["get_testing_data"]
+
+__all__ = ["config_override", "get_testing_data"]
 
 
 def get_testing_data(fname: str) -> Path:
@@ -27,3 +32,30 @@ def get_testing_data(fname: str) -> Path:
     if not t_path.exists():
         raise FileNotFoundError(f"File {fname} not found in testing data")
     return t_path.resolve()
+
+
+@contextmanager
+def config_override(key: str, val: ConfigVal) -> Generator[None, None, None]:
+    """Temporarily set a configuration parameter.
+
+    The previous state is restored on exit: the previous value if the key
+    was set, otherwise the key is deleted.
+
+    Parameters
+    ----------
+    key : str
+        The configuration key to set.
+    val : bool or int or float
+        The value to set ``key`` to.
+
+    """
+    missing = object()
+    prev = config.get(key, missing)
+    config.set(key, val)
+    try:
+        yield
+    finally:
+        if prev is missing:
+            config.delete(key)
+        else:
+            config.set(key, prev)

junifer/tests/test_stats.py

@@ -3,8 +3,6 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Optional
-
 import numpy as np
 import pytest
 from numpy.testing import assert_array_equal
@@ -25,7 +23,7 @@ from junifer.stats import count, get_aggfunc_by_name, select, winsorized_mean
         ("mode", {"keepdims": True}),
     ],
 )
-def test_get_aggfunc_by_name(name: str, params: Optional[dict]) -> None:
+def test_get_aggfunc_by_name(name: str, params: dict | None) -> None:
     """Test aggregation function retrieval by name.
 
     Parameters
@@ -41,8 +39,6 @@ def test_get_aggfunc_by_name(name: str, params: Optional[dict]) -> None:
 
 def test_get_aggfunc_by_name_errors() -> None:
     """Test aggregation function retrieval using wrong name."""
-    with pytest.raises(ValueError, match=r"unknown. Please provide any of"):
-        get_aggfunc_by_name(name="invalid", func_params=None)
 
     with pytest.raises(ValueError, match=r"list of limits"):
         get_aggfunc_by_name(name="winsorized_mean", func_params=None)

junifer/typing/__init__.pyi

@@ -12,6 +12,7 @@ __all__ = [
     "MarkerInOutMappings",
     "DataGrabberPatterns",
     "ConfigVal",
+    "SpaceLike",
     "Element",
     "Elements",
 ]
@@ -30,6 +31,7 @@ from ._typing import (
     MarkerInOutMappings,
     DataGrabberPatterns,
     ConfigVal,
+    SpaceLike,
     Element,
     Elements,
 )

junifer/typing/_typing.py

@@ -3,7 +3,16 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from collections.abc import MutableMapping, Sequence
+import sys
+
+
+if sys.version_info < (3, 12):  # pragma: no cover
+    from typing_extensions import TypedDict
+else:
+    from typing import TypedDict
+
+from collections.abc import Sequence
+from enum import Enum
 from typing import (
     TYPE_CHECKING,
     Union,
@@ -12,11 +21,12 @@ from typing import (
 
 if TYPE_CHECKING:
     from ..data import BasePipelineDataRegistry
-    from ..datagrabber import BaseDataGrabber
+    from ..datagrabber import BaseDataGrabber, DataType
     from ..datareader import DefaultDataReader
     from ..markers import BaseMarker
+    from ..pipeline import BaseDataDumpAsset, ExtDep
     from ..preprocess import BasePreprocessor
-    from ..storage import BaseFeatureStorage
+    from ..storage import BaseFeatureStorage, StorageType
 
 
 __all__ = [
@@ -34,11 +44,22 @@ __all__ = [
     "MarkerLike",
     "PipelineComponent",
     "PreprocessorLike",
+    "SpaceLike",
     "StorageLike",
 ]
 
 
-DataDumpAssetLike = type["DataDumpAssetLike"]
+class ExternalDependency(TypedDict):
+    name: "ExtDep"
+    commands: list[str]
+
+
+class ConditionalDependency(TypedDict):
+    using: object
+    depends_on: list[object]
+
+
+DataDumpAssetLike = type["BaseDataDumpAsset"]
 DataRegistryLike = type["BasePipelineDataRegistry"]
 DataGrabberLike = type["BaseDataGrabber"]
 PreprocessorLike = type["BasePreprocessor"]
@@ -52,20 +73,14 @@ PipelineComponent = Union[
     "StorageLike",
 ]
 Dependencies = set[str]
-ConditionalDependencies = Sequence[
-    MutableMapping[
-        str,
-        Union[
-            str,
-            PipelineComponent,
-            Sequence[str],
-            Sequence[PipelineComponent],
-        ],
-    ]
+ConditionalDependencies = list[ConditionalDependency]
+ExternalDependencies = list[ExternalDependency]
+MarkerInOutMappings = dict["DataType", dict[str, "StorageType"]]
+DataGrabberPatterns = dict[
+    str,
+    dict[str, dict[str, str] | str] | dict[str, str] | list[dict[str, str]],
 ]
-ExternalDependencies = Sequence[MutableMapping[str, Union[str, Sequence[str]]]]
-MarkerInOutMappings = MutableMapping[str, MutableMapping[str, str]]
-DataGrabberPatterns = dict[str, Union[dict[str, str], list[dict[str, str]]]]
-ConfigVal = Union[bool, int, float, str]
-Element = Union[str, tuple[str, ...]]
+ConfigVal = bool | int | float | str
+SpaceLike = str | Enum
+Element = str | tuple[str, ...]
 Elements = Sequence[Element]

junifer/utils/__init__.pyi

@@ -7,6 +7,9 @@ __all__ = [
     "warn_with_log",
     "run_ext_cmd",
     "deep_update",
+    "ensure_list",
+    "ensure_list_or_none",
+    "check_standardize",
     "yaml",
     "ConfigManager",
 ]
@@ -14,5 +17,11 @@ __all__ = [
 from .fs import make_executable
 from .logging import configure_logging, logger, raise_error, warn_with_log
 from ._config import config, ConfigManager
-from .helpers import run_ext_cmd, deep_update
+from .helpers import (
+    run_ext_cmd,
+    deep_update,
+    ensure_list,
+    ensure_list_or_none,
+    check_standardize,
+)
 from ._yaml import yaml

junifer/utils/_config.py

@@ -5,7 +5,6 @@
 # License: AGPL
 
 import os
-from typing import Optional
 
 from ..typing import ConfigVal
 from .logging import logger
@@ -61,7 +60,7 @@ class ConfigManager(metaclass=Singleton):
                 )
                 self._config[var_name] = var_value
 
-    def get(self, key: str, default: Optional[ConfigVal] = None) -> ConfigVal:
+    def get(self, key: str, default: ConfigVal | None = None) -> ConfigVal:
         """Get configuration parameter.
 
         Parameters

junifer/utils/helpers.py

@@ -6,14 +6,22 @@
 import collections.abc
 import subprocess
 import sys
+from pathlib import Path
+from typing import Any
 
 from .logging import logger, raise_error
 
 
-__all__ = ["deep_update", "run_ext_cmd"]
+__all__ = [
+    "check_standardize",
+    "deep_update",
+    "ensure_list",
+    "ensure_list_or_none",
+    "run_ext_cmd",
+]
 
 
-def run_ext_cmd(name: str, cmd: list[str]) -> None:
+def run_ext_cmd(name: str, cmd: list[str], cwd: Path | None = None) -> None:
     """Run external command via subprocess.
 
     Parameters
@@ -22,6 +30,10 @@ def run_ext_cmd(name: str, cmd: list[str]) -> None:
         The name of the command.
     cmd : list of str
         The command to run as list of string.
+    cwd : pathlib.Path or None, optional
+        The directory to run the command in, e.g., for commands that write
+        extra files in the current directory. If None, the current directory
+        is used (default None).
 
     Raises
     ------
@@ -40,6 +52,7 @@ def run_ext_cmd(name: str, cmd: list[str]) -> None:
         stderr=subprocess.STDOUT,
         shell=True,  # needed for respecting $PATH
         check=False,
+        cwd=cwd,
     )
     # Check for success or failure
     if process.returncode == 0:
@@ -82,3 +95,56 @@ def deep_update(d: dict, u: dict) -> dict:
         else:
             d[k] = v
     return d
+
+
+def ensure_list(value: Any) -> Any:
+    """Ensure list."""
+    if not isinstance(value, list):
+        return [value]
+    else:
+        return value
+
+
+def ensure_list_or_none(value: Any) -> Any:
+    """Ensure list or None."""
+    if value is None:
+        return None
+    if not isinstance(value, list):
+        return [value]
+    else:
+        return value
+
+
+def check_standardize(value: Any) -> Any:
+    """Check the value of a ``standardize`` parameter.
+
+    Booleans were replaced by the strategy names used by nilearn, so they
+    raise an error explaining the change instead of a generic validation
+    error.
+
+    Parameters
+    ----------
+    value : Any
+        The value of the parameter.
+
+    Returns
+    -------
+    Any
+        The value of the parameter.
+
+    Raises
+    ------
+    ValueError
+        If ``value`` is a boolean.
+
+    """
+    if isinstance(value, bool):
+        raise_error(
+            "`standardize` does not accept booleans anymore, following "
+            "nilearn, which is removing them: True used to z-score with the "
+            "population standard deviation, while nilearn now z-scores with "
+            "the sample standard deviation ('zscore_sample'). Use "
+            "'zscore_sample' instead of True and None instead of False "
+            f"(got {value})."
+        )
+    return value

junifer/utils/logging.py

@@ -14,8 +14,17 @@ else:
 
 import logging
 import logging.config
+import sys
 import warnings
-from typing import NoReturn, Optional, Union
+
+# Before Python 3.11, packages_distributions() only uses top_level.txt and
+# misses packages that do not ship it
+from importlib.metadata import (
+    PackageNotFoundError,
+    packages_distributions,
+    version,
+)
+from typing import NoReturn
 from warnings import warn
 
 import datalad
@@ -111,7 +120,8 @@ structlog.configure(
     cache_logger_on_first_use=True,
 )
 
-logger = logging.getLogger("junifer")
+_log = structlog.get_logger("junifer")
+logger = _log.bind(pkg="utils")
 
 # Remove datalad logger handlers to avoid duplicate logging
 _datalad_lgr_hdlrs = datalad.log.lgr.handlers
@@ -150,6 +160,8 @@ def get_versions() -> dict:
         The module names and corresponding versions.
 
     """
+    # Map top-level modules to their distributions
+    module_distributions = packages_distributions()
     # Setup dictionary to track versions of modules
     module_versions = {}
     for name, module in sys.modules.copy().items():
@@ -157,8 +169,20 @@ def get_versions() -> dict:
         # allowing ruamel.yaml
         if "." in name and name != "ruamel.yaml":
             continue
+        # Get version from distribution metadata, as some packages
+        # deprecated the module attribute (e.g. click, markupsafe)
+        vstring = None
+        for dist_name in module_distributions.get(name, []):
+            try:
+                vstring = version(dist_name)
+                break
+            except PackageNotFoundError:
+                continue
+        # Fallback to module attribute
+        if vstring is None:
+            vstring = getattr(module, "__version__", None)
         # Get version or None as string
-        vstring = str(getattr(module, "__version__", None))
+        vstring = str(vstring)
         # Get module version
         module_version = getattr(LooseVersion(vstring), "vstring", None)
         module_versions[name] = module_version
@@ -210,8 +234,8 @@ def log_versions() -> None:
 
 
 def configure_logging(
-    level: Union[int, str] = "WARNING",
-    level_datalad: Union[int, str, None] = None,
+    level: int | str = "WARNING",
+    level_datalad: int | str | None = None,
 ) -> None:
     """Configure the logging functionality.
 
@@ -246,7 +270,7 @@ def configure_logging(
 def raise_error(
     msg: str,
     klass: type[Exception] = ValueError,
-    exception: Optional[Exception] = None,
+    exception: Exception | None = None,
 ) -> NoReturn:
     """Raise error, but first log it.
 
@@ -268,7 +292,7 @@ def raise_error(
 
 
 def warn_with_log(
-    msg: str, category: Optional[type[Warning]] = RuntimeWarning
+    msg: str, category: type[Warning] | None = RuntimeWarning
 ) -> None:
     """Warn, but first log it.