junifer 0.0.7.dev332__py3-none-any.whl → 0.0.8__py3-none-any.whl

junifer/pipeline/tests/test_pipeline_component_registry.py

@@ -5,16 +5,19 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+import importlib
 import logging
+import pkgutil
 from abc import ABC
 
 import pytest
 
+import junifer
 from junifer.datagrabber.pattern import PatternDataGrabber
 from junifer.pipeline.pipeline_component_registry import (
     PipelineComponentRegistry,
 )
-from junifer.storage import SQLiteFeatureStorage
+from junifer.storage import HDF5FeatureStorage
 
 
 def test_pipeline_component_registry_singleton() -> None:
@@ -56,7 +59,7 @@ def test_pipeline_component_registry_components():
     "step, klass",
     [
         ("datagrabber", PatternDataGrabber),
-        ("storage", SQLiteFeatureStorage),
+        ("storage", HDF5FeatureStorage),
     ],
 )
 def test_pipeline_component_registry_register(
@@ -85,7 +88,7 @@ def test_pipeline_component_registry_register(
     "step, klass",
     [
         ("datagrabber", PatternDataGrabber),
-        ("storage", SQLiteFeatureStorage),
+        ("storage", HDF5FeatureStorage),
     ],
 )
 def test_pipeline_component_registry_deregister(
@@ -103,11 +106,16 @@ def test_pipeline_component_registry_deregister(
         The parametrized name of the class.
 
     """
-    with caplog.at_level(logging.INFO):
-        # Register
-        PipelineComponentRegistry().deregister(step=step, klass=klass)
-        # Check logging message
-        assert "De-registering" in caplog.text
+    registry = PipelineComponentRegistry()
+    try:
+        with caplog.at_level(logging.INFO):
+            registry.deregister(step=step, klass=klass)
+            # Check logging message
+            assert "De-registering" in caplog.text
+        assert klass.__name__ not in registry.step_components(step)
+    finally:
+        # Register again, as the registry is shared by all the tests
+        registry.register(step=step, klass=klass)
 
 
 def test_pipeline_component_registry_step_components() -> None:
@@ -149,6 +157,42 @@ def test_pipeline_component_registry_get_class():
     assert isinstance(obj, type(bar))
 
 
+def test_pipeline_component_registry_get_class_builtin() -> None:
+    """Test get class for the built-in components of the registry."""
+    # A new registry, with the built-in components not imported yet (the
+    # singleton might have imported some of them)
+    registry = object.__new__(PipelineComponentRegistry)
+    registry.__init__()
+    for step in registry.steps:
+        for name in registry.step_components(step=step):
+            assert isinstance(registry.components[step][name], str)
+            klass = registry.get_class(step=step, name=name)
+            assert klass.__name__ == name
+    # Import all the components of the sub-packages of the steps, which
+    # registers them, and check that they are all built-in components, except
+    # the ones with optional dependencies, which are registered by importing
+    # their sub-package (e.g., with ``with:`` in the YAML)
+    optional = ("junifer.markers.complexity",)
+    for module in pkgutil.walk_packages(junifer.__path__, "junifer."):
+        if ".tests" in module.name:
+            continue
+        try:
+            importlib.import_module(module.name)
+        except ImportError:
+            # Only if the optional dependencies are not installed
+            assert module.name.startswith(optional)
+    for step, subpkg in registry._step_to_subpkg_mappings.items():
+        for name, klass in (
+            PipelineComponentRegistry().components[step].items()
+        ):
+            if isinstance(klass, str):
+                continue
+            if klass.__module__.startswith(optional):
+                assert name not in registry.components[step]
+            elif klass.__module__.startswith(f"junifer.{subpkg}."):
+                assert name in registry.components[step]
+
+
 def test_pipeline_component_registry_build():
     """Test component instance building for PipelineComponentRegistry."""
     import numpy as np

junifer/pipeline/tests/test_pipeline_step_mixin.py

@@ -9,7 +9,7 @@ from typing import ClassVar
 
 import pytest
 
-from junifer.pipeline.pipeline_step_mixin import PipelineStepMixin
+from junifer.pipeline import ExtDep, PipelineStepMixin
 from junifer.pipeline.utils import _check_afni
 from junifer.typing import (
     ConditionalDependencies,
@@ -18,6 +18,9 @@ from junifer.typing import (
 )
 
 
+pytestmark = pytest.mark.external
+
+
 def test_PipelineStepMixin_correct_dependencies() -> None:
     """Test fit-transform with correct dependencies."""
 
@@ -64,7 +67,9 @@ def test_PipelineStepMixin_correct_ext_dependencies() -> None:
     class CorrectMixer(PipelineStepMixin):
         """Test class for validation."""
 
-        _EXT_DEPENDENCIES: ClassVar[ExternalDependencies] = [{"name": "afni"}]
+        _EXT_DEPENDENCIES: ClassVar[ExternalDependencies] = [
+            {"name": ExtDep.AFNI}
+        ]
 
         def validate_input(self, input: list[str]) -> list[str]:
             return input
@@ -86,7 +91,7 @@ def test_PipelineStepMixin_ext_deps_correct_commands() -> None:
         """Test class for validation."""
 
         _EXT_DEPENDENCIES: ClassVar[ExternalDependencies] = [
-            {"name": "afni", "commands": ["3dReHo"]}
+            {"name": ExtDep.AFNI, "commands": ["3dReHo"]}
         ]
 
         def validate_input(self, input: list[str]) -> list[str]:
@@ -111,7 +116,7 @@ def test_PipelineStepMixin_ext_deps_incorrect_commands() -> None:
         """Test class for validation."""
 
         _EXT_DEPENDENCIES: ClassVar[ExternalDependencies] = [
-            {"name": "afni", "commands": ["3d"]}
+            {"name": ExtDep.AFNI, "commands": ["3d"]}
         ]
 
         def validate_input(self, input: list[str]) -> list[str]:
@@ -125,27 +130,6 @@ def test_PipelineStepMixin_ext_deps_incorrect_commands() -> None:
         mixer.fit_transform({})
 
 
-def test_PipelineStepMixin_incorrect_ext_dependencies() -> None:
-    """Test fit-transform with incorrect external dependencies."""
-
-    class IncorrectMixer(PipelineStepMixin):
-        """Test class for validation."""
-
-        _EXT_DEPENDENCIES: ClassVar[ExternalDependencies] = [
-            {"name": "foobar", "optional": True}
-        ]
-
-        def validate_input(self, input: list[str]) -> list[str]:
-            return input
-
-        def _fit_transform(self, input: dict[str, dict]) -> dict[str, dict]:
-            return {"input": input}
-
-    mixer = IncorrectMixer()
-    with pytest.raises(ValueError, match="Invalid value"):
-        mixer.fit_transform({})
-
-
 def test_PipelineStepMixin_correct_conditional_dependencies() -> None:
     """Test fit-transform with correct conditional dependencies."""
 
@@ -208,7 +192,9 @@ def test_PipelineStepMixin_correct_conditional_ext_dependencies() -> None:
     """Test fit-transform with correct conditional external dependencies."""
 
     class ExternalDependency:
-        _EXT_DEPENDENCIES: ClassVar[ExternalDependencies] = [{"name": "afni"}]
+        _EXT_DEPENDENCIES: ClassVar[ExternalDependencies] = [
+            {"name": ExtDep.AFNI}
+        ]
 
     class CorrectMixer(PipelineStepMixin):
         """Test class for validation."""

junifer/pipeline/tests/test_update_meta_mixin.py

@@ -4,11 +4,10 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Union
-
 import pytest
+from pydantic import BaseModel
 
-from junifer.pipeline.update_meta_mixin import UpdateMetaMixin
+from junifer.pipeline import UpdateMetaMixin
 
 
 @pytest.mark.parametrize(
@@ -23,7 +22,7 @@ from junifer.pipeline.update_meta_mixin import UpdateMetaMixin
 def test_UpdateMetaMixin(
     input: dict,
     step_name: str,
-    dependencies: Union[set, list, str, None],
+    dependencies: set | list | str | None,
     expected: set,
 ) -> None:
     """Test UpdateMetaMixin.
@@ -41,7 +40,7 @@ def test_UpdateMetaMixin(
 
     """
 
-    class TestUpdateMetaMixin(UpdateMetaMixin):
+    class TestUpdateMetaMixin(BaseModel, UpdateMetaMixin):
         """Test UpdateMetaMixin."""
 
         _DEPENDENCIES = dependencies

junifer/pipeline/tests/test_workdir_manager.py

@@ -29,6 +29,21 @@ def test_workdir_manager_workdir(tmp_path: Path) -> None:
     assert workdir_mgr.workdir == tmp_path
 
 
+def test_workdir_manager_create_workdir(tmp_path: Path) -> None:
+    """Test WorkDirManager creates the workdir if it does not exist.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    workdir = tmp_path / "new" / "workdir"
+    workdir_mgr = WorkDirManager(workdir=workdir)
+    assert workdir_mgr.workdir == workdir
+    assert workdir.is_dir()
+
+
 def test_workdir_manager_get_and_delete_element_tempdir(
     tmp_path: Path,
 ) -> None:
@@ -80,6 +95,30 @@ def test_workdir_manager_cleanup_elementdir(
     assert workdir_mgr.elementdir is None
 
 
+def test_workdir_manager_get_elementdir(tmp_path: Path) -> None:
+    """Test WorkDirManager gets the element directory correctly.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    workdir_mgr = WorkDirManager()
+    workdir_mgr.workdir = tmp_path
+    assert workdir_mgr.elementdir is None
+    # Created on the first call, and the same afterwards
+    elementdir = workdir_mgr.get_elementdir()
+    assert elementdir.is_dir()
+    assert elementdir.parent == tmp_path
+    assert workdir_mgr.get_elementdir() == elementdir
+    # The element temporary directories are in it
+    assert workdir_mgr.get_element_tempdir().parent == elementdir
+    workdir_mgr.cleanup_elementdir()
+    assert not elementdir.exists()
+    assert workdir_mgr.elementdir is None
+
+
 def test_workdir_manager_get_and_delete_tempdir(tmp_path: Path) -> None:
     """Test WorkDirManager gets and deletes temporary directories correctly.
 

junifer/pipeline/update_meta_mixin.py

@@ -4,9 +4,6 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from typing import Union
-
-
 __all__ = ["UpdateMetaMixin"]
 
 
@@ -15,8 +12,9 @@ class UpdateMetaMixin:
 
     def update_meta(
         self,
-        input: Union[dict, list[dict]],
+        input: dict | list[dict],
         step_name: str,
+        append: bool = False,
     ) -> None:
         """Update metadata.
 
@@ -26,16 +24,18 @@ class UpdateMetaMixin:
             The data object to update.
         step_name : str
             The name of the pipeline step.
+        append : bool, optional
+            Whether to append the metadata of the step to a list, for the
+            steps that can be several (e.g., the preprocessors), instead of
+            replacing the metadata of the previous step (default False).
 
         """
         # Initialize empty dictionary for the step's metadata
         t_meta = {}
         # Set class name for the step
         t_meta["class"] = self.__class__.__name__
-        # Add object variables to metadata if name doesn't start with "_"
-        for k, v in vars(self).items():
-            if not k.startswith("_"):
-                t_meta[k] = v
+        # Dump model
+        t_meta.update(self.model_dump(mode="json"))
         # Conditional for list dtype vals like Warp
         if not isinstance(input, list):
             input = [input]
@@ -44,13 +44,16 @@ class UpdateMetaMixin:
             if "meta" not in entry:
                 entry["meta"] = {}
             # Add step name
-            entry["meta"][step_name] = t_meta
+            if append:
+                entry["meta"].setdefault(step_name, []).append(t_meta)
+            else:
+                entry["meta"][step_name] = t_meta
             # Add step dependencies
             if "dependencies" not in entry["meta"]:
                 entry["meta"]["dependencies"] = set()
             # Update step dependencies
             dependencies = getattr(self, "_DEPENDENCIES", set())
             if dependencies is not None:
-                if not isinstance(dependencies, (set, list)):
+                if not isinstance(dependencies, set | list):
                     dependencies = {dependencies}
                 entry["meta"]["dependencies"].update(dependencies)

junifer/pipeline/utils.py

@@ -5,22 +5,35 @@
 # License: AGPL
 
 import subprocess
-from typing import Any, Optional
+from enum import StrEnum
+from typing import Any
 
-from junifer.utils.logging import raise_error, warn_with_log
+from pydantic import validate_call
 
+from ..utils import raise_error, warn_with_log
 
-__all__ = ["check_ext_dependencies"]
 
+__all__ = ["ExtDep", "check_ext_dependencies"]
 
+
+class ExtDep(StrEnum):
+    """Accepted external dependencies."""
+
+    AFNI = "afni"
+    FSL = "fsl"
+    ANTs = "ants"
+    FreeSurfer = "freesurfer"
+
+
+@validate_call
 def check_ext_dependencies(
-    name: str, optional: bool = False, **kwargs: Any
+    name: ExtDep, optional: bool = False, **kwargs: Any
 ) -> bool:
     """Check if external dependency `name` is found if mandatory.
 
     Parameters
     ----------
-    name : str
+    name : :enum:`.ExtDep`
         The name of the dependency.
     optional : bool, optional
         Whether the dependency is optional (default False).
@@ -34,18 +47,10 @@ def check_ext_dependencies(
 
     Raises
     ------
-    ValueError
-        If ``name`` is invalid.
     RuntimeError
         If ``name`` is mandatory and is not found.
 
     """
-    valid_ext_dependencies = ("afni", "fsl", "ants", "freesurfer")
-    if name not in valid_ext_dependencies:
-        raise_error(
-            "Invalid value for `name`, should be one of: "
-            f"{valid_ext_dependencies}"
-        )
     # Check for afni
     if name == "afni":
         found = _check_afni(**kwargs)
@@ -71,7 +76,7 @@ def check_ext_dependencies(
     return found
 
 
-def _check_afni(commands: Optional[list[str]] = None) -> bool:
+def _check_afni(commands: list[str] | None = None) -> bool:
     """Check if AFNI is present in the system.
 
     Parameters
@@ -131,7 +136,7 @@ def _check_afni(commands: Optional[list[str]] = None) -> bool:
     return afni_found
 
 
-def _check_fsl(commands: Optional[list[str]] = None) -> bool:
+def _check_fsl(commands: list[str] | None = None) -> bool:
     """Check if FSL is present in the system.
 
     Parameters
@@ -195,7 +200,7 @@ def _check_fsl(commands: Optional[list[str]] = None) -> bool:
     return fsl_found
 
 
-def _check_ants(commands: Optional[list[str]] = None) -> bool:
+def _check_ants(commands: list[str] | None = None) -> bool:
     """Check if ANTs is present in the system.
 
     Parameters
@@ -255,7 +260,7 @@ def _check_ants(commands: Optional[list[str]] = None) -> bool:
     return ants_found
 
 
-def _check_freesurfer(commands: Optional[list[str]] = None) -> bool:
+def _check_freesurfer(commands: list[str] | None = None) -> bool:
     """Check if FreeSurfer is present in the system.
 
     Parameters

junifer/pipeline/workdir_manager.py

@@ -7,14 +7,17 @@
 import shutil
 import tempfile
 from pathlib import Path
-from typing import Optional, Union
 
-from ..utils import logger
+import structlog
+
 from ..utils.singleton import Singleton
 
 
 __all__ = ["WorkDirManager"]
 
+_log = structlog.get_logger("junifer")
+logger = _log.bind(pkg="pipeline")
+
 
 class WorkDirManager(metaclass=Singleton):
     """Class for working directory manager.
@@ -46,7 +49,7 @@ class WorkDirManager(metaclass=Singleton):
     """
 
     def __init__(
-        self, workdir: Optional[Union[str, Path]] = None, cleanup: bool = True
+        self, workdir: str | Path | None = None, cleanup: bool = True
     ) -> None:
         """Initialize the class."""
         self._workdir = Path(workdir) if isinstance(workdir, str) else workdir
@@ -60,20 +63,16 @@ class WorkDirManager(metaclass=Singleton):
         # Check and set topmost level directory if not provided
         if self._workdir is None:
             self._workdir = Path(tempfile.gettempdir()) / "junifer"
-            # Create directory if not found
-            if not self._workdir.is_dir():
-                logger.debug(
-                    "Creating working directory at "
-                    f"{self._workdir.resolve()!s}"
-                )
-                self._workdir.mkdir(parents=True)
+
+        # Create directory if not found
+        if not self._workdir.is_dir():
             logger.debug(
-                f"Setting working directory to {self._workdir.resolve()!s}"
+                f"Creating working directory at {self._workdir.resolve()!s}"
             )
-
-    def __del__(self) -> None:
-        """Destructor."""
-        self._cleanup()
+            self._workdir.mkdir(parents=True)
+        logger.debug(
+            f"Setting working directory to {self._workdir.resolve()!s}"
+        )
 
     def _cleanup(self) -> None:
         """Clean up the element and temporary directories."""
@@ -97,7 +96,7 @@ class WorkDirManager(metaclass=Singleton):
         return self._workdir  # type: ignore
 
     @workdir.setter
-    def workdir(self, path: Union[str, Path]) -> None:
+    def workdir(self, path: str | Path) -> None:
         """Set working directory.
 
         The directory path is created if it doesn't exist yet.
@@ -129,8 +128,28 @@ class WorkDirManager(metaclass=Singleton):
         """Get element directory."""
         return self._elementdir  # type: ignore
 
+    def get_elementdir(self) -> Path:
+        """Get the element directory, creating it if needed.
+
+        This directory should be available only for the lifetime of an
+        element.
+
+        Returns
+        -------
+        pathlib.Path
+            The path to the element directory.
+
+        """
+        if self._elementdir is None:
+            logger.debug(
+                "Setting up element directory under "
+                f"{self._workdir.resolve()!s}"  # type: ignore
+            )
+            self._elementdir = Path(tempfile.mkdtemp(dir=self._workdir))
+        return self._elementdir
+
     def get_element_tempdir(
-        self, prefix: Optional[str] = None, suffix: Optional[str] = None
+        self, prefix: str | None = None, suffix: str | None = None
     ) -> Path:
         """Get an element-scoped temporary directory.
 
@@ -152,22 +171,12 @@ class WorkDirManager(metaclass=Singleton):
             The path to the temporary directory.
 
         """
-        # Create element directory if not created already
-        if self._elementdir is None:
-            logger.debug(
-                "Setting up element directory under "
-                f"{self._workdir.resolve()!s}"  # type: ignore
-            )
-            self._elementdir = Path(tempfile.mkdtemp(dir=self._workdir))
-
+        elementdir = self.get_elementdir()
         logger.debug(
-            "Creating element temporary directory at "
-            f"{self._elementdir.resolve()!s}"
+            f"Creating element temporary directory at {elementdir.resolve()!s}"
         )
         return Path(
-            tempfile.mkdtemp(
-                dir=self._elementdir, prefix=prefix, suffix=suffix
-            )
+            tempfile.mkdtemp(dir=elementdir, prefix=prefix, suffix=suffix)
         )
 
     def delete_element_tempdir(self, tempdir: Path) -> None:
@@ -203,12 +212,12 @@ class WorkDirManager(metaclass=Singleton):
             self._elementdir = None
 
     @property
-    def root_tempdir(self) -> Optional[Path]:
+    def root_tempdir(self) -> Path | None:
         """Get root temporary directory."""
         return self._root_tempdir
 
     def get_tempdir(
-        self, prefix: Optional[str] = None, suffix: Optional[str] = None
+        self, prefix: str | None = None, suffix: str | None = None
     ) -> Path:
         """Get a component-scoped temporary directory.
 

junifer/preprocess/__init__.pyi

@@ -1,15 +1,20 @@
 __all__ = [
     "BasePreprocessor",
     "fMRIPrepConfoundRemover",
+    "Confounds",
+    "Strategy",
     "SpaceWarper",
+    "SpaceWarpingImpl",
     "Smoothing",
+    "SmoothingImpl",
     "TemporalSlicer",
     "TemporalFilter",
+    "logger",
 ]
 
-from .base import BasePreprocessor
-from .confounds import fMRIPrepConfoundRemover
-from .warping import SpaceWarper
-from .smoothing import Smoothing
+from .base import BasePreprocessor, logger
+from .confounds import fMRIPrepConfoundRemover, Confounds, Strategy
+from .warping import SpaceWarper, SpaceWarpingImpl
+from .smoothing import Smoothing, SmoothingImpl
 from ._temporal_slicer import TemporalSlicer
 from ._temporal_filter import TemporalFilter