junifer 0.0.5.dev242__py3-none-any.whl → 0.0.6__py3-none-any.whl

junifer/api/queue_context/gnu_parallel_local_adapter.py

@@ -6,8 +6,9 @@
 import shutil
 import textwrap
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Optional
 
+from ...typing import Elements
 from ...utils import logger, make_executable, raise_error, run_ext_cmd
 from .queue_context_adapter import QueueContextAdapter
 
@@ -37,6 +38,9 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
         virtual environment of any kind (default None).
     verbose : str, optional
         The level of verbosity (default "info").
+    verbose_datalad : str or None, optional
+        The level of verbosity for datalad. If None, will be the same
+        as ``verbose`` (default None).
     submit : bool, optional
         Whether to submit the jobs (default False).
 
@@ -60,11 +64,12 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
         job_name: str,
         job_dir: Path,
         yaml_config_path: Path,
-        elements: List[Union[str, Tuple]],
+        elements: Elements,
         pre_run: Optional[str] = None,
         pre_collect: Optional[str] = None,
-        env: Optional[Dict[str, str]] = None,
+        env: Optional[dict[str, str]] = None,
         verbose: str = "info",
+        verbose_datalad: Optional[str] = None,
         submit: bool = False,
     ) -> None:
         """Initialize the class."""
@@ -76,6 +81,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
         self._pre_collect = pre_collect
         self._check_env(env)
         self._verbose = verbose
+        self._verbose_datalad = verbose_datalad
         self._submit = submit
 
         self._log_dir = self._job_dir / "logs"
@@ -86,7 +92,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
         self._run_joblog_path = self._job_dir / f"run_{self._job_name}_joblog"
         self._elements_file_path = self._job_dir / "elements"
 
-    def _check_env(self, env: Optional[Dict[str, str]]) -> None:
+    def _check_env(self, env: Optional[dict[str, str]]) -> None:
         """Check value of env parameter on init.
 
         Parameters
@@ -155,6 +161,11 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
 
     def run(self) -> str:
         """Return run commands."""
+        verbose_args = f"--verbose {self._verbose}"
+        if self._verbose_datalad:
+            verbose_args = (
+                f"{verbose_args} --verbose-datalad {self._verbose_datalad}"
+            )
         return (
             f"#!/usr/bin/env {self._shell}\n\n"
             "# This script is auto-generated by junifer.\n\n"
@@ -169,7 +180,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
             f"{self._job_dir.resolve()!s}/{self._executable} "
             f"{self._arguments} run "
             f"{self._yaml_config_path.resolve()!s} "
-            f"--verbose {self._verbose} "
+            f"{verbose_args} "
             f"--element"
         )
 
@@ -184,6 +195,11 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
 
     def collect(self) -> str:
         """Return collect commands."""
+        verbose_args = f"--verbose {self._verbose}"
+        if self._verbose_datalad:
+            verbose_args = (
+                f"{verbose_args} --verbose-datalad {self._verbose_datalad}"
+            )
         return (
             f"#!/usr/bin/env {self._shell}\n\n"
             "# This script is auto-generated by junifer.\n\n"
@@ -193,7 +209,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
             f"{self._job_dir.resolve()!s}/{self._executable} "
             f"{self._arguments} collect "
             f"{self._yaml_config_path.resolve()!s} "
-            f"--verbose {self._verbose}"
+            f"{verbose_args}"
         )
 
     def prepare(self) -> None:

junifer/api/queue_context/htcondor_adapter.py

@@ -6,8 +6,9 @@
 import shutil
 import textwrap
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Optional
 
+from ...typing import Elements
 from ...utils import logger, make_executable, raise_error, run_ext_cmd
 from .queue_context_adapter import QueueContextAdapter
 
@@ -37,6 +38,9 @@ class HTCondorAdapter(QueueContextAdapter):
         virtual environment of any kind (default None).
     verbose : str, optional
         The level of verbosity (default "info").
+    verbose_datalad : str or None, optional
+        The level of verbosity for datalad. If None, will be the same
+        as ``verbose`` (default None).
     cpus : int, optional
         The number of CPU cores to use (default 1).
     mem : str, optional
@@ -78,11 +82,12 @@ class HTCondorAdapter(QueueContextAdapter):
         job_name: str,
         job_dir: Path,
         yaml_config_path: Path,
-        elements: List[Union[str, Tuple]],
+        elements: Elements,
         pre_run: Optional[str] = None,
         pre_collect: Optional[str] = None,
-        env: Optional[Dict[str, str]] = None,
+        env: Optional[dict[str, str]] = None,
         verbose: str = "info",
+        verbose_datalad: Optional[str] = None,
         cpus: int = 1,
         mem: str = "8G",
         disk: str = "1G",
@@ -99,6 +104,7 @@ class HTCondorAdapter(QueueContextAdapter):
         self._pre_collect = pre_collect
         self._check_env(env)
         self._verbose = verbose
+        self._verbose_datalad = verbose_datalad
         self._cpus = cpus
         self._mem = mem
         self._disk = disk
@@ -115,7 +121,7 @@ class HTCondorAdapter(QueueContextAdapter):
         )
         self._dag_path = self._job_dir / f"{self._job_name}.dag"
 
-    def _check_env(self, env: Optional[Dict[str, str]]) -> None:
+    def _check_env(self, env: Optional[dict[str, str]]) -> None:
         """Check value of env parameter on init.
 
         Parameters
@@ -201,10 +207,15 @@ class HTCondorAdapter(QueueContextAdapter):
 
     def run(self) -> str:
         """Return run commands."""
+        verbose_args = f"--verbose {self._verbose} "
+        if self._verbose_datalad is not None:
+            verbose_args = (
+                f"{verbose_args} --verbose-datalad {self._verbose_datalad} "
+            )
         junifer_run_args = (
             "run "
             f"{self._yaml_config_path.resolve()!s} "
-            f"--verbose {self._verbose} "
+            f"{verbose_args}"
             "--element $(element)"
         )
         log_dir_prefix = (
@@ -246,10 +257,16 @@ class HTCondorAdapter(QueueContextAdapter):
 
     def collect(self) -> str:
         """Return collect commands."""
+        verbose_args = f"--verbose {self._verbose} "
+        if self._verbose_datalad is not None:
+            verbose_args = (
+                f"{verbose_args} --verbose-datalad {self._verbose_datalad} "
+            )
+
         junifer_collect_args = (
             "collect "
             f"{self._yaml_config_path.resolve()!s} "
-            f"--verbose {self._verbose}"
+            f"{verbose_args}"
         )
         log_dir_prefix = f"{self._log_dir.resolve()!s}/junifer_collect"
         fixed = (

junifer/api/queue_context/tests/test_gnu_parallel_local_adapter.py

@@ -5,7 +5,7 @@
 
 import logging
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Optional, Union
 
 import pytest
 
@@ -44,7 +44,7 @@ def test_GnuParallelLocalAdapter_env_shell_error() -> None:
     ],
 )
 def test_GnuParallelLocalAdapter_elements(
-    elements: List[Union[str, Tuple]],
+    elements: list[Union[str, tuple]],
     expected_text: str,
 ) -> None:
     """Test GnuParallelLocalAdapter elements().
@@ -177,7 +177,7 @@ def test_GnuParallelLocalAdapter_prepare(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
-    env: Dict[str, str],
+    env: dict[str, str],
 ) -> None:
     """Test GnuParallelLocalAdapter prepare().
 

junifer/api/queue_context/tests/test_htcondor_adapter.py

@@ -5,7 +5,7 @@
 
 import logging
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Optional, Union
 
 import pytest
 
@@ -181,7 +181,7 @@ def test_HTCondorAdapter_run_collect(
     ],
 )
 def test_HTCondor_dag(
-    elements: List[Union[str, Tuple]], collect: str, expected_text: str
+    elements: list[Union[str, tuple]], collect: str, expected_text: str
 ) -> None:
     """Test HTCondorAdapter dag().
 
@@ -218,7 +218,7 @@ def test_HTCondorAdapter_prepare(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
-    env: Dict[str, str],
+    env: dict[str, str],
 ) -> None:
     """Test HTCondorAdapter prepare().
 

junifer/api/tests/test_functions.py

@@ -1,4 +1,4 @@
-"""Provide tests for functions."""
+"""Provide tests for API functions."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Leonard Sasse <l.sasse@fz-juelich.de>
@@ -6,16 +6,19 @@
 # License: AGPL
 
 import logging
+from contextlib import AbstractContextManager, nullcontext
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Any, Optional, Union
 
 import pytest
+from nibabel.filebasedimages import ImageFileError
 from ruamel.yaml import YAML
 
 import junifer.testing.registry  # noqa: F401
-from junifer.api.functions import collect, list_elements, queue, reset, run
+from junifer.api import collect, list_elements, queue, reset, run
 from junifer.datagrabber.base import BaseDataGrabber
-from junifer.pipeline.registry import build
+from junifer.pipeline import PipelineComponentRegistry
+from junifer.typing import Elements
 
 
 # Configure YAML class
@@ -25,16 +28,41 @@ yaml.allow_unicode = True
 yaml.indent(mapping=2, sequence=4, offset=2)
 
 
+# Kept for parametrizing
+_datagrabber = {
+    "kind": "PartlyCloudyTestingDataGrabber",
+}
+_bids_ses_datagrabber = {
+    "kind": "PatternDataladDataGrabber",
+    "uri": "https://gin.g-node.org/juaml/datalad-example-bids-ses",
+    "types": ["T1w", "BOLD"],
+    "patterns": {
+        "T1w": {
+            "pattern": (
+                "{subject}/{session}/anat/{subject}_{session}_T1w.nii.gz"
+            ),
+            "space": "MNI152NLin6Asym",
+        },
+        "BOLD": {
+            "pattern": (
+                "{subject}/{session}/func/{subject}_{session}_task-rest_bold.nii.gz"
+            ),
+            "space": "MNI152NLin6Asym",
+        },
+    },
+    "replacements": ["subject", "session"],
+    "rootdir": "example_bids_ses",
+}
+
+
 @pytest.fixture
-def datagrabber() -> Dict[str, str]:
+def datagrabber() -> dict[str, str]:
     """Return a datagrabber as a dictionary."""
-    return {
-        "kind": "PartlyCloudyTestingDataGrabber",
-    }
+    return _datagrabber.copy()
 
 
 @pytest.fixture
-def markers() -> List[Dict[str, str]]:
+def markers() -> list[dict[str, str]]:
     """Return markers as a list of dictionary."""
     return [
         {
@@ -53,18 +81,55 @@ def markers() -> List[Dict[str, str]]:
 
 
 @pytest.fixture
-def storage() -> Dict[str, str]:
+def storage() -> dict[str, str]:
     """Return a storage as a dictionary."""
     return {
         "kind": "SQLiteFeatureStorage",
     }
 
 
+@pytest.mark.parametrize(
+    "datagrabber, element, expect",
+    [
+        (
+            _datagrabber,
+            [("sub-01",)],
+            pytest.raises(RuntimeError, match="element selectors are invalid"),
+        ),
+        (
+            _datagrabber,
+            ["sub-01"],
+            nullcontext(),
+        ),
+        (
+            _bids_ses_datagrabber,
+            ["sub-01"],
+            pytest.raises(ImageFileError, match="is not a gzip file"),
+        ),
+        (
+            _bids_ses_datagrabber,
+            [("sub-01", "ses-01")],
+            pytest.raises(ImageFileError, match="is not a gzip file"),
+        ),
+        (
+            _bids_ses_datagrabber,
+            [("sub-01", "ses-100")],
+            pytest.raises(RuntimeError, match="element selectors are invalid"),
+        ),
+        (
+            _bids_ses_datagrabber,
+            [("sub-100", "ses-01")],
+            pytest.raises(RuntimeError, match="element selectors are invalid"),
+        ),
+    ],
+)
 def test_run_single_element(
     tmp_path: Path,
-    datagrabber: Dict[str, str],
-    markers: List[Dict[str, str]],
-    storage: Dict[str, str],
+    datagrabber: dict[str, Any],
+    markers: list[dict[str, str]],
+    storage: dict[str, str],
+    element: Elements,
+    expect: AbstractContextManager,
 ) -> None:
     """Test run function with single element.
 
@@ -78,27 +143,32 @@ def test_run_single_element(
         Testing markers as list of dictionary.
     storage : dict
         Testing storage as dictionary.
+    element : list of str or tuple
+        The parametrized element.
+    expect : typing.ContextManager
+        The parametrized ContextManager object.
 
     """
     # Set storage
     storage["uri"] = str((tmp_path / "out.sqlite").resolve())
     # Run operations
-    run(
-        workdir=tmp_path,
-        datagrabber=datagrabber,
-        markers=markers,
-        storage=storage,
-        elements=["sub-01"],
-    )
-    # Check files
-    files = list(tmp_path.glob("*.sqlite"))
-    assert len(files) == 1
+    with expect:
+        run(
+            workdir=tmp_path,
+            datagrabber=datagrabber,
+            markers=markers,
+            storage=storage,
+            elements=element,
+        )
+        # Check files
+        files = list(tmp_path.glob("*.sqlite"))
+        assert len(files) == 1
 
 
 def test_run_single_element_with_preprocessing(
     tmp_path: Path,
-    markers: List[Dict[str, str]],
-    storage: Dict[str, str],
+    markers: list[dict[str, str]],
+    storage: dict[str, str],
 ) -> None:
     """Test run function with single element and pre-processing.
 
@@ -116,7 +186,7 @@ def test_run_single_element_with_preprocessing(
     storage["uri"] = str((tmp_path / "out.sqlite").resolve())
     # Run operations
     run(
-        workdir=tmp_path,
+        workdir={"path": tmp_path, "cleanup": False},
         datagrabber={
             "kind": "PartlyCloudyTestingDataGrabber",
             "reduce_confounds": False,
@@ -135,11 +205,23 @@ def test_run_single_element_with_preprocessing(
     assert len(files) == 1
 
 
+@pytest.mark.parametrize(
+    "element, expect",
+    [
+        (
+            [("sub-01",), ("sub-03",)],
+            pytest.raises(RuntimeError, match="element selectors are invalid"),
+        ),
+        (["sub-01", "sub-03"], nullcontext()),
+    ],
+)
 def test_run_multi_element_multi_output(
     tmp_path: Path,
-    datagrabber: Dict[str, str],
-    markers: List[Dict[str, str]],
-    storage: Dict[str, str],
+    datagrabber: dict[str, str],
+    markers: list[dict[str, str]],
+    storage: dict[str, str],
+    element: Elements,
+    expect: AbstractContextManager,
 ) -> None:
     """Test run function with multi element and multi output.
 
@@ -153,29 +235,34 @@ def test_run_multi_element_multi_output(
         Testing markers as list of dictionary.
     storage : dict
         Testing storage as dictionary.
+    element : list of str or tuple
+        The parametrized element.
+    expect : typing.ContextManager
+        The parametrized ContextManager object.
 
     """
     # Set storage
     storage["uri"] = str((tmp_path / "out.sqlite").resolve())
     storage["single_output"] = False  # type: ignore
     # Run operations
-    run(
-        workdir=tmp_path,
-        datagrabber=datagrabber,
-        markers=markers,
-        storage=storage,
-        elements=["sub-01", "sub-03"],
-    )
-    # Check files
-    files = list(tmp_path.glob("*.sqlite"))
-    assert len(files) == 2
+    with expect:
+        run(
+            workdir=tmp_path,
+            datagrabber=datagrabber,
+            markers=markers,
+            storage=storage,
+            elements=element,
+        )
+        # Check files
+        files = list(tmp_path.glob("*.sqlite"))
+        assert len(files) == 2
 
 
 def test_run_multi_element_single_output(
     tmp_path: Path,
-    datagrabber: Dict[str, str],
-    markers: List[Dict[str, str]],
-    storage: Dict[str, str],
+    datagrabber: dict[str, str],
+    markers: list[dict[str, str]],
+    storage: dict[str, str],
 ) -> None:
     """Test run function with multi element and single output.
 
@@ -210,9 +297,9 @@ def test_run_multi_element_single_output(
 
 def test_run_and_collect(
     tmp_path: Path,
-    datagrabber: Dict[str, str],
-    markers: List[Dict[str, str]],
-    storage: Dict[str, str],
+    datagrabber: dict[str, str],
+    markers: list[dict[str, str]],
+    storage: dict[str, str],
 ) -> None:
     """Test run and collect functions.
 
@@ -240,7 +327,7 @@ def test_run_and_collect(
         storage=storage,
     )
     # Get datagrabber
-    dg = build(
+    dg = PipelineComponentRegistry().build_component_instance(
         step="datagrabber", name=datagrabber["kind"], baseclass=BaseDataGrabber
     )
     elements = dg.get_elements()  # type: ignore
@@ -259,9 +346,9 @@ def test_queue_correct_yaml_config(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
-    datagrabber: Dict[str, str],
-    markers: List[Dict[str, str]],
-    storage: Dict[str, str],
+    datagrabber: dict[str, str],
+    markers: list[dict[str, str]],
+    storage: dict[str, str],
 ) -> None:
     """Test proper YAML config generation for queueing.
 
@@ -287,7 +374,10 @@ def test_queue_correct_yaml_config(
             queue(
                 config={
                     "with": "junifer.testing.registry",
-                    "workdir": str(tmp_path.resolve()),
+                    "workdir": {
+                        "path": str(tmp_path.resolve()),
+                        "cleanup": True,
+                    },
                     "datagrabber": datagrabber,
                     "markers": markers,
                     "storage": storage,
@@ -342,7 +432,7 @@ def test_queue_invalid_job_queue(
         with monkeypatch.context() as m:
             m.chdir(tmp_path)
             queue(
-                config={"elements": ["sub-001"]},
+                config={"elements": [("sub-001",)]},
                 kind="ABC",
             )
 
@@ -366,13 +456,13 @@ def test_queue_assets_disallow_overwrite(
             m.chdir(tmp_path)
             # First generate assets
             queue(
-                config={"elements": ["sub-001"]},
+                config={"elements": [("sub-001",)]},
                 kind="HTCondor",
                 jobname="prevent_overwrite",
             )
             # Re-run to trigger error
             queue(
-                config={"elements": ["sub-001"]},
+                config={"elements": [("sub-001",)]},
                 kind="HTCondor",
                 jobname="prevent_overwrite",
             )
@@ -399,14 +489,14 @@ def test_queue_assets_allow_overwrite(
         m.chdir(tmp_path)
         # First generate assets
         queue(
-            config={"elements": ["sub-001"]},
+            config={"elements": [("sub-001",)]},
             kind="HTCondor",
             jobname="allow_overwrite",
         )
         with caplog.at_level(logging.INFO):
             # Re-run to overwrite
             queue(
-                config={"elements": ["sub-001"]},
+                config={"elements": [("sub-001",)]},
                 kind="HTCondor",
                 jobname="allow_overwrite",
                 overwrite=True,
@@ -426,7 +516,7 @@ def test_queue_with_imports(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
-    with_: Union[str, List[str]],
+    with_: Union[str, list[str]],
 ) -> None:
     """Test queue with `with` imports.
 
@@ -448,10 +538,16 @@ def test_queue_with_imports(
         (tmp_path / "a.py").touch()
         with caplog.at_level(logging.DEBUG):
             queue(
-                config={"with": with_},
+                config={
+                    "with": with_,
+                    "workdir": {
+                        "path": str(tmp_path.resolve()),
+                        "cleanup": False,
+                    },
+                },
                 kind="HTCondor",
                 jobname="with_import_check",
-                elements="sub-001",
+                elements=[("sub-001",)],
             )
             assert "Copying" in caplog.text
             assert "Queue done" in caplog.text
@@ -465,10 +561,8 @@ def test_queue_with_imports(
 @pytest.mark.parametrize(
     "elements",
     [
-        "sub-001",
-        ["sub-001"],
-        ["sub-001", "sub-002"],
-        ("sub-001", "ses-001"),
+        [("sub-001",)],
+        [("sub-001",), ("sub-002",)],
         [("sub-001", "ses-001")],
         [("sub-001", "ses-001"), ("sub-001", "ses-002")],
     ],
@@ -477,7 +571,7 @@ def test_queue_with_elements(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
-    elements: Union[str, List[Union[str, Tuple[str]]], Tuple[str]],
+    elements: list[tuple[str, ...]],
 ) -> None:
     """Test queue with elements.
 
@@ -489,7 +583,7 @@ def test_queue_with_elements(
         The pytest.MonkeyPatch object.
     caplog : pytest.LogCaptureFixture
         The pytest.LogCaptureFixture object.
-    elements : str of list of str
+    elements : list of tuple
         The parametrized elements for the queue.
 
     """
@@ -508,7 +602,7 @@ def test_queue_without_elements(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
     caplog: pytest.LogCaptureFixture,
-    datagrabber: Dict[str, str],
+    datagrabber: dict[str, str],
 ) -> None:
     """Test queue without elements.
 
@@ -536,9 +630,9 @@ def test_queue_without_elements(
 
 def test_reset_run(
     tmp_path: Path,
-    datagrabber: Dict[str, str],
-    markers: List[Dict[str, str]],
-    storage: Dict[str, str],
+    datagrabber: dict[str, str],
+    markers: list[dict[str, str]],
+    storage: dict[str, str],
 ) -> None:
     """Test reset function for run.
 
@@ -580,9 +674,9 @@ def test_reset_run(
 def test_reset_queue(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,
-    datagrabber: Dict[str, str],
-    markers: List[Dict[str, str]],
-    storage: Dict[str, str],
+    datagrabber: dict[str, str],
+    markers: list[dict[str, str]],
+    storage: dict[str, str],
     job_name: str,
 ) -> None:
     """Test reset function for queue.
@@ -642,13 +736,13 @@ def test_reset_queue(
 @pytest.mark.parametrize(
     "elements",
     [
-        ["sub-01"],
+        [("sub-01",)],
         None,
     ],
 )
 def test_list_elements(
-    datagrabber: Dict[str, str],
-    elements: Optional[List[str]],
+    datagrabber: dict[str, str],
+    elements: Optional[list[tuple[str, ...]]],
 ) -> None:
     """Test elements listing.