junifer 0.0.8.dev298__tar.gz → 0.0.8.dev303__tar.gz

{junifer-0.0.8.dev298/junifer.egg-info → junifer-0.0.8.dev303}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.8.dev298
+Version: 0.0.8.dev303
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

junifer-0.0.8.dev303/docs/changes/newsfragments/509.enh

@@ -0,0 +1 @@
+Set up the working directory first in all the commands of the CLI (from ``workdir`` in the YAML) and in :func:`.run`, :func:`.collect`, :func:`.queue`, :func:`.reset` and :func:`.list_elements`, adding the ``workdir`` parameter to :func:`.collect` and :func:`.list_elements`, and create the working directory if it does not exist by `Synchon Mandal`_ and `Fede Raimondo`_

{junifer-0.0.8.dev298 → junifer-0.0.8.dev303}/junifer/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.0.8.dev298'
-__version_tuple__ = version_tuple = (0, 0, 8, 'dev298')
+__version__ = version = '0.0.8.dev303'
+__version_tuple__ = version_tuple = (0, 0, 8, 'dev303')
 
-__commit_id__ = commit_id = 'g30ddf7410'
+__commit_id__ = commit_id = 'g4a74bdf26'

{junifer-0.0.8.dev298 → junifer-0.0.8.dev303}/junifer/api/functions.py

@@ -136,6 +136,45 @@ def _get_storage(storage_config: dict) -> StorageLike:
     )
 
 
+def _get_workdir_params(workdir: str | Path | dict | None) -> dict:
+    """Get the parameters of the working directory manager.
+
+    Parameters
+    ----------
+    workdir : str or pathlib.Path or dict or None
+        The working directory as a path, or as a dictionary with the keys
+        ``path`` and ``cleanup`` (default True). If None, the default of
+        :class:`.WorkDirManager` is used.
+
+    Returns
+    -------
+    dict
+        The parameters for :class:`.WorkDirManager`.
+
+    """
+    if isinstance(workdir, dict):
+        return {
+            "workdir": Path(workdir["path"]),
+            "cleanup": workdir.get("cleanup", True),
+        }
+    if workdir is None:
+        return {"workdir": None, "cleanup": True}
+    return {"workdir": Path(workdir), "cleanup": True}
+
+
+def _setup_workdir(workdir: str | Path | dict | None) -> None:
+    """Set up the working directory manager.
+
+    Parameters
+    ----------
+    workdir : str or pathlib.Path or dict or None
+        The working directory (see :func:`_get_workdir_params`).
+
+    """
+    WorkDirManager(**_get_workdir_params(workdir))
+    atexit.register(WorkDirManager()._cleanup)
+
+
 def run(
     workdir: str | Path | dict,
     datagrabber: dict,
@@ -180,25 +219,15 @@ def run(
         If invalid element selectors are found.
 
     """
-    # Conditional to handle workdir config
-    if isinstance(workdir, str | Path):
-        if isinstance(workdir, str):
-            workdir = {"workdir": Path(workdir), "cleanup": True}
-        else:
-            workdir = {"workdir": workdir, "cleanup": True}
-    elif isinstance(workdir, dict):
-        workdir["workdir"] = workdir.pop("path")
-
     # Initiate working directory manager with correct variation
-    if not workdir["cleanup"]:
+    if not _get_workdir_params(workdir)["cleanup"]:
         if elements is None or len(elements) > 1:
             raise_error(
                 "Cannot disable `workdir.cleanup` as "
                 f"{len(elements) if elements is not None else 'all'} "
                 "elements will be processed"
             )
-    WorkDirManager(**workdir)
-    atexit.register(WorkDirManager()._cleanup)
+    _setup_workdir(workdir)
 
     # Get datagrabber to use
     datagrabber_object = _get_datagrabber(datagrabber.copy())
@@ -253,7 +282,7 @@ def run(
                 mc.fit(datagrabber_object[t_element])
 
 
-def collect(storage: dict) -> None:
+def collect(storage: dict, workdir: str | Path | dict | None = None) -> None:
     """Collect and store data.
 
     Parameters
@@ -262,8 +291,12 @@ def collect(storage: dict) -> None:
         Storage to use. Must have a key ``kind`` with the kind of
         storage to use. All other keys are passed to the storage
         constructor.
+    workdir : str or pathlib.Path or dict or None, optional
+        The working directory, as in :func:`.run`. If None, the default of
+        :class:`.WorkDirManager` is used (default None).
 
     """
+    _setup_workdir(workdir)
     logger.info(f"Collecting data using {storage['kind']}")
     logger.debug(f"\tStorage params: {storage}")
     if "single_output" not in storage:
@@ -307,6 +340,7 @@ def queue(
         if the ``jobdir`` exists and ``overwrite = False``.
 
     """
+    _setup_workdir(config.get("workdir"))
     valid_kind = ["HTCondor", "GNUParallelLocal"]
     if kind not in valid_kind:
         raise_error(
@@ -410,6 +444,7 @@ def reset(config: dict) -> None:
         The configuration to be used for resetting.
 
     """
+    _setup_workdir(config.get("workdir"))
     # Fetch storage
     storage = config["storage"]
     storage_uri = Path(storage["uri"])
@@ -442,6 +477,7 @@ def reset(config: dict) -> None:
 def list_elements(
     datagrabber: dict,
     elements: Elements | None = None,
+    workdir: str | Path | dict | None = None,
 ) -> str:
     """List elements of the datagrabber filtered using `elements`.
 
@@ -454,8 +490,13 @@ def list_elements(
     elements : list or None, optional
         Element(s) to filter using. Will be used to index the DataGrabber
         (default None).
+    workdir : str or pathlib.Path or dict or None, optional
+        Directory where the DataGrabber will be used, as in :func:`.run`.
+        If None, the default of :class:`.WorkDirManager` is used
+        (default None).
 
     """
+    _setup_workdir(workdir)
     # Get datagrabber to use
     datagrabber_object = _get_datagrabber(datagrabber)
 

{junifer-0.0.8.dev298 → junifer-0.0.8.dev303}/junifer/api/tests/test_functions.py

@@ -7,6 +7,7 @@
 
 import logging
 import sys
+import tempfile
 from contextlib import AbstractContextManager, nullcontext
 from pathlib import Path
 from typing import Any
@@ -18,7 +19,7 @@ from ruamel.yaml import YAML
 import junifer.testing.registry  # noqa: F401
 from junifer.api import collect, list_elements, parse_yaml, queue, reset, run
 from junifer.datagrabber.base import BaseDataGrabber
-from junifer.pipeline import PipelineComponentRegistry
+from junifer.pipeline import PipelineComponentRegistry, WorkDirManager
 from junifer.typing import Elements
 
 
@@ -639,6 +640,143 @@ def test_queue_without_elements(
             assert "Queue done" in caplog.text
 
 
+def _workdir(tmp_path: Path, as_dict: bool) -> tuple[str | dict, Path]:
+    """Get a working directory in the test directory.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    as_dict : bool
+        Whether to give the working directory as a dictionary or a path.
+
+    Returns
+    -------
+    str or dict
+        The working directory, as given to the functions.
+    pathlib.Path
+        The path to the working directory.
+
+    """
+    path = tmp_path / "workdir"
+    workdir = {"path": str(path), "cleanup": True} if as_dict else str(path)
+    return workdir, path
+
+
+@pytest.mark.parametrize("as_dict", [False, True])
+@pytest.mark.parametrize("elements", [None, ["sub-01"]])
+def test_queue_workdir(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    datagrabber: dict[str, str],
+    elements: list[str] | None,
+    as_dict: bool,
+) -> None:
+    """Test queue sets up the working directory.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    monkeypatch : pytest.MonkeyPatch
+        The pytest.MonkeyPatch object.
+    datagrabber : dict
+        Testing datagrabber as dictionary.
+    elements : list of str or None
+        The parametrized elements to queue. If None, the elements are
+        listed using the datagrabber.
+    as_dict : bool
+        Whether to give the working directory as a dictionary or a path.
+
+    """
+    workdir, path = _workdir(tmp_path, as_dict)
+    with monkeypatch.context() as m:
+        m.chdir(tmp_path)
+        queue(
+            config={"workdir": workdir, "datagrabber": datagrabber},
+            kind="HTCondor",
+            elements=elements,
+        )
+    assert WorkDirManager().workdir == path
+    assert path.is_dir()
+    # The working directory is written to the job configuration as given
+    config = YAML().load(tmp_path / "junifer_jobs/junifer_job/config.yaml")
+    assert config["workdir"] == workdir
+
+
+def test_queue_workdir_cleanup(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    datagrabber: dict[str, str],
+) -> None:
+    """Test queued jobs clean up the working directory.
+
+    Disabling the cleanup only applies to queueing (e.g. to debug it), as
+    the jobs would otherwise leave the files of each element.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    monkeypatch : pytest.MonkeyPatch
+        The pytest.MonkeyPatch object.
+    datagrabber : dict
+        Testing datagrabber as dictionary.
+
+    """
+    workdir = {"path": str(tmp_path / "workdir"), "cleanup": False}
+    with monkeypatch.context() as m:
+        m.chdir(tmp_path)
+        with pytest.warns(RuntimeWarning, match="will be set to True"):
+            queue(
+                config={"workdir": workdir, "datagrabber": datagrabber},
+                kind="HTCondor",
+            )
+    # Queueing does not clean up
+    assert WorkDirManager()._cleanup_dirs is False
+    # The jobs clean up
+    config = YAML().load(tmp_path / "junifer_jobs/junifer_job/config.yaml")
+    assert config["workdir"]["cleanup"] is True
+
+
+@pytest.mark.parametrize("as_dict", [False, True])
+def test_collect_workdir(tmp_path: Path, as_dict: bool) -> None:
+    """Test collect sets up the working directory.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    as_dict : bool
+        Whether to give the working directory as a dictionary or a path.
+
+    """
+    workdir, path = _workdir(tmp_path, as_dict)
+    storage = {"kind": "HDF5FeatureStorage", "uri": str(tmp_path / "out.hdf5")}
+    collect(storage, workdir=workdir)
+    assert WorkDirManager().workdir == path
+    assert path.is_dir()
+
+
+@pytest.mark.parametrize("as_dict", [False, True])
+def test_reset_workdir(tmp_path: Path, as_dict: bool) -> None:
+    """Test reset sets up the working directory.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    as_dict : bool
+        Whether to give the working directory as a dictionary or a path.
+
+    """
+    workdir, path = _workdir(tmp_path, as_dict)
+    storage = {"uri": str(tmp_path / "out.hdf5")}
+    reset(config={"workdir": workdir, "storage": storage})
+    assert WorkDirManager().workdir == path
+    assert path.is_dir()
+
+
 def test_reset_run(
     tmp_path: Path,
     datagrabber: dict[str, str],
@@ -769,6 +907,40 @@ def test_list_elements(
     assert "sub-01" in listed_elements
 
 
+@pytest.mark.parametrize("as_dict", [False, True, None])
+def test_list_elements_workdir(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    datagrabber: dict[str, str],
+    as_dict: bool | None,
+) -> None:
+    """Test elements listing sets up the working directory.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    monkeypatch : pytest.MonkeyPatch
+        The pytest.MonkeyPatch object.
+    datagrabber : dict
+        Testing datagrabber as dictionary.
+    as_dict : bool or None
+        Whether to give the working directory as a dictionary or a path.
+        If None, no working directory is given and the default is used.
+
+    """
+    if as_dict is None:
+        # The default working directory is in the temporary directory
+        monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
+        workdir, path = None, tmp_path / "junifer"
+    else:
+        workdir, path = _workdir(tmp_path, as_dict)
+    listed_elements = list_elements(datagrabber, workdir=workdir)
+    assert "sub-01" in listed_elements
+    assert WorkDirManager().workdir == path
+    assert path.is_dir()
+
+
 def test_parse_yaml_failure() -> None:
     """Test YAML parsing failure."""
     with pytest.raises(ValueError, match="does not exist"):

{junifer-0.0.8.dev298 → junifer-0.0.8.dev303}/junifer/cli/cli.py

@@ -44,6 +44,28 @@ __all__ = [
 ]
 
 
+def _parse_yaml(filepath: click.Path) -> dict:
+    """Parse the YAML file and set up the working directory.
+
+    The working directory is set up before anything else, as all the
+    components assume it is available.
+
+    Parameters
+    ----------
+    filepath : click.Path
+        The filepath to the configuration file.
+
+    Returns
+    -------
+    dict
+        The configuration.
+
+    """
+    config = cli_func.parse_yaml(filepath)  # type: ignore
+    cli_func._setup_workdir(config.get("workdir"))
+    return config
+
+
 def _validate_optional_verbose(
     ctx: click.Context, param: str, value: str | None
 ):
@@ -162,7 +184,7 @@ def run(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO(synchon): add validation
     # Parse YAML
-    config = cli_func.parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Retrieve working directory
     workdir = config["workdir"]
     # Fetch datagrabber
@@ -232,7 +254,7 @@ def collect(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO: add validation
     # Parse YAML
-    config = cli_func.parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Fetch storage
     storage = config["storage"]
     # Perform operation
@@ -299,7 +321,7 @@ def queue(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO: add validation
     # Parse YAML
-    config = cli_func.parse_yaml(filepath)  # type: ignore
+    config = _parse_yaml(filepath)
     # Check queue section
     if "queue" not in config:
         raise_error(f"No queue configuration found in {filepath}.")
@@ -450,7 +472,7 @@ def reset(
     # Setup logging
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # Parse YAML
-    config = cli_func.parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Perform operation
     cli_func.reset(config)
 
@@ -510,7 +532,7 @@ def list_elements(
     # Setup logging
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # Parse YAML
-    config = cli_func.parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Fetch datagrabber
     datagrabber = config["datagrabber"]
     # Parse elements

{junifer-0.0.8.dev298 → junifer-0.0.8.dev303}/junifer/cli/tests/test_cli.py

@@ -4,6 +4,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+import tempfile
 from collections.abc import Callable
 from pathlib import Path
 
@@ -12,6 +13,7 @@ from click.testing import CliRunner
 from ruamel.yaml import YAML
 
 from junifer.cli.cli import (
+    _parse_yaml,
     collect,
     list_elements,
     queue,
@@ -21,6 +23,7 @@ from junifer.cli.cli import (
     wtf,
 )
 from junifer.cli.parser import _parse_elements_file
+from junifer.pipeline import WorkDirManager
 
 
 pytestmark = pytest.mark.external
@@ -383,6 +386,75 @@ def test_list_elements_output_file(
         assert f"{elements[0]}\n{elements[1]}" == f.read()
 
 
+@pytest.mark.parametrize(
+    "workdir, expected",
+    [
+        ("workdir", "workdir"),
+        ({"path": "workdir", "cleanup": False}, "workdir"),
+        (None, "junifer"),
+    ],
+    ids=["path", "dict", "default"],
+)
+def test_parse_yaml_workdir(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    workdir: str | dict | None,
+    expected: str,
+) -> None:
+    """Test the working directory is set up when parsing the YAML.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    monkeypatch : pytest.MonkeyPatch
+        The pytest.MonkeyPatch object.
+    workdir : str or dict or None
+        The parametrized working directory in the YAML. If None, the YAML
+        has no working directory.
+    expected : str
+        The parametrized working directory set up, relative to ``tmp_path``.
+
+    """
+    # Use the test directory for relative paths and as temporary directory,
+    # where the default working directory is
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
+    contents = {"datagrabber": {"kind": "PartlyCloudyTestingDataGrabber"}}
+    if workdir is not None:
+        contents["workdir"] = workdir
+    yaml.dump(contents, stream=tmp_path / "in.yaml")
+    # The configuration is not changed
+    assert _parse_yaml(tmp_path / "in.yaml") == contents  # type: ignore
+    assert WorkDirManager().workdir.resolve() == tmp_path / expected
+    assert (tmp_path / expected).is_dir()
+
+
+def test_list_elements_workdir(tmp_path: Path) -> None:
+    """Test elements listing in the working directory of the YAML.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    # Get test config
+    infile = Path(__file__).parent / "data" / "partly_cloudy_agg_mean_tian.yml"
+    contents = yaml.load(infile)
+    # Working directory
+    workdir = tmp_path / "workdir"
+    contents["workdir"] = str(workdir)
+    # Write new test config
+    outfile = tmp_path / "in.yaml"
+    yaml.dump(contents, stream=outfile)
+    # Invoke list elements command
+    list_elements_result = runner.invoke(list_elements, [str(outfile)])
+    # Check
+    assert list_elements_result.exit_code == 0
+    assert WorkDirManager().workdir == workdir
+
+
 def test_wtf_short() -> None:
     """Test short version of wtf command."""
     # Invoke wtf command

{junifer-0.0.8.dev298 → junifer-0.0.8.dev303}/junifer/pipeline/tests/test_workdir_manager.py

@@ -29,6 +29,21 @@ def test_workdir_manager_workdir(tmp_path: Path) -> None:
     assert workdir_mgr.workdir == tmp_path
 
 
+def test_workdir_manager_create_workdir(tmp_path: Path) -> None:
+    """Test WorkDirManager creates the workdir if it does not exist.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    workdir = tmp_path / "new" / "workdir"
+    workdir_mgr = WorkDirManager(workdir=workdir)
+    assert workdir_mgr.workdir == workdir
+    assert workdir.is_dir()
+
+
 def test_workdir_manager_get_and_delete_element_tempdir(
     tmp_path: Path,
 ) -> None:

{junifer-0.0.8.dev298 → junifer-0.0.8.dev303}/junifer/pipeline/workdir_manager.py

@@ -63,16 +63,16 @@ class WorkDirManager(metaclass=Singleton):
         # Check and set topmost level directory if not provided
         if self._workdir is None:
             self._workdir = Path(tempfile.gettempdir()) / "junifer"
-            # Create directory if not found
-            if not self._workdir.is_dir():
-                logger.debug(
-                    "Creating working directory at "
-                    f"{self._workdir.resolve()!s}"
-                )
-                self._workdir.mkdir(parents=True)
+
+        # Create directory if not found
+        if not self._workdir.is_dir():
             logger.debug(
-                f"Setting working directory to {self._workdir.resolve()!s}"
+                f"Creating working directory at {self._workdir.resolve()!s}"
             )
+            self._workdir.mkdir(parents=True)
+        logger.debug(
+            f"Setting working directory to {self._workdir.resolve()!s}"
+        )
 
     def _cleanup(self) -> None:
         """Clean up the element and temporary directories."""

{junifer-0.0.8.dev298 → junifer-0.0.8.dev303/junifer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.8.dev298
+Version: 0.0.8.dev303
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

{junifer-0.0.8.dev298 → junifer-0.0.8.dev303}/junifer.egg-info/SOURCES.txt

@@ -76,6 +76,7 @@ docs/changes/newsfragments/492.bugfix
 docs/changes/newsfragments/499.enh
 docs/changes/newsfragments/502.bugfix
 docs/changes/newsfragments/503.enh
+docs/changes/newsfragments/509.enh
 docs/changes/newsfragments/510.bugfix
 docs/changes/newsfragments/511.bugfix
 docs/changes/newsfragments/511.bugfix.1

{junifer-0.0.8.dev298 → junifer-0.0.8.dev303}/junifer.egg-info/scm_file_list.json

@@ -61,6 +61,7 @@
     "docs/changes/newsfragments/499.enh",
     "docs/changes/newsfragments/502.bugfix",
     "docs/changes/newsfragments/503.enh",
+    "docs/changes/newsfragments/509.enh",
     "docs/changes/newsfragments/510.bugfix",
     "docs/changes/newsfragments/511.bugfix",
     "docs/changes/newsfragments/511.bugfix.1",

{junifer-0.0.8.dev298 → junifer-0.0.8.dev303}/junifer.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
   "tag": "0.0.7",
-  "distance": 298,
-  "node": "g30ddf7410414faec3138ccbac70a8890ff609537",
+  "distance": 303,
+  "node": "g4a74bdf26b428460a90404d6c0246318c1f95723",
   "dirty": false,
   "branch": "main",
   "node_date": "2026-10-09"