junifer 0.0.8.dev295__tar.gz → 0.0.8.dev303__tar.gz

{junifer-0.0.8.dev295/junifer.egg-info → junifer-0.0.8.dev303}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.8.dev295
+Version: 0.0.8.dev303
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

junifer-0.0.8.dev303/docs/changes/newsfragments/509.enh

@@ -0,0 +1 @@
+Set up the working directory first in all the commands of the CLI (from ``workdir`` in the YAML) and in :func:`.run`, :func:`.collect`, :func:`.queue`, :func:`.reset` and :func:`.list_elements`, adding the ``workdir`` parameter to :func:`.collect` and :func:`.list_elements`, and create the working directory if it does not exist by `Synchon Mandal`_ and `Fede Raimondo`_

junifer-0.0.8.dev303/docs/changes/newsfragments/523.misc

@@ -0,0 +1 @@
+Use :class:`.HDF5FeatureStorage` instead of :class:`.SQLiteFeatureStorage` in the tests and documentation, as the latter is only kept for compatibility, and fix the component registry test leaving components de-registered by `Fede Raimondo`_

{junifer-0.0.8.dev295 → junifer-0.0.8.dev303}/docs/understanding/storage.rst

@@ -21,7 +21,7 @@ The :ref:`Markers <marker>` are responsible for mapping the input
 :ref:`data type <data_types>` to its output :ref:`storage type <storage_types>`
 as shown :ref:`here <extending_markers_input_output>`.
 The storage object in turn declares and provides implementation for
-specific *storage type*. For example, :class:`.SQLiteFeatureStorage` supports
+specific *storage type*. For example, :class:`.HDF5FeatureStorage` supports
 saving ``matrix``, ``vector`` and ``timeseries`` via ``store_matrix``,
 ``store_vector`` and ``store_timeseries`` methods respectively.
 
@@ -88,3 +88,10 @@ Storage Interfaces
      - ``.hdf5``
      - HDF5
      - ``matrix``, ``vector``, ``timeseries``, ``timeseries_2d``, ``scalar_table``
+
+.. note::
+
+   :class:`.HDF5FeatureStorage` is the recommended storage.
+   :class:`.SQLiteFeatureStorage` is only kept for compatibility with features
+   stored in previous versions of ``junifer`` and should not be used for new
+   analyses.

{junifer-0.0.8.dev295 → junifer-0.0.8.dev303}/docs/using/running.rst

@@ -130,8 +130,8 @@ First we load the storage like so:
 
     from junifer.storage import HDF5FeatureStorage
 
-    # You need to import and use SQLiteFeatureStorage if you chose that
-    # for storage while extracting features
+    # If the features were stored with the legacy SQLiteFeatureStorage,
+    # import and use that class instead
     storage = HDF5FeatureStorage("<path/to/your/collected/file>")
 
 The best way to start analysing would be to list all the extracted features

{junifer-0.0.8.dev295 → junifer-0.0.8.dev303}/junifer/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.0.8.dev295'
-__version_tuple__ = version_tuple = (0, 0, 8, 'dev295')
+__version__ = version = '0.0.8.dev303'
+__version_tuple__ = version_tuple = (0, 0, 8, 'dev303')
 
-__commit_id__ = commit_id = 'g4eb842a09'
+__commit_id__ = commit_id = 'g4a74bdf26'

{junifer-0.0.8.dev295 → junifer-0.0.8.dev303}/junifer/api/functions.py

@@ -136,6 +136,45 @@ def _get_storage(storage_config: dict) -> StorageLike:
     )
 
 
+def _get_workdir_params(workdir: str | Path | dict | None) -> dict:
+    """Get the parameters of the working directory manager.
+
+    Parameters
+    ----------
+    workdir : str or pathlib.Path or dict or None
+        The working directory as a path, or as a dictionary with the keys
+        ``path`` and ``cleanup`` (default True). If None, the default of
+        :class:`.WorkDirManager` is used.
+
+    Returns
+    -------
+    dict
+        The parameters for :class:`.WorkDirManager`.
+
+    """
+    if isinstance(workdir, dict):
+        return {
+            "workdir": Path(workdir["path"]),
+            "cleanup": workdir.get("cleanup", True),
+        }
+    if workdir is None:
+        return {"workdir": None, "cleanup": True}
+    return {"workdir": Path(workdir), "cleanup": True}
+
+
+def _setup_workdir(workdir: str | Path | dict | None) -> None:
+    """Set up the working directory manager.
+
+    Parameters
+    ----------
+    workdir : str or pathlib.Path or dict or None
+        The working directory (see :func:`_get_workdir_params`).
+
+    """
+    WorkDirManager(**_get_workdir_params(workdir))
+    atexit.register(WorkDirManager()._cleanup)
+
+
 def run(
     workdir: str | Path | dict,
     datagrabber: dict,
@@ -180,25 +219,15 @@ def run(
         If invalid element selectors are found.
 
     """
-    # Conditional to handle workdir config
-    if isinstance(workdir, str | Path):
-        if isinstance(workdir, str):
-            workdir = {"workdir": Path(workdir), "cleanup": True}
-        else:
-            workdir = {"workdir": workdir, "cleanup": True}
-    elif isinstance(workdir, dict):
-        workdir["workdir"] = workdir.pop("path")
-
     # Initiate working directory manager with correct variation
-    if not workdir["cleanup"]:
+    if not _get_workdir_params(workdir)["cleanup"]:
         if elements is None or len(elements) > 1:
             raise_error(
                 "Cannot disable `workdir.cleanup` as "
                 f"{len(elements) if elements is not None else 'all'} "
                 "elements will be processed"
             )
-    WorkDirManager(**workdir)
-    atexit.register(WorkDirManager()._cleanup)
+    _setup_workdir(workdir)
 
     # Get datagrabber to use
     datagrabber_object = _get_datagrabber(datagrabber.copy())
@@ -253,7 +282,7 @@ def run(
                 mc.fit(datagrabber_object[t_element])
 
 
-def collect(storage: dict) -> None:
+def collect(storage: dict, workdir: str | Path | dict | None = None) -> None:
     """Collect and store data.
 
     Parameters
@@ -262,8 +291,12 @@ def collect(storage: dict) -> None:
         Storage to use. Must have a key ``kind`` with the kind of
         storage to use. All other keys are passed to the storage
         constructor.
+    workdir : str or pathlib.Path or dict or None, optional
+        The working directory, as in :func:`.run`. If None, the default of
+        :class:`.WorkDirManager` is used (default None).
 
     """
+    _setup_workdir(workdir)
     logger.info(f"Collecting data using {storage['kind']}")
     logger.debug(f"\tStorage params: {storage}")
     if "single_output" not in storage:
@@ -307,6 +340,7 @@ def queue(
         if the ``jobdir`` exists and ``overwrite = False``.
 
     """
+    _setup_workdir(config.get("workdir"))
     valid_kind = ["HTCondor", "GNUParallelLocal"]
     if kind not in valid_kind:
         raise_error(
@@ -410,6 +444,7 @@ def reset(config: dict) -> None:
         The configuration to be used for resetting.
 
     """
+    _setup_workdir(config.get("workdir"))
     # Fetch storage
     storage = config["storage"]
     storage_uri = Path(storage["uri"])
@@ -442,6 +477,7 @@ def reset(config: dict) -> None:
 def list_elements(
     datagrabber: dict,
     elements: Elements | None = None,
+    workdir: str | Path | dict | None = None,
 ) -> str:
     """List elements of the datagrabber filtered using `elements`.
 
@@ -454,8 +490,13 @@ def list_elements(
     elements : list or None, optional
         Element(s) to filter using. Will be used to index the DataGrabber
         (default None).
+    workdir : str or pathlib.Path or dict or None, optional
+        Directory where the DataGrabber will be used, as in :func:`.run`.
+        If None, the default of :class:`.WorkDirManager` is used
+        (default None).
 
     """
+    _setup_workdir(workdir)
     # Get datagrabber to use
     datagrabber_object = _get_datagrabber(datagrabber)
 

{junifer-0.0.8.dev295 → junifer-0.0.8.dev303}/junifer/api/tests/test_functions.py

@@ -7,6 +7,7 @@
 
 import logging
 import sys
+import tempfile
 from contextlib import AbstractContextManager, nullcontext
 from pathlib import Path
 from typing import Any
@@ -18,7 +19,7 @@ from ruamel.yaml import YAML
 import junifer.testing.registry  # noqa: F401
 from junifer.api import collect, list_elements, parse_yaml, queue, reset, run
 from junifer.datagrabber.base import BaseDataGrabber
-from junifer.pipeline import PipelineComponentRegistry
+from junifer.pipeline import PipelineComponentRegistry, WorkDirManager
 from junifer.typing import Elements
 
 
@@ -92,7 +93,7 @@ def markers() -> list[dict[str, list[str] | str]]:
 def storage() -> dict[str, str]:
     """Return a storage as a dictionary."""
     return {
-        "kind": "SQLiteFeatureStorage",
+        "kind": "HDF5FeatureStorage",
     }
 
 
@@ -158,7 +159,7 @@ def test_run_single_element(
 
     """
     # Set storage
-    storage["uri"] = str((tmp_path / "out.sqlite").resolve())
+    storage["uri"] = str((tmp_path / "out.hdf5").resolve())
     # Run operations
     with expect:
         run(
@@ -169,7 +170,7 @@ def test_run_single_element(
             elements=element,
         )
         # Check files
-        files = list(tmp_path.glob("*.sqlite"))
+        files = list(tmp_path.glob("*.hdf5"))
         assert len(files) == 1
 
 
@@ -191,7 +192,7 @@ def test_run_single_element_with_preprocessing(
 
     """
     # Set storage
-    storage["uri"] = str((tmp_path / "out.sqlite").resolve())
+    storage["uri"] = str((tmp_path / "out.hdf5").resolve())
     # Run operations
     run(
         workdir={"path": tmp_path, "cleanup": False},
@@ -209,7 +210,7 @@ def test_run_single_element_with_preprocessing(
         elements=["sub-01"],
     )
     # Check files
-    files = list(tmp_path.glob("*.sqlite"))
+    files = list(tmp_path.glob("*.hdf5"))
     assert len(files) == 1
 
 
@@ -250,7 +251,7 @@ def test_run_multi_element_multi_output(
 
     """
     # Set storage
-    storage["uri"] = str((tmp_path / "out.sqlite").resolve())
+    storage["uri"] = str((tmp_path / "out.hdf5").resolve())
     storage["single_output"] = False  # type: ignore
     # Run operations
     with expect:
@@ -262,7 +263,7 @@ def test_run_multi_element_multi_output(
             elements=element,
         )
         # Check files
-        files = list(tmp_path.glob("*.sqlite"))
+        files = list(tmp_path.glob("*.hdf5"))
         assert len(files) == 2
 
 
@@ -287,7 +288,7 @@ def test_run_multi_element_single_output(
 
     """
     # Set storage
-    storage["uri"] = str((tmp_path / "out.sqlite").resolve())
+    storage["uri"] = str((tmp_path / "out.hdf5").resolve())
     storage["single_output"] = True  # type: ignore
     # Run operations
     run(
@@ -298,9 +299,9 @@ def test_run_multi_element_single_output(
         elements=["sub-01", "sub-03"],
     )
     # Check files
-    files = list(tmp_path.glob("*.sqlite"))
+    files = list(tmp_path.glob("*.hdf5"))
     assert len(files) == 1
-    assert files[0].name == "out.sqlite"
+    assert files[0].name == "out.hdf5"
 
 
 def test_run_and_collect(
@@ -324,7 +325,7 @@ def test_run_and_collect(
 
     """
     # Set storage
-    uri = tmp_path / "out.sqlite"
+    uri = tmp_path / "out.hdf5"
     storage["uri"] = str(uri.resolve())
     storage["single_output"] = False  # type: ignore
     # Run operations
@@ -343,9 +344,9 @@ def test_run_and_collect(
     )
     elements = dg.get_elements()  # type: ignore
     # This should create one file per element
-    files = list(tmp_path.glob("*.sqlite"))
+    files = list(tmp_path.glob("*.hdf5"))
     assert len(files) == len(elements)
-    # But the test.sqlite file should not exist
+    # But the test.hdf5 file should not exist
     assert not uri.exists()
     # Collect in storage
     collect(storage)
@@ -639,6 +640,143 @@ def test_queue_without_elements(
             assert "Queue done" in caplog.text
 
 
+def _workdir(tmp_path: Path, as_dict: bool) -> tuple[str | dict, Path]:
+    """Get a working directory in the test directory.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    as_dict : bool
+        Whether to give the working directory as a dictionary or a path.
+
+    Returns
+    -------
+    str or dict
+        The working directory, as given to the functions.
+    pathlib.Path
+        The path to the working directory.
+
+    """
+    path = tmp_path / "workdir"
+    workdir = {"path": str(path), "cleanup": True} if as_dict else str(path)
+    return workdir, path
+
+
+@pytest.mark.parametrize("as_dict", [False, True])
+@pytest.mark.parametrize("elements", [None, ["sub-01"]])
+def test_queue_workdir(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    datagrabber: dict[str, str],
+    elements: list[str] | None,
+    as_dict: bool,
+) -> None:
+    """Test queue sets up the working directory.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    monkeypatch : pytest.MonkeyPatch
+        The pytest.MonkeyPatch object.
+    datagrabber : dict
+        Testing datagrabber as dictionary.
+    elements : list of str or None
+        The parametrized elements to queue. If None, the elements are
+        listed using the datagrabber.
+    as_dict : bool
+        Whether to give the working directory as a dictionary or a path.
+
+    """
+    workdir, path = _workdir(tmp_path, as_dict)
+    with monkeypatch.context() as m:
+        m.chdir(tmp_path)
+        queue(
+            config={"workdir": workdir, "datagrabber": datagrabber},
+            kind="HTCondor",
+            elements=elements,
+        )
+    assert WorkDirManager().workdir == path
+    assert path.is_dir()
+    # The working directory is written to the job configuration as given
+    config = YAML().load(tmp_path / "junifer_jobs/junifer_job/config.yaml")
+    assert config["workdir"] == workdir
+
+
+def test_queue_workdir_cleanup(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    datagrabber: dict[str, str],
+) -> None:
+    """Test queued jobs clean up the working directory.
+
+    Disabling the cleanup only applies to queueing (e.g. to debug it), as
+    the jobs would otherwise leave the files of each element.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    monkeypatch : pytest.MonkeyPatch
+        The pytest.MonkeyPatch object.
+    datagrabber : dict
+        Testing datagrabber as dictionary.
+
+    """
+    workdir = {"path": str(tmp_path / "workdir"), "cleanup": False}
+    with monkeypatch.context() as m:
+        m.chdir(tmp_path)
+        with pytest.warns(RuntimeWarning, match="will be set to True"):
+            queue(
+                config={"workdir": workdir, "datagrabber": datagrabber},
+                kind="HTCondor",
+            )
+    # Queueing does not clean up
+    assert WorkDirManager()._cleanup_dirs is False
+    # The jobs clean up
+    config = YAML().load(tmp_path / "junifer_jobs/junifer_job/config.yaml")
+    assert config["workdir"]["cleanup"] is True
+
+
+@pytest.mark.parametrize("as_dict", [False, True])
+def test_collect_workdir(tmp_path: Path, as_dict: bool) -> None:
+    """Test collect sets up the working directory.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    as_dict : bool
+        Whether to give the working directory as a dictionary or a path.
+
+    """
+    workdir, path = _workdir(tmp_path, as_dict)
+    storage = {"kind": "HDF5FeatureStorage", "uri": str(tmp_path / "out.hdf5")}
+    collect(storage, workdir=workdir)
+    assert WorkDirManager().workdir == path
+    assert path.is_dir()
+
+
+@pytest.mark.parametrize("as_dict", [False, True])
+def test_reset_workdir(tmp_path: Path, as_dict: bool) -> None:
+    """Test reset sets up the working directory.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    as_dict : bool
+        Whether to give the working directory as a dictionary or a path.
+
+    """
+    workdir, path = _workdir(tmp_path, as_dict)
+    storage = {"uri": str(tmp_path / "out.hdf5")}
+    reset(config={"workdir": workdir, "storage": storage})
+    assert WorkDirManager().workdir == path
+    assert path.is_dir()
+
+
 def test_reset_run(
     tmp_path: Path,
     datagrabber: dict[str, str],
@@ -660,7 +798,7 @@ def test_reset_run(
 
     """
     # Create storage
-    storage["uri"] = tmp_path / "test_reset_run.sqlite"  # type: ignore
+    storage["uri"] = tmp_path / "test_reset_run.hdf5"  # type: ignore
     # Run operation to generate files
     run(
         workdir=tmp_path,
@@ -711,7 +849,7 @@ def test_reset_queue(
     with monkeypatch.context() as m:
         m.chdir(tmp_path)
         # Create storage
-        storage["uri"] = "test_reset_queue.sqlite"
+        storage["uri"] = "test_reset_queue.hdf5"
         # Set job name
         if job_name is None:
             job_name = "junifer_job"
@@ -769,6 +907,40 @@ def test_list_elements(
     assert "sub-01" in listed_elements
 
 
+@pytest.mark.parametrize("as_dict", [False, True, None])
+def test_list_elements_workdir(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    datagrabber: dict[str, str],
+    as_dict: bool | None,
+) -> None:
+    """Test elements listing sets up the working directory.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    monkeypatch : pytest.MonkeyPatch
+        The pytest.MonkeyPatch object.
+    datagrabber : dict
+        Testing datagrabber as dictionary.
+    as_dict : bool or None
+        Whether to give the working directory as a dictionary or a path.
+        If None, no working directory is given and the default is used.
+
+    """
+    if as_dict is None:
+        # The default working directory is in the temporary directory
+        monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
+        workdir, path = None, tmp_path / "junifer"
+    else:
+        workdir, path = _workdir(tmp_path, as_dict)
+    listed_elements = list_elements(datagrabber, workdir=workdir)
+    assert "sub-01" in listed_elements
+    assert WorkDirManager().workdir == path
+    assert path.is_dir()
+
+
 def test_parse_yaml_failure() -> None:
     """Test YAML parsing failure."""
     with pytest.raises(ValueError, match="does not exist"):

{junifer-0.0.8.dev295 → junifer-0.0.8.dev303}/junifer/cli/cli.py

@@ -44,6 +44,28 @@ __all__ = [
 ]
 
 
+def _parse_yaml(filepath: click.Path) -> dict:
+    """Parse the YAML file and set up the working directory.
+
+    The working directory is set up before anything else, as all the
+    components assume it is available.
+
+    Parameters
+    ----------
+    filepath : click.Path
+        The filepath to the configuration file.
+
+    Returns
+    -------
+    dict
+        The configuration.
+
+    """
+    config = cli_func.parse_yaml(filepath)  # type: ignore
+    cli_func._setup_workdir(config.get("workdir"))
+    return config
+
+
 def _validate_optional_verbose(
     ctx: click.Context, param: str, value: str | None
 ):
@@ -162,7 +184,7 @@ def run(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO(synchon): add validation
     # Parse YAML
-    config = cli_func.parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Retrieve working directory
     workdir = config["workdir"]
     # Fetch datagrabber
@@ -232,7 +254,7 @@ def collect(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO: add validation
     # Parse YAML
-    config = cli_func.parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Fetch storage
     storage = config["storage"]
     # Perform operation
@@ -299,7 +321,7 @@ def queue(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO: add validation
     # Parse YAML
-    config = cli_func.parse_yaml(filepath)  # type: ignore
+    config = _parse_yaml(filepath)
     # Check queue section
     if "queue" not in config:
         raise_error(f"No queue configuration found in {filepath}.")
@@ -450,7 +472,7 @@ def reset(
     # Setup logging
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # Parse YAML
-    config = cli_func.parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Perform operation
     cli_func.reset(config)
 
@@ -510,7 +532,7 @@ def list_elements(
     # Setup logging
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # Parse YAML
-    config = cli_func.parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Fetch datagrabber
     datagrabber = config["datagrabber"]
     # Parse elements

{junifer-0.0.8.dev295 → junifer-0.0.8.dev303}/junifer/cli/tests/data/gmd_mean.yaml

@@ -10,5 +10,5 @@ markers:
     parcellation: TianxS2x3TxMNInonlinear2009cAsym
     method: mean
 storage:
-  kind: SQLiteFeatureStorage
-  uri: /Users/fraimondo/dev/tbox/junifer/scratch/db/test.sqlite
+  kind: HDF5FeatureStorage
+  uri: /Users/fraimondo/dev/tbox/junifer/scratch/db/test.hdf5

{junifer-0.0.8.dev295 → junifer-0.0.8.dev303}/junifer/cli/tests/data/gmd_mean_htcondor.yaml

@@ -9,8 +9,8 @@ markers:
     parcellation: TianxS2x3TxMNInonlinear2009cAsym
     method: mean
 storage:
-  kind: SQLiteFeatureStorage
-  uri: /Users/fraimondo/dev/tbox/junifer/scratch/db/test.sqlite
+  kind: HDF5FeatureStorage
+  uri: /Users/fraimondo/dev/tbox/junifer/scratch/db/test.hdf5
 queue:
   jobname: TestHTCondorQueue
   kind: HTCondor