junifer 0.0.7.dev332__py3-none-any.whl → 0.0.8__py3-none-any.whl

junifer/cli/__init__.py

@@ -3,17 +3,11 @@
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-import sys
+from importlib.metadata import entry_points
 
 import lazy_loader as lazy
 
 
-if sys.version_info < (3, 11):  # pragma: no cover
-    from importlib_metadata import entry_points
-else:
-    from importlib.metadata import entry_points
-
-
 __getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)
 
 

junifer/cli/cli.py

@@ -8,7 +8,6 @@ import pathlib
 import subprocess
 import sys
 from pathlib import Path
-from typing import Optional, Union
 
 import click
 
@@ -18,7 +17,7 @@ from ..utils import (
     raise_error,
     yaml,
 )
-from .parser import parse_elements, parse_yaml
+from .parser import parse_elements
 from .utils import (
     _get_dependency_information,
     _get_environment_information,
@@ -45,8 +44,30 @@ __all__ = [
 ]
 
 
+def _parse_yaml(filepath: click.Path) -> dict:
+    """Parse the YAML file and set up the working directory.
+
+    The working directory is set up before anything else, as all the
+    components assume it is available.
+
+    Parameters
+    ----------
+    filepath : click.Path
+        The filepath to the configuration file.
+
+    Returns
+    -------
+    dict
+        The configuration.
+
+    """
+    config = cli_func.parse_yaml(filepath)  # type: ignore
+    cli_func._setup_workdir(config.get("workdir"))
+    return config
+
+
 def _validate_optional_verbose(
-    ctx: click.Context, param: str, value: Optional[str]
+    ctx: click.Context, param: str, value: str | None
 ):
     """Validate optional verbose option.
 
@@ -71,9 +92,7 @@ def _validate_optional_verbose(
         return _validate_verbose(ctx, param, value)
 
 
-def _validate_verbose(
-    ctx: click.Context, param: str, value: str
-) -> Union[str, int]:
+def _validate_verbose(ctx: click.Context, param: str, value: str) -> str | int:
     """Validate verbose option.
 
     Parameters
@@ -142,8 +161,8 @@ def cli() -> None:  # pragma: no cover
 def run(
     filepath: click.Path,
     element: tuple[str],
-    verbose: Union[str, int],
-    verbose_datalad: Optional[Union[str, int]],
+    verbose: str | int,
+    verbose_datalad: str | int | None,
 ) -> None:
     """Run feature extraction.
 
@@ -165,7 +184,7 @@ def run(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO(synchon): add validation
     # Parse YAML
-    config = parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Retrieve working directory
     workdir = config["workdir"]
     # Fetch datagrabber
@@ -214,8 +233,8 @@ def run(
 )
 def collect(
     filepath: click.Path,
-    verbose: Union[str, int],
-    verbose_datalad: Union[str, int, None],
+    verbose: str | int,
+    verbose_datalad: str | int | None,
 ) -> None:
     """Collect extracted features.
 
@@ -235,7 +254,7 @@ def collect(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO: add validation
     # Parse YAML
-    config = parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Fetch storage
     storage = config["storage"]
     # Perform operation
@@ -270,8 +289,8 @@ def queue(
     element: tuple[str],
     overwrite: bool,
     submit: bool,
-    verbose: Union[str, int],
-    verbose_datalad: Union[str, int, None],
+    verbose: str | int,
+    verbose_datalad: str | int | None,
 ) -> None:
     """Queue feature extraction.
 
@@ -302,7 +321,7 @@ def queue(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO: add validation
     # Parse YAML
-    config = parse_yaml(filepath)  # type: ignore
+    config = _parse_yaml(filepath)
     # Check queue section
     if "queue" not in config:
         raise_error(f"No queue configuration found in {filepath}.")
@@ -433,8 +452,8 @@ def selftest(subpkg: str) -> None:
 )
 def reset(
     filepath: click.Path,
-    verbose: Union[str, int],
-    verbose_datalad: Union[str, int, None],
+    verbose: str | int,
+    verbose_datalad: str | int | None,
 ) -> None:
     """Reset generated assets.
 
@@ -453,7 +472,7 @@ def reset(
     # Setup logging
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # Parse YAML
-    config = parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Perform operation
     cli_func.reset(config)
 
@@ -487,9 +506,9 @@ def reset(
 def list_elements(
     filepath: click.Path,
     element: tuple[str],
-    output_file: Optional[click.Path],
-    verbose: Union[str, int],
-    verbose_datalad: Union[str, int, None],
+    output_file: click.Path | None,
+    verbose: str | int,
+    verbose_datalad: str | int | None,
 ) -> None:
     """List elements of a dataset.
 
@@ -513,7 +532,7 @@ def list_elements(
     # Setup logging
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # Parse YAML
-    config = parse_yaml(filepath)
+    config = _parse_yaml(filepath)
     # Fetch datagrabber
     datagrabber = config["datagrabber"]
     # Parse elements

junifer/cli/parser.py

@@ -4,147 +4,19 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-import importlib
-import importlib.util
-import sys
 from pathlib import Path
-from typing import Union
 
 import pandas as pd
 
 from ..typing import Elements
-from ..utils import logger, raise_error, warn_with_log, yaml
+from ..utils import logger, raise_error, warn_with_log
 
 
-__all__ = ["parse_elements", "parse_yaml"]
+__all__ = ["parse_elements"]
 
 
-def parse_yaml(filepath: Union[str, Path]) -> dict:  # noqa: C901
-    """Parse YAML.
-
-    Parameters
-    ----------
-    filepath : str or pathlib.Path
-        The filepath to read from.
-
-    Returns
-    -------
-    dict
-        The contents represented as dictionary.
-
-    """
-    # Convert str to Path
-    if not isinstance(filepath, Path):
-        filepath = Path(filepath)
-
-    logger.info(f"Parsing yaml file: {filepath.absolute()!s}")
-    # Filepath existence check
-    if not filepath.exists():
-        raise_error(f"File does not exist: {filepath.absolute()!s}")
-    # Filepath reading
-    contents = yaml.load(filepath)
-    if "elements" in contents:
-        if contents["elements"] is None:
-            raise_error(
-                "The elements key was defined but its content is empty. "
-                "Please define the elements to operate on or remove the key."
-            )
-    # load modules
-    if "with" in contents:
-        to_load = contents["with"]
-        # Convert load modules to list
-        if not isinstance(to_load, list):
-            to_load = [to_load]
-        # Initialize list to have absolute paths for custom modules
-        final_to_load = []
-        for t_module in to_load:
-            if t_module.endswith(".py"):
-                logger.debug(f"Importing file: {t_module}")
-                # This resolves both absolute and relative paths
-                file_path = filepath.parent / t_module
-                if not file_path.exists():
-                    raise_error(
-                        f"File in 'with' section does not exist: {file_path}"
-                    )
-                # Add the parent directory to the sys.path so that the
-                # any imports from this module work correctly
-                t_path = str(file_path.parent)
-                if t_path not in sys.path:
-                    sys.path.append(str(file_path.parent))
-
-                spec = importlib.util.spec_from_file_location(
-                    t_module, file_path
-                )
-                module = importlib.util.module_from_spec(spec)  # type: ignore
-                sys.modules[t_module] = module
-                spec.loader.exec_module(module)  # type: ignore
-
-                # Add absolute path to final list
-                final_to_load.append(str(file_path.resolve()))
-
-                # Check if the module has junifer_module_deps function
-                if hasattr(module, "junifer_module_deps"):
-                    logger.debug(
-                        f"Module {t_module} has junifer_module_deps function"
-                    )
-                    # Get the dependencies
-                    deps = module.junifer_module_deps()
-                    # Add the dependencies to the final list
-                    for dep in deps:
-                        if dep not in final_to_load:
-                            final_to_load.append(
-                                str((file_path.parent / dep).resolve())
-                            )
-            else:
-                logger.info(f"Importing module: {t_module}")
-                importlib.import_module(t_module)
-                # Add module to final list
-                final_to_load.append(t_module)
-
-        # Replace modules to be loaded so that custom modules will take the
-        # absolute path. This was not the case as found in #224. Similar thing
-        # is done with the storage URI below.
-        contents["with"] = final_to_load
-
-    # Compute path for the URI parameter in storage files that are relative
-    # This is a tricky thing that appeared in #127. The problem is that
-    # the path in the URI parameter is relative to YAML file, not to the
-    # current working directory. If we leave it as is in the contents
-    # dict, then it will be used later in the ``build`` function as is,
-    # which will be computed relative to the current working directory.
-    # The solution is to compute the absolute path and replace the
-    # relative path in the contents dict with the absolute path.
-
-    # Check if the storage file is defined
-    if "storage" in contents:
-        if "uri" in contents["storage"]:
-            # Check if the storage file is relative
-            uri_path = Path(contents["storage"]["uri"])
-            if not uri_path.is_absolute():
-                # Compute the absolute path
-                contents["storage"]["uri"] = str(
-                    (filepath.parent / uri_path).resolve()
-                )
-
-    # Allow relative path if queue env kind is venv; same motivation as above
-    if "queue" in contents:
-        if "env" in contents["queue"]:
-            if "venv" == contents["queue"]["env"]["kind"]:
-                # Check if the env name is relative
-                venv_path = Path(contents["queue"]["env"]["name"])
-                if not venv_path.is_absolute():
-                    # Compute the absolute path
-                    contents["queue"]["env"]["name"] = str(
-                        (filepath.parent / venv_path).resolve()
-                    )
-
-    return contents
-
-
-def parse_elements(
-    element: tuple[str, ...], config: dict
-) -> Union[Elements, None]:
-    """Parse elements from cli.
+def parse_elements(element: tuple[str, ...], config: dict) -> Elements | None:
+    """Parse elements from cli, or from the configuration if not in the cli.
 
     Parameters
     ----------
@@ -156,13 +28,13 @@ def parse_elements(
     Returns
     -------
     list or None
-        The element(s) as list or None.
+        The element(s) as list, or None (all the elements) if they are not in
+        the command-line options nor in the configuration.
 
     Raises
     ------
     ValueError
-        If no element is found either in the command-line options or
-        the configuration file.
+        If the ``elements`` key of the configuration has no value.
 
     Warns
     -----
@@ -172,9 +44,22 @@ def parse_elements(
 
     """
     logger.debug(f"Parsing elements: {element}")
-    # Early return None to continue with all elements
+    # Use the elements of the configuration if not in the command line
     if not element:
-        return None
+        if "elements" not in config:
+            # None to continue with all elements
+            return None
+        elements = config["elements"]
+        if elements is None:
+            raise_error(
+                "The `elements` key is set in the configuration, but its value"
+                " is `None`. It is likely that there is an empty `elements` "
+                "section in the yaml configuration file."
+            )
+        # A single element, and the elements with multiple values as tuples
+        if not isinstance(elements, list):
+            elements = [elements]
+        return [tuple(x) if isinstance(x, list) else x for x in elements]
     # Check if the element is a file for single element;
     # if yes, then parse elements from it
     if len(element) == 1 and Path(element[0]).resolve().is_file():
@@ -183,7 +68,7 @@ def parse_elements(
         # Process multi-keyed elements
         elements = [tuple(x.split(",")) if "," in x else x for x in element]
     logger.debug(f"Parsed elements: {elements}")
-    if elements is not None and "elements" in config:
+    if "elements" in config:
         warn_with_log(
             "One or more elements have been specified in both the command "
             "line and in the config file. The command line has precedence "
@@ -192,15 +77,6 @@ def parse_elements(
             "the configuration file will be ignored. To remove this warning, "
             "please remove the `elements` item from the configuration file."
         )
-    elif elements is None:
-        # Check in config
-        elements = config.get("elements", None)
-        if elements is None:
-            raise_error(
-                "The `elements` key is set in the configuration, but its value"
-                " is `None`. It is likely that there is an empty `elements` "
-                "section in the yaml configuration file."
-            )
     return elements
 
 

junifer/cli/tests/data/gmd_mean.yaml

@@ -2,13 +2,13 @@ with: junifer.testing.registry
 workdir: /tmp
 
 datagrabber:
-  kind: OasisVBMTestingDataGrabber
-elements: [1, 2]
+  kind: UCLACNPVBMTestingDataGrabber
+elements: [sub-10206, sub-10339]
 markers:
-  - name: Schaefer1000x7_Mean
+  - name: TianxS2_Mean
     kind: ParcelAggregation
-    parcellation: Schaefer1000x7
+    parcellation: TianxS2x3TxMNInonlinear2009cAsym
     method: mean
 storage:
-  kind: SQLiteFeatureStorage
-  uri: /Users/fraimondo/dev/tbox/junifer/scratch/db/test.sqlite
+  kind: HDF5FeatureStorage
+  uri: /Users/fraimondo/dev/tbox/junifer/scratch/db/test.hdf5

junifer/cli/tests/data/gmd_mean_htcondor.yaml

@@ -2,19 +2,20 @@ with: junifer.testing.registry
 workdir: /tmp
 
 datagrabber:
-  kind: OasisVBMTestingDataGrabber
+  kind: UCLACNPVBMTestingDataGrabber
 markers:
-  - name: Schaefer1000x7_Mean
+  - name: TianxS2_Mean
     kind: ParcelAggregation
-    parcellation: Schaefer1000x7
+    parcellation: TianxS2x3TxMNInonlinear2009cAsym
     method: mean
 storage:
-  kind: SQLiteFeatureStorage
-  uri: /Users/fraimondo/dev/tbox/junifer/scratch/db/test.sqlite
+  kind: HDF5FeatureStorage
+  uri: /Users/fraimondo/dev/tbox/junifer/scratch/db/test.hdf5
 queue:
   jobname: TestHTCondorQueue
   kind: HTCondor
   env:
     kind: conda
     name: junifer
+    shell: bash
   mem: 8G

junifer/cli/tests/data/partly_cloudy_agg_mean_tian.yml

@@ -4,6 +4,7 @@ with: junifer.testing.registry
 
 datagrabber:
   kind: PartlyCloudyTestingDataGrabber
+  n_subjects: 4
 
 markers:
   - kind: ParcelAggregation

junifer/cli/tests/test_cli.py

@@ -4,14 +4,16 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+import tempfile
+from collections.abc import Callable
 from pathlib import Path
-from typing import Callable
 
 import pytest
 from click.testing import CliRunner
 from ruamel.yaml import YAML
 
 from junifer.cli.cli import (
+    _parse_yaml,
     collect,
     list_elements,
     queue,
@@ -20,7 +22,11 @@ from junifer.cli.cli import (
     selftest,
     wtf,
 )
-from junifer.cli.parser import _parse_elements_file
+from junifer.cli.parser import _parse_elements_file, parse_elements
+from junifer.pipeline import WorkDirManager
+
+
+pytestmark = pytest.mark.external
 
 
 # Configure YAML class
@@ -199,6 +205,47 @@ def test_multi_element_access(
     assert read_elements == expected_list
 
 
+@pytest.mark.parametrize(
+    "element, config, expected",
+    [
+        ((), {}, None),
+        ((), {"elements": ["sub-01", "sub-02"]}, ["sub-01", "sub-02"]),
+        ((), {"elements": "sub-01"}, ["sub-01"]),
+        ((), {"elements": [["sub-01", "ses-01"]]}, [("sub-01", "ses-01")]),
+        (("sub-01", "sub-02,ses-01"), {}, ["sub-01", ("sub-02", "ses-01")]),
+    ],
+)
+def test_parse_elements(
+    element: tuple[str, ...], config: dict, expected: list | None
+) -> None:
+    """Test parsing the elements from the command line or the configuration.
+
+    Parameters
+    ----------
+    element : tuple of str
+        The parametrized elements of the command line.
+    config : dict
+        The parametrized configuration.
+    expected : list or None
+        The parametrized elements to expect.
+
+    """
+    assert parse_elements(element, config) == expected
+
+
+def test_parse_elements_command_line_precedence() -> None:
+    """Test the elements of the command line have precedence."""
+    with pytest.warns(RuntimeWarning, match="command line has precedence"):
+        elements = parse_elements(("sub-01",), {"elements": ["sub-02"]})
+    assert elements == ["sub-01"]
+
+
+def test_parse_elements_empty_config() -> None:
+    """Test an empty elements key in the configuration."""
+    with pytest.raises(ValueError, match="its value is `None`"):
+        parse_elements((), {"elements": None})
+
+
 def test_queue(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     """Test queue command.
 
@@ -219,7 +266,7 @@ def test_queue(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
         # Working directory
         contents["workdir"] = str(tmp_path.resolve())
         # Storage
-        contents["storage"]["uri"] = str((tmp_path / "out.sqlite").resolve())
+        contents["storage"]["uri"] = str((tmp_path / "out.hdf5").resolve())
         # Write new test config
         outfile = tmp_path / "in.yaml"
         yaml.dump(contents, stream=outfile)
@@ -271,7 +318,7 @@ def test_reset(
         # Working directory
         contents["workdir"] = str(tmp_path.resolve())
         # Storage
-        contents["storage"]["uri"] = str((tmp_path / "out.sqlite").resolve())
+        contents["storage"]["uri"] = str((tmp_path / "out.hdf5").resolve())
         # Write new test config
         outfile = tmp_path / "in.yaml"
         yaml.dump(contents, stream=outfile)
@@ -380,6 +427,75 @@ def test_list_elements_output_file(
         assert f"{elements[0]}\n{elements[1]}" == f.read()
 
 
+@pytest.mark.parametrize(
+    "workdir, expected",
+    [
+        ("workdir", "workdir"),
+        ({"path": "workdir", "cleanup": False}, "workdir"),
+        (None, "junifer"),
+    ],
+    ids=["path", "dict", "default"],
+)
+def test_parse_yaml_workdir(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    workdir: str | dict | None,
+    expected: str,
+) -> None:
+    """Test the working directory is set up when parsing the YAML.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+    monkeypatch : pytest.MonkeyPatch
+        The pytest.MonkeyPatch object.
+    workdir : str or dict or None
+        The parametrized working directory in the YAML. If None, the YAML
+        has no working directory.
+    expected : str
+        The parametrized working directory set up, relative to ``tmp_path``.
+
+    """
+    # Use the test directory for relative paths and as temporary directory,
+    # where the default working directory is
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
+    contents = {"datagrabber": {"kind": "PartlyCloudyTestingDataGrabber"}}
+    if workdir is not None:
+        contents["workdir"] = workdir
+    yaml.dump(contents, stream=tmp_path / "in.yaml")
+    # The configuration is not changed
+    assert _parse_yaml(tmp_path / "in.yaml") == contents  # type: ignore
+    assert WorkDirManager().workdir.resolve() == tmp_path / expected
+    assert (tmp_path / expected).is_dir()
+
+
+def test_list_elements_workdir(tmp_path: Path) -> None:
+    """Test elements listing in the working directory of the YAML.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    # Get test config
+    infile = Path(__file__).parent / "data" / "partly_cloudy_agg_mean_tian.yml"
+    contents = yaml.load(infile)
+    # Working directory
+    workdir = tmp_path / "workdir"
+    contents["workdir"] = str(workdir)
+    # Write new test config
+    outfile = tmp_path / "in.yaml"
+    yaml.dump(contents, stream=outfile)
+    # Invoke list elements command
+    list_elements_result = runner.invoke(list_elements, [str(outfile)])
+    # Check
+    assert list_elements_result.exit_code == 0
+    assert WorkDirManager().workdir == workdir
+
+
 def test_wtf_short() -> None:
     """Test short version of wtf command."""
     # Invoke wtf command