junifer 0.0.8.dev195__tar.gz → 0.0.8.dev200__tar.gz

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/.github/workflows/ci.yml

@@ -61,7 +61,7 @@ jobs:
         with:
           submodules: true
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Python dependencies

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/.github/workflows/docker-ci-publish.yml

@@ -37,22 +37,22 @@ jobs:
         if: github.event_name != 'pull_request'
         uses: sigstore/cosign-installer@v4.1.2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4.1.0
+        uses: docker/setup-buildx-action@v4.2.0
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4.2.0
+        uses: docker/login-action@v4.5.1
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v6.1.0
+        uses: docker/metadata-action@v6.2.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@v7.2.0
+        uses: docker/build-push-action@v7.3.0
         with:
           context: .
           file: Dockerfile-ci

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/.github/workflows/docker-docs-publish.yml

@@ -37,22 +37,22 @@ jobs:
         if: github.event_name != 'pull_request'
         uses: sigstore/cosign-installer@v4.1.2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4.1.0
+        uses: docker/setup-buildx-action@v4.2.0
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4.2.0
+        uses: docker/login-action@v4.5.1
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v6.1.0
+        uses: docker/metadata-action@v6.2.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@v7.2.0
+        uses: docker/build-push-action@v7.3.0
         with:
           context: .
           file: Dockerfile-docs

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/.github/workflows/docker-publish.yml

@@ -37,22 +37,22 @@ jobs:
         if: github.event_name != 'pull_request'
         uses: sigstore/cosign-installer@v4.1.2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4.1.0
+        uses: docker/setup-buildx-action@v4.2.0
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4.2.0
+        uses: docker/login-action@v4.5.1
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v6.1.0
+        uses: docker/metadata-action@v6.2.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@v7.2.0
+        uses: docker/build-push-action@v7.3.0
         with:
           context: .
           file: Dockerfile

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/.github/workflows/docs-preview.yml

@@ -48,7 +48,7 @@ jobs:
         fetch-depth: 0
         submodules: true
     - name: Set up Python
-      uses: actions/setup-python@v6
+      uses: actions/setup-python@v7
       with:
         python-version: "3.12"
     - name: Install Python dependencies

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/.github/workflows/docs.yml

@@ -40,7 +40,7 @@ jobs:
         fetch-depth: 0
         submodules: true
     - name: Set up Python
-      uses: actions/setup-python@v6
+      uses: actions/setup-python@v7
       with:
         python-version: "3.12"
     - name: Install Python dependencies

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/.github/workflows/lint.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Install Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: "3.x"
       - name: Install dependencies

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/.github/workflows/publish.yml

@@ -23,7 +23,7 @@ jobs:
         fetch-depth: 0
         submodules: true
     - name: Set up Python
-      uses: actions/setup-python@v6
+      uses: actions/setup-python@v7
       with:
         python-version: "3.x"
     - name: Install build

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.8.dev195
+Version: 0.0.8.dev200
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

junifer-0.0.8.dev200/docs/changes/newsfragments/499.enh

@@ -0,0 +1 @@
+Move ``parse_yaml`` from ``junifer.cli`` to ``junifer.api`` by `Synchon Mandal`_

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/junifer/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.0.8.dev195'
-__version_tuple__ = version_tuple = (0, 0, 8, 'dev195')
+__version__ = version = '0.0.8.dev200'
+__version_tuple__ = version_tuple = (0, 0, 8, 'dev200')
 
-__commit_id__ = commit_id = 'g64d583140'
+__commit_id__ = commit_id = 'gf8293c860'

junifer-0.0.8.dev200/junifer/api/__init__.pyi

@@ -0,0 +1,19 @@
+__all__ = [
+    "decorators",
+    "collect",
+    "queue",
+    "run",
+    "reset",
+    "list_elements",
+    "parse_yaml",
+]
+
+from . import decorators
+from .functions import (
+    collect,
+    list_elements,
+    parse_yaml,
+    reset,
+    run,
+    queue,
+)

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/junifer/api/functions.py

@@ -6,8 +6,11 @@
 # License: AGPL
 
 import atexit
+import importlib
+import importlib.util
 import os
 import shutil
+import sys
 from pathlib import Path
 
 import structlog
@@ -32,7 +35,14 @@ from ..typing import (
 from ..utils import raise_error, warn_with_log, yaml
 
 
-__all__ = ["collect", "list_elements", "queue", "reset", "run"]
+__all__ = [
+    "collect",
+    "list_elements",
+    "parse_yaml",
+    "queue",
+    "reset",
+    "run",
+]
 
 _log = structlog.get_logger("junifer")
 logger = _log.bind(pkg="api")
@@ -468,3 +478,125 @@ def list_elements(
         elements_to_list.append(str_element)
 
     return "\n".join(elements_to_list)
+
+
+def parse_yaml(filepath: str | Path) -> dict:  # noqa: C901
+    """Parse YAML.
+
+    Parameters
+    ----------
+    filepath : str or pathlib.Path
+        The filepath to read from.
+
+    Returns
+    -------
+    dict
+        The contents represented as dictionary.
+
+    """
+    # Convert str to Path
+    if not isinstance(filepath, Path):
+        filepath = Path(filepath)
+
+    logger.info(f"Parsing yaml file: {filepath.absolute()!s}")
+    # Filepath existence check
+    if not filepath.exists():
+        raise_error(f"File does not exist: {filepath.absolute()!s}")
+    # Filepath reading
+    contents = yaml.load(filepath)
+    if "elements" in contents:
+        if contents["elements"] is None:
+            raise_error(
+                "The elements key was defined but its content is empty. "
+                "Please define the elements to operate on or remove the key."
+            )
+    # load modules
+    if "with" in contents:
+        to_load = contents["with"]
+        # Convert load modules to list
+        if not isinstance(to_load, list):
+            to_load = [to_load]
+        # Initialize list to have absolute paths for custom modules
+        final_to_load = []
+        for t_module in to_load:
+            if t_module.endswith(".py"):
+                logger.debug(f"Importing file: {t_module}")
+                # This resolves both absolute and relative paths
+                file_path = filepath.parent / t_module
+                if not file_path.exists():
+                    raise_error(
+                        f"File in 'with' section does not exist: {file_path}"
+                    )
+                # Add the parent directory to the sys.path so that the
+                # any imports from this module work correctly
+                t_path = str(file_path.parent)
+                if t_path not in sys.path:
+                    sys.path.append(str(file_path.parent))
+
+                spec = importlib.util.spec_from_file_location(
+                    t_module, file_path
+                )
+                module = importlib.util.module_from_spec(spec)  # type: ignore
+                sys.modules[t_module] = module
+                spec.loader.exec_module(module)  # type: ignore
+
+                # Add absolute path to final list
+                final_to_load.append(str(file_path.resolve()))
+
+                # Check if the module has junifer_module_deps function
+                if hasattr(module, "junifer_module_deps"):
+                    logger.debug(
+                        f"Module {t_module} has junifer_module_deps function"
+                    )
+                    # Get the dependencies
+                    deps = module.junifer_module_deps()
+                    # Add the dependencies to the final list
+                    for dep in deps:
+                        if dep not in final_to_load:
+                            final_to_load.append(
+                                str((file_path.parent / dep).resolve())
+                            )
+            else:
+                logger.info(f"Importing module: {t_module}")
+                importlib.import_module(t_module)
+                # Add module to final list
+                final_to_load.append(t_module)
+
+        # Replace modules to be loaded so that custom modules will take the
+        # absolute path. This was not the case as found in #224. Similar thing
+        # is done with the storage URI below.
+        contents["with"] = final_to_load
+
+    # Compute path for the URI parameter in storage files that are relative
+    # This is a tricky thing that appeared in #127. The problem is that
+    # the path in the URI parameter is relative to YAML file, not to the
+    # current working directory. If we leave it as is in the contents
+    # dict, then it will be used later in the ``build`` function as is,
+    # which will be computed relative to the current working directory.
+    # The solution is to compute the absolute path and replace the
+    # relative path in the contents dict with the absolute path.
+
+    # Check if the storage file is defined
+    if "storage" in contents:
+        if "uri" in contents["storage"]:
+            # Check if the storage file is relative
+            uri_path = Path(contents["storage"]["uri"])
+            if not uri_path.is_absolute():
+                # Compute the absolute path
+                contents["storage"]["uri"] = str(
+                    (filepath.parent / uri_path).resolve()
+                )
+
+    # Allow relative path if queue env kind is venv; same motivation as above
+    if "queue" in contents:
+        if "env" in contents["queue"]:
+            if "venv" == contents["queue"]["env"]["kind"]:
+                # Check if the env name is relative
+                venv_path = Path(contents["queue"]["env"]["name"])
+                if not venv_path.is_absolute():
+                    # Compute the absolute path
+                    contents["queue"]["env"]["name"] = str(
+                        (filepath.parent / venv_path).resolve()
+                    )
+
+    return contents

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/junifer/api/tests/test_functions.py

@@ -6,6 +6,7 @@
 # License: AGPL
 
 import logging
+import sys
 from contextlib import AbstractContextManager, nullcontext
 from pathlib import Path
 from typing import Any
@@ -15,7 +16,7 @@ from nibabel.filebasedimages import ImageFileError
 from ruamel.yaml import YAML
 
 import junifer.testing.registry  # noqa: F401
-from junifer.api import collect, list_elements, queue, reset, run
+from junifer.api import collect, list_elements, parse_yaml, queue, reset, run
 from junifer.datagrabber.base import BaseDataGrabber
 from junifer.pipeline import PipelineComponentRegistry
 from junifer.typing import Elements
@@ -756,3 +757,271 @@ def test_list_elements(
     """
     listed_elements = list_elements(datagrabber, elements)
     assert "sub-01" in listed_elements
+
+
+def test_parse_yaml_failure() -> None:
+    """Test YAML parsing failure."""
+    with pytest.raises(ValueError, match="does not exist"):
+        parse_yaml("foo.yaml")
+
+
+def test_parse_yaml_empty_elements_failure(tmp_path: Path) -> None:
+    """Test YAML parsing with empty elements failure.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    # Write test file
+    fname = tmp_path / "test_parse_yaml_empty_elements_failure.yaml"
+    fname.write_text("elements:")
+    # Check test file
+    with pytest.raises(ValueError, match="elements key was defined"):
+        parse_yaml(fname)
+
+
+def test_parse_yaml_success(tmp_path: Path) -> None:
+    """Test YAML parsing success.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    # Write test file
+    fname = tmp_path / "test_parse_yaml_success.yaml"
+    fname.write_text("foo: bar")
+    # Check test file
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+
+
+def test_parse_yaml_success_with_module_autoload(tmp_path: Path) -> None:
+    """Test YAML parsing with single module autoload success.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    # Write test file
+    fname = tmp_path / "test_parse_yaml_with_single_module_autoload.yaml"
+    fname.write_text("foo: bar\nwith: numpy")
+    # Check test file
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+    assert "with" in contents
+    assert contents["with"] == ["numpy"]
+    assert "numpy" in sys.modules
+    assert "junifer.configs.wrong_config" not in sys.modules
+
+
+def test_parse_yaml_failure_with_multi_module_autoload(tmp_path: Path) -> None:
+    """Test YAML parsing with multi module autoload failure.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    # Write test file
+    fname = tmp_path / "test_parse_yaml_with_multi_module_autoload.yaml"
+    fname.write_text(
+        "foo: bar\nwith:\n  - numpy\n  - junifer.testing.wrong_config"
+    )
+    # Check test file
+    with pytest.raises(ImportError, match="wrong_config"):
+        parse_yaml(fname)
+
+
+def test_parse_yaml_with_wrong_path(tmp_path: Path) -> None:
+    """Test YAML parsing with wrong paths in with.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    t_tmp_path = tmp_path / "test_relative_with"
+    # Write yaml that includes a relative path
+    yaml_path = t_tmp_path / "yamls"
+    yaml_path.mkdir(exist_ok=True, parents=True)
+    yaml_fname = yaml_path / "test_parse_yaml_wrong_path.yaml"
+
+    yaml_fname.write_text("foo: bar\nwith:\n  -  missingt.py\n  - scipy\n")
+
+    # Check test file
+    with pytest.raises(ValueError, match="does not exist"):
+        parse_yaml(yaml_fname)
+
+
+def test_parse_yaml_relative_path(tmp_path: Path) -> None:
+    """Test YAML parsing with relative paths in with.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    t_tmp_path = tmp_path / "test_relative_with"
+
+    # Write .py to include
+    py_path = t_tmp_path / "external"
+    py_path.mkdir(exist_ok=True, parents=True)
+    py_fname = py_path / "first.py"
+    py_fname.write_text("import numpy as np\n")
+
+    # Write yaml that includes a relative path
+    yaml_path = t_tmp_path / "yamls"
+    yaml_path.mkdir(exist_ok=True, parents=True)
+    yaml_fname = yaml_path / "test_parse_yaml_relative_path.yaml"
+
+    yaml_fname.write_text(
+        "foo: bar\nwith:\n  - ../external/first.py\n  - scipy\n"
+    )
+
+    # Check test file
+    parse_yaml(yaml_fname)
+
+
+def test_parse_yaml_absolute_path(tmp_path: Path) -> None:
+    """Test YAML parsing with absolute paths in with.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    t_tmp_path = tmp_path / "test_relative_with"
+
+    # Write .py to include
+    py_path = t_tmp_path / "external"
+    py_path.mkdir(exist_ok=True, parents=True)
+    py_fname = py_path / "first.py"
+    py_fname.write_text("import numpy as np\n")
+
+    # Write yaml that includes a relative path
+    yaml_path = t_tmp_path / "yamls"
+    yaml_path.mkdir(exist_ok=True, parents=True)
+    yaml_fname = yaml_path / "test_parse_yaml_relative_path.yaml"
+
+    yaml_fname.write_text(
+        f"foo: bar\nwith:\n  - {py_fname.absolute()}\n  - scipy\n"
+    )
+
+    # Check test file
+    parse_yaml(yaml_fname)
+
+
+def test_parse_yaml_multi_module_deps(tmp_path: Path) -> None:
+    """Test YAML parsing with multi-module import with deps.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    t_tmp_path = tmp_path / "test_with_multi_module"
+
+    # Write .py to include
+    py_path = t_tmp_path / "external"
+    py_path.mkdir(exist_ok=True, parents=True)
+    py_fname_1 = py_path / "first.py"
+    py_fname_1.write_text(
+        "import numpy as np\nfrom second import hej\n"
+        "def junifer_module_deps(): return ['second.py']\n"
+    )
+    py_fname_2 = py_path / "second.py"
+    py_fname_2.write_text("def hej(): print('hej')\n")
+
+    # Write yaml
+    yaml_path = t_tmp_path / "yamls"
+    yaml_path.mkdir(exist_ok=True, parents=True)
+    yaml_fname = yaml_path / "test_parse_yaml_multi_module.yaml"
+
+    yaml_fname.write_text(
+        "foo: bar\nwith:\n  - ../external/first.py\n  - scipy\n"
+    )
+
+    # Check test file
+    parse_yaml(yaml_fname)
+
+
+def test_parse_storage_uri_relative(tmp_path: Path) -> None:
+    """Test YAML parsing with storage and relative URI.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml"
+    fname.write_text("foo: bar\nwith: numpy\nstorage:\n  uri: test.db\n")
+
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+    assert "storage" in contents
+    assert "uri" in contents["storage"]
+    assert contents["storage"]["uri"] == str(tmp_path / "test.db")
+
+    fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml"
+    fname.write_text(
+        "foo: bar\nwith: numpy\nstorage:\n  uri: ../another/test.db\n"
+    )
+
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+    assert "storage" in contents
+    assert "uri" in contents["storage"]
+    assert contents["storage"]["uri"] == str(
+        (tmp_path / "../another/test.db").resolve()
+    )
+
+    fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml"
+    fname.write_text(
+        "foo: bar\nwith: numpy\nstorage:\n  uri: /absolute/test.db\n"
+    )
+
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+    assert "storage" in contents
+    assert "uri" in contents["storage"]
+    assert contents["storage"]["uri"] == "/absolute/test.db"
+
+    # Just to trick coverage
+    fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml"
+    fname.write_text("foo: bar\nwith: numpy\nstorage:\n  kind: SomeStorage\n")
+
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+    assert "storage" in contents
+
+
+def test_parse_yaml_queue_venv_relative(tmp_path: Path) -> None:
+    """Test YAML parsing with relative venv queue.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    fname = tmp_path / "test_parse_yaml_queue_venv_relative.yaml"
+    fname.write_text("queue:\n  env:\n    kind: venv\n    name: .venv\n")
+    _ = parse_yaml(fname)

{junifer-0.0.8.dev195 → junifer-0.0.8.dev200}/junifer/cli/cli.py

@@ -17,7 +17,7 @@ from ..utils import (
     raise_error,
     yaml,
 )
-from .parser import parse_elements, parse_yaml
+from .parser import parse_elements
 from .utils import (
     _get_dependency_information,
     _get_environment_information,
@@ -162,7 +162,7 @@ def run(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO(synchon): add validation
     # Parse YAML
-    config = parse_yaml(filepath)
+    config = cli_func.parse_yaml(filepath)
     # Retrieve working directory
     workdir = config["workdir"]
     # Fetch datagrabber
@@ -232,7 +232,7 @@ def collect(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO: add validation
     # Parse YAML
-    config = parse_yaml(filepath)
+    config = cli_func.parse_yaml(filepath)
     # Fetch storage
     storage = config["storage"]
     # Perform operation
@@ -299,7 +299,7 @@ def queue(
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # TODO: add validation
     # Parse YAML
-    config = parse_yaml(filepath)  # type: ignore
+    config = cli_func.parse_yaml(filepath)  # type: ignore
     # Check queue section
     if "queue" not in config:
         raise_error(f"No queue configuration found in {filepath}.")
@@ -450,7 +450,7 @@ def reset(
     # Setup logging
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # Parse YAML
-    config = parse_yaml(filepath)
+    config = cli_func.parse_yaml(filepath)
     # Perform operation
     cli_func.reset(config)
 
@@ -510,7 +510,7 @@ def list_elements(
     # Setup logging
     configure_logging(level=verbose, level_datalad=verbose_datalad)
     # Parse YAML
-    config = parse_yaml(filepath)
+    config = cli_func.parse_yaml(filepath)
     # Fetch datagrabber
     datagrabber = config["datagrabber"]
     # Parse elements