junifer 0.0.8.dev193__tar.gz → 0.0.8.dev198__tar.gz

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/.github/workflows/ci-docs.yml

@@ -48,7 +48,7 @@ jobs:
           echo '</coverage>' >> coverage.xml
         if: matrix.python-version == 3.12
       - name: Upload mock coverage to Codecov
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@v7
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           # Allow coverage upload failure

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/.github/workflows/ci.yml

@@ -57,7 +57,7 @@ jobs:
           mkdir -p $HOME/junifer_data/v7
           cp -ar /root/junifer_data/v7/. $HOME/junifer_data/v7/
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           submodules: true
       - name: Set up Python ${{ matrix.python-version }}
@@ -78,7 +78,7 @@ jobs:
           git config --system --add safe.directory $(pwd)
           tox
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v5
+        uses: codecov/codecov-action@v7
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/.github/workflows/docker-ci-publish.yml

@@ -32,27 +32,27 @@ jobs:
           sudo docker builder prune -a
       - run: df -h
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
       - name: Install cosign
         if: github.event_name != 'pull_request'
-        uses: sigstore/cosign-installer@v4.1.0
+        uses: sigstore/cosign-installer@v4.1.2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4.0.0
+        uses: docker/setup-buildx-action@v4.1.0
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4.0.0
+        uses: docker/login-action@v4.2.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@v6.1.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@v7.0.0
+        uses: docker/build-push-action@v7.2.0
         with:
           context: .
           file: Dockerfile-ci

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/.github/workflows/docker-docs-publish.yml

@@ -32,27 +32,27 @@ jobs:
           sudo docker builder prune -a
       - run: df -h
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
       - name: Install cosign
         if: github.event_name != 'pull_request'
-        uses: sigstore/cosign-installer@v4.1.0
+        uses: sigstore/cosign-installer@v4.1.2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4.0.0
+        uses: docker/setup-buildx-action@v4.1.0
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4.0.0
+        uses: docker/login-action@v4.2.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@v6.1.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@v7.0.0
+        uses: docker/build-push-action@v7.2.0
         with:
           context: .
           file: Dockerfile-docs

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/.github/workflows/docker-publish.yml

@@ -32,27 +32,27 @@ jobs:
           sudo docker builder prune -a
       - run: df -h
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
       - name: Install cosign
         if: github.event_name != 'pull_request'
-        uses: sigstore/cosign-installer@v4.1.0
+        uses: sigstore/cosign-installer@v4.1.2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4.0.0
+        uses: docker/setup-buildx-action@v4.1.0
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v4.0.0
+        uses: docker/login-action@v4.2.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v6.0.0
+        uses: docker/metadata-action@v6.1.0
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@v7.0.0
+        uses: docker/build-push-action@v7.2.0
         with:
           context: .
           file: Dockerfile

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/.github/workflows/docs-preview.yml

@@ -43,7 +43,7 @@ jobs:
         mkdir -p $HOME/junifer_data/v7
         cp -ar /root/junifer_data/v7/. $HOME/junifer_data/v7/
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
       with:
         fetch-depth: 0
         submodules: true

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/.github/workflows/docs.yml

@@ -34,7 +34,7 @@ jobs:
         mkdir -p $HOME/junifer_data/v7
         cp -ar /root/junifer_data/v7/. $HOME/junifer_data/v7/
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
       with:
         # require all of history to see all tagged versions' docs
         fetch-depth: 0
@@ -55,7 +55,7 @@ jobs:
     - name: Checkout gh-pages
       # As we already did a deploy of gh-pages above, it is guaranteed to be there
       # so check it out so we can selectively build docs below
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
       with:
           ref: gh-pages
           path: docs/_build

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/.github/workflows/lint.yml

@@ -8,7 +8,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Install Python
         uses: actions/setup-python@v6
         with:

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/.github/workflows/publish.yml

@@ -18,7 +18,7 @@ jobs:
       contents: write
     steps:
     - name: Checkout source
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
       with:
         fetch-depth: 0
         submodules: true
@@ -36,7 +36,7 @@ jobs:
     - name: Publish package to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
     - name: Make release on GitHub
-      uses: softprops/action-gh-release@v2
+      uses: softprops/action-gh-release@v3
       if: startsWith(github.ref, 'refs/tags/')
       with:
         files: |

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.8.dev193
+Version: 0.0.8.dev198
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

junifer-0.0.8.dev198/docs/changes/newsfragments/499.enh

@@ -0,0 +1 @@
+Move ``parse_yaml`` from ``junifer.cli`` to ``junifer.api`` by `Synchon Mandal`_

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/junifer/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.0.8.dev193'
-__version_tuple__ = version_tuple = (0, 0, 8, 'dev193')
+__version__ = version = '0.0.8.dev198'
+__version_tuple__ = version_tuple = (0, 0, 8, 'dev198')
 
-__commit_id__ = commit_id = 'g8d610dbb9'
+__commit_id__ = commit_id = 'g61fa4962c'

junifer-0.0.8.dev198/junifer/api/__init__.pyi

@@ -0,0 +1,19 @@
+__all__ = [
+    "decorators",
+    "collect",
+    "queue",
+    "run",
+    "reset",
+    "list_elements",
+    "parse_yaml",
+]
+
+from . import decorators
+from .functions import (
+    collect,
+    list_elements,
+    parse_yaml,
+    reset,
+    run,
+    queue,
+)

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/junifer/api/functions.py

@@ -6,8 +6,11 @@
 # License: AGPL
 
 import atexit
+import importlib
+import importlib.util
 import os
 import shutil
+import sys
 from pathlib import Path
 
 import structlog
@@ -32,7 +35,14 @@ from ..typing import (
 from ..utils import raise_error, warn_with_log, yaml
 
 
-__all__ = ["collect", "list_elements", "queue", "reset", "run"]
+__all__ = [
+    "collect",
+    "list_elements",
+    "parse_yaml",
+    "queue",
+    "reset",
+    "run",
+]
 
 _log = structlog.get_logger("junifer")
 logger = _log.bind(pkg="api")
@@ -468,3 +478,125 @@ def list_elements(
         elements_to_list.append(str_element)
 
     return "\n".join(elements_to_list)
+
+
+def parse_yaml(filepath: str | Path) -> dict:  # noqa: C901
+    """Parse YAML.
+
+    Parameters
+    ----------
+    filepath : str or pathlib.Path
+        The filepath to read from.
+
+    Returns
+    -------
+    dict
+        The contents represented as dictionary.
+
+    """
+    # Convert str to Path
+    if not isinstance(filepath, Path):
+        filepath = Path(filepath)
+
+    logger.info(f"Parsing yaml file: {filepath.absolute()!s}")
+    # Filepath existence check
+    if not filepath.exists():
+        raise_error(f"File does not exist: {filepath.absolute()!s}")
+    # Filepath reading
+    contents = yaml.load(filepath)
+    if "elements" in contents:
+        if contents["elements"] is None:
+            raise_error(
+                "The elements key was defined but its content is empty. "
+                "Please define the elements to operate on or remove the key."
+            )
+    # load modules
+    if "with" in contents:
+        to_load = contents["with"]
+        # Convert load modules to list
+        if not isinstance(to_load, list):
+            to_load = [to_load]
+        # Initialize list to have absolute paths for custom modules
+        final_to_load = []
+        for t_module in to_load:
+            if t_module.endswith(".py"):
+                logger.debug(f"Importing file: {t_module}")
+                # This resolves both absolute and relative paths
+                file_path = filepath.parent / t_module
+                if not file_path.exists():
+                    raise_error(
+                        f"File in 'with' section does not exist: {file_path}"
+                    )
+                # Add the parent directory to the sys.path so that the
+                # any imports from this module work correctly
+                t_path = str(file_path.parent)
+                if t_path not in sys.path:
+                    sys.path.append(str(file_path.parent))
+
+                spec = importlib.util.spec_from_file_location(
+                    t_module, file_path
+                )
+                module = importlib.util.module_from_spec(spec)  # type: ignore
+                sys.modules[t_module] = module
+                spec.loader.exec_module(module)  # type: ignore
+
+                # Add absolute path to final list
+                final_to_load.append(str(file_path.resolve()))
+
+                # Check if the module has junifer_module_deps function
+                if hasattr(module, "junifer_module_deps"):
+                    logger.debug(
+                        f"Module {t_module} has junifer_module_deps function"
+                    )
+                    # Get the dependencies
+                    deps = module.junifer_module_deps()
+                    # Add the dependencies to the final list
+                    for dep in deps:
+                        if dep not in final_to_load:
+                            final_to_load.append(
+                                str((file_path.parent / dep).resolve())
+                            )
+            else:
+                logger.info(f"Importing module: {t_module}")
+                importlib.import_module(t_module)
+                # Add module to final list
+                final_to_load.append(t_module)
+
+        # Replace modules to be loaded so that custom modules will take the
+        # absolute path. This was not the case as found in #224. Similar thing
+        # is done with the storage URI below.
+        contents["with"] = final_to_load
+
+    # Compute path for the URI parameter in storage files that are relative
+    # This is a tricky thing that appeared in #127. The problem is that
+    # the path in the URI parameter is relative to YAML file, not to the
+    # current working directory. If we leave it as is in the contents
+    # dict, then it will be used later in the ``build`` function as is,
+    # which will be computed relative to the current working directory.
+    # The solution is to compute the absolute path and replace the
+    # relative path in the contents dict with the absolute path.
+
+    # Check if the storage file is defined
+    if "storage" in contents:
+        if "uri" in contents["storage"]:
+            # Check if the storage file is relative
+            uri_path = Path(contents["storage"]["uri"])
+            if not uri_path.is_absolute():
+                # Compute the absolute path
+                contents["storage"]["uri"] = str(
+                    (filepath.parent / uri_path).resolve()
+                )
+
+    # Allow relative path if queue env kind is venv; same motivation as above
+    if "queue" in contents:
+        if "env" in contents["queue"]:
+            if "venv" == contents["queue"]["env"]["kind"]:
+                # Check if the env name is relative
+                venv_path = Path(contents["queue"]["env"]["name"])
+                if not venv_path.is_absolute():
+                    # Compute the absolute path
+                    contents["queue"]["env"]["name"] = str(
+                        (filepath.parent / venv_path).resolve()
+                    )
+
+    return contents

{junifer-0.0.8.dev193 → junifer-0.0.8.dev198}/junifer/api/tests/test_functions.py

@@ -6,6 +6,7 @@
 # License: AGPL
 
 import logging
+import sys
 from contextlib import AbstractContextManager, nullcontext
 from pathlib import Path
 from typing import Any
@@ -15,7 +16,7 @@ from nibabel.filebasedimages import ImageFileError
 from ruamel.yaml import YAML
 
 import junifer.testing.registry  # noqa: F401
-from junifer.api import collect, list_elements, queue, reset, run
+from junifer.api import collect, list_elements, parse_yaml, queue, reset, run
 from junifer.datagrabber.base import BaseDataGrabber
 from junifer.pipeline import PipelineComponentRegistry
 from junifer.typing import Elements
@@ -756,3 +757,271 @@ def test_list_elements(
     """
     listed_elements = list_elements(datagrabber, elements)
     assert "sub-01" in listed_elements
+
+
+def test_parse_yaml_failure() -> None:
+    """Test YAML parsing failure."""
+    with pytest.raises(ValueError, match="does not exist"):
+        parse_yaml("foo.yaml")
+
+
+def test_parse_yaml_empty_elements_failure(tmp_path: Path) -> None:
+    """Test YAML parsing with empty elements failure.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    # Write test file
+    fname = tmp_path / "test_parse_yaml_empty_elements_failure.yaml"
+    fname.write_text("elements:")
+    # Check test file
+    with pytest.raises(ValueError, match="elements key was defined"):
+        parse_yaml(fname)
+
+
+def test_parse_yaml_success(tmp_path: Path) -> None:
+    """Test YAML parsing success.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    # Write test file
+    fname = tmp_path / "test_parse_yaml_success.yaml"
+    fname.write_text("foo: bar")
+    # Check test file
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+
+
+def test_parse_yaml_success_with_module_autoload(tmp_path: Path) -> None:
+    """Test YAML parsing with single module autoload success.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    # Write test file
+    fname = tmp_path / "test_parse_yaml_with_single_module_autoload.yaml"
+    fname.write_text("foo: bar\nwith: numpy")
+    # Check test file
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+    assert "with" in contents
+    assert contents["with"] == ["numpy"]
+    assert "numpy" in sys.modules
+    assert "junifer.configs.wrong_config" not in sys.modules
+
+
+def test_parse_yaml_failure_with_multi_module_autoload(tmp_path: Path) -> None:
+    """Test YAML parsing with multi module autoload failure.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    # Write test file
+    fname = tmp_path / "test_parse_yaml_with_multi_module_autoload.yaml"
+    fname.write_text(
+        "foo: bar\nwith:\n  - numpy\n  - junifer.testing.wrong_config"
+    )
+    # Check test file
+    with pytest.raises(ImportError, match="wrong_config"):
+        parse_yaml(fname)
+
+
+def test_parse_yaml_with_wrong_path(tmp_path: Path) -> None:
+    """Test YAML parsing with wrong paths in with.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    t_tmp_path = tmp_path / "test_relative_with"
+    # Write yaml that includes a relative path
+    yaml_path = t_tmp_path / "yamls"
+    yaml_path.mkdir(exist_ok=True, parents=True)
+    yaml_fname = yaml_path / "test_parse_yaml_wrong_path.yaml"
+
+    yaml_fname.write_text("foo: bar\nwith:\n  -  missingt.py\n  - scipy\n")
+
+    # Check test file
+    with pytest.raises(ValueError, match="does not exist"):
+        parse_yaml(yaml_fname)
+
+
+def test_parse_yaml_relative_path(tmp_path: Path) -> None:
+    """Test YAML parsing with relative paths in with.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    t_tmp_path = tmp_path / "test_relative_with"
+
+    # Write .py to include
+    py_path = t_tmp_path / "external"
+    py_path.mkdir(exist_ok=True, parents=True)
+    py_fname = py_path / "first.py"
+    py_fname.write_text("import numpy as np\n")
+
+    # Write yaml that includes a relative path
+    yaml_path = t_tmp_path / "yamls"
+    yaml_path.mkdir(exist_ok=True, parents=True)
+    yaml_fname = yaml_path / "test_parse_yaml_relative_path.yaml"
+
+    yaml_fname.write_text(
+        "foo: bar\nwith:\n  - ../external/first.py\n  - scipy\n"
+    )
+
+    # Check test file
+    parse_yaml(yaml_fname)
+
+
+def test_parse_yaml_absolute_path(tmp_path: Path) -> None:
+    """Test YAML parsing with absolute paths in with.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    t_tmp_path = tmp_path / "test_relative_with"
+
+    # Write .py to include
+    py_path = t_tmp_path / "external"
+    py_path.mkdir(exist_ok=True, parents=True)
+    py_fname = py_path / "first.py"
+    py_fname.write_text("import numpy as np\n")
+
+    # Write yaml that includes a relative path
+    yaml_path = t_tmp_path / "yamls"
+    yaml_path.mkdir(exist_ok=True, parents=True)
+    yaml_fname = yaml_path / "test_parse_yaml_relative_path.yaml"
+
+    yaml_fname.write_text(
+        f"foo: bar\nwith:\n  - {py_fname.absolute()}\n  - scipy\n"
+    )
+
+    # Check test file
+    parse_yaml(yaml_fname)
+
+
+def test_parse_yaml_multi_module_deps(tmp_path: Path) -> None:
+    """Test YAML parsing with multi-module import with deps.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    t_tmp_path = tmp_path / "test_with_multi_module"
+
+    # Write .py to include
+    py_path = t_tmp_path / "external"
+    py_path.mkdir(exist_ok=True, parents=True)
+    py_fname_1 = py_path / "first.py"
+    py_fname_1.write_text(
+        "import numpy as np\nfrom second import hej\n"
+        "def junifer_module_deps(): return ['second.py']\n"
+    )
+    py_fname_2 = py_path / "second.py"
+    py_fname_2.write_text("def hej(): print('hej')\n")
+
+    # Write yaml
+    yaml_path = t_tmp_path / "yamls"
+    yaml_path.mkdir(exist_ok=True, parents=True)
+    yaml_fname = yaml_path / "test_parse_yaml_multi_module.yaml"
+
+    yaml_fname.write_text(
+        "foo: bar\nwith:\n  - ../external/first.py\n  - scipy\n"
+    )
+
+    # Check test file
+    parse_yaml(yaml_fname)
+
+
+def test_parse_storage_uri_relative(tmp_path: Path) -> None:
+    """Test YAML parsing with storage and relative URI.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml"
+    fname.write_text("foo: bar\nwith: numpy\nstorage:\n  uri: test.db\n")
+
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+    assert "storage" in contents
+    assert "uri" in contents["storage"]
+    assert contents["storage"]["uri"] == str(tmp_path / "test.db")
+
+    fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml"
+    fname.write_text(
+        "foo: bar\nwith: numpy\nstorage:\n  uri: ../another/test.db\n"
+    )
+
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+    assert "storage" in contents
+    assert "uri" in contents["storage"]
+    assert contents["storage"]["uri"] == str(
+        (tmp_path / "../another/test.db").resolve()
+    )
+
+    fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml"
+    fname.write_text(
+        "foo: bar\nwith: numpy\nstorage:\n  uri: /absolute/test.db\n"
+    )
+
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+    assert "storage" in contents
+    assert "uri" in contents["storage"]
+    assert contents["storage"]["uri"] == "/absolute/test.db"
+
+    # Just to trick coverage
+    fname = tmp_path / "test_parse_yaml_with_storage_uri.yaml"
+    fname.write_text("foo: bar\nwith: numpy\nstorage:\n  kind: SomeStorage\n")
+
+    contents = parse_yaml(fname)
+    assert "foo" in contents
+    assert contents["foo"] == "bar"
+    assert "storage" in contents
+
+
+def test_parse_yaml_queue_venv_relative(tmp_path: Path) -> None:
+    """Test YAML parsing with relative venv queue.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    fname = tmp_path / "test_parse_yaml_queue_venv_relative.yaml"
+    fname.write_text("queue:\n  env:\n    kind: venv\n    name: .venv\n")
+    _ = parse_yaml(fname)