junifer 0.0.4.dev829__py3-none-any.whl → 0.0.5__py3-none-any.whl

junifer/__init__.py

@@ -20,3 +20,20 @@ from . import (
     onthefly,
 )
 from ._version import __version__
+
+
+__all__ = [
+    "api",
+    "configs",
+    "data",
+    "datagrabber",
+    "datareader",
+    "markers",
+    "pipeline",
+    "preprocess",
+    "stats",
+    "storage",
+    "utils",
+    "external",
+    "onthefly",
+]

junifer/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.4.dev829'
-__version_tuple__ = version_tuple = (0, 0, 4, 'dev829')
+__version__ = version = '0.0.5'
+__version_tuple__ = version_tuple = (0, 0, 5)

junifer/api/__init__.py

@@ -1,4 +1,4 @@
-"""Provide imports for api sub-package."""
+"""Public API and CLI components."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Synchon Mandal <s.mandal@fz-juelich.de>
@@ -7,3 +7,6 @@
 from . import decorators
 from .cli import cli
 from .functions import collect, queue, run
+
+
+__all__ = ["decorators", "cli", "collect", "queue", "run"]

junifer/api/cli.py

@@ -8,7 +8,7 @@ import pathlib
 import subprocess
 import sys
 from pathlib import Path
-from typing import Dict, List, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 import click
 import pandas as pd
@@ -20,6 +20,7 @@ from ..utils.logging import (
     warn_with_log,
 )
 from .functions import collect as api_collect
+from .functions import list_elements as api_list_elements
 from .functions import queue as api_queue
 from .functions import reset as api_reset
 from .functions import run as api_run
@@ -451,6 +452,69 @@ def reset(
     api_reset(config)
 
 
+@cli.command()
+@click.argument(
+    "filepath",
+    type=click.Path(
+        exists=True, readable=True, dir_okay=False, path_type=pathlib.Path
+    ),
+)
+@click.option("--element", type=str, multiple=True)
+@click.option(
+    "-o",
+    "--output-file",
+    type=click.Path(dir_okay=False, writable=True, path_type=pathlib.Path),
+)
+@click.option(
+    "-v",
+    "--verbose",
+    type=click.UNPROCESSED,
+    callback=_validate_verbose,
+    default="info",
+)
+def list_elements(
+    filepath: click.Path,
+    element: Tuple[str],
+    output_file: Optional[click.Path],
+    verbose: Union[str, int],
+) -> None:
+    """Element listing command for CLI.
+
+    \f
+
+    Parameters
+    ----------
+    filepath : click.Path
+        The filepath to the configuration file.
+    element : tuple of str
+        The element to operate on.
+    output_file : click.Path or None
+        The path to write the output to. If not None, writing to
+        stdout is not performed.
+    verbose : click.Choice
+        The verbosity level: warning, info or debug (default "info").
+
+    """
+    configure_logging(level=verbose)
+    # Parse YAML
+    config = parse_yaml(filepath)  # type: ignore
+    # Fetch datagrabber
+    datagrabber = config["datagrabber"]
+    # Parse elements
+    elements = _parse_elements(element, config)
+    # Perform operation
+    listed_elements = api_list_elements(
+        datagrabber=datagrabber,
+        elements=elements,
+    )
+    # Check if output file is provided
+    if output_file is not None:
+        output_file.touch()
+        output_file.write_text(listed_elements)
+    else:
+        click.secho(listed_elements, fg="blue")
+
+
 @cli.group()
 def setup() -> None:  # pragma: no cover
     """Configure commands for Junifer."""
@@ -524,3 +588,29 @@ def ants_docker() -> None:  # pragma: no cover
     export PATH="$PATH:{ants_wrappers_path}"
     """
     click.secho(msg, fg="blue")
+
+
+@setup.command("freesurfer-docker")
+def freesurfer_docker() -> None:  # pragma: no cover
+    """Configure FreeSurfer-Docker wrappers."""
+    import junifer
+
+    pkg_path = Path(junifer.__path__[0])  # type: ignore
+    fs_wrappers_path = pkg_path / "api" / "res" / "freesurfer"
+    msg = f"""
+    Installation instructions for FreeSurfer-Docker wrappers:
+
+    1. Install Docker: https://docs.docker.com/get-docker/
+
+    2. Get the FreeSurfer-Docker image by running this on the command line:
+
+        docker pull freesurfer/freesurfer
+
+    3. Get license from: https://surfer.nmr.mgh.harvard.edu/registration.html .
+       You can skip this step if you already have one.
+
+    4. Add this line to the ~/.bashrc or ~/.zshrc file:
+
+    export PATH="$PATH:{fs_wrappers_path}"
+    """
+    click.secho(msg, fg="blue")

junifer/api/decorators.py

@@ -10,6 +10,15 @@ from typing import Type
 from ..pipeline.registry import register
 
 
+__all__ = [
+    "register_datagrabber",
+    "register_datareader",
+    "register_preprocessor",
+    "register_marker",
+    "register_storage",
+]
+
+
 def register_datagrabber(klass: Type) -> Type:
     """Register DataGrabber.
 

junifer/api/functions.py

@@ -5,6 +5,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+import os
 import shutil
 import typing
 from pathlib import Path
@@ -22,6 +23,9 @@ from .queue_context import GnuParallelLocalAdapter, HTCondorAdapter
 from .utils import yaml
 
 
+__all__ = ["run", "collect", "queue", "reset", "list_elements"]
+
+
 def _get_datagrabber(datagrabber_config: Dict) -> BaseDataGrabber:
     """Get DataGrabber.
 
@@ -91,7 +95,7 @@ def run(
     datagrabber : dict
         DataGrabber to use. Must have a key ``kind`` with the kind of
         DataGrabber to use. All other keys are passed to the DataGrabber
-        init function.
+        constructor.
     markers : list of dict
         List of markers to extract. Each marker is a dict with at least two
         keys: ``name`` and ``kind``. The ``name`` key is used to name the
@@ -101,11 +105,11 @@ def run(
     storage : dict
         Storage to use. Must have a key ``kind`` with the kind of
         storage to use. All other keys are passed to the storage
-        init function.
+        constructor.
     preprocessors : list of dict, optional
         List of preprocessors to use. Each preprocessor is a dict with at
         least a key ``kind`` specifying the preprocessor to use. All other keys
-        are passed to the preprocessor init function (default None).
+        are passed to the preprocessor constructor (default None).
     elements : str or tuple or list of str or tuple, optional
         Element(s) to process. Will be used to index the DataGrabber
         (default None).
@@ -187,7 +191,7 @@ def collect(storage: Dict) -> None:
     storage : dict
         Storage to use. Must have a key ``kind`` with the kind of
         storage to use. All other keys are passed to the storage
-        init function.
+        constructor.
 
     """
     storage_params = storage.copy()
@@ -339,12 +343,12 @@ def reset(config: Dict) -> None:
     storage = config["storage"]
     storage_uri = Path(storage["uri"])
     logger.info(f"Deleting {storage_uri.resolve()!s}")
-    # Delete storage; will be str
+    # Delete storage
     if storage_uri.exists():
-        # Delete files in the directory
-        for file in storage_uri.iterdir():
+        # Delete files in the job storage directory
+        for file in storage_uri.parent.iterdir():
             file.unlink(missing_ok=True)
-        # Remove directory
+        # Remove job storage directory
         storage_uri.parent.rmdir()
 
     # Fetch job name (if present)
@@ -359,5 +363,47 @@ def reset(config: Dict) -> None:
         if job_dir.exists():
             # Remove files and directories
             shutil.rmtree(job_dir)
-            # Remove directory
-            job_dir.parent.rmdir()
+            # Remove parent directory (if empty)
+            if not next(os.scandir(job_dir.parent), None):
+                job_dir.parent.rmdir()
+
+
+def list_elements(
+    datagrabber: Dict,
+    elements: Union[str, List[Union[str, Tuple]], Tuple, None] = None,
+) -> str:
+    """List elements of the datagrabber filtered using `elements`.
+
+    Parameters
+    ----------
+    datagrabber : dict
+        DataGrabber to index. Must have a key ``kind`` with the kind of
+        DataGrabber to use. All other keys are passed to the DataGrabber
+        constructor.
+    elements : str or tuple or list of str or tuple, optional
+        Element(s) to filter using. Will be used to index the DataGrabber
+        (default None).
+
+    """
+    # Get datagrabber to use
+    datagrabber_object = _get_datagrabber(datagrabber)
+
+    # Fetch elements
+    raw_elements_to_list = []
+    with datagrabber_object:
+        if elements is not None:
+            for element in datagrabber_object.filter(elements):
+                raw_elements_to_list.append(element)
+        else:
+            for element in datagrabber_object:
+                raw_elements_to_list.append(element)
+
+    elements_to_list = []
+    for element in raw_elements_to_list:
+        # Stringify elements if tuple for operation
+        str_element = (
+            ",".join(element) if isinstance(element, tuple) else element
+        )
+        elements_to_list.append(str_element)
+
+    return "\n".join(elements_to_list)

junifer/api/parser.py

@@ -14,6 +14,9 @@ from ..utils.logging import logger, raise_error
 from .utils import yaml
 
 
+__all__ = ["parse_yaml"]
+
+
 def parse_yaml(filepath: Union[str, Path]) -> Dict:
     """Parse YAML.
 

junifer/api/queue_context/__init__.py

@@ -1,4 +1,4 @@
-"""Provide imports for queue context sub-package."""
+"""Context adapters for queueing."""
 
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
@@ -6,3 +6,6 @@
 from .queue_context_adapter import QueueContextAdapter
 from .htcondor_adapter import HTCondorAdapter
 from .gnu_parallel_local_adapter import GnuParallelLocalAdapter
+
+
+__all__ = ["QueueContextAdapter", "HTCondorAdapter", "GnuParallelLocalAdapter"]

junifer/api/queue_context/gnu_parallel_local_adapter.py

@@ -43,7 +43,8 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
     Raises
     ------
     ValueError
-        If``env`` is invalid.
+        If ``env.kind`` is invalid or
+        if ``env.shell`` is invalid.
 
     See Also
     --------
@@ -110,13 +111,22 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
                 f"must be one of {valid_env_kinds}"
             )
         else:
+            # Check shell
+            shell = env.get("shell", "bash")
+            valid_shells = ["bash", "zsh"]
+            if shell not in valid_shells:
+                raise_error(
+                    f"Invalid value for `env.shell`: {shell}, "
+                    f"must be one of {valid_shells}"
+                )
+            self._shell = shell
             # Set variables
             if env["kind"] == "local":
                 # No virtual environment
                 self._executable = "junifer"
                 self._arguments = ""
             else:
-                self._executable = f"run_{env['kind']}.sh"
+                self._executable = f"run_{env['kind']}.{self._shell}"
                 self._arguments = f"{env['name']} junifer"
                 self._exec_path = self._job_dir / self._executable
 
@@ -135,7 +145,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
     def pre_run(self) -> str:
         """Return pre-run commands."""
         fixed = (
-            "#!/usr/bin/env bash\n\n"
+            f"#!/usr/bin/env {self._shell}\n\n"
             "# This script is auto-generated by junifer.\n\n"
             "# Force datalad to run in non-interactive mode\n"
             "DATALAD_UI_INTERACTIVE=false\n"
@@ -146,7 +156,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
     def run(self) -> str:
         """Return run commands."""
         return (
-            "#!/usr/bin/env bash\n\n"
+            f"#!/usr/bin/env {self._shell}\n\n"
             "# This script is auto-generated by junifer.\n\n"
             "# Run pre_run.sh\n"
             f"sh {self._pre_run_path.resolve()!s}\n\n"
@@ -166,7 +176,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
     def pre_collect(self) -> str:
         """Return pre-collect commands."""
         fixed = (
-            "#!/usr/bin/env bash\n\n"
+            f"#!/usr/bin/env {self._shell}\n\n"
             "# This script is auto-generated by junifer.\n"
         )
         var = self._pre_collect or ""
@@ -175,7 +185,7 @@ class GnuParallelLocalAdapter(QueueContextAdapter):
     def collect(self) -> str:
         """Return collect commands."""
         return (
-            "#!/usr/bin/env bash\n\n"
+            f"#!/usr/bin/env {self._shell}\n\n"
             "# This script is auto-generated by junifer.\n\n"
             "# Run pre_collect.sh\n"
             f"sh {self._pre_collect_path.resolve()!s}\n\n"

junifer/api/queue_context/htcondor_adapter.py

@@ -126,7 +126,8 @@ class HTCondorAdapter(QueueContextAdapter):
         Raises
         ------
         ValueError
-            If ``env.kind`` is invalid.
+            If ``env.kind`` is invalid or
+            if ``env.shell`` is invalid.
 
         """
         # Set env related variables
@@ -140,13 +141,22 @@ class HTCondorAdapter(QueueContextAdapter):
                 f"must be one of {valid_env_kinds}"
             )
         else:
+            # Check shell
+            shell = env.get("shell", "bash")
+            valid_shells = ["bash", "zsh"]
+            if shell not in valid_shells:
+                raise_error(
+                    f"Invalid value for `env.shell`: {shell}, "
+                    f"must be one of {valid_shells}"
+                )
+            self._shell = shell
             # Set variables
             if env["kind"] == "local":
                 # No virtual environment
                 self._executable = "junifer"
                 self._arguments = ""
             else:
-                self._executable = f"run_{env['kind']}.sh"
+                self._executable = f"run_{env['kind']}.{self._shell}"
                 self._arguments = f"{env['name']} junifer"
                 self._exec_path = self._job_dir / self._executable
 
@@ -181,7 +191,7 @@ class HTCondorAdapter(QueueContextAdapter):
     def pre_run(self) -> str:
         """Return pre-run commands."""
         fixed = (
-            "#!/bin/bash\n\n"
+            f"#!/usr/bin/env {self._shell}\n\n"
             "# This script is auto-generated by junifer.\n\n"
             "# Force datalad to run in non-interactive mode\n"
             "DATALAD_UI_INTERACTIVE=false\n"
@@ -225,12 +235,13 @@ class HTCondorAdapter(QueueContextAdapter):
     def pre_collect(self) -> str:
         """Return pre-collect commands."""
         fixed = (
-            "#!/bin/bash\n\n" "# This script is auto-generated by junifer.\n"
+            f"#!/usr/bin/env {self._shell}\n\n"
+            "# This script is auto-generated by junifer.\n"
         )
         var = self._pre_collect or ""
         # Add commands if collect="yes"
         if self._collect == "yes":
-            var += 'if [ "${1}" == "4" ]; then\n' "    exit 1\n" "fi\n"
+            var += 'if [ "${1}" == "4" ]; then\n    exit 1\nfi\n'
         return fixed + "\n" + var
 
     def collect(self) -> str:

junifer/api/queue_context/tests/test_gnu_parallel_local_adapter.py

@@ -12,11 +12,11 @@ import pytest
 from junifer.api.queue_context import GnuParallelLocalAdapter
 
 
-def test_GnuParallelLocalAdapter_env_error() -> None:
+def test_GnuParallelLocalAdapter_env_kind_error() -> None:
     """Test error for invalid env kind."""
     with pytest.raises(ValueError, match="Invalid value for `env.kind`"):
         GnuParallelLocalAdapter(
-            job_name="check_env",
+            job_name="check_env_kind",
             job_dir=Path("."),
             yaml_config_path=Path("."),
             elements=["sub01"],
@@ -24,6 +24,18 @@ def test_GnuParallelLocalAdapter_env_error() -> None:
         )
 
 
+def test_GnuParallelLocalAdapter_env_shell_error() -> None:
+    """Test error for invalid env shell."""
+    with pytest.raises(ValueError, match="Invalid value for `env.shell`"):
+        GnuParallelLocalAdapter(
+            job_name="check_env_shell",
+            job_dir=Path("."),
+            yaml_config_path=Path("."),
+            elements=["sub01"],
+            env={"kind": "conda", "shell": "fish"},
+        )
+
+
 @pytest.mark.parametrize(
     "elements, expected_text",
     [
@@ -55,14 +67,18 @@ def test_GnuParallelLocalAdapter_elements(
 
 
 @pytest.mark.parametrize(
-    "pre_run, expected_text",
+    "pre_run, expected_text, shell",
     [
-        (None, "# Force datalad"),
-        ("# Check this out\n", "# Check this out"),
+        (None, "# Force datalad", "bash"),
+        (None, "# Force datalad", "zsh"),
+        ("# Check this out\n", "# Check this out", "bash"),
+        ("# Check this out\n", "# Check this out", "zsh"),
     ],
 )
 def test_GnuParallelLocalAdapter_pre_run(
-    pre_run: Optional[str], expected_text: str
+    pre_run: Optional[str],
+    expected_text: str,
+    shell: str,
 ) -> None:
     """Test GnuParallelLocalAdapter pre_run().
 
@@ -72,6 +88,8 @@ def test_GnuParallelLocalAdapter_pre_run(
         The parametrized pre run text.
     expected_text : str
         The parametrized expected text.
+    shell : str
+        The parametrized expected shell.
 
     """
     adapter = GnuParallelLocalAdapter(
@@ -79,21 +97,26 @@ def test_GnuParallelLocalAdapter_pre_run(
         job_dir=Path("."),
         yaml_config_path=Path("."),
         elements=["sub01"],
+        env={"kind": "conda", "name": "junifer", "shell": shell},
         pre_run=pre_run,
     )
+    assert shell in adapter.pre_run()
     assert expected_text in adapter.pre_run()
 
 
 @pytest.mark.parametrize(
-    "pre_collect, expected_text",
+    "pre_collect, expected_text, shell",
     [
-        (None, "# This script"),
-        ("# Check this out\n", "# Check this out"),
+        (None, "# This script", "bash"),
+        (None, "# This script", "zsh"),
+        ("# Check this out\n", "# Check this out", "bash"),
+        ("# Check this out\n", "# Check this out", "zsh"),
     ],
 )
 def test_GnuParallelLocalAdapter_pre_collect(
     pre_collect: Optional[str],
     expected_text: str,
+    shell: str,
 ) -> None:
     """Test GnuParallelLocalAdapter pre_collect().
 
@@ -103,6 +126,8 @@ def test_GnuParallelLocalAdapter_pre_collect(
         The parametrized pre collect text.
     expected_text : str
         The parametrized expected text.
+    shell : str
+        The parametrized expected shell.
 
     """
     adapter = GnuParallelLocalAdapter(
@@ -110,8 +135,10 @@ def test_GnuParallelLocalAdapter_pre_collect(
         job_dir=Path("."),
         yaml_config_path=Path("."),
         elements=["sub01"],
+        env={"kind": "venv", "name": "junifer", "shell": shell},
         pre_collect=pre_collect,
     )
+    assert shell in adapter.pre_collect()
     assert expected_text in adapter.pre_collect()
 
 
@@ -140,8 +167,10 @@ def test_GnuParallelLocalAdapter_collect() -> None:
 @pytest.mark.parametrize(
     "env",
     [
-        {"kind": "conda", "name": "junifer"},
-        {"kind": "venv", "name": "./junifer"},
+        {"kind": "conda", "name": "junifer", "shell": "bash"},
+        {"kind": "conda", "name": "junifer", "shell": "zsh"},
+        {"kind": "venv", "name": "./junifer", "shell": "bash"},
+        {"kind": "venv", "name": "./junifer", "shell": "zsh"},
     ],
 )
 def test_GnuParallelLocalAdapter_prepare(
@@ -177,7 +206,7 @@ def test_GnuParallelLocalAdapter_prepare(
             adapter.prepare()
 
             assert "GNU parallel" in caplog.text
-            assert f"Copying run_{env['kind']}" in caplog.text
+            assert f"Copying run_{env['kind']}.{env['shell']}" in caplog.text
             assert "Writing pre_run.sh" in caplog.text
             assert "Writing run_test_prepare.sh" in caplog.text
             assert "Writing pre_collect.sh" in caplog.text