junifer 0.0.6.dev33__tar.gz → 0.0.6.dev51__tar.gz

{junifer-0.0.6.dev33 → junifer-0.0.6.dev51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: junifer
-Version: 0.0.6.dev33
+Version: 0.0.6.dev51
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

{junifer-0.0.6.dev33 → junifer-0.0.6.dev51}/docs/api/api.rst

@@ -1,10 +1,10 @@
 API Functions
 =============
 
-Main API functions
-------------------
+Main Functions
+--------------
 
-.. automodule:: junifer.api
+.. automodule:: junifer.api.functions
    :members:
    :imported-members:
 

junifer-0.0.6.dev51/docs/api/cli.rst

@@ -0,0 +1,6 @@
+CLI Functions
+=============
+
+.. automodule:: junifer.cli.parser
+   :members:
+   :imported-members:

{junifer-0.0.6.dev33 → junifer-0.0.6.dev51}/docs/api/index.rst

@@ -25,6 +25,7 @@ Utilities
    stats
    pipeline
    api
+   cli
    utils
    testing
    onthefly

junifer-0.0.6.dev51/docs/changes/newsfragments/361.misc

@@ -0,0 +1 @@
+Refactor ``api`` sub-package and introduce new ``cli`` sub-package by `Synchon Mandal`_

junifer-0.0.6.dev51/docs/changes/newsfragments/367.enh

@@ -0,0 +1 @@
+Allow for external python files in the ``with`` section of the yaml to import other files by `Fede Raimondo`_

{junifer-0.0.6.dev33 → junifer-0.0.6.dev51}/docs/extending/extension.rst

@@ -28,8 +28,25 @@ This is the ideal place to include ``junifer`` extensions.
 
 .. important::
 
-   Some ``junifer`` commands will not consider files imported from files included
-   in the ``with`` statement. If ``my_file.py`` imports ``my_other_file.py``,
-   some of the ``junifer`` commands will not consider ``my_other_file.py``. Either
-   place all the code in one file or add multiple files to the ``with``
-   statement.
+   Some ``junifer`` commands will not consider files imported from files
+   included in the ``with`` statement, unless this is known to junifer. If
+   ``my_file.py`` imports ``my_other_file.py``, the ``run`` command will work,
+   but ``queue`` will not create a proper job. This is because we need to 
+   let junifer know that ``my_other_file.py`` is also part of the code. To do
+   so, we need to include a special function in ``my_file.py`` which tells
+   ``junifer`` about the dependencies of the module:
+
+   .. code-block:: python
+
+      def junifer_module_deps() -> List[str]:
+        """Return the dependencies of the module.
+
+        Returns
+        -------
+        list of str
+            The list of dependencies.
+
+        """
+
+        return ["my_other_file.py"]
+

{junifer-0.0.6.dev33 → junifer-0.0.6.dev51}/junifer/__init__.py

@@ -6,6 +6,7 @@
 
 from . import (
     api,
+    cli,
     configs,
     data,
     datagrabber,
@@ -24,6 +25,7 @@ from ._version import __version__
 
 __all__ = [
     "api",
+    "cli",
     "configs",
     "data",
     "datagrabber",

{junifer-0.0.6.dev33 → junifer-0.0.6.dev51}/junifer/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.6.dev33'
-__version_tuple__ = version_tuple = (0, 0, 6, 'dev33')
+__version__ = version = '0.0.6.dev51'
+__version_tuple__ = version_tuple = (0, 0, 6, 'dev51')

junifer-0.0.6.dev51/junifer/api/__init__.py

@@ -0,0 +1,11 @@
+"""Public API components."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+#          Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+from . import decorators
+from .functions import collect, list_elements, reset, run, queue
+
+
+__all__ = ["decorators", "collect", "queue", "run", "reset", "list_elements"]

{junifer-0.0.6.dev33 → junifer-0.0.6.dev51}/junifer/api/decorators.py

@@ -1,4 +1,4 @@
-"""Provide decorators for api."""
+"""Provide API decorators."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Leonard Sasse <l.sasse@fz-juelich.de>

{junifer-0.0.6.dev33 → junifer-0.0.6.dev51}/junifer/api/functions.py

@@ -1,4 +1,4 @@
-"""Provide functions for cli."""
+"""Provide API functions."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Leonard Sasse <l.sasse@fz-juelich.de>
@@ -11,6 +11,7 @@ import typing
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
+from ..api.queue_context import GnuParallelLocalAdapter, HTCondorAdapter
 from ..datagrabber.base import BaseDataGrabber
 from ..markers.base import BaseMarker
 from ..markers.collection import MarkerCollection
@@ -18,9 +19,7 @@ from ..pipeline import WorkDirManager
 from ..pipeline.registry import build
 from ..preprocess.base import BasePreprocessor
 from ..storage.base import BaseFeatureStorage
-from ..utils import logger, raise_error
-from .queue_context import GnuParallelLocalAdapter, HTCondorAdapter
-from .utils import yaml
+from ..utils import logger, raise_error, yaml
 
 
 __all__ = ["run", "collect", "queue", "reset", "list_elements"]

{junifer-0.0.6.dev33 → junifer-0.0.6.dev51}/junifer/api/tests/test_functions.py

@@ -1,4 +1,4 @@
-"""Provide tests for functions."""
+"""Provide tests for API functions."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Leonard Sasse <l.sasse@fz-juelich.de>
@@ -13,7 +13,7 @@ import pytest
 from ruamel.yaml import YAML
 
 import junifer.testing.registry  # noqa: F401
-from junifer.api.functions import collect, list_elements, queue, reset, run
+from junifer.api import collect, list_elements, queue, reset, run
 from junifer.datagrabber.base import BaseDataGrabber
 from junifer.pipeline.registry import build
 

junifer-0.0.6.dev51/junifer/cli/__init__.py

@@ -0,0 +1,8 @@
+"""Junifer CLI."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+from .cli import cli
+
+__all__ = ["cli"]

{junifer-0.0.6.dev33/junifer/api → junifer-0.0.6.dev51/junifer/cli}/cli.py

@@ -1,4 +1,4 @@
-"""Provide functions for cli."""
+"""Provide functions for CLI."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Synchon Mandal <s.mandal@fz-juelich.de>
@@ -8,122 +8,26 @@ import pathlib
 import subprocess
 import sys
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Optional, Tuple, Union
 
 import click
-import pandas as pd
 
-from ..utils.logging import (
+from ..api import functions as cli_func
+from ..utils import (
     configure_logging,
-    logger,
     raise_error,
-    warn_with_log,
+    yaml,
 )
-from .functions import collect as api_collect
-from .functions import list_elements as api_list_elements
-from .functions import queue as api_queue
-from .functions import reset as api_reset
-from .functions import run as api_run
-from .parser import parse_yaml
+from .parser import parse_elements, parse_yaml
 from .utils import (
     _get_dependency_information,
     _get_environment_information,
     _get_junifer_version,
     _get_python_information,
     _get_system_information,
-    yaml,
 )
 
 
-def _parse_elements(element: Tuple[str], config: Dict) -> Union[List, None]:
-    """Parse elements from cli.
-
-    Parameters
-    ----------
-    element : tuple of str
-        The element(s) to operate on.
-    config : dict
-        The configuration to operate using.
-
-    Returns
-    -------
-    list or None
-        The element(s) as list or None.
-
-    Raises
-    ------
-    ValueError
-        If no element is found either in the command-line options or
-        the configuration file.
-
-    Warns
-    -----
-    RuntimeWarning
-        If elements are specified both via the command-line options and
-        the configuration file.
-
-    """
-    logger.debug(f"Parsing elements: {element}")
-    # Early return None to continue with all elements
-    if len(element) == 0:
-        return None
-    # Check if the element is a file for single element;
-    # if yes, then parse elements from it
-    if len(element) == 1 and Path(element[0]).resolve().is_file():
-        elements = _parse_elements_file(Path(element[0]).resolve())
-    else:
-        # Process multi-keyed elements
-        elements = [tuple(x.split(",")) if "," in x else x for x in element]
-    logger.debug(f"Parsed elements: {elements}")
-    if elements is not None and "elements" in config:
-        warn_with_log(
-            "One or more elements have been specified in both the command "
-            "line and in the config file. The command line has precedence "
-            "over the configuration file. That is, the elements specified "
-            "in the command line will be used. The elements specified in "
-            "the configuration file will be ignored. To remove this warning, "
-            "please remove the `elements` item from the configuration file."
-        )
-    elif elements is None:
-        # Check in config
-        elements = config.get("elements", None)
-        if elements is None:
-            raise_error(
-                "The `elements` key is set in the configuration, but its value"
-                " is `None`. It is likely that there is an empty `elements` "
-                "section in the yaml configuration file."
-            )
-    return elements
-
-
-def _parse_elements_file(filepath: Path) -> List[Tuple[str, ...]]:
-    """Parse elements from file.
-
-    Parameters
-    ----------
-    filepath : pathlib.Path
-        The path to the element file.
-
-    Returns
-    -------
-    list of tuple of str
-        The element(s) as list.
-
-    """
-    # Read CSV into dataframe
-    csv_df = pd.read_csv(
-        filepath,
-        header=None,  # no header  # type: ignore
-        index_col=False,  # no index column
-        dtype=str,
-        skipinitialspace=True,  # no leading space after delimiter
-    )
-    # Remove trailing whitespace in cell entries
-    csv_df_trimmed = csv_df.apply(lambda x: x.str.strip())
-    # Convert to list of tuple of str
-    return list(map(tuple, csv_df_trimmed.to_numpy()))
-
-
 def _validate_verbose(
     ctx: click.Context, param: str, value: str
 ) -> Union[str, int]:
@@ -166,7 +70,7 @@ def _validate_verbose(
 
 @click.group()
 def cli() -> None:  # pragma: no cover
-    """CLI for JUelich NeuroImaging FEature extractoR."""
+    """JUelich NeuroImaging FEature extractoR."""
 
 
 @cli.command()
@@ -187,7 +91,7 @@ def cli() -> None:  # pragma: no cover
 def run(
     filepath: click.Path, element: Tuple[str], verbose: Union[str, int]
 ) -> None:
-    """Run command for CLI.
+    """Run feature extraction.
 
     \f
 
@@ -219,9 +123,9 @@ def run(
     if preprocessors is not None and not isinstance(preprocessors, list):
         preprocessors = [preprocessors]
     # Parse elements
-    elements = _parse_elements(element, config)
+    elements = parse_elements(element, config)
     # Perform operation
-    api_run(
+    cli_func.run(
         workdir=workdir,
         datagrabber=datagrabber,
         markers=markers,
@@ -246,7 +150,7 @@ def run(
     default="info",
 )
 def collect(filepath: click.Path, verbose: Union[str, int]) -> None:
-    """Collect command for CLI.
+    """Collect extracted features.
 
     \f
 
@@ -263,7 +167,7 @@ def collect(filepath: click.Path, verbose: Union[str, int]) -> None:
     config = parse_yaml(filepath)  # type: ignore
     storage = config["storage"]
     # Perform operation
-    api_collect(storage=storage)
+    cli_func.collect(storage=storage)
 
 
 @cli.command()
@@ -290,7 +194,7 @@ def queue(
     submit: bool,
     verbose: Union[str, int],
 ) -> None:
-    """Queue command for CLI.
+    """Queue feature extraction.
 
     \f
 
@@ -311,12 +215,12 @@ def queue(
     configure_logging(level=verbose)
     # TODO: add validation
     config = parse_yaml(filepath)  # type: ignore
-    elements = _parse_elements(element, config)
+    elements = parse_elements(element, config)
     if "queue" not in config:
         raise_error(f"No queue configuration found in {filepath}.")
     queue_config = config.pop("queue")
     kind = queue_config.pop("kind")
-    api_queue(
+    cli_func.queue(
         config=config,
         kind=kind,
         overwrite=overwrite,
@@ -329,7 +233,7 @@ def queue(
 @cli.command()
 @click.option("--long", "long_", is_flag=True)
 def wtf(long_: bool) -> None:
-    """Wtf command for CLI.
+    """WTF?!.
 
     \f
 
@@ -352,7 +256,7 @@ def wtf(long_: bool) -> None:
 @cli.command()
 @click.argument("subpkg", type=str)
 def selftest(subpkg: str) -> None:
-    """Selftest command for CLI.
+    """Selftest.
 
     \f
 
@@ -433,7 +337,7 @@ def reset(
     filepath: click.Path,
     verbose: Union[str, int],
 ) -> None:
-    """Reset command for CLI.
+    """Reset generated assets.
 
     \f
 
@@ -449,7 +353,7 @@ def reset(
     # Parse YAML
     config = parse_yaml(filepath)
     # Perform operation
-    api_reset(config)
+    cli_func.reset(config)
 
 
 @cli.command()
@@ -478,7 +382,7 @@ def list_elements(
     output_file: Optional[click.Path],
     verbose: Union[str, int],
 ) -> None:
-    """Element listing command for CLI.
+    """List elements of a dataset.
 
     \f
 
@@ -501,9 +405,9 @@ def list_elements(
     # Fetch datagrabber
     datagrabber = config["datagrabber"]
     # Parse elements
-    elements = _parse_elements(element, config)
+    elements = parse_elements(element, config)
     # Perform operation
-    listed_elements = api_list_elements(
+    listed_elements = cli_func.list_elements(
         datagrabber=datagrabber,
         elements=elements,
     )
@@ -517,7 +421,7 @@ def list_elements(
 
 @cli.group()
 def setup() -> None:  # pragma: no cover
-    """Configure commands for Junifer."""
+    """Configure external tools."""
     pass
 
 

{junifer-0.0.6.dev33/junifer/api → junifer-0.0.6.dev51/junifer/cli}/parser.py

@@ -1,4 +1,4 @@
-"""Provide functions for parser."""
+"""Provide functions for CLI parser."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Synchon Mandal <s.mandal@fz-juelich.de>
@@ -8,16 +8,17 @@ import importlib
 import importlib.util
 import sys
 from pathlib import Path
-from typing import Dict, Union
+from typing import Dict, List, Tuple, Union
 
-from ..utils.logging import logger, raise_error
-from .utils import yaml
+import pandas as pd
 
+from ..utils import logger, raise_error, warn_with_log, yaml
 
-__all__ = ["parse_yaml"]
 
+__all__ = ["parse_yaml", "parse_elements"]
 
-def parse_yaml(filepath: Union[str, Path]) -> Dict:
+
+def parse_yaml(filepath: Union[str, Path]) -> Dict:  # noqa: C901
     """Parse YAML.
 
     Parameters
@@ -64,14 +65,35 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict:
                     raise_error(
                         f"File in 'with' section does not exist: {file_path}"
                     )
+                # Add the parent directory to the sys.path so that the
+                # any imports from this module work correctly
+                t_path = str(file_path.parent)
+                if t_path not in sys.path:
+                    sys.path.append(str(file_path.parent))
+
                 spec = importlib.util.spec_from_file_location(
                     t_module, file_path
                 )
                 module = importlib.util.module_from_spec(spec)  # type: ignore
                 sys.modules[t_module] = module
                 spec.loader.exec_module(module)  # type: ignore
+
                 # Add absolute path to final list
                 final_to_load.append(str(file_path.resolve()))
+
+                # Check if the module has junifer_module_deps function
+                if hasattr(module, "junifer_module_deps"):
+                    logger.debug(
+                        f"Module {t_module} has junifer_module_deps function"
+                    )
+                    # Get the dependencies
+                    deps = module.junifer_module_deps()
+                    # Add the dependencies to the final list
+                    for dep in deps:
+                        if dep not in final_to_load:
+                            final_to_load.append(
+                                str((file_path.parent / dep).resolve())
+                            )
             else:
                 logger.info(f"Importing module: {t_module}")
                 importlib.import_module(t_module)
@@ -116,3 +138,92 @@ def parse_yaml(filepath: Union[str, Path]) -> Dict:
                     )
 
     return contents
+
+
+def parse_elements(element: Tuple[str], config: Dict) -> Union[List, None]:
+    """Parse elements from cli.
+
+    Parameters
+    ----------
+    element : tuple of str
+        The element(s) to operate on.
+    config : dict
+        The configuration to operate using.
+
+    Returns
+    -------
+    list or None
+        The element(s) as list or None.
+
+    Raises
+    ------
+    ValueError
+        If no element is found either in the command-line options or
+        the configuration file.
+
+    Warns
+    -----
+    RuntimeWarning
+        If elements are specified both via the command-line options and
+        the configuration file.
+
+    """
+    logger.debug(f"Parsing elements: {element}")
+    # Early return None to continue with all elements
+    if len(element) == 0:
+        return None
+    # Check if the element is a file for single element;
+    # if yes, then parse elements from it
+    if len(element) == 1 and Path(element[0]).resolve().is_file():
+        elements = _parse_elements_file(Path(element[0]).resolve())
+    else:
+        # Process multi-keyed elements
+        elements = [tuple(x.split(",")) if "," in x else x for x in element]
+    logger.debug(f"Parsed elements: {elements}")
+    if elements is not None and "elements" in config:
+        warn_with_log(
+            "One or more elements have been specified in both the command "
+            "line and in the config file. The command line has precedence "
+            "over the configuration file. That is, the elements specified "
+            "in the command line will be used. The elements specified in "
+            "the configuration file will be ignored. To remove this warning, "
+            "please remove the `elements` item from the configuration file."
+        )
+    elif elements is None:
+        # Check in config
+        elements = config.get("elements", None)
+        if elements is None:
+            raise_error(
+                "The `elements` key is set in the configuration, but its value"
+                " is `None`. It is likely that there is an empty `elements` "
+                "section in the yaml configuration file."
+            )
+    return elements
+
+
+def _parse_elements_file(filepath: Path) -> List[Tuple[str, ...]]:
+    """Parse elements from file.
+
+    Parameters
+    ----------
+    filepath : pathlib.Path
+        The path to the element file.
+
+    Returns
+    -------
+    list of tuple of str
+        The element(s) as list.
+
+    """
+    # Read CSV into dataframe
+    csv_df = pd.read_csv(
+        filepath,
+        header=None,  # no header  # type: ignore
+        index_col=False,  # no index column
+        dtype=str,
+        skipinitialspace=True,  # no leading space after delimiter
+    )
+    # Remove trailing whitespace in cell entries
+    csv_df_trimmed = csv_df.apply(lambda x: x.str.strip())
+    # Convert to list of tuple of str
+    return list(map(tuple, csv_df_trimmed.to_numpy()))

{junifer-0.0.6.dev33/junifer/api → junifer-0.0.6.dev51/junifer/cli}/tests/test_cli.py

@@ -1,4 +1,4 @@
-"""Provide tests for cli."""
+"""Provide tests for CLI."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Synchon Mandal <s.mandal@fz-juelich.de>
@@ -11,8 +11,7 @@ import pytest
 from click.testing import CliRunner
 from ruamel.yaml import YAML
 
-from junifer.api.cli import (
-    _parse_elements_file,
+from junifer.cli.cli import (
     collect,
     list_elements,
     queue,
@@ -21,6 +20,7 @@ from junifer.api.cli import (
     selftest,
     wtf,
 )
+from junifer.cli.parser import _parse_elements_file
 
 
 # Configure YAML class

junifer-0.0.6.dev33/junifer/api/tests/test_api_utils.py → junifer-0.0.6.dev51/junifer/cli/tests/test_cli_utils.py

@@ -1,4 +1,4 @@
-"""Provide tests for utils."""
+"""Provide tests for CLI utils."""
 
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
@@ -9,7 +9,7 @@ import sys
 import pytest
 
 from junifer._version import __version__
-from junifer.api.utils import (
+from junifer.cli.utils import (
     _get_dependency_information,
     _get_environment_information,
     _get_junifer_version,

{junifer-0.0.6.dev33/junifer/api → junifer-0.0.6.dev51/junifer/cli}/tests/test_parser.py

@@ -1,4 +1,4 @@
-"""Provide tests for parser."""
+"""Provide tests for CLI parser."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Synchon Mandal <s.mandal@fz-juelich.de>
@@ -9,7 +9,7 @@ from pathlib import Path
 
 import pytest
 
-from junifer.api.parser import parse_yaml
+from junifer.cli.parser import parse_yaml
 
 
 def test_parse_yaml_failure() -> None:

{junifer-0.0.6.dev33/junifer/api → junifer-0.0.6.dev51/junifer/cli}/utils.py

@@ -1,4 +1,4 @@
-"""Provide utility functions for the api sub-package."""
+"""Provide utility functions for the cli sub-package."""
 
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
@@ -9,19 +9,10 @@ import re
 from importlib.metadata import distribution
 from typing import Dict
 
-from ruamel.yaml import YAML
-
 from .._version import __version__
 from ..utils.logging import get_versions
 
 
-# Configure YAML class once for further use
-yaml = YAML()
-yaml.default_flow_style = False
-yaml.allow_unicode = True
-yaml.indent(mapping=2, sequence=4, offset=2)
-
-
 def _get_junifer_version() -> Dict[str, str]:
     """Get junifer version information.