junifer 0.0.5.dev242__py3-none-any.whl → 0.0.6__py3-none-any.whl

junifer/cli/parser.py

@@ -0,0 +1,235 @@
+"""Provide functions for CLI parser."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+#          Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+import importlib
+import importlib.util
+import sys
+from pathlib import Path
+from typing import Union
+
+import pandas as pd
+
+from ..typing import Elements
+from ..utils import logger, raise_error, warn_with_log, yaml
+
+
+__all__ = ["parse_elements", "parse_yaml"]
+
+
+def parse_yaml(filepath: Union[str, Path]) -> dict:  # noqa: C901
+    """Parse YAML.
+
+    Parameters
+    ----------
+    filepath : str or pathlib.Path
+        The filepath to read from.
+
+    Returns
+    -------
+    dict
+        The contents represented as dictionary.
+
+    """
+    # Convert str to Path
+    if not isinstance(filepath, Path):
+        filepath = Path(filepath)
+
+    logger.info(f"Parsing yaml file: {filepath.absolute()!s}")
+    # Filepath existence check
+    if not filepath.exists():
+        raise_error(f"File does not exist: {filepath.absolute()!s}")
+    # Filepath reading
+    contents = yaml.load(filepath)
+    if "elements" in contents:
+        if contents["elements"] is None:
+            raise_error(
+                "The elements key was defined but its content is empty. "
+                "Please define the elements to operate on or remove the key."
+            )
+    # load modules
+    if "with" in contents:
+        to_load = contents["with"]
+        # Convert load modules to list
+        if not isinstance(to_load, list):
+            to_load = [to_load]
+        # Initialize list to have absolute paths for custom modules
+        final_to_load = []
+        for t_module in to_load:
+            if t_module.endswith(".py"):
+                logger.debug(f"Importing file: {t_module}")
+                # This resolves both absolute and relative paths
+                file_path = filepath.parent / t_module
+                if not file_path.exists():
+                    raise_error(
+                        f"File in 'with' section does not exist: {file_path}"
+                    )
+                # Add the parent directory to the sys.path so that the
+                # any imports from this module work correctly
+                t_path = str(file_path.parent)
+                if t_path not in sys.path:
+                    sys.path.append(str(file_path.parent))
+
+                spec = importlib.util.spec_from_file_location(
+                    t_module, file_path
+                )
+                module = importlib.util.module_from_spec(spec)  # type: ignore
+                sys.modules[t_module] = module
+                spec.loader.exec_module(module)  # type: ignore
+
+                # Add absolute path to final list
+                final_to_load.append(str(file_path.resolve()))
+
+                # Check if the module has junifer_module_deps function
+                if hasattr(module, "junifer_module_deps"):
+                    logger.debug(
+                        f"Module {t_module} has junifer_module_deps function"
+                    )
+                    # Get the dependencies
+                    deps = module.junifer_module_deps()
+                    # Add the dependencies to the final list
+                    for dep in deps:
+                        if dep not in final_to_load:
+                            final_to_load.append(
+                                str((file_path.parent / dep).resolve())
+                            )
+            else:
+                logger.info(f"Importing module: {t_module}")
+                importlib.import_module(t_module)
+                # Add module to final list
+                final_to_load.append(t_module)
+
+        # Replace modules to be loaded so that custom modules will take the
+        # absolute path. This was not the case as found in #224. Similar thing
+        # is done with the storage URI below.
+        contents["with"] = final_to_load
+
+    # Compute path for the URI parameter in storage files that are relative
+    # This is a tricky thing that appeared in #127. The problem is that
+    # the path in the URI parameter is relative to YAML file, not to the
+    # current working directory. If we leave it as is in the contents
+    # dict, then it will be used later in the ``build`` function as is,
+    # which will be computed relative to the current working directory.
+    # The solution is to compute the absolute path and replace the
+    # relative path in the contents dict with the absolute path.
+
+    # Check if the storage file is defined
+    if "storage" in contents:
+        if "uri" in contents["storage"]:
+            # Check if the storage file is relative
+            uri_path = Path(contents["storage"]["uri"])
+            if not uri_path.is_absolute():
+                # Compute the absolute path
+                contents["storage"]["uri"] = str(
+                    (filepath.parent / uri_path).resolve()
+                )
+
+    # Allow relative path if queue env kind is venv; same motivation as above
+    if "queue" in contents:
+        if "env" in contents["queue"]:
+            if "venv" == contents["queue"]["env"]["kind"]:
+                # Check if the env name is relative
+                venv_path = Path(contents["queue"]["env"]["name"])
+                if not venv_path.is_absolute():
+                    # Compute the absolute path
+                    contents["queue"]["env"]["name"] = str(
+                        (filepath.parent / venv_path).resolve()
+                    )
+
+    return contents
+
+
+def parse_elements(
+    element: tuple[str, ...], config: dict
+) -> Union[Elements, None]:
+    """Parse elements from cli.
+
+    Parameters
+    ----------
+    element : tuple of str
+        The element(s) to operate on.
+    config : dict
+        The configuration to operate using.
+
+    Returns
+    -------
+    list or None
+        The element(s) as list or None.
+
+    Raises
+    ------
+    ValueError
+        If no element is found either in the command-line options or
+        the configuration file.
+
+    Warns
+    -----
+    RuntimeWarning
+        If elements are specified both via the command-line options and
+        the configuration file.
+
+    """
+    logger.debug(f"Parsing elements: {element}")
+    # Early return None to continue with all elements
+    if not element:
+        return None
+    # Check if the element is a file for single element;
+    # if yes, then parse elements from it
+    if len(element) == 1 and Path(element[0]).resolve().is_file():
+        elements = _parse_elements_file(Path(element[0]).resolve())
+    else:
+        # Process multi-keyed elements
+        elements = [tuple(x.split(",")) if "," in x else x for x in element]
+    logger.debug(f"Parsed elements: {elements}")
+    if elements is not None and "elements" in config:
+        warn_with_log(
+            "One or more elements have been specified in both the command "
+            "line and in the config file. The command line has precedence "
+            "over the configuration file. That is, the elements specified "
+            "in the command line will be used. The elements specified in "
+            "the configuration file will be ignored. To remove this warning, "
+            "please remove the `elements` item from the configuration file."
+        )
+    elif elements is None:
+        # Check in config
+        elements = config.get("elements", None)
+        if elements is None:
+            raise_error(
+                "The `elements` key is set in the configuration, but its value"
+                " is `None`. It is likely that there is an empty `elements` "
+                "section in the yaml configuration file."
+            )
+    return elements
+
+
+def _parse_elements_file(filepath: Path) -> Elements:
+    """Parse elements from file.
+
+    Parameters
+    ----------
+    filepath : pathlib.Path
+        The path to the element file.
+
+    Returns
+    -------
+    list
+        The element(s) as list.
+
+    """
+    # Read CSV into dataframe
+    csv_df = pd.read_csv(
+        filepath,
+        header=None,  # no header  # type: ignore
+        index_col=False,  # no index column
+        dtype=str,
+        skipinitialspace=True,  # no leading space after delimiter
+    )
+    # Remove trailing whitespace in cell entries
+    csv_df_trimmed = csv_df.apply(lambda x: x.str.strip())
+    # Convert to list of tuple of str if more than one column else flatten
+    if len(csv_df_trimmed.columns) == 1:
+        return csv_df_trimmed.to_numpy().flatten().tolist()
+    else:
+        return list(map(tuple, csv_df_trimmed.to_numpy()))

junifer/{api → cli}/tests/test_cli.py

@@ -1,18 +1,17 @@
-"""Provide tests for cli."""
+"""Provide tests for CLI."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
 from pathlib import Path
-from typing import Callable, List, Tuple
+from typing import Callable
 
 import pytest
 from click.testing import CliRunner
 from ruamel.yaml import YAML
 
-from junifer.api.cli import (
-    _parse_elements_file,
+from junifer.cli.cli import (
     collect,
     list_elements,
     queue,
@@ -21,6 +20,7 @@ from junifer.api.cli import (
     selftest,
     wtf,
 )
+from junifer.cli.parser import _parse_elements_file
 
 
 # Configure YAML class
@@ -42,7 +42,7 @@ runner = CliRunner()
     ],
 )
 def test_run_and_collect_commands(
-    tmp_path: Path, elements: Tuple[str, ...]
+    tmp_path: Path, elements: tuple[str, ...]
 ) -> None:
     """Test run and collect commands.
 
@@ -175,7 +175,7 @@ def test_run_using_element_file(tmp_path: Path, elements: str) -> None:
     ],
 )
 def test_multi_element_access(
-    tmp_path: Path, elements: str, expected_list: List[Tuple[str, ...]]
+    tmp_path: Path, elements: str, expected_list: list[tuple[str, ...]]
 ) -> None:
     """Test mulit-element parsing.
 
@@ -306,7 +306,7 @@ def test_reset(
     ],
 )
 def test_list_elements_stdout(
-    elements: Tuple[str, ...],
+    elements: tuple[str, ...],
 ) -> None:
     """Test elements listing to stdout.
 
@@ -344,7 +344,7 @@ def test_list_elements_stdout(
 )
 def test_list_elements_output_file(
     tmp_path: Path,
-    elements: Tuple[str, ...],
+    elements: tuple[str, ...],
 ) -> None:
     """Test elements listing to output file.
 

junifer/{api/tests/test_api_utils.py → cli/tests/test_cli_utils.py}

@@ -1,4 +1,4 @@
-"""Provide tests for utils."""
+"""Provide tests for CLI utils."""
 
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
@@ -9,7 +9,7 @@ import sys
 import pytest
 
 from junifer._version import __version__
-from junifer.api.utils import (
+from junifer.cli.utils import (
     _get_dependency_information,
     _get_environment_information,
     _get_junifer_version,
@@ -43,11 +43,12 @@ def test_get_dependency_information_short() -> None:
         "nilearn",
         "sqlalchemy",
         "ruamel.yaml",
-        "httpx",
         "tqdm",
         "templateflow",
         "lapy",
+        "lazy_loader",
         "looseversion",
+        "junifer_data",
     ]
 
     if sys.version_info < (3, 11):
@@ -72,10 +73,10 @@ def test_get_dependency_information_long() -> None:
         "nilearn",
         "sqlalchemy",
         "ruamel.yaml",
-        "httpx",
         "tqdm",
         "templateflow",
         "lapy",
+        "lazy_loader",
     ]
     for key in dependency_list:
         assert key in dependency_information_keys

junifer/{api → cli}/tests/test_parser.py

@@ -1,4 +1,4 @@
-"""Provide tests for parser."""
+"""Provide tests for CLI parser."""
 
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 #          Synchon Mandal <s.mandal@fz-juelich.de>
@@ -9,7 +9,7 @@ from pathlib import Path
 
 import pytest
 
-from junifer.api.parser import parse_yaml
+from junifer.cli.parser import parse_yaml
 
 
 def test_parse_yaml_failure() -> None:

junifer/{api → cli}/utils.py

@@ -1,4 +1,4 @@
-"""Provide utility functions for the api sub-package."""
+"""Provide utility functions for the cli sub-package."""
 
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
@@ -7,22 +7,12 @@ import os
 import platform as pl
 import re
 from importlib.metadata import distribution
-from typing import Dict
-
-from ruamel.yaml import YAML
 
 from .._version import __version__
 from ..utils.logging import get_versions
 
 
-# Configure YAML class once for further use
-yaml = YAML()
-yaml.default_flow_style = False
-yaml.allow_unicode = True
-yaml.indent(mapping=2, sequence=4, offset=2)
-
-
-def _get_junifer_version() -> Dict[str, str]:
+def _get_junifer_version() -> dict[str, str]:
     """Get junifer version information.
 
     Returns
@@ -36,7 +26,7 @@ def _get_junifer_version() -> Dict[str, str]:
     }
 
 
-def _get_python_information() -> Dict[str, str]:
+def _get_python_information() -> dict[str, str]:
     """Get installed Python information.
 
     Parameters
@@ -51,7 +41,7 @@ def _get_python_information() -> Dict[str, str]:
     }
 
 
-def _get_dependency_information(long_: bool) -> Dict[str, str]:
+def _get_dependency_information(long_: bool) -> dict[str, str]:
     """Get Python environment dependency information.
 
     Parameters
@@ -96,7 +86,7 @@ def _get_dependency_information(long_: bool) -> Dict[str, str]:
     return pruned_dependency_versions
 
 
-def _get_system_information() -> Dict[str, str]:
+def _get_system_information() -> dict[str, str]:
     """Get system information.
 
     Returns
@@ -110,7 +100,7 @@ def _get_system_information() -> Dict[str, str]:
     }
 
 
-def _get_environment_information(long_: bool) -> Dict[str, str]:
+def _get_environment_information(long_: bool) -> dict[str, str]:
     """Get system environment information.
 
     Parameters

junifer/configs/juseless/__init__.py

@@ -3,7 +3,7 @@
 # Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
 # License: AGPL
 
-from . import datagrabbers
+import lazy_loader as lazy
 
 
-__all__ = ["datagrabbers"]
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)

junifer/configs/juseless/__init__.pyi

@@ -0,0 +1,3 @@
+__all__ = ["datagrabbers"]
+
+from . import datagrabbers

junifer/configs/juseless/datagrabbers/__init__.py

@@ -5,17 +5,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from .aomic_id1000_vbm import JuselessDataladAOMICID1000VBM
-from .camcan_vbm import JuselessDataladCamCANVBM
-from .ixi_vbm import JuselessDataladIXIVBM
-from .ucla import JuselessUCLA
-from .ukb_vbm import JuselessDataladUKBVBM
+import lazy_loader as lazy
 
 
-__all__ = [
-    "JuselessDataladAOMICID1000VBM",
-    "JuselessDataladCamCANVBM",
-    "JuselessDataladIXIVBM",
-    "JuselessUCLA",
-    "JuselessDataladUKBVBM",
-]
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)

junifer/configs/juseless/datagrabbers/__init__.pyi

@@ -0,0 +1,13 @@
+__all__ = [
+    "JuselessDataladAOMICID1000VBM",
+    "JuselessDataladCamCANVBM",
+    "JuselessDataladIXIVBM",
+    "JuselessUCLA",
+    "JuselessDataladUKBVBM",
+]
+
+from .aomic_id1000_vbm import JuselessDataladAOMICID1000VBM
+from .camcan_vbm import JuselessDataladCamCANVBM
+from .ixi_vbm import JuselessDataladIXIVBM
+from .ucla import JuselessUCLA
+from .ukb_vbm import JuselessDataladUKBVBM

junifer/configs/juseless/datagrabbers/ixi_vbm.py

@@ -6,7 +6,7 @@
 # License: AGPL
 
 from pathlib import Path
-from typing import List, Union
+from typing import Union
 
 from ....api.decorators import register_datagrabber
 from ....datagrabber import PatternDataladDataGrabber
@@ -37,7 +37,7 @@ class JuselessDataladIXIVBM(PatternDataladDataGrabber):
     def __init__(
         self,
         datadir: Union[str, Path, None] = None,
-        sites: Union[str, List[str], None] = None,
+        sites: Union[str, list[str], None] = None,
     ) -> None:
         uri = (
             "ria+http://cat_12.5.ds.inm7.de"

junifer/configs/juseless/datagrabbers/tests/test_ucla.py

@@ -6,7 +6,7 @@
 # License: AGPL
 
 import socket
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 import pytest
 
@@ -53,7 +53,7 @@ def test_JuselessUCLA() -> None:
     ],
 )
 def test_JuselessUCLA_partial_data_access(
-    types: Union[str, List[str]],
+    types: Union[str, list[str]],
 ) -> None:
     """Test JuselessUCLA DataGrabber partial data access.
 

junifer/configs/juseless/datagrabbers/ucla.py

@@ -5,7 +5,7 @@
 # License: AGPL
 
 from pathlib import Path
-from typing import List, Union
+from typing import Union
 
 from ....api.decorators import register_datagrabber
 from ....datagrabber import PatternDataGrabber
@@ -43,8 +43,8 @@ class JuselessUCLA(PatternDataGrabber):
         datadir: Union[
             str, Path
         ] = "/data/project/psychosis_thalamus/data/fmriprep",
-        types: Union[str, List[str], None] = None,
-        tasks: Union[str, List[str], None] = None,
+        types: Union[str, list[str], None] = None,
+        tasks: Union[str, list[str], None] = None,
     ) -> None:
         # Declare all tasks
         all_tasks = [
@@ -136,7 +136,7 @@ class JuselessUCLA(PatternDataGrabber):
             confounds_format="fmriprep",
         )
 
-    def get_elements(self) -> List:
+    def get_elements(self) -> list:
         """Implement fetching list of elements in the dataset.
 
         Returns

junifer/conftest.py

@@ -0,0 +1,25 @@
+"""Provide conftest for pytest."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+# License: AGPL
+
+import pytest
+
+from junifer.utils.singleton import Singleton
+
+
+@pytest.fixture(autouse=True)
+def reset_singletons() -> None:
+    """Reset all singletons."""
+    to_clean = ["WorkDirManager"]
+    to_remove = [
+        v for k, v in Singleton.instances.items() if k.__name__ in to_clean
+    ]
+    Singleton.instances = {
+        k: v
+        for k, v in Singleton.instances.items()
+        if k.__name__ not in to_clean
+    }
+    # Force deleting the singletons
+    for elem in to_remove:
+        del elem

junifer/data/__init__.py

@@ -4,47 +4,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from .coordinates import (
-    list_coordinates,
-    load_coordinates,
-    register_coordinates,
-    get_coordinates,
-)
-from .parcellations import (
-    list_parcellations,
-    load_parcellation,
-    register_parcellation,
-    merge_parcellations,
-    get_parcellation,
-)
+import lazy_loader as lazy
 
-from .masks import (
-    list_masks,
-    load_mask,
-    register_mask,
-    get_mask,
-)
 
-from .template_spaces import get_template, get_xfm
-
-from . import utils
-
-
-__all__ = [
-    "list_coordinates",
-    "load_coordinates",
-    "register_coordinates",
-    "get_coordinates",
-    "list_parcellations",
-    "load_parcellation",
-    "register_parcellation",
-    "merge_parcellations",
-    "get_parcellation",
-    "list_masks",
-    "load_mask",
-    "register_mask",
-    "get_mask",
-    "get_template",
-    "get_xfm",
-    "utils",
-]
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)

junifer/data/__init__.pyi

@@ -0,0 +1,29 @@
+__all__ = [
+    "CoordinatesRegistry",
+    "ParcellationRegistry",
+    "MaskRegistry",
+    "get_data",
+    "list_data",
+    "load_data",
+    "register_data",
+    "deregister_data",
+    "get_template",
+    "get_xfm",
+    "utils",
+]
+
+from .coordinates import CoordinatesRegistry
+from .parcellations import ParcellationRegistry
+from .masks import MaskRegistry
+
+from ._dispatch import (
+    get_data,
+    list_data,
+    load_data,
+    register_data,
+    deregister_data,
+)
+
+from .template_spaces import get_template, get_xfm
+
+from . import utils