junifer 0.0.5.dev240__py3-none-any.whl → 0.0.6__py3-none-any.whl

junifer/typing/__init__.pyi

@@ -0,0 +1,31 @@
+__all__ = [
+    "DataGrabberLike",
+    "PreprocessorLike",
+    "MarkerLike",
+    "StorageLike",
+    "PipelineComponent",
+    "Dependencies",
+    "ConditionalDependencies",
+    "ExternalDependencies",
+    "MarkerInOutMappings",
+    "DataGrabberPatterns",
+    "ConfigVal",
+    "Element",
+    "Elements",
+]
+
+from ._typing import (
+    DataGrabberLike,
+    PreprocessorLike,
+    MarkerLike,
+    StorageLike,
+    PipelineComponent,
+    Dependencies,
+    ConditionalDependencies,
+    ExternalDependencies,
+    MarkerInOutMappings,
+    DataGrabberPatterns,
+    ConfigVal,
+    Element,
+    Elements,
+)

junifer/typing/_typing.py

@@ -0,0 +1,68 @@
+"""Provide type hints for internal and external use."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+from collections.abc import MutableMapping, Sequence
+from typing import (
+    TYPE_CHECKING,
+    Union,
+)
+
+
+if TYPE_CHECKING:
+    from ..datagrabber import BaseDataGrabber
+    from ..datareader import DefaultDataReader
+    from ..markers import BaseMarker
+    from ..preprocess import BasePreprocessor
+    from ..storage import BaseFeatureStorage
+
+
+__all__ = [
+    "ConditionalDependencies",
+    "ConfigVal",
+    "DataGrabberLike",
+    "DataGrabberPatterns",
+    "Dependencies",
+    "Element",
+    "Elements",
+    "ExternalDependencies",
+    "MarkerInOutMappings",
+    "MarkerLike",
+    "PipelineComponent",
+    "PreprocessorLike",
+    "StorageLike",
+]
+
+
+DataGrabberLike = type["BaseDataGrabber"]
+PreprocessorLike = type["BasePreprocessor"]
+MarkerLike = type["BaseMarker"]
+StorageLike = type["BaseFeatureStorage"]
+PipelineComponent = Union[
+    "DataGrabberLike",
+    "DefaultDataReader",
+    "PreprocessorLike",
+    "MarkerLike",
+    "StorageLike",
+]
+Dependencies = set[str]
+ConditionalDependencies = Sequence[
+    MutableMapping[
+        str,
+        Union[
+            str,
+            PipelineComponent,
+            Sequence[str],
+            Sequence[PipelineComponent],
+        ],
+    ]
+]
+ExternalDependencies = Sequence[MutableMapping[str, Union[str, Sequence[str]]]]
+MarkerInOutMappings = MutableMapping[str, MutableMapping[str, str]]
+DataGrabberPatterns = dict[
+    str, Union[dict[str, str], Sequence[dict[str, str]]]
+]
+ConfigVal = Union[bool, int, float, str]
+Element = Union[str, tuple[str, ...]]
+Elements = Sequence[Element]

junifer/utils/__init__.py

@@ -4,17 +4,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
-from .fs import make_executable
-from .logging import configure_logging, logger, raise_error, warn_with_log
-from .helpers import run_ext_cmd, deep_update
+import lazy_loader as lazy
 
 
-__all__ = [
-    "make_executable",
-    "configure_logging",
-    "logger",
-    "raise_error",
-    "warn_with_log",
-    "run_ext_cmd",
-    "deep_update",
-]
+__getattr__, __dir__, __all__ = lazy.attach_stub(__name__, __file__)

junifer/utils/__init__.pyi

@@ -0,0 +1,18 @@
+__all__ = [
+    "make_executable",
+    "configure_logging",
+    "config",
+    "logger",
+    "raise_error",
+    "warn_with_log",
+    "run_ext_cmd",
+    "deep_update",
+    "yaml",
+    "ConfigManager",
+]
+
+from .fs import make_executable
+from .logging import configure_logging, logger, raise_error, warn_with_log
+from ._config import config, ConfigManager
+from .helpers import run_ext_cmd, deep_update
+from ._yaml import yaml

junifer/utils/_config.py

@@ -0,0 +1,110 @@
+"""Provide junifer global configuration."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+#          Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+import os
+from typing import Optional
+
+from ..typing import ConfigVal
+from .logging import logger
+from .singleton import Singleton
+
+
+__all__ = ["ConfigManager", "config"]
+
+
+class ConfigManager(metaclass=Singleton):
+    """Manage configuration parameters.
+
+    Attributes
+    ----------
+    _config : dict
+        Configuration parameters.
+
+    """
+
+    def __init__(self) -> None:
+        """Initialize the class."""
+        self._config = {}
+        # Initial setup from process env
+        self._reload()
+
+    def _reload(self) -> None:
+        """Reload env vars."""
+        for t_var in os.environ:
+            if t_var.startswith("JUNIFER_"):
+                # Set correct type
+                var_value = os.environ[t_var]
+                # bool
+                if var_value.lower() == "true":
+                    var_value = True
+                elif var_value.lower() == "false":
+                    var_value = False
+                # numeric
+                else:
+                    try:
+                        var_value = int(var_value)
+                    except ValueError:
+                        try:
+                            var_value = float(var_value)
+                        except ValueError:
+                            pass
+                # Set value
+                var_name = (
+                    t_var.replace("JUNIFER_", "").lower().replace("_", ".")
+                )
+                logger.debug(
+                    f"Setting `{var_name}` from environment to "
+                    f"`{var_value}` (type: {type(var_value)})"
+                )
+                self._config[var_name] = var_value
+
+    def get(self, key: str, default: Optional[ConfigVal] = None) -> ConfigVal:
+        """Get configuration parameter.
+
+        Parameters
+        ----------
+        key : str
+            The configuration key to get.
+        default : bool or int or float or None, optional
+            The default value to return if the key is not found (default None).
+
+        Returns
+        -------
+        bool or int or float
+            The configuration value.
+
+        """
+        return self._config.get(key, default)
+
+    def set(self, key: str, val: ConfigVal) -> None:
+        """Set configuration parameter.
+
+        Parameters
+        ----------
+        key : str
+            The configuration key to set.
+        val : bool or int or float
+            The value to set ``key`` to.
+
+        """
+        logger.debug(f"Setting `{key}` to `{val}` (type: {type(val)})")
+        self._config[key] = val
+
+    def delete(self, key: str) -> None:
+        """Delete configuration parameter.
+
+        Parameters
+        ----------
+        key : str
+            The configuration key to delete.
+
+        """
+        logger.debug(f"Deleting `{key}` from config")
+        _ = self._config.pop(key)
+
+
+# Initialize here to access from anywhere
+config = ConfigManager()

junifer/utils/_yaml.py

@@ -0,0 +1,16 @@
+"""Provide YAML config definition for junifer use."""
+
+# Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+from ruamel.yaml import YAML
+
+
+__all__ = ["yaml"]
+
+
+# Configure YAML class once for further use
+yaml = YAML()
+yaml.default_flow_style = False
+yaml.allow_unicode = True
+yaml.indent(mapping=2, sequence=4, offset=2)

junifer/utils/helpers.py

@@ -5,15 +5,15 @@
 
 import collections.abc
 import subprocess
-from typing import Dict, List
+import sys
 
 from .logging import logger, raise_error
 
 
-__all__ = ["run_ext_cmd", "deep_update"]
+__all__ = ["deep_update", "run_ext_cmd"]
 
 
-def run_ext_cmd(name: str, cmd: List[str]) -> None:
+def run_ext_cmd(name: str, cmd: list[str]) -> None:
     """Run external command via subprocess.
 
     Parameters
@@ -45,19 +45,19 @@ def run_ext_cmd(name: str, cmd: List[str]) -> None:
     if process.returncode == 0:
         logger.info(
             f"{name} command succeeded with the following output:\n"
-            f"{process.stdout}"
+            f"{process.stdout.decode(sys.stdout.encoding)}"
         )
     else:
         raise_error(
             msg=(
                 f"{name} command failed with the following error:\n"
-                f"{process.stdout}"
+                f"{process.stdout.decode(sys.stdout.encoding)}"
             ),
             klass=RuntimeError,
         )
 
 
-def deep_update(d: Dict, u: Dict) -> Dict:
+def deep_update(d: dict, u: dict) -> dict:
     """Deep update `d` with `u`.
 
     From: "https://stackoverflow.com/questions/3232943/update-value-of-a-nested

junifer/utils/logging.py

@@ -4,6 +4,7 @@
 #          Synchon Mandal <s.mandal@fz-juelich.de>
 # License: AGPL
 
+import os
 import sys
 
 
@@ -16,7 +17,7 @@ import logging
 import warnings
 from pathlib import Path
 from subprocess import PIPE, Popen, TimeoutExpired
-from typing import Dict, NoReturn, Optional, Type, Union
+from typing import ClassVar, NoReturn, Optional, Union
 from warnings import warn
 
 import datalad
@@ -24,9 +25,9 @@ import datalad
 
 __all__ = [
     "WrapStdOut",
+    "configure_logging",
     "get_versions",
     "log_versions",
-    "configure_logging",
     "raise_error",
     "warn_with_log",
 ]
@@ -80,6 +81,59 @@ class WrapStdOut(logging.StreamHandler):
             raise AttributeError(f"'file' object has not attribute '{name}'")
 
 
+class ColorFormatter(logging.Formatter):
+    """Color formatter for logging messages.
+
+    Parameters
+    ----------
+    fmt : str
+        The format string for the logging message.
+    datefmt : str, optional
+        The format string for the date.
+
+    """
+
+    BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)
+
+    COLORS: ClassVar[dict[str, int]] = {
+        "WARNING": YELLOW,
+        "INFO": GREEN,
+        "DEBUG": BLUE,
+        "CRITICAL": MAGENTA,
+        "ERROR": RED,
+    }
+
+    RESET_SEQ: str = "\033[0m"
+    COLOR_SEQ: str = "\033[1;%dm"
+    BOLD_SEQ: str = "\033[1m"
+
+    def __init__(self, fmt: str, datefmt: Optional[str] = None) -> None:
+        """Initialize the ColorFormatter."""
+        logging.Formatter.__init__(self, fmt, datefmt)
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Format the log record.
+
+        Parameters
+        ----------
+        record : logging.LogRecord
+            The log record to format.
+
+        Returns
+        -------
+        str
+            The formatted log record.
+
+        """
+        levelname = record.levelname
+        if levelname in self.COLORS:
+            levelname_color = (
+                self.COLOR_SEQ % (30 + self.COLORS[levelname]) + levelname
+            )
+            record.levelname = levelname_color + self.RESET_SEQ
+        return logging.Formatter.format(self, record)
+
+
 def _get_git_head(path: Path) -> str:
     """Aux function to read HEAD from git.
 
@@ -119,7 +173,7 @@ def _get_git_head(path: Path) -> str:
     return proc_stdout
 
 
-def get_versions() -> Dict:
+def get_versions() -> dict:
     """Import stuff and get versions if module.
 
     Returns
@@ -189,7 +243,7 @@ def _close_handlers(logger: logging.Logger) -> None:
             logger.removeHandler(handler)
 
 
-def _safe_log(versions: Dict, name: str) -> None:
+def _safe_log(versions: dict, name: str) -> None:
     """Log with safety.
 
     Parameters
@@ -237,11 +291,51 @@ def log_versions(tbox_path: Optional[Path] = None) -> None:
         pass
 
 
+def _can_use_color(handler: logging.Handler) -> bool:
+    """Check if color can be used in the logging output.
+
+    Parameters
+    ----------
+    handler : logging.Handler
+        The logging handler to check for color support.
+
+    Returns
+    -------
+    bool
+        Whether color can be used in the logging output.
+
+    """
+    if isinstance(handler, logging.FileHandler):
+        # Do not use colors in file handlers
+        return False
+    else:
+        stream = handler.stream
+        if hasattr(stream, "isatty") and stream.isatty():
+            valid_terms = [
+                "xterm-256color",
+                "xterm-kitty",
+                "xterm-color",
+            ]
+            this_term = os.getenv("TERM", None)
+            if this_term is not None:
+                if this_term in valid_terms:
+                    return True
+                if this_term.endswith("256color") or this_term.endswith("256"):
+                    return True
+                if this_term == "dumb" and os.getenv("CI", False):
+                    return True
+            if os.getenv("COLORTERM", False):
+                return True
+        # No TTY, no color
+        return False
+
+
 def configure_logging(
     level: Union[int, str] = "WARNING",
     fname: Optional[Union[str, Path]] = None,
     overwrite: Optional[bool] = None,
     output_format=None,
+    level_datalad: Union[int, str, None] = None,
 ) -> None:
     """Configure the logging functionality.
 
@@ -264,6 +358,10 @@ def configure_logging(
         e.g., ``"%(asctime)s - %(levelname)s - %(message)s"``.
         If None, default string format is used
         (default ``"%(asctime)s - %(name)s - %(levelname)s - %(message)s"``).
+    level_datalad : int or {"DEBUG", "INFO", "WARNING", "ERROR"}, optional
+        The level of the messages to print for datalad. If string, it will be
+        interpreted as elements of logging. If None, it will be set as the
+        ``level`` parameter (default None).
 
     """
     _close_handlers(logger)  # close relevant logger handlers
@@ -297,18 +395,29 @@ def configure_logging(
         #     "%(asctime)s [%(levelname)s] %(message)s "
         #     "(%(filename)s:%(lineno)s)"
         # )
-    formatter = logging.Formatter(fmt=output_format)
+    if _can_use_color(lh):
+        formatter = ColorFormatter(fmt=output_format)
+    else:
+        formatter = logging.Formatter(fmt=output_format)
 
     lh.setFormatter(formatter)  # set formatter
     logger.setLevel(level)  # set level
-    datalad.log.lgr.setLevel(level)  # set level for datalad
+
+    # Set datalad logging level accordingly
+    if level_datalad is not None:
+        if isinstance(level_datalad, str):
+            level_datalad = _logging_types[level_datalad]
+    else:
+        level_datalad = level
+    datalad.log.lgr.setLevel(level_datalad)  # set level for datalad
+
     logger.addHandler(lh)  # set handler
     log_versions()  # log versions of installed packages
 
 
 def raise_error(
     msg: str,
-    klass: Type[Exception] = ValueError,
+    klass: type[Exception] = ValueError,
     exception: Optional[Exception] = None,
 ) -> NoReturn:
     """Raise error, but first log it.
@@ -331,7 +440,7 @@ def raise_error(
 
 
 def warn_with_log(
-    msg: str, category: Optional[Type[Warning]] = RuntimeWarning
+    msg: str, category: Optional[type[Warning]] = RuntimeWarning
 ) -> None:
     """Warn, but first log it.
 

junifer/{pipeline → utils}/singleton.py

@@ -1,15 +1,17 @@
 """Provide a singleton class to be used by pipeline components."""
 
 # Authors: Synchon Mandal <s.mandal@fz-juelich.de>
+#          Federico Raimondo <f.raimondo@fz-juelich.de>
 # License: AGPL
 
-from typing import Any, Dict, Type
+from abc import ABCMeta
+from typing import Any, ClassVar
 
 
-__all__ = ["singleton"]
+__all__ = ["ABCSingleton", "Singleton"]
 
 
-def singleton(cls: Type) -> Type:
+class Singleton(type):
     """Make a class singleton.
 
     Parameters
@@ -17,15 +19,11 @@ def singleton(cls: Type) -> Type:
     cls : class
         The class to designate as singleton.
 
-    Returns
-    -------
-    class
-        The only instance of the class.
-
     """
-    instances: Dict = {}
 
-    def get_instance(*args: Any, **kwargs: Any) -> Type:
+    instances: ClassVar[dict] = {}
+
+    def __call__(cls, *args: Any, **kwargs: Any) -> type:
         """Get the only instance for a class.
 
         Parameters
@@ -41,8 +39,15 @@ def singleton(cls: Type) -> Type:
             The only instance of the class.
 
         """
-        if cls not in instances:
-            instances[cls] = cls(*args, **kwargs)
-        return instances[cls]
+        if cls not in cls.instances:
+            cls.instances[cls] = super(Singleton, cls).__call__(  # noqa: UP008
+                *args, **kwargs
+            )
+
+        return cls.instances[cls]
+
+
+class ABCSingleton(ABCMeta, Singleton):
+    """Make an abstract class a singleton."""
 
-    return get_instance
+    pass

junifer/utils/tests/test_config.py

@@ -0,0 +1,59 @@
+"""Provide tests for ConfigManager."""
+
+# Authors: Federico Raimondo <f.raimondo@fz-juelich.de>
+#          Synchon Mandal <s.mandal@fz-juelich.de>
+# License: AGPL
+
+import os
+
+import pytest
+
+from junifer.typing import ConfigVal
+from junifer.utils import config
+from junifer.utils._config import ConfigManager
+
+
+def test_config_manager_singleton() -> None:
+    """Test that ConfigManager is a singleton."""
+    config_mgr_1 = ConfigManager()
+    config_mgr_2 = ConfigManager()
+    assert id(config_mgr_1) == id(config_mgr_2)
+
+
+def test_config_manager() -> None:
+    """Test config operations for ConfigManager."""
+    # Get non-existing with default
+    assert config.get(key="scooby") is None
+    # Set
+    config.set(key="scooby", val=True)
+    # Get existing
+    assert config.get("scooby")
+    # Delete
+    config.delete("scooby")
+    # Get non-existing with default
+    assert config.get(key="scooby") is None
+
+
+@pytest.mark.parametrize(
+    "val, expected_val",
+    [("TRUE", True), ("FALSE", False), ("1", 1), ("0.0", 0.0)],
+)
+def test_config_manager_env_reload(val: str, expected_val: ConfigVal) -> None:
+    """Test config parsing from env reload.
+
+    Parameters
+    ----------
+    val : str
+        The parametrized values.
+    expected_val : bool or int or float
+        The parametrized expected value.
+
+    """
+    # Set env var
+    os.environ["JUNIFER_TESTME"] = val
+    # Check
+    config._reload()
+    assert config.get("testme") == expected_val
+    # Cleanup
+    del os.environ["JUNIFER_TESTME"]
+    config.delete("testme")