pythonwrench 0.6.4__py3-none-any.whl

pythonwrench/importlib.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import importlib
+import json
+import logging
+import sys
+from functools import wraps
+from importlib.metadata import Distribution, PackageNotFoundError
+from importlib.util import find_spec
+from types import ModuleType
+from typing import Any, Callable, Dict, Iterable, List, Union
+
+from typing_extensions import ParamSpec, TypeVar
+
+from pythonwrench.warnings import warn_once
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+_DEFAULT_SKIPPED = (
+    "reimport_all",
+    "get_ipython",
+    "exit",
+    "quit",
+    "__name__",
+    "__doc__",
+    "__package__",
+    "__loader__",
+    "__spec__",
+    "__builtin__",
+    "__builtins__",
+)
+
+logger = logging.getLogger(__name__)
+
+
+def is_available_package(package: str) -> bool:
+    """Returns True if package is installed in the current python environment."""
+    if "-" in package:
+        msg = f"Found character '-' in package name '{package}'. (it will be replaced by '_')"
+        warn_once(msg)
+        package = package.replace("-", "_")
+
+    try:
+        return find_spec(package) is not None
+    except AttributeError:
+        # Old support for Python <= 3.6
+        return False
+    except (ImportError, ModuleNotFoundError):
+        # Python >= 3.7
+        return False
+
+
+def is_editable_package(package: str) -> bool:
+    """Returns True if package is installed in editable mode in the current python environment."""
+    package = package.replace("_", "-")
+    try:
+        direct_url = Distribution.from_name(package).read_text("direct_url.json")
+    except PackageNotFoundError:
+        return False
+    if direct_url is None:
+        return False
+    editable = json.loads(direct_url).get("dir_info", {}).get("editable", False)
+    return editable
+
+
+def search_submodules(
+    root: ModuleType,
+    only_editable: bool = True,
+    only_loaded: bool = False,
+) -> List[ModuleType]:
+    """Return the submodules already imported."""
+
+    def _impl(
+        root: ModuleType,
+        accumulator: Dict[ModuleType, None],
+    ) -> Dict[ModuleType, None]:
+        """Perform the impl operation."""
+        attrs = [getattr(root, attr_name) for attr_name in dir(root)]
+        submodules = [
+            attr
+            for attr in attrs
+            if isinstance(attr, ModuleType) and attr not in accumulator
+        ]
+        submodules = {
+            submodule
+            for submodule in submodules
+            if (
+                (
+                    not only_editable
+                    or is_editable_package(submodule.__name__.split(".")[0])
+                )
+                and (not only_loaded or submodule.__name__ in sys.modules)
+            )
+        }
+        accumulator.update(dict.fromkeys(submodules))
+
+        for submodule in submodules:
+            accumulator = _impl(submodule, accumulator)
+        return accumulator
+
+    submodules = _impl(root, {root: None})
+    submodules = list(submodules)
+    submodules = submodules[::-1]
+    return submodules
+
+
+def reload_submodules(
+    *modules: ModuleType,
+    verbose: int = 0,
+    only_editable: bool = True,
+    only_loaded: bool = False,
+) -> List[ModuleType]:
+    """Reload all submodule recursively."""
+    candidates: Dict[ModuleType, None] = {}
+    for module in modules:
+        submodules = search_submodules(
+            module,
+            only_editable=only_editable,
+            only_loaded=only_loaded,
+        )
+        candidates.update(dict.fromkeys(submodules))
+
+    for candidate in candidates:
+        if verbose > 0:
+            logger.info(f"Reload '{candidate}'...")
+        try:
+            importlib.reload(candidate)
+        except ModuleNotFoundError as err:
+            msg = f"ModuleNotFound: did this module '{candidate.__name__}' has been renamed after starting execution?"
+            logger.warning(msg)
+            raise err
+
+    return list(candidates)
+
+
+def reload_editable_packages(*, verbose: int = 0) -> List[ModuleType]:
+    """Reload all submodules of editable packages already imported."""
+    pkg_names = {name.split(".")[0] for name in sys.modules.keys()}
+    editable_packages = [
+        sys.modules[name] for name in pkg_names if is_editable_package(name)
+    ]
+    if verbose >= 2:
+        msg = f"{len(editable_packages)}/{len(pkg_names)} editable packages found: {editable_packages}"
+        logger.debug(msg)
+
+    return reload_submodules(
+        *editable_packages,
+        verbose=verbose,
+        only_editable=True,
+        only_loaded=False,
+    )
+
+
+def requires_packages(
+    arg0: Union[Iterable[str], str],
+    /,
+    *args: str,
+) -> Callable[[Callable[P, T]], Callable[P, T]]:
+    """Decorator to wrap a function and raises an error if the function is called.
+
+    Example
+    -------
+    >>> @requires_packages("pandas")
+    >>> def f(x):
+    >>>     return x
+    >>> f(1)  # raises ImportError if pandas is not installed
+    """
+    if isinstance(arg0, str):
+        packages = [arg0] + list(args)
+    elif isinstance(arg0, Iterable):
+        packages = list(arg0) + list(args)
+    else:
+        raise TypeError(f"Invalid arguments types {(arg0,) + args}.")
+
+    def _wrap(fn: Callable[P, T]) -> Callable[P, T]:
+        """Perform the wrap operation."""
+
+        @wraps(fn)
+        def _impl(*args: P.args, **kwargs: P.kwargs) -> T:
+            """Perform the impl operation."""
+            missing = [pkg for pkg in packages if not is_available_package(pkg)]
+            if len(missing) == 0:
+                return fn(*args, **kwargs)
+            else:
+                prefix = "\n - "
+                missing_str = prefix.join(missing)
+                msg = (
+                    f"Cannot use/import objects because the following optionals dependencies are missing:"
+                    f"{prefix}{missing_str}\n"
+                )
+                raise ImportError(msg)
+
+        return _impl
+
+    return _wrap
+
+
+class Placeholder:
+    """Placeholder object. All instances attributes always returns the object itself."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        """Initialize the instance."""
+        super().__init__()
+        self.__excluded_self_attrs = [
+            "__file__",
+            "__eq__",
+            "__ne__",
+            "__str__",
+            "__repr__",
+        ]
+
+    def __getattr__(self, name: str) -> Any:
+        """Return a dynamically imported attribute."""
+        if name in self.__excluded_self_attrs:
+            return self.__getattribute__(name)
+        else:
+            return self
+
+    def __call__(self, *args, **kwargs) -> Any:
+        """Call the instance."""
+        return self
+
+    def __getitem__(self, *args, **kwargs) -> Any:
+        """Return the item at the requested index or key."""
+        return self
+
+
+class ModulePlaceholder(ModuleType, Placeholder):
+    def __init__(self, *args, **kwargs) -> None:
+        """Initialize the instance."""
+        Placeholder.__init__(self, *args, **kwargs)
+        ModuleType.__init__(self, *args, **kwargs)
+
+
+def import_if_available(name: str) -> ModuleType:
+    """Perform the import if available operation."""
+    if is_available_package(name):
+        return __import__(name)
+    else:
+        return ModulePlaceholder(name)

pythonwrench/inspect.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import inspect
+from typing import Any, Callable, List, TypeVar, Union, get_args
+
+T = TypeVar("T")
+
+
+def get_argnames(fn: Callable) -> List[str]:
+    """Get arguments names of a method, function or callable object. This function does not return the 'self' argument for methods."""
+    spec = inspect.getfullargspec(fn)
+    all_args = spec.args + spec.kwonlyargs
+    if inspect.ismethod(fn) or inspect.isclass(fn):
+        return all_args[1:]
+    else:
+        return all_args
+
+
+def get_current_fn_name(*, default: T = "") -> Union[str, T]:
+    """Get caller function name."""
+    try:
+        return inspect.currentframe().f_back.f_code.co_name  # type: ignore
+    except AttributeError:
+        return default
+
+
+def get_fullname(x: Any, *, inst_suffix: str = "(...)") -> str:
+    """Returns the classname of an object with parent modules.
+
+    Args:
+        obj: Object to scan.
+        inst_suffix: Suffix appended to the classname in case the object is an instance of a class.
+
+    Examples
+    --------
+    >>> get_fullname([0, 1, 2])
+    ... 'builtins.list(...)'
+    >>> get_fullname(1.0)
+    ... 'builtins.float(...)'
+    >>> class A: def f(self): return 0
+    >>> a = A()
+    >>> get_fullname(a)
+    ... '__main__.A(...)'
+    >>> get_fullname(A)
+    ... '__main__.A'
+    >>> get_fullname(a.f)
+    ... '__main__.A.f'
+    >>> get_fullname(A.f)
+    ... '__main__.A.f'
+    """
+    if hasattr(x, "__module__") and hasattr(x, "__qualname__"):
+        if x.__module__ is None:
+            name = f"{x.__qualname__}"
+        else:
+            name = f"{x.__module__}.{x.__qualname__}"
+    elif inspect.ismodule(x):
+        name = x.__name__
+    else:
+        cls = x.__class__
+        name = f"{cls.__module__}.{cls.__qualname__}{inst_suffix}"
+
+    cls_args = get_args(x)
+    if len(cls_args) != 0:
+        argsnames = [get_fullname(arg, inst_suffix=inst_suffix) for arg in cls_args]
+        argsnames_str = ", ".join(argsnames)
+        name = f"{name}[{argsnames_str}]"
+
+    return name

pythonwrench/json.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# for backward compatibility
+from pythonwrench.serialization.json import (  # noqa: F401
+    dump_json,
+    dumps_json,
+    load_json,
+    loads_json,
+    read_json,
+    save_json,
+)

pythonwrench/jsonl.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# for backward compatibility
+from pythonwrench.serialization.jsonl import (  # noqa: F401
+    dump_jsonl,
+    dumps_jsonl,
+    load_jsonl,
+    loads_jsonl,
+    read_jsonl,
+    save_jsonl,
+)

pythonwrench/logging.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import inspect
+import logging
+import sys
+from functools import lru_cache
+from logging import FileHandler, Formatter, Logger, StreamHandler
+from pathlib import Path
+from types import ModuleType
+from typing import IO, List, Literal, Optional, TypeVar, Union
+
+from typing_extensions import TypeAlias
+
+from pythonwrench.importlib import reload_submodules
+from pythonwrench.semver import Version
+from pythonwrench.typing import SupportsIterLen
+
+T = TypeVar("T", covariant=True)
+
+PackageOrLogger: TypeAlias = Union[
+    str,
+    ModuleType,
+    None,
+    Logger,
+    Literal["__parent_file__"],
+]
+PackageOrLoggerList: TypeAlias = Union[
+    PackageOrLogger,
+    SupportsIterLen[PackageOrLogger],
+]
+
+_PARENT_FILE_KEY = "__parent_file__"
+DEFAULT_FMT = "[%(asctime)s][%(name)s][%(levelname)s] - %(message)s"
+VERBOSE_DEBUG = 2
+VERBOSE_INFO = 1
+VERBOSE_WARNING = 0
+VERBOSE_ERROR = -1
+
+logger = logging.getLogger(__name__)
+
+
+@lru_cache(maxsize=None)
+def log_once(
+    msg: str,
+    logger: PackageOrLoggerList = _PARENT_FILE_KEY,
+    *,
+    level: int = logging.INFO,
+) -> None:
+    """Log message to loggers at the specified level."""
+    loggers = _get_loggers(logger)
+    for logger in loggers:
+        logger.log(level, msg)
+
+
+def setup_logging_verbose(
+    package_or_logger: PackageOrLoggerList = None,
+    verbose: Optional[int] = VERBOSE_INFO,
+    *,
+    fmt: Union[str, None, Formatter] = DEFAULT_FMT,
+    stream: Union[IO[str], Literal["auto"]] = "auto",
+    set_fmt: bool = True,
+    capture_warnings: bool = True,
+    autoreload: bool = True,
+) -> None:
+    """Helper function to customize logging messages using verbose_level.
+
+    Note: Higher verbose values means more debug messages.
+    """
+    if verbose is None:
+        level = None
+    else:
+        level = _verbose_to_logging_level(verbose)
+
+    return setup_logging_level(
+        package_or_logger,
+        level=level,
+        fmt=fmt,
+        stream=stream,
+        set_fmt=set_fmt,
+        capture_warnings=capture_warnings,
+        autoreload=autoreload,
+    )
+
+
+def setup_logging_level(
+    package_or_logger: PackageOrLoggerList = None,
+    level: Optional[int] = logging.INFO,
+    *,
+    fmt: Union[str, None, Formatter] = DEFAULT_FMT,
+    stream: Union[IO[str], Literal["auto"]] = "auto",
+    set_fmt: bool = True,
+    capture_warnings: bool = True,
+    autoreload: bool = True,
+) -> None:
+    """Helper function to customize logging messages using logging.level.
+
+    Note: Lower level values means more debug messages.
+    """
+    logging.captureWarnings(capture_warnings)
+
+    logger_lst = _get_loggers(package_or_logger)
+    if isinstance(fmt, str):
+        fmt = Formatter(fmt)
+
+    if stream == "auto":
+        if running_on_interpreter():
+            stream = sys.stdout
+        else:
+            stream = sys.stderr
+
+    for logger in logger_lst:
+        if set_fmt:
+            found = False
+
+            for handler in logger.handlers:
+                if isinstance(handler, StreamHandler) and handler.stream is stream:
+                    handler.setFormatter(fmt)
+                    found = True
+                    break
+
+            if not found:
+                handler = StreamHandler(stream)  # type: ignore
+                handler.setFormatter(fmt)
+                logger.addHandler(handler)
+
+        if level is not None:
+            logger.setLevel(level)
+
+    if autoreload:
+        for logger in logger_lst:
+            if logger.name not in sys.modules:
+                continue
+            reload_submodules(sys.modules[logger.name])
+
+
+def running_on_interpreter() -> bool:
+    """Return whether running on interpreter."""
+    return get_ipython_name() is None
+
+
+def running_on_terminal() -> bool:
+    """Return whether running on terminal."""
+    return get_ipython_name() == "TerminalInteractiveShell"
+
+
+def running_on_notebook() -> bool:
+    """Return whether running on notebook."""
+    return get_ipython_name() == "ZMQInteractiveShell"
+
+
+def get_ipython_name() -> Optional[
+    Literal["TerminalInteractiveShell", "ZMQInteractiveShell"]
+]:
+    """Return ipython name."""
+    try:
+        return get_ipython().__class__.__name__  # type: ignore
+    except NameError:
+        return None
+
+
+def get_current_file_logger(
+    *,
+    parent_deep: int = 1,
+    default: T = logging.root,
+) -> Union[Logger, T]:
+    """Returns the logger of the caller file. If this cannot be found, returns the root logger."""
+    try:
+        frame = inspect.currentframe()
+        for _ in range(parent_deep):
+            frame = frame.f_back  # type: ignore
+        parent_name = frame.f_globals["__name__"]  # type: ignore
+        return logging.getLogger(parent_name)
+    except (AttributeError, KeyError):
+        return default
+
+
+@lru_cache(maxsize=None)
+def get_null_logger() -> Logger:
+    """Return null logger."""
+    logger = logging.getLogger("null_logger")
+    logger.addHandler(logging.NullHandler())
+    logger.setLevel(logging.CRITICAL + 1)
+    return logger
+
+
+def _get_loggers(pkg_name_log_arg: PackageOrLoggerList) -> List[Logger]:
+    """Perform the get loggers operation."""
+    if pkg_name_log_arg is None or isinstance(
+        pkg_name_log_arg, (str, Logger, ModuleType)
+    ):
+        pkg_name_log_lst = [pkg_name_log_arg]
+    else:
+        pkg_name_log_lst = list(pkg_name_log_arg)
+
+    loggers: List[Logger] = []
+    for pkg_name_log in pkg_name_log_lst:
+        if isinstance(pkg_name_log, ModuleType):
+            logger = logging.getLogger(pkg_name_log.__name__)
+
+        elif pkg_name_log == _PARENT_FILE_KEY:
+            logger = get_current_file_logger(parent_deep=2)
+
+        elif isinstance(pkg_name_log, (type(None), str)):
+            logger = logging.getLogger(pkg_name_log)
+
+        else:
+            logger = pkg_name_log
+
+        loggers.append(logger)
+
+    return loggers
+
+
+class MkdirFileHandler(FileHandler):
+    """FileHandler that build intermediate directories to filename.
+
+    Used for export hydra logs to a file contained in a folder that does not exists yet at the start of the program.
+    """
+
+    def __init__(
+        self,
+        filename: Union[str, Path],
+        mode: str = "a",
+        encoding: Optional[str] = None,
+        delay: bool = True,
+        errors: Optional[str] = None,
+        *,
+        mkdir_parents: bool = True,
+        mkdir_exist_ok: bool = True,
+    ) -> None:
+        """Initialize the instance."""
+        filename = Path(filename)
+        filename.parent.mkdir(parents=mkdir_parents, exist_ok=mkdir_exist_ok)
+
+        if Version.python() < Version("3.9.0"):
+            super().__init__(filename, mode, encoding, delay)
+        else:
+            super().__init__(filename, mode, encoding, delay, errors)  # type: ignore
+
+
+def _verbose_to_logging_level(verbose: int) -> int:
+    """Perform the verbose to logging level operation."""
+    if verbose <= VERBOSE_ERROR:
+        level = logging.ERROR
+    elif verbose == VERBOSE_WARNING:
+        level = logging.WARNING
+    elif verbose == VERBOSE_INFO:
+        level = logging.INFO
+    else:
+        level = logging.DEBUG
+    return level

pythonwrench/math.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import math
+import struct
+from numbers import Real
+from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar
+
+from pythonwrench.functools import compose, function_alias
+
+T = TypeVar("T")
+T_Real = TypeVar("T_Real", bound=Real)
+
+
+def clip(
+    x: T_Real,
+    xmin: Optional[T_Real] = None,
+    xmax: Optional[T_Real] = None,
+) -> T_Real:
+    """Perform the clip operation."""
+    if xmin is not None:
+        x = max(x, xmin)
+    if xmax is not None:
+        x = min(x, xmax)
+    return x
+
+
+@function_alias(clip)
+def clamp(*args, **kwargs):
+    """Perform the clamp operation."""
+    ...
+
+
+def argmax(x: Iterable) -> int:
+    """Perform the argmax operation."""
+    max_index, _max_value = max(enumerate(x), key=lambda t: t[1])
+    return max_index
+
+
+def argmin(x: Iterable) -> int:
+    """Perform the argmin operation."""
+    min_index, _max_value = min(enumerate(x), key=lambda t: t[1])
+    return min_index
+
+
+def argsort(
+    x: Iterable[T],
+    *,
+    key: Optional[Callable[[T], Any]] = None,
+    reverse: bool = False,
+) -> List[int]:
+    """Perform the argsort operation."""
+
+    def get_second(t: Tuple[int, T]) -> T:
+        """Return second."""
+        return t[1]
+
+    if key is None:
+        key_fn = get_second
+    else:
+        key_fn = compose(get_second, key)
+
+    sorted_x = sorted(enumerate(x), key=key_fn, reverse=reverse)  # type: ignore
+    indices = [idx for idx, _ in sorted_x]
+    return indices
+
+
+def nextdown(x: float) -> float:
+    """Perform the nextdown operation."""
+    return -_nextup(-x)
+
+
+def nextafter(x: float, y: float) -> float:
+    """Equivalent to `math.nextafter` for python <=3.8."""
+
+    # BASED on https://stackoverflow.com/questions/10420848/how-do-you-get-the-next-value-in-the-floating-point-sequence/10426033#10426033
+    # If either argument is a NaN, return that argument.
+    # This matches the implementation in decimal.Decimal
+    if math.isnan(x):
+        return x
+    if math.isnan(y):
+        return y
+
+    if y == x:
+        return y
+    elif y > x:
+        return _nextup(x)
+    else:
+        return nextdown(x)
+
+
+def _nextup(x: float) -> float:
+    # NaNs and positive infinity map to themselves.
+    """Perform the nextup operation."""
+    if math.isnan(x) or (math.isinf(x) and x > 0):
+        return x
+
+    # 0.0 and -0.0 both map to the smallest +ve float.
+    if x == 0.0:
+        x = 0.0
+
+    n = struct.unpack("<q", struct.pack("<d", x))[0]
+    if n >= 0:
+        n += 1
+    else:
+        n -= 1
+    return struct.unpack("<d", struct.pack("<q", n))[0]