lsst-utils 25.2023.2800__py3-none-any.whl → 29.2025.4800__py3-none-any.whl

lsst/utils/timer.py

@@ -10,25 +10,29 @@
 # license that can be found in the LICENSE file.
 #
 
-"""Utilities for measuring execution time.
-"""
+"""Utilities for measuring execution time."""
 
 from __future__ import annotations
 
-__all__ = ["profile", "logInfo", "timeMethod", "time_this"]
+__all__ = ["duration_from_timeMethod", "logInfo", "profile", "timeMethod", "time_this"]
 
+import dataclasses
 import datetime
 import functools
 import logging
+import os
+import platform
+import sys
 import time
+import traceback
 from collections.abc import Callable, Collection, Iterable, Iterator, MutableMapping
-from contextlib import contextmanager
-from typing import TYPE_CHECKING, Any
+from contextlib import contextmanager, suppress
+from typing import TYPE_CHECKING, Any, TypeVar, overload
 
 from astropy import units as u
 
 from .introspection import find_outside_stacklevel
-from .logging import LsstLoggers
+from .logging import LsstLoggers, _is_structlog_logger
 from .usage import _get_current_rusage, get_current_mem_usage, get_peak_mem_usage
 
 if TYPE_CHECKING:
@@ -75,10 +79,10 @@ def logPairs(
     pairs: Collection[tuple[str, Any]],
     logLevel: int = logging.DEBUG,
     metadata: MutableMapping | None = None,
-    logger: logging.Logger | None = None,
+    logger: LsstLoggers | None = None,
     stacklevel: int | None = None,
 ) -> None:
-    """Log ``(name, value)`` pairs to ``obj.metadata`` and ``obj.log``
+    """Log ``(name, value)`` pairs to ``obj.metadata`` and ``obj.log``.
 
     Parameters
     ----------
@@ -98,7 +102,7 @@ def logPairs(
         Log level (an `logging` level constant, such as `logging.DEBUG`).
     metadata : `collections.abc.MutableMapping`, optional
         Metadata object to write entries to.  Ignored if `None`.
-    logger : `logging.Logger`
+    logger : `logging.Logger` or `lsst.utils.logging.LsstLogAdapter`
         Log object to write entries to.  Ignored if `None`.
     stacklevel : `int`, optional
         The stack level to pass to the logger to adjust which stack frame
@@ -109,15 +113,13 @@ def logPairs(
     """
     if obj is not None:
         if metadata is None:
-            try:
+            with suppress(AttributeError):
                 metadata = obj.metadata
-            except AttributeError:
-                pass
+
         if logger is None:
-            try:
+            with suppress(AttributeError):
                 logger = obj.log
-            except AttributeError:
-                pass
+
     strList = []
     for name, value in pairs:
         if metadata is not None:
@@ -142,7 +144,7 @@ def logInfo(
     prefix: str,
     logLevel: int = logging.DEBUG,
     metadata: MutableMapping | None = None,
-    logger: logging.Logger | None = None,
+    logger: LsstLoggers | None = None,
     stacklevel: int | None = None,
 ) -> None:
     """Log timer information to ``obj.metadata`` and ``obj.log``.
@@ -167,7 +169,7 @@ def logInfo(
         Log level (a `logging` level constant, such as `logging.DEBUG`).
     metadata : `collections.abc.MutableMapping`, optional
         Metadata object to write entries to, overriding ``obj.metadata``.
-    logger : `logging.Logger`
+    logger : `logging.Logger` or `lsst.utils.logging.LsstLogAdapter`
         Log object to write entries to, overriding ``obj.log``.
     stacklevel : `int`, optional
         The stack level to pass to the logger to adjust which stack frame
@@ -197,18 +199,24 @@ def logInfo(
     * Version 1: ``MaxResidentSetSize`` will be stored in bytes.
     """
     if metadata is None and obj is not None:
-        try:
+        with suppress(AttributeError):
             metadata = obj.metadata
-        except AttributeError:
-            pass
+
     if metadata is not None:
         # Log messages already have timestamps.
-        utcStr = datetime.datetime.utcnow().isoformat()
+        if sys.version_info < (3, 11, 0):
+            now = datetime.datetime.utcnow()
+        else:
+            now = datetime.datetime.now(datetime.UTC)
+        utcStr = now.isoformat()
         _add_to_metadata(metadata, name=prefix + "Utc", value=utcStr)
 
         # Force a version number into the metadata.
         # v1: Ensure that max_rss field is always bytes.
         metadata["__version__"] = 1
+
+        # Add node information.
+        metadata["nodeName"] = platform.node()
     if stacklevel is not None:
         # Account for the caller of this routine not knowing that we
         # are going one down in the stack.
@@ -225,23 +233,41 @@ def logInfo(
     )
 
 
+_F = TypeVar("_F", bound=Callable)
+
+
+@overload
+def timeMethod(_func: _F) -> _F: ...
+
+
+@overload
+def timeMethod(
+    *,
+    metadata: MutableMapping | None = None,
+    logger: LsstLoggers | None = None,
+    logLevel: int = logging.DEBUG,
+) -> Callable[[_F], _F]: ...
+
+
 def timeMethod(
     _func: Any | None = None,
     *,
     metadata: MutableMapping | None = None,
-    logger: logging.Logger | None = None,
+    logger: LsstLoggers | None = None,
     logLevel: int = logging.DEBUG,
 ) -> Callable:
     """Measure duration of a method.
 
+    Set LSST_UTILS_DISABLE_TIMER in your environment to disable this method.
+
     Parameters
     ----------
-    func
+    _func : `~collections.abc.Callable` or `None`
         The method to wrap.
     metadata : `collections.abc.MutableMapping`, optional
         Metadata to use as override if the instance object attached
         to this timer does not support a ``metadata`` property.
-    logger : `logging.Logger`, optional
+    logger : `logging.Logger` or `lsst.utils.logging.LsstLogAdapter`, optional
         Logger to use when the class containing the decorated method does not
         have a ``log`` property.
     logLevel : `int`, optional
@@ -309,12 +335,38 @@ def timeMethod(
 
         return timeMethod_wrapper
 
+    if "LSST_UTILS_DISABLE_TIMER" in os.environ:
+        if _func is not None:
+            return _func
+        else:
+
+            def pass_through(func: Callable) -> Callable:
+                """Pass through decorator.
+
+                This decorator will not appear in the call stack.
+                """
+                return func
+
+            return pass_through
+
     if _func is None:
         return decorator_timer
     else:
         return decorator_timer(_func)
 
 
+@dataclasses.dataclass
+class _TimerResult:
+    duration: float = 0.0
+    """Duration of context in seconds."""
+    mem_current_usage: float | None = None
+    """Memory usage at end of context in requested units."""
+    mem_peak_delta: float | None = None
+    """Peak differential between entering and leaving context."""
+    mem_current_delta: float | None = None
+    """Difference in usage between entering and leaving context."""
+
+
 @contextmanager
 def time_this(
     log: LsstLoggers | None = None,
@@ -322,24 +374,27 @@ def time_this(
     level: int = logging.DEBUG,
     prefix: str | None = "timer",
     args: Iterable[Any] = (),
+    kwargs: dict[str, Any] | None = None,
     mem_usage: bool = False,
     mem_child: bool = False,
-    mem_unit: u.Quantity = u.byte,
+    mem_unit: u.Unit = u.byte,
     mem_fmt: str = ".0f",
-) -> Iterator[None]:
+    force_mem_usage: bool = False,
+) -> Iterator[_TimerResult]:
     """Time the enclosed block and issue a log message.
 
     Parameters
     ----------
     log : `logging.Logger`, optional
         Logger to use to report the timer message. The root logger will
-        be used if none is given.
+        be used if none is given. Is also allowed to be a `structlog` bound
+        logger.
     msg : `str`, optional
         Context to include in log message.
     level : `int`, optional
         Python logging level to use to issue the log message. If the
-        code block raises an exception the log message will automatically
-        switch to level ERROR.
+        code block raises an exception the log message will include some
+        information about the exception that occurred.
     prefix : `str`, optional
         Prefix to use to prepend to the supplied logger to
         create a new logger to use instead. No prefix is used if the value
@@ -347,9 +402,15 @@ def time_this(
     args : iterable of any
         Additional parameters passed to the log command that should be
         written to ``msg``.
+    kwargs : `dict`, optional
+        Additional keyword parameters passed to the log command. If a Structlog
+        is used then these will be added to the structured data. Otherwise
+        they will be converted to a single string for inclusion in the log
+        message.
     mem_usage : `bool`, optional
         Flag indicating whether to include the memory usage in the report.
-        Defaults, to False.
+        Defaults, to False. Does nothing if the log message will not be
+        generated.
     mem_child : `bool`, optional
         Flag indication whether to include memory usage of the child processes.
     mem_unit : `astropy.units.Unit`, optional
@@ -357,26 +418,49 @@ def time_this(
     mem_fmt : `str`, optional
         Format specifier to use when displaying values related to memory usage.
         Defaults to '.0f'.
+    force_mem_usage : `bool`, optional
+        If `True` memory usage is always calculated even if the log message
+        will not be delivered. Use this if you want the information recorded
+        by the context manager.
+
+    Yields
+    ------
+    timer_result : `_TimerResult`
+        Simple data class containing the duration of the block in seconds via
+        the ``duration`` property.
     """
     if log is None:
         log = logging.getLogger()
-    if prefix:
+    is_structlog = _is_structlog_logger(log)
+    if prefix and not is_structlog:
+        # Struct log loggers do not have a name property and so the prefix
+        # is not applied to them.
         log_name = f"{prefix}.{log.name}" if not isinstance(log, logging.RootLogger) else prefix
         log = logging.getLogger(log_name)
 
-    success = False
+    # Some structured data that can be used if we have been given a
+    # structlog logger.
+    structured_args: dict[str, Any] = {}
+
     start = time.time()
 
     if mem_usage and not log.isEnabledFor(level):
         mem_usage = False
+    if force_mem_usage:
+        mem_usage = True
 
     if mem_usage:
         current_usages_start = get_current_mem_usage()
         peak_usages_start = get_peak_mem_usage()
 
+    timer_result = _TimerResult()
+    errmsg = ""
     try:
-        yield
-        success = True
+        yield timer_result
+    except BaseException as e:
+        frame, lineno = list(traceback.walk_tb(e.__traceback__))[-1]
+        errmsg = f"{e!r} @ {frame.f_code.co_filename}:{lineno}"
+        raise
     finally:
         end = time.time()
 
@@ -390,15 +474,17 @@ def time_this(
         # mypy stop complaining when additional parameters will be added below.
         params = list(args) if args else []
 
-        if not success:
-            # Something went wrong so change the log level to indicate
-            # this.
-            level = logging.ERROR
+        duration = end - start
+        timer_result.duration = duration
 
         # Specify stacklevel to ensure the message is reported from the
         # caller (1 is this file, 2 is contextlib, 3 is user)
-        params += (": " if msg else "", end - start)
+        params += (": " if msg else "", duration)
         msg += "%sTook %.4f seconds"
+        structured_args["duration"] = duration
+        if errmsg:
+            params += (f" (timed code triggered exception of {errmsg!r})",)
+            msg += "%s"
         if mem_usage:
             current_usages_end = get_current_mem_usage()
             peak_usages_end = get_peak_mem_usage()
@@ -418,16 +504,37 @@ def time_this(
                 _LOG.warning("Invalid memory unit '%s', using '%s' instead", mem_unit, u.byte)
                 mem_unit = u.byte
 
+            current_usage = current_usage.to(mem_unit)
+            current_delta = current_delta.to(mem_unit)
+            peak_delta = peak_delta.to(mem_unit)
+
+            timer_result.mem_current_usage = float(current_usage.value)
+            timer_result.mem_current_delta = float(current_delta.value)
+            timer_result.mem_peak_delta = float(peak_delta.value)
+
             msg += (
-                f"; current memory usage: {current_usage.to(mem_unit):{mem_fmt}}"
-                f", delta: {current_delta.to(mem_unit):{mem_fmt}}"
-                f", peak delta: {peak_delta.to(mem_unit):{mem_fmt}}"
+                f"; current memory usage: {current_usage:{mem_fmt}}"
+                f", delta: {current_delta:{mem_fmt}}"
+                f", peak delta: {peak_delta:{mem_fmt}}"
             )
-        log.log(level, msg, *params, stacklevel=3)
+            structured_args["mem_current_usage"] = float(current_usage.value)
+            structured_args["mem_current_delta"] = float(current_delta.value)
+            structured_args["mem_peak_delta"] = float(peak_delta.value)
+        if not is_structlog:
+            # Can only use the structured content if we have structlog logger
+            # but stacklevel is only supported by standard loggers.
+            structured_args = {"stacklevel": 3}
+            if kwargs is not None:
+                msg += " %s"
+                params += ("; ".join(f"{k}={v!r}" for k, v in kwargs.items()),)
+        elif kwargs:
+            structured_args.update(kwargs)
+
+        log.log(level, msg, *params, **structured_args)
 
 
 @contextmanager
-def profile(filename: str | None, log: logging.Logger | None = None) -> Iterator[cProfile.Profile | None]:
+def profile(filename: str | None, log: LsstLoggers | None = None) -> Iterator[cProfile.Profile | None]:
     """Profile the enclosed code block and save the result to a file.
 
     Parameters
@@ -435,7 +542,7 @@ def profile(filename: str | None, log: logging.Logger | None = None) -> Iterator
     filename : `str` or `None`
         Filename to which to write profile (profiling disabled if `None` or
         empty string).
-    log : `logging.Logger`, optional
+    log : `logging.Logger` or `lsst.utils.logging.LsstLogAdapter`, optional
         Log object for logging the profile operations.
 
     Yields
@@ -476,3 +583,40 @@ def profile(filename: str | None, log: logging.Logger | None = None) -> Iterator
     profile.dump_stats(filename)
     if log is not None:
         log.info("cProfile stats written to %s", filename)
+
+
+def duration_from_timeMethod(
+    metadata: MutableMapping | None, method_name: str, clock: str = "Cpu"
+) -> float | None:
+    """Parse the metadata entries from ``timeMethod`` and return a duration.
+
+    Parameters
+    ----------
+    metadata : `collections.abc.MutableMapping`
+        The Task metadata that timing metrics were added to.
+    method_name : `str`
+        Name of the timed method to extract a duration for.
+    clock : `str`, optional
+        Options are "Cpu", "User", "System", or "Utc".
+
+    Returns
+    -------
+    duration : `float`
+        The time elapsed between the start and end of the timed method.
+    """
+    if metadata is None:
+        return None
+    if clock.lower() == "utc":
+        start = metadata[method_name + "StartUtc"]
+        end = metadata[method_name + "EndUtc"]
+    else:
+        start = metadata[method_name + "Start" + clock + "Time"]
+        end = metadata[method_name + "End" + clock + "Time"]
+    if isinstance(start, list):
+        start = start[-1]
+    if isinstance(end, list):
+        end = end[-1]
+    if isinstance(start, str) and isinstance(end, str):
+        return (datetime.datetime.fromisoformat(end) - datetime.datetime.fromisoformat(start)).total_seconds()
+    else:
+        return end - start

lsst/utils/usage.py

@@ -9,8 +9,8 @@
 # Use of this source code is governed by a 3-clause BSD-style
 # license that can be found in the LICENSE file.
 
-"""Utilities for measuring resource consumption.
-"""
+"""Utilities for measuring resource consumption."""
+
 from __future__ import annotations
 
 __all__ = ["get_current_mem_usage", "get_peak_mem_usage"]

lsst/utils/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "25.2023.2800"
+__version__ = "29.2025.4800"

lsst/utils/wrappers.py

@@ -13,10 +13,11 @@ from __future__ import annotations
 
 import sys
 import types
+from typing import Any
 
 import numpy as np
 
-__all__ = ("continueClass", "inClass", "TemplateMeta")
+__all__ = ("TemplateMeta", "continueClass", "inClass")
 
 
 INTRINSIC_SPECIAL_ATTRIBUTES = frozenset(
@@ -34,13 +35,25 @@ INTRINSIC_SPECIAL_ATTRIBUTES = frozenset(
 )
 
 
-def isAttributeSafeToTransfer(name, value):
+def isAttributeSafeToTransfer(name: str, value: Any) -> bool:
     """Return True if an attribute is safe to monkeypatch-transfer to another
     class.
 
     This rejects special methods that are defined automatically for all
     classes, leaving only those explicitly defined in a class decorated by
     `continueClass` or registered with an instance of `TemplateMeta`.
+
+    Parameters
+    ----------
+    name : `str`
+        The name of the attribute to check.
+    value : `~typing.Any`
+        The value of the attribute.
+
+    Returns
+    -------
+    `bool`
+        Whether the attribute is safe to monkeypatch-transfer.
     """
     if name.startswith("__") and (
         value is getattr(object, name, None) or name in INTRINSIC_SPECIAL_ATTRIBUTES
@@ -60,6 +73,7 @@ def continueClass(cls):
         class Foo:
             pass
 
+
         @continueClass
         class Foo:
             def run(self):
@@ -78,7 +92,6 @@ def continueClass(cls):
         Python's built-in `super` function does not behave properly in classes
         decorated with `continueClass`.  Base class methods must be invoked
         directly using their explicit types instead.
-
     """
     orig = getattr(sys.modules[cls.__module__], cls.__name__)
     for name in dir(cls):
@@ -93,9 +106,17 @@ def continueClass(cls):
     return orig
 
 
-def inClass(cls, name=None):
+def inClass(cls, name: str | None = None):
     """Add the decorated function to the given class as a method.
 
+    Parameters
+    ----------
+    name : `str` or `None`, optional
+        Name to be associated with the decorated function if the default
+        can not be determined.
+
+    Examples
+    --------
     For example:
 
     .. code-block:: python
@@ -103,6 +124,7 @@ def inClass(cls, name=None):
         class Foo:
             pass
 
+
         @inClass(Foo)
         def run(self):
             return None
@@ -115,6 +137,8 @@ def inClass(cls, name=None):
             def run(self):
                 return None
 
+    Notes
+    -----
     Standard decorators like ``classmethod``, ``staticmethod``, and
     ``property`` may be used *after* this decorator.  Custom decorators
     may only be used if they return an object with a ``__name__`` attribute
@@ -169,9 +193,11 @@ class TemplateMeta(type):
         import numpy as np
         from ._image import ImageF, ImageD
 
+
         class Image(metaclass=TemplateMeta):
             pass
 
+
         Image.register(np.float32, ImageF)
         Image.register(np.float64, ImageD)
         Image.alias("F", ImageF)
@@ -212,10 +238,10 @@ class TemplateMeta(type):
     .. code-block:: python
 
         class Image(metaclass=TemplateMeta):
-
             def sum(self):
                 return np.sum(self.getArray())
 
+
         Image.register(np.float32, ImageF)
         Image.register(np.float64, ImageD)
 
@@ -250,7 +276,6 @@ class TemplateMeta(type):
         Python's built-in `super` function does not behave properly in classes
         that have `TemplateMeta` as their metaclass (which should be rare, as
         TemplateMeta ABCs will have base classes of their own)..
-
     """
 
     def __new__(cls, name, bases, attrs):
@@ -307,20 +332,14 @@ class TemplateMeta(type):
         # any registered type or true subclass thereof.
         if subclass in cls._registry.values():
             return True
-        for v in cls._registry.values():
-            if issubclass(subclass, v):
-                return True
-        return False
+        return any(issubclass(subclass, v) for v in cls._registry.values())
 
     def __instancecheck__(cls, instance):
         # Special method hook for the isinstance built-in: we return true for
         # an instance of any registered type or true subclass thereof.
         if type(instance) in cls._registry.values():
             return True
-        for v in cls._registry.values():
-            if isinstance(instance, v):
-                return True
-        return False
+        return any(isinstance(instance, v) for v in cls._registry.values())
 
     def __subclasses__(cls):
         """Return a tuple of all classes that inherit from this class."""
@@ -329,11 +348,18 @@ class TemplateMeta(type):
         # functionality.
         return tuple(set(cls._registry.values()))
 
-    def register(cls, key, subclass):
+    def register(cls, key, subclass) -> None:
         """Register a subclass of this ABC with the given key (a string,
         number, type, or other hashable).
 
         Register may only be called once for a given key or a given subclass.
+
+        Parameters
+        ----------
+        key : `str` or `numbers.Number` or `None` or `collections.abc.Hashable`
+            Key to use for registration.
+        subclass : `type`
+            Subclass to register.
         """
         if key is None:
             raise ValueError("None may not be used as a key.")
@@ -396,17 +422,22 @@ class TemplateMeta(type):
                 setattrSafe(p, k)
         else:
             raise ValueError(
-                "key must have {} elements (one for each of {})".format(
-                    len(cls.TEMPLATE_PARAMS), cls.TEMPLATE_PARAMS
-                )
+                f"key must have {len(cls.TEMPLATE_PARAMS)} elements (one for each of {cls.TEMPLATE_PARAMS})"
             )
 
         for name, attr in cls._inherited.items():
             setattr(subclass, name, attr)
 
-    def alias(cls, key, subclass):
+    def alias(cls, key, subclass) -> None:
         """Add an alias that allows an existing subclass to be accessed with a
         different key.
+
+        Parameters
+        ----------
+        key : `str` or `numbers.Number` or `None` or `collections.abc.Hashable`
+            Key to use for aliasing.
+        subclass : `type`
+            Subclass to alias.
         """
         if key is None:
             raise ValueError("None may not be used as a key.")
@@ -449,8 +480,20 @@ class TemplateMeta(type):
         """Return an iterable of (key, subclass) pairs."""
         return cls._registry.items()
 
-    def get(cls, key, default=None):
-        """Return the subclass associated with the given key (including
-        aliases), or ``default`` if the key is not recognized.
+    def get(cls, key, default=None) -> type:
+        """Return the subclass associated with the given key.
+
+        Parameters
+        ----------
+        key : `~collections.abc.Hashable`
+            Key to query.
+        default : `~typing.Any` or `None`, optional
+            Default value to return if ``key`` is not found.
+
+        Returns
+        -------
+        `type`
+            Subclass with the given key. Includes aliases. Returns ``default``
+            if the key is not recognized.
         """
         return cls._registry.get(key, default)