lsst-utils 25.2023.600__py3-none-any.whl → 29.2025.4800__py3-none-any.whl

lsst/utils/threads.py

@@ -9,11 +9,11 @@
 # Use of this source code is governed by a 3-clause BSD-style
 # license that can be found in the LICENSE file.
 #
-from __future__ import annotations
-
 """Support for threading and multi-processing."""
 
-__all__ = ["set_thread_envvars", "disable_implicit_threading"]
+from __future__ import annotations
+
+__all__ = ["disable_implicit_threading", "set_thread_envvars"]
 
 import os
 
@@ -50,6 +50,12 @@ def set_thread_envvars(num_threads: int = 1, override: bool = False) -> None:
         if override or var not in os.environ:
             os.environ[var] = str(num_threads)
 
+    # Also specify an explicit value for OMP_PROC_BIND to tell OpenMP not to
+    # set CPU affinity.
+    var = "OMP_PROC_BIND"
+    if override or var not in os.environ:
+        os.environ[var] = "false"
+
 
 def disable_implicit_threading() -> None:
     """Do whatever is necessary to try to prevent implicit threading.

lsst/utils/timer.py

@@ -10,40 +10,34 @@
 # license that can be found in the LICENSE file.
 #
 
-"""Utilities for measuring execution time.
-"""
+"""Utilities for measuring execution time."""
 
 from __future__ import annotations
 
-__all__ = ["profile", "logInfo", "timeMethod", "time_this"]
+__all__ = ["duration_from_timeMethod", "logInfo", "profile", "timeMethod", "time_this"]
 
+import dataclasses
 import datetime
 import functools
-import inspect
 import logging
+import os
+import platform
+import sys
 import time
-from contextlib import contextmanager
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    Collection,
-    Iterable,
-    Iterator,
-    MutableMapping,
-    Optional,
-    Tuple,
-)
+import traceback
+from collections.abc import Callable, Collection, Iterable, Iterator, MutableMapping
+from contextlib import contextmanager, suppress
+from typing import TYPE_CHECKING, Any, TypeVar, overload
 
 from astropy import units as u
 
+from .introspection import find_outside_stacklevel
+from .logging import LsstLoggers, _is_structlog_logger
 from .usage import _get_current_rusage, get_current_mem_usage, get_peak_mem_usage
 
 if TYPE_CHECKING:
     import cProfile
 
-    from .logging import LsstLoggers
-
 
 _LOG = logging.getLogger(__name__)
 
@@ -80,54 +74,15 @@ def _add_to_metadata(metadata: MutableMapping, name: str, value: Any) -> None:
     metadata[name].append(value)
 
 
-def _find_outside_stacklevel() -> int:
-    """Find the stack level corresponding to caller code outside of this
-    module.
-
-    This can be passed directly to `logging.Logger.log()` to ensure
-    that log messages are issued as if they are coming from caller code.
-
-    Returns
-    -------
-    stacklevel : `int`
-        The stack level to use to refer to a caller outside of this module.
-        A ``stacklevel`` of ``1`` corresponds to the caller of this internal
-        function and that is the default expected by `logging.Logger.log()`.
-
-    Notes
-    -----
-    Intended to be called from the function that is going to issue a log
-    message. The result should be passed into `~logging.Logger.log` via the
-    keyword parameter ``stacklevel``.
-    """
-    stacklevel = 1  # the default for `Logger.log`
-    for i, s in enumerate(inspect.stack()):
-        module = inspect.getmodule(s.frame)
-        if module is None:
-            # Stack frames sometimes hang around so explicilty delete.
-            del s
-            continue
-        if not module.__name__.startswith("lsst.utils"):
-            # 0 will be this function.
-            # 1 will be the caller which will be the default for `Logger.log`
-            # and so does not need adjustment.
-            stacklevel = i
-            break
-        # Stack frames sometimes hang around so explicilty delete.
-        del s
-
-    return stacklevel
-
-
 def logPairs(
     obj: Any,
-    pairs: Collection[Tuple[str, Any]],
+    pairs: Collection[tuple[str, Any]],
     logLevel: int = logging.DEBUG,
-    metadata: Optional[MutableMapping] = None,
-    logger: Optional[logging.Logger] = None,
-    stacklevel: Optional[int] = None,
+    metadata: MutableMapping | None = None,
+    logger: LsstLoggers | None = None,
+    stacklevel: int | None = None,
 ) -> None:
-    """Log ``(name, value)`` pairs to ``obj.metadata`` and ``obj.log``
+    """Log ``(name, value)`` pairs to ``obj.metadata`` and ``obj.log``.
 
     Parameters
     ----------
@@ -147,7 +102,7 @@ def logPairs(
         Log level (an `logging` level constant, such as `logging.DEBUG`).
     metadata : `collections.abc.MutableMapping`, optional
         Metadata object to write entries to.  Ignored if `None`.
-    logger : `logging.Logger`
+    logger : `logging.Logger` or `lsst.utils.logging.LsstLogAdapter`
         Log object to write entries to.  Ignored if `None`.
     stacklevel : `int`, optional
         The stack level to pass to the logger to adjust which stack frame
@@ -158,15 +113,13 @@ def logPairs(
     """
     if obj is not None:
         if metadata is None:
-            try:
+            with suppress(AttributeError):
                 metadata = obj.metadata
-            except AttributeError:
-                pass
+
         if logger is None:
-            try:
+            with suppress(AttributeError):
                 logger = obj.log
-            except AttributeError:
-                pass
+
     strList = []
     for name, value in pairs:
         if metadata is not None:
@@ -179,7 +132,7 @@ def logPairs(
         timer_logger = logging.getLogger("timer." + logger.name)
         if timer_logger.isEnabledFor(logLevel):
             if stacklevel is None:
-                stacklevel = _find_outside_stacklevel()
+                stacklevel = find_outside_stacklevel("lsst.utils")
             else:
                 # Account for the caller stack.
                 stacklevel += 1
@@ -190,9 +143,9 @@ def logInfo(
     obj: Any,
     prefix: str,
     logLevel: int = logging.DEBUG,
-    metadata: Optional[MutableMapping] = None,
-    logger: Optional[logging.Logger] = None,
-    stacklevel: Optional[int] = None,
+    metadata: MutableMapping | None = None,
+    logger: LsstLoggers | None = None,
+    stacklevel: int | None = None,
 ) -> None:
     """Log timer information to ``obj.metadata`` and ``obj.log``.
 
@@ -216,7 +169,7 @@ def logInfo(
         Log level (a `logging` level constant, such as `logging.DEBUG`).
     metadata : `collections.abc.MutableMapping`, optional
         Metadata object to write entries to, overriding ``obj.metadata``.
-    logger : `logging.Logger`
+    logger : `logging.Logger` or `lsst.utils.logging.LsstLogAdapter`
         Log object to write entries to, overriding ``obj.log``.
     stacklevel : `int`, optional
         The stack level to pass to the logger to adjust which stack frame
@@ -246,18 +199,24 @@ def logInfo(
     * Version 1: ``MaxResidentSetSize`` will be stored in bytes.
     """
     if metadata is None and obj is not None:
-        try:
+        with suppress(AttributeError):
             metadata = obj.metadata
-        except AttributeError:
-            pass
+
     if metadata is not None:
         # Log messages already have timestamps.
-        utcStr = datetime.datetime.utcnow().isoformat()
+        if sys.version_info < (3, 11, 0):
+            now = datetime.datetime.utcnow()
+        else:
+            now = datetime.datetime.now(datetime.UTC)
+        utcStr = now.isoformat()
         _add_to_metadata(metadata, name=prefix + "Utc", value=utcStr)
 
         # Force a version number into the metadata.
         # v1: Ensure that max_rss field is always bytes.
         metadata["__version__"] = 1
+
+        # Add node information.
+        metadata["nodeName"] = platform.node()
     if stacklevel is not None:
         # Account for the caller of this routine not knowing that we
         # are going one down in the stack.
@@ -274,23 +233,41 @@ def logInfo(
     )
 
 
+_F = TypeVar("_F", bound=Callable)
+
+
+@overload
+def timeMethod(_func: _F) -> _F: ...
+
+
+@overload
 def timeMethod(
-    _func: Optional[Any] = None,
     *,
-    metadata: Optional[MutableMapping] = None,
-    logger: Optional[logging.Logger] = None,
+    metadata: MutableMapping | None = None,
+    logger: LsstLoggers | None = None,
+    logLevel: int = logging.DEBUG,
+) -> Callable[[_F], _F]: ...
+
+
+def timeMethod(
+    _func: Any | None = None,
+    *,
+    metadata: MutableMapping | None = None,
+    logger: LsstLoggers | None = None,
     logLevel: int = logging.DEBUG,
 ) -> Callable:
     """Measure duration of a method.
 
+    Set LSST_UTILS_DISABLE_TIMER in your environment to disable this method.
+
     Parameters
     ----------
-    func
+    _func : `~collections.abc.Callable` or `None`
         The method to wrap.
     metadata : `collections.abc.MutableMapping`, optional
         Metadata to use as override if the instance object attached
         to this timer does not support a ``metadata`` property.
-    logger : `logging.Logger`, optional
+    logger : `logging.Logger` or `lsst.utils.logging.LsstLogAdapter`, optional
         Logger to use when the class containing the decorated method does not
         have a ``log`` property.
     logLevel : `int`, optional
@@ -358,37 +335,66 @@ def timeMethod(
 
         return timeMethod_wrapper
 
+    if "LSST_UTILS_DISABLE_TIMER" in os.environ:
+        if _func is not None:
+            return _func
+        else:
+
+            def pass_through(func: Callable) -> Callable:
+                """Pass through decorator.
+
+                This decorator will not appear in the call stack.
+                """
+                return func
+
+            return pass_through
+
     if _func is None:
         return decorator_timer
     else:
         return decorator_timer(_func)
 
 
+@dataclasses.dataclass
+class _TimerResult:
+    duration: float = 0.0
+    """Duration of context in seconds."""
+    mem_current_usage: float | None = None
+    """Memory usage at end of context in requested units."""
+    mem_peak_delta: float | None = None
+    """Peak differential between entering and leaving context."""
+    mem_current_delta: float | None = None
+    """Difference in usage between entering and leaving context."""
+
+
 @contextmanager
 def time_this(
-    log: Optional[LsstLoggers] = None,
-    msg: Optional[str] = None,
+    log: LsstLoggers | None = None,
+    msg: str | None = None,
     level: int = logging.DEBUG,
-    prefix: Optional[str] = "timer",
+    prefix: str | None = "timer",
     args: Iterable[Any] = (),
+    kwargs: dict[str, Any] | None = None,
     mem_usage: bool = False,
     mem_child: bool = False,
-    mem_unit: u.Quantity = u.byte,
+    mem_unit: u.Unit = u.byte,
     mem_fmt: str = ".0f",
-) -> Iterator[None]:
+    force_mem_usage: bool = False,
+) -> Iterator[_TimerResult]:
     """Time the enclosed block and issue a log message.
 
     Parameters
     ----------
     log : `logging.Logger`, optional
         Logger to use to report the timer message. The root logger will
-        be used if none is given.
+        be used if none is given. Is also allowed to be a `structlog` bound
+        logger.
     msg : `str`, optional
         Context to include in log message.
     level : `int`, optional
         Python logging level to use to issue the log message. If the
-        code block raises an exception the log message will automatically
-        switch to level ERROR.
+        code block raises an exception the log message will include some
+        information about the exception that occurred.
     prefix : `str`, optional
         Prefix to use to prepend to the supplied logger to
         create a new logger to use instead. No prefix is used if the value
@@ -396,9 +402,15 @@ def time_this(
     args : iterable of any
         Additional parameters passed to the log command that should be
         written to ``msg``.
+    kwargs : `dict`, optional
+        Additional keyword parameters passed to the log command. If a Structlog
+        is used then these will be added to the structured data. Otherwise
+        they will be converted to a single string for inclusion in the log
+        message.
     mem_usage : `bool`, optional
         Flag indicating whether to include the memory usage in the report.
-        Defaults, to False.
+        Defaults, to False. Does nothing if the log message will not be
+        generated.
     mem_child : `bool`, optional
         Flag indication whether to include memory usage of the child processes.
     mem_unit : `astropy.units.Unit`, optional
@@ -406,22 +418,49 @@ def time_this(
     mem_fmt : `str`, optional
         Format specifier to use when displaying values related to memory usage.
         Defaults to '.0f'.
+    force_mem_usage : `bool`, optional
+        If `True` memory usage is always calculated even if the log message
+        will not be delivered. Use this if you want the information recorded
+        by the context manager.
+
+    Yields
+    ------
+    timer_result : `_TimerResult`
+        Simple data class containing the duration of the block in seconds via
+        the ``duration`` property.
     """
     if log is None:
         log = logging.getLogger()
-    if prefix:
+    is_structlog = _is_structlog_logger(log)
+    if prefix and not is_structlog:
+        # Struct log loggers do not have a name property and so the prefix
+        # is not applied to them.
         log_name = f"{prefix}.{log.name}" if not isinstance(log, logging.RootLogger) else prefix
         log = logging.getLogger(log_name)
 
-    success = False
+    # Some structured data that can be used if we have been given a
+    # structlog logger.
+    structured_args: dict[str, Any] = {}
+
     start = time.time()
+
+    if mem_usage and not log.isEnabledFor(level):
+        mem_usage = False
+    if force_mem_usage:
+        mem_usage = True
+
     if mem_usage:
         current_usages_start = get_current_mem_usage()
         peak_usages_start = get_peak_mem_usage()
 
+    timer_result = _TimerResult()
+    errmsg = ""
     try:
-        yield
-        success = True
+        yield timer_result
+    except BaseException as e:
+        frame, lineno = list(traceback.walk_tb(e.__traceback__))[-1]
+        errmsg = f"{e!r} @ {frame.f_code.co_filename}:{lineno}"
+        raise
     finally:
         end = time.time()
 
@@ -435,16 +474,18 @@ def time_this(
         # mypy stop complaining when additional parameters will be added below.
         params = list(args) if args else []
 
-        if not success:
-            # Something went wrong so change the log level to indicate
-            # this.
-            level = logging.ERROR
+        duration = end - start
+        timer_result.duration = duration
 
         # Specify stacklevel to ensure the message is reported from the
         # caller (1 is this file, 2 is contextlib, 3 is user)
-        params += (": " if msg else "", end - start)
+        params += (": " if msg else "", duration)
         msg += "%sTook %.4f seconds"
-        if mem_usage and log.isEnabledFor(level):
+        structured_args["duration"] = duration
+        if errmsg:
+            params += (f" (timed code triggered exception of {errmsg!r})",)
+            msg += "%s"
+        if mem_usage:
             current_usages_end = get_current_mem_usage()
             peak_usages_end = get_peak_mem_usage()
 
@@ -463,18 +504,37 @@ def time_this(
                 _LOG.warning("Invalid memory unit '%s', using '%s' instead", mem_unit, u.byte)
                 mem_unit = u.byte
 
+            current_usage = current_usage.to(mem_unit)
+            current_delta = current_delta.to(mem_unit)
+            peak_delta = peak_delta.to(mem_unit)
+
+            timer_result.mem_current_usage = float(current_usage.value)
+            timer_result.mem_current_delta = float(current_delta.value)
+            timer_result.mem_peak_delta = float(peak_delta.value)
+
             msg += (
-                f"; current memory usage: {current_usage.to(mem_unit):{mem_fmt}}"
-                f", delta: {current_delta.to(mem_unit):{mem_fmt}}"
-                f", peak delta: {peak_delta.to(mem_unit):{mem_fmt}}"
+                f"; current memory usage: {current_usage:{mem_fmt}}"
+                f", delta: {current_delta:{mem_fmt}}"
+                f", peak delta: {peak_delta:{mem_fmt}}"
             )
-        log.log(level, msg, *params, stacklevel=3)
+            structured_args["mem_current_usage"] = float(current_usage.value)
+            structured_args["mem_current_delta"] = float(current_delta.value)
+            structured_args["mem_peak_delta"] = float(peak_delta.value)
+        if not is_structlog:
+            # Can only use the structured content if we have structlog logger
+            # but stacklevel is only supported by standard loggers.
+            structured_args = {"stacklevel": 3}
+            if kwargs is not None:
+                msg += " %s"
+                params += ("; ".join(f"{k}={v!r}" for k, v in kwargs.items()),)
+        elif kwargs:
+            structured_args.update(kwargs)
+
+        log.log(level, msg, *params, **structured_args)
 
 
 @contextmanager
-def profile(
-    filename: Optional[str], log: Optional[logging.Logger] = None
-) -> Iterator[Optional[cProfile.Profile]]:
+def profile(filename: str | None, log: LsstLoggers | None = None) -> Iterator[cProfile.Profile | None]:
     """Profile the enclosed code block and save the result to a file.
 
     Parameters
@@ -482,7 +542,7 @@ def profile(
     filename : `str` or `None`
         Filename to which to write profile (profiling disabled if `None` or
         empty string).
-    log : `logging.Logger`, optional
+    log : `logging.Logger` or `lsst.utils.logging.LsstLogAdapter`, optional
         Log object for logging the profile operations.
 
     Yields
@@ -523,3 +583,40 @@ def profile(
     profile.dump_stats(filename)
     if log is not None:
         log.info("cProfile stats written to %s", filename)
+
+
+def duration_from_timeMethod(
+    metadata: MutableMapping | None, method_name: str, clock: str = "Cpu"
+) -> float | None:
+    """Parse the metadata entries from ``timeMethod`` and return a duration.
+
+    Parameters
+    ----------
+    metadata : `collections.abc.MutableMapping`
+        The Task metadata that timing metrics were added to.
+    method_name : `str`
+        Name of the timed method to extract a duration for.
+    clock : `str`, optional
+        Options are "Cpu", "User", "System", or "Utc".
+
+    Returns
+    -------
+    duration : `float`
+        The time elapsed between the start and end of the timed method.
+    """
+    if metadata is None:
+        return None
+    if clock.lower() == "utc":
+        start = metadata[method_name + "StartUtc"]
+        end = metadata[method_name + "EndUtc"]
+    else:
+        start = metadata[method_name + "Start" + clock + "Time"]
+        end = metadata[method_name + "End" + clock + "Time"]
+    if isinstance(start, list):
+        start = start[-1]
+    if isinstance(end, list):
+        end = end[-1]
+    if isinstance(start, str) and isinstance(end, str):
+        return (datetime.datetime.fromisoformat(end) - datetime.datetime.fromisoformat(start)).total_seconds()
+    else:
+        return end - start

lsst/utils/usage.py

@@ -9,8 +9,9 @@
 # Use of this source code is governed by a 3-clause BSD-style
 # license that can be found in the LICENSE file.
 
-"""Utilities for measuring resource consumption.
-"""
+"""Utilities for measuring resource consumption."""
+
+from __future__ import annotations
 
 __all__ = ["get_current_mem_usage", "get_peak_mem_usage"]
 
@@ -18,7 +19,6 @@ import dataclasses
 import platform
 import resource
 import time
-from typing import Dict, Tuple, Union
 
 import astropy.units as u
 import psutil
@@ -48,7 +48,7 @@ def _get_rusage_multiplier() -> int:
 _RUSAGE_MEMORY_MULTIPLIER = _get_rusage_multiplier()
 
 
-def get_current_mem_usage() -> Tuple[u.Quantity, u.Quantity]:
+def get_current_mem_usage() -> tuple[u.Quantity, u.Quantity]:
     """Report current memory usage.
 
     Returns
@@ -72,7 +72,7 @@ def get_current_mem_usage() -> Tuple[u.Quantity, u.Quantity]:
     return usage_main, usage_child
 
 
-def get_peak_mem_usage() -> Tuple[u.Quantity, u.Quantity]:
+def get_peak_mem_usage() -> tuple[u.Quantity, u.Quantity]:
     """Report peak memory usage.
 
     Returns
@@ -112,7 +112,7 @@ class _UsageInfo:
     voluntaryContextSwitches: int
     involuntaryContextSwitches: int
 
-    def dict(self) -> Dict[str, Union[float, int]]:
+    def dict(self) -> dict[str, float | int]:
         return dataclasses.asdict(self)
 
 

lsst/utils/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "25.2023.600"
+__version__ = "29.2025.4800"