lsst-utils 25.2023.2800__py3-none-any.whl → 29.2025.4800__py3-none-any.whl

lsst/utils/logging.py

@@ -14,10 +14,11 @@ from __future__ import annotations
 __all__ = (
     "TRACE",
     "VERBOSE",
-    "getLogger",
-    "getTraceLogger",
     "LsstLogAdapter",
+    "LsstLoggers",
     "PeriodicLogger",
+    "getLogger",
+    "getTraceLogger",
     "trace_set_at",
 )
 
@@ -27,13 +28,24 @@ import time
 from collections.abc import Generator
 from contextlib import contextmanager
 from logging import LoggerAdapter
-from typing import Any, Union
+from typing import TYPE_CHECKING, Any, TypeAlias, TypeGuard
 
 try:
     import lsst.log.utils as logUtils
 except ImportError:
     logUtils = None
 
+try:
+    from structlog import get_context as get_structlog_context
+except ImportError:
+    get_structlog_context = None  # type: ignore[assignment]
+
+
+if TYPE_CHECKING:
+    try:
+        from structlog.typing import BindableLogger
+    except ImportError:
+        BindableLogger: TypeAlias = Any  # type: ignore[no-redef]
 
 # log level for trace (verbose debug).
 TRACE = 5
@@ -44,6 +56,23 @@ VERBOSE = (logging.INFO + logging.DEBUG) // 2
 logging.addLevelName(VERBOSE, "VERBOSE")
 
 
+def _is_structlog_logger(
+    logger: logging.Logger | LsstLogAdapter | BindableLogger,
+) -> TypeGuard[BindableLogger]:
+    """Check if the given logger is a structlog logger."""
+    if get_structlog_context is None:
+        return False  # type: ignore[unreachable]
+
+    try:
+        # Returns a dict for structlog loggers; raises for stdlib logger
+        # objects.
+        get_structlog_context(logger)  # type: ignore[arg-type]
+        return True
+    except Exception:
+        # In practice this is usually ValueError or AttributeError.
+        return False
+
+
 def _calculate_base_stacklevel(default: int, offset: int) -> int:
     """Calculate the default logging stacklevel to use.
 
@@ -236,6 +265,15 @@ class LsstLogAdapter(LoggerAdapter):
 
         Arguments are as for `logging.info`.
         ``VERBOSE`` is between ``DEBUG`` and ``INFO``.
+
+        Parameters
+        ----------
+        fmt : `str`
+            Log message.
+        *args : `~typing.Any`
+            Parameters references by log message.
+        **kwargs : `~typing.Any`
+            Parameters forwarded to `log`.
         """
         # There is no other way to achieve this other than a special logger
         # method.
@@ -248,6 +286,15 @@ class LsstLogAdapter(LoggerAdapter):
 
         Arguments are as for `logging.info`.
         ``TRACE`` is lower than ``DEBUG``.
+
+        Parameters
+        ----------
+        fmt : `str`
+            Log message.
+        *args : `~typing.Any`
+            Parameters references by log message.
+        **kwargs : `~typing.Any`
+            Parameters forwarded to `log`.
         """
         # There is no other way to achieve this other than a special logger
         # method.
@@ -280,12 +327,21 @@ class LsstLogAdapter(LoggerAdapter):
     def addHandler(self, handler: logging.Handler) -> None:
         """Add a handler to this logger.
 
-        The handler is forwarded to the underlying logger.
+        Parameters
+        ----------
+        handler : `logging.Handler`
+           Handler to add. The handler is forwarded to the underlying logger.
         """
         self.logger.addHandler(handler)
 
     def removeHandler(self, handler: logging.Handler) -> None:
-        """Remove the given handler from the underlying logger."""
+        """Remove the given handler from the underlying logger.
+
+        Parameters
+        ----------
+        handler : `logging.Handler`
+            Handler to remove.
+        """
         self.logger.removeHandler(handler)
 
 
@@ -321,7 +377,7 @@ def getLogger(name: str | None = None, logger: logging.Logger | None = None) ->
     return LsstLogAdapter(logger, {})
 
 
-LsstLoggers = Union[logging.Logger, LsstLogAdapter]
+LsstLoggers: TypeAlias = logging.Logger | LsstLogAdapter
 
 
 def getTraceLogger(logger: str | LsstLoggers, trace_level: int) -> LsstLogAdapter:
@@ -354,22 +410,28 @@ class PeriodicLogger:
     be useful to issue a log message periodically to show that the
     algorithm is progressing.
 
+    The first time threshold is counted from object construction, so in general
+    the first call to `log` does not log.
+
     Parameters
     ----------
     logger : `logging.Logger` or `LsstLogAdapter`
         Logger to use when issuing a message.
     interval : `float`
-        The minimum interval between log messages. If `None` the class
-        default will be used.
+        The minimum interval in seconds between log messages. If `None`,
+        `LOGGING_INTERVAL` will be used.
     level : `int`, optional
-        Log level to use when issuing messages.
+        Log level to use when issuing messages, default is
+        `~logging.INFO`.
     """
 
     LOGGING_INTERVAL = 600.0
-    """Default interval between log messages."""
+    """Default interval between log messages in seconds."""
 
-    def __init__(self, logger: LsstLoggers, interval: float | None = None, level: int = VERBOSE):
+    def __init__(self, logger: LsstLoggers, interval: float | None = None, level: int = logging.INFO):
         self.logger = logger
+        # None -> LOGGING_INTERVAL conversion done so that unit tests (e.g., in
+        # pipe_base) can tweak log interval without access to the constructor.
         self.interval = interval if interval is not None else self.LOGGING_INTERVAL
         self.level = level
         self.next_log_time = time.time() + self.interval
@@ -385,12 +447,16 @@ class PeriodicLogger:
     def log(self, msg: str, *args: Any) -> bool:
         """Issue a log message if the interval has elapsed.
 
+        The interval is measured from the previous call to ``log``, or from the
+        creation of this object.
+
         Parameters
         ----------
         msg : `str`
             Message to issue if the time has been exceeded.
         *args : Any
-            Parameters to be passed to the log system.
+            Arguments to be merged into the message string, as described under
+            `logging.Logger.debug`.
 
         Returns
         -------

lsst/utils/packages.py

@@ -9,11 +9,11 @@
 # Use of this source code is governed by a 3-clause BSD-style
 # license that can be found in the LICENSE file.
 #
-"""Determine which packages are being used in the system and their versions.
-"""
+"""Determine which packages are being used in the system and their versions."""
 
 from __future__ import annotations
 
+import contextlib
 import hashlib
 import importlib
 import io
@@ -26,19 +26,21 @@ import subprocess
 import sys
 import types
 from collections.abc import Mapping
-from functools import lru_cache
-from typing import Any
+from functools import cache, lru_cache
+from importlib.metadata import packages_distributions
+from typing import Any, ClassVar
 
 import yaml
 
 log = logging.getLogger(__name__)
 
 __all__ = [
-    "getVersionFromPythonModule",
-    "getPythonPackages",
-    "getEnvironmentPackages",
-    "getCondaPackages",
     "Packages",
+    "getAllPythonDistributions",
+    "getCondaPackages",
+    "getEnvironmentPackages",
+    "getPythonPackages",
+    "getVersionFromPythonModule",
 ]
 
 
@@ -48,7 +50,9 @@ BUILDTIME = {"boost", "eigen", "tmv"}
 # Python modules to attempt to load so we can try to get the version
 # We do this because the version only appears to be available from python,
 # but we use the library
-PYTHON = {"galsim"}
+PYTHON: set[str] = set()
+
+SPECIAL_NAMESPACES = {"lsst"}
 
 # Packages that don't seem to have a mechanism for reporting the runtime
 # version.  We need to guess the version from the environment
@@ -73,6 +77,7 @@ def getVersionFromPythonModule(module: types.ModuleType) -> str:
     Returns
     -------
     version : `str`
+        The version of the python module.
 
     Raises
     ------
@@ -96,14 +101,37 @@ def getVersionFromPythonModule(module: types.ModuleType) -> str:
     return str(version)
 
 
+@cache
+def getAllPythonDistributions() -> dict[str, str]:
+    """Get the versions for all Python distributions that are installed.
+
+    Returns
+    -------
+    packages : `dict` [ `str`, `str` ]
+        Keys are distribution names; values are their versions.
+        Unlike `getPythonPackages` this function will not include
+        standard library packages defined in `sys.stdlib_module_names` but
+        will include a special ``python`` key reporting the Python version.
+
+    Notes
+    -----
+    If this function is called a second time an identical result will be
+    returned even if a new distribution has been installed.
+    """
+    packages = {"python": sys.version}
+
+    for dist in importlib.metadata.distributions():
+        packages[dist.name] = dist.version
+    return _mangle_lsst_package_names(packages)
+
+
 def getPythonPackages() -> dict[str, str]:
     """Get imported python packages and their versions.
 
     Returns
     -------
-    packages : `dict`
-        Keys (type `str`) are package names; values (type `str`) are their
-        versions.
+    packages : `dict` [ `str`, `str` ]
+        Keys are package names; values are their versions.
 
     Notes
     -----
@@ -111,20 +139,29 @@ def getPythonPackages() -> dict[str, str]:
     module.  Note, therefore, that we can only report on modules that have
     *already* been imported.
 
+    Python standard library packages are not included in the report. A
+    ``python`` key is inserted that records the Python version.
+
     We don't include any module for which we cannot determine a version.
+
+    Whilst distribution names are used to determine package versions, the
+    key returned for the package version is the package name that was imported.
+    This means that ``yaml`` will appear as the version key even though the
+    distribution would be called ``PyYAML``.
     """
     # Attempt to import libraries that only report their version in python
     for module_name in PYTHON:
-        try:
+        # If it's not available we continue.
+        with contextlib.suppress(Exception):
             importlib.import_module(module_name)
-        except Exception:
-            pass  # It's not available, so don't care
+
+    package_dist = packages_distributions()
 
     packages = {"python": sys.version}
 
     # Not iterating with sys.modules.iteritems() because it's not atomic and
     # subject to race conditions
-    module_names = list(sys.modules.keys())
+    module_names = sorted(sys.modules.keys())
 
     # Use knowledge of package hierarchy to find the versions rather than
     # using each name independently. Group all the module names into the
@@ -144,7 +181,7 @@ def getPythonPackages() -> dict[str, str]:
     n_versions = 0
     n_checked = 0
     previous = ""
-    for name in sorted(module_names):
+    for name in module_names:
         if name.startswith("_") or "._" in name:
             # Refers to a private module so we can ignore it and assume
             # version has been lifted into parent or, if top level, not
@@ -152,20 +189,31 @@ def getPythonPackages() -> dict[str, str]:
             # packages such as _abc and __future__.
             continue
 
-        if name in _STDLIB:
-            # Assign all standard library packages the python version
-            # since they almost all lack explicit versions.
-            packages[name] = sys.version
-            previous = name
-            continue
-
         if name.startswith(previous + ".") and previous in packages:
             # Already have this version. Use the same previous name
             # for the line after this.
             continue
 
-        # Look for a version.
-        ver = _get_python_package_version(name, packages)
+        # Find the namespace which we need to use package_dist.
+        namespace = name.split(".")[0]
+
+        if namespace in _STDLIB:
+            # If this is an import from the standard library, skip it.
+            # Standard library names only refer to top-level namespace
+            # so "importlib" appears but "importlib.metadata" does not.
+            previous = name
+            continue
+
+        # package_dist is a mapping from import namespace to distribution
+        # package names. This may be a one-to-many mapping due to namespace
+        # packages. Note that package_dist does not know about editable
+        # installs or eups installs via path manipulation.
+        if namespace in package_dist:
+            dist_names = package_dist[namespace]
+        else:
+            dist_names = [name]
+
+        ver = _get_python_package_version(name, namespace, dist_names, packages)
 
         n_checked += 1
         if ver is not None:
@@ -179,26 +227,41 @@ def getPythonPackages() -> dict[str, str]:
         n_versions,
     )
 
+    return _mangle_lsst_package_names(packages)
+
+
+def _mangle_lsst_package_names(packages: dict[str, str]) -> dict[str, str]:
     for name in list(packages.keys()):
         # Use LSST package names instead of python module names
         # This matches the names we get from the environment (i.e., EUPS)
         # so we can clobber these build-time versions if the environment
         # reveals that we're not using the packages as-built.
         if name.startswith("lsst."):
-            new_name = name.replace("lsst.", "").replace(".", "_")
-            packages[new_name] = packages[name]
-            del packages[name]
+            sep = "."
+        elif name.startswith("lsst-"):
+            sep = "-"
+        else:
+            continue
+        new_name = name.replace(f"lsst{sep}", "").replace(sep, "_")
+        packages[new_name] = packages[name]
+        del packages[name]
 
     return packages
 
 
-def _get_python_package_version(name: str, packages: dict[str, str]) -> str | None:
+def _get_python_package_version(
+    name: str, namespace: str, dist_names: list[str], packages: dict[str, str]
+) -> str | None:
     """Given a package or module name, try to determine the version.
 
     Parameters
     ----------
     name : `str`
-        The name of the package or module to try.
+        The imported name of the package or module to try.
+    namespace : `str`
+        The namespace of the package or module.
+    dist_names : `list` [ `str` ]
+        The distribution names of the package or module.
     packages : `dict` [ `str`, `str` ]
         A dictionary mapping a name to a version. Modified in place.
         The key used might not match exactly the given key.
@@ -209,10 +272,44 @@ def _get_python_package_version(name: str, packages: dict[str, str]) -> str | No
         The version string stored in ``packages``. Nothing is stored if the
         value here is `None`.
     """
+    # We have certain special namespaces that are used via eups that
+    # need to be enumerated here.
+    if len(dist_names) > 1 or namespace in SPECIAL_NAMESPACES:
+        # Split the name into parts.
+        name_parts = re.split("[._-]", name)
+
+        found = False
+        for dist_name in dist_names:
+            # It should be impossible for this to happen but it has happened
+            # so check for it.
+            if dist_name is None:
+                continue  # type: ignore
+            dist_name_parts = re.split("[._-]", dist_name)
+
+            # Check if the components start with the namespace; this is
+            # needed because (at least) lsst.ts packages do not use
+            # ``lsst`` in the package name.
+            if dist_name_parts[0] != namespace:
+                dist_name_parts.insert(0, namespace)
+
+            if dist_name_parts == name_parts:
+                found = True
+                break
+
+        if not found:
+            # This fallback case occurs when (a) we are testing the overall
+            # namespace (e.g. "lsst" or "sphinxcontrib") and the code below
+            # will return None; or (b) for eups-installed and other
+            # "editable installations" that are not registered as part
+            # of importlib.packages_distributions().
+            dist_name = name
+    else:
+        dist_name = dist_names[0]
+
     try:
         # This is the Python standard way to find a package version.
         # It can be slow.
-        ver = importlib.metadata.version(name)
+        ver = importlib.metadata.version(dist_name)
     except Exception:
         # Fall back to using the module itself. There is no guarantee
         # that "a" exists for module "a.b" so if hierarchy has been expanded
@@ -235,7 +332,7 @@ def _get_python_package_version(name: str, packages: dict[str, str]) -> str | No
 _eups: Any | None = None  # Singleton Eups object
 
 
-@lru_cache(maxsize=1)
+@lru_cache(maxsize=2)
 def getEnvironmentPackages(include_all: bool = False) -> dict[str, str]:
     """Get products and their versions from the environment.
 
@@ -258,6 +355,9 @@ def getEnvironmentPackages(include_all: bool = False) -> dict[str, str]:
     provide a means to determine the version any other way) and to check if
     uninstalled packages are being used. We only report the product/version
     for these packages unless ``include_all`` is `True`.
+
+    Assumes that no new EUPS packages are set up after this function is
+    called the first time.
     """
     try:
         from eups import Eups
@@ -287,7 +387,7 @@ def getEnvironmentPackages(include_all: bool = False) -> dict[str, str]:
         if not prod.version.startswith(Product.LocalVersionPrefix):
             if include_all:
                 tags = {t for t in prod.tags if t != "current"}
-                tag_msg = " (" + " ".join(tags) + ")" if tags else ""
+                tag_msg = " (" + " ".join(sorted(tags)) + ")" if tags else ""
                 packages[prod.name] = prod.version + tag_msg
             continue
         ver = prod.version
@@ -336,21 +436,40 @@ def getCondaPackages() -> dict[str, str]:
     Returns empty result if a conda environment is not in use or can not
     be queried.
     """
+    if "CONDA_PREFIX" not in os.environ:
+        return {}
+
+    # conda list is very slow. Ten times faster to scan the directory
+    # directly. This will only find conda packages and not pip installed
+    # packages.
+    meta_path = os.path.join(os.environ["CONDA_PREFIX"], "conda-meta")
+
     try:
-        from conda.cli.python_api import Commands, run_command
-    except ImportError:
+        filenames = os.scandir(path=meta_path)
+    except FileNotFoundError:
         return {}
 
-    # Get the installed package list
-    versions_json = run_command(Commands.LIST, "--json")
-    packages = {pkg["name"]: pkg["version"] for pkg in json.loads(versions_json[0])}
+    packages = {}
+
+    for filename in filenames:
+        if not filename.name.endswith(".json"):
+            continue
+        with open(filename) as f:
+            try:
+                data = json.load(f)
+            except ValueError:
+                continue
+            try:
+                packages[data["name"]] = data["version"]
+            except KeyError:
+                continue
+
+    packages = dict(sorted(packages.items()))
 
     # Try to work out the conda environment name and include it as a fake
     # package. The "obvious" way of running "conda info --json" does give
     # access to the active_prefix but takes about 2 seconds to run.
-    # The equivalent to the code above would be:
-    #    info_json = run_command(Commands.INFO, "--json")
-    # As a comporomise look for the env name in the path to the python
+    # As a compromise look for the env name in the path to the python
     # executable
     match = re.search(r"/envs/(.*?)/bin/", sys.executable)
     if match:
@@ -398,19 +517,19 @@ class Packages(dict):
         print("Extra packages compared to before:", pkgs.extra(old))
         print("Different packages: ", pkgs.difference(old))
         old.update(pkgs)  # Include any new packages in the old
-        old.write("/path/to/packages.pickle")
-
-    Parameters
-    ----------
-    packages : `dict`
-        A mapping {package: version} where both keys and values are type `str`.
+        old.write("/path/to/packages.pickle").
 
     Notes
     -----
     This is a wrapper around a dict with some convenience methods.
     """
 
-    formats = {".pkl": "pickle", ".pickle": "pickle", ".yaml": "yaml", ".json": "json"}
+    formats: ClassVar[dict[str, str]] = {
+        ".pkl": "pickle",
+        ".pickle": "pickle",
+        ".yaml": "yaml",
+        ".json": "json",
+    }
 
     def __setstate__(self, state: dict[str, Any]) -> None:
         # This only seems to be called for old pickle files where
@@ -418,22 +537,43 @@ class Packages(dict):
         self.update(state["_packages"])
 
     @classmethod
-    def fromSystem(cls) -> Packages:
+    def fromSystem(cls, include_all: bool = False) -> Packages:
         """Construct a `Packages` by examining the system.
 
-        Determine packages by examining python's `sys.modules`, conda
+        Determine packages by examining python's installed packages
+        (by default filtered by `sys.modules`) or distributions, conda
         libraries and EUPS. EUPS packages take precedence over conda and
         general python packages.
 
+        Parameters
+        ----------
+        include_all : `bool`, optional
+            If `False`, will only include imported Python packages, installed
+            Conda packages and locally-setup EUPS packages. If `True` all
+            installed Python distributions and conda packages will be reported
+            as well as all EUPS packages that are set up.
+
         Returns
         -------
         packages : `Packages`
             All version package information that could be obtained.
+
+        Note
+        ----
+        The names of Python distributions can differ from the names of the
+        Python packages installed by those distributions. Since ``include_all``
+        set to `True` uses Python distributions and `False` uses Python
+        packages do not expect that the answers are directly comparable.
         """
         packages = {}
-        packages.update(getPythonPackages())
+        if include_all:
+            packages.update(getAllPythonDistributions())
+        else:
+            packages.update(getPythonPackages())
+        # Conda list always reports all Conda packages.
         packages.update(getCondaPackages())
-        packages.update(getEnvironmentPackages())  # Should be last, to override products with LOCAL versions
+        # Should be last, to override products with LOCAL versions
+        packages.update(getEnvironmentPackages(include_all=include_all))
         return cls(packages)
 
     @classmethod
@@ -523,7 +663,7 @@ class Packages(dict):
         filename : `str`
             Filename to which to write. The format of the data file
             is determined from the file extension. Currently supports
-            ``.pickle``, ``.json``, and ``.yaml``
+            ``.pickle``, ``.json``, and ``.yaml``.
         """
         _, ext = os.path.splitext(filename)
         if ext not in self.formats:
@@ -608,6 +748,18 @@ class _BackwardsCompatibilityUnpickler(pickle.Unpickler):
         """Return the class that should be used for unpickling.
 
         This is always known to be the class in this package.
+
+        Parameters
+        ----------
+        module : `str`
+            Ignored.
+        name : `str`
+            Ignored.
+
+        Returns
+        -------
+        `type` [`Packages`]
+            The Python type to use. Always returns `Packages`.
         """
         return Packages
 
@@ -628,7 +780,11 @@ def _pkg_constructor(loader: yaml.constructor.SafeConstructor, node: yaml.Node)
     yield Packages(loader.construct_mapping(node, deep=True))  # type: ignore
 
 
-for loader in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader):
+for loader_str in ("Loader", "CLoader", "UnsafeLoader", "SafeLoader", "FullLoader"):
+    loader = getattr(yaml, loader_str, None)
+    if loader is None:
+        continue
+
     yaml.add_constructor("lsst.utils.packages.Packages", _pkg_constructor, Loader=loader)
 
     # Register the old name with YAML.

lsst/utils/plotting/__init__.py

@@ -0,0 +1,15 @@
+# This file is part of utils.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# Use of this source code is governed by a 3-clause BSD-style
+# license that can be found in the LICENSE file.
+"""General LSST Plotting Utilities."""
+
+from .figures import *
+from .limits import *
+from .publication_plots import *