lsst-utils 25.2023.600__py3-none-any.whl → 29.2025.4800__py3-none-any.whl

lsst/utils/packages.py

@@ -9,11 +9,11 @@
 # Use of this source code is governed by a 3-clause BSD-style
 # license that can be found in the LICENSE file.
 #
+"""Determine which packages are being used in the system and their versions."""
+
 from __future__ import annotations
 
-"""
-Determine which packages are being used in the system and their versions
-"""
+import contextlib
 import hashlib
 import importlib
 import io
@@ -26,33 +26,44 @@ import subprocess
 import sys
 import types
 from collections.abc import Mapping
-from functools import lru_cache
-from typing import Any, Dict, Optional, Tuple, Type
+from functools import cache, lru_cache
+from importlib.metadata import packages_distributions
+from typing import Any, ClassVar
 
 import yaml
 
 log = logging.getLogger(__name__)
 
 __all__ = [
-    "getVersionFromPythonModule",
-    "getPythonPackages",
-    "getEnvironmentPackages",
-    "getCondaPackages",
     "Packages",
+    "getAllPythonDistributions",
+    "getCondaPackages",
+    "getEnvironmentPackages",
+    "getPythonPackages",
+    "getVersionFromPythonModule",
 ]
 
 
 # Packages used at build-time (e.g., header-only)
-BUILDTIME = set(["boost", "eigen", "tmv"])
+BUILDTIME = {"boost", "eigen", "tmv"}
 
 # Python modules to attempt to load so we can try to get the version
 # We do this because the version only appears to be available from python,
 # but we use the library
-PYTHON = set(["galsim"])
+PYTHON: set[str] = set()
+
+SPECIAL_NAMESPACES = {"lsst"}
 
 # Packages that don't seem to have a mechanism for reporting the runtime
 # version.  We need to guess the version from the environment
-ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"])
+ENVIRONMENT = {"astrometry_net", "astrometry_net_data", "minuit2", "xpa"}
+
+try:
+    # Python 3.10 includes a list of standard library modules.
+    # These will all have the same version number as Python itself.
+    _STDLIB = sys.stdlib_module_names
+except AttributeError:
+    _STDLIB = frozenset()
 
 
 def getVersionFromPythonModule(module: types.ModuleType) -> str:
@@ -60,17 +71,18 @@ def getVersionFromPythonModule(module: types.ModuleType) -> str:
 
     Parameters
     ----------
-    module : `module`
+    module : `~types.ModuleType`
         Module for which to get version.
 
     Returns
     -------
     version : `str`
+        The version of the python module.
 
     Raises
     ------
     AttributeError
-        Raised if __version__ attribute is not set.
+        Raised if ``__version__`` attribute is not set.
 
     Notes
     -----
@@ -85,18 +97,41 @@ def getVersionFromPythonModule(module: types.ModuleType) -> str:
         deps = module.__dependency_versions__
         buildtime = BUILDTIME & set(deps.keys())
         if buildtime:
-            version += " with " + " ".join("%s=%s" % (pkg, deps[pkg]) for pkg in sorted(buildtime))
+            version += " with " + " ".join(f"{pkg}={deps[pkg]}" for pkg in sorted(buildtime))
     return str(version)
 
 
-def getPythonPackages() -> Dict[str, str]:
+@cache
+def getAllPythonDistributions() -> dict[str, str]:
+    """Get the versions for all Python distributions that are installed.
+
+    Returns
+    -------
+    packages : `dict` [ `str`, `str` ]
+        Keys are distribution names; values are their versions.
+        Unlike `getPythonPackages` this function will not include
+        standard library packages defined in `sys.stdlib_module_names` but
+        will include a special ``python`` key reporting the Python version.
+
+    Notes
+    -----
+    If this function is called a second time an identical result will be
+    returned even if a new distribution has been installed.
+    """
+    packages = {"python": sys.version}
+
+    for dist in importlib.metadata.distributions():
+        packages[dist.name] = dist.version
+    return _mangle_lsst_package_names(packages)
+
+
+def getPythonPackages() -> dict[str, str]:
     """Get imported python packages and their versions.
 
     Returns
     -------
-    packages : `dict`
-        Keys (type `str`) are package names; values (type `str`) are their
-        versions.
+    packages : `dict` [ `str`, `str` ]
+        Keys are package names; values are their versions.
 
     Notes
     -----
@@ -104,54 +139,201 @@ def getPythonPackages() -> Dict[str, str]:
     module.  Note, therefore, that we can only report on modules that have
     *already* been imported.
 
+    Python standard library packages are not included in the report. A
+    ``python`` key is inserted that records the Python version.
+
     We don't include any module for which we cannot determine a version.
+
+    Whilst distribution names are used to determine package versions, the
+    key returned for the package version is the package name that was imported.
+    This means that ``yaml`` will appear as the version key even though the
+    distribution would be called ``PyYAML``.
     """
     # Attempt to import libraries that only report their version in python
     for module_name in PYTHON:
-        try:
+        # If it's not available we continue.
+        with contextlib.suppress(Exception):
             importlib.import_module(module_name)
-        except Exception:
-            pass  # It's not available, so don't care
+
+    package_dist = packages_distributions()
 
     packages = {"python": sys.version}
+
     # Not iterating with sys.modules.iteritems() because it's not atomic and
     # subject to race conditions
-    moduleNames = list(sys.modules.keys())
-    for name in moduleNames:
-        module = sys.modules[name]
-        try:
-            ver = getVersionFromPythonModule(module)
-        except Exception:
-            continue  # Can't get a version from it, don't care
-
-        # Remove "foo.bar.version" in favor of "foo.bar"
-        # This prevents duplication when the __init__.py includes
-        # "from .version import *"
-        for ending in (".version", "._version"):
-            if name.endswith(ending):
-                name = name[: -len(ending)]
-                if name in packages:
-                    assert ver == packages[name]
-            elif name in packages:
-                assert ver == packages[name]
+    module_names = sorted(sys.modules.keys())
+
+    # Use knowledge of package hierarchy to find the versions rather than
+    # using each name independently. Group all the module names into the
+    # hierarchy, splitting on dot, and skipping any component that starts
+    # with an underscore.
+
+    # Sorting the module names gives us:
+    # lsst
+    # lsst.afw
+    # lsst.afw.cameraGeom
+    # ...
+    # lsst.daf
+    # lsst.daf.butler
+    #
+    # and so we can use knowledge of the previous version to inform whether
+    # we need to look at the subsequent line.
+    n_versions = 0
+    n_checked = 0
+    previous = ""
+    for name in module_names:
+        if name.startswith("_") or "._" in name:
+            # Refers to a private module so we can ignore it and assume
+            # version has been lifted into parent or, if top level, not
+            # relevant for versioning. This applies also to standard library
+            # packages such as _abc and __future__.
+            continue
+
+        if name.startswith(previous + ".") and previous in packages:
+            # Already have this version. Use the same previous name
+            # for the line after this.
+            continue
+
+        # Find the namespace which we need to use package_dist.
+        namespace = name.split(".")[0]
+
+        if namespace in _STDLIB:
+            # If this is an import from the standard library, skip it.
+            # Standard library names only refer to top-level namespace
+            # so "importlib" appears but "importlib.metadata" does not.
+            previous = name
+            continue
+
+        # package_dist is a mapping from import namespace to distribution
+        # package names. This may be a one-to-many mapping due to namespace
+        # packages. Note that package_dist does not know about editable
+        # installs or eups installs via path manipulation.
+        if namespace in package_dist:
+            dist_names = package_dist[namespace]
+        else:
+            dist_names = [name]
+
+        ver = _get_python_package_version(name, namespace, dist_names, packages)
+
+        n_checked += 1
+        if ver is not None:
+            n_versions += 1
+        previous = name
+
+    log.debug(
+        "Given %d modules but checked %d in hierarchy and found versions for %d",
+        len(module_names),
+        n_checked,
+        n_versions,
+    )
+
+    return _mangle_lsst_package_names(packages)
+
 
+def _mangle_lsst_package_names(packages: dict[str, str]) -> dict[str, str]:
+    for name in list(packages.keys()):
         # Use LSST package names instead of python module names
         # This matches the names we get from the environment (i.e., EUPS)
         # so we can clobber these build-time versions if the environment
         # reveals that we're not using the packages as-built.
-        if "lsst" in name:
-            name = name.replace("lsst.", "").replace(".", "_")
+        if name.startswith("lsst."):
+            sep = "."
+        elif name.startswith("lsst-"):
+            sep = "-"
+        else:
+            continue
+        new_name = name.replace(f"lsst{sep}", "").replace(sep, "_")
+        packages[new_name] = packages[name]
+        del packages[name]
+
+    return packages
+
+
+def _get_python_package_version(
+    name: str, namespace: str, dist_names: list[str], packages: dict[str, str]
+) -> str | None:
+    """Given a package or module name, try to determine the version.
+
+    Parameters
+    ----------
+    name : `str`
+        The imported name of the package or module to try.
+    namespace : `str`
+        The namespace of the package or module.
+    dist_names : `list` [ `str` ]
+        The distribution names of the package or module.
+    packages : `dict` [ `str`, `str` ]
+        A dictionary mapping a name to a version. Modified in place.
+        The key used might not match exactly the given key.
+
+    Returns
+    -------
+    ver : `str` or `None`
+        The version string stored in ``packages``. Nothing is stored if the
+        value here is `None`.
+    """
+    # We have certain special namespaces that are used via eups that
+    # need to be enumerated here.
+    if len(dist_names) > 1 or namespace in SPECIAL_NAMESPACES:
+        # Split the name into parts.
+        name_parts = re.split("[._-]", name)
+
+        found = False
+        for dist_name in dist_names:
+            # It should be impossible for this to happen but it has happened
+            # so check for it.
+            if dist_name is None:
+                continue  # type: ignore
+            dist_name_parts = re.split("[._-]", dist_name)
+
+            # Check if the components start with the namespace; this is
+            # needed because (at least) lsst.ts packages do not use
+            # ``lsst`` in the package name.
+            if dist_name_parts[0] != namespace:
+                dist_name_parts.insert(0, namespace)
+
+            if dist_name_parts == name_parts:
+                found = True
+                break
+
+        if not found:
+            # This fallback case occurs when (a) we are testing the overall
+            # namespace (e.g. "lsst" or "sphinxcontrib") and the code below
+            # will return None; or (b) for eups-installed and other
+            # "editable installations" that are not registered as part
+            # of importlib.packages_distributions().
+            dist_name = name
+    else:
+        dist_name = dist_names[0]
+
+    try:
+        # This is the Python standard way to find a package version.
+        # It can be slow.
+        ver = importlib.metadata.version(dist_name)
+    except Exception:
+        # Fall back to using the module itself. There is no guarantee
+        # that "a" exists for module "a.b" so if hierarchy has been expanded
+        # this might fail. Check first.
+        if name not in sys.modules:
+            return None
+        module = sys.modules[name]
+        try:
+            ver = getVersionFromPythonModule(module)
+        except Exception:
+            return None  # Can't get a version from it, don't care
 
+    # Update the package information.
+    if ver is not None:
         packages[name] = ver
 
-    return packages
+    return ver
 
 
-_eups: Optional[Any] = None  # Singleton Eups object
+_eups: Any | None = None  # Singleton Eups object
 
 
-@lru_cache(maxsize=1)
-def getEnvironmentPackages(include_all: bool = False) -> Dict[str, str]:
+@lru_cache(maxsize=2)
+def getEnvironmentPackages(include_all: bool = False) -> dict[str, str]:
     """Get products and their versions from the environment.
 
     Parameters
@@ -173,6 +355,9 @@ def getEnvironmentPackages(include_all: bool = False) -> Dict[str, str]:
     provide a means to determine the version any other way) and to check if
     uninstalled packages are being used. We only report the product/version
     for these packages unless ``include_all`` is `True`.
+
+    Assumes that no new EUPS packages are set up after this function is
+    called the first time.
     """
     try:
         from eups import Eups
@@ -202,7 +387,7 @@ def getEnvironmentPackages(include_all: bool = False) -> Dict[str, str]:
         if not prod.version.startswith(Product.LocalVersionPrefix):
             if include_all:
                 tags = {t for t in prod.tags if t != "current"}
-                tag_msg = " (" + " ".join(tags) + ")" if tags else ""
+                tag_msg = " (" + " ".join(sorted(tags)) + ")" if tags else ""
                 packages[prod.name] = prod.version + tag_msg
             continue
         ver = prod.version
@@ -237,7 +422,7 @@ def getEnvironmentPackages(include_all: bool = False) -> Dict[str, str]:
 
 
 @lru_cache(maxsize=1)
-def getCondaPackages() -> Dict[str, str]:
+def getCondaPackages() -> dict[str, str]:
     """Get products and their versions from the conda environment.
 
     Returns
@@ -251,21 +436,40 @@ def getCondaPackages() -> Dict[str, str]:
     Returns empty result if a conda environment is not in use or can not
     be queried.
     """
+    if "CONDA_PREFIX" not in os.environ:
+        return {}
+
+    # conda list is very slow. Ten times faster to scan the directory
+    # directly. This will only find conda packages and not pip installed
+    # packages.
+    meta_path = os.path.join(os.environ["CONDA_PREFIX"], "conda-meta")
+
     try:
-        from conda.cli.python_api import Commands, run_command
-    except ImportError:
+        filenames = os.scandir(path=meta_path)
+    except FileNotFoundError:
         return {}
 
-    # Get the installed package list
-    versions_json = run_command(Commands.LIST, "--json")
-    packages = {pkg["name"]: pkg["version"] for pkg in json.loads(versions_json[0])}
+    packages = {}
+
+    for filename in filenames:
+        if not filename.name.endswith(".json"):
+            continue
+        with open(filename) as f:
+            try:
+                data = json.load(f)
+            except ValueError:
+                continue
+            try:
+                packages[data["name"]] = data["version"]
+            except KeyError:
+                continue
+
+    packages = dict(sorted(packages.items()))
 
     # Try to work out the conda environment name and include it as a fake
     # package. The "obvious" way of running "conda info --json" does give
     # access to the active_prefix but takes about 2 seconds to run.
-    # The equivalent to the code above would be:
-    #    info_json = run_command(Commands.INFO, "--json")
-    # As a comporomise look for the env name in the path to the python
+    # As a compromise look for the env name in the path to the python
     # executable
     match = re.search(r"/envs/(.*?)/bin/", sys.executable)
     if match:
@@ -313,42 +517,63 @@ class Packages(dict):
         print("Extra packages compared to before:", pkgs.extra(old))
         print("Different packages: ", pkgs.difference(old))
         old.update(pkgs)  # Include any new packages in the old
-        old.write("/path/to/packages.pickle")
-
-    Parameters
-    ----------
-    packages : `dict`
-        A mapping {package: version} where both keys and values are type `str`.
+        old.write("/path/to/packages.pickle").
 
     Notes
     -----
     This is a wrapper around a dict with some convenience methods.
     """
 
-    formats = {".pkl": "pickle", ".pickle": "pickle", ".yaml": "yaml", ".json": "json"}
+    formats: ClassVar[dict[str, str]] = {
+        ".pkl": "pickle",
+        ".pickle": "pickle",
+        ".yaml": "yaml",
+        ".json": "json",
+    }
 
-    def __setstate__(self, state: Dict[str, Any]) -> None:
+    def __setstate__(self, state: dict[str, Any]) -> None:
         # This only seems to be called for old pickle files where
         # the data was stored in _packages.
         self.update(state["_packages"])
 
     @classmethod
-    def fromSystem(cls) -> Packages:
+    def fromSystem(cls, include_all: bool = False) -> Packages:
         """Construct a `Packages` by examining the system.
 
-        Determine packages by examining python's `sys.modules`, conda
+        Determine packages by examining python's installed packages
+        (by default filtered by `sys.modules`) or distributions, conda
         libraries and EUPS. EUPS packages take precedence over conda and
         general python packages.
 
+        Parameters
+        ----------
+        include_all : `bool`, optional
+            If `False`, will only include imported Python packages, installed
+            Conda packages and locally-setup EUPS packages. If `True` all
+            installed Python distributions and conda packages will be reported
+            as well as all EUPS packages that are set up.
+
         Returns
         -------
         packages : `Packages`
             All version package information that could be obtained.
+
+        Note
+        ----
+        The names of Python distributions can differ from the names of the
+        Python packages installed by those distributions. Since ``include_all``
+        set to `True` uses Python distributions and `False` uses Python
+        packages do not expect that the answers are directly comparable.
         """
         packages = {}
-        packages.update(getPythonPackages())
+        if include_all:
+            packages.update(getAllPythonDistributions())
+        else:
+            packages.update(getPythonPackages())
+        # Conda list always reports all Conda packages.
         packages.update(getCondaPackages())
-        packages.update(getEnvironmentPackages())  # Should be last, to override products with LOCAL versions
+        # Should be last, to override products with LOCAL versions
+        packages.update(getEnvironmentPackages(include_all=include_all))
         return cls(packages)
 
     @classmethod
@@ -438,7 +663,7 @@ class Packages(dict):
         filename : `str`
             Filename to which to write. The format of the data file
             is determined from the file extension. Currently supports
-            ``.pickle``, ``.json``, and ``.yaml``
+            ``.pickle``, ``.json``, and ``.yaml``.
         """
         _, ext = os.path.splitext(filename)
         if ext not in self.formats:
@@ -449,7 +674,7 @@ class Packages(dict):
             ff.write(self.toBytes(self.formats[ext]))
 
     def __str__(self) -> str:
-        ss = "%s({\n" % self.__class__.__name__
+        ss = self.__class__.__name__ + "({\n"
         # Sort alphabetically by module name, for convenience in reading
         ss += ",\n".join(f"{prod!r}:{self[prod]!r}" for prod in sorted(self))
         ss += ",\n})"
@@ -459,7 +684,7 @@ class Packages(dict):
         # Default repr() does not report the class name.
         return f"{self.__class__.__name__}({super().__repr__()})"
 
-    def extra(self, other: Mapping) -> Dict[str, str]:
+    def extra(self, other: Mapping) -> dict[str, str]:
         """Get packages in self but not in another `Packages` object.
 
         Parameters
@@ -475,7 +700,7 @@ class Packages(dict):
         """
         return {pkg: self[pkg] for pkg in self.keys() - other.keys()}
 
-    def missing(self, other: Mapping) -> Dict[str, str]:
+    def missing(self, other: Mapping) -> dict[str, str]:
         """Get packages in another `Packages` object but missing from self.
 
         Parameters
@@ -491,7 +716,7 @@ class Packages(dict):
         """
         return {pkg: other[pkg] for pkg in other.keys() - self.keys()}
 
-    def difference(self, other: Mapping) -> Dict[str, Tuple[str, str]]:
+    def difference(self, other: Mapping) -> dict[str, tuple[str, str]]:
         """Get packages in symmetric difference of self and another `Packages`
         object.
 
@@ -502,9 +727,9 @@ class Packages(dict):
 
         Returns
         -------
-        difference : `dict` [`str`, `tuple` [`str`, `str`]]
+        difference : `dict` [`str`, `tuple` [ `str`, `str` ]]
             Packages in symmetric difference.  Keys (type `str`) are package
-            names; values (type `tuple`[`str`, `str`]) are their versions.
+            names; values (type `tuple` [ `str`, `str` ]) are their versions.
         """
         return {pkg: (self[pkg], other[pkg]) for pkg in self.keys() & other.keys() if self[pkg] != other[pkg]}
 
@@ -519,10 +744,22 @@ class _BackwardsCompatibilityUnpickler(pickle.Unpickler):
     `~lsst.utils.packages.Packages` instance.
     """
 
-    def find_class(self, module: str, name: str) -> Type:
+    def find_class(self, module: str, name: str) -> type:
         """Return the class that should be used for unpickling.
 
         This is always known to be the class in this package.
+
+        Parameters
+        ----------
+        module : `str`
+            Ignored.
+        name : `str`
+            Ignored.
+
+        Returns
+        -------
+        `type` [`Packages`]
+            The Python type to use. Always returns `Packages`.
         """
         return Packages
 
@@ -530,20 +767,25 @@ class _BackwardsCompatibilityUnpickler(pickle.Unpickler):
 # Register YAML representers
 
 
-def pkg_representer(dumper: yaml.Dumper, data: Any) -> yaml.MappingNode:
+def _pkg_representer(dumper: yaml.Dumper, data: Any) -> yaml.MappingNode:
     """Represent Packages as a simple dict"""
     return dumper.represent_mapping("lsst.utils.packages.Packages", data, flow_style=None)
 
 
-yaml.add_representer(Packages, pkg_representer)
+yaml.add_representer(Packages, _pkg_representer)
 
 
-def pkg_constructor(loader: yaml.constructor.SafeConstructor, node: yaml.Node) -> Any:
+def _pkg_constructor(loader: yaml.constructor.SafeConstructor, node: yaml.Node) -> Any:
+    """Convert YAML representation back to Python class."""
     yield Packages(loader.construct_mapping(node, deep=True))  # type: ignore
 
 
-for loader in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader):
-    yaml.add_constructor("lsst.utils.packages.Packages", pkg_constructor, Loader=loader)
+for loader_str in ("Loader", "CLoader", "UnsafeLoader", "SafeLoader", "FullLoader"):
+    loader = getattr(yaml, loader_str, None)
+    if loader is None:
+        continue
+
+    yaml.add_constructor("lsst.utils.packages.Packages", _pkg_constructor, Loader=loader)
 
     # Register the old name with YAML.
-    yaml.add_constructor("lsst.base.Packages", pkg_constructor, Loader=loader)
+    yaml.add_constructor("lsst.base.Packages", _pkg_constructor, Loader=loader)

lsst/utils/plotting/__init__.py

@@ -0,0 +1,15 @@
+# This file is part of utils.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# Use of this source code is governed by a 3-clause BSD-style
+# license that can be found in the LICENSE file.
+"""General LSST Plotting Utilities."""
+
+from .figures import *
+from .limits import *
+from .publication_plots import *