lsst-utils 25.2023.600__py3-none-any.whl → 29.2025.4800__py3-none-any.whl

lsst/utils/__init__.py

@@ -10,12 +10,9 @@
 # license that can be found in the LICENSE file.
 """General LSST Utilities."""
 
-from ._forwarded import *
 from ._packaging import *
-from .backtrace import *
 from .deprecated import *
 from .doImport import *
-from .get_caller_name import *
 from .inheritDoc import *
 from .version import *
 from .wrappers import *

lsst/utils/_packaging.py

@@ -11,6 +11,8 @@
 
 """Functions to help find packages."""
 
+from __future__ import annotations
+
 __all__ = ("getPackageDir",)
 
 import os

lsst/utils/argparsing.py

@@ -0,0 +1,79 @@
+# This file is part of utils.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# Use of this source code is governed by a 3-clause BSD-style
+# license that can be found in the LICENSE file.
+#
+
+"""Utilities to help with argument parsing in command line interfaces."""
+
+from __future__ import annotations
+
+__all__ = ["AppendDict"]
+
+import argparse
+import copy
+from collections.abc import Mapping
+from typing import Any
+
+
+class AppendDict(argparse.Action):
+    """An action analogous to the built-in 'append' that appends to a `dict`
+    instead of a `list`.
+
+    Inputs are assumed to be strings in the form "key=value"; any input that
+    does not contain exactly one "=" character is invalid. If the default value
+    is non-empty, the default key-value pairs may be overwritten by values from
+    the command line.
+    """
+
+    def __init__(
+        self,
+        option_strings: str | list[str],
+        dest: str,
+        nargs: int | str | None = None,
+        const: Any | None = None,
+        default: Any | None = None,
+        type: type | None = None,
+        choices: Any | None = None,
+        required: bool = False,
+        help: str | None = None,
+        metavar: str | None = None,
+    ):
+        if default is None:
+            default = {}
+        if not isinstance(default, Mapping):
+            argname = option_strings if option_strings else metavar if metavar else dest
+            raise TypeError(f"Default for {argname} must be a mapping or None, got {default!r}.")
+        super().__init__(option_strings, dest, nargs, const, default, type, choices, required, help, metavar)
+
+    def __call__(
+        self, parser: argparse.ArgumentParser, namespace: Any, values: Any, option_string: str | None = None
+    ) -> None:
+        # argparse doesn't make defensive copies, so namespace.dest may be
+        # the same object as self.default. Do the copy ourselves and avoid
+        # modifying the object previously in namespace.dest.
+        mapping = copy.copy(getattr(namespace, self.dest))
+
+        # Sometimes values is a copy of default instead of an input???
+        if isinstance(values, Mapping):
+            mapping.update(values)
+        else:
+            # values may be either a string or list of strings, depending on
+            # nargs. Unsafe to test for Sequence, because a scalar string
+            # passes.
+            if not isinstance(values, list):
+                values = [values]
+            for value in values:
+                vars = value.split("=")
+                if len(vars) != 2:
+                    raise ValueError(f"Argument {value!r} does not match format 'key=value'.")
+                mapping[vars[0]] = vars[1]
+
+        # Other half of the defensive copy.
+        setattr(namespace, self.dest, mapping)

lsst/utils/classes.py

@@ -10,15 +10,16 @@
 # license that can be found in the LICENSE file.
 #
 
-"""Utilities to help with class creation.
-"""
+"""Utilities to help with class creation."""
 
 from __future__ import annotations
 
 __all__ = ["Singleton", "cached_getter", "immutable"]
 
 import functools
-from typing import Any, Callable, Dict, Type, TypeVar
+from collections.abc import Callable
+from threading import RLock
+from typing import Any, ClassVar, TypeVar
 
 
 class Singleton(type):
@@ -32,18 +33,25 @@ class Singleton(type):
     adjust state of the singleton.
     """
 
-    _instances: Dict[Type, Any] = {}
+    _instances: ClassVar[dict[type, Any]] = {}
+    # This lock isn't ideal because it is shared between all classes using this
+    # metaclass, but current use cases don't do long-running I/O during
+    # initialization so the performance impact should be low.  It must be an
+    # RLock instead of a regular Lock because one singleton class might try to
+    # instantiate another as part of its initialization.
+    __lock: ClassVar[RLock] = RLock()
 
     # Signature is intentionally not substitutable for type.__call__ (no *args,
     # **kwargs) to require classes that use this metaclass to have no
     # constructor arguments.
     def __call__(cls) -> Any:
-        if cls not in cls._instances:
-            cls._instances[cls] = super(Singleton, cls).__call__()
-        return cls._instances[cls]
+        with cls.__lock:
+            if cls not in cls._instances:
+                cls._instances[cls] = super().__call__()
+            return cls._instances[cls]
 
 
-_T = TypeVar("_T", bound="Type")
+_T = TypeVar("_T", bound=Any)
 
 
 def immutable(cls: _T) -> _T:
@@ -80,7 +88,7 @@ def immutable(cls: _T) -> _T:
 
     # mypy says the variable here has signature (str, Any) i.e. no "self";
     # I think it's just confused by descriptor stuff.
-    cls.__setattr__ = __setattr__  # type: ignore
+    cls.__setattr__ = __setattr__
 
     def __getstate__(self: _T) -> dict:  # noqa: N807
         # Disable default state-setting when unpickled.
@@ -112,6 +120,16 @@ def cached_getter(func: Callable[[_S], _R]) -> Callable[[_S], _R]:
     Only works on methods that take only ``self``
     as an argument, and returns the cached result on subsequent calls.
 
+    Parameters
+    ----------
+    func : `~collections.abc.Callable`
+        Method from which the result should be cached.
+
+    Returns
+    -------
+    `~collections.abc.Callable`
+        Decorated method.
+
     Notes
     -----
     This is intended primarily as a stopgap for Python 3.8's more sophisticated

lsst/utils/db_auth.py

@@ -0,0 +1,339 @@
+# This file is part of utils.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+import fnmatch
+import json
+import os
+import stat
+import urllib.parse
+
+import yaml
+
+__all__ = ["DbAuth", "DbAuthError", "DbAuthPermissionsError"]
+
+_DEFAULT_PATH = "~/.lsst/db-auth.yaml"
+_DEFAULT_ENVVAR = "LSST_DB_AUTH"
+_DEFAULT_CREDS_ENVVAR = "LSST_DB_AUTH_CREDENTIALS"
+
+
+class DbAuthError(RuntimeError):
+    """Exception raised when a problem has occurred retrieving database
+    authentication information.
+    """
+
+    pass
+
+
+class DbAuthNotFoundError(DbAuthError):
+    """Credentials file does not exist or no match was found in it."""
+
+
+class DbAuthPermissionsError(DbAuthError):
+    """Credentials file has incorrect permissions."""
+
+
+class DbAuth:
+    """Retrieves authentication information for database connections.
+
+    The authorization configuration is taken from the ``authList`` parameter
+    or a (group- and world-inaccessible) YAML file located at a path specified
+    by the given environment variable or at a default path location.
+
+    Parameters
+    ----------
+    path : `str` or `None`, optional
+        Path to configuration file, default path is ``~/.lsst/db-auth.yaml``.
+        The default is used if `None`.
+    envVar : `str` or `None`, optional
+        Name of environment variable pointing to configuration file, default is
+        ``LSST_DB_AUTH``. This is used if the environment variable is available
+        even if the path is specified. The default is used if `None`.
+    authList : `list` [`dict`] or `None`, optional
+        Authentication configuration. Used directly if given without referring
+        to the environment.
+    credsEnvVar : `str` or `None`, optional
+        Name of environment variable containing the authentication
+        configuration in JSON format. Default is ``LSST_DB_AUTH_CREDENTIALS``.
+        If the environment variable is defined, this takes priority over
+        reading credentials from file. The default is used if `None`.
+
+    Notes
+    -----
+    Defaults will be used if no option is provided. If provided ``authList``
+    will be used directly. The JSON credentials environment variable will
+    be used if defined, in preference to reading from a file even if overrides
+    are given for ``path`` and ``envVar``.
+    """
+
+    def __init__(
+        self,
+        path: str | None = _DEFAULT_PATH,
+        envVar: str | None = _DEFAULT_ENVVAR,
+        authList: list[dict[str, str]] | None = None,
+        credsEnvVar: str | None = _DEFAULT_CREDS_ENVVAR,
+    ):
+        if authList is not None:
+            self._db_auth_path = "<auth-list>"
+            self.authList = authList
+            return
+
+        # Credentials in JSON takes priority.
+        if jsonCreds := os.getenv(credsEnvVar or _DEFAULT_CREDS_ENVVAR):
+            try:
+                self.authList = json.loads(jsonCreds)
+            except json.JSONDecodeError as exc:
+                raise DbAuthError(
+                    f"Unable to load DbAuth configuration using JSON environment variable {credsEnvVar}"
+                ) from exc
+            self._db_auth_path = "<json-env-var>"
+            return
+
+        secretPath = os.getenv(envVar or _DEFAULT_ENVVAR, path or _DEFAULT_PATH)
+        secretPath = os.path.expanduser(secretPath)
+        if not os.path.isfile(secretPath):
+            raise DbAuthNotFoundError(f"No DbAuth configuration file: {secretPath}")
+        mode = os.stat(secretPath).st_mode
+        if mode & (stat.S_IRWXG | stat.S_IRWXO) != 0:
+            raise DbAuthPermissionsError(
+                f"DbAuth configuration file {secretPath} has incorrect permissions: {mode:o}"
+            )
+
+        try:
+            with open(secretPath) as secretFile:
+                self.authList = yaml.safe_load(secretFile)
+        except Exception as exc:
+            raise DbAuthError(f"Unable to load DbAuth configuration file: {secretPath}.") from exc
+        self._db_auth_path = secretPath
+
+    @property
+    def db_auth_path(self) -> str:
+        """The path to the secrets file used to load credentials (`str`)."""
+        return self._db_auth_path
+
+    # dialectname, host, and database are tagged as Optional only because other
+    # routines delegate to this one in order to raise a consistent exception
+    # for that condition.
+    def getAuth(
+        self,
+        dialectname: str | None,
+        username: str | None,
+        host: str | None,
+        port: int | str | None,
+        database: str | None,
+    ) -> tuple[str | None, str]:
+        """Retrieve a username and password for a database connection.
+
+        This function matches elements from the database connection URL with
+        glob-like URL patterns in a list of configuration dictionaries.
+
+        Parameters
+        ----------
+        dialectname : `str`
+            Database dialect, for example sqlite, mysql, postgresql, oracle,
+            or mssql.
+        username : `str` or None
+            Username from connection URL if present.
+        host : `str`
+            Host name from connection URL if present.
+        port : `str` or `int` or None
+            Port from connection URL if present.
+        database : `str`
+            Database name from connection URL.
+
+        Returns
+        -------
+        username: `str`
+            Username to use for database connection; same as parameter if
+            present.
+        password: `str`
+            Password to use for database connection.
+
+        Raises
+        ------
+        DbAuthError
+            Raised if the input is missing elements, an authorization
+            dictionary is missing elements, the authorization file is
+            misconfigured, or no matching authorization is found.
+
+        Notes
+        -----
+        The list of authorization configuration dictionaries is tested in
+        order, with the first matching dictionary used.  Each dictionary must
+        contain a ``url`` item with a pattern to match against the database
+        connection URL and a ``password`` item.  If no username is provided in
+        the database connection URL, the dictionary must also contain a
+        ``username`` item.
+
+        The ``url`` item must begin with a dialect and is not allowed to
+        specify dialect+driver.
+
+        Glob-style patterns (using "``*``" and "``?``" as wildcards) can be
+        used to match the host and database name portions of the connection
+        URL.  For the username, port, and database name portions, omitting them
+        from the pattern matches against any value in the connection URL.
+
+        Examples
+        --------
+        The connection URL
+        ``postgresql://user@host.example.com:5432/my_database`` matches against
+        the identical string as a pattern.  Other patterns that would match
+        include:
+
+        * ``postgresql://*``
+        * ``postgresql://*.example.com``
+        * ``postgresql://*.example.com/my_*``
+        * ``postgresql://host.example.com/my_database``
+        * ``postgresql://host.example.com:5432/my_database``
+        * ``postgresql://user@host.example.com/my_database``
+
+        Note that the connection URL
+        ``postgresql://host.example.com/my_database`` would not match against
+        the pattern ``postgresql://host.example.com:5432``, even if the default
+        port for the connection is 5432.
+        """
+        # Check inputs, squashing MyPy warnings that they're unnecessary
+        # (since they're only unnecessary if everyone else runs MyPy).
+        if dialectname is None or dialectname == "":
+            raise DbAuthError("Missing dialectname parameter")
+        if host is None or host == "":
+            raise DbAuthError("Missing host parameter")
+        if database is None or database == "":
+            raise DbAuthError("Missing database parameter")
+
+        for authDict in self.authList:
+            # Check for mandatory entries
+            if "url" not in authDict:
+                raise DbAuthError("Missing URL in DbAuth configuration")
+
+            # Parse pseudo-URL from db-auth.yaml
+            components = urllib.parse.urlparse(authDict["url"])
+
+            # Check for same database backend type/dialect
+            if components.scheme == "":
+                raise DbAuthError("Missing database dialect in URL: " + authDict["url"])
+
+            if "+" in components.scheme:
+                raise DbAuthError(
+                    "Authorization dictionary URLs should only specify "
+                    f"dialects, got: {components.scheme}. instead."
+                )
+
+            # dialect and driver are allowed in db string, since functionality
+            # could change. Connecting to a DB using different driver does not
+            # change dbname/user/pass and other auth info so we ignore it.
+            # https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls
+            dialect = dialectname.split("+")[0]
+            if dialect != components.scheme:
+                continue
+
+            # Check for same database name
+            if (
+                components.path != ""
+                and components.path != "/"
+                and not fnmatch.fnmatch(database, components.path.lstrip("/"))
+            ):
+                continue
+
+            # Check username
+            if components.username is not None:
+                if username is None or username == "":
+                    continue
+                if username != components.username:
+                    continue
+
+            # Check hostname
+            if components.hostname is None:
+                raise DbAuthError("Missing host in URL: " + authDict["url"])
+            if not fnmatch.fnmatch(host, components.hostname):
+                continue
+
+            # Check port
+            if components.port is not None and (port is None or str(port) != str(components.port)):
+                continue
+
+            # Don't override username from connection string
+            if username is not None and username != "":
+                return (username, authDict["password"])
+            else:
+                if "username" not in authDict:
+                    return (None, authDict["password"])
+                return (authDict["username"], authDict["password"])
+
+        raise DbAuthNotFoundError(
+            f"No matching DbAuth configuration for: ({dialectname}, {username}, {host}, {port}, {database})"
+        )
+
+    def getUrl(self, url: str) -> str:
+        """Fill in a username and password in a database connection URL.
+
+        This function parses the URL and calls `getAuth`.
+
+        Parameters
+        ----------
+        url : `str`
+            Database connection URL.
+
+        Returns
+        -------
+        url : `str`
+            Database connection URL with username and password.
+
+        Raises
+        ------
+        DbAuthError
+            Raised if the input is missing elements, an authorization
+            dictionary is missing elements, the authorization file is
+            misconfigured, or no matching authorization is found.
+
+        See Also
+        --------
+        getAuth : Retrieve authentication credentials.
+        """
+        components = urllib.parse.urlparse(url)
+        username, password = self.getAuth(
+            components.scheme,
+            components.username,
+            components.hostname,
+            components.port,
+            components.path.lstrip("/"),
+        )
+        hostname = components.hostname
+        assert hostname is not None
+        if ":" in hostname:  # ipv6
+            hostname = f"[{hostname}]"
+        assert username is not None
+        netloc = "{}:{}@{}".format(
+            urllib.parse.quote(username, safe=""), urllib.parse.quote(password, safe=""), hostname
+        )
+        if components.port is not None:
+            netloc += ":" + str(components.port)
+        return urllib.parse.urlunparse(
+            (
+                components.scheme,
+                netloc,
+                components.path,
+                components.params,
+                components.query,
+                components.fragment,
+            )
+        )

lsst/utils/deprecated.py

@@ -9,18 +9,21 @@
 # Use of this source code is governed by a 3-clause BSD-style
 # license that can be found in the LICENSE file.
 
+from __future__ import annotations
+
 __all__ = ["deprecate_pybind11", "suppress_deprecations"]
 
 import functools
 import unittest.mock
 import warnings
+from collections.abc import Iterator
 from contextlib import contextmanager
-from typing import Any, Iterator, Type
+from typing import Any
 
 import deprecated.sphinx
 
 
-def deprecate_pybind11(obj: Any, reason: str, version: str, category: Type[Warning] = FutureWarning) -> Any:
+def deprecate_pybind11(obj: Any, reason: str, version: str, category: type[Warning] = FutureWarning) -> Any:
     """Deprecate a pybind11-wrapped C++ interface function, method or class.
 
     This needs to use a pass-through Python wrapper so that
@@ -35,17 +38,17 @@ def deprecate_pybind11(obj: Any, reason: str, version: str, category: Type[Warni
     obj : function, method, or class
         The function, method, or class to deprecate.
     reason : `str`
-        Reason for deprecation, passed to `~deprecated.sphinx.deprecated`
+        Reason for deprecation, passed to `~deprecated.sphinx.deprecated`.
     version : 'str'
         Next major version in which the interface will be deprecated,
-        passed to `~deprecated.sphinx.deprecated`
+        passed to `~deprecated.sphinx.deprecated`.
     category : `Warning`
-        Warning category, passed to `~deprecated.sphinx.deprecated`
+        Warning category, passed to `~deprecated.sphinx.deprecated`.
 
     Returns
     -------
     obj : function, method, or class
-        Wrapped function, method, or class
+        Wrapped function, method, or class.
 
     Examples
     --------
@@ -73,7 +76,7 @@ def deprecate_pybind11(obj: Any, reason: str, version: str, category: Type[Warni
 
 
 @contextmanager
-def suppress_deprecations(category: Type[Warning] = FutureWarning) -> Iterator[None]:
+def suppress_deprecations(category: type[Warning] = FutureWarning) -> Iterator[None]:
     """Suppress warnings generated by `deprecated.sphinx.deprecated`.
 
     Naively, one might attempt to suppress these warnings by using

lsst/utils/doImport.py

@@ -9,14 +9,15 @@
 # Use of this source code is governed by a 3-clause BSD-style
 # license that can be found in the LICENSE file.
 
+from __future__ import annotations
+
 __all__ = ("doImport", "doImportType")
 
 import importlib
 import types
-from typing import List, Optional, Type, Union
 
 
-def doImport(importable: str) -> Union[types.ModuleType, Type]:
+def doImport(importable: str) -> types.ModuleType | type:
     """Import a python object given an importable string and return it.
 
     Parameters
@@ -43,26 +44,24 @@ def doImport(importable: str) -> Union[types.ModuleType, Type]:
     if not isinstance(importable, str):
         raise TypeError(f"Unhandled type of importable, val: {importable}")
 
-    def tryImport(
-        module: str, fromlist: List[str], previousError: Optional[str]
-    ) -> Union[types.ModuleType, Type]:
+    def tryImport(module: str, fromlist: list[str], previousError: str | None) -> types.ModuleType | type:
         pytype = importlib.import_module(module)
         # Can have functions inside classes inside modules
         for f in fromlist:
             try:
                 pytype = getattr(pytype, f)
-            except AttributeError:
+            except AttributeError as e:
                 extra = f"({previousError})" if previousError is not None else ""
                 raise ImportError(
                     f"Could not get attribute '{f}' from '{module}' when importing '{importable}' {extra}"
-                )
+                ) from e
         return pytype
 
     # Go through the import path attempting to load the module
     # and retrieve the class or function as an attribute. Shift components
     # from the module list to the attribute list until something works.
     moduleComponents = importable.split(".")
-    infileComponents: List[str] = []
+    infileComponents: list[str] = []
     previousError = None
 
     while moduleComponents:
@@ -88,7 +87,7 @@ def doImport(importable: str) -> Union[types.ModuleType, Type]:
     raise ModuleNotFoundError(f"Unable to import {importable!r} {extra}")
 
 
-def doImportType(importable: str) -> Type:
+def doImportType(importable: str) -> type:
     """Import a python type given an importable string and return it.
 
     Parameters