lsst-utils 25.2023.600__py3-none-any.whl → 29.2025.4800__py3-none-any.whl

lsst/utils/inheritDoc.py

@@ -9,12 +9,15 @@
 # Use of this source code is governed by a 3-clause BSD-style
 # license that can be found in the LICENSE file.
 
+from __future__ import annotations
+
 __all__ = ("inheritDoc",)
 
-from typing import Callable, Type
+import inspect
+from collections.abc import Callable
 
 
-def inheritDoc(klass: Type) -> Callable:
+def inheritDoc(klass: type) -> Callable:
     """Extend existing documentation for a method that exists in another
     class and extend it with any additional documentation defined.
 
@@ -32,15 +35,40 @@ def inheritDoc(klass: Type) -> Callable:
     -------
     decorator : callable
         Intermediate decorator used in the documentation process.
+
+    Notes
+    -----
+    This method naively appends the doc string from the decorated method to the
+    doc string of the equivalent method from the given class. No attempt
+    is made to ensure that duplicated sections are merged together or
+    overwritten.
+
+    This decorator is not necessary to ensure that a parent doc string appears
+    in a subclass. Tools like ``pydoc`` and Sphinx handle that automatically.
+    This can, though, be used to ensure that a specific docstring from a
+    parent class appears if there is ambiguity from multiple inheritance.
     """
 
-    def tmpDecorator(method: Type) -> Callable:
+    def _tmpDecorator(method: type) -> Callable:
         """Update the documentation from a class with the same method."""
         methodName = method.__name__
         if not hasattr(klass, methodName):
             raise AttributeError(f"{klass} has no method named {methodName} to inherit from")
-        appendText = method.__doc__ or ""
-        method.__doc__ = getattr(klass, methodName).__doc__ + appendText
+
+        # To append reliably, the doc strings need to be cleaned to
+        # remove indents.
+        appendText = inspect.cleandoc(method.__doc__ or "")
+        parentText = inspect.cleandoc(getattr(klass, methodName).__doc__ or "")
+
+        if parentText:
+            if appendText:
+                # cleandoc() strips leading and trailing space so it is safe
+                # to add new lines.
+                parentText += "\n\n" + appendText
+            method.__doc__ = parentText
+        else:
+            # Do not update the doc string if there was no parent doc string.
+            pass
         return method
 
-    return tmpDecorator
+    return _tmpDecorator

lsst/utils/introspection.py

@@ -9,27 +9,40 @@
 # Use of this source code is governed by a 3-clause BSD-style
 # license that can be found in the LICENSE file.
 #
+"""Utilities relating to introspection in python."""
 
 from __future__ import annotations
 
-"""Utilities relating to introspection in python."""
-
-__all__ = ["get_class_of", "get_full_type_name", "get_instance_of", "get_caller_name"]
+__all__ = [
+    "find_outside_stacklevel",
+    "get_caller_name",
+    "get_class_of",
+    "get_full_type_name",
+    "get_instance_of",
+    "take_object_census",
+    "trace_object_references",
+]
 
 import builtins
+import collections
+import gc
 import inspect
+import itertools
+import sys
 import types
-from typing import Any, Type, Union
+import warnings
+from collections.abc import Set
+from typing import Any
 
 from .doImport import doImport, doImportType
 
 
-def get_full_type_name(cls: Any) -> str:
+def get_full_type_name(cls_: Any) -> str:
     """Return full type name of the supplied entity.
 
     Parameters
     ----------
-    cls : `type` or `object`
+    cls_ : `type` or `object`
         Entity from which to obtain the full name. Can be an instance
         or a `type`.
 
@@ -48,16 +61,16 @@ def get_full_type_name(cls: Any) -> str:
     """
     # If we have a module that needs to be converted directly
     # to a name.
-    if isinstance(cls, types.ModuleType):
-        return cls.__name__
+    if isinstance(cls_, types.ModuleType):
+        return cls_.__name__
     # If we have an instance we need to convert to a type
-    if not hasattr(cls, "__qualname__"):
-        cls = type(cls)
-    if hasattr(builtins, cls.__qualname__):
+    if not hasattr(cls_, "__qualname__"):
+        cls_ = type(cls_)
+    if hasattr(builtins, cls_.__qualname__):
         # Special case builtins such as str and dict
-        return cls.__qualname__
+        return cls_.__qualname__
 
-    real_name = cls.__module__ + "." + cls.__qualname__
+    real_name = cls_.__module__ + "." + cls_.__qualname__
 
     # Remove components with leading underscores
     cleaned_name = ".".join(c for c in real_name.split(".") if not c.startswith("_"))
@@ -72,13 +85,13 @@ def get_full_type_name(cls: Any) -> str:
 
         # The thing we imported should match the class we started with
         # despite the clean up. If it does not we return the real name
-        if test is not cls:
+        if test is not cls_:
             return real_name
 
     return cleaned_name
 
 
-def get_class_of(typeOrName: Union[Type, str]) -> Type:
+def get_class_of(typeOrName: type | str | types.ModuleType) -> type:
     """Given the type name or a type, return the python type.
 
     If a type name is given, an attempt will be made to import the type.
@@ -106,13 +119,13 @@ def get_class_of(typeOrName: Union[Type, str]) -> Type:
     if isinstance(typeOrName, str):
         cls = doImportType(typeOrName)
     else:
-        cls = typeOrName
-        if isinstance(cls, types.ModuleType):
+        if isinstance(typeOrName, types.ModuleType):
             raise TypeError(f"Can not get class of module {get_full_type_name(typeOrName)}")
+        cls = typeOrName
     return cls
 
 
-def get_instance_of(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) -> Any:
+def get_instance_of(typeOrName: type | str, *args: Any, **kwargs: Any) -> Any:
     """Given the type name or a type, instantiate an object of that type.
 
     If a type name is given, an attempt will be made to import the type.
@@ -121,7 +134,7 @@ def get_instance_of(typeOrName: Union[Type, str], *args: Any, **kwargs: Any) ->
     ----------
     typeOrName : `str` or Python class
         A string describing the Python class to load or a Python type.
-    args : `tuple`
+    *args : `tuple`
         Positional arguments to use pass to the object constructor.
     **kwargs
         Keyword arguments to pass to object constructor.
@@ -184,3 +197,256 @@ def get_caller_name(stacklevel: int = 2) -> str:
     if codename != "<module>":  # top level usually
         name.append(codename)  # function or a method
     return ".".join(name)
+
+
+def find_outside_stacklevel(
+    *module_names: str,
+    allow_modules: Set[str] = frozenset(),
+    allow_methods: Set[str] = frozenset(),
+    stack_info: dict[str, Any] | None = None,
+) -> int:
+    """Find the stacklevel for outside of the given module.
+
+    This can be used to determine the stacklevel parameter that should be
+    passed to log messages or warnings in order to make them appear to
+    come from external code and not this package.
+
+    Parameters
+    ----------
+    *module_names : `str`
+        The names of the modules to skip when calculating the relevant stack
+        level.
+    allow_modules : `set` [`str`]
+        Names that should not be skipped when calculating the stacklevel.
+        If the module name starts with any of the names in this set the
+        corresponding stacklevel is used.
+    allow_methods : `set` [`str`]
+        Method names that are allowed to be treated as "outside". Fully
+        qualified method names must match exactly. Method names without
+        path components will match solely the method name itself. On Python
+        3.10 fully qualified names are not supported.
+    stack_info : `dict` or `None`, optional
+        If given, the dictionary is filled with information from
+        the relevant stack frame. This can be used to form your own warning
+        message without having to call :func:`inspect.stack` yourself with
+        the stack level.
+
+    Returns
+    -------
+    stacklevel : `int`
+        The stacklevel to use matching the first stack frame outside of the
+        given module.
+
+    Examples
+    --------
+    .. code-block:: python
+
+        warnings.warn(
+            "A warning message", stacklevel=find_outside_stacklevel("lsst.daf")
+        )
+    """
+    if sys.version_info < (3, 11, 0):
+        short_names = {m for m in allow_methods if "." not in m}
+        if len(short_names) != len(allow_methods):
+            warnings.warn(
+                "Python 3.10 does not support fully qualified names in allow_methods. Dropping them.",
+                stacklevel=2,
+            )
+            allow_methods = short_names
+
+    need_full_names = any("." in m for m in allow_methods)
+
+    if stack_info is not None:
+        # Ensure it is empty when we start.
+        stack_info.clear()
+
+    stacklevel = -1
+    for i, s in enumerate(inspect.stack()):
+        # This function is never going to be the right answer.
+        if i == 0:
+            continue
+        module = inspect.getmodule(s.frame)
+        if module is None:
+            continue
+
+        if stack_info is not None:
+            stack_info["filename"] = s.filename
+            stack_info["lineno"] = s.lineno
+            stack_info["name"] = s.frame.f_code.co_name
+
+        if allow_methods:
+            code = s.frame.f_code
+            names = {code.co_name}  # The name of the function itself.
+            if need_full_names:
+                full_name = f"{module.__name__}.{code.co_qualname}"
+                names.add(full_name)
+            if names & allow_methods:
+                # Method name is allowed so we stop here.
+                del s
+                stacklevel = i
+                break
+
+        # Stack frames sometimes hang around so explicitly delete.
+        del s
+
+        if (
+            # The module does not match any of the skipped names.
+            not any(module.__name__.startswith(name) for name in module_names)
+            # This match is explicitly allowed to be treated as "outside".
+            or any(module.__name__.startswith(name) for name in allow_modules)
+        ):
+            # 0 will be this function.
+            # 1 will be the caller
+            # and so does not need adjustment.
+            stacklevel = i
+            break
+    else:
+        # The top can't be inside the module.
+        stacklevel = i
+
+    return stacklevel
+
+
+def take_object_census() -> collections.Counter[type]:
+    """Count the number of existing objects, by type.
+
+    The census is returned as a `~collections.Counter` object. Expected usage
+    involves taking the difference with a different `~collections.Counter` and
+    examining any changes.
+
+    Returns
+    -------
+    census : `collections.Counter` [`type`]
+        The number of objects found of each type.
+
+    Notes
+    -----
+    This function counts *all* Python objects in memory. To count only
+    reachable objects, run `gc.collect` first.
+    """
+    counts: collections.Counter[type] = collections.Counter()
+    for obj in gc.get_objects():
+        counts[type(obj)] += 1
+    return counts
+
+
+def trace_object_references(
+    target_class: type,
+    count: int = 5,
+    max_level: int = 10,
+) -> tuple[list[list], bool]:
+    """Find the chain(s) of references that make(s) objects of a class
+    reachable.
+
+    Parameters
+    ----------
+    target_class : `type`
+        The class whose objects need to be traced. This is typically a class
+        that is known to be leaking.
+    count : `int`, optional
+        The number of example objects to trace, if that many exist.
+    max_level : `int`, optional
+        The number of levels of references to trace. ``max_level=1`` means
+        finding only objects that directly refer to the examples.
+
+    Returns
+    -------
+    traces : `list` [`list`]
+        A sequence whose first element (index 0) is the set of example objects
+        of type ``target_class``, whose second element (index 1) is the set of
+        objects that refer to the examples, and so on. Contains at most
+        ``max_level + 1`` elements.
+    trace_complete : `bool`
+        `True` if the trace for all objects terminated in at most
+        ``max_level`` references, and `False` if more references exist.
+
+    Examples
+    --------
+    An example with two levels of references:
+
+    >>> from collections import namedtuple
+    >>> class Foo:
+    ...     pass
+    >>> holder = namedtuple("Holder", ["bar", "baz"])
+    >>> myholder = holder(bar={"object": Foo()}, baz=42)
+    >>> # In doctest, the trace extends up to the whole global dict
+    >>> # if you let it.
+    >>> trace_object_references(Foo, max_level=2)  # doctest: +ELLIPSIS
+    ... # doctest: +NORMALIZE_WHITESPACE
+    ([[<lsst.utils.introspection.Foo object at ...>],
+      [{'object': <lsst.utils.introspection.Foo object at ...>}],
+      [Holder(bar={'object': <lsst.utils.introspection.Foo object at ...>},
+              baz=42)]], False)
+    """
+
+    def class_filter(o: Any) -> bool:
+        return isinstance(o, target_class)
+
+    # set() would be more appropriate, but objects may not be hashable.
+    objs = list(itertools.islice(filter(class_filter, gc.get_objects()), count))
+    if objs:
+        return _recurse_trace(objs, remaining=max_level)
+    else:
+        return [objs], True
+
+
+def _recurse_trace(objs: list, remaining: int) -> tuple[list[list], bool]:
+    """Recursively find references to a set of objects.
+
+    Parameters
+    ----------
+    objs : `list`
+        The objects to trace.
+    remaining : `int`
+        The number of levels of references to trace.
+
+    Returns
+    -------
+    traces : `list` [`list`]
+        A sequence whose first element (index 0) is ``objs``, whose second
+        element (index 1) is the set of objects that refer to those, and so on.
+        Contains at most ``remaining + 1``.
+    trace_complete : `bool`
+        `True` if the trace for all objects terminated in at most
+        ``remaining`` references, and `False` if more references exist.
+    """
+    # Filter out our own references to the objects. This is needed to avoid
+    # circular recursion.
+    refs = _get_clean_refs(objs)
+
+    if refs:
+        if remaining > 1:
+            more_refs, complete = _recurse_trace(refs, remaining=remaining - 1)
+            more_refs.insert(0, objs)
+            return more_refs, complete
+        else:
+            more_refs = _get_clean_refs(refs)
+            return [objs, refs], (not more_refs)
+    else:
+        return [objs], True
+
+
+def _get_clean_refs(objects: list) -> list:
+    """Find references to a set of objects, excluding those needed to query
+    for references.
+
+    Parameters
+    ----------
+    objects : `list`
+        The objects to find references for.
+
+    Returns
+    -------
+    refs : `list`
+        The objects that refer to the elements of ``objects``, not counting
+        ``objects`` itself.
+    """
+    # Pre-create the tuple so we know its id() and can filter it out.
+    # This allows for difference in behavior between python 3.12 and 3.13
+    # when calling gc.get_referrers with multiple arguments.
+    objects_tuple = tuple(objects)
+    refs = gc.get_referrers(*objects_tuple)
+    ids_to_drop = {id(objects), id(objects_tuple)}
+    refs = [ref for ref in refs if id(ref) not in ids_to_drop]
+    refs = [ref for ref in refs if not type(ref).__name__.endswith("_iterator")]
+    return refs

lsst/utils/iteration.py

@@ -10,18 +10,18 @@
 # license that can be found in the LICENSE file.
 #
 
-from __future__ import annotations
-
 """Utilities relating to iterators."""
 
-__all__ = ["chunk_iterable", "ensure_iterable", "isplit"]
+from __future__ import annotations
+
+__all__ = ["chunk_iterable", "ensure_iterable", "isplit", "sequence_to_string"]
 
 import itertools
-from collections.abc import Mapping
-from typing import Any, Iterable, Iterator, Tuple, TypeVar
+from collections.abc import Iterable, Iterator, Mapping
+from typing import Any, TypeGuard, TypeVar
 
 
-def chunk_iterable(data: Iterable[Any], chunk_size: int = 1_000) -> Iterator[Tuple[Any, ...]]:
+def chunk_iterable(data: Iterable[Any], chunk_size: int = 1_000) -> Iterator[tuple[Any, ...]]:
     """Return smaller chunks of an iterable.
 
     Parameters
@@ -63,7 +63,7 @@ def ensure_iterable(a: Any) -> Iterable[Any]:
 
     Returns
     -------
-    i : `generator`
+    i : `~collections.abc.Iterable`
         Iterable version of the input value.
     """
     if isinstance(a, str):
@@ -107,3 +107,189 @@ def isplit(string: T, sep: T) -> Iterator[T]:
             return
         yield string[begin:end]
         begin = end + 1
+
+
+def _extract_numeric_suffix(s: str) -> tuple[str, int | None]:
+    """Extract the numeric suffix from a string.
+
+    Returns the prefix and the numeric suffix as an integer, if present.
+
+    For example:
+    'node1' -> ('node', 1)
+    'node' -> ('node', None)
+    'node123abc' -> ('node123abc', None)
+
+    Parameters
+    ----------
+    s : str
+        The string to extract the numeric suffix from.
+
+    Returns
+    -------
+    suffix : str
+        The numeric suffix of the string, if any.
+    """
+    index = len(s)
+    while index > 0 and s[index - 1].isdigit():
+        index -= 1
+    prefix = s[:index]
+    suffix = s[index:]
+    if suffix:
+        return prefix, int(suffix)
+    else:
+        return s, None
+
+
+def _add_pair_to_name(val_name: list[str], val0: int | str, val1: int | str, stride: int = 1) -> None:
+    """Format a pair of values (val0 and val1) and appends the result to
+    val_name.
+
+    This helper function takes the starting and ending values of a sequence
+    and formats them into a compact string representation, considering the
+    stride and whether the values are integers or strings with common
+    prefixes.
+
+    Parameters
+    ----------
+    val_name : List[str]
+        The list to append the formatted string to.
+    val0 : [int, str]
+        The starting value of the sequence.
+    val1 : [int, str]
+        The ending value of the sequence.
+    stride : int, optional
+        The stride or difference between consecutive numbers in the
+        sequence. Defaults to 1.
+    """
+    if isinstance(val0, str) and isinstance(val1, str):
+        prefix0, num_suffix0 = _extract_numeric_suffix(val0)
+        prefix1, num_suffix1 = _extract_numeric_suffix(val1)
+        if prefix0 == prefix1 and num_suffix0 is not None and num_suffix1 is not None:
+            if num_suffix0 == num_suffix1:
+                dvn = val0
+            else:
+                dvn = f"{val0}..{val1}"
+                if stride > 1:
+                    dvn += f":{stride}"
+        else:
+            dvn = val0 if val0 == val1 else f"{val0}^{val1}"
+    else:
+        sval0 = str(val0)
+        sval1 = str(val1)
+        if val0 == val1:
+            dvn = sval0
+        elif isinstance(val0, int) and isinstance(val1, int):
+            if val1 == val0 + stride:
+                dvn = f"{sval0}^{sval1}"
+            else:
+                dvn = f"{sval0}..{sval1}"
+                if stride > 1:
+                    dvn += f":{stride}"
+        else:
+            dvn = f"{sval0}^{sval1}"
+    val_name.append(dvn)
+
+
+def _is_list_of_ints(values: list[int | str]) -> TypeGuard[list[int]]:
+    """Check if a list is composed entirely of integers.
+
+    Parameters
+    ----------
+    values : List[int, str]:
+        The list of values to check.
+
+    Returns
+    -------
+    is_ints : bool
+        True if all values are integers, False otherwise.
+    """
+    return all(isinstance(v, int) for v in values)
+
+
+def sequence_to_string(values: list[int | str]) -> str:
+    """Convert a list of integers or strings into a compact string
+    representation by merging consecutive values or sequences.
+
+    This function takes a list of integers or strings, sorts them, identifies
+    sequences where consecutive numbers differ by a consistent stride, or
+    strings with common prefixes, and returns a string that compactly
+    represents these sequences. Consecutive numbers are merged into ranges, and
+    strings with common prefixes are handled to produce a concise
+    representation.
+
+    >>> getNameOfSet([1, 2, 3, 5, 7, 8, 9])
+    '1..3^5^7..9'
+    >>> getNameOfSet(["node1", "node2", "node3"])
+    'node1..node3'
+    >>> getNameOfSet([10, 20, 30, 40])
+    '10..40:10'
+
+    Parameters
+    ----------
+    values : list[int, str]:
+        A list of items to be compacted. Must all be of the same type.
+
+    Returns
+    -------
+    sequence_as_string : str
+        A compact string representation of the input list.
+
+    Notes
+    -----
+        - The function handles both integers and strings.
+        - For strings with common prefixes, only the differing suffixes are
+            considered.
+        - The stride is determined as the minimum difference between
+            consecutive numbers.
+        - Strings without common prefixes are listed individually.
+    """
+    if not values:
+        return ""
+
+    values = sorted(set(values))
+
+    pure_ints_or_pure_strings = all(isinstance(item, int) for item in values) or all(
+        isinstance(item, str) for item in values
+    )
+    if not pure_ints_or_pure_strings:
+        types = {type(item) for item in values}
+        raise TypeError(f"All items in the input list must be either integers or strings, got {types}")
+
+    # Determine the stride for integers
+    stride = 1
+    if len(values) > 1 and _is_list_of_ints(values):
+        differences = [values[i + 1] - values[i] for i in range(len(values) - 1)]
+        stride = min(differences) if differences else 1
+        stride = max(stride, 1)
+
+    val_name: list[str] = []
+    val0 = values[0]
+    val1 = val0
+    for val in values[1:]:
+        if isinstance(val, int):
+            assert isinstance(val1, int)
+            if val == val1 + stride:
+                val1 = val
+            else:
+                _add_pair_to_name(val_name, val0, val1, stride)
+                val0 = val
+                val1 = val0
+        elif isinstance(val, str):
+            assert isinstance(val1, str)
+            prefix1, num_suffix1 = _extract_numeric_suffix(val1)
+            prefix, num_suffix = _extract_numeric_suffix(val)
+            if prefix1 == prefix and num_suffix1 is not None and num_suffix is not None:
+                if num_suffix == num_suffix1 + stride:
+                    val1 = val
+                else:
+                    _add_pair_to_name(val_name, val0, val1)
+                    val0 = val
+                    val1 = val0
+            else:
+                _add_pair_to_name(val_name, val0, val1)
+                val0 = val
+                val1 = val0
+
+    _add_pair_to_name(val_name, val0, val1, stride)
+
+    return "^".join(val_name)