lsst-pipe-base 29.2025.1300__py3-none-any.whl → 29.2025.1500__py3-none-any.whl

lsst/pipe/base/pipeline_graph/expressions.py

@@ -0,0 +1,271 @@
+# This file is part of pipe_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Expressions that resolve to subsets of pipelines.
+
+See :ref:`pipeline-graph-subset-expressions`.
+"""
+
+from __future__ import annotations
+
+__all__ = (
+    "DirectionNode",
+    "IdentifierNode",
+    "IntersectionNode",
+    "Node",
+    "NotNode",
+    "UnionNode",
+    "parse",
+)
+
+import dataclasses
+import functools
+from typing import TYPE_CHECKING, Any, Literal, TypeAlias
+
+from lsst.daf.butler.registry.queries.expressions.parser.ply import lex, yacc
+
+from ._exceptions import InvalidExpressionError
+
+if TYPE_CHECKING:
+    from lsst.daf.butler.registry.queries.expressions.parser.parserLex import LexToken
+    from lsst.daf.butler.registry.queries.expressions.parser.parserYacc import YaccProduction
+
+
+class _ParserLex:
+    @classmethod
+    def make_lexer(cls) -> Any:  # unspecified PLY type.
+        return lex.lex(object=cls())
+
+    tokens = (
+        "IDENTIFIER",
+        "LPAREN",
+        "RPAREN",
+        "NOT",
+        "UNION",
+        "INTERSECTION",
+        "LT",
+        "LE",
+        "GT",
+        "GE",
+    )
+
+    t_LPAREN = r"\("
+    t_RPAREN = r"\)"
+    t_NOT = "~"
+    t_UNION = r"\|"
+    t_INTERSECTION = "&"
+    t_LT = "<"
+    t_LE = "<="
+    t_GT = ">"
+    t_GE = ">="
+
+    # Identifiers are alphanumeric, and may have a T:, D:, or S: prefix.
+    def t_IDENTIFIER(self, t: LexToken) -> LexToken:
+        r"""([TDS]:)?\w+"""
+        t.type = "IDENTIFIER"
+        return t
+
+    # Ignore spaces and tables.
+    t_ignore = " \t"
+
+    def t_error(self, t: LexToken) -> LexToken:
+        raise InvalidExpressionError(
+            f"invalid token in expression near character {t.lexer.lexpos}: {t.value[0]!r}"
+        )
+
+
+class _ParserYacc:
+    def __init__(self) -> None:
+        self.parser = self._parser_factory()
+
+    @staticmethod
+    @functools.cache
+    def _parser_factory() -> Any:  # unspecified PLY type.
+        return yacc.yacc(module=_ParserYacc, write_tables=False, debug=False)
+
+    def parse(self, input: str) -> Node:
+        """Parse input expression and return the parsed tree object.
+
+        Parameters
+        ----------
+        input : `str`
+            Expression to parse.
+
+        Returns
+        -------
+        node : `Node`
+            Root of the parsed expression tree.
+        """
+        lexer = _ParserLex.make_lexer()
+        tree = self.parser.parse(input=input, lexer=lexer)
+        return tree
+
+    tokens = _ParserLex.tokens[:]
+
+    start = "expr"
+
+    precedence = (
+        ("left", "UNION"),
+        ("left", "INTERSECTION"),
+        ("right", "NOT", "LT", "LE", "GT", "GE"),
+    )
+
+    # Ruff wants 'noqa' on the doc line, pydocstyle wants it on the function.
+
+    @classmethod
+    def p_expr_union(cls, p: YaccProduction) -> None:  # noqa: D403
+        """expr : expr UNION expr"""  # noqa: D403
+        p[0] = UnionNode(lhs=p[1], rhs=p[3])
+
+    @classmethod
+    def p_expr_intersection(cls, p: YaccProduction) -> None:  # noqa: D403
+        """expr : expr INTERSECTION expr"""  # noqa: D403
+        p[0] = IntersectionNode(lhs=p[1], rhs=p[3])
+
+    @classmethod
+    def p_expr_not(cls, p: YaccProduction) -> None:  # noqa: D403
+        """expr : NOT expr"""  # noqa: D403
+        p[0] = NotNode(operand=p[2])
+
+    @classmethod
+    def p_expr_parens(cls, p: YaccProduction) -> None:  # noqa: D403
+        """expr : LPAREN expr RPAREN"""  # noqa: D403
+        p[0] = p[2]
+
+    @classmethod
+    def p_expr_inequality(cls, p: YaccProduction) -> None:  # noqa: D403
+        """expr : LT identifier
+        | LE identifier
+        | GT identifier
+        | GE identifier
+        """  # noqa: D403
+        p[0] = DirectionNode(operator=p[1], start=p[2])
+
+    @classmethod
+    def p_expr_identifier(cls, p: YaccProduction) -> None:  # noqa: D403
+        """expr : identifier"""  # noqa: D403
+        p[0] = p[1]
+
+    @classmethod
+    def p_identifier_qualified(cls, p: YaccProduction) -> None:  # noqa: D403, D401
+        """identifier : IDENTIFIER"""  # noqa: D403, D401
+        match p[1].split(":"):
+            case [qualifier, label]:
+                p[0] = IdentifierNode(qualifier=qualifier, label=label)
+            case [label]:
+                p[0] = IdentifierNode(qualifier=None, label=label)
+            case _:  # pragma: no cover
+                raise AssertionError("Unexpected identifier form.")
+
+    @classmethod
+    def p_error(cls, p: YaccProduction | None) -> None:
+        if p is None:
+            raise InvalidExpressionError("Expression ended unexpectedly.")
+        else:
+            raise InvalidExpressionError(f"Syntax error near character {p.lexpos}: {p.value!r}")
+
+
+@dataclasses.dataclass
+class IdentifierNode:
+    """A node that corresponds to a task label, dataset type name, or labeled
+    subset.
+    """
+
+    qualifier: Literal["T", "D", "S"] | None
+    """Qualiifier that indicates whether this is a task (T), dataset type (T),
+    or labeled subset (S).
+
+    Unqualified identifiers (`None`) must resolve unambiguously.
+    """
+
+    label: str
+    """Task label, dataset type name, or subset label."""
+
+
+@dataclasses.dataclass
+class DirectionNode:
+    """A node that represents the ancestors or descendents of a task label or
+    dataset type.
+    """
+
+    operator: Literal["<", ">", "<=", ">="]
+    """Which direction to traverse the graph ('>' for descendents, '<' for
+    ancestors), and whether to include the operand ('=') or not.
+    """
+
+    start: IdentifierNode
+    """Node at which to start the DAG traversal."""
+
+
+@dataclasses.dataclass
+class NotNode:
+    """A node that represents set inversion (including all elements not in the
+    operand).
+    """
+
+    operand: Node
+    """Node representing the set to invert."""
+
+
+@dataclasses.dataclass
+class UnionNode:
+    """Node representing a set union."""
+
+    lhs: Node
+    rhs: Node
+
+
+@dataclasses.dataclass
+class IntersectionNode:
+    """Node representing a set intersection."""
+
+    lhs: Node
+    rhs: Node
+
+
+def parse(expression: str) -> Node:
+    """Parse an expression into a `Node` tree.
+
+    Parameters
+    ----------
+    expression : `str`
+        String expression to parse.  See
+        :ref:`pipeline-graph-subset-expressions`.
+
+    Returns
+    -------
+    node
+        Root node of the parsed expression tree.
+
+    Raises
+    ------
+    InvalidExpressionError
+        Raised if the expression could not be parsed.
+    """
+    return _ParserYacc().parse(expression)
+
+
+Node: TypeAlias = IdentifierNode | DirectionNode | NotNode | UnionNode | IntersectionNode

lsst/pipe/base/pipeline_graph/visualization/__init__.py

@@ -34,3 +34,4 @@ from ._mermaid import *
 from ._options import *
 from ._printer import *
 from ._show import *
+from ._status_annotator import *

lsst/pipe/base/pipeline_graph/visualization/_formatting.py

@@ -29,17 +29,20 @@ from __future__ import annotations
 __all__ = ("GetNodeText", "format_dimensions", "format_task_class", "get_node_symbol")
 
 import itertools
+import re
 import textwrap
 from collections.abc import Iterator
 
 import networkx
 import networkx.algorithms.community
+from wcwidth import wcswidth  # type: ignore
 
 from lsst.daf.butler import DimensionGroup
 
 from .._nodes import NodeKey, NodeType
 from ._merge import MergedNodeKey
 from ._options import NodeAttributeOptions
+from ._status_annotator import DatasetTypeStatusInfo, NodeStatusOptions, StatusColors, TaskStatusInfo
 
 DisplayNodeKey = NodeKey | MergedNodeKey
 """Type alias for graph keys that may be original task, task init, or dataset
@@ -47,6 +50,42 @@ type keys, or a merge of several keys for display purposes.
 """
 
 
+def strip_ansi(s: str) -> str:
+    """Remove ANSI escape codes from a string, so that `wcswidth()` measures
+    the real visible width of the string.
+
+    Parameters
+    ----------
+    s : `str`
+        String to strip of ANSI escape codes.
+
+    Returns
+    -------
+    stripped : `str`
+        String with ANSI escape codes removed.
+    """
+    # ANSI escape sequence remover
+    ansi_escape = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
+    return ansi_escape.sub("", s)
+
+
+def render_segment(f: float) -> str:
+    """Convert a float into a string of blocks, rounding to the nearest whole
+    number of columns.
+
+    Parameters
+    ----------
+    f : `float`
+        Number of columns to fill.
+
+    Returns
+    -------
+    blocks : `str`
+        String of blocks filling the specified number of columns.
+    """
+    return "█" * round(f)
+
+
 def get_node_symbol(node: DisplayNodeKey, x: int | None = None) -> str:
     """Return a single-character symbol for a particular node type.
 
@@ -107,23 +146,134 @@ class GetNodeText:
         self.deferred: list[tuple[str, tuple[str, str], list[str]]] = []
 
     def __call__(self, node: DisplayNodeKey, x: int, style: tuple[str, str]) -> str:
+        """Return a line of text describing a node.
+
+        Parameters
+        ----------
+        node : `DisplayNodeKey`
+            Named tuple used as the node key.
+        x : `int`
+            Ignored; may be passed for compatibility with the `Printer` class's
+            ``get_text`` callback.
+        style : `tuple` [`str`, `str`]
+            Tuple of ANSI color codes for overflow markers.
+        """
         state = self.xgraph.nodes[node]
-        terms: list[str] = [f"{node}:" if self.options.has_details(node.node_type) else str(node)]
+        has_status = "status" in state
+
+        # Build description.
+        description = self._build_description(node, state)
+
+        # Possibly build progress bar to append to description.
+        progress_portion = ""
+        if has_status:
+            progress_portion = self.format_node_status(description, state["status"])
+
+        # Stitch together the final line, handling overflow if needed.
+        final_line = self._stitch_node_text(description, progress_portion, style)
+        return final_line
+
+    def _build_description(self, node: DisplayNodeKey, state: dict) -> str:
+        """Build the node description, possibly with additional details.
+
+        Parameters
+        ----------
+        node : `DisplayNodeKey`
+            Named tuple used as the node key.
+        state : `dict`
+            Node attributes.
+
+        Returns
+        -------
+        description : `str`
+            The node description.
+        """
+        terms = [f"{node}:" if self.options.has_details(node.node_type) else str(node)]
+        # Optionally append dimension info.
         if self.options.dimensions and node.node_type != NodeType.TASK_INIT:
             terms.append(self.format_dimensions(state["dimensions"]))
+
+        # Optionally append task class name.
         if self.options.task_classes and (
             node.node_type is NodeType.TASK or node.node_type is NodeType.TASK_INIT
         ):
             terms.append(self.format_task_class(state["task_class_name"]))
+
+        # Optionally append storage class name.
         if self.options.storage_classes and node.node_type is NodeType.DATASET_TYPE:
             terms.append(state["storage_class_name"])
+
         description = " ".join(terms)
-        if self.width and len(description) > self.width:
-            index = f"[{len(self.deferred) + 1}]"
-            self.deferred.append((index, style, terms))
-            return f"{description[: self.width - len(index) - 6]}...{style[0]}{index}{style[1]} "
+
         return description
 
+    def _stitch_node_text(self, description: str, progress_portion: str, style: tuple[str, str]) -> str:
+        """Make the final line of node text to display given the description
+        and possibly a progress portion, and handle overflow.
+
+        It measures the total width of the description and progress portion,
+        and if it exceeds the screen width, it truncates the description and
+        appends a footnote.
+
+        Parameters
+        ----------
+        description : `str`
+            The node description.
+        progress_portion : `str`
+            The progress portion of the node.
+        style : `tuple` [`str`, `str`]
+            Tuple of ANSI color codes for overflow markers.
+
+        Returns
+        -------
+        final_line : `str`
+            The final line of text to display.
+        """
+        final_line = f"{description}{progress_portion}" if progress_portion else description
+        total_len = wcswidth(strip_ansi(final_line))
+
+        if self.width and total_len > self.width:
+            overflow_index = f"[{len(self.deferred) + 1}]"
+            overflow_marker = f"...{style[0]}{overflow_index}{style[1]}"
+
+            avail_desc_width = (
+                self.width - wcswidth(strip_ansi(progress_portion)) - wcswidth(strip_ansi(overflow_marker))
+            )
+            if avail_desc_width < 0:
+                avail_desc_width = 0
+
+            truncated_desc = description[:avail_desc_width] + overflow_marker
+            self.deferred.append((overflow_index, style, [description]))
+
+            return f"{truncated_desc}{progress_portion}"
+        else:
+            return final_line
+
+    def format_node_status(self, description: str, status: TaskStatusInfo | DatasetTypeStatusInfo) -> str:
+        """Format the status of a task node.
+
+        Parameters
+        ----------
+        description : `str`
+            The node description.
+        status : `TaskStatusInfo` or `DatasetTypeStatusInfo`
+            Holds status information for a task or dataset type.
+
+        Returns
+        -------
+        formatted : `str`
+            The formatted status string.
+        """
+        if not isinstance(self.options.status, NodeStatusOptions):
+            raise ValueError(f"Invalid node status options: {self.options.status!r}.")
+
+        return format_node_status(
+            description,
+            self.options.status,
+            status,
+            self.width,
+        )
+
     def format_dimensions(self, dimensions: DimensionGroup) -> str:
         """Format the dimensions of a task or dataset type node.
 
@@ -232,3 +382,148 @@ def format_task_class(options: NodeAttributeOptions, task_class_name: str) -> st
         case False:
             return ""
     raise ValueError(f"Invalid display option for task_classes: {options.task_classes!r}.")
+
+
+def _build_progress_bar(
+    description: str,
+    prefix: str,
+    suffix: str,
+    segments: list[tuple[str, float]],
+    width: int | None,
+    colors: StatusColors,
+    min_bar_width: int,
+) -> str:
+    """Shared helper that constructs a multi-segment progress bar.
+
+    Parameters
+    ----------
+    description : `str`
+        Main node description (used for measuring available space).
+    prefix : `str`
+        Text before the bar (e.g. ' · 42%▕').
+    suffix : `str`
+        Text after the bar (e.g. '▏exp: 107 ...').
+    segments : `list` of `tuple` [`str`, `float`]
+        Each tuple is (color_code, fractionOfBarWidth). We'll create these
+        segments in sequence.
+    width : `int` or None
+        Overall maximum line width. None for unlimited. We'll still respect the
+        minimum bar width.
+    colors : `StatusColors`
+        An instance containing .reset and color fields.
+    min_bar_width : `int`
+        Minimum number of display columns for the bar.
+
+    Returns
+    -------
+    formatted : str
+        The assembled prefix + bar + suffix, sized with respect to the width.
+    """
+    used_len = wcswidth(strip_ansi(prefix)) + wcswidth(strip_ansi(suffix))
+    if width is not None:
+        bar_width = max(width - wcswidth(strip_ansi(description)) - used_len, min_bar_width)
+    else:
+        bar_width = min_bar_width
+
+    # Build bar from fractional segments.
+    bar_str = ""
+    total_cols = 0
+    for ansi_color, fraction in segments:
+        cols = round(bar_width * fraction)
+        bar_str += f"{ansi_color}{render_segment(cols)}{colors.reset}"
+        total_cols += cols
+
+    # Pad the bar to the full width.
+    if total_cols < bar_width:
+        bar_str += " " * (bar_width - total_cols)
+
+    return prefix + bar_str + suffix
+
+
+def format_node_status(
+    description: str,
+    status_options: NodeStatusOptions,
+    status: TaskStatusInfo | DatasetTypeStatusInfo,
+    width: int | None,
+) -> str:
+    """Build a progress bar for a task or dataset type node.
+
+    Parameters
+    ----------
+    description : `str`
+        Node description for measuring leftover columns.
+    status_options : `NodeStatusOptions`
+        Options for node status visualization.
+    status : `TaskStatusInfo` or `DatasetTypeStatusInfo`
+        Holds status information for a task or dataset type.
+    width : `int` or None
+        Overall width limit (None => unlimited).
+
+    Returns
+    -------
+    formatted : str
+        The final prefix + bar + suffix line.
+    """
+    import dataclasses
+
+    status_abbreviations = {
+        "expected": "exp",
+        "succeeded": "suc",
+        "failed": "fail",
+        "blocked": "blk",
+        "ready": "rdy",
+        "running": "run",
+        "wonky": "wnk",
+        "unknown": "unk",
+        "produced": "prd",
+    }
+
+    colors = status_options.colors
+    expected = status.expected
+    done = status.succeeded if isinstance(status, TaskStatusInfo) else status.produced
+    full_success = done == expected
+    status_lookup = dataclasses.asdict(status)
+
+    percent = 100.0 * done / expected if expected else 0.0
+    total = float(expected) if expected else 1.0
+    prefix = ""
+
+    if status_options.display_percent or status_options.display_counts:
+        if not status_options.visualize or (status_options.visualize and status_options.display_percent):
+            full_success_color = colors.succeeded if isinstance(status, TaskStatusInfo) else colors.produced
+            color_code = full_success_color if full_success else colors.failed
+            prefix += f"{color_code} ▶ {colors.reset}"
+        if status_options.display_percent:
+            pct = round(percent)
+            if pct == 100 and not full_success:
+                pct == 99  # Avoid showing 100% if not fully successful.
+            prefix += f"{pct}%"
+        if not status_options.visualize and status_options.display_percent and status_options.display_counts:
+            prefix += " | "
+
+    if status_options.visualize:
+        prefix += "▕"
+
+    suffix_parts = []
+    segments = []
+
+    for key, value in status_lookup.items():
+        if value is not None:
+            color_code = getattr(colors, key)
+            if status_options.display_counts:
+                label = status_abbreviations[key] if status_options.abbreviate else key
+                suffix_parts.append(f"{label}:{color_code}{value}{colors.reset}")
+            if key != "expected":
+                # Build a progress bar segment.
+                segments.append((color_code, value / total))
+
+    # Produce suffix from the parts.
+    suffix = "▏" if status_options.visualize else ""
+    suffix += " | ".join(suffix_parts)
+
+    if status_options.visualize:
+        return _build_progress_bar(
+            description, prefix, suffix, segments, width, colors, status_options.min_bar_width
+        )
+    else:
+        return f"{prefix}{suffix}"

lsst/pipe/base/pipeline_graph/visualization/_options.py

@@ -32,6 +32,7 @@ import dataclasses
 from typing import Literal
 
 from .._nodes import NodeType
+from ._status_annotator import NodeStatusOptions
 
 
 @dataclasses.dataclass
@@ -71,8 +72,8 @@ class NodeAttributeOptions:
     - `None`: context-dependent default behavior.
     """
 
-    def __bool__(self) -> bool:
-        return bool(self.dimensions or self.storage_classes or self.task_classes)
+    status: NodeStatusOptions | None
+    """Options for displaying execution status."""
 
     def has_details(self, node_type: NodeType) -> bool:
         """Check whether there is any information beyond the node name for a
@@ -93,7 +94,10 @@ class NodeAttributeOptions:
         else:
             return bool(self.dimensions or self.task_classes)
 
-    def checked(self, is_resolved: bool) -> NodeAttributeOptions:
+    def __bool__(self) -> bool:
+        return bool(self.dimensions or self.storage_classes or self.task_classes or self.status)
+
+    def checked(self, is_resolved: bool, has_status: bool = False) -> NodeAttributeOptions:
         """Check these options against a pipeline graph's resolution status and
         fill in defaults.
 
@@ -102,6 +106,9 @@ class NodeAttributeOptions:
         is_resolved : `bool`
             Whether the pipeline graph to be displayed is resolved
             (`PipelineGraph.is_fully_resolved`).
+        has_status : `bool`
+            Whether the pipeline graph to be displayed has status information.
+            Defaults to `False`.
 
         Returns
         -------
@@ -127,4 +134,5 @@ class NodeAttributeOptions:
                 self.task_classes if self.task_classes is not None else ("concise" if is_resolved else False)
             ),
             storage_classes=(self.storage_classes if self.storage_classes is not None else is_resolved),
+            status=self.status if has_status else None,
         )