lsst-pipe-base 29.2025.1400__tar.gz → 29.2025.1600__tar.gz

{lsst_pipe_base-29.2025.1400/python/lsst_pipe_base.egg-info → lsst_pipe_base-29.2025.1600}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.1400
+Version: 29.2025.1600
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License
@@ -27,6 +27,7 @@ Requires-Dist: lsst-pex-config
 Requires-Dist: astropy
 Requires-Dist: pydantic<3.0,>=2
 Requires-Dist: networkx
+Requires-Dist: wcwidth
 Requires-Dist: pyyaml>=5.1
 Requires-Dist: numpy>=1.17
 Requires-Dist: frozendict

{lsst_pipe_base-29.2025.1400 → lsst_pipe_base-29.2025.1600}/doc/lsst.pipe.base/working-with-pipeline-graphs.rst

@@ -86,3 +86,102 @@ The export methods include:
    This is a `networkx.DiGraph`, because all dataset types that connect a pair of tasks are rolled into one edge, and edges have no state.
  - `~PipelineGraph.make_dataset_type_graph` exports just dataset type nodes; it is one "bipartite projection" of the full graph.
    This is a `networkx.DiGraph`, because all tasks that connect a pair of dataset types are rolled into one edge, and edges have no state.
+
+.. _pipeline-graph-subset-expressions:
+
+Pipeline graph subset expressions
+---------------------------------
+
+The `PipelineGraph.select` and `PipelineGraph.select_tasks` methods utilize a boolean expression language to select a subset of the tasks in a `PipelineGraph`.
+The language uses familiar set operators for union (``|``), intersection (``&``), and set-inversion (``~``), with the operands any of the following:
+
+- a task label
+- a task subset label
+- a dataset type name (resolves to the label of the producing task, or an empty set for overall inputs; may not be an init-output)
+- an ancestor or descendant search, starting from a task label or dataset type name (see below)
+- a nested expression.
+
+Parentheses may be used for grouping.
+
+Task labels, task subset labels, and dataset type names all appear as regular unquoted strings.
+In cases where dataset type name is the same as a task or task subset label, a prefix can be added to disambiguate: ``T:`` for task, ``D:`` for dataset type, and ``S:`` for task subset.
+
+An ancestor or descendant search uses ``<``, ``<=``, ``>``, and ``>=`` as *unary* operators, with the operands being task labels or dataset type names (which may be qualified with ``T:`` or ``D:``, respectively, as described above).
+For tasks these searches are straightforward:
+
+- ``<`` and ``<=`` select all tasks whose outputs are consumed by the operand task, recursively, with the operand task itself included only for ``<=``.
+
+- ``>`` and ``>=`` select all tasks that consume the outputs of the operand task, rescursively, with the operand task itself included only for ``>=``.
+
+Because the expressions are logically set operations on tasks, ancestor and descendant searches on dataset types work differently and are not quite symmetric:
+
+- ``<`` and ``<=`` act like an ancestor search on the task that produces the operand dataset type.
+  For overall inputs they yield empty sets.
+  Init-outputs are not permitted.
+
+- ``>`` and ``>=`` act like a union of descendant searches on all tasks that consume the operand dataset type.
+  This includes tasks that consume the operand dataset type as an init-input (this is the only context in which init-output dataset types can appear in expressions).
+  For ``>=`` only, the task that produces the operand dataset type is also included, but in this case it is an error for the operand to be an init-output.
+
+Note that these ancestor and descendant searches are not the only useful way to define the subset of a pipeline that is "before" or "after" a task; the ancestors ``<a`` of a task ``a`` are those that *must* be run before ``a``, while the inverse of the descendants ``~>=a`` are the tasks that *can* be run before ``a``.
+Similarly, the descendants ``>a`` of ``a`` are the tasks that can only be run after ``a``, while the inverse of the ancestors ``~<=a`` are all tasks that can be run after ``a``.
+
+Examples
+^^^^^^^^
+
+All tasks in subset ``s`` except task ``b``:
+
+.. code-block:: text
+
+  s & ~b
+
+All tasks in either subset ``r`` or subset ``s`` that would need to be re-run to pick up a change in the behavior of task ``a``:
+
+.. code-block:: text
+
+  (r | s) & >=a
+
+All tasks in subset ``s`` that need to be run to accept failures in task ``c`` as unrecoverable, after a previous run left some quanta of those tasks blocked:
+
+.. code-block:: text
+
+  s & >a
+
+All tasks needed to produce dataset type ``d`` or dataset type ``e``:
+
+.. code-block:: text
+
+  <d | <e
+
+All tasks except task ``a`` that can be run without producing dataset type ``f``:
+
+.. code-block:: text
+
+  ~a & ~>=f
+
+
+Formal grammar
+^^^^^^^^^^^^^^
+
+.. code-block:: bnf
+
+  <expression> ::= ~ <expression>
+                 | <expression> | <expression>
+                 | <expression> & <expression>
+                 | (<expression>)
+                 | S:<subset-label>
+                 | <subset-label>
+                 | < <node>
+                 | <= <node>
+                 | > <node>
+                 | <= <node>
+                 | <node>
+
+  <node> ::= T:<task-label>
+           | <task-label>
+           | D:<dataset-type-name>
+           | <dataset-type-name>
+
+Whitespace is ignored, but is not permitted before or after the ``:`` in qualified identifiers.
+
+The operator precedence in the absence of parenthesis is ``~``, ``&``, ``|`` ( highest to lowest).

{lsst_pipe_base-29.2025.1400 → lsst_pipe_base-29.2025.1600}/pyproject.toml

@@ -30,6 +30,7 @@ dependencies = [
     "astropy",
     "pydantic >=2,<3.0",
     "networkx",
+    "wcwidth",
     "pyyaml >= 5.1",
     "numpy >= 1.17",
     "frozendict",
@@ -205,4 +206,6 @@ exclude = [
     "^test_.*",  # Do not test docstrings in test code.
     '^commands\.',  # Click docstrings, not numpydoc
     '\._[a-zA-Z_]+$',  # Private methods.
+    '_ParserLex\.',  # Docstrings are not numpydoc
+    '_ParserYacc\.',  # Docstrings are not numpydoc
 ]

{lsst_pipe_base-29.2025.1400 → lsst_pipe_base-29.2025.1600}/python/lsst/pipe/base/all_dimensions_quantum_graph_builder.py

@@ -39,6 +39,8 @@ from collections import defaultdict
 from collections.abc import Iterable, Mapping
 from typing import TYPE_CHECKING, Any, TypeAlias, final
 
+import astropy.table
+
 from lsst.daf.butler import (
     Butler,
     DataCoordinate,
@@ -85,6 +87,11 @@ class AllDimensionsQuantumGraphBuilder(QuantumGraphBuilder):
         (sometimes catastrophically bad) query plan.
     bind : `~collections.abc.Mapping`, optional
         Variable substitutions for the ``where`` expression.
+    data_id_tables : `~collections.abc.Iterable` [ `astropy.table.Table` ],\
+            optional
+        Tables of data IDs to join in as constraints.  Missing dimensions that
+        are constrained by the ``where`` argument or pipeline data ID will be
+        filled in automatically.
     **kwargs
         Additional keyword arguments forwarded to `QuantumGraphBuilder`.
 
@@ -113,6 +120,7 @@ class AllDimensionsQuantumGraphBuilder(QuantumGraphBuilder):
         where: str = "",
         dataset_query_constraint: DatasetQueryConstraintVariant = DatasetQueryConstraintVariant.ALL,
         bind: Mapping[str, Any] | None = None,
+        data_id_tables: Iterable[astropy.table.Table] = (),
         **kwargs: Any,
     ):
         super().__init__(pipeline_graph, butler, **kwargs)
@@ -120,6 +128,7 @@ class AllDimensionsQuantumGraphBuilder(QuantumGraphBuilder):
         self.where = where
         self.dataset_query_constraint = dataset_query_constraint
         self.bind = bind
+        self.data_id_tables = list(data_id_tables)
 
     @timeMethod
     def process_subgraph(self, subgraph: PipelineGraph) -> QuantumGraphSkeleton:
@@ -194,6 +203,14 @@ class AllDimensionsQuantumGraphBuilder(QuantumGraphBuilder):
                 f"{self.where!r}, bind={self.bind!r})"
             )
             query = query.where(tree.subgraph.data_id, self.where, bind=self.bind)
+            # It's important for tables to be joined in last, so data IDs from
+            # pipeline and where can be used to fill in missing columns.
+            for table in self.data_id_tables:
+                # If this is from ctrl_mpexec's pipetask, it'll have added
+                # a filename to the metadata for us.
+                table_name = table.meta.get("filename", "unknown")
+                query_cmd.append(f"    query = query.join_data_coordinate_table(<{table_name}>)")
+                query = query.join_data_coordinate_table(table)
             self.log.verbose("Querying for data IDs via: %s", "\n".join(query_cmd))
             # Allow duplicates from common skypix overlaps to make some queries
             # run faster.

{lsst_pipe_base-29.2025.1400 → lsst_pipe_base-29.2025.1600}/python/lsst/pipe/base/graph/_loadHelpers.py

@@ -65,6 +65,7 @@ class LoadHelper(AbstractContextManager["LoadHelper"]):
     to upgrade them to the latest format before they can be used in
     production.
     """
+    fullRead: bool = False
 
     def __post_init__(self) -> None:
         self._resourceHandle: ResourceHandleProtocol | None = None
@@ -261,6 +262,9 @@ class LoadHelper(AbstractContextManager["LoadHelper"]):
     def __enter__(self) -> LoadHelper:
         if isinstance(self.uri, BinaryIO | BytesIO | BufferedRandom):
             self._resourceHandle = self.uri
+        elif self.fullRead:
+            local = self._exitStack.enter_context(self.uri.as_local())
+            self._resourceHandle = self._exitStack.enter_context(local.open("rb"))
         else:
             self._resourceHandle = self._exitStack.enter_context(self.uri.open("rb"))
         self._initialize()

{lsst_pipe_base-29.2025.1400 → lsst_pipe_base-29.2025.1600}/python/lsst/pipe/base/graph/graph.py

@@ -963,7 +963,7 @@ class QuantumGraph:
         """
         uri = ResourcePath(uri)
         if uri.getExtension() in {".qgraph"}:
-            with LoadHelper(uri, minimumVersion) as loader:
+            with LoadHelper(uri, minimumVersion, fullRead=(nodes is None)) as loader:
                 qgraph = loader.load(universe, nodes, graphID)
         else:
             raise ValueError(f"Only know how to handle files saved as `.qgraph`, not {uri}")
@@ -1230,7 +1230,7 @@ class QuantumGraph:
             being loaded or if the supplied uri does not point at a valid
             `QuantumGraph` save file.
         """
-        with LoadHelper(file, minimumVersion) as loader:
+        with LoadHelper(file, minimumVersion, fullRead=(nodes is None)) as loader:
             qgraph = loader.load(universe, nodes, graphID)
         if not isinstance(qgraph, QuantumGraph):
             raise TypeError(f"QuantumGraph file contains unexpected object type: {type(qgraph)}")

{lsst_pipe_base-29.2025.1400 → lsst_pipe_base-29.2025.1600}/python/lsst/pipe/base/pipeline.py

@@ -427,7 +427,7 @@ class Pipeline:
             if "," in label_subset:
                 if ".." in label_subset:
                     raise ValueError(
-                        "Can only specify a list of labels or a rangewhen loading a Pipline not both"
+                        "Can only specify a list of labels or a range when loading a Pipeline, not both."
                     )
                 args = {"labels": set(label_subset.split(","))}
             # labels supplied as a range

{lsst_pipe_base-29.2025.1400 → lsst_pipe_base-29.2025.1600}/python/lsst/pipe/base/pipelineIR.py

@@ -980,10 +980,19 @@ class PipelineIR:
             if extraTaskLabels := (labeled_subset.subset - pipeline.tasks.keys()):
                 match subsetCtrl:
                     case PipelineSubsetCtrl.DROP:
-                        pipeline.labeled_subsets.pop(label)
+                        del pipeline.labeled_subsets[label]
                     case PipelineSubsetCtrl.EDIT:
                         for extra in extraTaskLabels:
                             labeled_subset.subset.discard(extra)
+            elif subsetCtrl is PipelineSubsetCtrl.DROP and not labeled_subset.subset:
+                # When mode is DROP, also drop any subsets that were already
+                # empty.  This ensures we drop steps that were emptied-out by
+                # (earlier) imports with exclude in EDIT mode.  Note that we
+                # don't want to drop those steps when they're first excluded
+                # down to nothing, because the pipeline might be about to add
+                # new tasks back into them, and then we'd want to preserve the
+                # step definitions.
+                del pipeline.labeled_subsets[label]
 
         # remove any steps that correspond to removed subsets
         new_steps = []

{lsst_pipe_base-29.2025.1400 → lsst_pipe_base-29.2025.1600}/python/lsst/pipe/base/pipeline_graph/__main__.py

@@ -334,6 +334,7 @@ class DisplayArguments:
                 dimensions=args.dimensions,
                 task_classes=args.task_classes,
                 storage_classes=args.storage_classes,
+                status=None,
             ),
             merge_input_trees=args.merge_input_trees,
             merge_output_trees=args.merge_output_trees,

{lsst_pipe_base-29.2025.1400 → lsst_pipe_base-29.2025.1600}/python/lsst/pipe/base/pipeline_graph/_exceptions.py

@@ -31,6 +31,7 @@ __all__ = (
     "DuplicateOutputError",
     "EdgesChangedError",
     "IncompatibleDatasetTypeError",
+    "InvalidExpressionError",
     "InvalidStepsError",
     "PipelineDataCycleError",
     "PipelineGraphError",
@@ -102,5 +103,11 @@ class PipelineGraphExceptionSafetyError(PipelineGraphError):
     """
 
 
+class InvalidExpressionError(PipelineGraphError):
+    """Exception raised when a pipeline subset expression could not be parsed
+    or applied.
+    """
+
+
 class InvalidStepsError(PipelineGraphError):
     """Exception raised when the step definitions are invalid."""