lsst-ctrl-bps 29.2025.3900__tar.gz → 29.2025.4000__tar.gz

{lsst_ctrl_bps-29.2025.3900/python/lsst_ctrl_bps.egg-info → lsst_ctrl_bps-29.2025.4000}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps
-Version: 29.2025.3900
+Version: 29.2025.4000
 Summary: Pluggable execution of workflow graphs from Rubin pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4000}/doc/lsst.ctrl.bps/quickstart.rst

@@ -157,7 +157,7 @@ or a pre-made file containing a serialized QuantumGraph, for example
 
 .. code-block:: YAML
 
-   qgraphFile: pipelines_check_w_2020_45.qgraph
+   qgraphFile: pipelines_check_w_2020_45.qg
 
 .. warning::
 
@@ -846,7 +846,7 @@ Supported settings
     When to output job QuantumGraph files (default = TRANSFORM).
 
     * NEVER = all jobs will use full QuantumGraph file. (Warning: make sure
-      runQuantumCommand has ``--qgraph-id {qgraphId} --qgraph-node-id {qgraphNodeId}``.)
+      runQuantumCommand has ``--qgraph-node-id {qgraphNodeId}``.)
     * TRANSFORM = Output QuantumGraph files after creating GenericWorkflow.
     * PREPARE = QuantumGraph files are output after creating WMS submission.
 
@@ -904,7 +904,7 @@ Reserved keywords
     However, contrary to YAML specification, it is currently not portable.
 
 **qgraphId**
-    Internal ID for the full QuantumGraph (passed as ``--qgraph-id`` on pipetask command line).
+    Ignored; accepted for backwards compatibility.
 
 **qgraphNodeId**
     Comma-separated list of internal QuantumGraph node numbers to be
@@ -1079,13 +1079,12 @@ single full QuantumGraph file plus node numbers for each job. The default is
 using per-job QuantumGraph files.
 
 To use full QuantumGraph file, the submit YAML must set ``whenSaveJobQgraph`` to
-"NEVER" and the ``pipetask run`` command must include ``--qgraph-id {qgraphId}
---qgraph-node-id {qgraphNodeId}``.  For example:
+"NEVER" and the ``pipetask run`` command must include ``--qgraph-node-id {qgraphNodeId}``.  For example:
 
 .. code::
 
     whenSaveJobQgraph: "NEVER"
-    runQuantumCommand: "${CTRL_MPEXEC_DIR}/bin/pipetask --long-log run -b {butlerConfig} --output {output} --output-run {outputRun} --qgraph {qgraphFile} --qgraph-id {qgraphId} --qgraph-node-id {qgraphNodeId} --skip-init-writes --extend-run --clobber-outputs --skip-existing"
+    runQuantumCommand: "${CTRL_MPEXEC_DIR}/bin/pipetask --long-log run -b {butlerConfig} --output {output} --output-run {outputRun} --qgraph {qgraphFile} --qgraph-node-id {qgraphNodeId} --skip-init-writes --extend-run --clobber-outputs --skip-existing"
 
 
 .. warning::
@@ -1244,7 +1243,7 @@ The major differences to users are:
     the output run in the provided pre-existing quantum graph.
   - ``final_post_finalJob.out``: An internal file for debugging incorrect
     reporting of final run status.
-  - ``<qgraph_filename>_orig.qgraph``: A backup copy of the original
+  - ``<qgraph_filename>_orig.qg``: A backup copy of the original
     pre-existing quantum graph file that was used for submitting the run.  Note
     that this file will *not* be present in the submit directory if the
     pipeline YAML specification was used during the submission instead.

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4000}/python/lsst/ctrl/bps/bps_utils.py

@@ -146,7 +146,7 @@ def create_job_quantum_graph_filename(config, job, out_prefix=None):
     found, subdir = config.search("subDirTemplate", opt={"curvals": curvals})
     if not found:
         subdir = "{job.label}"
-    full_filename = Path("inputs") / subdir / f"quantum_{job.name}.qgraph"
+    full_filename = Path("inputs") / subdir / f"quantum_{job.name}.qg"
 
     if out_prefix is not None:
         full_filename = Path(out_prefix) / full_filename

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4000}/python/lsst/ctrl/bps/clustered_quantum_graph.py

@@ -29,18 +29,21 @@
 a QuantumGraph.
 """
 
-__all__ = ["ClusteredQuantumGraph", "QuantaCluster"]
+from __future__ import annotations
 
+__all__ = ["ClusteredQuantumGraph", "QuantaCluster"]
 
 import logging
 import pickle
 import re
+import uuid
 from collections import Counter, defaultdict
 from pathlib import Path
 
 from networkx import DiGraph, is_directed_acyclic_graph, is_isomorphic, topological_sort
 
-from lsst.pipe.base import NodeId, QuantumGraph
+from lsst.pipe.base.pipeline_graph import TaskImportMode
+from lsst.pipe.base.quantum_graph import PredictedQuantumGraph, QuantumInfo
 from lsst.utils.iteration import ensure_iterable
 
 from .bps_draw import draw_networkx_dot
@@ -79,13 +82,17 @@ class QuantaCluster:
             self.tags = {}
 
     @classmethod
-    def from_quantum_node(cls, quantum_node, template):
-        """Create single quantum cluster from given quantum node.
+    def from_quantum_info(
+        cls, quantum_id: uuid.UUID, quantum_info: QuantumInfo, template: str
+    ) -> QuantaCluster:
+        """Create single quantum cluster from the given quantum information.
 
         Parameters
         ----------
-        quantum_node : `lsst.pipe.base.QuantumNode`
-            QuantumNode for which to make into a single quantum cluster.
+        quantum_id : `uuid.UUID`
+            ID of the quantum.
+        quantum_info : `lsst.pipe.base.quantum_graph.QuantumInfo`
+            Dictionary of additional information about the quantum.
         template : `str`
             Template for creating cluster name.
 
@@ -94,14 +101,13 @@ class QuantaCluster:
         cluster : `QuantaCluster`
             Newly created cluster containing the given quantum.
         """
-        label = quantum_node.taskDef.label
-        node_id = quantum_node.nodeId
-        data_id = quantum_node.quantum.dataId
+        label = quantum_info["task_label"]
+        data_id = quantum_info["data_id"]
 
         # Gather info for name template into a dictionary.
         info = dict(data_id.required)
         info["label"] = label
-        info["node_number"] = node_id
+        info["node_number"] = quantum_id
         _LOG.debug("template = %s", template)
         _LOG.debug("info for template = %s", info)
 
@@ -116,7 +122,7 @@ class QuantaCluster:
         _LOG.debug("template name = %s", name)
 
         cluster = QuantaCluster(name, label, info)
-        cluster.add_quantum(quantum_node.nodeId, label)
+        cluster.add_quantum(quantum_id, label)
         return cluster
 
     @property
@@ -130,24 +136,12 @@ class QuantaCluster:
         """Counts of Quanta per taskDef.label in this cluster."""
         return Counter(self._task_label_counts)
 
-    def add_quantum_node(self, quantum_node):
-        """Add a quantumNode to this cluster.
-
-        Parameters
-        ----------
-        quantum_node : `lsst.pipe.base.QuantumNode`
-            Quantum node to add.
-        """
-        _LOG.debug("quantum_node = %s", quantum_node)
-        _LOG.debug("quantum_node.nodeId = %s", quantum_node.nodeId)
-        self.add_quantum(quantum_node.nodeId, quantum_node.taskDef.label)
-
     def add_quantum(self, node_id, task_label):
         """Add a quantumNode to this cluster.
 
         Parameters
         ----------
-        node_id : `lsst.pipe.base.NodeId`
+        node_id : `uuid.UUID`
             ID for quantumNode to be added to cluster.
         task_label : `str`
             Task label for quantumNode to be added to cluster.
@@ -185,11 +179,10 @@ class ClusteredQuantumGraph:
     ----------
     name : `str`
         Name to be given to the ClusteredQuantumGraph.
-    qgraph : `lsst.pipe.base.QuantumGraph`
-        The QuantumGraph to be clustered.
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
+        The quantum graph to be clustered.
     qgraph_filename : `str`
-        Filename for given QuantumGraph if it has already been
-        serialized.
+        Filename for given quantum graph.
 
     Raises
     ------
@@ -203,11 +196,12 @@ class ClusteredQuantumGraph:
     use API over totally minimized memory usage.
     """
 
-    def __init__(self, name, qgraph, qgraph_filename=None):
+    def __init__(self, name: str, qgraph: PredictedQuantumGraph, qgraph_filename: str):
         if "/" in name:
             raise ValueError(f"name cannot have a / ({name})")
         self._name = name
         self._quantum_graph = qgraph
+        self._quantum_only_xgraph = qgraph.quantum_only_xgraph
         self._quantum_graph_filename = Path(qgraph_filename).resolve()
         self._cluster_graph = DiGraph()
 
@@ -228,22 +222,27 @@ class ClusteredQuantumGraph:
             return False
         if len(self) != len(other):
             return False
-        return self._quantum_graph == other._quantum_graph and is_isomorphic(
+        return is_isomorphic(self.qxgraph, other.qxgraph) and is_isomorphic(
             self._cluster_graph, other._cluster_graph
         )
 
     @property
-    def name(self):
+    def name(self) -> str:
         """The name of the ClusteredQuantumGraph."""
         return self._name
 
     @property
-    def qgraph(self):
-        """The QuantumGraph associated with this Clustered
+    def qgraph(self) -> PredictedQuantumGraph:
+        """The quantum graph associated with this Clustered
         QuantumGraph.
         """
         return self._quantum_graph
 
+    @property
+    def qxgraph(self) -> DiGraph:
+        """A networkx graph of all quanta."""
+        return self._quantum_only_xgraph
+
     def add_cluster(self, clusters_for_adding):
         """Add a cluster of quanta as a node in the graph.
 
@@ -286,30 +285,26 @@ class ClusteredQuantumGraph:
             raise KeyError(f"{self.name} does not have a cluster named {name}") from ex
         return attr["cluster"]
 
-    def get_quantum_node(self, id_):
-        """Retrieve a QuantumNode from the ClusteredQuantumGraph by ID.
+    def get_quantum_info(self, id_: uuid.UUID) -> QuantumInfo:
+        """Retrieve a quantum info dict from the ClusteredQuantumGraph by ID.
 
         Parameters
         ----------
-        id_ : `lsst.pipe.base.NodeId` or int
-            ID of the QuantumNode to retrieve.
+        id_ : `uuid.UUID`
+            ID of the quantum to retrieve.
 
         Returns
         -------
-        quantum_node : `lsst.pipe.base.QuantumNode`
-            QuantumNode matching given ID.
+        quantum_info : `lsst.pipe.base.quantum_graph.QuantumInfo`
+            Quantum info dictionary for the given ID.
 
         Raises
         ------
         KeyError
             Raised if the ClusteredQuantumGraph does not contain
-            a QuantumNode with given ID.
+            a quantum with given ID.
         """
-        node_id = id_
-        if isinstance(id_, int):
-            node_id = NodeId(id, self._quantum_graph.graphID)
-        _LOG.debug("get_quantum_node: node_id = %s", node_id)
-        return self._quantum_graph.getQuantumNodeByNodeId(node_id)
+        return self._quantum_only_xgraph.nodes[id_]
 
     def __iter__(self):
         """Iterate over names of clusters.
@@ -414,8 +409,8 @@ class ClusteredQuantumGraph:
 
     def save(self, filename, format_=None):
         """Save the ClusteredQuantumGraph in a format that is loadable.
-        The QuantumGraph is saved separately if hasn't already been
-        serialized.
+
+        The quantum graph is assumed to have been saved separately.
 
         Parameters
         ----------
@@ -433,14 +428,6 @@ class ClusteredQuantumGraph:
         if format_ not in {"pickle"}:
             raise RuntimeError(f"Unknown format ({format_})")
 
-        if not self._quantum_graph_filename:
-            # Create filename based on given ClusteredQuantumGraph filename
-            self._quantum_graph_filename = path.with_suffix(".qgraph")
-
-        # If QuantumGraph file doesn't already exist, save it:
-        if not Path(self._quantum_graph_filename).exists():
-            self._quantum_graph.saveUri(self._quantum_graph_filename)
-
         if format_ == "pickle":
             # Don't save QuantumGraph in same file.
             tmp_qgraph = self._quantum_graph
@@ -503,14 +490,14 @@ class ClusteredQuantumGraph:
         cgraph = None
         if format_ == "pickle":
             with open(filename, "rb") as fh:
-                cgraph = pickle.load(fh)
+                cgraph: ClusteredQuantumGraph = pickle.load(fh)
 
             # The QuantumGraph was saved separately
-            try:
-                cgraph._quantum_graph = QuantumGraph.loadUri(cgraph._quantum_graph_filename)
-            except FileNotFoundError:  # Try same path as ClusteredQuantumGraph
-                new_filename = path.parent / Path(cgraph._quantum_graph_filename).name
-                cgraph._quantum_graph = QuantumGraph.loadUri(new_filename)
+            with PredictedQuantumGraph.open(
+                cgraph._quantum_graph_filename, import_mode=TaskImportMode.DO_NOT_IMPORT
+            ) as reader:
+                reader.read_thin_graph()
+                cgraph._quantum_graph = reader.finish()
 
         return cgraph
 

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4000}/python/lsst/ctrl/bps/drivers.py

@@ -50,7 +50,7 @@ import logging
 import os
 from pathlib import Path
 
-from lsst.pipe.base import QuantumGraph
+from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
 from lsst.utils.timer import time_this
 from lsst.utils.usage import get_peak_mem_usage
 
@@ -111,7 +111,7 @@ def _init_submission_driver(config_file: str, **kwargs) -> BpsConfig:
     return config
 
 
-def acquire_qgraph_driver(config_file: str, **kwargs) -> tuple[BpsConfig, QuantumGraph]:
+def acquire_qgraph_driver(config_file: str, **kwargs) -> tuple[BpsConfig, PredictedQuantumGraph]:
     """Read a quantum graph from a file or create one from pipeline definition.
 
     Parameters
@@ -125,7 +125,7 @@ def acquire_qgraph_driver(config_file: str, **kwargs) -> tuple[BpsConfig, Quantu
     -------
     config : `lsst.ctrl.bps.BpsConfig`
         Updated configuration.
-    qgraph : `lsst.pipe.base.graph.QuantumGraph`
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
         A graph representing quanta.
     """
     config = _init_submission_driver(config_file, **kwargs)

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4000}/python/lsst/ctrl/bps/etc/bps_defaults.yaml

@@ -1,6 +1,6 @@
 #USER pipelineYaml: "${OBS_SUBARU_DIR}/pipelines/DRP.yaml#processCcd"
 #  OR
-#USER qgraphFile: "/path/to/existing/file.qgraph"
+#USER qgraphFile: "/path/to/existing/file.qg"
 
 
 # At minimum, following group used in bps report and can be
@@ -94,7 +94,7 @@ clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.single_quantum_clusteri
 
 # Templates for bps filenames
 submitPath: ${PWD}/submit/{outputRun}
-qgraphFileTemplate: "{uniqProcName}.qgraph"
+qgraphFileTemplate: "{uniqProcName}.qg"
 subDirTemplate: "{label}/{tract}/{patch}/{band}/{subfilter}/{physical_filter}/{visit}/{exposure}"
 templateDataId: "{tract}_{patch}_{band}_{visit}_{exposure}_{detector}"
 

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4000}/python/lsst/ctrl/bps/pre_transform.py

@@ -38,7 +38,10 @@ import subprocess
 from pathlib import Path
 
 from lsst.ctrl.bps import BpsConfig, BpsSubprocessError
-from lsst.pipe.base.graph import QuantumGraph
+from lsst.pipe.base import QuantumGraph
+from lsst.pipe.base.pipeline_graph import TaskImportMode
+from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
+from lsst.resources import ResourcePath
 from lsst.utils import doImport
 from lsst.utils.logging import VERBOSE
 from lsst.utils.timer import time_this, timeMethod
@@ -47,7 +50,7 @@ _LOG = logging.getLogger(__name__)
 
 
 @timeMethod(logger=_LOG, logLevel=VERBOSE)
-def acquire_quantum_graph(config: BpsConfig, out_prefix: str = "") -> tuple[str, QuantumGraph]:
+def acquire_quantum_graph(config: BpsConfig, out_prefix: str = "") -> tuple[str, PredictedQuantumGraph]:
     """Read a quantum graph from a file or create one from scratch.
 
     Parameters
@@ -62,8 +65,8 @@ def acquire_quantum_graph(config: BpsConfig, out_prefix: str = "") -> tuple[str,
     -------
     qgraph_filename : `str`
         Name of file containing QuantumGraph that was read into qgraph.
-    qgraph : `lsst.pipe.base.graph.QuantumGraph`
-        A QuantumGraph read in from pre-generated file or one that is the
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
+        A quantum graph read in from pre-generated file or one that is the
         result of running code that generates it.
     """
     # Check to see if user provided pre-generated QuantumGraph.
@@ -90,8 +93,15 @@ def acquire_quantum_graph(config: BpsConfig, out_prefix: str = "") -> tuple[str,
 
     _LOG.info("Reading quantum graph from '%s'", qgraph_filename)
     with time_this(log=_LOG, level=logging.INFO, prefix=None, msg="Completed reading quantum graph"):
-        qgraph = QuantumGraph.loadUri(qgraph_filename)
-
+        qgraph_path = ResourcePath(qgraph_filename)
+        if qgraph_path.getExtension() == ".qg":
+            with PredictedQuantumGraph.open(qgraph_path, import_mode=TaskImportMode.DO_NOT_IMPORT) as reader:
+                reader.read_thin_graph()
+                qgraph = reader.finish()
+        elif qgraph_path.getExtension() == ".qgraph":
+            qgraph = PredictedQuantumGraph.from_old_quantum_graph(QuantumGraph.loadUri(qgraph_path))
+        else:
+            raise ValueError(f"Unrecognized extension for quantum graph file: {qgraph_filename}.")
     return qgraph_filename, qgraph
 
 
@@ -244,8 +254,8 @@ def cluster_quanta(config, qgraph, name):
     ----------
     config : `lsst.ctrl.bps.BpsConfig`
         BPS configuration.
-    qgraph : `lsst.pipe.base.QuantumGraph`
-        Original full QuantumGraph for the run.
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
+        Original full quantum graph for the run.
     name : `str`
         Name for the ClusteredQuantumGraph that will be generated.
 

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4000}/python/lsst/ctrl/bps/quantum_clustering_funcs.py

@@ -31,28 +31,31 @@ __all__ = ["check_clustering_config"]
 
 import logging
 import re
+import uuid
 from collections import defaultdict
 from typing import Any
 from uuid import UUID
 
 from networkx import DiGraph, NetworkXNoCycle, find_cycle, topological_sort
 
-from lsst.pipe.base import QuantumGraph, QuantumNode
+from lsst.pipe.base.quantum_graph import PredictedQuantumGraph, QuantumInfo
 
 from . import BpsConfig, ClusteredQuantumGraph, QuantaCluster
 
 _LOG = logging.getLogger(__name__)
 
 
-def single_quantum_clustering(config: BpsConfig, qgraph: QuantumGraph, name: str) -> ClusteredQuantumGraph:
+def single_quantum_clustering(
+    config: BpsConfig, qgraph: PredictedQuantumGraph, name: str
+) -> ClusteredQuantumGraph:
     """Create clusters with only single quantum.
 
     Parameters
     ----------
     config : `lsst.ctrl.bps.BpsConfig`
         BPS configuration.
-    qgraph : `lsst.pipe.base.QuantumGraph`
-        QuantumGraph to break into clusters for ClusteredQuantumGraph.
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
+        Quantum graph to break into clusters for ClusteredQuantumGraph.
     name : `str`
         Name to give to ClusteredQuantumGraph.
 
@@ -76,18 +79,19 @@ def single_quantum_clustering(config: BpsConfig, qgraph: QuantumGraph, name: str
     cached_template = {}
 
     # Create cluster of single quantum.
-    for qnode in qgraph:
-        if qnode.taskDef.label not in cached_template:
+    for quantum_id, quantum_info in cqgraph.qxgraph.nodes.items():
+        task_label = quantum_info["task_label"]
+        if task_label not in cached_template:
             found, template_data_id = config.search(
                 "templateDataId",
-                opt={"curvals": {"curr_pipetask": qnode.taskDef.label}, "replaceVars": False},
+                opt={"curvals": {"curr_pipetask": task_label}, "replaceVars": False},
             )
             if found:
                 template = "{label}_" + template_data_id
                 _, use_node_number = config.search(
                     "useNodeIdInClusterName",
                     opt={
-                        "curvals": {"curr_pipetask": qnode.taskDef.label},
+                        "curvals": {"curr_pipetask": task_label},
                         "replaceVars": False,
                         "default": True,
                     },
@@ -96,21 +100,21 @@ def single_quantum_clustering(config: BpsConfig, qgraph: QuantumGraph, name: str
                     template = "{node_number}_" + template
             else:
                 template = "{node_number}"
-            cached_template[qnode.taskDef.label] = template
+            cached_template[task_label] = template
 
-        cluster = QuantaCluster.from_quantum_node(qnode, cached_template[qnode.taskDef.label])
+        cluster = QuantaCluster.from_quantum_info(quantum_id, quantum_info, cached_template[task_label])
 
         # Save mapping for use when creating dependencies.
-        number_to_name[qnode.nodeId] = cluster.name
+        number_to_name[quantum_id] = cluster.name
 
         cqgraph.add_cluster(cluster)
 
     # Add cluster dependencies.
-    for qnode in qgraph:
+    for quantum_id in cqgraph.qxgraph:
         # Get child nodes.
-        children = qgraph.determineOutputsOfQuantumNode(qnode)
+        children = cqgraph.qxgraph.successors(quantum_id)
         for child in children:
-            cqgraph.add_dependency(number_to_name[qnode.nodeId], number_to_name[child.nodeId])
+            cqgraph.add_dependency(number_to_name[quantum_id], number_to_name[child])
 
     return cqgraph
 
@@ -209,15 +213,17 @@ def check_clustering_config(
     return list(topological_sort(clustered_task_graph)), ordered_tasks
 
 
-def dimension_clustering(config: BpsConfig, qgraph: QuantumGraph, name: str) -> ClusteredQuantumGraph:
+def dimension_clustering(
+    config: BpsConfig, qgraph: PredictedQuantumGraph, name: str
+) -> ClusteredQuantumGraph:
     """Follow config instructions to make clusters based upon dimensions.
 
     Parameters
     ----------
     config : `lsst.ctrl.bps.BpsConfig`
         BPS configuration.
-    qgraph : `lsst.pipe.base.QuantumGraph`
-        QuantumGraph to break into clusters for ClusteredQuantumGraph.
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
+        Quantum graph to break into clusters for ClusteredQuantumGraph.
     name : `str`
         Name to give to ClusteredQuantumGraph.
 
@@ -264,7 +270,7 @@ def dimension_clustering(config: BpsConfig, qgraph: QuantumGraph, name: str) ->
 def add_clusters_per_quantum(
     config: BpsConfig,
     label: str,
-    qgraph: QuantumGraph,
+    qgraph: PredictedQuantumGraph,
     cqgraph: ClusteredQuantumGraph,
     quantum_to_cluster: dict[UUID, str],
 ) -> None:
@@ -276,8 +282,8 @@ def add_clusters_per_quantum(
         BPS configuration.
     label : `str`
         The taskDef label for which to add clusters.
-    qgraph : `lsst.pipe.base.QuantumGraph`
-        QuantumGraph providing quanta for the clusters.
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
+        Quantum graph providing quanta for the clusters.
     cqgraph : `lsst.ctrl.bps.ClusteredQuantumGraph`
         The ClusteredQuantumGraph to which the new 1-quantum
         clusters are added (modified in method).
@@ -304,16 +310,10 @@ def add_clusters_per_quantum(
     else:
         template = "{node_number}"
 
-    # Currently getQuantaForTask is currently a mapping taskDef to
-    # Quanta, so quick enough to call repeatedly.
-    task_def = qgraph.findTaskDefByLabel(label)
-    assert task_def is not None, f"Given taskDef label ({label}) not found in QuantumGraph"  # for mypy
-    quantum_nodes = qgraph.getNodesForTask(task_def)
-
-    for qnode in quantum_nodes:
-        cluster = QuantaCluster.from_quantum_node(qnode, template)
+    for quantum_id in qgraph.quanta_by_task[label].values():
+        cluster = QuantaCluster.from_quantum_info(quantum_id, cqgraph.qxgraph.nodes[quantum_id], template)
         cqgraph.add_cluster(cluster)
-        quantum_to_cluster[qnode.nodeId] = cluster.name
+        quantum_to_cluster[quantum_id] = cluster.name
         add_cluster_dependencies(cqgraph, cluster, quantum_to_cluster)
 
 
@@ -467,7 +467,7 @@ def partition_cluster_values(
 def add_dim_clusters(
     cluster_config: BpsConfig,
     cluster_label: str,
-    qgraph: QuantumGraph,
+    qgraph: PredictedQuantumGraph,
     ordered_tasks: dict[str, DiGraph],
     cqgraph: ClusteredQuantumGraph,
     quantum_to_cluster: dict[UUID, str],
@@ -505,14 +505,15 @@ def add_dim_clusters(
     partition_values: set[str] = set()
     quanta_info: dict[str, list[dict[str, Any]]] = {}
     for task_label in topological_sort(ordered_tasks[cluster_label]):
-        # Determine cluster for each node
-        task_def = qgraph.findTaskDefByLabel(task_label)
-        assert task_def is not None  # for mypy
-        quantum_nodes = qgraph.getNodesForTask(task_def)
-
-        for qnode in quantum_nodes:
-            cluster_name, info = get_cluster_name_from_node(
-                qnode, cluster_dims, cluster_label, template, equal_dims, partition_dims
+        for quantum_id in qgraph.quanta_by_task[task_label].values():
+            cluster_name, info = get_cluster_name_from_info(
+                quantum_id,
+                cqgraph.qxgraph.nodes[quantum_id],
+                cluster_dims,
+                cluster_label,
+                template,
+                equal_dims,
+                partition_dims,
             )
             if "partition_key" in info:
                 partition_values.add(info["partition_key"])
@@ -551,23 +552,22 @@ def add_cluster_dependencies(
         from quantum_to_cluster or if their parent quantum node ids
         are missing from quantum_to_cluster.
     """
-    qgraph = cqgraph.qgraph
     for node_id in cluster.qgraph_node_ids:
-        cluster_node = qgraph.getQuantumNodeByNodeId(node_id)
-        parents = qgraph.determineInputsToQuantumNode(cluster_node)
-        for parent in parents:
+        cluster_node_info = cqgraph.get_quantum_info(node_id)
+        parents = cqgraph.qxgraph.predecessors(node_id)
+        for parent_id in parents:
             try:
-                if quantum_to_cluster[parent.nodeId] != quantum_to_cluster[node_id]:
-                    cqgraph.add_dependency(quantum_to_cluster[parent.nodeId], quantum_to_cluster[node_id])
+                if quantum_to_cluster[parent_id] != quantum_to_cluster[node_id]:
+                    cqgraph.add_dependency(quantum_to_cluster[parent_id], quantum_to_cluster[node_id])
             except KeyError as e:  # pragma: no cover
                 # For debugging a problem internal to method
-                qnode = qgraph.getQuantumNodeByNodeId(e.args[0])
+                qnode_info = cqgraph.get_quantum_info(e.args[0])
                 _LOG.error(
                     "Quanta missing when clustering: cluster node = %s, %s; missing = %s, %s",
-                    cluster_node.taskDef.label,
-                    cluster_node.quantum.dataId,
-                    qnode.taskDef.label,
-                    qnode.quantum.dataId,
+                    cluster_node_info["task_label"],
+                    cluster_node_info["data_id"],
+                    qnode_info["task_label"],
+                    qnode_info["data_id"],
                 )
                 _LOG.error(quantum_to_cluster)
                 raise
@@ -576,13 +576,13 @@ def add_cluster_dependencies(
 def add_dim_clusters_dependency(
     cluster_config: BpsConfig,
     cluster_label: str,
-    qgraph: QuantumGraph,
+    qgraph: PredictedQuantumGraph,
     ordered_tasks: dict[str, DiGraph],
     cqgraph: ClusteredQuantumGraph,
     quantum_to_cluster: dict[UUID, str],
 ) -> None:
     """Add clusters for a cluster label to a ClusteredQuantumGraph using
-       QuantumGraph dependencies as well as dimension values to help when
+       quantum graph dependencies as well as dimension values to help when
        some do not have particular dimension value.
 
     Parameters
@@ -591,8 +591,8 @@ def add_dim_clusters_dependency(
         BPS configuration for specific cluster label.
     cluster_label : `str`
         Cluster label for which to add clusters.
-    qgraph : `lsst.pipe.base.QuantumGraph`
-        QuantumGraph providing quanta for the clusters.
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
+        Quantum graph providing quanta for the clusters.
     ordered_tasks : `dict` [`str`, `networkx.DiGraph`]
         Mapping of cluster label to task label subgraph.
     cqgraph : `lsst.ctrl.bps.ClusteredQuantumGraph`
@@ -618,9 +618,9 @@ def add_dim_clusters_dependency(
     method = cluster_config["findDependencyMethod"]
     match method:
         case "source":
-            find_possible_nodes = qgraph.determineOutputsOfQuantumNode
+            find_possible_nodes = cqgraph.qxgraph.successors
         case "sink":
-            find_possible_nodes = qgraph.determineInputsToQuantumNode
+            find_possible_nodes = cqgraph.qxgraph.predecessors
             label_search_order.reverse()
         case _:
             raise RuntimeError(f"Invalid findDependencyMethod ({method})")
@@ -634,49 +634,60 @@ def add_dim_clusters_dependency(
     # quicker to check
     quanta_visited = set()
     for task_label in label_search_order:
-        task_def = qgraph.findTaskDefByLabel(task_label)
-        assert task_def is not None  # for mypy
-        for node in qgraph.getNodesForTask(task_def):
+        for quantum_id in qgraph.quanta_by_task[task_label].values():
             # skip if visited before
-            if node.nodeId in quanta_visited:
+            if quantum_id in quanta_visited:
                 continue
 
-            cluster_name, info = get_cluster_name_from_node(
-                node, cluster_dims, cluster_label, template, equal_dims, partition_dims
+            cluster_name, info = get_cluster_name_from_info(
+                quantum_id,
+                cqgraph.qxgraph.nodes[quantum_id],
+                cluster_dims,
+                cluster_label,
+                template,
+                equal_dims,
+                partition_dims,
             )
             if "partition_key" in info:
                 partition_values.add(info["partition_key"])
             quanta_info.setdefault(cluster_name, []).append(info)
-            quanta_visited.add(node.nodeId)
+            quanta_visited.add(quantum_id)
 
             # Use dependencies to find other quantum to add
             # Note: in testing, using the following code was faster than
             # using networkx descendants and ancestors functions
             # While traversing the QuantumGraph, nodes may appear
             # repeatedly in possible_nodes.
-            nodes_to_use = [node]
+            nodes_to_use = [quantum_id]
             while nodes_to_use:
                 node_to_use = nodes_to_use.pop()
-                possible_nodes = find_possible_nodes(node_to_use)
-                for possible_node in possible_nodes:
+                possible_node_ids = find_possible_nodes(node_to_use)
+                for possible_node_id in possible_node_ids:
                     # skip if visited before
-                    if possible_node.nodeId in quanta_visited:
+                    if possible_node_id in quanta_visited:
                         continue
-                    quanta_visited.add(possible_node.nodeId)
-
-                    if possible_node.taskDef.label in ordered_tasks[cluster_label]:
-                        cluster_name, info = get_cluster_name_from_node(
-                            possible_node, cluster_dims, cluster_label, template, equal_dims, partition_dims
+                    quanta_visited.add(possible_node_id)
+
+                    possible_node_info = cqgraph.qxgraph.nodes[possible_node_id]
+                    if possible_node_info["task_label"] in ordered_tasks[cluster_label]:
+                        cluster_name, info = get_cluster_name_from_info(
+                            possible_node_id,
+                            possible_node_info,
+                            cluster_dims,
+                            cluster_label,
+                            template,
+                            equal_dims,
+                            partition_dims,
                         )
                         if "partition_key" in info:
                             partition_values.add(info["partition_key"])
                         quanta_info.setdefault(cluster_name, []).append(info)
-                        nodes_to_use.append(possible_node)
+                        nodes_to_use.append(possible_node_id)
                     else:
                         _LOG.debug(
                             "label (%s) not in ordered_tasks. Not adding possible quantum %s",
-                            possible_node.taskDef.label,
-                            possible_node.nodeId,
+                            possible_node_info["task_label"],
+                            possible_node_id,
                         )
 
     make_and_add_clusters(
@@ -746,8 +757,9 @@ def make_and_add_clusters(
         add_cluster_dependencies(cqgraph, cluster, quantum_to_cluster)
 
 
-def get_cluster_name_from_node(
-    node: QuantumNode,
+def get_cluster_name_from_info(
+    quantum_id: uuid.UUID,
+    quantum_info: QuantumInfo,
     cluster_dims: list[str],
     cluster_label: str,
     template: str,
@@ -758,8 +770,10 @@ def get_cluster_name_from_node(
 
     Parameters
     ----------
-    node : `lsst.pipe.base.QuantumNode`
-        QuantumNode from which to create the cluster.
+    quantum_id : `uuid.UUID`
+        Unique ID for the quantum.
+    quantum_info : `lsst.pipe.base.quantum_graph.QuantumInfo`
+        Info dictionary from which to create the cluster.
     cluster_dims : `list` [`str`]
         Dimension names to be used when clustering.
     cluster_label : `str`
@@ -780,8 +794,8 @@ def get_cluster_name_from_node(
     """
     # Gather info for cluster name template into a dictionary.
     info: dict[str, Any] = {
-        "node_number": node.nodeId,
-        "node_label": node.taskDef.label,
+        "node_number": quantum_id,
+        "node_label": quantum_info["task_label"],
         "label": cluster_label,
     }
 
@@ -789,8 +803,7 @@ def get_cluster_name_from_node(
     all_dims = cluster_dims + partition_dims
 
     missing_info = set()
-    assert node.quantum.dataId is not None  # for mypy
-    data_id_info = dict(node.quantum.dataId.mapping)
+    data_id_info = dict(quantum_info["data_id"].mapping)
     for dim_name in all_dims:
         _LOG.debug("dim_name = %s", dim_name)
         if dim_name in data_id_info:
@@ -807,7 +820,7 @@ def get_cluster_name_from_node(
 
     if missing_info:
         raise RuntimeError(
-            f"Quantum {node.nodeId} ({data_id_info}) missing dimensions: {','.join(missing_info)}; "
+            f"Quantum {quantum_id} ({data_id_info}) missing dimensions: {','.join(missing_info)}; "
             f"required for cluster {cluster_label}"
         )
 

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4000}/python/lsst/ctrl/bps/transform.py

@@ -190,16 +190,6 @@ def create_init_workflow(
     # Adjust job attributes values if necessary.
     _handle_job_values(job_values, gwjob)
 
-    # Pick a node id for each task (not quantum!) to avoid reading the entire
-    # quantum graph during the initialization stage.
-    node_ids = []
-    for task_label in qgraph.pipeline_graph.tasks:
-        task_def = qgraph.findTaskDefByLabel(task_label)
-        node = next(iter(qgraph.getNodesForTask(task_def)))
-        node_ids.append(node.nodeId)
-    gwjob.cmdvals["qgraphId"] = qgraph.graphID
-    gwjob.cmdvals["qgraphNodeId"] = ",".join(sorted([f"{node_id}" for node_id in node_ids]))
-
     init_workflow.add_job(gwjob)
     init_workflow.add_job_inputs(gwjob.name, [qgraph_gwfile])
     _enhance_command(config, init_workflow, gwjob, {})
@@ -682,14 +672,13 @@ def create_generic_workflow(
         # either common or aggregate for all Quanta in cluster.
         for node_id in iter(cluster.qgraph_node_ids):
             _LOG.debug("node_id=%s", node_id)
-            qnode = cqgraph.get_quantum_node(node_id)
+            quantum_info = cqgraph.get_quantum_info(node_id)
 
-            if qnode.taskDef.label not in cached_pipetask_values:
-                search_opt["curvals"]["curr_pipetask"] = qnode.taskDef.label
-                cached_pipetask_values[qnode.taskDef.label] = _get_job_values(
-                    config, search_opt, "runQuantumCommand"
-                )
-            _handle_job_values(cached_pipetask_values[qnode.taskDef.label], gwjob, unset_attributes)
+            task_label = quantum_info["task_label"]
+            if task_label not in cached_pipetask_values:
+                search_opt["curvals"]["curr_pipetask"] = task_label
+                cached_pipetask_values[task_label] = _get_job_values(config, search_opt, "runQuantumCommand")
+            _handle_job_values(cached_pipetask_values[task_label], gwjob, unset_attributes)
 
         # Update job with workflow attribute and profile values.
         qgraph_gwfile = _get_qgraph_gwfile(
@@ -699,7 +688,6 @@ def create_generic_workflow(
         generic_workflow.add_job(gwjob)
         generic_workflow.add_job_inputs(gwjob.name, [qgraph_gwfile])
 
-        gwjob.cmdvals["qgraphId"] = cqgraph.qgraph.graphID
         gwjob.cmdvals["qgraphNodeId"] = ",".join(
             sorted([f"{node_id}" for node_id in cluster.qgraph_node_ids])
         )

lsst_ctrl_bps-29.2025.4000/python/lsst/ctrl/bps/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.4000"

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4000/python/lsst_ctrl_bps.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps
-Version: 29.2025.3900
+Version: 29.2025.4000
 Summary: Pluggable execution of workflow graphs from Rubin pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4000}/tests/test_clustered_quantum_graph.py

@@ -50,54 +50,49 @@ class TestQuantaCluster(unittest.TestCase):
 
     def setUp(self):
         self.qgraph = make_test_quantum_graph()
-        nodes = list(self.qgraph.getNodesForTask(self.qgraph.findTaskDefByLabel("T1")))
-        self.qnode1 = nodes[0]
-        self.qnode2 = nodes[1]
+        self.id1, self.id2, *_ = self.qgraph.quanta_by_task["T1"].values()
+        self.info1 = self.qgraph.quantum_only_xgraph.nodes[self.id1]
+        self.info2 = self.qgraph.quantum_only_xgraph.nodes[self.id2]
 
     def tearDown(self):
         pass
 
     def testQgraphNodeIds(self):
-        qc = QuantaCluster.from_quantum_node(self.qnode1, "{node_number}")
-        self.assertEqual(qc.qgraph_node_ids, frozenset([self.qnode1.nodeId]))
+        qc = QuantaCluster.from_quantum_info(self.id1, self.info1, "{node_number}")
+        self.assertEqual(qc.qgraph_node_ids, frozenset([self.id1]))
 
     def testQuantaCountsNone(self):
         qc = QuantaCluster("NoQuanta", "the_label")
         self.assertEqual(qc.quanta_counts, Counter())
 
     def testQuantaCounts(self):
-        qc = QuantaCluster.from_quantum_node(self.qnode1, "{node_number}")
+        qc = QuantaCluster.from_quantum_info(self.id1, self.info1, "{node_number}")
         self.assertEqual(qc.quanta_counts, Counter({"T1": 1}))
 
-    def testAddQuantumNode(self):
-        qc = QuantaCluster.from_quantum_node(self.qnode1, "{node_number}")
-        qc.add_quantum_node(self.qnode2)
-        self.assertEqual(qc.quanta_counts, Counter({"T1": 2}))
-
     def testAddQuantum(self):
-        qc = QuantaCluster.from_quantum_node(self.qnode1, "{node_number}")
-        qc.add_quantum(self.qnode2.quantum, self.qnode2.taskDef.label)
+        qc = QuantaCluster.from_quantum_info(self.id1, self.info1, "{node_number}")
+        qc.add_quantum(self.id2, self.info2["task_label"])
         self.assertEqual(qc.quanta_counts, Counter({"T1": 2}))
 
     def testStr(self):
-        qc = QuantaCluster.from_quantum_node(self.qnode1, "{node_number}")
+        qc = QuantaCluster.from_quantum_info(self.id1, self.info1, "{node_number}")
         self.assertIn(qc.name, str(qc))
         self.assertIn("T1", str(qc))
         self.assertIn("tags", str(qc))
 
     def testEqual(self):
-        qc1 = QuantaCluster.from_quantum_node(self.qnode1, "{node_number}")
-        qc2 = QuantaCluster.from_quantum_node(self.qnode1, "{node_number}")
+        qc1 = QuantaCluster.from_quantum_info(self.id1, self.info1, "{node_number}")
+        qc2 = QuantaCluster.from_quantum_info(self.id1, self.info1, "{node_number}")
         self.assertEqual(qc1, qc2)
 
     def testNotEqual(self):
-        qc1 = QuantaCluster.from_quantum_node(self.qnode1, "{node_number}")
-        qc2 = QuantaCluster.from_quantum_node(self.qnode2, "{node_number}")
+        qc1 = QuantaCluster.from_quantum_info(self.id1, self.info1, "{node_number}")
+        qc2 = QuantaCluster.from_quantum_info(self.id2, self.info2, "{node_number}")
         self.assertNotEqual(qc1, qc2)
 
     def testHash(self):
-        qc1 = QuantaCluster.from_quantum_node(self.qnode1, "{node_number}")
-        qc2 = QuantaCluster.from_quantum_node(self.qnode2, "{node_number}")
+        qc1 = QuantaCluster.from_quantum_info(self.id1, self.info1, "{node_number}")
+        qc2 = QuantaCluster.from_quantum_info(self.id2, self.info2, "{node_number}")
         self.assertNotEqual(hash(qc1), hash(qc2))
 
 
@@ -196,8 +191,9 @@ class TestClusteredQuantumGraph(unittest.TestCase):
     def testValidateDuplicateId(self):
         # Add new Quanta with duplicate Quantum
         qc1 = self.cqg1.get_cluster("T1_1_2")
-        qnode = self.cqg1.get_quantum_node(next(iter(qc1.qgraph_node_ids)))
-        qc = QuantaCluster.from_quantum_node(qnode, "DuplicateId")
+        quantum_id = next(iter(qc1.qgraph_node_ids))
+        quantum_info = self.cqg1.get_quantum_info(quantum_id)
+        qc = QuantaCluster.from_quantum_info(quantum_id, quantum_info, "DuplicateId")
         self.cqg1.add_cluster(qc)
         qc2 = self.cqg1.get_cluster("T23_1_2")
         self.cqg1.add_dependency(qc2, qc)

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4000}/tests/test_pre_transform.py

@@ -35,8 +35,7 @@ from pathlib import Path
 
 from lsst.ctrl.bps import BpsConfig, BpsSubprocessError, ClusteredQuantumGraph
 from lsst.ctrl.bps.pre_transform import cluster_quanta, create_quantum_graph, execute, update_quantum_graph
-from lsst.daf.butler import DimensionUniverse
-from lsst.pipe.base import QuantumGraph
+from lsst.pipe.base.tests.mocks import InMemoryRepo
 
 TESTDIR = os.path.abspath(os.path.dirname(__file__))
 _LOG = logging.getLogger(__name__)
@@ -82,7 +81,7 @@ class TestCreatingQuantumGraph(unittest.TestCase):
             "submitPath": self.tmpdir,
             "whenSaveJobQgraph": "NEVER",
             "uniqProcName": "my_test",
-            "qgraphFileTemplate": "{uniqProcName}.qgraph",
+            "qgraphFileTemplate": "{uniqProcName}.qg",
         }
         self.logger = logging.getLogger("lsst.ctrl.bps")
 
@@ -125,8 +124,8 @@ class TestUpdatingQuantumGraph(unittest.TestCase):
             "submitPath": self.tmpdir,
             "whenSaveJobQgraph": "NEVER",
             "uniqProcName": "my_test",
-            "qgraphFileTemplate": "{uniqProcName}.qgraph",
-            "inputQgraphFile": f"{self.tmpdir}/src.qgraph",
+            "qgraphFileTemplate": "{uniqProcName}.qg",
+            "inputQgraphFile": f"{self.tmpdir}/src.qg",
         }
         self.logger = logging.getLogger("lsst.ctrl.bps")
 
@@ -195,7 +194,7 @@ class TestClusterQuanta(unittest.TestCase):
             "validateClusteredQgraph": True,
         }
         config = BpsConfig(settings, search_order=[])
-        qgraph = QuantumGraph({}, universe=DimensionUniverse())
+        qgraph = InMemoryRepo().make_quantum_graph()
         with self.assertRaisesRegex(RuntimeError, "Fake error"):
             _ = cluster_quanta(config, qgraph, "a_name")
 
@@ -209,7 +208,7 @@ class TestClusterQuanta(unittest.TestCase):
             "validateClusteredQgraph": False,
         }
         config = BpsConfig(settings, search_order=[])
-        qgraph = QuantumGraph({}, universe=DimensionUniverse())
+        qgraph = InMemoryRepo().make_quantum_graph()
         _ = cluster_quanta(config, qgraph, "a_name")
 
 

lsst_ctrl_bps-29.2025.3900/python/lsst/ctrl/bps/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.3900"