lsst-ctrl-bps 29.2025.3900__tar.gz → 29.2025.4100__tar.gz

{lsst_ctrl_bps-29.2025.3900/python/lsst_ctrl_bps.egg-info → lsst_ctrl_bps-29.2025.4100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-bps
-Version: 29.2025.3900
+Version: 29.2025.4100
 Summary: Pluggable execution of workflow graphs from Rubin pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4100}/doc/lsst.ctrl.bps/quickstart.rst

@@ -157,7 +157,7 @@ or a pre-made file containing a serialized QuantumGraph, for example
 
 .. code-block:: YAML
 
-   qgraphFile: pipelines_check_w_2020_45.qgraph
+   qgraphFile: pipelines_check_w_2020_45.qg
 
 .. warning::
 
@@ -846,7 +846,7 @@ Supported settings
     When to output job QuantumGraph files (default = TRANSFORM).
 
     * NEVER = all jobs will use full QuantumGraph file. (Warning: make sure
-      runQuantumCommand has ``--qgraph-id {qgraphId} --qgraph-node-id {qgraphNodeId}``.)
+      runQuantumCommand has ``--qgraph-node-id {qgraphNodeId}``.)
     * TRANSFORM = Output QuantumGraph files after creating GenericWorkflow.
     * PREPARE = QuantumGraph files are output after creating WMS submission.
 
@@ -904,7 +904,7 @@ Reserved keywords
     However, contrary to YAML specification, it is currently not portable.
 
 **qgraphId**
-    Internal ID for the full QuantumGraph (passed as ``--qgraph-id`` on pipetask command line).
+    Ignored; accepted for backwards compatibility.
 
 **qgraphNodeId**
     Comma-separated list of internal QuantumGraph node numbers to be
@@ -1079,13 +1079,12 @@ single full QuantumGraph file plus node numbers for each job. The default is
 using per-job QuantumGraph files.
 
 To use full QuantumGraph file, the submit YAML must set ``whenSaveJobQgraph`` to
-"NEVER" and the ``pipetask run`` command must include ``--qgraph-id {qgraphId}
---qgraph-node-id {qgraphNodeId}``.  For example:
+"NEVER" and the ``pipetask run`` command must include ``--qgraph-node-id {qgraphNodeId}``.  For example:
 
 .. code::
 
     whenSaveJobQgraph: "NEVER"
-    runQuantumCommand: "${CTRL_MPEXEC_DIR}/bin/pipetask --long-log run -b {butlerConfig} --output {output} --output-run {outputRun} --qgraph {qgraphFile} --qgraph-id {qgraphId} --qgraph-node-id {qgraphNodeId} --skip-init-writes --extend-run --clobber-outputs --skip-existing"
+    runQuantumCommand: "${CTRL_MPEXEC_DIR}/bin/pipetask --long-log run -b {butlerConfig} --output {output} --output-run {outputRun} --qgraph {qgraphFile} --qgraph-node-id {qgraphNodeId} --skip-init-writes --extend-run --clobber-outputs --skip-existing"
 
 
 .. warning::
@@ -1244,7 +1243,7 @@ The major differences to users are:
     the output run in the provided pre-existing quantum graph.
   - ``final_post_finalJob.out``: An internal file for debugging incorrect
     reporting of final run status.
-  - ``<qgraph_filename>_orig.qgraph``: A backup copy of the original
+  - ``<qgraph_filename>_orig.qg``: A backup copy of the original
     pre-existing quantum graph file that was used for submitting the run.  Note
     that this file will *not* be present in the submit directory if the
     pipeline YAML specification was used during the submission instead.

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4100}/python/lsst/ctrl/bps/bps_utils.py

@@ -146,7 +146,7 @@ def create_job_quantum_graph_filename(config, job, out_prefix=None):
     found, subdir = config.search("subDirTemplate", opt={"curvals": curvals})
     if not found:
         subdir = "{job.label}"
-    full_filename = Path("inputs") / subdir / f"quantum_{job.name}.qgraph"
+    full_filename = Path("inputs") / subdir / f"quantum_{job.name}.qg"
 
     if out_prefix is not None:
         full_filename = Path(out_prefix) / full_filename

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4100}/python/lsst/ctrl/bps/clustered_quantum_graph.py

@@ -29,18 +29,21 @@
 a QuantumGraph.
 """
 
-__all__ = ["ClusteredQuantumGraph", "QuantaCluster"]
+from __future__ import annotations
 
+__all__ = ["ClusteredQuantumGraph", "QuantaCluster"]
 
 import logging
 import pickle
 import re
+import uuid
 from collections import Counter, defaultdict
 from pathlib import Path
 
 from networkx import DiGraph, is_directed_acyclic_graph, is_isomorphic, topological_sort
 
-from lsst.pipe.base import NodeId, QuantumGraph
+from lsst.pipe.base.pipeline_graph import TaskImportMode
+from lsst.pipe.base.quantum_graph import PredictedQuantumGraph, QuantumInfo
 from lsst.utils.iteration import ensure_iterable
 
 from .bps_draw import draw_networkx_dot
@@ -79,13 +82,17 @@ class QuantaCluster:
             self.tags = {}
 
     @classmethod
-    def from_quantum_node(cls, quantum_node, template):
-        """Create single quantum cluster from given quantum node.
+    def from_quantum_info(
+        cls, quantum_id: uuid.UUID, quantum_info: QuantumInfo, template: str
+    ) -> QuantaCluster:
+        """Create single quantum cluster from the given quantum information.
 
         Parameters
         ----------
-        quantum_node : `lsst.pipe.base.QuantumNode`
-            QuantumNode for which to make into a single quantum cluster.
+        quantum_id : `uuid.UUID`
+            ID of the quantum.
+        quantum_info : `lsst.pipe.base.quantum_graph.QuantumInfo`
+            Dictionary of additional information about the quantum.
         template : `str`
             Template for creating cluster name.
 
@@ -94,14 +101,13 @@ class QuantaCluster:
         cluster : `QuantaCluster`
             Newly created cluster containing the given quantum.
         """
-        label = quantum_node.taskDef.label
-        node_id = quantum_node.nodeId
-        data_id = quantum_node.quantum.dataId
+        label = quantum_info["task_label"]
+        data_id = quantum_info["data_id"]
 
         # Gather info for name template into a dictionary.
         info = dict(data_id.required)
         info["label"] = label
-        info["node_number"] = node_id
+        info["node_number"] = quantum_id
         _LOG.debug("template = %s", template)
         _LOG.debug("info for template = %s", info)
 
@@ -116,7 +122,7 @@ class QuantaCluster:
         _LOG.debug("template name = %s", name)
 
         cluster = QuantaCluster(name, label, info)
-        cluster.add_quantum(quantum_node.nodeId, label)
+        cluster.add_quantum(quantum_id, label)
         return cluster
 
     @property
@@ -130,24 +136,12 @@ class QuantaCluster:
         """Counts of Quanta per taskDef.label in this cluster."""
         return Counter(self._task_label_counts)
 
-    def add_quantum_node(self, quantum_node):
-        """Add a quantumNode to this cluster.
-
-        Parameters
-        ----------
-        quantum_node : `lsst.pipe.base.QuantumNode`
-            Quantum node to add.
-        """
-        _LOG.debug("quantum_node = %s", quantum_node)
-        _LOG.debug("quantum_node.nodeId = %s", quantum_node.nodeId)
-        self.add_quantum(quantum_node.nodeId, quantum_node.taskDef.label)
-
     def add_quantum(self, node_id, task_label):
         """Add a quantumNode to this cluster.
 
         Parameters
         ----------
-        node_id : `lsst.pipe.base.NodeId`
+        node_id : `uuid.UUID`
             ID for quantumNode to be added to cluster.
         task_label : `str`
             Task label for quantumNode to be added to cluster.
@@ -185,11 +179,10 @@ class ClusteredQuantumGraph:
     ----------
     name : `str`
         Name to be given to the ClusteredQuantumGraph.
-    qgraph : `lsst.pipe.base.QuantumGraph`
-        The QuantumGraph to be clustered.
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
+        The quantum graph to be clustered.
     qgraph_filename : `str`
-        Filename for given QuantumGraph if it has already been
-        serialized.
+        Filename for given quantum graph.
 
     Raises
     ------
@@ -203,11 +196,12 @@ class ClusteredQuantumGraph:
     use API over totally minimized memory usage.
     """
 
-    def __init__(self, name, qgraph, qgraph_filename=None):
+    def __init__(self, name: str, qgraph: PredictedQuantumGraph, qgraph_filename: str):
         if "/" in name:
             raise ValueError(f"name cannot have a / ({name})")
         self._name = name
         self._quantum_graph = qgraph
+        self._quantum_only_xgraph = qgraph.quantum_only_xgraph
         self._quantum_graph_filename = Path(qgraph_filename).resolve()
         self._cluster_graph = DiGraph()
 
@@ -228,22 +222,27 @@ class ClusteredQuantumGraph:
             return False
         if len(self) != len(other):
             return False
-        return self._quantum_graph == other._quantum_graph and is_isomorphic(
+        return is_isomorphic(self.qxgraph, other.qxgraph) and is_isomorphic(
             self._cluster_graph, other._cluster_graph
         )
 
     @property
-    def name(self):
+    def name(self) -> str:
         """The name of the ClusteredQuantumGraph."""
         return self._name
 
     @property
-    def qgraph(self):
-        """The QuantumGraph associated with this Clustered
+    def qgraph(self) -> PredictedQuantumGraph:
+        """The quantum graph associated with this Clustered
         QuantumGraph.
         """
         return self._quantum_graph
 
+    @property
+    def qxgraph(self) -> DiGraph:
+        """A networkx graph of all quanta."""
+        return self._quantum_only_xgraph
+
     def add_cluster(self, clusters_for_adding):
         """Add a cluster of quanta as a node in the graph.
 
@@ -286,30 +285,26 @@ class ClusteredQuantumGraph:
             raise KeyError(f"{self.name} does not have a cluster named {name}") from ex
         return attr["cluster"]
 
-    def get_quantum_node(self, id_):
-        """Retrieve a QuantumNode from the ClusteredQuantumGraph by ID.
+    def get_quantum_info(self, id_: uuid.UUID) -> QuantumInfo:
+        """Retrieve a quantum info dict from the ClusteredQuantumGraph by ID.
 
         Parameters
         ----------
-        id_ : `lsst.pipe.base.NodeId` or int
-            ID of the QuantumNode to retrieve.
+        id_ : `uuid.UUID`
+            ID of the quantum to retrieve.
 
         Returns
         -------
-        quantum_node : `lsst.pipe.base.QuantumNode`
-            QuantumNode matching given ID.
+        quantum_info : `lsst.pipe.base.quantum_graph.QuantumInfo`
+            Quantum info dictionary for the given ID.
 
         Raises
         ------
         KeyError
             Raised if the ClusteredQuantumGraph does not contain
-            a QuantumNode with given ID.
+            a quantum with given ID.
         """
-        node_id = id_
-        if isinstance(id_, int):
-            node_id = NodeId(id, self._quantum_graph.graphID)
-        _LOG.debug("get_quantum_node: node_id = %s", node_id)
-        return self._quantum_graph.getQuantumNodeByNodeId(node_id)
+        return self._quantum_only_xgraph.nodes[id_]
 
     def __iter__(self):
         """Iterate over names of clusters.
@@ -414,8 +409,8 @@ class ClusteredQuantumGraph:
 
     def save(self, filename, format_=None):
         """Save the ClusteredQuantumGraph in a format that is loadable.
-        The QuantumGraph is saved separately if hasn't already been
-        serialized.
+
+        The quantum graph is assumed to have been saved separately.
 
         Parameters
         ----------
@@ -433,14 +428,6 @@ class ClusteredQuantumGraph:
         if format_ not in {"pickle"}:
             raise RuntimeError(f"Unknown format ({format_})")
 
-        if not self._quantum_graph_filename:
-            # Create filename based on given ClusteredQuantumGraph filename
-            self._quantum_graph_filename = path.with_suffix(".qgraph")
-
-        # If QuantumGraph file doesn't already exist, save it:
-        if not Path(self._quantum_graph_filename).exists():
-            self._quantum_graph.saveUri(self._quantum_graph_filename)
-
         if format_ == "pickle":
             # Don't save QuantumGraph in same file.
             tmp_qgraph = self._quantum_graph
@@ -503,14 +490,14 @@ class ClusteredQuantumGraph:
         cgraph = None
         if format_ == "pickle":
             with open(filename, "rb") as fh:
-                cgraph = pickle.load(fh)
+                cgraph: ClusteredQuantumGraph = pickle.load(fh)
 
             # The QuantumGraph was saved separately
-            try:
-                cgraph._quantum_graph = QuantumGraph.loadUri(cgraph._quantum_graph_filename)
-            except FileNotFoundError:  # Try same path as ClusteredQuantumGraph
-                new_filename = path.parent / Path(cgraph._quantum_graph_filename).name
-                cgraph._quantum_graph = QuantumGraph.loadUri(new_filename)
+            with PredictedQuantumGraph.open(
+                cgraph._quantum_graph_filename, import_mode=TaskImportMode.DO_NOT_IMPORT
+            ) as reader:
+                reader.read_thin_graph()
+                cgraph._quantum_graph = reader.finish()
 
         return cgraph
 

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4100}/python/lsst/ctrl/bps/drivers.py

@@ -50,7 +50,7 @@ import logging
 import os
 from pathlib import Path
 
-from lsst.pipe.base import QuantumGraph
+from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
 from lsst.utils.timer import time_this
 from lsst.utils.usage import get_peak_mem_usage
 
@@ -111,7 +111,7 @@ def _init_submission_driver(config_file: str, **kwargs) -> BpsConfig:
     return config
 
 
-def acquire_qgraph_driver(config_file: str, **kwargs) -> tuple[BpsConfig, QuantumGraph]:
+def acquire_qgraph_driver(config_file: str, **kwargs) -> tuple[BpsConfig, PredictedQuantumGraph]:
     """Read a quantum graph from a file or create one from pipeline definition.
 
     Parameters
@@ -125,7 +125,7 @@ def acquire_qgraph_driver(config_file: str, **kwargs) -> tuple[BpsConfig, Quantu
     -------
     config : `lsst.ctrl.bps.BpsConfig`
         Updated configuration.
-    qgraph : `lsst.pipe.base.graph.QuantumGraph`
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
         A graph representing quanta.
     """
     config = _init_submission_driver(config_file, **kwargs)
@@ -451,6 +451,7 @@ def report_driver(wms_service, run_id, user, hist_days, pass_thru, is_global=Fal
         hist=hist_days,
         pass_thru=pass_thru,
         is_global=is_global,
+        return_exit_codes=return_exit_codes,
         postprocessors=postprocessors,
     )
 

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4100}/python/lsst/ctrl/bps/etc/bps_defaults.yaml

@@ -1,6 +1,6 @@
 #USER pipelineYaml: "${OBS_SUBARU_DIR}/pipelines/DRP.yaml#processCcd"
 #  OR
-#USER qgraphFile: "/path/to/existing/file.qgraph"
+#USER qgraphFile: "/path/to/existing/file.qg"
 
 
 # At minimum, following group used in bps report and can be
@@ -94,7 +94,7 @@ clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.single_quantum_clusteri
 
 # Templates for bps filenames
 submitPath: ${PWD}/submit/{outputRun}
-qgraphFileTemplate: "{uniqProcName}.qgraph"
+qgraphFileTemplate: "{uniqProcName}.qg"
 subDirTemplate: "{label}/{tract}/{patch}/{band}/{subfilter}/{physical_filter}/{visit}/{exposure}"
 templateDataId: "{tract}_{patch}_{band}_{visit}_{exposure}_{detector}"
 

{lsst_ctrl_bps-29.2025.3900 → lsst_ctrl_bps-29.2025.4100}/python/lsst/ctrl/bps/pre_transform.py

@@ -38,7 +38,10 @@ import subprocess
 from pathlib import Path
 
 from lsst.ctrl.bps import BpsConfig, BpsSubprocessError
-from lsst.pipe.base.graph import QuantumGraph
+from lsst.pipe.base import QuantumGraph
+from lsst.pipe.base.pipeline_graph import TaskImportMode
+from lsst.pipe.base.quantum_graph import PredictedQuantumGraph
+from lsst.resources import ResourcePath
 from lsst.utils import doImport
 from lsst.utils.logging import VERBOSE
 from lsst.utils.timer import time_this, timeMethod
@@ -47,7 +50,7 @@ _LOG = logging.getLogger(__name__)
 
 
 @timeMethod(logger=_LOG, logLevel=VERBOSE)
-def acquire_quantum_graph(config: BpsConfig, out_prefix: str = "") -> tuple[str, QuantumGraph]:
+def acquire_quantum_graph(config: BpsConfig, out_prefix: str = "") -> tuple[str, PredictedQuantumGraph]:
     """Read a quantum graph from a file or create one from scratch.
 
     Parameters
@@ -62,8 +65,8 @@ def acquire_quantum_graph(config: BpsConfig, out_prefix: str = "") -> tuple[str,
     -------
     qgraph_filename : `str`
         Name of file containing QuantumGraph that was read into qgraph.
-    qgraph : `lsst.pipe.base.graph.QuantumGraph`
-        A QuantumGraph read in from pre-generated file or one that is the
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
+        A quantum graph read in from pre-generated file or one that is the
         result of running code that generates it.
     """
     # Check to see if user provided pre-generated QuantumGraph.
@@ -90,8 +93,15 @@ def acquire_quantum_graph(config: BpsConfig, out_prefix: str = "") -> tuple[str,
 
     _LOG.info("Reading quantum graph from '%s'", qgraph_filename)
     with time_this(log=_LOG, level=logging.INFO, prefix=None, msg="Completed reading quantum graph"):
-        qgraph = QuantumGraph.loadUri(qgraph_filename)
-
+        qgraph_path = ResourcePath(qgraph_filename)
+        if qgraph_path.getExtension() == ".qg":
+            with PredictedQuantumGraph.open(qgraph_path, import_mode=TaskImportMode.DO_NOT_IMPORT) as reader:
+                reader.read_thin_graph()
+                qgraph = reader.finish()
+        elif qgraph_path.getExtension() == ".qgraph":
+            qgraph = PredictedQuantumGraph.from_old_quantum_graph(QuantumGraph.loadUri(qgraph_path))
+        else:
+            raise ValueError(f"Unrecognized extension for quantum graph file: {qgraph_filename}.")
     return qgraph_filename, qgraph
 
 
@@ -244,8 +254,8 @@ def cluster_quanta(config, qgraph, name):
     ----------
     config : `lsst.ctrl.bps.BpsConfig`
         BPS configuration.
-    qgraph : `lsst.pipe.base.QuantumGraph`
-        Original full QuantumGraph for the run.
+    qgraph : `lsst.pipe.base.quantum_graph.PredictedQuantumGraph`
+        Original full quantum graph for the run.
     name : `str`
         Name for the ClusteredQuantumGraph that will be generated.