lsst-ctrl-mpexec 29.2025.1600__py3-none-any.whl → 29.2025.1800__py3-none-any.whl

lsst/ctrl/mpexec/cli/cmd/commands.py

@@ -311,6 +311,7 @@ def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
 @ctrlMpExecOpts.enable_implicit_threading_option()
 @ctrlMpExecOpts.cores_per_quantum_option()
 @ctrlMpExecOpts.memory_per_quantum_option()
+@ctrlMpExecOpts.no_existing_outputs_option()
 def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
     """Execute pipeline using Quantum-Backed Butler.
 

lsst/ctrl/mpexec/cli/opt/options.py

@@ -396,6 +396,16 @@ clobber_outputs_option = MWOptionDecorator(
 )
 
 
+no_existing_outputs_option = MWOptionDecorator(
+    "--no-existing-outputs",
+    help=(
+        "Assume that no predicted outputs already exist in the output run collection. "
+        "This will eliminate existence checks that otherwise run before each quantum is executed."
+    ),
+    is_flag=True,
+)
+
+
 skip_init_writes_option = MWOptionDecorator(
     "--skip-init-writes",
     help=unwrap(

lsst/ctrl/mpexec/cli/script/run_qbb.py

@@ -53,6 +53,7 @@ def run_qbb(
     cores_per_quantum: int,
     memory_per_quantum: str,
     raise_on_partial_outputs: bool,
+    no_existing_outputs: bool,
 ) -> None:
     """Implement the command line interface ``pipetask run-qbb`` subcommand.
 
@@ -104,6 +105,9 @@ def run_qbb(
         units of MB) or a combination of number and unit.
     raise_on_partial_outputs : `bool`
         Consider partial outputs an error instead of a success.
+    no_existing_outputs : `bool`
+        Whether to assume that no predicted outputs for these quanta already
+        exist in the output run collection.
     """
     # Fork option still exists for compatibility but we use spawn instead.
     if start_method == "fork":
@@ -131,6 +135,7 @@ def run_qbb(
         cores_per_quantum=cores_per_quantum,
         memory_per_quantum=memory_per_quantum,
         raise_on_partial_outputs=raise_on_partial_outputs,
+        no_existing_outputs=no_existing_outputs,
     )
 
     f = CmdLineFwk()

lsst/ctrl/mpexec/cmdLineFwk.py

@@ -75,7 +75,7 @@ from lsst.pipe.base.mermaid_tools import graph2mermaid
 from lsst.pipe.base.pipeline_graph import NodeType
 from lsst.resources import ResourcePath
 from lsst.utils import doImportType
-from lsst.utils.logging import getLogger
+from lsst.utils.logging import VERBOSE, getLogger
 from lsst.utils.threads import disable_implicit_threading
 
 from ._pipeline_graph_factory import PipelineGraphFactory
@@ -844,7 +844,7 @@ class CmdLineFwk:
                 _LOG.debug("Will try to import debug.py")
                 import debug  # type: ignore # noqa:F401
             except ImportError:
-                _LOG.warn("No 'debug' module found.")
+                _LOG.warning("No 'debug' module found.")
 
         # Save all InitOutputs, configs, etc.
         preExecInit = PreExecInit(butler, taskFactory, extendRun=args.extend_run)
@@ -977,6 +977,7 @@ class CmdLineFwk:
         return None
 
     def preExecInitQBB(self, task_factory: TaskFactory, args: SimpleNamespace) -> None:
+        _LOG.verbose("Reading full quantum graph from %s.", args.qgraph)
         # Load quantum graph. We do not really need individual Quanta here,
         # but we need datastore records for initInputs, and those are only
         # available from Quanta, so load the whole thing.
@@ -986,8 +987,10 @@ class CmdLineFwk:
         _ButlerFactory.defineDatastoreCache()
 
         # Make QBB.
+        _LOG.verbose("Initializing quantum-backed butler.")
         butler = qgraph.make_init_qbb(args.butler_config, config_search_paths=args.config_search_path)
         # Save all InitOutputs, configs, etc.
+        _LOG.verbose("Instantiating tasks and saving init-outputs.")
         preExecInit = PreExecInitLimited(butler, task_factory)
         preExecInit.initialize(qgraph)
 
@@ -997,7 +1000,13 @@ class CmdLineFwk:
 
         # Load quantum graph.
         nodes = args.qgraph_node_id or None
-        qgraph = QuantumGraph.loadUri(args.qgraph, nodes=nodes, graphID=args.qgraph_id)
+        with lsst.utils.timer.time_this(
+            _LOG,
+            msg=f"Reading {str(len(nodes)) if nodes is not None else 'all'} quanta.",
+            level=VERBOSE,
+        ) as qg_read_time:
+            qgraph = QuantumGraph.loadUri(args.qgraph, nodes=nodes, graphID=args.qgraph_id)
+        job_metadata = {"qg_read_time": qg_read_time.duration, "qg_size": len(qgraph)}
 
         if qgraph.metadata is None:
             raise ValueError("QuantumGraph is missing metadata, cannot continue.")
@@ -1023,8 +1032,11 @@ class CmdLineFwk:
             enableLsstDebug=args.enableLsstDebug,
             limited_butler_factory=_butler_factory,
             resources=resources,
-            assumeNoExistingOutputs=True,
+            assumeNoExistingOutputs=args.no_existing_outputs,
+            skipExisting=True,
+            clobberOutputs=True,
             raise_on_partial_outputs=args.raise_on_partial_outputs,
+            job_metadata=job_metadata,
         )
 
         timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout

lsst/ctrl/mpexec/singleQuantumExecutor.py

@@ -34,12 +34,13 @@ import logging
 import time
 import uuid
 from collections import defaultdict
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from itertools import chain
 from typing import Any, cast
 
 from lsst.daf.butler import (
     Butler,
+    ButlerMetrics,
     CollectionType,
     DatasetRef,
     DatasetType,
@@ -127,6 +128,11 @@ class SingleQuantumExecutor(QuantumExecutor):
         `lsst.pipe.base.AnnotatedPartialOutputError` immediately, instead of
         considering the partial result a success and continuing to run
         downstream tasks.
+    job_metadata : `~collections.abc.Mapping`
+        Mapping with extra metadata to embed within the quantum metadata under
+        the "job" key.  This is intended to correspond to information common
+        to all quanta being executed in a single process, such as the time
+        taken to load the quantum graph in a BPS job.
     """
 
     def __init__(
@@ -141,6 +147,7 @@ class SingleQuantumExecutor(QuantumExecutor):
         skipExisting: bool = False,
         assumeNoExistingOutputs: bool = False,
         raise_on_partial_outputs: bool = True,
+        job_metadata: Mapping[str, int | str | float] | None = None,
     ):
         self.butler = butler
         self.taskFactory = taskFactory
@@ -150,6 +157,7 @@ class SingleQuantumExecutor(QuantumExecutor):
         self.resources = resources
         self.assumeNoExistingOutputs = assumeNoExistingOutputs
         self.raise_on_partial_outputs = raise_on_partial_outputs
+        self.job_metadata = job_metadata
 
         if self.butler is None:
             assert limited_butler_factory is not None, "limited_butler_factory is needed when butler is None"
@@ -246,6 +254,8 @@ class SingleQuantumExecutor(QuantumExecutor):
                 fullMetadata = _TASK_FULL_METADATA_TYPE()
                 fullMetadata[task_node.label] = _TASK_METADATA_TYPE()
                 fullMetadata["quantum"] = quantumMetadata
+                if self.job_metadata is not None:
+                    fullMetadata["job"] = self.job_metadata
                 self.writeMetadata(quantum, fullMetadata, task_node, limited_butler)
                 return quantum
 
@@ -255,7 +265,7 @@ class SingleQuantumExecutor(QuantumExecutor):
                     _LOG.debug("Will try to import debug.py")
                     import debug  # type: ignore # noqa:F401
                 except ImportError:
-                    _LOG.warn("No 'debug' module found.")
+                    _LOG.warning("No 'debug' module found.")
 
             # initialize global state
             self.initGlobals(quantum)
@@ -273,7 +283,7 @@ class SingleQuantumExecutor(QuantumExecutor):
             task = self.taskFactory.makeTask(task_node, limited_butler, init_input_refs)
             logInfo(None, "start", metadata=quantumMetadata)  # type: ignore[arg-type]
             try:
-                caveats, outputsPut = self.runQuantum(
+                caveats, outputsPut, butler_metrics = self.runQuantum(
                     task, quantum, task_node, limited_butler, quantum_id=quantum_id
                 )
             except Exception as e:
@@ -286,6 +296,7 @@ class SingleQuantumExecutor(QuantumExecutor):
                 )
                 raise
             else:
+                quantumMetadata["butler_metrics"] = butler_metrics.model_dump()
                 quantumMetadata["caveats"] = caveats.value
                 # Stringify the UUID for easier compatibility with
                 # PropertyList.
@@ -293,6 +304,8 @@ class SingleQuantumExecutor(QuantumExecutor):
             logInfo(None, "end", metadata=quantumMetadata)  # type: ignore[arg-type]
             fullMetadata = task.getFullMetadata()
             fullMetadata["quantum"] = quantumMetadata
+            if self.job_metadata is not None:
+                fullMetadata["job"] = self.job_metadata
             self.writeMetadata(quantum, fullMetadata, task_node, limited_butler)
             stopTime = time.time()
             _LOG.info(
@@ -488,7 +501,7 @@ class SingleQuantumExecutor(QuantumExecutor):
         /,
         limited_butler: LimitedButler,
         quantum_id: uuid.UUID | None = None,
-    ) -> tuple[QuantumSuccessCaveats, list[uuid.UUID]]:
+    ) -> tuple[QuantumSuccessCaveats, list[uuid.UUID], ButlerMetrics]:
         """Execute task on a single quantum.
 
         Parameters
@@ -511,6 +524,8 @@ class SingleQuantumExecutor(QuantumExecutor):
         ids_put : list[ `uuid.UUID` ]
             Record of all the dataset IDs that were written by this quantum
             being executed.
+        metrics : `lsst.daf.butler.ButlerMetrics`
+            Butler metrics recorded for this quantum.
         """
         flags = QuantumSuccessCaveats.NO_CAVEATS
 
@@ -522,7 +537,8 @@ class SingleQuantumExecutor(QuantumExecutor):
 
         # Call task runQuantum() method.
         try:
-            task.runQuantum(butlerQC, inputRefs, outputRefs)
+            with limited_butler.record_metrics() as butler_metrics:
+                task.runQuantum(butlerQC, inputRefs, outputRefs)
         except NoWorkFound as err:
             # Not an error, just an early exit.
             _LOG.info(
@@ -565,7 +581,7 @@ class SingleQuantumExecutor(QuantumExecutor):
         if not butlerQC.outputsPut == butlerQC.allOutputs:
             flags |= QuantumSuccessCaveats.ANY_OUTPUTS_MISSING
         ids_put = [output[2] for output in butlerQC.outputsPut]
-        return flags, ids_put
+        return flags, ids_put, butler_metrics
 
     def writeMetadata(
         self, quantum: Quantum, metadata: Any, task_node: TaskNode, /, limited_butler: LimitedButler

lsst/ctrl/mpexec/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.1600"
+__version__ = "29.2025.1800"

{lsst_ctrl_mpexec-29.2025.1600.dist-info → lsst_ctrl_mpexec-29.2025.1800.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-mpexec
-Version: 29.2025.1600
+Version: 29.2025.1800
 Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_mpexec-29.2025.1600.dist-info → lsst_ctrl_mpexec-29.2025.1800.dist-info}/RECORD

@@ -2,7 +2,7 @@ lsst/__init__.py,sha256=aXdEOZVrBQISQi6XPS9s1NhBjIJaIwNNxCFRiGchRAw,1369
 lsst/ctrl/__init__.py,sha256=aXdEOZVrBQISQi6XPS9s1NhBjIJaIwNNxCFRiGchRAw,1369
 lsst/ctrl/mpexec/__init__.py,sha256=c2mK9--wemp2AEa81CSMkrqaIjJ1_iOs_M-O8CHJxB4,1757
 lsst/ctrl/mpexec/_pipeline_graph_factory.py,sha256=suzWUn9YGn0CTA_3N1Wd-sUo7TFxuo_6VZ2nO0CJ5a8,3552
-lsst/ctrl/mpexec/cmdLineFwk.py,sha256=L8CaSTLZDe9cz4KGSlelMGDjpTKXxC4DTBEw6u9YPS8,43132
+lsst/ctrl/mpexec/cmdLineFwk.py,sha256=hrjDfsaBO0oTqliksQgZywyN2uVY7E7DLZzUSvAOA4Y,43759
 lsst/ctrl/mpexec/dotTools.py,sha256=nx31DC7NkQM766sC97lPQMI22snVaWS7USFMFVLChqk,3421
 lsst/ctrl/mpexec/execFixupDataId.py,sha256=MUirmuoQiWj7C7wQ23QGkgMqx_5_CP4_UMV6h_gVfxk,5056
 lsst/ctrl/mpexec/executionGraphFixup.py,sha256=1z4P4YmtLTMPoMxmFiOnzljLcP9B5V1SCG9haqR_TyI,2830
@@ -15,19 +15,19 @@ lsst/ctrl/mpexec/reports.py,sha256=zxZOkJWhTFdl4eJTOYsM-58tqRbbyF5-cQ-zF-zIR9Y,7
 lsst/ctrl/mpexec/separablePipelineExecutor.py,sha256=6QvZ9EFHoqXwYjHbcNe9wWMssFbAg2I9mNpjFnTSiM0,12074
 lsst/ctrl/mpexec/showInfo.py,sha256=elrWQ8QPusZguLqYhvmMjYaNmoJgWpQDr3uQ0QhB1HI,16407
 lsst/ctrl/mpexec/simple_pipeline_executor.py,sha256=scqQ1MDS6FcZvFTwxtCkmTSg0zJmKIfah91lawr7Dhc,20454
-lsst/ctrl/mpexec/singleQuantumExecutor.py,sha256=mlv3nF29N1meiHie3r072Ynz5k6XTpFO8jfhq7BufF8,28036
+lsst/ctrl/mpexec/singleQuantumExecutor.py,sha256=sYYdUe6gKRZELFMVHxQLS5ZnBcrKJpxKnXFwchn-7Z8,29023
 lsst/ctrl/mpexec/taskFactory.py,sha256=c4xj8cR_Ts5uzzGovh87ZKdVeeXy5E3lIjCwJnDuOqg,2720
 lsst/ctrl/mpexec/util.py,sha256=y2Rw5PL40_EuLtVxiqSVX0JfPV4IrFl1LfOMUWx2u30,4236
-lsst/ctrl/mpexec/version.py,sha256=DWDCyavYRMZByGPGMYcnpz6piNv1sQvCKH8yohX_F3c,55
+lsst/ctrl/mpexec/version.py,sha256=uRdffTNnwH4GQy77H1Wk3Md64craDCIuRxmCtF7ON0g,55
 lsst/ctrl/mpexec/cli/__init__.py,sha256=6dpDHNBzyicVpFi1fsaiYVbYEMeoL57IHKkPaej24gs,1301
 lsst/ctrl/mpexec/cli/pipetask.py,sha256=4HnhX9dCizCihVbpHVJX5WXO9TEli9oL6wA-tPh1_vA,2209
 lsst/ctrl/mpexec/cli/utils.py,sha256=5iOrlj5jJTWtZS0BMLsuiCGAvxbfrjd1MSyXxBthWcc,6503
 lsst/ctrl/mpexec/cli/cmd/__init__.py,sha256=nRmwwW5d55gAEkyE7NpSK8mxa56HcfEta2r-Y9I07F8,1661
-lsst/ctrl/mpexec/cli/cmd/commands.py,sha256=Wevv7Wu3hgLY2tNA4RJdMpOFur9rbH5JtwgtNLPnqiY,17811
+lsst/ctrl/mpexec/cli/cmd/commands.py,sha256=hb4o2xaGRcImIv4FzHw7ABOdqTJWmlB07EWL8layVlM,17856
 lsst/ctrl/mpexec/cli/opt/__init__.py,sha256=IzUInuJj9igiaNcEqMx0adelgJtQC5_XMYnaiizBn0A,1378
 lsst/ctrl/mpexec/cli/opt/arguments.py,sha256=vjUw0ZN_4HStp-_3ne6AT5S_eH7sly3OVfL07tgrJnY,1572
 lsst/ctrl/mpexec/cli/opt/optionGroups.py,sha256=qtpwlgfvEzAs0CNav7t9ylywWBavoeWlZfYhRQEuZTw,8132
-lsst/ctrl/mpexec/cli/opt/options.py,sha256=8gu-QYCblBO_RlPGYFw__teRWxtRCyasGKz7wNhv_dc,20563
+lsst/ctrl/mpexec/cli/opt/options.py,sha256=1JK67cMx9F2poURXkA-q3Uru4B86CZoNtSzM6vmZ7pw,20867
 lsst/ctrl/mpexec/cli/script/__init__.py,sha256=eCuF4FAI5D3pl05IMJj7TCkZq-hireua2mA5Ui-mKSI,1624
 lsst/ctrl/mpexec/cli/script/build.py,sha256=Ff_8ffBmxuJi45x7-rUb-f0axucagYs6rNvBowwVmq0,6149
 lsst/ctrl/mpexec/cli/script/cleanup.py,sha256=D7W-azf4mNJcIWhbU5uCRCi94mkb8-Q2ksRFblQGrUw,4990
@@ -37,15 +37,15 @@ lsst/ctrl/mpexec/cli/script/purge.py,sha256=gYwSsZfTBP6oDcDp_YdqQEKGvAStvsj5hwNw
 lsst/ctrl/mpexec/cli/script/qgraph.py,sha256=ncjbtJ7AMp-Obo524HAC2E4Hnqyk1Aji4mJPAZUGMDM,10048
 lsst/ctrl/mpexec/cli/script/report.py,sha256=ItJitmYmWIDjj7PxRtP4qXLx-z5FAU6nSfI2gx0GS5k,12800
 lsst/ctrl/mpexec/cli/script/run.py,sha256=22g7vBhGDGsFSQq7CtruTf-bivo3t53decBAV0wAg1g,9767
-lsst/ctrl/mpexec/cli/script/run_qbb.py,sha256=oCEydg13zNAtci8XB0IwXx_D5PPsHbjCPATWoBlhcPk,5294
+lsst/ctrl/mpexec/cli/script/run_qbb.py,sha256=v05gQvngvKCWvFkRifQoUoPnGnkQ57ZBGWaNVFnmCB8,5528
 lsst/ctrl/mpexec/cli/script/update_graph_run.py,sha256=v_EdOaD6jR_vSlgm_5-pwUjoNEFMrAuYFM1xIaHVU3Q,2597
-lsst_ctrl_mpexec-29.2025.1600.dist-info/licenses/COPYRIGHT,sha256=pGCjnRAnyt02a6_9PLzXQikpvYmvMmK9fCdOKlRSV6k,369
-lsst_ctrl_mpexec-29.2025.1600.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_ctrl_mpexec-29.2025.1600.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_ctrl_mpexec-29.2025.1600.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_ctrl_mpexec-29.2025.1600.dist-info/METADATA,sha256=4w8JjdeOxf8HVCuGdbBxHxYx0DTb-RiKmQZ9rqkLLlk,2302
-lsst_ctrl_mpexec-29.2025.1600.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-lsst_ctrl_mpexec-29.2025.1600.dist-info/entry_points.txt,sha256=aYE38yqZU8qvpLUUkXzgmUxDJYYknEqPxgxYkowrL4s,64
-lsst_ctrl_mpexec-29.2025.1600.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_ctrl_mpexec-29.2025.1600.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_ctrl_mpexec-29.2025.1600.dist-info/RECORD,,
+lsst_ctrl_mpexec-29.2025.1800.dist-info/licenses/COPYRIGHT,sha256=pGCjnRAnyt02a6_9PLzXQikpvYmvMmK9fCdOKlRSV6k,369
+lsst_ctrl_mpexec-29.2025.1800.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_ctrl_mpexec-29.2025.1800.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_ctrl_mpexec-29.2025.1800.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_ctrl_mpexec-29.2025.1800.dist-info/METADATA,sha256=6AvPdrrvpzEHHIb2g9-CN7_O0BsC15DHhSNM0i0tUpU,2302
+lsst_ctrl_mpexec-29.2025.1800.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
+lsst_ctrl_mpexec-29.2025.1800.dist-info/entry_points.txt,sha256=aYE38yqZU8qvpLUUkXzgmUxDJYYknEqPxgxYkowrL4s,64
+lsst_ctrl_mpexec-29.2025.1800.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_ctrl_mpexec-29.2025.1800.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_ctrl_mpexec-29.2025.1800.dist-info/RECORD,,

{lsst_ctrl_mpexec-29.2025.1600.dist-info → lsst_ctrl_mpexec-29.2025.1800.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any