lsst-ctrl-mpexec 29.2025.1600__tar.gz → 29.2025.1800__tar.gz

{lsst_ctrl_mpexec-29.2025.1600/python/lsst_ctrl_mpexec.egg-info → lsst_ctrl_mpexec-29.2025.1800}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-mpexec
-Version: 29.2025.1600
+Version: 29.2025.1800
 Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_mpexec-29.2025.1600 → lsst_ctrl_mpexec-29.2025.1800}/doc/lsst.ctrl.mpexec/configuring-pipetask-tasks.rst

@@ -285,7 +285,7 @@ The ``taskName`` is the pipeline task label and can either come from the name se
 
 Here are two examples:
 
-- :file:`obs_lsst/config/makeWarp.py`: specifies which parameters are preferred when warping images using the ``obs_lsst`` observatory package.
+- :file:`obs_lsst/config/makeDirectWarp.py`: specifies which parameters are preferred when warping images using the ``obs_lsst`` observatory package.
 - :file:`obs_lsst/config/latiss/isr.py``: provides overrides for the instrument signature removal (aka detrending) task for the ``latiss`` camera in the ``obs_lsst`` observatory package.
 
 Overall, the priority order for setting task configurations is configurations is (highest priority first):

{lsst_ctrl_mpexec-29.2025.1600 → lsst_ctrl_mpexec-29.2025.1800}/python/lsst/ctrl/mpexec/cli/cmd/commands.py

@@ -311,6 +311,7 @@ def pre_exec_init_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
 @ctrlMpExecOpts.enable_implicit_threading_option()
 @ctrlMpExecOpts.cores_per_quantum_option()
 @ctrlMpExecOpts.memory_per_quantum_option()
+@ctrlMpExecOpts.no_existing_outputs_option()
 def run_qbb(repo: str, qgraph: str, **kwargs: Any) -> None:
     """Execute pipeline using Quantum-Backed Butler.
 

{lsst_ctrl_mpexec-29.2025.1600 → lsst_ctrl_mpexec-29.2025.1800}/python/lsst/ctrl/mpexec/cli/opt/options.py

@@ -396,6 +396,16 @@ clobber_outputs_option = MWOptionDecorator(
 )
 
 
+no_existing_outputs_option = MWOptionDecorator(
+    "--no-existing-outputs",
+    help=(
+        "Assume that no predicted outputs already exist in the output run collection. "
+        "This will eliminate existence checks that otherwise run before each quantum is executed."
+    ),
+    is_flag=True,
+)
+
+
 skip_init_writes_option = MWOptionDecorator(
     "--skip-init-writes",
     help=unwrap(

{lsst_ctrl_mpexec-29.2025.1600 → lsst_ctrl_mpexec-29.2025.1800}/python/lsst/ctrl/mpexec/cli/script/run_qbb.py

@@ -53,6 +53,7 @@ def run_qbb(
     cores_per_quantum: int,
     memory_per_quantum: str,
     raise_on_partial_outputs: bool,
+    no_existing_outputs: bool,
 ) -> None:
     """Implement the command line interface ``pipetask run-qbb`` subcommand.
 
@@ -104,6 +105,9 @@ def run_qbb(
         units of MB) or a combination of number and unit.
     raise_on_partial_outputs : `bool`
         Consider partial outputs an error instead of a success.
+    no_existing_outputs : `bool`
+        Whether to assume that no predicted outputs for these quanta already
+        exist in the output run collection.
     """
     # Fork option still exists for compatibility but we use spawn instead.
     if start_method == "fork":
@@ -131,6 +135,7 @@ def run_qbb(
         cores_per_quantum=cores_per_quantum,
         memory_per_quantum=memory_per_quantum,
         raise_on_partial_outputs=raise_on_partial_outputs,
+        no_existing_outputs=no_existing_outputs,
     )
 
     f = CmdLineFwk()

{lsst_ctrl_mpexec-29.2025.1600 → lsst_ctrl_mpexec-29.2025.1800}/python/lsst/ctrl/mpexec/cmdLineFwk.py

@@ -75,7 +75,7 @@ from lsst.pipe.base.mermaid_tools import graph2mermaid
 from lsst.pipe.base.pipeline_graph import NodeType
 from lsst.resources import ResourcePath
 from lsst.utils import doImportType
-from lsst.utils.logging import getLogger
+from lsst.utils.logging import VERBOSE, getLogger
 from lsst.utils.threads import disable_implicit_threading
 
 from ._pipeline_graph_factory import PipelineGraphFactory
@@ -844,7 +844,7 @@ class CmdLineFwk:
                 _LOG.debug("Will try to import debug.py")
                 import debug  # type: ignore # noqa:F401
             except ImportError:
-                _LOG.warn("No 'debug' module found.")
+                _LOG.warning("No 'debug' module found.")
 
         # Save all InitOutputs, configs, etc.
         preExecInit = PreExecInit(butler, taskFactory, extendRun=args.extend_run)
@@ -977,6 +977,7 @@ class CmdLineFwk:
         return None
 
     def preExecInitQBB(self, task_factory: TaskFactory, args: SimpleNamespace) -> None:
+        _LOG.verbose("Reading full quantum graph from %s.", args.qgraph)
         # Load quantum graph. We do not really need individual Quanta here,
         # but we need datastore records for initInputs, and those are only
         # available from Quanta, so load the whole thing.
@@ -986,8 +987,10 @@ class CmdLineFwk:
         _ButlerFactory.defineDatastoreCache()
 
         # Make QBB.
+        _LOG.verbose("Initializing quantum-backed butler.")
         butler = qgraph.make_init_qbb(args.butler_config, config_search_paths=args.config_search_path)
         # Save all InitOutputs, configs, etc.
+        _LOG.verbose("Instantiating tasks and saving init-outputs.")
         preExecInit = PreExecInitLimited(butler, task_factory)
         preExecInit.initialize(qgraph)
 
@@ -997,7 +1000,13 @@ class CmdLineFwk:
 
         # Load quantum graph.
         nodes = args.qgraph_node_id or None
-        qgraph = QuantumGraph.loadUri(args.qgraph, nodes=nodes, graphID=args.qgraph_id)
+        with lsst.utils.timer.time_this(
+            _LOG,
+            msg=f"Reading {str(len(nodes)) if nodes is not None else 'all'} quanta.",
+            level=VERBOSE,
+        ) as qg_read_time:
+            qgraph = QuantumGraph.loadUri(args.qgraph, nodes=nodes, graphID=args.qgraph_id)
+        job_metadata = {"qg_read_time": qg_read_time.duration, "qg_size": len(qgraph)}
 
         if qgraph.metadata is None:
             raise ValueError("QuantumGraph is missing metadata, cannot continue.")
@@ -1023,8 +1032,11 @@ class CmdLineFwk:
             enableLsstDebug=args.enableLsstDebug,
             limited_butler_factory=_butler_factory,
             resources=resources,
-            assumeNoExistingOutputs=True,
+            assumeNoExistingOutputs=args.no_existing_outputs,
+            skipExisting=True,
+            clobberOutputs=True,
             raise_on_partial_outputs=args.raise_on_partial_outputs,
+            job_metadata=job_metadata,
         )
 
         timeout = self.MP_TIMEOUT if args.timeout is None else args.timeout

{lsst_ctrl_mpexec-29.2025.1600 → lsst_ctrl_mpexec-29.2025.1800}/python/lsst/ctrl/mpexec/singleQuantumExecutor.py

@@ -34,12 +34,13 @@ import logging
 import time
 import uuid
 from collections import defaultdict
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from itertools import chain
 from typing import Any, cast
 
 from lsst.daf.butler import (
     Butler,
+    ButlerMetrics,
     CollectionType,
     DatasetRef,
     DatasetType,
@@ -127,6 +128,11 @@ class SingleQuantumExecutor(QuantumExecutor):
         `lsst.pipe.base.AnnotatedPartialOutputError` immediately, instead of
         considering the partial result a success and continuing to run
         downstream tasks.
+    job_metadata : `~collections.abc.Mapping`
+        Mapping with extra metadata to embed within the quantum metadata under
+        the "job" key.  This is intended to correspond to information common
+        to all quanta being executed in a single process, such as the time
+        taken to load the quantum graph in a BPS job.
     """
 
     def __init__(
@@ -141,6 +147,7 @@ class SingleQuantumExecutor(QuantumExecutor):
         skipExisting: bool = False,
         assumeNoExistingOutputs: bool = False,
         raise_on_partial_outputs: bool = True,
+        job_metadata: Mapping[str, int | str | float] | None = None,
     ):
         self.butler = butler
         self.taskFactory = taskFactory
@@ -150,6 +157,7 @@ class SingleQuantumExecutor(QuantumExecutor):
         self.resources = resources
         self.assumeNoExistingOutputs = assumeNoExistingOutputs
         self.raise_on_partial_outputs = raise_on_partial_outputs
+        self.job_metadata = job_metadata
 
         if self.butler is None:
             assert limited_butler_factory is not None, "limited_butler_factory is needed when butler is None"
@@ -246,6 +254,8 @@ class SingleQuantumExecutor(QuantumExecutor):
                 fullMetadata = _TASK_FULL_METADATA_TYPE()
                 fullMetadata[task_node.label] = _TASK_METADATA_TYPE()
                 fullMetadata["quantum"] = quantumMetadata
+                if self.job_metadata is not None:
+                    fullMetadata["job"] = self.job_metadata
                 self.writeMetadata(quantum, fullMetadata, task_node, limited_butler)
                 return quantum
 
@@ -255,7 +265,7 @@ class SingleQuantumExecutor(QuantumExecutor):
                     _LOG.debug("Will try to import debug.py")
                     import debug  # type: ignore # noqa:F401
                 except ImportError:
-                    _LOG.warn("No 'debug' module found.")
+                    _LOG.warning("No 'debug' module found.")
 
             # initialize global state
             self.initGlobals(quantum)
@@ -273,7 +283,7 @@ class SingleQuantumExecutor(QuantumExecutor):
             task = self.taskFactory.makeTask(task_node, limited_butler, init_input_refs)
             logInfo(None, "start", metadata=quantumMetadata)  # type: ignore[arg-type]
             try:
-                caveats, outputsPut = self.runQuantum(
+                caveats, outputsPut, butler_metrics = self.runQuantum(
                     task, quantum, task_node, limited_butler, quantum_id=quantum_id
                 )
             except Exception as e:
@@ -286,6 +296,7 @@ class SingleQuantumExecutor(QuantumExecutor):
                 )
                 raise
             else:
+                quantumMetadata["butler_metrics"] = butler_metrics.model_dump()
                 quantumMetadata["caveats"] = caveats.value
                 # Stringify the UUID for easier compatibility with
                 # PropertyList.
@@ -293,6 +304,8 @@ class SingleQuantumExecutor(QuantumExecutor):
             logInfo(None, "end", metadata=quantumMetadata)  # type: ignore[arg-type]
             fullMetadata = task.getFullMetadata()
             fullMetadata["quantum"] = quantumMetadata
+            if self.job_metadata is not None:
+                fullMetadata["job"] = self.job_metadata
             self.writeMetadata(quantum, fullMetadata, task_node, limited_butler)
             stopTime = time.time()
             _LOG.info(
@@ -488,7 +501,7 @@ class SingleQuantumExecutor(QuantumExecutor):
         /,
         limited_butler: LimitedButler,
         quantum_id: uuid.UUID | None = None,
-    ) -> tuple[QuantumSuccessCaveats, list[uuid.UUID]]:
+    ) -> tuple[QuantumSuccessCaveats, list[uuid.UUID], ButlerMetrics]:
         """Execute task on a single quantum.
 
         Parameters
@@ -511,6 +524,8 @@ class SingleQuantumExecutor(QuantumExecutor):
         ids_put : list[ `uuid.UUID` ]
             Record of all the dataset IDs that were written by this quantum
             being executed.
+        metrics : `lsst.daf.butler.ButlerMetrics`
+            Butler metrics recorded for this quantum.
         """
         flags = QuantumSuccessCaveats.NO_CAVEATS
 
@@ -522,7 +537,8 @@ class SingleQuantumExecutor(QuantumExecutor):
 
         # Call task runQuantum() method.
         try:
-            task.runQuantum(butlerQC, inputRefs, outputRefs)
+            with limited_butler.record_metrics() as butler_metrics:
+                task.runQuantum(butlerQC, inputRefs, outputRefs)
         except NoWorkFound as err:
             # Not an error, just an early exit.
             _LOG.info(
@@ -565,7 +581,7 @@ class SingleQuantumExecutor(QuantumExecutor):
         if not butlerQC.outputsPut == butlerQC.allOutputs:
             flags |= QuantumSuccessCaveats.ANY_OUTPUTS_MISSING
         ids_put = [output[2] for output in butlerQC.outputsPut]
-        return flags, ids_put
+        return flags, ids_put, butler_metrics
 
     def writeMetadata(
         self, quantum: Quantum, metadata: Any, task_node: TaskNode, /, limited_butler: LimitedButler

lsst_ctrl_mpexec-29.2025.1800/python/lsst/ctrl/mpexec/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.1800"

{lsst_ctrl_mpexec-29.2025.1600 → lsst_ctrl_mpexec-29.2025.1800/python/lsst_ctrl_mpexec.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-mpexec
-Version: 29.2025.1600
+Version: 29.2025.1800
 Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_mpexec-29.2025.1600 → lsst_ctrl_mpexec-29.2025.1800}/tests/test_cmdLineFwk.py

@@ -656,7 +656,12 @@ class CmdLineFwkTestCaseWithButler(unittest.TestCase):
         with tempfile.NamedTemporaryFile(suffix=".qgraph") as temp_graph:
             qgraph.saveUri(temp_graph.name)
 
-            args = _makeArgs(butler_config=self.root, qgraph=temp_graph.name, config_search_path=[])
+            args = _makeArgs(
+                butler_config=self.root,
+                qgraph=temp_graph.name,
+                config_search_path=[],
+                no_existing_outputs=False,
+            )
 
             # Check that pre-exec-init can run.
             fwk.preExecInitQBB(taskFactory, args)
@@ -670,6 +675,16 @@ class CmdLineFwkTestCaseWithButler(unittest.TestCase):
 
         self.assertEqual(taskFactory.countExec, self.nQuanta)
 
+        some_task_label = next(iter(qgraph.pipeline_graph.tasks))
+        (some_metadata_ref,) = butler.query_datasets(
+            f"{some_task_label}_metadata",
+            limit=1,
+            collections=output_run,
+        )
+        some_metadata = butler.get(some_metadata_ref)
+        self.assertIn("qg_read_time", some_metadata["job"])
+        self.assertIn("qg_size", some_metadata["job"])
+
         # Update the output run and try again.
         new_output_run = output_run + "_new"
         qgraph.updateRun(new_output_run, metadata_key="output_run", update_graph_id=True)
@@ -679,7 +694,12 @@ class CmdLineFwkTestCaseWithButler(unittest.TestCase):
         with tempfile.NamedTemporaryFile(suffix=".qgraph") as temp_graph:
             qgraph.saveUri(temp_graph.name)
 
-            args = _makeArgs(butler_config=self.root, qgraph=temp_graph.name, config_search_path=[])
+            args = _makeArgs(
+                butler_config=self.root,
+                qgraph=temp_graph.name,
+                config_search_path=[],
+                no_existing_outputs=False,
+            )
 
             # Check that pre-exec-init can run.
             fwk.preExecInitQBB(taskFactory, args)

lsst_ctrl_mpexec-29.2025.1600/python/lsst/ctrl/mpexec/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.1600"