lsst-ctrl-mpexec 29.2025.2000__tar.gz → 29.2025.2400__tar.gz

{lsst_ctrl_mpexec-29.2025.2000/python/lsst_ctrl_mpexec.egg-info → lsst_ctrl_mpexec-29.2025.2400}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-mpexec
-Version: 29.2025.2000
+Version: 29.2025.2400
 Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_mpexec-29.2025.2000 → lsst_ctrl_mpexec-29.2025.2400}/python/lsst/ctrl/mpexec/simple_pipeline_executor.py

@@ -31,10 +31,11 @@ __all__ = ("SimplePipelineExecutor",)
 
 import datetime
 import getpass
+import os
 from collections.abc import Iterable, Iterator, Mapping
 from typing import Any
 
-from lsst.daf.butler import Butler, CollectionType, Quantum
+from lsst.daf.butler import Butler, CollectionType, DataCoordinate, DatasetRef, Quantum
 from lsst.pex.config import Config
 from lsst.pipe.base import ExecutionResources, Instrument, Pipeline, PipelineGraph, PipelineTask, QuantumGraph
 from lsst.pipe.base.all_dimensions_quantum_graph_builder import AllDimensionsQuantumGraphBuilder
@@ -152,6 +153,8 @@ class SimplePipelineExecutor:
         resources: ExecutionResources | None = None,
         raise_on_partial_outputs: bool = True,
         attach_datastore_records: bool = False,
+        output: str | None = None,
+        output_run: str | None = None,
     ) -> SimplePipelineExecutor:
         """Create an executor by building a QuantumGraph from an on-disk
         pipeline YAML file.
@@ -179,6 +182,13 @@ class SimplePipelineExecutor:
             Whether to attach datastore records to the quantum graph.  This is
             usually unnecessary, unless the executor is used to test behavior
             that depends on datastore records.
+        output : `str`, optional
+            Name of a new output `~lsst.daf.butler.CollectionType.CHAINED`
+            collection to create that will combine both inputs and outputs.
+        output_run : `str`, optional
+            Name of the output `~lsst.daf.butler.CollectionType.RUN` that will
+            directly hold all output datasets.  If not provided, a name will
+            be created from ``output`` and a timestamp.
 
         Returns
         -------
@@ -196,6 +206,8 @@ class SimplePipelineExecutor:
             resources=resources,
             raise_on_partial_outputs=raise_on_partial_outputs,
             attach_datastore_records=attach_datastore_records,
+            output=output,
+            output_run=output_run,
         )
 
     @classmethod
@@ -211,6 +223,8 @@ class SimplePipelineExecutor:
         resources: ExecutionResources | None = None,
         raise_on_partial_outputs: bool = True,
         attach_datastore_records: bool = False,
+        output: str | None = None,
+        output_run: str | None = None,
     ) -> SimplePipelineExecutor:
         """Create an executor by building a QuantumGraph from a pipeline
         containing a single task.
@@ -244,6 +258,13 @@ class SimplePipelineExecutor:
             Whether to attach datastore records to the quantum graph.  This is
             usually unnecessary, unless the executor is used to test behavior
             that depends on datastore records.
+        output : `str`, optional
+            Name of a new output `~lsst.daf.butler.CollectionType.CHAINED`
+            collection to create that will combine both inputs and outputs.
+        output_run : `str`, optional
+            Name of the output `~lsst.daf.butler.CollectionType.RUN` that will
+            directly hold all output datasets.  If not provided, a name will
+            be created from ``output`` and a timestamp.
 
         Returns
         -------
@@ -271,6 +292,8 @@ class SimplePipelineExecutor:
             resources=resources,
             raise_on_partial_outputs=raise_on_partial_outputs,
             attach_datastore_records=attach_datastore_records,
+            output=output,
+            output_run=output_run,
         )
 
     @classmethod
@@ -284,6 +307,8 @@ class SimplePipelineExecutor:
         resources: ExecutionResources | None = None,
         raise_on_partial_outputs: bool = True,
         attach_datastore_records: bool = False,
+        output: str | None = None,
+        output_run: str | None = None,
     ) -> SimplePipelineExecutor:
         """Create an executor by building a QuantumGraph from an in-memory
         pipeline.
@@ -313,6 +338,13 @@ class SimplePipelineExecutor:
             Whether to attach datastore records to the quantum graph.  This is
             usually unnecessary, unless the executor is used to test behavior
             that depends on datastore records.
+        output : `str`, optional
+            Name of a new output `~lsst.daf.butler.CollectionType.CHAINED`
+            collection to create that will combine both inputs and outputs.
+        output_run : `str`, optional
+            Name of the output `~lsst.daf.butler.CollectionType.RUN` that will
+            directly hold all output datasets.  If not provided, a name will
+            be created from ``output`` and a timestamp.
 
         Returns
         -------
@@ -330,6 +362,8 @@ class SimplePipelineExecutor:
             resources=resources,
             raise_on_partial_outputs=raise_on_partial_outputs,
             attach_datastore_records=attach_datastore_records,
+            output=output,
+            output_run=output_run,
         )
 
     @classmethod
@@ -343,6 +377,8 @@ class SimplePipelineExecutor:
         resources: ExecutionResources | None = None,
         raise_on_partial_outputs: bool = True,
         attach_datastore_records: bool = False,
+        output: str | None = None,
+        output_run: str | None = None,
     ) -> SimplePipelineExecutor:
         """Create an executor by building a QuantumGraph from an in-memory
         pipeline graph.
@@ -373,6 +409,13 @@ class SimplePipelineExecutor:
             Whether to attach datastore records to the quantum graph.  This is
             usually unnecessary, unless the executor is used to test behavior
             that depends on datastore records.
+        output : `str`, optional
+            Name of a new output `~lsst.daf.butler.CollectionType.CHAINED`
+            collection to create that will combine both inputs and outputs.
+        output_run : `str`, optional
+            Name of the output `~lsst.daf.butler.CollectionType.RUN` that will
+            directly hold all output datasets.  If not provided, a name will
+            be created from ``output`` and a timestamp.
 
         Returns
         -------
@@ -381,12 +424,20 @@ class SimplePipelineExecutor:
             `~lsst.pipe.base.QuantumGraph` and `~lsst.daf.butler.Butler`, ready
             for `run` to be called.
         """
+        if output_run is None:
+            output_run = butler.run
+            if output_run is None:
+                if output is None:
+                    raise TypeError("At least one of output or output_run must be provided.")
+                output_run = f"{output}/{Instrument.makeCollectionTimestamp()}"
+
         quantum_graph_builder = AllDimensionsQuantumGraphBuilder(
-            pipeline_graph, butler, where=where, bind=bind
+            pipeline_graph, butler, where=where, bind=bind, output_run=output_run
         )
         metadata = {
             "input": list(butler.collections.defaults),
-            "output_run": butler.run,
+            "output": output,
+            "output_run": output_run,
             "skip_existing_in": [],
             "skip_existing": False,
             "data_query": where,
@@ -403,6 +454,92 @@ class SimplePipelineExecutor:
             raise_on_partial_outputs=raise_on_partial_outputs,
         )
 
+    def use_local_butler(
+        self, root: str, register_dataset_types: bool = True, transfer_dimensions: bool = True
+    ) -> Butler:
+        """Transfer all inputs to a local data repository. and set the executor
+        to write outputs to it.
+
+        Parameters
+        ----------
+        root : `str`
+            Path to the local data repository; created if it does not exist.
+        register_dataset_types : `bool`, optional
+            Whether to register dataset types in the new repository.  If
+            `False`, the local data repository must already exist and already
+            have all input dataset types registered.
+        transfer_dimensions : `bool`, optional
+            Whether to transfer dimension records to the new repository.  If
+            `False`, the local data repository must already exist and already
+            have all needed dimension records.
+
+        Returns
+        -------
+        butler : `lsst.daf.butler.Butler`
+            Writeable butler for local data repository.
+
+        Notes
+        -----
+        The input collection structure from the original data repository is not
+        preserved by this method (it cannot be reconstructed from the quantum
+        graph).  Instead, a `~lsst.daf.butler.CollectionType.TAGGED` collection
+        is created to gather all inputs, and appended to the output
+        `~lsst.daf.butler.CollectionType.CHAINED` collection after the output
+        `~lsst.daf.butler.CollectionType.RUN` collection.  Calibration inputs
+        with the same data ID but multiple validity ranges are *not* included
+        in that `~lsst.daf.butler.CollectionType.TAGGED`; they are still
+        transferred to the local data repository, but can only be found via the
+        quantum graph or their original `~lsst.daf.butler.CollectionType.RUN`
+        collections.
+        """
+        if not os.path.exists(root):
+            Butler.makeRepo(root)
+        out_butler = Butler.from_config(root, writeable=True)
+
+        output_run = self.quantum_graph.metadata["output_run"]
+        output = self.quantum_graph.metadata["output"]
+        inputs = f"{output}/inputs"
+        out_butler.collections.register(output_run, CollectionType.RUN)
+        out_butler.collections.register(output, CollectionType.CHAINED)
+        out_butler.collections.register(inputs, CollectionType.TAGGED)
+        out_butler.collections.redefine_chain(output, [output_run, inputs])
+
+        refs: set[DatasetRef] = set()
+        to_tag_by_type: dict[str, dict[DataCoordinate, DatasetRef | None]] = {}
+        pipeline_graph = self.quantum_graph.pipeline_graph
+        for name, _ in pipeline_graph.iter_overall_inputs():
+            to_tag_for_type = to_tag_by_type.setdefault(name, {})
+            for task_node in pipeline_graph.consumers_of(name):
+                for quantum in self.quantum_graph.get_task_quanta(task_node.label).values():
+                    refs.update(quantum.inputs[name])
+                    for ref in quantum.inputs[name]:
+                        if to_tag_for_type.setdefault(ref.dataId, ref) != ref:
+                            # There is already a dataset with the same data ID
+                            # and dataset type, but a different UUID/run.  This
+                            # can only happen for calibrations found in
+                            # calibration collections, and for now we have no
+                            # choice but to leave them out of the TAGGED inputs
+                            # collection in the local butler.
+                            to_tag_for_type[ref.dataId] = None
+
+        out_butler.transfer_from(
+            self.butler,
+            refs,
+            register_dataset_types=register_dataset_types,
+            transfer_dimensions=transfer_dimensions,
+        )
+
+        to_tag_flat: list[DatasetRef] = []
+        for ref_map in to_tag_by_type.values():
+            for tag_ref in ref_map.values():
+                if tag_ref is not None:
+                    to_tag_flat.append(tag_ref)
+        out_butler.registry.associate(inputs, to_tag_flat)
+
+        out_butler.registry.defaults = self.butler.registry.defaults.clone(collections=output, run=output_run)
+        self.butler = out_butler
+        return self.butler
+
     def run(self, register_dataset_types: bool = False, save_versions: bool = True) -> list[Quantum]:
         """Run all the quanta in the `~lsst.pipe.base.QuantumGraph` in
         topological order.

lsst_ctrl_mpexec-29.2025.2400/python/lsst/ctrl/mpexec/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2400"

{lsst_ctrl_mpexec-29.2025.2000 → lsst_ctrl_mpexec-29.2025.2400/python/lsst_ctrl_mpexec.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-ctrl-mpexec
-Version: 29.2025.2000
+Version: 29.2025.2400
 Summary: Pipeline execution infrastructure for the Rubin Observatory LSST Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_ctrl_mpexec-29.2025.2000 → lsst_ctrl_mpexec-29.2025.2400}/tests/test_cmdLineFwk.py

@@ -670,7 +670,7 @@ class CmdLineFwkTestCaseWithButler(unittest.TestCase):
             fwk.runGraphQBB(taskFactory, args)
 
             # Transfer the datasets to the butler.
-            n1 = transfer_from_graph(temp_graph.name, self.root, True, False, False, False)
+            n1 = transfer_from_graph(temp_graph.name, self.root, True, False, False, False, ())
             self.assertEqual(n1, 31)
 
         self.assertEqual(taskFactory.countExec, self.nQuanta)
@@ -708,7 +708,7 @@ class CmdLineFwkTestCaseWithButler(unittest.TestCase):
             fwk.runGraphQBB(taskFactory, args)
 
             # Transfer the datasets to the butler.
-            n2 = transfer_from_graph(temp_graph.name, self.root, True, False, False, False)
+            n2 = transfer_from_graph(temp_graph.name, self.root, True, False, False, False, ())
             self.assertEqual(n1, n2)
 
     def testEmptyQGraph(self):

{lsst_ctrl_mpexec-29.2025.2000 → lsst_ctrl_mpexec-29.2025.2400}/tests/test_simple_pipeline_executor.py

@@ -179,51 +179,35 @@ class SimplePipelineExecutorTests(lsst.utils.tests.TestCase):
         # The final output was written.
         self.assertTrue(self.butler.exists("output"))
 
-    def test_from_pipeline_file(self):
+    def test_from_pipeline_file(self) -> None:
         """Test executing a two quanta from different configurations of the
         same task, with an executor created by the `from_pipeline_filename`
         factory method, and the `SimplePipelineExecutor.run` method.
         """
-        filename = os.path.join(self.path, "pipeline.yaml")
-        with open(filename, "w") as f:
-            f.write(
-                """
-                description: test
-                tasks:
-                    a:
-                        class: "lsst.pipe.base.tests.mocks.DynamicTestPipelineTask"
-                        config:
-                            python: |
-                                from lsst.pipe.base.tests.mocks import DynamicConnectionConfig
-                                config.inputs["i"] = DynamicConnectionConfig(
-                                    dataset_type_name="input",
-                                    storage_class="StructuredDataDict",
-                                    mock_storage_class=False,
-                                )
-                                config.outputs["o"] = DynamicConnectionConfig(
-                                    dataset_type_name="intermediate",
-                                    storage_class="StructuredDataDict",
-                                )
-                    b:
-                        class: "lsst.pipe.base.tests.mocks.DynamicTestPipelineTask"
-                        config:
-                            python: |
-                                from lsst.pipe.base.tests.mocks import DynamicConnectionConfig
-                                config.inputs["i"] = DynamicConnectionConfig(
-                                    dataset_type_name="intermediate",
-                                    storage_class="StructuredDataDict",
-                                )
-                                config.outputs["o"] = DynamicConnectionConfig(
-                                    dataset_type_name="output",
-                                    storage_class="StructuredDataDict",
-                                )
-                """
-            )
+        filename = os.path.join(TESTDIR, "pipeline_simple.yaml")
         executor = SimplePipelineExecutor.from_pipeline_filename(filename, butler=self.butler)
+        self._test_pipeline_file(executor)
+
+    def test_use_local_butler(self) -> None:
+        """Test generating a local butler repository from a pipeline, then
+        running that pipeline using the local butler.
+        """
+        filename = os.path.join(TESTDIR, "pipeline_simple.yaml")
+        executor = SimplePipelineExecutor.from_pipeline_filename(
+            filename, butler=self.butler, output="u/someone/pipeline"
+        )
+        with tempfile.TemporaryDirectory() as tempdir:
+            root = os.path.join(tempdir, "butler_root")
+            executor.use_local_butler(root)
+            self._test_pipeline_file(executor)
+
+    def _test_pipeline_file(self, executor: SimplePipelineExecutor) -> None:
         quanta = executor.run(register_dataset_types=True, save_versions=False)
         self.assertEqual(len(quanta), 2)
-        self.assertEqual(self.butler.get("intermediate").storage_class, get_mock_name("StructuredDataDict"))
-        self.assertEqual(self.butler.get("output").storage_class, get_mock_name("StructuredDataDict"))
+        self.assertEqual(
+            executor.butler.get("intermediate").storage_class, get_mock_name("StructuredDataDict")
+        )
+        self.assertEqual(executor.butler.get("output").storage_class, get_mock_name("StructuredDataDict"))
 
     def test_partial_outputs_success(self):
         """Test executing two quanta where the first raises

lsst_ctrl_mpexec-29.2025.2000/python/lsst/ctrl/mpexec/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2000"