lsst-pipe-base 29.2025.2800__py3-none-any.whl → 29.2025.3000__py3-none-any.whl

lsst/pipe/base/connections.py

@@ -45,6 +45,7 @@ __all__ = [
 import dataclasses
 import itertools
 import string
+import uuid
 import warnings
 from collections import UserDict, defaultdict
 from collections.abc import Collection, Generator, Iterable, Iterator, Mapping, Sequence, Set
@@ -1271,6 +1272,52 @@ class QuantaAdjuster:
             for edge in self._task_node.iter_all_inputs()
         }
 
+    def get_prerequisite_inputs(
+        self,
+        quantum_data_id: DataCoordinate,
+    ) -> dict[str, dict[uuid.UUID, DataCoordinate]]:
+        """Return the data IDs of all prerequisite inputs to a quantum.
+
+        Parameters
+        ----------
+        quantum_data_id : `~lsst.daf.butler.DataCoordinate`
+            Data ID of the quantum to get the inputs of.
+
+        Returns
+        -------
+        inputs : `dict` [ `str`, \
+            `dict` [ `uuid.UUID`, `~lsst.daf.butler.DataCoordinate` ] ]
+            Dataset IDs and and data IDs of prerequisite inputs, keyed by the
+            connection name (the internal task name, not the dataset type
+            name).  This only contains prerequisite inputs, not init-inputs or
+            regular inputs.
+
+        Notes
+        -----
+        If two connections have the same dataset type, the current
+        implementation assumes the set of datasets is the same for the two
+        connections.  This limitation may be removed in the future.
+
+        Unlike regular inputs, prerequisite inputs are not looked up from input
+        collections or indexed by data ID.  Instead, they are uniquely
+        identified by dataset UUID and reused directly between quanta.
+        """
+        from .quantum_graph_skeleton import PrerequisiteDatasetKey, QuantumKey
+
+        by_dataset_type_name: defaultdict[str, dict[uuid.UUID, DataCoordinate]] = defaultdict(dict)
+        quantum_key = QuantumKey(self._task_node.label, quantum_data_id.required_values)
+
+        for dataset_key in self._skeleton.iter_inputs_of(quantum_key):
+            if not isinstance(dataset_key, PrerequisiteDatasetKey):
+                continue
+            by_dataset_type_name[dataset_key.parent_dataset_type_name][
+                uuid.UUID(bytes=dataset_key.dataset_id_bytes)
+            ] = self._skeleton.get_data_id(dataset_key)
+        return {
+            edge.connection_name: by_dataset_type_name[edge.parent_dataset_type_name]
+            for edge in self._task_node.iter_all_inputs()
+        }
+
     def get_outputs(self, quantum_data_id: DataCoordinate) -> dict[str, list[DataCoordinate]]:
         """Return the data IDs of all regular outputs to a quantum.
 
@@ -1340,6 +1387,45 @@ class QuantaAdjuster:
             )
         self._skeleton.add_input_edge(quantum_key, dataset_key)
 
+    def add_prerequisite_input(
+        self, quantum_data_id: DataCoordinate, connection_name: str, dataset_uuid: uuid.UUID
+    ) -> None:
+        """Add a new prerequisite input to a quantum.
+
+        Parameters
+        ----------
+        quantum_data_id : `~lsst.daf.butler.DataCoordinate`
+            Data ID of the quantum to add an input to.
+        connection_name : `str`
+            Name of the connection (the task-internal name, not the butler
+            dataset type name).
+        dataset_uuid : `uuid.UUID`
+            UUID of the prerequisite input dataset.  Must already exist in the
+            graph as an input to a different quantum of this task, and must be
+            a prerequisite input, not a regular input or init-input.
+
+        Notes
+        -----
+        If two connections have the same dataset type, the current
+        implementation assumes the set of datasets is the same for the two
+        connections.  This limitation may be removed in the future.
+
+        Unlike regular inputs, prerequisite inputs are not looked up from input
+        collections or indexed by data ID.  Instead, they are uniquely
+        identified by dataset UUID and reused directly between quanta.
+        """
+        from .quantum_graph_skeleton import PrerequisiteDatasetKey, QuantumKey
+
+        quantum_key = QuantumKey(self._task_node.label, quantum_data_id.required_values)
+        read_edge = self._task_node.prerequisite_inputs[connection_name]
+        dataset_key = PrerequisiteDatasetKey(read_edge.parent_dataset_type_name, dataset_uuid.bytes)
+        if dataset_key not in self._skeleton:
+            raise LookupError(
+                f"Prerequisite Dataset {read_edge.parent_dataset_type_name}@{dataset_uuid} "
+                "is not already in the graph."
+            )
+        self._skeleton.add_input_edge(quantum_key, dataset_key)
+
     def move_output(
         self, quantum_data_id: DataCoordinate, connection_name: str, dataset_data_id: DataCoordinate
     ) -> None:

lsst/pipe/base/prerequisite_helpers.py

@@ -353,6 +353,8 @@ class PrerequisiteFinder:
                         where=" AND ".join(where_terms),
                         bind=bind,
                         with_dimension_records=True,
+                        limit=None,
+                        explain=False,
                     )
                 except MissingDatasetTypeError:
                     return []
@@ -383,6 +385,8 @@ class PrerequisiteFinder:
                 collections=input_collections,
                 data_id=data_id,
                 with_dimension_records=True,
+                limit=None,
+                explain=False,
             )
         except MissingDatasetTypeError:
             return []

lsst/pipe/base/script/transfer_from_graph.py

@@ -29,8 +29,7 @@ __all__ = ["transfer_from_graph"]
 
 import math
 
-from lsst.daf.butler import Butler, CollectionType, QuantumBackedButler, Registry
-from lsst.daf.butler.registry import MissingCollectionError
+from lsst.daf.butler import Butler, CollectionType, MissingCollectionError, QuantumBackedButler
 from lsst.pipe.base import QuantumGraph
 from lsst.utils.iteration import chunk_iterable
 from lsst.utils.logging import getLogger
@@ -124,45 +123,55 @@ def transfer_from_graph(
         )
         count += len(transferred)
 
-    # If anything was transferred then update output chain definition if asked.
-    if count > 0 and update_output_chain and (metadata := qgraph.metadata) is not None:
+    # If asked to do so, update output chain definition.
+    if update_output_chain and (metadata := qgraph.metadata) is not None:
         # These are defined in CmdLineFwk.
         output_run = metadata.get("output_run")
         output = metadata.get("output")
         input = metadata.get("input")
         if output_run is not None and output is not None:
-            _update_chain(dest_butler.registry, output, output_run, input)
+            _update_chain(dest_butler, output, output_run, input)
 
     return count
 
 
-def _update_chain(registry: Registry, output_chain: str, output_run: str, inputs: list[str] | None) -> None:
+def _update_chain(butler: Butler, output_chain: str, output_run: str, inputs: list[str] | None) -> None:
     """Update chain definition if it exists to include run as the first item
     in a chain. If it does not exist then create it to include all inputs and
     output.
+
+    Parameters
+    ----------
+    butler : `lsst.daf.butler.Butler`
+        Butler where to update the collection chain.
+    output_chain : `str`
+        Name of the output CHAINED collection.
+    output_run : `str`
+        Name of the output RUN collection.
+    inputs : `list` [`str`] | None
+        All the input collections to be included in the chain if the
+        chain is created.
     """
+    # Do not need to update chain if output_run does not already exist.
     try:
-        # If output_chain is not a chain the exception will be raised.
-        chain_definition = list(registry.getCollectionChain(output_chain))
+        _ = butler.collections.get_info(output_run)
     except MissingCollectionError:
-        # We have to create chained collection to include inputs and output run
-        # (this reproduces logic in CmdLineFwk).
-        registry.registerCollection(output_chain, type=CollectionType.CHAINED)
-        chain_definition = list(registry.queryCollections(inputs, flattenChains=True)) if inputs else []
-        chain_definition = [output_run] + [run for run in chain_definition if run != output_run]
-        registry.setCollectionChain(output_chain, chain_definition)
-    else:
-        # If run is in the chain but not the first item then remove it, will
-        # re-insert at front below.
-        try:
-            index = chain_definition.index(output_run)
-            if index == 0:
-                # It is already at the top.
-                return
-            else:
-                del chain_definition[index]
-        except ValueError:
-            pass
-
-        chain_definition.insert(0, output_run)
-        registry.setCollectionChain(output_chain, chain_definition)
+        _LOG.verbose(
+            "Output RUN collection (%s) does not exist.  Skipping updating the output chain.",
+            output_run,
+        )
+        return
+
+    # Make chain collection if doesn't exist before calling prepend_chain.
+    created_now = butler.collections.register(output_chain, CollectionType.CHAINED)
+    if created_now:
+        _LOG.verbose("Registered chain collection: %s", output_chain)
+        if inputs:
+            # Add input collections to chain collection just made.  Using
+            # extend instead of prepend in case of race condition where another
+            # execution adds a run before this adds the inputs to the chain.
+            butler.collections.extend_chain(output_chain, inputs)
+    _LOG.verbose(
+        "Prepending output chain collection (%s) with output RUN collection (%s)", output_chain, output_run
+    )
+    butler.collections.prepend_chain(output_chain, output_run)

lsst/pipe/base/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.2800"
+__version__ = "29.2025.3000"

{lsst_pipe_base-29.2025.2800.dist-info → lsst_pipe_base-29.2025.3000.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2800
+Version: 29.2025.3000
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2800.dist-info → lsst_pipe_base-29.2025.3000.dist-info}/RECORD

@@ -14,7 +14,7 @@ lsst/pipe/base/caching_limited_butler.py,sha256=u1uJYzCE7OxW8MW8Xv2LDB9-Nuj-Ao8l
 lsst/pipe/base/config.py,sha256=yNipVEc6awwhU_O9I01g20OnvQrs28dAwkXuI1hrlYE,11982
 lsst/pipe/base/configOverrides.py,sha256=B0An8EaX76VzWnC5dJxvyZ2AhVzawMtq7qlE9ma5lkc,14661
 lsst/pipe/base/connectionTypes.py,sha256=5QbyL7MAikohjXimXbmtPWryjLCZZXlS_vPzIWy5mzI,16708
-lsst/pipe/base/connections.py,sha256=248r_HT3mNDPrJzTviq3dp3zGKdfUchSPWZRupKAkV0,62795
+lsst/pipe/base/connections.py,sha256=_tr_9RWs-LWetDW1tJX3ZROE5ZqvlLdIRStnz_11J5U,66707
 lsst/pipe/base/dot_tools.py,sha256=o_bDp9vW-4PelE7kWodH6pWVIRuyGlTRFv-kR_YKfLo,13824
 lsst/pipe/base/executionButlerBuilder.py,sha256=-vv-1aGm06RM4fJECjvJL0ZXHrwth7Hjt0jIfkKYY18,21254
 lsst/pipe/base/execution_reports.py,sha256=jYtWCD4PkEAeVUpKIxuiJJVgsCm7qiwCorWVgNHkVgU,17270
@@ -22,7 +22,7 @@ lsst/pipe/base/mermaid_tools.py,sha256=b_15oqCcxSom4ecMTDX8tfEtxe8W-juPVL65HOMyw
 lsst/pipe/base/pipeline.py,sha256=FVaiLhgw9Pzo-nzXKS0dLNafegP0AMZKLtPlSvOSkRU,37563
 lsst/pipe/base/pipelineIR.py,sha256=DDOAYHnMP-iw021RDMYsZnvb21tWumLjYqO5d38q_Zk,44300
 lsst/pipe/base/pipelineTask.py,sha256=K3GdjJLvy8A7I-jzQiERQZaYF7mC1LM3iB5TmUtbOCI,8394
-lsst/pipe/base/prerequisite_helpers.py,sha256=WxfIGkF0Wlucp9mE3Wp3E6K2M6d66O0oZrWecRqn5CI,28312
+lsst/pipe/base/prerequisite_helpers.py,sha256=bmiebQ4veSrypZgAXjmCBFfj8fUtPW9eRQaVShhxdBQ,28446
 lsst/pipe/base/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/pipe/base/quantum_graph_builder.py,sha256=ugTOV5Qe6iWlaX9Zm1HoNWfnFFhohSmIrSFkCFUVOZM,54808
 lsst/pipe/base/quantum_graph_skeleton.py,sha256=iBSVCAV1OznYd70Lg4bLn0nMqjW5m9-kH5eecPn8DYw,24964
@@ -32,7 +32,7 @@ lsst/pipe/base/task.py,sha256=XHBd-7m1a4-6LgobBYA1DgY4H7EV-_RWKfxbhZbMmD4,15145
 lsst/pipe/base/taskFactory.py,sha256=4GhN2DozPM8suBYIvoKN4E6VP0I3mYZHBjCUO5JcCGk,2901
 lsst/pipe/base/testUtils.py,sha256=lSBKMhoKflbi8JkMNYfEqqHNl-rtFI8UYT3QneDYpLo,18477
 lsst/pipe/base/utils.py,sha256=JmEt3l0xrh9uayKrSXuQEq12aXOhDr2YXmbYduaxCko,1940
-lsst/pipe/base/version.py,sha256=zkbtcIsz6tbu7rmamcut51DoG62vyYwam95urGw5Lis,55
+lsst/pipe/base/version.py,sha256=GKnpddvvmGJ8yPsnT--mhT32jeA31BaGZsJcw6cTK7k,55
 lsst/pipe/base/cli/__init__.py,sha256=861tXIAW7SqtqNUYkjbeEdfg8lDswXsjJQca0gVCFz4,54
 lsst/pipe/base/cli/_get_cli_subcommands.py,sha256=g_af64klRybBGKAg7fmBSZBdw2LYBAsFON_yQIMZON0,1289
 lsst/pipe/base/cli/cmd/__init__.py,sha256=BGicstnryQ48rYcNRh4fa6Vy63ZIlZ_pPAEa17jhkwY,1519
@@ -74,7 +74,7 @@ lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py,sha256=dp7PXl9C
 lsst/pipe/base/script/__init__.py,sha256=cLEXE7aq5UZ0juL_ScmRw0weFgp4tDgwEX_ts-NEYic,1522
 lsst/pipe/base/script/register_instrument.py,sha256=TRC2r2tSoYBNWNVQya01ELxAtGH8WVk9Ya-uNgCIL5U,2426
 lsst/pipe/base/script/retrieve_artifacts_for_quanta.py,sha256=pYI0wNl5PU8ImgzWfGEDrRz3PSKSg2szWLEIVKdm7Og,3939
-lsst/pipe/base/script/transfer_from_graph.py,sha256=WMpmzaN8hdW4FoZPNL6LRiPW9gvxf9mw20Z1i-KCkWI,6740
+lsst/pipe/base/script/transfer_from_graph.py,sha256=_7zuKf6NdeIzXafkOAFwdkx2zM1H-2uXJSi676WaSdM,6954
 lsst/pipe/base/script/utils.py,sha256=zNqpHG3kXA8OaNXnwYIo0Hu_LCie1qoBAARAME3WEjs,3739
 lsst/pipe/base/script/zip_from_graph.py,sha256=rbH_5Jk7Yc-YFD3X4mbDE4Vzddtu5y90Z77wha94mdM,3228
 lsst/pipe/base/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -86,13 +86,13 @@ lsst/pipe/base/tests/mocks/__init__.py,sha256=NrIJYDeYgR3HsOJXBEXi8EXDhhV7iw7dgw
 lsst/pipe/base/tests/mocks/_data_id_match.py,sha256=v33QZhZm-srXZAXZ8NbNKGN-_ql4AzaArBUk1lxhyss,7474
 lsst/pipe/base/tests/mocks/_pipeline_task.py,sha256=fqaJ-tB7K3jxlfCvCSnVd_GNrz-JhX7FB914h7nHLXc,29366
 lsst/pipe/base/tests/mocks/_storage_class.py,sha256=gC0czHURMk7PWj8N6dLxnY5V4HWX5i8ukb5SZbgWKy8,25257
-lsst_pipe_base-29.2025.2800.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
-lsst_pipe_base-29.2025.2800.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_pipe_base-29.2025.2800.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_pipe_base-29.2025.2800.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_pipe_base-29.2025.2800.dist-info/METADATA,sha256=rpezbW4ETyiwGgw7ttMrlVru-eZA_KG-Xax1jGXA5lE,2195
-lsst_pipe_base-29.2025.2800.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_pipe_base-29.2025.2800.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
-lsst_pipe_base-29.2025.2800.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_pipe_base-29.2025.2800.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_pipe_base-29.2025.2800.dist-info/RECORD,,
+lsst_pipe_base-29.2025.3000.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
+lsst_pipe_base-29.2025.3000.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_pipe_base-29.2025.3000.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_pipe_base-29.2025.3000.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_pipe_base-29.2025.3000.dist-info/METADATA,sha256=ErHeFechV8GyMQBh9Se8mLflvi9VDImXeOt5AAUhHt4,2195
+lsst_pipe_base-29.2025.3000.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_pipe_base-29.2025.3000.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
+lsst_pipe_base-29.2025.3000.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_pipe_base-29.2025.3000.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_pipe_base-29.2025.3000.dist-info/RECORD,,