lsst-pipe-base 29.2025.1900__py3-none-any.whl → 29.2025.2100__py3-none-any.whl

lsst/pipe/base/all_dimensions_quantum_graph_builder.py

@@ -48,7 +48,7 @@ from lsst.daf.butler import (
     DimensionRecordSet,
     MissingDatasetTypeError,
 )
-from lsst.utils.logging import LsstLogAdapter
+from lsst.utils.logging import LsstLogAdapter, PeriodicLogger
 from lsst.utils.timer import timeMethod
 
 from ._datasetQueryConstraints import DatasetQueryConstraintVariant
@@ -215,17 +215,26 @@ class AllDimensionsQuantumGraphBuilder(QuantumGraphBuilder):
             # Allow duplicates from common skypix overlaps to make some queries
             # run faster.
             query._allow_duplicate_overlaps = True
-            self.log.info("Iterating over query results to associate quanta with datasets.")
             # Iterate over query results, populating data IDs for datasets,
             # quanta, and edges.  We populate only the first level of the tree
             # in the first pass, so we can be done with the query results as
             # quickly as possible in case that holds a connection/cursor open.
             n_rows = 0
+            progress_logger: PeriodicLogger | None = None
             for common_data_id in query.data_ids(tree.all_dimensions):
+                if progress_logger is None:
+                    # There can be a long wait between submitting the query and
+                    # returning the first row, so we want to make sure we log
+                    # when we get it; note that PeriodicLogger is not going to
+                    # do that for us, as it waits for its interval _after_ the
+                    # first log is seen.
+                    self.log.info("Iterating over data ID query results.")
+                    progress_logger = PeriodicLogger(self.log)
                 for branch_dimensions, branch in tree.trunk_branches.items():
                     data_id = common_data_id.subset(branch_dimensions)
                     branch.data_ids.add(data_id)
                 n_rows += 1
+                progress_logger.log("Iterating over data ID query results: %d rows processed so far.", n_rows)
             if n_rows == 0:
                 # A single multiline log plays better with log aggregators like
                 # Loki.
@@ -242,7 +251,7 @@ class AllDimensionsQuantumGraphBuilder(QuantumGraphBuilder):
                     # If an exception was raised, write a partial.
                     self.log.error("\n".join(lines))
                 return
-        self.log.verbose("Processed %s initial data ID query rows.", n_rows)
+        self.log.verbose("Done iterating over query results: %d rows processed in total.", n_rows)
         # We now recursively populate the data IDs of the rest of the tree.
         tree.project_data_ids(self.log)
 

lsst/pipe/base/cli/cmd/commands.py

@@ -61,6 +61,7 @@ def register_instrument(*args: Any, **kwargs: Any) -> None:
 @click.option(
     "--dry-run", is_flag=True, default=False, help="Run the transfer but do not update the destination butler"
 )
+@dataset_type_option(help="Subset of dataset types to transfer from graph.")
 @options_file_option()
 def transfer_from_graph(**kwargs: Any) -> None:
     """Transfer datasets from a quantum graph to a destination butler.

lsst/pipe/base/script/transfer_from_graph.py

@@ -31,6 +31,8 @@ from lsst.daf.butler import Butler, CollectionType, QuantumBackedButler, Registr
 from lsst.daf.butler.registry import MissingCollectionError
 from lsst.pipe.base import QuantumGraph
 
+from .utils import filter_by_dataset_type_glob
+
 
 def transfer_from_graph(
     graph: str,
@@ -39,6 +41,7 @@ def transfer_from_graph(
     transfer_dimensions: bool,
     update_output_chain: bool,
     dry_run: bool,
+    dataset_type: tuple[str, ...],
 ) -> int:
     """Transfer output datasets from quantum graph to dest.
 
@@ -60,6 +63,9 @@ def transfer_from_graph(
         name as a the first collection in the chain.
     dry_run : `bool`
         Run the transfer without updating the destination butler.
+    dataset_type : `tuple` of `str`
+        Dataset type names. An empty tuple implies all dataset types.
+        Can include globs.
 
     Returns
     -------
@@ -84,11 +90,14 @@ def transfer_from_graph(
         dataset_types=dataset_types,
     )
 
+    # Filter the refs based on requested dataset types.
+    filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
+
     dest_butler = Butler.from_config(dest, writeable=True)
 
     transferred = dest_butler.transfer_from(
         qbb,
-        output_refs,
+        filtered_refs,
         transfer="auto",
         register_dataset_types=register_dataset_types,
         transfer_dimensions=transfer_dimensions,

lsst/pipe/base/script/utils.py

@@ -0,0 +1,74 @@
+# This file is part of pipe_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+__all__ = [
+    "filter_by_dataset_type_glob",
+]
+
+import re
+from collections.abc import Collection
+
+from lsst.daf.butler import DatasetRef
+from lsst.daf.butler.utils import globToRegex
+
+
+def _matches_dataset_type(dataset_type_name: str, regexes: list[str | re.Pattern]) -> bool:
+    for regex in regexes:
+        if isinstance(regex, str):
+            if dataset_type_name == regex:
+                return True
+        elif regex.search(dataset_type_name):
+            return True
+    return False
+
+
+def filter_by_dataset_type_glob(
+    refs: Collection[DatasetRef], dataset_types: tuple[str, ...]
+) -> Collection[DatasetRef]:
+    """Filter the refs based on requested dataset types.
+
+    Parameters
+    ----------
+    refs : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Datasets to be filtered.
+    dataset_types : `tuple` [ `str`, ...]
+        Dataset type names or globs to use for filtering. Empty tuple implies
+        no filtering.
+
+    Returns
+    -------
+    filtered : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Filter datasets.
+    """
+    regexes = globToRegex(dataset_types)
+    if regexes is ...:
+        # Nothing to do.
+        return refs
+
+    return {ref for ref in refs if _matches_dataset_type(ref.datasetType.name, regexes)}

lsst/pipe/base/script/zip_from_graph.py

@@ -28,13 +28,13 @@
 __all__ = ["zip_from_graph"]
 
 import logging
-import re
 
 from lsst.daf.butler import QuantumBackedButler
-from lsst.daf.butler.utils import globToRegex
 from lsst.pipe.base import QuantumGraph
 from lsst.resources import ResourcePath
 
+from .utils import filter_by_dataset_type_glob
+
 _LOG = logging.getLogger(__name__)
 
 
@@ -83,22 +83,8 @@ def zip_from_graph(
     )
 
     # Filter the refs based on requested dataset types.
-    regexes = globToRegex(dataset_type)
-    if regexes is ...:
-        filtered_refs = output_refs
-    else:
-
-        def _matches(dataset_type_name: str, regexes: list[str | re.Pattern]) -> bool:
-            for regex in regexes:
-                if isinstance(regex, str):
-                    if dataset_type_name == regex:
-                        return True
-                elif regex.search(dataset_type_name):
-                    return True
-            return False
-
-        filtered_refs = {ref for ref in output_refs if _matches(ref.datasetType.name, regexes)}
+    filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
 
-    _LOG.info("Retrieving artifacts for %d datasets and storing in Zip file.", len(output_refs))
+    _LOG.info("Retrieving artifacts for %d datasets and storing in Zip file.", len(filtered_refs))
     zip = qbb.retrieve_artifacts_zip(filtered_refs, dest)
     return zip

lsst/pipe/base/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.1900"
+__version__ = "29.2025.2100"

{lsst_pipe_base-29.2025.1900.dist-info → lsst_pipe_base-29.2025.2100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.1900
+Version: 29.2025.2100
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.1900.dist-info → lsst_pipe_base-29.2025.2100.dist-info}/RECORD

@@ -8,7 +8,7 @@ lsst/pipe/base/_observation_dimension_packer.py,sha256=78Jg2OVFOdXIK62TS2Y3X4095
 lsst/pipe/base/_quantumContext.py,sha256=gb60mTHbgOIEptYvJ64SaChvViXyeKJlG6kEHq4nYVw,19345
 lsst/pipe/base/_status.py,sha256=tvKm-z_haZGksOR4nQ-ePJgbLag-e3t4nQY47yLFP2M,15741
 lsst/pipe/base/_task_metadata.py,sha256=SfM4L07Pe5itBDCsNh-XjdQvlKLY5LBcG6lpdtpMxHA,24968
-lsst/pipe/base/all_dimensions_quantum_graph_builder.py,sha256=b_MVBsQcLERTOoitO0swJHdGK36P4-1CpeeD_ohU9Ug,45105
+lsst/pipe/base/all_dimensions_quantum_graph_builder.py,sha256=YCgDh-P1HxgS2zH2fP57uYCCZGI9DxBLIcXihnc-czQ,45758
 lsst/pipe/base/automatic_connection_constants.py,sha256=H5uuh1rYRpjndgPdb0dh1L_-OyLKdT6VWOZTAb__xCU,3298
 lsst/pipe/base/caching_limited_butler.py,sha256=u1uJYzCE7OxW8MW8Xv2LDB9-Nuj-Ao8lBJcDttKrc1Y,7700
 lsst/pipe/base/config.py,sha256=yNipVEc6awwhU_O9I01g20OnvQrs28dAwkXuI1hrlYE,11982
@@ -32,11 +32,11 @@ lsst/pipe/base/task.py,sha256=XHBd-7m1a4-6LgobBYA1DgY4H7EV-_RWKfxbhZbMmD4,15145
 lsst/pipe/base/taskFactory.py,sha256=4GhN2DozPM8suBYIvoKN4E6VP0I3mYZHBjCUO5JcCGk,2901
 lsst/pipe/base/testUtils.py,sha256=lSBKMhoKflbi8JkMNYfEqqHNl-rtFI8UYT3QneDYpLo,18477
 lsst/pipe/base/utils.py,sha256=JmEt3l0xrh9uayKrSXuQEq12aXOhDr2YXmbYduaxCko,1940
-lsst/pipe/base/version.py,sha256=duakYcQBT5t4KBaOX14WCtq_sSvpoXvNcxi3sUb8GvI,55
+lsst/pipe/base/version.py,sha256=czd5myijXlfzN1l1OFP7x6hn5ASHoE02wXhI32IqLec,55
 lsst/pipe/base/cli/__init__.py,sha256=861tXIAW7SqtqNUYkjbeEdfg8lDswXsjJQca0gVCFz4,54
 lsst/pipe/base/cli/_get_cli_subcommands.py,sha256=g_af64klRybBGKAg7fmBSZBdw2LYBAsFON_yQIMZON0,1289
 lsst/pipe/base/cli/cmd/__init__.py,sha256=BGicstnryQ48rYcNRh4fa6Vy63ZIlZ_pPAEa17jhkwY,1519
-lsst/pipe/base/cli/cmd/commands.py,sha256=noP1fU7mlNUwxzJGtYy5CF457iQMl3O4W2X0sYQNIYc,5535
+lsst/pipe/base/cli/cmd/commands.py,sha256=t91PZoIQwgstzONNbqpFJMWpaxeHesVP6EtgkMJYrcE,5612
 lsst/pipe/base/cli/opt/__init__.py,sha256=DN17wUbMwNIgbDBfF35sdyGfaMT81f3b_CA5Pp8POdk,1347
 lsst/pipe/base/cli/opt/arguments.py,sha256=9LhDnsM98_2zOqqm-eyv_nnZmAQcBG5OpHzeJYw_eTw,1484
 lsst/pipe/base/cli/opt/options.py,sha256=d5mC2WXZJiUbkdaIo_VUsvNxShD3GmredbEN5jQD64Q,1900
@@ -74,8 +74,9 @@ lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py,sha256=dp7PXl9C
 lsst/pipe/base/script/__init__.py,sha256=cLEXE7aq5UZ0juL_ScmRw0weFgp4tDgwEX_ts-NEYic,1522
 lsst/pipe/base/script/register_instrument.py,sha256=TRC2r2tSoYBNWNVQya01ELxAtGH8WVk9Ya-uNgCIL5U,2426
 lsst/pipe/base/script/retrieve_artifacts_for_quanta.py,sha256=pYI0wNl5PU8ImgzWfGEDrRz3PSKSg2szWLEIVKdm7Og,3939
-lsst/pipe/base/script/transfer_from_graph.py,sha256=OE_ruTiaDMDKe05RA_y7sbDawmQzdKVEtGj_cmHY2tM,5599
-lsst/pipe/base/script/zip_from_graph.py,sha256=E4_ziLRQCKEwQ0rWIXt9evJoi8FDghG4zDrDXg277Kg,3728
+lsst/pipe/base/script/transfer_from_graph.py,sha256=YkOFszLJuaqD2vl1izhSWemZzGCxDxUSdZNDoDyFEfo,5949
+lsst/pipe/base/script/utils.py,sha256=E00wb3usCuL3-3rJHZeqLJ_kwHV9V7Vaj6SdUe1VxU4,2631
+lsst/pipe/base/script/zip_from_graph.py,sha256=rbH_5Jk7Yc-YFD3X4mbDE4Vzddtu5y90Z77wha94mdM,3228
 lsst/pipe/base/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/pipe/base/tests/no_dimensions.py,sha256=58UpyRN8cLAMZtkOmjTm3dJZyRFRekotQ-7-OgEfiAI,4710
 lsst/pipe/base/tests/pipelineStepTester.py,sha256=KGxdB8gdVpSey2RUGURDIzIfPL-4qvQCsBpMrhG4Z2M,7208
@@ -85,13 +86,13 @@ lsst/pipe/base/tests/mocks/__init__.py,sha256=NrIJYDeYgR3HsOJXBEXi8EXDhhV7iw7dgw
 lsst/pipe/base/tests/mocks/_data_id_match.py,sha256=WU0-5cPsU4565UhlsvQwhZKP5RzPJPyZ8sGio1CKAPI,6813
 lsst/pipe/base/tests/mocks/_pipeline_task.py,sha256=fqaJ-tB7K3jxlfCvCSnVd_GNrz-JhX7FB914h7nHLXc,29366
 lsst/pipe/base/tests/mocks/_storage_class.py,sha256=JvUmyXQC5DwOkFoLHkJk6u0ldn1VgZVVjh9GjpXRGg0,25211
-lsst_pipe_base-29.2025.1900.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
-lsst_pipe_base-29.2025.1900.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_pipe_base-29.2025.1900.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_pipe_base-29.2025.1900.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_pipe_base-29.2025.1900.dist-info/METADATA,sha256=gGserBL9JVxpAxDR1NTfZ_AVFs-J4JnlG5l69qDgZNg,2195
-lsst_pipe_base-29.2025.1900.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-lsst_pipe_base-29.2025.1900.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
-lsst_pipe_base-29.2025.1900.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_pipe_base-29.2025.1900.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_pipe_base-29.2025.1900.dist-info/RECORD,,
+lsst_pipe_base-29.2025.2100.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
+lsst_pipe_base-29.2025.2100.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_pipe_base-29.2025.2100.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_pipe_base-29.2025.2100.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_pipe_base-29.2025.2100.dist-info/METADATA,sha256=4CN9L65r6qXfM8HHmng1_umH1vLoq6CW_kjzbJvlLx8,2195
+lsst_pipe_base-29.2025.2100.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+lsst_pipe_base-29.2025.2100.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
+lsst_pipe_base-29.2025.2100.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_pipe_base-29.2025.2100.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_pipe_base-29.2025.2100.dist-info/RECORD,,

{lsst_pipe_base-29.2025.1900.dist-info → lsst_pipe_base-29.2025.2100.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any