lsst-pipe-base 29.2025.2000__tar.gz → 29.2025.2100__tar.gz

{lsst_pipe_base-29.2025.2000/python/lsst_pipe_base.egg-info → lsst_pipe_base-29.2025.2100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2000
+Version: 29.2025.2100
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2100}/python/lsst/pipe/base/all_dimensions_quantum_graph_builder.py

@@ -48,7 +48,7 @@ from lsst.daf.butler import (
     DimensionRecordSet,
     MissingDatasetTypeError,
 )
-from lsst.utils.logging import LsstLogAdapter
+from lsst.utils.logging import LsstLogAdapter, PeriodicLogger
 from lsst.utils.timer import timeMethod
 
 from ._datasetQueryConstraints import DatasetQueryConstraintVariant
@@ -215,17 +215,26 @@ class AllDimensionsQuantumGraphBuilder(QuantumGraphBuilder):
             # Allow duplicates from common skypix overlaps to make some queries
             # run faster.
             query._allow_duplicate_overlaps = True
-            self.log.info("Iterating over query results to associate quanta with datasets.")
             # Iterate over query results, populating data IDs for datasets,
             # quanta, and edges.  We populate only the first level of the tree
             # in the first pass, so we can be done with the query results as
             # quickly as possible in case that holds a connection/cursor open.
             n_rows = 0
+            progress_logger: PeriodicLogger | None = None
             for common_data_id in query.data_ids(tree.all_dimensions):
+                if progress_logger is None:
+                    # There can be a long wait between submitting the query and
+                    # returning the first row, so we want to make sure we log
+                    # when we get it; note that PeriodicLogger is not going to
+                    # do that for us, as it waits for its interval _after_ the
+                    # first log is seen.
+                    self.log.info("Iterating over data ID query results.")
+                    progress_logger = PeriodicLogger(self.log)
                 for branch_dimensions, branch in tree.trunk_branches.items():
                     data_id = common_data_id.subset(branch_dimensions)
                     branch.data_ids.add(data_id)
                 n_rows += 1
+                progress_logger.log("Iterating over data ID query results: %d rows processed so far.", n_rows)
             if n_rows == 0:
                 # A single multiline log plays better with log aggregators like
                 # Loki.
@@ -242,7 +251,7 @@ class AllDimensionsQuantumGraphBuilder(QuantumGraphBuilder):
                     # If an exception was raised, write a partial.
                     self.log.error("\n".join(lines))
                 return
-        self.log.verbose("Processed %s initial data ID query rows.", n_rows)
+        self.log.verbose("Done iterating over query results: %d rows processed in total.", n_rows)
         # We now recursively populate the data IDs of the rest of the tree.
         tree.project_data_ids(self.log)
 

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2100}/python/lsst/pipe/base/cli/cmd/commands.py

@@ -61,6 +61,7 @@ def register_instrument(*args: Any, **kwargs: Any) -> None:
 @click.option(
     "--dry-run", is_flag=True, default=False, help="Run the transfer but do not update the destination butler"
 )
+@dataset_type_option(help="Subset of dataset types to transfer from graph.")
 @options_file_option()
 def transfer_from_graph(**kwargs: Any) -> None:
     """Transfer datasets from a quantum graph to a destination butler.

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2100}/python/lsst/pipe/base/script/transfer_from_graph.py

@@ -31,6 +31,8 @@ from lsst.daf.butler import Butler, CollectionType, QuantumBackedButler, Registr
 from lsst.daf.butler.registry import MissingCollectionError
 from lsst.pipe.base import QuantumGraph
 
+from .utils import filter_by_dataset_type_glob
+
 
 def transfer_from_graph(
     graph: str,
@@ -39,6 +41,7 @@ def transfer_from_graph(
     transfer_dimensions: bool,
     update_output_chain: bool,
     dry_run: bool,
+    dataset_type: tuple[str, ...],
 ) -> int:
     """Transfer output datasets from quantum graph to dest.
 
@@ -60,6 +63,9 @@ def transfer_from_graph(
         name as a the first collection in the chain.
     dry_run : `bool`
         Run the transfer without updating the destination butler.
+    dataset_type : `tuple` of `str`
+        Dataset type names. An empty tuple implies all dataset types.
+        Can include globs.
 
     Returns
     -------
@@ -84,11 +90,14 @@ def transfer_from_graph(
         dataset_types=dataset_types,
     )
 
+    # Filter the refs based on requested dataset types.
+    filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
+
     dest_butler = Butler.from_config(dest, writeable=True)
 
     transferred = dest_butler.transfer_from(
         qbb,
-        output_refs,
+        filtered_refs,
         transfer="auto",
         register_dataset_types=register_dataset_types,
         transfer_dimensions=transfer_dimensions,

lsst_pipe_base-29.2025.2100/python/lsst/pipe/base/script/utils.py

@@ -0,0 +1,74 @@
+# This file is part of pipe_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+__all__ = [
+    "filter_by_dataset_type_glob",
+]
+
+import re
+from collections.abc import Collection
+
+from lsst.daf.butler import DatasetRef
+from lsst.daf.butler.utils import globToRegex
+
+
+def _matches_dataset_type(dataset_type_name: str, regexes: list[str | re.Pattern]) -> bool:
+    for regex in regexes:
+        if isinstance(regex, str):
+            if dataset_type_name == regex:
+                return True
+        elif regex.search(dataset_type_name):
+            return True
+    return False
+
+
+def filter_by_dataset_type_glob(
+    refs: Collection[DatasetRef], dataset_types: tuple[str, ...]
+) -> Collection[DatasetRef]:
+    """Filter the refs based on requested dataset types.
+
+    Parameters
+    ----------
+    refs : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Datasets to be filtered.
+    dataset_types : `tuple` [ `str`, ...]
+        Dataset type names or globs to use for filtering. Empty tuple implies
+        no filtering.
+
+    Returns
+    -------
+    filtered : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Filter datasets.
+    """
+    regexes = globToRegex(dataset_types)
+    if regexes is ...:
+        # Nothing to do.
+        return refs
+
+    return {ref for ref in refs if _matches_dataset_type(ref.datasetType.name, regexes)}

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2100}/python/lsst/pipe/base/script/zip_from_graph.py

@@ -28,13 +28,13 @@
 __all__ = ["zip_from_graph"]
 
 import logging
-import re
 
 from lsst.daf.butler import QuantumBackedButler
-from lsst.daf.butler.utils import globToRegex
 from lsst.pipe.base import QuantumGraph
 from lsst.resources import ResourcePath
 
+from .utils import filter_by_dataset_type_glob
+
 _LOG = logging.getLogger(__name__)
 
 
@@ -83,22 +83,8 @@ def zip_from_graph(
     )
 
     # Filter the refs based on requested dataset types.
-    regexes = globToRegex(dataset_type)
-    if regexes is ...:
-        filtered_refs = output_refs
-    else:
-
-        def _matches(dataset_type_name: str, regexes: list[str | re.Pattern]) -> bool:
-            for regex in regexes:
-                if isinstance(regex, str):
-                    if dataset_type_name == regex:
-                        return True
-                elif regex.search(dataset_type_name):
-                    return True
-            return False
-
-        filtered_refs = {ref for ref in output_refs if _matches(ref.datasetType.name, regexes)}
+    filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
 
-    _LOG.info("Retrieving artifacts for %d datasets and storing in Zip file.", len(output_refs))
+    _LOG.info("Retrieving artifacts for %d datasets and storing in Zip file.", len(filtered_refs))
     zip = qbb.retrieve_artifacts_zip(filtered_refs, dest)
     return zip

lsst_pipe_base-29.2025.2100/python/lsst/pipe/base/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2100"

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2100/python/lsst_pipe_base.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2000
+Version: 29.2025.2100
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2100}/python/lsst_pipe_base.egg-info/SOURCES.txt

@@ -93,6 +93,7 @@ python/lsst/pipe/base/script/__init__.py
 python/lsst/pipe/base/script/register_instrument.py
 python/lsst/pipe/base/script/retrieve_artifacts_for_quanta.py
 python/lsst/pipe/base/script/transfer_from_graph.py
+python/lsst/pipe/base/script/utils.py
 python/lsst/pipe/base/script/zip_from_graph.py
 python/lsst/pipe/base/tests/__init__.py
 python/lsst/pipe/base/tests/no_dimensions.py

lsst_pipe_base-29.2025.2000/python/lsst/pipe/base/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2000"