lsst-pipe-base 29.2025.2300__tar.gz → 29.2025.2500__tar.gz

{lsst_pipe_base-29.2025.2300/python/lsst_pipe_base.egg-info → lsst_pipe_base-29.2025.2500}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2300
+Version: 29.2025.2500
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2300 → lsst_pipe_base-29.2025.2500}/doc/lsst.pipe.base/CHANGES.rst

@@ -1,3 +1,42 @@
+lsst-pipe-base v29.1.0 (2025-06-13)
+===================================
+
+New Features
+------------
+
+- Introduced a graph-style visualization of pipeline processing that annotates nodes with their status using ``QuantumProvenanceGraph`` data. (`DM-48206 <https://rubinobs.atlassian.net/browse/DM-48206>`_)
+- Integrated ``mermaid-py`` for rendering pipeline visualizations as SVG/PNG images. (`DM-48482 <https://rubinobs.atlassian.net/browse/DM-48482>`_)
+- Added ``PipelineTask.select`` and and ``PipelineTask.select_tasks``, which use a small custom expression language to select subsets of a pipeline.
+
+  This functionality is intended to eventually be made available on the ``pipetask`` command-line and possibly in ``Pipeline`` YAML definitions, but this change just adds it to the ``PipelineGraph`` Python interface while design work continues on making it available more broadly. (`DM-48913 <https://rubinobs.atlassian.net/browse/DM-48913>`_)
+- Added a quantum graph generation hook that allows tasks to adjust all of their quanta (in limited ways) at once. (`DM-49300 <https://rubinobs.atlassian.net/browse/DM-49300>`_)
+- ``AllDimensionQuantumGraphBuilder`` now accepts one or more Astropy tables of data IDs as constraints. (`DM-49949 <https://rubinobs.atlassian.net/browse/DM-49949>`_)
+- Added support for moving outputs from one quantum to another in the ``adjust_all_quanta`` hook, as well as access to a ``Butler`` instance with implementations of that hook. (`DM-50539 <https://rubinobs.atlassian.net/browse/DM-50539>`_)
+
+
+Bug Fixes
+---------
+
+- Fixed unintentional exceptions from building an empty ``QuantumGraph``. (`DM-49763 <https://rubinobs.atlassian.net/browse/DM-49763>`_)
+- Fixed pipeline incorrect step-validation failures caused by further subsetting a pipeline with an empty step. (`DM-50107 <https://rubinobs.atlassian.net/browse/DM-50107>`_)
+- Fixed quantum graph builder logging to correctly identify when the initial butler query actually starts returning rows. (`DM-50969 <https://rubinobs.atlassian.net/browse/DM-50969>`_)
+
+
+Performance Enhancement
+-----------------------
+
+- Added parallelization and quantum-backed butler usage to ``QuantumProvenanceGraph`` to speed up metadata reads.
+
+  By default, ``QuantumProvenanceGraph`` now uses quantum-backed butler and only reads metadata for quanta for which at least one predicted output was not produced (``read_caveats='lazy'``). (`DM-49440 <https://rubinobs.atlassian.net/browse/DM-49440>`_)
+- When reading a full ``QuantumGraph`` file from a remote location (i.e. a non-POSIX URI), first copy the file into a temporary local location. (`DM-50041 <https://rubinobs.atlassian.net/browse/DM-50041>`_)
+
+
+Other Changes and Additions
+---------------------------
+
+- Added ability to select specific dataset types when transferring datasets from a graph. (`DM-50885 <https://rubinobs.atlassian.net/browse/DM-50885>`_)
+
+
 lsst-pipe-base v29.0.0 (2025-03-25)
 ===================================
 

{lsst_pipe_base-29.2025.2300 → lsst_pipe_base-29.2025.2500}/python/lsst/pipe/base/script/transfer_from_graph.py

@@ -27,11 +27,17 @@
 
 __all__ = ["transfer_from_graph"]
 
+import math
+
 from lsst.daf.butler import Butler, CollectionType, QuantumBackedButler, Registry
 from lsst.daf.butler.registry import MissingCollectionError
 from lsst.pipe.base import QuantumGraph
+from lsst.utils.iteration import chunk_iterable
+from lsst.utils.logging import getLogger
+
+from .utils import filter_by_dataset_type_glob, filter_by_existence
 
-from .utils import filter_by_dataset_type_glob
+_LOG = getLogger(__name__)
 
 
 def transfer_from_graph(
@@ -92,18 +98,31 @@ def transfer_from_graph(
 
     # Filter the refs based on requested dataset types.
     filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
+    _LOG.verbose("After filtering by dataset_type, number of datasets to transfer: %d", len(filtered_refs))
 
     dest_butler = Butler.from_config(dest, writeable=True)
 
-    transferred = dest_butler.transfer_from(
-        qbb,
-        filtered_refs,
-        transfer="auto",
-        register_dataset_types=register_dataset_types,
-        transfer_dimensions=transfer_dimensions,
-        dry_run=dry_run,
-    )
-    count = len(transferred)
+    # For faster restarts, filter out those the destination already knows.
+    filtered_refs = filter_by_existence(dest_butler, filtered_refs)
+
+    # Transfer in chunks
+    chunk_size = 50_000
+    n_chunks = math.ceil(len(filtered_refs) / chunk_size)
+    chunk_num = 0
+    count = 0
+    for chunk in chunk_iterable(filtered_refs, chunk_size=chunk_size):
+        chunk_num += 1
+        if n_chunks > 1:
+            _LOG.verbose("Transferring %d datasets in chunk %d/%d", len(chunk), chunk_num, n_chunks)
+        transferred = dest_butler.transfer_from(
+            qbb,
+            chunk,
+            transfer="auto",
+            register_dataset_types=register_dataset_types,
+            transfer_dimensions=transfer_dimensions,
+            dry_run=dry_run,
+        )
+        count += len(transferred)
 
     # If anything was transferred then update output chain definition if asked.
     if count > 0 and update_output_chain and (metadata := qgraph.metadata) is not None:

{lsst_pipe_base-29.2025.2300 → lsst_pipe_base-29.2025.2500}/python/lsst/pipe/base/script/utils.py

@@ -27,15 +27,17 @@
 
 from __future__ import annotations
 
-__all__ = [
-    "filter_by_dataset_type_glob",
-]
+__all__ = ["filter_by_dataset_type_glob", "filter_by_existence"]
 
 import re
 from collections.abc import Collection
 
-from lsst.daf.butler import DatasetRef
+from lsst.daf.butler import Butler, DatasetRef
 from lsst.daf.butler.utils import globToRegex
+from lsst.utils.logging import getLogger
+from lsst.utils.timer import time_this
+
+_LOG = getLogger(__name__)
 
 
 def _matches_dataset_type(dataset_type_name: str, regexes: list[str | re.Pattern]) -> bool:
@@ -72,3 +74,30 @@ def filter_by_dataset_type_glob(
         return refs
 
     return {ref for ref in refs if _matches_dataset_type(ref.datasetType.name, regexes)}
+
+
+def filter_by_existence(butler: Butler, refs: Collection[DatasetRef]) -> Collection[DatasetRef]:
+    """Filter out the refs that the butler already knows exist.
+
+    Parameters
+    ----------
+    butler : `lsst.daf.butler.Butler`
+        Butler in which to check existence of given datarefs.
+    refs : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Datasets to be filtered.
+
+    Returns
+    -------
+    filtered : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Filter datasets.
+    """
+    _LOG.verbose("Filtering out datasets already known to the target butler...")
+    with time_this(log=_LOG, msg="Completed checking existence"):
+        existence = butler._datastore.knows_these(refs)
+        filtered = [ref for ref in existence if not existence[ref]]
+    _LOG.verbose(
+        "After filtering out those already in the target butler, number of datasets to transfer: %d",
+        len(filtered),
+    )
+
+    return filtered

lsst_pipe_base-29.2025.2500/python/lsst/pipe/base/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2500"

{lsst_pipe_base-29.2025.2300 → lsst_pipe_base-29.2025.2500/python/lsst_pipe_base.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2300
+Version: 29.2025.2500
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2300 → lsst_pipe_base-29.2025.2500}/python/lsst_pipe_base.egg-info/SOURCES.txt

@@ -136,6 +136,7 @@ tests/test_pipeline_graph_expressions.py
 tests/test_quantumGraph.py
 tests/test_quantum_provenance_graph.py
 tests/test_quantum_success_caveats.py
+tests/test_script_utils.py
 tests/test_struct.py
 tests/test_task.py
 tests/test_taskmetadata.py

{lsst_pipe_base-29.2025.2300 → lsst_pipe_base-29.2025.2500}/tests/test_adjust_all_quanta.py

@@ -30,7 +30,6 @@ from __future__ import annotations
 __all__ = ()
 
 import operator
-import os
 import unittest
 from collections import defaultdict
 
@@ -45,7 +44,7 @@ from lsst.pipe.base import (
     QuantaAdjuster,
 )
 from lsst.pipe.base.all_dimensions_quantum_graph_builder import AllDimensionsQuantumGraphBuilder
-from lsst.utils import getPackageDir
+from lsst.resources import ResourcePath
 
 
 class GroupTestConnections(PipelineTaskConnections, dimensions=("detector",)):
@@ -111,13 +110,9 @@ class AdjustAllQuantaTestCase(unittest.TestCase):
 
     @staticmethod
     def make_butler() -> Butler:
-        try:
-            DATA_DIR = os.path.join(getPackageDir("daf_butler"), "tests", "data", "registry")
-        except LookupError:
-            raise unittest.SkipTest("Test data is not available with pip-provided daf_butler.") from None
-
+        DATA_ROOT = ResourcePath("resource://lsst.daf.butler/tests/registry_data", forceDirectory=True)
         return create_populated_sqlite_registry(
-            *[os.path.join(DATA_DIR, filename) for filename in ("base.yaml", "datasets.yaml")]
+            *[DATA_ROOT.join(filename) for filename in ("base.yaml", "datasets.yaml")]
         )
 
     def test_adjust_all_quanta(self) -> None:

lsst_pipe_base-29.2025.2500/tests/test_script_utils.py

@@ -0,0 +1,62 @@
+# This file is part of pipe_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Tests for scripts/utils.py functions."""
+
+import logging
+import unittest
+
+from lsst.pipe.base.script.utils import filter_by_existence
+
+
+class FilterByExistenceTestCase(unittest.TestCase):
+    """Test filter_by_existence function."""
+
+    def setUp(self):
+        # Currently function itself does not need real DatasetRefs.
+        # Only the butler calls requires DatasetRefs, but butler
+        # is being mocked in these tests. So using strings in place
+        # of DatasetRefs.
+        self.orig_refs = [f"{i:05}" for i in range(1, 11)]
+        self.mock_exists = {}
+        for i in range(1, 11):
+            self.mock_exists[f"{i:05}"] = True if i % 3 == 0 else False
+        self.filtered_refs = [f"{i:05}" for i in range(1, 11) if i % 3 != 0]
+
+    def test_success(self):
+        butler_mock = unittest.mock.Mock()
+        butler_mock._datastore.knows_these.return_value = self.mock_exists
+        with self.assertLogs(logging.getLogger("lsst.pipe.base"), level="VERBOSE") as cm:
+            filtered = filter_by_existence(butler_mock, self.orig_refs)
+            self.assertCountEqual(self.filtered_refs, filtered)
+        log_messages = " ".join(cm.output)
+        self.assertIn("Filtering out datasets already known to the target butler", log_messages)
+        self.assertIn("number of datasets to transfer", log_messages)
+
+
+if __name__ == "__main__":
+    unittest.main()

lsst_pipe_base-29.2025.2300/python/lsst/pipe/base/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2300"