lsst-pipe-base 29.2025.2300__tar.gz → 29.2025.2400__tar.gz

{lsst_pipe_base-29.2025.2300/python/lsst_pipe_base.egg-info → lsst_pipe_base-29.2025.2400}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2300
+Version: 29.2025.2400
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2300 → lsst_pipe_base-29.2025.2400}/python/lsst/pipe/base/script/transfer_from_graph.py

@@ -27,11 +27,17 @@
 
 __all__ = ["transfer_from_graph"]
 
+import math
+
 from lsst.daf.butler import Butler, CollectionType, QuantumBackedButler, Registry
 from lsst.daf.butler.registry import MissingCollectionError
 from lsst.pipe.base import QuantumGraph
+from lsst.utils.iteration import chunk_iterable
+from lsst.utils.logging import getLogger
+
+from .utils import filter_by_dataset_type_glob, filter_by_existence
 
-from .utils import filter_by_dataset_type_glob
+_LOG = getLogger(__name__)
 
 
 def transfer_from_graph(
@@ -92,18 +98,31 @@ def transfer_from_graph(
 
     # Filter the refs based on requested dataset types.
     filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
+    _LOG.verbose("After filtering by dataset_type, number of datasets to transfer: %d", len(filtered_refs))
 
     dest_butler = Butler.from_config(dest, writeable=True)
 
-    transferred = dest_butler.transfer_from(
-        qbb,
-        filtered_refs,
-        transfer="auto",
-        register_dataset_types=register_dataset_types,
-        transfer_dimensions=transfer_dimensions,
-        dry_run=dry_run,
-    )
-    count = len(transferred)
+    # For faster restarts, filter out those the destination already knows.
+    filtered_refs = filter_by_existence(dest_butler, filtered_refs)
+
+    # Transfer in chunks
+    chunk_size = 50_000
+    n_chunks = math.ceil(len(filtered_refs) / chunk_size)
+    chunk_num = 0
+    count = 0
+    for chunk in chunk_iterable(filtered_refs, chunk_size=chunk_size):
+        chunk_num += 1
+        if n_chunks > 1:
+            _LOG.verbose("Transferring %d datasets in chunk %d/%d", len(chunk), chunk_num, n_chunks)
+        transferred = dest_butler.transfer_from(
+            qbb,
+            chunk,
+            transfer="auto",
+            register_dataset_types=register_dataset_types,
+            transfer_dimensions=transfer_dimensions,
+            dry_run=dry_run,
+        )
+        count += len(transferred)
 
     # If anything was transferred then update output chain definition if asked.
     if count > 0 and update_output_chain and (metadata := qgraph.metadata) is not None:

{lsst_pipe_base-29.2025.2300 → lsst_pipe_base-29.2025.2400}/python/lsst/pipe/base/script/utils.py

@@ -27,15 +27,17 @@
 
 from __future__ import annotations
 
-__all__ = [
-    "filter_by_dataset_type_glob",
-]
+__all__ = ["filter_by_dataset_type_glob", "filter_by_existence"]
 
 import re
 from collections.abc import Collection
 
-from lsst.daf.butler import DatasetRef
+from lsst.daf.butler import Butler, DatasetRef
 from lsst.daf.butler.utils import globToRegex
+from lsst.utils.logging import getLogger
+from lsst.utils.timer import time_this
+
+_LOG = getLogger(__name__)
 
 
 def _matches_dataset_type(dataset_type_name: str, regexes: list[str | re.Pattern]) -> bool:
@@ -72,3 +74,30 @@ def filter_by_dataset_type_glob(
         return refs
 
     return {ref for ref in refs if _matches_dataset_type(ref.datasetType.name, regexes)}
+
+
+def filter_by_existence(butler: Butler, refs: Collection[DatasetRef]) -> Collection[DatasetRef]:
+    """Filter out the refs that the butler already knows exist.
+
+    Parameters
+    ----------
+    butler : `lsst.daf.butler.Butler`
+        Butler in which to check existence of given datarefs.
+    refs : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Datasets to be filtered.
+
+    Returns
+    -------
+    filtered : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Filter datasets.
+    """
+    _LOG.verbose("Filtering out datasets already known to the target butler...")
+    with time_this(log=_LOG, msg="Completed checking existence"):
+        existence = butler._datastore.knows_these(refs)
+        filtered = [ref for ref in existence if not existence[ref]]
+    _LOG.verbose(
+        "After filtering out those already in the target butler, number of datasets to transfer: %d",
+        len(filtered),
+    )
+
+    return filtered

lsst_pipe_base-29.2025.2400/python/lsst/pipe/base/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2400"

{lsst_pipe_base-29.2025.2300 → lsst_pipe_base-29.2025.2400/python/lsst_pipe_base.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2300
+Version: 29.2025.2400
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2300 → lsst_pipe_base-29.2025.2400}/python/lsst_pipe_base.egg-info/SOURCES.txt

@@ -136,6 +136,7 @@ tests/test_pipeline_graph_expressions.py
 tests/test_quantumGraph.py
 tests/test_quantum_provenance_graph.py
 tests/test_quantum_success_caveats.py
+tests/test_script_utils.py
 tests/test_struct.py
 tests/test_task.py
 tests/test_taskmetadata.py

lsst_pipe_base-29.2025.2400/tests/test_script_utils.py

@@ -0,0 +1,62 @@
+# This file is part of pipe_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Tests for scripts/utils.py functions."""
+
+import logging
+import unittest
+
+from lsst.pipe.base.script.utils import filter_by_existence
+
+
+class FilterByExistenceTestCase(unittest.TestCase):
+    """Test filter_by_existence function."""
+
+    def setUp(self):
+        # Currently function itself does not need real DatasetRefs.
+        # Only the butler calls requires DatasetRefs, but butler
+        # is being mocked in these tests. So using strings in place
+        # of DatasetRefs.
+        self.orig_refs = [f"{i:05}" for i in range(1, 11)]
+        self.mock_exists = {}
+        for i in range(1, 11):
+            self.mock_exists[f"{i:05}"] = True if i % 3 == 0 else False
+        self.filtered_refs = [f"{i:05}" for i in range(1, 11) if i % 3 != 0]
+
+    def test_success(self):
+        butler_mock = unittest.mock.Mock()
+        butler_mock._datastore.knows_these.return_value = self.mock_exists
+        with self.assertLogs(logging.getLogger("lsst.pipe.base"), level="VERBOSE") as cm:
+            filtered = filter_by_existence(butler_mock, self.orig_refs)
+            self.assertCountEqual(self.filtered_refs, filtered)
+        log_messages = " ".join(cm.output)
+        self.assertIn("Filtering out datasets already known to the target butler", log_messages)
+        self.assertIn("number of datasets to transfer", log_messages)
+
+
+if __name__ == "__main__":
+    unittest.main()

lsst_pipe_base-29.2025.2300/python/lsst/pipe/base/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2300"