lsst-pipe-base 29.2025.2300__py3-none-any.whl → 29.2025.2400__py3-none-any.whl

lsst/pipe/base/script/transfer_from_graph.py

@@ -27,11 +27,17 @@
 
 __all__ = ["transfer_from_graph"]
 
+import math
+
 from lsst.daf.butler import Butler, CollectionType, QuantumBackedButler, Registry
 from lsst.daf.butler.registry import MissingCollectionError
 from lsst.pipe.base import QuantumGraph
+from lsst.utils.iteration import chunk_iterable
+from lsst.utils.logging import getLogger
+
+from .utils import filter_by_dataset_type_glob, filter_by_existence
 
-from .utils import filter_by_dataset_type_glob
+_LOG = getLogger(__name__)
 
 
 def transfer_from_graph(
@@ -92,18 +98,31 @@ def transfer_from_graph(
 
     # Filter the refs based on requested dataset types.
     filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
+    _LOG.verbose("After filtering by dataset_type, number of datasets to transfer: %d", len(filtered_refs))
 
     dest_butler = Butler.from_config(dest, writeable=True)
 
-    transferred = dest_butler.transfer_from(
-        qbb,
-        filtered_refs,
-        transfer="auto",
-        register_dataset_types=register_dataset_types,
-        transfer_dimensions=transfer_dimensions,
-        dry_run=dry_run,
-    )
-    count = len(transferred)
+    # For faster restarts, filter out those the destination already knows.
+    filtered_refs = filter_by_existence(dest_butler, filtered_refs)
+
+    # Transfer in chunks
+    chunk_size = 50_000
+    n_chunks = math.ceil(len(filtered_refs) / chunk_size)
+    chunk_num = 0
+    count = 0
+    for chunk in chunk_iterable(filtered_refs, chunk_size=chunk_size):
+        chunk_num += 1
+        if n_chunks > 1:
+            _LOG.verbose("Transferring %d datasets in chunk %d/%d", len(chunk), chunk_num, n_chunks)
+        transferred = dest_butler.transfer_from(
+            qbb,
+            chunk,
+            transfer="auto",
+            register_dataset_types=register_dataset_types,
+            transfer_dimensions=transfer_dimensions,
+            dry_run=dry_run,
+        )
+        count += len(transferred)
 
     # If anything was transferred then update output chain definition if asked.
     if count > 0 and update_output_chain and (metadata := qgraph.metadata) is not None:

lsst/pipe/base/script/utils.py

@@ -27,15 +27,17 @@
 
 from __future__ import annotations
 
-__all__ = [
-    "filter_by_dataset_type_glob",
-]
+__all__ = ["filter_by_dataset_type_glob", "filter_by_existence"]
 
 import re
 from collections.abc import Collection
 
-from lsst.daf.butler import DatasetRef
+from lsst.daf.butler import Butler, DatasetRef
 from lsst.daf.butler.utils import globToRegex
+from lsst.utils.logging import getLogger
+from lsst.utils.timer import time_this
+
+_LOG = getLogger(__name__)
 
 
 def _matches_dataset_type(dataset_type_name: str, regexes: list[str | re.Pattern]) -> bool:
@@ -72,3 +74,30 @@ def filter_by_dataset_type_glob(
         return refs
 
     return {ref for ref in refs if _matches_dataset_type(ref.datasetType.name, regexes)}
+
+
+def filter_by_existence(butler: Butler, refs: Collection[DatasetRef]) -> Collection[DatasetRef]:
+    """Filter out the refs that the butler already knows exist.
+
+    Parameters
+    ----------
+    butler : `lsst.daf.butler.Butler`
+        Butler in which to check existence of given datarefs.
+    refs : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Datasets to be filtered.
+
+    Returns
+    -------
+    filtered : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Filter datasets.
+    """
+    _LOG.verbose("Filtering out datasets already known to the target butler...")
+    with time_this(log=_LOG, msg="Completed checking existence"):
+        existence = butler._datastore.knows_these(refs)
+        filtered = [ref for ref in existence if not existence[ref]]
+    _LOG.verbose(
+        "After filtering out those already in the target butler, number of datasets to transfer: %d",
+        len(filtered),
+    )
+
+    return filtered

lsst/pipe/base/version.py

@@ -1,2 +1,2 @@
 __all__ = ["__version__"]
-__version__ = "29.2025.2300"
+__version__ = "29.2025.2400"

{lsst_pipe_base-29.2025.2300.dist-info → lsst_pipe_base-29.2025.2400.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2300
+Version: 29.2025.2400
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2300.dist-info → lsst_pipe_base-29.2025.2400.dist-info}/RECORD

@@ -32,7 +32,7 @@ lsst/pipe/base/task.py,sha256=XHBd-7m1a4-6LgobBYA1DgY4H7EV-_RWKfxbhZbMmD4,15145
 lsst/pipe/base/taskFactory.py,sha256=4GhN2DozPM8suBYIvoKN4E6VP0I3mYZHBjCUO5JcCGk,2901
 lsst/pipe/base/testUtils.py,sha256=lSBKMhoKflbi8JkMNYfEqqHNl-rtFI8UYT3QneDYpLo,18477
 lsst/pipe/base/utils.py,sha256=JmEt3l0xrh9uayKrSXuQEq12aXOhDr2YXmbYduaxCko,1940
-lsst/pipe/base/version.py,sha256=GnyaHNlpLZJmdCekPuL3VEJCw758_rgDNpEpGPT5mRw,55
+lsst/pipe/base/version.py,sha256=AyivLkwA4FcYO6eiFsfTA_EuqMhydgvzBupNZlQZL-E,55
 lsst/pipe/base/cli/__init__.py,sha256=861tXIAW7SqtqNUYkjbeEdfg8lDswXsjJQca0gVCFz4,54
 lsst/pipe/base/cli/_get_cli_subcommands.py,sha256=g_af64klRybBGKAg7fmBSZBdw2LYBAsFON_yQIMZON0,1289
 lsst/pipe/base/cli/cmd/__init__.py,sha256=BGicstnryQ48rYcNRh4fa6Vy63ZIlZ_pPAEa17jhkwY,1519
@@ -74,8 +74,8 @@ lsst/pipe/base/pipeline_graph/visualization/_status_annotator.py,sha256=dp7PXl9C
 lsst/pipe/base/script/__init__.py,sha256=cLEXE7aq5UZ0juL_ScmRw0weFgp4tDgwEX_ts-NEYic,1522
 lsst/pipe/base/script/register_instrument.py,sha256=TRC2r2tSoYBNWNVQya01ELxAtGH8WVk9Ya-uNgCIL5U,2426
 lsst/pipe/base/script/retrieve_artifacts_for_quanta.py,sha256=pYI0wNl5PU8ImgzWfGEDrRz3PSKSg2szWLEIVKdm7Og,3939
-lsst/pipe/base/script/transfer_from_graph.py,sha256=YkOFszLJuaqD2vl1izhSWemZzGCxDxUSdZNDoDyFEfo,5949
-lsst/pipe/base/script/utils.py,sha256=E00wb3usCuL3-3rJHZeqLJ_kwHV9V7Vaj6SdUe1VxU4,2631
+lsst/pipe/base/script/transfer_from_graph.py,sha256=WMpmzaN8hdW4FoZPNL6LRiPW9gvxf9mw20Z1i-KCkWI,6740
+lsst/pipe/base/script/utils.py,sha256=zNqpHG3kXA8OaNXnwYIo0Hu_LCie1qoBAARAME3WEjs,3739
 lsst/pipe/base/script/zip_from_graph.py,sha256=rbH_5Jk7Yc-YFD3X4mbDE4Vzddtu5y90Z77wha94mdM,3228
 lsst/pipe/base/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lsst/pipe/base/tests/no_dimensions.py,sha256=58UpyRN8cLAMZtkOmjTm3dJZyRFRekotQ-7-OgEfiAI,4710
@@ -86,13 +86,13 @@ lsst/pipe/base/tests/mocks/__init__.py,sha256=NrIJYDeYgR3HsOJXBEXi8EXDhhV7iw7dgw
 lsst/pipe/base/tests/mocks/_data_id_match.py,sha256=WU0-5cPsU4565UhlsvQwhZKP5RzPJPyZ8sGio1CKAPI,6813
 lsst/pipe/base/tests/mocks/_pipeline_task.py,sha256=fqaJ-tB7K3jxlfCvCSnVd_GNrz-JhX7FB914h7nHLXc,29366
 lsst/pipe/base/tests/mocks/_storage_class.py,sha256=gC0czHURMk7PWj8N6dLxnY5V4HWX5i8ukb5SZbgWKy8,25257
-lsst_pipe_base-29.2025.2300.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
-lsst_pipe_base-29.2025.2300.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
-lsst_pipe_base-29.2025.2300.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
-lsst_pipe_base-29.2025.2300.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-lsst_pipe_base-29.2025.2300.dist-info/METADATA,sha256=qN3C-TyotJss0dGGTai0UtAR3E-x-T3BKgV7BOAihZI,2195
-lsst_pipe_base-29.2025.2300.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lsst_pipe_base-29.2025.2300.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
-lsst_pipe_base-29.2025.2300.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
-lsst_pipe_base-29.2025.2300.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-lsst_pipe_base-29.2025.2300.dist-info/RECORD,,
+lsst_pipe_base-29.2025.2400.dist-info/licenses/COPYRIGHT,sha256=kB3Z9_f6a6uFLGpEmNJT_n186CE65H6wHu4F6BNt_zA,368
+lsst_pipe_base-29.2025.2400.dist-info/licenses/LICENSE,sha256=pRExkS03v0MQW-neNfIcaSL6aiAnoLxYgtZoFzQ6zkM,232
+lsst_pipe_base-29.2025.2400.dist-info/licenses/bsd_license.txt,sha256=7MIcv8QRX9guUtqPSBDMPz2SnZ5swI-xZMqm_VDSfxY,1606
+lsst_pipe_base-29.2025.2400.dist-info/licenses/gpl-v3.0.txt,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+lsst_pipe_base-29.2025.2400.dist-info/METADATA,sha256=J-bk-ErrNnOs-UY8uVxZhvQVUmZTJ2X4CeBBITdsgIc,2195
+lsst_pipe_base-29.2025.2400.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lsst_pipe_base-29.2025.2400.dist-info/entry_points.txt,sha256=bnmUhJBsChxMdqST9VmFBYYKxLQoToOfqW1wjW7khjk,64
+lsst_pipe_base-29.2025.2400.dist-info/top_level.txt,sha256=eUWiOuVVm9wwTrnAgiJT6tp6HQHXxIhj2QSZ7NYZH80,5
+lsst_pipe_base-29.2025.2400.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+lsst_pipe_base-29.2025.2400.dist-info/RECORD,,