lsst-pipe-base 29.2025.2100__tar.gz → 29.2025.2400__tar.gz

{lsst_pipe_base-29.2025.2100/python/lsst_pipe_base.egg-info → lsst_pipe_base-29.2025.2400}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2100
+Version: 29.2025.2400
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2100 → lsst_pipe_base-29.2025.2400}/python/lsst/pipe/base/_task_metadata.py

@@ -677,7 +677,7 @@ class TaskMetadata(BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc]
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc, override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 

{lsst_pipe_base-29.2025.2100 → lsst_pipe_base-29.2025.2400}/python/lsst/pipe/base/graph/graphSummary.py

@@ -66,7 +66,7 @@ class QgraphTaskSummary(pydantic.BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 
@@ -121,7 +121,7 @@ class QgraphSummary(pydantic.BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 

{lsst_pipe_base-29.2025.2100 → lsst_pipe_base-29.2025.2400}/python/lsst/pipe/base/pipeline_graph/_dataset_types.py

@@ -60,12 +60,13 @@ class DatasetTypeNode:
     """
 
     dataset_type: DatasetType
-    """Common definition of this dataset type for the graph.
+    """Common definition of this dataset type for the graph
+    (`~lsst.daf.butler.DatasetType`).
     """
 
     is_initial_query_constraint: bool
     """Whether this dataset should be included as a constraint in the initial
-    query for data IDs in QuantumGraph generation.
+    query for data IDs in QuantumGraph generation (`bool`).
 
     This is only `True` for dataset types that are overall regular inputs, and
     only if none of those input connections had ``deferQueryConstraint=True``.
@@ -73,14 +74,16 @@ class DatasetTypeNode:
 
     is_prerequisite: bool
     """Whether this dataset type is a prerequisite input that must exist in
-    the Registry before graph creation.
+    the Registry before graph creation (`bool`).
     """
 
     producing_edge: WriteEdge | None
-    """The edge to the task that produces this dataset type."""
+    """The edge to the task that produces this dataset type
+    (`lsst.pipe.base.pipeline_graph.WriteEdge` or `None`)."""
 
     consuming_edges: Collection[ReadEdge]
-    """The edges to tasks that consume this dataset type."""
+    """The edges to tasks that consume this dataset type
+    (collection [`lsst.pipe.base.pipeline_graph.ReadEdge`])."""
 
     @classmethod
     def _from_edges(
@@ -184,7 +187,7 @@ class DatasetTypeNode:
 
     @property
     def name(self) -> str:
-        """Name of the dataset type.
+        """Name of the dataset type (`str`).
 
         This is always the parent dataset type, never that of a component.
         """
@@ -193,29 +196,33 @@ class DatasetTypeNode:
     @property
     def key(self) -> NodeKey:
         """Key that identifies this dataset type in internal and exported
-        networkx graphs.
+        networkx graphs (`~lsst.pipe.base.pipeline_graph.NodeKey`).
         """
         return NodeKey(NodeType.DATASET_TYPE, self.dataset_type.name)
 
     @property
     def dimensions(self) -> DimensionGroup:
-        """Dimensions of the dataset type."""
+        """Dimensions of the dataset type
+        (`~lsst.daf.butler.DimensionGroup`).
+        """
         return self.dataset_type.dimensions
 
     @property
     def storage_class_name(self) -> str:
-        """String name of the storage class for this dataset type."""
+        """String name of the storage class for this dataset type (`str`)."""
         return self.dataset_type.storageClass_name
 
     @property
     def storage_class(self) -> StorageClass:
-        """Storage class for this dataset type."""
+        """Storage class for this dataset type
+        (`~lsst.daf.butler.StorageClass`).
+        """
         return self.dataset_type.storageClass
 
     @property
     def is_calibration(self) -> bool:
         """Whether this dataset type can be included in
-        `~lsst.daf.butler.CollectionType.CALIBRATION` collections.
+        `~lsst.daf.butler.CollectionType.CALIBRATION` collections (`bool`).
         """
         return self.dataset_type.isCalibration()
 

{lsst_pipe_base-29.2025.2100 → lsst_pipe_base-29.2025.2400}/python/lsst/pipe/base/pipeline_graph/_edges.py

@@ -90,16 +90,16 @@ class Edge(ABC):
     """
 
     task_key: NodeKey
-    """Task part of the key for this edge in networkx graphs."""
+    """Task part of the key for this edge in networkx graphs (`NodeKey`)."""
 
     dataset_type_key: NodeKey
-    """Task part of the key for this edge in networkx graphs."""
+    """Task part of the key for this edge in networkx graphs (`NodeKey`)."""
 
     connection_name: str
-    """Name used by the task to refer to this dataset type."""
+    """Name used by the task to refer to this dataset type (`str`)."""
 
     storage_class_name: str
-    """Storage class expected by this task.
+    """Storage class expected by this task (`str`).
 
     If `ReadEdge.component` is not `None`, this is the component storage class,
     not the parent storage class.
@@ -111,7 +111,7 @@ class Edge(ABC):
     """
 
     raw_dimensions: frozenset[str]
-    """Raw dimensions in the task declaration.
+    """Raw dimensions in the task declaration (`frozenset` [`str`]).
 
     This can only be used safely for partial comparisons: two edges with the
     same ``raw_dimensions`` (and the same parent dataset type name) always have
@@ -128,12 +128,12 @@ class Edge(ABC):
 
     @property
     def task_label(self) -> str:
-        """Label of the task."""
+        """Label of the task (`str`)."""
         return str(self.task_key)
 
     @property
     def parent_dataset_type_name(self) -> str:
-        """Name of the parent dataset type.
+        """Name of the parent dataset type (`str`).
 
         All dataset type nodes in a pipeline graph are for parent dataset
         types; components are represented by additional `ReadEdge` state.
@@ -154,7 +154,9 @@ class Edge(ABC):
     @property
     def key(self) -> tuple[NodeKey, NodeKey, str]:
         """Ordered tuple of node keys and connection name that uniquely
-        identifies this edge in a pipeline graph.
+        identifies this edge in a pipeline graph (`NodeKey`, `NodeKey`, `str`).
+
+        The nodes are ordered in the same sense as for `nodes`.
         """
         return self.nodes + (self.connection_name,)
 
@@ -163,7 +165,7 @@ class Edge(ABC):
 
     @property
     def dataset_type_name(self) -> str:
-        """Dataset type name seen by the task.
+        """Dataset type name seen by the task (`str`).
 
         This defaults to the parent dataset type name, which is appropriate
         for all writes and most reads.
@@ -355,7 +357,7 @@ class ReadEdge(Edge):
 
     component: str | None
     """Component to add to `parent_dataset_type_name` to form the dataset type
-    name seen by this task.
+    name seen by this task (`str` or `None`).
     """
 
     is_prerequisite: bool
@@ -378,7 +380,7 @@ class ReadEdge(Edge):
 
     @property
     def dataset_type_name(self) -> str:
-        """Complete dataset type name, as seen by the task."""
+        """Complete dataset type name, as seen by the task (`str`)."""
         if self.component is not None:
             return f"{self.parent_dataset_type_name}.{self.component}"
         return self.parent_dataset_type_name

{lsst_pipe_base-29.2025.2100 → lsst_pipe_base-29.2025.2400}/python/lsst/pipe/base/pipeline_graph/_pipeline_graph.py

@@ -1546,7 +1546,7 @@ class PipelineGraph:
 
             - the parent dataset type name;
             - the resolved `DatasetTypeNode`, or `None` if the dataset type has
-            - not been resolved.
+              not been resolved.
         """
         for generation in networkx.algorithms.dag.topological_generations(self._xgraph):
             key: NodeKey

{lsst_pipe_base-29.2025.2100 → lsst_pipe_base-29.2025.2400}/python/lsst/pipe/base/script/transfer_from_graph.py

@@ -27,11 +27,17 @@
 
 __all__ = ["transfer_from_graph"]
 
+import math
+
 from lsst.daf.butler import Butler, CollectionType, QuantumBackedButler, Registry
 from lsst.daf.butler.registry import MissingCollectionError
 from lsst.pipe.base import QuantumGraph
+from lsst.utils.iteration import chunk_iterable
+from lsst.utils.logging import getLogger
+
+from .utils import filter_by_dataset_type_glob, filter_by_existence
 
-from .utils import filter_by_dataset_type_glob
+_LOG = getLogger(__name__)
 
 
 def transfer_from_graph(
@@ -92,18 +98,31 @@ def transfer_from_graph(
 
     # Filter the refs based on requested dataset types.
     filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
+    _LOG.verbose("After filtering by dataset_type, number of datasets to transfer: %d", len(filtered_refs))
 
     dest_butler = Butler.from_config(dest, writeable=True)
 
-    transferred = dest_butler.transfer_from(
-        qbb,
-        filtered_refs,
-        transfer="auto",
-        register_dataset_types=register_dataset_types,
-        transfer_dimensions=transfer_dimensions,
-        dry_run=dry_run,
-    )
-    count = len(transferred)
+    # For faster restarts, filter out those the destination already knows.
+    filtered_refs = filter_by_existence(dest_butler, filtered_refs)
+
+    # Transfer in chunks
+    chunk_size = 50_000
+    n_chunks = math.ceil(len(filtered_refs) / chunk_size)
+    chunk_num = 0
+    count = 0
+    for chunk in chunk_iterable(filtered_refs, chunk_size=chunk_size):
+        chunk_num += 1
+        if n_chunks > 1:
+            _LOG.verbose("Transferring %d datasets in chunk %d/%d", len(chunk), chunk_num, n_chunks)
+        transferred = dest_butler.transfer_from(
+            qbb,
+            chunk,
+            transfer="auto",
+            register_dataset_types=register_dataset_types,
+            transfer_dimensions=transfer_dimensions,
+            dry_run=dry_run,
+        )
+        count += len(transferred)
 
     # If anything was transferred then update output chain definition if asked.
     if count > 0 and update_output_chain and (metadata := qgraph.metadata) is not None:

{lsst_pipe_base-29.2025.2100 → lsst_pipe_base-29.2025.2400}/python/lsst/pipe/base/script/utils.py

@@ -27,15 +27,17 @@
 
 from __future__ import annotations
 
-__all__ = [
-    "filter_by_dataset_type_glob",
-]
+__all__ = ["filter_by_dataset_type_glob", "filter_by_existence"]
 
 import re
 from collections.abc import Collection
 
-from lsst.daf.butler import DatasetRef
+from lsst.daf.butler import Butler, DatasetRef
 from lsst.daf.butler.utils import globToRegex
+from lsst.utils.logging import getLogger
+from lsst.utils.timer import time_this
+
+_LOG = getLogger(__name__)
 
 
 def _matches_dataset_type(dataset_type_name: str, regexes: list[str | re.Pattern]) -> bool:
@@ -72,3 +74,30 @@ def filter_by_dataset_type_glob(
         return refs
 
     return {ref for ref in refs if _matches_dataset_type(ref.datasetType.name, regexes)}
+
+
+def filter_by_existence(butler: Butler, refs: Collection[DatasetRef]) -> Collection[DatasetRef]:
+    """Filter out the refs that the butler already knows exist.
+
+    Parameters
+    ----------
+    butler : `lsst.daf.butler.Butler`
+        Butler in which to check existence of given datarefs.
+    refs : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Datasets to be filtered.
+
+    Returns
+    -------
+    filtered : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Filter datasets.
+    """
+    _LOG.verbose("Filtering out datasets already known to the target butler...")
+    with time_this(log=_LOG, msg="Completed checking existence"):
+        existence = butler._datastore.knows_these(refs)
+        filtered = [ref for ref in existence if not existence[ref]]
+    _LOG.verbose(
+        "After filtering out those already in the target butler, number of datasets to transfer: %d",
+        len(filtered),
+    )
+
+    return filtered

{lsst_pipe_base-29.2025.2100 → lsst_pipe_base-29.2025.2400}/python/lsst/pipe/base/tests/mocks/_storage_class.py

@@ -227,7 +227,7 @@ class MockDataset(pydantic.BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 
@@ -266,7 +266,7 @@ class ConvertedUnmockedDataset(pydantic.BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc]
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc, override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 
@@ -316,7 +316,7 @@ class MockDatasetQuantum(pydantic.BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc]
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc, override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 

lsst_pipe_base-29.2025.2400/python/lsst/pipe/base/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2400"

{lsst_pipe_base-29.2025.2100 → lsst_pipe_base-29.2025.2400/python/lsst_pipe_base.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2100
+Version: 29.2025.2400
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2100 → lsst_pipe_base-29.2025.2400}/python/lsst_pipe_base.egg-info/SOURCES.txt

@@ -136,6 +136,7 @@ tests/test_pipeline_graph_expressions.py
 tests/test_quantumGraph.py
 tests/test_quantum_provenance_graph.py
 tests/test_quantum_success_caveats.py
+tests/test_script_utils.py
 tests/test_struct.py
 tests/test_task.py
 tests/test_taskmetadata.py

lsst_pipe_base-29.2025.2400/tests/test_script_utils.py

@@ -0,0 +1,62 @@
+# This file is part of pipe_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+"""Tests for scripts/utils.py functions."""
+
+import logging
+import unittest
+
+from lsst.pipe.base.script.utils import filter_by_existence
+
+
+class FilterByExistenceTestCase(unittest.TestCase):
+    """Test filter_by_existence function."""
+
+    def setUp(self):
+        # Currently function itself does not need real DatasetRefs.
+        # Only the butler calls requires DatasetRefs, but butler
+        # is being mocked in these tests. So using strings in place
+        # of DatasetRefs.
+        self.orig_refs = [f"{i:05}" for i in range(1, 11)]
+        self.mock_exists = {}
+        for i in range(1, 11):
+            self.mock_exists[f"{i:05}"] = True if i % 3 == 0 else False
+        self.filtered_refs = [f"{i:05}" for i in range(1, 11) if i % 3 != 0]
+
+    def test_success(self):
+        butler_mock = unittest.mock.Mock()
+        butler_mock._datastore.knows_these.return_value = self.mock_exists
+        with self.assertLogs(logging.getLogger("lsst.pipe.base"), level="VERBOSE") as cm:
+            filtered = filter_by_existence(butler_mock, self.orig_refs)
+            self.assertCountEqual(self.filtered_refs, filtered)
+        log_messages = " ".join(cm.output)
+        self.assertIn("Filtering out datasets already known to the target butler", log_messages)
+        self.assertIn("number of datasets to transfer", log_messages)
+
+
+if __name__ == "__main__":
+    unittest.main()

lsst_pipe_base-29.2025.2100/python/lsst/pipe/base/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2100"