lsst-pipe-base 29.2025.2000__tar.gz → 29.2025.2300__tar.gz

{lsst_pipe_base-29.2025.2000/python/lsst_pipe_base.egg-info → lsst_pipe_base-29.2025.2300}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2000
+Version: 29.2025.2300
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300}/python/lsst/pipe/base/_task_metadata.py

@@ -677,7 +677,7 @@ class TaskMetadata(BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc]
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc, override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300}/python/lsst/pipe/base/all_dimensions_quantum_graph_builder.py

@@ -48,7 +48,7 @@ from lsst.daf.butler import (
     DimensionRecordSet,
     MissingDatasetTypeError,
 )
-from lsst.utils.logging import LsstLogAdapter
+from lsst.utils.logging import LsstLogAdapter, PeriodicLogger
 from lsst.utils.timer import timeMethod
 
 from ._datasetQueryConstraints import DatasetQueryConstraintVariant
@@ -215,17 +215,26 @@ class AllDimensionsQuantumGraphBuilder(QuantumGraphBuilder):
             # Allow duplicates from common skypix overlaps to make some queries
             # run faster.
             query._allow_duplicate_overlaps = True
-            self.log.info("Iterating over query results to associate quanta with datasets.")
             # Iterate over query results, populating data IDs for datasets,
             # quanta, and edges.  We populate only the first level of the tree
             # in the first pass, so we can be done with the query results as
             # quickly as possible in case that holds a connection/cursor open.
             n_rows = 0
+            progress_logger: PeriodicLogger | None = None
             for common_data_id in query.data_ids(tree.all_dimensions):
+                if progress_logger is None:
+                    # There can be a long wait between submitting the query and
+                    # returning the first row, so we want to make sure we log
+                    # when we get it; note that PeriodicLogger is not going to
+                    # do that for us, as it waits for its interval _after_ the
+                    # first log is seen.
+                    self.log.info("Iterating over data ID query results.")
+                    progress_logger = PeriodicLogger(self.log)
                 for branch_dimensions, branch in tree.trunk_branches.items():
                     data_id = common_data_id.subset(branch_dimensions)
                     branch.data_ids.add(data_id)
                 n_rows += 1
+                progress_logger.log("Iterating over data ID query results: %d rows processed so far.", n_rows)
             if n_rows == 0:
                 # A single multiline log plays better with log aggregators like
                 # Loki.
@@ -242,7 +251,7 @@ class AllDimensionsQuantumGraphBuilder(QuantumGraphBuilder):
                     # If an exception was raised, write a partial.
                     self.log.error("\n".join(lines))
                 return
-        self.log.verbose("Processed %s initial data ID query rows.", n_rows)
+        self.log.verbose("Done iterating over query results: %d rows processed in total.", n_rows)
         # We now recursively populate the data IDs of the rest of the tree.
         tree.project_data_ids(self.log)
 

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300}/python/lsst/pipe/base/cli/cmd/commands.py

@@ -61,6 +61,7 @@ def register_instrument(*args: Any, **kwargs: Any) -> None:
 @click.option(
     "--dry-run", is_flag=True, default=False, help="Run the transfer but do not update the destination butler"
 )
+@dataset_type_option(help="Subset of dataset types to transfer from graph.")
 @options_file_option()
 def transfer_from_graph(**kwargs: Any) -> None:
     """Transfer datasets from a quantum graph to a destination butler.

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300}/python/lsst/pipe/base/graph/graphSummary.py

@@ -66,7 +66,7 @@ class QgraphTaskSummary(pydantic.BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 
@@ -121,7 +121,7 @@ class QgraphSummary(pydantic.BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300}/python/lsst/pipe/base/pipeline_graph/_dataset_types.py

@@ -60,12 +60,13 @@ class DatasetTypeNode:
     """
 
     dataset_type: DatasetType
-    """Common definition of this dataset type for the graph.
+    """Common definition of this dataset type for the graph
+    (`~lsst.daf.butler.DatasetType`).
     """
 
     is_initial_query_constraint: bool
     """Whether this dataset should be included as a constraint in the initial
-    query for data IDs in QuantumGraph generation.
+    query for data IDs in QuantumGraph generation (`bool`).
 
     This is only `True` for dataset types that are overall regular inputs, and
     only if none of those input connections had ``deferQueryConstraint=True``.
@@ -73,14 +74,16 @@ class DatasetTypeNode:
 
     is_prerequisite: bool
     """Whether this dataset type is a prerequisite input that must exist in
-    the Registry before graph creation.
+    the Registry before graph creation (`bool`).
     """
 
     producing_edge: WriteEdge | None
-    """The edge to the task that produces this dataset type."""
+    """The edge to the task that produces this dataset type
+    (`lsst.pipe.base.pipeline_graph.WriteEdge` or `None`)."""
 
     consuming_edges: Collection[ReadEdge]
-    """The edges to tasks that consume this dataset type."""
+    """The edges to tasks that consume this dataset type
+    (collection [`lsst.pipe.base.pipeline_graph.ReadEdge`])."""
 
     @classmethod
     def _from_edges(
@@ -184,7 +187,7 @@ class DatasetTypeNode:
 
     @property
     def name(self) -> str:
-        """Name of the dataset type.
+        """Name of the dataset type (`str`).
 
         This is always the parent dataset type, never that of a component.
         """
@@ -193,29 +196,33 @@ class DatasetTypeNode:
     @property
     def key(self) -> NodeKey:
         """Key that identifies this dataset type in internal and exported
-        networkx graphs.
+        networkx graphs (`~lsst.pipe.base.pipeline_graph.NodeKey`).
         """
         return NodeKey(NodeType.DATASET_TYPE, self.dataset_type.name)
 
     @property
     def dimensions(self) -> DimensionGroup:
-        """Dimensions of the dataset type."""
+        """Dimensions of the dataset type
+        (`~lsst.daf.butler.DimensionGroup`).
+        """
         return self.dataset_type.dimensions
 
     @property
     def storage_class_name(self) -> str:
-        """String name of the storage class for this dataset type."""
+        """String name of the storage class for this dataset type (`str`)."""
         return self.dataset_type.storageClass_name
 
     @property
     def storage_class(self) -> StorageClass:
-        """Storage class for this dataset type."""
+        """Storage class for this dataset type
+        (`~lsst.daf.butler.StorageClass`).
+        """
         return self.dataset_type.storageClass
 
     @property
     def is_calibration(self) -> bool:
         """Whether this dataset type can be included in
-        `~lsst.daf.butler.CollectionType.CALIBRATION` collections.
+        `~lsst.daf.butler.CollectionType.CALIBRATION` collections (`bool`).
         """
         return self.dataset_type.isCalibration()
 

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300}/python/lsst/pipe/base/pipeline_graph/_edges.py

@@ -90,16 +90,16 @@ class Edge(ABC):
     """
 
     task_key: NodeKey
-    """Task part of the key for this edge in networkx graphs."""
+    """Task part of the key for this edge in networkx graphs (`NodeKey`)."""
 
     dataset_type_key: NodeKey
-    """Task part of the key for this edge in networkx graphs."""
+    """Task part of the key for this edge in networkx graphs (`NodeKey`)."""
 
     connection_name: str
-    """Name used by the task to refer to this dataset type."""
+    """Name used by the task to refer to this dataset type (`str`)."""
 
     storage_class_name: str
-    """Storage class expected by this task.
+    """Storage class expected by this task (`str`).
 
     If `ReadEdge.component` is not `None`, this is the component storage class,
     not the parent storage class.
@@ -111,7 +111,7 @@ class Edge(ABC):
     """
 
     raw_dimensions: frozenset[str]
-    """Raw dimensions in the task declaration.
+    """Raw dimensions in the task declaration (`frozenset` [`str`]).
 
     This can only be used safely for partial comparisons: two edges with the
     same ``raw_dimensions`` (and the same parent dataset type name) always have
@@ -128,12 +128,12 @@ class Edge(ABC):
 
     @property
     def task_label(self) -> str:
-        """Label of the task."""
+        """Label of the task (`str`)."""
         return str(self.task_key)
 
     @property
     def parent_dataset_type_name(self) -> str:
-        """Name of the parent dataset type.
+        """Name of the parent dataset type (`str`).
 
         All dataset type nodes in a pipeline graph are for parent dataset
         types; components are represented by additional `ReadEdge` state.
@@ -154,7 +154,9 @@ class Edge(ABC):
     @property
     def key(self) -> tuple[NodeKey, NodeKey, str]:
         """Ordered tuple of node keys and connection name that uniquely
-        identifies this edge in a pipeline graph.
+        identifies this edge in a pipeline graph (`NodeKey`, `NodeKey`, `str`).
+
+        The nodes are ordered in the same sense as for `nodes`.
         """
         return self.nodes + (self.connection_name,)
 
@@ -163,7 +165,7 @@ class Edge(ABC):
 
     @property
     def dataset_type_name(self) -> str:
-        """Dataset type name seen by the task.
+        """Dataset type name seen by the task (`str`).
 
         This defaults to the parent dataset type name, which is appropriate
         for all writes and most reads.
@@ -355,7 +357,7 @@ class ReadEdge(Edge):
 
     component: str | None
     """Component to add to `parent_dataset_type_name` to form the dataset type
-    name seen by this task.
+    name seen by this task (`str` or `None`).
     """
 
     is_prerequisite: bool
@@ -378,7 +380,7 @@ class ReadEdge(Edge):
 
     @property
     def dataset_type_name(self) -> str:
-        """Complete dataset type name, as seen by the task."""
+        """Complete dataset type name, as seen by the task (`str`)."""
         if self.component is not None:
             return f"{self.parent_dataset_type_name}.{self.component}"
         return self.parent_dataset_type_name

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300}/python/lsst/pipe/base/pipeline_graph/_pipeline_graph.py

@@ -1546,7 +1546,7 @@ class PipelineGraph:
 
             - the parent dataset type name;
             - the resolved `DatasetTypeNode`, or `None` if the dataset type has
-            - not been resolved.
+              not been resolved.
         """
         for generation in networkx.algorithms.dag.topological_generations(self._xgraph):
             key: NodeKey

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300}/python/lsst/pipe/base/script/transfer_from_graph.py

@@ -31,6 +31,8 @@ from lsst.daf.butler import Butler, CollectionType, QuantumBackedButler, Registr
 from lsst.daf.butler.registry import MissingCollectionError
 from lsst.pipe.base import QuantumGraph
 
+from .utils import filter_by_dataset_type_glob
+
 
 def transfer_from_graph(
     graph: str,
@@ -39,6 +41,7 @@ def transfer_from_graph(
     transfer_dimensions: bool,
     update_output_chain: bool,
     dry_run: bool,
+    dataset_type: tuple[str, ...],
 ) -> int:
     """Transfer output datasets from quantum graph to dest.
 
@@ -60,6 +63,9 @@ def transfer_from_graph(
         name as a the first collection in the chain.
     dry_run : `bool`
         Run the transfer without updating the destination butler.
+    dataset_type : `tuple` of `str`
+        Dataset type names. An empty tuple implies all dataset types.
+        Can include globs.
 
     Returns
     -------
@@ -84,11 +90,14 @@ def transfer_from_graph(
         dataset_types=dataset_types,
     )
 
+    # Filter the refs based on requested dataset types.
+    filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
+
     dest_butler = Butler.from_config(dest, writeable=True)
 
     transferred = dest_butler.transfer_from(
         qbb,
-        output_refs,
+        filtered_refs,
         transfer="auto",
         register_dataset_types=register_dataset_types,
         transfer_dimensions=transfer_dimensions,

lsst_pipe_base-29.2025.2300/python/lsst/pipe/base/script/utils.py

@@ -0,0 +1,74 @@
+# This file is part of pipe_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+__all__ = [
+    "filter_by_dataset_type_glob",
+]
+
+import re
+from collections.abc import Collection
+
+from lsst.daf.butler import DatasetRef
+from lsst.daf.butler.utils import globToRegex
+
+
+def _matches_dataset_type(dataset_type_name: str, regexes: list[str | re.Pattern]) -> bool:
+    for regex in regexes:
+        if isinstance(regex, str):
+            if dataset_type_name == regex:
+                return True
+        elif regex.search(dataset_type_name):
+            return True
+    return False
+
+
+def filter_by_dataset_type_glob(
+    refs: Collection[DatasetRef], dataset_types: tuple[str, ...]
+) -> Collection[DatasetRef]:
+    """Filter the refs based on requested dataset types.
+
+    Parameters
+    ----------
+    refs : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Datasets to be filtered.
+    dataset_types : `tuple` [ `str`, ...]
+        Dataset type names or globs to use for filtering. Empty tuple implies
+        no filtering.
+
+    Returns
+    -------
+    filtered : `collections.abc.Collection` [ `lsst.daf.butler.DatasetRef` ]
+        Filter datasets.
+    """
+    regexes = globToRegex(dataset_types)
+    if regexes is ...:
+        # Nothing to do.
+        return refs
+
+    return {ref for ref in refs if _matches_dataset_type(ref.datasetType.name, regexes)}

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300}/python/lsst/pipe/base/script/zip_from_graph.py

@@ -28,13 +28,13 @@
 __all__ = ["zip_from_graph"]
 
 import logging
-import re
 
 from lsst.daf.butler import QuantumBackedButler
-from lsst.daf.butler.utils import globToRegex
 from lsst.pipe.base import QuantumGraph
 from lsst.resources import ResourcePath
 
+from .utils import filter_by_dataset_type_glob
+
 _LOG = logging.getLogger(__name__)
 
 
@@ -83,22 +83,8 @@ def zip_from_graph(
     )
 
     # Filter the refs based on requested dataset types.
-    regexes = globToRegex(dataset_type)
-    if regexes is ...:
-        filtered_refs = output_refs
-    else:
-
-        def _matches(dataset_type_name: str, regexes: list[str | re.Pattern]) -> bool:
-            for regex in regexes:
-                if isinstance(regex, str):
-                    if dataset_type_name == regex:
-                        return True
-                elif regex.search(dataset_type_name):
-                    return True
-            return False
-
-        filtered_refs = {ref for ref in output_refs if _matches(ref.datasetType.name, regexes)}
+    filtered_refs = filter_by_dataset_type_glob(output_refs, dataset_type)
 
-    _LOG.info("Retrieving artifacts for %d datasets and storing in Zip file.", len(output_refs))
+    _LOG.info("Retrieving artifacts for %d datasets and storing in Zip file.", len(filtered_refs))
     zip = qbb.retrieve_artifacts_zip(filtered_refs, dest)
     return zip

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300}/python/lsst/pipe/base/tests/mocks/_storage_class.py

@@ -227,7 +227,7 @@ class MockDataset(pydantic.BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 
@@ -266,7 +266,7 @@ class ConvertedUnmockedDataset(pydantic.BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc]
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc, override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 
@@ -316,7 +316,7 @@ class MockDatasetQuantum(pydantic.BaseModel):
             return super().model_copy(*args, **kwargs)
 
         @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc]
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc, override]
             """See `pydantic.BaseModel.model_construct`."""
             return super().model_construct(*args, **kwargs)
 

lsst_pipe_base-29.2025.2300/python/lsst/pipe/base/version.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "29.2025.2300"

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300/python/lsst_pipe_base.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lsst-pipe-base
-Version: 29.2025.2000
+Version: 29.2025.2300
 Summary: Pipeline infrastructure for the Rubin Science Pipelines.
 Author-email: Rubin Observatory Data Management <dm-admin@lists.lsst.org>
 License: BSD 3-Clause License

{lsst_pipe_base-29.2025.2000 → lsst_pipe_base-29.2025.2300}/python/lsst_pipe_base.egg-info/SOURCES.txt

@@ -93,6 +93,7 @@ python/lsst/pipe/base/script/__init__.py
 python/lsst/pipe/base/script/register_instrument.py
 python/lsst/pipe/base/script/retrieve_artifacts_for_quanta.py
 python/lsst/pipe/base/script/transfer_from_graph.py
+python/lsst/pipe/base/script/utils.py
 python/lsst/pipe/base/script/zip_from_graph.py
 python/lsst/pipe/base/tests/__init__.py
 python/lsst/pipe/base/tests/no_dimensions.py

lsst_pipe_base-29.2025.2000/python/lsst/pipe/base/version.py

@@ -1,2 +0,0 @@
-__all__ = ["__version__"]
-__version__ = "29.2025.2000"