lsst-pipe-base 30.0.1rc1__py3-none-any.whl → 30.2025.5200__py3-none-any.whl

lsst/pipe/base/connectionTypes.py

@@ -41,36 +41,35 @@ from lsst.utils.introspection import find_outside_stacklevel
 
 @dataclasses.dataclass(frozen=True)
 class BaseConnection:
-    """Base class used for declaring `PipelineTask` connections."""
+    """Base class used for declaring `PipelineTask` connections.
+
+    Attributes
+    ----------
+    name : `str`
+        The name used to identify the dataset type.
+    storageClass : `str`
+        The storage class used when (un)/persisting the dataset type.
+    multiple : `bool`
+        Indicates if this connection should expect to contain multiple objects
+        of the given dataset type.  Tasks with more than one connection with
+        ``multiple=True`` with the same dimensions may want to implement
+        `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
+        consistent (i.e. zip-iterable) in `PipelineTask.runQuantum()` and
+        notify the execution system as early as possible of outputs that will
+        not be produced because the corresponding input is missing.
+    deprecated : `str`, optional
+        A description of why this connection is deprecated, including the
+        version after which it may be removed.
+
+        If not `None`, the string is appended to the docstring for this
+        connection and the corresponding config Field.
+    """
 
     name: str
-    """The name used to identify the dataset type."""
-
     storageClass: str
-    """The storage class used when (un)/persisting the dataset type."""
-
     doc: str = ""
-    """Documentation for this connection."""
-
     multiple: bool = False
-    """Indicates if this connection should expect to contain multiple objects
-    of the given dataset type.
-
-    Tasks with more than one connection with ``multiple=True`` with the same
-    dimensions may want to implement `.PipelineTaskConnections.adjustQuantum`
-    to ensure those datasets are consistent (i.e. zip-iterable) in
-    `PipelineTask.runQuantum()` and notify the execution system as early as
-    possible of outputs that will not be produced because the corresponding
-    input is missing.
-    """
-
     deprecated: str | None = dataclasses.field(default=None, kw_only=True)
-    """A description of why this connection is deprecated, including the
-    version after which it may be removed.
-
-    If not `None`, the string is appended to the docstring for this
-    connection and the corresponding config Field.
-    """
 
     _connection_type_set: ClassVar[str]
     _deprecation_context: str = ""
@@ -111,15 +110,32 @@ class BaseConnection:
 class DimensionedConnection(BaseConnection):
     """Class used for declaring PipelineTask connections that includes
     dimensions.
+
+    Attributes
+    ----------
+    name : `str`
+        The name used to identify the dataset type.
+    storageClass : `str`
+        The storage class used when (un)/persisting the dataset type.
+    multiple : `bool`
+        Indicates if this connection should expect to contain multiple objects
+        of the given dataset type.  Tasks with more than one connection with
+        ``multiple=True`` with the same dimensions may want to implement
+        `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
+        consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
+        the execution system as early as possible of outputs that will not be
+        produced because the corresponding input is missing.
+    dimensions : iterable of `str`
+        The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
+        to identify the dataset type identified by the specified name.
+    isCalibration : `bool`, optional
+        `True` if this dataset type may be included in CALIBRATION-type
+        collections to associate it with a validity range, `False` (default)
+        otherwise.
     """
 
     dimensions: Iterable[str] = ()
-    """The keys of the butler data coordinates for this dataset type."""
-
     isCalibration: bool = False
-    """ `True` if this dataset type may be included in
-    `~lsst.daf.butler.CollectionType.CALIBRATION` collections to associate it
-    with a validity range, `False` (default) otherwise."""
 
     def __post_init__(self):
         super().__post_init__()
@@ -135,6 +151,39 @@ class DimensionedConnection(BaseConnection):
 class BaseInput(DimensionedConnection):
     """Class used for declaring PipelineTask input connections.
 
+    Attributes
+    ----------
+    name : `str`
+        The default name used to identify the dataset type.
+    storageClass : `str`
+        The storage class used when (un)/persisting the dataset type.
+    multiple : `bool`
+        Indicates if this connection should expect to contain multiple objects
+        of the given dataset type.  Tasks with more than one connection with
+        ``multiple=True`` with the same dimensions may want to implement
+        `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
+        consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
+        the execution system as early as possible of outputs that will not be
+        produced because the corresponding input is missing.
+    dimensions : iterable of `str`
+        The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
+        to identify the dataset type identified by the specified name.
+    deferLoad : `bool`
+        Indicates that this dataset type will be loaded as a
+        `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
+        object to load the object at a later time.
+    minimum : `bool`
+        Minimum number of datasets required for this connection, per quantum.
+        This is checked in the base implementation of
+        `.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if
+        the minimum is not met for `Input` connections (causing the quantum to
+        be pruned, skipped, or never created, depending on the context), and
+        `FileNotFoundError` for `PrerequisiteInput` connections (causing
+        QuantumGraph generation to fail).  `PipelineTask` implementations may
+        provide custom `~.PipelineTaskConnections.adjustQuantum`
+        implementations for more fine-grained or configuration-driven
+        constraints, as long as they are compatible with this minium.
+
     Raises
     ------
     TypeError
@@ -145,24 +194,7 @@ class BaseInput(DimensionedConnection):
     """
 
     deferLoad: bool = False
-    """Whether this dataset type will be loaded as a
-    `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
-    object to load the object at a later time.
-    """
-
     minimum: int = 1
-    """Minimum number of datasets required for this connection, per quantum.
-
-    This is checked in the base implementation of
-    `.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if the
-    minimum is not met for `Input` connections (causing the quantum to be
-    pruned, skipped, or never created, depending on the context), and
-    `FileNotFoundError` for `PrerequisiteInput` connections (causing
-    QuantumGraph generation to fail).  `PipelineTask` implementations may
-    provide custom `~.PipelineTaskConnections.adjustQuantum` implementations
-    for more fine-grained or configuration-driven constraints, as long as they
-    are compatible with this minimum.
-    """
 
     def __post_init__(self) -> None:
         super().__post_init__()
@@ -174,6 +206,56 @@ class BaseInput(DimensionedConnection):
 class Input(BaseInput):
     """Class used for declaring PipelineTask input connections.
 
+    Attributes
+    ----------
+    name : `str`
+        The default name used to identify the dataset type.
+    storageClass : `str`
+        The storage class used when (un)/persisting the dataset type.
+    multiple : `bool`
+        Indicates if this connection should expect to contain multiple objects
+        of the given dataset type.  Tasks with more than one connection with
+        ``multiple=True`` with the same dimensions may want to implement
+        `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
+        consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
+        the execution system as early as possible of outputs that will not be
+        produced because the corresponding input is missing.
+    dimensions : iterable of `str`
+        The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
+        to identify the dataset type identified by the specified name.
+    deferLoad : `bool`
+        Indicates that this dataset type will be loaded as a
+        `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
+        object to load the object at a later time.
+    minimum : `bool`
+        Minimum number of datasets required for this connection, per quantum.
+        This is checked in the base implementation of
+        `.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if
+        the minimum is not met for `Input` connections (causing the quantum to
+        be pruned, skipped, or never created, depending on the context), and
+        `FileNotFoundError` for `PrerequisiteInput` connections (causing
+        QuantumGraph generation to fail).  `PipelineTask` implementations may
+        provide custom `~.PipelineTaskConnections.adjustQuantum`
+        implementations for more fine-grained or configuration-driven
+        constraints, as long as they are compatible with this minium.
+    deferGraphConstraint : `bool`, optional
+        If `True`, do not include this dataset type's existence in the initial
+        query that starts the QuantumGraph generation process.  This can be
+        used to make QuantumGraph generation faster by avoiding redundant
+        datasets, and in certain cases it can (along with careful attention to
+        which tasks are included in the same QuantumGraph) be used to work
+        around the QuantumGraph generation algorithm's inflexible handling of
+        spatial overlaps.  This option has no effect when the connection is not
+        an overall input of the pipeline (or subset thereof) for which a graph
+        is being created, and it never affects the ordering of quanta.
+    deferBinding : `bool`, optional
+        If `True`, the dataset will not be automatically included in
+        the pipeline graph, ``deferGraphConstraint`` is implied.
+        The custom QuantumGraphBuilder is required to bind it and add a
+        corresponding edge to the pipeline graph.
+        This option allows to have the same dataset type as both
+        input and output of a quantum.
+
     Raises
     ------
     TypeError
@@ -184,27 +266,8 @@ class Input(BaseInput):
     """
 
     deferGraphConstraint: bool = False
-    """If `True`, do not include this dataset type's existence in the initial
-    query that starts the QuantumGraph generation process.
-
-    This can be used to make QuantumGraph generation faster by avoiding
-    redundant datasets, and in certain cases it can (along with careful
-    attention to which tasks are included in the same QuantumGraph) be used to
-    work around the QuantumGraph generation algorithm's inflexible handling of
-    spatial overlaps.  This option has no effect when the connection is not an
-    overall input of the pipeline (or subset thereof) for which a graph is
-    being created, and it never affects the ordering of quanta.
-    """
 
     deferBinding: bool = False
-    """If `True`, the dataset will not be automatically included in the
-    pipeline graph (``deferGraphConstraint=True`` is implied).
-
-    A custom `~.quantum_graph_builder.QuantumGraphBuilder` is required to bind
-    it and add a corresponding edge to the pipeline graph. This option allows
-    the same dataset type to be used as both an input and an output of a
-    quantum.
-    """
 
     _connection_type_set: ClassVar[str] = "inputs"
 
@@ -213,6 +276,38 @@ class Input(BaseInput):
 class PrerequisiteInput(BaseInput):
     """Class used for declaring PipelineTask prerequisite connections.
 
+    Attributes
+    ----------
+    name : `str`
+        The default name used to identify the dataset type.
+    storageClass : `str`
+        The storage class used when (un)/persisting the dataset type.
+    multiple : `bool`
+        Indicates if this connection should expect to contain multiple objects
+        of the given dataset type.  Tasks with more than one connection with
+        ``multiple=True`` with the same dimensions may want to implement
+        `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
+        consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
+        the execution system as early as possible of outputs that will not be
+        produced because the corresponding input is missing.
+    dimensions : iterable of `str`
+        The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
+        to identify the dataset type identified by the specified name.
+    minimum : `bool`
+        Minimum number of datasets required for this connection, per quantum.
+        This is checked in the base implementation of
+        `.PipelineTaskConnections.adjustQuantum`, which raises
+        `FileNotFoundError` (causing QuantumGraph generation to fail).
+        `PipelineTask` implementations may provide custom
+        `~.PipelineTaskConnections.adjustQuantum` implementations for more
+        fine-grained or configuration-driven constraints, as long as they are
+        compatible with this minium.
+    lookupFunction : `typing.Callable`, optional
+        An optional callable function that will look up PrerequisiteInputs
+        using the DatasetType, registry, quantum dataId, and input collections
+        passed to it. If no function is specified, the default temporal spatial
+        lookup will be used.
+
     Raises
     ------
     TypeError
@@ -247,13 +342,6 @@ class PrerequisiteInput(BaseInput):
     lookupFunction: (
         Callable[[DatasetType, Registry, DataCoordinate, Sequence[str]], Iterable[DatasetRef]] | None
     ) = None
-    """An optional callable function that will look up PrerequisiteInputs
-    using the DatasetType, registry, quantum dataId, and input collections
-    passed to it.
-
-    If no function is specified, the default temporal/spatial lookup will be
-    used.
-    """
 
     _connection_type_set: ClassVar[str] = "prerequisiteInputs"
 

lsst/pipe/base/connections.py

@@ -461,7 +461,7 @@ class QuantizedConnection(SimpleNamespace):
 
     def __iter__(
         self,
-    ) -> Generator[tuple[str, DatasetRef | list[DatasetRef]]]:
+    ) -> Generator[tuple[str, DatasetRef | list[DatasetRef]], None, None]:
         """Make an iterator for this `QuantizedConnection`.
 
         Iterating over a `QuantizedConnection` will yield a tuple with the name
@@ -471,7 +471,7 @@ class QuantizedConnection(SimpleNamespace):
         """
         yield from ((name, getattr(self, name)) for name in self._attributes)
 
-    def keys(self) -> Generator[str]:
+    def keys(self) -> Generator[str, None, None]:
         """Return an iterator over all the attributes added to a
         `QuantizedConnection` class.
         """
@@ -495,12 +495,15 @@ class DeferredDatasetRef:
     """A wrapper class for `~lsst.daf.butler.DatasetRef` that indicates that a
     `PipelineTask` should receive a `~lsst.daf.butler.DeferredDatasetHandle`
     instead of an in-memory dataset.
+
+    Attributes
+    ----------
+    datasetRef : `lsst.daf.butler.DatasetRef`
+        The `lsst.daf.butler.DatasetRef` that will be eventually used to
+        resolve a dataset.
     """
 
     datasetRef: DatasetRef
-    """The `lsst.daf.butler.DatasetRef` that will be eventually used to
-    resolve a dataset.
-    """
 
     def __getattr__(self, name: str) -> Any:
         # make sure reduce is called on DeferredDatasetRef and not on
@@ -1041,7 +1044,7 @@ class PipelineTaskConnections(metaclass=PipelineTaskConnectionsMetaclass):
 
 def iterConnections(
     connections: PipelineTaskConnections, connectionType: str | Iterable[str]
-) -> Generator[BaseConnection]:
+) -> Generator[BaseConnection, None, None]:
     """Create an iterator over the selected connections type which yields
     all the defined connections of that type.
 

lsst/pipe/base/execution_reports.py

@@ -299,6 +299,11 @@ class QuantumGraphExecutionReport:
     produced DatasetTypes for each task. This report can be output as a
     dictionary or a yaml file.
 
+    Attributes
+    ----------
+    tasks : `dict`
+        A dictionary of TaskExecutionReports by task label.
+
     See Also
     --------
     TaskExecutionReport : A task report.

lsst/pipe/base/graph/graph.py

@@ -136,14 +136,13 @@ class QuantumGraph:
         Maps tasks to their InitOutput dataset refs. Dataset refs can be either
         resolved or non-resolved. For intermediate resolved refs their dataset
         ID must match ``initInputs`` and Quantum ``initInputs``.
-    globalInitOutputs : `~collections.abc.Iterable` \
-            [ `~lsst.daf.butler.DatasetRef` ], optional
+    globalInitOutputs : iterable [ `~lsst.daf.butler.DatasetRef` ], optional
         Dataset refs for some global objects produced by pipeline. These
         objects include task configurations and package versions. Typically
         they have an empty DataId, but there is no real restriction on what
         can appear here.
-    registryDatasetTypes : `~collections.abc.Iterable` \
-            [ `~lsst.daf.butler.DatasetType` ],  optional
+    registryDatasetTypes : iterable [ `~lsst.daf.butler.DatasetType` ], \
+            optional
         Dataset types which are used by this graph, their definitions must
         match registry. If registry does not define dataset type yet, then
         it should match one that will be created later.
@@ -489,7 +488,7 @@ class QuantumGraph:
 
         Returns
         -------
-        tasks : `~collections.abc.Iterable` [ `TaskDef` ]
+        tasks : iterable of `TaskDef`
             `TaskDef` objects that have the specified `DatasetTypeName` as an
             input, list will be empty if no tasks use specified
             `DatasetTypeName` as an input.
@@ -538,7 +537,7 @@ class QuantumGraph:
 
         Returns
         -------
-        result : `~collections.abc.Iterable` [`TaskDef`]
+        result : iterable of `TaskDef`
             `TaskDef` objects that are associated with the specified
             `DatasetTypeName`.
 
@@ -936,7 +935,7 @@ class QuantumGraph:
             saved structure. If supplied, the
             `~lsst.daf.butler.DimensionUniverse` from the loaded `QuantumGraph`
             will be validated against the supplied argument for compatibility.
-        nodes : `~collections.abc.Iterable` [ `uuid.UUID` | `str` ] or `None`
+        nodes : iterable of [ `uuid.UUID` | `str` ] or `None`
             UUIDs that correspond to nodes in the graph. If specified, only
             these nodes will be loaded. Defaults to None, in which case all
             nodes will be loaded.
@@ -1221,7 +1220,7 @@ class QuantumGraph:
             saved structure. If supplied, the
             `~lsst.daf.butler.DimensionUniverse` from the loaded `QuantumGraph`
             will be validated against the supplied argument for compatibility.
-        nodes : `~collections.abc.Iterable` [`uuid.UUID`] or `None`
+        nodes : iterable of `uuid.UUID` or `None`
             UUIDs that correspond to nodes in the graph. If specified, only
             these nodes will be loaded. Defaults to None, in which case all
             nodes will be loaded.
@@ -1257,7 +1256,7 @@ class QuantumGraph:
             raise TypeError(f"QuantumGraph file contains unexpected object type: {type(qgraph)}")
         return qgraph
 
-    def iterTaskGraph(self) -> Generator[TaskDef]:
+    def iterTaskGraph(self) -> Generator[TaskDef, None, None]:
         """Iterate over the `taskGraph` attribute in topological order.
 
         Yields
@@ -1375,7 +1374,7 @@ class QuantumGraph:
         """
         return self._universe
 
-    def __iter__(self) -> Generator[QuantumNode]:
+    def __iter__(self) -> Generator[QuantumNode, None, None]:
         yield from nx.topological_sort(self._connectedQuanta)
 
     def __len__(self) -> int:
@@ -1439,7 +1438,7 @@ class QuantumGraph:
         Returns
         -------
         summary : `QgraphSummary`
-            Summary of QuantumGraph.
+           Summary of QuantumGraph.
         """
         inCollection = self.metadata.get("input", None)
         if isinstance(inCollection, str):

lsst/pipe/base/graph/quantumNode.py

@@ -54,7 +54,7 @@ def _hashDsRef(ref: DatasetRef) -> int:
 
 @dataclass(frozen=True, eq=True)
 class NodeId:
-    r"""Deprecated, this class is used with QuantumGraph save formats of
+    """Deprecated, this class is used with QuantumGraph save formats of
     1 and 2 when unpicking objects and must be retained until those formats
     are considered unloadable.
 
@@ -66,9 +66,9 @@ class NodeId:
     A `NodeId` will not be the same if a new graph is built containing the same
     information in a `QuantumNode`, or even built from exactly the same inputs.
 
-    `NodeId`\ s do not play any role in deciding the equality or identity
-    (hash) of a `QuantumNode`, and are mainly useful in debugging or working
-    with various subsets of the same graph.
+    `NodeId`s do not play any role in deciding the equality or identity (hash)
+    of a `QuantumNode`, and are mainly useful in debugging or working with
+    various subsets of the same graph.
 
     This interface is a convenance only, and no guarantees on long term
     stability are made. New implementations might change the `NodeId`, or

lsst/pipe/base/graph_walker.py

@@ -29,12 +29,14 @@ from __future__ import annotations
 
 __all__ = ("GraphWalker",)
 
-from typing import Self
+from typing import Generic, Self, TypeVar
 
 import networkx
 
+_T = TypeVar("_T")
 
-class GraphWalker[T]:
+
+class GraphWalker(Generic[_T]):
     """A helper for traversing directed acyclic graphs.
 
     Parameters
@@ -57,14 +59,14 @@ class GraphWalker[T]:
 
     def __init__(self, xgraph: networkx.DiGraph | networkx.MultiDiGraph):
         self._xgraph = xgraph
-        self._ready: set[T] = set(next(iter(networkx.dag.topological_generations(self._xgraph)), []))
-        self._active: set[T] = set()
-        self._incomplete: set[T] = set(self._xgraph)
+        self._ready: set[_T] = set(next(iter(networkx.dag.topological_generations(self._xgraph)), []))
+        self._active: set[_T] = set()
+        self._incomplete: set[_T] = set(self._xgraph)
 
     def __iter__(self) -> Self:
         return self
 
-    def __next__(self) -> frozenset[T]:
+    def __next__(self) -> frozenset[_T]:
         if not self._incomplete:
             raise StopIteration()
         new_active = frozenset(self._ready)
@@ -72,7 +74,7 @@ class GraphWalker[T]:
         self._ready.clear()
         return new_active
 
-    def finish(self, key: T) -> None:
+    def finish(self, key: _T) -> None:
         """Mark a node as successfully processed, unblocking (at least in part)
         iteration over successor nodes.
 
@@ -95,7 +97,7 @@ class GraphWalker[T]:
             ):
                 self._ready.add(successor)
 
-    def fail(self, key: T) -> list[T]:
+    def fail(self, key: _T) -> list[_T]:
         """Mark a node as unsuccessfully processed, permanently blocking all
         recursive descendants.
 

lsst/pipe/base/log_capture.py

@@ -103,7 +103,7 @@ class _ExecutionLogRecordsExtra(pydantic.BaseModel):
 
         Parameters
         ----------
-        log_records : `lsst.daf.butler.ButlerLogRecords`
+        log_records : `ButlerLogRecords`
             Logs from a past attempt to run a quantum.
         """
         previous = self.model_validate(log_records.extra)
@@ -163,9 +163,7 @@ class LogCapture:
         return cls(butler, butler)
 
     @contextmanager
-    def capture_logging(
-        self, task_node: TaskNode, /, quantum: Quantum, records: ButlerLogRecords | None = None
-    ) -> Iterator[_LogCaptureContext]:
+    def capture_logging(self, task_node: TaskNode, /, quantum: Quantum) -> Iterator[_LogCaptureContext]:
         """Configure logging system to capture logs for execution of this task.
 
         Parameters
@@ -174,9 +172,6 @@ class LogCapture:
             The task definition.
         quantum : `~lsst.daf.butler.Quantum`
             Single Quantum instance.
-        records : `lsst.daf.butler.logging.ButlerLogRecords`, optional
-            Log record container to append to and save.  If provided, streaming
-            mode is disabled (since we'll be saving logs in memory anyway).
 
         Notes
         -----
@@ -218,7 +213,7 @@ class LogCapture:
                 ) from exc
             # Either accumulate into ButlerLogRecords or stream JSON records to
             # file and ingest that (ingest is possible only with full butler).
-            if self.stream_json_logs and self.full_butler is not None and records is None:
+            if self.stream_json_logs and self.full_butler is not None:
                 with TemporaryForIngest(self.full_butler, ref) as temporary:
                     log_handler_file = FileHandler(temporary.ospath)
                     log_handler_file.setFormatter(JsonLogFormatter())
@@ -241,7 +236,7 @@ class LogCapture:
                             temporary.ingest()
 
             else:
-                log_handler_memory = ButlerLogRecordHandler(records)
+                log_handler_memory = ButlerLogRecordHandler()
                 logging.getLogger().addHandler(log_handler_memory)
 
                 try:
@@ -260,6 +255,7 @@ class LogCapture:
                     logging.getLogger().removeHandler(log_handler_memory)
                     if ctx.store:
                         self._store_log_records(quantum, log_dataset_name, log_handler_memory)
+                    log_handler_memory.records.clear()
 
         else:
             with ButlerMDC.set_mdc(mdc):