lsst-pipe-base 30.2026.200__py3-none-any.whl → 30.2026.400__py3-none-any.whl

lsst/pipe/base/_instrument.py

@@ -31,7 +31,6 @@ __all__ = ("Instrument",)
 
 import contextlib
 import datetime
-import os.path
 from abc import ABCMeta, abstractmethod
 from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any, Self, cast, final
@@ -39,6 +38,7 @@ from typing import TYPE_CHECKING, Any, Self, cast, final
 from lsst.daf.butler import DataCoordinate, DataId, DimensionPacker, DimensionRecord, Formatter
 from lsst.daf.butler.registry import DataIdError
 from lsst.pex.config import Config, RegistryField
+from lsst.resources import ResourcePath, ResourcePathExpression
 from lsst.utils import doImportType
 from lsst.utils.introspection import get_full_type_name
 
@@ -65,7 +65,7 @@ class Instrument(metaclass=ABCMeta):
     the base class.
     """
 
-    configPaths: Sequence[str] = ()
+    configPaths: Sequence[ResourcePathExpression] = ()
     """Paths to config files to read for specific Tasks.
 
     The paths in this list should contain files of the form `task.py`, for
@@ -109,6 +109,10 @@ class Instrument(metaclass=ABCMeta):
             If `True` (`False` is default), update existing records if they
             differ from the new ones.
 
+        Returns
+        -------
+        None
+
         Raises
         ------
         lsst.daf.butler.registry.ConflictingDefinitionError
@@ -127,13 +131,6 @@ class Instrument(metaclass=ABCMeta):
         the level of individual dimension entries; new detectors and filters
         should be added, but changes to any existing record should not be.
         This can generally be achieved via a block like
-
-        .. code-block:: python
-
-            with registry.transaction():
-                registry.syncDimensionData("instrument", ...)
-                registry.syncDimensionData("detector", ...)
-                self.registerFilters(registry)
         """
         raise NotImplementedError()
 
@@ -366,9 +363,10 @@ class Instrument(metaclass=ABCMeta):
             Config instance to which overrides should be applied.
         """
         for root in self.configPaths:
-            path = os.path.join(root, f"{name}.py")
-            if os.path.exists(path):
-                config.load(path)
+            resource = ResourcePath(root, forceDirectory=True, forceAbsolute=True)
+            uri = resource.join(f"{name}.py", forceDirectory=False)
+            if uri.exists():
+                config.load(uri)
 
     @staticmethod
     def formatCollectionTimestamp(timestamp: str | datetime.datetime) -> str:

lsst/pipe/base/_status.py

@@ -275,15 +275,23 @@ class ExceptionInfo(pydantic.BaseModel):
 class QuantumAttemptStatus(enum.Enum):
     """Enum summarizing an attempt to run a quantum."""
 
+    ABORTED = -4
+    """The quantum failed with a hard error that prevented both logs and
+    metadata from being written.
+
+    This state is only set if information from higher-level tooling (e.g. BPS)
+    is available to distinguish it from ``UNKNOWN``.
+    """
+
     UNKNOWN = -3
     """The status of this attempt is unknown.
 
-    This usually means no logs or metadata were written, and it at least could
-    not be determined whether the quantum was blocked by an upstream failure
-    (if it was definitely blocked, `BLOCKED` is set instead).
+    This means no logs or metadata were written, and it at least could not be
+    determined whether the quantum was blocked by an upstream failure (if it
+    was definitely blocked, `BLOCKED` is set instead).
     """
 
-    LOGS_MISSING = -2
+    ABORTED_SUCCESS = -2
     """Task metadata was written for this attempt but logs were not.
 
     This is a rare condition that requires a hard failure (i.e. the kind that
@@ -292,20 +300,21 @@ class QuantumAttemptStatus(enum.Enum):
     """
 
     FAILED = -1
-    """Execution of the quantum failed.
+    """Execution of the quantum failed gracefully.
 
     This is always set if the task metadata dataset was not written but logs
     were, as is the case when a Python exception is caught and handled by the
-    execution system.  It may also be set in cases where logs were not written
-    either, but other information was available (e.g. from higher-level
-    orchestration tooling) to mark it as a failure.
+    execution system.
+
+    This status guarantees that the task log dataset was produced but the
+    metadata dataset was not.
     """
 
     BLOCKED = 0
     """This quantum was not executed because an upstream quantum failed.
 
-    Upstream quanta with status `UNKNOWN` or `FAILED` are considered blockers;
-    `LOGS_MISSING` is not.
+    Upstream quanta with status `UNKNOWN`, `FAILED`, or `ABORTED` are
+    considered blockers; `ABORTED_SUCCESS` is not.
     """
 
     SUCCESSFUL = 1
@@ -319,6 +328,16 @@ class QuantumAttemptStatus(enum.Enum):
     these "successes with caveats" are reported.
     """
 
+    @property
+    def has_metadata(self) -> bool:
+        """Whether the task metadata dataset was produced."""
+        return self is self.SUCCESSFUL or self is self.ABORTED_SUCCESS
+
+    @property
+    def has_log(self) -> bool:
+        """Whether the log dataset was produced."""
+        return self is self.SUCCESSFUL or self is self.FAILED
+
 
 class GetSetDictMetadataHolder(Protocol):
     """Protocol for objects that have a ``metadata`` attribute that satisfies

lsst/pipe/base/automatic_connection_constants.py

@@ -26,7 +26,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """Constants used to define the connections automatically added for each
-PipelineTask by the execution system.
+PipelineTask by the execution system, as well as other special dataset types.
 """
 
 from __future__ import annotations
@@ -43,6 +43,8 @@ __all__ = (
     "METADATA_OUTPUT_TEMPLATE",
     "PACKAGES_INIT_OUTPUT_NAME",
     "PACKAGES_INIT_OUTPUT_STORAGE_CLASS",
+    "PROVENANCE_DATASET_TYPE_NAME",
+    "PROVENANCE_STORAGE_CLASS",
 )
 
 
@@ -91,3 +93,9 @@ type names.
 METADATA_OUTPUT_STORAGE_CLASS: str = "TaskMetadata"
 """Name of the storage class for task metadata output datasets.
 """
+
+PROVENANCE_DATASET_TYPE_NAME: str = "run_provenance"
+"""Name of the dataset used to store per-RUN provenance."""
+
+PROVENANCE_STORAGE_CLASS: str = "ProvenanceQuantumGraph"
+"""Name of the storage class used to store provenance."""

lsst/pipe/base/cli/cmd/__init__.py

@@ -25,6 +25,20 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-__all__ = ["register_instrument", "transfer_from_graph", "zip_from_graph", "retrieve_artifacts_for_quanta", "aggregate_graph"]
+__all__ = [
+    "register_instrument",
+    "transfer_from_graph",
+    "zip_from_graph",
+    "retrieve_artifacts_for_quanta",
+    "aggregate_graph",
+    "ingest_graph",
+]
 
-from .commands import (register_instrument, retrieve_artifacts_for_quanta, transfer_from_graph, zip_from_graph, aggregate_graph)
+from .commands import (
+    register_instrument,
+    retrieve_artifacts_for_quanta,
+    transfer_from_graph,
+    zip_from_graph,
+    aggregate_graph,
+    ingest_graph,
+)

lsst/pipe/base/cli/cmd/commands.py

@@ -161,7 +161,7 @@ _AGGREGATOR_DEFAULTS = aggregator.AggregatorConfig()
 
 @click.command(short_help="Scan for the outputs of an active or completed quantum graph.", cls=ButlerCommand)
 @click.argument("predicted_graph", required=True)
-@repo_argument(required=True, help="Path to the central butler repository.")
+@repo_argument(required=True, help="Path or alias for the butler repository.")
 @click.option(
     "-o",
     "--output",
@@ -181,9 +181,9 @@ _AGGREGATOR_DEFAULTS = aggregator.AggregatorConfig()
     help="Number of processes to use.",
 )
 @click.option(
-    "--complete/--incomplete",
-    "assume_complete",
-    default=_AGGREGATOR_DEFAULTS.assume_complete,
+    "--incomplete/--complete",
+    "incomplete",
+    default=_AGGREGATOR_DEFAULTS.incomplete,
     help="Whether execution has completed (and failures cannot be retried).",
 )
 @click.option(
@@ -249,6 +249,14 @@ _AGGREGATOR_DEFAULTS = aggregator.AggregatorConfig()
     default=_AGGREGATOR_DEFAULTS.mock_storage_classes,
     help="Enable support for storage classes created by the lsst.pipe.base.tests.mocks package.",
 )
+@click.option(
+    "--promise-ingest-graph/--no-promise-ingest-graph",
+    default=_AGGREGATOR_DEFAULTS.promise_ingest_graph,
+    help=(
+        "Promise to run 'butler ingest-graph' later, allowing aggregate-graph "
+        "to skip metadata/log/config ingestion for now."
+    ),
+)
 def aggregate_graph(predicted_graph: str, repo: str, **kwargs: Any) -> None:
     """Scan for quantum graph's outputs to gather provenance, ingest datasets
     into the central butler repository, and delete datasets that are no
@@ -268,3 +276,33 @@ def aggregate_graph(predicted_graph: str, repo: str, **kwargs: Any) -> None:
         # When this exception is raised, we'll have already logged the relevant
         # traceback from a separate worker.
         raise click.ClickException(str(err)) from None
+
+
+@click.command(
+    short_help="Ingest a provenance quantum graph into a butler, finalizing a RUN collection.",
+    cls=ButlerCommand,
+)
+@repo_argument(required=True, help="Path or alias for the butler repository.")
+@click.argument("provenance_graph", required=False)
+@transfer_option(default="move")
+@click.option("--batch-size", default=10000, help="How many datasets to process in each transaction.")
+@click.option(
+    "--output-run",
+    default=None,
+    help=(
+        "Name of the output RUN collection.  Must be provided if the provenance graph is not"
+        " provided (so the graph can be found in the butler)."
+    ),
+)
+def ingest_graph(
+    *,
+    repo: str,
+    provenance_graph: str | None,
+    transfer: str | None,
+    batch_size: int,
+    output_run: str | None,
+) -> None:
+    """Ingest a provenance graph into a butler repository."""
+    from ...quantum_graph.ingest_graph import ingest_graph as ingest_graph_py
+
+    ingest_graph_py(repo, provenance_graph, transfer=transfer, batch_size=batch_size, output_run=output_run)

lsst/pipe/base/connectionTypes.py

@@ -41,35 +41,36 @@ from lsst.utils.introspection import find_outside_stacklevel
 
 @dataclasses.dataclass(frozen=True)
 class BaseConnection:
-    """Base class used for declaring `PipelineTask` connections.
-
-    Attributes
-    ----------
-    name : `str`
-        The name used to identify the dataset type.
-    storageClass : `str`
-        The storage class used when (un)/persisting the dataset type.
-    multiple : `bool`
-        Indicates if this connection should expect to contain multiple objects
-        of the given dataset type.  Tasks with more than one connection with
-        ``multiple=True`` with the same dimensions may want to implement
-        `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
-        consistent (i.e. zip-iterable) in `PipelineTask.runQuantum()` and
-        notify the execution system as early as possible of outputs that will
-        not be produced because the corresponding input is missing.
-    deprecated : `str`, optional
-        A description of why this connection is deprecated, including the
-        version after which it may be removed.
-
-        If not `None`, the string is appended to the docstring for this
-        connection and the corresponding config Field.
-    """
+    """Base class used for declaring `PipelineTask` connections."""
 
     name: str
+    """The name used to identify the dataset type."""
+
     storageClass: str
+    """The storage class used when (un)/persisting the dataset type."""
+
     doc: str = ""
+    """Documentation for this connection."""
+
     multiple: bool = False
+    """Indicates if this connection should expect to contain multiple objects
+    of the given dataset type.
+
+    Tasks with more than one connection with ``multiple=True`` with the same
+    dimensions may want to implement `.PipelineTaskConnections.adjustQuantum`
+    to ensure those datasets are consistent (i.e. zip-iterable) in
+    `PipelineTask.runQuantum()` and notify the execution system as early as
+    possible of outputs that will not be produced because the corresponding
+    input is missing.
+    """
+
     deprecated: str | None = dataclasses.field(default=None, kw_only=True)
+    """A description of why this connection is deprecated, including the
+    version after which it may be removed.
+
+    If not `None`, the string is appended to the docstring for this
+    connection and the corresponding config Field.
+    """
 
     _connection_type_set: ClassVar[str]
     _deprecation_context: str = ""
@@ -110,32 +111,15 @@ class BaseConnection:
 class DimensionedConnection(BaseConnection):
     """Class used for declaring PipelineTask connections that includes
     dimensions.
-
-    Attributes
-    ----------
-    name : `str`
-        The name used to identify the dataset type.
-    storageClass : `str`
-        The storage class used when (un)/persisting the dataset type.
-    multiple : `bool`
-        Indicates if this connection should expect to contain multiple objects
-        of the given dataset type.  Tasks with more than one connection with
-        ``multiple=True`` with the same dimensions may want to implement
-        `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
-        consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
-        the execution system as early as possible of outputs that will not be
-        produced because the corresponding input is missing.
-    dimensions : iterable of `str`
-        The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
-        to identify the dataset type identified by the specified name.
-    isCalibration : `bool`, optional
-        `True` if this dataset type may be included in CALIBRATION-type
-        collections to associate it with a validity range, `False` (default)
-        otherwise.
     """
 
     dimensions: Iterable[str] = ()
+    """The keys of the butler data coordinates for this dataset type."""
+
     isCalibration: bool = False
+    """ `True` if this dataset type may be included in
+    `~lsst.daf.butler.CollectionType.CALIBRATION` collections to associate it
+    with a validity range, `False` (default) otherwise."""
 
     def __post_init__(self):
         super().__post_init__()
@@ -151,39 +135,6 @@ class DimensionedConnection(BaseConnection):
 class BaseInput(DimensionedConnection):
     """Class used for declaring PipelineTask input connections.
 
-    Attributes
-    ----------
-    name : `str`
-        The default name used to identify the dataset type.
-    storageClass : `str`
-        The storage class used when (un)/persisting the dataset type.
-    multiple : `bool`
-        Indicates if this connection should expect to contain multiple objects
-        of the given dataset type.  Tasks with more than one connection with
-        ``multiple=True`` with the same dimensions may want to implement
-        `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
-        consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
-        the execution system as early as possible of outputs that will not be
-        produced because the corresponding input is missing.
-    dimensions : iterable of `str`
-        The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
-        to identify the dataset type identified by the specified name.
-    deferLoad : `bool`
-        Indicates that this dataset type will be loaded as a
-        `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
-        object to load the object at a later time.
-    minimum : `bool`
-        Minimum number of datasets required for this connection, per quantum.
-        This is checked in the base implementation of
-        `.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if
-        the minimum is not met for `Input` connections (causing the quantum to
-        be pruned, skipped, or never created, depending on the context), and
-        `FileNotFoundError` for `PrerequisiteInput` connections (causing
-        QuantumGraph generation to fail).  `PipelineTask` implementations may
-        provide custom `~.PipelineTaskConnections.adjustQuantum`
-        implementations for more fine-grained or configuration-driven
-        constraints, as long as they are compatible with this minium.
-
     Raises
     ------
     TypeError
@@ -194,7 +145,24 @@ class BaseInput(DimensionedConnection):
     """
 
     deferLoad: bool = False
+    """Whether this dataset type will be loaded as a
+    `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
+    object to load the object at a later time.
+    """
+
     minimum: int = 1
+    """Minimum number of datasets required for this connection, per quantum.
+
+    This is checked in the base implementation of
+    `.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if the
+    minimum is not met for `Input` connections (causing the quantum to be
+    pruned, skipped, or never created, depending on the context), and
+    `FileNotFoundError` for `PrerequisiteInput` connections (causing
+    QuantumGraph generation to fail).  `PipelineTask` implementations may
+    provide custom `~.PipelineTaskConnections.adjustQuantum` implementations
+    for more fine-grained or configuration-driven constraints, as long as they
+    are compatible with this minimum.
+    """
 
     def __post_init__(self) -> None:
         super().__post_init__()
@@ -206,56 +174,6 @@ class BaseInput(DimensionedConnection):
 class Input(BaseInput):
     """Class used for declaring PipelineTask input connections.
 
-    Attributes
-    ----------
-    name : `str`
-        The default name used to identify the dataset type.
-    storageClass : `str`
-        The storage class used when (un)/persisting the dataset type.
-    multiple : `bool`
-        Indicates if this connection should expect to contain multiple objects
-        of the given dataset type.  Tasks with more than one connection with
-        ``multiple=True`` with the same dimensions may want to implement
-        `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
-        consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
-        the execution system as early as possible of outputs that will not be
-        produced because the corresponding input is missing.
-    dimensions : iterable of `str`
-        The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
-        to identify the dataset type identified by the specified name.
-    deferLoad : `bool`
-        Indicates that this dataset type will be loaded as a
-        `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
-        object to load the object at a later time.
-    minimum : `bool`
-        Minimum number of datasets required for this connection, per quantum.
-        This is checked in the base implementation of
-        `.PipelineTaskConnections.adjustQuantum`, which raises `NoWorkFound` if
-        the minimum is not met for `Input` connections (causing the quantum to
-        be pruned, skipped, or never created, depending on the context), and
-        `FileNotFoundError` for `PrerequisiteInput` connections (causing
-        QuantumGraph generation to fail).  `PipelineTask` implementations may
-        provide custom `~.PipelineTaskConnections.adjustQuantum`
-        implementations for more fine-grained or configuration-driven
-        constraints, as long as they are compatible with this minium.
-    deferGraphConstraint : `bool`, optional
-        If `True`, do not include this dataset type's existence in the initial
-        query that starts the QuantumGraph generation process.  This can be
-        used to make QuantumGraph generation faster by avoiding redundant
-        datasets, and in certain cases it can (along with careful attention to
-        which tasks are included in the same QuantumGraph) be used to work
-        around the QuantumGraph generation algorithm's inflexible handling of
-        spatial overlaps.  This option has no effect when the connection is not
-        an overall input of the pipeline (or subset thereof) for which a graph
-        is being created, and it never affects the ordering of quanta.
-    deferBinding : `bool`, optional
-        If `True`, the dataset will not be automatically included in
-        the pipeline graph, ``deferGraphConstraint`` is implied.
-        The custom QuantumGraphBuilder is required to bind it and add a
-        corresponding edge to the pipeline graph.
-        This option allows to have the same dataset type as both
-        input and output of a quantum.
-
     Raises
     ------
     TypeError
@@ -266,8 +184,27 @@ class Input(BaseInput):
     """
 
     deferGraphConstraint: bool = False
+    """If `True`, do not include this dataset type's existence in the initial
+    query that starts the QuantumGraph generation process.
+
+    This can be used to make QuantumGraph generation faster by avoiding
+    redundant datasets, and in certain cases it can (along with careful
+    attention to which tasks are included in the same QuantumGraph) be used to
+    work around the QuantumGraph generation algorithm's inflexible handling of
+    spatial overlaps.  This option has no effect when the connection is not an
+    overall input of the pipeline (or subset thereof) for which a graph is
+    being created, and it never affects the ordering of quanta.
+    """
 
     deferBinding: bool = False
+    """If `True`, the dataset will not be automatically included in the
+    pipeline graph (``deferGraphConstraint=True`` is implied).
+
+    A custom `~.quantum_graph_builder.QuantumGraphBuilder` is required to bind
+    it and add a corresponding edge to the pipeline graph. This option allows
+    the same dataset type to be used as both an input and an output of a
+    quantum.
+    """
 
     _connection_type_set: ClassVar[str] = "inputs"
 
@@ -276,38 +213,6 @@ class Input(BaseInput):
 class PrerequisiteInput(BaseInput):
     """Class used for declaring PipelineTask prerequisite connections.
 
-    Attributes
-    ----------
-    name : `str`
-        The default name used to identify the dataset type.
-    storageClass : `str`
-        The storage class used when (un)/persisting the dataset type.
-    multiple : `bool`
-        Indicates if this connection should expect to contain multiple objects
-        of the given dataset type.  Tasks with more than one connection with
-        ``multiple=True`` with the same dimensions may want to implement
-        `.PipelineTaskConnections.adjustQuantum` to ensure those datasets are
-        consistent (i.e. zip-iterable) in `PipelineTask.runQuantum` and notify
-        the execution system as early as possible of outputs that will not be
-        produced because the corresponding input is missing.
-    dimensions : iterable of `str`
-        The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
-        to identify the dataset type identified by the specified name.
-    minimum : `bool`
-        Minimum number of datasets required for this connection, per quantum.
-        This is checked in the base implementation of
-        `.PipelineTaskConnections.adjustQuantum`, which raises
-        `FileNotFoundError` (causing QuantumGraph generation to fail).
-        `PipelineTask` implementations may provide custom
-        `~.PipelineTaskConnections.adjustQuantum` implementations for more
-        fine-grained or configuration-driven constraints, as long as they are
-        compatible with this minium.
-    lookupFunction : `typing.Callable`, optional
-        An optional callable function that will look up PrerequisiteInputs
-        using the DatasetType, registry, quantum dataId, and input collections
-        passed to it. If no function is specified, the default temporal spatial
-        lookup will be used.
-
     Raises
     ------
     TypeError
@@ -342,6 +247,13 @@ class PrerequisiteInput(BaseInput):
     lookupFunction: (
         Callable[[DatasetType, Registry, DataCoordinate, Sequence[str]], Iterable[DatasetRef]] | None
     ) = None
+    """An optional callable function that will look up PrerequisiteInputs
+    using the DatasetType, registry, quantum dataId, and input collections
+    passed to it.
+
+    If no function is specified, the default temporal/spatial lookup will be
+    used.
+    """
 
     _connection_type_set: ClassVar[str] = "prerequisiteInputs"
 

lsst/pipe/base/connections.py

@@ -495,15 +495,12 @@ class DeferredDatasetRef:
     """A wrapper class for `~lsst.daf.butler.DatasetRef` that indicates that a
     `PipelineTask` should receive a `~lsst.daf.butler.DeferredDatasetHandle`
     instead of an in-memory dataset.
-
-    Attributes
-    ----------
-    datasetRef : `lsst.daf.butler.DatasetRef`
-        The `lsst.daf.butler.DatasetRef` that will be eventually used to
-        resolve a dataset.
     """
 
     datasetRef: DatasetRef
+    """The `lsst.daf.butler.DatasetRef` that will be eventually used to
+    resolve a dataset.
+    """
 
     def __getattr__(self, name: str) -> Any:
         # make sure reduce is called on DeferredDatasetRef and not on

lsst/pipe/base/execution_reports.py

@@ -299,11 +299,6 @@ class QuantumGraphExecutionReport:
     produced DatasetTypes for each task. This report can be output as a
     dictionary or a yaml file.
 
-    Attributes
-    ----------
-    tasks : `dict`
-        A dictionary of TaskExecutionReports by task label.
-
     See Also
     --------
     TaskExecutionReport : A task report.

lsst/pipe/base/log_capture.py

@@ -163,7 +163,9 @@ class LogCapture:
         return cls(butler, butler)
 
     @contextmanager
-    def capture_logging(self, task_node: TaskNode, /, quantum: Quantum) -> Iterator[_LogCaptureContext]:
+    def capture_logging(
+        self, task_node: TaskNode, /, quantum: Quantum, records: ButlerLogRecords | None = None
+    ) -> Iterator[_LogCaptureContext]:
         """Configure logging system to capture logs for execution of this task.
 
         Parameters
@@ -172,6 +174,9 @@ class LogCapture:
             The task definition.
         quantum : `~lsst.daf.butler.Quantum`
             Single Quantum instance.
+        records : `lsst.daf.butler.logging.ButlerLogRecords`, optional
+            Log record container to append to and save.  If provided, streaming
+            mode is disabled (since we'll be saving logs in memory anyway).
 
         Notes
         -----
@@ -213,7 +218,7 @@ class LogCapture:
                 ) from exc
             # Either accumulate into ButlerLogRecords or stream JSON records to
             # file and ingest that (ingest is possible only with full butler).
-            if self.stream_json_logs and self.full_butler is not None:
+            if self.stream_json_logs and self.full_butler is not None and records is None:
                 with TemporaryForIngest(self.full_butler, ref) as temporary:
                     log_handler_file = FileHandler(temporary.ospath)
                     log_handler_file.setFormatter(JsonLogFormatter())
@@ -236,7 +241,7 @@ class LogCapture:
                             temporary.ingest()
 
             else:
-                log_handler_memory = ButlerLogRecordHandler()
+                log_handler_memory = ButlerLogRecordHandler(records)
                 logging.getLogger().addHandler(log_handler_memory)
 
                 try:
@@ -255,7 +260,6 @@ class LogCapture:
                     logging.getLogger().removeHandler(log_handler_memory)
                     if ctx.store:
                         self._store_log_records(quantum, log_dataset_name, log_handler_memory)
-                    log_handler_memory.records.clear()
 
         else:
             with ButlerMDC.set_mdc(mdc):