lsst-pipe-base 30.2026.200__py3-none-any.whl → 30.2026.400__py3-none-any.whl

lsst/pipe/base/quantum_graph/_provenance.py

@@ -35,19 +35,25 @@ __all__ = (
     "ProvenanceLogRecordsModel",
     "ProvenanceQuantumGraph",
     "ProvenanceQuantumGraphReader",
+    "ProvenanceQuantumGraphWriter",
     "ProvenanceQuantumInfo",
     "ProvenanceQuantumModel",
+    "ProvenanceQuantumScanData",
+    "ProvenanceQuantumScanModels",
+    "ProvenanceQuantumScanStatus",
     "ProvenanceTaskMetadataModel",
 )
 
 
 import dataclasses
+import enum
+import itertools
 import sys
 import uuid
 from collections import Counter
-from collections.abc import Iterable, Iterator, Mapping
-from contextlib import contextmanager
-from typing import TYPE_CHECKING, Any, TypedDict, TypeVar
+from collections.abc import Callable, Iterable, Iterator, Mapping
+from contextlib import ExitStack, contextmanager
+from typing import TYPE_CHECKING, Any, TypeAlias, TypedDict, TypeVar
 
 import astropy.table
 import networkx
@@ -57,15 +63,21 @@ import pydantic
 from lsst.daf.butler import DataCoordinate
 from lsst.daf.butler.logging import ButlerLogRecord, ButlerLogRecords
 from lsst.resources import ResourcePathExpression
+from lsst.utils.iteration import ensure_iterable
+from lsst.utils.logging import LsstLogAdapter, getLogger
 from lsst.utils.packages import Packages
 
+from .. import automatic_connection_constants as acc
 from .._status import ExceptionInfo, QuantumAttemptStatus, QuantumSuccessCaveats
 from .._task_metadata import TaskMetadata
+from ..log_capture import _ExecutionLogRecordsExtra
+from ..log_on_close import LogOnClose
 from ..pipeline_graph import PipelineGraph, TaskImportMode, TaskInitNode
 from ..resource_usage import QuantumResourceUsage
 from ._common import (
     BaseQuantumGraph,
     BaseQuantumGraphReader,
+    BaseQuantumGraphWriter,
     ConnectionName,
     DataCoordinateValues,
     DatasetInfo,
@@ -74,8 +86,26 @@ from ._common import (
     QuantumInfo,
     TaskLabel,
 )
-from ._multiblock import MultiblockReader
-from ._predicted import PredictedDatasetModel, PredictedQuantumDatasetsModel
+from ._multiblock import Compressor, MultiblockReader, MultiblockWriter
+from ._predicted import (
+    PredictedDatasetModel,
+    PredictedQuantumDatasetsModel,
+    PredictedQuantumGraph,
+    PredictedQuantumGraphComponents,
+)
+
+# Sphinx needs imports for type annotations of base class members.
+if "sphinx" in sys.modules:
+    import zipfile  # noqa: F401
+
+    from ._multiblock import AddressReader, Decompressor  # noqa: F401
+
+
+_T = TypeVar("_T")
+
+LoopWrapper: TypeAlias = Callable[[Iterable[_T]], Iterable[_T]]
+
+_LOG = getLogger(__file__)
 
 DATASET_ADDRESS_INDEX = 0
 QUANTUM_ADDRESS_INDEX = 1
@@ -87,7 +117,9 @@ QUANTUM_MB_NAME = "quanta"
 LOG_MB_NAME = "logs"
 METADATA_MB_NAME = "metadata"
 
-_I = TypeVar("_I", bound=uuid.UUID | int)
+
+def pass_through(arg: _T) -> _T:
+    return arg
 
 
 class ProvenanceDatasetInfo(DatasetInfo):
@@ -161,6 +193,12 @@ class ProvenanceQuantumInfo(QuantumInfo):
     failure.
     """
 
+    metadata_id: uuid.UUID
+    """ID of this quantum's metadata dataset."""
+
+    log_id: uuid.UUID
+    """ID of this quantum's log dataset."""
+
 
 class ProvenanceInitQuantumInfo(TypedDict):
     """A typed dictionary that annotates the attributes of the NetworkX graph
@@ -187,6 +225,9 @@ class ProvenanceInitQuantumInfo(TypedDict):
     pipeline_node: TaskInitNode
     """Node in the pipeline graph for this task's init-only step."""
 
+    config_id: uuid.UUID
+    """ID of this task's config dataset."""
+
 
 class ProvenanceDatasetModel(PredictedDatasetModel):
     """Data model for the datasets in a provenance quantum graph file."""
@@ -621,6 +662,8 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
             resource_usage=last_attempt.resource_usage,
             attempts=self.attempts,
         )
+        graph._quanta_by_task_label[self.task_label][data_id] = self.quantum_id
+        graph._quantum_only_xgraph.add_node(self.quantum_id, **graph._bipartite_xgraph.nodes[self.quantum_id])
         for connection_name, dataset_ids in self.inputs.items():
             read_edge = task_node.get_input_edge(connection_name)
             for dataset_id in dataset_ids:
@@ -630,6 +673,30 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
                 ).append(read_edge)
         for connection_name, dataset_ids in self.outputs.items():
             write_edge = task_node.get_output_edge(connection_name)
+            if connection_name == acc.METADATA_OUTPUT_CONNECTION_NAME:
+                graph._bipartite_xgraph.add_node(
+                    dataset_ids[0],
+                    data_id=data_id,
+                    dataset_type_name=write_edge.dataset_type_name,
+                    pipeline_node=graph.pipeline_graph.dataset_types[write_edge.dataset_type_name],
+                    run=graph.header.output_run,
+                    produced=last_attempt.status.has_metadata,
+                )
+                graph._datasets_by_type[write_edge.dataset_type_name][data_id] = dataset_ids[0]
+                graph._bipartite_xgraph.nodes[self.quantum_id]["metadata_id"] = dataset_ids[0]
+                graph._quantum_only_xgraph.nodes[self.quantum_id]["metadata_id"] = dataset_ids[0]
+            if connection_name == acc.LOG_OUTPUT_CONNECTION_NAME:
+                graph._bipartite_xgraph.add_node(
+                    dataset_ids[0],
+                    data_id=data_id,
+                    dataset_type_name=write_edge.dataset_type_name,
+                    pipeline_node=graph.pipeline_graph.dataset_types[write_edge.dataset_type_name],
+                    run=graph.header.output_run,
+                    produced=last_attempt.status.has_log,
+                )
+                graph._datasets_by_type[write_edge.dataset_type_name][data_id] = dataset_ids[0]
+                graph._bipartite_xgraph.nodes[self.quantum_id]["log_id"] = dataset_ids[0]
+                graph._quantum_only_xgraph.nodes[self.quantum_id]["log_id"] = dataset_ids[0]
             for dataset_id in dataset_ids:
                 graph._bipartite_xgraph.add_edge(
                     self.quantum_id,
@@ -638,8 +705,6 @@ class ProvenanceQuantumModel(pydantic.BaseModel):
                     # There can only be one pipeline edge for an output.
                     pipeline_edges=[write_edge],
                 )
-        graph._quanta_by_task_label[self.task_label][data_id] = self.quantum_id
-        graph._quantum_only_xgraph.add_node(self.quantum_id, **graph._bipartite_xgraph.nodes[self.quantum_id])
         for dataset_id in graph._bipartite_xgraph.predecessors(self.quantum_id):
             for upstream_quantum_id in graph._bipartite_xgraph.predecessors(dataset_id):
                 graph._quantum_only_xgraph.add_edge(upstream_quantum_id, self.quantum_id)
@@ -778,6 +843,15 @@ class ProvenanceInitQuantumModel(pydantic.BaseModel):
             ).append(read_edge)
         for connection_name, dataset_id in self.outputs.items():
             write_edge = task_init_node.get_output_edge(connection_name)
+            graph._bipartite_xgraph.add_node(
+                dataset_id,
+                data_id=empty_data_id,
+                dataset_type_name=write_edge.dataset_type_name,
+                pipeline_node=graph.pipeline_graph.dataset_types[write_edge.dataset_type_name],
+                run=graph.header.output_run,
+                produced=True,
+            )
+            graph._datasets_by_type[write_edge.dataset_type_name][empty_data_id] = dataset_id
             graph._bipartite_xgraph.add_edge(
                 self.quantum_id,
                 dataset_id,
@@ -785,6 +859,8 @@ class ProvenanceInitQuantumModel(pydantic.BaseModel):
                 # There can only be one pipeline edge for an output.
                 pipeline_edges=[write_edge],
             )
+            if write_edge.connection_name == acc.CONFIG_INIT_OUTPUT_CONNECTION_NAME:
+                graph._bipartite_xgraph.nodes[self.quantum_id]["config_id"] = dataset_id
         graph._init_quanta[self.task_label] = self.quantum_id
 
     # Work around the fact that Sphinx chokes on Pydantic docstring formatting,
@@ -969,6 +1045,8 @@ class ProvenanceQuantumGraph(BaseQuantumGraph):
         types in the pipeline graph are included, even if none of their
         datasets were loaded (i.e. nested mappings may be empty).
 
+        Reading a quantum also populates its log and metadata datasets.
+
         The returned object may be an internal dictionary; as the type
         annotation indicates, it should not be modified in place.
         """
@@ -1007,7 +1085,8 @@ class ProvenanceQuantumGraph(BaseQuantumGraph):
         `ProvenanceQuantumGraphReader.read_quanta`) or datasets (via
         `ProvenanceQuantumGraphReader.read_datasets`) will load those nodes
         with full attributes and edges to adjacent nodes with no attributes.
-        Loading quanta necessary to populate edge attributes.
+        Loading quanta is necessary to populate edge attributes.
+        Reading a quantum also populates its log and metadata datasets.
 
         Node attributes are described by the
         `ProvenanceQuantumInfo`, `ProvenanceInitQuantumInfo`, and
@@ -1079,10 +1158,6 @@ class ProvenanceQuantumGraph(BaseQuantumGraph):
         """Construct an `astropy.table.Table` with counts for each exception
         type raised by each task.
 
-        At present this only includes information from partial-outputs-error
-        successes, since exception information for failures is not tracked.
-        This may change in the future.
-
         Returns
         -------
         table : `astropy.table.Table`
@@ -1269,19 +1344,19 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
                     # also have other outstanding reference holders).
                     continue
                 node._add_to_graph(self.graph)
-            return
-        with MultiblockReader.open_in_zip(self.zf, mb_name, int_size=self.header.int_size) as mb_reader:
-            for node_id_or_index in nodes:
-                address_row = self.address_reader.find(node_id_or_index)
-                if "pipeline_node" in self.graph._bipartite_xgraph.nodes.get(address_row.key, {}):
-                    # Use the old node to reduce memory usage (since it might
-                    # also have other outstanding reference holders).
-                    continue
-                node = mb_reader.read_model(
-                    address_row.addresses[address_index], model_type, self.decompressor
-                )
-                if node is not None:
-                    node._add_to_graph(self.graph)
+        else:
+            with MultiblockReader.open_in_zip(self.zf, mb_name, int_size=self.header.int_size) as mb_reader:
+                for node_id_or_index in nodes:
+                    address_row = self.address_reader.find(node_id_or_index)
+                    if "pipeline_node" in self.graph._bipartite_xgraph.nodes.get(address_row.key, {}):
+                        # Use the old node to reduce memory usage (since it
+                        # might also have other outstanding reference holders).
+                        continue
+                    node = mb_reader.read_model(
+                        address_row.addresses[address_index], model_type, self.decompressor
+                    )
+                    if node is not None:
+                        node._add_to_graph(self.graph)
 
     def fetch_logs(self, nodes: Iterable[uuid.UUID]) -> dict[uuid.UUID, list[ButlerLogRecords | None]]:
         """Fetch log datasets.
@@ -1352,3 +1427,629 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
         """Fetch package version information."""
         data = self._read_single_block_raw("packages")
         return Packages.fromBytes(data, format="json")
+
+
+class ProvenanceQuantumGraphWriter:
+    """A struct of low-level writer objects for the main components of a
+    provenance quantum graph.
+
+    Parameters
+    ----------
+    output_path : `str`
+        Path to write the graph to.
+    exit_stack : `contextlib.ExitStack`
+        Object that can be used to manage multiple context managers.
+    log_on_close : `LogOnClose`
+        Factory for context managers that log when closed.
+    predicted : `.PredictedQuantumGraphComponents`
+        Components of the predicted graph.
+    zstd_level : `int`, optional
+        Compression level.
+    cdict_data : `bytes` or `None`, optional
+        Bytes representation of the compression dictionary used by the
+        compressor.
+    loop_wrapper : `~collections.abc.Callable`, optional
+        A callable that takes an iterable and returns an equivalent one, to be
+        used in all potentially-large loops.  This can be used to add progress
+        reporting or check for cancelation signals.
+    log : `LsstLogAdapter`, optional
+        Logger to use for debug messages.
+    """
+
+    def __init__(
+        self,
+        output_path: str,
+        *,
+        exit_stack: ExitStack,
+        log_on_close: LogOnClose,
+        predicted: PredictedQuantumGraphComponents | PredictedQuantumGraph,
+        zstd_level: int = 10,
+        cdict_data: bytes | None = None,
+        loop_wrapper: LoopWrapper = pass_through,
+        log: LsstLogAdapter | None = None,
+    ) -> None:
+        header = predicted.header.model_copy()
+        header.graph_type = "provenance"
+        if log is None:
+            log = _LOG
+        self.log = log
+        self._base_writer = exit_stack.enter_context(
+            log_on_close.wrap(
+                BaseQuantumGraphWriter.open(
+                    output_path,
+                    header,
+                    predicted.pipeline_graph,
+                    address_filename="nodes",
+                    zstd_level=zstd_level,
+                    cdict_data=cdict_data,
+                ),
+                "Finishing writing provenance quantum graph.",
+            )
+        )
+        self._base_writer.address_writer.addresses = [{}, {}, {}, {}]
+        self._log_writer = exit_stack.enter_context(
+            log_on_close.wrap(
+                MultiblockWriter.open_in_zip(
+                    self._base_writer.zf, LOG_MB_NAME, header.int_size, use_tempfile=True
+                ),
+                "Copying logs into zip archive.",
+            ),
+        )
+        self._base_writer.address_writer.addresses[LOG_ADDRESS_INDEX] = self._log_writer.addresses
+        self._metadata_writer = exit_stack.enter_context(
+            log_on_close.wrap(
+                MultiblockWriter.open_in_zip(
+                    self._base_writer.zf, METADATA_MB_NAME, header.int_size, use_tempfile=True
+                ),
+                "Copying metadata into zip archive.",
+            )
+        )
+        self._base_writer.address_writer.addresses[METADATA_ADDRESS_INDEX] = self._metadata_writer.addresses
+        self._dataset_writer = exit_stack.enter_context(
+            log_on_close.wrap(
+                MultiblockWriter.open_in_zip(
+                    self._base_writer.zf, DATASET_MB_NAME, header.int_size, use_tempfile=True
+                ),
+                "Copying dataset provenance into zip archive.",
+            )
+        )
+        self._base_writer.address_writer.addresses[DATASET_ADDRESS_INDEX] = self._dataset_writer.addresses
+        self._quantum_writer = exit_stack.enter_context(
+            log_on_close.wrap(
+                MultiblockWriter.open_in_zip(
+                    self._base_writer.zf, QUANTUM_MB_NAME, header.int_size, use_tempfile=True
+                ),
+                "Copying quantum provenance into zip archive.",
+            )
+        )
+        self._base_writer.address_writer.addresses[QUANTUM_ADDRESS_INDEX] = self._quantum_writer.addresses
+        self._init_predicted_quanta(predicted)
+        self._populate_xgraph_and_inputs(loop_wrapper)
+        self._existing_init_outputs: set[uuid.UUID] = set()
+
+    def _init_predicted_quanta(
+        self, predicted: PredictedQuantumGraph | PredictedQuantumGraphComponents
+    ) -> None:
+        self._predicted_init_quanta: list[PredictedQuantumDatasetsModel] = []
+        self._predicted_quanta: dict[uuid.UUID, PredictedQuantumDatasetsModel] = {}
+        if isinstance(predicted, PredictedQuantumGraph):
+            self._predicted_init_quanta.extend(predicted._init_quanta.values())
+            self._predicted_quanta.update(predicted._quantum_datasets)
+        else:
+            self._predicted_init_quanta.extend(predicted.init_quanta.root)
+            self._predicted_quanta.update(predicted.quantum_datasets)
+        self._predicted_quanta.update({q.quantum_id: q for q in self._predicted_init_quanta})
+
+    def _populate_xgraph_and_inputs(self, loop_wrapper: LoopWrapper = pass_through) -> None:
+        self._xgraph = networkx.DiGraph()
+        self._overall_inputs: dict[uuid.UUID, PredictedDatasetModel] = {}
+        output_dataset_ids: set[uuid.UUID] = set()
+        for predicted_quantum in loop_wrapper(self._predicted_quanta.values()):
+            if not predicted_quantum.task_label:
+                # Skip the 'packages' producer quantum.
+                continue
+            output_dataset_ids.update(predicted_quantum.iter_output_dataset_ids())
+        for predicted_quantum in loop_wrapper(self._predicted_quanta.values()):
+            if not predicted_quantum.task_label:
+                # Skip the 'packages' producer quantum.
+                continue
+            for predicted_input in itertools.chain.from_iterable(predicted_quantum.inputs.values()):
+                self._xgraph.add_edge(predicted_input.dataset_id, predicted_quantum.quantum_id)
+                if predicted_input.dataset_id not in output_dataset_ids:
+                    self._overall_inputs.setdefault(predicted_input.dataset_id, predicted_input)
+            for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
+                self._xgraph.add_edge(predicted_quantum.quantum_id, predicted_output.dataset_id)
+
+    @property
+    def compressor(self) -> Compressor:
+        """Object that should be used to compress all JSON blocks."""
+        return self._base_writer.compressor
+
+    def write_packages(self) -> None:
+        """Write package version information to the provenance graph."""
+        packages = Packages.fromSystem(include_all=True)
+        data = packages.toBytes("json")
+        self._base_writer.write_single_block("packages", data)
+
+    def write_overall_inputs(self, loop_wrapper: LoopWrapper = pass_through) -> None:
+        """Write provenance for overall-input datasets.
+
+        Parameters
+        ----------
+        loop_wrapper : `~collections.abc.Callable`, optional
+            A callable that takes an iterable and returns an equivalent one, to
+            be used in all potentially-large loops.  This can be used to add
+            progress reporting or check for cancelation signals.
+        """
+        for predicted_input in loop_wrapper(self._overall_inputs.values()):
+            if predicted_input.dataset_id not in self._dataset_writer.addresses:
+                self._dataset_writer.write_model(
+                    predicted_input.dataset_id,
+                    ProvenanceDatasetModel.from_predicted(
+                        predicted_input,
+                        producer=None,
+                        consumers=self._xgraph.successors(predicted_input.dataset_id),
+                    ),
+                    self.compressor,
+                )
+        del self._overall_inputs
+
+    def write_init_outputs(self, assume_existence: bool = True) -> None:
+        """Write provenance for init-output datasets and init-quanta.
+
+        Parameters
+        ----------
+        assume_existence : `bool`, optional
+            If `True`, just assume all init-outputs exist.
+        """
+        init_quanta = ProvenanceInitQuantaModel()
+        for predicted_init_quantum in self._predicted_init_quanta:
+            if not predicted_init_quantum.task_label:
+                # Skip the 'packages' producer quantum.
+                continue
+            for predicted_output in itertools.chain.from_iterable(predicted_init_quantum.outputs.values()):
+                provenance_output = ProvenanceDatasetModel.from_predicted(
+                    predicted_output,
+                    producer=predicted_init_quantum.quantum_id,
+                    consumers=self._xgraph.successors(predicted_output.dataset_id),
+                )
+                provenance_output.produced = assume_existence or (
+                    provenance_output.dataset_id in self._existing_init_outputs
+                )
+                self._dataset_writer.write_model(
+                    provenance_output.dataset_id, provenance_output, self.compressor
+                )
+            init_quanta.root.append(ProvenanceInitQuantumModel.from_predicted(predicted_init_quantum))
+        self._base_writer.write_single_model("init_quanta", init_quanta)
+
+    def write_quantum_provenance(
+        self, quantum_id: uuid.UUID, metadata: TaskMetadata | None, logs: ButlerLogRecords | None
+    ) -> None:
+        """Gather and write provenance for a quantum.
+
+        Parameters
+        ----------
+        quantum_id : `uuid.UUID`
+            Unique ID for the quantum.
+        metadata : `..TaskMetadata` or `None`
+            Task metadata.
+        logs : `lsst.daf.butler.logging.ButlerLogRecords` or `None`
+            Task logs.
+        """
+        predicted_quantum = self._predicted_quanta[quantum_id]
+        provenance_models = ProvenanceQuantumScanModels.from_metadata_and_logs(
+            predicted_quantum, metadata, logs, incomplete=False
+        )
+        scan_data = provenance_models.to_scan_data(predicted_quantum, compressor=self.compressor)
+        self.write_scan_data(scan_data)
+
+    def write_scan_data(self, scan_data: ProvenanceQuantumScanData) -> None:
+        """Write the output of a quantum provenance scan to disk.
+
+        Parameters
+        ----------
+        scan_data : `ProvenanceQuantumScanData`
+            Result of a quantum provenance scan.
+        """
+        if scan_data.status is ProvenanceQuantumScanStatus.INIT:
+            self.log.debug("Handling init-output scan for %s.", scan_data.quantum_id)
+            self._existing_init_outputs.update(scan_data.existing_outputs)
+            return
+        self.log.debug("Handling quantum scan for %s.", scan_data.quantum_id)
+        # We shouldn't need this predicted quantum after this method runs; pop
+        # from the dict it in the hopes that'll free up some memory when we're
+        # done.
+        predicted_quantum = self._predicted_quanta.pop(scan_data.quantum_id)
+        outputs: dict[uuid.UUID, bytes] = {}
+        for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
+            provenance_output = ProvenanceDatasetModel.from_predicted(
+                predicted_output,
+                producer=predicted_quantum.quantum_id,
+                consumers=self._xgraph.successors(predicted_output.dataset_id),
+            )
+            provenance_output.produced = provenance_output.dataset_id in scan_data.existing_outputs
+            outputs[provenance_output.dataset_id] = self.compressor.compress(
+                provenance_output.model_dump_json().encode()
+            )
+        if not scan_data.quantum:
+            scan_data.quantum = (
+                ProvenanceQuantumModel.from_predicted(predicted_quantum).model_dump_json().encode()
+            )
+            if scan_data.is_compressed:
+                scan_data.quantum = self.compressor.compress(scan_data.quantum)
+        if not scan_data.is_compressed:
+            scan_data.quantum = self.compressor.compress(scan_data.quantum)
+            if scan_data.metadata:
+                scan_data.metadata = self.compressor.compress(scan_data.metadata)
+            if scan_data.logs:
+                scan_data.logs = self.compressor.compress(scan_data.logs)
+        self.log.debug("Writing quantum %s.", scan_data.quantum_id)
+        self._quantum_writer.write_bytes(scan_data.quantum_id, scan_data.quantum)
+        for dataset_id, dataset_data in outputs.items():
+            self._dataset_writer.write_bytes(dataset_id, dataset_data)
+        if scan_data.metadata:
+            (metadata_output,) = predicted_quantum.outputs[acc.METADATA_OUTPUT_CONNECTION_NAME]
+            address = self._metadata_writer.write_bytes(scan_data.quantum_id, scan_data.metadata)
+            self._metadata_writer.addresses[metadata_output.dataset_id] = address
+        if scan_data.logs:
+            (log_output,) = predicted_quantum.outputs[acc.LOG_OUTPUT_CONNECTION_NAME]
+            address = self._log_writer.write_bytes(scan_data.quantum_id, scan_data.logs)
+            self._log_writer.addresses[log_output.dataset_id] = address
+
+
+class ProvenanceQuantumScanStatus(enum.Enum):
+    """Status enum for quantum scanning.
+
+    Note that this records the status for the *scanning* which is distinct
+    from the status of the quantum's execution.
+    """
+
+    INCOMPLETE = enum.auto()
+    """The quantum is not necessarily done running, and cannot be scanned
+    conclusively yet.
+    """
+
+    ABANDONED = enum.auto()
+    """The quantum's execution appears to have failed but we cannot rule out
+    the possibility that it could be recovered, but we've also waited long
+    enough (according to `ScannerTimeConfigDict.retry_timeout`) that it's time
+    to stop trying for now.
+
+    This state means `ProvenanceQuantumScanModels.from_metadata_and_logs` must
+    be run again with ``incomplete=False``.
+    """
+
+    SUCCESSFUL = enum.auto()
+    """The quantum was conclusively scanned and was executed successfully,
+    unblocking scans for downstream quanta.
+    """
+
+    FAILED = enum.auto()
+    """The quantum was conclusively scanned and failed execution, blocking
+    scans for downstream quanta.
+    """
+
+    BLOCKED = enum.auto()
+    """A quantum upstream of this one failed."""
+
+    INIT = enum.auto()
+    """Init quanta need special handling, because they don't have logs and
+    metadata.
+    """
+
+
+@dataclasses.dataclass
+class ProvenanceQuantumScanModels:
+    """A struct that represents provenance information for a single quantum."""
+
+    quantum_id: uuid.UUID
+    """Unique ID for the quantum."""
+
+    status: ProvenanceQuantumScanStatus = ProvenanceQuantumScanStatus.INCOMPLETE
+    """Combined status for the scan and the execution of the quantum."""
+
+    attempts: list[ProvenanceQuantumAttemptModel] = dataclasses.field(default_factory=list)
+    """Provenance information about each attempt to run the quantum."""
+
+    output_existence: dict[uuid.UUID, bool] = dataclasses.field(default_factory=dict)
+    """Unique IDs of the output datasets mapped to whether they were actually
+    produced.
+    """
+
+    metadata: ProvenanceTaskMetadataModel = dataclasses.field(default_factory=ProvenanceTaskMetadataModel)
+    """Task metadata information for each attempt.
+    """
+
+    logs: ProvenanceLogRecordsModel = dataclasses.field(default_factory=ProvenanceLogRecordsModel)
+    """Log records for each attempt.
+    """
+
+    @classmethod
+    def from_metadata_and_logs(
+        cls,
+        predicted: PredictedQuantumDatasetsModel,
+        metadata: TaskMetadata | None,
+        logs: ButlerLogRecords | None,
+        *,
+        incomplete: bool = False,
+    ) -> ProvenanceQuantumScanModels:
+        """Construct provenance information from task metadata and logs.
+
+        Parameters
+        ----------
+        predicted : `PredictedQuantumDatasetsModel`
+            Information about the predicted quantum.
+        metadata : `..TaskMetadata` or `None`
+            Task metadata.
+        logs : `lsst.daf.butler.logging.ButlerLogRecords` or `None`
+            Task logs.
+        incomplete : `bool`, optional
+            If `True`, treat execution failures as possibly-incomplete quanta
+            and do not fully process them; instead just set the status to
+            `ProvenanceQuantumScanStatus.ABANDONED` and return.
+
+        Returns
+        -------
+        scan_models : `ProvenanceQuantumScanModels`
+            Struct of models that describe quantum provenance.
+
+        Notes
+        -----
+        This method does not necessarily fully populate the `output_existence`
+        field; it does what it can given the information in the metadata and
+        logs, but the caller is responsible for filling in the existence status
+        for any predicted outputs that are not present at all in that `dict`.
+        """
+        self = ProvenanceQuantumScanModels(predicted.quantum_id)
+        last_attempt = ProvenanceQuantumAttemptModel()
+        self._process_logs(predicted, logs, last_attempt, incomplete=incomplete)
+        self._process_metadata(predicted, metadata, last_attempt, incomplete=incomplete)
+        if self.status is ProvenanceQuantumScanStatus.ABANDONED:
+            return self
+        self._reconcile_attempts(last_attempt)
+        self._extract_output_existence(predicted)
+        return self
+
+    def _process_logs(
+        self,
+        predicted: PredictedQuantumDatasetsModel,
+        logs: ButlerLogRecords | None,
+        last_attempt: ProvenanceQuantumAttemptModel,
+        *,
+        incomplete: bool,
+    ) -> None:
+        (predicted_log_dataset,) = predicted.outputs[acc.LOG_OUTPUT_CONNECTION_NAME]
+        if logs is None:
+            self.output_existence[predicted_log_dataset.dataset_id] = False
+            if incomplete:
+                self.status = ProvenanceQuantumScanStatus.ABANDONED
+            else:
+                self.status = ProvenanceQuantumScanStatus.FAILED
+        else:
+            # Set the attempt's run status to FAILED, since the default is
+            # UNKNOWN (i.e. logs *and* metadata are missing) and we now know
+            # the logs exist.  This will usually get replaced by SUCCESSFUL
+            # when we look for metadata next.
+            last_attempt.status = QuantumAttemptStatus.FAILED
+            self.output_existence[predicted_log_dataset.dataset_id] = True
+            if logs.extra:
+                log_extra = _ExecutionLogRecordsExtra.model_validate(logs.extra)
+                self._extract_from_log_extra(log_extra, last_attempt=last_attempt)
+            self.logs.attempts.append(list(logs))
+
+    def _extract_from_log_extra(
+        self,
+        log_extra: _ExecutionLogRecordsExtra,
+        last_attempt: ProvenanceQuantumAttemptModel | None,
+    ) -> None:
+        for previous_attempt_log_extra in log_extra.previous_attempts:
+            self._extract_from_log_extra(
+                previous_attempt_log_extra,
+                last_attempt=None,
+            )
+        quantum_attempt: ProvenanceQuantumAttemptModel
+        if last_attempt is None:
+            # This is not the last attempt, so it must be a failure.
+            quantum_attempt = ProvenanceQuantumAttemptModel(
+                attempt=len(self.attempts), status=QuantumAttemptStatus.FAILED
+            )
+            # We also need to get the logs from this extra provenance, since
+            # they won't be the main section of the log records.
+            self.logs.attempts.append(log_extra.logs)
+            # The special last attempt is only appended after we attempt to
+            # read metadata later, but we have to append this one now.
+            self.attempts.append(quantum_attempt)
+        else:
+            assert not log_extra.logs, "Logs for the last attempt should not be stored in the extra JSON."
+            quantum_attempt = last_attempt
+        if log_extra.exception is not None or log_extra.metadata is not None or last_attempt is None:
+            # We won't be getting a separate metadata dataset, so anything we
+            # might get from the metadata has to come from this extra
+            # provenance in the logs.
+            quantum_attempt.exception = log_extra.exception
+            if log_extra.metadata is not None:
+                quantum_attempt.resource_usage = QuantumResourceUsage.from_task_metadata(log_extra.metadata)
+                self.metadata.attempts.append(log_extra.metadata)
+            else:
+                self.metadata.attempts.append(None)
+        # Regardless of whether this is the last attempt or not, we can only
+        # get the previous_process_quanta from the log extra.
+        quantum_attempt.previous_process_quanta.extend(log_extra.previous_process_quanta)
+
+    def _process_metadata(
+        self,
+        predicted: PredictedQuantumDatasetsModel,
+        metadata: TaskMetadata | None,
+        last_attempt: ProvenanceQuantumAttemptModel,
+        *,
+        incomplete: bool,
+    ) -> None:
+        (predicted_metadata_dataset,) = predicted.outputs[acc.METADATA_OUTPUT_CONNECTION_NAME]
+        if metadata is None:
+            self.output_existence[predicted_metadata_dataset.dataset_id] = False
+            if incomplete:
+                self.status = ProvenanceQuantumScanStatus.ABANDONED
+            else:
+                self.status = ProvenanceQuantumScanStatus.FAILED
+        else:
+            self.status = ProvenanceQuantumScanStatus.SUCCESSFUL
+            self.output_existence[predicted_metadata_dataset.dataset_id] = True
+            last_attempt.status = QuantumAttemptStatus.SUCCESSFUL
+            try:
+                # Int conversion guards against spurious conversion to
+                # float that can apparently sometimes happen in
+                # TaskMetadata.
+                last_attempt.caveats = QuantumSuccessCaveats(int(metadata["quantum"]["caveats"]))
+            except LookupError:
+                pass
+            try:
+                last_attempt.exception = ExceptionInfo._from_metadata(
+                    metadata[predicted.task_label]["failure"]
+                )
+            except LookupError:
+                pass
+            last_attempt.resource_usage = QuantumResourceUsage.from_task_metadata(metadata)
+            self.metadata.attempts.append(metadata)
+
+    def _reconcile_attempts(self, last_attempt: ProvenanceQuantumAttemptModel) -> None:
+        last_attempt.attempt = len(self.attempts)
+        self.attempts.append(last_attempt)
+        assert self.status is not ProvenanceQuantumScanStatus.INCOMPLETE
+        assert self.status is not ProvenanceQuantumScanStatus.ABANDONED
+        if len(self.logs.attempts) < len(self.attempts):
+            # Logs were not found for this attempt; must have been a hard error
+            # that kept the `finally` block from running or otherwise
+            # interrupted the writing of the logs.
+            self.logs.attempts.append(None)
+            if self.status is ProvenanceQuantumScanStatus.SUCCESSFUL:
+                # But we found the metadata!  Either that hard error happened
+                # at a very unlucky time (in between those two writes), or
+                # something even weirder happened.
+                self.attempts[-1].status = QuantumAttemptStatus.ABORTED_SUCCESS
+            else:
+                self.attempts[-1].status = QuantumAttemptStatus.FAILED
+        if len(self.metadata.attempts) < len(self.attempts):
+            # Metadata missing usually just means a failure.  In any case, the
+            # status will already be correct, either because it was set to a
+            # failure when we read the logs, or left at UNKNOWN if there were
+            # no logs.  Note that scanners never process BLOCKED quanta at all.
+            self.metadata.attempts.append(None)
+        assert len(self.logs.attempts) == len(self.attempts) or len(self.metadata.attempts) == len(
+            self.attempts
+        ), (
+            "The only way we can add more than one quantum attempt is by "
+            "extracting info stored with the logs, and that always appends "
+            "a log attempt and a metadata attempt, so this must be a bug in "
+            "this class."
+        )
+
+    def _extract_output_existence(self, predicted: PredictedQuantumDatasetsModel) -> None:
+        try:
+            outputs_put = self.metadata.attempts[-1]["quantum"].getArray("outputs")  # type: ignore[index]
+        except (
+            IndexError,  # metadata.attempts is empty
+            TypeError,  # metadata.attempts[-1] is None
+            LookupError,  # no 'quantum' entry in metadata or 'outputs' in that
+        ):
+            pass
+        else:
+            for id_str in ensure_iterable(outputs_put):
+                self.output_existence[uuid.UUID(id_str)] = True
+            # If the metadata told us what it wrote, anything not in that
+            # list was not written.
+            for predicted_output in itertools.chain.from_iterable(predicted.outputs.values()):
+                self.output_existence.setdefault(predicted_output.dataset_id, False)
+
+    def to_scan_data(
+        self: ProvenanceQuantumScanModels,
+        predicted_quantum: PredictedQuantumDatasetsModel,
+        compressor: Compressor | None = None,
+    ) -> ProvenanceQuantumScanData:
+        """Convert these models to JSON data.
+
+        Parameters
+        ----------
+        predicted_quantum : `PredictedQuantumDatasetsModel`
+            Information about the predicted quantum.
+        compressor : `Compressor`
+            Object that can compress bytes.
+
+        Returns
+        -------
+        scan_data : `ProvenanceQuantumScanData`
+            Scan information ready for serialization.
+        """
+        quantum: ProvenanceInitQuantumModel | ProvenanceQuantumModel
+        if self.status is ProvenanceQuantumScanStatus.INIT:
+            quantum = ProvenanceInitQuantumModel.from_predicted(predicted_quantum)
+        else:
+            quantum = ProvenanceQuantumModel.from_predicted(predicted_quantum)
+            quantum.attempts = self.attempts
+        for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
+            if predicted_output.dataset_id not in self.output_existence:
+                raise RuntimeError(
+                    "Logic bug in provenance gathering or execution invariants: "
+                    f"no existence information for output {predicted_output.dataset_id} "
+                    f"({predicted_output.dataset_type_name}@{predicted_output.data_coordinate})."
+                )
+        data = ProvenanceQuantumScanData(
+            self.quantum_id,
+            self.status,
+            existing_outputs={
+                dataset_id for dataset_id, was_produced in self.output_existence.items() if was_produced
+            },
+            quantum=quantum.model_dump_json().encode(),
+            logs=self.logs.model_dump_json().encode() if self.logs.attempts else b"",
+            metadata=self.metadata.model_dump_json().encode() if self.metadata.attempts else b"",
+        )
+        if compressor is not None:
+            data.compress(compressor)
+        return data
+
+
+@dataclasses.dataclass
+class ProvenanceQuantumScanData:
+    """A struct that represents ready-for-serialization provenance information
+    for a single quantum.
+    """
+
+    quantum_id: uuid.UUID
+    """Unique ID for the quantum."""
+
+    status: ProvenanceQuantumScanStatus
+    """Combined status for the scan and the execution of the quantum."""
+
+    existing_outputs: set[uuid.UUID] = dataclasses.field(default_factory=set)
+    """Unique IDs of the output datasets that were actually written."""
+
+    quantum: bytes = b""
+    """Serialized quantum provenance model.
+
+    This may be empty for quanta that had no attempts.
+    """
+
+    metadata: bytes = b""
+    """Serialized task metadata."""
+
+    logs: bytes = b""
+    """Serialized logs."""
+
+    is_compressed: bool = False
+    """Whether the `quantum`, `metadata`, and `log` attributes are
+    compressed.
+    """
+
+    def compress(self, compressor: Compressor) -> None:
+        """Compress the data in this struct if it has not been compressed
+        already.
+
+        Parameters
+        ----------
+        compressor : `Compressor`
+            Object with a ``compress`` method that takes and returns `bytes`.
+        """
+        if not self.is_compressed:
+            self.quantum = compressor.compress(self.quantum)
+            self.logs = compressor.compress(self.logs) if self.logs else b""
+            self.metadata = compressor.compress(self.metadata) if self.metadata else b""
+            self.is_compressed = True