lsst-pipe-base 30.0.0rc3__py3-none-any.whl → 30.2025.5100__py3-none-any.whl

lsst/pipe/base/quantum_graph/_provenance.py

@@ -35,25 +35,19 @@ __all__ = (
     "ProvenanceLogRecordsModel",
     "ProvenanceQuantumGraph",
     "ProvenanceQuantumGraphReader",
-    "ProvenanceQuantumGraphWriter",
     "ProvenanceQuantumInfo",
     "ProvenanceQuantumModel",
-    "ProvenanceQuantumScanData",
-    "ProvenanceQuantumScanModels",
-    "ProvenanceQuantumScanStatus",
     "ProvenanceTaskMetadataModel",
 )
 
 
 import dataclasses
-import enum
-import itertools
 import sys
 import uuid
 from collections import Counter
-from collections.abc import Callable, Iterable, Iterator, Mapping
-from contextlib import ExitStack, contextmanager
-from typing import TYPE_CHECKING, Any, TypeAlias, TypedDict, TypeVar
+from collections.abc import Iterable, Iterator, Mapping
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any, TypedDict, TypeVar
 
 import astropy.table
 import networkx
@@ -63,21 +57,15 @@ import pydantic
 from lsst.daf.butler import DataCoordinate
 from lsst.daf.butler.logging import ButlerLogRecord, ButlerLogRecords
 from lsst.resources import ResourcePathExpression
-from lsst.utils.iteration import ensure_iterable
-from lsst.utils.logging import LsstLogAdapter, getLogger
 from lsst.utils.packages import Packages
 
-from .. import automatic_connection_constants as acc
 from .._status import ExceptionInfo, QuantumAttemptStatus, QuantumSuccessCaveats
 from .._task_metadata import TaskMetadata
-from ..log_capture import _ExecutionLogRecordsExtra
-from ..log_on_close import LogOnClose
 from ..pipeline_graph import PipelineGraph, TaskImportMode, TaskInitNode
 from ..resource_usage import QuantumResourceUsage
 from ._common import (
     BaseQuantumGraph,
     BaseQuantumGraphReader,
-    BaseQuantumGraphWriter,
     ConnectionName,
     DataCoordinateValues,
     DatasetInfo,
@@ -86,19 +74,8 @@ from ._common import (
     QuantumInfo,
     TaskLabel,
 )
-from ._multiblock import Compressor, MultiblockReader, MultiblockWriter
-from ._predicted import (
-    PredictedDatasetModel,
-    PredictedQuantumDatasetsModel,
-    PredictedQuantumGraph,
-    PredictedQuantumGraphComponents,
-)
-
-_T = TypeVar("_T")
-
-LoopWrapper: TypeAlias = Callable[[Iterable[_T]], Iterable[_T]]
-
-_LOG = getLogger(__file__)
+from ._multiblock import MultiblockReader
+from ._predicted import PredictedDatasetModel, PredictedQuantumDatasetsModel
 
 DATASET_ADDRESS_INDEX = 0
 QUANTUM_ADDRESS_INDEX = 1
@@ -110,9 +87,7 @@ QUANTUM_MB_NAME = "quanta"
 LOG_MB_NAME = "logs"
 METADATA_MB_NAME = "metadata"
 
-
-def pass_through(arg: _T) -> _T:
-    return arg
+_I = TypeVar("_I", bound=uuid.UUID | int)
 
 
 class ProvenanceDatasetInfo(DatasetInfo):
@@ -1377,629 +1352,3 @@ class ProvenanceQuantumGraphReader(BaseQuantumGraphReader):
         """Fetch package version information."""
         data = self._read_single_block_raw("packages")
         return Packages.fromBytes(data, format="json")
-
-
-class ProvenanceQuantumGraphWriter:
-    """A struct of low-level writer objects for the main components of a
-    provenance quantum graph.
-
-    Parameters
-    ----------
-    output_path : `str`
-        Path to write the graph to.
-    exit_stack : `contextlib.ExitStack`
-        Object that can be used to manage multiple context managers.
-    log_on_close : `LogOnClose`
-        Factory for context managers that log when closed.
-    predicted : `.PredictedQuantumGraphComponents`
-        Components of the predicted graph.
-    zstd_level : `int`, optional
-        Compression level.
-    cdict_data : `bytes` or `None`, optional
-        Bytes representation of the compression dictionary used by the
-        compressor.
-    loop_wrapper : `~collections.abc.Callable`, optional
-        A callable that takes an iterable and returns an equivalent one, to be
-        used in all potentially-large loops.  This can be used to add progress
-        reporting or check for cancelation signals.
-    log : `LsstLogAdapter`, optional
-        Logger to use for debug messages.
-    """
-
-    def __init__(
-        self,
-        output_path: str,
-        *,
-        exit_stack: ExitStack,
-        log_on_close: LogOnClose,
-        predicted: PredictedQuantumGraphComponents | PredictedQuantumGraph,
-        zstd_level: int = 10,
-        cdict_data: bytes | None = None,
-        loop_wrapper: LoopWrapper = pass_through,
-        log: LsstLogAdapter | None = None,
-    ) -> None:
-        header = predicted.header.model_copy()
-        header.graph_type = "provenance"
-        if log is None:
-            log = _LOG
-        self.log = log
-        self._base_writer = exit_stack.enter_context(
-            log_on_close.wrap(
-                BaseQuantumGraphWriter.open(
-                    output_path,
-                    header,
-                    predicted.pipeline_graph,
-                    address_filename="nodes",
-                    zstd_level=zstd_level,
-                    cdict_data=cdict_data,
-                ),
-                "Finishing writing provenance quantum graph.",
-            )
-        )
-        self._base_writer.address_writer.addresses = [{}, {}, {}, {}]
-        self._log_writer = exit_stack.enter_context(
-            log_on_close.wrap(
-                MultiblockWriter.open_in_zip(
-                    self._base_writer.zf, LOG_MB_NAME, header.int_size, use_tempfile=True
-                ),
-                "Copying logs into zip archive.",
-            ),
-        )
-        self._base_writer.address_writer.addresses[LOG_ADDRESS_INDEX] = self._log_writer.addresses
-        self._metadata_writer = exit_stack.enter_context(
-            log_on_close.wrap(
-                MultiblockWriter.open_in_zip(
-                    self._base_writer.zf, METADATA_MB_NAME, header.int_size, use_tempfile=True
-                ),
-                "Copying metadata into zip archive.",
-            )
-        )
-        self._base_writer.address_writer.addresses[METADATA_ADDRESS_INDEX] = self._metadata_writer.addresses
-        self._dataset_writer = exit_stack.enter_context(
-            log_on_close.wrap(
-                MultiblockWriter.open_in_zip(
-                    self._base_writer.zf, DATASET_MB_NAME, header.int_size, use_tempfile=True
-                ),
-                "Copying dataset provenance into zip archive.",
-            )
-        )
-        self._base_writer.address_writer.addresses[DATASET_ADDRESS_INDEX] = self._dataset_writer.addresses
-        self._quantum_writer = exit_stack.enter_context(
-            log_on_close.wrap(
-                MultiblockWriter.open_in_zip(
-                    self._base_writer.zf, QUANTUM_MB_NAME, header.int_size, use_tempfile=True
-                ),
-                "Copying quantum provenance into zip archive.",
-            )
-        )
-        self._base_writer.address_writer.addresses[QUANTUM_ADDRESS_INDEX] = self._quantum_writer.addresses
-        self._init_predicted_quanta(predicted)
-        self._populate_xgraph_and_inputs(loop_wrapper)
-        self._existing_init_outputs: set[uuid.UUID] = set()
-
-    def _init_predicted_quanta(
-        self, predicted: PredictedQuantumGraph | PredictedQuantumGraphComponents
-    ) -> None:
-        self._predicted_init_quanta: list[PredictedQuantumDatasetsModel] = []
-        self._predicted_quanta: dict[uuid.UUID, PredictedQuantumDatasetsModel] = {}
-        if isinstance(predicted, PredictedQuantumGraph):
-            self._predicted_init_quanta.extend(predicted._init_quanta.values())
-            self._predicted_quanta.update(predicted._quantum_datasets)
-        else:
-            self._predicted_init_quanta.extend(predicted.init_quanta.root)
-            self._predicted_quanta.update(predicted.quantum_datasets)
-        self._predicted_quanta.update({q.quantum_id: q for q in self._predicted_init_quanta})
-
-    def _populate_xgraph_and_inputs(self, loop_wrapper: LoopWrapper = pass_through) -> None:
-        self._xgraph = networkx.DiGraph()
-        self._overall_inputs: dict[uuid.UUID, PredictedDatasetModel] = {}
-        output_dataset_ids: set[uuid.UUID] = set()
-        for predicted_quantum in loop_wrapper(self._predicted_quanta.values()):
-            if not predicted_quantum.task_label:
-                # Skip the 'packages' producer quantum.
-                continue
-            output_dataset_ids.update(predicted_quantum.iter_output_dataset_ids())
-        for predicted_quantum in loop_wrapper(self._predicted_quanta.values()):
-            if not predicted_quantum.task_label:
-                # Skip the 'packages' producer quantum.
-                continue
-            for predicted_input in itertools.chain.from_iterable(predicted_quantum.inputs.values()):
-                self._xgraph.add_edge(predicted_input.dataset_id, predicted_quantum.quantum_id)
-                if predicted_input.dataset_id not in output_dataset_ids:
-                    self._overall_inputs.setdefault(predicted_input.dataset_id, predicted_input)
-            for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
-                self._xgraph.add_edge(predicted_quantum.quantum_id, predicted_output.dataset_id)
-
-    @property
-    def compressor(self) -> Compressor:
-        """Object that should be used to compress all JSON blocks."""
-        return self._base_writer.compressor
-
-    def write_packages(self) -> None:
-        """Write package version information to the provenance graph."""
-        packages = Packages.fromSystem(include_all=True)
-        data = packages.toBytes("json")
-        self._base_writer.write_single_block("packages", data)
-
-    def write_overall_inputs(self, loop_wrapper: LoopWrapper = pass_through) -> None:
-        """Write provenance for overall-input datasets.
-
-        Parameters
-        ----------
-        loop_wrapper : `~collections.abc.Callable`, optional
-            A callable that takes an iterable and returns an equivalent one, to
-            be used in all potentially-large loops.  This can be used to add
-            progress reporting or check for cancelation signals.
-        """
-        for predicted_input in loop_wrapper(self._overall_inputs.values()):
-            if predicted_input.dataset_id not in self._dataset_writer.addresses:
-                self._dataset_writer.write_model(
-                    predicted_input.dataset_id,
-                    ProvenanceDatasetModel.from_predicted(
-                        predicted_input,
-                        producer=None,
-                        consumers=self._xgraph.successors(predicted_input.dataset_id),
-                    ),
-                    self.compressor,
-                )
-        del self._overall_inputs
-
-    def write_init_outputs(self, assume_existence: bool = True) -> None:
-        """Write provenance for init-output datasets and init-quanta.
-
-        Parameters
-        ----------
-        assume_existence : `bool`, optional
-            If `True`, just assume all init-outputs exist.
-        """
-        init_quanta = ProvenanceInitQuantaModel()
-        for predicted_init_quantum in self._predicted_init_quanta:
-            if not predicted_init_quantum.task_label:
-                # Skip the 'packages' producer quantum.
-                continue
-            for predicted_output in itertools.chain.from_iterable(predicted_init_quantum.outputs.values()):
-                provenance_output = ProvenanceDatasetModel.from_predicted(
-                    predicted_output,
-                    producer=predicted_init_quantum.quantum_id,
-                    consumers=self._xgraph.successors(predicted_output.dataset_id),
-                )
-                provenance_output.produced = assume_existence or (
-                    provenance_output.dataset_id in self._existing_init_outputs
-                )
-                self._dataset_writer.write_model(
-                    provenance_output.dataset_id, provenance_output, self.compressor
-                )
-            init_quanta.root.append(ProvenanceInitQuantumModel.from_predicted(predicted_init_quantum))
-        self._base_writer.write_single_model("init_quanta", init_quanta)
-
-    def write_quantum_provenance(
-        self, quantum_id: uuid.UUID, metadata: TaskMetadata | None, logs: ButlerLogRecords | None
-    ) -> None:
-        """Gather and write provenance for a quantum.
-
-        Parameters
-        ----------
-        quantum_id : `uuid.UUID`
-            Unique ID for the quantum.
-        metadata : `..TaskMetadata` or `None`
-            Task metadata.
-        logs : `lsst.daf.butler.logging.ButlerLogRecords` or `None`
-            Task logs.
-        """
-        predicted_quantum = self._predicted_quanta[quantum_id]
-        provenance_models = ProvenanceQuantumScanModels.from_metadata_and_logs(
-            predicted_quantum, metadata, logs, assume_complete=True
-        )
-        scan_data = provenance_models.to_scan_data(predicted_quantum, compressor=self.compressor)
-        self.write_scan_data(scan_data)
-
-    def write_scan_data(self, scan_data: ProvenanceQuantumScanData) -> None:
-        """Write the output of a quantum provenance scan to disk.
-
-        Parameters
-        ----------
-        scan_data : `ProvenanceQuantumScanData`
-            Result of a quantum provenance scan.
-        """
-        if scan_data.status is ProvenanceQuantumScanStatus.INIT:
-            self.log.debug("Handling init-output scan for %s.", scan_data.quantum_id)
-            self._existing_init_outputs.update(scan_data.existing_outputs)
-            return
-        self.log.debug("Handling quantum scan for %s.", scan_data.quantum_id)
-        # We shouldn't need this predicted quantum after this method runs; pop
-        # from the dict it in the hopes that'll free up some memory when we're
-        # done.
-        predicted_quantum = self._predicted_quanta.pop(scan_data.quantum_id)
-        outputs: dict[uuid.UUID, bytes] = {}
-        for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
-            provenance_output = ProvenanceDatasetModel.from_predicted(
-                predicted_output,
-                producer=predicted_quantum.quantum_id,
-                consumers=self._xgraph.successors(predicted_output.dataset_id),
-            )
-            provenance_output.produced = provenance_output.dataset_id in scan_data.existing_outputs
-            outputs[provenance_output.dataset_id] = self.compressor.compress(
-                provenance_output.model_dump_json().encode()
-            )
-        if not scan_data.quantum:
-            scan_data.quantum = (
-                ProvenanceQuantumModel.from_predicted(predicted_quantum).model_dump_json().encode()
-            )
-            if scan_data.is_compressed:
-                scan_data.quantum = self.compressor.compress(scan_data.quantum)
-        if not scan_data.is_compressed:
-            scan_data.quantum = self.compressor.compress(scan_data.quantum)
-            if scan_data.metadata:
-                scan_data.metadata = self.compressor.compress(scan_data.metadata)
-            if scan_data.logs:
-                scan_data.logs = self.compressor.compress(scan_data.logs)
-        self.log.debug("Writing quantum %s.", scan_data.quantum_id)
-        self._quantum_writer.write_bytes(scan_data.quantum_id, scan_data.quantum)
-        for dataset_id, dataset_data in outputs.items():
-            self._dataset_writer.write_bytes(dataset_id, dataset_data)
-        if scan_data.metadata:
-            (metadata_output,) = predicted_quantum.outputs[acc.METADATA_OUTPUT_CONNECTION_NAME]
-            address = self._metadata_writer.write_bytes(scan_data.quantum_id, scan_data.metadata)
-            self._metadata_writer.addresses[metadata_output.dataset_id] = address
-        if scan_data.logs:
-            (log_output,) = predicted_quantum.outputs[acc.LOG_OUTPUT_CONNECTION_NAME]
-            address = self._log_writer.write_bytes(scan_data.quantum_id, scan_data.logs)
-            self._log_writer.addresses[log_output.dataset_id] = address
-
-
-class ProvenanceQuantumScanStatus(enum.Enum):
-    """Status enum for quantum scanning.
-
-    Note that this records the status for the *scanning* which is distinct
-    from the status of the quantum's execution.
-    """
-
-    INCOMPLETE = enum.auto()
-    """The quantum is not necessarily done running, and cannot be scanned
-    conclusively yet.
-    """
-
-    ABANDONED = enum.auto()
-    """The quantum's execution appears to have failed but we cannot rule out
-    the possibility that it could be recovered, but we've also waited long
-    enough (according to `ScannerTimeConfigDict.retry_timeout`) that it's time
-    to stop trying for now.
-
-    This state means a later run with `ScannerConfig.assume_complete` is
-    required.
-    """
-
-    SUCCESSFUL = enum.auto()
-    """The quantum was conclusively scanned and was executed successfully,
-    unblocking scans for downstream quanta.
-    """
-
-    FAILED = enum.auto()
-    """The quantum was conclusively scanned and failed execution, blocking
-    scans for downstream quanta.
-    """
-
-    BLOCKED = enum.auto()
-    """A quantum upstream of this one failed."""
-
-    INIT = enum.auto()
-    """Init quanta need special handling, because they don't have logs and
-    metadata.
-    """
-
-
-@dataclasses.dataclass
-class ProvenanceQuantumScanModels:
-    """A struct that represents provenance information for a single quantum."""
-
-    quantum_id: uuid.UUID
-    """Unique ID for the quantum."""
-
-    status: ProvenanceQuantumScanStatus = ProvenanceQuantumScanStatus.INCOMPLETE
-    """Combined status for the scan and the execution of the quantum."""
-
-    attempts: list[ProvenanceQuantumAttemptModel] = dataclasses.field(default_factory=list)
-    """Provenance information about each attempt to run the quantum."""
-
-    output_existence: dict[uuid.UUID, bool] = dataclasses.field(default_factory=dict)
-    """Unique IDs of the output datasets mapped to whether they were actually
-    produced.
-    """
-
-    metadata: ProvenanceTaskMetadataModel = dataclasses.field(default_factory=ProvenanceTaskMetadataModel)
-    """Task metadata information for each attempt.
-    """
-
-    logs: ProvenanceLogRecordsModel = dataclasses.field(default_factory=ProvenanceLogRecordsModel)
-    """Log records for each attempt.
-    """
-
-    @classmethod
-    def from_metadata_and_logs(
-        cls,
-        predicted: PredictedQuantumDatasetsModel,
-        metadata: TaskMetadata | None,
-        logs: ButlerLogRecords | None,
-        *,
-        assume_complete: bool = True,
-    ) -> ProvenanceQuantumScanModels:
-        """Construct provenance information from task metadata and logs.
-
-        Parameters
-        ----------
-        predicted : `PredictedQuantumDatasetsModel`
-            Information about the predicted quantum.
-        metadata : `..TaskMetadata` or `None`
-            Task metadata.
-        logs : `lsst.daf.butler.logging.ButlerLogRecords` or `None`
-            Task logs.
-        assume_complete : `bool`, optional
-            If `False`, treat execution failures as possibly-incomplete quanta
-            and do not fully process them; instead just set the status to
-            `ProvenanceQuantumScanStatus.ABANDONED` and return.
-
-        Returns
-        -------
-        scan_models : `ProvenanceQuantumScanModels`
-            Struct of models that describe quantum provenance.
-
-        Notes
-        -----
-        This method does not necessarily fully populate the `output_existence`
-        field; it does what it can given the information in the metadata and
-        logs, but the caller is responsible for filling in the existence status
-        for any predicted outputs that are not present at all in that `dict`.
-        """
-        self = ProvenanceQuantumScanModels(predicted.quantum_id)
-        last_attempt = ProvenanceQuantumAttemptModel()
-        self._process_logs(predicted, logs, last_attempt, assume_complete=assume_complete)
-        self._process_metadata(predicted, metadata, last_attempt, assume_complete=assume_complete)
-        if self.status is ProvenanceQuantumScanStatus.ABANDONED:
-            return self
-        self._reconcile_attempts(last_attempt)
-        self._extract_output_existence(predicted)
-        return self
-
-    def _process_logs(
-        self,
-        predicted: PredictedQuantumDatasetsModel,
-        logs: ButlerLogRecords | None,
-        last_attempt: ProvenanceQuantumAttemptModel,
-        *,
-        assume_complete: bool,
-    ) -> None:
-        (predicted_log_dataset,) = predicted.outputs[acc.LOG_OUTPUT_CONNECTION_NAME]
-        if logs is None:
-            self.output_existence[predicted_log_dataset.dataset_id] = False
-            if assume_complete:
-                self.status = ProvenanceQuantumScanStatus.FAILED
-            else:
-                self.status = ProvenanceQuantumScanStatus.ABANDONED
-        else:
-            # Set the attempt's run status to FAILED, since the default is
-            # UNKNOWN (i.e. logs *and* metadata are missing) and we now know
-            # the logs exist.  This will usually get replaced by SUCCESSFUL
-            # when we look for metadata next.
-            last_attempt.status = QuantumAttemptStatus.FAILED
-            self.output_existence[predicted_log_dataset.dataset_id] = True
-            if logs.extra:
-                log_extra = _ExecutionLogRecordsExtra.model_validate(logs.extra)
-                self._extract_from_log_extra(log_extra, last_attempt=last_attempt)
-            self.logs.attempts.append(list(logs))
-
-    def _extract_from_log_extra(
-        self,
-        log_extra: _ExecutionLogRecordsExtra,
-        last_attempt: ProvenanceQuantumAttemptModel | None,
-    ) -> None:
-        for previous_attempt_log_extra in log_extra.previous_attempts:
-            self._extract_from_log_extra(
-                previous_attempt_log_extra,
-                last_attempt=None,
-            )
-        quantum_attempt: ProvenanceQuantumAttemptModel
-        if last_attempt is None:
-            # This is not the last attempt, so it must be a failure.
-            quantum_attempt = ProvenanceQuantumAttemptModel(
-                attempt=len(self.attempts), status=QuantumAttemptStatus.FAILED
-            )
-            # We also need to get the logs from this extra provenance, since
-            # they won't be the main section of the log records.
-            self.logs.attempts.append(log_extra.logs)
-            # The special last attempt is only appended after we attempt to
-            # read metadata later, but we have to append this one now.
-            self.attempts.append(quantum_attempt)
-        else:
-            assert not log_extra.logs, "Logs for the last attempt should not be stored in the extra JSON."
-            quantum_attempt = last_attempt
-        if log_extra.exception is not None or log_extra.metadata is not None or last_attempt is None:
-            # We won't be getting a separate metadata dataset, so anything we
-            # might get from the metadata has to come from this extra
-            # provenance in the logs.
-            quantum_attempt.exception = log_extra.exception
-            if log_extra.metadata is not None:
-                quantum_attempt.resource_usage = QuantumResourceUsage.from_task_metadata(log_extra.metadata)
-                self.metadata.attempts.append(log_extra.metadata)
-            else:
-                self.metadata.attempts.append(None)
-        # Regardless of whether this is the last attempt or not, we can only
-        # get the previous_process_quanta from the log extra.
-        quantum_attempt.previous_process_quanta.extend(log_extra.previous_process_quanta)
-
-    def _process_metadata(
-        self,
-        predicted: PredictedQuantumDatasetsModel,
-        metadata: TaskMetadata | None,
-        last_attempt: ProvenanceQuantumAttemptModel,
-        *,
-        assume_complete: bool,
-    ) -> None:
-        (predicted_metadata_dataset,) = predicted.outputs[acc.METADATA_OUTPUT_CONNECTION_NAME]
-        if metadata is None:
-            self.output_existence[predicted_metadata_dataset.dataset_id] = False
-            if assume_complete:
-                self.status = ProvenanceQuantumScanStatus.FAILED
-            else:
-                self.status = ProvenanceQuantumScanStatus.ABANDONED
-        else:
-            self.status = ProvenanceQuantumScanStatus.SUCCESSFUL
-            self.output_existence[predicted_metadata_dataset.dataset_id] = True
-            last_attempt.status = QuantumAttemptStatus.SUCCESSFUL
-            try:
-                # Int conversion guards against spurious conversion to
-                # float that can apparently sometimes happen in
-                # TaskMetadata.
-                last_attempt.caveats = QuantumSuccessCaveats(int(metadata["quantum"]["caveats"]))
-            except LookupError:
-                pass
-            try:
-                last_attempt.exception = ExceptionInfo._from_metadata(
-                    metadata[predicted.task_label]["failure"]
-                )
-            except LookupError:
-                pass
-            last_attempt.resource_usage = QuantumResourceUsage.from_task_metadata(metadata)
-            self.metadata.attempts.append(metadata)
-
-    def _reconcile_attempts(self, last_attempt: ProvenanceQuantumAttemptModel) -> None:
-        last_attempt.attempt = len(self.attempts)
-        self.attempts.append(last_attempt)
-        assert self.status is not ProvenanceQuantumScanStatus.INCOMPLETE
-        assert self.status is not ProvenanceQuantumScanStatus.ABANDONED
-        if len(self.logs.attempts) < len(self.attempts):
-            # Logs were not found for this attempt; must have been a hard error
-            # that kept the `finally` block from running or otherwise
-            # interrupted the writing of the logs.
-            self.logs.attempts.append(None)
-            if self.status is ProvenanceQuantumScanStatus.SUCCESSFUL:
-                # But we found the metadata!  Either that hard error happened
-                # at a very unlucky time (in between those two writes), or
-                # something even weirder happened.
-                self.attempts[-1].status = QuantumAttemptStatus.LOGS_MISSING
-            else:
-                self.attempts[-1].status = QuantumAttemptStatus.FAILED
-        if len(self.metadata.attempts) < len(self.attempts):
-            # Metadata missing usually just means a failure.  In any case, the
-            # status will already be correct, either because it was set to a
-            # failure when we read the logs, or left at UNKNOWN if there were
-            # no logs.  Note that scanners never process BLOCKED quanta at all.
-            self.metadata.attempts.append(None)
-        assert len(self.logs.attempts) == len(self.attempts) or len(self.metadata.attempts) == len(
-            self.attempts
-        ), (
-            "The only way we can add more than one quantum attempt is by "
-            "extracting info stored with the logs, and that always appends "
-            "a log attempt and a metadata attempt, so this must be a bug in "
-            "this class."
-        )
-
-    def _extract_output_existence(self, predicted: PredictedQuantumDatasetsModel) -> None:
-        try:
-            outputs_put = self.metadata.attempts[-1]["quantum"].getArray("outputs")  # type: ignore[index]
-        except (
-            IndexError,  # metadata.attempts is empty
-            TypeError,  # metadata.attempts[-1] is None
-            LookupError,  # no 'quantum' entry in metadata or 'outputs' in that
-        ):
-            pass
-        else:
-            for id_str in ensure_iterable(outputs_put):
-                self.output_existence[uuid.UUID(id_str)] = True
-            # If the metadata told us what it wrote, anything not in that
-            # list was not written.
-            for predicted_output in itertools.chain.from_iterable(predicted.outputs.values()):
-                self.output_existence.setdefault(predicted_output.dataset_id, False)
-
-    def to_scan_data(
-        self: ProvenanceQuantumScanModels,
-        predicted_quantum: PredictedQuantumDatasetsModel,
-        compressor: Compressor | None = None,
-    ) -> ProvenanceQuantumScanData:
-        """Convert these models to JSON data.
-
-        Parameters
-        ----------
-        predicted_quantum : `PredictedQuantumDatasetsModel`
-            Information about the predicted quantum.
-        compressor : `Compressor`
-            Object that can compress bytes.
-
-        Returns
-        -------
-        scan_data : `ProvenanceQuantumScanData`
-            Scan information ready for serialization.
-        """
-        quantum: ProvenanceInitQuantumModel | ProvenanceQuantumModel
-        if self.status is ProvenanceQuantumScanStatus.INIT:
-            quantum = ProvenanceInitQuantumModel.from_predicted(predicted_quantum)
-        else:
-            quantum = ProvenanceQuantumModel.from_predicted(predicted_quantum)
-            quantum.attempts = self.attempts
-        for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
-            if predicted_output.dataset_id not in self.output_existence:
-                raise RuntimeError(
-                    "Logic bug in provenance gathering or execution invariants: "
-                    f"no existence information for output {predicted_output.dataset_id} "
-                    f"({predicted_output.dataset_type_name}@{predicted_output.data_coordinate})."
-                )
-        data = ProvenanceQuantumScanData(
-            self.quantum_id,
-            self.status,
-            existing_outputs={
-                dataset_id for dataset_id, was_produced in self.output_existence.items() if was_produced
-            },
-            quantum=quantum.model_dump_json().encode(),
-            logs=self.logs.model_dump_json().encode() if self.logs.attempts else b"",
-            metadata=self.metadata.model_dump_json().encode() if self.metadata.attempts else b"",
-        )
-        if compressor is not None:
-            data.compress(compressor)
-        return data
-
-
-@dataclasses.dataclass
-class ProvenanceQuantumScanData:
-    """A struct that represents ready-for-serialization provenance information
-    for a single quantum.
-    """
-
-    quantum_id: uuid.UUID
-    """Unique ID for the quantum."""
-
-    status: ProvenanceQuantumScanStatus
-    """Combined status for the scan and the execution of the quantum."""
-
-    existing_outputs: set[uuid.UUID] = dataclasses.field(default_factory=set)
-    """Unique IDs of the output datasets that were actually written."""
-
-    quantum: bytes = b""
-    """Serialized quantum provenance model.
-
-    This may be empty for quanta that had no attempts.
-    """
-
-    metadata: bytes = b""
-    """Serialized task metadata."""
-
-    logs: bytes = b""
-    """Serialized logs."""
-
-    is_compressed: bool = False
-    """Whether the `quantum`, `metadata`, and `log` attributes are
-    compressed.
-    """
-
-    def compress(self, compressor: Compressor) -> None:
-        """Compress the data in this struct if it has not been compressed
-        already.
-
-        Parameters
-        ----------
-        compressor : `Compressor`
-            Object with a ``compress`` method that takes and returns `bytes`.
-        """
-        if not self.is_compressed:
-            self.quantum = compressor.compress(self.quantum)
-            self.logs = compressor.compress(self.logs) if self.logs else b""
-            self.metadata = compressor.compress(self.metadata) if self.metadata else b""
-            self.is_compressed = True