lsst-pipe-base 30.0.1rc1__py3-none-any.whl → 30.2025.5100__py3-none-any.whl

lsst/pipe/base/quantum_graph/aggregator/_config.py

@@ -29,8 +29,6 @@ from __future__ import annotations
 
 __all__ = ("AggregatorConfig",)
 
-import sys
-from typing import TYPE_CHECKING, Any
 
 import pydantic
 
@@ -62,13 +60,11 @@ class AggregatorConfig(pydantic.BaseModel):
     n_processes: int = 1
     """Number of processes the scanner should use."""
 
-    incomplete: bool = False
-    """If `True`, do not expect the graph to have been executed to completion
-    yet, and only ingest the outputs of successful quanta.
-
-    This disables writing the provenance quantum graph, since this is likely to
-    be wasted effort that just complicates a follow-up run with
-    ``incomplete=False`` later.
+    assume_complete: bool = True
+    """If `True`, the aggregator can assume all quanta have run to completion
+    (including any automatic retries).  If `False`, only successes can be
+    considered final, and quanta that appear to have failed or to have not been
+    executed are ignored.
     """
 
     defensive_ingest: bool = False
@@ -99,10 +95,11 @@ class AggregatorConfig(pydantic.BaseModel):
     """
 
     dry_run: bool = False
-    """If `True`, do not actually perform any central butler ingests.
+    """If `True`, do not actually perform any deletions or central butler
+    ingests.
 
-    Most log messages concerning ingests will still be emitted in order to
-    provide a better emulation of a real run.
+    Most log messages concerning deletions and ingests will still be emitted in
+    order to provide a better emulation of a real run.
     """
 
     interactive_status: bool = False
@@ -140,78 +137,3 @@ class AggregatorConfig(pydantic.BaseModel):
     """Enable support for storage classes by created by the
     lsst.pipe.base.tests.mocks package.
     """
-
-    promise_ingest_graph: bool = False
-    """If `True`, the aggregator will assume that `~.ingest_graph.ingest_graph`
-    will be run later to ingest metadata/log/config datasets, and will not
-    ingest them itself.  This means that if `~.ingest_graph.ingest_graph` is
-    not run, those files will be abandoned in the butler storage root without
-    being present in the butler database, but it will speed up both processes.
-
-    It is *usually* safe to build a quantum graph for downstream processing
-    before or while running `~.ingest_graph.ingest_graph`, because
-    metadata/log/config datasets are rarely used as inputs.  To check, use
-    ``pipetask build ... --show inputs`` to show the overall-inputs to the
-    graph and scan for these dataset types.
-    """
-
-    worker_check_timeout: float = 5.0
-    """Time to wait (s) for reports from subprocesses before running
-    process-alive checks.
-
-    These checks are designed to kill the main aggregator process when a
-    subprocess has been unexpectedly killed (e.g. for for using too much
-    memory).
-    """
-
-    @property
-    def is_writing_provenance(self) -> bool:
-        """Whether the aggregator is configured to write the provenance quantum
-        graph.
-        """
-        return self.output_path is not None and not self.incomplete
-
-    # Work around the fact that Sphinx chokes on Pydantic docstring formatting,
-    # when we inherit those docstrings in our public classes.
-    if "sphinx" in sys.modules and not TYPE_CHECKING:
-
-        def copy(self, *args: Any, **kwargs: Any) -> Any:
-            """See `pydantic.BaseModel.copy`."""
-            return super().copy(*args, **kwargs)
-
-        def model_dump(self, *args: Any, **kwargs: Any) -> Any:
-            """See `pydantic.BaseModel.model_dump`."""
-            return super().model_dump(*args, **kwargs)
-
-        def model_dump_json(self, *args: Any, **kwargs: Any) -> Any:
-            """See `pydantic.BaseModel.model_dump_json`."""
-            return super().model_dump(*args, **kwargs)
-
-        def model_copy(self, *args: Any, **kwargs: Any) -> Any:
-            """See `pydantic.BaseModel.model_copy`."""
-            return super().model_copy(*args, **kwargs)
-
-        @classmethod
-        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc, override]
-            """See `pydantic.BaseModel.model_construct`."""
-            return super().model_construct(*args, **kwargs)
-
-        @classmethod
-        def model_json_schema(cls, *args: Any, **kwargs: Any) -> Any:
-            """See `pydantic.BaseModel.model_json_schema`."""
-            return super().model_json_schema(*args, **kwargs)
-
-        @classmethod
-        def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
-            """See `pydantic.BaseModel.model_validate`."""
-            return super().model_validate(*args, **kwargs)
-
-        @classmethod
-        def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
-            """See `pydantic.BaseModel.model_validate_json`."""
-            return super().model_validate_json(*args, **kwargs)
-
-        @classmethod
-        def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
-            """See `pydantic.BaseModel.model_validate_strings`."""
-            return super().model_validate_strings(*args, **kwargs)

lsst/pipe/base/quantum_graph/aggregator/_ingester.py

@@ -43,7 +43,7 @@ from lsst.daf.butler.registry import ConflictingDefinitionError
 
 from ...pipeline_graph import TaskImportMode
 from .._common import DatastoreName
-from .._predicted import PredictedQuantumGraphComponents, PredictedQuantumGraphReader
+from .._predicted import PredictedDatasetModel, PredictedQuantumGraphComponents, PredictedQuantumGraphReader
 from ._communicators import IngesterCommunicator
 
 
@@ -140,7 +140,7 @@ class Ingester(AbstractContextManager):
         Notes
         -----
         This method is designed to run as the ``target`` in
-        `WorkerFactory.make_worker`.
+        `WorkerContext.make_worker`.
         """
         with comms, Ingester(predicted_path, butler_path, comms) as ingester:
             ingester.loop()
@@ -170,7 +170,7 @@ class Ingester(AbstractContextManager):
             for ingest_request in self.comms.poll():
                 self.n_producers_pending += 1
                 self.comms.log.debug(f"Got ingest request for producer {ingest_request.producer_id}.")
-                self.update_outputs_pending(refs=ingest_request.refs, records=ingest_request.records)
+                self.update_pending(ingest_request.datasets, ingest_request.records)
                 if self.n_datasets_pending > self.comms.config.ingest_batch_size:
                     self.ingest()
             self.comms.log.info("All ingest requests received.")
@@ -266,32 +266,31 @@ class Ingester(AbstractContextManager):
             else:
                 del self.records_pending[datastore_name]
 
-    def update_outputs_pending(
-        self,
-        refs: list[DatasetRef],
-        records: dict[DatastoreName, DatastoreRecordData],
+    def update_pending(
+        self, datasets: list[PredictedDatasetModel], records: dict[DatastoreName, DatastoreRecordData]
     ) -> None:
         """Add an ingest request to the pending-ingest data structures.
 
         Parameters
         ----------
-        refs : `list` [ `lsst.daf.butler.DatasetRef` ]
-            Registry information about regular quantum-output datasets.
+        datasets : `list` [ `PredictedDatasetModel` ]
+            Registry information about the datasets.
         records : `dict` [ `str`, \
                 `lsst.daf.butler.datastore.record_data.DatastoreRecordData` ]
             Datastore information about the datasets.
         """
-        n_given = len(refs)
+        n_given = len(datasets)
         if self.already_ingested is not None:
-            refs = [ref for ref in refs if ref.id not in self.already_ingested]
-            kept = {ref.id for ref in refs}
+            datasets = [d for d in datasets if d.dataset_id not in self.already_ingested]
+            kept = {d.dataset_id for d in datasets}
             self.n_datasets_skipped += n_given - len(kept)
             records = {
                 datastore_name: filtered_records
                 for datastore_name, original_records in records.items()
                 if (filtered_records := original_records.subset(kept)) is not None
             }
-        for ref in refs:
+        for dataset in datasets:
+            ref = self.predicted.make_dataset_ref(dataset)
             self.refs_pending[ref.datasetType.dimensions].append(ref)
         for datastore_name, datastore_records in records.items():
             if (existing_records := self.records_pending.get(datastore_name)) is not None:

lsst/pipe/base/quantum_graph/aggregator/_scanner.py

@@ -38,19 +38,23 @@ from typing import Any, Literal, Self
 import zstandard
 
 from lsst.daf.butler import ButlerLogRecords, DatasetRef, QuantumBackedButler
+from lsst.utils.iteration import ensure_iterable
 
 from ... import automatic_connection_constants as acc
+from ..._status import ExceptionInfo, QuantumAttemptStatus, QuantumSuccessCaveats
 from ..._task_metadata import TaskMetadata
+from ...log_capture import _ExecutionLogRecordsExtra
 from ...pipeline_graph import PipelineGraph, TaskImportMode
+from ...resource_usage import QuantumResourceUsage
 from .._multiblock import Compressor
 from .._predicted import (
     PredictedDatasetModel,
     PredictedQuantumDatasetsModel,
     PredictedQuantumGraphReader,
 )
-from .._provenance import ProvenanceQuantumScanModels, ProvenanceQuantumScanStatus
+from .._provenance import ProvenanceInitQuantumModel, ProvenanceQuantumAttemptModel, ProvenanceQuantumModel
 from ._communicators import ScannerCommunicator
-from ._structs import IngestRequest, ScanReport
+from ._structs import IngestRequest, InProgressScan, ScanReport, ScanStatus, WriteRequest
 
 
 @dataclasses.dataclass
@@ -90,7 +94,7 @@ class Scanner(AbstractContextManager):
         if self.comms.config.mock_storage_classes:
             import lsst.pipe.base.tests.mocks  # noqa: F401
         self.comms.log.verbose("Reading from predicted quantum graph.")
-        self.reader = self.comms.exit_stack.enter_context(
+        self.reader = self.comms.enter(
             PredictedQuantumGraphReader.open(self.predicted_path, import_mode=TaskImportMode.DO_NOT_IMPORT)
         )
         self.reader.read_dimension_data()
@@ -161,7 +165,7 @@ class Scanner(AbstractContextManager):
         Notes
         -----
         This method is designed to run as the ``target`` in
-        `WorkerFactory.make_worker`.
+        `WorkerContext.make_worker`.
         """
         with comms, Scanner(predicted_path, butler_path, comms) as scanner:
             scanner.loop()
@@ -192,7 +196,7 @@ class Scanner(AbstractContextManager):
         ref = self.reader.components.make_dataset_ref(predicted)
         return self.qbb.stored(ref)
 
-    def scan_quantum(self, quantum_id: uuid.UUID) -> ProvenanceQuantumScanModels:
+    def scan_quantum(self, quantum_id: uuid.UUID) -> InProgressScan:
         """Scan for a quantum's completion and error status, and its output
         datasets' existence.
 
@@ -203,38 +207,76 @@ class Scanner(AbstractContextManager):
 
         Returns
         -------
-        result : `ProvenanceQuantumScanModels`
+        result : `InProgressScan`
             Scan result struct.
         """
         if (predicted_quantum := self.init_quanta.get(quantum_id)) is not None:
-            result = ProvenanceQuantumScanModels(
-                predicted_quantum.quantum_id, status=ProvenanceQuantumScanStatus.INIT
-            )
+            result = InProgressScan(predicted_quantum.quantum_id, status=ScanStatus.INIT)
             self.comms.log.debug("Created init scan for %s (%s)", quantum_id, predicted_quantum.task_label)
         else:
             self.reader.read_quantum_datasets([quantum_id])
-            predicted_quantum = self.reader.components.quantum_datasets.pop(quantum_id)
+            predicted_quantum = self.reader.components.quantum_datasets[quantum_id]
             self.comms.log.debug(
                 "Scanning %s (%s@%s)",
                 quantum_id,
                 predicted_quantum.task_label,
                 predicted_quantum.data_coordinate,
             )
-            logs = self._read_log(predicted_quantum)
-            metadata = self._read_metadata(predicted_quantum)
-            result = ProvenanceQuantumScanModels.from_metadata_and_logs(
-                predicted_quantum, metadata, logs, incomplete=self.comms.config.incomplete
-            )
-            if result.status is ProvenanceQuantumScanStatus.ABANDONED:
-                self.comms.log.debug("Abandoning scan for failed quantum %s.", quantum_id)
+            result = InProgressScan(predicted_quantum.quantum_id, ScanStatus.INCOMPLETE)
+            del self.reader.components.quantum_datasets[quantum_id]
+            last_attempt = ProvenanceQuantumAttemptModel()
+            if not self._read_log(predicted_quantum, result, last_attempt):
+                self.comms.log.debug("Abandoning scan for %s; no log dataset.", quantum_id)
+                self.comms.report_scan(ScanReport(result.quantum_id, result.status))
+                return result
+            if not self._read_metadata(predicted_quantum, result, last_attempt):
+                # We found the log dataset, but no metadata; this means the
+                # quantum failed, but a retry might still happen that could
+                # turn it into a success if we can't yet assume the run is
+                # complete.
+                self.comms.log.debug("Abandoning scan for %s.", quantum_id)
                 self.comms.report_scan(ScanReport(result.quantum_id, result.status))
                 return result
+            last_attempt.attempt = len(result.attempts)
+            result.attempts.append(last_attempt)
+        assert result.status is not ScanStatus.INCOMPLETE
+        assert result.status is not ScanStatus.ABANDONED
+
+        if len(result.logs.attempts) < len(result.attempts):
+            # Logs were not found for this attempt; must have been a hard error
+            # that kept the `finally` block from running or otherwise
+            # interrupted the writing of the logs.
+            result.logs.attempts.append(None)
+            if result.status is ScanStatus.SUCCESSFUL:
+                # But we found the metadata!  Either that hard error happened
+                # at a very unlucky time (in between those two writes), or
+                # something even weirder happened.
+                result.attempts[-1].status = QuantumAttemptStatus.LOGS_MISSING
+            else:
+                result.attempts[-1].status = QuantumAttemptStatus.FAILED
+        if len(result.metadata.attempts) < len(result.attempts):
+            # Metadata missing usually just means a failure.  In any case, the
+            # status will already be correct, either because it was set to a
+            # failure when we read the logs, or left at UNKNOWN if there were
+            # no logs.  Note that scanners never process BLOCKED quanta at all.
+            result.metadata.attempts.append(None)
+        assert len(result.logs.attempts) == len(result.attempts) or len(result.metadata.attempts) == len(
+            result.attempts
+        ), (
+            "The only way we can add more than one quantum attempt is by "
+            "extracting info stored with the logs, and that always appends "
+            "a log attempt and a metadata attempt, so this must be a bug in "
+            "the scanner."
+        )
+        # Scan for output dataset existence, skipping any the metadata reported
+        # on as well as and the metadata and logs themselves (since we just
+        # checked those).
         for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
-            if predicted_output.dataset_id not in result.output_existence:
-                result.output_existence[predicted_output.dataset_id] = self.scan_dataset(predicted_output)
+            if predicted_output.dataset_id not in result.outputs:
+                result.outputs[predicted_output.dataset_id] = self.scan_dataset(predicted_output)
         to_ingest = self._make_ingest_request(predicted_quantum, result)
-        if self.comms.config.is_writing_provenance:
-            to_write = result.to_scan_data(predicted_quantum, compressor=self.compressor)
+        if self.comms.config.output_path is not None:
+            to_write = self._make_write_request(predicted_quantum, result)
             self.comms.request_write(to_write)
         self.comms.request_ingest(to_ingest)
         self.comms.report_scan(ScanReport(result.quantum_id, result.status))
@@ -242,7 +284,7 @@ class Scanner(AbstractContextManager):
         return result
 
     def _make_ingest_request(
-        self, predicted_quantum: PredictedQuantumDatasetsModel, result: ProvenanceQuantumScanModels
+        self, predicted_quantum: PredictedQuantumDatasetsModel, result: InProgressScan
     ) -> IngestRequest:
         """Make an ingest request from a quantum scan.
 
@@ -250,7 +292,7 @@ class Scanner(AbstractContextManager):
         ----------
         predicted_quantum : `PredictedQuantumDatasetsModel`
             Information about the predicted quantum.
-        result : `ProvenanceQuantumScanModels`
+        result : `InProgressScan`
             Result of a quantum scan.
 
         Returns
@@ -261,36 +303,79 @@ class Scanner(AbstractContextManager):
         predicted_outputs_by_id = {
             d.dataset_id: d for d in itertools.chain.from_iterable(predicted_quantum.outputs.values())
         }
+        to_ingest_predicted: list[PredictedDatasetModel] = []
         to_ingest_refs: list[DatasetRef] = []
-        to_ignore: set[uuid.UUID] = set()
-        if self.comms.config.promise_ingest_graph:
-            if result.status is ProvenanceQuantumScanStatus.INIT:
-                if predicted_quantum.task_label:  # i.e. not the 'packages' producer
-                    to_ignore.add(
-                        predicted_quantum.outputs[acc.CONFIG_INIT_OUTPUT_CONNECTION_NAME][0].dataset_id
-                    )
-            else:
-                to_ignore.add(predicted_quantum.outputs[acc.METADATA_OUTPUT_CONNECTION_NAME][0].dataset_id)
-                to_ignore.add(predicted_quantum.outputs[acc.LOG_OUTPUT_CONNECTION_NAME][0].dataset_id)
-        for dataset_id, was_produced in result.output_existence.items():
-            if was_produced and dataset_id not in to_ignore:
+        for dataset_id, was_produced in result.outputs.items():
+            if was_produced:
                 predicted_output = predicted_outputs_by_id[dataset_id]
+                to_ingest_predicted.append(predicted_output)
                 to_ingest_refs.append(self.reader.components.make_dataset_ref(predicted_output))
         to_ingest_records = self.qbb._datastore.export_predicted_records(to_ingest_refs)
-        return IngestRequest(result.quantum_id, to_ingest_refs, to_ingest_records)
+        return IngestRequest(result.quantum_id, to_ingest_predicted, to_ingest_records)
 
-    def _read_metadata(self, predicted_quantum: PredictedQuantumDatasetsModel) -> TaskMetadata | None:
-        """Attempt to read the metadata dataset for a quantum.
+    def _make_write_request(
+        self, predicted_quantum: PredictedQuantumDatasetsModel, result: InProgressScan
+    ) -> WriteRequest:
+        """Make a write request from a quantum scan.
 
         Parameters
         ----------
         predicted_quantum : `PredictedQuantumDatasetsModel`
             Information about the predicted quantum.
+        result : `InProgressScan`
+            Result of a quantum scan.
 
         Returns
         -------
-        metadata : `...TaskMetadata` or `None`
-            Task metadata.
+        write_request : `WriteRequest`
+            A request to be sent to the writer.
+        """
+        quantum: ProvenanceInitQuantumModel | ProvenanceQuantumModel
+        if result.status is ScanStatus.INIT:
+            quantum = ProvenanceInitQuantumModel.from_predicted(predicted_quantum)
+        else:
+            quantum = ProvenanceQuantumModel.from_predicted(predicted_quantum)
+            quantum.attempts = result.attempts
+        request = WriteRequest(
+            result.quantum_id,
+            result.status,
+            existing_outputs={
+                dataset_id for dataset_id, was_produced in result.outputs.items() if was_produced
+            },
+            quantum=quantum.model_dump_json().encode(),
+            logs=result.logs.model_dump_json().encode() if result.logs.attempts else b"",
+            metadata=result.metadata.model_dump_json().encode() if result.metadata.attempts else b"",
+        )
+        if self.compressor is not None:
+            request.quantum = self.compressor.compress(request.quantum)
+            request.logs = self.compressor.compress(request.logs) if request.logs else b""
+            request.metadata = self.compressor.compress(request.metadata) if request.metadata else b""
+            request.is_compressed = True
+        return request
+
+    def _read_metadata(
+        self,
+        predicted_quantum: PredictedQuantumDatasetsModel,
+        result: InProgressScan,
+        last_attempt: ProvenanceQuantumAttemptModel,
+    ) -> bool:
+        """Attempt to read the metadata dataset for a quantum to extract
+        provenance information from it.
+
+        Parameters
+        ----------
+        predicted_quantum : `PredictedQuantumDatasetsModel`
+            Information about the predicted quantum.
+        result : `InProgressScan`
+            Result object to be modified in-place.
+        last_attempt : `ScanningProvenanceQuantumAttemptModel`
+            Structure to fill in with information about the last attempt to
+            run this quantum.
+
+        Returns
+        -------
+        complete : `bool`
+            Whether the quantum is complete.
         """
         (predicted_dataset,) = predicted_quantum.outputs[acc.METADATA_OUTPUT_CONNECTION_NAME]
         ref = self.reader.components.make_dataset_ref(predicted_dataset)
@@ -298,28 +383,129 @@ class Scanner(AbstractContextManager):
             # This assumes QBB metadata writes are atomic, which should be the
             # case. If it's not we'll probably get pydantic validation errors
             # here.
-            return self.qbb.get(ref, storageClass="TaskMetadata")
+            metadata: TaskMetadata = self.qbb.get(ref, storageClass="TaskMetadata")
         except FileNotFoundError:
-            return None
-
-    def _read_log(self, predicted_quantum: PredictedQuantumDatasetsModel) -> ButlerLogRecords | None:
-        """Attempt to read the log dataset for a quantum.
+            result.outputs[ref.id] = False
+            if self.comms.config.assume_complete:
+                result.status = ScanStatus.FAILED
+            else:
+                result.status = ScanStatus.ABANDONED
+                return False
+        else:
+            result.status = ScanStatus.SUCCESSFUL
+            result.outputs[ref.id] = True
+            last_attempt.status = QuantumAttemptStatus.SUCCESSFUL
+            try:
+                # Int conversion guards against spurious conversion to
+                # float that can apparently sometimes happen in
+                # TaskMetadata.
+                last_attempt.caveats = QuantumSuccessCaveats(int(metadata["quantum"]["caveats"]))
+            except LookupError:
+                pass
+            try:
+                last_attempt.exception = ExceptionInfo._from_metadata(
+                    metadata[predicted_quantum.task_label]["failure"]
+                )
+            except LookupError:
+                pass
+            try:
+                for id_str in ensure_iterable(metadata["quantum"].getArray("outputs")):
+                    result.outputs[uuid.UUID(id_str)]
+            except LookupError:
+                pass
+            else:
+                # If the metadata told us what it wrote, anything not in that
+                # list was not written.
+                for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
+                    result.outputs.setdefault(predicted_output.dataset_id, False)
+            last_attempt.resource_usage = QuantumResourceUsage.from_task_metadata(metadata)
+            result.metadata.attempts.append(metadata)
+        return True
+
+    def _read_log(
+        self,
+        predicted_quantum: PredictedQuantumDatasetsModel,
+        result: InProgressScan,
+        last_attempt: ProvenanceQuantumAttemptModel,
+    ) -> bool:
+        """Attempt to read the log dataset for a quantum to test for the
+        quantum's completion (the log is always written last) and aggregate
+        the log content in the provenance quantum graph.
 
         Parameters
         ----------
         predicted_quantum : `PredictedQuantumDatasetsModel`
             Information about the predicted quantum.
+        result : `InProgressScan`
+            Result object to be modified in-place.
+        last_attempt : `ScanningProvenanceQuantumAttemptModel`
+            Structure to fill in with information about the last attempt to
+            run this quantum.
 
         Returns
         -------
-        logs : `lsst.daf.butler.logging.ButlerLogRecords` or `None`
-            Task logs.
+        complete : `bool`
+            Whether the quantum is complete.
         """
         (predicted_dataset,) = predicted_quantum.outputs[acc.LOG_OUTPUT_CONNECTION_NAME]
         ref = self.reader.components.make_dataset_ref(predicted_dataset)
         try:
             # This assumes QBB log writes are atomic, which should be the case.
             # If it's not we'll probably get pydantic validation errors here.
-            return self.qbb.get(ref)
+            log_records: ButlerLogRecords = self.qbb.get(ref)
         except FileNotFoundError:
-            return None
+            result.outputs[ref.id] = False
+            if self.comms.config.assume_complete:
+                result.status = ScanStatus.FAILED
+            else:
+                result.status = ScanStatus.ABANDONED
+                return False
+        else:
+            # Set the attempt's run status to FAILED, since the default is
+            # UNKNOWN (i.e. logs *and* metadata are missing) and we now know
+            # the logs exist.  This will usually get replaced by SUCCESSFUL
+            # when we look for metadata next.
+            last_attempt.status = QuantumAttemptStatus.FAILED
+            result.outputs[ref.id] = True
+            if log_records.extra:
+                log_extra = _ExecutionLogRecordsExtra.model_validate(log_records.extra)
+                self._extract_from_log_extra(log_extra, result, last_attempt=last_attempt)
+            result.logs.attempts.append(list(log_records))
+        return True
+
+    def _extract_from_log_extra(
+        self,
+        log_extra: _ExecutionLogRecordsExtra,
+        result: InProgressScan,
+        last_attempt: ProvenanceQuantumAttemptModel | None,
+    ) -> None:
+        for previous_attempt_log_extra in log_extra.previous_attempts:
+            self._extract_from_log_extra(previous_attempt_log_extra, result, last_attempt=None)
+        quantum_attempt: ProvenanceQuantumAttemptModel
+        if last_attempt is None:
+            # This is not the last attempt, so it must be a failure.
+            quantum_attempt = ProvenanceQuantumAttemptModel(
+                attempt=len(result.attempts), status=QuantumAttemptStatus.FAILED
+            )
+            # We also need to get the logs from this extra provenance, since
+            # they won't be the main section of the log records.
+            result.logs.attempts.append(log_extra.logs)
+            # The special last attempt is only appended after we attempt to
+            # read metadata later, but we have to append this one now.
+            result.attempts.append(quantum_attempt)
+        else:
+            assert not log_extra.logs, "Logs for the last attempt should not be stored in the extra JSON."
+            quantum_attempt = last_attempt
+        if log_extra.exception is not None or log_extra.metadata is not None or last_attempt is None:
+            # We won't be getting a separate metadata dataset, so anything we
+            # might get from the metadata has to come from this extra
+            # provenance in the logs.
+            quantum_attempt.exception = log_extra.exception
+            if log_extra.metadata is not None:
+                quantum_attempt.resource_usage = QuantumResourceUsage.from_task_metadata(log_extra.metadata)
+                result.metadata.attempts.append(log_extra.metadata)
+            else:
+                result.metadata.attempts.append(None)
+        # Regardless of whether this is the last attempt or not, we can only
+        # get the previous_process_quanta from the log extra.
+        quantum_attempt.previous_process_quanta.extend(log_extra.previous_process_quanta)