lsst-pipe-base 29.2025.4400__py3-none-any.whl → 29.2025.4600__py3-none-any.whl

lsst/pipe/base/quantum_graph/aggregator/_communicators.py

@@ -59,7 +59,7 @@ from typing import Any, Literal, Self, TypeAlias, TypeVar, Union
 from lsst.utils.logging import VERBOSE, LsstLogAdapter
 
 from ._config import AggregatorConfig
-from ._progress import Progress, make_worker_log
+from ._progress import ProgressManager, make_worker_log
 from ._structs import IngestRequest, ScanReport, ScanResult
 
 _T = TypeVar("_T")
@@ -340,7 +340,7 @@ class SupervisorCommunicator:
         config: AggregatorConfig,
     ) -> None:
         self.config = config
-        self.progress = Progress(log, config)
+        self.progress = ProgressManager(log, config)
         self.n_scanners = n_scanners
         # The supervisor sends scan requests to scanners on this queue.
         # When complete, the supervisor sends n_scanners sentinals and each
@@ -406,13 +406,13 @@ class SupervisorCommunicator:
                     pass
                 case _Sentinel.INGESTER_DONE:
                     self._ingester_done = True
-                    self.progress.finish_ingests()
+                    self.progress.quantum_ingests.close()
                 case _Sentinel.SCANNER_DONE:
                     self._n_scanners_done += 1
-                    self.progress.finish_scans()
+                    self.progress.scans.close()
                 case _Sentinel.WRITER_DONE:
                     self._writer_done = True
-                    self.progress.finish_writes()
+                    self.progress.writes.close()
                 case unexpected:
                     raise AssertionError(f"Unexpected message {unexpected!r} to supervisor.")
             self.log.verbose(
@@ -530,9 +530,9 @@ class SupervisorCommunicator:
                 if not already_failing:
                     raise FatalWorkerError()
             case _IngestReport(n_producers=n_producers):
-                self.progress.report_ingests(n_producers)
+                self.progress.quantum_ingests.update(n_producers)
             case _Sentinel.WRITE_REPORT:
-                self.progress.report_write()
+                self.progress.writes.update(1)
             case _ProgressLog(message=message, level=level):
                 self.progress.log.log(level, "%s [after %0.1fs]", message, self.progress.elapsed_time)
             case _:
@@ -626,10 +626,10 @@ class WorkerCommunicator:
 
         Parameters
         ----------
-        message : `str`
-            Log message.
         level : `int`
             Log level.  Should be ``VERBOSE`` or higher.
+        message : `str`
+            Log message.
         """
         self._reports.put(_ProgressLog(message=message, level=level), block=False)
 

lsst/pipe/base/quantum_graph/aggregator/_progress.py

@@ -27,20 +27,86 @@
 
 from __future__ import annotations
 
-__all__ = ("Progress", "make_worker_log")
+__all__ = ("ProgressCounter", "ProgressManager", "make_worker_log")
 
 import logging
 import os
 import time
 from types import TracebackType
-from typing import Self
+from typing import Any, Self
 
 from lsst.utils.logging import TRACE, VERBOSE, LsstLogAdapter, PeriodicLogger, getLogger
 
 from ._config import AggregatorConfig
 
 
-class Progress:
+class ProgressCounter:
+    """A progress tracker for an individual aspect of the aggregation process.
+
+    Parameters
+    ----------
+    parent : `ProgressManager`
+        The parent progress manager object.
+    description : `str`
+        Human-readable description of this aspect.
+    unit : `str`
+        Unit (in plural form) for the items being counted.
+    total : `int`, optional
+        Expected total number of items.  May be set later.
+    """
+
+    def __init__(self, parent: ProgressManager, description: str, unit: str, total: int | None = None):
+        self._parent = parent
+        self.total = total
+        self._description = description
+        self._current = 0
+        self._unit = unit
+        self._bar: Any = None
+
+    def update(self, n: int) -> None:
+        """Report that ``n`` new items have been processed.
+
+        Parameters
+        ----------
+        n : `int`
+            Number of new items processed.
+        """
+        self._current += n
+        if self._parent.interactive:
+            if self._bar is None:
+                if n == self.total:
+                    return
+                from tqdm import tqdm
+
+                self._bar = tqdm(desc=self._description, total=self.total, leave=False, unit=f" {self._unit}")
+            else:
+                self._bar.update(n)
+                if self._current == self.total:
+                    self._bar.close()
+        self._parent._log_status()
+
+    def close(self) -> None:
+        """Close the counter, guaranteeing that `update` will not be called
+        again.
+        """
+        if self._bar is not None:
+            self._bar.close()
+            self._bar = None
+
+    def append_log_terms(self, msg: list[str]) -> None:
+        """Append a log message for this counter to a list if it is active.
+
+        Parameters
+        ----------
+        msg : `list` [ `str` ]
+            List of messages to concatenate into a single line and log
+            together, to be modified in-place.
+        """
+        if self.total is not None and self._current > 0 and self._current < self.total:
+            msg.append(f"{self._description} ({self._current} of {self.total} {self._unit})")
+
+
+class ProgressManager:
     """A helper class for the provenance aggregator that handles reporting
     progress to the user.
 
@@ -66,10 +132,9 @@ class Progress:
         self.log = log
         self.config = config
         self._periodic_log = PeriodicLogger(self.log, config.log_status_interval)
-        self._n_scanned: int = 0
-        self._n_ingested: int = 0
-        self._n_written: int = 0
-        self._n_quanta: int | None = None
+        self.scans = ProgressCounter(self, "scanning", "quanta")
+        self.writes = ProgressCounter(self, "writing", "quanta")
+        self.quantum_ingests = ProgressCounter(self, "ingesting outputs", "quanta")
         self.interactive = config.interactive_status
 
     def __enter__(self) -> Self:
@@ -90,29 +155,6 @@ class Progress:
             self._logging_redirect.__exit__(exc_type, exc_value, traceback)
         return None
 
-    def set_n_quanta(self, n_quanta: int) -> None:
-        """Set the total number of quanta.
-
-        Parameters
-        ----------
-        n_quanta : `int`
-            Total number of quanta, including special "init" quanta.
-
-        Notes
-        -----
-        This method must be called before any of the ``report_*`` methods.
-        """
-        self._n_quanta = n_quanta
-        if self.interactive:
-            from tqdm import tqdm
-
-            self._scan_progress = tqdm(desc="Scanning", total=n_quanta, leave=False, unit="quanta")
-            self._ingest_progress = tqdm(
-                desc="Ingesting", total=n_quanta, leave=False, smoothing=0.1, unit="quanta"
-            )
-            if self.config.output_path is not None:
-                self._write_progress = tqdm(desc="Writing", total=n_quanta, leave=False, unit="quanta")
-
     @property
     def elapsed_time(self) -> float:
         """The time in seconds since the start of the aggregator."""
@@ -120,60 +162,11 @@ class Progress:
 
     def _log_status(self) -> None:
         """Invoke the periodic logger with the current status."""
-        self._periodic_log.log(
-            "%s quanta scanned, %s quantum outputs ingested, "
-            "%s provenance quanta written (of %s) after %0.1fs.",
-            self._n_scanned,
-            self._n_ingested,
-            self._n_written,
-            self._n_quanta,
-            self.elapsed_time,
-        )
-
-    def report_scan(self) -> None:
-        """Report that a quantum was scanned."""
-        self._n_scanned += 1
-        if self.interactive:
-            self._scan_progress.update(1)
-        else:
-            self._log_status()
-
-    def finish_scans(self) -> None:
-        """Report that all scanning is done."""
-        if self.interactive:
-            self._scan_progress.close()
-
-    def report_ingests(self, n_quanta: int) -> None:
-        """Report that ingests for multiple quanta were completed.
-
-        Parameters
-        ----------
-        n_quanta : `int`
-            Number of quanta whose outputs were ingested.
-        """
-        self._n_ingested += n_quanta
-        if self.interactive:
-            self._ingest_progress.update(n_quanta)
-        else:
-            self._log_status()
-
-    def finish_ingests(self) -> None:
-        """Report that all ingests are done."""
-        if self.interactive:
-            self._ingest_progress.close()
-
-    def report_write(self) -> None:
-        """Report that a quantum's provenance was written."""
-        self._n_written += 1
-        if self.interactive:
-            self._write_progress.update()
-        else:
-            self._log_status()
-
-    def finish_writes(self) -> None:
-        """Report that all writes are done."""
-        if self.interactive:
-            self._write_progress.close()
+        log_terms: list[str] = []
+        self.scans.append_log_terms(log_terms)
+        self.writes.append_log_terms(log_terms)
+        self.quantum_ingests.append_log_terms(log_terms)
+        self._periodic_log.log("Status after %0.1fs: %s.", self.elapsed_time, "; ".join(log_terms))
 
 
 def make_worker_log(name: str, config: AggregatorConfig) -> LsstLogAdapter:

lsst/pipe/base/quantum_graph/aggregator/_scanner.py

@@ -39,10 +39,10 @@ from lsst.daf.butler import ButlerLogRecords, DatasetRef, QuantumBackedButler
 from lsst.utils.iteration import ensure_iterable
 
 from ... import automatic_connection_constants as acc
-from ..._status import QuantumSuccessCaveats
+from ..._status import ExceptionInfo, QuantumAttemptStatus, QuantumSuccessCaveats
 from ..._task_metadata import TaskMetadata
+from ...log_capture import _ExecutionLogRecordsExtra
 from ...pipeline_graph import PipelineGraph, TaskImportMode
-from ...quantum_provenance_graph import ExceptionInfo
 from ...resource_usage import QuantumResourceUsage
 from .._multiblock import Compressor
 from .._predicted import (
@@ -50,6 +50,7 @@ from .._predicted import (
     PredictedQuantumDatasetsModel,
     PredictedQuantumGraphReader,
 )
+from .._provenance import ProvenanceQuantumAttemptModel
 from ._communicators import ScannerCommunicator
 from ._structs import IngestRequest, ScanReport, ScanResult, ScanStatus
 
@@ -179,7 +180,7 @@ class Scanner:
         Returns
         -------
         exists : `bool``
-            Whether the dataset exists
+            Whether the dataset exists.
         """
         ref = self.reader.components.make_dataset_ref(predicted)
         return self.qbb.stored(ref)
@@ -212,29 +213,67 @@ class Scanner:
             )
             result = ScanResult(predicted_quantum.quantum_id, ScanStatus.INCOMPLETE)
             del self.reader.components.quantum_datasets[quantum_id]
-            log_id = self._read_and_compress_log(predicted_quantum, result)
-            if not self.comms.config.assume_complete and not result.log:
+            last_attempt = ProvenanceQuantumAttemptModel()
+            if not self._read_log(predicted_quantum, result, last_attempt):
                 self.comms.log.debug("Abandoning scan for %s; no log dataset.", quantum_id)
-                result.status = ScanStatus.ABANDONED
                 self.comms.report_scan(ScanReport(result.quantum_id, result.status))
                 return result
-            metadata_id = self._read_and_compress_metadata(predicted_quantum, result)
-            if result.metadata:
-                result.status = ScanStatus.SUCCESSFUL
-                result.existing_outputs.add(metadata_id)
-            elif self.comms.config.assume_complete:
-                result.status = ScanStatus.FAILED
-            else:
+            if not self._read_metadata(predicted_quantum, result, last_attempt):
                 # We found the log dataset, but no metadata; this means the
                 # quantum failed, but a retry might still happen that could
                 # turn it into a success if we can't yet assume the run is
                 # complete.
                 self.comms.log.debug("Abandoning scan for %s.", quantum_id)
-                result.status = ScanStatus.ABANDONED
                 self.comms.report_scan(ScanReport(result.quantum_id, result.status))
                 return result
-            if result.log:
-                result.existing_outputs.add(log_id)
+            last_attempt.attempt = len(result.attempts)
+            result.attempts.append(last_attempt)
+        assert result.status is not ScanStatus.INCOMPLETE
+        assert result.status is not ScanStatus.ABANDONED
+        assert result.log_model is not None, "Only set to None after converting to JSON."
+        assert result.metadata_model is not None, "Only set to None after converting to JSON."
+
+        if len(result.log_model.attempts) < len(result.attempts):
+            # Logs were not found for this attempt; must have been a hard error
+            # that kept the `finally` block from running or otherwise
+            # interrupted the writing of the logs.
+            result.log_model.attempts.append(None)
+            if result.status is ScanStatus.SUCCESSFUL:
+                # But we found the metadata!  Either that hard error happened
+                # at a very unlucky time (in between those two writes), or
+                # something even weirder happened.
+                result.attempts[-1].status = QuantumAttemptStatus.LOGS_MISSING
+            else:
+                result.attempts[-1].status = QuantumAttemptStatus.FAILED
+        if len(result.metadata_model.attempts) < len(result.attempts):
+            # Metadata missing usually just means a failure.  In any case, the
+            # status will already be correct, either because it was set to a
+            # failure when we read the logs, or left at UNKNOWN if there were
+            # no logs.  Note that scanners never process BLOCKED quanta at all.
+            result.metadata_model.attempts.append(None)
+        assert len(result.log_model.attempts) == len(result.attempts) or len(
+            result.metadata_model.attempts
+        ) == len(result.attempts), (
+            "The only way we can add more than one quantum attempt is by "
+            "extracting info stored with the logs, and that always appends "
+            "a log attempt and a metadata attempt, so this must be a bug in "
+            "the scanner."
+        )
+        # Now that we're done gathering the log and metadata information into
+        # models, dump them to JSON and delete the originals.
+        result.log_content = result.log_model.model_dump_json().encode()
+        result.log_model = None
+        result.metadata_content = result.metadata_model.model_dump_json().encode()
+        result.metadata_model = None
+        if self.compressor is not None:
+            if result.log_content is not None:
+                result.log_content = self.compressor.compress(result.log_content)
+            if result.metadata_content is not None:
+                result.metadata_content = self.compressor.compress(result.metadata_content)
+            result.is_compressed = True
+        # Scan for output dataset existence, skipping any the metadata reported
+        # as having been definitively written, as well as and the metadata and
+        # logs themselves (since we just checked those).
         for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
             if predicted_output.dataset_id not in result.existing_outputs and self.scan_dataset(
                 predicted_output
@@ -242,8 +281,6 @@ class Scanner:
                 result.existing_outputs.add(predicted_output.dataset_id)
         to_ingest = self._make_ingest_request(predicted_quantum, result)
         self.comms.report_scan(ScanReport(result.quantum_id, result.status))
-        assert result.status is not ScanStatus.INCOMPLETE
-        assert result.status is not ScanStatus.ABANDONED
         if self.comms.config.output_path is not None:
             self.comms.request_write(result)
         self.comms.request_ingest(to_ingest)
@@ -279,9 +316,12 @@ class Scanner:
         to_ingest_records = self.qbb._datastore.export_predicted_records(to_ingest_refs)
         return IngestRequest(result.quantum_id, to_ingest_predicted, to_ingest_records)
 
-    def _read_and_compress_metadata(
-        self, predicted_quantum: PredictedQuantumDatasetsModel, result: ScanResult
-    ) -> uuid.UUID:
+    def _read_metadata(
+        self,
+        predicted_quantum: PredictedQuantumDatasetsModel,
+        result: ScanResult,
+        last_attempt: ProvenanceQuantumAttemptModel,
+    ) -> bool:
         """Attempt to read the metadata dataset for a quantum to extract
         provenance information from it.
 
@@ -291,53 +331,62 @@ class Scanner:
             Information about the predicted quantum.
         result : `ScanResult`
             Result object to be modified in-place.
+        last_attempt : `ScanningProvenanceQuantumAttemptModel`
+            Structure to fill in with information about the last attempt to
+            run this quantum.
 
         Returns
         -------
-        dataset_id : `uuid.UUID`
-            UUID of the metadata dataset.
+        complete : `bool`
+            Whether the quantum is complete.
         """
-        assert not result.metadata, "We shouldn't be scanning again if we already read the metadata."
         (predicted_dataset,) = predicted_quantum.outputs[acc.METADATA_OUTPUT_CONNECTION_NAME]
         ref = self.reader.components.make_dataset_ref(predicted_dataset)
         try:
             # This assumes QBB metadata writes are atomic, which should be the
             # case. If it's not we'll probably get pydantic validation errors
             # here.
-            content: TaskMetadata = self.qbb.get(ref, storageClass="TaskMetadata")
+            metadata: TaskMetadata = self.qbb.get(ref, storageClass="TaskMetadata")
         except FileNotFoundError:
-            if not self.comms.config.assume_complete:
-                return ref.id
+            if self.comms.config.assume_complete:
+                result.status = ScanStatus.FAILED
+            else:
+                result.status = ScanStatus.ABANDONED
+                return False
         else:
+            result.status = ScanStatus.SUCCESSFUL
+            result.existing_outputs.add(ref.id)
+            last_attempt.status = QuantumAttemptStatus.SUCCESSFUL
             try:
                 # Int conversion guards against spurious conversion to
                 # float that can apparently sometimes happen in
                 # TaskMetadata.
-                result.caveats = QuantumSuccessCaveats(int(content["quantum"]["caveats"]))
+                last_attempt.caveats = QuantumSuccessCaveats(int(metadata["quantum"]["caveats"]))
             except LookupError:
                 pass
             try:
-                result.exception = ExceptionInfo._from_metadata(
-                    content[predicted_quantum.task_label]["failure"]
+                last_attempt.exception = ExceptionInfo._from_metadata(
+                    metadata[predicted_quantum.task_label]["failure"]
                 )
             except LookupError:
                 pass
             try:
-                result.existing_outputs = {
-                    uuid.UUID(id_str) for id_str in ensure_iterable(content["quantum"].getArray("outputs"))
-                }
+                result.existing_outputs.update(
+                    uuid.UUID(id_str) for id_str in ensure_iterable(metadata["quantum"].getArray("outputs"))
+                )
             except LookupError:
                 pass
-            result.resource_usage = QuantumResourceUsage.from_task_metadata(content)
-            result.metadata = content.model_dump_json().encode()
-            if self.compressor is not None:
-                result.metadata = self.compressor.compress(result.metadata)
-                result.is_compressed = True
-        return ref.id
-
-    def _read_and_compress_log(
-        self, predicted_quantum: PredictedQuantumDatasetsModel, result: ScanResult
-    ) -> uuid.UUID:
+            last_attempt.resource_usage = QuantumResourceUsage.from_task_metadata(metadata)
+            assert result.metadata_model is not None, "Only set to None after converting to JSON."
+            result.metadata_model.attempts.append(metadata)
+        return True
+
+    def _read_log(
+        self,
+        predicted_quantum: PredictedQuantumDatasetsModel,
+        result: ScanResult,
+        last_attempt: ProvenanceQuantumAttemptModel,
+    ) -> bool:
         """Attempt to read the log dataset for a quantum to test for the
         quantum's completion (the log is always written last) and aggregate
         the log content in the provenance quantum graph.
@@ -348,24 +397,76 @@ class Scanner:
             Information about the predicted quantum.
         result : `ScanResult`
             Result object to be modified in-place.
+        last_attempt : `ScanningProvenanceQuantumAttemptModel`
+            Structure to fill in with information about the last attempt to
+            run this quantum.
 
         Returns
         -------
-        dataset_id : `uuid.UUID`
-            UUID of the log dataset.
+        complete : `bool`
+            Whether the quantum is complete.
         """
         (predicted_dataset,) = predicted_quantum.outputs[acc.LOG_OUTPUT_CONNECTION_NAME]
         ref = self.reader.components.make_dataset_ref(predicted_dataset)
         try:
             # This assumes QBB log writes are atomic, which should be the case.
             # If it's not we'll probably get pydantic validation errors here.
-            content: ButlerLogRecords = self.qbb.get(ref)
+            log_records: ButlerLogRecords = self.qbb.get(ref)
         except FileNotFoundError:
-            if not self.comms.config.assume_complete:
-                return ref.id
+            if self.comms.config.assume_complete:
+                result.status = ScanStatus.FAILED
+            else:
+                result.status = ScanStatus.ABANDONED
+                return False
         else:
-            result.log = content.model_dump_json().encode()
-            if self.compressor is not None:
-                result.log = self.compressor.compress(result.log)
-                result.is_compressed = True
-        return ref.id
+            # Set the attempt's run status to FAILED, since the default is
+            # UNKNOWN (i.e. logs *and* metadata are missing) and we now know
+            # the logs exist.  This will usually get replaced by SUCCESSFUL
+            # when we look for metadata next.
+            last_attempt.status = QuantumAttemptStatus.FAILED
+            result.existing_outputs.add(ref.id)
+            if log_records.extra:
+                log_extra = _ExecutionLogRecordsExtra.model_validate(log_records.extra)
+                self._extract_from_log_extra(log_extra, result, last_attempt=last_attempt)
+            assert result.log_model is not None, "Only set to None after converting to JSON."
+            result.log_model.attempts.append(list(log_records))
+        return True
+
+    def _extract_from_log_extra(
+        self,
+        log_extra: _ExecutionLogRecordsExtra,
+        result: ScanResult,
+        last_attempt: ProvenanceQuantumAttemptModel | None,
+    ) -> None:
+        for previous_attempt_log_extra in log_extra.previous_attempts:
+            self._extract_from_log_extra(previous_attempt_log_extra, result, last_attempt=None)
+        quantum_attempt: ProvenanceQuantumAttemptModel
+        if last_attempt is None:
+            # This is not the last attempt, so it must be a failure.
+            quantum_attempt = ProvenanceQuantumAttemptModel(
+                attempt=len(result.attempts), status=QuantumAttemptStatus.FAILED
+            )
+            # We also need to get the logs from this extra provenance, since
+            # they won't be the main section of the log records.
+            assert result.log_model is not None, "Only set to None after converting to JSON."
+            result.log_model.attempts.append(log_extra.logs)
+            # The special last attempt is only appended after we attempt to
+            # read metadata later, but we have to append this one now.
+            result.attempts.append(quantum_attempt)
+        else:
+            assert not log_extra.logs, "Logs for the last attempt should not be stored in the extra JSON."
+            quantum_attempt = last_attempt
+        if log_extra.exception is not None or log_extra.metadata is not None or last_attempt is None:
+            # We won't be getting a separate metadata dataset, so anything we
+            # might get from the metadata has to come from this extra
+            # provenance in the logs.
+            quantum_attempt.exception = log_extra.exception
+            assert result.metadata_model is not None, "Only set to None after converting to JSON."
+            if log_extra.metadata is not None:
+                quantum_attempt.resource_usage = QuantumResourceUsage.from_task_metadata(log_extra.metadata)
+                result.metadata_model.attempts.append(log_extra.metadata)
+            else:
+                result.metadata_model.attempts.append(None)
+        # Regardless of whether this is the last attempt or not, we can only
+        # get the previous_process_quanta from the log extra.
+        quantum_attempt.previous_process_quanta.extend(log_extra.previous_process_quanta)

lsst/pipe/base/quantum_graph/aggregator/_structs.py

@@ -40,11 +40,13 @@ import uuid
 
 from lsst.daf.butler.datastore.record_data import DatastoreRecordData
 
-from ..._status import QuantumSuccessCaveats
-from ...quantum_provenance_graph import ExceptionInfo, QuantumRunStatus
-from ...resource_usage import QuantumResourceUsage
 from .._common import DatastoreName
 from .._predicted import PredictedDatasetModel
+from .._provenance import (
+    ProvenanceLogRecordsModel,
+    ProvenanceQuantumAttemptModel,
+    ProvenanceTaskMetadataModel,
+)
 
 
 class ScanStatus(enum.Enum):
@@ -126,42 +128,33 @@ class ScanResult:
     status: ScanStatus
     """Combined status for the scan and the execution of the quantum."""
 
-    caveats: QuantumSuccessCaveats | None = None
-    """Flags indicating caveats on successful quanta."""
-
-    exception: ExceptionInfo | None = None
-    """Information about an exception raised when the quantum was executing."""
-
-    resource_usage: QuantumResourceUsage | None = None
-    """Resource usage information (timing, memory use) for this quantum."""
+    attempts: list[ProvenanceQuantumAttemptModel] = dataclasses.field(default_factory=list)
+    """Provenance information about each attempt to run the quantum."""
 
     existing_outputs: set[uuid.UUID] = dataclasses.field(default_factory=set)
     """Unique IDs of the output datasets that were actually written."""
 
-    metadata: bytes = b""
-    """Raw content of the metadata dataset."""
+    metadata_model: ProvenanceTaskMetadataModel | None = dataclasses.field(
+        default_factory=ProvenanceTaskMetadataModel
+    )
+    """Task metadata information for each attempt.
+
+    This is set to `None` to keep the pickle size small after it is saved
+    to `metadata_content`.
+    """
+
+    metadata_content: bytes = b""
+    """Serialized form of `metadata_model`."""
+
+    log_model: ProvenanceLogRecordsModel | None = dataclasses.field(default_factory=ProvenanceLogRecordsModel)
+    """Log records for each attempt.
 
-    log: bytes = b""
-    """Raw content of the log dataset."""
+    This is set to `None` to keep the pickle size small after it is saved
+    to `log_content`.
+    """
+
+    log_content: bytes = b""
+    """Serialized form of `logs_model`."""
 
     is_compressed: bool = False
     """Whether the `metadata` and `log` attributes are compressed."""
-
-    def get_run_status(self) -> QuantumRunStatus:
-        """Translate the scan status and metadata/log presence into a run
-        status.
-        """
-        if self.status is ScanStatus.BLOCKED:
-            return QuantumRunStatus.BLOCKED
-        if self.status is ScanStatus.INIT:
-            return QuantumRunStatus.SUCCESSFUL
-        if self.log:
-            if self.metadata:
-                return QuantumRunStatus.SUCCESSFUL
-            else:
-                return QuantumRunStatus.FAILED
-        else:
-            if self.metadata:
-                return QuantumRunStatus.LOGS_MISSING
-            else:
-                return QuantumRunStatus.METADATA_MISSING