lsst-pipe-base 29.2025.4500__py3-none-any.whl → 29.2025.4700__py3-none-any.whl

lsst/pipe/base/quantum_graph/aggregator/_scanner.py

@@ -39,10 +39,10 @@ from lsst.daf.butler import ButlerLogRecords, DatasetRef, QuantumBackedButler
 from lsst.utils.iteration import ensure_iterable
 
 from ... import automatic_connection_constants as acc
-from ..._status import QuantumSuccessCaveats
+from ..._status import ExceptionInfo, QuantumAttemptStatus, QuantumSuccessCaveats
 from ..._task_metadata import TaskMetadata
+from ...log_capture import _ExecutionLogRecordsExtra
 from ...pipeline_graph import PipelineGraph, TaskImportMode
-from ...quantum_provenance_graph import ExceptionInfo
 from ...resource_usage import QuantumResourceUsage
 from .._multiblock import Compressor
 from .._predicted import (
@@ -50,8 +50,9 @@ from .._predicted import (
     PredictedQuantumDatasetsModel,
     PredictedQuantumGraphReader,
 )
+from .._provenance import ProvenanceInitQuantumModel, ProvenanceQuantumAttemptModel, ProvenanceQuantumModel
 from ._communicators import ScannerCommunicator
-from ._structs import IngestRequest, ScanReport, ScanResult, ScanStatus
+from ._structs import IngestRequest, InProgressScan, ScanReport, ScanStatus, WriteRequest
 
 
 @dataclasses.dataclass
@@ -179,12 +180,12 @@ class Scanner:
         Returns
         -------
         exists : `bool``
-            Whether the dataset exists
+            Whether the dataset exists.
         """
         ref = self.reader.components.make_dataset_ref(predicted)
         return self.qbb.stored(ref)
 
-    def scan_quantum(self, quantum_id: uuid.UUID) -> ScanResult:
+    def scan_quantum(self, quantum_id: uuid.UUID) -> InProgressScan:
         """Scan for a quantum's completion and error status, and its output
         datasets' existence.
 
@@ -195,11 +196,11 @@ class Scanner:
 
         Returns
         -------
-        result : `ScanResult`
+        result : `InProgressScan`
             Scan result struct.
         """
         if (predicted_quantum := self.init_quanta.get(quantum_id)) is not None:
-            result = ScanResult(predicted_quantum.quantum_id, status=ScanStatus.INIT)
+            result = InProgressScan(predicted_quantum.quantum_id, status=ScanStatus.INIT)
             self.comms.log.debug("Created init scan for %s (%s)", quantum_id, predicted_quantum.task_label)
         else:
             self.reader.read_quantum_datasets([quantum_id])
@@ -210,48 +211,69 @@ class Scanner:
                 predicted_quantum.task_label,
                 predicted_quantum.data_coordinate,
             )
-            result = ScanResult(predicted_quantum.quantum_id, ScanStatus.INCOMPLETE)
+            result = InProgressScan(predicted_quantum.quantum_id, ScanStatus.INCOMPLETE)
             del self.reader.components.quantum_datasets[quantum_id]
-            log_id = self._read_and_compress_log(predicted_quantum, result)
-            if not self.comms.config.assume_complete and not result.log:
+            last_attempt = ProvenanceQuantumAttemptModel()
+            if not self._read_log(predicted_quantum, result, last_attempt):
                 self.comms.log.debug("Abandoning scan for %s; no log dataset.", quantum_id)
-                result.status = ScanStatus.ABANDONED
                 self.comms.report_scan(ScanReport(result.quantum_id, result.status))
                 return result
-            metadata_id = self._read_and_compress_metadata(predicted_quantum, result)
-            if result.metadata:
-                result.status = ScanStatus.SUCCESSFUL
-                result.existing_outputs.add(metadata_id)
-            elif self.comms.config.assume_complete:
-                result.status = ScanStatus.FAILED
-            else:
+            if not self._read_metadata(predicted_quantum, result, last_attempt):
                 # We found the log dataset, but no metadata; this means the
                 # quantum failed, but a retry might still happen that could
                 # turn it into a success if we can't yet assume the run is
                 # complete.
                 self.comms.log.debug("Abandoning scan for %s.", quantum_id)
-                result.status = ScanStatus.ABANDONED
                 self.comms.report_scan(ScanReport(result.quantum_id, result.status))
                 return result
-            if result.log:
-                result.existing_outputs.add(log_id)
-        for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
-            if predicted_output.dataset_id not in result.existing_outputs and self.scan_dataset(
-                predicted_output
-            ):
-                result.existing_outputs.add(predicted_output.dataset_id)
-        to_ingest = self._make_ingest_request(predicted_quantum, result)
-        self.comms.report_scan(ScanReport(result.quantum_id, result.status))
+            last_attempt.attempt = len(result.attempts)
+            result.attempts.append(last_attempt)
         assert result.status is not ScanStatus.INCOMPLETE
         assert result.status is not ScanStatus.ABANDONED
+
+        if len(result.logs.attempts) < len(result.attempts):
+            # Logs were not found for this attempt; must have been a hard error
+            # that kept the `finally` block from running or otherwise
+            # interrupted the writing of the logs.
+            result.logs.attempts.append(None)
+            if result.status is ScanStatus.SUCCESSFUL:
+                # But we found the metadata!  Either that hard error happened
+                # at a very unlucky time (in between those two writes), or
+                # something even weirder happened.
+                result.attempts[-1].status = QuantumAttemptStatus.LOGS_MISSING
+            else:
+                result.attempts[-1].status = QuantumAttemptStatus.FAILED
+        if len(result.metadata.attempts) < len(result.attempts):
+            # Metadata missing usually just means a failure.  In any case, the
+            # status will already be correct, either because it was set to a
+            # failure when we read the logs, or left at UNKNOWN if there were
+            # no logs.  Note that scanners never process BLOCKED quanta at all.
+            result.metadata.attempts.append(None)
+        assert len(result.logs.attempts) == len(result.attempts) or len(result.metadata.attempts) == len(
+            result.attempts
+        ), (
+            "The only way we can add more than one quantum attempt is by "
+            "extracting info stored with the logs, and that always appends "
+            "a log attempt and a metadata attempt, so this must be a bug in "
+            "the scanner."
+        )
+        # Scan for output dataset existence, skipping any the metadata reported
+        # on as well as and the metadata and logs themselves (since we just
+        # checked those).
+        for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
+            if predicted_output.dataset_id not in result.outputs:
+                result.outputs[predicted_output.dataset_id] = self.scan_dataset(predicted_output)
+        to_ingest = self._make_ingest_request(predicted_quantum, result)
         if self.comms.config.output_path is not None:
-            self.comms.request_write(result)
+            to_write = self._make_write_request(predicted_quantum, result)
+            self.comms.request_write(to_write)
         self.comms.request_ingest(to_ingest)
+        self.comms.report_scan(ScanReport(result.quantum_id, result.status))
         self.comms.log.debug("Finished scan for %s.", quantum_id)
         return result
 
     def _make_ingest_request(
-        self, predicted_quantum: PredictedQuantumDatasetsModel, result: ScanResult
+        self, predicted_quantum: PredictedQuantumDatasetsModel, result: InProgressScan
     ) -> IngestRequest:
         """Make an ingest request from a quantum scan.
 
@@ -259,7 +281,7 @@ class Scanner:
         ----------
         predicted_quantum : `PredictedQuantumDatasetsModel`
             Information about the predicted quantum.
-        result : `ScanResult`
+        result : `InProgressScan`
             Result of a quantum scan.
 
         Returns
@@ -272,16 +294,60 @@ class Scanner:
         }
         to_ingest_predicted: list[PredictedDatasetModel] = []
         to_ingest_refs: list[DatasetRef] = []
-        for dataset_id in result.existing_outputs:
-            predicted_output = predicted_outputs_by_id[dataset_id]
-            to_ingest_predicted.append(predicted_output)
-            to_ingest_refs.append(self.reader.components.make_dataset_ref(predicted_output))
+        for dataset_id, was_produced in result.outputs.items():
+            if was_produced:
+                predicted_output = predicted_outputs_by_id[dataset_id]
+                to_ingest_predicted.append(predicted_output)
+                to_ingest_refs.append(self.reader.components.make_dataset_ref(predicted_output))
         to_ingest_records = self.qbb._datastore.export_predicted_records(to_ingest_refs)
         return IngestRequest(result.quantum_id, to_ingest_predicted, to_ingest_records)
 
-    def _read_and_compress_metadata(
-        self, predicted_quantum: PredictedQuantumDatasetsModel, result: ScanResult
-    ) -> uuid.UUID:
+    def _make_write_request(
+        self, predicted_quantum: PredictedQuantumDatasetsModel, result: InProgressScan
+    ) -> WriteRequest:
+        """Make a write request from a quantum scan.
+
+        Parameters
+        ----------
+        predicted_quantum : `PredictedQuantumDatasetsModel`
+            Information about the predicted quantum.
+        result : `InProgressScan`
+            Result of a quantum scan.
+
+        Returns
+        -------
+        write_request : `WriteRequest`
+            A request to be sent to the writer.
+        """
+        quantum: ProvenanceInitQuantumModel | ProvenanceQuantumModel
+        if result.status is ScanStatus.INIT:
+            quantum = ProvenanceInitQuantumModel.from_predicted(predicted_quantum)
+        else:
+            quantum = ProvenanceQuantumModel.from_predicted(predicted_quantum)
+            quantum.attempts = result.attempts
+        request = WriteRequest(
+            result.quantum_id,
+            result.status,
+            existing_outputs={
+                dataset_id for dataset_id, was_produced in result.outputs.items() if was_produced
+            },
+            quantum=quantum.model_dump_json().encode(),
+            logs=result.logs.model_dump_json().encode() if result.logs.attempts else b"",
+            metadata=result.metadata.model_dump_json().encode() if result.metadata.attempts else b"",
+        )
+        if self.compressor is not None:
+            request.quantum = self.compressor.compress(request.quantum)
+            request.logs = self.compressor.compress(request.logs) if request.logs else b""
+            request.metadata = self.compressor.compress(request.metadata) if request.metadata else b""
+            request.is_compressed = True
+        return request
+
+    def _read_metadata(
+        self,
+        predicted_quantum: PredictedQuantumDatasetsModel,
+        result: InProgressScan,
+        last_attempt: ProvenanceQuantumAttemptModel,
+    ) -> bool:
         """Attempt to read the metadata dataset for a quantum to extract
         provenance information from it.
 
@@ -289,55 +355,68 @@ class Scanner:
         ----------
         predicted_quantum : `PredictedQuantumDatasetsModel`
             Information about the predicted quantum.
-        result : `ScanResult`
+        result : `InProgressScan`
             Result object to be modified in-place.
+        last_attempt : `ScanningProvenanceQuantumAttemptModel`
+            Structure to fill in with information about the last attempt to
+            run this quantum.
 
         Returns
         -------
-        dataset_id : `uuid.UUID`
-            UUID of the metadata dataset.
+        complete : `bool`
+            Whether the quantum is complete.
         """
-        assert not result.metadata, "We shouldn't be scanning again if we already read the metadata."
         (predicted_dataset,) = predicted_quantum.outputs[acc.METADATA_OUTPUT_CONNECTION_NAME]
         ref = self.reader.components.make_dataset_ref(predicted_dataset)
         try:
             # This assumes QBB metadata writes are atomic, which should be the
             # case. If it's not we'll probably get pydantic validation errors
             # here.
-            content: TaskMetadata = self.qbb.get(ref, storageClass="TaskMetadata")
+            metadata: TaskMetadata = self.qbb.get(ref, storageClass="TaskMetadata")
         except FileNotFoundError:
-            if not self.comms.config.assume_complete:
-                return ref.id
+            result.outputs[ref.id] = False
+            if self.comms.config.assume_complete:
+                result.status = ScanStatus.FAILED
+            else:
+                result.status = ScanStatus.ABANDONED
+                return False
         else:
+            result.status = ScanStatus.SUCCESSFUL
+            result.outputs[ref.id] = True
+            last_attempt.status = QuantumAttemptStatus.SUCCESSFUL
             try:
                 # Int conversion guards against spurious conversion to
                 # float that can apparently sometimes happen in
                 # TaskMetadata.
-                result.caveats = QuantumSuccessCaveats(int(content["quantum"]["caveats"]))
+                last_attempt.caveats = QuantumSuccessCaveats(int(metadata["quantum"]["caveats"]))
             except LookupError:
                 pass
             try:
-                result.exception = ExceptionInfo._from_metadata(
-                    content[predicted_quantum.task_label]["failure"]
+                last_attempt.exception = ExceptionInfo._from_metadata(
+                    metadata[predicted_quantum.task_label]["failure"]
                 )
             except LookupError:
                 pass
             try:
-                result.existing_outputs = {
-                    uuid.UUID(id_str) for id_str in ensure_iterable(content["quantum"].getArray("outputs"))
-                }
+                for id_str in ensure_iterable(metadata["quantum"].getArray("outputs")):
+                    result.outputs[uuid.UUID(id_str)]
             except LookupError:
                 pass
-            result.resource_usage = QuantumResourceUsage.from_task_metadata(content)
-            result.metadata = content.model_dump_json().encode()
-            if self.compressor is not None:
-                result.metadata = self.compressor.compress(result.metadata)
-                result.is_compressed = True
-        return ref.id
-
-    def _read_and_compress_log(
-        self, predicted_quantum: PredictedQuantumDatasetsModel, result: ScanResult
-    ) -> uuid.UUID:
+            else:
+                # If the metadata told us what it wrote, anything not in that
+                # list was not written.
+                for predicted_output in itertools.chain.from_iterable(predicted_quantum.outputs.values()):
+                    result.outputs.setdefault(predicted_output.dataset_id, False)
+            last_attempt.resource_usage = QuantumResourceUsage.from_task_metadata(metadata)
+            result.metadata.attempts.append(metadata)
+        return True
+
+    def _read_log(
+        self,
+        predicted_quantum: PredictedQuantumDatasetsModel,
+        result: InProgressScan,
+        last_attempt: ProvenanceQuantumAttemptModel,
+    ) -> bool:
         """Attempt to read the log dataset for a quantum to test for the
         quantum's completion (the log is always written last) and aggregate
         the log content in the provenance quantum graph.
@@ -346,26 +425,76 @@ class Scanner:
         ----------
         predicted_quantum : `PredictedQuantumDatasetsModel`
             Information about the predicted quantum.
-        result : `ScanResult`
+        result : `InProgressScan`
             Result object to be modified in-place.
+        last_attempt : `ScanningProvenanceQuantumAttemptModel`
+            Structure to fill in with information about the last attempt to
+            run this quantum.
 
         Returns
         -------
-        dataset_id : `uuid.UUID`
-            UUID of the log dataset.
+        complete : `bool`
+            Whether the quantum is complete.
         """
         (predicted_dataset,) = predicted_quantum.outputs[acc.LOG_OUTPUT_CONNECTION_NAME]
         ref = self.reader.components.make_dataset_ref(predicted_dataset)
         try:
             # This assumes QBB log writes are atomic, which should be the case.
             # If it's not we'll probably get pydantic validation errors here.
-            content: ButlerLogRecords = self.qbb.get(ref)
+            log_records: ButlerLogRecords = self.qbb.get(ref)
         except FileNotFoundError:
-            if not self.comms.config.assume_complete:
-                return ref.id
+            result.outputs[ref.id] = False
+            if self.comms.config.assume_complete:
+                result.status = ScanStatus.FAILED
+            else:
+                result.status = ScanStatus.ABANDONED
+                return False
+        else:
+            # Set the attempt's run status to FAILED, since the default is
+            # UNKNOWN (i.e. logs *and* metadata are missing) and we now know
+            # the logs exist.  This will usually get replaced by SUCCESSFUL
+            # when we look for metadata next.
+            last_attempt.status = QuantumAttemptStatus.FAILED
+            result.outputs[ref.id] = True
+            if log_records.extra:
+                log_extra = _ExecutionLogRecordsExtra.model_validate(log_records.extra)
+                self._extract_from_log_extra(log_extra, result, last_attempt=last_attempt)
+            result.logs.attempts.append(list(log_records))
+        return True
+
+    def _extract_from_log_extra(
+        self,
+        log_extra: _ExecutionLogRecordsExtra,
+        result: InProgressScan,
+        last_attempt: ProvenanceQuantumAttemptModel | None,
+    ) -> None:
+        for previous_attempt_log_extra in log_extra.previous_attempts:
+            self._extract_from_log_extra(previous_attempt_log_extra, result, last_attempt=None)
+        quantum_attempt: ProvenanceQuantumAttemptModel
+        if last_attempt is None:
+            # This is not the last attempt, so it must be a failure.
+            quantum_attempt = ProvenanceQuantumAttemptModel(
+                attempt=len(result.attempts), status=QuantumAttemptStatus.FAILED
+            )
+            # We also need to get the logs from this extra provenance, since
+            # they won't be the main section of the log records.
+            result.logs.attempts.append(log_extra.logs)
+            # The special last attempt is only appended after we attempt to
+            # read metadata later, but we have to append this one now.
+            result.attempts.append(quantum_attempt)
         else:
-            result.log = content.model_dump_json().encode()
-            if self.compressor is not None:
-                result.log = self.compressor.compress(result.log)
-                result.is_compressed = True
-        return ref.id
+            assert not log_extra.logs, "Logs for the last attempt should not be stored in the extra JSON."
+            quantum_attempt = last_attempt
+        if log_extra.exception is not None or log_extra.metadata is not None or last_attempt is None:
+            # We won't be getting a separate metadata dataset, so anything we
+            # might get from the metadata has to come from this extra
+            # provenance in the logs.
+            quantum_attempt.exception = log_extra.exception
+            if log_extra.metadata is not None:
+                quantum_attempt.resource_usage = QuantumResourceUsage.from_task_metadata(log_extra.metadata)
+                result.metadata.attempts.append(log_extra.metadata)
+            else:
+                result.metadata.attempts.append(None)
+        # Regardless of whether this is the last attempt or not, we can only
+        # get the previous_process_quanta from the log extra.
+        quantum_attempt.previous_process_quanta.extend(log_extra.previous_process_quanta)

lsst/pipe/base/quantum_graph/aggregator/_structs.py

@@ -28,10 +28,11 @@
 from __future__ import annotations
 
 __all__ = (
+    "InProgressScan",
     "IngestRequest",
     "ScanReport",
-    "ScanResult",
     "ScanStatus",
+    "WriteRequest",
 )
 
 import dataclasses
@@ -40,11 +41,13 @@ import uuid
 
 from lsst.daf.butler.datastore.record_data import DatastoreRecordData
 
-from ..._status import QuantumSuccessCaveats
-from ...quantum_provenance_graph import ExceptionInfo, QuantumRunStatus
-from ...resource_usage import QuantumResourceUsage
 from .._common import DatastoreName
 from .._predicted import PredictedDatasetModel
+from .._provenance import (
+    ProvenanceLogRecordsModel,
+    ProvenanceQuantumAttemptModel,
+    ProvenanceTaskMetadataModel,
+)
 
 
 class ScanStatus(enum.Enum):
@@ -117,8 +120,8 @@ class IngestRequest:
 
 
 @dataclasses.dataclass
-class ScanResult:
-    """A struct that represents the result of scanning a quantum."""
+class InProgressScan:
+    """A struct that represents a quantum that is being scanned."""
 
     quantum_id: uuid.UUID
     """Unique ID for the quantum."""
@@ -126,42 +129,49 @@ class ScanResult:
     status: ScanStatus
     """Combined status for the scan and the execution of the quantum."""
 
-    caveats: QuantumSuccessCaveats | None = None
-    """Flags indicating caveats on successful quanta."""
+    attempts: list[ProvenanceQuantumAttemptModel] = dataclasses.field(default_factory=list)
+    """Provenance information about each attempt to run the quantum."""
+
+    outputs: dict[uuid.UUID, bool] = dataclasses.field(default_factory=dict)
+    """Unique IDs of the output datasets mapped to whether they were actually
+    produced.
+    """
+
+    metadata: ProvenanceTaskMetadataModel = dataclasses.field(default_factory=ProvenanceTaskMetadataModel)
+    """Task metadata information for each attempt.
+    """
+
+    logs: ProvenanceLogRecordsModel = dataclasses.field(default_factory=ProvenanceLogRecordsModel)
+    """Log records for each attempt.
+    """
+
+
+@dataclasses.dataclass
+class WriteRequest:
+    """A struct that represents a request to write provenance for a quantum."""
 
-    exception: ExceptionInfo | None = None
-    """Information about an exception raised when the quantum was executing."""
+    quantum_id: uuid.UUID
+    """Unique ID for the quantum."""
 
-    resource_usage: QuantumResourceUsage | None = None
-    """Resource usage information (timing, memory use) for this quantum."""
+    status: ScanStatus
+    """Combined status for the scan and the execution of the quantum."""
 
     existing_outputs: set[uuid.UUID] = dataclasses.field(default_factory=set)
     """Unique IDs of the output datasets that were actually written."""
 
+    quantum: bytes = b""
+    """Serialized quantum provenance model.
+
+    This may be empty for quanta that had no attempts.
+    """
+
     metadata: bytes = b""
-    """Raw content of the metadata dataset."""
+    """Serialized task metadata."""
 
-    log: bytes = b""
-    """Raw content of the log dataset."""
+    logs: bytes = b""
+    """Serialized logs."""
 
     is_compressed: bool = False
-    """Whether the `metadata` and `log` attributes are compressed."""
-
-    def get_run_status(self) -> QuantumRunStatus:
-        """Translate the scan status and metadata/log presence into a run
-        status.
-        """
-        if self.status is ScanStatus.BLOCKED:
-            return QuantumRunStatus.BLOCKED
-        if self.status is ScanStatus.INIT:
-            return QuantumRunStatus.SUCCESSFUL
-        if self.log:
-            if self.metadata:
-                return QuantumRunStatus.SUCCESSFUL
-            else:
-                return QuantumRunStatus.FAILED
-        else:
-            if self.metadata:
-                return QuantumRunStatus.LOGS_MISSING
-            else:
-                return QuantumRunStatus.METADATA_MISSING
+    """Whether the `quantum`, `metadata`, and `log` attributes are
+    compressed.
+    """

lsst/pipe/base/quantum_graph/aggregator/_supervisor.py

@@ -30,6 +30,7 @@ from __future__ import annotations
 __all__ = ("aggregate_graph",)
 
 import dataclasses
+import itertools
 import uuid
 
 import astropy.units as u
@@ -53,7 +54,7 @@ from ._communicators import (
 from ._config import AggregatorConfig
 from ._ingester import Ingester
 from ._scanner import Scanner
-from ._structs import ScanReport, ScanResult, ScanStatus
+from ._structs import ScanReport, ScanStatus, WriteRequest
 from ._writer import Writer
 
 
@@ -87,19 +88,15 @@ class Supervisor:
             reader.read_init_quanta()
             self.predicted = reader.components
         self.comms.progress.log.info("Analyzing predicted graph.")
-        uuid_by_index = {
-            quantum_index: quantum_id for quantum_id, quantum_index in self.predicted.quantum_indices.items()
-        }
-        xgraph = networkx.DiGraph(
-            [(uuid_by_index[a], uuid_by_index[b]) for a, b in self.predicted.thin_graph.edges]
-        )
+        xgraph = networkx.DiGraph(self.predicted.thin_graph.edges)
         # Make sure all quanta are in the graph, even if they don't have any
         # quantum-only edges.
-        xgraph.add_nodes_from(uuid_by_index.values())
+        for thin_quantum in itertools.chain.from_iterable(self.predicted.thin_graph.quanta.values()):
+            xgraph.add_node(thin_quantum.quantum_id)
         # Add init quanta as nodes without edges, because the scanner should
         # only be run after init outputs are all written and hence we don't
         # care when we process them.
-        for init_quantum in self.predicted.init_quanta.root[1:]:  # skip 'packages' producer
+        for init_quantum in self.predicted.init_quanta.root:
             xgraph.add_node(init_quantum.quantum_id)
         self.walker = GraphWalker(xgraph)
 
@@ -107,9 +104,10 @@ class Supervisor:
         """Scan the outputs of the quantum graph to gather provenance and
         ingest outputs.
         """
-        self.comms.progress.set_n_quanta(
-            self.predicted.header.n_quanta + len(self.predicted.init_quanta.root)
-        )
+        n_quanta = self.predicted.header.n_quanta + len(self.predicted.init_quanta.root)
+        self.comms.progress.scans.total = n_quanta
+        self.comms.progress.writes.total = n_quanta
+        self.comms.progress.quantum_ingests.total = n_quanta
         ready_set: set[uuid.UUID] = set()
         for ready_quanta in self.walker:
             self.comms.log.debug("Sending %d new quanta to scan queue.", len(ready_quanta))
@@ -136,9 +134,9 @@ class Supervisor:
                 blocked_quanta = self.walker.fail(scan_report.quantum_id)
                 for blocked_quantum_id in blocked_quanta:
                     if self.comms.config.output_path is not None:
-                        self.comms.request_write(ScanResult(blocked_quantum_id, status=ScanStatus.BLOCKED))
-                    self.comms.progress.report_scan()
-                self.comms.progress.report_ingests(len(blocked_quanta))
+                        self.comms.request_write(WriteRequest(blocked_quantum_id, status=ScanStatus.BLOCKED))
+                    self.comms.progress.scans.update(1)
+                self.comms.progress.quantum_ingests.update(len(blocked_quanta))
             case ScanStatus.ABANDONED:
                 self.comms.log.debug("Abandoning scan for %s: quantum has not succeeded (yet).")
                 self.walker.fail(scan_report.quantum_id)
@@ -147,7 +145,7 @@ class Supervisor:
                 raise AssertionError(
                     f"Unexpected status {unexpected!r} in scanner loop for {scan_report.quantum_id}."
                 )
-        self.comms.progress.report_scan()
+        self.comms.progress.scans.update(1)
 
 
 def aggregate_graph(predicted_path: str, butler_path: str, config: AggregatorConfig) -> None:
@@ -159,7 +157,7 @@ def aggregate_graph(predicted_path: str, butler_path: str, config: AggregatorCon
         Path to the predicted quantum graph.
     butler_path : `str`
         Path or alias to the central butler repository.
-    config: `AggregatorConfig`
+    config : `AggregatorConfig`
         Configuration for the aggregator.
     """
     log = getLogger("lsst.pipe.base.quantum_graph.aggregator")