lsst-pipe-base 30.0.1rc1__py3-none-any.whl → 30.2025.5100__py3-none-any.whl

lsst/pipe/base/quantum_graph/aggregator/_communicators.py

@@ -31,33 +31,154 @@ __all__ = (
     "FatalWorkerError",
     "IngesterCommunicator",
     "ScannerCommunicator",
+    "SpawnProcessContext",
     "SupervisorCommunicator",
+    "ThreadingContext",
+    "WorkerContext",
 )
 
 import cProfile
 import dataclasses
 import enum
 import logging
+import multiprocessing.context
+import multiprocessing.synchronize
 import os
+import queue
 import signal
+import threading
 import time
 import uuid
-from collections.abc import Iterable, Iterator
-from contextlib import ExitStack
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Iterable, Iterator
+from contextlib import AbstractContextManager, ExitStack, contextmanager
 from traceback import format_exception
 from types import TracebackType
-from typing import Literal, Self, overload
+from typing import Any, Literal, Self, TypeAlias, TypeVar, Union
 
-from lsst.utils.logging import LsstLogAdapter
+from lsst.utils.logging import VERBOSE, LsstLogAdapter
 
-from .._provenance import ProvenanceQuantumScanData
 from ._config import AggregatorConfig
 from ._progress import ProgressManager, make_worker_log
-from ._structs import IngestRequest, ScanReport
-from ._workers import Event, Queue, Worker, WorkerFactory
+from ._structs import IngestRequest, ScanReport, WriteRequest
+
+_T = TypeVar("_T")
 
 _TINY_TIMEOUT = 0.01
 
+# multiprocessing.Queue is a type according to the standard library type stubs,
+# but it's really a function at runtime.  But since the Python <= 3.11 type
+# alias syntax uses the real runtime things we need to use strings, and hence
+# we need to use Union.  With Python 3.12's 'type' statement this gets cleaner.
+Queue: TypeAlias = Union["queue.Queue[_T]", "multiprocessing.Queue[_T]"]
+
+Event: TypeAlias = threading.Event | multiprocessing.synchronize.Event
+
+Worker: TypeAlias = threading.Thread | multiprocessing.context.SpawnProcess
+
+
+class WorkerContext(ABC):
+    """A simple abstract interface that can be implemented by both threading
+    and multiprocessing.
+    """
+
+    @abstractmethod
+    def make_queue(self) -> Queue[Any]:
+        """Make an empty queue that can be used to pass objects between
+        workers in this context.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def make_event(self) -> Event:
+        """Make an event that can be used to communicate a boolean state change
+        to workers in this context.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def make_worker(
+        self, target: Callable[..., None], args: tuple[Any, ...], name: str | None = None
+    ) -> Worker:
+        """Make a worker that runs the given callable.
+
+        Parameters
+        ----------
+        target : `~collections.abc.Callable`
+            A callable to invoke on the worker.
+        args : `tuple`
+            Positional arguments to pass to the callable.
+        name : `str`, optional
+            Human-readable name for the worker.
+
+        Returns
+        -------
+        worker : `threading.Thread` or `multiprocessing.Process`
+            Process or thread.  Will need to have its ``start`` method called
+            to actually begin.
+        """
+        raise NotImplementedError()
+
+
+class ThreadingContext(WorkerContext):
+    """An implementation of `WorkerContext` backed by the `threading`
+    module.
+    """
+
+    def make_queue(self) -> Queue[Any]:
+        return queue.Queue()
+
+    def make_event(self) -> Event:
+        return threading.Event()
+
+    def make_worker(
+        self, target: Callable[..., None], args: tuple[Any, ...], name: str | None = None
+    ) -> Worker:
+        return threading.Thread(target=target, args=args, name=name)
+
+
+class SpawnProcessContext(WorkerContext):
+    """An implementation of `WorkerContext` backed by the `multiprocessing`
+    module, with new processes started by spawning.
+    """
+
+    def __init__(self) -> None:
+        self._ctx = multiprocessing.get_context("spawn")
+
+    def make_queue(self) -> Queue[Any]:
+        return self._ctx.Queue()
+
+    def make_event(self) -> Event:
+        return self._ctx.Event()
+
+    def make_worker(
+        self, target: Callable[..., None], args: tuple[Any, ...], name: str | None = None
+    ) -> Worker:
+        return self._ctx.Process(target=target, args=args, name=name)
+
+
+def _get_from_queue(q: Queue[_T], block: bool = False, timeout: float | None = None) -> _T | None:
+    """Get an object from a queue and return `None` if it is empty.
+
+    Parameters
+    ----------
+    q : `Queue`
+        Queue to get an object from.
+    block : `bool`
+        Whether to block until an object is available.
+    timeout : `float` or `None`, optional
+        Maximum number of seconds to wait while blocking.
+
+    Returns
+    -------
+    obj : `object` or `None`
+        Object from the queue, or `None` if it was empty.
+    """
+    try:
+        return q.get(block=block, timeout=timeout)
+    except queue.Empty:
+        return None
+
 
 class FatalWorkerError(BaseException):
     """An exception raised by communicators when one worker (including the
@@ -66,12 +187,6 @@ class FatalWorkerError(BaseException):
     """
 
 
-class _WorkerCommunicationError(Exception):
-    """An exception raised by communicators when a worker has died unexpectedly
-    or become unresponsive.
-    """
-
-
 class _Sentinel(enum.Enum):
     """Sentinel values used to indicate sequence points or worker shutdown
     conditions.
@@ -98,6 +213,21 @@ class _Sentinel(enum.Enum):
     quantum's provenance was written.
     """
 
+    SCANNER_DONE = enum.auto()
+    """Sentinel sent from scanners to the supervisor to report that they are
+    done and shutting down.
+    """
+
+    INGESTER_DONE = enum.auto()
+    """Sentinel sent from the ingester to the supervisor to report that it is
+    done and shutting down.
+    """
+
+    WRITER_DONE = enum.auto()
+    """Sentinel sent from the writer to the supervisor to report that it is
+    done and shutting down.
+    """
+
 
 @dataclasses.dataclass
 class _WorkerErrorMessage:
@@ -147,16 +277,6 @@ class _IngestReport:
     """
 
 
-@dataclasses.dataclass
-class _WorkerDone:
-    """An internal struct passed from a worker to the supervisor when it has
-    successfully completed all work.
-    """
-
-    name: str
-    """Name of the worker reporting completion."""
-
-
 @dataclasses.dataclass
 class _ProgressLog:
     """A high-level log message sent from a worker to the supervisor.
@@ -183,22 +303,20 @@ class _CompressionDictionary:
     """
 
 
-type Report = (
+Report: TypeAlias = (
     ScanReport
     | _IngestReport
     | _WorkerErrorMessage
     | _ProgressLog
-    | _WorkerDone
-    | Literal[_Sentinel.WRITE_REPORT]
+    | Literal[
+        _Sentinel.WRITE_REPORT,
+        _Sentinel.SCANNER_DONE,
+        _Sentinel.INGESTER_DONE,
+        _Sentinel.WRITER_DONE,
+    ]
 )
 
 
-def _disable_resources_parallelism() -> None:
-    os.environ["LSST_RESOURCES_NUM_WORKERS"] = "1"
-    os.environ.pop("LSST_RESOURCES_EXECUTOR", None)
-    os.environ["LSST_S3_USE_THREADS"] = "False"
-
-
 class SupervisorCommunicator:
     """A helper object that lets the supervisor direct the other workers.
 
@@ -208,7 +326,7 @@ class SupervisorCommunicator:
         LSST-customized logger.
     n_scanners : `int`
         Number of scanner workers.
-    worker_factory : `WorkerFactory`
+    context : `WorkerContext`
         Abstraction over threading vs. multiprocessing.
     config : `AggregatorConfig`
         Configuration for the aggregator.
@@ -218,7 +336,7 @@ class SupervisorCommunicator:
         self,
         log: LsstLogAdapter,
         n_scanners: int,
-        worker_factory: WorkerFactory,
+        context: WorkerContext,
         config: AggregatorConfig,
     ) -> None:
         self.config = config
@@ -228,14 +346,14 @@ class SupervisorCommunicator:
         # When complete, the supervisor sends n_scanners sentinals and each
         # scanner is careful to only take one before it starts its shutdown.
         self._scan_requests: Queue[_ScanRequest | Literal[_Sentinel.NO_MORE_SCAN_REQUESTS]] = (
-            worker_factory.make_queue()
+            context.make_queue()
         )
         # The scanners send ingest requests to the ingester on this queue. Each
         # scanner sends one sentinal when it is done, and the ingester is
         # careful to wait for n_scanners sentinals to arrive before it starts
         # its shutdown.
         self._ingest_requests: Queue[IngestRequest | Literal[_Sentinel.NO_MORE_INGEST_REQUESTS]] = (
-            worker_factory.make_queue()
+            context.make_queue()
         )
         # The scanners send write requests to the writer on this queue (which
         # will be `None` if we're not writing).  The supervisor also sends
@@ -243,24 +361,24 @@ class SupervisorCommunicator:
         # scanner and the supervisor send one sentinal when done, and the
         # writer waits for (n_scanners + 1) sentinals to arrive before it
         # starts its shutdown.
-        self._write_requests: (
-            Queue[ProvenanceQuantumScanData | Literal[_Sentinel.NO_MORE_WRITE_REQUESTS]] | None
-        ) = worker_factory.make_queue() if config.is_writing_provenance else None
+        self._write_requests: Queue[WriteRequest | Literal[_Sentinel.NO_MORE_WRITE_REQUESTS]] | None = (
+            context.make_queue() if config.output_path is not None else None
+        )
         # All other workers use this queue to send many different kinds of
         # reports the supervisor.  The supervisor waits for a _DONE sentinal
         # from each worker before it finishes its shutdown.
-        self._reports: Queue[Report] = worker_factory.make_queue()
+        self._reports: Queue[Report] = context.make_queue()
         # The writer sends the compression dictionary to the scanners on this
         # queue.  It puts n_scanners copies on the queue, and each scanner only
         # takes one.  The compression_dict queue has no sentinal because it is
         # only used at most once; the supervisor takes responsibility for
         # clearing it out shutting down.
-        self._compression_dict: Queue[_CompressionDictionary] = worker_factory.make_queue()
+        self._compression_dict: Queue[_CompressionDictionary] = context.make_queue()
         # The supervisor sets this event when it receives an interrupt request
         # from an exception in the main process (usually KeyboardInterrupt).
         # Worker communicators check this in their polling loops and raise
         # FatalWorkerError when they see it set.
-        self._cancel_event: Event = worker_factory.make_event()
+        self._cancel_event: Event = context.make_event()
         # Track what state we are in closing down, so we can start at the right
         # point if we're interrupted and __exit__ needs to clean up.  Note that
         # we can't rely on a non-exception __exit__ to do any shutdown work
@@ -269,77 +387,51 @@ class SupervisorCommunicator:
         self._sent_no_more_scan_requests = False
         self._sent_no_more_write_requests = False
         self._n_scanners_done = 0
-        self.workers: dict[str, Worker] = {}
+        self._ingester_done = False
+        self._writer_done = self._write_requests is None
 
-    def _wait_for_workers_to_finish(self, already_failing: bool = False) -> None:
-        # Orderly shutdown, including exceptions: let workers clear out the
-        # queues they're responsible for reading from.
+    def wait_for_workers_to_finish(self, already_failing: bool = False) -> None:
         if not self._sent_no_more_scan_requests:
             for _ in range(self.n_scanners):
-                self._scan_requests.put(_Sentinel.NO_MORE_SCAN_REQUESTS)
+                self._scan_requests.put(_Sentinel.NO_MORE_SCAN_REQUESTS, block=False)
             self._sent_no_more_scan_requests = True
         if not self._sent_no_more_write_requests and self._write_requests is not None:
-            self._write_requests.put(_Sentinel.NO_MORE_WRITE_REQUESTS)
+            self._write_requests.put(_Sentinel.NO_MORE_WRITE_REQUESTS, block=False)
             self._sent_no_more_write_requests = True
-        while not all(w.successful for w in self.workers.values()):
+        while not (self._ingester_done and self._writer_done and self._n_scanners_done == self.n_scanners):
             match self._handle_progress_reports(
-                self._get_report(block=True), already_failing=already_failing
+                self._reports.get(block=True), already_failing=already_failing
             ):
-                case None | ScanReport():
+                case None | ScanReport() | _IngestReport():
                     pass
-                case _WorkerDone(name=worker_name):
-                    self.workers[worker_name].successful = True
-                    if worker_name == IngesterCommunicator.get_worker_name():
-                        self.progress.quantum_ingests.close()
-                    elif worker_name == WriterCommunicator.get_worker_name():
-                        self.progress.writes.close()
-                    else:
-                        self._n_scanners_done += 1
-                        if self._n_scanners_done == self.n_scanners:
-                            self.progress.scans.close()
+                case _Sentinel.INGESTER_DONE:
+                    self._ingester_done = True
+                    self.progress.quantum_ingests.close()
+                case _Sentinel.SCANNER_DONE:
+                    self._n_scanners_done += 1
+                    self.progress.scans.close()
+                case _Sentinel.WRITER_DONE:
+                    self._writer_done = True
+                    self.progress.writes.close()
                 case unexpected:
                     raise AssertionError(f"Unexpected message {unexpected!r} to supervisor.")
             self.log.verbose(
-                "Waiting for workers [%s] to report successful completion.",
-                ", ".join(w.name for w in self.workers.values() if not w.successful),
+                "Blocking on reports queue: ingester_done=%s, writer_done=%s, n_scanners_done=%s.",
+                self._ingester_done,
+                self._writer_done,
+                self._n_scanners_done,
             )
+        while _get_from_queue(self._compression_dict) is not None:
+            self.log.verbose("Flushing compression dict queue.")
         self.log.verbose("Checking that all queues are empty.")
-        if self._scan_requests.clear():
-            self.progress.log.warning("Scan request queue was not empty at shutdown.")
-            self._scan_requests.kill()
-        if self._ingest_requests.clear():
-            self.progress.log.warning("Ingest request queue was not empty at shutdown.")
-            self._ingest_requests.kill()
-        if self._write_requests is not None and self._write_requests.clear():
-            self.progress.log.warning("Write request queue was not empty at shutdown.")
-            self._write_requests.kill()
-        if self._reports.clear():
-            self.progress.log.warning("Reports queue was not empty at shutdown.")
-            self._reports.kill()
-        if self._compression_dict.clear():
-            self.progress.log.warning("Compression dictionary queue was not empty at shutdown.")
-            self._compression_dict.kill()
-        for worker in self.workers.values():
-            self.log.verbose("Waiting for %s to shut down.", worker.name)
-            worker.join()
-
-    def _terminate(self) -> None:
-        # Disorderly shutdown: we cannot assume any of the
-        # multiprocessing.Queue object work, and in fact they may hang
-        # if we try to do anything with them.
-        self._scan_requests.kill()
-        self._ingest_requests.kill()
+        self._expect_empty_queue(self._scan_requests)
+        self._expect_empty_queue(self._ingest_requests)
         if self._write_requests is not None:
-            self._write_requests.kill()
-        self._compression_dict.kill()
-        self._reports.kill()
-        for name, worker in self.workers.items():
-            if worker.is_alive():
-                self.progress.log.critical("Terminating worker %r.", name)
-                worker.kill()
+            self._expect_empty_queue(self._write_requests)
+        self._expect_empty_queue(self._reports)
+        self._expect_empty_queue(self._compression_dict)
 
     def __enter__(self) -> Self:
-        _disable_resources_parallelism()
         self.progress.__enter__()
         # We make the low-level logger in __enter__ instead of __init__ only
         # because that's the pattern used by true workers (where it matters).
@@ -353,23 +445,11 @@ class SupervisorCommunicator:
         traceback: TracebackType | None,
     ) -> None:
         if exc_type is not None:
-            self._cancel_event.set()
-            if exc_type is _WorkerCommunicationError:
-                self.progress.log.critical("Worker '%s' was terminated before it could finish.", exc_value)
-                self._terminate()
-                return None
             if exc_type is not FatalWorkerError:
-                self.progress.log.critical("Caught %s; attempting to shut down cleanly.", exc_type)
-        try:
-            self._wait_for_workers_to_finish(already_failing=exc_type is not None)
-        except _WorkerCommunicationError as err:
-            self.progress.log.critical(
-                "Worker '%s' was terminated before it could finish (after scanning).", err
-            )
-            self._terminate()
-            raise
+                self.progress.log.critical(f"Caught {exc_type.__name__}; attempting to shut down cleanly.")
+            self._cancel_event.set()
+        self.wait_for_workers_to_finish(already_failing=exc_type is not None)
         self.progress.__exit__(exc_type, exc_value, traceback)
-        return None
 
     def request_scan(self, quantum_id: uuid.UUID) -> None:
         """Send a request to the scanners to scan the given quantum.
@@ -379,19 +459,19 @@ class SupervisorCommunicator:
         quantum_id : `uuid.UUID`
             ID of the quantum to scan.
         """
-        self._scan_requests.put(_ScanRequest(quantum_id))
+        self._scan_requests.put(_ScanRequest(quantum_id), block=False)
 
-    def request_write(self, request: ProvenanceQuantumScanData) -> None:
+    def request_write(self, request: WriteRequest) -> None:
         """Send a request to the writer to write provenance for the given scan.
 
         Parameters
         ----------
-        request : `ProvenanceQuantumScanData`
+        request : `WriteRequest`
             Information from scanning a quantum (or knowing you don't have to,
             in the case of blocked quanta).
         """
         assert self._write_requests is not None, "Writer should not be used if writing is disabled."
-        self._write_requests.put(request)
+        self._write_requests.put(request, block=False)
 
     def poll(self) -> Iterator[ScanReport]:
         """Poll for reports from workers while sending scan requests.
@@ -407,8 +487,9 @@ class SupervisorCommunicator:
         it continues until the report queue is empty.
         """
         block = True
-        while report := self._get_report(block=block):
-            match self._handle_progress_reports(report):
+        msg = _get_from_queue(self._reports, block=block)
+        while msg is not None:
+            match self._handle_progress_reports(msg):
                 case ScanReport() as scan_report:
                     block = False
                     yield scan_report
@@ -416,40 +497,19 @@ class SupervisorCommunicator:
                     pass
                 case unexpected:
                     raise AssertionError(f"Unexpected message {unexpected!r} to supervisor.")
-
-    @overload
-    def _get_report(self, block: Literal[True]) -> Report: ...
-
-    @overload
-    def _get_report(self, block: bool) -> Report | None: ...
-
-    def _get_report(self, block: bool) -> Report | None:
-        """Get a report from the reports queue, with timeout guards on
-        blocking requests.
-
-        This method may *return* WorkerCommunicatorError (rather than raise it)
-        when a serious error occurred communicating with a subprocess.  This
-        is to avoid raising an exception in an __exit__ method (which calls
-        method).
-        """
-        report = self._reports.get(block=block, timeout=self.config.worker_check_timeout)
-        while report is None and block:
-            # We hit the timeout; make sure all of the workers
-            # that should be alive actually are.
-            for name, worker in self.workers.items():
-                if not worker.successful and not worker.is_alive():
-                    # Delete this worker from the list of workers so we don't
-                    # hit this condition again when we try to handle the
-                    # exception we raise.
-                    raise _WorkerCommunicationError(name)
-            # If nothing is dead and we didn't hit the hang timeout, keep
-            # trying.
-            report = self._reports.get(block=block, timeout=self.config.worker_check_timeout)
-        return report
+            msg = _get_from_queue(self._reports, block=block)
 
     def _handle_progress_reports(
         self, report: Report, already_failing: bool = False
-    ) -> ScanReport | _WorkerDone | None:
+    ) -> (
+        ScanReport
+        | Literal[
+            _Sentinel.SCANNER_DONE,
+            _Sentinel.INGESTER_DONE,
+            _Sentinel.WRITER_DONE,
+        ]
+        | None
+    ):
         """Handle reports to the supervisor that can appear at any time, and
         are typically just updates to the progress we've made.
 
@@ -479,9 +539,15 @@ class SupervisorCommunicator:
                 return report
         return None
 
+    @staticmethod
+    def _expect_empty_queue(queue: Queue[Any]) -> None:
+        """Assert that the given queue is empty."""
+        if (msg := _get_from_queue(queue, block=False, timeout=0)) is not None:
+            raise AssertionError(f"Queue is not empty; found {msg!r}.")
+
 
 class WorkerCommunicator:
-    """A base class for non-supervisor worker communicators.
+    """A base class for non-supervisor workers.
 
     Parameters
     ----------
@@ -493,8 +559,8 @@ class WorkerCommunicator:
     Notes
     -----
     Each worker communicator is constructed in the main process and entered as
-    a context manager *only* on the actual worker process, so attributes that
-    cannot be pickled are constructed in ``__enter__`` instead of ``__init__``.
+    a context manager on the actual worker process, so attributes that cannot
+    be pickled are constructed in ``__enter__`` instead of ``__init__``.
 
     Worker communicators provide access to an `AggregatorConfig` and a logger
     to their workers.  As context managers, they handle exceptions and ensure
@@ -514,7 +580,6 @@ class WorkerCommunicator:
         self._cancel_event = supervisor._cancel_event
 
     def __enter__(self) -> Self:
-        _disable_resources_parallelism()
         self.log = make_worker_log(self.name, self.config)
         self.log.verbose("%s has PID %s (parent is %s).", self.name, os.getpid(), os.getppid())
         self._exit_stack = ExitStack().__enter__()
@@ -547,7 +612,8 @@ class WorkerCommunicator:
                     _WorkerErrorMessage(
                         self.name,
                         "".join(format_exception(exc_type, exc_value, traceback)),
-                    )
+                    ),
+                    block=False,
                 )
                 self.log.debug("Error message sent to supervisor.")
             else:
@@ -555,11 +621,6 @@ class WorkerCommunicator:
         self._exit_stack.__exit__(exc_type, exc_value, traceback)
         return True
 
-    @property
-    def exit_stack(self) -> ExitStack:
-        """A `contextlib.ExitStack` tied to the communicator."""
-        return self._exit_stack
-
     def log_progress(self, level: int, message: str) -> None:
         """Send a high-level log message to the supervisor.
 
@@ -570,7 +631,45 @@ class WorkerCommunicator:
         message : `str`
             Log message.
         """
-        self._reports.put(_ProgressLog(message=message, level=level))
+        self._reports.put(_ProgressLog(message=message, level=level), block=False)
+
+    def enter(
+        self,
+        cm: AbstractContextManager[_T],
+        on_close: str | None = None,
+        level: int = VERBOSE,
+        is_progress_log: bool = False,
+    ) -> _T:
+        """Enter a context manager that will be exited when the communicator's
+        context is exited.
+
+        Parameters
+        ----------
+        cm : `contextlib.AbstractContextManager`
+            A context manager to enter.
+        on_close : `str`, optional
+            A log message to emit (on the worker's logger) just before the
+            given context manager is exited.  This can be used to indicate
+            what's going on when an ``__exit__`` implementation has a lot of
+            work to do (e.g. moving a large file into a zip archive).
+        level : `int`, optional
+            Level for the ``on_close`` log message.
+        is_progress_log : `bool`, optional
+            If `True`, send the ``on_close`` message to the supervisor via
+            `log_progress` as well as the worker's logger.
+        """
+        if on_close is None:
+            return self._exit_stack.enter_context(cm)
+
+        @contextmanager
+        def wrapper() -> Iterator[_T]:
+            with cm as result:
+                yield result
+                self.log.log(level, on_close)
+                if is_progress_log:
+                    self.log_progress(level, on_close)
+
+        return self._exit_stack.enter_context(wrapper())
 
     def check_for_cancel(self) -> None:
         """Check for a cancel signal from the supervisor and raise
@@ -592,7 +691,7 @@ class ScannerCommunicator(WorkerCommunicator):
     """
 
     def __init__(self, supervisor: SupervisorCommunicator, scanner_id: int):
-        super().__init__(supervisor, self.get_worker_name(scanner_id))
+        super().__init__(supervisor, f"scanner-{scanner_id:03d}")
         self.scanner_id = scanner_id
         self._scan_requests = supervisor._scan_requests
         self._ingest_requests = supervisor._ingest_requests
@@ -601,10 +700,6 @@ class ScannerCommunicator(WorkerCommunicator):
         self._got_no_more_scan_requests: bool = False
         self._sent_no_more_ingest_requests: bool = False
 
-    @staticmethod
-    def get_worker_name(scanner_id: int) -> str:
-        return f"scanner-{scanner_id:03d}"
-
     def report_scan(self, msg: ScanReport) -> None:
         """Report a completed scan to the supervisor.
 
@@ -613,7 +708,7 @@ class ScannerCommunicator(WorkerCommunicator):
         msg : `ScanReport`
             Report to send.
         """
-        self._reports.put(msg)
+        self._reports.put(msg, block=False)
 
     def request_ingest(self, request: IngestRequest) -> None:
         """Ask the ingester to ingest a quantum's outputs.
@@ -629,20 +724,20 @@ class ScannerCommunicator(WorkerCommunicator):
         as complete to the supervisor instead of sending it to the ingester.
         """
         if request:
-            self._ingest_requests.put(request)
+            self._ingest_requests.put(request, block=False)
         else:
-            self._reports.put(_IngestReport(1))
+            self._reports.put(_IngestReport(1), block=False)
 
-    def request_write(self, request: ProvenanceQuantumScanData) -> None:
+    def request_write(self, request: WriteRequest) -> None:
         """Ask the writer to write provenance for a quantum.
 
         Parameters
         ----------
-        request : `ProvenanceQuantumScanData`
+        request : `WriteRequest`
             Result of scanning a quantum.
         """
         assert self._write_requests is not None, "Writer should not be used if writing is disabled."
-        self._write_requests.put(request)
+        self._write_requests.put(request, block=False)
 
     def get_compression_dict(self) -> bytes | None:
         """Attempt to get the compression dict from the writer.
@@ -658,7 +753,7 @@ class ScannerCommunicator(WorkerCommunicator):
         A scanner should only call this method before it actually has the
         compression dict.
         """
-        if (cdict := self._compression_dict.get()) is not None:
+        if (cdict := _get_from_queue(self._compression_dict)) is not None:
             return cdict.data
         return None
 
@@ -677,7 +772,7 @@ class ScannerCommunicator(WorkerCommunicator):
         """
         while True:
             self.check_for_cancel()
-            scan_request = self._scan_requests.get(block=True, timeout=self.config.worker_sleep)
+            scan_request = _get_from_queue(self._scan_requests, block=True, timeout=self.config.worker_sleep)
             if scan_request is _Sentinel.NO_MORE_SCAN_REQUESTS:
                 self._got_no_more_scan_requests = True
                 return
@@ -691,18 +786,20 @@ class ScannerCommunicator(WorkerCommunicator):
         traceback: TracebackType | None,
     ) -> bool | None:
         result = super().__exit__(exc_type, exc_value, traceback)
-        self._ingest_requests.put(_Sentinel.NO_MORE_INGEST_REQUESTS)
+        self._ingest_requests.put(_Sentinel.NO_MORE_INGEST_REQUESTS, block=False)
         if self._write_requests is not None:
-            self._write_requests.put(_Sentinel.NO_MORE_WRITE_REQUESTS)
+            self._write_requests.put(_Sentinel.NO_MORE_WRITE_REQUESTS, block=False)
         while not self._got_no_more_scan_requests:
+            self.log.debug("Clearing scan request queue (~%d remaining)", self._scan_requests.qsize())
             if (
                 not self._got_no_more_scan_requests
-                and self._scan_requests.get(block=True) is _Sentinel.NO_MORE_SCAN_REQUESTS
+                and self._scan_requests.get() is _Sentinel.NO_MORE_SCAN_REQUESTS
             ):
                 self._got_no_more_scan_requests = True
-        # We let the writer clear out the compression dict queue.
-        self.log.verbose("Sending completion message.")
-        self._reports.put(_WorkerDone(self.name))
+        # We let the supervisor clear out the compression dict queue, because
+        # a single scanner can't know if it ever got sent out or not.
+        self.log.verbose("Sending done sentinal.")
+        self._reports.put(_Sentinel.SCANNER_DONE, block=False)
         return result
 
 
@@ -716,15 +813,11 @@ class IngesterCommunicator(WorkerCommunicator):
     """
 
     def __init__(self, supervisor: SupervisorCommunicator):
-        super().__init__(supervisor, self.get_worker_name())
+        super().__init__(supervisor, "ingester")
         self.n_scanners = supervisor.n_scanners
         self._ingest_requests = supervisor._ingest_requests
         self._n_requesters_done = 0
 
-    @staticmethod
-    def get_worker_name() -> str:
-        return "ingester"
-
     def __exit__(
         self,
         exc_type: type[BaseException] | None,
@@ -740,8 +833,8 @@ class IngesterCommunicator(WorkerCommunicator):
             )
             if self._ingest_requests.get(block=True) is _Sentinel.NO_MORE_INGEST_REQUESTS:
                 self._n_requesters_done += 1
-        self.log.verbose("Sending completion message.")
-        self._reports.put(_WorkerDone(self.name))
+        self.log.verbose("Sending done sentinal.")
+        self._reports.put(_Sentinel.INGESTER_DONE, block=False)
         return result
 
     def report_ingest(self, n_producers: int) -> None:
@@ -752,7 +845,7 @@ class IngesterCommunicator(WorkerCommunicator):
         n_producers : `int`
             Number of producing quanta whose datasets were ingested.
         """
-        self._reports.put(_IngestReport(n_producers))
+        self._reports.put(_IngestReport(n_producers), block=False)
 
     def poll(self) -> Iterator[IngestRequest]:
         """Poll for ingest requests from the scanner workers.
@@ -769,7 +862,7 @@ class IngesterCommunicator(WorkerCommunicator):
         """
         while True:
             self.check_for_cancel()
-            ingest_request = self._ingest_requests.get(block=True, timeout=_TINY_TIMEOUT)
+            ingest_request = _get_from_queue(self._ingest_requests, block=True, timeout=_TINY_TIMEOUT)
             if ingest_request is _Sentinel.NO_MORE_INGEST_REQUESTS:
                 self._n_requesters_done += 1
                 if self._n_requesters_done == self.n_scanners:
@@ -791,7 +884,7 @@ class WriterCommunicator(WorkerCommunicator):
 
     def __init__(self, supervisor: SupervisorCommunicator):
         assert supervisor._write_requests is not None
-        super().__init__(supervisor, self.get_worker_name())
+        super().__init__(supervisor, "writer")
         self.n_scanners = supervisor.n_scanners
         self._write_requests = supervisor._write_requests
         self._compression_dict = supervisor._compression_dict
@@ -799,10 +892,6 @@ class WriterCommunicator(WorkerCommunicator):
         self._n_requesters_done = 0
         self._sent_compression_dict = False
 
-    @staticmethod
-    def get_worker_name() -> str:
-        return "writer"
-
     def __exit__(
         self,
         exc_type: type[BaseException] | None,
@@ -820,20 +909,16 @@ class WriterCommunicator(WorkerCommunicator):
             )
             if self._write_requests.get(block=True) is _Sentinel.NO_MORE_WRITE_REQUESTS:
                 self._n_requesters_done += 1
-        if self._compression_dict.clear():
-            self.log.verbose("Cleared out compression dictionary queue.")
-        else:
-            self.log.verbose("Compression dictionary queue was already empty.")
-        self.log.verbose("Sending completion message.")
-        self._reports.put(_WorkerDone(self.name))
+        self.log.verbose("Sending done sentinal.")
+        self._reports.put(_Sentinel.WRITER_DONE, block=False)
         return result
 
-    def poll(self) -> Iterator[ProvenanceQuantumScanData]:
+    def poll(self) -> Iterator[WriteRequest]:
         """Poll for writer requests from the scanner workers and supervisor.
 
         Yields
         ------
-        request : `ProvenanceQuantumScanData`
+        request : `WriteRequest`
             The result of a quantum scan.
 
         Notes
@@ -843,7 +928,7 @@ class WriterCommunicator(WorkerCommunicator):
         """
         while True:
             self.check_for_cancel()
-            write_request = self._write_requests.get(block=True, timeout=_TINY_TIMEOUT)
+            write_request = _get_from_queue(self._write_requests, block=True, timeout=_TINY_TIMEOUT)
             if write_request is _Sentinel.NO_MORE_WRITE_REQUESTS:
                 self._n_requesters_done += 1
                 if self._n_requesters_done == self._n_requesters:
@@ -863,16 +948,16 @@ class WriterCommunicator(WorkerCommunicator):
         """
         self.log.debug("Sending compression dictionary.")
         for _ in range(self.n_scanners):
-            self._compression_dict.put(_CompressionDictionary(cdict_data))
+            self._compression_dict.put(_CompressionDictionary(cdict_data), block=False)
         self._sent_compression_dict = True
 
     def report_write(self) -> None:
         """Report to the supervisor that provenance for a quantum was written
         to the graph.
         """
-        self._reports.put(_Sentinel.WRITE_REPORT)
+        self._reports.put(_Sentinel.WRITE_REPORT, block=False)
 
-    def periodically_check_for_cancel[T](self, iterable: Iterable[T], n: int = 100) -> Iterator[T]:
+    def periodically_check_for_cancel(self, iterable: Iterable[_T], n: int = 100) -> Iterator[_T]:
         """Iterate while checking for a cancellation signal every ``n``
         iterations.