lsst-pipe-base 30.0.0rc3__py3-none-any.whl → 30.0.1__py3-none-any.whl

lsst/pipe/base/quantum_graph/aggregator/_config.py

@@ -29,6 +29,8 @@ from __future__ import annotations
 
 __all__ = ("AggregatorConfig",)
 
+import sys
+from typing import TYPE_CHECKING, Any
 
 import pydantic
 
@@ -60,11 +62,13 @@ class AggregatorConfig(pydantic.BaseModel):
     n_processes: int = 1
     """Number of processes the scanner should use."""
 
-    assume_complete: bool = True
-    """If `True`, the aggregator can assume all quanta have run to completion
-    (including any automatic retries).  If `False`, only successes can be
-    considered final, and quanta that appear to have failed or to have not been
-    executed are ignored.
+    incomplete: bool = False
+    """If `True`, do not expect the graph to have been executed to completion
+    yet, and only ingest the outputs of successful quanta.
+
+    This disables writing the provenance quantum graph, since this is likely to
+    be wasted effort that just complicates a follow-up run with
+    ``incomplete=False`` later.
     """
 
     defensive_ingest: bool = False
@@ -95,11 +99,10 @@ class AggregatorConfig(pydantic.BaseModel):
     """
 
     dry_run: bool = False
-    """If `True`, do not actually perform any deletions or central butler
-    ingests.
+    """If `True`, do not actually perform any central butler ingests.
 
-    Most log messages concerning deletions and ingests will still be emitted in
-    order to provide a better emulation of a real run.
+    Most log messages concerning ingests will still be emitted in order to
+    provide a better emulation of a real run.
     """
 
     interactive_status: bool = False
@@ -137,3 +140,78 @@ class AggregatorConfig(pydantic.BaseModel):
     """Enable support for storage classes by created by the
     lsst.pipe.base.tests.mocks package.
     """
+
+    promise_ingest_graph: bool = False
+    """If `True`, the aggregator will assume that `~.ingest_graph.ingest_graph`
+    will be run later to ingest metadata/log/config datasets, and will not
+    ingest them itself.  This means that if `~.ingest_graph.ingest_graph` is
+    not run, those files will be abandoned in the butler storage root without
+    being present in the butler database, but it will speed up both processes.
+
+    It is *usually* safe to build a quantum graph for downstream processing
+    before or while running `~.ingest_graph.ingest_graph`, because
+    metadata/log/config datasets are rarely used as inputs.  To check, use
+    ``pipetask build ... --show inputs`` to show the overall-inputs to the
+    graph and scan for these dataset types.
+    """
+
+    worker_check_timeout: float = 5.0
+    """Time to wait (s) for reports from subprocesses before running
+    process-alive checks.
+
+    These checks are designed to kill the main aggregator process when a
+    subprocess has been unexpectedly killed (e.g. for for using too much
+    memory).
+    """
+
+    @property
+    def is_writing_provenance(self) -> bool:
+        """Whether the aggregator is configured to write the provenance quantum
+        graph.
+        """
+        return self.output_path is not None and not self.incomplete
+
+    # Work around the fact that Sphinx chokes on Pydantic docstring formatting,
+    # when we inherit those docstrings in our public classes.
+    if "sphinx" in sys.modules and not TYPE_CHECKING:
+
+        def copy(self, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.copy`."""
+            return super().copy(*args, **kwargs)
+
+        def model_dump(self, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_dump`."""
+            return super().model_dump(*args, **kwargs)
+
+        def model_dump_json(self, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_dump_json`."""
+            return super().model_dump(*args, **kwargs)
+
+        def model_copy(self, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_copy`."""
+            return super().model_copy(*args, **kwargs)
+
+        @classmethod
+        def model_construct(cls, *args: Any, **kwargs: Any) -> Any:  # type: ignore[misc, override]
+            """See `pydantic.BaseModel.model_construct`."""
+            return super().model_construct(*args, **kwargs)
+
+        @classmethod
+        def model_json_schema(cls, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_json_schema`."""
+            return super().model_json_schema(*args, **kwargs)
+
+        @classmethod
+        def model_validate(cls, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_validate`."""
+            return super().model_validate(*args, **kwargs)
+
+        @classmethod
+        def model_validate_json(cls, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_validate_json`."""
+            return super().model_validate_json(*args, **kwargs)
+
+        @classmethod
+        def model_validate_strings(cls, *args: Any, **kwargs: Any) -> Any:
+            """See `pydantic.BaseModel.model_validate_strings`."""
+            return super().model_validate_strings(*args, **kwargs)

lsst/pipe/base/quantum_graph/aggregator/_ingester.py

@@ -43,7 +43,7 @@ from lsst.daf.butler.registry import ConflictingDefinitionError
 
 from ...pipeline_graph import TaskImportMode
 from .._common import DatastoreName
-from .._predicted import PredictedDatasetModel, PredictedQuantumGraphComponents, PredictedQuantumGraphReader
+from .._predicted import PredictedQuantumGraphComponents, PredictedQuantumGraphReader
 from ._communicators import IngesterCommunicator
 
 
@@ -140,7 +140,7 @@ class Ingester(AbstractContextManager):
         Notes
         -----
         This method is designed to run as the ``target`` in
-        `WorkerContext.make_worker`.
+        `WorkerFactory.make_worker`.
         """
         with comms, Ingester(predicted_path, butler_path, comms) as ingester:
             ingester.loop()
@@ -170,7 +170,7 @@ class Ingester(AbstractContextManager):
             for ingest_request in self.comms.poll():
                 self.n_producers_pending += 1
                 self.comms.log.debug(f"Got ingest request for producer {ingest_request.producer_id}.")
-                self.update_pending(ingest_request.datasets, ingest_request.records)
+                self.update_outputs_pending(refs=ingest_request.refs, records=ingest_request.records)
                 if self.n_datasets_pending > self.comms.config.ingest_batch_size:
                     self.ingest()
             self.comms.log.info("All ingest requests received.")
@@ -266,31 +266,32 @@ class Ingester(AbstractContextManager):
             else:
                 del self.records_pending[datastore_name]
 
-    def update_pending(
-        self, datasets: list[PredictedDatasetModel], records: dict[DatastoreName, DatastoreRecordData]
+    def update_outputs_pending(
+        self,
+        refs: list[DatasetRef],
+        records: dict[DatastoreName, DatastoreRecordData],
     ) -> None:
         """Add an ingest request to the pending-ingest data structures.
 
         Parameters
         ----------
-        datasets : `list` [ `PredictedDatasetModel` ]
-            Registry information about the datasets.
+        refs : `list` [ `lsst.daf.butler.DatasetRef` ]
+            Registry information about regular quantum-output datasets.
         records : `dict` [ `str`, \
                 `lsst.daf.butler.datastore.record_data.DatastoreRecordData` ]
             Datastore information about the datasets.
         """
-        n_given = len(datasets)
+        n_given = len(refs)
         if self.already_ingested is not None:
-            datasets = [d for d in datasets if d.dataset_id not in self.already_ingested]
-            kept = {d.dataset_id for d in datasets}
+            refs = [ref for ref in refs if ref.id not in self.already_ingested]
+            kept = {ref.id for ref in refs}
             self.n_datasets_skipped += n_given - len(kept)
             records = {
                 datastore_name: filtered_records
                 for datastore_name, original_records in records.items()
                 if (filtered_records := original_records.subset(kept)) is not None
             }
-        for dataset in datasets:
-            ref = self.predicted.make_dataset_ref(dataset)
+        for ref in refs:
             self.refs_pending[ref.datasetType.dimensions].append(ref)
         for datastore_name, datastore_records in records.items():
             if (existing_records := self.records_pending.get(datastore_name)) is not None:

lsst/pipe/base/quantum_graph/aggregator/_scanner.py

@@ -161,7 +161,7 @@ class Scanner(AbstractContextManager):
         Notes
         -----
         This method is designed to run as the ``target`` in
-        `WorkerContext.make_worker`.
+        `WorkerFactory.make_worker`.
         """
         with comms, Scanner(predicted_path, butler_path, comms) as scanner:
             scanner.loop()
@@ -223,7 +223,7 @@ class Scanner(AbstractContextManager):
             logs = self._read_log(predicted_quantum)
             metadata = self._read_metadata(predicted_quantum)
             result = ProvenanceQuantumScanModels.from_metadata_and_logs(
-                predicted_quantum, metadata, logs, assume_complete=self.comms.config.assume_complete
+                predicted_quantum, metadata, logs, incomplete=self.comms.config.incomplete
             )
             if result.status is ProvenanceQuantumScanStatus.ABANDONED:
                 self.comms.log.debug("Abandoning scan for failed quantum %s.", quantum_id)
@@ -233,7 +233,7 @@ class Scanner(AbstractContextManager):
             if predicted_output.dataset_id not in result.output_existence:
                 result.output_existence[predicted_output.dataset_id] = self.scan_dataset(predicted_output)
         to_ingest = self._make_ingest_request(predicted_quantum, result)
-        if self.comms.config.output_path is not None:
+        if self.comms.config.is_writing_provenance:
             to_write = result.to_scan_data(predicted_quantum, compressor=self.compressor)
             self.comms.request_write(to_write)
         self.comms.request_ingest(to_ingest)
@@ -261,15 +261,23 @@ class Scanner(AbstractContextManager):
         predicted_outputs_by_id = {
             d.dataset_id: d for d in itertools.chain.from_iterable(predicted_quantum.outputs.values())
         }
-        to_ingest_predicted: list[PredictedDatasetModel] = []
         to_ingest_refs: list[DatasetRef] = []
+        to_ignore: set[uuid.UUID] = set()
+        if self.comms.config.promise_ingest_graph:
+            if result.status is ProvenanceQuantumScanStatus.INIT:
+                if predicted_quantum.task_label:  # i.e. not the 'packages' producer
+                    to_ignore.add(
+                        predicted_quantum.outputs[acc.CONFIG_INIT_OUTPUT_CONNECTION_NAME][0].dataset_id
+                    )
+            else:
+                to_ignore.add(predicted_quantum.outputs[acc.METADATA_OUTPUT_CONNECTION_NAME][0].dataset_id)
+                to_ignore.add(predicted_quantum.outputs[acc.LOG_OUTPUT_CONNECTION_NAME][0].dataset_id)
         for dataset_id, was_produced in result.output_existence.items():
-            if was_produced:
+            if was_produced and dataset_id not in to_ignore:
                 predicted_output = predicted_outputs_by_id[dataset_id]
-                to_ingest_predicted.append(predicted_output)
                 to_ingest_refs.append(self.reader.components.make_dataset_ref(predicted_output))
         to_ingest_records = self.qbb._datastore.export_predicted_records(to_ingest_refs)
-        return IngestRequest(result.quantum_id, to_ingest_predicted, to_ingest_records)
+        return IngestRequest(result.quantum_id, to_ingest_refs, to_ingest_records)
 
     def _read_metadata(self, predicted_quantum: PredictedQuantumDatasetsModel) -> TaskMetadata | None:
         """Attempt to read the metadata dataset for a quantum.

lsst/pipe/base/quantum_graph/aggregator/_structs.py

@@ -32,10 +32,10 @@ __all__ = ("IngestRequest", "ScanReport")
 import dataclasses
 import uuid
 
+from lsst.daf.butler import DatasetRef
 from lsst.daf.butler.datastore.record_data import DatastoreRecordData
 
 from .._common import DatastoreName
-from .._predicted import PredictedDatasetModel
 from .._provenance import ProvenanceQuantumScanStatus
 
 
@@ -57,11 +57,11 @@ class IngestRequest:
     producer_id: uuid.UUID
     """ID of the quantum that produced these datasets."""
 
-    datasets: list[PredictedDatasetModel]
+    refs: list[DatasetRef]
     """Registry information about the datasets."""
 
     records: dict[DatastoreName, DatastoreRecordData]
     """Datastore information about the datasets."""
 
     def __bool__(self) -> bool:
-        return bool(self.datasets or self.records)
+        return bool(self.refs or self.records)

lsst/pipe/base/quantum_graph/aggregator/_supervisor.py

@@ -46,16 +46,14 @@ from .._provenance import ProvenanceQuantumScanData, ProvenanceQuantumScanStatus
 from ._communicators import (
     IngesterCommunicator,
     ScannerCommunicator,
-    SpawnProcessContext,
     SupervisorCommunicator,
-    ThreadingContext,
-    Worker,
     WriterCommunicator,
 )
 from ._config import AggregatorConfig
 from ._ingester import Ingester
 from ._scanner import Scanner
 from ._structs import ScanReport
+from ._workers import SpawnWorkerFactory, ThreadWorkerFactory
 from ._writer import Writer
 
 
@@ -117,6 +115,17 @@ class Supervisor:
                 self.comms.request_scan(ready_set.pop())
             for scan_return in self.comms.poll():
                 self.handle_report(scan_return)
+        if self.comms.config.incomplete:
+            quantum_or_quanta = "quanta" if self.n_abandoned != 1 else "quantum"
+            self.comms.progress.log.info(
+                "%d %s incomplete/failed abandoned; re-run with incomplete=False to finish.",
+                self.n_abandoned,
+                quantum_or_quanta,
+            )
+        self.comms.progress.log.info(
+            "Scanning complete after %0.1fs; waiting for workers to finish.",
+            self.comms.progress.elapsed_time,
+        )
 
     def handle_report(self, scan_report: ScanReport) -> None:
         """Handle a report from a scanner.
@@ -134,7 +143,7 @@ class Supervisor:
                 self.comms.log.debug("Scan complete for %s: quantum failed.", scan_report.quantum_id)
                 blocked_quanta = self.walker.fail(scan_report.quantum_id)
                 for blocked_quantum_id in blocked_quanta:
-                    if self.comms.config.output_path is not None:
+                    if self.comms.config.is_writing_provenance:
                         self.comms.request_write(
                             ProvenanceQuantumScanData(
                                 blocked_quantum_id, status=ProvenanceQuantumScanStatus.BLOCKED
@@ -166,55 +175,31 @@ def aggregate_graph(predicted_path: str, butler_path: str, config: AggregatorCon
         Configuration for the aggregator.
     """
     log = getLogger("lsst.pipe.base.quantum_graph.aggregator")
-    ctx = ThreadingContext() if config.n_processes == 1 else SpawnProcessContext()
-    scanners: list[Worker] = []
-    ingester: Worker
-    writer: Worker | None = None
-    with SupervisorCommunicator(log, config.n_processes, ctx, config) as comms:
+    worker_factory = ThreadWorkerFactory() if config.n_processes == 1 else SpawnWorkerFactory()
+    with SupervisorCommunicator(log, config.n_processes, worker_factory, config) as comms:
         comms.progress.log.verbose("Starting workers.")
-        if config.output_path is not None:
+        if config.is_writing_provenance:
             writer_comms = WriterCommunicator(comms)
-            writer = ctx.make_worker(
+            comms.workers[writer_comms.name] = worker_factory.make_worker(
                 target=Writer.run,
                 args=(predicted_path, writer_comms),
                 name=writer_comms.name,
             )
-            writer.start()
         for scanner_id in range(config.n_processes):
             scanner_comms = ScannerCommunicator(comms, scanner_id)
-            worker = ctx.make_worker(
+            comms.workers[scanner_comms.name] = worker_factory.make_worker(
                 target=Scanner.run,
                 args=(predicted_path, butler_path, scanner_comms),
                 name=scanner_comms.name,
             )
-            worker.start()
-            scanners.append(worker)
         ingester_comms = IngesterCommunicator(comms)
-        ingester = ctx.make_worker(
+        comms.workers[ingester_comms.name] = worker_factory.make_worker(
             target=Ingester.run,
             args=(predicted_path, butler_path, ingester_comms),
             name=ingester_comms.name,
         )
-        ingester.start()
         supervisor = Supervisor(predicted_path, comms)
         supervisor.loop()
-        log.info(
-            "Scanning complete after %0.1fs; waiting for workers to finish.",
-            comms.progress.elapsed_time,
-        )
-        comms.wait_for_workers_to_finish()
-        if supervisor.n_abandoned:
-            raise RuntimeError(
-                f"{supervisor.n_abandoned} {'quanta' if supervisor.n_abandoned > 1 else 'quantum'} "
-                "abandoned because they did not succeed.  Re-run with assume_complete=True after all retry "
-                "attempts have been exhausted."
-            )
-    for w in scanners:
-        w.join()
-    ingester.join()
-    if writer is not None and writer.is_alive():
-        log.info("Waiting for writer process to close (garbage collecting can be very slow).")
-        writer.join()
     # We can't get memory usage for children until they've joined.
     parent_mem, child_mem = get_peak_mem_usage()
     # This is actually an upper bound on the peak (since the peaks could be

lsst/pipe/base/quantum_graph/aggregator/_workers.py

@@ -0,0 +1,303 @@
+# This file is part of pipe_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This software is dual licensed under the GNU General Public License and also
+# under a 3-clause BSD license. Recipients may choose which of these licenses
+# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
+# respectively.  If you choose the GPL option then the following text applies
+# (but note that there is still no warranty even if you opt for BSD instead):
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+__all__ = ("Event", "Queue", "SpawnWorkerFactory", "ThreadWorkerFactory", "Worker", "WorkerFactory")
+
+import multiprocessing.context
+import multiprocessing.synchronize
+import queue
+import threading
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from typing import Any, Literal, overload
+
+_TINY_TIMEOUT = 0.01
+
+type Event = threading.Event | multiprocessing.synchronize.Event
+
+
+class Worker(ABC):
+    """A thin abstraction over `threading.Thread` and `multiprocessing.Process`
+    that also provides a variable to track whether it reported successful
+    completion.
+    """
+
+    def __init__(self) -> None:
+        self.successful = False
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Name of the worker, as assigned at creation."""
+        raise NotImplementedError()
+
+    @abstractmethod
+    def join(self, timeout: float | None = None) -> None:
+        """Wait for the worker to finish.
+
+        Parameters
+        ----------
+        timeout : `float`, optional
+            How long to wait in seconds.  If the timeout is exceeded,
+            `is_alive` can be used to see whether the worker finished or not.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def is_alive(self) -> bool:
+        """Return whether the worker is still running."""
+        raise NotImplementedError()
+
+    def kill(self) -> None:
+        """Kill the worker, if possible."""
+
+
+class Queue[T](ABC):
+    """A thin abstraction over `queue.Queue` and `multiprocessing.Queue` that
+    provides better control over disorderly shutdowns.
+    """
+
+    @overload
+    def get(self, *, block: Literal[True]) -> T: ...
+
+    @overload
+    def get(self, *, timeout: float | None = None, block: bool = False) -> T | None: ...
+
+    @abstractmethod
+    def get(self, *, timeout: float | None = None, block: bool = False) -> T | None:
+        """Get an object or return `None` if the queue is empty.
+
+        Parameters
+        ----------
+        timeout : `float` or `None`, optional
+            Maximum number of seconds to wait while blocking.
+        block : `bool`, optional
+            Whether to block until an object is available.
+
+        Returns
+        -------
+        obj : `object` or `None`
+            Object from the queue, or `None` if it was empty.  Note that this
+            is different from the behavior of the built-in Python queues,
+            which raise `queue.Empty` instead.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def put(self, item: T) -> None:
+        """Add an object to the queue.
+
+        Parameters
+        ----------
+        item : `object`
+            Item to add.
+        """
+        raise NotImplementedError()
+
+    def clear(self) -> bool:
+        """Clear out all objects currently on the queue.
+
+        This does not guarantee that more objects will not be added later.
+        """
+        found_anything: bool = False
+        while self.get() is not None:
+            found_anything = True
+        return found_anything
+
+    def kill(self) -> None:
+        """Prepare a queue for a disorderly shutdown, without assuming that
+        any other workers using it are still alive and functioning.
+        """
+
+
+class WorkerFactory(ABC):
+    """A simple abstract interface that can be implemented by both threading
+    and multiprocessing.
+    """
+
+    @abstractmethod
+    def make_queue(self) -> Queue[Any]:
+        """Make an empty queue that can be used to pass objects between
+        workers created by this factory.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def make_event(self) -> Event:
+        """Make an event that can be used to communicate a boolean state change
+        to workers created by this factory.
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def make_worker(
+        self, target: Callable[..., None], args: tuple[Any, ...], name: str | None = None
+    ) -> Worker:
+        """Make a worker that runs the given callable.
+
+        Parameters
+        ----------
+        target : `~collections.abc.Callable`
+            A callable to invoke on the worker.
+        args : `tuple`
+            Positional arguments to pass to the callable.
+        name : `str`, optional
+            Human-readable name for the worker.
+
+        Returns
+        -------
+        worker : `Worker`
+            Process or thread that is already running the given callable.
+        """
+        raise NotImplementedError()
+
+
+class _ThreadWorker(Worker):
+    """An implementation of `Worker` backed by the `threading` module."""
+
+    def __init__(self, thread: threading.Thread):
+        super().__init__()
+        self._thread = thread
+
+    @property
+    def name(self) -> str:
+        return self._thread.name
+
+    def join(self, timeout: float | None = None) -> None:
+        self._thread.join(timeout=timeout)
+
+    def is_alive(self) -> bool:
+        return self._thread.is_alive()
+
+
+class _ThreadQueue[T](Queue[T]):
+    def __init__(self) -> None:
+        self._impl = queue.Queue[T]()
+
+    @overload
+    def get(self, *, block: Literal[True]) -> T: ...
+
+    @overload
+    def get(self, *, timeout: float | None = None, block: bool = False) -> T | None: ...
+
+    def get(self, *, timeout: float | None = None, block: bool = False) -> T | None:
+        try:
+            return self._impl.get(block=block, timeout=timeout)
+        except queue.Empty:
+            return None
+
+    def put(self, item: T) -> None:
+        self._impl.put(item, block=False)
+
+
+class ThreadWorkerFactory(WorkerFactory):
+    """An implementation of `WorkerFactory` backed by the `threading`
+    module.
+    """
+
+    def make_queue(self) -> Queue[Any]:
+        return _ThreadQueue()
+
+    def make_event(self) -> Event:
+        return threading.Event()
+
+    def make_worker(
+        self, target: Callable[..., None], args: tuple[Any, ...], name: str | None = None
+    ) -> Worker:
+        thread = threading.Thread(target=target, args=args, name=name)
+        thread.start()
+        return _ThreadWorker(thread)
+
+
+class _ProcessWorker(Worker):
+    """An implementation of `Worker` backed by the `multiprocessing` module."""
+
+    def __init__(self, process: multiprocessing.context.SpawnProcess):
+        super().__init__()
+        self._process = process
+
+    @property
+    def name(self) -> str:
+        return self._process.name
+
+    def join(self, timeout: float | None = None) -> None:
+        self._process.join(timeout=timeout)
+
+    def is_alive(self) -> bool:
+        return self._process.is_alive()
+
+    def kill(self) -> None:
+        """Kill the worker, if possible."""
+        self._process.kill()
+
+
+class _ProcessQueue[T](Queue[T]):
+    def __init__(self, impl: multiprocessing.Queue):
+        self._impl = impl
+
+    @overload
+    def get(self, *, block: Literal[True]) -> T: ...
+
+    @overload
+    def get(self, *, timeout: float | None = None, block: bool = False) -> T | None: ...
+
+    def get(self, *, timeout: float | None = None, block: bool = False) -> T | None:
+        try:
+            return self._impl.get(block=block, timeout=timeout)
+        except queue.Empty:
+            return None
+
+    def put(self, item: T) -> None:
+        self._impl.put(item, block=False)
+
+    def kill(self) -> None:
+        self._impl.cancel_join_thread()
+        self._impl.close()
+
+
+class SpawnWorkerFactory(WorkerFactory):
+    """An implementation of `WorkerFactory` backed by the `multiprocessing`
+    module, with new processes started by spawning.
+    """
+
+    def __init__(self) -> None:
+        self._ctx = multiprocessing.get_context("spawn")
+
+    def make_queue(self) -> Queue[Any]:
+        return _ProcessQueue(self._ctx.Queue())
+
+    def make_event(self) -> Event:
+        return self._ctx.Event()
+
+    def make_worker(
+        self, target: Callable[..., None], args: tuple[Any, ...], name: str | None = None
+    ) -> Worker:
+        process = self._ctx.Process(target=target, args=args, name=name)
+        process.start()
+        return _ProcessWorker(process)