csu 2.2.3__tar.gz → 2.2.5__tar.gz

{csu-2.2.3 → csu-2.2.5}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.2.3
+current_version = 2.2.5
 commit = True
 tag = True
 

{csu-2.2.3 → csu-2.2.5}/.cookiecutterrc

@@ -40,7 +40,7 @@ default_context:
     sphinx_theme: furo
     test_matrix_separate_coverage: 'no'
     tests_inside_package: 'yes'
-    version: 2.2.3
+    version: 2.2.5
     version_manager: bump2version
     website: https://blog.ionelmc.ro
     year_from: '2024'

{csu-2.2.3 → csu-2.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csu
-Version: 2.2.3
+Version: 2.2.5
 Summary: Clean Slate Utils - bunch of utility code, mostly Django/DRF specific.
 Author-email: Ionel Cristian Mărieș <contact@ionelmc.ro>
 License-Expression: BSD-2-Clause
@@ -77,9 +77,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.2.3.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.2.5.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.2.3...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.2.5...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

{csu-2.2.3 → csu-2.2.5}/README.rst

@@ -34,9 +34,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.2.3.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.2.5.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.2.3...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.2.5...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

{csu-2.2.3 → csu-2.2.5}/pyproject.toml

@@ -12,7 +12,7 @@ dynamic = [
     "readme",
 ]
 name = "csu"
-version = "2.2.3"
+version = "2.2.5"
 license = "BSD-2-Clause"
 license-files = ["LICENSE"]
 description = "Clean Slate Utils - bunch of utility code, mostly Django/DRF specific."

{csu-2.2.3 → csu-2.2.5}/src/csu/__init__.py

@@ -2,4 +2,4 @@
 a.k.a Clean Slate Utils
 """
 
-__version__ = "2.2.3"
+__version__ = "2.2.5"

{csu-2.2.3 → csu-2.2.5}/src/csu/worker/__init__.py

@@ -17,7 +17,3 @@ Typical batch processing workload::
 
 The database is completely optional.
 """
-
-from logging import getLogger
-
-logger = getLogger("worker")

csu-2.2.5/src/csu/worker/asgi.py

@@ -0,0 +1,74 @@
+from logging import getLogger
+
+import django
+from asgiref.typing import ASGIReceiveCallable
+from asgiref.typing import ASGIReceiveEvent
+from asgiref.typing import ASGISendCallable
+from asgiref.typing import LifespanShutdownCompleteEvent
+from asgiref.typing import LifespanShutdownFailedEvent
+from asgiref.typing import LifespanStartupCompleteEvent
+from asgiref.typing import LifespanStartupFailedEvent
+from django.core.handlers.asgi import ASGIHandler
+
+from ..utils import lazy_import_classproperty
+
+logger = getLogger(__name__)
+
+
+class WorkerLifespanASGIHandler(ASGIHandler):
+    async def __call__(self, scope, receive: ASGIReceiveCallable, send) -> None:
+        if scope["type"] == "lifespan":
+            await self.handle_lifespan(receive, send)
+        else:
+            await super().__call__(scope, receive, send)
+
+    engine_registry = lazy_import_classproperty("csu.worker.registry.REGISTRY")
+
+    async def handle_start_engines(self):
+        registry = self.engine_registry
+        logger.info("Found %s registered engines.", len(registry))
+        for module_name, engine in registry.items():
+            logger.info("Starting engine for %s: %s...", module_name, engine)
+            if engine.shutdown_on:
+                raise RuntimeError(f"Engine {type(engine)} shutdown_on should be empty, not {engine.shutdown_on}!")
+            await engine.start()
+        logger.info(
+            "Started %s engines: \n   %s",
+            len(registry),
+            "\n   ".join(f"{module_name}: {engine}" for module_name, engine in registry.items()),
+        )
+
+    async def handle_stop_engines(self):
+        registry = self.engine_registry
+        logger.info("Stopping %s registered engines.", len(registry))
+        for engine in registry.values():
+            await engine.stop(graceful=True)
+
+    async def handle_lifespan(self, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None:
+        while True:
+            message: ASGIReceiveEvent = await receive()
+            match message["type"]:
+                case "lifespan.startup":
+                    try:
+                        await self.handle_start_engines()
+                    except Exception as exc:
+                        await send(LifespanStartupFailedEvent(type="lifespan.startup.failed", message=str(exc)))
+                        raise
+                    else:
+                        await send(LifespanStartupCompleteEvent(type="lifespan.startup.complete"))
+                case "lifespan.shutdown":
+                    try:
+                        await self.handle_stop_engines()
+                    except Exception as exc:
+                        await send(LifespanShutdownFailedEvent(type="lifespan.shutdown.failed", message=str(exc)))
+                        raise
+                    else:
+                        await send(LifespanShutdownCompleteEvent(type="lifespan.shutdown.complete"))
+                        return
+                case _:
+                    raise ValueError(f"Unexpected message type: {message['type']!r}", message)
+
+
+def get_worker_lifespan_application() -> WorkerLifespanASGIHandler:
+    django.setup(set_prefix=False)
+    return WorkerLifespanASGIHandler()

{csu-2.2.3 → csu-2.2.5}/src/csu/worker/engine.py

@@ -1,5 +1,6 @@
 import asyncio
 import signal
+import sys
 import traceback
 from abc import ABC
 from abc import abstractmethod
@@ -17,6 +18,7 @@ from datetime import timedelta
 from enum import Flag
 from enum import auto
 from functools import partial
+from logging import getLogger
 from random import randint
 from typing import ClassVar
 from typing import Protocol
@@ -25,10 +27,11 @@ from datetimerange import DateTimeRange
 
 from ..exceptions import InternalServiceError
 from ..timezones import naivenow
-from . import logger
 from .enums import WorkerState
 from .job import Job
 
+logger = getLogger(__name__)
+
 
 class TaskProtocol(Protocol):
     job_kwargs: dict
@@ -269,9 +272,11 @@ class AbstractEngine(ABC):
     workers: list[AbstractWorker]
     queue: Queue = None
     queue_sem: Semaphore
-    shutdown_on: Iterable[int] = (signal.SIGINT, signal.SIGTERM)
+    shutdown_on: Iterable[signal.Signals] = (signal.SIGINT, signal.SIGTERM)
     shutdown_requested = False
     shutdown_tasks: ClassVar[dict[bool, asyncio.Task]] = {}
+    if sys.platform == "win32":
+        shutdown_on += (signal.SIGBREAK,)
 
     def __init__(self):
         self.workers = []
@@ -314,11 +319,13 @@ class AbstractEngine(ABC):
         for worker in self.workers:
             assert worker.runner is None, f"{worker=} should not be started"
             worker.start()
+
+        loop = asyncio.get_event_loop()
         for sig in self.shutdown_on:
-            signal.signal(sig, self.shutdown_handler)
+            loop.add_signal_handler(sig, self.shutdown_handler, sig)
 
-    def shutdown_handler(self, sig, frame):
-        sig_name = signal.Signals(sig).name
+    def shutdown_handler(self, sig: signal.Signals) -> None:
+        sig_name = sig.name
         graceful = not self.shutdown_requested
         logger.info("%s.shutdown_handler(%s) %s", self, sig_name, "GRACEFUL" if graceful else "FORCED")
         if not graceful:

{csu-2.2.3 → csu-2.2.5}/src/csu/worker/models.py

@@ -1,6 +1,7 @@
 from abc import abstractmethod
 from asyncio import CancelledError
 from datetime import timedelta
+from logging import getLogger
 from random import SystemRandom
 
 from asgiref.sync import sync_to_async
@@ -11,11 +12,12 @@ from django.db.models import F
 from django.db.models import Q
 
 from ..timezones import utcnow
-from . import logger
 from .engine import AbstractConsumer
 from .engine import AbstractProducer
 from .job import Job
 
+logger = getLogger(__name__)
+
 
 def get_random_float(system_random=SystemRandom()):  # noqa: B008
     return system_random.random()

{csu-2.2.3 → csu-2.2.5}/src/csu.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csu
-Version: 2.2.3
+Version: 2.2.5
 Summary: Clean Slate Utils - bunch of utility code, mostly Django/DRF specific.
 Author-email: Ionel Cristian Mărieș <contact@ionelmc.ro>
 License-Expression: BSD-2-Clause
@@ -77,9 +77,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.2.3.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.2.5.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.2.3...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.2.5...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

csu-2.2.5/tests/test_exampleworker.py

@@ -0,0 +1,164 @@
+import signal
+import sys
+import time
+
+from process_tests import TestProcess
+from process_tests import dump_on_error
+from process_tests import wait_for_strings
+
+
+def test_noskip():
+    with TestProcess(sys.executable, "-mexampleworker", "3", "0", "1000", "4") as target, dump_on_error(target.read):
+        wait_for_strings(
+            target.read,
+            5,
+            "INFO:csu.worker.engine:ExampleConsumer(1/UNKNOWN) started.",
+            "INFO:csu.worker.engine:ExampleConsumer(2/UNKNOWN) started.",
+            "INFO:csu.worker.engine:ExampleConsumer(3/UNKNOWN) started.",
+            "INFO:csu.worker.engine:ExampleProducer(1/UNKNOWN) started.",
+            "INFO:csu.worker.engine:ExampleProducer(1/UNKNOWN).run() started.",
+            "PRODUCER fetch_task @ 1 RETURN",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=1))",
+            "INFO:csu.worker.engine:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=1).done of NoOpTask(id=1).",
+            "PRODUCER fetch_task @ 2 RETURN",
+            "INFO:csu.worker.engine:ExampleEngine(4w/1q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=2))",
+            "INFO:csu.worker.engine:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=2).done of NoOpTask(id=2).",
+            "PRODUCER fetch_task @ 3 RETURN",
+            "INFO:csu.worker.engine:ExampleEngine(4w/2q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=3))",
+            "INFO:csu.worker.engine:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=3).done of NoOpTask(id=3).",
+            "INFO:csu.worker.engine:ExampleConsumer(1/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=1)",
+            "INFO:csu.worker.engine:ExampleConsumer(1/WORKING).run() - performing...",
+            "CONSUMER 0 perform_work JOB 1 START",
+            "INFO:csu.worker.engine:ExampleConsumer(2/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=2)",
+            "INFO:csu.worker.engine:ExampleConsumer(2/WORKING).run() - performing...",
+            "CONSUMER 1 perform_work JOB 2 START",
+            "INFO:csu.worker.engine:ExampleConsumer(3/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=3)",
+            "INFO:csu.worker.engine:ExampleConsumer(3/WORKING).run() - performing...",
+            "CONSUMER 2 perform_work JOB 3 START",
+            "CONSUMER 0 perform_work JOB 1 DONE",
+            "CONSUMER 0 save_result JOB 1",
+            "CONSUMER 1 perform_work JOB 2 DONE",
+            "CONSUMER 1 save_result JOB 2",
+            "CONSUMER 2 perform_work JOB 3 DONE",
+            "CONSUMER 2 save_result JOB 3",
+            "PRODUCER fetch_task @ 4 RETURN",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=4))",
+            "INFO:csu.worker.engine:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=4).done of NoOpTask(id=4).",
+            "PRODUCER fetch_task @ 5 EXHAUSTED",
+            "INFO:csu.worker.engine:ExampleConsumer(1/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=4)",
+            "INFO:csu.worker.engine:ExampleConsumer(1/WORKING).run() - performing...",
+            "CONSUMER 0 perform_work JOB 4 START",
+            "PRODUCER fetch_task @ 6 EXHAUSTED",
+            "CONSUMER 0 perform_work JOB 4 DONE",
+            "CONSUMER 0 save_result JOB 4",
+            "PRODUCER fetch_task @ 7 EXHAUSTED",
+        )
+
+
+def test_skip():
+    with TestProcess(sys.executable, "-mexampleworker", "3", "2", "100", "4") as target, dump_on_error(target.read):
+        wait_for_strings(
+            target.read,
+            5,
+            "INFO:csu.worker.engine:ExampleConsumer(1/UNKNOWN) started.",
+            "INFO:csu.worker.engine:ExampleConsumer(2/UNKNOWN) started.",
+            "INFO:csu.worker.engine:ExampleConsumer(3/UNKNOWN) started.",
+            "INFO:csu.worker.engine:ExampleProducer(1/UNKNOWN) started.",
+            "INFO:csu.worker.engine:ExampleProducer(1/UNKNOWN).run() started.",
+            "PRODUCER fetch_task @ 1 RETURN",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=1))",
+            "INFO:csu.worker.engine:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=1).done of NoOpTask(id=1).",
+            "PRODUCER fetch_task @ 2 SKIP",
+            "INFO:csu.worker.engine:ExampleConsumer(1/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=1)",
+            "INFO:csu.worker.engine:ExampleConsumer(1/WORKING).run() - performing...",
+            "CONSUMER 0 perform_work JOB 1 START",
+            "CONSUMER 0 perform_work JOB 1 DONE",
+            "CONSUMER 0 save_result JOB 1",
+            "PRODUCER fetch_task @ 3 RETURN",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=3))",
+            "INFO:csu.worker.engine:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=3).done of NoOpTask(id=3).",
+            "PRODUCER fetch_task @ 4 SKIP",
+            "INFO:csu.worker.engine:ExampleConsumer(2/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=3)",
+            "INFO:csu.worker.engine:ExampleConsumer(2/WORKING).run() - performing...",
+            "CONSUMER 1 perform_work JOB 3 START",
+            "CONSUMER 1 perform_work JOB 3 DONE",
+            "CONSUMER 1 save_result JOB 3",
+            "PRODUCER fetch_task @ 5 EXHAUSTED",
+        )
+
+
+def test_graceful_shutdown():
+    with TestProcess(sys.executable, "-mexampleworker", "3", "0", "1000", "100") as target, dump_on_error(target.read):
+        wait_for_strings(
+            target.read,
+            15,
+            "PRODUCER fetch_task @ 1 RETURN",
+        )
+        target.proc.send_signal(signal.SIGINT)
+        wait_for_strings(
+            target.read,
+            15,
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q).shutdown_handler(SIGINT) GRACEFUL",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q).stop(graceful=True)",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q) stats: {'ExampleConsumer': {'WORKING': 3}, 'ExampleProducer': {'WAITING': 1}, 'ExampleEngine': {'JOBS': 3, 'QSIZE': 0, 'SEMVAL': 0}}",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q) worker 1: <__main__.ExampleConsumer object at ",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q) worker 2: <__main__.ExampleConsumer object at ",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q) worker 3: <__main__.ExampleConsumer object at ",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q) worker 4: <__main__.ExampleProducer object at ",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q) requesting workers to stop and awaiting 1 producers...",
+            "CONSUMER 0 perform_work JOB 1 DONE",
+            "CONSUMER 0 save_result JOB 1",
+            "INFO:csu.worker.engine:ExampleProducer(1/WAITING).on_exit(",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q) flushing queue (0 items) and awaiting consumers...",
+            "INFO:csu.worker.engine:ExampleConsumer(1/SHUTDOWN).on_exit(",
+            "INFO:csu.worker.engine:ExampleConsumer(2/SHUTDOWN).on_exit(",
+            "INFO:csu.worker.engine:ExampleConsumer(3/SHUTDOWN).on_exit(",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q) shutdown complete.",
+            "DONE: {'ExampleConsumer': {'EXITED': 3}, 'ExampleProducer': {'EXITED': 1}, 'ExampleEngine': {'JOBS': 3, 'QSIZE': 0, 'SEMVAL': 2}}",
+        )
+
+
+def test_forced_shutdown():
+    with TestProcess(sys.executable, "-mexampleworker", "3", "0", "2000", "100") as target, dump_on_error(target.read):
+        wait_for_strings(
+            target.read,
+            15,
+            "PRODUCER fetch_task @ 1 RETURN",
+        )
+        target.proc.send_signal(signal.SIGINT)
+        time.sleep(1)
+        target.proc.send_signal(signal.SIGTERM)
+        wait_for_strings(
+            target.read,
+            15,
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q).shutdown_handler(SIGINT) GRACEFUL",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q).stop(graceful=True)",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q) requesting workers to stop and awaiting 1 producers...",
+            "INFO:csu.worker.engine:ExampleEngine(4w/0q).shutdown_handler(SIGTERM) FORCED",
+            "CONSUMER 0 perform_work JOB 1 DONE",
+            "CONSUMER 1 perform_work JOB 2 DONE",
+            "CONSUMER 2 perform_work JOB 3 DONE",
+            "DONE: {'ExampleConsumer': {'CANCELED': 3}, 'ExampleProducer': {'CANCELED': 1}, 'ExampleEngine': {'JOBS': 3, 'QSIZE': 0, 'SEMVAL': 0}}",
+        )
+
+
+def test_sem_rollback():
+    with TestProcess(sys.executable, "-mexampleworker", "1", "2", "10", "10") as target, dump_on_error(target.read):
+        wait_for_strings(
+            target.read,
+            15,
+            "PRODUCER fetch_task @ 1 RETURN",
+            "CONSUMER 0 perform_work JOB 1 START",
+            "CONSUMER 0 perform_work JOB 1 DONE",
+            "CONSUMER 0 save_result JOB 1",
+            "PRODUCER fetch_task @ 2 SKIP",
+            "PRODUCER fetch_task @ 3 RETURN",
+            "CONSUMER 0 perform_work JOB 3 START",
+            "CONSUMER 0 perform_work JOB 3 DONE",
+            "CONSUMER 0 save_result JOB 3",
+            "PRODUCER fetch_task @ 4 SKIP",
+            "PRODUCER fetch_task @ 5 RETURN",
+            "CONSUMER 0 perform_work JOB 5 START",
+            "CONSUMER 0 perform_work JOB 5 DONE",
+            "CONSUMER 0 save_result JOB 5",
+        )

csu-2.2.3/src/csu/worker/asgi.py

@@ -1,36 +0,0 @@
-from asgiref.typing import ASGIReceiveCallable
-from asgiref.typing import ASGISendCallable
-from asgiref.typing import Scope
-from django.core.asgi import get_asgi_application
-
-from . import logger
-
-
-def get_worker_lifespan_application():
-    django_application = get_asgi_application()
-
-    async def application_with_worker_lifespan(scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None:
-        if scope["type"] == "lifespan":
-            try:
-                from .registry import REGISTRY  # noqa:PLC0415
-
-                while True:
-                    message = await receive()
-                    message_type = message["type"]
-                    if message_type == "lifespan.startup":
-                        logger.info("Found %s registered engines.", len(REGISTRY))
-                        for module_name, engine in REGISTRY.items():
-                            logger.info("Starting engine for %s: %s...", module_name, engine)
-                            await engine.start()
-                        await send({"type": "lifespan.startup.complete"})
-                    if message_type == "lifespan.shutdown":
-                        for engine in REGISTRY.values():
-                            await engine.stop()
-                        await send({"type": "lifespan.shutdown.complete"})
-                        return
-            except Exception as exc:
-                logger.exception("Failed setting up lifespan: %r", exc)
-        else:
-            await django_application(scope, receive, send)
-
-    return application_with_worker_lifespan

csu-2.2.3/tests/test_exampleworker.py

@@ -1,164 +0,0 @@
-import signal
-import sys
-import time
-
-from process_tests import TestProcess
-from process_tests import dump_on_error
-from process_tests import wait_for_strings
-
-
-def test_noskip():
-    with TestProcess(sys.executable, "-mexampleworker", "3", "0", "1000", "4") as target, dump_on_error(target.read):
-        wait_for_strings(
-            target.read,
-            5,
-            "INFO:worker:ExampleConsumer(1/UNKNOWN) started.",
-            "INFO:worker:ExampleConsumer(2/UNKNOWN) started.",
-            "INFO:worker:ExampleConsumer(3/UNKNOWN) started.",
-            "INFO:worker:ExampleProducer(1/UNKNOWN) started.",
-            "INFO:worker:ExampleProducer(1/UNKNOWN).run() started.",
-            "PRODUCER fetch_task @ 1 RETURN",
-            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=1))",
-            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=1).done of NoOpTask(id=1).",
-            "PRODUCER fetch_task @ 2 RETURN",
-            "INFO:worker:ExampleEngine(4w/1q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=2))",
-            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=2).done of NoOpTask(id=2).",
-            "PRODUCER fetch_task @ 3 RETURN",
-            "INFO:worker:ExampleEngine(4w/2q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=3))",
-            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=3).done of NoOpTask(id=3).",
-            "INFO:worker:ExampleConsumer(1/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=1)",
-            "INFO:worker:ExampleConsumer(1/WORKING).run() - performing...",
-            "CONSUMER 0 perform_work JOB 1 START",
-            "INFO:worker:ExampleConsumer(2/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=2)",
-            "INFO:worker:ExampleConsumer(2/WORKING).run() - performing...",
-            "CONSUMER 1 perform_work JOB 2 START",
-            "INFO:worker:ExampleConsumer(3/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=3)",
-            "INFO:worker:ExampleConsumer(3/WORKING).run() - performing...",
-            "CONSUMER 2 perform_work JOB 3 START",
-            "CONSUMER 0 perform_work JOB 1 DONE",
-            "CONSUMER 0 save_result JOB 1",
-            "CONSUMER 1 perform_work JOB 2 DONE",
-            "CONSUMER 1 save_result JOB 2",
-            "CONSUMER 2 perform_work JOB 3 DONE",
-            "CONSUMER 2 save_result JOB 3",
-            "PRODUCER fetch_task @ 4 RETURN",
-            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=4))",
-            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=4).done of NoOpTask(id=4).",
-            "PRODUCER fetch_task @ 5 EXHAUSTED",
-            "INFO:worker:ExampleConsumer(1/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=4)",
-            "INFO:worker:ExampleConsumer(1/WORKING).run() - performing...",
-            "CONSUMER 0 perform_work JOB 4 START",
-            "PRODUCER fetch_task @ 6 EXHAUSTED",
-            "CONSUMER 0 perform_work JOB 4 DONE",
-            "CONSUMER 0 save_result JOB 4",
-            "PRODUCER fetch_task @ 7 EXHAUSTED",
-        )
-
-
-def test_skip():
-    with TestProcess(sys.executable, "-mexampleworker", "3", "2", "100", "4") as target, dump_on_error(target.read):
-        wait_for_strings(
-            target.read,
-            5,
-            "INFO:worker:ExampleConsumer(1/UNKNOWN) started.",
-            "INFO:worker:ExampleConsumer(2/UNKNOWN) started.",
-            "INFO:worker:ExampleConsumer(3/UNKNOWN) started.",
-            "INFO:worker:ExampleProducer(1/UNKNOWN) started.",
-            "INFO:worker:ExampleProducer(1/UNKNOWN).run() started.",
-            "PRODUCER fetch_task @ 1 RETURN",
-            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=1))",
-            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=1).done of NoOpTask(id=1).",
-            "PRODUCER fetch_task @ 2 SKIP",
-            "INFO:worker:ExampleConsumer(1/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=1)",
-            "INFO:worker:ExampleConsumer(1/WORKING).run() - performing...",
-            "CONSUMER 0 perform_work JOB 1 START",
-            "CONSUMER 0 perform_work JOB 1 DONE",
-            "CONSUMER 0 save_result JOB 1",
-            "PRODUCER fetch_task @ 3 RETURN",
-            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=3))",
-            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=3).done of NoOpTask(id=3).",
-            "PRODUCER fetch_task @ 4 SKIP",
-            "INFO:worker:ExampleConsumer(2/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=3)",
-            "INFO:worker:ExampleConsumer(2/WORKING).run() - performing...",
-            "CONSUMER 1 perform_work JOB 3 START",
-            "CONSUMER 1 perform_work JOB 3 DONE",
-            "CONSUMER 1 save_result JOB 3",
-            "PRODUCER fetch_task @ 5 EXHAUSTED",
-        )
-
-
-def test_graceful_shutdown():
-    with TestProcess(sys.executable, "-mexampleworker", "3", "0", "1000", "100") as target, dump_on_error(target.read):
-        wait_for_strings(
-            target.read,
-            15,
-            "PRODUCER fetch_task @ 1 RETURN",
-        )
-        target.proc.send_signal(signal.SIGINT)
-        wait_for_strings(
-            target.read,
-            15,
-            "INFO:worker:ExampleEngine(4w/0q).shutdown_handler(SIGINT) GRACEFUL",
-            "INFO:worker:ExampleEngine(4w/0q).stop(graceful=True)",
-            "INFO:worker:ExampleEngine(4w/0q) stats: {'ExampleConsumer': {'WORKING': 3}, 'ExampleProducer': {'WAITING': 1}, 'ExampleEngine': {'JOBS': 3, 'QSIZE': 0, 'SEMVAL': 0}}",
-            "INFO:worker:ExampleEngine(4w/0q) worker 1: <__main__.ExampleConsumer object at ",
-            "INFO:worker:ExampleEngine(4w/0q) worker 2: <__main__.ExampleConsumer object at ",
-            "INFO:worker:ExampleEngine(4w/0q) worker 3: <__main__.ExampleConsumer object at ",
-            "INFO:worker:ExampleEngine(4w/0q) worker 4: <__main__.ExampleProducer object at ",
-            "INFO:worker:ExampleEngine(4w/0q) requesting workers to stop and awaiting 1 producers...",
-            "CONSUMER 0 perform_work JOB 1 DONE",
-            "CONSUMER 0 save_result JOB 1",
-            "INFO:worker:ExampleProducer(1/WAITING).on_exit(",
-            "INFO:worker:ExampleEngine(4w/0q) flushing queue (0 items) and awaiting consumers...",
-            "INFO:worker:ExampleConsumer(1/SHUTDOWN).on_exit(",
-            "INFO:worker:ExampleConsumer(2/SHUTDOWN).on_exit(",
-            "INFO:worker:ExampleConsumer(3/SHUTDOWN).on_exit(",
-            "INFO:worker:ExampleEngine(4w/0q) shutdown complete.",
-            "DONE: {'ExampleConsumer': {'EXITED': 3}, 'ExampleProducer': {'EXITED': 1}, 'ExampleEngine': {'JOBS': 3, 'QSIZE': 0, 'SEMVAL': 2}}",
-        )
-
-
-def test_forced_shutdown():
-    with TestProcess(sys.executable, "-mexampleworker", "3", "0", "2000", "100") as target, dump_on_error(target.read):
-        wait_for_strings(
-            target.read,
-            15,
-            "PRODUCER fetch_task @ 1 RETURN",
-        )
-        target.proc.send_signal(signal.SIGINT)
-        time.sleep(1)
-        target.proc.send_signal(signal.SIGTERM)
-        wait_for_strings(
-            target.read,
-            15,
-            "INFO:worker:ExampleEngine(4w/0q).shutdown_handler(SIGINT) GRACEFUL",
-            "INFO:worker:ExampleEngine(4w/0q).stop(graceful=True)",
-            "INFO:worker:ExampleEngine(4w/0q) requesting workers to stop and awaiting 1 producers...",
-            "INFO:worker:ExampleEngine(4w/0q).shutdown_handler(SIGTERM) FORCED",
-            "CONSUMER 0 perform_work JOB 1 DONE",
-            "CONSUMER 1 perform_work JOB 2 DONE",
-            "CONSUMER 2 perform_work JOB 3 DONE",
-            "DONE: {'ExampleConsumer': {'CANCELED': 3}, 'ExampleProducer': {'CANCELED': 1}, 'ExampleEngine': {'JOBS': 3, 'QSIZE': 0, 'SEMVAL': 0}}",
-        )
-
-
-def test_sem_rollback():
-    with TestProcess(sys.executable, "-mexampleworker", "1", "2", "10", "10") as target, dump_on_error(target.read):
-        wait_for_strings(
-            target.read,
-            15,
-            "PRODUCER fetch_task @ 1 RETURN",
-            "CONSUMER 0 perform_work JOB 1 START",
-            "CONSUMER 0 perform_work JOB 1 DONE",
-            "CONSUMER 0 save_result JOB 1",
-            "PRODUCER fetch_task @ 2 SKIP",
-            "PRODUCER fetch_task @ 3 RETURN",
-            "CONSUMER 0 perform_work JOB 3 START",
-            "CONSUMER 0 perform_work JOB 3 DONE",
-            "CONSUMER 0 save_result JOB 3",
-            "PRODUCER fetch_task @ 4 SKIP",
-            "PRODUCER fetch_task @ 5 RETURN",
-            "CONSUMER 0 perform_work JOB 5 START",
-            "CONSUMER 0 perform_work JOB 5 DONE",
-            "CONSUMER 0 save_result JOB 5",
-        )