csu 2.1.5__tar.gz → 2.2.0__tar.gz

{csu-2.1.5 → csu-2.2.0}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.1.5
+current_version = 2.2.0
 commit = True
 tag = True
 

{csu-2.1.5 → csu-2.2.0}/.cookiecutterrc

@@ -40,7 +40,7 @@ default_context:
     sphinx_theme: furo
     test_matrix_separate_coverage: 'no'
     tests_inside_package: 'yes'
-    version: 2.1.5
+    version: 2.2.0
     version_manager: bump2version
     website: https://blog.ionelmc.ro
     year_from: '2024'

{csu-2.1.5 → csu-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csu
-Version: 2.1.5
+Version: 2.2.0
 Summary: Clean Slate Utils - bunch of utility code, mostly Django/DRF specific.
 Author-email: Ionel Cristian Mărieș <contact@ionelmc.ro>
 License-Expression: BSD-2-Clause
@@ -77,9 +77,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.5.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.2.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.1.5...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.2.0...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

{csu-2.1.5 → csu-2.2.0}/README.rst

@@ -34,9 +34,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.5.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.2.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.1.5...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.2.0...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

{csu-2.1.5 → csu-2.2.0}/pyproject.toml

@@ -12,7 +12,7 @@ dynamic = [
     "readme",
 ]
 name = "csu"
-version = "2.1.5"
+version = "2.2.0"
 license = "BSD-2-Clause"
 license-files = ["LICENSE"]
 description = "Clean Slate Utils - bunch of utility code, mostly Django/DRF specific."

{csu-2.1.5 → csu-2.2.0}/src/csu/__init__.py

@@ -2,4 +2,4 @@
 a.k.a Clean Slate Utils
 """
 
-__version__ = "2.1.5"
+__version__ = "2.2.0"

{csu-2.1.5 → csu-2.2.0}/src/csu/worker/engine.py

@@ -4,9 +4,9 @@ import traceback
 from abc import ABC
 from abc import abstractmethod
 from asyncio import CancelledError
-from asyncio import Event
 from asyncio import Future
 from asyncio import Queue
+from asyncio import Semaphore
 from collections import defaultdict
 from collections.abc import AsyncIterator
 from collections.abc import Iterable
@@ -18,6 +18,7 @@ from enum import Flag
 from enum import auto
 from functools import partial
 from random import randint
+from typing import ClassVar
 from typing import Protocol
 
 from datetimerange import DateTimeRange
@@ -47,17 +48,19 @@ class AbstractWorker(ABC):
     state: WorkerState = WorkerState.UNKNOWN
     runner: asyncio.Task | None = None
     shutdown_requested = False
+    id_counter: ClassVar = 0
+    id = "?"
 
     def __str__(self):
-        return f"{type(self).__name__}({self.state.name})"
-
-    def __repr__(self):
-        return f"{type(self).__name__}({id(self)}, state={self.state.name})"
+        return f"{type(self).__name__}({self.id}/{self.state.name})"
 
     def report(self, inspect: defaultdict[str, int]):
         inspect[self.state.name] += 1
 
     def start(self):
+        cls = type(self)
+        cls.id_counter += 1
+        self.id = cls.id_counter
         self.runner = asyncio.create_task(self.run())
         # noinspection PyTypeChecker
         self.runner.add_done_callback(self.on_exit)
@@ -161,9 +164,6 @@ class AbstractProducer(AbstractWorker):
     def __init__(self, engine: AbstractEngine):
         self.engine = engine
 
-    def __str__(self):
-        return f"{type(self).__name__}()"
-
     async def run(self):
         task: TaskProtocol | None
         logger.info("%s.run() started.", self)
@@ -211,15 +211,14 @@ class AbstractConsumer(AbstractWorker):
     def __init__(self, engine: AbstractEngine):
         self.engine = engine
 
-    def __str__(self):
-        return f"{type(self).__name__}({id(self)})"
-
     async def run(self):
         while True:
-            if not self.shutdown_requested:
+            if self.shutdown_requested:
+                self.state = WorkerState.SHUTDOWN
+            else:
                 self.state = WorkerState.BEFORE
                 await self.before()
-            self.state = WorkerState.READY
+                self.state = WorkerState.READY
             job: Job | None
             async with self.engine.consume() as job:
                 if job is None:
@@ -255,8 +254,9 @@ class AbstractConsumer(AbstractWorker):
 
             if self.shutdown_requested:
                 continue
-            self.state = WorkerState.AFTER
-            await self.after(idle=idle)
+            else:
+                self.state = WorkerState.AFTER
+                await self.after(idle=idle)
 
     @abstractmethod
     async def perform_work(self, job: Job) -> dict:
@@ -266,15 +266,13 @@ class AbstractConsumer(AbstractWorker):
 class AbstractEngine(ABC):
     workers: list[AbstractWorker]
     queue: Queue = None
-    queue_maxsize: int
+    queue_sem: Semaphore
     shutdown_on: Iterable[int] = (signal.SIGINT, signal.SIGTERM)
     shutdown_requested = False
-    shutdown_task = None
+    shutdown_tasks: ClassVar[dict[bool, asyncio.Task]] = {}
 
     def __init__(self):
         self.workers = []
-        self.queue_awaited = Event()
-        self.queue_maxsize = 0
 
     def __str__(self):
         return f"{type(self).__name__}({len(self.workers)}w/{self.queue.qsize() if self.queue else '-'}q)"
@@ -288,28 +286,27 @@ class AbstractEngine(ABC):
 
     async def push(self, job: Job | None) -> None:
         logger.info("%s.push(%s)", self, job)
-        self.queue_awaited.clear()  # clear it here, it's the shortest path to "consumer wakes up and calls .set()"
         await self.queue.put(job)
 
     @asynccontextmanager
     async def consume(self) -> AsyncIterator[Job | None]:
-        self.queue_awaited.set()
+        self.queue_sem.release()
         yield await self.queue.get()
         # if raised then there's no task to mark as done (this won't be reached on exception)
         self.queue.task_done()
 
     async def wait_worker_ready(self):
         # event set after consuming, cleared right before there's a task
-        await self.queue_awaited.wait()
+        await self.queue_sem.acquire()
 
     def add_worker(self, worker: AbstractWorker):
-        assert self.queue is None, f"{self} is already started"
-        if isinstance(worker, AbstractConsumer):
-            self.queue_maxsize += 1
+        assert self.queue is None, f"{self} is already started, {self.queue=}"
         self.workers.append(worker)
 
     async def start(self):
-        self.queue = Queue(maxsize=self.queue_maxsize)
+        assert self.queue is None, f"{self} is already started, {self.queue=}"
+        self.queue = Queue()
+        self.queue_sem = Semaphore(value=0)
         for worker in self.workers:
             assert worker.runner is None, f"{worker=} should not be started"
             worker.start()
@@ -317,8 +314,15 @@ class AbstractEngine(ABC):
             signal.signal(sig, self.shutdown_handler)
 
     def shutdown_handler(self, sig, frame):
-        logger.info("%s.shutdown_handler(%s)", self, sig)
-        self.shutdown_task = asyncio.create_task(self.stop(graceful=not self.shutdown_requested))
+        sig_name = signal.Signals(sig).name
+        graceful = not self.shutdown_requested
+        logger.info("%s.shutdown_handler(%s) %s", self, sig_name, "GRACEFUL" if graceful else "FORCED")
+        if not graceful:
+            self.shutdown_tasks[True].cancel()
+        if graceful in self.shutdown_tasks:
+            logger.warn("%s.shutdown_handler(%s) already requested", self, sig_name)
+        else:
+            self.shutdown_tasks[graceful] = asyncio.create_task(self.stop(graceful=graceful))
         self.shutdown_requested = True
 
     async def stop(self, graceful=False):
@@ -327,16 +331,19 @@ class AbstractEngine(ABC):
         for i, worker in enumerate(self.workers, 1):
             logger.info("%s worker %s: %r", self, i, worker)
         if graceful:
-            logger.info("%s waiting producers to stop...", self)
-            producers = [worker for worker in self.workers if isinstance(worker, AbstractProducer)]
-            for producer in producers:
-                producer.shutdown_requested = True
-            await asyncio.gather(*[producer.runner for producer in producers], return_exceptions=True)
-            logger.info("%s requesting all workers to shutdown...", self)
             for worker in self.workers:
                 worker.shutdown_requested = True
+
+            producers = [worker.runner for worker in self.workers if isinstance(worker, AbstractProducer)]
+            logger.info("%s requesting workers to stop and awaiting %s producers...", self, len(producers))
+            await asyncio.gather(
+                *producers,
+                return_exceptions=True,
+            )
+            logger.info("%s flushing queue (%s items) and awaiting consumers...", self, self.queue.qsize())
+            for worker in self.workers:
                 if isinstance(worker, AbstractConsumer):
-                    await self.push(None)
+                    self.queue.put_nowait(None)
         else:
             while self.queue.qsize() > 0:
                 item = self.queue.get_nowait()
@@ -349,7 +356,10 @@ class AbstractEngine(ABC):
         logger.info("%s shutdown complete.", self)
 
     def report(self, inspect: defaultdict[str, object]):
-        inspect["QSIZE"] = "UNSTARTED" if self.queue is None else self.queue.qsize()
+        inspect.update(
+            QSIZE="UNSTARTED" if self.queue is None else self.queue.qsize(),
+            SEMVAL=self.queue_sem._value,
+        )
 
     def inspect(self) -> dict[str, dict[str, int]]:
         result = defaultdict(partial(defaultdict, int))

{csu-2.1.5 → csu-2.2.0}/src/csu/worker/enums.py

@@ -16,3 +16,4 @@ class WorkerState(Enum):
     CANCELED = auto()
     FAILED = auto()
     EXITED = auto()
+    SHUTDOWN = auto()

{csu-2.1.5 → csu-2.2.0}/src/csu.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csu
-Version: 2.1.5
+Version: 2.2.0
 Summary: Clean Slate Utils - bunch of utility code, mostly Django/DRF specific.
 Author-email: Ionel Cristian Mărieș <contact@ionelmc.ro>
 License-Expression: BSD-2-Clause
@@ -77,9 +77,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.5.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.2.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.1.5...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.2.0...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

csu-2.2.0/tests/test_exampleworker.py

@@ -0,0 +1,143 @@
+import signal
+import sys
+import time
+
+from process_tests import TestProcess
+from process_tests import dump_on_error
+from process_tests import wait_for_strings
+
+
+def test_noskip():
+    with TestProcess(sys.executable, "-mexampleworker", "3", "0", "1000", "4") as target, dump_on_error(target.read):
+        wait_for_strings(
+            target.read,
+            5,
+            "INFO:worker:ExampleConsumer(1/UNKNOWN) started.",
+            "INFO:worker:ExampleConsumer(2/UNKNOWN) started.",
+            "INFO:worker:ExampleConsumer(3/UNKNOWN) started.",
+            "INFO:worker:ExampleProducer(1/UNKNOWN) started.",
+            "INFO:worker:ExampleProducer(1/UNKNOWN).run() started.",
+            "PRODUCER fetch_task @ 1 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=1))",
+            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=1).done of NoOpTask(id=1).",
+            "PRODUCER fetch_task @ 2 RETURN",
+            "INFO:worker:ExampleEngine(4w/1q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=2))",
+            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=2).done of NoOpTask(id=2).",
+            "PRODUCER fetch_task @ 3 RETURN",
+            "INFO:worker:ExampleEngine(4w/2q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=3))",
+            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=3).done of NoOpTask(id=3).",
+            "INFO:worker:ExampleConsumer(1/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=1)",
+            "INFO:worker:ExampleConsumer(1/WORKING).run() - performing...",
+            "CONSUMER 0 perform_work JOB 1 START",
+            "INFO:worker:ExampleConsumer(2/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=2)",
+            "INFO:worker:ExampleConsumer(2/WORKING).run() - performing...",
+            "CONSUMER 1 perform_work JOB 2 START",
+            "INFO:worker:ExampleConsumer(3/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=3)",
+            "INFO:worker:ExampleConsumer(3/WORKING).run() - performing...",
+            "CONSUMER 2 perform_work JOB 3 START",
+            "CONSUMER 0 perform_work JOB 1 DONE",
+            "CONSUMER 0 save_result JOB 1",
+            "CONSUMER 1 perform_work JOB 2 DONE",
+            "CONSUMER 1 save_result JOB 2",
+            "CONSUMER 2 perform_work JOB 3 DONE",
+            "CONSUMER 2 save_result JOB 3",
+            "PRODUCER fetch_task @ 4 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=4))",
+            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=4).done of NoOpTask(id=4).",
+            "PRODUCER fetch_task @ 5 EXHAUSTED",
+            "INFO:worker:ExampleConsumer(1/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=4)",
+            "INFO:worker:ExampleConsumer(1/WORKING).run() - performing...",
+            "CONSUMER 0 perform_work JOB 4 START",
+            "PRODUCER fetch_task @ 6 EXHAUSTED",
+            "CONSUMER 0 perform_work JOB 4 DONE",
+            "CONSUMER 0 save_result JOB 4",
+            "PRODUCER fetch_task @ 7 EXHAUSTED",
+        )
+
+
+def test_skip():
+    with TestProcess(sys.executable, "-mexampleworker", "3", "2", "100", "4") as target, dump_on_error(target.read):
+        wait_for_strings(
+            target.read,
+            5,
+            "INFO:worker:ExampleConsumer(1/UNKNOWN) started.",
+            "INFO:worker:ExampleConsumer(2/UNKNOWN) started.",
+            "INFO:worker:ExampleConsumer(3/UNKNOWN) started.",
+            "INFO:worker:ExampleProducer(1/UNKNOWN) started.",
+            "INFO:worker:ExampleProducer(1/UNKNOWN).run() started.",
+            "PRODUCER fetch_task @ 1 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=1))",
+            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=1).done of NoOpTask(id=1).",
+            "PRODUCER fetch_task @ 2 SKIP",
+            "INFO:worker:ExampleConsumer(1/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=1)",
+            "INFO:worker:ExampleConsumer(1/WORKING).run() - performing...",
+            "CONSUMER 0 perform_work JOB 1 START",
+            "CONSUMER 0 perform_work JOB 1 DONE",
+            "CONSUMER 0 save_result JOB 1",
+            "PRODUCER fetch_task @ 3 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=3))",
+            "INFO:worker:ExampleProducer(1/WORKING).run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=3).done of NoOpTask(id=3).",
+            "PRODUCER fetch_task @ 4 SKIP",
+            "INFO:worker:ExampleConsumer(2/READY).run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=3)",
+            "INFO:worker:ExampleConsumer(2/WORKING).run() - performing...",
+            "CONSUMER 1 perform_work JOB 3 START",
+            "CONSUMER 1 perform_work JOB 3 DONE",
+            "CONSUMER 1 save_result JOB 3",
+            "PRODUCER fetch_task @ 5 EXHAUSTED",
+        )
+
+
+def test_graceful_shutdown():
+    with TestProcess(sys.executable, "-mexampleworker", "3", "0", "1000", "100") as target, dump_on_error(target.read):
+        wait_for_strings(
+            target.read,
+            15,
+            "PRODUCER fetch_task @ 1 RETURN",
+        )
+        target.proc.send_signal(signal.SIGINT)
+        wait_for_strings(
+            target.read,
+            15,
+            "INFO:worker:ExampleEngine(4w/0q).shutdown_handler(SIGINT) GRACEFUL",
+            "INFO:worker:ExampleEngine(4w/0q).stop(graceful=True)",
+            "INFO:worker:ExampleEngine(4w/0q) stats: {'ExampleConsumer': {'WORKING': 3}, 'ExampleProducer': {'WAITING': 1}, 'ExampleEngine': {'JOBS': 3, 'QSIZE': 0, 'SEMVAL': 0}}",
+            "INFO:worker:ExampleEngine(4w/0q) worker 1: <__main__.ExampleConsumer object at ",
+            "INFO:worker:ExampleEngine(4w/0q) worker 2: <__main__.ExampleConsumer object at ",
+            "INFO:worker:ExampleEngine(4w/0q) worker 3: <__main__.ExampleConsumer object at ",
+            "INFO:worker:ExampleEngine(4w/0q) worker 4: <__main__.ExampleProducer object at ",
+            "INFO:worker:ExampleEngine(4w/0q) requesting workers to stop and awaiting 1 producers...",
+            "CONSUMER 0 perform_work JOB 1 DONE",
+            "CONSUMER 0 save_result JOB 1",
+            "INFO:worker:ExampleProducer(1/WAITING).on_exit(<Task finished name='Task-5' coro=<AbstractProducer.run() done, defined at /home/ionel/open-source/python-csu/src/csu/worker/engine.py:167> result=None>) exit with result: None",
+            "INFO:worker:ExampleEngine(4w/0q) flushing queue (0 items) and awaiting consumers...",
+            "INFO:worker:ExampleConsumer(1/SHUTDOWN).on_exit(<Task finished name='Task-2' coro=<AbstractConsumer.run() done, defined at /home/ionel/open-source/python-csu/src/csu/worker/engine.py:214> result=None>) exit with result: None",
+            "INFO:worker:ExampleConsumer(2/SHUTDOWN).on_exit(<Task finished name='Task-3' coro=<AbstractConsumer.run() done, defined at /home/ionel/open-source/python-csu/src/csu/worker/engine.py:214> result=None>) exit with result: None",
+            "INFO:worker:ExampleConsumer(3/SHUTDOWN).on_exit(<Task finished name='Task-4' coro=<AbstractConsumer.run() done, defined at /home/ionel/open-source/python-csu/src/csu/worker/engine.py:214> result=None>) exit with result: None",
+            "INFO:worker:ExampleEngine(0w/0q) shutdown complete.",
+            "DONE: {'ExampleEngine':",
+        )
+
+
+def test_forced_shutdown():
+    with TestProcess(sys.executable, "-mexampleworker", "3", "0", "2000", "100") as target, dump_on_error(target.read):
+        wait_for_strings(
+            target.read,
+            15,
+            "PRODUCER fetch_task @ 1 RETURN",
+        )
+        target.proc.send_signal(signal.SIGINT)
+        time.sleep(1)
+        target.proc.send_signal(signal.SIGTERM)
+        wait_for_strings(
+            target.read,
+            15,
+            "INFO:worker:ExampleEngine(4w/0q).shutdown_handler(SIGINT) GRACEFUL",
+            "INFO:worker:ExampleEngine(4w/0q).stop(graceful=True)",
+            "INFO:worker:ExampleEngine(4w/0q) requesting workers to stop and awaiting 1 producers...",
+            "INFO:worker:ExampleEngine(4w/0q).shutdown_handler(SIGTERM) FORCED",
+            "CONSUMER 0 perform_work JOB 1 DONE",
+            "CONSUMER 1 perform_work JOB 2 DONE",
+            "CONSUMER 2 perform_work JOB 3 DONE",
+            "DONE: {'ExampleConsumer': {'CANCELED': 3}, 'ExampleProducer': {'CANCELED': 1}, 'ExampleEngine': {'JOBS': 3, 'QSIZE': 0, 'SEMVAL': 0}}",
+            "INFO:worker:ExampleEngine(0w/0q) shutdown complete.",
+        )

csu-2.1.5/tests/test_exampleworker.py

@@ -1,104 +0,0 @@
-import signal
-import sys
-
-from process_tests import TestProcess
-from process_tests import dump_on_error
-from process_tests import wait_for_strings
-
-
-def test_noskip():
-    with TestProcess(sys.executable, "-mexampleworker", "3", "0", "1000", "4") as target, dump_on_error(target.read):
-        wait_for_strings(
-            target.read,
-            5,
-            "PRODUCER fetch_task @ 1 RETURN",
-            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=1))",
-            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=1).done of NoOpTask(id=1).",
-            "working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=1)",
-            "- performing...",
-            "CONSUMER 0 perform_work JOB 1 START",
-            "CONSUMER 0 perform_work JOB 1 DONE",
-            "CONSUMER 0 save_result JOB 1",
-            "PRODUCER fetch_task @ 2 RETURN",
-            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=2))",
-            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=2).done of NoOpTask(id=2).",
-            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=2)",
-            ").run() - performing...",
-            "CONSUMER 1 perform_work JOB 2 START",
-            "CONSUMER 1 perform_work JOB 2 DONE",
-            "CONSUMER 1 save_result JOB 2",
-            "PRODUCER fetch_task @ 3 RETURN",
-            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=3))",
-            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=3).done of NoOpTask(id=3).",
-            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=3)",
-            ").run() - performing...",
-            "CONSUMER 2 perform_work JOB 3 START",
-            "CONSUMER 2 perform_work JOB 3 DONE",
-            "CONSUMER 2 save_result JOB 3",
-            "PRODUCER fetch_task @ 4 RETURN",
-            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=4))",
-            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=4).done of NoOpTask(id=4).",
-            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=4)",
-            ").run() - performing...",
-            "CONSUMER 0 perform_work JOB 4 START",
-            "CONSUMER 0 perform_work JOB 4 DONE",
-            "CONSUMER 0 save_result JOB 4",
-            "EXHAUSTED",
-        )
-
-
-def test_skip():
-    with TestProcess(sys.executable, "-mexampleworker", "3", "2", "100", "4") as target, dump_on_error(target.read):
-        wait_for_strings(
-            target.read,
-            5,
-            "INFO:worker:ExampleProducer() started.",
-            "INFO:worker:ExampleProducer().run() started.",
-            "PRODUCER fetch_task @ 1 RETURN",
-            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=1))",
-            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=1).done of NoOpTask(id=1).",
-            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=1)",
-            ").run() - performing...",
-            "CONSUMER 0 perform_work JOB 1 START",
-            "CONSUMER 0 perform_work JOB 1 DONE",
-            "CONSUMER 0 save_result JOB 1",
-            "PRODUCER fetch_task @ 2 SKIP",
-            "PRODUCER fetch_task @ 3 RETURN",
-            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=3))",
-            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=3).done of NoOpTask(id=3).",
-            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=3)",
-            ").run() - performing...",
-            "CONSUMER 1 perform_work JOB 3 START",
-            "CONSUMER 1 perform_work JOB 3 DONE",
-            "CONSUMER 1 save_result JOB 3",
-            "PRODUCER fetch_task @ 4 SKIP",
-            "PRODUCER fetch_task @ 5 EXHAUSTED",
-        )
-
-
-def test_sigterm():
-    with TestProcess(sys.executable, "-mexampleworker", "3", "0", "1000", "100") as target, dump_on_error(target.read):
-        wait_for_strings(
-            target.read,
-            15,
-            "PRODUCER fetch_task @ 1 RETURN",
-            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=1))",
-            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=1).done of NoOpTask(id=1).",
-            "working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=1)",
-            "- performing...",
-            "CONSUMER 0 perform_work JOB 1 START",
-        )
-        target.proc.send_signal(signal.SIGINT)
-        wait_for_strings(
-            target.read,
-            15,
-            "INFO:worker:ExampleEngine(4w/0q).stop(graceful=True)",
-            "CONSUMER 0 perform_work JOB 1 DONE",
-            "CONSUMER 0 save_result JOB 1",
-            "INFO:worker:ExampleEngine(4w/0q) requesting all workers to shutdown...",
-            "INFO:worker:ExampleEngine(4w/0q).push(None)",
-            "INFO:worker:ExampleEngine(4w/1q).push(None)",
-            "INFO:worker:ExampleEngine(4w/2q).push(None)",
-            "INFO:worker:ExampleEngine(0w/0q) shutdown complete.",
-            "DONE: {'ExampleEngine': {'JOBS': 4, 'QSIZE': 0}}",
-        )