csu 2.0.8__tar.gz → 2.1.0__tar.gz

{csu-2.0.8 → csu-2.1.0}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.0.8
+current_version = 2.1.0
 commit = True
 tag = True
 

{csu-2.0.8 → csu-2.1.0}/.cookiecutterrc

@@ -40,7 +40,7 @@ default_context:
     sphinx_theme: furo
     test_matrix_separate_coverage: 'no'
     tests_inside_package: 'yes'
-    version: 2.0.8
+    version: 2.1.0
     version_manager: bump2version
     website: https://blog.ionelmc.ro
     year_from: '2024'

{csu-2.0.8 → csu-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csu
-Version: 2.0.8
+Version: 2.1.0
 Summary: Clean Slate Utils - bunch of utility code, mostly Django/DRF specific.
 Author-email: Ionel Cristian Mărieș <contact@ionelmc.ro>
 License-Expression: BSD-2-Clause
@@ -77,9 +77,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.0.8.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.0.8...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.1.0...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

{csu-2.0.8 → csu-2.1.0}/README.rst

@@ -34,9 +34,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.0.8.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.0.8...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.1.0...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

{csu-2.0.8 → csu-2.1.0}/pyproject.toml

@@ -12,7 +12,7 @@ dynamic = [
     "readme",
 ]
 name = "csu"
-version = "2.0.8"
+version = "2.1.0"
 license = "BSD-2-Clause"
 license-files = ["LICENSE"]
 description = "Clean Slate Utils - bunch of utility code, mostly Django/DRF specific."

{csu-2.0.8 → csu-2.1.0}/src/csu/__init__.py

@@ -2,4 +2,4 @@
 a.k.a Clean Slate Utils
 """
 
-__version__ = "2.0.8"
+__version__ = "2.1.0"

{csu-2.0.8 → csu-2.1.0}/src/csu/worker/engine.py

@@ -4,11 +4,12 @@ import traceback
 from abc import ABC
 from abc import abstractmethod
 from asyncio import CancelledError
+from asyncio import Event
 from asyncio import Future
 from asyncio import Queue
-from asyncio import Semaphore
 from collections import defaultdict
-from dataclasses import dataclass
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
 from datetime import datetime
 from datetime import time
 from datetime import timedelta
@@ -121,7 +122,7 @@ class NoPrefetchingProducerMixin:
         # noinspection PyUnresolvedReferences
         await super().before()
         self.state = WorkerState.WAITING
-        await self.engine.wait_pending()
+        await self.engine.wait_worker_ready()
 
 
 class SleepingCooldown(Flag):
@@ -153,7 +154,6 @@ class SleepingCooldownMixin:
             await asyncio.sleep(sleep_seconds)
 
 
-@dataclass(repr=False)
 class AbstractProducer(AbstractWorker):
     state = WorkerState.UNKNOWN
     engine: AbstractEngine
@@ -169,6 +169,9 @@ class AbstractProducer(AbstractWorker):
         Should raise error if there would be a spin-loop condition.
         """
 
+    def __init__(self, engine: AbstractEngine):
+        self.engine = engine
+
     def __str__(self):
         return f"{type(self).__name__}()"
 
@@ -198,7 +201,6 @@ class AbstractProducer(AbstractWorker):
             await self.after(idle=not task)
 
 
-@dataclass(repr=False)
 class AbstractConsumer(AbstractWorker):
     engine: AbstractEngine
 
@@ -211,6 +213,9 @@ class AbstractConsumer(AbstractWorker):
     async def save_result(self, job: Job, /, **kwargs):
         pass
 
+    def __init__(self, engine: AbstractEngine):
+        self.engine = engine
+
     def __str__(self):
         return f"{type(self).__name__}({id(self)})"
 
@@ -220,38 +225,39 @@ class AbstractConsumer(AbstractWorker):
                 self.state = WorkerState.BEFORE
                 await self.before()
             self.state = WorkerState.READY
-            job: Job = await self.engine.pull()
-            if job is None:
-                break
-            job.started.set_result(True)
-            logger.info("%s.run() working on: %s", self, job)
-            self.state = WorkerState.WORKING
-            logger.info("%s.run() - performing...", self)
-            try:
-                fields = await self.perform_work(job)
-            except Exception as exc:
-                self.state = WorkerState.SAVING
-                logger.error("%s.perform_work(%s) failed: %r", self, job, exc)
-                result = await self.save_result(
-                    job,
-                    result_type=self.result_type_enum.ERROR,
-                    result_details=f"{exc!r}\n{traceback.format_exc()}",
-                )
-                idle = True
-            else:
-                self.state = WorkerState.SAVING
-                result = await self.save_result(
-                    job,
-                    result_type=self.result_type_enum.SUCCESS if fields else self.result_type_enum.ABSENT,
-                    result_details=None if fields else repr(fields),
-                    **fields or {},
-                )
-                idle = False
-            if job.done.cancelled():
-                logger.warning("%s.run() - %s is cancelled.", self, job)
-            else:
-                job.done.set_result(result)
-            self.engine.mark_done()
+            job: Job | None
+            async with self.engine.consume() as job:
+                if job is None:
+                    break
+                job.started.set_result(True)
+                logger.info("%s.run() working on: %s", self, job)
+                self.state = WorkerState.WORKING
+                logger.info("%s.run() - performing...", self)
+                try:
+                    fields = await self.perform_work(job)
+                except Exception as exc:
+                    self.state = WorkerState.SAVING
+                    logger.error("%s.perform_work(%s) failed: %r", self, job, exc)
+                    result = await self.save_result(
+                        job,
+                        result_type=self.result_type_enum.ERROR,
+                        result_details=f"{exc!r}\n{traceback.format_exc()}",
+                    )
+                    idle = True
+                else:
+                    self.state = WorkerState.SAVING
+                    result = await self.save_result(
+                        job,
+                        result_type=self.result_type_enum.SUCCESS if fields else self.result_type_enum.ABSENT,
+                        result_details=None if fields else repr(fields),
+                        **fields or {},
+                    )
+                    idle = False
+                if job.done.cancelled():
+                    logger.warning("%s.run() - %s is cancelled.", self, job)
+                else:
+                    job.done.set_result(result)
+
             if self.shutdown_requested:
                 continue
             self.state = WorkerState.AFTER
@@ -262,21 +268,21 @@ class AbstractConsumer(AbstractWorker):
         pass
 
 
-@dataclass
 class AbstractEngine(ABC):
     workers: list[AbstractWorker]
-    queue: Queue
+    queue: Queue = None
+    queue_maxsize: int
     shutdown_signal = signal.SIGINT
     shutdown_requested = False
     shutdown_task = None
 
     def __init__(self):
         self.workers = []
-        self.queue = Queue()
-        self.semaphore = Semaphore(value=0)
+        self.queue_awaited = Event()
+        self.queue_maxsize = 0
 
     def __str__(self):
-        return f"{type(self).__name__}({len(self.workers)}w/{self.queue.qsize()}q)"
+        return f"{type(self).__name__}({len(self.workers)}w/{self.queue.qsize() if self.queue else '-'}q)"
 
     def is_work_pending(self):
         if any(worker for worker in self.workers if worker.state == WorkerState.WORKING):
@@ -285,29 +291,31 @@ class AbstractEngine(ABC):
             return True
         return not self.shutdown_requested
 
-    async def wait_pending(self):
-        await self.queue.join()
-
-    async def push(self, job: Job) -> None:
+    async def push(self, job: Job | None) -> None:
         logger.info("%s.push(%s)", self, job)
-        await self.semaphore.acquire()
         await self.queue.put(job)
 
-    def push_nowait(self, job: Job | None) -> None:
-        """
-        None means a shutdown request.
-        """
-        logger.info("%s.push_nowait(%s)", self, job)
-        self.queue.put_nowait(job)
-
-    async def pull(self) -> Job | None:
-        self.semaphore.release()
-        return await self.queue.get()
-
-    def mark_done(self):
-        self.queue.task_done()
+    @asynccontextmanager
+    async def consume(self) -> AsyncIterator[Job | None]:
+        try:
+            self.queue_awaited.set()
+            yield await self.queue.get()
+        finally:
+            self.queue.task_done()
+
+    async def wait_worker_ready(self):
+        await self.queue_awaited.wait()
+        self.queue_awaited.clear()
+
+    def add_worker(self, worker: AbstractWorker):
+        if self.queue is not None:
+            raise RuntimeError(f"Engine {self} is already started!")
+        if isinstance(worker, AbstractConsumer):
+            self.queue_maxsize += 1
+        self.workers.append(worker)
 
     async def start(self):
+        self.queue = Queue(maxsize=self.queue_maxsize)
         signal.signal(self.shutdown_signal, self.shutdown_handler)
 
     def shutdown_handler(self, sig, frame):
@@ -329,20 +337,20 @@ class AbstractEngine(ABC):
             for worker in self.workers:
                 worker.shutdown_requested = True
                 if isinstance(worker, AbstractConsumer):
-                    self.push_nowait(None)
+                    await self.push(None)
         else:
             while self.queue.qsize() > 0:
                 item = self.queue.get_nowait()
                 if item is not None:
-                    logger.critical("%s unprocessed queue item: %r", item)
+                    logger.critical("%s unprocessed queue item: %r", self, item)
             for worker in self.workers:
                 worker.runner.cancel()
         await asyncio.gather(*[worker.runner for worker in self.workers], return_exceptions=True)
         self.workers.clear()
         logger.info("%s shutdown complete.", self)
 
-    def report(self, inspect: defaultdict[str, int]):
-        inspect["QSIZE"] = self.queue.qsize()
+    def report(self, inspect: defaultdict[str, object]):
+        inspect["QSIZE"] = "UNSTARTED" if self.queue is None else self.queue.qsize()
 
     def inspect(self) -> dict[str, dict[str, int]]:
         result = defaultdict(partial(defaultdict, int))

{csu-2.0.8 → csu-2.1.0}/src/csu.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csu
-Version: 2.0.8
+Version: 2.1.0
 Summary: Clean Slate Utils - bunch of utility code, mostly Django/DRF specific.
 Author-email: Ionel Cristian Mărieș <contact@ionelmc.ro>
 License-Expression: BSD-2-Clause
@@ -77,9 +77,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.0.8.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.0.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.0.8...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.1.0...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

{csu-2.0.8 → csu-2.1.0}/tests/exampleworker.py

@@ -8,7 +8,6 @@ from logging import getLogger
 from csu.worker.engine import AbstractConsumer
 from csu.worker.engine import AbstractEngine
 from csu.worker.engine import AbstractProducer
-from csu.worker.engine import NoPrefetchingProducerMixin
 from csu.worker.engine import TaskProtocol
 from csu.worker.job import Job
 
@@ -45,7 +44,10 @@ class ExampleResultType(StrEnum):
     ABSENT = "ABSENT"
 
 
-class ExampleProducer(NoPrefetchingProducerMixin, AbstractProducer):
+class ExampleProducer(
+    # NoPrefetchingProducerMixin,
+    AbstractProducer,
+):
     engine: ExampleEngine
     job_class = ExampleJob
     counter = 0
@@ -69,24 +71,25 @@ class ExampleConsumer(AbstractConsumer):
         logger.info("%s.save_result(%s)", self, job)
 
 
-WORKERS = 1
+WORKERS = 5
 
 
 class ExampleEngine(AbstractEngine):
     queue_maxsize = WORKERS
 
     async def start(self):
-        await super().start()
         logger.info("Starting workers...")
 
         for _ in range(WORKERS):
             consumer = ExampleConsumer(engine=self)
             consumer.start()
-            self.workers.append(consumer)
+            self.add_worker(consumer)
 
         producer = ExampleProducer(engine=self)
         producer.start()
-        self.workers.append(producer)
+        self.add_worker(producer)
+
+        await super().start()
 
         while self.is_work_pending():
             await asyncio.sleep(1)

{csu-2.0.8 → csu-2.1.0}/tests/test_worker.py

@@ -43,7 +43,8 @@ class Engine(AbstractEngine):
     async def start(self):
         worker = Consumer(engine=self)
         worker.start()
-        self.workers.append(worker)
+        self.add_worker(worker)
+        await super().start()
         logging.info("engine started")
 
 
@@ -52,9 +53,9 @@ async def test_cancel():
     engine = Engine()
     await engine.start()
     job1 = SleepyJob(0.2)
-    engine.push_nowait(job1)
+    await engine.push(job1)
     job2 = SleepyJob(0)
-    engine.push_nowait(job2)
+    await engine.push(job2)
     await asyncio.sleep(0.1)
     with pytest.raises(TimeoutError):
         await asyncio.wait_for(job1.done, 0.1)