csu 2.1.2__tar.gz → 2.1.4__tar.gz

{csu-2.1.2 → csu-2.1.4}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.1.2
+current_version = 2.1.4
 commit = True
 tag = True
 

{csu-2.1.2 → csu-2.1.4}/.cookiecutterrc

@@ -40,7 +40,7 @@ default_context:
     sphinx_theme: furo
     test_matrix_separate_coverage: 'no'
     tests_inside_package: 'yes'
-    version: 2.1.2
+    version: 2.1.4
     version_manager: bump2version
     website: https://blog.ionelmc.ro
     year_from: '2024'

{csu-2.1.2 → csu-2.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csu
-Version: 2.1.2
+Version: 2.1.4
 Summary: Clean Slate Utils - bunch of utility code, mostly Django/DRF specific.
 Author-email: Ionel Cristian Mărieș <contact@ionelmc.ro>
 License-Expression: BSD-2-Clause
@@ -77,9 +77,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.2.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.4.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.1.2...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.1.4...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

{csu-2.1.2 → csu-2.1.4}/README.rst

@@ -34,9 +34,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.2.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.4.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.1.2...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.1.4...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

{csu-2.1.2 → csu-2.1.4}/pyproject.toml

@@ -12,7 +12,7 @@ dynamic = [
     "readme",
 ]
 name = "csu"
-version = "2.1.2"
+version = "2.1.4"
 license = "BSD-2-Clause"
 license-files = ["LICENSE"]
 description = "Clean Slate Utils - bunch of utility code, mostly Django/DRF specific."

{csu-2.1.2 → csu-2.1.4}/src/csu/__init__.py

@@ -2,4 +2,4 @@
 a.k.a Clean Slate Utils
 """
 
-__version__ = "2.1.2"
+__version__ = "2.1.4"

{csu-2.1.2 → csu-2.1.4}/src/csu/worker/engine.py

@@ -293,19 +293,19 @@ class AbstractEngine(ABC):
 
     async def push(self, job: Job | None) -> None:
         logger.info("%s.push(%s)", self, job)
+        self.queue_awaited.clear()  # clear it here, it's the shortest path to "consumer wakes up and calls .set()"
         await self.queue.put(job)
 
     @asynccontextmanager
     async def consume(self) -> AsyncIterator[Job | None]:
-        try:
-            self.queue_awaited.set()
-            yield await self.queue.get()
-        finally:
-            self.queue.task_done()
+        self.queue_awaited.set()
+        yield await self.queue.get()
+        # if raised then there's no task to mark as done
+        self.queue.task_done()
 
     async def wait_worker_ready(self):
+        # event set after consuming, cleared right before there's a task
         await self.queue_awaited.wait()
-        self.queue_awaited.clear()
 
     def add_worker(self, worker: AbstractWorker):
         assert self.queue is None, f"{self} is already started"

{csu-2.1.2 → csu-2.1.4}/src/csu.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csu
-Version: 2.1.2
+Version: 2.1.4
 Summary: Clean Slate Utils - bunch of utility code, mostly Django/DRF specific.
 Author-email: Ionel Cristian Mărieș <contact@ionelmc.ro>
 License-Expression: BSD-2-Clause
@@ -77,9 +77,9 @@ Overview
 .. |supported-implementations| image:: https://img.shields.io/pypi/implementation/csu.svg
     :alt: Supported implementations
     :target: https://pypi.org/project/csu
-.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.2.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-csu/v2.1.4.svg
     :alt: Commits since latest release
-    :target: https://github.com/ionelmc/python-csu/compare/v2.1.2...main
+    :target: https://github.com/ionelmc/python-csu/compare/v2.1.4...main
 .. |scrutinizer| image:: https://img.shields.io/scrutinizer/quality/g/ionelmc/python-csu/main.svg
     :alt: Scrutinizer Status
     :target: https://scrutinizer-ci.com/g/ionelmc/python-csu/

{csu-2.1.2 → csu-2.1.4}/src/csu.egg-info/SOURCES.txt

@@ -83,6 +83,7 @@ src/csu/worker/models.py
 src/csu/worker/registry.py
 tests/conftest.py
 tests/exampleworker.py
+tests/test_exampleworker.py
 tests/test_models.py
 tests/test_service.py
 tests/test_service_auth.py

csu-2.1.4/tests/exampleworker.py

@@ -0,0 +1,126 @@
+import asyncio
+import logging
+import sys
+from dataclasses import asdict
+from dataclasses import dataclass
+from enum import StrEnum
+from logging import getLogger
+
+from csu.worker.engine import AbstractConsumer
+from csu.worker.engine import AbstractEngine
+from csu.worker.engine import AbstractProducer
+from csu.worker.engine import NoPrefetchingProducerMixin
+from csu.worker.engine import SleepingCooldown
+from csu.worker.engine import SleepingCooldownMixin
+from csu.worker.engine import TaskProtocol
+from csu.worker.job import Job
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class NoOpTask:
+    """
+    Simple task model that doesn't store anything.
+    """
+
+    id: int
+
+    @property
+    def job_kwargs(self):
+        return asdict(self)
+
+    async def done_watchdog(self, job: Job):
+        """
+        Nothing done as there's no storage for this job.
+        """
+
+
+@dataclass
+class ExampleJob[RequestStruct](Job):
+    id: int
+
+
+class ExampleResultType(StrEnum):
+    PENDING = "PENDING"
+    SUCCESS = "SUCCESS"
+    ERROR = "ERROR"
+    ABSENT = "ABSENT"
+
+
+if sys.argv[1] == "prefetch":
+
+    class NoPrefetchingProducerMixin:
+        pass
+
+
+class ExampleProducer(
+    NoPrefetchingProducerMixin,
+    SleepingCooldownMixin,
+    AbstractProducer,
+):
+    engine: ExampleEngine
+    job_class = ExampleJob
+    counter = 0
+    cooldown_min_seconds = cooldown_max_seconds = 1
+    cooldown_on = SleepingCooldown.IDLE
+
+    async def fetch_task(self) -> TaskProtocol | None:
+        self.counter += 1
+        if self.counter > self.engine.max_jobs:
+            print(f"PRODUCER fetch_task @ {self.counter} EXHAUSTED")
+            return
+        if not self.engine.producer_skip or self.counter % self.engine.producer_skip:
+            print(f"PRODUCER fetch_task @ {self.counter} RETURN")
+            return NoOpTask(self.counter)
+        else:
+            print(f"PRODUCER fetch_task @ {self.counter} SKIP")
+
+
+class ExampleConsumer(AbstractConsumer):
+    result_type_enum = ExampleResultType
+    engine: ExampleEngine
+
+    def __init__(self, engine: ExampleEngine, name):
+        super().__init__(engine)
+        self.name = name
+
+    async def perform_work(self, job: ExampleJob):
+        print(f"CONSUMER {self.name} perform_work JOB {job.id} START")
+        try:
+            await asyncio.sleep(self.engine.consumer_sleep)
+            return {"foo": "bar"}
+        finally:
+            print(f"CONSUMER {self.name} perform_work JOB {job.id} DONE")
+
+    async def save_result(self, job: ExampleJob, /, **kwargs):
+        print(f"CONSUMER {self.name} save_result JOB {job.id}")
+
+
+class ExampleEngine(AbstractEngine):
+    def __init__(self, consumers, producer_skip, consumer_sleep, max_jobs):
+        super().__init__()
+        self.consumers = int(consumers)
+        self.producer_skip = int(producer_skip)
+        self.consumer_sleep = int(consumer_sleep) / 1000
+        self.max_jobs = int(max_jobs)
+        print(self.__dict__)
+
+    async def start(self):
+        for i in range(self.consumers):
+            self.add_worker(ExampleConsumer(engine=self, name=i))
+
+        self.add_worker(ExampleProducer(engine=self))
+
+        await super().start()
+
+        while self.is_work_pending():
+            await asyncio.sleep(1)
+            print(self.inspect())
+        print(f"DONE: {self.inspect()}")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG)
+    engine = ExampleEngine(*sys.argv[2:])
+    asyncio.run(engine.start())

csu-2.1.4/tests/test_exampleworker.py

@@ -0,0 +1,87 @@
+import sys
+
+from process_tests import TestProcess
+from process_tests import dump_on_error
+from process_tests import wait_for_strings
+
+
+def test_noprefetch_noskip():
+    with TestProcess(sys.executable, "-mexampleworker", "nope", "3", "0", "1000", "4") as target, dump_on_error(target.read):
+        wait_for_strings(
+            target.read,
+            5,
+            "PRODUCER fetch_task @ 1 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=1))",
+            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=1).done of NoOpTask(id=1).",
+            "working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=1)",
+            "- performing...",
+            "CONSUMER 0 perform_work JOB 1 START",
+            "CONSUMER 0 perform_work JOB 1 DONE",
+            "CONSUMER 0 save_result JOB 1",
+            "PRODUCER fetch_task @ 2 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=2))",
+            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=2).done of NoOpTask(id=2).",
+            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=2)",
+            ").run() - performing...",
+            "CONSUMER 1 perform_work JOB 2 START",
+            "CONSUMER 1 perform_work JOB 2 DONE",
+            "CONSUMER 1 save_result JOB 2",
+            "PRODUCER fetch_task @ 3 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=3))",
+            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=3).done of NoOpTask(id=3).",
+            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=3)",
+            ").run() - performing...",
+            "CONSUMER 2 perform_work JOB 3 START",
+            "CONSUMER 2 perform_work JOB 3 DONE",
+            "CONSUMER 2 save_result JOB 3",
+            "PRODUCER fetch_task @ 4 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=4))",
+            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=4).done of NoOpTask(id=4).",
+            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=4)",
+            ").run() - performing...",
+            "CONSUMER 0 perform_work JOB 4 START",
+            "CONSUMER 0 perform_work JOB 4 DONE",
+            "CONSUMER 0 save_result JOB 4",
+            "EXHAUSTED",
+        )
+
+
+def test_prefetch_skip():
+    with TestProcess(sys.executable, "-mexampleworker", "nope", "3", "0", "100", "4") as target, dump_on_error(target.read):
+        wait_for_strings(
+            target.read,
+            5,
+            "PRODUCER fetch_task @ 1 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=1))",
+            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=1).done of NoOpTask(id=1).",
+            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=1)",
+            ").run() - performing...",
+            "CONSUMER 0 perform_work JOB 1 START",
+            "CONSUMER 0 perform_work JOB 1 DONE",
+            "CONSUMER 0 save_result JOB 1",
+            "PRODUCER fetch_task @ 2 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=2))",
+            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=2).done of NoOpTask(id=2).",
+            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=2)",
+            ").run() - performing...",
+            "CONSUMER 1 perform_work JOB 2 START",
+            "CONSUMER 1 perform_work JOB 2 DONE",
+            "CONSUMER 1 save_result JOB 2",
+            "PRODUCER fetch_task @ 3 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=3))",
+            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=3).done of NoOpTask(id=3).",
+            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=3)",
+            ").run() - performing...",
+            "CONSUMER 2 perform_work JOB 3 START",
+            "CONSUMER 2 perform_work JOB 3 DONE",
+            "CONSUMER 2 save_result JOB 3",
+            "PRODUCER fetch_task @ 4 RETURN",
+            "INFO:worker:ExampleEngine(4w/0q).push(ExampleJob(done=<Future pending>, started=<Future pending>, id=4))",
+            "INFO:worker:ExampleProducer().run() awaiting ExampleJob(done=<Future pending>, started=<Future pending>, id=4).done of NoOpTask(id=4).",
+            ").run() working on: ExampleJob(done=<Future pending>, started=<Future finished result=True>, id=4)",
+            ").run() - performing...",
+            "CONSUMER 0 perform_work JOB 4 START",
+            "CONSUMER 0 perform_work JOB 4 DONE",
+            "CONSUMER 0 save_result JOB 4",
+            "PRODUCER fetch_task @ 5 EXHAUSTED",
+        )

{csu-2.1.2 → csu-2.1.4}/tests/test_worker.py

@@ -42,7 +42,6 @@ class Consumer(AbstractConsumer):
 class Engine(AbstractEngine):
     async def start(self):
         worker = Consumer(engine=self)
-        worker.start()
         self.add_worker(worker)
         await super().start()
         logging.info("engine started")

{csu-2.1.2 → csu-2.1.4}/tox.ini

@@ -35,6 +35,7 @@ deps =
     pytest-django
     pytest-httpbin
     pytest-recording
+    process-tests
     inline-snapshot
     time-machine
     hunter

csu-2.1.2/tests/exampleworker.py

@@ -1,104 +0,0 @@
-import asyncio
-import logging
-from dataclasses import asdict
-from dataclasses import dataclass
-from enum import StrEnum
-from logging import getLogger
-
-from csu.worker.engine import AbstractConsumer
-from csu.worker.engine import AbstractEngine
-from csu.worker.engine import AbstractProducer
-from csu.worker.engine import TaskProtocol
-from csu.worker.job import Job
-
-logger = getLogger(__name__)
-
-
-@dataclass
-class NoOpTask:
-    """
-    Simple task model that doesn't store anything.
-    """
-
-    id: int
-
-    @property
-    def job_kwargs(self):
-        return asdict(self)
-
-    async def done_watchdog(self, job: Job):
-        """
-        Nothing done as there's no storage for this job.
-        """
-
-
-@dataclass
-class ExampleJob[RequestStruct](Job):
-    id: int
-
-
-class ExampleResultType(StrEnum):
-    PENDING = "PENDING"
-    SUCCESS = "SUCCESS"
-    ERROR = "ERROR"
-    ABSENT = "ABSENT"
-
-
-class ExampleProducer(
-    # NoPrefetchingProducerMixin,
-    AbstractProducer,
-):
-    engine: ExampleEngine
-    job_class = ExampleJob
-    counter = 0
-
-    async def fetch_task(self) -> TaskProtocol | None:
-        self.counter += 1
-        logger.info("%s.fetch_task() -> %s", self, self.counter)
-        return NoOpTask(self.counter)
-
-
-class ExampleConsumer(AbstractConsumer):
-    result_type_enum = ExampleResultType
-    engine: ExampleEngine
-
-    async def perform_work(self, job: ExampleJob):
-        logger.info("%s.perform_work(%s)", self, job)
-        await asyncio.sleep(1)
-        return {"foo": "bar"}
-
-    async def save_result(self, job: ExampleJob, /, **kwargs):
-        logger.info("%s.save_result(%s)", self, job)
-
-
-WORKERS = 5
-
-
-class ExampleEngine(AbstractEngine):
-    queue_maxsize = WORKERS
-
-    async def start(self):
-        logger.info("Starting workers...")
-
-        for _ in range(WORKERS):
-            consumer = ExampleConsumer(engine=self)
-            consumer.start()
-            self.add_worker(consumer)
-
-        producer = ExampleProducer(engine=self)
-        producer.start()
-        self.add_worker(producer)
-
-        await super().start()
-
-        while self.is_work_pending():
-            await asyncio.sleep(1)
-            logger.info("%s", self.inspect())
-        logger.info("DONE: %s", self.inspect())
-
-
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.DEBUG)
-    engine = ExampleEngine()
-    print(engine)
-    asyncio.run(engine.start())