ez-a-sync 0.27.5__tar.gz → 0.27.6__tar.gz

{ez_a_sync-0.27.5 → ez_a_sync-0.27.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ez-a-sync
-Version: 0.27.5
+Version: 0.27.6
 Summary: A library that makes it easy to define objects that can be used for both sync and async use cases.
 Home-page: https://github.com/BobTheBuidler/a-sync
 Author: BobTheBuidler

{ez_a_sync-0.27.5 → ez_a_sync-0.27.6}/a_sync/primitives/queue.py

@@ -15,7 +15,7 @@ See Also:
 
 import asyncio
 import sys
-from asyncio import InvalidStateError, QueueEmpty
+from asyncio import AbstractEventLoop, Future, InvalidStateError, QueueEmpty, Task, get_event_loop
 from asyncio.events import _get_running_loop
 from functools import wraps
 from heapq import heappop, heappush, heappushpop
@@ -271,7 +271,7 @@ _put_nowait = asyncio.Queue.put_nowait
 _loop_kwarg_deprecated = sys.version_info >= (3, 10)
 
 
-class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
+class ProcessingQueue(_Queue[Tuple[P, "Future[V]"]], Generic[P, V]):
     """
     A queue designed for processing tasks asynchronously with multiple workers.
 
@@ -299,7 +299,7 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
         *,
         return_data: bool = True,
         name: str = "",
-        loop: Optional[asyncio.AbstractEventLoop] = None,
+        loop: Optional[AbstractEventLoop] = None,
     ) -> None:
         """
         Initializes a processing queue with the given worker function and worker count.
@@ -367,7 +367,7 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
             repr_string += f" pending={self._unfinished_tasks}"
         return f"{repr_string}>"
 
-    def __call__(self, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
+    def __call__(self, *args: P.args, **kwargs: P.kwargs) -> "Future[V]":
         """
         Submits a task to the queue.
 
@@ -409,7 +409,7 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
         """
         self._closed = True
 
-    async def put(self, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
+    async def put(self, *args: P.args, **kwargs: P.kwargs) -> "Future[V]":
         # sourcery skip: use-contextlib-suppress
         """
         Asynchronously submits a task to the queue.
@@ -446,7 +446,7 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
                 raise
         return self.put_nowait(*args, **kwargs)
 
-    def put_nowait(self, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
+    def put_nowait(self, *args: P.args, **kwargs: P.kwargs) -> "Future[V]":
         """
         Immediately submits a task to the queue without waiting.
 
@@ -468,7 +468,7 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
         _put_nowait(self, (args, kwargs, proxy(fut)))
         return fut
 
-    def _create_future(self) -> "asyncio.Future[V]":
+    def _create_future(self) -> "Future[V]":
         """Creates a future for the task."""
         return _get_running_loop().create_future()
 
@@ -477,7 +477,7 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
         if self._closed:
             raise RuntimeError(f"{type(self).__name__} is closed: ", self) from None
         if self._workers.done():
-            worker_subtasks: List["asyncio.Task[NoReturn]"] = self._workers._workers
+            worker_subtasks: List["Task[NoReturn]"] = self._workers._workers
             for worker in worker_subtasks:
                 if worker.done():  # its only done if its broken
                     exc = worker.exception()
@@ -489,10 +489,10 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
             raise exc.with_traceback(exc.__traceback__) from exc.__cause__
 
     @cached_property_unsafe
-    def _workers(self) -> "asyncio.Task[NoReturn]":
+    def _workers(self) -> "Task[NoReturn]":
         """Creates and manages the worker tasks for the queue."""
         log_debug("starting worker task for %s", self)
-        name = self.name
+        name = str(self.name)
         workers = tuple(
             create_task(
                 coro=self._worker_coro(),
@@ -530,33 +530,32 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
                     logger.exception(e)
                 task_done()
         else:
-            fut: asyncio.Future[V]
+            fut: Future[V]
             while True:
-                args, kwargs, fut = await get_next_job()
                 try:
-                    if fut is None:
-                        # the weakref was already cleaned up, we don't need to process this item
-                        task_done()
-                        continue
+                    args, kwargs, fut = await get_next_job()
+                except RuntimeError as e:
+                    if _check_loop_is_closed(self, e):
+                        return
+                    raise
+
+                if fut is None:
+                    # the weakref was already cleaned up, we don't need to process this item
+                    task_done()
+                    continue
+
+                try:
                     result = await func(*args, **kwargs)
-                    fut.set_result(result)
-                except InvalidStateError:
-                    logger.error(
-                        "cannot set result for %s %s: %s",
-                        func.__name__,
-                        fut,
-                        result,
-                    )
                 except Exception as e:
                     try:
                         fut.set_exception(e)
                     except InvalidStateError:
-                        logger.error(
-                            "cannot set exception for %s %s: %s",
-                            func.__name__,
-                            fut,
-                            e,
-                        )
+                        _log_invalid_state_err("exception", func, fut, e)
+                else:
+                    try:
+                        fut.set_result(result)
+                    except InvalidStateError:
+                        _log_invalid_state_err("result", func, fut, result)
                 task_done()
 
 
@@ -674,7 +673,8 @@ class PriorityProcessingQueue(_PriorityQueueMixin[T], ProcessingQueue[T, V]):
         :class:`~ProcessingQueue`
     """
 
-    async def put(self, priority: Any, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
+    async def put(self, priority: Any, *args: P.args, **kwargs: P.kwargs) -> "Future[V]":
+        # sourcery skip: use-contextlib-suppress
         """
         Asynchronously adds a task with priority to the queue.
 
@@ -690,12 +690,29 @@ class PriorityProcessingQueue(_PriorityQueueMixin[T], ProcessingQueue[T, V]):
             >>> fut = await queue.put(priority=1, item='task')
             >>> print(await fut)
         """
-        self._ensure_workers()
-        fut = asyncio.get_event_loop().create_future()
-        await ProcessingQueue.put(self, (priority, args, kwargs, fut))
-        return fut
+        while self.full():
+            putter = self._get_loop().create_future()
+            self._putters.append(putter)
+            try:
+                await putter
+            except:
+                putter.cancel()  # Just in case putter is not done yet.
+                try:
+                    # Clean self._putters from canceled putters.
+                    self._putters.remove(putter)
+                except ValueError:
+                    # The putter could be removed from self._putters by a
+                    # previous get_nowait call.
+                    pass
+                if not self.full() and not putter.cancelled():
+                    # We were woken up by get_nowait(), but can't take
+                    # the call.  Wake up the next in line.
+                    self._wakeup_next(self._putters)
+                raise
+
+        return self.put_nowait(priority, *args, **kwargs)
 
-    def put_nowait(self, priority: Any, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
+    def put_nowait(self, priority: Any, *args: P.args, **kwargs: P.kwargs) -> "Future[V]":
         """
         Immediately adds a task with priority to the queue without waiting.
 
@@ -713,7 +730,7 @@ class PriorityProcessingQueue(_PriorityQueueMixin[T], ProcessingQueue[T, V]):
         """
         self._ensure_workers()
         fut = self._create_future()
-        ProcessingQueue.put_nowait(self, (priority, args, kwargs, fut))
+        _Queue.put_nowait(self, (priority, args, kwargs, fut))
         return fut
 
     def _get(self, heappop=heappop):
@@ -727,8 +744,10 @@ class PriorityProcessingQueue(_PriorityQueueMixin[T], ProcessingQueue[T, V]):
             >>> task = queue._get()
             >>> print(task)
         """
-        priority, args, kwargs, fut = heappop(self._queue)
-        return args, kwargs, fut
+        # For readability, what we're really doing is this:
+        # priority, args, kwargs, fut = heappop(self._queue)
+        # return args, kwargs, fut
+        return heappop(self._queue)[1:]
 
 
 class _VariablePriorityQueueMixin(_PriorityQueueMixin[T]):
@@ -756,8 +775,7 @@ class _VariablePriorityQueueMixin(_PriorityQueueMixin[T]):
         # NOTE: Since waiter priorities can change, heappop might not return the job with the
         #       most waiters if `self._queue` is not currently in order, but we can use `heappushpop`,
         #       to ensure we get the job with the most waiters.
-        queue = self._queue
-        return heappushpop(queue, heappop(queue))
+        return heappushpop(queue := self._queue, heappop(queue))
 
     def _get_key(self, *args, **kwargs) -> _SmartKey:
         """
@@ -774,7 +792,7 @@ class _VariablePriorityQueueMixin(_PriorityQueueMixin[T]):
             >>> key = queue._get_key(*args, **kwargs)
             >>> print(key)
         """
-        return (args, tuple((kwarg, kwargs[kwarg]) for kwarg in sorted(kwargs)))
+        return args, tuple(sorted(kwargs.items()))
 
 
 class VariablePriorityQueue(_VariablePriorityQueueMixin[T], asyncio.PriorityQueue):
@@ -830,7 +848,7 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
         num_workers: int,
         *,
         name: str = "",
-        loop: Optional[asyncio.AbstractEventLoop] = None,
+        loop: Optional[AbstractEventLoop] = None,
     ) -> None:
         """
         Initializes a smart processing queue with the given worker function.
@@ -849,6 +867,7 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
         self._futs = WeakValueDictionary()
 
     async def put(self, *args: P.args, **kwargs: P.kwargs) -> SmartFuture[V]:
+        # sourcery skip: use-contextlib-suppress
         """
         Asynchronously adds a task with smart future handling to the queue.
 
@@ -863,14 +882,26 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
             >>> fut = await queue.put(item='task')
             >>> print(await fut)
         """
-        self._ensure_workers()
-        key = self._get_key(*args, **kwargs)
-        if fut := self._futs.get(key, None):
-            return fut
-        fut = self._create_future(key)
-        self._futs[key] = fut
-        await Queue.put(self, (_SmartFutureRef(fut), args, kwargs))
-        return fut
+        while self.full():
+            putter = self._loop.create_future()
+            self._putters.append(putter)
+            try:
+                await putter
+            except:
+                putter.cancel()  # Just in case putter is not done yet.
+                try:
+                    # Clean self._putters from canceled putters.
+                    self._putters.remove(putter)
+                except ValueError:
+                    # The putter could be removed from self._putters by a
+                    # previous get_nowait call.
+                    pass
+                if not self.full() and not putter.cancelled():
+                    # We were woken up by get_nowait(), but can't take
+                    # the call.  Wake up the next in line.
+                    self._wakeup_next(self._putters)
+                raise
+        return self.put_nowait(*args, **kwargs)
 
     def put_nowait(self, *args: P.args, **kwargs: P.kwargs) -> SmartFuture[V]:
         """
@@ -896,7 +927,7 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
         Queue.put_nowait(self, (_SmartFutureRef(fut), args, kwargs))
         return fut
 
-    def _create_future(self, key: _SmartKey) -> "asyncio.Future[V]":
+    def _create_future(self, key: _SmartKey) -> "Future[V]":
         """Creates a smart future for the task."""
         return create_future(queue=self, key=key, loop=self._loop)
 
@@ -929,7 +960,7 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
         get_next_job = self.get
         func = self.func
         task_done = self.task_done
-        log_debug = log_debug
+        log = log_debug
 
         args: P.args
         kwargs: P.kwargs
@@ -938,34 +969,58 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
             while True:
                 try:
                     args, kwargs, fut = await get_next_job()
-                    if fut is None:
-                        # the weakref was already cleaned up, we don't need to process this item
-                        task_done()
-                        continue
-                    log_debug("processing %s", fut)
+                except RuntimeError as e:
+                    if _check_loop_is_closed(self, e):
+                        return
+                    raise
+
+                if fut is None:
+                    # the weakref was already cleaned up, we don't need to process this item
+                    task_done()
+                    continue
+
+                log("processing %s", fut)
+
+                try:
                     result = await func(*args, **kwargs)
-                    fut.set_result(result)
-                except InvalidStateError:
-                    logger.error(
-                        "cannot set result for %s %s: %s",
-                        func.__name__,
-                        fut,
-                        result,
-                    )
                 except Exception as e:
-                    log_debug("%s: %s", type(e).__name__, e)
+                    log("%s: %s", type(e).__name__, e)
                     try:
                         fut.set_exception(e)
                     except InvalidStateError:
-                        logger.error(
-                            "cannot set exception for %s %s: %s",
-                            func.__name__,
-                            fut,
-                            e,
-                        )
+                        _log_invalid_state_err("exception", func, fut, e)
+                else:
+                    try:
+                        fut.set_result(result)
+                    except InvalidStateError:
+                        _log_invalid_state_err("result", func, fut, result)
                 task_done()
 
         except Exception as e:
             logger.error("%s is broken!!!", self)
             logger.exception(e)
             raise
+
+
+def _log_invalid_state_err(
+    typ: Literal["result", "exception"], func: Callable, fut: Future, value: Any
+) -> None:
+    logger.error(
+        "cannot set %s for %s %s: %s",
+        typ,
+        func.__name__,
+        fut,
+        value,
+    )
+
+
+def _check_loop_is_closed(queue: Queue, e: Exception) -> bool:
+    if "Event loop is closed" not in str(e):
+        return False
+    if queue._unfinished_tasks:
+        logger.error(
+            "Event loop is closed. Closing %s with %s unfinished tasks",
+            queue,
+            queue._unfinished_tasks,
+        )
+    return True

{ez_a_sync-0.27.5 → ez_a_sync-0.27.6}/ez_a_sync.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ez-a-sync
-Version: 0.27.5
+Version: 0.27.6
 Summary: A library that makes it easy to define objects that can be used for both sync and async use cases.
 Home-page: https://github.com/BobTheBuidler/a-sync
 Author: BobTheBuidler

{ez_a_sync-0.27.5 → ez_a_sync-0.27.6}/tests/primitives/test_queue.py

@@ -1,6 +1,11 @@
 import pytest
 import asyncio
-from a_sync.primitives.queue import ProcessingQueue, Queue
+from a_sync.primitives.queue import (
+    PriorityProcessingQueue,
+    ProcessingQueue,
+    Queue,
+    SmartProcessingQueue,
+)
 
 
 @pytest.mark.asyncio_cooperative
@@ -366,3 +371,81 @@ async def test_processing_call():
     results = await asyncio.gather(*big_work)
     assert results == list(range(100))
     assert queue.empty()
+
+
+@pytest.mark.asyncio_cooperative
+async def test_smart_processing_queue_initialization():
+    queue = SmartProcessingQueue(coro_fn, 2)
+    assert isinstance(queue, ProcessingQueue)
+    assert queue.func == coro_fn
+    assert queue.num_workers == 2
+    assert queue.empty()
+
+
+@pytest.mark.asyncio_cooperative
+async def test_priority_processing_queue_initialization():
+    queue = PriorityProcessingQueue(coro_fn, 2)
+    assert isinstance(queue, PriorityProcessingQueue)
+    assert queue.func == coro_fn
+    assert queue.num_workers == 2
+    assert queue.empty()
+
+
+@pytest.mark.asyncio_cooperative
+async def test_smart_processing_put_and_await():
+    queue = SmartProcessingQueue(coro_fn, 2)
+    fut = await queue.put("1")
+    assert isinstance(fut, asyncio.Future)
+    assert not queue.empty()
+    assert await fut == 1
+    assert queue.empty()
+
+
+@pytest.mark.asyncio_cooperative
+async def test_priority_processing_put_and_await():
+    queue = PriorityProcessingQueue(coro_fn, 2)
+    fut = await queue.put(5, "1")
+    assert isinstance(fut, asyncio.Future)
+    assert not queue.empty()
+    assert await fut == 1
+    assert queue.empty()
+
+
+@pytest.mark.asyncio_cooperative
+async def test_smart_processing_put_nowait_and_await():
+    queue = SmartProcessingQueue(coro_fn, 2)
+    fut = queue.put_nowait("2")
+    assert isinstance(fut, asyncio.Future)
+    assert not queue.empty()
+    assert await fut == 2
+    with pytest.raises(asyncio.QueueEmpty):
+        queue.get_nowait()
+
+
+@pytest.mark.asyncio_cooperative
+async def test_priority_processing_put_nowait_and_await():
+    queue = PriorityProcessingQueue(coro_fn, 2)
+    fut = queue.put_nowait(5, "2")
+    assert isinstance(fut, asyncio.Future)
+    assert not queue.empty()
+    assert await fut == 2
+    with pytest.raises(asyncio.QueueEmpty):
+        queue.get_nowait()
+
+
+@pytest.mark.asyncio_cooperative
+async def test_smart_processing_call():
+    queue = SmartProcessingQueue(coro_fn, 10)
+    big_work = map(queue, map(str, range(100)))
+    results = await asyncio.gather(*big_work)
+    assert results == list(range(100))
+    assert queue.empty()
+
+
+@pytest.mark.asyncio_cooperative
+async def test_priority_processing_call():
+    queue = PriorityProcessingQueue(coro_fn, 10)
+    big_work = (queue(i, str(i)) for i in range(100))
+    results = await asyncio.gather(*big_work)
+    assert results == list(range(100))
+    assert queue.empty()