queutils 0.9.3__py3-none-any.whl → 0.10.0__py3-none-any.whl

queutils/__init__.py

@@ -6,6 +6,10 @@ from .eventcounterqueue import (
     QCounter as QCounter,
     EventCounterQueue as EventCounterQueue,
 )
+from .awrap import (
+    awrap as awrap,
+    abatch as abatch,
+)
 
 __all__ = [
     "asyncqueue",
@@ -13,4 +17,5 @@ __all__ = [
     "eventcounterqueue",
     "filequeue",
     "iterablequeue",
+    "awrap",
 ]

queutils/asyncqueue.py

@@ -19,18 +19,10 @@ from queue import Full, Empty, Queue
 from asyncio.queues import QueueEmpty, QueueFull
 import asyncio
 from typing import Generic, TypeVar
-import logging
 from asyncio import sleep
 
 T = TypeVar("T")
 
-logger = logging.getLogger(__name__)
-
-debug = logger.debug
-message = logger.warning
-verbose = logger.info
-error = logger.error
-
 
 class AsyncQueue(asyncio.Queue, Generic[T]):
     """

queutils/awrap.py

@@ -0,0 +1,45 @@
+"""awrap() is a async wrapper for Iterables
+
+It converts an Iterable[T] to AsyncGenerator[T].
+AsyncGenerator[T] is also AsyncIterable[T] allowing it to be used in async for
+"""
+
+from typing import Iterable, Sequence, TypeVar, AsyncGenerator, AsyncIterable
+
+T = TypeVar("T")
+
+
+async def awrap(iterable: Iterable[T]) -> AsyncGenerator[T, None]:
+    """
+    Async wrapper for Iterable[T] so it can be used in async for
+    Can be used in async for loop
+    """
+    for item in iter(iterable):
+        yield item
+
+
+async def abatch(
+    iterable: Iterable[T] | AsyncIterable[T], size: int
+) -> AsyncGenerator[Sequence[T], None]:
+    """
+    Async wrapper reads batches from a AsyncIterable[T] or Iterable[T]
+    Can be used in async for loop
+    """
+    batch: list[T] = []
+    if isinstance(iterable, AsyncIterable):
+        async for item in iterable:
+            batch.append(item)
+            if len(batch) == size:
+                yield batch
+                batch = []
+    elif isinstance(iterable, Iterable):
+        for item in iterable:
+            batch.append(item)
+            if len(batch) == size:
+                yield batch
+                batch = []
+    else:
+        raise TypeError(f"Expected Iterable or AsyncIterable, got {type(iterable)}")
+
+    if batch:
+        yield batch

queutils/eventcounterqueue.py

@@ -4,13 +4,6 @@ from deprecated import deprecated
 from .countable import Countable
 from .iterablequeue import IterableQueue, QueueDone
 from collections import defaultdict
-import logging
-
-logger = logging.getLogger()
-error = logger.error
-message = logger.warning
-verbose = logger.info
-debug = logger.debug
 
 ###########################################
 #

queutils/filequeue.py

@@ -29,8 +29,6 @@ from .iterablequeue import IterableQueue
 
 logger = logging.getLogger(__name__)
 error = logger.error
-message = logger.warning
-verbose = logger.info
 debug = logger.debug
 
 
@@ -47,8 +45,8 @@ def str2path(filename: str | Path, suffix: str | None = None) -> Path:
 
 class FileQueue(IterableQueue[Path]):
     """
-    Class to create a IterableQueue(asyncio.Queue) of filenames based on 
-    given directories and files as arguments. 
+    Class to create a IterableQueue(asyncio.Queue) of filenames based on
+    given directories and files as arguments.
     Supports include/exclude filters based on filenames.
     """
 
@@ -58,21 +56,20 @@ class FileQueue(IterableQueue[Path]):
         filter: str = "*",
         exclude: bool = False,
         case_sensitive: bool = True,
-        follow_symlinks: bool = False, 
+        follow_symlinks: bool = False,
         **kwargs,
     ):
         assert base is None or isinstance(base, Path), "base has to be Path or None"
         assert isinstance(filter, str), "filter has to be string"
         assert isinstance(case_sensitive, bool), "case_sensitive has to be bool"
         assert isinstance(follow_symlinks, bool), "follow_symlinks has to be bool"
-        
+
         # debug(f"maxsize={str(maxsize)}, filter='{filter}'")
-        super().__init__(count_items=True, **kwargs)
+        super().__init__(**kwargs)
         self._base: Optional[Path] = base
-        # self._done: bool = False
         self._case_sensitive: bool = False
         self._exclude: bool = False
-        self._follow_symlinks : bool = follow_symlinks
+        self._follow_symlinks: bool = follow_symlinks
         self.set_filter(filter=filter, exclude=exclude, case_sensitive=case_sensitive)
 
     def set_filter(
@@ -119,8 +116,7 @@ class FileQueue(IterableQueue[Path]):
                     await self.put(path)
         except Exception as err:
             error(f"{err}")
-        return await self.finish()
-
+        return await self.finish_producer()
 
     async def put(self, path: Path) -> None:
         """Recursive function to build process queue. Sanitize filename"""

queutils/iterablequeue.py

@@ -17,16 +17,14 @@ __email__ = "Jylpah@gmail.com"
 __status__ = "Production"
 
 
-from asyncio import Queue, QueueFull, QueueEmpty, Event, Lock
+from asyncio import Queue, QueueFull, Event, Lock
 from typing import AsyncIterable, TypeVar, Optional
+from deprecated import deprecated
 from .countable import Countable
 import logging
 
 # Setup logging
 logger = logging.getLogger(__name__)
-error = logger.error
-message = logger.warning
-verbose = logger.info
 debug = logger.debug
 
 T = TypeVar("T")
@@ -45,34 +43,44 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
     IterableQueue is asyncio.Queue subclass that can be iterated asynchronusly.
 
     IterableQueue terminates automatically when the queue has been
-    filled and emptied. Supports:
+    filled and emptied.
+
+    Supports:
     - asyncio.Queue() interface, _nowait() methods are experimental
     - AsyncIterable(): async for item in queue:
     - Automatic termination of the consumers when the queue has been emptied with QueueDone exception
     - Producers must be registered with add_producer() and they must notify the queue
       with finish() once they have finished adding items
     - Countable interface to count number of items task_done() through 'count' property
-    - Countable property can be disabled with count_items=False. This is useful when you
-    want to sum the count of multiple IterableQueues
+
+    IterableQueue stages:
+
+    1) Initialized: Queue has been created and it is empty
+    2) is_illed: All producers have finished adding items to the queue
+    3) empty/has_wip: Queue has been emptied
+    4) is_done: All items have been marked with task_done()
     """
 
-    def __init__(self, count_items: bool = True, **kwargs):
+    def __init__(self, **kwargs) -> None:
         # _Q is required instead of inheriting from Queue()
         # using super() since Queue is Optional[T], not [T]
         self._Q: Queue[Optional[T]] = Queue(**kwargs)
+        self._maxsize: int = self._Q.maxsize  # Asyncio.Queue has _maxsize
         self._producers: int = 0
-        self._count_items: bool = count_items
         self._count: int = 0
         self._wip: int = 0
 
         self._modify: Lock = Lock()
         self._put_lock: Lock = Lock()
 
+        # the last producer has finished
         self._filled: Event = Event()
+        # the last producer has finished and the queue is empty
         self._empty: Event = Event()
+        # the queue is done, all items have been marked with task_done()
         self._done: Event = Event()
 
-        self._empty.set()
+        self._empty.clear()  # this will be tested only after queue is filled
 
     @property
     def is_filled(self) -> bool:
@@ -99,12 +107,6 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         """
         return self._Q.full()
 
-    def check_done(self) -> bool:
-        if self.is_filled and self.empty() and not self.has_wip:
-            self._done.set()
-            return True
-        return False
-
     def empty(self) -> bool:
         """
         Queue has no items except None as a sentinel
@@ -137,101 +139,95 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
 
     @property
     def count(self) -> int:
-        if self._count_items:
-            return self._count
-        else:
-            return 0
+        return self._count
 
     async def add_producer(self, N: int = 1) -> int:
         """
         Add producer(s) to the queue
         """
-        assert N > 0, "N has to be positive"
+        if N <= 0:
+            raise ValueError("N has to be positive")
         async with self._modify:
             if self.is_filled:
                 raise QueueDone
             self._producers += N
         return self._producers
 
-    async def finish(self, all: bool = False, empty: bool = False) -> bool:
+    @deprecated(version="0.10.0", reason="Use finish_producer() instead for clarity")
+    async def finish(self, all: bool = False) -> bool:
+        """
+        Finish producer
+
+        Depreciated function, use finish_producer() instead
+        """
+        return await self.finish_producer(all=all)
+
+    async def finish_producer(self, all: bool = False) -> bool:
         """
         Producer has finished adding items to the queue.
         Once the last producers has finished, the queue is_filled.
         - all: finish() queue for all producers at once
+
+        Return True if the last producer is 'finished'
         """
         async with self._modify:
-            if self._producers <= 0 or self.is_filled:
-                # raise ValueError("finish() called more than the is producers")
-                self._producers = 0
-                return False
+            if self.is_filled:
+                return True
+
             self._producers -= 1
 
-            if all or self._producers <= 0:
+            if self._producers < 0:
+                raise ValueError("Too many finish() calls")
+            elif all or self._producers == 0:
                 self._filled.set()
                 self._producers = 0
 
         if self._producers == 0:
-            if empty:
-                try:
-                    while True:
-                        _ = self.get_nowait()
-                        self.task_done()
-                except (QueueDone, QueueEmpty):
-                    pass
-
             async with self._put_lock:
-                if empty:
-                    try:
-                        _ = self.get_nowait()
-                        self.task_done()
-                    except (QueueDone, QueueEmpty):
-                        pass
-                self.check_done()
+                if self._Q.qsize() == 0:
+                    self._empty.set()
+                    if not self.has_wip:
+                        self._done.set()
                 await self._Q.put(None)
                 return True
         return False
 
     async def put(self, item: T) -> None:
+        if item is None:
+            raise ValueError("Cannot add None to IterableQueue")
         async with self._put_lock:
             if self.is_filled:  # should this be inside put_lock?
                 raise QueueDone
             if self._producers <= 0:
                 raise ValueError("No registered producers")
-            elif item is None:
-                raise ValueError("Cannot add None to IterableQueue")
             await self._Q.put(item=item)
-            self._empty.clear()
         return None
 
     def put_nowait(self, item: T) -> None:
         """
         Experimental asyncio.Queue.put_nowait() implementation
         """
-        # raise NotImplementedError
         if self.is_filled:
             raise QueueDone
         if self._producers <= 0:
             raise ValueError("No registered producers")
-        elif item is None:
+        if item is None:
             raise ValueError("Cannot add None to IterableQueue")
         self._Q.put_nowait(item=item)
-        self._empty.clear()
         return None
 
     async def get(self) -> T:
         item = await self._Q.get()
         if item is None:
             self._empty.set()
+            if not self.has_wip:
+                self._done.set()
             self._Q.task_done()
-            self.check_done()
             async with self._put_lock:
                 await self._Q.put(None)
                 raise QueueDone
         else:
-            if self._Q.qsize() == 0:
-                self._empty.set()
-            async with self._modify:
-                self._wip += 1
+            self._wip += 1
             return item
 
     def get_nowait(self) -> T:
@@ -241,16 +237,15 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         item: T | None = self._Q.get_nowait()
         if item is None:
             self._empty.set()
+            if not self.has_wip:
+                self._done.set()
             self._Q.task_done()
-            self.check_done()
             try:
                 self._Q.put_nowait(None)
             except QueueFull:
                 pass
             raise QueueDone
         else:
-            if self._Q.qsize() == 0:
-                self._empty.set()
             self._wip += 1
             return item
 
@@ -260,7 +255,8 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         self._wip -= 1
         if self._wip < 0:
             raise ValueError("task_done() called more than tasks open")
-        self.check_done()
+        if self.is_filled and self._empty.is_set() and not self.has_wip:
+            self._done.set()
 
     async def join(self) -> None:
         debug("Waiting queue to be filled")
@@ -271,14 +267,18 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         return None
 
     def __aiter__(self):
+        """
+        Return ASyncIterator to be able iterate the queue using async for
+        """
         return self
 
     async def __anext__(self) -> T:
-        async with self._modify:
-            if self._wip > 0:  # do not mark task_done() at first call
-                self.task_done()
+        """
+        Async iterator for IterableQueue
+        """
+        if self._wip > 0:  # do not mark task_done() at first call
+            self.task_done()
         try:
-            item = await self.get()
-            return item
+            return await self.get()
         except QueueDone:
             raise StopAsyncIteration

{queutils-0.9.3.dist-info → queutils-0.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queutils
-Version: 0.9.3
+Version: 0.10.0
 Summary: Handy Python Queue utilies
 Project-URL: Homepage, https://github.com/Jylpah/queutils
 Project-URL: Bug Tracker, https://github.com/Jylpah/queutils/issues
@@ -38,13 +38,13 @@ Queutils *[Queue Utils]* is a package of handy Python queue classes:
 - **AsyncQueue** - An `async` wrapper for non-async `queue.Queue`
 - **IterableQueue** - An `AsyncIterable` queue that terminates when finished
 - **EventCounterQueue** - An `IterableQueue` for counting events in `async` threads
-- **FileQueue** - Builds an iterable queue of filenames from files/dirs given as input
+- **FileQueue** - Builds an `IterableQueue[pathlib.Path]` of filenames from files/dirs given as input
 
 
 # AsyncQueue
 
 `AsyncQueue` is a async wrapper for non-async `queue.Queue`. It can be used to create 
-an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessing.Queue` and thus enable `async` code in parent/child processes to communicate over  `multiprocessing.Queue` as it were an `asyncio.Queue`. 
+an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessing.Queue` and thus enable `async` code in parent/child processes to communicate over  `multiprocessing.Queue` as it were an `asyncio.Queue`. Uses `sleep()` for `get()`/`put()` if the queue is empty/full.
 
 ## Features 
 
@@ -58,7 +58,7 @@ an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessin
 `IterableQueue` is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be 
 iterated in `async for` loop. `IterableQueue` terminates automatically when the queue has been filled and emptied. 
 
-The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves and it 
+The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves with `add_producer()` call and it 
 keeps count of registered producers which are "finished" adding items to the queue. Once all the registered 
 producers are "finished", the queue enters into "filled" state and no new items can be added. Once an 
 "filled" queue is emptied, the queue becomes "done" and all new `get()` calls to the queue will 
@@ -72,8 +72,6 @@ producers are "finished", the queue enters into "filled" state and no new items
 - Producers must be registered with `add_producer()` and they must notify the queue
   with `finish()` once they have finished adding items 
 - Countable interface to count number of items task_done() through `count` property
-- Countable property can be disabled with count_items=False. This is useful when you
-    want to sum the count of multiple IterableQueues 
 
 # EventCounterQueue
 

queutils-0.10.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+queutils/__init__.py,sha256=EGFMKYqmOdB22HjQH62w5eWmncr9IxC9_QAlT-4R9cY,519
+queutils/asyncqueue.py,sha256=qYtrF03u954B7NdCOIIW9s1_Lpw38BmGgB-Pin8y4n0,2646
+queutils/awrap.py,sha256=QydWBLmyNIHpYrRwVFn1RWjbQKRnJr61u-DYl7d3F7w,1293
+queutils/countable.py,sha256=YSi7ILf9CuB5Tm3T4UUMEFlveqzqcmomfqJAlLGHEz8,172
+queutils/eventcounterqueue.py,sha256=9CvgfnWmvXrMr3h-RiQyIVIBXqOnGvohql_ADvdlmxo,2913
+queutils/filequeue.py,sha256=J_UK3VHKaM4ELFzedtj6lDmlXas8O732aC0nO8zNzNo,5256
+queutils/iterablequeue.py,sha256=_aAtitQROpUHglcmGCAJkOErqDjibbiALp2ewnzf2U8,8699
+queutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+queutils-0.10.0.dist-info/METADATA,sha256=_6RN7Pde53fUpvEOgVn7Y3LV-EkVfeGtX4OfB9N3SLc,4630
+queutils-0.10.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+queutils-0.10.0.dist-info/licenses/LICENSE,sha256=J1zeIKU2JVQmhwO2hHQDK8WR6zjVZ-wX8r7ZlL45AbI,1063
+queutils-0.10.0.dist-info/RECORD,,

queutils-0.9.3.dist-info/RECORD

@@ -1,11 +0,0 @@
-queutils/__init__.py,sha256=rbDSh9513Dday-MfWzPOUfN7Aj2x4IqrvCmKbBj7has,441
-queutils/asyncqueue.py,sha256=GZRmlWTBQoKzTf7xr4MI-qhNqvIiaNWswAxFokP91Lg,2789
-queutils/countable.py,sha256=YSi7ILf9CuB5Tm3T4UUMEFlveqzqcmomfqJAlLGHEz8,172
-queutils/eventcounterqueue.py,sha256=NjT8FqEskZhjmus7L333DZU9kXoZCNsn7ydCI5HSe1U,3047
-queutils/filequeue.py,sha256=q2ly9H-lSCq6xuOqT1IlWgyCVyLoZiKbc0NuzmkF4aw,5360
-queutils/iterablequeue.py,sha256=ZXwa9040yTawL1DMMNBXgQKiYr32nmzuSbgi-raAtZc,8664
-queutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-queutils-0.9.3.dist-info/METADATA,sha256=VGe0K6t0aepLD6DlXlwjFiifK8IUZfcmQlOUXVCKFbA,4664
-queutils-0.9.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-queutils-0.9.3.dist-info/licenses/LICENSE,sha256=J1zeIKU2JVQmhwO2hHQDK8WR6zjVZ-wX8r7ZlL45AbI,1063
-queutils-0.9.3.dist-info/RECORD,,