queutils 0.9.4__py3-none-any.whl → 0.10.0__py3-none-any.whl

queutils/__init__.py

@@ -6,6 +6,10 @@ from .eventcounterqueue import (
     QCounter as QCounter,
     EventCounterQueue as EventCounterQueue,
 )
+from .awrap import (
+    awrap as awrap,
+    abatch as abatch,
+)
 
 __all__ = [
     "asyncqueue",
@@ -13,4 +17,5 @@ __all__ = [
     "eventcounterqueue",
     "filequeue",
     "iterablequeue",
+    "awrap",
 ]

queutils/asyncqueue.py

@@ -19,18 +19,10 @@ from queue import Full, Empty, Queue
 from asyncio.queues import QueueEmpty, QueueFull
 import asyncio
 from typing import Generic, TypeVar
-import logging
 from asyncio import sleep
 
 T = TypeVar("T")
 
-logger = logging.getLogger(__name__)
-
-debug = logger.debug
-message = logger.warning
-verbose = logger.info
-error = logger.error
-
 
 class AsyncQueue(asyncio.Queue, Generic[T]):
     """

queutils/awrap.py

@@ -0,0 +1,45 @@
+"""awrap() is a async wrapper for Iterables
+
+It converts an Iterable[T] to AsyncGenerator[T].
+AsyncGenerator[T] is also AsyncIterable[T] allowing it to be used in async for
+"""
+
+from typing import Iterable, Sequence, TypeVar, AsyncGenerator, AsyncIterable
+
+T = TypeVar("T")
+
+
+async def awrap(iterable: Iterable[T]) -> AsyncGenerator[T, None]:
+    """
+    Async wrapper for Iterable[T] so it can be used in async for
+    Can be used in async for loop
+    """
+    for item in iter(iterable):
+        yield item
+
+
+async def abatch(
+    iterable: Iterable[T] | AsyncIterable[T], size: int
+) -> AsyncGenerator[Sequence[T], None]:
+    """
+    Async wrapper reads batches from a AsyncIterable[T] or Iterable[T]
+    Can be used in async for loop
+    """
+    batch: list[T] = []
+    if isinstance(iterable, AsyncIterable):
+        async for item in iterable:
+            batch.append(item)
+            if len(batch) == size:
+                yield batch
+                batch = []
+    elif isinstance(iterable, Iterable):
+        for item in iterable:
+            batch.append(item)
+            if len(batch) == size:
+                yield batch
+                batch = []
+    else:
+        raise TypeError(f"Expected Iterable or AsyncIterable, got {type(iterable)}")
+
+    if batch:
+        yield batch

queutils/eventcounterqueue.py

@@ -4,13 +4,6 @@ from deprecated import deprecated
 from .countable import Countable
 from .iterablequeue import IterableQueue, QueueDone
 from collections import defaultdict
-import logging
-
-logger = logging.getLogger()
-error = logger.error
-message = logger.warning
-verbose = logger.info
-debug = logger.debug
 
 ###########################################
 #

queutils/filequeue.py

@@ -29,8 +29,6 @@ from .iterablequeue import IterableQueue
 
 logger = logging.getLogger(__name__)
 error = logger.error
-message = logger.warning
-verbose = logger.info
 debug = logger.debug
 
 
@@ -47,8 +45,8 @@ def str2path(filename: str | Path, suffix: str | None = None) -> Path:
 
 class FileQueue(IterableQueue[Path]):
     """
-    Class to create a IterableQueue(asyncio.Queue) of filenames based on 
-    given directories and files as arguments. 
+    Class to create a IterableQueue(asyncio.Queue) of filenames based on
+    given directories and files as arguments.
     Supports include/exclude filters based on filenames.
     """
 
@@ -58,21 +56,20 @@ class FileQueue(IterableQueue[Path]):
         filter: str = "*",
         exclude: bool = False,
         case_sensitive: bool = True,
-        follow_symlinks: bool = False, 
+        follow_symlinks: bool = False,
         **kwargs,
     ):
         assert base is None or isinstance(base, Path), "base has to be Path or None"
         assert isinstance(filter, str), "filter has to be string"
         assert isinstance(case_sensitive, bool), "case_sensitive has to be bool"
         assert isinstance(follow_symlinks, bool), "follow_symlinks has to be bool"
-        
+
         # debug(f"maxsize={str(maxsize)}, filter='{filter}'")
-        super().__init__(count_items=True, **kwargs)
+        super().__init__(**kwargs)
         self._base: Optional[Path] = base
-        # self._done: bool = False
         self._case_sensitive: bool = False
         self._exclude: bool = False
-        self._follow_symlinks : bool = follow_symlinks
+        self._follow_symlinks: bool = follow_symlinks
         self.set_filter(filter=filter, exclude=exclude, case_sensitive=case_sensitive)
 
     def set_filter(
@@ -119,8 +116,7 @@ class FileQueue(IterableQueue[Path]):
                     await self.put(path)
         except Exception as err:
             error(f"{err}")
-        return await self.finish()
-
+        return await self.finish_producer()
 
     async def put(self, path: Path) -> None:
         """Recursive function to build process queue. Sanitize filename"""

queutils/iterablequeue.py

@@ -17,16 +17,14 @@ __email__ = "Jylpah@gmail.com"
 __status__ = "Production"
 
 
-from asyncio import Queue, QueueFull, QueueEmpty, Event, Lock
+from asyncio import Queue, QueueFull, Event, Lock
 from typing import AsyncIterable, TypeVar, Optional
+from deprecated import deprecated
 from .countable import Countable
 import logging
 
 # Setup logging
 logger = logging.getLogger(__name__)
-error = logger.error
-message = logger.warning
-verbose = logger.info
 debug = logger.debug
 
 T = TypeVar("T")
@@ -45,34 +43,44 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
     IterableQueue is asyncio.Queue subclass that can be iterated asynchronusly.
 
     IterableQueue terminates automatically when the queue has been
-    filled and emptied. Supports:
+    filled and emptied.
+
+    Supports:
     - asyncio.Queue() interface, _nowait() methods are experimental
     - AsyncIterable(): async for item in queue:
     - Automatic termination of the consumers when the queue has been emptied with QueueDone exception
     - Producers must be registered with add_producer() and they must notify the queue
       with finish() once they have finished adding items
     - Countable interface to count number of items task_done() through 'count' property
-    - Countable property can be disabled with count_items=False. This is useful when you
-    want to sum the count of multiple IterableQueues
+
+    IterableQueue stages:
+
+    1) Initialized: Queue has been created and it is empty
+    2) is_illed: All producers have finished adding items to the queue
+    3) empty/has_wip: Queue has been emptied
+    4) is_done: All items have been marked with task_done()
     """
 
-    def __init__(self, count_items: bool = True, **kwargs):
+    def __init__(self, **kwargs) -> None:
         # _Q is required instead of inheriting from Queue()
         # using super() since Queue is Optional[T], not [T]
         self._Q: Queue[Optional[T]] = Queue(**kwargs)
+        self._maxsize: int = self._Q.maxsize  # Asyncio.Queue has _maxsize
         self._producers: int = 0
-        self._count_items: bool = count_items
         self._count: int = 0
         self._wip: int = 0
 
         self._modify: Lock = Lock()
         self._put_lock: Lock = Lock()
 
+        # the last producer has finished
         self._filled: Event = Event()
+        # the last producer has finished and the queue is empty
         self._empty: Event = Event()
+        # the queue is done, all items have been marked with task_done()
         self._done: Event = Event()
 
-        self._empty.set()
+        self._empty.clear()  # this will be tested only after queue is filled
 
     @property
     def is_filled(self) -> bool:
@@ -99,12 +107,6 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         """
         return self._Q.full()
 
-    def check_done(self) -> bool:
-        if self.is_filled and self.empty() and not self.has_wip:
-            self._done.set()
-            return True
-        return False
-
     def empty(self) -> bool:
         """
         Queue has no items except None as a sentinel
@@ -137,101 +139,95 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
 
     @property
     def count(self) -> int:
-        if self._count_items:
-            return self._count
-        else:
-            return 0
+        return self._count
 
     async def add_producer(self, N: int = 1) -> int:
         """
         Add producer(s) to the queue
         """
-        assert N > 0, "N has to be positive"
+        if N <= 0:
+            raise ValueError("N has to be positive")
         async with self._modify:
             if self.is_filled:
                 raise QueueDone
             self._producers += N
         return self._producers
 
-    async def finish(self, all: bool = False, empty: bool = False) -> bool:
+    @deprecated(version="0.10.0", reason="Use finish_producer() instead for clarity")
+    async def finish(self, all: bool = False) -> bool:
+        """
+        Finish producer
+
+        Depreciated function, use finish_producer() instead
+        """
+        return await self.finish_producer(all=all)
+
+    async def finish_producer(self, all: bool = False) -> bool:
         """
         Producer has finished adding items to the queue.
         Once the last producers has finished, the queue is_filled.
         - all: finish() queue for all producers at once
+
+        Return True if the last producer is 'finished'
         """
         async with self._modify:
-            if self._producers <= 0 or self.is_filled:
-                # raise ValueError("finish() called more than the is producers")
-                self._producers = 0
-                return False
+            if self.is_filled:
+                return True
+
             self._producers -= 1
 
-            if all or self._producers <= 0:
+            if self._producers < 0:
+                raise ValueError("Too many finish() calls")
+            elif all or self._producers == 0:
                 self._filled.set()
                 self._producers = 0
 
         if self._producers == 0:
-            if empty:
-                try:
-                    while True:
-                        _ = self.get_nowait()
-                        self.task_done()
-                except (QueueDone, QueueEmpty):
-                    pass
-
             async with self._put_lock:
-                if empty:
-                    try:
-                        _ = self.get_nowait()
-                        self.task_done()
-                    except (QueueDone, QueueEmpty):
-                        pass
-                self.check_done()
+                if self._Q.qsize() == 0:
+                    self._empty.set()
+                    if not self.has_wip:
+                        self._done.set()
                 await self._Q.put(None)
                 return True
         return False
 
     async def put(self, item: T) -> None:
+        if item is None:
+            raise ValueError("Cannot add None to IterableQueue")
         async with self._put_lock:
             if self.is_filled:  # should this be inside put_lock?
                 raise QueueDone
             if self._producers <= 0:
                 raise ValueError("No registered producers")
-            elif item is None:
-                raise ValueError("Cannot add None to IterableQueue")
             await self._Q.put(item=item)
-            self._empty.clear()
         return None
 
     def put_nowait(self, item: T) -> None:
         """
         Experimental asyncio.Queue.put_nowait() implementation
         """
-        # raise NotImplementedError
         if self.is_filled:
             raise QueueDone
         if self._producers <= 0:
             raise ValueError("No registered producers")
-        elif item is None:
+        if item is None:
             raise ValueError("Cannot add None to IterableQueue")
         self._Q.put_nowait(item=item)
-        self._empty.clear()
         return None
 
     async def get(self) -> T:
         item = await self._Q.get()
         if item is None:
             self._empty.set()
+            if not self.has_wip:
+                self._done.set()
             self._Q.task_done()
-            self.check_done()
             async with self._put_lock:
                 await self._Q.put(None)
                 raise QueueDone
         else:
-            if self._Q.qsize() == 0:
-                self._empty.set()
-            async with self._modify:
-                self._wip += 1
+            self._wip += 1
             return item
 
     def get_nowait(self) -> T:
@@ -241,16 +237,15 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         item: T | None = self._Q.get_nowait()
         if item is None:
             self._empty.set()
+            if not self.has_wip:
+                self._done.set()
             self._Q.task_done()
-            self.check_done()
             try:
                 self._Q.put_nowait(None)
             except QueueFull:
                 pass
             raise QueueDone
         else:
-            if self._Q.qsize() == 0:
-                self._empty.set()
             self._wip += 1
             return item
 
@@ -260,7 +255,8 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         self._wip -= 1
         if self._wip < 0:
             raise ValueError("task_done() called more than tasks open")
-        self.check_done()
+        if self.is_filled and self._empty.is_set() and not self.has_wip:
+            self._done.set()
 
     async def join(self) -> None:
         debug("Waiting queue to be filled")
@@ -271,14 +267,18 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         return None
 
     def __aiter__(self):
+        """
+        Return ASyncIterator to be able iterate the queue using async for
+        """
         return self
 
     async def __anext__(self) -> T:
-        async with self._modify:
-            if self._wip > 0:  # do not mark task_done() at first call
-                self.task_done()
+        """
+        Async iterator for IterableQueue
+        """
+        if self._wip > 0:  # do not mark task_done() at first call
+            self.task_done()
         try:
-            item = await self.get()
-            return item
+            return await self.get()
         except QueueDone:
             raise StopAsyncIteration

{queutils-0.9.4.dist-info → queutils-0.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queutils
-Version: 0.9.4
+Version: 0.10.0
 Summary: Handy Python Queue utilies
 Project-URL: Homepage, https://github.com/Jylpah/queutils
 Project-URL: Bug Tracker, https://github.com/Jylpah/queutils/issues
@@ -72,8 +72,6 @@ producers are "finished", the queue enters into "filled" state and no new items
 - Producers must be registered with `add_producer()` and they must notify the queue
   with `finish()` once they have finished adding items 
 - Countable interface to count number of items task_done() through `count` property
-- Countable property can be disabled with count_items=False. This is useful when you
-    want to sum the count of multiple IterableQueues 
 
 # EventCounterQueue
 

queutils-0.10.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+queutils/__init__.py,sha256=EGFMKYqmOdB22HjQH62w5eWmncr9IxC9_QAlT-4R9cY,519
+queutils/asyncqueue.py,sha256=qYtrF03u954B7NdCOIIW9s1_Lpw38BmGgB-Pin8y4n0,2646
+queutils/awrap.py,sha256=QydWBLmyNIHpYrRwVFn1RWjbQKRnJr61u-DYl7d3F7w,1293
+queutils/countable.py,sha256=YSi7ILf9CuB5Tm3T4UUMEFlveqzqcmomfqJAlLGHEz8,172
+queutils/eventcounterqueue.py,sha256=9CvgfnWmvXrMr3h-RiQyIVIBXqOnGvohql_ADvdlmxo,2913
+queutils/filequeue.py,sha256=J_UK3VHKaM4ELFzedtj6lDmlXas8O732aC0nO8zNzNo,5256
+queutils/iterablequeue.py,sha256=_aAtitQROpUHglcmGCAJkOErqDjibbiALp2ewnzf2U8,8699
+queutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+queutils-0.10.0.dist-info/METADATA,sha256=_6RN7Pde53fUpvEOgVn7Y3LV-EkVfeGtX4OfB9N3SLc,4630
+queutils-0.10.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+queutils-0.10.0.dist-info/licenses/LICENSE,sha256=J1zeIKU2JVQmhwO2hHQDK8WR6zjVZ-wX8r7ZlL45AbI,1063
+queutils-0.10.0.dist-info/RECORD,,

queutils-0.9.4.dist-info/RECORD

@@ -1,11 +0,0 @@
-queutils/__init__.py,sha256=rbDSh9513Dday-MfWzPOUfN7Aj2x4IqrvCmKbBj7has,441
-queutils/asyncqueue.py,sha256=GZRmlWTBQoKzTf7xr4MI-qhNqvIiaNWswAxFokP91Lg,2789
-queutils/countable.py,sha256=YSi7ILf9CuB5Tm3T4UUMEFlveqzqcmomfqJAlLGHEz8,172
-queutils/eventcounterqueue.py,sha256=NjT8FqEskZhjmus7L333DZU9kXoZCNsn7ydCI5HSe1U,3047
-queutils/filequeue.py,sha256=q2ly9H-lSCq6xuOqT1IlWgyCVyLoZiKbc0NuzmkF4aw,5360
-queutils/iterablequeue.py,sha256=ZXwa9040yTawL1DMMNBXgQKiYr32nmzuSbgi-raAtZc,8664
-queutils/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-queutils-0.9.4.dist-info/METADATA,sha256=v3MC3cKun7jnGfnR9WFs7PYQHiCEDu398YBBVEWVcqE,4768
-queutils-0.9.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-queutils-0.9.4.dist-info/licenses/LICENSE,sha256=J1zeIKU2JVQmhwO2hHQDK8WR6zjVZ-wX8r7ZlL45AbI,1063
-queutils-0.9.4.dist-info/RECORD,,