queutils 0.9.4__tar.gz → 0.10.0__tar.gz

{queutils-0.9.4 → queutils-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queutils
-Version: 0.9.4
+Version: 0.10.0
 Summary: Handy Python Queue utilies
 Project-URL: Homepage, https://github.com/Jylpah/queutils
 Project-URL: Bug Tracker, https://github.com/Jylpah/queutils/issues
@@ -72,8 +72,6 @@ producers are "finished", the queue enters into "filled" state and no new items
 - Producers must be registered with `add_producer()` and they must notify the queue
   with `finish()` once they have finished adding items 
 - Countable interface to count number of items task_done() through `count` property
-- Countable property can be disabled with count_items=False. This is useful when you
-    want to sum the count of multiple IterableQueues 
 
 # EventCounterQueue
 

{queutils-0.9.4 → queutils-0.10.0}/README.md

@@ -41,8 +41,6 @@ producers are "finished", the queue enters into "filled" state and no new items
 - Producers must be registered with `add_producer()` and they must notify the queue
   with `finish()` once they have finished adding items 
 - Countable interface to count number of items task_done() through `count` property
-- Countable property can be disabled with count_items=False. This is useful when you
-    want to sum the count of multiple IterableQueues 
 
 # EventCounterQueue
 

{queutils-0.9.4 → queutils-0.10.0}/docs/iterablequeue.md

@@ -11,8 +11,6 @@ iterated in `async for` loop. The great benefit of `IterableQueue` is that it te
 - Producers must be registered with `add_producer()` and they must notify the queue
   with `finish()` once they have finished adding items 
 - Countable interface to count number of items task_done() through `count` property
-- Countable property can be disabled with count_items=False. This is useful when you
-    want to sum the count of multiple IterableQueues 
 
 ### Experimental
 

{queutils-0.9.4 → queutils-0.10.0}/pypi.md

@@ -41,8 +41,6 @@ producers are "finished", the queue enters into "filled" state and no new items
 - Producers must be registered with `add_producer()` and they must notify the queue
   with `finish()` once they have finished adding items 
 - Countable interface to count number of items task_done() through `count` property
-- Countable property can be disabled with count_items=False. This is useful when you
-    want to sum the count of multiple IterableQueues 
 
 # EventCounterQueue
 

{queutils-0.9.4 → queutils-0.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "queutils"
-version = "0.9.4"
+version = "0.10.0"
 authors = [{ name = "Jylpah", email = "jylpah@gmail.com" }]
 description = "Handy Python Queue utilies"
 readme = { file = "pypi.md", content-type = "text/markdown" }
@@ -67,3 +67,8 @@ asyncio_default_fixture_loop_scope = "function"
 reportGeneralTypeIssues = false
 reportInvalidStringEscapeSequence = false
 typeCheckingMode = "off"
+include = ["src", "tests"]
+extraPaths = ["src"]
+
+[tool.coverage.run]
+omit = ['tests/*']

{queutils-0.9.4 → queutils-0.10.0}/src/queutils/__init__.py

@@ -6,6 +6,10 @@ from .eventcounterqueue import (
     QCounter as QCounter,
     EventCounterQueue as EventCounterQueue,
 )
+from .awrap import (
+    awrap as awrap,
+    abatch as abatch,
+)
 
 __all__ = [
     "asyncqueue",
@@ -13,4 +17,5 @@ __all__ = [
     "eventcounterqueue",
     "filequeue",
     "iterablequeue",
+    "awrap",
 ]

{queutils-0.9.4 → queutils-0.10.0}/src/queutils/asyncqueue.py

@@ -19,18 +19,10 @@ from queue import Full, Empty, Queue
 from asyncio.queues import QueueEmpty, QueueFull
 import asyncio
 from typing import Generic, TypeVar
-import logging
 from asyncio import sleep
 
 T = TypeVar("T")
 
-logger = logging.getLogger(__name__)
-
-debug = logger.debug
-message = logger.warning
-verbose = logger.info
-error = logger.error
-
 
 class AsyncQueue(asyncio.Queue, Generic[T]):
     """

queutils-0.10.0/src/queutils/awrap.py

@@ -0,0 +1,45 @@
+"""awrap() is a async wrapper for Iterables
+
+It converts an Iterable[T] to AsyncGenerator[T].
+AsyncGenerator[T] is also AsyncIterable[T] allowing it to be used in async for
+"""
+
+from typing import Iterable, Sequence, TypeVar, AsyncGenerator, AsyncIterable
+
+T = TypeVar("T")
+
+
+async def awrap(iterable: Iterable[T]) -> AsyncGenerator[T, None]:
+    """
+    Async wrapper for Iterable[T] so it can be used in async for
+    Can be used in async for loop
+    """
+    for item in iter(iterable):
+        yield item
+
+
+async def abatch(
+    iterable: Iterable[T] | AsyncIterable[T], size: int
+) -> AsyncGenerator[Sequence[T], None]:
+    """
+    Async wrapper reads batches from a AsyncIterable[T] or Iterable[T]
+    Can be used in async for loop
+    """
+    batch: list[T] = []
+    if isinstance(iterable, AsyncIterable):
+        async for item in iterable:
+            batch.append(item)
+            if len(batch) == size:
+                yield batch
+                batch = []
+    elif isinstance(iterable, Iterable):
+        for item in iterable:
+            batch.append(item)
+            if len(batch) == size:
+                yield batch
+                batch = []
+    else:
+        raise TypeError(f"Expected Iterable or AsyncIterable, got {type(iterable)}")
+
+    if batch:
+        yield batch

{queutils-0.9.4 → queutils-0.10.0}/src/queutils/eventcounterqueue.py

@@ -4,13 +4,6 @@ from deprecated import deprecated
 from .countable import Countable
 from .iterablequeue import IterableQueue, QueueDone
 from collections import defaultdict
-import logging
-
-logger = logging.getLogger()
-error = logger.error
-message = logger.warning
-verbose = logger.info
-debug = logger.debug
 
 ###########################################
 #

{queutils-0.9.4 → queutils-0.10.0}/src/queutils/filequeue.py

@@ -29,8 +29,6 @@ from .iterablequeue import IterableQueue
 
 logger = logging.getLogger(__name__)
 error = logger.error
-message = logger.warning
-verbose = logger.info
 debug = logger.debug
 
 
@@ -47,8 +45,8 @@ def str2path(filename: str | Path, suffix: str | None = None) -> Path:
 
 class FileQueue(IterableQueue[Path]):
     """
-    Class to create a IterableQueue(asyncio.Queue) of filenames based on 
-    given directories and files as arguments. 
+    Class to create a IterableQueue(asyncio.Queue) of filenames based on
+    given directories and files as arguments.
     Supports include/exclude filters based on filenames.
     """
 
@@ -58,21 +56,20 @@ class FileQueue(IterableQueue[Path]):
         filter: str = "*",
         exclude: bool = False,
         case_sensitive: bool = True,
-        follow_symlinks: bool = False, 
+        follow_symlinks: bool = False,
         **kwargs,
     ):
         assert base is None or isinstance(base, Path), "base has to be Path or None"
         assert isinstance(filter, str), "filter has to be string"
         assert isinstance(case_sensitive, bool), "case_sensitive has to be bool"
         assert isinstance(follow_symlinks, bool), "follow_symlinks has to be bool"
-        
+
         # debug(f"maxsize={str(maxsize)}, filter='{filter}'")
-        super().__init__(count_items=True, **kwargs)
+        super().__init__(**kwargs)
         self._base: Optional[Path] = base
-        # self._done: bool = False
         self._case_sensitive: bool = False
         self._exclude: bool = False
-        self._follow_symlinks : bool = follow_symlinks
+        self._follow_symlinks: bool = follow_symlinks
         self.set_filter(filter=filter, exclude=exclude, case_sensitive=case_sensitive)
 
     def set_filter(
@@ -119,8 +116,7 @@ class FileQueue(IterableQueue[Path]):
                     await self.put(path)
         except Exception as err:
             error(f"{err}")
-        return await self.finish()
-
+        return await self.finish_producer()
 
     async def put(self, path: Path) -> None:
         """Recursive function to build process queue. Sanitize filename"""

{queutils-0.9.4 → queutils-0.10.0}/src/queutils/iterablequeue.py

@@ -17,16 +17,14 @@ __email__ = "Jylpah@gmail.com"
 __status__ = "Production"
 
 
-from asyncio import Queue, QueueFull, QueueEmpty, Event, Lock
+from asyncio import Queue, QueueFull, Event, Lock
 from typing import AsyncIterable, TypeVar, Optional
+from deprecated import deprecated
 from .countable import Countable
 import logging
 
 # Setup logging
 logger = logging.getLogger(__name__)
-error = logger.error
-message = logger.warning
-verbose = logger.info
 debug = logger.debug
 
 T = TypeVar("T")
@@ -45,34 +43,44 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
     IterableQueue is asyncio.Queue subclass that can be iterated asynchronusly.
 
     IterableQueue terminates automatically when the queue has been
-    filled and emptied. Supports:
+    filled and emptied.
+
+    Supports:
     - asyncio.Queue() interface, _nowait() methods are experimental
     - AsyncIterable(): async for item in queue:
     - Automatic termination of the consumers when the queue has been emptied with QueueDone exception
     - Producers must be registered with add_producer() and they must notify the queue
       with finish() once they have finished adding items
     - Countable interface to count number of items task_done() through 'count' property
-    - Countable property can be disabled with count_items=False. This is useful when you
-    want to sum the count of multiple IterableQueues
+
+    IterableQueue stages:
+
+    1) Initialized: Queue has been created and it is empty
+    2) is_illed: All producers have finished adding items to the queue
+    3) empty/has_wip: Queue has been emptied
+    4) is_done: All items have been marked with task_done()
     """
 
-    def __init__(self, count_items: bool = True, **kwargs):
+    def __init__(self, **kwargs) -> None:
         # _Q is required instead of inheriting from Queue()
         # using super() since Queue is Optional[T], not [T]
         self._Q: Queue[Optional[T]] = Queue(**kwargs)
+        self._maxsize: int = self._Q.maxsize  # Asyncio.Queue has _maxsize
         self._producers: int = 0
-        self._count_items: bool = count_items
         self._count: int = 0
         self._wip: int = 0
 
         self._modify: Lock = Lock()
         self._put_lock: Lock = Lock()
 
+        # the last producer has finished
         self._filled: Event = Event()
+        # the last producer has finished and the queue is empty
         self._empty: Event = Event()
+        # the queue is done, all items have been marked with task_done()
         self._done: Event = Event()
 
-        self._empty.set()
+        self._empty.clear()  # this will be tested only after queue is filled
 
     @property
     def is_filled(self) -> bool:
@@ -99,12 +107,6 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         """
         return self._Q.full()
 
-    def check_done(self) -> bool:
-        if self.is_filled and self.empty() and not self.has_wip:
-            self._done.set()
-            return True
-        return False
-
     def empty(self) -> bool:
         """
         Queue has no items except None as a sentinel
@@ -137,101 +139,95 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
 
     @property
     def count(self) -> int:
-        if self._count_items:
-            return self._count
-        else:
-            return 0
+        return self._count
 
     async def add_producer(self, N: int = 1) -> int:
         """
         Add producer(s) to the queue
         """
-        assert N > 0, "N has to be positive"
+        if N <= 0:
+            raise ValueError("N has to be positive")
         async with self._modify:
             if self.is_filled:
                 raise QueueDone
             self._producers += N
         return self._producers
 
-    async def finish(self, all: bool = False, empty: bool = False) -> bool:
+    @deprecated(version="0.10.0", reason="Use finish_producer() instead for clarity")
+    async def finish(self, all: bool = False) -> bool:
+        """
+        Finish producer
+
+        Depreciated function, use finish_producer() instead
+        """
+        return await self.finish_producer(all=all)
+
+    async def finish_producer(self, all: bool = False) -> bool:
         """
         Producer has finished adding items to the queue.
         Once the last producers has finished, the queue is_filled.
         - all: finish() queue for all producers at once
+
+        Return True if the last producer is 'finished'
         """
         async with self._modify:
-            if self._producers <= 0 or self.is_filled:
-                # raise ValueError("finish() called more than the is producers")
-                self._producers = 0
-                return False
+            if self.is_filled:
+                return True
+
             self._producers -= 1
 
-            if all or self._producers <= 0:
+            if self._producers < 0:
+                raise ValueError("Too many finish() calls")
+            elif all or self._producers == 0:
                 self._filled.set()
                 self._producers = 0
 
         if self._producers == 0:
-            if empty:
-                try:
-                    while True:
-                        _ = self.get_nowait()
-                        self.task_done()
-                except (QueueDone, QueueEmpty):
-                    pass
-
             async with self._put_lock:
-                if empty:
-                    try:
-                        _ = self.get_nowait()
-                        self.task_done()
-                    except (QueueDone, QueueEmpty):
-                        pass
-                self.check_done()
+                if self._Q.qsize() == 0:
+                    self._empty.set()
+                    if not self.has_wip:
+                        self._done.set()
                 await self._Q.put(None)
                 return True
         return False
 
     async def put(self, item: T) -> None:
+        if item is None:
+            raise ValueError("Cannot add None to IterableQueue")
         async with self._put_lock:
             if self.is_filled:  # should this be inside put_lock?
                 raise QueueDone
             if self._producers <= 0:
                 raise ValueError("No registered producers")
-            elif item is None:
-                raise ValueError("Cannot add None to IterableQueue")
             await self._Q.put(item=item)
-            self._empty.clear()
         return None
 
     def put_nowait(self, item: T) -> None:
         """
         Experimental asyncio.Queue.put_nowait() implementation
         """
-        # raise NotImplementedError
         if self.is_filled:
             raise QueueDone
         if self._producers <= 0:
             raise ValueError("No registered producers")
-        elif item is None:
+        if item is None:
             raise ValueError("Cannot add None to IterableQueue")
         self._Q.put_nowait(item=item)
-        self._empty.clear()
         return None
 
     async def get(self) -> T:
         item = await self._Q.get()
         if item is None:
             self._empty.set()
+            if not self.has_wip:
+                self._done.set()
             self._Q.task_done()
-            self.check_done()
             async with self._put_lock:
                 await self._Q.put(None)
                 raise QueueDone
         else:
-            if self._Q.qsize() == 0:
-                self._empty.set()
-            async with self._modify:
-                self._wip += 1
+            self._wip += 1
             return item
 
     def get_nowait(self) -> T:
@@ -241,16 +237,15 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         item: T | None = self._Q.get_nowait()
         if item is None:
             self._empty.set()
+            if not self.has_wip:
+                self._done.set()
             self._Q.task_done()
-            self.check_done()
             try:
                 self._Q.put_nowait(None)
             except QueueFull:
                 pass
             raise QueueDone
         else:
-            if self._Q.qsize() == 0:
-                self._empty.set()
             self._wip += 1
             return item
 
@@ -260,7 +255,8 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         self._wip -= 1
         if self._wip < 0:
             raise ValueError("task_done() called more than tasks open")
-        self.check_done()
+        if self.is_filled and self._empty.is_set() and not self.has_wip:
+            self._done.set()
 
     async def join(self) -> None:
         debug("Waiting queue to be filled")
@@ -271,14 +267,18 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
         return None
 
     def __aiter__(self):
+        """
+        Return ASyncIterator to be able iterate the queue using async for
+        """
         return self
 
     async def __anext__(self) -> T:
-        async with self._modify:
-            if self._wip > 0:  # do not mark task_done() at first call
-                self.task_done()
+        """
+        Async iterator for IterableQueue
+        """
+        if self._wip > 0:  # do not mark task_done() at first call
+            self.task_done()
         try:
-            item = await self.get()
-            return item
+            return await self.get()
         except QueueDone:
             raise StopAsyncIteration

queutils-0.10.0/tests/test_awrap.py

@@ -0,0 +1,100 @@
+import pytest  # type: ignore
+from asyncio import (
+    Task,
+    create_task,
+)
+
+from queutils import IterableQueue, QueueDone, abatch, awrap
+
+
+async def _producer_int(
+    Q: IterableQueue[int], n: int, finish: bool = False, wait: float = 0
+) -> None:
+    await Q.add_producer(N=1)
+    try:
+        for i in range(n):
+            await Q.put(i)
+    except QueueDone:
+        raise ValueError("Queue is done even no one closed it")
+    await Q.finish_producer()
+    return None
+
+
+@pytest.mark.timeout(5)
+@pytest.mark.parametrize(
+    "N",
+    [
+        (int(10e4)),
+        (int(10e4)),
+    ],
+)
+@pytest.mark.asyncio
+async def test_1_awrap_iterable(N: int):
+    """Test for awrap() with AsyncIterable"""
+
+    iterable: list[int] = [i for i in range(N)]
+
+    max_elem: int = -1
+
+    async for elem in awrap(iterable):
+        assert elem > max_elem, "outcome"
+        max_elem = elem
+    assert max_elem == N - 1, f"last element is incorrect: {max_elem} != {N - 1}"
+
+
+@pytest.mark.timeout(5)
+@pytest.mark.parametrize(
+    "bsize, N",
+    [
+        (17, int(10e4)),
+        (89, int(10e4)),
+    ],
+)
+@pytest.mark.asyncio
+async def test_2_abatch_iterable(bsize: int, N: int):
+    """Test for abatch() with AsyncIterable"""
+
+    iterable: list[int] = [i for i in range(N)]
+
+    max_elem: int = -1
+    last: bool = False
+
+    async for batch in abatch(iterable, bsize):
+        assert not last, "abatch() returned mismatched batch that was not the last"
+        if len(batch) < bsize:
+            last = True
+        assert last or len(batch) == bsize, f"Batch size is {len(batch)} != {bsize}"
+        for elem in batch:
+            assert elem > max_elem, f"batch is not sorted: {batch}"
+            max_elem = elem
+
+
+@pytest.mark.timeout(10)
+@pytest.mark.parametrize(
+    "Qsize, bsize, N",
+    [
+        (101, 17, int(10e4)),
+        (11, 89, int(10e4)),
+    ],
+)
+@pytest.mark.asyncio
+async def test_3_abatch_asynciterable(Qsize: int, bsize: int, N: int):
+    """Test for abatch() with AsyncIterable"""
+
+    Q = IterableQueue[int](maxsize=Qsize)
+    producer: Task = create_task(_producer_int(Q, N))
+
+    max_elem: int = -1
+    last: bool = False
+
+    async for batch in abatch(Q, bsize):
+        assert not last, "abatch() returned mismatched batch that was not the last"
+        if len(batch) < bsize:
+            last = True
+        assert last or len(batch) == bsize, f"Batch size is {len(batch)} != {bsize}"
+        for elem in batch:
+            assert elem > max_elem, f"batch is not sorted: {batch}"
+            max_elem = elem
+    assert Q.empty(), f"Queue is not empty after async for: Q.size={Q.qsize()}"
+    assert Q.is_done, "Queue is not done"
+    producer.cancel()

{queutils-0.9.4 → queutils-0.10.0}/tests/test_eventcounterqueue.py

@@ -54,7 +54,7 @@ async def test_1_category_counter_queue(
             count: int = randint(1, 10)
             await Q.send(cat, count)
             _counter[cat] += count
-        await Q.finish()
+        await Q.finish_producer()
         return _counter
 
     senders: list[Task] = list()

{queutils-0.9.4 → queutils-0.10.0}/tests/test_iterablequeue.py

@@ -13,9 +13,10 @@ from random import random
 
 from queutils import IterableQueue, QueueDone
 
-QSIZE: int = 10
-N: int = 100  # N >> QSIZE
-THREADS: int = 4
+QSIZE: int = 37
+N: int = 500  # N >> QSIZE
+THREADS: int = 17
+TIMEOUT: int = 30
 # N : int = int(1e10)
 
 
@@ -36,7 +37,7 @@ async def _producer_int(
     except QueueDone:
         pass
     if finish:
-        await Q.finish()
+        await Q.finish_producer()
     return None
 
 
@@ -54,17 +55,19 @@ async def _consumer_int(Q: IterableQueue[int], n: int = -1, wait: float = 0) ->
     return True
 
 
-@pytest.mark.timeout(10)
+@pytest.mark.timeout(TIMEOUT)
 @pytest.mark.asyncio
 async def test_1_put_get_async(test_interablequeue_int: IterableQueue[int]):
     """Test: put(), get(), join(), qsize(), empty() == True"""
     Q = test_interablequeue_int
     try:
-        async with timeout(5):
+        async with timeout(TIMEOUT / 2):
             await _producer_int(Q, QSIZE - 1, finish=True)
     except TimeoutError:
         assert False, "IterableQueue got stuck"
-    assert Q.qsize() == QSIZE - 1, f"qsize() returned {Q.qsize()}, should be {QSIZE-1}"
+    assert Q.qsize() == QSIZE - 1, (
+        f"qsize() returned {Q.qsize()}, should be {QSIZE - 1}"
+    )
     try:
         await Q.put(1)
         assert False, "Queue is filled and put() should raise an exception"
@@ -73,10 +76,10 @@ async def test_1_put_get_async(test_interablequeue_int: IterableQueue[int]):
     assert not Q.is_done, "is_done returned True even queue is not finished"
     consumer: Task = create_task(_consumer_int(Q))
     try:
-        async with timeout(5):
+        async with timeout(TIMEOUT / 2):
             await Q.join()
         await Q.get()
-        assert False, "Queue is done and put() should raise an exception"
+        assert False, "Queue is done and get() should raise an exception"
     except TimeoutError:
         assert False, "IterableQueue.join() took too long"
     except QueueDone:
@@ -86,13 +89,13 @@ async def test_1_put_get_async(test_interablequeue_int: IterableQueue[int]):
     consumer.cancel()
 
 
-@pytest.mark.timeout(10)
+@pytest.mark.timeout(TIMEOUT)
 @pytest.mark.asyncio
 async def test_2_put_get_nowait(test_interablequeue_int: IterableQueue[int]):
     """Test put_nowait() and get_nowait() methods"""
     Q = test_interablequeue_int
     producer: Task = create_task(_producer_int(Q, N))
-    await sleep(1)
+    await sleep(N / 500)
     # In theory this could fail without a real error
     # if QSIZE is huge and/or system is slow
     assert Q.qsize() == Q.maxsize, "Queue was supposed to be at maxsize"
@@ -105,35 +108,44 @@ async def test_2_put_get_nowait(test_interablequeue_int: IterableQueue[int]):
     except QueueFull:
         pass  # OK, Queue was supposed to be full
 
+    finisher = create_task(Q.finish_producer(all=True))
+
     try:
         while True:
-            _ = Q.get_nowait()
-            Q.task_done()
-            await sleep(0.01)
-    except QueueEmpty:
-        assert Q.qsize() == 0, "Queue size should be zero"
+            try:
+                _ = Q.get_nowait()
+                Q.task_done()
+            except QueueEmpty:
+                assert Q.qsize() == 0, "Queue size should be zero"
+                await sleep(0.01)
+    except QueueDone:
+        pass
 
     try:
-        async with timeout(5):
-            await Q.finish()
+        async with timeout(3):
+            await Q.finish_producer()
             await Q.join()
     except TimeoutError:
         assert False, "Queue.join() took longer than it should"
-    assert Q.qsize() == 0, "queue size is > 0 even it should be empty"
+    assert Q.qsize() == 0, f"queue size is {Q.qsize()} > 0 even it should be empty"
     assert Q.empty(), "queue not empty()"
     producer.cancel()
+    finisher.cancel()
+    await sleep(0.1)
+    assert producer.done(), "producer has not finished"
+    assert finisher.done(), "finisher has not finished"
 
 
-@pytest.mark.timeout(10)
+@pytest.mark.timeout(TIMEOUT)
 @pytest.mark.asyncio
 async def test_3_multiple_producers(test_interablequeue_int: IterableQueue[int]):
     Q = test_interablequeue_int
     workers: list[Task] = list()
     for _ in range(THREADS):
-        workers.append(create_task(_producer_int(Q, N, finish=True, wait=0.05)))
+        workers.append(create_task(_producer_int(Q, N, finish=True, wait=0.01)))
     try:
         assert not Q.is_done, "is_done returned True even queue is not finished"
-        async with timeout(10):
+        async with timeout(TIMEOUT):
             async for _ in Q:
                 pass
     except TimeoutError:
@@ -147,7 +159,7 @@ async def test_3_multiple_producers(test_interablequeue_int: IterableQueue[int])
         w.cancel()
 
 
-@pytest.mark.timeout(10)
+@pytest.mark.timeout(TIMEOUT)
 @pytest.mark.asyncio
 async def test_4_multiple_producers_consumers(
     test_interablequeue_int: IterableQueue[int],
@@ -157,26 +169,26 @@ async def test_4_multiple_producers_consumers(
     consumers: list[Task] = list()
 
     for _ in range(THREADS):
-        producers.append(create_task(_producer_int(Q, N, finish=False, wait=0.05)))
-        consumers.append(create_task(_consumer_int(Q, 2 * N, wait=0.06)))
+        producers.append(create_task(_producer_int(Q, N, finish=False, wait=0.01)))
+        consumers.append(create_task(_consumer_int(Q, 2 * N, wait=0.01)))
     try:
-        async with timeout(10):
+        async with timeout(TIMEOUT):
             await gather(*producers)
-            await Q.finish(all=True)
+            await Q.finish_producer(all=True)
             await Q.join()
             assert not Q.has_wip, "Queue should not have any items WIP"
     except TimeoutError:
         assert False, "IterableQueue.join() took too long"
-    assert (
-        Q.count == THREADS * N
-    ), f"count returned wrong value {Q.count}, should be {THREADS*N}"
+    assert Q.count == THREADS * N, (
+        f"count returned wrong value {Q.count}, should be {THREADS * N}"
+    )
     assert Q.qsize() == 0, "queue size is > 0 even it should be empty"
     assert Q.empty(), "queue not empty"
     for p in consumers:
         p.cancel()
 
 
-@pytest.mark.timeout(10)
+@pytest.mark.timeout(TIMEOUT)
 @pytest.mark.asyncio
 async def test_5_empty_join(test_interablequeue_int: IterableQueue[int]):
     """Test for await join when an empty queue is finished"""
@@ -185,48 +197,52 @@ async def test_5_empty_join(test_interablequeue_int: IterableQueue[int]):
     assert not Q.is_done, "is_done returned True even queue is not finished"
     consumer: Task = create_task(_consumer_int(Q))
     try:
-        
         async with timeout(3):
             await Q.join()
-        assert (
-            Q.empty()
-        ), "Queue is done after 3 secs and the join() should finish before timeout(5)"
+        assert Q.empty(), (
+            "Queue is done after 3 secs and the join() should finish before timeout(5)"
+        )
     except TimeoutError:
         assert False, "await IterableQueue.join() failed with an empty queue finished"
     await sleep(0.1)
 
     try:
         consumer.cancel()
-        assert (
-            not consumer.cancelled()
-        ), "consumer task was cancelled and did not complete even it should have"
+        assert not consumer.cancelled(), (
+            "consumer task was cancelled and did not complete even it should have"
+        )
     except Exception as err:
         assert False, f"Unknown Exception caught: {err}"
     assert producer.done(), "producer has not finished"
 
 
-@pytest.mark.timeout(10)
+@pytest.mark.timeout(TIMEOUT)
 @pytest.mark.asyncio
 async def test_6_finish_full_queue(test_interablequeue_int: IterableQueue[int]):
     """Test for await join when an empty queue is finished"""
     Q = test_interablequeue_int
     producer: Task = create_task(_producer_int(Q, n=QSIZE * 2))
     try:
-        await sleep(0.5)
-        async with timeout(3):
-            await Q.finish(all=True, empty=True)
-        assert (
-            Q.empty()
-        ), f"Queue should be empty: qsize={Q._Q.qsize()}: {Q._Q.get_nowait()}, {Q._Q.get_nowait()}"
+        await sleep(1)
+        finisher: Task = create_task(Q.finish_producer(all=True))
+        async for _ in Q:
+            pass
+        assert Q.empty(), (
+            f"Queue should be empty: qsize={Q.qsize()}: {Q._Q.get_nowait()}, {Q._Q.get_nowait()}"
+        )
         assert Q.is_done, "Queue is not done"
     except TimeoutError:
-        assert False, "await IterableQueue.join() failed with an empty queue finished"
+        assert False, f"await IterableQueue.finish() failed. qsize={Q.qsize()}"
     await sleep(0.1)
     assert Q.is_done, "Queue is not done"
     producer.cancel()
+    finisher.cancel()
+    await sleep(0.1)
+    assert producer.done(), "producer has not finished"
+    assert finisher.done(), "finisher has not finished"
 
 
-@pytest.mark.timeout(10)
+@pytest.mark.timeout(TIMEOUT)
 @pytest.mark.asyncio
 async def test_7_aiter(test_interablequeue_int: IterableQueue[int]):
     """Test for await join when an empty queue is finished"""
@@ -245,7 +261,7 @@ async def test_7_aiter(test_interablequeue_int: IterableQueue[int]):
         assert False, "await IterableQueue.join() failed with an empty queue finished"
 
 
-@pytest.mark.timeout(10)
+@pytest.mark.timeout(TIMEOUT)
 @pytest.mark.asyncio
 async def test_8_aiter_1_item(test_interablequeue_int: IterableQueue[int]):
     """Test for await join when an empty queue is finished"""
@@ -260,8 +276,8 @@ async def test_8_aiter_1_item(test_interablequeue_int: IterableQueue[int]):
             count += 1
             assert i >= 0, "Did not receive an int"
         assert count == 1, f"Did not receive correct number of elements {count} != 1"
-        assert (
-            True
-        ), "Queue is done after 3 secs and the join() should finish before timeout(5)"
+        assert True, (
+            "Queue is done after 3 secs and the join() should finish before timeout(5)"
+        )
     except TimeoutError:
         assert False, "await IterableQueue.join() failed with an empty queue finished"