queutils 0.8.1__py3-none-any.whl

queutils/__init__.py

@@ -0,0 +1,11 @@
+from .countable import Countable as Countable
+from .asyncqueue import AsyncQueue as AsyncQueue
+from .iterablequeue import IterableQueue as IterableQueue, QueueDone as QueueDone
+from .filequeue import FileQueue as FileQueue
+
+__all__ = [
+    "asyncqueue",
+    "countable",
+    "filequeue",
+    "iterablequeue",
+]

queutils/asyncqueue.py

@@ -0,0 +1,112 @@
+# -----------------------------------------------------------
+#  Class AsyncQueue(asyncio.Queue, Generic[T])
+#
+#  asyncio wrapper for non-async queue.Queue. Can be used to create 
+#  an asyncio.Queue out of a (non-async) multiprocessing.Queue.
+#
+# -----------------------------------------------------------
+
+__author__ = "Jylpah"
+__copyright__ = "Copyright 2024, Jylpah <Jylpah@gmail.com>"
+__credits__ = ["Jylpah"]
+__license__ = "MIT"
+# __version__ = "1.0"
+__maintainer__ = "Jylpah"
+__email__ = "Jylpah@gmail.com"
+__status__ = "Production"
+
+
+from queue import Full, Empty, Queue
+from asyncio.queues import QueueEmpty, QueueFull
+import asyncio
+from typing import Generic, TypeVar
+import logging
+from asyncio import sleep
+
+T = TypeVar("T")
+
+logger = logging.getLogger(__name__)
+
+debug = logger.debug
+message = logger.warning
+verbose = logger.info
+error = logger.error
+
+
+class AsyncQueue(asyncio.Queue, Generic[T]):
+    """
+    Async wrapper for non-async queue.Queue. Can be used to create 
+    an asyncio.Queue out of a (non-async) multiprocessing.Queue.
+    """
+
+    def __init__(self, queue: Queue[T], asleep: float = 0.01):
+        self._Q: Queue[T] = queue
+        # self._maxsize: int = queue.maxsize
+        self._done: int = 0
+        self._items: int = 0
+        self._sleep: float = asleep
+
+
+    @property
+    def maxsize(self) -> int:
+        """not supported by queue.Queue"""
+        return self._Q.maxsize
+
+    async def get(self) -> T:
+        while True:
+            try:
+                return self.get_nowait()
+            except QueueEmpty:
+                await sleep(self._sleep)
+
+    def get_nowait(self) -> T:
+        try:
+            return self._Q.get_nowait()
+        except Empty:
+            raise QueueEmpty
+
+    async def put(self, item: T) -> None:
+        while True:
+            try:
+                return self.put_nowait(item)
+            except QueueFull:
+                await sleep(self._sleep)
+
+    def put_nowait(self, item: T) -> None:
+        try:
+            self._Q.put_nowait(item)
+            self._items += 1
+            return None
+        except Full:
+            raise QueueFull
+
+    async def join(self) -> None:
+        while True:
+            if self._Q.empty():
+                return None
+            else:
+                await sleep(self._sleep)
+
+    def task_done(self) -> None:
+        self._Q.task_done()
+        self._done += 1
+        return None
+
+    def qsize(self) -> int:
+        return self._Q.qsize()
+
+    @property
+    def done(self) -> int:
+        """Number of done items"""
+        return self._done
+
+    @property
+    def items(self) -> int:
+        """Number of items ever put to the queue"""
+        return self._items
+
+    def empty(self) -> bool:
+        return self._Q.empty()
+
+    def full(self) -> bool:
+        return self._Q.full()

queutils/countable.py

@@ -0,0 +1,11 @@
+
+
+from typing import Protocol
+
+class Countable(Protocol):
+    """"
+    Protocol interface for count @property
+    """
+    @property
+    def count(self) -> int:
+        ...

queutils/filequeue.py

@@ -0,0 +1,156 @@
+# -----------------------------------------------------------
+#  Class FileQueue(asyncio.Queue)
+#
+#  Class to build asyncio.Queue of filename (pathlib.Path) based on arguments given.
+#  - Inherits from queutils.IterableQueue
+#  - Supports include/exclude filters
+#
+# -----------------------------------------------------------
+
+__author__ = "Jylpah"
+__copyright__ = "Copyright 2024, Jylpah <Jylpah@gmail.com>"
+__credits__ = ["Jylpah"]
+__license__ = "MIT"
+# __version__ = "1.0"
+__maintainer__ = "Jylpah"
+__email__ = "Jylpah@gmail.com"
+__status__ = "Production"
+
+
+import logging
+
+# from asyncio import Queue
+import aioconsole  # type: ignore
+from fnmatch import fnmatch, fnmatchcase
+from pathlib import Path
+from typing import Optional, Sequence
+
+from .iterablequeue import IterableQueue
+
+logger = logging.getLogger(__name__)
+error = logger.error
+message = logger.warning
+verbose = logger.info
+debug = logger.debug
+
+
+def str2path(filename: str | Path, suffix: str | None = None) -> Path:
+    """
+    Convert filename (str) to pathlib.Path
+    """
+    if isinstance(filename, str):
+        filename = Path(filename)
+    if suffix is not None and not filename.name.lower().endswith(suffix):
+        filename = filename.with_suffix(suffix)
+    return filename
+
+
+class FileQueue(IterableQueue[Path]):
+    """
+    Class to create a IterableQueue(asyncio.Queue) of filenames based on 
+    given directories and files as arguments. 
+    Supports include/exclude filters based on filenames.
+    """
+
+    def __init__(
+        self,
+        base: Optional[Path] = None,
+        filter: str = "*",
+        exclude: bool = False,
+        case_sensitive: bool = True,
+        follow_symlinks: bool = False, 
+        **kwargs,
+    ):
+        assert base is None or isinstance(base, Path), "base has to be Path or None"
+        assert isinstance(filter, str), "filter has to be string"
+        assert isinstance(case_sensitive, bool), "case_sensitive has to be bool"
+        assert isinstance(follow_symlinks, bool), "follow_symlinks has to be bool"
+        
+        # debug(f"maxsize={str(maxsize)}, filter='{filter}'")
+        super().__init__(count_items=True, **kwargs)
+        self._base: Optional[Path] = base
+        # self._done: bool = False
+        self._case_sensitive: bool = False
+        self._exclude: bool = False
+        self._follow_symlinks : bool = follow_symlinks
+        self.set_filter(filter=filter, exclude=exclude, case_sensitive=case_sensitive)
+
+    def set_filter(
+        self,
+        filter: str = "*",
+        exclude: bool = False,
+        case_sensitive: bool = False,
+    ):
+        """set filtering logic. Only set (!= None) params are changed"""
+        assert isinstance(case_sensitive, bool), "case_sensitive must be type of bool"
+        assert isinstance(exclude, bool), "exclude must be type of bool"
+
+        self._case_sensitive = case_sensitive
+        self._exclude = exclude
+        self._filter = filter
+        debug(
+            "filter=%s exclude=%s, case_sensitive=%s",
+            str(self._filter),
+            self._exclude,
+            self._case_sensitive,
+        )
+
+    async def mk_queue(self, files: Sequence[str | Path]) -> bool:
+        """Create file queue from arguments given
+        '-' denotes for STDIN
+        """
+        assert files is not None and len(files) > 0, "No files given to process"
+        await self.add_producer()
+        path: Path
+        file: str | Path
+        try:
+            if isinstance(files[0], str) and files[0] == "-":
+                stdin, _ = await aioconsole.get_standard_streams()
+                while (line := await stdin.readline()) is not None:
+                    path = Path(line.decode("utf-8").removesuffix("\n"))
+                    if self._base is not None:
+                        path = self._base / path
+                    await self.put(path)
+            else:
+                for file in files:
+                    path = str2path(file)
+                    if self._base is not None:
+                        path = self._base / path
+                    await self.put(path)
+        except Exception as err:
+            error(f"{err}")
+        return await self.finish()
+
+
+    async def put(self, path: Path) -> None:
+        """Recursive function to build process queue. Sanitize filename"""
+        assert isinstance(path, Path), "path has to be type Path()"
+        try:
+            if path.is_symlink() and not self._follow_symlinks:
+                return None
+            if path.is_dir():
+                for child in path.iterdir():
+                    await self.put(child)
+            elif path.is_file() and self.match(path):
+                debug("Adding file to queue: %s", str(path))
+                await super().put(path)
+        except Exception as err:
+            error(f"{err}")
+        return None
+
+    def match(self, path: Path) -> bool:
+        """ "Match file name with filter
+
+        https://docs.python.org/3/library/fnmatch.html
+        """
+        assert isinstance(path, Path), "path has to be type Path()"
+        try:
+            m: bool
+            if self._case_sensitive:
+                m = fnmatch(path.name, self._filter)
+            else:
+                m = fnmatchcase(path.name, self._filter)
+            return m != self._exclude
+        except Exception as err:
+            error(f"{err}")
+        return False

queutils/iterablequeue.py

@@ -0,0 +1,283 @@
+# -----------------------------------------------------------
+#  Class IterableQueue(asyncio.Queue[T], AsyncIterable[T], Countable)
+#
+#  IterableQueue is asyncio.Queue subclass that can be iterated asynchronusly. 
+#  IterableQueue terminates automatically when the queue has been
+#  filled and emptied.
+#
+# -----------------------------------------------------------
+
+__author__ = "Jylpah"
+__copyright__ = "Copyright 2024, Jylpah <Jylpah@gmail.com>"
+__credits__ = ["Jylpah"]
+__license__ = "MIT"
+# __version__ = "1.0"
+__maintainer__ = "Jylpah"
+__email__ = "Jylpah@gmail.com"
+__status__ = "Production"
+
+
+from asyncio import Queue, QueueFull, QueueEmpty, Event, Lock
+from typing import AsyncIterable, TypeVar, Optional
+from .countable import Countable
+import logging
+
+# Setup logging
+logger = logging.getLogger()
+error = logger.error
+message = logger.warning
+verbose = logger.info
+debug = logger.debug
+
+T = TypeVar("T")
+
+class QueueDone(Exception):
+    """
+    Exception to mark an IterableQueue as filled and emptied i.e. done
+    """
+    pass
+
+
+class IterableQueue(Queue[T], AsyncIterable[T], Countable):
+    """
+    IterableQueue is asyncio.Queue subclass that can be iterated asynchronusly. 
+    
+    IterableQueue terminates automatically when the queue has been
+    filled and emptied. Supports:
+    - asyncio.Queue() interface, _nowait() methods are experimental
+    - AsyncIterable(): async for item in queue:
+    - Automatic termination of the consumers when the queue has been emptied with QueueDone exception
+    - Producers must be registered with add_producer() and they must notify the queue
+      with finish() once they have finished adding items 
+    - Countable interface to count number of items task_done() through 'count' property
+    - Countable property can be disabled with count_items=False. This is useful when you
+    want to sum the count of multiple IterableQueues
+    """
+
+    def __init__(self, count_items: bool = True, **kwargs):
+        # _Q is required instead of inheriting from Queue()
+        # using super() since Queue is Optional[T], not [T]
+        self._Q: Queue[Optional[T]] = Queue(**kwargs)
+        self._producers: int = 0
+        self._count_items: bool = count_items
+        self._count: int = 0
+        self._wip: int = 0
+
+        self._modify: Lock = Lock()
+        self._put_lock: Lock = Lock()
+
+        self._filled: Event = Event()
+        self._empty: Event = Event()
+        self._done: Event = Event()
+
+        self._empty.set()
+
+    @property
+    def is_filled(self) -> bool:
+        """"
+        True if all the producers finished filling the queue. 
+        New items cannot be added to a filled queue nor can new producers can be added.
+        """
+        return self._filled.is_set()
+    
+    @property
+    def is_done(self) -> bool:
+        """
+        Has the queue been filled, emptied and all the items have been marked with task_done() 
+        """
+        return self.is_filled and self.empty() and not self.has_wip
+
+    @property
+    def maxsize(self) -> int:
+        return self._Q.maxsize
+
+
+    def full(self) -> bool:
+        """
+        True if the queue is full
+        """
+        return self._Q.full()
+
+    def check_done(self) -> bool:
+        if self.is_filled and self.empty() and not self.has_wip:
+            self._done.set()
+            return True
+        return False
+
+    def empty(self) -> bool:
+        """
+        Queue has no items except None as a sentinel
+        """
+        return self._empty.is_set() or self.qsize() == 0
+
+    def qsize(self) -> int:
+        """
+        asyncio.Queue.qsize()
+        """
+        if self.is_filled:
+            return self._Q.qsize() - 1
+        else:
+            return self._Q.qsize()
+
+    @property
+    def wip(self) -> int:
+        """
+        Number of items in progress i.e. items that have been 
+        read from the queue, but not marked with task_done() 
+        """
+        return self._wip
+
+    @property
+    def has_wip(self) -> bool:
+        """
+        True if queue has items in progress
+        """
+        return self._wip > 0
+
+    @property
+    def count(self) -> int:
+        if self._count_items:
+            return self._count
+        else:
+            return 0
+
+    async def add_producer(self, N: int = 1) -> int:
+        """
+        Add producer(s) to the queue
+        """
+        assert N > 0, "N has to be positive"
+        async with self._modify:
+            if self.is_filled:
+                raise QueueDone
+            self._producers += N
+        return self._producers
+
+    async def finish(self, all: bool = False, empty: bool = False) -> bool:
+        """
+        Producer has finished adding items to the queue.
+        Once the last producers has finished, the queue is_filled.
+        - all: finish() queue for all producers at once
+        """
+        async with self._modify:
+            if self._producers <= 0 or self.is_filled:
+                # raise ValueError("finish() called more than the is producers")
+                self._producers = 0
+                return False
+            self._producers -= 1
+
+            if all or self._producers <= 0:
+                self._filled.set()
+                self._producers = 0
+
+        if self._producers == 0:
+            if empty:
+                try:
+                    while True:
+                        _ = self.get_nowait()
+                        self.task_done()
+                except (QueueDone, QueueEmpty):
+                    pass
+
+            async with self._put_lock:
+                if empty:
+                    try:
+                        _ = self.get_nowait()
+                        self.task_done()
+                    except (QueueDone, QueueEmpty):
+                        pass
+                self.check_done()
+                await self._Q.put(None)
+                return True
+        return False
+
+    async def put(self, item: T) -> None:
+        async with self._put_lock:
+            if self.is_filled:  # should this be inside put_lock?
+                raise QueueDone
+            if self._producers <= 0:
+                raise ValueError("No registered producers")
+            elif item is None:
+                raise ValueError("Cannot add None to IterableQueue")
+            await self._Q.put(item=item)
+            self._empty.clear()
+        return None
+
+    def put_nowait(self, item: T) -> None:
+        """
+        Experimental asyncio.Queue.put_nowait() implementation
+        """
+        # raise NotImplementedError
+        if self.is_filled:
+            raise QueueDone
+        if self._producers <= 0:
+            raise ValueError("No registered producers")
+        elif item is None:
+            raise ValueError("Cannot add None to IterableQueue")
+        self._Q.put_nowait(item=item)
+        self._empty.clear()
+        return None
+
+    async def get(self) -> T:
+        item = await self._Q.get()
+        if item is None:
+            self._empty.set()
+            self._Q.task_done()
+            self.check_done()
+            async with self._put_lock:
+                await self._Q.put(None)
+                raise QueueDone
+        else:
+            if self._Q.qsize() == 0:
+                self._empty.set()
+            async with self._modify:
+                self._wip += 1
+            return item
+
+    def get_nowait(self) -> T:
+        """
+        Experimental asyncio.Queue.get_nowait() implementation
+        """        
+        item: T | None = self._Q.get_nowait()
+        if item is None:
+            self._empty.set()
+            self._Q.task_done()
+            self.check_done()
+            try:
+                self._Q.put_nowait(None)
+            except QueueFull:
+                pass
+            raise QueueDone
+        else:
+            if self._Q.qsize() == 0:
+                self._empty.set()
+            self._wip += 1
+            return item
+
+    def task_done(self) -> None:
+        self._Q.task_done()
+        self._count += 1
+        self._wip -= 1
+        if self._wip < 0:
+            raise ValueError("task_done() called more than tasks open")
+        self.check_done()
+
+    async def join(self) -> None:
+        debug("Waiting queue to be filled")
+        await self._filled.wait()
+        debug("Queue filled, waiting when queue is done")
+        await self._done.wait()
+        debug("queue is done")
+        return None
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self) -> T:
+        async with self._modify:
+            if self._wip > 0:  # do not mark task_done() at first call
+                self.task_done()
+        try:
+            item = await self.get()
+            return item
+        except QueueDone:
+            raise StopAsyncIteration

queutils-0.8.1.dist-info/METADATA

@@ -0,0 +1,78 @@
+Metadata-Version: 2.3
+Name: queutils
+Version: 0.8.1
+Summary: Handy Python Queue utilies
+Project-URL: Homepage, https://github.com/Jylpah/queutils
+Project-URL: Bug Tracker, https://github.com/Jylpah/queutils/issues
+Author-email: Jylpah <jylpah@gmail.com>
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: AsyncIO
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.11
+Requires-Dist: aioconsole>=0.6
+Provides-Extra: dev
+Requires-Dist: build>=0.10; extra == 'dev'
+Requires-Dist: hatchling>=1.22.4; extra == 'dev'
+Requires-Dist: mypy>=1.8; extra == 'dev'
+Requires-Dist: pip-chill>=1.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest-cov>=4.1; extra == 'dev'
+Requires-Dist: pytest-datafiles>=3.0; extra == 'dev'
+Requires-Dist: pytest-timeout>=2.2; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.9; extra == 'dev'
+Description-Content-Type: text/markdown
+
+[![Python package](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml/badge.svg)](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml)  [![codecov](https://codecov.io/gh/Jylpah/queutils/graph/badge.svg?token=rMKdbfHOFs)](https://codecov.io/gh/Jylpah/queutils)
+
+# Queutils
+
+Queutils *[Queue Utils]* is a package if handy Python queue classes:
+- **[AsyncQueue](docs/asyncqueue.md)** - `async` wrapper for `queue.Queue`
+- **[IterableQueue](docs/iterablequeue.md)** - `AsyncIterable` queue
+- **[FileQueue](docs/filequeue.md)** - builds a queue of filenames from input
+
+
+# AsyncQueue
+
+[`AsyncQueue`](docs/asyncqueue.md) is a async wrapper for non-async `queue.Queue`. It can be used to create 
+an `asyncio.Queue` compatible out of a (non-async) `multiprocessing.Queue`. This is handy to have `async` code running in `multiprocessing` processes and yet be able to communicate with the parent via (non-async) managed `multiprocessing.Queue` queue. 
+
+## Features 
+
+- `asyncio.Queue` compatible
+- `queue.Queue` support
+- `multiprocessing.Queue` support
+
+
+# IterableQueue
+
+[`IterableQueue`](docs/iterablequeue.md) is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be 
+iterated in `async for` loop. `IterableQueue` terminates automatically when the queue has been filled and emptied. 
+    
+## Features
+
+- `asyncio.Queue` interface, `_nowait()` methods are experimental
+- `AsyncIterable` support: `async for item in queue:`
+- Automatic termination of the consumers with `QueueDone` exception when the queue has been emptied 
+- Producers must be registered with `add_producer()` and they must notify the queue
+  with `finish()` once they have finished adding items 
+- Countable interface to count number of items task_done() through `count` property
+- Countable property can be disabled with count_items=False. This is useful when you
+    want to sum the count of multiple IterableQueues 
+
+# FileQueue
+
+[`FileQueue`](docs/filequeue.md) builds a queue (`IterableQueue[pathlib.Path]`) of the matching files found based on search parameters given. It can search both list of files or directories or mixed. Async method `FileQueue.mk_queue()` searches subdirectories of given directories.  
+
+## Features
+
+- Input can be given both as `str` and `pathlib.Path`
+- `exclude: bool` exclusive or  inclusive filtering. Default is `False`.
+- `case_sensitive: bool` case sensitive filtering (use of `fnmatch` or `fnmatchcase`). Default is `True`.
+- `follow_symlinks: bool` whether to follow symlinks. Default is `False`.
+

queutils-0.8.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+queutils/__init__.py,sha256=MboAom80iQTQURz0liGv_f2ae2Np8YA5ORX1LwsNa50,311
+queutils/asyncqueue.py,sha256=cZPrXJgRCK0iOtE6k-VTzA9kcDDg818h9v0cwYzm2g0,2813
+queutils/countable.py,sha256=YSi7ILf9CuB5Tm3T4UUMEFlveqzqcmomfqJAlLGHEz8,172
+queutils/filequeue.py,sha256=q2ly9H-lSCq6xuOqT1IlWgyCVyLoZiKbc0NuzmkF4aw,5360
+queutils/iterablequeue.py,sha256=hygOtgRRuqaPebcK27Ixm_KOOW0MnlKmBNcYreQoWOA,8677
+queutils-0.8.1.dist-info/METADATA,sha256=IcC2JgXacCtVK-NWXMP5T1ehHqJVWzDMuaBr93tvS-0,3694
+queutils-0.8.1.dist-info/WHEEL,sha256=uNdcs2TADwSd5pVaP0Z_kcjcvvTUklh2S7bxZMF8Uj0,87
+queutils-0.8.1.dist-info/licenses/LICENSE,sha256=J1zeIKU2JVQmhwO2hHQDK8WR6zjVZ-wX8r7ZlL45AbI,1063
+queutils-0.8.1.dist-info/RECORD,,

queutils-0.8.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.22.4
+Root-Is-Purelib: true
+Tag: py3-none-any

queutils-0.8.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Jylpah
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.