queutils 0.8.1__tar.gz

queutils-0.8.1/.github/workflows/python-package.yml

@@ -0,0 +1,45 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install .
+        python -m pip install ".[dev]" 
+    - name: Lint with ruff
+      run: | 
+        ruff check src tests
+    - name: Test with mypy
+      run: |
+        mypy src
+    - name: Test with pytest
+      run: |
+        pytest --cov=src tests
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v4
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

queutils-0.8.1/.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+.pypirc
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# VS Code
+*.code-workspace
+.vscode/
+
+# Tmp
+tmp/
+*.swp

queutils-0.8.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Jylpah
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

queutils-0.8.1/PKG-INFO

@@ -0,0 +1,78 @@
+Metadata-Version: 2.3
+Name: queutils
+Version: 0.8.1
+Summary: Handy Python Queue utilies
+Project-URL: Homepage, https://github.com/Jylpah/queutils
+Project-URL: Bug Tracker, https://github.com/Jylpah/queutils/issues
+Author-email: Jylpah <jylpah@gmail.com>
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: AsyncIO
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.11
+Requires-Dist: aioconsole>=0.6
+Provides-Extra: dev
+Requires-Dist: build>=0.10; extra == 'dev'
+Requires-Dist: hatchling>=1.22.4; extra == 'dev'
+Requires-Dist: mypy>=1.8; extra == 'dev'
+Requires-Dist: pip-chill>=1.0; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest-cov>=4.1; extra == 'dev'
+Requires-Dist: pytest-datafiles>=3.0; extra == 'dev'
+Requires-Dist: pytest-timeout>=2.2; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.9; extra == 'dev'
+Description-Content-Type: text/markdown
+
+[![Python package](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml/badge.svg)](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml)  [![codecov](https://codecov.io/gh/Jylpah/queutils/graph/badge.svg?token=rMKdbfHOFs)](https://codecov.io/gh/Jylpah/queutils)
+
+# Queutils
+
+Queutils *[Queue Utils]* is a package if handy Python queue classes:
+- **[AsyncQueue](docs/asyncqueue.md)** - `async` wrapper for `queue.Queue`
+- **[IterableQueue](docs/iterablequeue.md)** - `AsyncIterable` queue
+- **[FileQueue](docs/filequeue.md)** - builds a queue of filenames from input
+
+
+# AsyncQueue
+
+[`AsyncQueue`](docs/asyncqueue.md) is a async wrapper for non-async `queue.Queue`. It can be used to create 
+an `asyncio.Queue` compatible out of a (non-async) `multiprocessing.Queue`. This is handy to have `async` code running in `multiprocessing` processes and yet be able to communicate with the parent via (non-async) managed `multiprocessing.Queue` queue. 
+
+## Features 
+
+- `asyncio.Queue` compatible
+- `queue.Queue` support
+- `multiprocessing.Queue` support
+
+
+# IterableQueue
+
+[`IterableQueue`](docs/iterablequeue.md) is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be 
+iterated in `async for` loop. `IterableQueue` terminates automatically when the queue has been filled and emptied. 
+    
+## Features
+
+- `asyncio.Queue` interface, `_nowait()` methods are experimental
+- `AsyncIterable` support: `async for item in queue:`
+- Automatic termination of the consumers with `QueueDone` exception when the queue has been emptied 
+- Producers must be registered with `add_producer()` and they must notify the queue
+  with `finish()` once they have finished adding items 
+- Countable interface to count number of items task_done() through `count` property
+- Countable property can be disabled with count_items=False. This is useful when you
+    want to sum the count of multiple IterableQueues 
+
+# FileQueue
+
+[`FileQueue`](docs/filequeue.md) builds a queue (`IterableQueue[pathlib.Path]`) of the matching files found based on search parameters given. It can search both list of files or directories or mixed. Async method `FileQueue.mk_queue()` searches subdirectories of given directories.  
+
+## Features
+
+- Input can be given both as `str` and `pathlib.Path`
+- `exclude: bool` exclusive or  inclusive filtering. Default is `False`.
+- `case_sensitive: bool` case sensitive filtering (use of `fnmatch` or `fnmatchcase`). Default is `True`.
+- `follow_symlinks: bool` whether to follow symlinks. Default is `False`.
+

queutils-0.8.1/README.md

@@ -0,0 +1,49 @@
+[![Python package](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml/badge.svg)](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml)  [![codecov](https://codecov.io/gh/Jylpah/queutils/graph/badge.svg?token=rMKdbfHOFs)](https://codecov.io/gh/Jylpah/queutils)
+
+# Queutils
+
+Queutils *[Queue Utils]* is a package if handy Python queue classes:
+- **[AsyncQueue](docs/asyncqueue.md)** - `async` wrapper for `queue.Queue`
+- **[IterableQueue](docs/iterablequeue.md)** - `AsyncIterable` queue
+- **[FileQueue](docs/filequeue.md)** - builds a queue of filenames from input
+
+
+# AsyncQueue
+
+[`AsyncQueue`](docs/asyncqueue.md) is a async wrapper for non-async `queue.Queue`. It can be used to create 
+an `asyncio.Queue` compatible out of a (non-async) `multiprocessing.Queue`. This is handy to have `async` code running in `multiprocessing` processes and yet be able to communicate with the parent via (non-async) managed `multiprocessing.Queue` queue. 
+
+## Features 
+
+- `asyncio.Queue` compatible
+- `queue.Queue` support
+- `multiprocessing.Queue` support
+
+
+# IterableQueue
+
+[`IterableQueue`](docs/iterablequeue.md) is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be 
+iterated in `async for` loop. `IterableQueue` terminates automatically when the queue has been filled and emptied. 
+    
+## Features
+
+- `asyncio.Queue` interface, `_nowait()` methods are experimental
+- `AsyncIterable` support: `async for item in queue:`
+- Automatic termination of the consumers with `QueueDone` exception when the queue has been emptied 
+- Producers must be registered with `add_producer()` and they must notify the queue
+  with `finish()` once they have finished adding items 
+- Countable interface to count number of items task_done() through `count` property
+- Countable property can be disabled with count_items=False. This is useful when you
+    want to sum the count of multiple IterableQueues 
+
+# FileQueue
+
+[`FileQueue`](docs/filequeue.md) builds a queue (`IterableQueue[pathlib.Path]`) of the matching files found based on search parameters given. It can search both list of files or directories or mixed. Async method `FileQueue.mk_queue()` searches subdirectories of given directories.  
+
+## Features
+
+- Input can be given both as `str` and `pathlib.Path`
+- `exclude: bool` exclusive or  inclusive filtering. Default is `False`.
+- `case_sensitive: bool` case sensitive filtering (use of `fnmatch` or `fnmatchcase`). Default is `True`.
+- `follow_symlinks: bool` whether to follow symlinks. Default is `False`.
+

queutils-0.8.1/codecov.yml

@@ -0,0 +1,9 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+        threshold: 5%
+    patch:
+      default:
+        informational: true

queutils-0.8.1/demos/asyncqueue_demo.py

@@ -0,0 +1,90 @@
+from multiprocessing import Manager
+from multiprocessing.pool import Pool, AsyncResult
+from queutils import AsyncQueue
+import queue
+from asyncio import run, sleep, CancelledError, create_task, Task, gather
+import asyncio
+from random import random, sample
+from typing import List
+from time import time
+
+start : float = time()
+
+def since() -> float:
+    return time() - start
+
+asyncQ : AsyncQueue
+
+async def main() -> None:
+
+    with Manager() as manager:
+        inQ: queue.Queue[int] = manager.Queue(maxsize=5)
+        ainQ: AsyncQueue[int] = AsyncQueue(inQ)
+        reader : Task[int] = create_task(consumer(ainQ))
+        N_process : int = 3
+        count : int = 0
+        work : List[int] = list(sample(range(3,7), N_process))
+        print(f"{N_process} producers will put {', '.join([str(i) for i in  work])} items to the queue")
+        with Pool(
+            processes=N_process,
+            initializer=producer_init,
+            initargs=[inQ],
+        ) as pool:
+            results: AsyncResult = pool.map_async(producer, work) 
+            pool.close()
+            
+            while not results.ready():
+                await sleep(0.2)
+                print(f"{since():.3f} waiting")
+            await ainQ.join()
+            reader.cancel()
+
+            for res in results.get():
+                count += res
+
+        print(f"{since():.3f} :: {N_process} producers added {count} items to the queue")
+        reader_count : int = 0
+        for res in await gather(*[reader]):
+            reader_count += res
+        print(f"{since():.3f} :: reader read {reader_count} items from the queue")
+
+
+def  producer_init(outQ: queue.Queue[int]):
+    global asyncQ
+    asyncQ = AsyncQueue(outQ)
+
+
+def producer(N :int) -> int:
+    return run(producer_async(N), debug=True)
+
+async def  producer_async(N: int) -> int:
+    """
+    async function to add N items to the shared queue
+    """
+    global asyncQ
+    count : int = 0
+    for i in range(N):
+        await sleep(0.1 * random())   
+        await asyncQ.put(i)
+        print(f"{since():.3f} put {i} to the queue")
+        count += 1
+    return count
+
+async def consumer(inQ: asyncio.queues.Queue[int]) -> int:
+    """
+    Async consumer of an async queue 
+    """
+    try:
+        count : int = 0
+        while True:
+            await sleep(0.1 * random())
+            print(f"{since():.3f} awating to read the queue")
+            i : int =  await inQ.get()
+            print(f"{since():.3f} read {i} from the queue")
+            count += 1
+    except CancelledError:
+        pass
+    return count
+
+if __name__ == "__main__":
+    run(main())

queutils-0.8.1/demos/filequeue_demo.py

@@ -0,0 +1,22 @@
+from queutils import FileQueue
+from pathlib import Path
+from asyncio import Task, create_task, run
+
+async def main() -> None:
+    fileQ = FileQueue(filter="*.py",  case_sensitive=False)
+    current_dir = Path(__file__).parent
+    spider : Task = create_task(fileQ.mk_queue(files=[current_dir]))
+    async for filename in fileQ:
+        try:
+            rel_path : Path = filename.relative_to(current_dir)
+            print(f"found {rel_path}")
+        except ValueError as err:
+            print(f"{err}")
+    
+    if spider.done():
+        print("finished, no need to use fileQ.join()")
+    else:
+        print("Oops, it did not work as promised")
+
+if __name__ == "__main__":
+    run(main())

queutils-0.8.1/demos/iterablequeue_demo.py

@@ -0,0 +1,53 @@
+from asyncio import sleep, run, TaskGroup
+from random import random
+from queutils import IterableQueue, QueueDone
+from time import time
+
+start : float = time()
+
+def since() -> float:
+    return time() - start
+
+async def producer(
+    Q: IterableQueue[int], N: int, id: int
+) -> None:
+    """
+    Fill the queue with N items
+    """
+    await Q.add_producer()
+    try:
+        for i in range(N):
+            await sleep(0.5 * random())
+            print(f"{since():.2f} producer {id}: awaiting to put {i} to queue")
+            await Q.put(i)
+            print(f"{since():.2f} producer {id}: put {i} to queue")
+        await Q.finish()
+    except QueueDone:
+        print(f"ERROR: producer {id}, this should not happen")
+    return None
+
+async def consumer(Q: IterableQueue[int], id: int = 1):
+    """
+    Consume the queue
+    """
+    async for i in Q:
+        print(f"{since():.2f} consumer {id}: got {i} from queue")
+        await sleep(0.5 * random())
+    print(f"{since():.2f} consumer {id}: queue is done")
+
+async def main() -> None:
+    """
+    Create a queue with maxsize and have multiple producers to fill it and 
+    multiple consumers to consume it over async for loop
+    """
+    queue : IterableQueue[int] = IterableQueue(maxsize=5)
+
+    async with TaskGroup() as tg:
+        for i in range(1,3):
+            tg.create_task(producer(Q=queue, N=5, id=i))
+        await sleep(2)
+        for  i in range(1,4):
+            tg.create_task(consumer(Q=queue, id=i))
+
+if __name__ == "__main__":
+    run(main())

queutils-0.8.1/docs/asyncqueue.md

@@ -0,0 +1,183 @@
+# AsyncQueue
+
+`AsyncQueue` is a async wrapper for non-async `queue.Queue`. It can be used to create 
+an `asyncio.Queue` compatible out of a (non-async) `multiprocessing.Queue`. This is handy to have `async` code running in `multiprocessing` processes and yet be able to communicate with the parent via (non-async) managed `multiprocessing.Queue` queue. 
+
+
+## Features 
+
+- `asyncio.Queue` compatible
+- `queue.Queue` support
+- `multiprocessing.Queue` support
+  
+## Example
+
+### Create async wrapper
+
+```python
+import queue
+from queutils import AsyncQueue
+
+syncQ: queue.Queue[int] = queue.Queue(maxsize=5)
+asyncQ: AsyncQueue[int] = AsyncQueue(syncQ)
+
+# asyncQ can not be used as any asyncio.Queue
+```
+
+### Full example
+
+Below is example code where a `multiprocessing.Manager.queue` is used to communicate between three child producer processes and a parent process that reads the queue. The code in both the parent and the child processes is `async`.
+
+```python
+from multiprocessing import Manager
+from multiprocessing.pool import Pool, AsyncResult
+from queutils import AsyncQueue
+import queue
+from asyncio import run, sleep, CancelledError, create_task, Task, gather
+import asyncio
+from random import random, sample
+from typing import List
+from time import time
+
+start : float = time()
+
+def since() -> float:
+    return time() - start
+
+asyncQ : AsyncQueue
+
+async def main() -> None:
+    """
+    Creates a multiprocessing.pool.Pool inside a multiprocessing.Manager() context manager and 
+    uses multiprocessing.Manager.Queue() to communicate between the parent process (consumer) 
+    and three child processes (producers). 
+    """
+
+    with Manager() as manager:
+        inQ: queue.Queue[int] = manager.Queue(maxsize=5)
+        ainQ: AsyncQueue[int] = AsyncQueue(inQ)
+        reader : Task[int] = create_task(consumer(ainQ))
+        N_process : int = 3
+        count : int = 0
+        work : List[int] = list(sample(range(3,7), N_process))
+        print(f"work to do {work}")
+        with Pool(
+            processes=N_process,
+            initializer=producer_init,
+            initargs=[inQ],
+        ) as pool:
+            results: AsyncResult = pool.map_async(producer, work) 
+            pool.close()
+            
+            # wait the child processes to finish
+            while not results.ready():
+                await sleep(0.2)
+                print(f"{since():.3f} waiting")
+            await ainQ.join()
+            reader.cancel()
+
+            for res in results.get():
+                count += res
+
+        print(f"{since():.3f} :: {N_process} producers added {count} items to the queue")
+        reader_count : int = 0
+        for res in await gather(*[reader]):
+            reader_count += res
+        print(f"{since():.3f} :: reader read {reader_count} items from the queue")
+
+
+def  producer_init(outQ: queue.Queue[int]):
+    """
+    multiprocessing init function to assing the created AsyncQueue to a global variable that is 
+    then available in the forked child process
+    """
+    global asyncQ
+    asyncQ = AsyncQueue(outQ)
+
+
+def producer(N :int) -> int:
+    return run(producer_async(N), debug=True)
+
+async def  producer_async(N: int) -> int:
+    """
+    async function to add N items to the shared queue
+    """
+    global asyncQ
+    count : int = 0
+    for i in range(N):
+        await sleep(0.1 * random())   
+        await asyncQ.put(i)
+        print(f"{since():.3f} put {i} to the queue")
+        count += 1
+    return count
+
+async def consumer(inQ: asyncio.queues.Queue[int]) -> int:
+    """
+    Async consumer of an async queue 
+    """
+    try:
+        count : int = 0
+        while True:
+            await sleep(0.1 * random())
+            print(f"{since():.3f} awating to read the queue")
+            i : int =  await inQ.get()
+            print(f"{since():.3f} read {i} from the queue")
+            count += 1
+    except CancelledError:
+        pass
+    return count
+
+if __name__ == "__main__":
+    run(main())
+```
+
+## Results
+
+```bash
+% python asyncqueue_demo.py                                                                queutils/demos
+3 producers will put 3, 6, 4 items to the queue
+0.039 awating to read the queue
+0.089 put 0 to the queue
+0.092 read 0 from the queue
+0.100 put 0 to the queue
+0.108 put 0 to the queue
+0.108 put 1 to the queue
+0.155 put 1 to the queue
+0.158 put 2 to the queue
+0.192 awating to read the queue
+0.193 read 0 from the queue
+0.196 put 3 to the queue
+0.203 awating to read the queue
+0.203 read 0 from the queue
+0.210 waiting
+0.210 put 1 to the queue
+0.285 awating to read the queue
+0.286 read 1 from the queue
+0.293 put 2 to the queue
+0.348 awating to read the queue
+0.349 read 1 from the queue
+0.350 put 3 to the queue
+0.411 waiting
+0.425 awating to read the queue
+0.426 read 2 from the queue
+0.427 put 2 to the queue
+0.466 awating to read the queue
+0.466 read 3 from the queue
+0.475 put 4 to the queue
+0.486 awating to read the queue
+0.486 read 1 from the queue
+0.521 awating to read the queue
+0.522 read 2 from the queue
+0.539 awating to read the queue
+0.540 read 3 from the queue
+0.567 put 5 to the queue
+0.613 waiting
+0.634 awating to read the queue
+0.634 read 2 from the queue
+0.655 awating to read the queue
+0.655 read 4 from the queue
+0.690 awating to read the queue
+0.691 read 5 from the queue
+0.697 :: 3 producers added 13 items to the queue
+0.697 :: reader read 13 items from the queue
+```