queutils 0.8.5__tar.gz → 0.9.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. queutils-0.9.3/.github/workflows/codeql.yml +100 -0
  2. queutils-0.9.3/.github/workflows/python-publish.yml +39 -0
  3. {queutils-0.8.5 → queutils-0.9.3}/PKG-INFO +22 -8
  4. {queutils-0.8.5 → queutils-0.9.3}/README.md +11 -0
  5. {queutils-0.8.5 → queutils-0.9.3}/docs/filequeue.md +14 -0
  6. queutils-0.9.3/docs/rm_links +14 -0
  7. queutils-0.9.3/pypi.md +69 -0
  8. {queutils-0.8.5 → queutils-0.9.3}/pyproject.toml +8 -4
  9. {queutils-0.8.5 → queutils-0.9.3}/src/queutils/__init__.py +5 -0
  10. queutils-0.9.3/src/queutils/eventcounterqueue.py +116 -0
  11. queutils-0.9.3/tests/test_eventcounterqueue.py +80 -0
  12. queutils-0.8.5/.github/workflows/python-publish.yml +0 -39
  13. {queutils-0.8.5 → queutils-0.9.3}/.github/workflows/dependency-review.yml +0 -0
  14. {queutils-0.8.5 → queutils-0.9.3}/.github/workflows/python-package.yml +0 -0
  15. {queutils-0.8.5 → queutils-0.9.3}/.gitignore +0 -0
  16. {queutils-0.8.5 → queutils-0.9.3}/LICENSE +0 -0
  17. {queutils-0.8.5 → queutils-0.9.3}/codecov.yml +0 -0
  18. {queutils-0.8.5 → queutils-0.9.3}/demos/asyncqueue_demo.py +0 -0
  19. {queutils-0.8.5 → queutils-0.9.3}/demos/filequeue_demo.py +0 -0
  20. {queutils-0.8.5 → queutils-0.9.3}/demos/iterablequeue_demo.py +0 -0
  21. {queutils-0.8.5 → queutils-0.9.3}/docs/asyncqueue.md +0 -0
  22. {queutils-0.8.5 → queutils-0.9.3}/docs/iterablequeue.md +0 -0
  23. {queutils-0.8.5 → queutils-0.9.3}/src/queutils/asyncqueue.py +0 -0
  24. {queutils-0.8.5 → queutils-0.9.3}/src/queutils/countable.py +0 -0
  25. {queutils-0.8.5 → queutils-0.9.3}/src/queutils/filequeue.py +0 -0
  26. {queutils-0.8.5 → queutils-0.9.3}/src/queutils/iterablequeue.py +0 -0
  27. {queutils-0.8.5 → queutils-0.9.3}/src/queutils/py.typed +0 -0
  28. {queutils-0.8.5 → queutils-0.9.3}/tests/test_asyncqueue.py +0 -0
  29. {queutils-0.8.5 → queutils-0.9.3}/tests/test_demos.py +0 -0
  30. {queutils-0.8.5 → queutils-0.9.3}/tests/test_filequeue.py +0 -0
  31. {queutils-0.8.5 → queutils-0.9.3}/tests/test_iterablequeue.py +0 -0
@@ -0,0 +1,100 @@
1
+ # For most projects, this workflow file will not need changing; you simply need
2
+ # to commit it to your repository.
3
+ #
4
+ # You may wish to alter this file to override the set of languages analyzed,
5
+ # or to provide custom queries or build logic.
6
+ #
7
+ # ******** NOTE ********
8
+ # We have attempted to detect the languages in your repository. Please check
9
+ # the `language` matrix defined below to confirm you have the correct set of
10
+ # supported CodeQL languages.
11
+ #
12
+ name: "CodeQL Advanced"
13
+
14
+ on:
15
+ push:
16
+ branches: [ "main" ]
17
+ pull_request:
18
+ branches: [ "main" ]
19
+ schedule:
20
+ - cron: '39 23 8 * *'
21
+
22
+ jobs:
23
+ analyze:
24
+ name: Analyze (${{ matrix.language }})
25
+ # Runner size impacts CodeQL analysis time. To learn more, please see:
26
+ # - https://gh.io/recommended-hardware-resources-for-running-codeql
27
+ # - https://gh.io/supported-runners-and-hardware-resources
28
+ # - https://gh.io/using-larger-runners (GitHub.com only)
29
+ # Consider using larger runners or machines with greater resources for possible analysis time improvements.
30
+ runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
31
+ permissions:
32
+ # required for all workflows
33
+ security-events: write
34
+
35
+ # required to fetch internal or private CodeQL packs
36
+ packages: read
37
+
38
+ # only required for workflows in private repositories
39
+ actions: read
40
+ contents: read
41
+
42
+ strategy:
43
+ fail-fast: false
44
+ matrix:
45
+ include:
46
+ - language: actions
47
+ build-mode: none
48
+ - language: python
49
+ build-mode: none
50
+ # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
51
+ # Use `c-cpp` to analyze code written in C, C++ or both
52
+ # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
53
+ # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
54
+ # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
55
+ # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
56
+ # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
57
+ # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
58
+ steps:
59
+ - name: Checkout repository
60
+ uses: actions/checkout@v4
61
+
62
+ # Add any setup steps before running the `github/codeql-action/init` action.
63
+ # This includes steps like installing compilers or runtimes (`actions/setup-node`
64
+ # or others). This is typically only required for manual builds.
65
+ # - name: Setup runtime (example)
66
+ # uses: actions/setup-example@v1
67
+
68
+ # Initializes the CodeQL tools for scanning.
69
+ - name: Initialize CodeQL
70
+ uses: github/codeql-action/init@v3
71
+ with:
72
+ languages: ${{ matrix.language }}
73
+ build-mode: ${{ matrix.build-mode }}
74
+ # If you wish to specify custom queries, you can do so here or in a config file.
75
+ # By default, queries listed here will override any specified in a config file.
76
+ # Prefix the list here with "+" to use these queries and those in the config file.
77
+
78
+ # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
79
+ # queries: security-extended,security-and-quality
80
+
81
+ # If the analyze step fails for one of the languages you are analyzing with
82
+ # "We were unable to automatically build your code", modify the matrix above
83
+ # to set the build mode to "manual" for that language. Then modify this step
84
+ # to build your code.
85
+ # ℹ️ Command-line programs to run using the OS shell.
86
+ # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
87
+ - if: matrix.build-mode == 'manual'
88
+ shell: bash
89
+ run: |
90
+ echo 'If you are using a "manual" build mode for one or more of the' \
91
+ 'languages you are analyzing, replace this with the commands to build' \
92
+ 'your code, for example:'
93
+ echo ' make bootstrap'
94
+ echo ' make release'
95
+ exit 1
96
+
97
+ - name: Perform CodeQL Analysis
98
+ uses: github/codeql-action/analyze@v3
99
+ with:
100
+ category: "/language:${{matrix.language}}"
@@ -0,0 +1,39 @@
1
+ # This workflow will upload a Python Package using Twine when a release is created
2
+ # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
3
+
4
+ # This workflow uses actions that are not certified by GitHub.
5
+ # They are provided by a third-party and are governed by
6
+ # separate terms of service, privacy policy, and support
7
+ # documentation.
8
+
9
+ name: Upload Python Package
10
+
11
+ on:
12
+ release:
13
+ types: [published]
14
+
15
+ jobs:
16
+ deploy:
17
+ runs-on: ubuntu-latest
18
+ environment: production
19
+ permissions:
20
+ id-token: write
21
+ contents: read
22
+
23
+ steps:
24
+ - uses: actions/checkout@v4
25
+ - name: Set up Python
26
+ uses: actions/setup-python@v5
27
+ with:
28
+ python-version: "3.12"
29
+ - name: Install dependencies
30
+ run: |
31
+ python -m pip install --upgrade pip
32
+ pip install build
33
+ - name: Build package
34
+ run: python -m build
35
+ - name: Publish package
36
+ uses: pypa/gh-action-pypi-publish@v1.12.4
37
+ # with:
38
+ # user: __token__
39
+ # password: ${{ secrets.PYPI_API_TOKEN }}
@@ -1,10 +1,11 @@
1
- Metadata-Version: 2.3
1
+ Metadata-Version: 2.4
2
2
  Name: queutils
3
- Version: 0.8.5
3
+ Version: 0.9.3
4
4
  Summary: Handy Python Queue utilies
5
5
  Project-URL: Homepage, https://github.com/Jylpah/queutils
6
6
  Project-URL: Bug Tracker, https://github.com/Jylpah/queutils/issues
7
7
  Author-email: Jylpah <jylpah@gmail.com>
8
+ License-File: LICENSE
8
9
  Classifier: Development Status :: 4 - Beta
9
10
  Classifier: Framework :: AsyncIO
10
11
  Classifier: License :: OSI Approved :: MIT License
@@ -13,6 +14,7 @@ Classifier: Programming Language :: Python :: 3
13
14
  Classifier: Topic :: Software Development :: Libraries
14
15
  Requires-Python: >=3.11
15
16
  Requires-Dist: aioconsole>=0.6
17
+ Requires-Dist: deprecated>=1.2.18
16
18
  Provides-Extra: dev
17
19
  Requires-Dist: build>=0.10; extra == 'dev'
18
20
  Requires-Dist: hatchling>=1.22.4; extra == 'dev'
@@ -24,6 +26,7 @@ Requires-Dist: pytest-datafiles>=3.0; extra == 'dev'
24
26
  Requires-Dist: pytest-timeout>=2.2; extra == 'dev'
25
27
  Requires-Dist: pytest>=8.0; extra == 'dev'
26
28
  Requires-Dist: ruff>=0.1.9; extra == 'dev'
29
+ Requires-Dist: types-deprecated>=1.2.15; extra == 'dev'
27
30
  Description-Content-Type: text/markdown
28
31
 
29
32
  [![Python package](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml/badge.svg)](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml) [![codecov](https://codecov.io/gh/Jylpah/queutils/graph/badge.svg?token=rMKdbfHOFs)](https://codecov.io/gh/Jylpah/queutils)
@@ -32,14 +35,15 @@ Description-Content-Type: text/markdown
32
35
 
33
36
  Queutils *[Queue Utils]* is a package of handy Python queue classes:
34
37
 
35
- - **[AsyncQueue](docs/asyncqueue.md)** - An `async` wrapper for non-async `queue.Queue`
36
- - **[IterableQueue](docs/iterablequeue.md)** - An `AsyncIterable` queue that terminates when finished
37
- - **[FileQueue](docs/filequeue.md)** - Builds an iterable queue of filenames from files/dirs given as input
38
+ - **AsyncQueue** - An `async` wrapper for non-async `queue.Queue`
39
+ - **IterableQueue** - An `AsyncIterable` queue that terminates when finished
40
+ - **EventCounterQueue** - An `IterableQueue` for counting events in `async` threads
41
+ - **FileQueue** - Builds an iterable queue of filenames from files/dirs given as input
38
42
 
39
43
 
40
44
  # AsyncQueue
41
45
 
42
- [`AsyncQueue`](docs/asyncqueue.md) is a async wrapper for non-async `queue.Queue`. It can be used to create
46
+ `AsyncQueue` is a async wrapper for non-async `queue.Queue`. It can be used to create
43
47
  an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessing.Queue` and thus enable `async` code in parent/child processes to communicate over `multiprocessing.Queue` as it were an `asyncio.Queue`.
44
48
 
45
49
  ## Features
@@ -51,7 +55,7 @@ an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessin
51
55
 
52
56
  # IterableQueue
53
57
 
54
- [`IterableQueue`](docs/iterablequeue.md) is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be
58
+ `IterableQueue` is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be
55
59
  iterated in `async for` loop. `IterableQueue` terminates automatically when the queue has been filled and emptied.
56
60
 
57
61
  The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves and it
@@ -71,9 +75,19 @@ producers are "finished", the queue enters into "filled" state and no new items
71
75
  - Countable property can be disabled with count_items=False. This is useful when you
72
76
  want to sum the count of multiple IterableQueues
73
77
 
78
+ # EventCounterQueue
79
+
80
+ `EventCounterQueue` can be used to count named events (default event is `count`) between `async` threads. `async` worker threads call `queue.send(event="event_name", N=amount)`. The receving end can either `receive()` a single event or `listen()` all events and return `collections.defaultdict[str, int]` as a result.
81
+
82
+ ## Features
83
+
84
+ - Supports multiple producers and a single listener
85
+ - Default event is `count`
86
+
87
+
74
88
  # FileQueue
75
89
 
76
- [`FileQueue`](docs/filequeue.md) builds a queue (`IterableQueue[pathlib.Path]`) of the matching
90
+ `FileQueue` builds a queue (`IterableQueue[pathlib.Path]`) of the matching
77
91
  files found based on search parameters given. It can search both list of files or directories or
78
92
  mixed. Async method `FileQueue.mk_queue()` searches subdirectories of given directories.
79
93
 
@@ -6,6 +6,7 @@ Queutils *[Queue Utils]* is a package of handy Python queue classes:
6
6
 
7
7
  - **[AsyncQueue](docs/asyncqueue.md)** - An `async` wrapper for non-async `queue.Queue`
8
8
  - **[IterableQueue](docs/iterablequeue.md)** - An `AsyncIterable` queue that terminates when finished
9
+ - **EventCounterQueue** - An `IterableQueue` for counting events in `async` threads
9
10
  - **[FileQueue](docs/filequeue.md)** - Builds an iterable queue of filenames from files/dirs given as input
10
11
 
11
12
 
@@ -43,6 +44,16 @@ producers are "finished", the queue enters into "filled" state and no new items
43
44
  - Countable property can be disabled with count_items=False. This is useful when you
44
45
  want to sum the count of multiple IterableQueues
45
46
 
47
+ # EventCounterQueue
48
+
49
+ `EventCounterQueue` can be used to count named events (default event is `count`) between `async` threads. `async` worker threads call `queue.send(event="event_name", N=amount)`. The receving end can either `receive()` a single event or `listen()` all events and return `collections.defaultdict[str, int]` as a result.
50
+
51
+ ## Features
52
+
53
+ - Supports multiple producers and a single listener
54
+ - Default event is `count`
55
+
56
+
46
57
  # FileQueue
47
58
 
48
59
  [`FileQueue`](docs/filequeue.md) builds a queue (`IterableQueue[pathlib.Path]`) of the matching
@@ -35,4 +35,18 @@ async def main() -> None:
35
35
 
36
36
  if __name__ == "__main__":
37
37
  run(main())
38
+ ```
39
+
40
+ ### Run
41
+
42
+ ```bash
43
+ cd demos
44
+ python -m filequeue_demo
45
+ ```
46
+ Output
47
+ ```text
48
+ found asyncqueue_demo.py
49
+ found iterablequeue_demo.py
50
+ found filequeue_demo.py
51
+ finished, no need to use fileQ.join()
38
52
  ```
@@ -0,0 +1,14 @@
1
+ #!/bin/bash
2
+
3
+ # Usage: ./rmlinks.sh input.md > output.md
4
+
5
+ if [ "$#" -ne 1 ]; then
6
+ echo "Usage: $0 <markdown-file>"
7
+ exit 1
8
+ fi
9
+
10
+ INPUT_FILE="$1"
11
+
12
+ # Use sed to remove markdown links to /docs but keep the link text
13
+ # Matches: [link text](/docs/whatever.md) and turns into: link text
14
+ perl -pe 's/\[([^\]]+)\]\((\/?docs\/[^)]+)\)/\1/g' "$INPUT_FILE"
queutils-0.9.3/pypi.md ADDED
@@ -0,0 +1,69 @@
1
+ [![Python package](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml/badge.svg)](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml) [![codecov](https://codecov.io/gh/Jylpah/queutils/graph/badge.svg?token=rMKdbfHOFs)](https://codecov.io/gh/Jylpah/queutils)
2
+
3
+ # Queutils
4
+
5
+ Queutils *[Queue Utils]* is a package of handy Python queue classes:
6
+
7
+ - **AsyncQueue** - An `async` wrapper for non-async `queue.Queue`
8
+ - **IterableQueue** - An `AsyncIterable` queue that terminates when finished
9
+ - **EventCounterQueue** - An `IterableQueue` for counting events in `async` threads
10
+ - **FileQueue** - Builds an iterable queue of filenames from files/dirs given as input
11
+
12
+
13
+ # AsyncQueue
14
+
15
+ `AsyncQueue` is a async wrapper for non-async `queue.Queue`. It can be used to create
16
+ an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessing.Queue` and thus enable `async` code in parent/child processes to communicate over `multiprocessing.Queue` as it were an `asyncio.Queue`.
17
+
18
+ ## Features
19
+
20
+ - `asyncio.Queue` compatible
21
+ - `queue.Queue` support
22
+ - `multiprocessing.Queue` support
23
+
24
+
25
+ # IterableQueue
26
+
27
+ `IterableQueue` is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be
28
+ iterated in `async for` loop. `IterableQueue` terminates automatically when the queue has been filled and emptied.
29
+
30
+ The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves and it
31
+ keeps count of registered producers which are "finished" adding items to the queue. Once all the registered
32
+ producers are "finished", the queue enters into "filled" state and no new items can be added. Once an
33
+ "filled" queue is emptied, the queue becomes "done" and all new `get()` calls to the queue will
34
+ `raise QueueDone` exception.
35
+
36
+ ## Features
37
+
38
+ - `asyncio.Queue` interface, `_nowait()` methods are experimental
39
+ - `AsyncIterable` support: `async for item in queue:`
40
+ - Automatic termination of the consumers with `QueueDone` exception when the queue has been emptied
41
+ - Producers must be registered with `add_producer()` and they must notify the queue
42
+ with `finish()` once they have finished adding items
43
+ - Countable interface to count number of items task_done() through `count` property
44
+ - Countable property can be disabled with count_items=False. This is useful when you
45
+ want to sum the count of multiple IterableQueues
46
+
47
+ # EventCounterQueue
48
+
49
+ `EventCounterQueue` can be used to count named events (default event is `count`) between `async` threads. `async` worker threads call `queue.send(event="event_name", N=amount)`. The receving end can either `receive()` a single event or `listen()` all events and return `collections.defaultdict[str, int]` as a result.
50
+
51
+ ## Features
52
+
53
+ - Supports multiple producers and a single listener
54
+ - Default event is `count`
55
+
56
+
57
+ # FileQueue
58
+
59
+ `FileQueue` builds a queue (`IterableQueue[pathlib.Path]`) of the matching
60
+ files found based on search parameters given. It can search both list of files or directories or
61
+ mixed. Async method `FileQueue.mk_queue()` searches subdirectories of given directories.
62
+
63
+ ## Features
64
+
65
+ - Input can be given both as `str` and `pathlib.Path`
66
+ - `exclude: bool` exclusive or inclusive filtering. Default is `False`.
67
+ - `case_sensitive: bool` case sensitive filtering (use of `fnmatch` or `fnmatchcase`). Default is `True`.
68
+ - `follow_symlinks: bool` whether to follow symlinks. Default is `False`.
69
+
@@ -1,9 +1,9 @@
1
1
  [project]
2
2
  name = "queutils"
3
- version = "0.8.5"
3
+ version = "0.9.3"
4
4
  authors = [{ name = "Jylpah", email = "jylpah@gmail.com" }]
5
5
  description = "Handy Python Queue utilies"
6
- readme = "README.md"
6
+ readme = { file = "pypi.md", content-type = "text/markdown" }
7
7
  requires-python = ">=3.11"
8
8
  classifiers = [
9
9
  "Programming Language :: Python :: 3",
@@ -13,7 +13,7 @@ classifiers = [
13
13
  "Framework :: AsyncIO",
14
14
  "Topic :: Software Development :: Libraries",
15
15
  ]
16
- dependencies = ["aioconsole>=0.6"]
16
+ dependencies = ["aioconsole>=0.6", "Deprecated>=1.2.18"]
17
17
 
18
18
  [project.optional-dependencies]
19
19
  dev = [
@@ -27,6 +27,7 @@ dev = [
27
27
  "pytest-cov>=4.1",
28
28
  "pytest-timeout>=2.2",
29
29
  "ruff>=0.1.9",
30
+ "types-Deprecated>=1.2.15",
30
31
  ]
31
32
 
32
33
 
@@ -59,7 +60,10 @@ lint.fixable = ["ALL"]
59
60
  minversion = "7.4"
60
61
  addopts = ["-v", "--cov=src"]
61
62
  testpaths = ["tests", "demos"]
62
- pythonpath = "src" # avoid import path append in test files
63
+ pythonpath = "src" # avoid import path append in test files
64
+ asyncio_default_fixture_loop_scope = "function"
63
65
 
64
66
  [tool.pyright]
65
67
  reportGeneralTypeIssues = false
68
+ reportInvalidStringEscapeSequence = false
69
+ typeCheckingMode = "off"
@@ -2,10 +2,15 @@ from .countable import Countable as Countable
2
2
  from .asyncqueue import AsyncQueue as AsyncQueue
3
3
  from .iterablequeue import IterableQueue as IterableQueue, QueueDone as QueueDone
4
4
  from .filequeue import FileQueue as FileQueue
5
+ from .eventcounterqueue import (
6
+ QCounter as QCounter,
7
+ EventCounterQueue as EventCounterQueue,
8
+ )
5
9
 
6
10
  __all__ = [
7
11
  "asyncqueue",
8
12
  "countable",
13
+ "eventcounterqueue",
9
14
  "filequeue",
10
15
  "iterablequeue",
11
16
  ]
@@ -0,0 +1,116 @@
1
+ from asyncio import Queue
2
+ from typing import TypeVar
3
+ from deprecated import deprecated
4
+ from .countable import Countable
5
+ from .iterablequeue import IterableQueue, QueueDone
6
+ from collections import defaultdict
7
+ import logging
8
+
9
+ logger = logging.getLogger()
10
+ error = logger.error
11
+ message = logger.warning
12
+ verbose = logger.info
13
+ debug = logger.debug
14
+
15
+ ###########################################
16
+ #
17
+ # class CounterQueue
18
+ #
19
+ ###########################################
20
+ T = TypeVar("T")
21
+
22
+
23
+ @deprecated(version="0.9.1", reason="Use EventCounterQueue instead")
24
+ class CounterQueue(Queue[T], Countable):
25
+ """
26
+ CounterQueue is a asyncio.Queue for counting items
27
+ """
28
+
29
+ _counter: int
30
+ _count_items: bool
31
+ _batch: int
32
+
33
+ def __init__(
34
+ self, *args, count_items: bool = True, batch: int = 1, **kwargs
35
+ ) -> None:
36
+ super().__init__(*args, **kwargs)
37
+ self._counter = 0
38
+ self._count_items = count_items
39
+ self._batch = batch
40
+
41
+ def task_done(self) -> None:
42
+ super().task_done()
43
+ if self._count_items:
44
+ self._counter += 1
45
+ return None
46
+
47
+ @property
48
+ def count(self) -> int:
49
+ """Return number of completed tasks"""
50
+ return self._counter * self._batch
51
+
52
+ @property
53
+ def count_items(self) -> bool:
54
+ """Whether or not count items"""
55
+ return self._count_items
56
+
57
+
58
+ class EventCounterQueue(IterableQueue[tuple[str, int]]):
59
+ """
60
+ EventCounterQueue is a asyncio.Queue for counting events by name
61
+ """
62
+
63
+ _counter: defaultdict[str, int]
64
+ _batch: int
65
+
66
+ def __init__(self, *args, batch: int = 1, **kwargs) -> None:
67
+ super().__init__(*args, **kwargs)
68
+ self._batch = batch
69
+ self._counter = defaultdict(int)
70
+
71
+ async def receive(self) -> tuple[str, int]:
72
+ """Receive an event value from the queue and sum it"""
73
+ event: str
74
+ value: int
75
+ event, value = await super().get()
76
+ self._counter[event] += value
77
+ super().task_done()
78
+ return (event, value)
79
+
80
+ async def send(self, event: str = "count", value: int = 1) -> None:
81
+ """Send count of an event"""
82
+ await super().put((event, value))
83
+ return None
84
+
85
+ def get_count(self, event: str = "count") -> int:
86
+ """Return count for an event"""
87
+ return self._counter[event]
88
+
89
+ def get_counts(self) -> defaultdict[str, int]:
90
+ """Return counts of all events"""
91
+ return self._counter
92
+
93
+ async def listen(self) -> defaultdict[str, int]:
94
+ """Listen for event values"""
95
+ try:
96
+ while True:
97
+ await self.receive()
98
+ except QueueDone:
99
+ pass
100
+ return self.get_counts()
101
+
102
+
103
+ class QCounter:
104
+ def __init__(self, Q: Queue[int]):
105
+ self._count = 0
106
+ self._Q: Queue[int] = Q
107
+
108
+ @property
109
+ def count(self) -> int:
110
+ return self._count
111
+
112
+ async def start(self) -> None:
113
+ """Read and count items from Q"""
114
+ while True:
115
+ self._count += await self._Q.get()
116
+ self._Q.task_done()
@@ -0,0 +1,80 @@
1
+ import pytest # type: ignore
2
+ from asyncio import (
3
+ Task,
4
+ create_task,
5
+ gather,
6
+ TimeoutError,
7
+ )
8
+ from random import choice, randint
9
+ import string
10
+ from collections import defaultdict
11
+
12
+ from queutils import EventCounterQueue
13
+
14
+
15
+ def randomword(length: int) -> str:
16
+ """Generate a random word of fixed length"""
17
+ # https://stackoverflow.com/a/2030081/12946084
18
+ letters: str = string.ascii_lowercase
19
+ return "".join(choice(letters) for i in range(length))
20
+
21
+
22
+ QSIZE: int = 10
23
+ N: int = 100 # N >> QSIZE
24
+ THREADS: int = 4
25
+ # N : int = int(1e10)
26
+
27
+
28
+ @pytest.mark.parametrize(
29
+ "events,N, producers",
30
+ [
31
+ ([randomword(5) for _ in range(10)], 1000, 1),
32
+ ([randomword(5) for _ in range(20)], 10000, 1),
33
+ ([randomword(5) for _ in range(5)], 1000, 3),
34
+ ],
35
+ )
36
+ @pytest.mark.timeout(10)
37
+ @pytest.mark.asyncio
38
+ async def test_1_category_counter_queue(
39
+ events: list[str], N: int, producers: int
40
+ ) -> None:
41
+ """Test EventCounterQueue"""
42
+ Q = EventCounterQueue(maxsize=QSIZE)
43
+
44
+ async def producer(
45
+ Q: EventCounterQueue, events: list[str], N: int = 100
46
+ ) -> defaultdict[str, int]:
47
+ """
48
+ Test Producer for EventCounterQueue
49
+ """
50
+ _counter: defaultdict[str, int] = defaultdict(int)
51
+ await Q.add_producer()
52
+ for _ in range(N):
53
+ cat: str = choice(events)
54
+ count: int = randint(1, 10)
55
+ await Q.send(cat, count)
56
+ _counter[cat] += count
57
+ await Q.finish()
58
+ return _counter
59
+
60
+ senders: list[Task] = list()
61
+
62
+ for _ in range(producers):
63
+ senders.append(create_task(producer(Q, events, N)))
64
+
65
+ try:
66
+ res_in: defaultdict[str, int] = await Q.listen()
67
+ res_out: defaultdict[str, int] = defaultdict(int)
68
+ for res in await gather(*senders):
69
+ for event, count in res.items():
70
+ res_out[event] += count
71
+
72
+ assert res_in == res_out, f"EventCounterQueue: {res_in} != {res_out}"
73
+ assert Q.qsize() == 0, "queue size is > 0 even it should be empty"
74
+ assert Q.empty(), "queue not empty"
75
+ assert Q.count == N * producers, (
76
+ f"count returned wrong value {Q.count}, should be {N * producers}"
77
+ )
78
+
79
+ except TimeoutError:
80
+ assert False, "await IterableQueue.join() failed with an empty queue finished"
@@ -1,39 +0,0 @@
1
- # This workflow will upload a Python Package using Twine when a release is created
2
- # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
3
-
4
- # This workflow uses actions that are not certified by GitHub.
5
- # They are provided by a third-party and are governed by
6
- # separate terms of service, privacy policy, and support
7
- # documentation.
8
-
9
- name: Upload Python Package
10
-
11
- on:
12
- release:
13
- types: [published]
14
-
15
- permissions:
16
- contents: read
17
-
18
- jobs:
19
- deploy:
20
-
21
- runs-on: ubuntu-latest
22
-
23
- steps:
24
- - uses: actions/checkout@v4
25
- - name: Set up Python
26
- uses: actions/setup-python@v5
27
- with:
28
- python-version: '3.11'
29
- - name: Install dependencies
30
- run: |
31
- python -m pip install --upgrade pip
32
- pip install build
33
- - name: Build package
34
- run: python -m build
35
- - name: Publish package
36
- uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
37
- with:
38
- user: __token__
39
- password: ${{ secrets.PYPI_API_TOKEN }}
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes