queutils 0.8.5__tar.gz → 0.9.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- queutils-0.9.3/.github/workflows/codeql.yml +100 -0
- queutils-0.9.3/.github/workflows/python-publish.yml +39 -0
- {queutils-0.8.5 → queutils-0.9.3}/PKG-INFO +22 -8
- {queutils-0.8.5 → queutils-0.9.3}/README.md +11 -0
- {queutils-0.8.5 → queutils-0.9.3}/docs/filequeue.md +14 -0
- queutils-0.9.3/docs/rm_links +14 -0
- queutils-0.9.3/pypi.md +69 -0
- {queutils-0.8.5 → queutils-0.9.3}/pyproject.toml +8 -4
- {queutils-0.8.5 → queutils-0.9.3}/src/queutils/__init__.py +5 -0
- queutils-0.9.3/src/queutils/eventcounterqueue.py +116 -0
- queutils-0.9.3/tests/test_eventcounterqueue.py +80 -0
- queutils-0.8.5/.github/workflows/python-publish.yml +0 -39
- {queutils-0.8.5 → queutils-0.9.3}/.github/workflows/dependency-review.yml +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/.github/workflows/python-package.yml +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/.gitignore +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/LICENSE +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/codecov.yml +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/demos/asyncqueue_demo.py +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/demos/filequeue_demo.py +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/demos/iterablequeue_demo.py +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/docs/asyncqueue.md +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/docs/iterablequeue.md +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/src/queutils/asyncqueue.py +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/src/queutils/countable.py +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/src/queutils/filequeue.py +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/src/queutils/iterablequeue.py +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/src/queutils/py.typed +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/tests/test_asyncqueue.py +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/tests/test_demos.py +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/tests/test_filequeue.py +0 -0
- {queutils-0.8.5 → queutils-0.9.3}/tests/test_iterablequeue.py +0 -0
@@ -0,0 +1,100 @@
|
|
1
|
+
# For most projects, this workflow file will not need changing; you simply need
|
2
|
+
# to commit it to your repository.
|
3
|
+
#
|
4
|
+
# You may wish to alter this file to override the set of languages analyzed,
|
5
|
+
# or to provide custom queries or build logic.
|
6
|
+
#
|
7
|
+
# ******** NOTE ********
|
8
|
+
# We have attempted to detect the languages in your repository. Please check
|
9
|
+
# the `language` matrix defined below to confirm you have the correct set of
|
10
|
+
# supported CodeQL languages.
|
11
|
+
#
|
12
|
+
name: "CodeQL Advanced"
|
13
|
+
|
14
|
+
on:
|
15
|
+
push:
|
16
|
+
branches: [ "main" ]
|
17
|
+
pull_request:
|
18
|
+
branches: [ "main" ]
|
19
|
+
schedule:
|
20
|
+
- cron: '39 23 8 * *'
|
21
|
+
|
22
|
+
jobs:
|
23
|
+
analyze:
|
24
|
+
name: Analyze (${{ matrix.language }})
|
25
|
+
# Runner size impacts CodeQL analysis time. To learn more, please see:
|
26
|
+
# - https://gh.io/recommended-hardware-resources-for-running-codeql
|
27
|
+
# - https://gh.io/supported-runners-and-hardware-resources
|
28
|
+
# - https://gh.io/using-larger-runners (GitHub.com only)
|
29
|
+
# Consider using larger runners or machines with greater resources for possible analysis time improvements.
|
30
|
+
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
|
31
|
+
permissions:
|
32
|
+
# required for all workflows
|
33
|
+
security-events: write
|
34
|
+
|
35
|
+
# required to fetch internal or private CodeQL packs
|
36
|
+
packages: read
|
37
|
+
|
38
|
+
# only required for workflows in private repositories
|
39
|
+
actions: read
|
40
|
+
contents: read
|
41
|
+
|
42
|
+
strategy:
|
43
|
+
fail-fast: false
|
44
|
+
matrix:
|
45
|
+
include:
|
46
|
+
- language: actions
|
47
|
+
build-mode: none
|
48
|
+
- language: python
|
49
|
+
build-mode: none
|
50
|
+
# CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
|
51
|
+
# Use `c-cpp` to analyze code written in C, C++ or both
|
52
|
+
# Use 'java-kotlin' to analyze code written in Java, Kotlin or both
|
53
|
+
# Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
|
54
|
+
# To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
|
55
|
+
# see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
|
56
|
+
# If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
|
57
|
+
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
|
58
|
+
steps:
|
59
|
+
- name: Checkout repository
|
60
|
+
uses: actions/checkout@v4
|
61
|
+
|
62
|
+
# Add any setup steps before running the `github/codeql-action/init` action.
|
63
|
+
# This includes steps like installing compilers or runtimes (`actions/setup-node`
|
64
|
+
# or others). This is typically only required for manual builds.
|
65
|
+
# - name: Setup runtime (example)
|
66
|
+
# uses: actions/setup-example@v1
|
67
|
+
|
68
|
+
# Initializes the CodeQL tools for scanning.
|
69
|
+
- name: Initialize CodeQL
|
70
|
+
uses: github/codeql-action/init@v3
|
71
|
+
with:
|
72
|
+
languages: ${{ matrix.language }}
|
73
|
+
build-mode: ${{ matrix.build-mode }}
|
74
|
+
# If you wish to specify custom queries, you can do so here or in a config file.
|
75
|
+
# By default, queries listed here will override any specified in a config file.
|
76
|
+
# Prefix the list here with "+" to use these queries and those in the config file.
|
77
|
+
|
78
|
+
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
|
79
|
+
# queries: security-extended,security-and-quality
|
80
|
+
|
81
|
+
# If the analyze step fails for one of the languages you are analyzing with
|
82
|
+
# "We were unable to automatically build your code", modify the matrix above
|
83
|
+
# to set the build mode to "manual" for that language. Then modify this step
|
84
|
+
# to build your code.
|
85
|
+
# ℹ️ Command-line programs to run using the OS shell.
|
86
|
+
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
87
|
+
- if: matrix.build-mode == 'manual'
|
88
|
+
shell: bash
|
89
|
+
run: |
|
90
|
+
echo 'If you are using a "manual" build mode for one or more of the' \
|
91
|
+
'languages you are analyzing, replace this with the commands to build' \
|
92
|
+
'your code, for example:'
|
93
|
+
echo ' make bootstrap'
|
94
|
+
echo ' make release'
|
95
|
+
exit 1
|
96
|
+
|
97
|
+
- name: Perform CodeQL Analysis
|
98
|
+
uses: github/codeql-action/analyze@v3
|
99
|
+
with:
|
100
|
+
category: "/language:${{matrix.language}}"
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# This workflow will upload a Python Package using Twine when a release is created
|
2
|
+
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
|
3
|
+
|
4
|
+
# This workflow uses actions that are not certified by GitHub.
|
5
|
+
# They are provided by a third-party and are governed by
|
6
|
+
# separate terms of service, privacy policy, and support
|
7
|
+
# documentation.
|
8
|
+
|
9
|
+
name: Upload Python Package
|
10
|
+
|
11
|
+
on:
|
12
|
+
release:
|
13
|
+
types: [published]
|
14
|
+
|
15
|
+
jobs:
|
16
|
+
deploy:
|
17
|
+
runs-on: ubuntu-latest
|
18
|
+
environment: production
|
19
|
+
permissions:
|
20
|
+
id-token: write
|
21
|
+
contents: read
|
22
|
+
|
23
|
+
steps:
|
24
|
+
- uses: actions/checkout@v4
|
25
|
+
- name: Set up Python
|
26
|
+
uses: actions/setup-python@v5
|
27
|
+
with:
|
28
|
+
python-version: "3.12"
|
29
|
+
- name: Install dependencies
|
30
|
+
run: |
|
31
|
+
python -m pip install --upgrade pip
|
32
|
+
pip install build
|
33
|
+
- name: Build package
|
34
|
+
run: python -m build
|
35
|
+
- name: Publish package
|
36
|
+
uses: pypa/gh-action-pypi-publish@v1.12.4
|
37
|
+
# with:
|
38
|
+
# user: __token__
|
39
|
+
# password: ${{ secrets.PYPI_API_TOKEN }}
|
@@ -1,10 +1,11 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.4
|
2
2
|
Name: queutils
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.9.3
|
4
4
|
Summary: Handy Python Queue utilies
|
5
5
|
Project-URL: Homepage, https://github.com/Jylpah/queutils
|
6
6
|
Project-URL: Bug Tracker, https://github.com/Jylpah/queutils/issues
|
7
7
|
Author-email: Jylpah <jylpah@gmail.com>
|
8
|
+
License-File: LICENSE
|
8
9
|
Classifier: Development Status :: 4 - Beta
|
9
10
|
Classifier: Framework :: AsyncIO
|
10
11
|
Classifier: License :: OSI Approved :: MIT License
|
@@ -13,6 +14,7 @@ Classifier: Programming Language :: Python :: 3
|
|
13
14
|
Classifier: Topic :: Software Development :: Libraries
|
14
15
|
Requires-Python: >=3.11
|
15
16
|
Requires-Dist: aioconsole>=0.6
|
17
|
+
Requires-Dist: deprecated>=1.2.18
|
16
18
|
Provides-Extra: dev
|
17
19
|
Requires-Dist: build>=0.10; extra == 'dev'
|
18
20
|
Requires-Dist: hatchling>=1.22.4; extra == 'dev'
|
@@ -24,6 +26,7 @@ Requires-Dist: pytest-datafiles>=3.0; extra == 'dev'
|
|
24
26
|
Requires-Dist: pytest-timeout>=2.2; extra == 'dev'
|
25
27
|
Requires-Dist: pytest>=8.0; extra == 'dev'
|
26
28
|
Requires-Dist: ruff>=0.1.9; extra == 'dev'
|
29
|
+
Requires-Dist: types-deprecated>=1.2.15; extra == 'dev'
|
27
30
|
Description-Content-Type: text/markdown
|
28
31
|
|
29
32
|
[](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml) [](https://codecov.io/gh/Jylpah/queutils)
|
@@ -32,14 +35,15 @@ Description-Content-Type: text/markdown
|
|
32
35
|
|
33
36
|
Queutils *[Queue Utils]* is a package of handy Python queue classes:
|
34
37
|
|
35
|
-
- **
|
36
|
-
- **
|
37
|
-
- **
|
38
|
+
- **AsyncQueue** - An `async` wrapper for non-async `queue.Queue`
|
39
|
+
- **IterableQueue** - An `AsyncIterable` queue that terminates when finished
|
40
|
+
- **EventCounterQueue** - An `IterableQueue` for counting events in `async` threads
|
41
|
+
- **FileQueue** - Builds an iterable queue of filenames from files/dirs given as input
|
38
42
|
|
39
43
|
|
40
44
|
# AsyncQueue
|
41
45
|
|
42
|
-
|
46
|
+
`AsyncQueue` is a async wrapper for non-async `queue.Queue`. It can be used to create
|
43
47
|
an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessing.Queue` and thus enable `async` code in parent/child processes to communicate over `multiprocessing.Queue` as it were an `asyncio.Queue`.
|
44
48
|
|
45
49
|
## Features
|
@@ -51,7 +55,7 @@ an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessin
|
|
51
55
|
|
52
56
|
# IterableQueue
|
53
57
|
|
54
|
-
|
58
|
+
`IterableQueue` is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be
|
55
59
|
iterated in `async for` loop. `IterableQueue` terminates automatically when the queue has been filled and emptied.
|
56
60
|
|
57
61
|
The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves and it
|
@@ -71,9 +75,19 @@ producers are "finished", the queue enters into "filled" state and no new items
|
|
71
75
|
- Countable property can be disabled with count_items=False. This is useful when you
|
72
76
|
want to sum the count of multiple IterableQueues
|
73
77
|
|
78
|
+
# EventCounterQueue
|
79
|
+
|
80
|
+
`EventCounterQueue` can be used to count named events (default event is `count`) between `async` threads. `async` worker threads call `queue.send(event="event_name", N=amount)`. The receving end can either `receive()` a single event or `listen()` all events and return `collections.defaultdict[str, int]` as a result.
|
81
|
+
|
82
|
+
## Features
|
83
|
+
|
84
|
+
- Supports multiple producers and a single listener
|
85
|
+
- Default event is `count`
|
86
|
+
|
87
|
+
|
74
88
|
# FileQueue
|
75
89
|
|
76
|
-
|
90
|
+
`FileQueue` builds a queue (`IterableQueue[pathlib.Path]`) of the matching
|
77
91
|
files found based on search parameters given. It can search both list of files or directories or
|
78
92
|
mixed. Async method `FileQueue.mk_queue()` searches subdirectories of given directories.
|
79
93
|
|
@@ -6,6 +6,7 @@ Queutils *[Queue Utils]* is a package of handy Python queue classes:
|
|
6
6
|
|
7
7
|
- **[AsyncQueue](docs/asyncqueue.md)** - An `async` wrapper for non-async `queue.Queue`
|
8
8
|
- **[IterableQueue](docs/iterablequeue.md)** - An `AsyncIterable` queue that terminates when finished
|
9
|
+
- **EventCounterQueue** - An `IterableQueue` for counting events in `async` threads
|
9
10
|
- **[FileQueue](docs/filequeue.md)** - Builds an iterable queue of filenames from files/dirs given as input
|
10
11
|
|
11
12
|
|
@@ -43,6 +44,16 @@ producers are "finished", the queue enters into "filled" state and no new items
|
|
43
44
|
- Countable property can be disabled with count_items=False. This is useful when you
|
44
45
|
want to sum the count of multiple IterableQueues
|
45
46
|
|
47
|
+
# EventCounterQueue
|
48
|
+
|
49
|
+
`EventCounterQueue` can be used to count named events (default event is `count`) between `async` threads. `async` worker threads call `queue.send(event="event_name", N=amount)`. The receving end can either `receive()` a single event or `listen()` all events and return `collections.defaultdict[str, int]` as a result.
|
50
|
+
|
51
|
+
## Features
|
52
|
+
|
53
|
+
- Supports multiple producers and a single listener
|
54
|
+
- Default event is `count`
|
55
|
+
|
56
|
+
|
46
57
|
# FileQueue
|
47
58
|
|
48
59
|
[`FileQueue`](docs/filequeue.md) builds a queue (`IterableQueue[pathlib.Path]`) of the matching
|
@@ -35,4 +35,18 @@ async def main() -> None:
|
|
35
35
|
|
36
36
|
if __name__ == "__main__":
|
37
37
|
run(main())
|
38
|
+
```
|
39
|
+
|
40
|
+
### Run
|
41
|
+
|
42
|
+
```bash
|
43
|
+
cd demos
|
44
|
+
python -m filequeue_demo
|
45
|
+
```
|
46
|
+
Output
|
47
|
+
```text
|
48
|
+
found asyncqueue_demo.py
|
49
|
+
found iterablequeue_demo.py
|
50
|
+
found filequeue_demo.py
|
51
|
+
finished, no need to use fileQ.join()
|
38
52
|
```
|
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
# Usage: ./rmlinks.sh input.md > output.md
|
4
|
+
|
5
|
+
if [ "$#" -ne 1 ]; then
|
6
|
+
echo "Usage: $0 <markdown-file>"
|
7
|
+
exit 1
|
8
|
+
fi
|
9
|
+
|
10
|
+
INPUT_FILE="$1"
|
11
|
+
|
12
|
+
# Use sed to remove markdown links to /docs but keep the link text
|
13
|
+
# Matches: [link text](/docs/whatever.md) and turns into: link text
|
14
|
+
perl -pe 's/\[([^\]]+)\]\((\/?docs\/[^)]+)\)/\1/g' "$INPUT_FILE"
|
queutils-0.9.3/pypi.md
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
[](https://github.com/Jylpah/queutils/actions/workflows/python-package.yml) [](https://codecov.io/gh/Jylpah/queutils)
|
2
|
+
|
3
|
+
# Queutils
|
4
|
+
|
5
|
+
Queutils *[Queue Utils]* is a package of handy Python queue classes:
|
6
|
+
|
7
|
+
- **AsyncQueue** - An `async` wrapper for non-async `queue.Queue`
|
8
|
+
- **IterableQueue** - An `AsyncIterable` queue that terminates when finished
|
9
|
+
- **EventCounterQueue** - An `IterableQueue` for counting events in `async` threads
|
10
|
+
- **FileQueue** - Builds an iterable queue of filenames from files/dirs given as input
|
11
|
+
|
12
|
+
|
13
|
+
# AsyncQueue
|
14
|
+
|
15
|
+
`AsyncQueue` is a async wrapper for non-async `queue.Queue`. It can be used to create
|
16
|
+
an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessing.Queue` and thus enable `async` code in parent/child processes to communicate over `multiprocessing.Queue` as it were an `asyncio.Queue`.
|
17
|
+
|
18
|
+
## Features
|
19
|
+
|
20
|
+
- `asyncio.Queue` compatible
|
21
|
+
- `queue.Queue` support
|
22
|
+
- `multiprocessing.Queue` support
|
23
|
+
|
24
|
+
|
25
|
+
# IterableQueue
|
26
|
+
|
27
|
+
`IterableQueue` is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be
|
28
|
+
iterated in `async for` loop. `IterableQueue` terminates automatically when the queue has been filled and emptied.
|
29
|
+
|
30
|
+
The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves and it
|
31
|
+
keeps count of registered producers which are "finished" adding items to the queue. Once all the registered
|
32
|
+
producers are "finished", the queue enters into "filled" state and no new items can be added. Once an
|
33
|
+
"filled" queue is emptied, the queue becomes "done" and all new `get()` calls to the queue will
|
34
|
+
`raise QueueDone` exception.
|
35
|
+
|
36
|
+
## Features
|
37
|
+
|
38
|
+
- `asyncio.Queue` interface, `_nowait()` methods are experimental
|
39
|
+
- `AsyncIterable` support: `async for item in queue:`
|
40
|
+
- Automatic termination of the consumers with `QueueDone` exception when the queue has been emptied
|
41
|
+
- Producers must be registered with `add_producer()` and they must notify the queue
|
42
|
+
with `finish()` once they have finished adding items
|
43
|
+
- Countable interface to count number of items task_done() through `count` property
|
44
|
+
- Countable property can be disabled with count_items=False. This is useful when you
|
45
|
+
want to sum the count of multiple IterableQueues
|
46
|
+
|
47
|
+
# EventCounterQueue
|
48
|
+
|
49
|
+
`EventCounterQueue` can be used to count named events (default event is `count`) between `async` threads. `async` worker threads call `queue.send(event="event_name", N=amount)`. The receving end can either `receive()` a single event or `listen()` all events and return `collections.defaultdict[str, int]` as a result.
|
50
|
+
|
51
|
+
## Features
|
52
|
+
|
53
|
+
- Supports multiple producers and a single listener
|
54
|
+
- Default event is `count`
|
55
|
+
|
56
|
+
|
57
|
+
# FileQueue
|
58
|
+
|
59
|
+
`FileQueue` builds a queue (`IterableQueue[pathlib.Path]`) of the matching
|
60
|
+
files found based on search parameters given. It can search both list of files or directories or
|
61
|
+
mixed. Async method `FileQueue.mk_queue()` searches subdirectories of given directories.
|
62
|
+
|
63
|
+
## Features
|
64
|
+
|
65
|
+
- Input can be given both as `str` and `pathlib.Path`
|
66
|
+
- `exclude: bool` exclusive or inclusive filtering. Default is `False`.
|
67
|
+
- `case_sensitive: bool` case sensitive filtering (use of `fnmatch` or `fnmatchcase`). Default is `True`.
|
68
|
+
- `follow_symlinks: bool` whether to follow symlinks. Default is `False`.
|
69
|
+
|
@@ -1,9 +1,9 @@
|
|
1
1
|
[project]
|
2
2
|
name = "queutils"
|
3
|
-
version = "0.
|
3
|
+
version = "0.9.3"
|
4
4
|
authors = [{ name = "Jylpah", email = "jylpah@gmail.com" }]
|
5
5
|
description = "Handy Python Queue utilies"
|
6
|
-
readme = "
|
6
|
+
readme = { file = "pypi.md", content-type = "text/markdown" }
|
7
7
|
requires-python = ">=3.11"
|
8
8
|
classifiers = [
|
9
9
|
"Programming Language :: Python :: 3",
|
@@ -13,7 +13,7 @@ classifiers = [
|
|
13
13
|
"Framework :: AsyncIO",
|
14
14
|
"Topic :: Software Development :: Libraries",
|
15
15
|
]
|
16
|
-
dependencies = ["aioconsole>=0.6"]
|
16
|
+
dependencies = ["aioconsole>=0.6", "Deprecated>=1.2.18"]
|
17
17
|
|
18
18
|
[project.optional-dependencies]
|
19
19
|
dev = [
|
@@ -27,6 +27,7 @@ dev = [
|
|
27
27
|
"pytest-cov>=4.1",
|
28
28
|
"pytest-timeout>=2.2",
|
29
29
|
"ruff>=0.1.9",
|
30
|
+
"types-Deprecated>=1.2.15",
|
30
31
|
]
|
31
32
|
|
32
33
|
|
@@ -59,7 +60,10 @@ lint.fixable = ["ALL"]
|
|
59
60
|
minversion = "7.4"
|
60
61
|
addopts = ["-v", "--cov=src"]
|
61
62
|
testpaths = ["tests", "demos"]
|
62
|
-
pythonpath = "src"
|
63
|
+
pythonpath = "src" # avoid import path append in test files
|
64
|
+
asyncio_default_fixture_loop_scope = "function"
|
63
65
|
|
64
66
|
[tool.pyright]
|
65
67
|
reportGeneralTypeIssues = false
|
68
|
+
reportInvalidStringEscapeSequence = false
|
69
|
+
typeCheckingMode = "off"
|
@@ -2,10 +2,15 @@ from .countable import Countable as Countable
|
|
2
2
|
from .asyncqueue import AsyncQueue as AsyncQueue
|
3
3
|
from .iterablequeue import IterableQueue as IterableQueue, QueueDone as QueueDone
|
4
4
|
from .filequeue import FileQueue as FileQueue
|
5
|
+
from .eventcounterqueue import (
|
6
|
+
QCounter as QCounter,
|
7
|
+
EventCounterQueue as EventCounterQueue,
|
8
|
+
)
|
5
9
|
|
6
10
|
__all__ = [
|
7
11
|
"asyncqueue",
|
8
12
|
"countable",
|
13
|
+
"eventcounterqueue",
|
9
14
|
"filequeue",
|
10
15
|
"iterablequeue",
|
11
16
|
]
|
@@ -0,0 +1,116 @@
|
|
1
|
+
from asyncio import Queue
|
2
|
+
from typing import TypeVar
|
3
|
+
from deprecated import deprecated
|
4
|
+
from .countable import Countable
|
5
|
+
from .iterablequeue import IterableQueue, QueueDone
|
6
|
+
from collections import defaultdict
|
7
|
+
import logging
|
8
|
+
|
9
|
+
logger = logging.getLogger()
|
10
|
+
error = logger.error
|
11
|
+
message = logger.warning
|
12
|
+
verbose = logger.info
|
13
|
+
debug = logger.debug
|
14
|
+
|
15
|
+
###########################################
|
16
|
+
#
|
17
|
+
# class CounterQueue
|
18
|
+
#
|
19
|
+
###########################################
|
20
|
+
T = TypeVar("T")
|
21
|
+
|
22
|
+
|
23
|
+
@deprecated(version="0.9.1", reason="Use EventCounterQueue instead")
|
24
|
+
class CounterQueue(Queue[T], Countable):
|
25
|
+
"""
|
26
|
+
CounterQueue is a asyncio.Queue for counting items
|
27
|
+
"""
|
28
|
+
|
29
|
+
_counter: int
|
30
|
+
_count_items: bool
|
31
|
+
_batch: int
|
32
|
+
|
33
|
+
def __init__(
|
34
|
+
self, *args, count_items: bool = True, batch: int = 1, **kwargs
|
35
|
+
) -> None:
|
36
|
+
super().__init__(*args, **kwargs)
|
37
|
+
self._counter = 0
|
38
|
+
self._count_items = count_items
|
39
|
+
self._batch = batch
|
40
|
+
|
41
|
+
def task_done(self) -> None:
|
42
|
+
super().task_done()
|
43
|
+
if self._count_items:
|
44
|
+
self._counter += 1
|
45
|
+
return None
|
46
|
+
|
47
|
+
@property
|
48
|
+
def count(self) -> int:
|
49
|
+
"""Return number of completed tasks"""
|
50
|
+
return self._counter * self._batch
|
51
|
+
|
52
|
+
@property
|
53
|
+
def count_items(self) -> bool:
|
54
|
+
"""Whether or not count items"""
|
55
|
+
return self._count_items
|
56
|
+
|
57
|
+
|
58
|
+
class EventCounterQueue(IterableQueue[tuple[str, int]]):
|
59
|
+
"""
|
60
|
+
EventCounterQueue is a asyncio.Queue for counting events by name
|
61
|
+
"""
|
62
|
+
|
63
|
+
_counter: defaultdict[str, int]
|
64
|
+
_batch: int
|
65
|
+
|
66
|
+
def __init__(self, *args, batch: int = 1, **kwargs) -> None:
|
67
|
+
super().__init__(*args, **kwargs)
|
68
|
+
self._batch = batch
|
69
|
+
self._counter = defaultdict(int)
|
70
|
+
|
71
|
+
async def receive(self) -> tuple[str, int]:
|
72
|
+
"""Receive an event value from the queue and sum it"""
|
73
|
+
event: str
|
74
|
+
value: int
|
75
|
+
event, value = await super().get()
|
76
|
+
self._counter[event] += value
|
77
|
+
super().task_done()
|
78
|
+
return (event, value)
|
79
|
+
|
80
|
+
async def send(self, event: str = "count", value: int = 1) -> None:
|
81
|
+
"""Send count of an event"""
|
82
|
+
await super().put((event, value))
|
83
|
+
return None
|
84
|
+
|
85
|
+
def get_count(self, event: str = "count") -> int:
|
86
|
+
"""Return count for an event"""
|
87
|
+
return self._counter[event]
|
88
|
+
|
89
|
+
def get_counts(self) -> defaultdict[str, int]:
|
90
|
+
"""Return counts of all events"""
|
91
|
+
return self._counter
|
92
|
+
|
93
|
+
async def listen(self) -> defaultdict[str, int]:
|
94
|
+
"""Listen for event values"""
|
95
|
+
try:
|
96
|
+
while True:
|
97
|
+
await self.receive()
|
98
|
+
except QueueDone:
|
99
|
+
pass
|
100
|
+
return self.get_counts()
|
101
|
+
|
102
|
+
|
103
|
+
class QCounter:
|
104
|
+
def __init__(self, Q: Queue[int]):
|
105
|
+
self._count = 0
|
106
|
+
self._Q: Queue[int] = Q
|
107
|
+
|
108
|
+
@property
|
109
|
+
def count(self) -> int:
|
110
|
+
return self._count
|
111
|
+
|
112
|
+
async def start(self) -> None:
|
113
|
+
"""Read and count items from Q"""
|
114
|
+
while True:
|
115
|
+
self._count += await self._Q.get()
|
116
|
+
self._Q.task_done()
|
@@ -0,0 +1,80 @@
|
|
1
|
+
import pytest # type: ignore
|
2
|
+
from asyncio import (
|
3
|
+
Task,
|
4
|
+
create_task,
|
5
|
+
gather,
|
6
|
+
TimeoutError,
|
7
|
+
)
|
8
|
+
from random import choice, randint
|
9
|
+
import string
|
10
|
+
from collections import defaultdict
|
11
|
+
|
12
|
+
from queutils import EventCounterQueue
|
13
|
+
|
14
|
+
|
15
|
+
def randomword(length: int) -> str:
|
16
|
+
"""Generate a random word of fixed length"""
|
17
|
+
# https://stackoverflow.com/a/2030081/12946084
|
18
|
+
letters: str = string.ascii_lowercase
|
19
|
+
return "".join(choice(letters) for i in range(length))
|
20
|
+
|
21
|
+
|
22
|
+
QSIZE: int = 10
|
23
|
+
N: int = 100 # N >> QSIZE
|
24
|
+
THREADS: int = 4
|
25
|
+
# N : int = int(1e10)
|
26
|
+
|
27
|
+
|
28
|
+
@pytest.mark.parametrize(
|
29
|
+
"events,N, producers",
|
30
|
+
[
|
31
|
+
([randomword(5) for _ in range(10)], 1000, 1),
|
32
|
+
([randomword(5) for _ in range(20)], 10000, 1),
|
33
|
+
([randomword(5) for _ in range(5)], 1000, 3),
|
34
|
+
],
|
35
|
+
)
|
36
|
+
@pytest.mark.timeout(10)
|
37
|
+
@pytest.mark.asyncio
|
38
|
+
async def test_1_category_counter_queue(
|
39
|
+
events: list[str], N: int, producers: int
|
40
|
+
) -> None:
|
41
|
+
"""Test EventCounterQueue"""
|
42
|
+
Q = EventCounterQueue(maxsize=QSIZE)
|
43
|
+
|
44
|
+
async def producer(
|
45
|
+
Q: EventCounterQueue, events: list[str], N: int = 100
|
46
|
+
) -> defaultdict[str, int]:
|
47
|
+
"""
|
48
|
+
Test Producer for EventCounterQueue
|
49
|
+
"""
|
50
|
+
_counter: defaultdict[str, int] = defaultdict(int)
|
51
|
+
await Q.add_producer()
|
52
|
+
for _ in range(N):
|
53
|
+
cat: str = choice(events)
|
54
|
+
count: int = randint(1, 10)
|
55
|
+
await Q.send(cat, count)
|
56
|
+
_counter[cat] += count
|
57
|
+
await Q.finish()
|
58
|
+
return _counter
|
59
|
+
|
60
|
+
senders: list[Task] = list()
|
61
|
+
|
62
|
+
for _ in range(producers):
|
63
|
+
senders.append(create_task(producer(Q, events, N)))
|
64
|
+
|
65
|
+
try:
|
66
|
+
res_in: defaultdict[str, int] = await Q.listen()
|
67
|
+
res_out: defaultdict[str, int] = defaultdict(int)
|
68
|
+
for res in await gather(*senders):
|
69
|
+
for event, count in res.items():
|
70
|
+
res_out[event] += count
|
71
|
+
|
72
|
+
assert res_in == res_out, f"EventCounterQueue: {res_in} != {res_out}"
|
73
|
+
assert Q.qsize() == 0, "queue size is > 0 even it should be empty"
|
74
|
+
assert Q.empty(), "queue not empty"
|
75
|
+
assert Q.count == N * producers, (
|
76
|
+
f"count returned wrong value {Q.count}, should be {N * producers}"
|
77
|
+
)
|
78
|
+
|
79
|
+
except TimeoutError:
|
80
|
+
assert False, "await IterableQueue.join() failed with an empty queue finished"
|
@@ -1,39 +0,0 @@
|
|
1
|
-
# This workflow will upload a Python Package using Twine when a release is created
|
2
|
-
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
|
3
|
-
|
4
|
-
# This workflow uses actions that are not certified by GitHub.
|
5
|
-
# They are provided by a third-party and are governed by
|
6
|
-
# separate terms of service, privacy policy, and support
|
7
|
-
# documentation.
|
8
|
-
|
9
|
-
name: Upload Python Package
|
10
|
-
|
11
|
-
on:
|
12
|
-
release:
|
13
|
-
types: [published]
|
14
|
-
|
15
|
-
permissions:
|
16
|
-
contents: read
|
17
|
-
|
18
|
-
jobs:
|
19
|
-
deploy:
|
20
|
-
|
21
|
-
runs-on: ubuntu-latest
|
22
|
-
|
23
|
-
steps:
|
24
|
-
- uses: actions/checkout@v4
|
25
|
-
- name: Set up Python
|
26
|
-
uses: actions/setup-python@v5
|
27
|
-
with:
|
28
|
-
python-version: '3.11'
|
29
|
-
- name: Install dependencies
|
30
|
-
run: |
|
31
|
-
python -m pip install --upgrade pip
|
32
|
-
pip install build
|
33
|
-
- name: Build package
|
34
|
-
run: python -m build
|
35
|
-
- name: Publish package
|
36
|
-
uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
|
37
|
-
with:
|
38
|
-
user: __token__
|
39
|
-
password: ${{ secrets.PYPI_API_TOKEN }}
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|