queutils 0.10.0__tar.gz → 0.10.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {queutils-0.10.0 → queutils-0.10.1}/PKG-INFO +6 -7
- {queutils-0.10.0 → queutils-0.10.1}/README.md +5 -6
- {queutils-0.10.0 → queutils-0.10.1}/demos/iterablequeue_demo.py +12 -9
- {queutils-0.10.0 → queutils-0.10.1}/docs/iterablequeue.md +4 -4
- {queutils-0.10.0 → queutils-0.10.1}/pypi.md +5 -6
- {queutils-0.10.0 → queutils-0.10.1}/pyproject.toml +1 -1
- {queutils-0.10.0 → queutils-0.10.1}/src/queutils/iterablequeue.py +3 -3
- {queutils-0.10.0 → queutils-0.10.1}/tests/test_iterablequeue.py +1 -1
- {queutils-0.10.0 → queutils-0.10.1}/.github/workflows/codeql.yml +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/.github/workflows/dependency-review.yml +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/.github/workflows/python-package.yml +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/.github/workflows/python-publish.yml +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/.gitignore +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/LICENSE +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/codecov.yml +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/demos/asyncqueue_demo.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/demos/filequeue_demo.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/docs/asyncqueue.md +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/docs/filequeue.md +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/docs/rm_links +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/src/queutils/__init__.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/src/queutils/asyncqueue.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/src/queutils/awrap.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/src/queutils/countable.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/src/queutils/eventcounterqueue.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/src/queutils/filequeue.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/src/queutils/py.typed +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/tests/test_asyncqueue.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/tests/test_awrap.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/tests/test_demos.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/tests/test_eventcounterqueue.py +0 -0
- {queutils-0.10.0 → queutils-0.10.1}/tests/test_filequeue.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: queutils
|
3
|
-
Version: 0.10.
|
3
|
+
Version: 0.10.1
|
4
4
|
Summary: Handy Python Queue utilies
|
5
5
|
Project-URL: Homepage, https://github.com/Jylpah/queutils
|
6
6
|
Project-URL: Bug Tracker, https://github.com/Jylpah/queutils/issues
|
@@ -58,11 +58,10 @@ an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessin
|
|
58
58
|
`IterableQueue` is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be
|
59
59
|
iterated in `async for` loop. `IterableQueue` terminates automatically when the queue has been filled and emptied.
|
60
60
|
|
61
|
-
The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves with `add_producer()` call
|
62
|
-
|
63
|
-
producers are "finished", the queue enters into "filled" state and no new items can be added. Once
|
64
|
-
|
65
|
-
`raise QueueDone` exception.
|
61
|
+
The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves with `add_producer()` call. It keeps count of registered producers. When a producer "finishes" adding items to the queue,
|
62
|
+
it needs to unregister itself with `finish_producer()` call. Once all the registered
|
63
|
+
producers are "finished", the queue enters into "filled" state and no new items can be added. Once a "filled" queue has been emptied, the queue becomes "done" and
|
64
|
+
all new `get()` calls to the queue will `raise QueueDone` exception.
|
66
65
|
|
67
66
|
## Features
|
68
67
|
|
@@ -70,7 +69,7 @@ producers are "finished", the queue enters into "filled" state and no new items
|
|
70
69
|
- `AsyncIterable` support: `async for item in queue:`
|
71
70
|
- Automatic termination of the consumers with `QueueDone` exception when the queue has been emptied
|
72
71
|
- Producers must be registered with `add_producer()` and they must notify the queue
|
73
|
-
with `
|
72
|
+
with `finish_producer()` once they have finished adding items
|
74
73
|
- Countable interface to count number of items task_done() through `count` property
|
75
74
|
|
76
75
|
# EventCounterQueue
|
@@ -27,11 +27,10 @@ an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessin
|
|
27
27
|
[`IterableQueue`](docs/iterablequeue.md) is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be
|
28
28
|
iterated in `async for` loop. `IterableQueue` terminates automatically when the queue has been filled and emptied.
|
29
29
|
|
30
|
-
The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves with `add_producer()` call
|
31
|
-
|
32
|
-
producers are "finished", the queue enters into "filled" state and no new items can be added. Once
|
33
|
-
|
34
|
-
`raise QueueDone` exception.
|
30
|
+
The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves with `add_producer()` call. It keeps count of registered producers. When a producer "finishes" adding items to the queue,
|
31
|
+
it needs to unregister itself with `finish_producer()` call. Once all the registered
|
32
|
+
producers are "finished", the queue enters into "filled" state and no new items can be added. Once a "filled" queue has been emptied, the queue becomes "done" and
|
33
|
+
all new `get()` calls to the queue will `raise QueueDone` exception.
|
35
34
|
|
36
35
|
## Features
|
37
36
|
|
@@ -39,7 +38,7 @@ producers are "finished", the queue enters into "filled" state and no new items
|
|
39
38
|
- `AsyncIterable` support: `async for item in queue:`
|
40
39
|
- Automatic termination of the consumers with `QueueDone` exception when the queue has been emptied
|
41
40
|
- Producers must be registered with `add_producer()` and they must notify the queue
|
42
|
-
with `
|
41
|
+
with `finish_producer()` once they have finished adding items
|
43
42
|
- Countable interface to count number of items task_done() through `count` property
|
44
43
|
|
45
44
|
# EventCounterQueue
|
@@ -3,14 +3,14 @@ from random import random
|
|
3
3
|
from queutils import IterableQueue, QueueDone
|
4
4
|
from time import time
|
5
5
|
|
6
|
-
start
|
6
|
+
start: float = time()
|
7
|
+
|
7
8
|
|
8
9
|
def since() -> float:
|
9
10
|
return time() - start
|
10
11
|
|
11
|
-
|
12
|
-
|
13
|
-
) -> None:
|
12
|
+
|
13
|
+
async def producer(Q: IterableQueue[int], N: int, id: int) -> None:
|
14
14
|
"""
|
15
15
|
Fill the queue with N items
|
16
16
|
"""
|
@@ -21,11 +21,12 @@ async def producer(
|
|
21
21
|
print(f"{since():.2f} producer {id}: awaiting to put {i} to queue")
|
22
22
|
await Q.put(i)
|
23
23
|
print(f"{since():.2f} producer {id}: put {i} to queue")
|
24
|
-
await Q.
|
24
|
+
await Q.finish_producer()
|
25
25
|
except QueueDone:
|
26
26
|
print(f"ERROR: producer {id}, this should not happen")
|
27
27
|
return None
|
28
28
|
|
29
|
+
|
29
30
|
async def consumer(Q: IterableQueue[int], id: int = 1):
|
30
31
|
"""
|
31
32
|
Consume the queue
|
@@ -35,19 +36,21 @@ async def consumer(Q: IterableQueue[int], id: int = 1):
|
|
35
36
|
await sleep(0.5 * random())
|
36
37
|
print(f"{since():.2f} consumer {id}: queue is done")
|
37
38
|
|
39
|
+
|
38
40
|
async def main() -> None:
|
39
41
|
"""
|
40
|
-
Create a queue with maxsize and have multiple producers to fill it and
|
42
|
+
Create a queue with maxsize and have multiple producers to fill it and
|
41
43
|
multiple consumers to consume it over async for loop
|
42
44
|
"""
|
43
|
-
queue
|
45
|
+
queue: IterableQueue[int] = IterableQueue(maxsize=5)
|
44
46
|
|
45
47
|
async with TaskGroup() as tg:
|
46
|
-
for i in range(1,3):
|
48
|
+
for i in range(1, 3):
|
47
49
|
tg.create_task(producer(Q=queue, N=5, id=i))
|
48
50
|
await sleep(2)
|
49
|
-
for
|
51
|
+
for i in range(1, 4):
|
50
52
|
tg.create_task(consumer(Q=queue, id=i))
|
51
53
|
|
54
|
+
|
52
55
|
if __name__ == "__main__":
|
53
56
|
run(main())
|
@@ -9,7 +9,7 @@ iterated in `async for` loop. The great benefit of `IterableQueue` is that it te
|
|
9
9
|
- `AsyncIterable` support: `async for item in queue:`
|
10
10
|
- Automatic termination of the consumers with `QueueDone` exception when the queue has been emptied
|
11
11
|
- Producers must be registered with `add_producer()` and they must notify the queue
|
12
|
-
with `
|
12
|
+
with `finish_producer()` once they have finished adding items
|
13
13
|
- Countable interface to count number of items task_done() through `count` property
|
14
14
|
|
15
15
|
### Experimental
|
@@ -21,7 +21,7 @@ iterated in `async for` loop. The great benefit of `IterableQueue` is that it te
|
|
21
21
|
|
22
22
|
### Producers fill a queue
|
23
23
|
|
24
|
-
A *Producer* is "process" that adds items to the queue. A producer needs to be registered to the queue with `add_producer()` coroutine. Once a producer has added all the items it intends to, it notifies the queue with `
|
24
|
+
A *Producer* is "process" that adds items to the queue. A producer needs to be registered to the queue with `add_producer()` coroutine. Once a producer has added all the items it intends to, it notifies the queue with `finish_producer()`
|
25
25
|
|
26
26
|
```python
|
27
27
|
from queutils.iterablequeue import IterableQueue
|
@@ -37,7 +37,7 @@ async def producer(
|
|
37
37
|
await Q.put(i)
|
38
38
|
|
39
39
|
# notify the queue that this producer does not add more
|
40
|
-
await Q.
|
40
|
+
await Q.finish_producer()
|
41
41
|
|
42
42
|
return None
|
43
43
|
```
|
@@ -88,7 +88,7 @@ async def producer(
|
|
88
88
|
print(f"{since():.2f} producer {id}: awaiting to put {i} to queue")
|
89
89
|
await Q.put(i)
|
90
90
|
print(f"{since():.2f} producer {id}: put {i} to queue")
|
91
|
-
await Q.
|
91
|
+
await Q.finish_producer()
|
92
92
|
except QueueDone:
|
93
93
|
print(f"ERROR: producer {id}, this should not happen")
|
94
94
|
return None
|
@@ -27,11 +27,10 @@ an `asyncio.Queue` compatible interface to a (non-async) managed `multiprocessin
|
|
27
27
|
`IterableQueue` is an `asyncio.Queue` subclass that is `AsyncIterable[T]` i.e. it can be
|
28
28
|
iterated in `async for` loop. `IterableQueue` terminates automatically when the queue has been filled and emptied.
|
29
29
|
|
30
|
-
The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves with `add_producer()` call
|
31
|
-
|
32
|
-
producers are "finished", the queue enters into "filled" state and no new items can be added. Once
|
33
|
-
|
34
|
-
`raise QueueDone` exception.
|
30
|
+
The `IterableQueue` requires "producers" (functions adding items to the queue) to register themselves with `add_producer()` call. It keeps count of registered producers. When a producer "finishes" adding items to the queue,
|
31
|
+
it needs to unregister itself with `finish_producer()` call. Once all the registered
|
32
|
+
producers are "finished", the queue enters into "filled" state and no new items can be added. Once a "filled" queue has been emptied, the queue becomes "done" and
|
33
|
+
all new `get()` calls to the queue will `raise QueueDone` exception.
|
35
34
|
|
36
35
|
## Features
|
37
36
|
|
@@ -39,7 +38,7 @@ producers are "finished", the queue enters into "filled" state and no new items
|
|
39
38
|
- `AsyncIterable` support: `async for item in queue:`
|
40
39
|
- Automatic termination of the consumers with `QueueDone` exception when the queue has been emptied
|
41
40
|
- Producers must be registered with `add_producer()` and they must notify the queue
|
42
|
-
with `
|
41
|
+
with `finish_producer()` once they have finished adding items
|
43
42
|
- Countable interface to count number of items task_done() through `count` property
|
44
43
|
|
45
44
|
# EventCounterQueue
|
@@ -50,7 +50,7 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
|
|
50
50
|
- AsyncIterable(): async for item in queue:
|
51
51
|
- Automatic termination of the consumers when the queue has been emptied with QueueDone exception
|
52
52
|
- Producers must be registered with add_producer() and they must notify the queue
|
53
|
-
with
|
53
|
+
with finish_producer() once they have finished adding items
|
54
54
|
- Countable interface to count number of items task_done() through 'count' property
|
55
55
|
|
56
56
|
IterableQueue stages:
|
@@ -166,7 +166,7 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
|
|
166
166
|
"""
|
167
167
|
Producer has finished adding items to the queue.
|
168
168
|
Once the last producers has finished, the queue is_filled.
|
169
|
-
- all:
|
169
|
+
- all: finish_producer() queue for all producers at once
|
170
170
|
|
171
171
|
Return True if the last producer is 'finished'
|
172
172
|
"""
|
@@ -177,7 +177,7 @@ class IterableQueue(Queue[T], AsyncIterable[T], Countable):
|
|
177
177
|
self._producers -= 1
|
178
178
|
|
179
179
|
if self._producers < 0:
|
180
|
-
raise ValueError("Too many
|
180
|
+
raise ValueError("Too many finish_producer() calls")
|
181
181
|
elif all or self._producers == 0:
|
182
182
|
self._filled.set()
|
183
183
|
self._producers = 0
|
@@ -232,7 +232,7 @@ async def test_6_finish_full_queue(test_interablequeue_int: IterableQueue[int]):
|
|
232
232
|
)
|
233
233
|
assert Q.is_done, "Queue is not done"
|
234
234
|
except TimeoutError:
|
235
|
-
assert False, f"await IterableQueue.
|
235
|
+
assert False, f"await IterableQueue.finish_producer() failed. qsize={Q.qsize()}"
|
236
236
|
await sleep(0.1)
|
237
237
|
assert Q.is_done, "Queue is not done"
|
238
238
|
producer.cancel()
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|