asyncio-extensions 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/.github/workflows/release.yml +1 -1
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/.github/workflows/tests.yml +10 -6
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/.pre-commit-config.yaml +1 -1
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/AGENTS.md +2 -2
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/CHANGELOG.md +12 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/PKG-INFO +45 -2
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/README.md +42 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/pyproject.toml +12 -2
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/src/asyncio_extensions/__init__.py +2 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/src/asyncio_extensions/_compat.py +7 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/src/asyncio_extensions/_iterators.py +60 -23
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/src/asyncio_extensions/_scheduling.py +2 -2
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/src/asyncio_extensions/_sync.py +1 -1
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/src/asyncio_extensions/_task_groups.py +24 -18
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/tests/test_iterators.py +93 -5
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/tests/test_task_groups.py +1 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/.github/dependabot.yml +0 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/.gitignore +0 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/CLAUDE.md +0 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/LICENSE.txt +0 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/codecov.yml +0 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/src/asyncio_extensions/py.typed +0 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/tests/__init__.py +0 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/tests/test_scheduling.py +0 -0
- {asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/tests/test_sync.py +0 -0
|
@@ -19,7 +19,7 @@ jobs:
|
|
|
19
19
|
strategy:
|
|
20
20
|
fail-fast: false
|
|
21
21
|
steps:
|
|
22
|
-
- uses: actions/checkout@
|
|
22
|
+
- uses: actions/checkout@v7
|
|
23
23
|
|
|
24
24
|
- name: Set up Python 3.13
|
|
25
25
|
uses: actions/setup-python@v6
|
|
@@ -32,23 +32,27 @@ jobs:
|
|
|
32
32
|
- name: Run static analysis
|
|
33
33
|
run: hatch fmt --check
|
|
34
34
|
|
|
35
|
-
- name:
|
|
35
|
+
- name: Type checking with MyPy
|
|
36
36
|
run: hatch run mypy src/asyncio_extensions
|
|
37
37
|
|
|
38
|
+
- name: Type checking with Pyrefly
|
|
39
|
+
run: hatch run pyrefly check src/asyncio_extensions
|
|
40
|
+
|
|
38
41
|
tests:
|
|
39
42
|
name: Run tests on ${{ matrix.python-version }}
|
|
40
43
|
runs-on: ubuntu-latest
|
|
41
44
|
strategy:
|
|
42
45
|
fail-fast: false
|
|
43
46
|
matrix:
|
|
44
|
-
python-version: ["3.11", "3.12", "3.13", "3.14"]
|
|
47
|
+
python-version: ["3.11", "3.12", "3.13", "3.14", "3.15"]
|
|
45
48
|
steps:
|
|
46
|
-
- uses: actions/checkout@
|
|
49
|
+
- uses: actions/checkout@v7
|
|
47
50
|
|
|
48
51
|
- name: Set up Python ${{ matrix.python-version }}
|
|
49
52
|
uses: actions/setup-python@v6
|
|
50
53
|
with:
|
|
51
54
|
python-version: ${{ matrix.python-version }}
|
|
55
|
+
allow-prereleases: true
|
|
52
56
|
|
|
53
57
|
- name: Setup Hatch
|
|
54
58
|
run: pipx install hatch
|
|
@@ -58,13 +62,13 @@ jobs:
|
|
|
58
62
|
hatch test -i py=${{ matrix.python-version }} --cover --junitxml=junit.xml
|
|
59
63
|
|
|
60
64
|
- name: Upload results to Codecov
|
|
61
|
-
uses: codecov/codecov-action@
|
|
65
|
+
uses: codecov/codecov-action@v7
|
|
62
66
|
with:
|
|
63
67
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
64
68
|
|
|
65
69
|
- name: Upload test results to Codecov
|
|
66
70
|
if: ${{ !cancelled() }}
|
|
67
|
-
uses: codecov/codecov-action@
|
|
71
|
+
uses: codecov/codecov-action@v7
|
|
68
72
|
with:
|
|
69
73
|
report_type: test_results
|
|
70
74
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
@@ -12,7 +12,7 @@ All implementation lives under `src/asyncio_extensions/`. Every submodule is **p
|
|
|
12
12
|
|
|
13
13
|
| Module | Theme |
|
|
14
14
|
|---|---|
|
|
15
|
-
| `_compat.py` | Version compatibility shims (Python 3.11–3.
|
|
15
|
+
| `_compat.py` | Version compatibility shims (Python 3.11–3.15). No new features here. |
|
|
16
16
|
| `_sync.py` | Bridging sync and async code. |
|
|
17
17
|
| `_scheduling.py` | Event-loop control utilities. |
|
|
18
18
|
| `_iterators.py` | Async iteration utilities. |
|
|
@@ -55,7 +55,7 @@ from asyncio_extensions import checkpoint # helper from another module
|
|
|
55
55
|
|
|
56
56
|
Maintain the same order as the source module: tests for each symbol should appear in the order the symbols are defined, and within a symbol's tests, individual cases should follow the order of branches and conditions in the implementation.
|
|
57
57
|
|
|
58
|
-
The test suite runs against Python 3.11, 3.12, 3.13, and 3.
|
|
58
|
+
The test suite runs against Python 3.11, 3.12, 3.13, 3.14 and 3.15 in CI. When writing version-specific tests, use `@pytest.mark.skipif(sys.version_info < (3, X), ...)`.
|
|
59
59
|
|
|
60
60
|
## Compatibility
|
|
61
61
|
|
|
@@ -8,6 +8,18 @@ This project adheres to both [Semantic Versioning](https://semver.org/spec/v2.0.
|
|
|
8
8
|
|
|
9
9
|
## [UNRELEASED]
|
|
10
10
|
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- `drain` coroutine to consume and discard remaining items from an async iterator or async iterable.
|
|
14
|
+
- `merge_streams` managed-stream helper to merge multiple `ManagedStream` sources into a single interleaved stream.
|
|
15
|
+
|
|
16
|
+
### Changed
|
|
17
|
+
|
|
18
|
+
- Type validation now includes Pyrefly checks for enhanced type safety.
|
|
19
|
+
- Enabled mypy strict mode to enforce stricter type checking standards.
|
|
20
|
+
- `STOP` is now a proper sentinel instance (via `typing_extensions.sentinel` on Python < 3.15) instead of a class used as a sentinel. Behaviour is unchanged; only the type changes.
|
|
21
|
+
- Use built-in TaskGroup on Python 3.15+
|
|
22
|
+
|
|
11
23
|
## [0.2.0] - 2026-05-15
|
|
12
24
|
|
|
13
25
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: asyncio-extensions
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Some (maybe) useful extensions for asyncio
|
|
5
5
|
Project-URL: Changelog, https://github.com/hartungstenio/asyncio-extensions/blob/main/CHANGELOG.md
|
|
6
6
|
Project-URL: Documentation, https://github.com/hartungstenio/asyncio-extensions#readme
|
|
@@ -19,12 +19,13 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
22
23
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
23
24
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
24
25
|
Classifier: Topic :: Software Development :: Libraries
|
|
25
26
|
Classifier: Typing :: Typed
|
|
26
27
|
Requires-Python: >=3.11
|
|
27
|
-
Requires-Dist: typing-extensions; python_version < '3.
|
|
28
|
+
Requires-Dist: typing-extensions>=4.16; python_version < '3.15'
|
|
28
29
|
Description-Content-Type: text/markdown
|
|
29
30
|
|
|
30
31
|
# asyncio-extensions
|
|
@@ -269,6 +270,25 @@ async def source():
|
|
|
269
270
|
await fill_queue(source(), queue)
|
|
270
271
|
```
|
|
271
272
|
|
|
273
|
+
### drain
|
|
274
|
+
|
|
275
|
+
The `drain` coroutine consumes and discards the remaining items from an async iterator or async iterable. This is useful when you want to stop processing early but must ensure the source is exhausted so producers can finish and resources can be released.
|
|
276
|
+
|
|
277
|
+
Example — process only the first N items, then discard the rest so the producer can complete:
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
from asyncio_extensions import drain
|
|
281
|
+
|
|
282
|
+
async def iter_items():
|
|
283
|
+
... # yields items that need to process
|
|
284
|
+
|
|
285
|
+
async for idx, item in enumerate(iter_items()):
|
|
286
|
+
if idx >= 5:
|
|
287
|
+
break
|
|
288
|
+
print(item)
|
|
289
|
+
# ensure the remaining items are consumed and the producer can finish
|
|
290
|
+
await drain(stream)
|
|
291
|
+
```
|
|
272
292
|
### merge_iterables
|
|
273
293
|
|
|
274
294
|
The `merge_iterables` async context manager merges multiple sync or async iterables into a single interleaved stream, feeding all sources into a shared queue concurrently.
|
|
@@ -306,6 +326,29 @@ def my_stream() -> ManagedStream[int]:
|
|
|
306
326
|
...
|
|
307
327
|
```
|
|
308
328
|
|
|
329
|
+
### merge_streams
|
|
330
|
+
|
|
331
|
+
`merge_streams` is the managed-stream equivalent of `merge_iterables`. It accepts one or more `ManagedStream` objects (context managers that yield `AsyncIterator`s) and returns a single managed stream that interleaves items from all sources. Use it when each source requires context-managed setup/cleanup (for example, generators wrapped with `safe_gen` or `asynccontextmanager`-based streams).
|
|
332
|
+
|
|
333
|
+
Example — merging two managed streams produced by `safe_gen`:
|
|
334
|
+
|
|
335
|
+
```python
|
|
336
|
+
from asyncio_extensions import merge_streams, safe_gen, checkpoint
|
|
337
|
+
|
|
338
|
+
@safe_gen
|
|
339
|
+
async def source(name: str):
|
|
340
|
+
for i in range(3):
|
|
341
|
+
yield f"{name}-{i}"
|
|
342
|
+
await checkpoint()
|
|
343
|
+
|
|
344
|
+
async def main():
|
|
345
|
+
async with merge_streams(source("a"), source("b")) as stream:
|
|
346
|
+
async for item in stream:
|
|
347
|
+
print(item)
|
|
348
|
+
```
|
|
349
|
+
|
|
350
|
+
Each input managed stream is guaranteed to be closed when the merged context exits, ensuring proper cleanup of resources.
|
|
351
|
+
|
|
309
352
|
### safe_gen
|
|
310
353
|
|
|
311
354
|
The `safe_gen` decorator converts an async generator function into a context manager, enforcing correct handling of early exits. A plain async generator abandoned before exhaustion leaks resources and keeps running indefinitely — the caller has no way to know it needs to call `aclose()`. By returning a context manager instead, `safe_gen` makes cleanup syntactically mandatory: callers must use `async with`, which guarantees `aclose()` is called on exit regardless of how iteration ends. It also suppresses `GeneratorExit` raised inside an exception group, so it composes safely with `TaskGroup`.
|
|
@@ -240,6 +240,25 @@ async def source():
|
|
|
240
240
|
await fill_queue(source(), queue)
|
|
241
241
|
```
|
|
242
242
|
|
|
243
|
+
### drain
|
|
244
|
+
|
|
245
|
+
The `drain` coroutine consumes and discards the remaining items from an async iterator or async iterable. This is useful when you want to stop processing early but must ensure the source is exhausted so producers can finish and resources can be released.
|
|
246
|
+
|
|
247
|
+
Example — process only the first N items, then discard the rest so the producer can complete:
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
from asyncio_extensions import drain
|
|
251
|
+
|
|
252
|
+
async def iter_items():
|
|
253
|
+
... # yields items that need to process
|
|
254
|
+
|
|
255
|
+
async for idx, item in enumerate(iter_items()):
|
|
256
|
+
if idx >= 5:
|
|
257
|
+
break
|
|
258
|
+
print(item)
|
|
259
|
+
# ensure the remaining items are consumed and the producer can finish
|
|
260
|
+
await drain(stream)
|
|
261
|
+
```
|
|
243
262
|
### merge_iterables
|
|
244
263
|
|
|
245
264
|
The `merge_iterables` async context manager merges multiple sync or async iterables into a single interleaved stream, feeding all sources into a shared queue concurrently.
|
|
@@ -277,6 +296,29 @@ def my_stream() -> ManagedStream[int]:
|
|
|
277
296
|
...
|
|
278
297
|
```
|
|
279
298
|
|
|
299
|
+
### merge_streams
|
|
300
|
+
|
|
301
|
+
`merge_streams` is the managed-stream equivalent of `merge_iterables`. It accepts one or more `ManagedStream` objects (context managers that yield `AsyncIterator`s) and returns a single managed stream that interleaves items from all sources. Use it when each source requires context-managed setup/cleanup (for example, generators wrapped with `safe_gen` or `asynccontextmanager`-based streams).
|
|
302
|
+
|
|
303
|
+
Example — merging two managed streams produced by `safe_gen`:
|
|
304
|
+
|
|
305
|
+
```python
|
|
306
|
+
from asyncio_extensions import merge_streams, safe_gen, checkpoint
|
|
307
|
+
|
|
308
|
+
@safe_gen
|
|
309
|
+
async def source(name: str):
|
|
310
|
+
for i in range(3):
|
|
311
|
+
yield f"{name}-{i}"
|
|
312
|
+
await checkpoint()
|
|
313
|
+
|
|
314
|
+
async def main():
|
|
315
|
+
async with merge_streams(source("a"), source("b")) as stream:
|
|
316
|
+
async for item in stream:
|
|
317
|
+
print(item)
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
Each input managed stream is guaranteed to be closed when the merged context exits, ensuring proper cleanup of resources.
|
|
321
|
+
|
|
280
322
|
### safe_gen
|
|
281
323
|
|
|
282
324
|
The `safe_gen` decorator converts an async generator function into a context manager, enforcing correct handling of early exits. A plain async generator abandoned before exhaustion leaks resources and keeps running indefinitely — the caller has no way to know it needs to call `aclose()`. By returning a context manager instead, `safe_gen` makes cleanup syntactically mandatory: callers must use `async with`, which guarantees `aclose()` is called on exit regardless of how iteration ends. It also suppresses `GeneratorExit` raised inside an exception group, so it composes safely with `TaskGroup`.
|
|
@@ -18,13 +18,14 @@ classifiers = [
|
|
|
18
18
|
"Programming Language :: Python :: 3.12",
|
|
19
19
|
"Programming Language :: Python :: 3.13",
|
|
20
20
|
"Programming Language :: Python :: 3.14",
|
|
21
|
+
"Programming Language :: Python :: 3.15",
|
|
21
22
|
"Programming Language :: Python :: Implementation :: CPython",
|
|
22
23
|
"Programming Language :: Python :: Implementation :: PyPy",
|
|
23
24
|
"Topic :: Software Development :: Libraries",
|
|
24
25
|
"Typing :: Typed",
|
|
25
26
|
]
|
|
26
27
|
dependencies = [
|
|
27
|
-
"typing-extensions; python_version < '3.
|
|
28
|
+
"typing-extensions (>= 4.16); python_version < '3.15'",
|
|
28
29
|
]
|
|
29
30
|
dynamic = ["version"]
|
|
30
31
|
|
|
@@ -40,6 +41,7 @@ dev = [
|
|
|
40
41
|
"faker",
|
|
41
42
|
"mypy",
|
|
42
43
|
"pre-commit",
|
|
44
|
+
"pyrefly",
|
|
43
45
|
"pytest",
|
|
44
46
|
"pytest-asyncio",
|
|
45
47
|
"pytest-cov",
|
|
@@ -87,7 +89,7 @@ extra-dependencies = [
|
|
|
87
89
|
]
|
|
88
90
|
|
|
89
91
|
[[tool.hatch.envs.hatch-test.matrix]]
|
|
90
|
-
python = ["3.11", "3.12", "3.13", "3.14"]
|
|
92
|
+
python = ["3.11", "3.12", "3.13", "3.14", "3.15"]
|
|
91
93
|
|
|
92
94
|
[tool.hatch.envs.hatch-test.scripts]
|
|
93
95
|
run = "pytest{env:HATCH_TEST_ARGS:} {args}"
|
|
@@ -95,6 +97,14 @@ run-cov = "coverage run -m pytest{env:HATCH_TEST_ARGS:} {args}"
|
|
|
95
97
|
cov-combine = "coverage combine"
|
|
96
98
|
cov-report = "coverage xml"
|
|
97
99
|
|
|
100
|
+
[tool.mypy]
|
|
101
|
+
strict = true
|
|
102
|
+
enable_error_code = ["deprecated"]
|
|
103
|
+
|
|
104
|
+
[tool.pyrefly]
|
|
105
|
+
preset = "strict"
|
|
106
|
+
min-severity = "ignore"
|
|
107
|
+
|
|
98
108
|
[tool.pytest]
|
|
99
109
|
asyncio_default_fixture_loop_scope = "function"
|
|
100
110
|
|
|
@@ -4,10 +4,12 @@ from ._compat import CreateTaskParams as CreateTaskParams
|
|
|
4
4
|
from ._compat import iscoroutinefunction as iscoroutinefunction
|
|
5
5
|
from ._iterators import STOP as STOP
|
|
6
6
|
from ._iterators import ManagedStream as ManagedStream
|
|
7
|
+
from ._iterators import drain as drain
|
|
7
8
|
from ._iterators import fill_queue as fill_queue
|
|
8
9
|
from ._iterators import flatten_stream as flatten_stream
|
|
9
10
|
from ._iterators import iterate_queue as iterate_queue
|
|
10
11
|
from ._iterators import merge_iterables as merge_iterables
|
|
12
|
+
from ._iterators import merge_streams as merge_streams
|
|
11
13
|
from ._iterators import safe_gen as safe_gen
|
|
12
14
|
from ._scheduling import checkpoint as checkpoint
|
|
13
15
|
from ._scheduling import heartbeat as heartbeat
|
|
@@ -45,6 +45,12 @@ if sys.version_info >= (3, 14):
|
|
|
45
45
|
else:
|
|
46
46
|
CreateTaskParams = _CreateTaskParams
|
|
47
47
|
|
|
48
|
+
if sys.version_info >= (3, 15):
|
|
49
|
+
import builtins
|
|
50
|
+
|
|
51
|
+
sentinel = builtins.sentinel
|
|
52
|
+
else:
|
|
53
|
+
from typing_extensions import sentinel # type: ignore[attr-defined]
|
|
48
54
|
|
|
49
55
|
__all__ = [
|
|
50
56
|
"CreateTaskParams",
|
|
@@ -53,4 +59,5 @@ __all__ = [
|
|
|
53
59
|
"iscoroutinefunction",
|
|
54
60
|
"markcoroutinefunction",
|
|
55
61
|
"override",
|
|
62
|
+
"sentinel",
|
|
56
63
|
]
|
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import sys
|
|
5
5
|
from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Callable, Iterable
|
|
6
|
-
from contextlib import AbstractAsyncContextManager, aclosing, asynccontextmanager
|
|
6
|
+
from contextlib import AbstractAsyncContextManager, AsyncExitStack, aclosing, asynccontextmanager
|
|
7
7
|
from functools import wraps
|
|
8
|
-
from typing import
|
|
8
|
+
from typing import ParamSpec, TypeAlias, TypeVar
|
|
9
9
|
|
|
10
|
-
from ._compat import QueueShutDown
|
|
10
|
+
from ._compat import QueueShutDown, sentinel
|
|
11
11
|
from ._sync import asyncify_iterable
|
|
12
12
|
from ._task_groups import TaskGroup
|
|
13
13
|
|
|
@@ -25,30 +25,26 @@ that return a context-managed async stream::
|
|
|
25
25
|
"""
|
|
26
26
|
|
|
27
27
|
if sys.version_info >= (3, 13):
|
|
28
|
-
|
|
28
|
+
STOP = sentinel("STOP")
|
|
29
|
+
"""Sentinel that signals the end of an :func:`iterate_queue` stream.
|
|
29
30
|
|
|
30
|
-
|
|
31
|
-
class STOP:
|
|
32
|
-
"""Sentinel that signals the end of an :func:`iterate_queue` stream.
|
|
31
|
+
Put this class in the queue to stop iteration::
|
|
33
32
|
|
|
34
|
-
|
|
33
|
+
await queue.put(STOP)
|
|
35
34
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
Prefer :meth:`asyncio.Queue.shutdown` on Python 3.13+.
|
|
40
|
-
"""
|
|
35
|
+
Deprecated:
|
|
36
|
+
Prefer :meth:`asyncio.Queue.shutdown` on Python 3.13+.
|
|
37
|
+
"""
|
|
41
38
|
|
|
42
39
|
|
|
43
40
|
else:
|
|
41
|
+
STOP = sentinel("STOP")
|
|
42
|
+
"""Sentinel that signals the end of an :func:`iterate_queue` stream.
|
|
44
43
|
|
|
45
|
-
class
|
|
46
|
-
"""Sentinel that signals the end of an :func:`iterate_queue` stream.
|
|
44
|
+
Put this class in the queue to stop iteration::
|
|
47
45
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
await queue.put(STOP)
|
|
51
|
-
"""
|
|
46
|
+
await queue.put(STOP)
|
|
47
|
+
"""
|
|
52
48
|
|
|
53
49
|
|
|
54
50
|
async def iterate_queue(queue: asyncio.Queue[T]) -> AsyncGenerator[T]:
|
|
@@ -100,10 +96,26 @@ async def fill_queue(itr: AsyncIterable[T] | Iterable[T], queue: asyncio.Queue[T
|
|
|
100
96
|
await queue.put(it)
|
|
101
97
|
|
|
102
98
|
|
|
99
|
+
async def drain(itr: AsyncIterable[T] | Iterable[T]) -> None:
|
|
100
|
+
"""Consume the remaining items from *itr*.
|
|
101
|
+
|
|
102
|
+
Accepts both sync and async iterables.
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
itr: The iterable (sync or async) to consume.
|
|
106
|
+
|
|
107
|
+
Example::
|
|
108
|
+
|
|
109
|
+
await drain(range(10))
|
|
110
|
+
"""
|
|
111
|
+
async for _ in asyncify_iterable(itr):
|
|
112
|
+
pass
|
|
113
|
+
|
|
114
|
+
|
|
103
115
|
@asynccontextmanager
|
|
104
116
|
async def merge_iterables(
|
|
105
117
|
*itrs: AsyncIterable[T] | Iterable[T],
|
|
106
|
-
) ->
|
|
118
|
+
) -> AsyncGenerator[AsyncGenerator[T]]:
|
|
107
119
|
"""Merge multiple iterables into a single async stream.
|
|
108
120
|
|
|
109
121
|
Feeds all *itrs* into a shared queue concurrently and yields a single
|
|
@@ -133,7 +145,7 @@ async def merge_iterables(
|
|
|
133
145
|
|
|
134
146
|
async def join() -> None:
|
|
135
147
|
await asyncio.wait(tasks)
|
|
136
|
-
await queue.put(STOP)
|
|
148
|
+
await queue.put(STOP)
|
|
137
149
|
|
|
138
150
|
tg.create_task(join())
|
|
139
151
|
|
|
@@ -143,7 +155,32 @@ async def merge_iterables(
|
|
|
143
155
|
tg.cancel()
|
|
144
156
|
|
|
145
157
|
|
|
146
|
-
|
|
158
|
+
@asynccontextmanager
|
|
159
|
+
async def merge_streams(*streams: ManagedStream[T]) -> AsyncGenerator[AsyncGenerator[T]]:
|
|
160
|
+
"""Merge multiple managed streams into a single async stream.
|
|
161
|
+
|
|
162
|
+
Similar to :func:`merge_iterables`, but work with managed streams.
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
*streams: Any number of managed streams to merge.
|
|
166
|
+
|
|
167
|
+
Yields:
|
|
168
|
+
An async generator producing items from all *itrs* interleaved.
|
|
169
|
+
|
|
170
|
+
Example::
|
|
171
|
+
|
|
172
|
+
async with merge_streams(source_a, source_b) as stream:
|
|
173
|
+
async for item in stream:
|
|
174
|
+
process(item)
|
|
175
|
+
"""
|
|
176
|
+
async with AsyncExitStack() as exit_stack:
|
|
177
|
+
itrs = [await exit_stack.enter_async_context(stream) for stream in streams]
|
|
178
|
+
merged = await exit_stack.enter_async_context(merge_iterables(*itrs))
|
|
179
|
+
|
|
180
|
+
yield merged
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def safe_gen(fn: Callable[P, AsyncGenerator[T]]) -> Callable[P, ManagedStream[T]]:
|
|
147
184
|
"""Wrap an async generator function so it is always closed on exit.
|
|
148
185
|
|
|
149
186
|
Async generators must be explicitly closed when iteration is abandoned early — otherwise
|
|
@@ -169,7 +206,7 @@ def safe_gen(fn: Callable[P, AsyncGenerator[T, Any]]) -> Callable[P, ManagedStre
|
|
|
169
206
|
|
|
170
207
|
@asynccontextmanager
|
|
171
208
|
@wraps(fn)
|
|
172
|
-
async def decorator(*args: P.args, **kwargs: P.kwargs) ->
|
|
209
|
+
async def decorator(*args: P.args, **kwargs: P.kwargs) -> AsyncGenerator[AsyncGenerator[T]]:
|
|
173
210
|
try:
|
|
174
211
|
async with aclosing(fn(*args, **kwargs)) as agen:
|
|
175
212
|
yield agen
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
4
|
from collections.abc import Awaitable, Callable, Generator
|
|
5
|
-
from typing import
|
|
5
|
+
from typing import Never, ParamSpec, TypeVar
|
|
6
6
|
|
|
7
7
|
T = TypeVar("T")
|
|
8
8
|
P = ParamSpec("P")
|
|
@@ -11,7 +11,7 @@ P = ParamSpec("P")
|
|
|
11
11
|
class _YieldToEventLoop:
|
|
12
12
|
"""Helper class to give control back to the event loop."""
|
|
13
13
|
|
|
14
|
-
def __await__(self) -> Generator[None
|
|
14
|
+
def __await__(self) -> Generator[None]:
|
|
15
15
|
"""Yield control to the event loop."""
|
|
16
16
|
yield
|
|
17
17
|
|
|
@@ -53,7 +53,7 @@ def asyncify(func: Callable[_P, _R] | Callable[_P, Awaitable[_R]], /) -> Callabl
|
|
|
53
53
|
return wrapper
|
|
54
54
|
|
|
55
55
|
|
|
56
|
-
def markcoroutinefunction(f: Callable[_P, _R]) -> Callable[_P, Coroutine[
|
|
56
|
+
def markcoroutinefunction(f: Callable[_P, _R]) -> Callable[_P, Coroutine[None, None, _R]]:
|
|
57
57
|
"""Mark *f* as a coroutine function without making it async.
|
|
58
58
|
|
|
59
59
|
After marking, :func:`asyncio.iscoroutinefunction` returns ``True``
|
{asyncio_extensions-0.2.0 → asyncio_extensions-0.3.0}/src/asyncio_extensions/_task_groups.py
RENAMED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
"""Extensions to :mod:`asyncio` task groups."""
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
|
+
import sys
|
|
4
5
|
from collections.abc import Coroutine
|
|
5
6
|
from types import TracebackType
|
|
6
|
-
from typing import
|
|
7
|
+
from typing import Never, TypeVar, Unpack
|
|
7
8
|
|
|
8
9
|
from ._compat import CreateTaskParams, override
|
|
9
10
|
|
|
@@ -24,23 +25,28 @@ async def force_terminate_task_group() -> Never:
|
|
|
24
25
|
raise TerminateTaskGroup
|
|
25
26
|
|
|
26
27
|
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
if sys.version_info >= (3, 15):
|
|
29
|
+
from asyncio import TaskGroup
|
|
30
|
+
else:
|
|
29
31
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
32
|
+
class TaskGroup(asyncio.TaskGroup):
|
|
33
|
+
"""An :class:`asyncio.TaskGroup` extended with a :meth:`cancel` method."""
|
|
34
|
+
|
|
35
|
+
@override
|
|
36
|
+
async def __aexit__(
|
|
37
|
+
self,
|
|
38
|
+
et: type[BaseException] | None,
|
|
39
|
+
exc: BaseException | None,
|
|
40
|
+
tb: TracebackType | None,
|
|
41
|
+
) -> None:
|
|
42
|
+
try:
|
|
43
|
+
await super().__aexit__(et, exc, tb)
|
|
44
|
+
except* TerminateTaskGroup:
|
|
45
|
+
pass
|
|
40
46
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
47
|
+
def cancel(self) -> None:
|
|
48
|
+
"""Schedule cancellation of all remaining tasks in the group."""
|
|
49
|
+
self.create_task(force_terminate_task_group())
|
|
44
50
|
|
|
45
51
|
|
|
46
52
|
class LimitedTaskGroup(TaskGroup):
|
|
@@ -54,14 +60,14 @@ class LimitedTaskGroup(TaskGroup):
|
|
|
54
60
|
super().__init__()
|
|
55
61
|
self._semaphore = asyncio.Semaphore(max_concurrent)
|
|
56
62
|
|
|
57
|
-
async def _task_wrapper(self, coro: Coroutine[
|
|
63
|
+
async def _task_wrapper(self, coro: Coroutine[None, None, T]) -> T:
|
|
58
64
|
async with self._semaphore:
|
|
59
65
|
return await coro
|
|
60
66
|
|
|
61
67
|
@override
|
|
62
68
|
def create_task(
|
|
63
69
|
self,
|
|
64
|
-
coro: Coroutine[
|
|
70
|
+
coro: Coroutine[None, None, T],
|
|
65
71
|
**kwargs: Unpack[CreateTaskParams],
|
|
66
72
|
) -> asyncio.Task[T]:
|
|
67
73
|
"""Schedule *coro* as a task, subject to the concurrency limit.
|
|
@@ -1,20 +1,31 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import sys
|
|
3
3
|
from collections.abc import AsyncGenerator
|
|
4
|
+
from contextlib import aclosing
|
|
4
5
|
from itertools import count
|
|
5
6
|
|
|
6
7
|
import pytest
|
|
7
8
|
|
|
8
9
|
from asyncio_extensions import checkpoint
|
|
9
|
-
from asyncio_extensions._iterators import
|
|
10
|
+
from asyncio_extensions._iterators import (
|
|
11
|
+
STOP,
|
|
12
|
+
drain,
|
|
13
|
+
fill_queue,
|
|
14
|
+
flatten_stream,
|
|
15
|
+
iterate_queue,
|
|
16
|
+
merge_iterables,
|
|
17
|
+
merge_streams,
|
|
18
|
+
safe_gen,
|
|
19
|
+
)
|
|
10
20
|
|
|
11
21
|
pytestmark = pytest.mark.asyncio
|
|
12
22
|
|
|
13
23
|
|
|
14
|
-
def
|
|
24
|
+
def _drain_queue(queue: asyncio.Queue[int]) -> list[int]:
|
|
15
25
|
items = []
|
|
16
26
|
while not queue.empty():
|
|
17
27
|
items.append(queue.get_nowait())
|
|
28
|
+
queue.task_done()
|
|
18
29
|
return items
|
|
19
30
|
|
|
20
31
|
|
|
@@ -88,7 +99,7 @@ async def test_fill_queue_sync_iterable_fills_queue() -> None:
|
|
|
88
99
|
|
|
89
100
|
await fill_queue([1, 2, 3], queue)
|
|
90
101
|
|
|
91
|
-
assert
|
|
102
|
+
assert _drain_queue(queue) == [1, 2, 3]
|
|
92
103
|
|
|
93
104
|
|
|
94
105
|
async def test_fill_queue_async_iterable_fills_queue() -> None:
|
|
@@ -100,7 +111,7 @@ async def test_fill_queue_async_iterable_fills_queue() -> None:
|
|
|
100
111
|
|
|
101
112
|
await fill_queue(source(), queue)
|
|
102
113
|
|
|
103
|
-
assert
|
|
114
|
+
assert _drain_queue(queue) == [4, 5, 6]
|
|
104
115
|
|
|
105
116
|
|
|
106
117
|
async def test_fill_queue_empty_iterable_queue_remains_empty() -> None:
|
|
@@ -123,7 +134,32 @@ async def test_fill_queue_full_queue_blocks_until_space_available() -> None:
|
|
|
123
134
|
|
|
124
135
|
queue.get_nowait()
|
|
125
136
|
|
|
126
|
-
assert
|
|
137
|
+
assert _drain_queue(queue) == [1]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# drain
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
async def test_drain_sync_iterator() -> None:
|
|
144
|
+
itr = iter([1, 2, 3])
|
|
145
|
+
|
|
146
|
+
await drain(itr)
|
|
147
|
+
|
|
148
|
+
with pytest.raises(StopIteration):
|
|
149
|
+
next(itr)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
async def test_drain_async_iterator() -> None:
|
|
153
|
+
async def gen() -> AsyncGenerator[int]:
|
|
154
|
+
for i in range(3):
|
|
155
|
+
yield i
|
|
156
|
+
|
|
157
|
+
itr = aiter(gen())
|
|
158
|
+
|
|
159
|
+
await drain(itr)
|
|
160
|
+
|
|
161
|
+
with pytest.raises(StopAsyncIteration):
|
|
162
|
+
await anext(itr)
|
|
127
163
|
|
|
128
164
|
|
|
129
165
|
# merge_iterables
|
|
@@ -174,6 +210,58 @@ async def test_merge_iterables_early_exit_cancels_background_tasks() -> None:
|
|
|
174
210
|
assert len(asyncio.all_tasks()) == initial_tasks
|
|
175
211
|
|
|
176
212
|
|
|
213
|
+
# merge_streams
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
async def test_merge_streams_yields_all_items() -> None:
|
|
217
|
+
async def source(values: list[int]) -> AsyncGenerator[int]:
|
|
218
|
+
for v in values:
|
|
219
|
+
yield v
|
|
220
|
+
|
|
221
|
+
async with merge_streams(aclosing(source([10, 20])), aclosing(source([30, 40]))) as stream:
|
|
222
|
+
results = [item async for item in stream]
|
|
223
|
+
|
|
224
|
+
assert sorted(results) == [10, 20, 30, 40]
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
async def test_merge_streams_single_source_yields_all_items() -> None:
|
|
228
|
+
async def source(values: list[int]) -> AsyncGenerator[int]:
|
|
229
|
+
for v in values:
|
|
230
|
+
yield v
|
|
231
|
+
|
|
232
|
+
async with merge_streams(aclosing(source([1, 2, 3]))) as stream:
|
|
233
|
+
results = [item async for item in stream]
|
|
234
|
+
|
|
235
|
+
assert results == [1, 2, 3]
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
async def test_merge_streams_empty_sources_yields_nothing() -> None:
|
|
239
|
+
async def source() -> AsyncGenerator[int]:
|
|
240
|
+
if False:
|
|
241
|
+
yield
|
|
242
|
+
|
|
243
|
+
async with merge_streams(aclosing(source())) as stream:
|
|
244
|
+
results = [item async for item in stream]
|
|
245
|
+
|
|
246
|
+
assert results == []
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
async def test_merge_streams_early_exit_cancels_background_tasks() -> None:
|
|
250
|
+
initial_tasks = len(asyncio.all_tasks())
|
|
251
|
+
|
|
252
|
+
async def source(values: list[int]) -> AsyncGenerator[int]:
|
|
253
|
+
for v in values:
|
|
254
|
+
yield v
|
|
255
|
+
|
|
256
|
+
async with merge_streams(aclosing(source([10, 20])), aclosing(source([30, 40]))) as stream:
|
|
257
|
+
assert len(asyncio.all_tasks()) > initial_tasks
|
|
258
|
+
|
|
259
|
+
async for _ in stream:
|
|
260
|
+
break
|
|
261
|
+
|
|
262
|
+
assert len(asyncio.all_tasks()) == initial_tasks
|
|
263
|
+
|
|
264
|
+
|
|
177
265
|
async def test_safe_gen_full_iteration() -> None:
|
|
178
266
|
@safe_gen
|
|
179
267
|
async def gen(items: int) -> AsyncGenerator[int]:
|
|
@@ -24,6 +24,7 @@ async def test_force_terminate_task_group() -> None:
|
|
|
24
24
|
tg.create_task(force_terminate_task_group())
|
|
25
25
|
|
|
26
26
|
|
|
27
|
+
@pytest.mark.skipif(sys.version_info >= (3, 15), reason="Skipping testing built-in TaskGroup")
|
|
27
28
|
class TestTaskGroup:
|
|
28
29
|
async def test_runs_tasks(self) -> None:
|
|
29
30
|
async with TaskGroup() as tg:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|