ez-a-sync 0.22.14__py3-none-any.whl → 0.22.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ez-a-sync might be problematic. Click here for more details.
- a_sync/ENVIRONMENT_VARIABLES.py +4 -3
- a_sync/__init__.py +30 -12
- a_sync/_smart.py +132 -28
- a_sync/_typing.py +56 -12
- a_sync/a_sync/__init__.py +35 -10
- a_sync/a_sync/_descriptor.py +74 -26
- a_sync/a_sync/_flags.py +14 -6
- a_sync/a_sync/_helpers.py +8 -7
- a_sync/a_sync/_kwargs.py +3 -2
- a_sync/a_sync/_meta.py +120 -28
- a_sync/a_sync/abstract.py +102 -28
- a_sync/a_sync/base.py +34 -16
- a_sync/a_sync/config.py +47 -13
- a_sync/a_sync/decorator.py +239 -117
- a_sync/a_sync/function.py +416 -146
- a_sync/a_sync/method.py +197 -59
- a_sync/a_sync/modifiers/__init__.py +47 -5
- a_sync/a_sync/modifiers/cache/__init__.py +46 -17
- a_sync/a_sync/modifiers/cache/memory.py +86 -20
- a_sync/a_sync/modifiers/limiter.py +52 -22
- a_sync/a_sync/modifiers/manager.py +98 -16
- a_sync/a_sync/modifiers/semaphores.py +48 -15
- a_sync/a_sync/property.py +383 -82
- a_sync/a_sync/singleton.py +1 -0
- a_sync/aliases.py +0 -1
- a_sync/asyncio/__init__.py +4 -1
- a_sync/asyncio/as_completed.py +177 -49
- a_sync/asyncio/create_task.py +31 -17
- a_sync/asyncio/gather.py +72 -52
- a_sync/asyncio/utils.py +3 -3
- a_sync/exceptions.py +78 -23
- a_sync/executor.py +118 -71
- a_sync/future.py +575 -158
- a_sync/iter.py +110 -50
- a_sync/primitives/__init__.py +14 -2
- a_sync/primitives/_debug.py +13 -13
- a_sync/primitives/_loggable.py +5 -4
- a_sync/primitives/locks/__init__.py +5 -2
- a_sync/primitives/locks/counter.py +38 -36
- a_sync/primitives/locks/event.py +21 -7
- a_sync/primitives/locks/prio_semaphore.py +182 -62
- a_sync/primitives/locks/semaphore.py +78 -77
- a_sync/primitives/queue.py +560 -58
- a_sync/sphinx/__init__.py +0 -1
- a_sync/sphinx/ext.py +160 -50
- a_sync/task.py +262 -97
- a_sync/utils/__init__.py +12 -6
- a_sync/utils/iterators.py +127 -43
- {ez_a_sync-0.22.14.dist-info → ez_a_sync-0.22.15.dist-info}/METADATA +1 -1
- ez_a_sync-0.22.15.dist-info/RECORD +74 -0
- {ez_a_sync-0.22.14.dist-info → ez_a_sync-0.22.15.dist-info}/WHEEL +1 -1
- tests/conftest.py +1 -2
- tests/executor.py +112 -9
- tests/fixtures.py +61 -32
- tests/test_abstract.py +7 -4
- tests/test_as_completed.py +54 -21
- tests/test_base.py +66 -17
- tests/test_cache.py +31 -15
- tests/test_decorator.py +54 -28
- tests/test_executor.py +8 -13
- tests/test_future.py +45 -8
- tests/test_gather.py +8 -2
- tests/test_helpers.py +2 -0
- tests/test_iter.py +55 -13
- tests/test_limiter.py +5 -3
- tests/test_meta.py +23 -9
- tests/test_modified.py +4 -1
- tests/test_semaphore.py +15 -8
- tests/test_singleton.py +15 -10
- tests/test_task.py +126 -28
- ez_a_sync-0.22.14.dist-info/RECORD +0 -74
- {ez_a_sync-0.22.14.dist-info → ez_a_sync-0.22.15.dist-info}/LICENSE.txt +0 -0
- {ez_a_sync-0.22.14.dist-info → ez_a_sync-0.22.15.dist-info}/top_level.txt +0 -0
a_sync/a_sync/singleton.py
CHANGED
a_sync/aliases.py
CHANGED
a_sync/asyncio/__init__.py
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
"""
|
|
2
|
-
This package
|
|
2
|
+
This package provides custom utilities and extensions to the builtin `asyncio` package.
|
|
3
|
+
|
|
4
|
+
These utilities include enhanced versions of common asyncio functions, offering additional
|
|
5
|
+
features and improved functionality for asynchronous programming.
|
|
3
6
|
"""
|
|
4
7
|
|
|
5
8
|
from a_sync.asyncio.as_completed import as_completed
|
a_sync/asyncio/as_completed.py
CHANGED
|
@@ -7,26 +7,65 @@ import asyncio
|
|
|
7
7
|
try:
|
|
8
8
|
from tqdm.asyncio import tqdm_asyncio
|
|
9
9
|
except ImportError as e:
|
|
10
|
+
|
|
10
11
|
class tqdm_asyncio: # type: ignore [no-redef]
|
|
11
12
|
def as_completed(*args, **kwargs):
|
|
12
13
|
raise ImportError("You must have tqdm installed to use this feature")
|
|
13
|
-
|
|
14
|
+
|
|
15
|
+
|
|
14
16
|
from a_sync._typing import *
|
|
15
17
|
from a_sync.iter import ASyncIterator
|
|
16
18
|
|
|
19
|
+
|
|
17
20
|
@overload
|
|
18
|
-
def as_completed(
|
|
19
|
-
|
|
21
|
+
def as_completed(
|
|
22
|
+
fs: Iterable[Awaitable[T]],
|
|
23
|
+
*,
|
|
24
|
+
timeout: Optional[float] = None,
|
|
25
|
+
return_exceptions: bool = False,
|
|
26
|
+
aiter: Literal[False] = False,
|
|
27
|
+
tqdm: bool = False,
|
|
28
|
+
**tqdm_kwargs: Any
|
|
29
|
+
) -> Iterator[Coroutine[Any, Any, T]]: ...
|
|
20
30
|
@overload
|
|
21
|
-
def as_completed(
|
|
22
|
-
|
|
31
|
+
def as_completed(
|
|
32
|
+
fs: Iterable[Awaitable[T]],
|
|
33
|
+
*,
|
|
34
|
+
timeout: Optional[float] = None,
|
|
35
|
+
return_exceptions: bool = False,
|
|
36
|
+
aiter: Literal[True] = True,
|
|
37
|
+
tqdm: bool = False,
|
|
38
|
+
**tqdm_kwargs: Any
|
|
39
|
+
) -> ASyncIterator[T]: ...
|
|
23
40
|
@overload
|
|
24
|
-
def as_completed(
|
|
25
|
-
|
|
41
|
+
def as_completed(
|
|
42
|
+
fs: Mapping[K, Awaitable[V]],
|
|
43
|
+
*,
|
|
44
|
+
timeout: Optional[float] = None,
|
|
45
|
+
return_exceptions: bool = False,
|
|
46
|
+
aiter: Literal[False] = False,
|
|
47
|
+
tqdm: bool = False,
|
|
48
|
+
**tqdm_kwargs: Any
|
|
49
|
+
) -> Iterator[Coroutine[Any, Any, Tuple[K, V]]]: ...
|
|
26
50
|
@overload
|
|
27
|
-
def as_completed(
|
|
28
|
-
|
|
29
|
-
|
|
51
|
+
def as_completed(
|
|
52
|
+
fs: Mapping[K, Awaitable[V]],
|
|
53
|
+
*,
|
|
54
|
+
timeout: Optional[float] = None,
|
|
55
|
+
return_exceptions: bool = False,
|
|
56
|
+
aiter: Literal[True] = True,
|
|
57
|
+
tqdm: bool = False,
|
|
58
|
+
**tqdm_kwargs: Any
|
|
59
|
+
) -> ASyncIterator[Tuple[K, V]]: ...
|
|
60
|
+
def as_completed(
|
|
61
|
+
fs,
|
|
62
|
+
*,
|
|
63
|
+
timeout: Optional[float] = None,
|
|
64
|
+
return_exceptions: bool = False,
|
|
65
|
+
aiter: bool = False,
|
|
66
|
+
tqdm: bool = False,
|
|
67
|
+
**tqdm_kwargs: Any
|
|
68
|
+
):
|
|
30
69
|
"""
|
|
31
70
|
Concurrently awaits a list of awaitable objects or mappings of awaitables and returns an iterator of results.
|
|
32
71
|
|
|
@@ -39,16 +78,13 @@ def as_completed(fs, *, timeout: Optional[float] = None, return_exceptions: bool
|
|
|
39
78
|
- Provides progress reporting using tqdm if 'tqdm' is set to True.
|
|
40
79
|
|
|
41
80
|
Args:
|
|
42
|
-
fs
|
|
43
|
-
timeout
|
|
44
|
-
return_exceptions
|
|
45
|
-
aiter
|
|
46
|
-
tqdm
|
|
81
|
+
fs: The awaitables to await concurrently. It can be a list of individual awaitables or a mapping of awaitables.
|
|
82
|
+
timeout: The maximum time, in seconds, to wait for the completion of awaitables. Defaults to None (no timeout).
|
|
83
|
+
return_exceptions: If True, exceptions are returned as results instead of raising them. Defaults to False. Note: This parameter is not currently implemented in the function logic.
|
|
84
|
+
aiter: If True, returns an async iterator of results. Defaults to False.
|
|
85
|
+
tqdm: If True, enables progress reporting using tqdm. Defaults to False.
|
|
47
86
|
**tqdm_kwargs: Additional keyword arguments for tqdm if progress reporting is enabled.
|
|
48
87
|
|
|
49
|
-
Returns:
|
|
50
|
-
Iterator[Coroutine[Any, Any, T] or ASyncIterator[Tuple[K, V]]]: An iterator of results when awaiting individual awaitables or an async iterator when awaiting mappings.
|
|
51
|
-
|
|
52
88
|
Examples:
|
|
53
89
|
Awaiting individual awaitables:
|
|
54
90
|
```
|
|
@@ -56,85 +92,176 @@ def as_completed(fs, *, timeout: Optional[float] = None, return_exceptions: bool
|
|
|
56
92
|
for coro in as_completed(awaitables):
|
|
57
93
|
val = await coro
|
|
58
94
|
...
|
|
59
|
-
|
|
95
|
+
|
|
60
96
|
async for val in as_completed(awaitables, aiter=True):
|
|
61
97
|
...
|
|
62
98
|
```
|
|
63
|
-
|
|
99
|
+
|
|
64
100
|
Awaiting mappings of awaitables:
|
|
65
101
|
```
|
|
66
102
|
mapping = {'key1': async_function1(), 'key2': async_function2()}
|
|
67
|
-
|
|
103
|
+
|
|
68
104
|
for coro in as_completed(mapping):
|
|
69
105
|
k, v = await coro
|
|
70
106
|
...
|
|
71
|
-
|
|
107
|
+
|
|
72
108
|
async for k, v in as_completed(mapping, aiter=True):
|
|
73
109
|
...
|
|
74
110
|
```
|
|
75
111
|
"""
|
|
76
112
|
if isinstance(fs, Mapping):
|
|
77
|
-
return as_completed_mapping(
|
|
113
|
+
return as_completed_mapping(
|
|
114
|
+
fs,
|
|
115
|
+
timeout=timeout,
|
|
116
|
+
return_exceptions=return_exceptions,
|
|
117
|
+
aiter=aiter,
|
|
118
|
+
tqdm=tqdm,
|
|
119
|
+
**tqdm_kwargs
|
|
120
|
+
)
|
|
78
121
|
if return_exceptions:
|
|
79
122
|
fs = [_exc_wrap(f) for f in fs]
|
|
80
123
|
return (
|
|
81
|
-
ASyncIterator(
|
|
82
|
-
|
|
83
|
-
|
|
124
|
+
ASyncIterator(
|
|
125
|
+
__yield_as_completed(fs, timeout=timeout, tqdm=tqdm, **tqdm_kwargs)
|
|
126
|
+
)
|
|
127
|
+
if aiter
|
|
128
|
+
else (
|
|
129
|
+
tqdm_asyncio.as_completed(fs, timeout=timeout, **tqdm_kwargs)
|
|
130
|
+
if tqdm
|
|
131
|
+
else asyncio.as_completed(fs, timeout=timeout)
|
|
132
|
+
)
|
|
84
133
|
)
|
|
85
134
|
|
|
135
|
+
|
|
86
136
|
@overload
|
|
87
|
-
def as_completed_mapping(
|
|
88
|
-
|
|
137
|
+
def as_completed_mapping(
|
|
138
|
+
mapping: Mapping[K, Awaitable[V]],
|
|
139
|
+
*,
|
|
140
|
+
timeout: Optional[float] = None,
|
|
141
|
+
return_exceptions: bool = False,
|
|
142
|
+
aiter: Literal[True] = True,
|
|
143
|
+
tqdm: bool = False,
|
|
144
|
+
**tqdm_kwargs: Any
|
|
145
|
+
) -> ASyncIterator[Tuple[K, V]]: ...
|
|
89
146
|
@overload
|
|
90
|
-
def as_completed_mapping(
|
|
91
|
-
|
|
92
|
-
|
|
147
|
+
def as_completed_mapping(
|
|
148
|
+
mapping: Mapping[K, Awaitable[V]],
|
|
149
|
+
*,
|
|
150
|
+
timeout: Optional[float] = None,
|
|
151
|
+
return_exceptions: bool = False,
|
|
152
|
+
aiter: Literal[False] = False,
|
|
153
|
+
tqdm: bool = False,
|
|
154
|
+
**tqdm_kwargs: Any
|
|
155
|
+
) -> Iterator[Coroutine[Any, Any, Tuple[K, V]]]: ...
|
|
156
|
+
def as_completed_mapping(
|
|
157
|
+
mapping: Mapping[K, Awaitable[V]],
|
|
158
|
+
*,
|
|
159
|
+
timeout: Optional[float] = None,
|
|
160
|
+
return_exceptions: bool = False,
|
|
161
|
+
aiter: bool = False,
|
|
162
|
+
tqdm: bool = False,
|
|
163
|
+
**tqdm_kwargs: Any
|
|
164
|
+
) -> Union[Iterator[Coroutine[Any, Any, Tuple[K, V]]], ASyncIterator[Tuple[K, V]]]:
|
|
93
165
|
"""
|
|
94
166
|
Concurrently awaits a mapping of awaitable objects and returns an iterator or async iterator of results.
|
|
95
167
|
|
|
96
168
|
This function is designed to await a mapping of awaitable objects, where each key-value pair represents a unique awaitable. It enables concurrent execution and gathers results into an iterator or an async iterator.
|
|
97
169
|
|
|
98
170
|
Args:
|
|
99
|
-
mapping
|
|
100
|
-
timeout
|
|
101
|
-
return_exceptions
|
|
102
|
-
aiter
|
|
103
|
-
tqdm
|
|
171
|
+
mapping: A dictionary-like object where keys are of type K and values are awaitable objects of type V.
|
|
172
|
+
timeout: The maximum time, in seconds, to wait for the completion of awaitables. Defaults to None (no timeout).
|
|
173
|
+
return_exceptions: If True, exceptions are returned as results instead of raising them. Defaults to False.
|
|
174
|
+
aiter: If True, returns an async iterator of results. Defaults to False.
|
|
175
|
+
tqdm: If True, enables progress reporting using tqdm. Defaults to False.
|
|
104
176
|
**tqdm_kwargs: Additional keyword arguments for tqdm if progress reporting is enabled.
|
|
105
177
|
|
|
106
|
-
Returns:
|
|
107
|
-
Union[Iterator[Coroutine[Any, Any, Tuple[K, V]]] or ASyncIterator[Tuple[K, V]]]: An iterator of results or an async iterator when awaiting mappings.
|
|
108
|
-
|
|
109
178
|
Example:
|
|
110
179
|
```
|
|
111
180
|
mapping = {'key1': async_function1(), 'key2': async_function2()}
|
|
112
|
-
|
|
181
|
+
|
|
113
182
|
for coro in as_completed_mapping(mapping):
|
|
114
183
|
k, v = await coro
|
|
115
184
|
...
|
|
116
|
-
|
|
185
|
+
|
|
117
186
|
async for k, v in as_completed_mapping(mapping, aiter=True):
|
|
118
187
|
...
|
|
119
188
|
```
|
|
120
189
|
"""
|
|
121
|
-
return as_completed(
|
|
190
|
+
return as_completed(
|
|
191
|
+
[
|
|
192
|
+
__mapping_wrap(k, v, return_exceptions=return_exceptions)
|
|
193
|
+
for k, v in mapping.items()
|
|
194
|
+
],
|
|
195
|
+
timeout=timeout,
|
|
196
|
+
aiter=aiter,
|
|
197
|
+
tqdm=tqdm,
|
|
198
|
+
**tqdm_kwargs
|
|
199
|
+
)
|
|
200
|
+
|
|
122
201
|
|
|
123
202
|
async def _exc_wrap(awaitable: Awaitable[T]) -> Union[T, Exception]:
|
|
203
|
+
"""Wraps an awaitable to catch exceptions and return them instead of raising.
|
|
204
|
+
|
|
205
|
+
Args:
|
|
206
|
+
awaitable: The awaitable to wrap.
|
|
207
|
+
|
|
208
|
+
Returns:
|
|
209
|
+
The result of the awaitable or the exception if one is raised.
|
|
210
|
+
"""
|
|
124
211
|
try:
|
|
125
212
|
return await awaitable
|
|
126
213
|
except Exception as e:
|
|
127
214
|
return e
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
async def __yield_as_completed(
|
|
218
|
+
futs: Iterable[Awaitable[T]],
|
|
219
|
+
*,
|
|
220
|
+
timeout: Optional[float] = None,
|
|
221
|
+
return_exceptions: bool = False,
|
|
222
|
+
tqdm: bool = False,
|
|
223
|
+
**tqdm_kwargs: Any
|
|
224
|
+
) -> AsyncIterator[T]:
|
|
225
|
+
"""Yields results from awaitables as they complete.
|
|
226
|
+
|
|
227
|
+
Args:
|
|
228
|
+
futs: The awaitables to await.
|
|
229
|
+
timeout: The maximum time, in seconds, to wait for the completion of awaitables. Defaults to None (no timeout).
|
|
230
|
+
return_exceptions: If True, exceptions are returned as results instead of raising them. Defaults to False.
|
|
231
|
+
tqdm: If True, enables progress reporting using tqdm. Defaults to False.
|
|
232
|
+
**tqdm_kwargs: Additional keyword arguments for tqdm if progress reporting is enabled.
|
|
233
|
+
"""
|
|
234
|
+
for fut in as_completed(
|
|
235
|
+
futs,
|
|
236
|
+
timeout=timeout,
|
|
237
|
+
return_exceptions=return_exceptions,
|
|
238
|
+
tqdm=tqdm,
|
|
239
|
+
**tqdm_kwargs
|
|
240
|
+
):
|
|
131
241
|
yield await fut
|
|
132
242
|
|
|
243
|
+
|
|
133
244
|
@overload
|
|
134
|
-
async def __mapping_wrap(
|
|
245
|
+
async def __mapping_wrap(
|
|
246
|
+
k: K, v: Awaitable[V], return_exceptions: Literal[True] = True
|
|
247
|
+
) -> Union[V, Exception]: ...
|
|
135
248
|
@overload
|
|
136
|
-
async def __mapping_wrap(
|
|
137
|
-
|
|
249
|
+
async def __mapping_wrap(
|
|
250
|
+
k: K, v: Awaitable[V], return_exceptions: Literal[False] = False
|
|
251
|
+
) -> V: ...
|
|
252
|
+
async def __mapping_wrap(
|
|
253
|
+
k: K, v: Awaitable[V], return_exceptions: bool = False
|
|
254
|
+
) -> Union[V, Exception]:
|
|
255
|
+
"""Wraps a key-value pair of awaitable to catch exceptions and return them with the key.
|
|
256
|
+
|
|
257
|
+
Args:
|
|
258
|
+
k: The key associated with the awaitable.
|
|
259
|
+
v: The awaitable to wrap.
|
|
260
|
+
return_exceptions: If True, exceptions are returned as results instead of raising them. Defaults to False.
|
|
261
|
+
|
|
262
|
+
Returns:
|
|
263
|
+
A tuple of the key and the result of the awaitable or the exception if one is raised.
|
|
264
|
+
"""
|
|
138
265
|
try:
|
|
139
266
|
return k, await v
|
|
140
267
|
except Exception as e:
|
|
@@ -142,4 +269,5 @@ async def __mapping_wrap(k: K, v: Awaitable[V], return_exceptions: bool = False)
|
|
|
142
269
|
return k, e
|
|
143
270
|
raise
|
|
144
271
|
|
|
145
|
-
|
|
272
|
+
|
|
273
|
+
__all__ = ["as_completed", "as_completed_mapping"]
|
a_sync/asyncio/create_task.py
CHANGED
|
@@ -12,24 +12,26 @@ from a_sync._typing import *
|
|
|
12
12
|
|
|
13
13
|
logger = logging.getLogger(__name__)
|
|
14
14
|
|
|
15
|
+
|
|
15
16
|
def create_task(
|
|
16
|
-
coro: Awaitable[T],
|
|
17
|
-
*,
|
|
18
|
-
name: Optional[str] = None,
|
|
17
|
+
coro: Awaitable[T],
|
|
18
|
+
*,
|
|
19
|
+
name: Optional[str] = None,
|
|
19
20
|
skip_gc_until_done: bool = False,
|
|
20
21
|
log_destroy_pending: bool = True,
|
|
21
22
|
) -> "asyncio.Task[T]":
|
|
22
23
|
"""
|
|
23
24
|
Extends asyncio.create_task to support any Awaitable, manage task lifecycle, and enhance error handling.
|
|
24
25
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
enhanced error management by wrapping exceptions in a custom exception.
|
|
26
|
+
This function accepts any Awaitable, ensuring broader compatibility. If the Awaitable is not a coroutine,
|
|
27
|
+
it attempts to convert it to one. It optionally prevents the task from being garbage-collected until completion
|
|
28
|
+
and provides enhanced error management by wrapping exceptions in a custom exception when skip_gc_until_done is True.
|
|
28
29
|
|
|
29
30
|
Args:
|
|
30
|
-
coro: An Awaitable object from which to create the task.
|
|
31
|
+
coro: An Awaitable object from which to create the task. If not a coroutine, it will be converted.
|
|
31
32
|
name: Optional name for the task, aiding in debugging.
|
|
32
33
|
skip_gc_until_done: If True, the task is kept alive until it completes, preventing garbage collection.
|
|
34
|
+
Exceptions are wrapped in PersistedTaskException for special handling.
|
|
33
35
|
log_destroy_pending: If False, asyncio's default error log when a pending task is destroyed is suppressed.
|
|
34
36
|
|
|
35
37
|
Returns:
|
|
@@ -50,8 +52,16 @@ def create_task(
|
|
|
50
52
|
|
|
51
53
|
__persisted_tasks: Set["asyncio.Task[Any]"] = set()
|
|
52
54
|
|
|
55
|
+
|
|
53
56
|
async def __await(awaitable: Awaitable[T]) -> T:
|
|
54
|
-
"""Wait for the completion of an Awaitable.
|
|
57
|
+
"""Wait for the completion of an Awaitable.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
awaitable: The Awaitable object to wait for.
|
|
61
|
+
|
|
62
|
+
Raises:
|
|
63
|
+
RuntimeError: If a RuntimeError occurs during the await, it is raised with additional context.
|
|
64
|
+
"""
|
|
55
65
|
try:
|
|
56
66
|
return await awaitable
|
|
57
67
|
except RuntimeError as e:
|
|
@@ -60,26 +70,33 @@ async def __await(awaitable: Awaitable[T]) -> T:
|
|
|
60
70
|
args.append(awaitable._children)
|
|
61
71
|
raise RuntimeError(*args) from None
|
|
62
72
|
|
|
73
|
+
|
|
63
74
|
def __prune_persisted_tasks():
|
|
64
|
-
"""Remove completed tasks from the set of persisted tasks.
|
|
75
|
+
"""Remove completed tasks from the set of persisted tasks.
|
|
76
|
+
|
|
77
|
+
This function checks each task in the persisted tasks set. If a task is done and has an exception,
|
|
78
|
+
it logs the exception and raises it if it's not a PersistedTaskException. It also logs the traceback
|
|
79
|
+
manually since the usual handler will not run after retrieving the exception.
|
|
80
|
+
"""
|
|
65
81
|
for task in tuple(__persisted_tasks):
|
|
66
82
|
if task.done() and (e := task.exception()):
|
|
67
83
|
# force exceptions related to this lib to bubble up
|
|
68
84
|
if not isinstance(e, exceptions.PersistedTaskException):
|
|
69
85
|
logger.exception(e)
|
|
70
86
|
raise e
|
|
71
|
-
# we have to manually log the traceback that asyncio would usually log
|
|
87
|
+
# we have to manually log the traceback that asyncio would usually log
|
|
72
88
|
# since we already got the exception from the task and the usual handler will now not run
|
|
73
89
|
context = {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
90
|
+
"message": f"{task.__class__.__name__} exception was never retrieved",
|
|
91
|
+
"exception": e,
|
|
92
|
+
"future": task,
|
|
77
93
|
}
|
|
78
94
|
if task._source_traceback:
|
|
79
|
-
context[
|
|
95
|
+
context["source_traceback"] = task._source_traceback
|
|
80
96
|
task._loop.call_exception_handler(context)
|
|
81
97
|
__persisted_tasks.discard(task)
|
|
82
98
|
|
|
99
|
+
|
|
83
100
|
async def __persisted_task_exc_wrap(task: "asyncio.Task[T]") -> T:
|
|
84
101
|
"""
|
|
85
102
|
Wrap a task to handle its exception in a specialized manner.
|
|
@@ -87,9 +104,6 @@ async def __persisted_task_exc_wrap(task: "asyncio.Task[T]") -> T:
|
|
|
87
104
|
Args:
|
|
88
105
|
task: The asyncio Task to wrap.
|
|
89
106
|
|
|
90
|
-
Returns:
|
|
91
|
-
The result of the task, if successful.
|
|
92
|
-
|
|
93
107
|
Raises:
|
|
94
108
|
PersistedTaskException: Wraps any exception raised by the task for special handling.
|
|
95
109
|
"""
|
a_sync/asyncio/gather.py
CHANGED
|
@@ -3,15 +3,18 @@ This module provides an enhanced version of :func:`asyncio.gather`.
|
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
5
|
import asyncio
|
|
6
|
-
from typing import
|
|
7
|
-
overload)
|
|
6
|
+
from typing import Any, Awaitable, Dict, List, Mapping, TypeVar, Union, overload
|
|
8
7
|
|
|
9
8
|
try:
|
|
10
9
|
from tqdm.asyncio import tqdm_asyncio
|
|
11
10
|
except ImportError as e:
|
|
11
|
+
|
|
12
12
|
class tqdm_asyncio: # type: ignore [no-redef]
|
|
13
13
|
async def gather(*args, **kwargs):
|
|
14
|
-
raise ImportError(
|
|
14
|
+
raise ImportError(
|
|
15
|
+
"You must have tqdm installed in order to use this feature"
|
|
16
|
+
)
|
|
17
|
+
|
|
15
18
|
|
|
16
19
|
from a_sync._typing import *
|
|
17
20
|
from a_sync.asyncio.as_completed import as_completed_mapping, _exc_wrap
|
|
@@ -19,87 +22,99 @@ from a_sync.asyncio.as_completed import as_completed_mapping, _exc_wrap
|
|
|
19
22
|
|
|
20
23
|
Excluder = Callable[[T], bool]
|
|
21
24
|
|
|
25
|
+
|
|
22
26
|
@overload
|
|
23
27
|
async def gather(
|
|
24
|
-
*awaitables: Mapping[K, Awaitable[V]],
|
|
25
|
-
return_exceptions: bool = False,
|
|
26
|
-
exclude_if: Optional[Excluder[V]] = None,
|
|
27
|
-
tqdm: bool = False,
|
|
28
|
+
*awaitables: Mapping[K, Awaitable[V]],
|
|
29
|
+
return_exceptions: bool = False,
|
|
30
|
+
exclude_if: Optional[Excluder[V]] = None,
|
|
31
|
+
tqdm: bool = False,
|
|
28
32
|
**tqdm_kwargs: Any,
|
|
29
|
-
) -> Dict[K, V]:
|
|
30
|
-
...
|
|
33
|
+
) -> Dict[K, V]: ...
|
|
31
34
|
@overload
|
|
32
35
|
async def gather(
|
|
33
|
-
*awaitables: Awaitable[T],
|
|
34
|
-
return_exceptions: bool = False,
|
|
36
|
+
*awaitables: Awaitable[T],
|
|
37
|
+
return_exceptions: bool = False,
|
|
35
38
|
exclude_if: Optional[Excluder[T]] = None,
|
|
36
39
|
tqdm: bool = False,
|
|
37
40
|
**tqdm_kwargs: Any,
|
|
38
|
-
) -> List[T]:
|
|
39
|
-
...
|
|
41
|
+
) -> List[T]: ...
|
|
40
42
|
async def gather(
|
|
41
|
-
*awaitables: Union[Awaitable[T], Mapping[K, Awaitable[V]]],
|
|
42
|
-
return_exceptions: bool = False,
|
|
43
|
-
exclude_if: Optional[Excluder[T]] = None,
|
|
44
|
-
tqdm: bool = False,
|
|
43
|
+
*awaitables: Union[Awaitable[T], Mapping[K, Awaitable[V]]],
|
|
44
|
+
return_exceptions: bool = False,
|
|
45
|
+
exclude_if: Optional[Excluder[T]] = None,
|
|
46
|
+
tqdm: bool = False,
|
|
45
47
|
**tqdm_kwargs: Any,
|
|
46
48
|
) -> Union[List[T], Dict[K, V]]:
|
|
47
49
|
"""
|
|
48
|
-
Concurrently awaits a list of awaitable objects or
|
|
50
|
+
Concurrently awaits a list of awaitable objects or a mapping of awaitables and returns the results.
|
|
49
51
|
|
|
50
|
-
This function extends Python's asyncio.gather, providing additional features for
|
|
52
|
+
This function extends Python's asyncio.gather, providing additional features for handling either individual awaitable objects or a mapping of awaitables.
|
|
51
53
|
|
|
52
54
|
Differences from asyncio.gather:
|
|
53
55
|
- Uses type hints for use with static type checkers.
|
|
54
56
|
- Supports gathering either individual awaitables or a k:v mapping of awaitables.
|
|
55
57
|
- Provides progress reporting using tqdm if 'tqdm' is set to True.
|
|
56
|
-
|
|
58
|
+
- Allows exclusion of results based on a condition using the 'exclude_if' parameter.
|
|
59
|
+
|
|
57
60
|
Args:
|
|
58
|
-
*awaitables: The awaitables to await concurrently. It can be a
|
|
59
|
-
return_exceptions
|
|
60
|
-
|
|
61
|
+
*awaitables: The awaitables to await concurrently. It can be a list of individual awaitables or a mapping of awaitables.
|
|
62
|
+
return_exceptions: If True, exceptions are returned as results instead of raising them. Defaults to False.
|
|
63
|
+
exclude_if: A callable that takes a result and returns True if the result should be excluded from the final output. Defaults to None.
|
|
64
|
+
tqdm: If True, enables progress reporting using tqdm. Defaults to False.
|
|
61
65
|
**tqdm_kwargs: Additional keyword arguments for tqdm if progress reporting is enabled.
|
|
62
66
|
|
|
63
|
-
Returns:
|
|
64
|
-
A list of results when awaiting individual awaitables or a dictionary of results when awaiting mappings.
|
|
65
|
-
|
|
66
67
|
Examples:
|
|
67
68
|
Awaiting individual awaitables:
|
|
68
|
-
|
|
69
|
+
|
|
69
70
|
- Results will be a list containing the result of each awaitable in sequential order.
|
|
70
71
|
|
|
71
|
-
```
|
|
72
72
|
>>> results = await gather(thing1(), thing2())
|
|
73
73
|
>>> results
|
|
74
74
|
['result', 123]
|
|
75
|
-
```
|
|
76
75
|
|
|
77
|
-
Awaiting
|
|
78
|
-
|
|
76
|
+
Awaiting a mapping of awaitables:
|
|
77
|
+
|
|
79
78
|
- Results will be a dictionary with 'key1' mapped to the result of thing1() and 'key2' mapped to the result of thing2.
|
|
80
|
-
|
|
81
|
-
```
|
|
79
|
+
|
|
82
80
|
>>> mapping = {'key1': thing1(), 'key2': thing2()}
|
|
83
81
|
>>> results = await gather(mapping)
|
|
84
82
|
>>> results
|
|
85
83
|
{'key1': 'result', 'key2': 123}
|
|
86
|
-
```
|
|
87
84
|
"""
|
|
88
85
|
is_mapping = _is_mapping(awaitables)
|
|
89
86
|
results = await (
|
|
90
|
-
gather_mapping(
|
|
91
|
-
|
|
92
|
-
|
|
87
|
+
gather_mapping(
|
|
88
|
+
awaitables[0],
|
|
89
|
+
return_exceptions=return_exceptions,
|
|
90
|
+
exclude_if=exclude_if,
|
|
91
|
+
tqdm=tqdm,
|
|
92
|
+
**tqdm_kwargs,
|
|
93
|
+
)
|
|
94
|
+
if is_mapping
|
|
95
|
+
else (
|
|
96
|
+
tqdm_asyncio.gather(
|
|
97
|
+
*(
|
|
98
|
+
(_exc_wrap(a) for a in awaitables)
|
|
99
|
+
if return_exceptions
|
|
100
|
+
else awaitables
|
|
101
|
+
),
|
|
102
|
+
**tqdm_kwargs,
|
|
103
|
+
)
|
|
104
|
+
if tqdm
|
|
105
|
+
else asyncio.gather(*awaitables, return_exceptions=return_exceptions)
|
|
106
|
+
) # type: ignore [arg-type]
|
|
93
107
|
)
|
|
94
108
|
if exclude_if and not is_mapping:
|
|
95
109
|
results = [r for r in results if not exclude_if(r)]
|
|
96
110
|
return results
|
|
97
|
-
|
|
111
|
+
|
|
112
|
+
|
|
98
113
|
async def gather_mapping(
|
|
99
|
-
mapping: Mapping[K, Awaitable[V]],
|
|
100
|
-
return_exceptions: bool = False,
|
|
114
|
+
mapping: Mapping[K, Awaitable[V]],
|
|
115
|
+
return_exceptions: bool = False,
|
|
101
116
|
exclude_if: Optional[Excluder[V]] = None,
|
|
102
|
-
tqdm: bool = False,
|
|
117
|
+
tqdm: bool = False,
|
|
103
118
|
**tqdm_kwargs: Any,
|
|
104
119
|
) -> Dict[K, V]:
|
|
105
120
|
"""
|
|
@@ -109,31 +124,36 @@ async def gather_mapping(
|
|
|
109
124
|
|
|
110
125
|
Args:
|
|
111
126
|
mapping: A dictionary-like object where keys are of type K and values are awaitable objects of type V.
|
|
112
|
-
return_exceptions
|
|
113
|
-
|
|
127
|
+
return_exceptions: If True, exceptions are returned as results instead of raising them. Defaults to False.
|
|
128
|
+
exclude_if: A callable that takes a result and returns True if the result should be excluded from the final output. Defaults to None.
|
|
129
|
+
tqdm: If True, enables progress reporting using tqdm. Defaults to False.
|
|
114
130
|
**tqdm_kwargs: Additional keyword arguments for tqdm if progress reporting is enabled.
|
|
115
131
|
|
|
116
|
-
Returns:
|
|
117
|
-
A dictionary with keys corresponding to the keys of the input mapping and values containing the results of the corresponding awaitables.
|
|
118
|
-
|
|
119
132
|
Example:
|
|
120
133
|
The 'results' dictionary will contain the awaited results, where keys match the keys in the 'mapping' and values contain the results of the corresponding awaitables.
|
|
121
|
-
|
|
134
|
+
|
|
122
135
|
>>> mapping = {'task1': async_function1(), 'task2': async_function2(), 'task3': async_function3()}
|
|
123
136
|
>>> results = await gather_mapping(mapping)
|
|
124
137
|
>>> results
|
|
125
138
|
{'task1': "result", 'task2': 123, 'task3': None}
|
|
126
|
-
```
|
|
127
139
|
"""
|
|
128
140
|
results = {
|
|
129
|
-
k: v
|
|
130
|
-
async for k, v in as_completed_mapping(
|
|
141
|
+
k: v
|
|
142
|
+
async for k, v in as_completed_mapping(
|
|
143
|
+
mapping,
|
|
144
|
+
return_exceptions=return_exceptions,
|
|
145
|
+
aiter=True,
|
|
146
|
+
tqdm=tqdm,
|
|
147
|
+
**tqdm_kwargs,
|
|
148
|
+
)
|
|
131
149
|
if exclude_if is None or not exclude_if(v)
|
|
132
150
|
}
|
|
133
151
|
# return data in same order as input mapping
|
|
134
|
-
return {k: results[k] for k in mapping}
|
|
152
|
+
return {k: results[k] for k in mapping}
|
|
135
153
|
|
|
136
154
|
|
|
137
|
-
_is_mapping = lambda awaitables: len(awaitables) == 1 and isinstance(
|
|
155
|
+
_is_mapping = lambda awaitables: len(awaitables) == 1 and isinstance(
|
|
156
|
+
awaitables[0], Mapping
|
|
157
|
+
)
|
|
138
158
|
|
|
139
159
|
__all__ = ["gather", "gather_mapping"]
|
a_sync/asyncio/utils.py
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
|
|
2
1
|
import asyncio
|
|
3
2
|
|
|
3
|
+
|
|
4
4
|
def get_event_loop() -> asyncio.AbstractEventLoop:
|
|
5
5
|
try:
|
|
6
6
|
loop = asyncio.get_event_loop()
|
|
7
|
-
except RuntimeError as e:
|
|
7
|
+
except RuntimeError as e: # Necessary for use with multi-threaded applications.
|
|
8
8
|
if not str(e).startswith("There is no current event loop in thread"):
|
|
9
9
|
raise
|
|
10
10
|
loop = asyncio.new_event_loop()
|
|
11
11
|
asyncio.set_event_loop(loop)
|
|
12
|
-
return loop
|
|
12
|
+
return loop
|