ez-a-sync 0.22.13__py3-none-any.whl → 0.22.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ez-a-sync might be problematic. Click here for more details.
- a_sync/ENVIRONMENT_VARIABLES.py +4 -3
- a_sync/__init__.py +30 -12
- a_sync/_smart.py +132 -28
- a_sync/_typing.py +56 -12
- a_sync/a_sync/__init__.py +35 -10
- a_sync/a_sync/_descriptor.py +74 -26
- a_sync/a_sync/_flags.py +14 -6
- a_sync/a_sync/_helpers.py +8 -7
- a_sync/a_sync/_kwargs.py +3 -2
- a_sync/a_sync/_meta.py +120 -28
- a_sync/a_sync/abstract.py +102 -28
- a_sync/a_sync/base.py +34 -16
- a_sync/a_sync/config.py +47 -13
- a_sync/a_sync/decorator.py +239 -117
- a_sync/a_sync/function.py +416 -146
- a_sync/a_sync/method.py +197 -59
- a_sync/a_sync/modifiers/__init__.py +47 -5
- a_sync/a_sync/modifiers/cache/__init__.py +46 -17
- a_sync/a_sync/modifiers/cache/memory.py +86 -20
- a_sync/a_sync/modifiers/limiter.py +52 -22
- a_sync/a_sync/modifiers/manager.py +98 -16
- a_sync/a_sync/modifiers/semaphores.py +48 -15
- a_sync/a_sync/property.py +383 -82
- a_sync/a_sync/singleton.py +1 -0
- a_sync/aliases.py +0 -1
- a_sync/asyncio/__init__.py +4 -1
- a_sync/asyncio/as_completed.py +177 -49
- a_sync/asyncio/create_task.py +31 -17
- a_sync/asyncio/gather.py +72 -52
- a_sync/asyncio/utils.py +3 -3
- a_sync/exceptions.py +78 -23
- a_sync/executor.py +120 -71
- a_sync/future.py +575 -158
- a_sync/iter.py +110 -50
- a_sync/primitives/__init__.py +14 -2
- a_sync/primitives/_debug.py +13 -13
- a_sync/primitives/_loggable.py +5 -4
- a_sync/primitives/locks/__init__.py +5 -2
- a_sync/primitives/locks/counter.py +38 -36
- a_sync/primitives/locks/event.py +21 -7
- a_sync/primitives/locks/prio_semaphore.py +182 -62
- a_sync/primitives/locks/semaphore.py +78 -77
- a_sync/primitives/queue.py +560 -58
- a_sync/sphinx/__init__.py +0 -1
- a_sync/sphinx/ext.py +160 -50
- a_sync/task.py +262 -97
- a_sync/utils/__init__.py +12 -6
- a_sync/utils/iterators.py +127 -43
- {ez_a_sync-0.22.13.dist-info → ez_a_sync-0.22.15.dist-info}/METADATA +1 -1
- ez_a_sync-0.22.15.dist-info/RECORD +74 -0
- {ez_a_sync-0.22.13.dist-info → ez_a_sync-0.22.15.dist-info}/WHEEL +1 -1
- tests/conftest.py +1 -2
- tests/executor.py +112 -9
- tests/fixtures.py +61 -32
- tests/test_abstract.py +7 -4
- tests/test_as_completed.py +54 -21
- tests/test_base.py +66 -17
- tests/test_cache.py +31 -15
- tests/test_decorator.py +54 -28
- tests/test_executor.py +8 -13
- tests/test_future.py +45 -8
- tests/test_gather.py +8 -2
- tests/test_helpers.py +2 -0
- tests/test_iter.py +55 -13
- tests/test_limiter.py +5 -3
- tests/test_meta.py +23 -9
- tests/test_modified.py +4 -1
- tests/test_semaphore.py +15 -8
- tests/test_singleton.py +15 -10
- tests/test_task.py +126 -28
- ez_a_sync-0.22.13.dist-info/RECORD +0 -74
- {ez_a_sync-0.22.13.dist-info → ez_a_sync-0.22.15.dist-info}/LICENSE.txt +0 -0
- {ez_a_sync-0.22.13.dist-info → ez_a_sync-0.22.15.dist-info}/top_level.txt +0 -0
a_sync/primitives/queue.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"""
|
|
2
|
-
This module provides various queue implementations for managing asynchronous tasks,
|
|
3
|
-
priority queues, and processing queues.
|
|
4
|
-
|
|
2
|
+
This module provides various queue implementations for managing asynchronous tasks,
|
|
3
|
+
including standard FIFO queues, priority queues, and processing queues.
|
|
4
|
+
# TODO specify a list of specific objects with a brief description + use case example instead of being vague like this
|
|
5
|
+
|
|
5
6
|
"""
|
|
6
7
|
|
|
7
8
|
import asyncio
|
|
@@ -18,32 +19,149 @@ from a_sync._typing import *
|
|
|
18
19
|
logger = logging.getLogger(__name__)
|
|
19
20
|
|
|
20
21
|
if sys.version_info < (3, 9):
|
|
22
|
+
|
|
21
23
|
class _Queue(asyncio.Queue, Generic[T]):
|
|
22
|
-
__slots__ =
|
|
24
|
+
__slots__ = (
|
|
25
|
+
"_maxsize",
|
|
26
|
+
"_loop",
|
|
27
|
+
"_getters",
|
|
28
|
+
"_putters",
|
|
29
|
+
"_unfinished_tasks",
|
|
30
|
+
"_finished",
|
|
31
|
+
)
|
|
32
|
+
|
|
23
33
|
else:
|
|
34
|
+
|
|
24
35
|
class _Queue(asyncio.Queue[T]):
|
|
25
36
|
__slots__ = "_maxsize", "_getters", "_putters", "_unfinished_tasks", "_finished"
|
|
26
37
|
|
|
38
|
+
|
|
27
39
|
class Queue(_Queue[T]):
|
|
28
|
-
|
|
40
|
+
"""
|
|
41
|
+
A generic asynchronous queue that extends the functionality of asyncio.Queue.
|
|
42
|
+
|
|
43
|
+
This implementation supports retrieving multiple items at once and handling
|
|
44
|
+
task processing in both FIFO and LIFO order. It provides enhanced type hinting
|
|
45
|
+
support and additional methods for bulk operations.
|
|
46
|
+
|
|
47
|
+
Inherits from:
|
|
48
|
+
- :class:`~asyncio.Queue`
|
|
49
|
+
|
|
50
|
+
Example:
|
|
51
|
+
>>> queue = Queue()
|
|
52
|
+
>>> await queue.put(item='task1')
|
|
53
|
+
>>> await queue.put(item='task2')
|
|
54
|
+
>>> result = await queue.get()
|
|
55
|
+
>>> print(result)
|
|
56
|
+
task1
|
|
57
|
+
>>> all_tasks = await queue.get_all()
|
|
58
|
+
>>> print(all_tasks)
|
|
59
|
+
['task2']
|
|
60
|
+
"""
|
|
61
|
+
|
|
29
62
|
async def get(self) -> T:
|
|
30
63
|
self._queue
|
|
64
|
+
"""
|
|
65
|
+
Asynchronously retrieves and removes the next item from the queue.
|
|
66
|
+
|
|
67
|
+
If the queue is empty, this method will block until an item is available.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
T: The next item in the queue.
|
|
71
|
+
|
|
72
|
+
Example:
|
|
73
|
+
>>> result = await queue.get()
|
|
74
|
+
>>> print(result)
|
|
75
|
+
"""
|
|
31
76
|
return await _Queue.get(self)
|
|
77
|
+
|
|
32
78
|
def get_nowait(self) -> T:
|
|
79
|
+
"""
|
|
80
|
+
Retrieves and removes the next item from the queue without blocking.
|
|
81
|
+
|
|
82
|
+
This method does not wait for an item to be available and will raise
|
|
83
|
+
an exception if the queue is empty.
|
|
84
|
+
|
|
85
|
+
Raises:
|
|
86
|
+
:exc:`~asyncio.QueueEmpty`: If the queue is empty.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
T: The next item in the queue.
|
|
90
|
+
|
|
91
|
+
Example:
|
|
92
|
+
>>> result = queue.get_nowait()
|
|
93
|
+
>>> print(result)
|
|
94
|
+
"""
|
|
33
95
|
return _Queue.get_nowait(self)
|
|
96
|
+
|
|
34
97
|
async def put(self, item: T) -> None:
|
|
98
|
+
"""
|
|
99
|
+
Asynchronously adds an item to the queue.
|
|
100
|
+
|
|
101
|
+
If the queue is full, this method will block until space is available.
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
item: The item to add to the queue.
|
|
105
|
+
|
|
106
|
+
Example:
|
|
107
|
+
>>> await queue.put(item='task')
|
|
108
|
+
"""
|
|
35
109
|
return _Queue.put(self, item)
|
|
110
|
+
|
|
36
111
|
def put_nowait(self, item: T) -> None:
|
|
112
|
+
"""
|
|
113
|
+
Adds an item to the queue without blocking.
|
|
114
|
+
|
|
115
|
+
This method does not wait for space to be available and will raise
|
|
116
|
+
an exception if the queue is full.
|
|
117
|
+
|
|
118
|
+
Args:
|
|
119
|
+
item: The item to add to the queue.
|
|
120
|
+
|
|
121
|
+
Raises:
|
|
122
|
+
:exc:`~asyncio.QueueFull`: If the queue is full.
|
|
123
|
+
|
|
124
|
+
Example:
|
|
125
|
+
>>> queue.put_nowait(item='task')
|
|
126
|
+
"""
|
|
37
127
|
return _Queue.put_nowait(self, item)
|
|
38
|
-
|
|
128
|
+
|
|
39
129
|
async def get_all(self) -> List[T]:
|
|
40
|
-
"""
|
|
130
|
+
"""
|
|
131
|
+
Asynchronously retrieves and removes all available items from the queue.
|
|
132
|
+
|
|
133
|
+
If the queue is empty, this method will wait until at least one item
|
|
134
|
+
is available before returning.
|
|
135
|
+
|
|
136
|
+
Returns:
|
|
137
|
+
List[T]: A list of all items that were in the queue.
|
|
138
|
+
|
|
139
|
+
Example:
|
|
140
|
+
>>> tasks = await queue.get_all()
|
|
141
|
+
>>> print(tasks)
|
|
142
|
+
"""
|
|
41
143
|
try:
|
|
42
144
|
return self.get_all_nowait()
|
|
43
145
|
except asyncio.QueueEmpty:
|
|
44
146
|
return [await self.get()]
|
|
147
|
+
|
|
45
148
|
def get_all_nowait(self) -> List[T]:
|
|
46
|
-
"""
|
|
149
|
+
"""
|
|
150
|
+
Retrieves and removes all available items from the queue without waiting.
|
|
151
|
+
|
|
152
|
+
This method does not wait for items to be available and will raise
|
|
153
|
+
an exception if the queue is empty.
|
|
154
|
+
|
|
155
|
+
Raises:
|
|
156
|
+
:exc:`~asyncio.QueueEmpty`: If the queue is empty.
|
|
157
|
+
|
|
158
|
+
Returns:
|
|
159
|
+
List[T]: A list of all items that were in the queue.
|
|
160
|
+
|
|
161
|
+
Example:
|
|
162
|
+
>>> tasks = queue.get_all_nowait()
|
|
163
|
+
>>> print(tasks)
|
|
164
|
+
"""
|
|
47
165
|
values: List[T] = []
|
|
48
166
|
while True:
|
|
49
167
|
try:
|
|
@@ -52,20 +170,53 @@ class Queue(_Queue[T]):
|
|
|
52
170
|
if not values:
|
|
53
171
|
raise asyncio.QueueEmpty from e
|
|
54
172
|
return values
|
|
55
|
-
|
|
173
|
+
|
|
56
174
|
async def get_multi(self, i: int, can_return_less: bool = False) -> List[T]:
|
|
175
|
+
"""
|
|
176
|
+
Asynchronously retrieves up to `i` items from the queue.
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
i: The number of items to retrieve.
|
|
180
|
+
can_return_less: If True, may return fewer than `i` items if queue is emptied.
|
|
181
|
+
|
|
182
|
+
Returns:
|
|
183
|
+
List[T]: A list containing the retrieved items.
|
|
184
|
+
|
|
185
|
+
Raises:
|
|
186
|
+
:exc:`~asyncio.QueueEmpty`: If no items are available and fewer items cannot be returned.
|
|
187
|
+
|
|
188
|
+
Example:
|
|
189
|
+
>>> tasks = await queue.get_multi(i=2, can_return_less=True)
|
|
190
|
+
>>> print(tasks)
|
|
191
|
+
"""
|
|
57
192
|
_validate_args(i, can_return_less)
|
|
58
193
|
items = []
|
|
59
194
|
while len(items) < i and not can_return_less:
|
|
60
195
|
try:
|
|
61
|
-
items.extend(
|
|
196
|
+
items.extend(
|
|
197
|
+
self.get_multi_nowait(i - len(items), can_return_less=True)
|
|
198
|
+
)
|
|
62
199
|
except asyncio.QueueEmpty:
|
|
63
200
|
items = [await self.get()]
|
|
64
201
|
return items
|
|
202
|
+
|
|
65
203
|
def get_multi_nowait(self, i: int, can_return_less: bool = False) -> List[T]:
|
|
66
204
|
"""
|
|
67
|
-
|
|
68
|
-
|
|
205
|
+
Retrieves up to `i` items from the queue without waiting.
|
|
206
|
+
|
|
207
|
+
Args:
|
|
208
|
+
i: The number of items to retrieve.
|
|
209
|
+
can_return_less: If True, may return fewer than `i` items if queue is emptied.
|
|
210
|
+
|
|
211
|
+
Raises:
|
|
212
|
+
:exc:`~asyncio.QueueEmpty`: If no items are available and fewer items cannot be returned.
|
|
213
|
+
|
|
214
|
+
Returns:
|
|
215
|
+
List[T]: A list containing the retrieved items.
|
|
216
|
+
|
|
217
|
+
Example:
|
|
218
|
+
>>> tasks = queue.get_multi_nowait(i=3, can_return_less=True)
|
|
219
|
+
>>> print(tasks)
|
|
69
220
|
"""
|
|
70
221
|
_validate_args(i, can_return_less)
|
|
71
222
|
items = []
|
|
@@ -83,35 +234,84 @@ class Queue(_Queue[T]):
|
|
|
83
234
|
|
|
84
235
|
|
|
85
236
|
class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
|
|
237
|
+
"""
|
|
238
|
+
A queue designed for processing tasks asynchronously with multiple workers.
|
|
239
|
+
|
|
240
|
+
Each item in the queue is processed by a worker, and tasks can return results
|
|
241
|
+
via asynchronous futures. This queue is ideal for scenarios where tasks need
|
|
242
|
+
to be processed concurrently with a fixed number of workers.
|
|
243
|
+
|
|
244
|
+
Example:
|
|
245
|
+
>>> async def process_task(data): return data.upper()
|
|
246
|
+
>>> queue = ProcessingQueue(func=process_task, num_workers=5)
|
|
247
|
+
>>> fut = await queue.put(item='task')
|
|
248
|
+
>>> print(await fut)
|
|
249
|
+
TASK
|
|
250
|
+
"""
|
|
251
|
+
|
|
86
252
|
_closed: bool = False
|
|
253
|
+
"""Indicates whether the queue is closed."""
|
|
254
|
+
|
|
87
255
|
__slots__ = "func", "num_workers", "_worker_coro"
|
|
256
|
+
|
|
88
257
|
def __init__(
|
|
89
|
-
self,
|
|
90
|
-
func: Callable[P, Awaitable[V]],
|
|
91
|
-
num_workers: int,
|
|
92
|
-
*,
|
|
93
|
-
return_data: bool = True,
|
|
258
|
+
self,
|
|
259
|
+
func: Callable[P, Awaitable[V]],
|
|
260
|
+
num_workers: int,
|
|
261
|
+
*,
|
|
262
|
+
return_data: bool = True,
|
|
94
263
|
name: str = "",
|
|
95
264
|
loop: Optional[asyncio.AbstractEventLoop] = None,
|
|
96
265
|
) -> None:
|
|
266
|
+
"""
|
|
267
|
+
Initializes a processing queue with the given worker function and worker count.
|
|
268
|
+
|
|
269
|
+
Args:
|
|
270
|
+
func: The task function to process.
|
|
271
|
+
num_workers: Number of workers to process tasks.
|
|
272
|
+
return_data: Whether tasks should return data via futures. Defaults to True.
|
|
273
|
+
name: Name of the queue. Defaults to an empty string.
|
|
274
|
+
loop: Optional event loop for the queue.
|
|
275
|
+
|
|
276
|
+
Example:
|
|
277
|
+
>>> queue = ProcessingQueue(func=my_task_func, num_workers=3, name='myqueue')
|
|
278
|
+
"""
|
|
97
279
|
if sys.version_info < (3, 10):
|
|
98
280
|
super().__init__(loop=loop)
|
|
99
281
|
elif loop:
|
|
100
|
-
raise NotImplementedError(
|
|
282
|
+
raise NotImplementedError(
|
|
283
|
+
f"You cannot pass a value for `loop` in python {sys.version_info}"
|
|
284
|
+
)
|
|
101
285
|
else:
|
|
102
286
|
super().__init__()
|
|
103
|
-
|
|
287
|
+
|
|
104
288
|
self.func = func
|
|
289
|
+
"""The function that each worker will process."""
|
|
290
|
+
|
|
105
291
|
self.num_workers = num_workers
|
|
292
|
+
"""The number of worker tasks for processing."""
|
|
293
|
+
|
|
106
294
|
self._name = name
|
|
295
|
+
"""Optional name for the queue."""
|
|
296
|
+
|
|
107
297
|
self._no_futs = not return_data
|
|
298
|
+
"""Indicates whether tasks will return data via futures."""
|
|
299
|
+
|
|
108
300
|
@functools.wraps(func)
|
|
109
301
|
async def _worker_coro() -> NoReturn:
|
|
110
|
-
|
|
302
|
+
"""Worker coroutine for processing tasks."""
|
|
111
303
|
return await self.__worker_coro()
|
|
304
|
+
|
|
112
305
|
self._worker_coro = _worker_coro
|
|
306
|
+
|
|
113
307
|
# NOTE: asyncio defines both this and __str__
|
|
114
308
|
def __repr__(self) -> str:
|
|
309
|
+
"""
|
|
310
|
+
Provides a detailed string representation of the queue.
|
|
311
|
+
|
|
312
|
+
Example:
|
|
313
|
+
>>> print(queue)
|
|
314
|
+
"""
|
|
115
315
|
repr_string = f"<{type(self).__name__} at {hex(id(self))}"
|
|
116
316
|
if self._name:
|
|
117
317
|
repr_string += f" name={self._name}"
|
|
@@ -119,8 +319,15 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
|
|
|
119
319
|
if self._unfinished_tasks:
|
|
120
320
|
repr_string += f" pending={self._unfinished_tasks}"
|
|
121
321
|
return f"{repr_string}>"
|
|
322
|
+
|
|
122
323
|
# NOTE: asyncio defines both this and __repr__
|
|
123
324
|
def __str__(self) -> str:
|
|
325
|
+
"""
|
|
326
|
+
Provides a string representation of the queue.
|
|
327
|
+
|
|
328
|
+
Example:
|
|
329
|
+
>>> print(queue)
|
|
330
|
+
"""
|
|
124
331
|
repr_string = f"<{type(self).__name__}"
|
|
125
332
|
if self._name:
|
|
126
333
|
repr_string += f" name={self._name}"
|
|
@@ -128,38 +335,98 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
|
|
|
128
335
|
if self._unfinished_tasks:
|
|
129
336
|
repr_string += f" pending={self._unfinished_tasks}"
|
|
130
337
|
return f"{repr_string}>"
|
|
338
|
+
|
|
131
339
|
def __call__(self, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
|
|
340
|
+
"""
|
|
341
|
+
Submits a task to the queue.
|
|
342
|
+
|
|
343
|
+
Example:
|
|
344
|
+
>>> fut = queue(*args, **kwargs)
|
|
345
|
+
>>> print(fut)
|
|
346
|
+
"""
|
|
132
347
|
return self.put_nowait(*args, **kwargs)
|
|
348
|
+
|
|
133
349
|
def __del__(self) -> None:
|
|
350
|
+
"""
|
|
351
|
+
Handles the deletion of the queue, ensuring tasks are handled.
|
|
352
|
+
"""
|
|
134
353
|
if self._closed:
|
|
135
354
|
return
|
|
136
355
|
if self._unfinished_tasks > 0:
|
|
137
356
|
context = {
|
|
138
|
-
|
|
357
|
+
"message": f"{self} was destroyed but has work pending!",
|
|
139
358
|
}
|
|
140
359
|
asyncio.get_event_loop().call_exception_handler(context)
|
|
360
|
+
|
|
141
361
|
@property
|
|
142
362
|
def name(self) -> str:
|
|
363
|
+
"""
|
|
364
|
+
Returns the name of the queue, or its representation.
|
|
365
|
+
|
|
366
|
+
Example:
|
|
367
|
+
>>> print(queue.name)
|
|
368
|
+
"""
|
|
143
369
|
return self._name or repr(self)
|
|
370
|
+
|
|
144
371
|
def close(self) -> None:
|
|
372
|
+
"""
|
|
373
|
+
Closes the queue, preventing further task submissions.
|
|
374
|
+
|
|
375
|
+
Example:
|
|
376
|
+
>>> queue.close()
|
|
377
|
+
"""
|
|
145
378
|
self._closed = True
|
|
379
|
+
|
|
146
380
|
async def put(self, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
|
|
381
|
+
"""
|
|
382
|
+
Asynchronously submits a task to the queue.
|
|
383
|
+
|
|
384
|
+
Args:
|
|
385
|
+
args: Positional arguments for the task.
|
|
386
|
+
kwargs: Keyword arguments for the task.
|
|
387
|
+
|
|
388
|
+
Returns:
|
|
389
|
+
The future result of the task.
|
|
390
|
+
|
|
391
|
+
Example:
|
|
392
|
+
>>> fut = await queue.put(item='task')
|
|
393
|
+
>>> print(await fut)
|
|
394
|
+
"""
|
|
147
395
|
self._ensure_workers()
|
|
148
396
|
if self._no_futs:
|
|
149
397
|
return await super().put((args, kwargs))
|
|
150
398
|
fut = self._create_future()
|
|
151
399
|
await super().put((args, kwargs, fut))
|
|
152
400
|
return fut
|
|
401
|
+
|
|
153
402
|
def put_nowait(self, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
|
|
403
|
+
"""
|
|
404
|
+
Immediately submits a task to the queue without waiting.
|
|
405
|
+
|
|
406
|
+
Args:
|
|
407
|
+
args: Positional arguments for the task.
|
|
408
|
+
kwargs: Keyword arguments for the task.
|
|
409
|
+
|
|
410
|
+
Returns:
|
|
411
|
+
The future result of the task.
|
|
412
|
+
|
|
413
|
+
Example:
|
|
414
|
+
>>> fut = queue.put_nowait(item='task')
|
|
415
|
+
>>> print(await fut)
|
|
416
|
+
"""
|
|
154
417
|
self._ensure_workers()
|
|
155
418
|
if self._no_futs:
|
|
156
419
|
return super().put_nowait((args, kwargs))
|
|
157
420
|
fut = self._create_future()
|
|
158
421
|
super().put_nowait((args, kwargs, weakref.proxy(fut)))
|
|
159
422
|
return fut
|
|
423
|
+
|
|
160
424
|
def _create_future(self) -> "asyncio.Future[V]":
|
|
425
|
+
"""Creates a future for the task."""
|
|
161
426
|
return asyncio.get_event_loop().create_future()
|
|
427
|
+
|
|
162
428
|
def _ensure_workers(self) -> None:
|
|
429
|
+
"""Ensures that the worker tasks are running."""
|
|
163
430
|
if self._closed:
|
|
164
431
|
raise RuntimeError(f"{type(self).__name__} is closed: ", self) from None
|
|
165
432
|
if self._workers.done():
|
|
@@ -170,29 +437,40 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
|
|
|
170
437
|
# re-raise with clean traceback
|
|
171
438
|
try:
|
|
172
439
|
raise type(exc)(*exc.args).with_traceback(exc.__traceback__) # type: ignore [union-attr]
|
|
173
|
-
except TypeError:
|
|
174
|
-
raise exc.with_traceback(exc.__traceback__)
|
|
440
|
+
except TypeError as e:
|
|
441
|
+
raise exc.with_traceback(exc.__traceback__) from e
|
|
175
442
|
# this should never be reached, but just in case
|
|
176
443
|
exc = self._workers.exception()
|
|
177
444
|
try:
|
|
178
445
|
# re-raise with clean traceback
|
|
179
446
|
raise type(exc)(*exc.args).with_traceback(exc.__traceback__) # type: ignore [union-attr]
|
|
180
|
-
except TypeError:
|
|
181
|
-
raise exc.with_traceback(exc.__traceback__)
|
|
447
|
+
except TypeError as e:
|
|
448
|
+
raise exc.with_traceback(exc.__traceback__) from e
|
|
449
|
+
|
|
182
450
|
@functools.cached_property
|
|
183
451
|
def _workers(self) -> "asyncio.Task[NoReturn]":
|
|
452
|
+
"""Creates and manages the worker tasks for the queue."""
|
|
184
453
|
logger.debug("starting worker task for %s", self)
|
|
185
454
|
workers = [
|
|
186
455
|
create_task(
|
|
187
|
-
coro=self._worker_coro(),
|
|
456
|
+
coro=self._worker_coro(),
|
|
188
457
|
name=f"{self.name} [Task-{i}]",
|
|
189
458
|
log_destroy_pending=False,
|
|
190
|
-
)
|
|
459
|
+
)
|
|
460
|
+
for i in range(self.num_workers)
|
|
191
461
|
]
|
|
192
|
-
task = create_task(
|
|
462
|
+
task = create_task(
|
|
463
|
+
asyncio.gather(*workers),
|
|
464
|
+
name=f"{self.name} worker main Task",
|
|
465
|
+
log_destroy_pending=False,
|
|
466
|
+
)
|
|
193
467
|
task._workers = workers
|
|
194
468
|
return task
|
|
469
|
+
|
|
195
470
|
async def __worker_coro(self) -> NoReturn:
|
|
471
|
+
"""
|
|
472
|
+
The coroutine executed by worker tasks to process the queue.
|
|
473
|
+
"""
|
|
196
474
|
args: P.args
|
|
197
475
|
kwargs: P.kwargs
|
|
198
476
|
if self._no_futs:
|
|
@@ -217,15 +495,27 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
|
|
|
217
495
|
result = await self.func(*args, **kwargs)
|
|
218
496
|
fut.set_result(result)
|
|
219
497
|
except asyncio.exceptions.InvalidStateError:
|
|
220
|
-
logger.error(
|
|
498
|
+
logger.error(
|
|
499
|
+
"cannot set result for %s %s: %s",
|
|
500
|
+
self.func.__name__,
|
|
501
|
+
fut,
|
|
502
|
+
result,
|
|
503
|
+
)
|
|
221
504
|
except Exception as e:
|
|
222
505
|
try:
|
|
223
506
|
fut.set_exception(e)
|
|
224
507
|
except asyncio.exceptions.InvalidStateError:
|
|
225
|
-
logger.error(
|
|
508
|
+
logger.error(
|
|
509
|
+
"cannot set exception for %s %s: %s",
|
|
510
|
+
self.func.__name__,
|
|
511
|
+
fut,
|
|
512
|
+
e,
|
|
513
|
+
)
|
|
226
514
|
self.task_done()
|
|
227
515
|
except Exception as e:
|
|
228
|
-
logger.error(
|
|
516
|
+
logger.error(
|
|
517
|
+
"%s for %s is broken!!!", type(self).__name__, self.func
|
|
518
|
+
)
|
|
229
519
|
logger.exception(e)
|
|
230
520
|
raise
|
|
231
521
|
|
|
@@ -235,97 +525,259 @@ def _validate_args(i: int, can_return_less: bool) -> None:
|
|
|
235
525
|
Validates the arguments for methods that retrieve multiple items from the queue.
|
|
236
526
|
|
|
237
527
|
Args:
|
|
238
|
-
i
|
|
239
|
-
can_return_less
|
|
528
|
+
i: The number of items to retrieve.
|
|
529
|
+
can_return_less: Whether the method is allowed to return fewer than `i` items.
|
|
240
530
|
|
|
241
531
|
Raises:
|
|
242
|
-
TypeError
|
|
243
|
-
ValueError
|
|
532
|
+
:exc:`~TypeError`: If `i` is not an integer or `can_return_less` is not a boolean.
|
|
533
|
+
:exc:`~ValueError`: If `i` is not greater than 1.
|
|
534
|
+
|
|
535
|
+
Example:
|
|
536
|
+
>>> _validate_args(i=2, can_return_less=False)
|
|
244
537
|
"""
|
|
245
538
|
if not isinstance(i, int):
|
|
246
539
|
raise TypeError(f"`i` must be an integer greater than 1. You passed {i}")
|
|
247
540
|
if not isinstance(can_return_less, bool):
|
|
248
|
-
raise TypeError(
|
|
541
|
+
raise TypeError(
|
|
542
|
+
f"`can_return_less` must be boolean. You passed {can_return_less}"
|
|
543
|
+
)
|
|
249
544
|
if i <= 1:
|
|
250
545
|
raise ValueError(f"`i` must be an integer greater than 1. You passed {i}")
|
|
251
546
|
|
|
252
547
|
|
|
253
|
-
|
|
254
548
|
class _SmartFutureRef(weakref.ref, Generic[T]):
|
|
549
|
+
"""
|
|
550
|
+
Weak reference for :class:`~_smart.SmartFuture` objects used in priority queues.
|
|
551
|
+
"""
|
|
552
|
+
|
|
255
553
|
def __lt__(self, other: "_SmartFutureRef[T]") -> bool:
|
|
256
554
|
"""
|
|
257
555
|
Compares two weak references to SmartFuture objects for ordering.
|
|
258
556
|
|
|
259
|
-
This comparison is used in priority queues to determine the order of processing. A SmartFuture
|
|
557
|
+
This comparison is used in priority queues to determine the order of processing. A SmartFuture
|
|
260
558
|
reference is considered less than another if it has more waiters or if it has been garbage collected.
|
|
261
559
|
|
|
262
560
|
Args:
|
|
263
|
-
other
|
|
561
|
+
other: The other SmartFuture reference to compare with.
|
|
264
562
|
|
|
265
563
|
Returns:
|
|
266
564
|
bool: True if this reference is less than the other, False otherwise.
|
|
565
|
+
|
|
566
|
+
Example:
|
|
567
|
+
>>> ref1 = _SmartFutureRef(fut1)
|
|
568
|
+
>>> ref2 = _SmartFutureRef(fut2)
|
|
569
|
+
>>> print(ref1 < ref2)
|
|
267
570
|
"""
|
|
268
571
|
strong_self = self()
|
|
269
572
|
if strong_self is None:
|
|
270
573
|
return True
|
|
271
574
|
strong_other = other()
|
|
272
|
-
if strong_other is None
|
|
273
|
-
|
|
274
|
-
return strong_self < strong_other
|
|
575
|
+
return False if strong_other is None else strong_self < strong_other
|
|
576
|
+
|
|
275
577
|
|
|
276
578
|
class _PriorityQueueMixin(Generic[T]):
|
|
579
|
+
"""
|
|
580
|
+
Mixin for creating priority queue functionality with support for custom comparison.
|
|
581
|
+
"""
|
|
582
|
+
|
|
277
583
|
def _init(self, maxsize):
|
|
584
|
+
"""
|
|
585
|
+
Initializes the priority queue.
|
|
586
|
+
|
|
587
|
+
Example:
|
|
588
|
+
>>> queue._init(maxsize=10)
|
|
589
|
+
"""
|
|
278
590
|
self._queue: List[T] = []
|
|
591
|
+
|
|
279
592
|
def _put(self, item, heappush=heapq.heappush):
|
|
593
|
+
"""
|
|
594
|
+
Adds an item to the priority queue based on its priority.
|
|
595
|
+
|
|
596
|
+
Example:
|
|
597
|
+
>>> queue._put(item='task')
|
|
598
|
+
"""
|
|
280
599
|
heappush(self._queue, item)
|
|
600
|
+
|
|
281
601
|
def _get(self, heappop=heapq.heappop):
|
|
602
|
+
"""
|
|
603
|
+
Retrieves the highest priority item from the queue.
|
|
604
|
+
|
|
605
|
+
Example:
|
|
606
|
+
>>> task = queue._get()
|
|
607
|
+
>>> print(task)
|
|
608
|
+
"""
|
|
282
609
|
return heappop(self._queue)
|
|
283
610
|
|
|
611
|
+
|
|
284
612
|
class PriorityProcessingQueue(_PriorityQueueMixin[T], ProcessingQueue[T, V]):
|
|
613
|
+
"""
|
|
614
|
+
A priority-based processing queue where tasks are processed based on priority.
|
|
285
615
|
# NOTE: WIP
|
|
286
|
-
|
|
616
|
+
"""
|
|
617
|
+
|
|
618
|
+
async def put(
|
|
619
|
+
self, priority: Any, *args: P.args, **kwargs: P.kwargs
|
|
620
|
+
) -> "asyncio.Future[V]":
|
|
621
|
+
"""
|
|
622
|
+
Asynchronously adds a task with priority to the queue.
|
|
623
|
+
|
|
624
|
+
Args:
|
|
625
|
+
priority: The priority of the task.
|
|
626
|
+
args: Positional arguments for the task.
|
|
627
|
+
kwargs: Keyword arguments for the task.
|
|
628
|
+
|
|
629
|
+
Returns:
|
|
630
|
+
The future representing the result of the task.
|
|
631
|
+
|
|
632
|
+
Example:
|
|
633
|
+
>>> fut = await queue.put(priority=1, item='task')
|
|
634
|
+
>>> print(await fut)
|
|
635
|
+
"""
|
|
287
636
|
self._ensure_workers()
|
|
288
637
|
fut = asyncio.get_event_loop().create_future()
|
|
289
638
|
await super().put(self, (priority, args, kwargs, fut))
|
|
290
639
|
return fut
|
|
291
|
-
|
|
640
|
+
|
|
641
|
+
def put_nowait(
|
|
642
|
+
self, priority: Any, *args: P.args, **kwargs: P.kwargs
|
|
643
|
+
) -> "asyncio.Future[V]":
|
|
644
|
+
"""
|
|
645
|
+
Immediately adds a task with priority to the queue without waiting.
|
|
646
|
+
|
|
647
|
+
Args:
|
|
648
|
+
priority: The priority of the task.
|
|
649
|
+
args: Positional arguments for the task.
|
|
650
|
+
kwargs: Keyword arguments for the task.
|
|
651
|
+
|
|
652
|
+
Returns:
|
|
653
|
+
The future representing the result of the task.
|
|
654
|
+
|
|
655
|
+
Example:
|
|
656
|
+
>>> fut = queue.put_nowait(priority=1, item='task')
|
|
657
|
+
>>> print(await fut)
|
|
658
|
+
"""
|
|
292
659
|
self._ensure_workers()
|
|
293
660
|
fut = self._create_future()
|
|
294
661
|
super().put_nowait(self, (priority, args, kwargs, fut))
|
|
295
662
|
return fut
|
|
663
|
+
|
|
296
664
|
def _get(self, heappop=heapq.heappop):
|
|
665
|
+
"""
|
|
666
|
+
Retrieves the highest priority task from the queue.
|
|
667
|
+
|
|
668
|
+
Returns:
|
|
669
|
+
The priority, task arguments, keyword arguments, and future of the task.
|
|
670
|
+
|
|
671
|
+
Example:
|
|
672
|
+
>>> task = queue._get()
|
|
673
|
+
>>> print(task)
|
|
674
|
+
"""
|
|
297
675
|
priority, args, kwargs, fut = heappop(self._queue)
|
|
298
676
|
return args, kwargs, fut
|
|
299
677
|
|
|
678
|
+
|
|
300
679
|
class _VariablePriorityQueueMixin(_PriorityQueueMixin[T]):
|
|
680
|
+
"""
|
|
681
|
+
Mixin for priority queues where task priorities can be updated dynamically.
|
|
682
|
+
"""
|
|
683
|
+
|
|
301
684
|
def _get(self, heapify=heapq.heapify, heappop=heapq.heappop):
|
|
302
|
-
"
|
|
303
|
-
|
|
685
|
+
"""
|
|
686
|
+
Resorts the priority queue to consider any changes in priorities and retrieves the task with the highest updated priority.
|
|
687
|
+
|
|
688
|
+
Args:
|
|
689
|
+
heapify: Function to resort the heap.
|
|
690
|
+
heappop: Function to pop the highest priority task.
|
|
691
|
+
|
|
692
|
+
Returns:
|
|
693
|
+
The highest priority task in the queue.
|
|
694
|
+
|
|
695
|
+
Example:
|
|
696
|
+
>>> task = queue._get()
|
|
697
|
+
>>> print(task)
|
|
698
|
+
"""
|
|
304
699
|
heapify(self._queue)
|
|
305
700
|
# take the job with the most waiters
|
|
306
701
|
return heappop(self._queue)
|
|
702
|
+
|
|
307
703
|
def _get_key(self, *args, **kwargs) -> _smart._Key:
|
|
704
|
+
"""
|
|
705
|
+
Generates a unique key for task identification based on arguments.
|
|
706
|
+
|
|
707
|
+
Args:
|
|
708
|
+
args: Positional arguments for the task.
|
|
709
|
+
kwargs: Keyword arguments for the task.
|
|
710
|
+
|
|
711
|
+
Returns:
|
|
712
|
+
The generated key for the task.
|
|
713
|
+
|
|
714
|
+
Example:
|
|
715
|
+
>>> key = queue._get_key(*args, **kwargs)
|
|
716
|
+
>>> print(key)
|
|
717
|
+
"""
|
|
308
718
|
return (args, tuple((kwarg, kwargs[kwarg]) for kwarg in sorted(kwargs)))
|
|
309
719
|
|
|
720
|
+
|
|
310
721
|
class VariablePriorityQueue(_VariablePriorityQueueMixin[T], asyncio.PriorityQueue):
|
|
311
|
-
"""
|
|
722
|
+
"""
|
|
723
|
+
A :class:`~asyncio.PriorityQueue` subclass that allows priorities to be updated (or computed) on the fly.
|
|
312
724
|
# NOTE: WIP
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
725
|
+
"""
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
class SmartProcessingQueue(
|
|
729
|
+
_VariablePriorityQueueMixin[T], ProcessingQueue[Concatenate[T, P], V]
|
|
730
|
+
):
|
|
731
|
+
"""
|
|
732
|
+
A PriorityProcessingQueue subclass that will execute jobs with the most waiters first
|
|
733
|
+
"""
|
|
734
|
+
|
|
316
735
|
_no_futs = False
|
|
317
|
-
|
|
736
|
+
"""Whether smart futures are used."""
|
|
737
|
+
|
|
738
|
+
_futs: "weakref.WeakValueDictionary[_smart._Key[T], _smart.SmartFuture[T]]"
|
|
739
|
+
"""
|
|
740
|
+
Weak reference dictionary for managing smart futures.
|
|
741
|
+
"""
|
|
742
|
+
|
|
318
743
|
def __init__(
|
|
319
|
-
self,
|
|
320
|
-
func: Callable[Concatenate[T, P], Awaitable[V]],
|
|
321
|
-
num_workers: int,
|
|
322
|
-
*,
|
|
744
|
+
self,
|
|
745
|
+
func: Callable[Concatenate[T, P], Awaitable[V]],
|
|
746
|
+
num_workers: int,
|
|
747
|
+
*,
|
|
323
748
|
name: str = "",
|
|
324
749
|
loop: Optional[asyncio.AbstractEventLoop] = None,
|
|
325
750
|
) -> None:
|
|
751
|
+
"""
|
|
752
|
+
Initializes a smart processing queue with the given worker function.
|
|
753
|
+
|
|
754
|
+
Args:
|
|
755
|
+
func: The worker function.
|
|
756
|
+
num_workers: Number of worker tasks.
|
|
757
|
+
name: Optional name for the queue.
|
|
758
|
+
loop: Optional event loop.
|
|
759
|
+
|
|
760
|
+
Example:
|
|
761
|
+
>>> queue = SmartProcessingQueue(func=my_task_func, num_workers=3, name='smart_queue')
|
|
762
|
+
"""
|
|
326
763
|
super().__init__(func, num_workers, return_data=True, name=name, loop=loop)
|
|
327
|
-
self._futs
|
|
764
|
+
self._futs = weakref.WeakValueDictionary()
|
|
765
|
+
|
|
328
766
|
async def put(self, *args: P.args, **kwargs: P.kwargs) -> _smart.SmartFuture[V]:
|
|
767
|
+
"""
|
|
768
|
+
Asynchronously adds a task with smart future handling to the queue.
|
|
769
|
+
|
|
770
|
+
Args:
|
|
771
|
+
args: Positional arguments for the task.
|
|
772
|
+
kwargs: Keyword arguments for the task.
|
|
773
|
+
|
|
774
|
+
Returns:
|
|
775
|
+
The future representing the task's result.
|
|
776
|
+
|
|
777
|
+
Example:
|
|
778
|
+
>>> fut = await queue.put(item='task')
|
|
779
|
+
>>> print(await fut)
|
|
780
|
+
"""
|
|
329
781
|
self._ensure_workers()
|
|
330
782
|
key = self._get_key(*args, **kwargs)
|
|
331
783
|
if fut := self._futs.get(key, None):
|
|
@@ -334,7 +786,22 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
|
|
|
334
786
|
self._futs[key] = fut
|
|
335
787
|
await Queue.put(self, (_SmartFutureRef(fut), args, kwargs))
|
|
336
788
|
return fut
|
|
789
|
+
|
|
337
790
|
def put_nowait(self, *args: P.args, **kwargs: P.kwargs) -> _smart.SmartFuture[V]:
|
|
791
|
+
"""
|
|
792
|
+
Immediately adds a task with smart future handling to the queue without waiting.
|
|
793
|
+
|
|
794
|
+
Args:
|
|
795
|
+
args: Positional arguments for the task.
|
|
796
|
+
kwargs: Keyword arguments for the task.
|
|
797
|
+
|
|
798
|
+
Returns:
|
|
799
|
+
The future representing the task's result.
|
|
800
|
+
|
|
801
|
+
Example:
|
|
802
|
+
>>> fut = queue.put_nowait(item='task')
|
|
803
|
+
>>> print(await fut)
|
|
804
|
+
"""
|
|
338
805
|
self._ensure_workers()
|
|
339
806
|
key = self._get_key(*args, **kwargs)
|
|
340
807
|
if fut := self._futs.get(key, None):
|
|
@@ -343,12 +810,37 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
|
|
|
343
810
|
self._futs[key] = fut
|
|
344
811
|
Queue.put_nowait(self, (_SmartFutureRef(fut), args, kwargs))
|
|
345
812
|
return fut
|
|
813
|
+
|
|
346
814
|
def _create_future(self, key: _smart._Key) -> "asyncio.Future[V]":
|
|
815
|
+
"""Creates a smart future for the task."""
|
|
347
816
|
return _smart.create_future(queue=self, key=key, loop=self._loop)
|
|
817
|
+
|
|
348
818
|
def _get(self):
|
|
819
|
+
"""
|
|
820
|
+
Retrieves the task with the highest priority from the queue.
|
|
821
|
+
|
|
822
|
+
Returns:
|
|
823
|
+
The priority, task arguments, keyword arguments, and future of the task.
|
|
824
|
+
|
|
825
|
+
Example:
|
|
826
|
+
>>> task = queue._get()
|
|
827
|
+
>>> print(task)
|
|
828
|
+
"""
|
|
349
829
|
fut, args, kwargs = super()._get()
|
|
350
830
|
return args, kwargs, fut()
|
|
831
|
+
|
|
351
832
|
async def __worker_coro(self) -> NoReturn:
|
|
833
|
+
"""
|
|
834
|
+
Worker coroutine responsible for processing tasks in the queue.
|
|
835
|
+
|
|
836
|
+
Retrieves tasks, executes them, and sets the results or exceptions for the futures.
|
|
837
|
+
|
|
838
|
+
Raises:
|
|
839
|
+
Any: Exceptions raised during task processing are logged.
|
|
840
|
+
|
|
841
|
+
Example:
|
|
842
|
+
>>> await queue.__worker_coro()
|
|
843
|
+
"""
|
|
352
844
|
args: P.args
|
|
353
845
|
kwargs: P.kwargs
|
|
354
846
|
fut: _smart.SmartFuture[V]
|
|
@@ -364,13 +856,23 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
|
|
|
364
856
|
result = await self.func(*args, **kwargs)
|
|
365
857
|
fut.set_result(result)
|
|
366
858
|
except asyncio.exceptions.InvalidStateError:
|
|
367
|
-
logger.error(
|
|
859
|
+
logger.error(
|
|
860
|
+
"cannot set result for %s %s: %s",
|
|
861
|
+
self.func.__name__,
|
|
862
|
+
fut,
|
|
863
|
+
result,
|
|
864
|
+
)
|
|
368
865
|
except Exception as e:
|
|
369
866
|
logger.debug("%s: %s", type(e).__name__, e)
|
|
370
867
|
try:
|
|
371
868
|
fut.set_exception(e)
|
|
372
869
|
except asyncio.exceptions.InvalidStateError:
|
|
373
|
-
logger.error(
|
|
870
|
+
logger.error(
|
|
871
|
+
"cannot set exception for %s %s: %s",
|
|
872
|
+
self.func.__name__,
|
|
873
|
+
fut,
|
|
874
|
+
e,
|
|
875
|
+
)
|
|
374
876
|
self.task_done()
|
|
375
877
|
except Exception as e:
|
|
376
878
|
logger.error("%s for %s is broken!!!", type(self).__name__, self.func)
|