ez-a-sync 0.22.13__py3-none-any.whl → 0.22.15__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ez-a-sync might be problematic. Click here for more details.

Files changed (73) hide show
  1. a_sync/ENVIRONMENT_VARIABLES.py +4 -3
  2. a_sync/__init__.py +30 -12
  3. a_sync/_smart.py +132 -28
  4. a_sync/_typing.py +56 -12
  5. a_sync/a_sync/__init__.py +35 -10
  6. a_sync/a_sync/_descriptor.py +74 -26
  7. a_sync/a_sync/_flags.py +14 -6
  8. a_sync/a_sync/_helpers.py +8 -7
  9. a_sync/a_sync/_kwargs.py +3 -2
  10. a_sync/a_sync/_meta.py +120 -28
  11. a_sync/a_sync/abstract.py +102 -28
  12. a_sync/a_sync/base.py +34 -16
  13. a_sync/a_sync/config.py +47 -13
  14. a_sync/a_sync/decorator.py +239 -117
  15. a_sync/a_sync/function.py +416 -146
  16. a_sync/a_sync/method.py +197 -59
  17. a_sync/a_sync/modifiers/__init__.py +47 -5
  18. a_sync/a_sync/modifiers/cache/__init__.py +46 -17
  19. a_sync/a_sync/modifiers/cache/memory.py +86 -20
  20. a_sync/a_sync/modifiers/limiter.py +52 -22
  21. a_sync/a_sync/modifiers/manager.py +98 -16
  22. a_sync/a_sync/modifiers/semaphores.py +48 -15
  23. a_sync/a_sync/property.py +383 -82
  24. a_sync/a_sync/singleton.py +1 -0
  25. a_sync/aliases.py +0 -1
  26. a_sync/asyncio/__init__.py +4 -1
  27. a_sync/asyncio/as_completed.py +177 -49
  28. a_sync/asyncio/create_task.py +31 -17
  29. a_sync/asyncio/gather.py +72 -52
  30. a_sync/asyncio/utils.py +3 -3
  31. a_sync/exceptions.py +78 -23
  32. a_sync/executor.py +120 -71
  33. a_sync/future.py +575 -158
  34. a_sync/iter.py +110 -50
  35. a_sync/primitives/__init__.py +14 -2
  36. a_sync/primitives/_debug.py +13 -13
  37. a_sync/primitives/_loggable.py +5 -4
  38. a_sync/primitives/locks/__init__.py +5 -2
  39. a_sync/primitives/locks/counter.py +38 -36
  40. a_sync/primitives/locks/event.py +21 -7
  41. a_sync/primitives/locks/prio_semaphore.py +182 -62
  42. a_sync/primitives/locks/semaphore.py +78 -77
  43. a_sync/primitives/queue.py +560 -58
  44. a_sync/sphinx/__init__.py +0 -1
  45. a_sync/sphinx/ext.py +160 -50
  46. a_sync/task.py +262 -97
  47. a_sync/utils/__init__.py +12 -6
  48. a_sync/utils/iterators.py +127 -43
  49. {ez_a_sync-0.22.13.dist-info → ez_a_sync-0.22.15.dist-info}/METADATA +1 -1
  50. ez_a_sync-0.22.15.dist-info/RECORD +74 -0
  51. {ez_a_sync-0.22.13.dist-info → ez_a_sync-0.22.15.dist-info}/WHEEL +1 -1
  52. tests/conftest.py +1 -2
  53. tests/executor.py +112 -9
  54. tests/fixtures.py +61 -32
  55. tests/test_abstract.py +7 -4
  56. tests/test_as_completed.py +54 -21
  57. tests/test_base.py +66 -17
  58. tests/test_cache.py +31 -15
  59. tests/test_decorator.py +54 -28
  60. tests/test_executor.py +8 -13
  61. tests/test_future.py +45 -8
  62. tests/test_gather.py +8 -2
  63. tests/test_helpers.py +2 -0
  64. tests/test_iter.py +55 -13
  65. tests/test_limiter.py +5 -3
  66. tests/test_meta.py +23 -9
  67. tests/test_modified.py +4 -1
  68. tests/test_semaphore.py +15 -8
  69. tests/test_singleton.py +15 -10
  70. tests/test_task.py +126 -28
  71. ez_a_sync-0.22.13.dist-info/RECORD +0 -74
  72. {ez_a_sync-0.22.13.dist-info → ez_a_sync-0.22.15.dist-info}/LICENSE.txt +0 -0
  73. {ez_a_sync-0.22.13.dist-info → ez_a_sync-0.22.15.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,8 @@
1
1
  """
2
- This module provides various queue implementations for managing asynchronous tasks, including standard FIFO queues,
3
- priority queues, and processing queues. These queues support advanced features like waiting for multiple items,
4
- handling priority tasks, and processing tasks with multiple workers.
2
+ This module provides various queue implementations for managing asynchronous tasks,
3
+ including standard FIFO queues, priority queues, and processing queues.
4
+ # TODO specify a list of specific objects with a brief description + use case example instead of being vague like this
5
+
5
6
  """
6
7
 
7
8
  import asyncio
@@ -18,32 +19,149 @@ from a_sync._typing import *
18
19
  logger = logging.getLogger(__name__)
19
20
 
20
21
  if sys.version_info < (3, 9):
22
+
21
23
  class _Queue(asyncio.Queue, Generic[T]):
22
- __slots__ = "_maxsize", "_loop", "_getters", "_putters", "_unfinished_tasks", "_finished"
24
+ __slots__ = (
25
+ "_maxsize",
26
+ "_loop",
27
+ "_getters",
28
+ "_putters",
29
+ "_unfinished_tasks",
30
+ "_finished",
31
+ )
32
+
23
33
  else:
34
+
24
35
  class _Queue(asyncio.Queue[T]):
25
36
  __slots__ = "_maxsize", "_getters", "_putters", "_unfinished_tasks", "_finished"
26
37
 
38
+
27
39
  class Queue(_Queue[T]):
28
- # for type hint support, no functional difference
40
+ """
41
+ A generic asynchronous queue that extends the functionality of asyncio.Queue.
42
+
43
+ This implementation supports retrieving multiple items at once and handling
44
+ task processing in both FIFO and LIFO order. It provides enhanced type hinting
45
+ support and additional methods for bulk operations.
46
+
47
+ Inherits from:
48
+ - :class:`~asyncio.Queue`
49
+
50
+ Example:
51
+ >>> queue = Queue()
52
+ >>> await queue.put(item='task1')
53
+ >>> await queue.put(item='task2')
54
+ >>> result = await queue.get()
55
+ >>> print(result)
56
+ task1
57
+ >>> all_tasks = await queue.get_all()
58
+ >>> print(all_tasks)
59
+ ['task2']
60
+ """
61
+
29
62
  async def get(self) -> T:
30
63
  self._queue
64
+ """
65
+ Asynchronously retrieves and removes the next item from the queue.
66
+
67
+ If the queue is empty, this method will block until an item is available.
68
+
69
+ Returns:
70
+ T: The next item in the queue.
71
+
72
+ Example:
73
+ >>> result = await queue.get()
74
+ >>> print(result)
75
+ """
31
76
  return await _Queue.get(self)
77
+
32
78
  def get_nowait(self) -> T:
79
+ """
80
+ Retrieves and removes the next item from the queue without blocking.
81
+
82
+ This method does not wait for an item to be available and will raise
83
+ an exception if the queue is empty.
84
+
85
+ Raises:
86
+ :exc:`~asyncio.QueueEmpty`: If the queue is empty.
87
+
88
+ Returns:
89
+ T: The next item in the queue.
90
+
91
+ Example:
92
+ >>> result = queue.get_nowait()
93
+ >>> print(result)
94
+ """
33
95
  return _Queue.get_nowait(self)
96
+
34
97
  async def put(self, item: T) -> None:
98
+ """
99
+ Asynchronously adds an item to the queue.
100
+
101
+ If the queue is full, this method will block until space is available.
102
+
103
+ Args:
104
+ item: The item to add to the queue.
105
+
106
+ Example:
107
+ >>> await queue.put(item='task')
108
+ """
35
109
  return _Queue.put(self, item)
110
+
36
111
  def put_nowait(self, item: T) -> None:
112
+ """
113
+ Adds an item to the queue without blocking.
114
+
115
+ This method does not wait for space to be available and will raise
116
+ an exception if the queue is full.
117
+
118
+ Args:
119
+ item: The item to add to the queue.
120
+
121
+ Raises:
122
+ :exc:`~asyncio.QueueFull`: If the queue is full.
123
+
124
+ Example:
125
+ >>> queue.put_nowait(item='task')
126
+ """
37
127
  return _Queue.put_nowait(self, item)
38
-
128
+
39
129
  async def get_all(self) -> List[T]:
40
- """returns 1 or more items"""
130
+ """
131
+ Asynchronously retrieves and removes all available items from the queue.
132
+
133
+ If the queue is empty, this method will wait until at least one item
134
+ is available before returning.
135
+
136
+ Returns:
137
+ List[T]: A list of all items that were in the queue.
138
+
139
+ Example:
140
+ >>> tasks = await queue.get_all()
141
+ >>> print(tasks)
142
+ """
41
143
  try:
42
144
  return self.get_all_nowait()
43
145
  except asyncio.QueueEmpty:
44
146
  return [await self.get()]
147
+
45
148
  def get_all_nowait(self) -> List[T]:
46
- """returns 1 or more items, or raises asyncio.QueueEmpty"""
149
+ """
150
+ Retrieves and removes all available items from the queue without waiting.
151
+
152
+ This method does not wait for items to be available and will raise
153
+ an exception if the queue is empty.
154
+
155
+ Raises:
156
+ :exc:`~asyncio.QueueEmpty`: If the queue is empty.
157
+
158
+ Returns:
159
+ List[T]: A list of all items that were in the queue.
160
+
161
+ Example:
162
+ >>> tasks = queue.get_all_nowait()
163
+ >>> print(tasks)
164
+ """
47
165
  values: List[T] = []
48
166
  while True:
49
167
  try:
@@ -52,20 +170,53 @@ class Queue(_Queue[T]):
52
170
  if not values:
53
171
  raise asyncio.QueueEmpty from e
54
172
  return values
55
-
173
+
56
174
  async def get_multi(self, i: int, can_return_less: bool = False) -> List[T]:
175
+ """
176
+ Asynchronously retrieves up to `i` items from the queue.
177
+
178
+ Args:
179
+ i: The number of items to retrieve.
180
+ can_return_less: If True, may return fewer than `i` items if queue is emptied.
181
+
182
+ Returns:
183
+ List[T]: A list containing the retrieved items.
184
+
185
+ Raises:
186
+ :exc:`~asyncio.QueueEmpty`: If no items are available and fewer items cannot be returned.
187
+
188
+ Example:
189
+ >>> tasks = await queue.get_multi(i=2, can_return_less=True)
190
+ >>> print(tasks)
191
+ """
57
192
  _validate_args(i, can_return_less)
58
193
  items = []
59
194
  while len(items) < i and not can_return_less:
60
195
  try:
61
- items.extend(self.get_multi_nowait(i - len(items), can_return_less=True))
196
+ items.extend(
197
+ self.get_multi_nowait(i - len(items), can_return_less=True)
198
+ )
62
199
  except asyncio.QueueEmpty:
63
200
  items = [await self.get()]
64
201
  return items
202
+
65
203
  def get_multi_nowait(self, i: int, can_return_less: bool = False) -> List[T]:
66
204
  """
67
- Just like `asyncio.Queue.get_nowait`, but will return `i` items instead of 1.
68
- Set `can_return_less` to True if you want to receive up to `i` items.
205
+ Retrieves up to `i` items from the queue without waiting.
206
+
207
+ Args:
208
+ i: The number of items to retrieve.
209
+ can_return_less: If True, may return fewer than `i` items if queue is emptied.
210
+
211
+ Raises:
212
+ :exc:`~asyncio.QueueEmpty`: If no items are available and fewer items cannot be returned.
213
+
214
+ Returns:
215
+ List[T]: A list containing the retrieved items.
216
+
217
+ Example:
218
+ >>> tasks = queue.get_multi_nowait(i=3, can_return_less=True)
219
+ >>> print(tasks)
69
220
  """
70
221
  _validate_args(i, can_return_less)
71
222
  items = []
@@ -83,35 +234,84 @@ class Queue(_Queue[T]):
83
234
 
84
235
 
85
236
  class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
237
+ """
238
+ A queue designed for processing tasks asynchronously with multiple workers.
239
+
240
+ Each item in the queue is processed by a worker, and tasks can return results
241
+ via asynchronous futures. This queue is ideal for scenarios where tasks need
242
+ to be processed concurrently with a fixed number of workers.
243
+
244
+ Example:
245
+ >>> async def process_task(data): return data.upper()
246
+ >>> queue = ProcessingQueue(func=process_task, num_workers=5)
247
+ >>> fut = await queue.put(item='task')
248
+ >>> print(await fut)
249
+ TASK
250
+ """
251
+
86
252
  _closed: bool = False
253
+ """Indicates whether the queue is closed."""
254
+
87
255
  __slots__ = "func", "num_workers", "_worker_coro"
256
+
88
257
  def __init__(
89
- self,
90
- func: Callable[P, Awaitable[V]],
91
- num_workers: int,
92
- *,
93
- return_data: bool = True,
258
+ self,
259
+ func: Callable[P, Awaitable[V]],
260
+ num_workers: int,
261
+ *,
262
+ return_data: bool = True,
94
263
  name: str = "",
95
264
  loop: Optional[asyncio.AbstractEventLoop] = None,
96
265
  ) -> None:
266
+ """
267
+ Initializes a processing queue with the given worker function and worker count.
268
+
269
+ Args:
270
+ func: The task function to process.
271
+ num_workers: Number of workers to process tasks.
272
+ return_data: Whether tasks should return data via futures. Defaults to True.
273
+ name: Name of the queue. Defaults to an empty string.
274
+ loop: Optional event loop for the queue.
275
+
276
+ Example:
277
+ >>> queue = ProcessingQueue(func=my_task_func, num_workers=3, name='myqueue')
278
+ """
97
279
  if sys.version_info < (3, 10):
98
280
  super().__init__(loop=loop)
99
281
  elif loop:
100
- raise NotImplementedError(f"You cannot pass a value for `loop` in python {sys.version_info}")
282
+ raise NotImplementedError(
283
+ f"You cannot pass a value for `loop` in python {sys.version_info}"
284
+ )
101
285
  else:
102
286
  super().__init__()
103
-
287
+
104
288
  self.func = func
289
+ """The function that each worker will process."""
290
+
105
291
  self.num_workers = num_workers
292
+ """The number of worker tasks for processing."""
293
+
106
294
  self._name = name
295
+ """Optional name for the queue."""
296
+
107
297
  self._no_futs = not return_data
298
+ """Indicates whether tasks will return data via futures."""
299
+
108
300
  @functools.wraps(func)
109
301
  async def _worker_coro() -> NoReturn:
110
- # we use this little helper so we can have context of `func` in any err logs
302
+ """Worker coroutine for processing tasks."""
111
303
  return await self.__worker_coro()
304
+
112
305
  self._worker_coro = _worker_coro
306
+
113
307
  # NOTE: asyncio defines both this and __str__
114
308
  def __repr__(self) -> str:
309
+ """
310
+ Provides a detailed string representation of the queue.
311
+
312
+ Example:
313
+ >>> print(queue)
314
+ """
115
315
  repr_string = f"<{type(self).__name__} at {hex(id(self))}"
116
316
  if self._name:
117
317
  repr_string += f" name={self._name}"
@@ -119,8 +319,15 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
119
319
  if self._unfinished_tasks:
120
320
  repr_string += f" pending={self._unfinished_tasks}"
121
321
  return f"{repr_string}>"
322
+
122
323
  # NOTE: asyncio defines both this and __repr__
123
324
  def __str__(self) -> str:
325
+ """
326
+ Provides a string representation of the queue.
327
+
328
+ Example:
329
+ >>> print(queue)
330
+ """
124
331
  repr_string = f"<{type(self).__name__}"
125
332
  if self._name:
126
333
  repr_string += f" name={self._name}"
@@ -128,38 +335,98 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
128
335
  if self._unfinished_tasks:
129
336
  repr_string += f" pending={self._unfinished_tasks}"
130
337
  return f"{repr_string}>"
338
+
131
339
  def __call__(self, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
340
+ """
341
+ Submits a task to the queue.
342
+
343
+ Example:
344
+ >>> fut = queue(*args, **kwargs)
345
+ >>> print(fut)
346
+ """
132
347
  return self.put_nowait(*args, **kwargs)
348
+
133
349
  def __del__(self) -> None:
350
+ """
351
+ Handles the deletion of the queue, ensuring tasks are handled.
352
+ """
134
353
  if self._closed:
135
354
  return
136
355
  if self._unfinished_tasks > 0:
137
356
  context = {
138
- 'message': f'{self} was destroyed but has work pending!',
357
+ "message": f"{self} was destroyed but has work pending!",
139
358
  }
140
359
  asyncio.get_event_loop().call_exception_handler(context)
360
+
141
361
  @property
142
362
  def name(self) -> str:
363
+ """
364
+ Returns the name of the queue, or its representation.
365
+
366
+ Example:
367
+ >>> print(queue.name)
368
+ """
143
369
  return self._name or repr(self)
370
+
144
371
  def close(self) -> None:
372
+ """
373
+ Closes the queue, preventing further task submissions.
374
+
375
+ Example:
376
+ >>> queue.close()
377
+ """
145
378
  self._closed = True
379
+
146
380
  async def put(self, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
381
+ """
382
+ Asynchronously submits a task to the queue.
383
+
384
+ Args:
385
+ args: Positional arguments for the task.
386
+ kwargs: Keyword arguments for the task.
387
+
388
+ Returns:
389
+ The future result of the task.
390
+
391
+ Example:
392
+ >>> fut = await queue.put(item='task')
393
+ >>> print(await fut)
394
+ """
147
395
  self._ensure_workers()
148
396
  if self._no_futs:
149
397
  return await super().put((args, kwargs))
150
398
  fut = self._create_future()
151
399
  await super().put((args, kwargs, fut))
152
400
  return fut
401
+
153
402
  def put_nowait(self, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
403
+ """
404
+ Immediately submits a task to the queue without waiting.
405
+
406
+ Args:
407
+ args: Positional arguments for the task.
408
+ kwargs: Keyword arguments for the task.
409
+
410
+ Returns:
411
+ The future result of the task.
412
+
413
+ Example:
414
+ >>> fut = queue.put_nowait(item='task')
415
+ >>> print(await fut)
416
+ """
154
417
  self._ensure_workers()
155
418
  if self._no_futs:
156
419
  return super().put_nowait((args, kwargs))
157
420
  fut = self._create_future()
158
421
  super().put_nowait((args, kwargs, weakref.proxy(fut)))
159
422
  return fut
423
+
160
424
  def _create_future(self) -> "asyncio.Future[V]":
425
+ """Creates a future for the task."""
161
426
  return asyncio.get_event_loop().create_future()
427
+
162
428
  def _ensure_workers(self) -> None:
429
+ """Ensures that the worker tasks are running."""
163
430
  if self._closed:
164
431
  raise RuntimeError(f"{type(self).__name__} is closed: ", self) from None
165
432
  if self._workers.done():
@@ -170,29 +437,40 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
170
437
  # re-raise with clean traceback
171
438
  try:
172
439
  raise type(exc)(*exc.args).with_traceback(exc.__traceback__) # type: ignore [union-attr]
173
- except TypeError:
174
- raise exc.with_traceback(exc.__traceback__)
440
+ except TypeError as e:
441
+ raise exc.with_traceback(exc.__traceback__) from e
175
442
  # this should never be reached, but just in case
176
443
  exc = self._workers.exception()
177
444
  try:
178
445
  # re-raise with clean traceback
179
446
  raise type(exc)(*exc.args).with_traceback(exc.__traceback__) # type: ignore [union-attr]
180
- except TypeError:
181
- raise exc.with_traceback(exc.__traceback__)
447
+ except TypeError as e:
448
+ raise exc.with_traceback(exc.__traceback__) from e
449
+
182
450
  @functools.cached_property
183
451
  def _workers(self) -> "asyncio.Task[NoReturn]":
452
+ """Creates and manages the worker tasks for the queue."""
184
453
  logger.debug("starting worker task for %s", self)
185
454
  workers = [
186
455
  create_task(
187
- coro=self._worker_coro(),
456
+ coro=self._worker_coro(),
188
457
  name=f"{self.name} [Task-{i}]",
189
458
  log_destroy_pending=False,
190
- ) for i in range(self.num_workers)
459
+ )
460
+ for i in range(self.num_workers)
191
461
  ]
192
- task = create_task(asyncio.gather(*workers), name=f"{self.name} worker main Task", log_destroy_pending=False)
462
+ task = create_task(
463
+ asyncio.gather(*workers),
464
+ name=f"{self.name} worker main Task",
465
+ log_destroy_pending=False,
466
+ )
193
467
  task._workers = workers
194
468
  return task
469
+
195
470
  async def __worker_coro(self) -> NoReturn:
471
+ """
472
+ The coroutine executed by worker tasks to process the queue.
473
+ """
196
474
  args: P.args
197
475
  kwargs: P.kwargs
198
476
  if self._no_futs:
@@ -217,15 +495,27 @@ class ProcessingQueue(_Queue[Tuple[P, "asyncio.Future[V]"]], Generic[P, V]):
217
495
  result = await self.func(*args, **kwargs)
218
496
  fut.set_result(result)
219
497
  except asyncio.exceptions.InvalidStateError:
220
- logger.error("cannot set result for %s %s: %s", self.func.__name__, fut, result)
498
+ logger.error(
499
+ "cannot set result for %s %s: %s",
500
+ self.func.__name__,
501
+ fut,
502
+ result,
503
+ )
221
504
  except Exception as e:
222
505
  try:
223
506
  fut.set_exception(e)
224
507
  except asyncio.exceptions.InvalidStateError:
225
- logger.error("cannot set exception for %s %s: %s", self.func.__name__, fut, e)
508
+ logger.error(
509
+ "cannot set exception for %s %s: %s",
510
+ self.func.__name__,
511
+ fut,
512
+ e,
513
+ )
226
514
  self.task_done()
227
515
  except Exception as e:
228
- logger.error("%s for %s is broken!!!", type(self).__name__, self.func)
516
+ logger.error(
517
+ "%s for %s is broken!!!", type(self).__name__, self.func
518
+ )
229
519
  logger.exception(e)
230
520
  raise
231
521
 
@@ -235,97 +525,259 @@ def _validate_args(i: int, can_return_less: bool) -> None:
235
525
  Validates the arguments for methods that retrieve multiple items from the queue.
236
526
 
237
527
  Args:
238
- i (int): The number of items to retrieve.
239
- can_return_less (bool): Whether the method is allowed to return fewer than `i` items.
528
+ i: The number of items to retrieve.
529
+ can_return_less: Whether the method is allowed to return fewer than `i` items.
240
530
 
241
531
  Raises:
242
- TypeError: If `i` is not an integer or `can_return_less` is not a boolean.
243
- ValueError: If `i` is not greater than 1.
532
+ :exc:`~TypeError`: If `i` is not an integer or `can_return_less` is not a boolean.
533
+ :exc:`~ValueError`: If `i` is not greater than 1.
534
+
535
+ Example:
536
+ >>> _validate_args(i=2, can_return_less=False)
244
537
  """
245
538
  if not isinstance(i, int):
246
539
  raise TypeError(f"`i` must be an integer greater than 1. You passed {i}")
247
540
  if not isinstance(can_return_less, bool):
248
- raise TypeError(f"`can_return_less` must be boolean. You passed {can_return_less}")
541
+ raise TypeError(
542
+ f"`can_return_less` must be boolean. You passed {can_return_less}"
543
+ )
249
544
  if i <= 1:
250
545
  raise ValueError(f"`i` must be an integer greater than 1. You passed {i}")
251
546
 
252
547
 
253
-
254
548
  class _SmartFutureRef(weakref.ref, Generic[T]):
549
+ """
550
+ Weak reference for :class:`~_smart.SmartFuture` objects used in priority queues.
551
+ """
552
+
255
553
  def __lt__(self, other: "_SmartFutureRef[T]") -> bool:
256
554
  """
257
555
  Compares two weak references to SmartFuture objects for ordering.
258
556
 
259
- This comparison is used in priority queues to determine the order of processing. A SmartFuture
557
+ This comparison is used in priority queues to determine the order of processing. A SmartFuture
260
558
  reference is considered less than another if it has more waiters or if it has been garbage collected.
261
559
 
262
560
  Args:
263
- other (_SmartFutureRef[T]): The other SmartFuture reference to compare with.
561
+ other: The other SmartFuture reference to compare with.
264
562
 
265
563
  Returns:
266
564
  bool: True if this reference is less than the other, False otherwise.
565
+
566
+ Example:
567
+ >>> ref1 = _SmartFutureRef(fut1)
568
+ >>> ref2 = _SmartFutureRef(fut2)
569
+ >>> print(ref1 < ref2)
267
570
  """
268
571
  strong_self = self()
269
572
  if strong_self is None:
270
573
  return True
271
574
  strong_other = other()
272
- if strong_other is None:
273
- return False
274
- return strong_self < strong_other
575
+ return False if strong_other is None else strong_self < strong_other
576
+
275
577
 
276
578
  class _PriorityQueueMixin(Generic[T]):
579
+ """
580
+ Mixin for creating priority queue functionality with support for custom comparison.
581
+ """
582
+
277
583
  def _init(self, maxsize):
584
+ """
585
+ Initializes the priority queue.
586
+
587
+ Example:
588
+ >>> queue._init(maxsize=10)
589
+ """
278
590
  self._queue: List[T] = []
591
+
279
592
  def _put(self, item, heappush=heapq.heappush):
593
+ """
594
+ Adds an item to the priority queue based on its priority.
595
+
596
+ Example:
597
+ >>> queue._put(item='task')
598
+ """
280
599
  heappush(self._queue, item)
600
+
281
601
  def _get(self, heappop=heapq.heappop):
602
+ """
603
+ Retrieves the highest priority item from the queue.
604
+
605
+ Example:
606
+ >>> task = queue._get()
607
+ >>> print(task)
608
+ """
282
609
  return heappop(self._queue)
283
610
 
611
+
284
612
  class PriorityProcessingQueue(_PriorityQueueMixin[T], ProcessingQueue[T, V]):
613
+ """
614
+ A priority-based processing queue where tasks are processed based on priority.
285
615
  # NOTE: WIP
286
- async def put(self, priority: Any, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
616
+ """
617
+
618
+ async def put(
619
+ self, priority: Any, *args: P.args, **kwargs: P.kwargs
620
+ ) -> "asyncio.Future[V]":
621
+ """
622
+ Asynchronously adds a task with priority to the queue.
623
+
624
+ Args:
625
+ priority: The priority of the task.
626
+ args: Positional arguments for the task.
627
+ kwargs: Keyword arguments for the task.
628
+
629
+ Returns:
630
+ The future representing the result of the task.
631
+
632
+ Example:
633
+ >>> fut = await queue.put(priority=1, item='task')
634
+ >>> print(await fut)
635
+ """
287
636
  self._ensure_workers()
288
637
  fut = asyncio.get_event_loop().create_future()
289
638
  await super().put(self, (priority, args, kwargs, fut))
290
639
  return fut
291
- def put_nowait(self, priority: Any, *args: P.args, **kwargs: P.kwargs) -> "asyncio.Future[V]":
640
+
641
+ def put_nowait(
642
+ self, priority: Any, *args: P.args, **kwargs: P.kwargs
643
+ ) -> "asyncio.Future[V]":
644
+ """
645
+ Immediately adds a task with priority to the queue without waiting.
646
+
647
+ Args:
648
+ priority: The priority of the task.
649
+ args: Positional arguments for the task.
650
+ kwargs: Keyword arguments for the task.
651
+
652
+ Returns:
653
+ The future representing the result of the task.
654
+
655
+ Example:
656
+ >>> fut = queue.put_nowait(priority=1, item='task')
657
+ >>> print(await fut)
658
+ """
292
659
  self._ensure_workers()
293
660
  fut = self._create_future()
294
661
  super().put_nowait(self, (priority, args, kwargs, fut))
295
662
  return fut
663
+
296
664
  def _get(self, heappop=heapq.heappop):
665
+ """
666
+ Retrieves the highest priority task from the queue.
667
+
668
+ Returns:
669
+ The priority, task arguments, keyword arguments, and future of the task.
670
+
671
+ Example:
672
+ >>> task = queue._get()
673
+ >>> print(task)
674
+ """
297
675
  priority, args, kwargs, fut = heappop(self._queue)
298
676
  return args, kwargs, fut
299
677
 
678
+
300
679
  class _VariablePriorityQueueMixin(_PriorityQueueMixin[T]):
680
+ """
681
+ Mixin for priority queues where task priorities can be updated dynamically.
682
+ """
683
+
301
684
  def _get(self, heapify=heapq.heapify, heappop=heapq.heappop):
302
- "Resort the heap to consider any changes in priorities and pop the smallest value"
303
- # resort the heap
685
+ """
686
+ Resorts the priority queue to consider any changes in priorities and retrieves the task with the highest updated priority.
687
+
688
+ Args:
689
+ heapify: Function to resort the heap.
690
+ heappop: Function to pop the highest priority task.
691
+
692
+ Returns:
693
+ The highest priority task in the queue.
694
+
695
+ Example:
696
+ >>> task = queue._get()
697
+ >>> print(task)
698
+ """
304
699
  heapify(self._queue)
305
700
  # take the job with the most waiters
306
701
  return heappop(self._queue)
702
+
307
703
  def _get_key(self, *args, **kwargs) -> _smart._Key:
704
+ """
705
+ Generates a unique key for task identification based on arguments.
706
+
707
+ Args:
708
+ args: Positional arguments for the task.
709
+ kwargs: Keyword arguments for the task.
710
+
711
+ Returns:
712
+ The generated key for the task.
713
+
714
+ Example:
715
+ >>> key = queue._get_key(*args, **kwargs)
716
+ >>> print(key)
717
+ """
308
718
  return (args, tuple((kwarg, kwargs[kwarg]) for kwarg in sorted(kwargs)))
309
719
 
720
+
310
721
  class VariablePriorityQueue(_VariablePriorityQueueMixin[T], asyncio.PriorityQueue):
311
- """A PriorityQueue subclass that allows priorities to be updated (or computed) on the fly"""
722
+ """
723
+ A :class:`~asyncio.PriorityQueue` subclass that allows priorities to be updated (or computed) on the fly.
312
724
  # NOTE: WIP
313
-
314
- class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Concatenate[T, P], V]):
315
- """A PriorityProcessingQueue subclass that will execute jobs with the most waiters first"""
725
+ """
726
+
727
+
728
+ class SmartProcessingQueue(
729
+ _VariablePriorityQueueMixin[T], ProcessingQueue[Concatenate[T, P], V]
730
+ ):
731
+ """
732
+ A PriorityProcessingQueue subclass that will execute jobs with the most waiters first
733
+ """
734
+
316
735
  _no_futs = False
317
- _futs: "weakref.WeakValueDictionary[_smart._Key, _smart.SmartFuture[T]]"
736
+ """Whether smart futures are used."""
737
+
738
+ _futs: "weakref.WeakValueDictionary[_smart._Key[T], _smart.SmartFuture[T]]"
739
+ """
740
+ Weak reference dictionary for managing smart futures.
741
+ """
742
+
318
743
  def __init__(
319
- self,
320
- func: Callable[Concatenate[T, P], Awaitable[V]],
321
- num_workers: int,
322
- *,
744
+ self,
745
+ func: Callable[Concatenate[T, P], Awaitable[V]],
746
+ num_workers: int,
747
+ *,
323
748
  name: str = "",
324
749
  loop: Optional[asyncio.AbstractEventLoop] = None,
325
750
  ) -> None:
751
+ """
752
+ Initializes a smart processing queue with the given worker function.
753
+
754
+ Args:
755
+ func: The worker function.
756
+ num_workers: Number of worker tasks.
757
+ name: Optional name for the queue.
758
+ loop: Optional event loop.
759
+
760
+ Example:
761
+ >>> queue = SmartProcessingQueue(func=my_task_func, num_workers=3, name='smart_queue')
762
+ """
326
763
  super().__init__(func, num_workers, return_data=True, name=name, loop=loop)
327
- self._futs: Dict[_smart._Key[T], _smart.SmartFuture[T]] = weakref.WeakValueDictionary()
764
+ self._futs = weakref.WeakValueDictionary()
765
+
328
766
  async def put(self, *args: P.args, **kwargs: P.kwargs) -> _smart.SmartFuture[V]:
767
+ """
768
+ Asynchronously adds a task with smart future handling to the queue.
769
+
770
+ Args:
771
+ args: Positional arguments for the task.
772
+ kwargs: Keyword arguments for the task.
773
+
774
+ Returns:
775
+ The future representing the task's result.
776
+
777
+ Example:
778
+ >>> fut = await queue.put(item='task')
779
+ >>> print(await fut)
780
+ """
329
781
  self._ensure_workers()
330
782
  key = self._get_key(*args, **kwargs)
331
783
  if fut := self._futs.get(key, None):
@@ -334,7 +786,22 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
334
786
  self._futs[key] = fut
335
787
  await Queue.put(self, (_SmartFutureRef(fut), args, kwargs))
336
788
  return fut
789
+
337
790
  def put_nowait(self, *args: P.args, **kwargs: P.kwargs) -> _smart.SmartFuture[V]:
791
+ """
792
+ Immediately adds a task with smart future handling to the queue without waiting.
793
+
794
+ Args:
795
+ args: Positional arguments for the task.
796
+ kwargs: Keyword arguments for the task.
797
+
798
+ Returns:
799
+ The future representing the task's result.
800
+
801
+ Example:
802
+ >>> fut = queue.put_nowait(item='task')
803
+ >>> print(await fut)
804
+ """
338
805
  self._ensure_workers()
339
806
  key = self._get_key(*args, **kwargs)
340
807
  if fut := self._futs.get(key, None):
@@ -343,12 +810,37 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
343
810
  self._futs[key] = fut
344
811
  Queue.put_nowait(self, (_SmartFutureRef(fut), args, kwargs))
345
812
  return fut
813
+
346
814
  def _create_future(self, key: _smart._Key) -> "asyncio.Future[V]":
815
+ """Creates a smart future for the task."""
347
816
  return _smart.create_future(queue=self, key=key, loop=self._loop)
817
+
348
818
  def _get(self):
819
+ """
820
+ Retrieves the task with the highest priority from the queue.
821
+
822
+ Returns:
823
+ The priority, task arguments, keyword arguments, and future of the task.
824
+
825
+ Example:
826
+ >>> task = queue._get()
827
+ >>> print(task)
828
+ """
349
829
  fut, args, kwargs = super()._get()
350
830
  return args, kwargs, fut()
831
+
351
832
  async def __worker_coro(self) -> NoReturn:
833
+ """
834
+ Worker coroutine responsible for processing tasks in the queue.
835
+
836
+ Retrieves tasks, executes them, and sets the results or exceptions for the futures.
837
+
838
+ Raises:
839
+ Any: Exceptions raised during task processing are logged.
840
+
841
+ Example:
842
+ >>> await queue.__worker_coro()
843
+ """
352
844
  args: P.args
353
845
  kwargs: P.kwargs
354
846
  fut: _smart.SmartFuture[V]
@@ -364,13 +856,23 @@ class SmartProcessingQueue(_VariablePriorityQueueMixin[T], ProcessingQueue[Conca
364
856
  result = await self.func(*args, **kwargs)
365
857
  fut.set_result(result)
366
858
  except asyncio.exceptions.InvalidStateError:
367
- logger.error("cannot set result for %s %s: %s", self.func.__name__, fut, result)
859
+ logger.error(
860
+ "cannot set result for %s %s: %s",
861
+ self.func.__name__,
862
+ fut,
863
+ result,
864
+ )
368
865
  except Exception as e:
369
866
  logger.debug("%s: %s", type(e).__name__, e)
370
867
  try:
371
868
  fut.set_exception(e)
372
869
  except asyncio.exceptions.InvalidStateError:
373
- logger.error("cannot set exception for %s %s: %s", self.func.__name__, fut, e)
870
+ logger.error(
871
+ "cannot set exception for %s %s: %s",
872
+ self.func.__name__,
873
+ fut,
874
+ e,
875
+ )
374
876
  self.task_done()
375
877
  except Exception as e:
376
878
  logger.error("%s for %s is broken!!!", type(self).__name__, self.func)