dycw-utilities 0.123.0__py3-none-any.whl → 0.124.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dycw_utilities-0.123.0.dist-info → dycw_utilities-0.124.1.dist-info}/METADATA +3 -3
- {dycw_utilities-0.123.0.dist-info → dycw_utilities-0.124.1.dist-info}/RECORD +10 -10
- utilities/__init__.py +1 -1
- utilities/asyncio.py +179 -41
- utilities/more_itertools.py +62 -6
- utilities/redis.py +2 -2
- utilities/slack_sdk.py +6 -10
- utilities/sqlalchemy.py +2 -1
- {dycw_utilities-0.123.0.dist-info → dycw_utilities-0.124.1.dist-info}/WHEEL +0 -0
- {dycw_utilities-0.123.0.dist-info → dycw_utilities-0.124.1.dist-info}/licenses/LICENSE +0 -0
@@ -1,12 +1,12 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: dycw-utilities
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.124.1
|
4
4
|
Author-email: Derek Wan <d.wan@icloud.com>
|
5
5
|
License-File: LICENSE
|
6
6
|
Requires-Python: >=3.12
|
7
7
|
Requires-Dist: typing-extensions<4.14,>=4.13.1
|
8
8
|
Provides-Extra: test
|
9
|
-
Requires-Dist: hypothesis<6.132,>=6.131.
|
9
|
+
Requires-Dist: hypothesis<6.132,>=6.131.24; extra == 'test'
|
10
10
|
Requires-Dist: pytest-asyncio<0.27,>=0.26.0; extra == 'test'
|
11
11
|
Requires-Dist: pytest-cov<6.2,>=6.1.1; extra == 'test'
|
12
12
|
Requires-Dist: pytest-instafail<0.6,>=0.5.0; extra == 'test'
|
@@ -80,7 +80,7 @@ Provides-Extra: zzz-test-hypothesis
|
|
80
80
|
Requires-Dist: aiosqlite<0.22,>=0.21.0; extra == 'zzz-test-hypothesis'
|
81
81
|
Requires-Dist: asyncpg<0.31,>=0.30.0; extra == 'zzz-test-hypothesis'
|
82
82
|
Requires-Dist: greenlet<3.3,>=3.2.0; extra == 'zzz-test-hypothesis'
|
83
|
-
Requires-Dist: hypothesis<6.132,>=6.131.
|
83
|
+
Requires-Dist: hypothesis<6.132,>=6.131.24; extra == 'zzz-test-hypothesis'
|
84
84
|
Requires-Dist: luigi<3.7,>=3.6.0; extra == 'zzz-test-hypothesis'
|
85
85
|
Requires-Dist: numpy<2.3,>=2.2.6; extra == 'zzz-test-hypothesis'
|
86
86
|
Requires-Dist: pathvalidate<3.3,>=3.2.3; extra == 'zzz-test-hypothesis'
|
@@ -1,7 +1,7 @@
|
|
1
|
-
utilities/__init__.py,sha256=
|
1
|
+
utilities/__init__.py,sha256=dKnQvkqkUeTn_Fv1OMEAfo6Y3pQb7Z1vdJNhQAe_NYM,60
|
2
2
|
utilities/altair.py,sha256=Gpja-flOo-Db0PIPJLJsgzAlXWoKUjPU1qY-DQ829ek,9156
|
3
3
|
utilities/astor.py,sha256=xuDUkjq0-b6fhtwjhbnebzbqQZAjMSHR1IIS5uOodVg,777
|
4
|
-
utilities/asyncio.py,sha256=
|
4
|
+
utilities/asyncio.py,sha256=joGmwv-WiDLYoK4q41TtVgfIT23s8Ok46jT_yEQGjaM,28240
|
5
5
|
utilities/atomicwrites.py,sha256=geFjn9Pwn-tTrtoGjDDxWli9NqbYfy3gGL6ZBctiqSo,5393
|
6
6
|
utilities/atools.py,sha256=IYMuFSFGSKyuQmqD6v5IUtDlz8PPw0Sr87Cub_gRU3M,1168
|
7
7
|
utilities/cachetools.py,sha256=C1zqOg7BYz0IfQFK8e3qaDDgEZxDpo47F15RTfJM37Q,2910
|
@@ -36,7 +36,7 @@ utilities/luigi.py,sha256=fpH9MbxJDuo6-k9iCXRayFRtiVbUtibCJKugf7ygpv0,5988
|
|
36
36
|
utilities/math.py,sha256=-mQgbah-dPJwOEWf3SonrFoVZ2AVxMgpeQ3dfVa-oJA,26764
|
37
37
|
utilities/memory_profiler.py,sha256=tf2C51P2lCujPGvRt2Rfc7VEw5LDXmVPCG3z_AvBmbU,962
|
38
38
|
utilities/modules.py,sha256=iuvLluJya-hvl1Q25-Jk3dLgx2Es3ck4SjJiEkAlVTs,3195
|
39
|
-
utilities/more_itertools.py,sha256=
|
39
|
+
utilities/more_itertools.py,sha256=fYsGyIPB2s1KRWoH3AkV3CxJxnMLvmidqBf8l1VQnyE,7193
|
40
40
|
utilities/numpy.py,sha256=Xn23sA2ZbVNqwUYEgNJD3XBYH6IbCri_WkHSNhg3NkY,26122
|
41
41
|
utilities/operator.py,sha256=0M2yZJ0PODH47ogFEnkGMBe_cfxwZR02T_92LZVZvHo,3715
|
42
42
|
utilities/optuna.py,sha256=loyJGWTzljgdJaoLhP09PT8Jz6o_pwBOwehY33lHkhw,1923
|
@@ -59,15 +59,15 @@ utilities/pytest_regressions.py,sha256=-SVT9647Dg6-JcdsiaDKXe3NdOmmrvGevLKWwGjxq
|
|
59
59
|
utilities/python_dotenv.py,sha256=iWcnpXbH7S6RoXHiLlGgyuH6udCupAcPd_gQ0eAenQ0,3190
|
60
60
|
utilities/random.py,sha256=lYdjgxB7GCfU_fwFVl5U-BIM_HV3q6_urL9byjrwDM8,4157
|
61
61
|
utilities/re.py,sha256=5J4d8VwIPFVrX2Eb8zfoxImDv7IwiN_U7mJ07wR2Wvs,3958
|
62
|
-
utilities/redis.py,sha256=
|
62
|
+
utilities/redis.py,sha256=fMVKsVCjCv63m_JgoOKqj3wHVzzHR0s4suvZ-mBy1gk,26615
|
63
63
|
utilities/reprlib.py,sha256=Re9bk3n-kC__9DxQmRlevqFA86pE6TtVfWjUgpbVOv0,1849
|
64
64
|
utilities/rich.py,sha256=t50MwwVBsoOLxzmeVFSVpjno4OW6Ufum32skXbV8-Bs,1911
|
65
65
|
utilities/scipy.py,sha256=X6ROnHwiUhAmPhM0jkfEh0-Fd9iRvwiqtCQMOLmOQF8,945
|
66
66
|
utilities/sentinel.py,sha256=3jIwgpMekWgDAxPDA_hXMP2St43cPhciKN3LWiZ7kv0,1248
|
67
67
|
utilities/shelve.py,sha256=HZsMwK4tcIfg3sh0gApx4-yjQnrY4o3V3ZRimvRhoW0,738
|
68
|
-
utilities/slack_sdk.py,sha256=
|
68
|
+
utilities/slack_sdk.py,sha256=Q8UakiB7qo6SUfaBDB0j1N4b8MuFzaD9lG5HGq7rtuw,3200
|
69
69
|
utilities/socket.py,sha256=K77vfREvzoVTrpYKo6MZakol0EYu2q1sWJnnZqL0So0,118
|
70
|
-
utilities/sqlalchemy.py,sha256=
|
70
|
+
utilities/sqlalchemy.py,sha256=p8vsHaNRoeq5zJouIKyp9piFM26wtm5yR4DkzCMFDSw,35471
|
71
71
|
utilities/sqlalchemy_polars.py,sha256=wjJpoUo-yO9E2ujpG_06vV5r2OdvBiQ4yvV6wKCa2Tk,15605
|
72
72
|
utilities/statsmodels.py,sha256=koyiBHvpMcSiBfh99wFUfSggLNx7cuAw3rwyfAhoKpQ,3410
|
73
73
|
utilities/streamlit.py,sha256=U9PJBaKP1IdSykKhPZhIzSPTZsmLsnwbEPZWzNhJPKk,2955
|
@@ -88,7 +88,7 @@ utilities/warnings.py,sha256=un1LvHv70PU-LLv8RxPVmugTzDJkkGXRMZTE2-fTQHw,1771
|
|
88
88
|
utilities/whenever.py,sha256=jS31ZAY5OMxFxLja_Yo5Fidi87Pd-GoVZ7Vi_teqVDA,16743
|
89
89
|
utilities/zipfile.py,sha256=24lQc9ATcJxHXBPc_tBDiJk48pWyRrlxO2fIsFxU0A8,699
|
90
90
|
utilities/zoneinfo.py,sha256=-5j7IQ9nb7gR43rdgA7ms05im-XuqhAk9EJnQBXxCoQ,1874
|
91
|
-
dycw_utilities-0.
|
92
|
-
dycw_utilities-0.
|
93
|
-
dycw_utilities-0.
|
94
|
-
dycw_utilities-0.
|
91
|
+
dycw_utilities-0.124.1.dist-info/METADATA,sha256=oaCBE9DgOXYJkT6-mZs_e6X54BEOtn_snXqk5zrtmNQ,12943
|
92
|
+
dycw_utilities-0.124.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
93
|
+
dycw_utilities-0.124.1.dist-info/licenses/LICENSE,sha256=gppZp16M6nSVpBbUBrNL6JuYfvKwZiKgV7XoKKsHzqo,1066
|
94
|
+
dycw_utilities-0.124.1.dist-info/RECORD,,
|
utilities/__init__.py
CHANGED
utilities/asyncio.py
CHANGED
@@ -8,6 +8,7 @@ from asyncio import (
|
|
8
8
|
PriorityQueue,
|
9
9
|
Queue,
|
10
10
|
QueueEmpty,
|
11
|
+
QueueFull,
|
11
12
|
Semaphore,
|
12
13
|
StreamReader,
|
13
14
|
Task,
|
@@ -27,6 +28,7 @@ from contextlib import (
|
|
27
28
|
)
|
28
29
|
from dataclasses import dataclass, field
|
29
30
|
from io import StringIO
|
31
|
+
from itertools import chain
|
30
32
|
from logging import getLogger
|
31
33
|
from subprocess import PIPE
|
32
34
|
from sys import stderr, stdout
|
@@ -43,6 +45,8 @@ from typing import (
|
|
43
45
|
override,
|
44
46
|
)
|
45
47
|
|
48
|
+
from typing_extensions import deprecated
|
49
|
+
|
46
50
|
from utilities.datetime import (
|
47
51
|
MINUTE,
|
48
52
|
SECOND,
|
@@ -53,7 +57,6 @@ from utilities.datetime import (
|
|
53
57
|
)
|
54
58
|
from utilities.errors import ImpossibleCaseError, repr_error
|
55
59
|
from utilities.functions import ensure_int, ensure_not_none, get_class_name
|
56
|
-
from utilities.reprlib import get_repr
|
57
60
|
from utilities.sentinel import Sentinel, sentinel
|
58
61
|
from utilities.types import (
|
59
62
|
Coroutine1,
|
@@ -67,6 +70,7 @@ from utilities.types import (
|
|
67
70
|
if TYPE_CHECKING:
|
68
71
|
from asyncio import _CoroutineLike
|
69
72
|
from asyncio.subprocess import Process
|
73
|
+
from collections import deque
|
70
74
|
from collections.abc import AsyncIterator, Sequence
|
71
75
|
from contextvars import Context
|
72
76
|
from types import TracebackType
|
@@ -77,6 +81,164 @@ if TYPE_CHECKING:
|
|
77
81
|
_T = TypeVar("_T")
|
78
82
|
|
79
83
|
|
84
|
+
class EnhancedQueue(Queue[_T]):
|
85
|
+
"""An asynchronous deque."""
|
86
|
+
|
87
|
+
@override
|
88
|
+
def __init__(self, maxsize: int = 0) -> None:
|
89
|
+
super().__init__(maxsize=maxsize)
|
90
|
+
self._finished: Event
|
91
|
+
self._getters: deque[Any]
|
92
|
+
self._putters: deque[Any]
|
93
|
+
self._queue: deque[_T]
|
94
|
+
self._unfinished_tasks: int
|
95
|
+
|
96
|
+
@override
|
97
|
+
@deprecated("Use `get_left`/`get_right` instead")
|
98
|
+
async def get(self) -> _T:
|
99
|
+
raise RuntimeError # pragma: no cover
|
100
|
+
|
101
|
+
@override
|
102
|
+
@deprecated("Use `get_left_nowait`/`get_right_nowait` instead")
|
103
|
+
def get_nowait(self) -> _T:
|
104
|
+
raise RuntimeError # pragma: no cover
|
105
|
+
|
106
|
+
@override
|
107
|
+
@deprecated("Use `put_left`/`put_right` instead")
|
108
|
+
async def put(self, item: _T) -> None:
|
109
|
+
raise RuntimeError(item) # pragma: no cover
|
110
|
+
|
111
|
+
@override
|
112
|
+
@deprecated("Use `put_left_nowait`/`put_right_nowait` instead")
|
113
|
+
def put_nowait(self, item: _T) -> None:
|
114
|
+
raise RuntimeError(item) # pragma: no cover
|
115
|
+
|
116
|
+
# get all
|
117
|
+
|
118
|
+
async def get_all(self, *, reverse: bool = False) -> Sequence[_T]:
|
119
|
+
"""Remove and return all items from the queue."""
|
120
|
+
first = await (self.get_right() if reverse else self.get_left())
|
121
|
+
return list(chain([first], self.get_all_nowait(reverse=reverse)))
|
122
|
+
|
123
|
+
def get_all_nowait(self, *, reverse: bool = False) -> Sequence[_T]:
|
124
|
+
"""Remove and return all items from the queue without blocking."""
|
125
|
+
items: Sequence[_T] = []
|
126
|
+
while True:
|
127
|
+
try:
|
128
|
+
items.append(
|
129
|
+
self.get_right_nowait() if reverse else self.get_left_nowait()
|
130
|
+
)
|
131
|
+
except QueueEmpty:
|
132
|
+
return items
|
133
|
+
|
134
|
+
# get left/right
|
135
|
+
|
136
|
+
async def get_left(self) -> _T:
|
137
|
+
"""Remove and return an item from the start of the queue."""
|
138
|
+
return await self._get_left_or_right(self._get)
|
139
|
+
|
140
|
+
async def get_right(self) -> _T:
|
141
|
+
"""Remove and return an item from the end of the queue."""
|
142
|
+
return await self._get_left_or_right(self._get_right)
|
143
|
+
|
144
|
+
def get_left_nowait(self) -> _T:
|
145
|
+
"""Remove and return an item from the start of the queue without blocking."""
|
146
|
+
return self._get_left_or_right_nowait(self._get)
|
147
|
+
|
148
|
+
def get_right_nowait(self) -> _T:
|
149
|
+
"""Remove and return an item from the end of the queue without blocking."""
|
150
|
+
return self._get_left_or_right_nowait(self._get_right)
|
151
|
+
|
152
|
+
# put left/right
|
153
|
+
|
154
|
+
async def put_left(self, *items: _T) -> None:
|
155
|
+
"""Put items into the queue at the start."""
|
156
|
+
return await self._put_left_or_right(self._put_left, *items)
|
157
|
+
|
158
|
+
async def put_right(self, *items: _T) -> None:
|
159
|
+
"""Put items into the queue at the end."""
|
160
|
+
return await self._put_left_or_right(self._put, *items)
|
161
|
+
|
162
|
+
def put_left_nowait(self, *items: _T) -> None:
|
163
|
+
"""Put items into the queue at the start without blocking."""
|
164
|
+
self._put_left_or_right_nowait(self._put_left, *items)
|
165
|
+
|
166
|
+
def put_right_nowait(self, *items: _T) -> None:
|
167
|
+
"""Put items into the queue at the end without blocking."""
|
168
|
+
self._put_left_or_right_nowait(self._put, *items)
|
169
|
+
|
170
|
+
# private
|
171
|
+
|
172
|
+
def _put_left(self, item: _T) -> None:
|
173
|
+
self._queue.appendleft(item)
|
174
|
+
|
175
|
+
def _get_right(self) -> _T:
|
176
|
+
return self._queue.pop()
|
177
|
+
|
178
|
+
async def _get_left_or_right(self, getter_use: Callable[[], _T], /) -> _T:
|
179
|
+
while self.empty(): # pragma: no cover
|
180
|
+
getter = self._get_loop().create_future() # pyright: ignore[reportAttributeAccessIssue]
|
181
|
+
self._getters.append(getter)
|
182
|
+
try:
|
183
|
+
await getter
|
184
|
+
except:
|
185
|
+
getter.cancel()
|
186
|
+
with suppress(ValueError):
|
187
|
+
self._getters.remove(getter)
|
188
|
+
if not self.empty() and not getter.cancelled():
|
189
|
+
self._wakeup_next(self._getters) # pyright: ignore[reportAttributeAccessIssue]
|
190
|
+
raise
|
191
|
+
return getter_use()
|
192
|
+
|
193
|
+
def _get_left_or_right_nowait(self, getter: Callable[[], _T], /) -> _T:
|
194
|
+
if self.empty():
|
195
|
+
raise QueueEmpty
|
196
|
+
item = getter()
|
197
|
+
self._wakeup_next(self._putters) # pyright: ignore[reportAttributeAccessIssue]
|
198
|
+
return item
|
199
|
+
|
200
|
+
async def _put_left_or_right(
|
201
|
+
self, putter_use: Callable[[_T], None], /, *items: _T
|
202
|
+
) -> None:
|
203
|
+
"""Put an item into the queue."""
|
204
|
+
for item in items:
|
205
|
+
await self._put_left_or_right_one(putter_use, item)
|
206
|
+
|
207
|
+
async def _put_left_or_right_one(
|
208
|
+
self, putter_use: Callable[[_T], None], item: _T, /
|
209
|
+
) -> None:
|
210
|
+
"""Put an item into the queue."""
|
211
|
+
while self.full(): # pragma: no cover
|
212
|
+
putter = self._get_loop().create_future() # pyright: ignore[reportAttributeAccessIssue]
|
213
|
+
self._putters.append(putter)
|
214
|
+
try:
|
215
|
+
await putter
|
216
|
+
except:
|
217
|
+
putter.cancel()
|
218
|
+
with suppress(ValueError):
|
219
|
+
self._putters.remove(putter)
|
220
|
+
if not self.full() and not putter.cancelled():
|
221
|
+
self._wakeup_next(self._putters) # pyright: ignore[reportAttributeAccessIssue]
|
222
|
+
raise
|
223
|
+
return putter_use(item)
|
224
|
+
|
225
|
+
def _put_left_or_right_nowait(
|
226
|
+
self, putter: Callable[[_T], None], /, *items: _T
|
227
|
+
) -> None:
|
228
|
+
for item in items:
|
229
|
+
self._put_left_or_right_nowait_one(putter, item)
|
230
|
+
|
231
|
+
def _put_left_or_right_nowait_one(
|
232
|
+
self, putter: Callable[[_T], None], item: _T, /
|
233
|
+
) -> None:
|
234
|
+
if self.full(): # pragma: no cover
|
235
|
+
raise QueueFull
|
236
|
+
putter(item)
|
237
|
+
self._unfinished_tasks += 1
|
238
|
+
self._finished.clear()
|
239
|
+
self._wakeup_next(self._getters) # pyright: ignore[reportAttributeAccessIssue]
|
240
|
+
|
241
|
+
|
80
242
|
##
|
81
243
|
|
82
244
|
|
@@ -428,14 +590,13 @@ class _InfiniteLooperDefaultEventError(InfiniteLooperError):
|
|
428
590
|
class InfiniteQueueLooper(InfiniteLooper[THashable], Generic[THashable, _T]):
|
429
591
|
"""An infinite loop which processes a queue."""
|
430
592
|
|
431
|
-
queue_type: type[Queue[_T]] = field(default=Queue, repr=False)
|
432
593
|
_await_upon_aenter: bool = field(default=False, init=False, repr=False)
|
433
|
-
_queue:
|
594
|
+
_queue: EnhancedQueue[_T] = field(init=False, repr=False)
|
434
595
|
|
435
596
|
@override
|
436
597
|
def __post_init__(self) -> None:
|
437
598
|
super().__post_init__()
|
438
|
-
self._queue =
|
599
|
+
self._queue = EnhancedQueue()
|
439
600
|
|
440
601
|
def __len__(self) -> int:
|
441
602
|
return self._queue.qsize()
|
@@ -443,55 +604,32 @@ class InfiniteQueueLooper(InfiniteLooper[THashable], Generic[THashable, _T]):
|
|
443
604
|
@override
|
444
605
|
async def _core(self) -> None:
|
445
606
|
"""Run the core part of the loop."""
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
except Exception as error: # noqa: BLE001
|
450
|
-
raise InfiniteQueueLooperError(
|
451
|
-
looper=self, items=items, error=error
|
452
|
-
) from None
|
607
|
+
first = await self._queue.get_left()
|
608
|
+
self._queue.put_left_nowait(first)
|
609
|
+
await self._process_queue()
|
453
610
|
|
454
611
|
@abstractmethod
|
455
|
-
async def
|
456
|
-
"""Process the
|
612
|
+
async def _process_queue(self) -> None:
|
613
|
+
"""Process the queue."""
|
457
614
|
|
458
615
|
def empty(self) -> bool:
|
459
616
|
"""Check if the queue is empty."""
|
460
617
|
return self._queue.empty()
|
461
618
|
|
462
|
-
def
|
463
|
-
"""Put items into the queue."""
|
464
|
-
|
619
|
+
def put_left_nowait(self, *items: _T) -> None:
|
620
|
+
"""Put items into the queue at the start without blocking."""
|
621
|
+
self._queue.put_left_nowait(*items) # pragma: no cover
|
622
|
+
|
623
|
+
def put_right_nowait(self, *items: _T) -> None:
|
624
|
+
"""Put items into the queue at the end without blocking."""
|
625
|
+
self._queue.put_right_nowait(*items) # pragma: no cover
|
465
626
|
|
466
627
|
async def run_until_empty(self) -> None:
|
467
628
|
"""Run until the queue is empty."""
|
468
629
|
while not self.empty():
|
469
|
-
await self.
|
630
|
+
await self._process_queue()
|
470
631
|
await self.stop()
|
471
632
|
|
472
|
-
@override
|
473
|
-
def _error_upon_core(self, error: Exception, /) -> None:
|
474
|
-
"""Handle any errors upon running the core function."""
|
475
|
-
if self.logger is not None:
|
476
|
-
if isinstance(error, InfiniteQueueLooperError):
|
477
|
-
getLogger(name=self.logger).error(
|
478
|
-
"%r encountered %s whilst processing %d item(s) %s; sleeping %s...",
|
479
|
-
get_class_name(self),
|
480
|
-
repr_error(error.error),
|
481
|
-
len(error.items),
|
482
|
-
get_repr(error.items),
|
483
|
-
self._sleep_restart_desc,
|
484
|
-
)
|
485
|
-
else:
|
486
|
-
super()._error_upon_core(error) # pragma: no cover
|
487
|
-
|
488
|
-
|
489
|
-
@dataclass(kw_only=True, slots=True)
|
490
|
-
class InfiniteQueueLooperError(Exception, Generic[_T]):
|
491
|
-
looper: InfiniteQueueLooper[Any, Any]
|
492
|
-
items: Sequence[_T]
|
493
|
-
error: Exception
|
494
|
-
|
495
633
|
|
496
634
|
##
|
497
635
|
|
@@ -715,11 +853,11 @@ async def timeout_dur(
|
|
715
853
|
|
716
854
|
|
717
855
|
__all__ = [
|
856
|
+
"EnhancedQueue",
|
718
857
|
"EnhancedTaskGroup",
|
719
858
|
"InfiniteLooper",
|
720
859
|
"InfiniteLooperError",
|
721
860
|
"InfiniteQueueLooper",
|
722
|
-
"InfiniteQueueLooperError",
|
723
861
|
"StreamCommandOutput",
|
724
862
|
"UniquePriorityQueue",
|
725
863
|
"UniqueQueue",
|
utilities/more_itertools.py
CHANGED
@@ -37,25 +37,81 @@ _U = TypeVar("_U")
|
|
37
37
|
|
38
38
|
@overload
|
39
39
|
def bucket_mapping(
|
40
|
-
iterable: Iterable[_T],
|
40
|
+
iterable: Iterable[_T],
|
41
|
+
func: Callable[[_T], THashable],
|
42
|
+
/,
|
43
|
+
*,
|
44
|
+
transform: Callable[[_T], _U],
|
45
|
+
list: Literal[True],
|
46
|
+
) -> Mapping[THashable, Sequence[_U]]: ...
|
47
|
+
@overload
|
48
|
+
def bucket_mapping(
|
49
|
+
iterable: Iterable[_T],
|
50
|
+
func: Callable[[_T], THashable],
|
51
|
+
/,
|
52
|
+
*,
|
53
|
+
transform: Callable[[_T], _U],
|
54
|
+
list: bool = False,
|
55
|
+
) -> Mapping[THashable, Iterator[_U]]: ...
|
56
|
+
@overload
|
57
|
+
def bucket_mapping(
|
58
|
+
iterable: Iterable[_T],
|
59
|
+
func: Callable[[_T], THashable],
|
60
|
+
/,
|
61
|
+
*,
|
62
|
+
transform: Callable[[_T], _U] | None = None,
|
63
|
+
list: Literal[True],
|
41
64
|
) -> Mapping[THashable, Sequence[_T]]: ...
|
42
65
|
@overload
|
43
66
|
def bucket_mapping(
|
44
|
-
iterable: Iterable[_T],
|
67
|
+
iterable: Iterable[_T],
|
68
|
+
func: Callable[[_T], THashable],
|
69
|
+
/,
|
70
|
+
*,
|
71
|
+
transform: Callable[[_T], _U] | None = None,
|
72
|
+
list: bool = False,
|
45
73
|
) -> Mapping[THashable, Iterator[_T]]: ...
|
74
|
+
@overload
|
75
|
+
def bucket_mapping(
|
76
|
+
iterable: Iterable[_T],
|
77
|
+
func: Callable[[_T], THashable],
|
78
|
+
/,
|
79
|
+
*,
|
80
|
+
transform: Callable[[_T], _U] | None = None,
|
81
|
+
list: bool = False,
|
82
|
+
) -> (
|
83
|
+
Mapping[THashable, Iterator[_T]]
|
84
|
+
| Mapping[THashable, Iterator[_U]]
|
85
|
+
| Mapping[THashable, Sequence[_T]]
|
86
|
+
| Mapping[THashable, Sequence[_U]]
|
87
|
+
): ...
|
46
88
|
def bucket_mapping(
|
47
89
|
iterable: Iterable[_T],
|
48
90
|
func: Callable[[_T], THashable],
|
49
91
|
/,
|
50
92
|
*,
|
93
|
+
transform: Callable[[_T], _U] | None = None,
|
51
94
|
list: bool = False, # noqa: A002
|
52
|
-
) ->
|
95
|
+
) -> (
|
96
|
+
Mapping[THashable, Iterator[_T]]
|
97
|
+
| Mapping[THashable, Iterator[_U]]
|
98
|
+
| Mapping[THashable, Sequence[_T]]
|
99
|
+
| Mapping[THashable, Sequence[_U]]
|
100
|
+
):
|
53
101
|
"""Bucket the values of iterable into a mapping."""
|
54
102
|
b = bucket(iterable, func)
|
55
103
|
mapping = {key: b[key] for key in b}
|
56
|
-
|
57
|
-
|
58
|
-
|
104
|
+
match transform, list:
|
105
|
+
case None, False:
|
106
|
+
return mapping
|
107
|
+
case None, True:
|
108
|
+
return {k: builtins.list(v) for k, v in mapping.items()}
|
109
|
+
case _, False:
|
110
|
+
return {k: map(transform, v) for k, v in mapping.items()}
|
111
|
+
case _, True:
|
112
|
+
return {k: builtins.list(map(transform, v)) for k, v in mapping.items()}
|
113
|
+
case _ as never:
|
114
|
+
assert_never(never)
|
59
115
|
|
60
116
|
|
61
117
|
##
|
utilities/redis.py
CHANGED
@@ -597,8 +597,8 @@ class Publisher(InfiniteQueueLooper[None, tuple[str, EncodableT]]):
|
|
597
597
|
timeout: Duration = _PUBLISH_TIMEOUT
|
598
598
|
|
599
599
|
@override
|
600
|
-
async def
|
601
|
-
for item in
|
600
|
+
async def _process_queue(self) -> None:
|
601
|
+
for item in self._queue.get_all_nowait(): # skipif-ci-and-not-linux
|
602
602
|
channel, data = item
|
603
603
|
_ = await publish(
|
604
604
|
self.redis,
|
utilities/slack_sdk.py
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
-
from asyncio import Queue
|
4
3
|
from dataclasses import dataclass
|
5
4
|
from http import HTTPStatus
|
6
5
|
from logging import NOTSET, Handler, LogRecord
|
@@ -47,15 +46,12 @@ class SlackHandler(Handler, InfiniteQueueLooper[None, str]):
|
|
47
46
|
level: int = NOTSET,
|
48
47
|
sleep_core: Duration = _SLEEP,
|
49
48
|
sleep_restart: Duration = _SLEEP,
|
50
|
-
queue_type: type[Queue[str]] = Queue,
|
51
49
|
sender: Callable[[str, str], Coroutine1[None]] = _send_adapter,
|
52
50
|
timeout: Duration = _TIMEOUT,
|
53
51
|
) -> None:
|
54
|
-
InfiniteQueueLooper.__init__( # InfiniteQueueLooper first
|
55
|
-
self, queue_type=queue_type
|
56
|
-
)
|
52
|
+
InfiniteQueueLooper.__init__(self) # InfiniteQueueLooper first
|
57
53
|
InfiniteQueueLooper.__post_init__(self)
|
58
|
-
Handler.__init__(self, level=level)
|
54
|
+
Handler.__init__(self, level=level) # Handler next
|
59
55
|
self.url = url
|
60
56
|
self.sender = sender
|
61
57
|
self.timeout = timeout
|
@@ -65,14 +61,14 @@ class SlackHandler(Handler, InfiniteQueueLooper[None, str]):
|
|
65
61
|
@override
|
66
62
|
def emit(self, record: LogRecord) -> None:
|
67
63
|
try:
|
68
|
-
self.
|
64
|
+
self.put_right_nowait(self.format(record))
|
69
65
|
except Exception: # noqa: BLE001 # pragma: no cover
|
70
66
|
self.handleError(record)
|
71
67
|
|
72
68
|
@override
|
73
|
-
async def
|
74
|
-
|
75
|
-
text = "\n".join(
|
69
|
+
async def _process_queue(self) -> None:
|
70
|
+
messages = self._queue.get_all_nowait()
|
71
|
+
text = "\n".join(messages)
|
76
72
|
async with timeout_dur(duration=self.timeout):
|
77
73
|
await self.sender(self.url, text)
|
78
74
|
|
utilities/sqlalchemy.py
CHANGED
@@ -623,7 +623,8 @@ class Upserter(InfiniteQueueLooper[None, _InsertItem]):
|
|
623
623
|
error_insert: type[Exception] = TimeoutError
|
624
624
|
|
625
625
|
@override
|
626
|
-
async def
|
626
|
+
async def _process_queue(self) -> None:
|
627
|
+
items = self._queue.get_all_nowait()
|
627
628
|
await upsert_items(
|
628
629
|
self.engine,
|
629
630
|
*items,
|
File without changes
|
File without changes
|