dycw-utilities 0.129.10__py3-none-any.whl → 0.175.17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dycw_utilities-0.175.17.dist-info/METADATA +34 -0
- dycw_utilities-0.175.17.dist-info/RECORD +103 -0
- dycw_utilities-0.175.17.dist-info/WHEEL +4 -0
- dycw_utilities-0.175.17.dist-info/entry_points.txt +4 -0
- utilities/__init__.py +1 -1
- utilities/altair.py +14 -14
- utilities/asyncio.py +350 -819
- utilities/atomicwrites.py +18 -6
- utilities/atools.py +77 -22
- utilities/cachetools.py +24 -29
- utilities/click.py +393 -237
- utilities/concurrent.py +8 -11
- utilities/contextlib.py +216 -17
- utilities/contextvars.py +20 -1
- utilities/cryptography.py +3 -3
- utilities/dataclasses.py +83 -118
- utilities/docker.py +293 -0
- utilities/enum.py +26 -23
- utilities/errors.py +17 -3
- utilities/fastapi.py +29 -65
- utilities/fpdf2.py +3 -3
- utilities/functions.py +169 -416
- utilities/functools.py +18 -19
- utilities/git.py +9 -30
- utilities/grp.py +28 -0
- utilities/gzip.py +31 -0
- utilities/http.py +3 -2
- utilities/hypothesis.py +738 -589
- utilities/importlib.py +17 -1
- utilities/inflect.py +25 -0
- utilities/iterables.py +194 -262
- utilities/jinja2.py +148 -0
- utilities/json.py +70 -0
- utilities/libcst.py +38 -17
- utilities/lightweight_charts.py +5 -9
- utilities/logging.py +345 -543
- utilities/math.py +18 -13
- utilities/memory_profiler.py +11 -15
- utilities/more_itertools.py +200 -131
- utilities/operator.py +33 -29
- utilities/optuna.py +6 -6
- utilities/orjson.py +272 -137
- utilities/os.py +61 -4
- utilities/parse.py +59 -61
- utilities/pathlib.py +281 -40
- utilities/permissions.py +298 -0
- utilities/pickle.py +2 -2
- utilities/platform.py +24 -5
- utilities/polars.py +1214 -430
- utilities/polars_ols.py +1 -1
- utilities/postgres.py +408 -0
- utilities/pottery.py +113 -26
- utilities/pqdm.py +10 -11
- utilities/psutil.py +6 -57
- utilities/pwd.py +28 -0
- utilities/pydantic.py +4 -54
- utilities/pydantic_settings.py +240 -0
- utilities/pydantic_settings_sops.py +76 -0
- utilities/pyinstrument.py +8 -10
- utilities/pytest.py +227 -121
- utilities/pytest_plugins/__init__.py +1 -0
- utilities/pytest_plugins/pytest_randomly.py +23 -0
- utilities/pytest_plugins/pytest_regressions.py +56 -0
- utilities/pytest_regressions.py +26 -46
- utilities/random.py +13 -9
- utilities/re.py +58 -28
- utilities/redis.py +401 -550
- utilities/scipy.py +1 -1
- utilities/sentinel.py +10 -0
- utilities/shelve.py +4 -1
- utilities/shutil.py +25 -0
- utilities/slack_sdk.py +36 -106
- utilities/sqlalchemy.py +502 -473
- utilities/sqlalchemy_polars.py +38 -94
- utilities/string.py +2 -3
- utilities/subprocess.py +1572 -0
- utilities/tempfile.py +86 -4
- utilities/testbook.py +50 -0
- utilities/text.py +165 -42
- utilities/timer.py +37 -65
- utilities/traceback.py +158 -929
- utilities/types.py +146 -116
- utilities/typing.py +531 -71
- utilities/tzdata.py +1 -53
- utilities/tzlocal.py +6 -23
- utilities/uuid.py +43 -5
- utilities/version.py +27 -26
- utilities/whenever.py +1776 -386
- utilities/zoneinfo.py +84 -22
- dycw_utilities-0.129.10.dist-info/METADATA +0 -241
- dycw_utilities-0.129.10.dist-info/RECORD +0 -96
- dycw_utilities-0.129.10.dist-info/WHEEL +0 -4
- dycw_utilities-0.129.10.dist-info/licenses/LICENSE +0 -21
- utilities/datetime.py +0 -1409
- utilities/eventkit.py +0 -402
- utilities/loguru.py +0 -144
- utilities/luigi.py +0 -228
- utilities/period.py +0 -324
- utilities/pyrsistent.py +0 -89
- utilities/python_dotenv.py +0 -105
- utilities/streamlit.py +0 -105
- utilities/sys.py +0 -87
- utilities/tenacity.py +0 -145
utilities/asyncio.py
CHANGED
|
@@ -1,22 +1,17 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import asyncio
|
|
3
4
|
from asyncio import (
|
|
4
|
-
Event,
|
|
5
5
|
Lock,
|
|
6
|
-
PriorityQueue,
|
|
7
6
|
Queue,
|
|
8
7
|
QueueEmpty,
|
|
9
|
-
QueueFull,
|
|
10
8
|
Semaphore,
|
|
11
9
|
StreamReader,
|
|
12
10
|
Task,
|
|
13
11
|
TaskGroup,
|
|
14
12
|
create_subprocess_shell,
|
|
15
|
-
create_task,
|
|
16
13
|
sleep,
|
|
17
|
-
timeout,
|
|
18
14
|
)
|
|
19
|
-
from collections.abc import Callable, Iterable, Iterator
|
|
20
15
|
from contextlib import (
|
|
21
16
|
AbstractAsyncContextManager,
|
|
22
17
|
AsyncExitStack,
|
|
@@ -24,211 +19,206 @@ from contextlib import (
|
|
|
24
19
|
asynccontextmanager,
|
|
25
20
|
suppress,
|
|
26
21
|
)
|
|
27
|
-
from dataclasses import dataclass
|
|
22
|
+
from dataclasses import dataclass
|
|
28
23
|
from io import StringIO
|
|
29
|
-
from
|
|
30
|
-
from logging import DEBUG, Logger, getLogger
|
|
24
|
+
from pathlib import Path
|
|
31
25
|
from subprocess import PIPE
|
|
32
26
|
from sys import stderr, stdout
|
|
33
27
|
from typing import (
|
|
34
28
|
TYPE_CHECKING,
|
|
35
29
|
Any,
|
|
36
|
-
|
|
30
|
+
ClassVar,
|
|
37
31
|
Self,
|
|
38
32
|
TextIO,
|
|
39
|
-
TypeVar,
|
|
40
33
|
assert_never,
|
|
34
|
+
cast,
|
|
41
35
|
overload,
|
|
42
36
|
override,
|
|
43
37
|
)
|
|
44
38
|
|
|
45
|
-
from typing_extensions import deprecated
|
|
46
|
-
|
|
47
|
-
from utilities.dataclasses import replace_non_sentinel
|
|
48
|
-
from utilities.datetime import (
|
|
49
|
-
SECOND,
|
|
50
|
-
datetime_duration_to_float,
|
|
51
|
-
get_now,
|
|
52
|
-
round_datetime,
|
|
53
|
-
)
|
|
54
|
-
from utilities.errors import repr_error
|
|
55
39
|
from utilities.functions import ensure_int, ensure_not_none
|
|
40
|
+
from utilities.os import is_pytest
|
|
56
41
|
from utilities.random import SYSTEM_RANDOM
|
|
42
|
+
from utilities.reprlib import get_repr
|
|
57
43
|
from utilities.sentinel import Sentinel, sentinel
|
|
58
|
-
from utilities.
|
|
44
|
+
from utilities.shelve import yield_shelf
|
|
45
|
+
from utilities.text import to_bool
|
|
46
|
+
from utilities.warnings import suppress_warnings
|
|
47
|
+
from utilities.whenever import get_now, round_date_or_date_time, to_nanoseconds
|
|
59
48
|
|
|
60
49
|
if TYPE_CHECKING:
|
|
61
|
-
import datetime as dt
|
|
62
50
|
from asyncio import _CoroutineLike
|
|
63
51
|
from asyncio.subprocess import Process
|
|
64
|
-
from collections import
|
|
65
|
-
|
|
52
|
+
from collections.abc import (
|
|
53
|
+
AsyncIterable,
|
|
54
|
+
AsyncIterator,
|
|
55
|
+
Callable,
|
|
56
|
+
ItemsView,
|
|
57
|
+
Iterable,
|
|
58
|
+
Iterator,
|
|
59
|
+
KeysView,
|
|
60
|
+
Sequence,
|
|
61
|
+
ValuesView,
|
|
62
|
+
)
|
|
66
63
|
from contextvars import Context
|
|
67
64
|
from random import Random
|
|
65
|
+
from shelve import Shelf
|
|
68
66
|
from types import TracebackType
|
|
69
67
|
|
|
70
|
-
from
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
_T = TypeVar("_T")
|
|
68
|
+
from whenever import ZonedDateTime
|
|
74
69
|
|
|
70
|
+
from utilities.shelve import _Flag
|
|
71
|
+
from utilities.types import (
|
|
72
|
+
Coro,
|
|
73
|
+
Delta,
|
|
74
|
+
MaybeCallableBoolLike,
|
|
75
|
+
MaybeType,
|
|
76
|
+
PathLike,
|
|
77
|
+
SupportsKeysAndGetItem,
|
|
78
|
+
)
|
|
75
79
|
|
|
76
|
-
class EnhancedQueue(Queue[_T]):
|
|
77
|
-
"""An asynchronous deque."""
|
|
78
80
|
|
|
81
|
+
class AsyncDict[K, V]:
|
|
82
|
+
@overload
|
|
83
|
+
def __init__(self) -> None: ...
|
|
84
|
+
@overload
|
|
85
|
+
def __init__(self, map: SupportsKeysAndGetItem[K, V], /) -> None: ...
|
|
86
|
+
@overload
|
|
87
|
+
def __init__(self, iterable: Iterable[tuple[K, V]], /) -> None: ...
|
|
79
88
|
@override
|
|
80
|
-
def __init__(self,
|
|
81
|
-
super().__init__(
|
|
82
|
-
self.
|
|
83
|
-
self.
|
|
84
|
-
self._putters: deque[Any]
|
|
85
|
-
self._queue: deque[_T]
|
|
86
|
-
self._unfinished_tasks: int
|
|
89
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
90
|
+
super().__init__()
|
|
91
|
+
self._dict = dict[K, V](*args, **kwargs)
|
|
92
|
+
self._lock = Lock()
|
|
87
93
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
raise RuntimeError # pragma: no cover
|
|
94
|
+
async def __aenter__(self) -> dict[K, V]:
|
|
95
|
+
await self._lock.__aenter__()
|
|
96
|
+
return self._dict
|
|
92
97
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
98
|
+
async def __aexit__(
|
|
99
|
+
self,
|
|
100
|
+
exc_type: type[BaseException] | None,
|
|
101
|
+
exc: BaseException | None,
|
|
102
|
+
tb: TracebackType | None,
|
|
103
|
+
/,
|
|
104
|
+
) -> None:
|
|
105
|
+
await self._lock.__aexit__(exc_type, exc, tb)
|
|
97
106
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
async def put(self, item: _T) -> None:
|
|
101
|
-
raise RuntimeError(item) # pragma: no cover
|
|
107
|
+
def __contains__(self, key: Any, /) -> bool:
|
|
108
|
+
return key in self._dict
|
|
102
109
|
|
|
103
110
|
@override
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
raise RuntimeError(item) # pragma: no cover
|
|
111
|
+
def __eq__(self, other: Any, /) -> bool:
|
|
112
|
+
return self._dict == other
|
|
107
113
|
|
|
108
|
-
#
|
|
114
|
+
__hash__: ClassVar[None] = None # pyright: ignore[reportIncompatibleMethodOverride]
|
|
109
115
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
first = await (self.get_right() if reverse else self.get_left())
|
|
113
|
-
return list(chain([first], self.get_all_nowait(reverse=reverse)))
|
|
116
|
+
def __getitem__(self, key: K, /) -> V:
|
|
117
|
+
return self._dict[key]
|
|
114
118
|
|
|
115
|
-
def
|
|
116
|
-
|
|
117
|
-
items: Sequence[_T] = []
|
|
118
|
-
while True:
|
|
119
|
-
try:
|
|
120
|
-
items.append(
|
|
121
|
-
self.get_right_nowait() if reverse else self.get_left_nowait()
|
|
122
|
-
)
|
|
123
|
-
except QueueEmpty:
|
|
124
|
-
return items
|
|
119
|
+
def __iter__(self) -> Iterator[K]:
|
|
120
|
+
yield from self._dict
|
|
125
121
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
async def get_left(self) -> _T:
|
|
129
|
-
"""Remove and return an item from the start of the queue."""
|
|
130
|
-
return await self._get_left_or_right(self._get)
|
|
131
|
-
|
|
132
|
-
async def get_right(self) -> _T:
|
|
133
|
-
"""Remove and return an item from the end of the queue."""
|
|
134
|
-
return await self._get_left_or_right(self._get_right)
|
|
135
|
-
|
|
136
|
-
def get_left_nowait(self) -> _T:
|
|
137
|
-
"""Remove and return an item from the start of the queue without blocking."""
|
|
138
|
-
return self._get_left_or_right_nowait(self._get)
|
|
139
|
-
|
|
140
|
-
def get_right_nowait(self) -> _T:
|
|
141
|
-
"""Remove and return an item from the end of the queue without blocking."""
|
|
142
|
-
return self._get_left_or_right_nowait(self._get_right)
|
|
122
|
+
def __len__(self) -> int:
|
|
123
|
+
return len(self._dict)
|
|
143
124
|
|
|
144
|
-
|
|
125
|
+
@override
|
|
126
|
+
def __repr__(self) -> str:
|
|
127
|
+
return repr(self._dict)
|
|
145
128
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
return await self._put_left_or_right(self._put_left, *items)
|
|
129
|
+
def __reversed__(self) -> Iterator[K]:
|
|
130
|
+
return reversed(self._dict)
|
|
149
131
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
return
|
|
132
|
+
@override
|
|
133
|
+
def __str__(self) -> str:
|
|
134
|
+
return str(self._dict)
|
|
153
135
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
136
|
+
@property
|
|
137
|
+
def empty(self) -> bool:
|
|
138
|
+
return len(self) == 0
|
|
139
|
+
|
|
140
|
+
@classmethod
|
|
141
|
+
@overload
|
|
142
|
+
def fromkeys[T](
|
|
143
|
+
cls, iterable: Iterable[T], value: None = None, /
|
|
144
|
+
) -> AsyncDict[T, Any | None]: ...
|
|
145
|
+
@classmethod
|
|
146
|
+
@overload
|
|
147
|
+
def fromkeys[K2, V2](
|
|
148
|
+
cls, iterable: Iterable[K2], value: V2, /
|
|
149
|
+
) -> AsyncDict[K2, V2]: ...
|
|
150
|
+
@classmethod
|
|
151
|
+
def fromkeys(
|
|
152
|
+
cls, iterable: Iterable[Any], value: Any = None, /
|
|
153
|
+
) -> AsyncDict[Any, Any]:
|
|
154
|
+
return cls(dict.fromkeys(iterable, value))
|
|
155
|
+
|
|
156
|
+
async def clear(self) -> None:
|
|
157
|
+
async with self._lock:
|
|
158
|
+
self._dict.clear()
|
|
157
159
|
|
|
158
|
-
def
|
|
159
|
-
|
|
160
|
-
self._put_left_or_right_nowait(self._put, *items)
|
|
160
|
+
def copy(self) -> Self:
|
|
161
|
+
return type(self)(self._dict.items())
|
|
161
162
|
|
|
162
|
-
|
|
163
|
+
async def del_(self, key: K, /) -> None:
|
|
164
|
+
async with self._lock:
|
|
165
|
+
del self._dict[key]
|
|
166
|
+
|
|
167
|
+
@overload
|
|
168
|
+
def get(self, key: K, default: None = None, /) -> V | None: ...
|
|
169
|
+
@overload
|
|
170
|
+
def get(self, key: K, default: V, /) -> V: ...
|
|
171
|
+
@overload
|
|
172
|
+
def get[V2](self, key: K, default: V2, /) -> V | V2: ...
|
|
173
|
+
def get(self, key: K, default: Any = sentinel, /) -> Any:
|
|
174
|
+
match default:
|
|
175
|
+
case Sentinel():
|
|
176
|
+
return self._dict.get(key)
|
|
177
|
+
case _:
|
|
178
|
+
return self._dict.get(key, default)
|
|
179
|
+
|
|
180
|
+
def keys(self) -> KeysView[K]:
|
|
181
|
+
return self._dict.keys()
|
|
182
|
+
|
|
183
|
+
def items(self) -> ItemsView[K, V]:
|
|
184
|
+
return self._dict.items()
|
|
185
|
+
|
|
186
|
+
@overload
|
|
187
|
+
async def pop(self, key: K, /) -> V: ...
|
|
188
|
+
@overload
|
|
189
|
+
async def pop(self, key: K, default: V, /) -> V: ...
|
|
190
|
+
@overload
|
|
191
|
+
async def pop[V2](self, key: K, default: V2, /) -> V | V2: ...
|
|
192
|
+
async def pop(self, key: K, default: Any = sentinel, /) -> Any:
|
|
193
|
+
async with self._lock:
|
|
194
|
+
match default:
|
|
195
|
+
case Sentinel():
|
|
196
|
+
return self._dict.pop(key)
|
|
197
|
+
case _:
|
|
198
|
+
return self._dict.pop(key, default)
|
|
163
199
|
|
|
164
|
-
def
|
|
165
|
-
self.
|
|
200
|
+
async def popitem(self) -> tuple[K, V]:
|
|
201
|
+
async with self._lock:
|
|
202
|
+
return self._dict.popitem()
|
|
166
203
|
|
|
167
|
-
def
|
|
168
|
-
|
|
204
|
+
async def set(self, key: K, value: V, /) -> None:
|
|
205
|
+
async with self._lock:
|
|
206
|
+
self._dict[key] = value
|
|
169
207
|
|
|
170
|
-
async def
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
self._getters.append(getter)
|
|
174
|
-
try:
|
|
175
|
-
await getter
|
|
176
|
-
except:
|
|
177
|
-
getter.cancel()
|
|
178
|
-
with suppress(ValueError):
|
|
179
|
-
self._getters.remove(getter)
|
|
180
|
-
if not self.empty() and not getter.cancelled():
|
|
181
|
-
self._wakeup_next(self._getters) # pyright: ignore[reportAttributeAccessIssue]
|
|
182
|
-
raise
|
|
183
|
-
return getter_use()
|
|
184
|
-
|
|
185
|
-
def _get_left_or_right_nowait(self, getter: Callable[[], _T], /) -> _T:
|
|
186
|
-
if self.empty():
|
|
187
|
-
raise QueueEmpty
|
|
188
|
-
item = getter()
|
|
189
|
-
self._wakeup_next(self._putters) # pyright: ignore[reportAttributeAccessIssue]
|
|
190
|
-
return item
|
|
191
|
-
|
|
192
|
-
async def _put_left_or_right(
|
|
193
|
-
self, putter_use: Callable[[_T], None], /, *items: _T
|
|
194
|
-
) -> None:
|
|
195
|
-
"""Put an item into the queue."""
|
|
196
|
-
for item in items:
|
|
197
|
-
await self._put_left_or_right_one(putter_use, item)
|
|
208
|
+
async def setdefault(self, key: K, default: V, /) -> V:
|
|
209
|
+
async with self._lock:
|
|
210
|
+
return self._dict.setdefault(key, default)
|
|
198
211
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
self.
|
|
206
|
-
try:
|
|
207
|
-
await putter
|
|
208
|
-
except:
|
|
209
|
-
putter.cancel()
|
|
210
|
-
with suppress(ValueError):
|
|
211
|
-
self._putters.remove(putter)
|
|
212
|
-
if not self.full() and not putter.cancelled():
|
|
213
|
-
self._wakeup_next(self._putters) # pyright: ignore[reportAttributeAccessIssue]
|
|
214
|
-
raise
|
|
215
|
-
return putter_use(item)
|
|
216
|
-
|
|
217
|
-
def _put_left_or_right_nowait(
|
|
218
|
-
self, putter: Callable[[_T], None], /, *items: _T
|
|
219
|
-
) -> None:
|
|
220
|
-
for item in items:
|
|
221
|
-
self._put_left_or_right_nowait_one(putter, item)
|
|
212
|
+
@overload
|
|
213
|
+
async def update(self, m: SupportsKeysAndGetItem[K, V], /) -> None: ...
|
|
214
|
+
@overload
|
|
215
|
+
async def update(self, m: Iterable[tuple[K, V]], /) -> None: ...
|
|
216
|
+
async def update(self, *args: Any, **kwargs: V) -> None:
|
|
217
|
+
async with self._lock:
|
|
218
|
+
self._dict.update(*args, **kwargs)
|
|
222
219
|
|
|
223
|
-
def
|
|
224
|
-
self
|
|
225
|
-
) -> None:
|
|
226
|
-
if self.full(): # pragma: no cover
|
|
227
|
-
raise QueueFull
|
|
228
|
-
putter(item)
|
|
229
|
-
self._unfinished_tasks += 1
|
|
230
|
-
self._finished.clear()
|
|
231
|
-
self._wakeup_next(self._getters) # pyright: ignore[reportAttributeAccessIssue]
|
|
220
|
+
def values(self) -> ValuesView[V]:
|
|
221
|
+
return self._dict.values()
|
|
232
222
|
|
|
233
223
|
|
|
234
224
|
##
|
|
@@ -237,9 +227,11 @@ class EnhancedQueue(Queue[_T]):
|
|
|
237
227
|
class EnhancedTaskGroup(TaskGroup):
|
|
238
228
|
"""Task group with enhanced features."""
|
|
239
229
|
|
|
230
|
+
_max_tasks: int | None
|
|
240
231
|
_semaphore: Semaphore | None
|
|
241
|
-
_timeout:
|
|
242
|
-
_error:
|
|
232
|
+
_timeout: Delta | None
|
|
233
|
+
_error: MaybeType[BaseException]
|
|
234
|
+
_debug: MaybeCallableBoolLike
|
|
243
235
|
_stack: AsyncExitStack
|
|
244
236
|
_timeout_cm: _AsyncGeneratorContextManager[None] | None
|
|
245
237
|
|
|
@@ -248,13 +240,19 @@ class EnhancedTaskGroup(TaskGroup):
|
|
|
248
240
|
self,
|
|
249
241
|
*,
|
|
250
242
|
max_tasks: int | None = None,
|
|
251
|
-
timeout:
|
|
252
|
-
error:
|
|
243
|
+
timeout: Delta | None = None,
|
|
244
|
+
error: MaybeType[BaseException] = TimeoutError,
|
|
245
|
+
debug: MaybeCallableBoolLike = False,
|
|
253
246
|
) -> None:
|
|
254
247
|
super().__init__()
|
|
255
|
-
self.
|
|
248
|
+
self._max_tasks = max_tasks
|
|
249
|
+
if (max_tasks is None) or (max_tasks <= 0):
|
|
250
|
+
self._semaphore = None
|
|
251
|
+
else:
|
|
252
|
+
self._semaphore = Semaphore(max_tasks)
|
|
256
253
|
self._timeout = timeout
|
|
257
254
|
self._error = error
|
|
255
|
+
self._debug = debug
|
|
258
256
|
self._stack = AsyncExitStack()
|
|
259
257
|
self._timeout_cm = None
|
|
260
258
|
|
|
@@ -271,16 +269,23 @@ class EnhancedTaskGroup(TaskGroup):
|
|
|
271
269
|
tb: TracebackType | None,
|
|
272
270
|
) -> None:
|
|
273
271
|
_ = await self._stack.__aexit__(et, exc, tb)
|
|
274
|
-
|
|
272
|
+
match self._is_debug():
|
|
273
|
+
case True:
|
|
274
|
+
with suppress(Exception):
|
|
275
|
+
_ = await super().__aexit__(et, exc, tb)
|
|
276
|
+
case False:
|
|
277
|
+
_ = await super().__aexit__(et, exc, tb)
|
|
278
|
+
case never:
|
|
279
|
+
assert_never(never)
|
|
275
280
|
|
|
276
281
|
@override
|
|
277
|
-
def create_task(
|
|
282
|
+
def create_task[T](
|
|
278
283
|
self,
|
|
279
|
-
coro: _CoroutineLike[
|
|
284
|
+
coro: _CoroutineLike[T],
|
|
280
285
|
*,
|
|
281
286
|
name: str | None = None,
|
|
282
287
|
context: Context | None = None,
|
|
283
|
-
) -> Task[
|
|
288
|
+
) -> Task[T]:
|
|
284
289
|
if self._semaphore is None:
|
|
285
290
|
coroutine = coro
|
|
286
291
|
else:
|
|
@@ -288,641 +293,110 @@ class EnhancedTaskGroup(TaskGroup):
|
|
|
288
293
|
coroutine = self._wrap_with_timeout(coroutine)
|
|
289
294
|
return super().create_task(coroutine, name=name, context=context)
|
|
290
295
|
|
|
291
|
-
def create_task_context(self, cm: AbstractAsyncContextManager[
|
|
296
|
+
def create_task_context[T](self, cm: AbstractAsyncContextManager[T], /) -> Task[T]:
|
|
292
297
|
"""Have the TaskGroup start an asynchronous context manager."""
|
|
293
298
|
_ = self._stack.push_async_callback(cm.__aexit__, None, None, None)
|
|
294
299
|
return self.create_task(cm.__aenter__())
|
|
295
300
|
|
|
296
|
-
async def
|
|
297
|
-
self, semaphore: Semaphore, coroutine: _CoroutineLike[_T], /
|
|
298
|
-
) -> _T:
|
|
299
|
-
async with semaphore:
|
|
300
|
-
return await coroutine
|
|
301
|
-
|
|
302
|
-
async def _wrap_with_timeout(self, coroutine: _CoroutineLike[_T], /) -> _T:
|
|
303
|
-
async with timeout_dur(duration=self._timeout, error=self._error):
|
|
304
|
-
return await coroutine
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
##
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
@dataclass(kw_only=True, slots=True)
|
|
311
|
-
class LooperError(Exception): ...
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
@dataclass(kw_only=True, slots=True)
|
|
315
|
-
class _LooperNoTaskError(LooperError):
|
|
316
|
-
looper: Looper
|
|
317
|
-
|
|
318
|
-
@override
|
|
319
|
-
def __str__(self) -> str:
|
|
320
|
-
return f"{self.looper} has no running task"
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
@dataclass(kw_only=True, unsafe_hash=True)
|
|
324
|
-
class Looper(Generic[_T]):
|
|
325
|
-
"""A looper of a core coroutine, handling errors."""
|
|
326
|
-
|
|
327
|
-
auto_start: bool = field(default=False, repr=False)
|
|
328
|
-
freq: Duration = field(default=SECOND, repr=False)
|
|
329
|
-
backoff: Duration = field(default=10 * SECOND, repr=False)
|
|
330
|
-
empty_upon_exit: bool = field(default=False, repr=False)
|
|
331
|
-
logger: str | None = field(default=None, repr=False)
|
|
332
|
-
timeout: Duration | None = field(default=None, repr=False)
|
|
333
|
-
# settings
|
|
334
|
-
_backoff: float = field(init=False, repr=False)
|
|
335
|
-
_debug: bool = field(default=False, repr=False)
|
|
336
|
-
_freq: float = field(init=False, repr=False)
|
|
337
|
-
# counts
|
|
338
|
-
_entries: int = field(default=0, init=False, repr=False)
|
|
339
|
-
_core_attempts: int = field(default=0, init=False, repr=False)
|
|
340
|
-
_core_successes: int = field(default=0, init=False, repr=False)
|
|
341
|
-
_core_failures: int = field(default=0, init=False, repr=False)
|
|
342
|
-
_initialization_attempts: int = field(default=0, init=False, repr=False)
|
|
343
|
-
_initialization_successes: int = field(default=0, init=False, repr=False)
|
|
344
|
-
_initialization_failures: int = field(default=0, init=False, repr=False)
|
|
345
|
-
_tear_down_attempts: int = field(default=0, init=False, repr=False)
|
|
346
|
-
_tear_down_successes: int = field(default=0, init=False, repr=False)
|
|
347
|
-
_tear_down_failures: int = field(default=0, init=False, repr=False)
|
|
348
|
-
_restart_attempts: int = field(default=0, init=False, repr=False)
|
|
349
|
-
_restart_successes: int = field(default=0, init=False, repr=False)
|
|
350
|
-
_restart_failures: int = field(default=0, init=False, repr=False)
|
|
351
|
-
_stops: int = field(default=0, init=False, repr=False)
|
|
352
|
-
# flags
|
|
353
|
-
_is_entered: Event = field(default_factory=Event, init=False, repr=False)
|
|
354
|
-
_is_initialized: Event = field(default_factory=Event, init=False, repr=False)
|
|
355
|
-
_is_initializing: Event = field(default_factory=Event, init=False, repr=False)
|
|
356
|
-
_is_pending_back_off: Event = field(default_factory=Event, init=False, repr=False)
|
|
357
|
-
_is_pending_restart: Event = field(default_factory=Event, init=False, repr=False)
|
|
358
|
-
_is_pending_stop: Event = field(default_factory=Event, init=False, repr=False)
|
|
359
|
-
_is_pending_stop_when_empty: Event = field(
|
|
360
|
-
default_factory=Event, init=False, repr=False
|
|
361
|
-
)
|
|
362
|
-
_is_stopped: Event = field(default_factory=Event, init=False, repr=False)
|
|
363
|
-
_is_tearing_down: Event = field(default_factory=Event, init=False, repr=False)
|
|
364
|
-
# internal objects
|
|
365
|
-
_lock: Lock = field(default_factory=Lock, init=False, repr=False, hash=False)
|
|
366
|
-
_logger: Logger = field(init=False, repr=False, hash=False)
|
|
367
|
-
_queue: EnhancedQueue[_T] = field(
|
|
368
|
-
default_factory=EnhancedQueue, init=False, repr=False, hash=False
|
|
369
|
-
)
|
|
370
|
-
_stack: AsyncExitStack = field(
|
|
371
|
-
default_factory=AsyncExitStack, init=False, repr=False, hash=False
|
|
372
|
-
)
|
|
373
|
-
_task: Task[None] | None = field(default=None, init=False, repr=False, hash=False)
|
|
374
|
-
|
|
375
|
-
def __post_init__(self) -> None:
|
|
376
|
-
self._backoff = datetime_duration_to_float(self.backoff)
|
|
377
|
-
self._freq = datetime_duration_to_float(self.freq)
|
|
378
|
-
self._logger = getLogger(name=self.logger)
|
|
379
|
-
self._logger.setLevel(DEBUG)
|
|
380
|
-
|
|
381
|
-
async def __aenter__(self) -> Self:
|
|
382
|
-
"""Enter the context manager."""
|
|
383
|
-
match self._is_entered.is_set():
|
|
384
|
-
case True:
|
|
385
|
-
_ = self._debug and self._logger.debug("%s: already entered", self)
|
|
386
|
-
case False:
|
|
387
|
-
_ = self._debug and self._logger.debug("%s: entering context...", self)
|
|
388
|
-
self._is_entered.set()
|
|
389
|
-
async with self._lock:
|
|
390
|
-
self._entries += 1
|
|
391
|
-
self._task = create_task(self.run_looper())
|
|
392
|
-
for looper in self._yield_sub_loopers():
|
|
393
|
-
_ = self._debug and self._logger.debug(
|
|
394
|
-
"%s: adding sub-looper %s", self, looper
|
|
395
|
-
)
|
|
396
|
-
_ = await self._stack.enter_async_context(looper)
|
|
397
|
-
if self.auto_start:
|
|
398
|
-
_ = self._debug and self._logger.debug("%s: auto-starting...", self)
|
|
399
|
-
with suppress(TimeoutError):
|
|
400
|
-
await self._task
|
|
401
|
-
case _ as never:
|
|
402
|
-
assert_never(never)
|
|
403
|
-
return self
|
|
404
|
-
|
|
405
|
-
async def __aexit__(
|
|
301
|
+
async def run_or_create_many_tasks[**P, T](
|
|
406
302
|
self,
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
) ->
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
):
|
|
421
|
-
_ = self._debug and self._logger.warning(
|
|
422
|
-
"%s: encountered %s whilst in context",
|
|
423
|
-
self,
|
|
424
|
-
repr_error(exc_value),
|
|
425
|
-
)
|
|
426
|
-
_ = await self._stack.__aexit__(exc_type, exc_value, traceback)
|
|
427
|
-
await self.stop()
|
|
428
|
-
if self.empty_upon_exit:
|
|
429
|
-
await self.run_until_empty()
|
|
430
|
-
case False:
|
|
431
|
-
_ = self._debug and self._logger.debug("%s: already exited", self)
|
|
432
|
-
case _ as never:
|
|
433
|
-
assert_never(never)
|
|
434
|
-
|
|
435
|
-
def __await__(self) -> Any:
|
|
436
|
-
if (task := self._task) is None: # cannot use match
|
|
437
|
-
raise _LooperNoTaskError(looper=self)
|
|
438
|
-
return task.__await__()
|
|
439
|
-
|
|
440
|
-
def __len__(self) -> int:
|
|
441
|
-
return self._queue.qsize()
|
|
442
|
-
|
|
443
|
-
async def _apply_back_off(self) -> None:
|
|
444
|
-
"""Apply a back off period."""
|
|
445
|
-
await sleep(self._backoff)
|
|
446
|
-
self._is_pending_back_off.clear()
|
|
447
|
-
|
|
448
|
-
async def core(self) -> None:
|
|
449
|
-
"""Core part of running the looper."""
|
|
450
|
-
|
|
451
|
-
def empty(self) -> bool:
|
|
452
|
-
"""Check if the queue is empty."""
|
|
453
|
-
return self._queue.empty()
|
|
454
|
-
|
|
455
|
-
def get_all_nowait(self, *, reverse: bool = False) -> Sequence[_T]:
|
|
456
|
-
"""Remove and return all items from the queue without blocking."""
|
|
457
|
-
return self._queue.get_all_nowait(reverse=reverse)
|
|
458
|
-
|
|
459
|
-
def get_left_nowait(self) -> _T:
|
|
460
|
-
"""Remove and return an item from the start of the queue without blocking."""
|
|
461
|
-
return self._queue.get_left_nowait()
|
|
462
|
-
|
|
463
|
-
def get_right_nowait(self) -> _T:
|
|
464
|
-
"""Remove and return an item from the end of the queue without blocking."""
|
|
465
|
-
return self._queue.get_right_nowait()
|
|
466
|
-
|
|
467
|
-
async def initialize(
|
|
468
|
-
self, *, skip_sleep_if_failure: bool = False
|
|
469
|
-
) -> Exception | None:
|
|
470
|
-
"""Initialize the looper."""
|
|
471
|
-
match self._is_initializing.is_set():
|
|
472
|
-
case True:
|
|
473
|
-
_ = self._debug and self._logger.debug("%s: already initializing", self)
|
|
474
|
-
return None
|
|
475
|
-
case False:
|
|
476
|
-
_ = self._debug and self._logger.debug("%s: initializing...", self)
|
|
477
|
-
self._is_initializing.set()
|
|
478
|
-
self._is_initialized.clear()
|
|
479
|
-
async with self._lock:
|
|
480
|
-
self._initialization_attempts += 1
|
|
481
|
-
try:
|
|
482
|
-
await self._initialize_core()
|
|
483
|
-
except Exception as error: # noqa: BLE001
|
|
484
|
-
async with self._lock:
|
|
485
|
-
self._initialization_failures += 1
|
|
486
|
-
ret = error
|
|
487
|
-
match skip_sleep_if_failure:
|
|
488
|
-
case True:
|
|
489
|
-
_ = self._logger.warning(
|
|
490
|
-
"%s: encountered %s whilst initializing",
|
|
491
|
-
self,
|
|
492
|
-
repr_error(error),
|
|
493
|
-
)
|
|
494
|
-
case False:
|
|
495
|
-
_ = self._logger.warning(
|
|
496
|
-
"%s: encountered %s whilst initializing; sleeping for %s...",
|
|
497
|
-
self,
|
|
498
|
-
repr_error(error),
|
|
499
|
-
self.backoff,
|
|
500
|
-
)
|
|
501
|
-
await self._apply_back_off()
|
|
502
|
-
case _ as never:
|
|
503
|
-
assert_never(never)
|
|
504
|
-
else:
|
|
505
|
-
_ = self._debug and self._logger.debug(
|
|
506
|
-
"%s: finished initializing", self
|
|
507
|
-
)
|
|
508
|
-
self._is_initialized.set()
|
|
509
|
-
async with self._lock:
|
|
510
|
-
self._initialization_successes += 1
|
|
511
|
-
ret = None
|
|
512
|
-
finally:
|
|
513
|
-
self._is_initializing.clear()
|
|
514
|
-
return ret
|
|
515
|
-
case _ as never:
|
|
303
|
+
make_coro: Callable[P, _CoroutineLike[T]],
|
|
304
|
+
*args: P.args,
|
|
305
|
+
**kwargs: P.kwargs,
|
|
306
|
+
) -> T | Sequence[Task[T]]:
|
|
307
|
+
match self._is_debug(), self._max_tasks:
|
|
308
|
+
case (True, _) | (False, None):
|
|
309
|
+
return await make_coro(*args, **kwargs)
|
|
310
|
+
case False, int():
|
|
311
|
+
return [
|
|
312
|
+
self.create_task(make_coro(*args, **kwargs))
|
|
313
|
+
for _ in range(self._max_tasks)
|
|
314
|
+
]
|
|
315
|
+
case never:
|
|
516
316
|
assert_never(never)
|
|
517
317
|
|
|
518
|
-
async def
|
|
519
|
-
"""Core part of initializing the looper."""
|
|
520
|
-
|
|
521
|
-
def put_left_nowait(self, *items: _T) -> None:
|
|
522
|
-
"""Put items into the queue at the start without blocking."""
|
|
523
|
-
self._queue.put_left_nowait(*items)
|
|
524
|
-
|
|
525
|
-
def put_right_nowait(self, *items: _T) -> None:
|
|
526
|
-
"""Put items into the queue at the end without blocking."""
|
|
527
|
-
self._queue.put_right_nowait(*items)
|
|
528
|
-
|
|
529
|
-
def qsize(self) -> int:
|
|
530
|
-
"""Get the number of items in the queue."""
|
|
531
|
-
return self._queue.qsize()
|
|
532
|
-
|
|
533
|
-
def replace(
|
|
318
|
+
async def run_or_create_task[T](
|
|
534
319
|
self,
|
|
320
|
+
coro: _CoroutineLike[T],
|
|
535
321
|
*,
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
logger: str | None | Sentinel = sentinel,
|
|
541
|
-
timeout: Duration | None | Sentinel = sentinel,
|
|
542
|
-
_debug: bool | Sentinel = sentinel,
|
|
543
|
-
**kwargs: Any,
|
|
544
|
-
) -> Self:
|
|
545
|
-
"""Replace elements of the looper."""
|
|
546
|
-
return replace_non_sentinel(
|
|
547
|
-
self,
|
|
548
|
-
auto_start=auto_start,
|
|
549
|
-
empty_upon_exit=empty_upon_exit,
|
|
550
|
-
freq=freq,
|
|
551
|
-
backoff=backoff,
|
|
552
|
-
logger=logger,
|
|
553
|
-
timeout=timeout,
|
|
554
|
-
_debug=_debug,
|
|
555
|
-
**kwargs,
|
|
556
|
-
)
|
|
557
|
-
|
|
558
|
-
def request_back_off(self) -> None:
|
|
559
|
-
"""Request the looper to back off."""
|
|
560
|
-
match self._is_pending_back_off.is_set():
|
|
561
|
-
case True:
|
|
562
|
-
_ = self._debug and self._logger.debug(
|
|
563
|
-
"%s: already requested back off", self
|
|
564
|
-
)
|
|
565
|
-
case False:
|
|
566
|
-
_ = self._debug and self._logger.debug(
|
|
567
|
-
"%s: requesting back off...", self
|
|
568
|
-
)
|
|
569
|
-
self._is_pending_back_off.set()
|
|
570
|
-
case _ as never:
|
|
571
|
-
assert_never(never)
|
|
572
|
-
|
|
573
|
-
def request_restart(self) -> None:
|
|
574
|
-
"""Request the looper to restart."""
|
|
575
|
-
match self._is_pending_restart.is_set():
|
|
576
|
-
case True:
|
|
577
|
-
_ = self._debug and self._logger.debug(
|
|
578
|
-
"%s: already requested restart", self
|
|
579
|
-
)
|
|
580
|
-
case False:
|
|
581
|
-
_ = self._debug and self._logger.debug(
|
|
582
|
-
"%s: requesting restart...", self
|
|
583
|
-
)
|
|
584
|
-
self._is_pending_restart.set()
|
|
585
|
-
case _ as never:
|
|
586
|
-
assert_never(never)
|
|
587
|
-
self.request_back_off()
|
|
588
|
-
|
|
589
|
-
def request_stop(self) -> None:
|
|
590
|
-
"""Request the looper to stop."""
|
|
591
|
-
match self._is_pending_stop.is_set():
|
|
592
|
-
case True:
|
|
593
|
-
_ = self._debug and self._logger.debug(
|
|
594
|
-
"%s: already requested stop", self
|
|
595
|
-
)
|
|
596
|
-
case False:
|
|
597
|
-
_ = self._debug and self._logger.debug("%s: requesting stop...", self)
|
|
598
|
-
self._is_pending_stop.set()
|
|
599
|
-
case _ as never:
|
|
600
|
-
assert_never(never)
|
|
601
|
-
|
|
602
|
-
def request_stop_when_empty(self) -> None:
|
|
603
|
-
"""Request the looper to stop when the queue is empty."""
|
|
604
|
-
match self._is_pending_stop_when_empty.is_set():
|
|
322
|
+
name: str | None = None,
|
|
323
|
+
context: Context | None = None,
|
|
324
|
+
) -> T | Task[T]:
|
|
325
|
+
match self._is_debug():
|
|
605
326
|
case True:
|
|
606
|
-
|
|
607
|
-
"%s: already requested stop when empty", self
|
|
608
|
-
)
|
|
327
|
+
return await coro
|
|
609
328
|
case False:
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
)
|
|
613
|
-
self._is_pending_stop_when_empty.set()
|
|
614
|
-
case _ as never:
|
|
615
|
-
assert_never(never)
|
|
616
|
-
|
|
617
|
-
async def restart(self) -> None:
|
|
618
|
-
"""Restart the looper."""
|
|
619
|
-
_ = self._debug and self._logger.debug("%s: restarting...", self)
|
|
620
|
-
self._is_pending_restart.clear()
|
|
621
|
-
async with self._lock:
|
|
622
|
-
self._restart_attempts += 1
|
|
623
|
-
tear_down = await self.tear_down(skip_sleep_if_failure=True)
|
|
624
|
-
initialization = await self.initialize(skip_sleep_if_failure=True)
|
|
625
|
-
match tear_down, initialization:
|
|
626
|
-
case None, None:
|
|
627
|
-
_ = self._debug and self._logger.debug("%s: finished restarting", self)
|
|
628
|
-
async with self._lock:
|
|
629
|
-
self._restart_successes += 1
|
|
630
|
-
case Exception(), None:
|
|
631
|
-
async with self._lock:
|
|
632
|
-
self._restart_failures += 1
|
|
633
|
-
_ = self._logger.warning(
|
|
634
|
-
"%s: encountered %s whilst restarting (tear down); sleeping for %s...",
|
|
635
|
-
self,
|
|
636
|
-
repr_error(tear_down),
|
|
637
|
-
self.backoff,
|
|
638
|
-
)
|
|
639
|
-
await self._apply_back_off()
|
|
640
|
-
case None, Exception():
|
|
641
|
-
async with self._lock:
|
|
642
|
-
self._restart_failures += 1
|
|
643
|
-
_ = self._logger.warning(
|
|
644
|
-
"%s: encountered %s whilst restarting (initialize); sleeping for %s...",
|
|
645
|
-
self,
|
|
646
|
-
repr_error(initialization),
|
|
647
|
-
self.backoff,
|
|
648
|
-
)
|
|
649
|
-
await self._apply_back_off()
|
|
650
|
-
case Exception(), Exception():
|
|
651
|
-
async with self._lock:
|
|
652
|
-
self._restart_failures += 1
|
|
653
|
-
_ = self._logger.warning(
|
|
654
|
-
"%s: encountered %s (tear down) and then %s (initialization) whilst restarting; sleeping for %s...",
|
|
655
|
-
self,
|
|
656
|
-
repr_error(tear_down),
|
|
657
|
-
repr_error(initialization),
|
|
658
|
-
self.backoff,
|
|
659
|
-
)
|
|
660
|
-
await self._apply_back_off()
|
|
661
|
-
case _ as never:
|
|
329
|
+
return self.create_task(coro, name=name, context=context)
|
|
330
|
+
case never:
|
|
662
331
|
assert_never(never)
|
|
663
332
|
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
async with timeout_dur(duration=self.timeout):
|
|
668
|
-
while True:
|
|
669
|
-
if self._is_stopped.is_set():
|
|
670
|
-
_ = self._debug and self._logger.debug("%s: stopped", self)
|
|
671
|
-
return
|
|
672
|
-
if (self._is_pending_stop.is_set()) or (
|
|
673
|
-
self._is_pending_stop_when_empty.is_set() and self.empty()
|
|
674
|
-
):
|
|
675
|
-
await self.stop()
|
|
676
|
-
elif self._is_pending_back_off.is_set():
|
|
677
|
-
await self._apply_back_off()
|
|
678
|
-
elif self._is_pending_restart.is_set():
|
|
679
|
-
await self.restart()
|
|
680
|
-
elif not self._is_initialized.is_set():
|
|
681
|
-
_ = await self.initialize()
|
|
682
|
-
else:
|
|
683
|
-
_ = self._debug and self._logger.debug(
|
|
684
|
-
"%s: running core...", self
|
|
685
|
-
)
|
|
686
|
-
async with self._lock:
|
|
687
|
-
self._core_attempts += 1
|
|
688
|
-
try:
|
|
689
|
-
await self.core()
|
|
690
|
-
except Exception as error: # noqa: BLE001
|
|
691
|
-
_ = self._logger.warning(
|
|
692
|
-
"%s: encountered %s whilst running core...",
|
|
693
|
-
self,
|
|
694
|
-
repr_error(error),
|
|
695
|
-
)
|
|
696
|
-
async with self._lock:
|
|
697
|
-
self._core_failures += 1
|
|
698
|
-
self.request_restart()
|
|
699
|
-
else:
|
|
700
|
-
async with self._lock:
|
|
701
|
-
self._core_successes += 1
|
|
702
|
-
await sleep(self._freq)
|
|
703
|
-
except RuntimeError as error: # pragma: no cover
|
|
704
|
-
if error.args[0] == "generator didn't stop after athrow()":
|
|
705
|
-
return
|
|
706
|
-
raise
|
|
707
|
-
except TimeoutError:
|
|
708
|
-
pass
|
|
709
|
-
|
|
710
|
-
async def run_until_empty(self) -> None:
|
|
711
|
-
"""Run until the queue is empty."""
|
|
712
|
-
while not self.empty():
|
|
713
|
-
await self.core()
|
|
714
|
-
if not self.empty():
|
|
715
|
-
await sleep(self._freq)
|
|
716
|
-
|
|
717
|
-
@property
|
|
718
|
-
def stats(self) -> _LooperStats:
|
|
719
|
-
"""Return the statistics."""
|
|
720
|
-
return _LooperStats(
|
|
721
|
-
entries=self._entries,
|
|
722
|
-
core_attempts=self._core_attempts,
|
|
723
|
-
core_successes=self._core_successes,
|
|
724
|
-
core_failures=self._core_failures,
|
|
725
|
-
initialization_attempts=self._initialization_attempts,
|
|
726
|
-
initialization_successes=self._initialization_successes,
|
|
727
|
-
initialization_failures=self._initialization_failures,
|
|
728
|
-
tear_down_attempts=self._tear_down_attempts,
|
|
729
|
-
tear_down_successes=self._tear_down_successes,
|
|
730
|
-
tear_down_failures=self._tear_down_failures,
|
|
731
|
-
restart_attempts=self._restart_attempts,
|
|
732
|
-
restart_successes=self._restart_successes,
|
|
733
|
-
restart_failures=self._restart_failures,
|
|
734
|
-
stops=self._stops,
|
|
333
|
+
def _is_debug(self) -> bool:
|
|
334
|
+
return to_bool(self._debug) or (
|
|
335
|
+
(self._max_tasks is not None) and (self._max_tasks <= 0)
|
|
735
336
|
)
|
|
736
337
|
|
|
737
|
-
async def
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
case False:
|
|
743
|
-
_ = self._debug and self._logger.debug("%s: stopping...", self)
|
|
744
|
-
self._is_pending_stop.clear()
|
|
745
|
-
self._is_stopped.set()
|
|
746
|
-
async with self._lock:
|
|
747
|
-
self._stops += 1
|
|
748
|
-
_ = self._debug and self._logger.debug("%s: stopped", self)
|
|
749
|
-
case _ as never:
|
|
750
|
-
assert_never(never)
|
|
751
|
-
|
|
752
|
-
async def tear_down(
|
|
753
|
-
self, *, skip_sleep_if_failure: bool = False
|
|
754
|
-
) -> Exception | None:
|
|
755
|
-
"""Tear down the looper."""
|
|
756
|
-
match self._is_tearing_down.is_set():
|
|
757
|
-
case True:
|
|
758
|
-
_ = self._debug and self._logger.debug("%s: already tearing down", self)
|
|
759
|
-
return None
|
|
760
|
-
case False:
|
|
761
|
-
_ = self._debug and self._logger.debug("%s: tearing down...", self)
|
|
762
|
-
self._is_tearing_down.set()
|
|
763
|
-
async with self._lock:
|
|
764
|
-
self._tear_down_attempts += 1
|
|
765
|
-
try:
|
|
766
|
-
await self._tear_down_core()
|
|
767
|
-
except Exception as error: # noqa: BLE001
|
|
768
|
-
async with self._lock:
|
|
769
|
-
self._tear_down_failures += 1
|
|
770
|
-
ret = error
|
|
771
|
-
match skip_sleep_if_failure:
|
|
772
|
-
case True:
|
|
773
|
-
_ = self._logger.warning(
|
|
774
|
-
"%s: encountered %s whilst tearing down",
|
|
775
|
-
self,
|
|
776
|
-
repr_error(error),
|
|
777
|
-
)
|
|
778
|
-
case False:
|
|
779
|
-
_ = self._logger.warning(
|
|
780
|
-
"%s: encountered %s whilst tearing down; sleeping for %s...",
|
|
781
|
-
self,
|
|
782
|
-
repr_error(error),
|
|
783
|
-
self.backoff,
|
|
784
|
-
)
|
|
785
|
-
await self._apply_back_off()
|
|
786
|
-
case _ as never:
|
|
787
|
-
assert_never(never)
|
|
788
|
-
else:
|
|
789
|
-
_ = self._debug and self._logger.debug(
|
|
790
|
-
"%s: finished tearing down", self
|
|
791
|
-
)
|
|
792
|
-
async with self._lock:
|
|
793
|
-
self._tear_down_successes += 1
|
|
794
|
-
ret = None
|
|
795
|
-
finally:
|
|
796
|
-
self._is_tearing_down.clear()
|
|
797
|
-
return ret
|
|
798
|
-
case _ as never:
|
|
799
|
-
assert_never(never)
|
|
800
|
-
|
|
801
|
-
async def _tear_down_core(self) -> None:
|
|
802
|
-
"""Core part of tearing down the looper."""
|
|
803
|
-
|
|
804
|
-
@property
|
|
805
|
-
def with_auto_start(self) -> Self:
|
|
806
|
-
"""Replace the auto start flag of the looper."""
|
|
807
|
-
return self.replace(auto_start=True)
|
|
808
|
-
|
|
809
|
-
def _yield_sub_loopers(self) -> Iterator[Looper]:
|
|
810
|
-
"""Yield all sub-loopers."""
|
|
811
|
-
yield from []
|
|
812
|
-
|
|
338
|
+
async def _wrap_with_semaphore[T](
|
|
339
|
+
self, semaphore: Semaphore, coroutine: _CoroutineLike[T], /
|
|
340
|
+
) -> T:
|
|
341
|
+
async with semaphore:
|
|
342
|
+
return await coroutine
|
|
813
343
|
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
core_attempts: int = 0
|
|
818
|
-
core_successes: int = 0
|
|
819
|
-
core_failures: int = 0
|
|
820
|
-
initialization_attempts: int = 0
|
|
821
|
-
initialization_successes: int = 0
|
|
822
|
-
initialization_failures: int = 0
|
|
823
|
-
tear_down_attempts: int = 0
|
|
824
|
-
tear_down_successes: int = 0
|
|
825
|
-
tear_down_failures: int = 0
|
|
826
|
-
restart_attempts: int = 0
|
|
827
|
-
restart_successes: int = 0
|
|
828
|
-
restart_failures: int = 0
|
|
829
|
-
stops: int = 0
|
|
344
|
+
async def _wrap_with_timeout[T](self, coroutine: _CoroutineLike[T], /) -> T:
|
|
345
|
+
async with timeout_td(self._timeout, error=self._error):
|
|
346
|
+
return await coroutine
|
|
830
347
|
|
|
831
348
|
|
|
832
349
|
##
|
|
833
350
|
|
|
834
351
|
|
|
835
|
-
|
|
836
|
-
"""
|
|
837
|
-
|
|
838
|
-
@override
|
|
839
|
-
def __init__(self, maxsize: int = 0) -> None:
|
|
840
|
-
super().__init__(maxsize)
|
|
841
|
-
self._set: set[THashable] = set()
|
|
842
|
-
|
|
843
|
-
@override
|
|
844
|
-
def _get(self) -> tuple[TSupportsRichComparison, THashable]:
|
|
845
|
-
item = super()._get()
|
|
846
|
-
_, value = item
|
|
847
|
-
self._set.remove(value)
|
|
848
|
-
return item
|
|
849
|
-
|
|
850
|
-
@override
|
|
851
|
-
def _put(self, item: tuple[TSupportsRichComparison, THashable]) -> None:
|
|
852
|
-
_, value = item
|
|
853
|
-
if value not in self._set:
|
|
854
|
-
super()._put(item)
|
|
855
|
-
self._set.add(value)
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
class UniqueQueue(Queue[THashable]):
|
|
859
|
-
"""Queue with unique tasks."""
|
|
352
|
+
def chain_async[T](*iterables: Iterable[T] | AsyncIterable[T]) -> AsyncIterator[T]:
|
|
353
|
+
"""Asynchronous version of `chain`."""
|
|
860
354
|
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
self._set.remove(item)
|
|
870
|
-
return item
|
|
355
|
+
async def iterator() -> AsyncIterator[T]:
|
|
356
|
+
for it in iterables:
|
|
357
|
+
try:
|
|
358
|
+
async for item in cast("AsyncIterable[T]", it):
|
|
359
|
+
yield item
|
|
360
|
+
except TypeError:
|
|
361
|
+
for item in cast("Iterable[T]", it):
|
|
362
|
+
yield item
|
|
871
363
|
|
|
872
|
-
|
|
873
|
-
def _put(self, item: THashable) -> None:
|
|
874
|
-
if item not in self._set:
|
|
875
|
-
super()._put(item)
|
|
876
|
-
self._set.add(item)
|
|
364
|
+
return iterator()
|
|
877
365
|
|
|
878
366
|
|
|
879
367
|
##
|
|
880
368
|
|
|
881
369
|
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
def get_event(
|
|
892
|
-
*, event: MaybeCallableEvent | None | Sentinel = sentinel
|
|
893
|
-
) -> Event | None | Sentinel: ...
|
|
894
|
-
def get_event(
|
|
895
|
-
*, event: MaybeCallableEvent | None | Sentinel = sentinel
|
|
896
|
-
) -> Event | None | Sentinel:
|
|
897
|
-
"""Get the event."""
|
|
898
|
-
match event:
|
|
899
|
-
case Event() | None | Sentinel():
|
|
900
|
-
return event
|
|
901
|
-
case Callable() as func:
|
|
902
|
-
return get_event(event=func())
|
|
903
|
-
case _ as never:
|
|
904
|
-
assert_never(never)
|
|
370
|
+
def get_coroutine_name(func: Callable[[], Coro[Any]], /) -> str:
|
|
371
|
+
"""Get the name of a coroutine, and then dispose of it gracefully."""
|
|
372
|
+
coro = func()
|
|
373
|
+
name = coro.__name__
|
|
374
|
+
with suppress_warnings(
|
|
375
|
+
message="coroutine '.*' was never awaited", category=RuntimeWarning
|
|
376
|
+
):
|
|
377
|
+
del coro
|
|
378
|
+
return name
|
|
905
379
|
|
|
906
380
|
|
|
907
381
|
##
|
|
908
382
|
|
|
909
383
|
|
|
910
|
-
async def get_items(queue: Queue[
|
|
384
|
+
async def get_items[T](queue: Queue[T], /, *, max_size: int | None = None) -> list[T]:
|
|
911
385
|
"""Get items from a queue; if empty then wait."""
|
|
912
386
|
try:
|
|
913
387
|
items = [await queue.get()]
|
|
914
388
|
except RuntimeError as error: # pragma: no cover
|
|
915
|
-
if error.args[0]
|
|
916
|
-
|
|
917
|
-
|
|
389
|
+
if (not is_pytest()) or (error.args[0] != "Event loop is closed"):
|
|
390
|
+
raise
|
|
391
|
+
return []
|
|
918
392
|
max_size_use = None if max_size is None else (max_size - 1)
|
|
919
393
|
items.extend(get_items_nowait(queue, max_size=max_size_use))
|
|
920
394
|
return items
|
|
921
395
|
|
|
922
396
|
|
|
923
|
-
def get_items_nowait(queue: Queue[
|
|
397
|
+
def get_items_nowait[T](queue: Queue[T], /, *, max_size: int | None = None) -> list[T]:
|
|
924
398
|
"""Get items from a queue; no waiting."""
|
|
925
|
-
items: list[
|
|
399
|
+
items: list[T] = []
|
|
926
400
|
if max_size is None:
|
|
927
401
|
while True:
|
|
928
402
|
try:
|
|
@@ -941,13 +415,50 @@ def get_items_nowait(queue: Queue[_T], /, *, max_size: int | None = None) -> lis
|
|
|
941
415
|
##
|
|
942
416
|
|
|
943
417
|
|
|
944
|
-
async def
|
|
418
|
+
async def one_async[T](*iterables: Iterable[T] | AsyncIterable[T]) -> T:
|
|
419
|
+
"""Asynchronous version of `one`."""
|
|
420
|
+
result: T | Sentinel = sentinel
|
|
421
|
+
async for item in chain_async(*iterables):
|
|
422
|
+
if not isinstance(result, Sentinel):
|
|
423
|
+
raise OneAsyncNonUniqueError(iterables=iterables, first=result, second=item)
|
|
424
|
+
result = item
|
|
425
|
+
if isinstance(result, Sentinel):
|
|
426
|
+
raise OneAsyncEmptyError(iterables=iterables)
|
|
427
|
+
return result
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
@dataclass(kw_only=True, slots=True)
|
|
431
|
+
class OneAsyncError[T](Exception):
|
|
432
|
+
iterables: tuple[Iterable[T] | AsyncIterable[T], ...]
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
@dataclass(kw_only=True, slots=True)
|
|
436
|
+
class OneAsyncEmptyError[T](OneAsyncError[T]):
|
|
437
|
+
@override
|
|
438
|
+
def __str__(self) -> str:
|
|
439
|
+
return f"Iterable(s) {get_repr(self.iterables)} must not be empty"
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
@dataclass(kw_only=True, slots=True)
|
|
443
|
+
class OneAsyncNonUniqueError[T](OneAsyncError):
|
|
444
|
+
first: T
|
|
445
|
+
second: T
|
|
446
|
+
|
|
447
|
+
@override
|
|
448
|
+
def __str__(self) -> str:
|
|
449
|
+
return f"Iterable(s) {get_repr(self.iterables)} must contain exactly one item; got {self.first}, {self.second} and perhaps more"
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
##
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
async def put_items[T](items: Iterable[T], queue: Queue[T], /) -> None:
|
|
945
456
|
"""Put items into a queue; if full then wait."""
|
|
946
457
|
for item in items:
|
|
947
458
|
await queue.put(item)
|
|
948
459
|
|
|
949
460
|
|
|
950
|
-
def put_items_nowait(items: Iterable[
|
|
461
|
+
def put_items_nowait[T](items: Iterable[T], queue: Queue[T], /) -> None:
|
|
951
462
|
"""Put items into a queue; no waiting."""
|
|
952
463
|
for item in items:
|
|
953
464
|
queue.put_nowait(item)
|
|
@@ -956,44 +467,39 @@ def put_items_nowait(items: Iterable[_T], queue: Queue[_T], /) -> None:
|
|
|
956
467
|
##
|
|
957
468
|
|
|
958
469
|
|
|
959
|
-
async def
|
|
960
|
-
|
|
961
|
-
|
|
470
|
+
async def sleep_max(
|
|
471
|
+
sleep: Delta | None = None, /, *, random: Random = SYSTEM_RANDOM
|
|
472
|
+
) -> None:
|
|
473
|
+
"""Sleep which accepts deltas."""
|
|
474
|
+
if sleep is None:
|
|
962
475
|
return
|
|
963
|
-
await sleep(
|
|
476
|
+
await asyncio.sleep(random.uniform(0.0, to_nanoseconds(sleep) / 1e9))
|
|
964
477
|
|
|
965
478
|
|
|
966
479
|
##
|
|
967
480
|
|
|
968
481
|
|
|
969
|
-
async def
|
|
970
|
-
|
|
971
|
-
)
|
|
972
|
-
"""Sleep which accepts max durations."""
|
|
973
|
-
if duration is None:
|
|
974
|
-
return
|
|
975
|
-
await sleep(random.uniform(0.0, datetime_duration_to_float(duration)))
|
|
482
|
+
async def sleep_rounded(delta: Delta, /) -> None:
|
|
483
|
+
"""Sleep until a rounded time."""
|
|
484
|
+
await sleep_until(round_date_or_date_time(get_now(), delta, mode="ceil"))
|
|
976
485
|
|
|
977
486
|
|
|
978
487
|
##
|
|
979
488
|
|
|
980
489
|
|
|
981
|
-
async def
|
|
982
|
-
"""Sleep
|
|
983
|
-
|
|
490
|
+
async def sleep_td(delta: Delta | None = None, /) -> None:
|
|
491
|
+
"""Sleep which accepts deltas."""
|
|
492
|
+
if delta is None:
|
|
493
|
+
return
|
|
494
|
+
await sleep(to_nanoseconds(delta) / 1e9)
|
|
984
495
|
|
|
985
496
|
|
|
986
497
|
##
|
|
987
498
|
|
|
988
499
|
|
|
989
|
-
async def
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
"""Sleep until a rounded time; accepts durations."""
|
|
993
|
-
datetime = round_datetime(
|
|
994
|
-
get_now(), duration, mode="ceil", rel_tol=rel_tol, abs_tol=abs_tol
|
|
995
|
-
)
|
|
996
|
-
await sleep_until(datetime)
|
|
500
|
+
async def sleep_until(datetime: ZonedDateTime, /) -> None:
|
|
501
|
+
"""Sleep until a given time."""
|
|
502
|
+
await sleep_td(datetime - get_now())
|
|
997
503
|
|
|
998
504
|
|
|
999
505
|
##
|
|
@@ -1007,27 +513,21 @@ class StreamCommandOutput:
|
|
|
1007
513
|
|
|
1008
514
|
@property
|
|
1009
515
|
def return_code(self) -> int:
|
|
1010
|
-
return ensure_int(self.process.returncode)
|
|
516
|
+
return ensure_int(self.process.returncode)
|
|
1011
517
|
|
|
1012
518
|
|
|
1013
519
|
async def stream_command(cmd: str, /) -> StreamCommandOutput:
|
|
1014
520
|
"""Run a shell command asynchronously and stream its output in real time."""
|
|
1015
|
-
process = await create_subprocess_shell(
|
|
1016
|
-
|
|
1017
|
-
)
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
)
|
|
1021
|
-
proc_stderr = ensure_not_none( # skipif-not-windows
|
|
1022
|
-
process.stderr, desc="process.stderr"
|
|
1023
|
-
)
|
|
1024
|
-
ret_stdout = StringIO() # skipif-not-windows
|
|
1025
|
-
ret_stderr = StringIO() # skipif-not-windows
|
|
1026
|
-
async with TaskGroup() as tg: # skipif-not-windows
|
|
521
|
+
process = await create_subprocess_shell(cmd, stdout=PIPE, stderr=PIPE)
|
|
522
|
+
proc_stdout = ensure_not_none(process.stdout, desc="process.stdout")
|
|
523
|
+
proc_stderr = ensure_not_none(process.stderr, desc="process.stderr")
|
|
524
|
+
ret_stdout = StringIO()
|
|
525
|
+
ret_stderr = StringIO()
|
|
526
|
+
async with TaskGroup() as tg:
|
|
1027
527
|
_ = tg.create_task(_stream_one(proc_stdout, stdout, ret_stdout))
|
|
1028
528
|
_ = tg.create_task(_stream_one(proc_stderr, stderr, ret_stderr))
|
|
1029
|
-
_ = await process.wait()
|
|
1030
|
-
return StreamCommandOutput(
|
|
529
|
+
_ = await process.wait()
|
|
530
|
+
return StreamCommandOutput(
|
|
1031
531
|
process=process, stdout=ret_stdout.getvalue(), stderr=ret_stderr.getvalue()
|
|
1032
532
|
)
|
|
1033
533
|
|
|
@@ -1036,7 +536,7 @@ async def _stream_one(
|
|
|
1036
536
|
input_: StreamReader, out_stream: TextIO, ret_stream: StringIO, /
|
|
1037
537
|
) -> None:
|
|
1038
538
|
"""Asynchronously read from a stream and write to the target output stream."""
|
|
1039
|
-
while True:
|
|
539
|
+
while True:
|
|
1040
540
|
line = await input_.readline()
|
|
1041
541
|
if not line:
|
|
1042
542
|
break
|
|
@@ -1050,35 +550,66 @@ async def _stream_one(
|
|
|
1050
550
|
|
|
1051
551
|
|
|
1052
552
|
@asynccontextmanager
|
|
1053
|
-
async def
|
|
1054
|
-
|
|
553
|
+
async def timeout_td(
|
|
554
|
+
timeout: Delta | None = None, /, *, error: MaybeType[BaseException] = TimeoutError
|
|
1055
555
|
) -> AsyncIterator[None]:
|
|
1056
|
-
"""Timeout context manager which accepts
|
|
1057
|
-
|
|
556
|
+
"""Timeout context manager which accepts deltas."""
|
|
557
|
+
timeout_use = None if timeout is None else (to_nanoseconds(timeout) / 1e9)
|
|
1058
558
|
try:
|
|
1059
|
-
async with timeout(
|
|
559
|
+
async with asyncio.timeout(timeout_use):
|
|
1060
560
|
yield
|
|
1061
561
|
except TimeoutError:
|
|
1062
562
|
raise error from None
|
|
1063
563
|
|
|
1064
564
|
|
|
565
|
+
##
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
_LOCKS: AsyncDict[Path, Lock] = AsyncDict()
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
@asynccontextmanager
|
|
572
|
+
async def yield_locked_shelf(
|
|
573
|
+
path: PathLike,
|
|
574
|
+
/,
|
|
575
|
+
*,
|
|
576
|
+
flag: _Flag = "c",
|
|
577
|
+
protocol: int | None = None,
|
|
578
|
+
writeback: bool = False,
|
|
579
|
+
) -> AsyncIterator[Shelf[Any]]:
|
|
580
|
+
"""Yield a shelf, behind a lock."""
|
|
581
|
+
path = Path(path)
|
|
582
|
+
try:
|
|
583
|
+
lock = _LOCKS[path]
|
|
584
|
+
except KeyError:
|
|
585
|
+
lock = Lock()
|
|
586
|
+
await _LOCKS.set(path, lock)
|
|
587
|
+
async with lock:
|
|
588
|
+
with yield_shelf(
|
|
589
|
+
path, flag=flag, protocol=protocol, writeback=writeback
|
|
590
|
+
) as shelf:
|
|
591
|
+
yield shelf
|
|
592
|
+
|
|
593
|
+
|
|
1065
594
|
__all__ = [
|
|
1066
|
-
"
|
|
595
|
+
"AsyncDict",
|
|
1067
596
|
"EnhancedTaskGroup",
|
|
1068
|
-
"
|
|
1069
|
-
"
|
|
597
|
+
"OneAsyncEmptyError",
|
|
598
|
+
"OneAsyncError",
|
|
599
|
+
"OneAsyncNonUniqueError",
|
|
1070
600
|
"StreamCommandOutput",
|
|
1071
|
-
"
|
|
1072
|
-
"
|
|
1073
|
-
"get_event",
|
|
601
|
+
"chain_async",
|
|
602
|
+
"get_coroutine_name",
|
|
1074
603
|
"get_items",
|
|
1075
604
|
"get_items_nowait",
|
|
605
|
+
"one_async",
|
|
1076
606
|
"put_items",
|
|
1077
607
|
"put_items_nowait",
|
|
1078
|
-
"
|
|
1079
|
-
"
|
|
608
|
+
"sleep_max",
|
|
609
|
+
"sleep_rounded",
|
|
610
|
+
"sleep_td",
|
|
1080
611
|
"sleep_until",
|
|
1081
|
-
"sleep_until_rounded",
|
|
1082
612
|
"stream_command",
|
|
1083
|
-
"
|
|
613
|
+
"timeout_td",
|
|
614
|
+
"yield_locked_shelf",
|
|
1084
615
|
]
|