dycw-utilities 0.128.0__py3-none-any.whl → 0.129.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dycw_utilities-0.128.0.dist-info → dycw_utilities-0.129.1.dist-info}/METADATA +4 -4
- {dycw_utilities-0.128.0.dist-info → dycw_utilities-0.129.1.dist-info}/RECORD +10 -10
- utilities/__init__.py +1 -1
- utilities/asyncio.py +5 -360
- utilities/fastapi.py +26 -12
- utilities/redis.py +4 -43
- utilities/slack_sdk.py +2 -45
- utilities/sqlalchemy.py +2 -56
- {dycw_utilities-0.128.0.dist-info → dycw_utilities-0.129.1.dist-info}/WHEEL +0 -0
- {dycw_utilities-0.128.0.dist-info → dycw_utilities-0.129.1.dist-info}/licenses/LICENSE +0 -0
@@ -1,13 +1,13 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: dycw-utilities
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.129.1
|
4
4
|
Author-email: Derek Wan <d.wan@icloud.com>
|
5
5
|
License-File: LICENSE
|
6
6
|
Requires-Python: >=3.12
|
7
|
-
Requires-Dist: typing-extensions<4.
|
7
|
+
Requires-Dist: typing-extensions<4.15,>=4.14.0
|
8
8
|
Provides-Extra: test
|
9
9
|
Requires-Dist: dycw-pytest-only<2.2,>=2.1.1; extra == 'test'
|
10
|
-
Requires-Dist: hypothesis<6.
|
10
|
+
Requires-Dist: hypothesis<6.136,>=6.135.0; extra == 'test'
|
11
11
|
Requires-Dist: pytest-asyncio<1.1,>=1.0.0; extra == 'test'
|
12
12
|
Requires-Dist: pytest-cov<6.2,>=6.1.1; extra == 'test'
|
13
13
|
Requires-Dist: pytest-instafail<0.6,>=0.5.0; extra == 'test'
|
@@ -77,7 +77,7 @@ Provides-Extra: zzz-test-hypothesis
|
|
77
77
|
Requires-Dist: aiosqlite<0.22,>=0.21.0; extra == 'zzz-test-hypothesis'
|
78
78
|
Requires-Dist: asyncpg<0.31,>=0.30.0; extra == 'zzz-test-hypothesis'
|
79
79
|
Requires-Dist: greenlet<3.3,>=3.2.0; extra == 'zzz-test-hypothesis'
|
80
|
-
Requires-Dist: hypothesis<6.
|
80
|
+
Requires-Dist: hypothesis<6.136,>=6.135.0; extra == 'zzz-test-hypothesis'
|
81
81
|
Requires-Dist: luigi<3.7,>=3.6.0; extra == 'zzz-test-hypothesis'
|
82
82
|
Requires-Dist: numpy<2.3,>=2.2.6; extra == 'zzz-test-hypothesis'
|
83
83
|
Requires-Dist: pathvalidate<3.3,>=3.2.3; extra == 'zzz-test-hypothesis'
|
@@ -1,6 +1,6 @@
|
|
1
|
-
utilities/__init__.py,sha256=
|
1
|
+
utilities/__init__.py,sha256=SD3J-kHQFgs1qNwFwV6LhMTAXj4jsrYuL05YluwVtcE,60
|
2
2
|
utilities/altair.py,sha256=Gpja-flOo-Db0PIPJLJsgzAlXWoKUjPU1qY-DQ829ek,9156
|
3
|
-
utilities/asyncio.py,sha256=
|
3
|
+
utilities/asyncio.py,sha256=OIQ4JddpQw8tSubzwDR0WyqQ-uE-L5DdbwuTqRQK5MQ,38202
|
4
4
|
utilities/atomicwrites.py,sha256=geFjn9Pwn-tTrtoGjDDxWli9NqbYfy3gGL6ZBctiqSo,5393
|
5
5
|
utilities/atools.py,sha256=IYMuFSFGSKyuQmqD6v5IUtDlz8PPw0Sr87Cub_gRU3M,1168
|
6
6
|
utilities/cachetools.py,sha256=C1zqOg7BYz0IfQFK8e3qaDDgEZxDpo47F15RTfJM37Q,2910
|
@@ -15,7 +15,7 @@ utilities/datetime.py,sha256=aiPh2OZK2g9gn4yEeSO0lODOmvx8U_rGn6XeSzyk4VY,38738
|
|
15
15
|
utilities/enum.py,sha256=HoRwVCWzsnH0vpO9ZEcAAIZLMv0Sn2vJxxA4sYMQgDs,5793
|
16
16
|
utilities/errors.py,sha256=nC7ZYtxxDBMfrTHtT_MByBfup_wfGQFRo3eDt-0ZPe8,1045
|
17
17
|
utilities/eventkit.py,sha256=6M5Xu1SzN-juk9PqBHwy5dS-ta7T0qA6SMpDsakOJ0E,13039
|
18
|
-
utilities/fastapi.py,sha256=
|
18
|
+
utilities/fastapi.py,sha256=gZrXYxKAc7ZEAL_tDmkcbqebkm-KfMCY0X8r-1HF5dI,2962
|
19
19
|
utilities/fpdf2.py,sha256=y1NGXR5chWqLXWpewGV3hlRGMr_5yV1lVRkPBhPEgJI,1843
|
20
20
|
utilities/functions.py,sha256=jgt592voaHNtX56qX0SRvFveVCRmSIxCZmqvpLZCnY8,27305
|
21
21
|
utilities/functools.py,sha256=WrpHt7NLNWSUn9A1Q_ZIWlNaYZOEI4IFKyBG9HO3BC4,1643
|
@@ -60,14 +60,14 @@ utilities/pytest_regressions.py,sha256=YI55B7EtLjhz7zPJZ6NK9bWrxrKCKabWZJe1cwcbA
|
|
60
60
|
utilities/python_dotenv.py,sha256=edXsvHZhZnYeqfMfrsRRpj7_9eJI6uizh3xLx8Q9B3w,3228
|
61
61
|
utilities/random.py,sha256=lYdjgxB7GCfU_fwFVl5U-BIM_HV3q6_urL9byjrwDM8,4157
|
62
62
|
utilities/re.py,sha256=5J4d8VwIPFVrX2Eb8zfoxImDv7IwiN_U7mJ07wR2Wvs,3958
|
63
|
-
utilities/redis.py,sha256=
|
63
|
+
utilities/redis.py,sha256=7Sc-G43VXVzFQU7MHKyI1y3u7My3oT1UoWXPGcKM_-0,36008
|
64
64
|
utilities/reprlib.py,sha256=ssYTcBW-TeRh3fhCJv57sopTZHF5FrPyyUg9yp5XBlo,3953
|
65
65
|
utilities/scipy.py,sha256=X6ROnHwiUhAmPhM0jkfEh0-Fd9iRvwiqtCQMOLmOQF8,945
|
66
66
|
utilities/sentinel.py,sha256=3jIwgpMekWgDAxPDA_hXMP2St43cPhciKN3LWiZ7kv0,1248
|
67
67
|
utilities/shelve.py,sha256=HZsMwK4tcIfg3sh0gApx4-yjQnrY4o3V3ZRimvRhoW0,738
|
68
|
-
utilities/slack_sdk.py,sha256=
|
68
|
+
utilities/slack_sdk.py,sha256=ltmzv68aa73CJGqTDvt8L9vDm22YU9iOCo3NCiNd3vA,4347
|
69
69
|
utilities/socket.py,sha256=K77vfREvzoVTrpYKo6MZakol0EYu2q1sWJnnZqL0So0,118
|
70
|
-
utilities/sqlalchemy.py,sha256=
|
70
|
+
utilities/sqlalchemy.py,sha256=I81qR7JtS-q1sLnw42p7L0FC0imT98gJHGte_KOjpAA,37890
|
71
71
|
utilities/sqlalchemy_polars.py,sha256=s7hQNep2O5DTgIRXyN_JRQma7a4DAtNd25tshaZW8iw,15490
|
72
72
|
utilities/statsmodels.py,sha256=koyiBHvpMcSiBfh99wFUfSggLNx7cuAw3rwyfAhoKpQ,3410
|
73
73
|
utilities/streamlit.py,sha256=U9PJBaKP1IdSykKhPZhIzSPTZsmLsnwbEPZWzNhJPKk,2955
|
@@ -89,7 +89,7 @@ utilities/warnings.py,sha256=un1LvHv70PU-LLv8RxPVmugTzDJkkGXRMZTE2-fTQHw,1771
|
|
89
89
|
utilities/whenever.py,sha256=jS31ZAY5OMxFxLja_Yo5Fidi87Pd-GoVZ7Vi_teqVDA,16743
|
90
90
|
utilities/zipfile.py,sha256=24lQc9ATcJxHXBPc_tBDiJk48pWyRrlxO2fIsFxU0A8,699
|
91
91
|
utilities/zoneinfo.py,sha256=-5j7IQ9nb7gR43rdgA7ms05im-XuqhAk9EJnQBXxCoQ,1874
|
92
|
-
dycw_utilities-0.
|
93
|
-
dycw_utilities-0.
|
94
|
-
dycw_utilities-0.
|
95
|
-
dycw_utilities-0.
|
92
|
+
dycw_utilities-0.129.1.dist-info/METADATA,sha256=5y1_o16EDF5UAXj97fupC2XxHo_SqHwLxYDo22_4xe0,12803
|
93
|
+
dycw_utilities-0.129.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
94
|
+
dycw_utilities-0.129.1.dist-info/licenses/LICENSE,sha256=gppZp16M6nSVpBbUBrNL6JuYfvKwZiKgV7XoKKsHzqo,1066
|
95
|
+
dycw_utilities-0.129.1.dist-info/RECORD,,
|
utilities/__init__.py
CHANGED
utilities/asyncio.py
CHANGED
@@ -1,9 +1,6 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
-
import datetime as dt
|
4
|
-
from abc import ABC, abstractmethod
|
5
3
|
from asyncio import (
|
6
|
-
CancelledError,
|
7
4
|
Event,
|
8
5
|
Lock,
|
9
6
|
PriorityQueue,
|
@@ -19,7 +16,7 @@ from asyncio import (
|
|
19
16
|
sleep,
|
20
17
|
timeout,
|
21
18
|
)
|
22
|
-
from collections.abc import Callable,
|
19
|
+
from collections.abc import Callable, Iterable, Iterator
|
23
20
|
from contextlib import (
|
24
21
|
AbstractAsyncContextManager,
|
25
22
|
AsyncExitStack,
|
@@ -37,7 +34,6 @@ from typing import (
|
|
37
34
|
TYPE_CHECKING,
|
38
35
|
Any,
|
39
36
|
Generic,
|
40
|
-
NoReturn,
|
41
37
|
Self,
|
42
38
|
TextIO,
|
43
39
|
TypeVar,
|
@@ -50,27 +46,19 @@ from typing_extensions import deprecated
|
|
50
46
|
|
51
47
|
from utilities.dataclasses import replace_non_sentinel
|
52
48
|
from utilities.datetime import (
|
53
|
-
MINUTE,
|
54
49
|
SECOND,
|
55
50
|
datetime_duration_to_float,
|
56
|
-
datetime_duration_to_timedelta,
|
57
51
|
get_now,
|
58
52
|
round_datetime,
|
59
53
|
)
|
60
|
-
from utilities.errors import
|
61
|
-
from utilities.functions import ensure_int, ensure_not_none
|
54
|
+
from utilities.errors import repr_error
|
55
|
+
from utilities.functions import ensure_int, ensure_not_none
|
62
56
|
from utilities.random import SYSTEM_RANDOM
|
63
57
|
from utilities.sentinel import Sentinel, sentinel
|
64
|
-
from utilities.types import
|
65
|
-
Coroutine1,
|
66
|
-
DurationOrEveryDuration,
|
67
|
-
MaybeCallableEvent,
|
68
|
-
MaybeType,
|
69
|
-
THashable,
|
70
|
-
TSupportsRichComparison,
|
71
|
-
)
|
58
|
+
from utilities.types import MaybeCallableEvent, THashable, TSupportsRichComparison
|
72
59
|
|
73
60
|
if TYPE_CHECKING:
|
61
|
+
import datetime as dt
|
74
62
|
from asyncio import _CoroutineLike
|
75
63
|
from asyncio.subprocess import Process
|
76
64
|
from collections import deque
|
@@ -319,346 +307,6 @@ class EnhancedTaskGroup(TaskGroup):
|
|
319
307
|
##
|
320
308
|
|
321
309
|
|
322
|
-
@dataclass(kw_only=True, unsafe_hash=True)
|
323
|
-
class InfiniteLooper(ABC, Generic[THashable]):
|
324
|
-
"""An infinite loop which can throw exceptions by setting events."""
|
325
|
-
|
326
|
-
sleep_core: DurationOrEveryDuration = field(default=SECOND, repr=False)
|
327
|
-
sleep_restart: DurationOrEveryDuration = field(default=MINUTE, repr=False)
|
328
|
-
duration: Duration | None = field(default=None, repr=False)
|
329
|
-
logger: str | None = field(default=None, repr=False)
|
330
|
-
_await_upon_aenter: bool = field(default=True, init=False, repr=False)
|
331
|
-
_depth: int = field(default=0, init=False, repr=False)
|
332
|
-
_events: Mapping[THashable | None, Event] = field(
|
333
|
-
default_factory=dict, init=False, repr=False, hash=False
|
334
|
-
)
|
335
|
-
_stack: AsyncExitStack = field(
|
336
|
-
default_factory=AsyncExitStack, init=False, repr=False
|
337
|
-
)
|
338
|
-
_task: Task[None] | None = field(default=None, init=False, repr=False)
|
339
|
-
|
340
|
-
def __post_init__(self) -> None:
|
341
|
-
self._events = {
|
342
|
-
event: Event() for event, _ in self._yield_events_and_exceptions()
|
343
|
-
}
|
344
|
-
|
345
|
-
async def __aenter__(self) -> Self:
|
346
|
-
"""Context manager entry."""
|
347
|
-
if self._depth == 0:
|
348
|
-
self._task = create_task(self._run_looper())
|
349
|
-
if self._await_upon_aenter:
|
350
|
-
with suppress(CancelledError):
|
351
|
-
await self._task
|
352
|
-
_ = await self._stack.__aenter__()
|
353
|
-
self._depth += 1
|
354
|
-
return self
|
355
|
-
|
356
|
-
async def __aexit__(
|
357
|
-
self,
|
358
|
-
exc_type: type[BaseException] | None = None,
|
359
|
-
exc_value: BaseException | None = None,
|
360
|
-
traceback: TracebackType | None = None,
|
361
|
-
) -> None:
|
362
|
-
"""Context manager exit."""
|
363
|
-
_ = (exc_type, exc_value, traceback)
|
364
|
-
self._depth = max(self._depth - 1, 0)
|
365
|
-
if (self._depth == 0) and (self._task is not None):
|
366
|
-
with suppress(CancelledError):
|
367
|
-
await self._task
|
368
|
-
self._task = None
|
369
|
-
try:
|
370
|
-
await self._teardown()
|
371
|
-
except Exception as error: # noqa: BLE001
|
372
|
-
self._error_upon_teardown(error)
|
373
|
-
_ = await self._stack.__aexit__(exc_type, exc_value, traceback)
|
374
|
-
|
375
|
-
async def stop(self) -> None:
|
376
|
-
"""Stop the service."""
|
377
|
-
if self._task is None:
|
378
|
-
raise ImpossibleCaseError(case=[f"{self._task=}"]) # pragma: no cover
|
379
|
-
with suppress(CancelledError):
|
380
|
-
_ = self._task.cancel()
|
381
|
-
|
382
|
-
async def _run_looper(self) -> None:
|
383
|
-
"""Run the looper."""
|
384
|
-
match self.duration:
|
385
|
-
case None:
|
386
|
-
await self._run_looper_without_timeout()
|
387
|
-
case int() | float() | dt.timedelta() as duration:
|
388
|
-
try:
|
389
|
-
async with timeout_dur(duration=duration):
|
390
|
-
return await self._run_looper_without_timeout()
|
391
|
-
except TimeoutError:
|
392
|
-
await self.stop()
|
393
|
-
case _ as never:
|
394
|
-
assert_never(never)
|
395
|
-
return None
|
396
|
-
|
397
|
-
async def _run_looper_without_timeout(self) -> None:
|
398
|
-
"""Run the looper without a timeout."""
|
399
|
-
coroutines = list(self._yield_coroutines())
|
400
|
-
loopers = list(self._yield_loopers())
|
401
|
-
if (len(coroutines) == 0) and (len(loopers) == 0):
|
402
|
-
return await self._run_looper_by_itself()
|
403
|
-
return await self._run_looper_with_others(coroutines, loopers)
|
404
|
-
|
405
|
-
async def _run_looper_by_itself(self) -> None:
|
406
|
-
"""Run the looper by itself."""
|
407
|
-
whitelisted = tuple(self._yield_whitelisted_errors())
|
408
|
-
blacklisted = tuple(self._yield_blacklisted_errors())
|
409
|
-
while True:
|
410
|
-
try:
|
411
|
-
self._reset_events()
|
412
|
-
try:
|
413
|
-
await self._initialize()
|
414
|
-
except Exception as error: # noqa: BLE001
|
415
|
-
self._error_upon_initialize(error)
|
416
|
-
await self._run_sleep(self.sleep_restart)
|
417
|
-
else:
|
418
|
-
while True:
|
419
|
-
try:
|
420
|
-
event = next(
|
421
|
-
key
|
422
|
-
for (key, value) in self._events.items()
|
423
|
-
if value.is_set()
|
424
|
-
)
|
425
|
-
except StopIteration:
|
426
|
-
await self._core()
|
427
|
-
await self._run_sleep(self.sleep_core)
|
428
|
-
else:
|
429
|
-
self._raise_error(event)
|
430
|
-
except InfiniteLooperError:
|
431
|
-
raise
|
432
|
-
except BaseException as error1:
|
433
|
-
match error1:
|
434
|
-
case Exception():
|
435
|
-
if isinstance(error1, blacklisted):
|
436
|
-
raise
|
437
|
-
case BaseException():
|
438
|
-
if not isinstance(error1, whitelisted):
|
439
|
-
raise
|
440
|
-
case _ as never:
|
441
|
-
assert_never(never)
|
442
|
-
self._error_upon_core(error1)
|
443
|
-
try:
|
444
|
-
await self._teardown()
|
445
|
-
except BaseException as error2: # noqa: BLE001
|
446
|
-
self._error_upon_teardown(error2)
|
447
|
-
finally:
|
448
|
-
await self._run_sleep(self.sleep_restart)
|
449
|
-
|
450
|
-
async def _run_looper_with_others(
|
451
|
-
self,
|
452
|
-
coroutines: Iterable[Callable[[], Coroutine1[None]]],
|
453
|
-
loopers: Iterable[InfiniteLooper[Any]],
|
454
|
-
/,
|
455
|
-
) -> None:
|
456
|
-
"""Run multiple loopers."""
|
457
|
-
while True:
|
458
|
-
self._reset_events()
|
459
|
-
try:
|
460
|
-
async with TaskGroup() as tg, AsyncExitStack() as stack:
|
461
|
-
_ = tg.create_task(self._run_looper_by_itself())
|
462
|
-
_ = [tg.create_task(c()) for c in coroutines]
|
463
|
-
_ = [
|
464
|
-
tg.create_task(stack.enter_async_context(lo)) for lo in loopers
|
465
|
-
]
|
466
|
-
except ExceptionGroup as error:
|
467
|
-
self._error_group_upon_others(error)
|
468
|
-
await self._run_sleep(self.sleep_restart)
|
469
|
-
|
470
|
-
async def _initialize(self) -> None:
|
471
|
-
"""Initialize the loop."""
|
472
|
-
|
473
|
-
async def _core(self) -> None:
|
474
|
-
"""Run the core part of the loop."""
|
475
|
-
|
476
|
-
async def _teardown(self) -> None:
|
477
|
-
"""Tear down the loop."""
|
478
|
-
|
479
|
-
def _error_upon_initialize(self, error: Exception, /) -> None:
|
480
|
-
"""Handle any errors upon initializing the looper."""
|
481
|
-
if self.logger is not None:
|
482
|
-
getLogger(name=self.logger).error(
|
483
|
-
"%r encountered %r whilst initializing; sleeping %s...",
|
484
|
-
get_class_name(self),
|
485
|
-
repr_error(error),
|
486
|
-
self._sleep_restart_desc,
|
487
|
-
)
|
488
|
-
|
489
|
-
def _error_upon_core(self, error: BaseException, /) -> None:
|
490
|
-
"""Handle any errors upon running the core function."""
|
491
|
-
if self.logger is not None:
|
492
|
-
getLogger(name=self.logger).error(
|
493
|
-
"%r encountered %r; sleeping %s...",
|
494
|
-
get_class_name(self),
|
495
|
-
repr_error(error),
|
496
|
-
self._sleep_restart_desc,
|
497
|
-
)
|
498
|
-
|
499
|
-
def _error_upon_teardown(self, error: BaseException, /) -> None:
|
500
|
-
"""Handle any errors upon tearing down the looper."""
|
501
|
-
if self.logger is not None:
|
502
|
-
getLogger(name=self.logger).error(
|
503
|
-
"%r encountered %r whilst tearing down; sleeping %s...",
|
504
|
-
get_class_name(self),
|
505
|
-
repr_error(error),
|
506
|
-
self._sleep_restart_desc,
|
507
|
-
)
|
508
|
-
|
509
|
-
def _error_group_upon_others(self, group: ExceptionGroup, /) -> None:
|
510
|
-
"""Handle any errors upon running the core function."""
|
511
|
-
if self.logger is not None:
|
512
|
-
errors = group.exceptions
|
513
|
-
n = len(errors)
|
514
|
-
msgs = [f"{get_class_name(self)!r} encountered {n} error(s):"]
|
515
|
-
msgs.extend(
|
516
|
-
f"- Error #{i}/{n}: {repr_error(e)}"
|
517
|
-
for i, e in enumerate(errors, start=1)
|
518
|
-
)
|
519
|
-
msgs.append(f"Sleeping {self._sleep_restart_desc}...")
|
520
|
-
getLogger(name=self.logger).error("\n".join(msgs))
|
521
|
-
|
522
|
-
def _raise_error(self, event: THashable | None, /) -> NoReturn:
|
523
|
-
"""Raise the error corresponding to given event."""
|
524
|
-
mapping = dict(self._yield_events_and_exceptions())
|
525
|
-
error = mapping.get(event, InfiniteLooperError)
|
526
|
-
raise error
|
527
|
-
|
528
|
-
def _reset_events(self) -> None:
|
529
|
-
"""Reset the events."""
|
530
|
-
self._events = {
|
531
|
-
event: Event() for event, _ in self._yield_events_and_exceptions()
|
532
|
-
}
|
533
|
-
|
534
|
-
async def _run_sleep(self, sleep: DurationOrEveryDuration, /) -> None:
|
535
|
-
"""Sleep until the next part of the loop."""
|
536
|
-
match sleep:
|
537
|
-
case int() | float() | dt.timedelta() as duration:
|
538
|
-
await sleep_dur(duration=duration)
|
539
|
-
case "every", (int() | float() | dt.timedelta()) as duration:
|
540
|
-
await sleep_until_rounded(duration)
|
541
|
-
case _ as never:
|
542
|
-
assert_never(never)
|
543
|
-
|
544
|
-
@property
|
545
|
-
def _sleep_restart_desc(self) -> str:
|
546
|
-
"""Get a description of the sleep until restart."""
|
547
|
-
match self.sleep_restart:
|
548
|
-
case int() | float() | dt.timedelta() as duration:
|
549
|
-
timedelta = datetime_duration_to_timedelta(duration)
|
550
|
-
return f"for {timedelta}"
|
551
|
-
case "every", (int() | float() | dt.timedelta()) as duration:
|
552
|
-
timedelta = datetime_duration_to_timedelta(duration)
|
553
|
-
return f"until next {timedelta}"
|
554
|
-
case _ as never:
|
555
|
-
assert_never(never)
|
556
|
-
|
557
|
-
def _set_event(self, *, event: THashable | None = None) -> None:
|
558
|
-
"""Set the given event."""
|
559
|
-
try:
|
560
|
-
event_obj = self._events[event]
|
561
|
-
except KeyError:
|
562
|
-
raise _InfiniteLooperNoSuchEventError(looper=self, event=event) from None
|
563
|
-
event_obj.set()
|
564
|
-
|
565
|
-
def _yield_events_and_exceptions(
|
566
|
-
self,
|
567
|
-
) -> Iterator[tuple[THashable | None, MaybeType[Exception]]]:
|
568
|
-
"""Yield the events & exceptions."""
|
569
|
-
yield (None, _InfiniteLooperDefaultEventError(looper=self))
|
570
|
-
|
571
|
-
def _yield_coroutines(self) -> Iterator[Callable[[], Coroutine1[None]]]:
|
572
|
-
"""Yield any other coroutines which must also be run."""
|
573
|
-
yield from []
|
574
|
-
|
575
|
-
def _yield_loopers(self) -> Iterator[InfiniteLooper[Any]]:
|
576
|
-
"""Yield any other loopers which must also be run."""
|
577
|
-
yield from []
|
578
|
-
|
579
|
-
def _yield_blacklisted_errors(self) -> Iterator[type[Exception]]:
|
580
|
-
"""Yield any exceptions which the looper ought to catch terminate upon."""
|
581
|
-
yield from []
|
582
|
-
|
583
|
-
def _yield_whitelisted_errors(self) -> Iterator[type[BaseException]]:
|
584
|
-
"""Yield any exceptions which the looper ought to catch and allow running."""
|
585
|
-
yield from []
|
586
|
-
|
587
|
-
|
588
|
-
@dataclass(kw_only=True, slots=True)
|
589
|
-
class InfiniteLooperError(Exception):
|
590
|
-
looper: InfiniteLooper[Any]
|
591
|
-
|
592
|
-
|
593
|
-
@dataclass(kw_only=True, slots=True)
|
594
|
-
class _InfiniteLooperNoSuchEventError(InfiniteLooperError):
|
595
|
-
event: Hashable
|
596
|
-
|
597
|
-
@override
|
598
|
-
def __str__(self) -> str:
|
599
|
-
return f"{get_class_name(self.looper)!r} does not have an event {self.event!r}"
|
600
|
-
|
601
|
-
|
602
|
-
@dataclass(kw_only=True, slots=True)
|
603
|
-
class _InfiniteLooperDefaultEventError(InfiniteLooperError):
|
604
|
-
@override
|
605
|
-
def __str__(self) -> str:
|
606
|
-
return f"{get_class_name(self.looper)!r} default event error"
|
607
|
-
|
608
|
-
|
609
|
-
##
|
610
|
-
|
611
|
-
|
612
|
-
@dataclass(kw_only=True)
|
613
|
-
class InfiniteQueueLooper(InfiniteLooper[THashable], Generic[THashable, _T]):
|
614
|
-
"""An infinite loop which processes a queue."""
|
615
|
-
|
616
|
-
_await_upon_aenter: bool = field(default=False, init=False, repr=False)
|
617
|
-
_queue: EnhancedQueue[_T] = field(
|
618
|
-
default_factory=EnhancedQueue, init=False, repr=False
|
619
|
-
)
|
620
|
-
|
621
|
-
def __len__(self) -> int:
|
622
|
-
return self._queue.qsize()
|
623
|
-
|
624
|
-
@override
|
625
|
-
async def _core(self) -> None:
|
626
|
-
"""Run the core part of the loop."""
|
627
|
-
if self.empty():
|
628
|
-
return
|
629
|
-
await self._process_queue()
|
630
|
-
|
631
|
-
@abstractmethod
|
632
|
-
async def _process_queue(self) -> None:
|
633
|
-
"""Process the queue."""
|
634
|
-
|
635
|
-
def empty(self) -> bool:
|
636
|
-
"""Check if the queue is empty."""
|
637
|
-
return self._queue.empty()
|
638
|
-
|
639
|
-
def put_left_nowait(self, *items: _T) -> None:
|
640
|
-
"""Put items into the queue at the start without blocking."""
|
641
|
-
self._queue.put_left_nowait(*items) # pragma: no cover
|
642
|
-
|
643
|
-
def put_right_nowait(self, *items: _T) -> None:
|
644
|
-
"""Put items into the queue at the end without blocking."""
|
645
|
-
self._queue.put_right_nowait(*items) # pragma: no cover
|
646
|
-
|
647
|
-
def qsize(self) -> int:
|
648
|
-
"""Get the number of items in the queue."""
|
649
|
-
return self._queue.qsize()
|
650
|
-
|
651
|
-
async def run_until_empty(self, *, stop: bool = False) -> None:
|
652
|
-
"""Run until the queue is empty."""
|
653
|
-
while not self.empty():
|
654
|
-
await self._process_queue()
|
655
|
-
if stop:
|
656
|
-
await self.stop()
|
657
|
-
|
658
|
-
|
659
|
-
##
|
660
|
-
|
661
|
-
|
662
310
|
@dataclass(kw_only=True, slots=True)
|
663
311
|
class LooperError(Exception): ...
|
664
312
|
|
@@ -1415,9 +1063,6 @@ async def timeout_dur(
|
|
1415
1063
|
__all__ = [
|
1416
1064
|
"EnhancedQueue",
|
1417
1065
|
"EnhancedTaskGroup",
|
1418
|
-
"InfiniteLooper",
|
1419
|
-
"InfiniteLooperError",
|
1420
|
-
"InfiniteQueueLooper",
|
1421
1066
|
"Looper",
|
1422
1067
|
"LooperError",
|
1423
1068
|
"StreamCommandOutput",
|
utilities/fastapi.py
CHANGED
@@ -1,15 +1,18 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
+
from asyncio import Task, create_task
|
3
4
|
from dataclasses import InitVar, dataclass, field
|
4
|
-
from typing import TYPE_CHECKING, Any, Literal, override
|
5
|
+
from typing import TYPE_CHECKING, Any, Literal, Self, override
|
5
6
|
|
6
7
|
from fastapi import FastAPI
|
7
8
|
from uvicorn import Config, Server
|
8
9
|
|
9
|
-
from utilities.asyncio import
|
10
|
+
from utilities.asyncio import Looper
|
10
11
|
from utilities.datetime import SECOND, datetime_duration_to_float
|
11
12
|
|
12
13
|
if TYPE_CHECKING:
|
14
|
+
from types import TracebackType
|
15
|
+
|
13
16
|
from utilities.types import Duration
|
14
17
|
|
15
18
|
|
@@ -36,7 +39,7 @@ class _PingerReceiverApp(FastAPI):
|
|
36
39
|
|
37
40
|
|
38
41
|
@dataclass(kw_only=True)
|
39
|
-
class PingReceiver(
|
42
|
+
class PingReceiver(Looper[None]):
|
40
43
|
"""A ping receiver."""
|
41
44
|
|
42
45
|
host: InitVar[str] = _LOCALHOST
|
@@ -44,12 +47,31 @@ class PingReceiver(InfiniteLooper):
|
|
44
47
|
_app: _PingerReceiverApp = field(
|
45
48
|
default_factory=_PingerReceiverApp, init=False, repr=False
|
46
49
|
)
|
47
|
-
_await_upon_aenter: bool = field(default=False, init=False, repr=False)
|
48
50
|
_server: Server = field(init=False, repr=False)
|
51
|
+
_server_task: Task[None] | None = field(default=None, init=False, repr=False)
|
49
52
|
|
53
|
+
@override
|
50
54
|
def __post_init__(self, host: str, port: int, /) -> None:
|
55
|
+
super().__post_init__() # skipif-ci
|
51
56
|
self._server = Server(Config(self._app, host=host, port=port)) # skipif-ci
|
52
57
|
|
58
|
+
@override
|
59
|
+
async def __aenter__(self) -> Self:
|
60
|
+
_ = await super().__aenter__() # skipif-ci
|
61
|
+
async with self._lock: # skipif-ci
|
62
|
+
self._server_task = create_task(self._server.serve())
|
63
|
+
return self # skipif-ci
|
64
|
+
|
65
|
+
@override
|
66
|
+
async def __aexit__(
|
67
|
+
self,
|
68
|
+
exc_type: type[BaseException] | None = None,
|
69
|
+
exc_value: BaseException | None = None,
|
70
|
+
traceback: TracebackType | None = None,
|
71
|
+
) -> None:
|
72
|
+
await super().__aexit__(exc_type, exc_value, traceback) # skipif-ci
|
73
|
+
await self._server.shutdown() # skipif-ci
|
74
|
+
|
53
75
|
@classmethod
|
54
76
|
async def ping(
|
55
77
|
cls, port: int, /, *, host: str = _LOCALHOST, timeout: Duration = _TIMEOUT
|
@@ -66,13 +88,5 @@ class PingReceiver(InfiniteLooper):
|
|
66
88
|
return False
|
67
89
|
return response.text if response.status_code == 200 else False # skipif-ci
|
68
90
|
|
69
|
-
@override
|
70
|
-
async def _initialize(self) -> None:
|
71
|
-
await self._server.serve() # skipif-ci
|
72
|
-
|
73
|
-
@override
|
74
|
-
async def _teardown(self) -> None:
|
75
|
-
await self._server.shutdown() # skipif-ci
|
76
|
-
|
77
91
|
|
78
92
|
__all__ = ["PingReceiver"]
|
utilities/redis.py
CHANGED
@@ -23,9 +23,8 @@ from typing import (
|
|
23
23
|
)
|
24
24
|
|
25
25
|
from redis.asyncio import Redis
|
26
|
-
from redis.typing import EncodableT
|
27
26
|
|
28
|
-
from utilities.asyncio import EnhancedQueue,
|
27
|
+
from utilities.asyncio import EnhancedQueue, Looper, timeout_dur
|
29
28
|
from utilities.contextlib import suppress_super_object_attribute_error
|
30
29
|
from utilities.datetime import (
|
31
30
|
MILLISECOND,
|
@@ -34,7 +33,7 @@ from utilities.datetime import (
|
|
34
33
|
datetime_duration_to_timedelta,
|
35
34
|
)
|
36
35
|
from utilities.errors import ImpossibleCaseError
|
37
|
-
from utilities.functions import ensure_int,
|
36
|
+
from utilities.functions import ensure_int, identity
|
38
37
|
from utilities.iterables import always_iterable, one
|
39
38
|
from utilities.orjson import deserialize, serialize
|
40
39
|
|
@@ -51,10 +50,10 @@ if TYPE_CHECKING:
|
|
51
50
|
|
52
51
|
from redis.asyncio import ConnectionPool
|
53
52
|
from redis.asyncio.client import PubSub
|
54
|
-
from redis.typing import ResponseT
|
53
|
+
from redis.typing import EncodableT, ResponseT
|
55
54
|
|
56
55
|
from utilities.iterables import MaybeIterable
|
57
|
-
from utilities.types import Duration,
|
56
|
+
from utilities.types import Duration, TypeLike
|
58
57
|
|
59
58
|
|
60
59
|
_K = TypeVar("_K")
|
@@ -620,42 +619,6 @@ class PublishError(Exception):
|
|
620
619
|
##
|
621
620
|
|
622
621
|
|
623
|
-
@dataclass(kw_only=True)
|
624
|
-
class Publisher(InfiniteQueueLooper[None, tuple[str, EncodableT]]):
|
625
|
-
"""Publish a set of messages to Redis."""
|
626
|
-
|
627
|
-
redis: Redis
|
628
|
-
serializer: Callable[[Any], EncodableT] | None = None
|
629
|
-
timeout: Duration = _PUBLISH_TIMEOUT
|
630
|
-
|
631
|
-
@override
|
632
|
-
async def _process_queue(self) -> None:
|
633
|
-
for item in self._queue.get_all_nowait(): # skipif-ci-and-not-linux
|
634
|
-
channel, data = item
|
635
|
-
_ = await publish(
|
636
|
-
self.redis,
|
637
|
-
channel,
|
638
|
-
data,
|
639
|
-
serializer=self.serializer,
|
640
|
-
timeout=self.timeout,
|
641
|
-
)
|
642
|
-
|
643
|
-
@override
|
644
|
-
def _yield_events_and_exceptions(
|
645
|
-
self,
|
646
|
-
) -> Iterator[tuple[None, MaybeType[Exception]]]:
|
647
|
-
yield (None, PublisherError) # skipif-ci-and-not-linux
|
648
|
-
|
649
|
-
|
650
|
-
@dataclass(kw_only=True)
|
651
|
-
class PublisherError(Exception):
|
652
|
-
publisher: Publisher
|
653
|
-
|
654
|
-
@override
|
655
|
-
def __str__(self) -> str:
|
656
|
-
return f"Error running {get_class_name(self.publisher)!r}" # skipif-ci-and-not-linux
|
657
|
-
|
658
|
-
|
659
622
|
@dataclass(kw_only=True)
|
660
623
|
class PublishService(Looper[tuple[str, _T]]):
|
661
624
|
"""Service to publish items to Redis."""
|
@@ -1095,8 +1058,6 @@ def _deserialize(
|
|
1095
1058
|
__all__ = [
|
1096
1059
|
"PublishService",
|
1097
1060
|
"PublishServiceMixin",
|
1098
|
-
"Publisher",
|
1099
|
-
"PublisherError",
|
1100
1061
|
"RedisHashMapKey",
|
1101
1062
|
"RedisKey",
|
1102
1063
|
"SubscribeService",
|
utilities/slack_sdk.py
CHANGED
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Self, override
|
|
7
7
|
|
8
8
|
from slack_sdk.webhook.async_client import AsyncWebhookClient
|
9
9
|
|
10
|
-
from utilities.asyncio import
|
10
|
+
from utilities.asyncio import Looper, timeout_dur
|
11
11
|
from utilities.datetime import MINUTE, SECOND, datetime_duration_to_float
|
12
12
|
from utilities.functools import cache
|
13
13
|
from utilities.math import safe_round
|
@@ -27,53 +27,10 @@ _TIMEOUT: Duration = MINUTE
|
|
27
27
|
##
|
28
28
|
|
29
29
|
|
30
|
-
_SLEEP: Duration = SECOND
|
31
|
-
|
32
|
-
|
33
30
|
async def _send_adapter(url: str, text: str, /) -> None:
|
34
31
|
await send_to_slack(url, text) # pragma: no cover
|
35
32
|
|
36
33
|
|
37
|
-
@dataclass(init=False, unsafe_hash=True)
|
38
|
-
class SlackHandler(Handler, InfiniteQueueLooper[None, str]):
|
39
|
-
"""Handler for sending messages to Slack."""
|
40
|
-
|
41
|
-
@override
|
42
|
-
def __init__(
|
43
|
-
self,
|
44
|
-
url: str,
|
45
|
-
/,
|
46
|
-
*,
|
47
|
-
level: int = NOTSET,
|
48
|
-
sleep_core: Duration = _SLEEP,
|
49
|
-
sleep_restart: Duration = _SLEEP,
|
50
|
-
sender: Callable[[str, str], Coroutine1[None]] = _send_adapter,
|
51
|
-
timeout: Duration = _TIMEOUT,
|
52
|
-
) -> None:
|
53
|
-
InfiniteQueueLooper.__init__(self) # InfiniteQueueLooper first
|
54
|
-
InfiniteQueueLooper.__post_init__(self)
|
55
|
-
Handler.__init__(self, level=level) # Handler next
|
56
|
-
self.url = url
|
57
|
-
self.sender = sender
|
58
|
-
self.timeout = timeout
|
59
|
-
self.sleep_core = sleep_core
|
60
|
-
self.sleep_restart = sleep_restart
|
61
|
-
|
62
|
-
@override
|
63
|
-
def emit(self, record: LogRecord) -> None:
|
64
|
-
try:
|
65
|
-
self.put_right_nowait(self.format(record))
|
66
|
-
except Exception: # noqa: BLE001 # pragma: no cover
|
67
|
-
self.handleError(record)
|
68
|
-
|
69
|
-
@override
|
70
|
-
async def _process_queue(self) -> None:
|
71
|
-
messages = self._queue.get_all_nowait()
|
72
|
-
text = "\n".join(messages)
|
73
|
-
async with timeout_dur(duration=self.timeout):
|
74
|
-
await self.sender(self.url, text)
|
75
|
-
|
76
|
-
|
77
34
|
@dataclass(init=False, unsafe_hash=True)
|
78
35
|
class SlackHandlerService(Handler, Looper[str]):
|
79
36
|
"""Service to send messages to Slack."""
|
@@ -187,4 +144,4 @@ def _get_client(url: str, /, *, timeout: Duration = _TIMEOUT) -> AsyncWebhookCli
|
|
187
144
|
return AsyncWebhookClient(url, timeout=timeout_use)
|
188
145
|
|
189
146
|
|
190
|
-
__all__ = ["SendToSlackError", "
|
147
|
+
__all__ = ["SendToSlackError", "SlackHandlerService", "send_to_slack"]
|
utilities/sqlalchemy.py
CHANGED
@@ -57,7 +57,7 @@ from sqlalchemy.orm import (
|
|
57
57
|
from sqlalchemy.orm.exc import UnmappedClassError
|
58
58
|
from sqlalchemy.pool import NullPool, Pool
|
59
59
|
|
60
|
-
from utilities.asyncio import
|
60
|
+
from utilities.asyncio import Looper, timeout_dur
|
61
61
|
from utilities.contextlib import suppress_super_object_attribute_error
|
62
62
|
from utilities.datetime import SECOND
|
63
63
|
from utilities.functions import (
|
@@ -82,13 +82,7 @@ from utilities.iterables import (
|
|
82
82
|
)
|
83
83
|
from utilities.reprlib import get_repr
|
84
84
|
from utilities.text import snake_case
|
85
|
-
from utilities.types import
|
86
|
-
Duration,
|
87
|
-
MaybeIterable,
|
88
|
-
MaybeType,
|
89
|
-
StrMapping,
|
90
|
-
TupleOrStrMapping,
|
91
|
-
)
|
85
|
+
from utilities.types import Duration, MaybeIterable, StrMapping, TupleOrStrMapping
|
92
86
|
|
93
87
|
_T = TypeVar("_T")
|
94
88
|
type _EngineOrConnectionOrAsync = Engine | Connection | AsyncEngine | AsyncConnection
|
@@ -610,52 +604,6 @@ class TablenameMixin:
|
|
610
604
|
##
|
611
605
|
|
612
606
|
|
613
|
-
@dataclass(kw_only=True)
|
614
|
-
class Upserter(InfiniteQueueLooper[None, _InsertItem]):
|
615
|
-
"""Upsert a set of items to a database."""
|
616
|
-
|
617
|
-
engine: AsyncEngine
|
618
|
-
snake: bool = False
|
619
|
-
selected_or_all: _SelectedOrAll = "selected"
|
620
|
-
chunk_size_frac: float = CHUNK_SIZE_FRAC
|
621
|
-
assume_tables_exist: bool = False
|
622
|
-
timeout_create: Duration | None = None
|
623
|
-
error_create: type[Exception] = TimeoutError
|
624
|
-
timeout_insert: Duration | None = None
|
625
|
-
error_insert: type[Exception] = TimeoutError
|
626
|
-
|
627
|
-
@override
|
628
|
-
async def _process_queue(self) -> None:
|
629
|
-
items = self._queue.get_all_nowait()
|
630
|
-
await upsert_items(
|
631
|
-
self.engine,
|
632
|
-
*items,
|
633
|
-
snake=self.snake,
|
634
|
-
selected_or_all=self.selected_or_all,
|
635
|
-
chunk_size_frac=self.chunk_size_frac,
|
636
|
-
assume_tables_exist=self.assume_tables_exist,
|
637
|
-
timeout_create=self.timeout_create,
|
638
|
-
error_create=self.error_create,
|
639
|
-
timeout_insert=self.timeout_insert,
|
640
|
-
error_insert=self.error_insert,
|
641
|
-
)
|
642
|
-
|
643
|
-
@override
|
644
|
-
def _yield_events_and_exceptions(
|
645
|
-
self,
|
646
|
-
) -> Iterator[tuple[None, MaybeType[Exception]]]:
|
647
|
-
yield (None, UpserterError)
|
648
|
-
|
649
|
-
|
650
|
-
@dataclass(kw_only=True)
|
651
|
-
class UpserterError(Exception):
|
652
|
-
upserter: Upserter
|
653
|
-
|
654
|
-
@override
|
655
|
-
def __str__(self) -> str:
|
656
|
-
return f"Error running {get_class_name(self.upserter)!r}"
|
657
|
-
|
658
|
-
|
659
607
|
@dataclass(kw_only=True)
|
660
608
|
class UpsertService(Looper[_InsertItem]):
|
661
609
|
"""Service to upsert items to a database."""
|
@@ -1202,8 +1150,6 @@ __all__ = [
|
|
1202
1150
|
"UpsertItemsError",
|
1203
1151
|
"UpsertService",
|
1204
1152
|
"UpsertServiceMixin",
|
1205
|
-
"Upserter",
|
1206
|
-
"UpserterError",
|
1207
1153
|
"check_engine",
|
1208
1154
|
"columnwise_max",
|
1209
1155
|
"columnwise_min",
|
File without changes
|
File without changes
|