dycw-utilities 0.126.12__py3-none-any.whl → 0.129.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dycw_utilities-0.126.12.dist-info → dycw_utilities-0.129.13.dist-info}/METADATA +16 -10
- {dycw_utilities-0.126.12.dist-info → dycw_utilities-0.129.13.dist-info}/RECORD +24 -23
- utilities/__init__.py +1 -1
- utilities/aiolimiter.py +25 -0
- utilities/asyncio.py +62 -426
- utilities/datetime.py +0 -8
- utilities/fastapi.py +26 -12
- utilities/git.py +5 -58
- utilities/hypothesis.py +1 -11
- utilities/logging.py +69 -56
- utilities/pathlib.py +83 -13
- utilities/pyinstrument.py +6 -4
- utilities/pytest_regressions.py +2 -2
- utilities/python_dotenv.py +10 -6
- utilities/redis.py +5 -58
- utilities/scipy.py +1 -1
- utilities/slack_sdk.py +2 -54
- utilities/sqlalchemy.py +2 -65
- utilities/traceback.py +278 -12
- utilities/types.py +2 -2
- utilities/version.py +0 -8
- utilities/whenever.py +64 -1
- {dycw_utilities-0.126.12.dist-info → dycw_utilities-0.129.13.dist-info}/WHEEL +0 -0
- {dycw_utilities-0.126.12.dist-info → dycw_utilities-0.129.13.dist-info}/licenses/LICENSE +0 -0
utilities/redis.py
CHANGED
@@ -23,15 +23,8 @@ from typing import (
|
|
23
23
|
)
|
24
24
|
|
25
25
|
from redis.asyncio import Redis
|
26
|
-
|
27
|
-
|
28
|
-
from utilities.asyncio import (
|
29
|
-
EnhancedQueue,
|
30
|
-
InfiniteQueueLooper,
|
31
|
-
Looper,
|
32
|
-
LooperTimeoutError,
|
33
|
-
timeout_dur,
|
34
|
-
)
|
26
|
+
|
27
|
+
from utilities.asyncio import EnhancedQueue, Looper, timeout_dur
|
35
28
|
from utilities.contextlib import suppress_super_object_attribute_error
|
36
29
|
from utilities.datetime import (
|
37
30
|
MILLISECOND,
|
@@ -40,7 +33,7 @@ from utilities.datetime import (
|
|
40
33
|
datetime_duration_to_timedelta,
|
41
34
|
)
|
42
35
|
from utilities.errors import ImpossibleCaseError
|
43
|
-
from utilities.functions import ensure_int,
|
36
|
+
from utilities.functions import ensure_int, identity
|
44
37
|
from utilities.iterables import always_iterable, one
|
45
38
|
from utilities.orjson import deserialize, serialize
|
46
39
|
|
@@ -57,10 +50,10 @@ if TYPE_CHECKING:
|
|
57
50
|
|
58
51
|
from redis.asyncio import ConnectionPool
|
59
52
|
from redis.asyncio.client import PubSub
|
60
|
-
from redis.typing import ResponseT
|
53
|
+
from redis.typing import EncodableT, ResponseT
|
61
54
|
|
62
55
|
from utilities.iterables import MaybeIterable
|
63
|
-
from utilities.types import Duration,
|
56
|
+
from utilities.types import Duration, TypeLike
|
64
57
|
|
65
58
|
|
66
59
|
_K = TypeVar("_K")
|
@@ -626,42 +619,6 @@ class PublishError(Exception):
|
|
626
619
|
##
|
627
620
|
|
628
621
|
|
629
|
-
@dataclass(kw_only=True)
|
630
|
-
class Publisher(InfiniteQueueLooper[None, tuple[str, EncodableT]]):
|
631
|
-
"""Publish a set of messages to Redis."""
|
632
|
-
|
633
|
-
redis: Redis
|
634
|
-
serializer: Callable[[Any], EncodableT] | None = None
|
635
|
-
timeout: Duration = _PUBLISH_TIMEOUT
|
636
|
-
|
637
|
-
@override
|
638
|
-
async def _process_queue(self) -> None:
|
639
|
-
for item in self._queue.get_all_nowait(): # skipif-ci-and-not-linux
|
640
|
-
channel, data = item
|
641
|
-
_ = await publish(
|
642
|
-
self.redis,
|
643
|
-
channel,
|
644
|
-
data,
|
645
|
-
serializer=self.serializer,
|
646
|
-
timeout=self.timeout,
|
647
|
-
)
|
648
|
-
|
649
|
-
@override
|
650
|
-
def _yield_events_and_exceptions(
|
651
|
-
self,
|
652
|
-
) -> Iterator[tuple[None, MaybeType[Exception]]]:
|
653
|
-
yield (None, PublisherError) # skipif-ci-and-not-linux
|
654
|
-
|
655
|
-
|
656
|
-
@dataclass(kw_only=True)
|
657
|
-
class PublisherError(Exception):
|
658
|
-
publisher: Publisher
|
659
|
-
|
660
|
-
@override
|
661
|
-
def __str__(self) -> str:
|
662
|
-
return f"Error running {get_class_name(self.publisher)!r}" # skipif-ci-and-not-linux
|
663
|
-
|
664
|
-
|
665
622
|
@dataclass(kw_only=True)
|
666
623
|
class PublishService(Looper[tuple[str, _T]]):
|
667
624
|
"""Service to publish items to Redis."""
|
@@ -702,9 +659,6 @@ class PublishServiceMixin(Generic[_T]):
|
|
702
659
|
publish_service_empty_upon_exit: bool = field(default=False, repr=False)
|
703
660
|
publish_service_logger: str | None = field(default=None, repr=False)
|
704
661
|
publish_service_timeout: Duration | None = field(default=None, repr=False)
|
705
|
-
publish_service_timeout_error: type[Exception] = field(
|
706
|
-
default=LooperTimeoutError, repr=False
|
707
|
-
)
|
708
662
|
publish_service_debug: bool = field(default=False, repr=False)
|
709
663
|
_is_pending_restart: Event = field(default_factory=Event, init=False, repr=False)
|
710
664
|
# base - publish service
|
@@ -724,7 +678,6 @@ class PublishServiceMixin(Generic[_T]):
|
|
724
678
|
empty_upon_exit=self.publish_service_empty_upon_exit,
|
725
679
|
logger=self.publish_service_logger,
|
726
680
|
timeout=self.publish_service_timeout,
|
727
|
-
timeout_error=self.publish_service_timeout_error,
|
728
681
|
_debug=self.publish_service_debug,
|
729
682
|
# publish service
|
730
683
|
redis=self.publish_service_redis,
|
@@ -986,9 +939,6 @@ class SubscribeServiceMixin(Generic[_T]):
|
|
986
939
|
subscribe_service_empty_upon_exit: bool = field(default=False, repr=False)
|
987
940
|
subscribe_service_logger: str | None = field(default=None, repr=False)
|
988
941
|
subscribe_service_timeout: Duration | None = field(default=None, repr=False)
|
989
|
-
subscribe_service_timeout_error: type[Exception] = field(
|
990
|
-
default=LooperTimeoutError, repr=False
|
991
|
-
)
|
992
942
|
subscribe_service_debug: bool = field(default=False, repr=False)
|
993
943
|
# base - looper
|
994
944
|
subscribe_service_redis: Redis
|
@@ -1009,7 +959,6 @@ class SubscribeServiceMixin(Generic[_T]):
|
|
1009
959
|
empty_upon_exit=self.subscribe_service_empty_upon_exit,
|
1010
960
|
logger=self.subscribe_service_logger,
|
1011
961
|
timeout=self.subscribe_service_timeout,
|
1012
|
-
timeout_error=self.subscribe_service_timeout_error,
|
1013
962
|
_debug=self.subscribe_service_debug,
|
1014
963
|
# subscribe service
|
1015
964
|
redis=self.subscribe_service_redis,
|
@@ -1109,8 +1058,6 @@ def _deserialize(
|
|
1109
1058
|
__all__ = [
|
1110
1059
|
"PublishService",
|
1111
1060
|
"PublishServiceMixin",
|
1112
|
-
"Publisher",
|
1113
|
-
"PublisherError",
|
1114
1061
|
"RedisHashMapKey",
|
1115
1062
|
"RedisKey",
|
1116
1063
|
"SubscribeService",
|
utilities/scipy.py
CHANGED
@@ -19,7 +19,7 @@ def _ppf_1d(array: NDArrayF, cutoff: float, /) -> NDArrayF:
|
|
19
19
|
out = full_like(array, nan, dtype=float)
|
20
20
|
out[j] = _ppf_1d(array[j], cutoff)
|
21
21
|
return out
|
22
|
-
low, high = min(
|
22
|
+
low, high = array.min(), array.max()
|
23
23
|
if is_zero(span := high - low):
|
24
24
|
return zeros_like(array, dtype=float)
|
25
25
|
centred = (array - low) / span
|
utilities/slack_sdk.py
CHANGED
@@ -7,12 +7,7 @@ from typing import TYPE_CHECKING, Any, Self, override
|
|
7
7
|
|
8
8
|
from slack_sdk.webhook.async_client import AsyncWebhookClient
|
9
9
|
|
10
|
-
from utilities.asyncio import
|
11
|
-
InfiniteQueueLooper,
|
12
|
-
Looper,
|
13
|
-
LooperTimeoutError,
|
14
|
-
timeout_dur,
|
15
|
-
)
|
10
|
+
from utilities.asyncio import Looper, timeout_dur
|
16
11
|
from utilities.datetime import MINUTE, SECOND, datetime_duration_to_float
|
17
12
|
from utilities.functools import cache
|
18
13
|
from utilities.math import safe_round
|
@@ -32,53 +27,10 @@ _TIMEOUT: Duration = MINUTE
|
|
32
27
|
##
|
33
28
|
|
34
29
|
|
35
|
-
_SLEEP: Duration = SECOND
|
36
|
-
|
37
|
-
|
38
30
|
async def _send_adapter(url: str, text: str, /) -> None:
|
39
31
|
await send_to_slack(url, text) # pragma: no cover
|
40
32
|
|
41
33
|
|
42
|
-
@dataclass(init=False, unsafe_hash=True)
|
43
|
-
class SlackHandler(Handler, InfiniteQueueLooper[None, str]):
|
44
|
-
"""Handler for sending messages to Slack."""
|
45
|
-
|
46
|
-
@override
|
47
|
-
def __init__(
|
48
|
-
self,
|
49
|
-
url: str,
|
50
|
-
/,
|
51
|
-
*,
|
52
|
-
level: int = NOTSET,
|
53
|
-
sleep_core: Duration = _SLEEP,
|
54
|
-
sleep_restart: Duration = _SLEEP,
|
55
|
-
sender: Callable[[str, str], Coroutine1[None]] = _send_adapter,
|
56
|
-
timeout: Duration = _TIMEOUT,
|
57
|
-
) -> None:
|
58
|
-
InfiniteQueueLooper.__init__(self) # InfiniteQueueLooper first
|
59
|
-
InfiniteQueueLooper.__post_init__(self)
|
60
|
-
Handler.__init__(self, level=level) # Handler next
|
61
|
-
self.url = url
|
62
|
-
self.sender = sender
|
63
|
-
self.timeout = timeout
|
64
|
-
self.sleep_core = sleep_core
|
65
|
-
self.sleep_restart = sleep_restart
|
66
|
-
|
67
|
-
@override
|
68
|
-
def emit(self, record: LogRecord) -> None:
|
69
|
-
try:
|
70
|
-
self.put_right_nowait(self.format(record))
|
71
|
-
except Exception: # noqa: BLE001 # pragma: no cover
|
72
|
-
self.handleError(record)
|
73
|
-
|
74
|
-
@override
|
75
|
-
async def _process_queue(self) -> None:
|
76
|
-
messages = self._queue.get_all_nowait()
|
77
|
-
text = "\n".join(messages)
|
78
|
-
async with timeout_dur(duration=self.timeout):
|
79
|
-
await self.sender(self.url, text)
|
80
|
-
|
81
|
-
|
82
34
|
@dataclass(init=False, unsafe_hash=True)
|
83
35
|
class SlackHandlerService(Handler, Looper[str]):
|
84
36
|
"""Service to send messages to Slack."""
|
@@ -94,7 +46,6 @@ class SlackHandlerService(Handler, Looper[str]):
|
|
94
46
|
backoff: Duration = SECOND,
|
95
47
|
logger: str | None = None,
|
96
48
|
timeout: Duration | None = None,
|
97
|
-
timeout_error: type[Exception] = LooperTimeoutError,
|
98
49
|
_debug: bool = False,
|
99
50
|
level: int = NOTSET,
|
100
51
|
sender: Callable[[str, str], Coroutine1[None]] = _send_adapter,
|
@@ -108,7 +59,6 @@ class SlackHandlerService(Handler, Looper[str]):
|
|
108
59
|
backoff=backoff,
|
109
60
|
logger=logger,
|
110
61
|
timeout=timeout,
|
111
|
-
timeout_error=timeout_error,
|
112
62
|
_debug=_debug,
|
113
63
|
)
|
114
64
|
Looper.__post_init__(self)
|
@@ -144,7 +94,6 @@ class SlackHandlerService(Handler, Looper[str]):
|
|
144
94
|
backoff: Duration | Sentinel = sentinel,
|
145
95
|
logger: str | None | Sentinel = sentinel,
|
146
96
|
timeout: Duration | None | Sentinel = sentinel,
|
147
|
-
timeout_error: type[Exception] | Sentinel = sentinel,
|
148
97
|
_debug: bool | Sentinel = sentinel,
|
149
98
|
**kwargs: Any,
|
150
99
|
) -> Self:
|
@@ -157,7 +106,6 @@ class SlackHandlerService(Handler, Looper[str]):
|
|
157
106
|
backoff=backoff,
|
158
107
|
logger=logger,
|
159
108
|
timeout=timeout,
|
160
|
-
timeout_error=timeout_error,
|
161
109
|
_debug=_debug,
|
162
110
|
**kwargs,
|
163
111
|
)
|
@@ -196,4 +144,4 @@ def _get_client(url: str, /, *, timeout: Duration = _TIMEOUT) -> AsyncWebhookCli
|
|
196
144
|
return AsyncWebhookClient(url, timeout=timeout_use)
|
197
145
|
|
198
146
|
|
199
|
-
__all__ = ["SendToSlackError", "
|
147
|
+
__all__ = ["SendToSlackError", "SlackHandlerService", "send_to_slack"]
|
utilities/sqlalchemy.py
CHANGED
@@ -57,12 +57,7 @@ from sqlalchemy.orm import (
|
|
57
57
|
from sqlalchemy.orm.exc import UnmappedClassError
|
58
58
|
from sqlalchemy.pool import NullPool, Pool
|
59
59
|
|
60
|
-
from utilities.asyncio import
|
61
|
-
InfiniteQueueLooper,
|
62
|
-
Looper,
|
63
|
-
LooperTimeoutError,
|
64
|
-
timeout_dur,
|
65
|
-
)
|
60
|
+
from utilities.asyncio import Looper, timeout_dur
|
66
61
|
from utilities.contextlib import suppress_super_object_attribute_error
|
67
62
|
from utilities.datetime import SECOND
|
68
63
|
from utilities.functions import (
|
@@ -87,13 +82,7 @@ from utilities.iterables import (
|
|
87
82
|
)
|
88
83
|
from utilities.reprlib import get_repr
|
89
84
|
from utilities.text import snake_case
|
90
|
-
from utilities.types import
|
91
|
-
Duration,
|
92
|
-
MaybeIterable,
|
93
|
-
MaybeType,
|
94
|
-
StrMapping,
|
95
|
-
TupleOrStrMapping,
|
96
|
-
)
|
85
|
+
from utilities.types import Duration, MaybeIterable, StrMapping, TupleOrStrMapping
|
97
86
|
|
98
87
|
_T = TypeVar("_T")
|
99
88
|
type _EngineOrConnectionOrAsync = Engine | Connection | AsyncEngine | AsyncConnection
|
@@ -615,52 +604,6 @@ class TablenameMixin:
|
|
615
604
|
##
|
616
605
|
|
617
606
|
|
618
|
-
@dataclass(kw_only=True)
|
619
|
-
class Upserter(InfiniteQueueLooper[None, _InsertItem]):
|
620
|
-
"""Upsert a set of items to a database."""
|
621
|
-
|
622
|
-
engine: AsyncEngine
|
623
|
-
snake: bool = False
|
624
|
-
selected_or_all: _SelectedOrAll = "selected"
|
625
|
-
chunk_size_frac: float = CHUNK_SIZE_FRAC
|
626
|
-
assume_tables_exist: bool = False
|
627
|
-
timeout_create: Duration | None = None
|
628
|
-
error_create: type[Exception] = TimeoutError
|
629
|
-
timeout_insert: Duration | None = None
|
630
|
-
error_insert: type[Exception] = TimeoutError
|
631
|
-
|
632
|
-
@override
|
633
|
-
async def _process_queue(self) -> None:
|
634
|
-
items = self._queue.get_all_nowait()
|
635
|
-
await upsert_items(
|
636
|
-
self.engine,
|
637
|
-
*items,
|
638
|
-
snake=self.snake,
|
639
|
-
selected_or_all=self.selected_or_all,
|
640
|
-
chunk_size_frac=self.chunk_size_frac,
|
641
|
-
assume_tables_exist=self.assume_tables_exist,
|
642
|
-
timeout_create=self.timeout_create,
|
643
|
-
error_create=self.error_create,
|
644
|
-
timeout_insert=self.timeout_insert,
|
645
|
-
error_insert=self.error_insert,
|
646
|
-
)
|
647
|
-
|
648
|
-
@override
|
649
|
-
def _yield_events_and_exceptions(
|
650
|
-
self,
|
651
|
-
) -> Iterator[tuple[None, MaybeType[Exception]]]:
|
652
|
-
yield (None, UpserterError)
|
653
|
-
|
654
|
-
|
655
|
-
@dataclass(kw_only=True)
|
656
|
-
class UpserterError(Exception):
|
657
|
-
upserter: Upserter
|
658
|
-
|
659
|
-
@override
|
660
|
-
def __str__(self) -> str:
|
661
|
-
return f"Error running {get_class_name(self.upserter)!r}"
|
662
|
-
|
663
|
-
|
664
607
|
@dataclass(kw_only=True)
|
665
608
|
class UpsertService(Looper[_InsertItem]):
|
666
609
|
"""Service to upsert items to a database."""
|
@@ -707,9 +650,6 @@ class UpsertServiceMixin:
|
|
707
650
|
upsert_service_empty_upon_exit: bool = field(default=False, repr=False)
|
708
651
|
upsert_service_logger: str | None = field(default=None, repr=False)
|
709
652
|
upsert_service_timeout: Duration | None = field(default=None, repr=False)
|
710
|
-
upsert_service_timeout_error: type[Exception] = field(
|
711
|
-
default=LooperTimeoutError, repr=False
|
712
|
-
)
|
713
653
|
upsert_service_debug: bool = field(default=False, repr=False)
|
714
654
|
# base - upsert service
|
715
655
|
upsert_service_database: AsyncEngine
|
@@ -734,7 +674,6 @@ class UpsertServiceMixin:
|
|
734
674
|
empty_upon_exit=self.upsert_service_empty_upon_exit,
|
735
675
|
logger=self.upsert_service_logger,
|
736
676
|
timeout=self.upsert_service_timeout,
|
737
|
-
timeout_error=self.upsert_service_timeout_error,
|
738
677
|
_debug=self.upsert_service_debug,
|
739
678
|
# upsert service
|
740
679
|
engine=self.upsert_service_database,
|
@@ -1211,8 +1150,6 @@ __all__ = [
|
|
1211
1150
|
"UpsertItemsError",
|
1212
1151
|
"UpsertService",
|
1213
1152
|
"UpsertServiceMixin",
|
1214
|
-
"Upserter",
|
1215
|
-
"UpserterError",
|
1216
1153
|
"check_engine",
|
1217
1154
|
"columnwise_max",
|
1218
1155
|
"columnwise_min",
|