dycw-utilities 0.135.0__py3-none-any.whl → 0.178.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dycw-utilities might be problematic. Click here for more details.
- dycw_utilities-0.178.1.dist-info/METADATA +34 -0
- dycw_utilities-0.178.1.dist-info/RECORD +105 -0
- dycw_utilities-0.178.1.dist-info/WHEEL +4 -0
- dycw_utilities-0.178.1.dist-info/entry_points.txt +4 -0
- utilities/__init__.py +1 -1
- utilities/altair.py +13 -10
- utilities/asyncio.py +312 -787
- utilities/atomicwrites.py +18 -6
- utilities/atools.py +64 -4
- utilities/cachetools.py +9 -6
- utilities/click.py +195 -77
- utilities/concurrent.py +1 -1
- utilities/contextlib.py +216 -17
- utilities/contextvars.py +20 -1
- utilities/cryptography.py +3 -3
- utilities/dataclasses.py +15 -28
- utilities/docker.py +387 -0
- utilities/enum.py +2 -2
- utilities/errors.py +17 -3
- utilities/fastapi.py +28 -59
- utilities/fpdf2.py +2 -2
- utilities/functions.py +24 -269
- utilities/git.py +9 -30
- utilities/grp.py +28 -0
- utilities/gzip.py +31 -0
- utilities/http.py +3 -2
- utilities/hypothesis.py +513 -159
- utilities/importlib.py +17 -1
- utilities/inflect.py +12 -4
- utilities/iterables.py +33 -58
- utilities/jinja2.py +148 -0
- utilities/json.py +70 -0
- utilities/libcst.py +38 -17
- utilities/lightweight_charts.py +4 -7
- utilities/logging.py +136 -93
- utilities/math.py +8 -4
- utilities/more_itertools.py +43 -45
- utilities/operator.py +27 -27
- utilities/orjson.py +189 -36
- utilities/os.py +61 -4
- utilities/packaging.py +115 -0
- utilities/parse.py +8 -5
- utilities/pathlib.py +269 -40
- utilities/permissions.py +298 -0
- utilities/platform.py +7 -6
- utilities/polars.py +1205 -413
- utilities/polars_ols.py +1 -1
- utilities/postgres.py +408 -0
- utilities/pottery.py +43 -19
- utilities/pqdm.py +3 -3
- utilities/psutil.py +5 -57
- utilities/pwd.py +28 -0
- utilities/pydantic.py +4 -52
- utilities/pydantic_settings.py +240 -0
- utilities/pydantic_settings_sops.py +76 -0
- utilities/pyinstrument.py +7 -7
- utilities/pytest.py +104 -143
- utilities/pytest_plugins/__init__.py +1 -0
- utilities/pytest_plugins/pytest_randomly.py +23 -0
- utilities/pytest_plugins/pytest_regressions.py +56 -0
- utilities/pytest_regressions.py +26 -46
- utilities/random.py +11 -6
- utilities/re.py +1 -1
- utilities/redis.py +220 -343
- utilities/sentinel.py +10 -0
- utilities/shelve.py +4 -1
- utilities/shutil.py +25 -0
- utilities/slack_sdk.py +35 -104
- utilities/sqlalchemy.py +496 -471
- utilities/sqlalchemy_polars.py +29 -54
- utilities/string.py +2 -3
- utilities/subprocess.py +1977 -0
- utilities/tempfile.py +112 -4
- utilities/testbook.py +50 -0
- utilities/text.py +174 -42
- utilities/throttle.py +158 -0
- utilities/timer.py +2 -2
- utilities/traceback.py +70 -35
- utilities/types.py +102 -30
- utilities/typing.py +479 -19
- utilities/uuid.py +42 -5
- utilities/version.py +27 -26
- utilities/whenever.py +1559 -361
- utilities/zoneinfo.py +80 -22
- dycw_utilities-0.135.0.dist-info/METADATA +0 -39
- dycw_utilities-0.135.0.dist-info/RECORD +0 -96
- dycw_utilities-0.135.0.dist-info/WHEEL +0 -4
- dycw_utilities-0.135.0.dist-info/licenses/LICENSE +0 -21
- utilities/aiolimiter.py +0 -25
- utilities/arq.py +0 -216
- utilities/eventkit.py +0 -388
- utilities/luigi.py +0 -183
- utilities/period.py +0 -152
- utilities/pudb.py +0 -62
- utilities/python_dotenv.py +0 -101
- utilities/streamlit.py +0 -105
- utilities/typed_settings.py +0 -123
utilities/eventkit.py
DELETED
|
@@ -1,388 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from asyncio import iscoroutinefunction
|
|
4
|
-
from dataclasses import dataclass
|
|
5
|
-
from functools import wraps
|
|
6
|
-
from typing import TYPE_CHECKING, Any, Self, assert_never, cast, override
|
|
7
|
-
|
|
8
|
-
from eventkit import (
|
|
9
|
-
Constant,
|
|
10
|
-
Count,
|
|
11
|
-
DropWhile,
|
|
12
|
-
Enumerate,
|
|
13
|
-
Event,
|
|
14
|
-
Filter,
|
|
15
|
-
Fork,
|
|
16
|
-
Iterate,
|
|
17
|
-
Map,
|
|
18
|
-
Pack,
|
|
19
|
-
Partial,
|
|
20
|
-
PartialRight,
|
|
21
|
-
Pluck,
|
|
22
|
-
Skip,
|
|
23
|
-
Star,
|
|
24
|
-
Take,
|
|
25
|
-
TakeUntil,
|
|
26
|
-
TakeWhile,
|
|
27
|
-
Timestamp,
|
|
28
|
-
)
|
|
29
|
-
|
|
30
|
-
from utilities.functions import apply_decorators
|
|
31
|
-
from utilities.iterables import always_iterable
|
|
32
|
-
from utilities.logging import get_logger
|
|
33
|
-
|
|
34
|
-
if TYPE_CHECKING:
|
|
35
|
-
from collections.abc import Callable
|
|
36
|
-
|
|
37
|
-
from utilities.types import Coro, LoggerOrName, MaybeCoro, MaybeIterable, TypeLike
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
##
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def add_listener[E: Event, F: Callable](
|
|
44
|
-
event: E,
|
|
45
|
-
listener: Callable[..., MaybeCoro[None]],
|
|
46
|
-
/,
|
|
47
|
-
*,
|
|
48
|
-
error: Callable[[Event, BaseException], MaybeCoro[None]] | None = None,
|
|
49
|
-
ignore: TypeLike[BaseException] | None = None,
|
|
50
|
-
logger: LoggerOrName | None = None,
|
|
51
|
-
decorators: MaybeIterable[Callable[[F], F]] | None = None,
|
|
52
|
-
done: Callable[..., MaybeCoro[None]] | None = None,
|
|
53
|
-
keep_ref: bool = False,
|
|
54
|
-
) -> E:
|
|
55
|
-
"""Connect a listener to an event."""
|
|
56
|
-
lifted = lift_listener(
|
|
57
|
-
listener,
|
|
58
|
-
event,
|
|
59
|
-
error=error,
|
|
60
|
-
ignore=ignore,
|
|
61
|
-
logger=logger,
|
|
62
|
-
decorators=decorators,
|
|
63
|
-
)
|
|
64
|
-
return cast("E", event.connect(lifted, done=done, keep_ref=keep_ref))
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
##
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
@dataclass(repr=False, kw_only=True)
|
|
71
|
-
class LiftedEvent[F: Callable[..., MaybeCoro[None]]]:
|
|
72
|
-
"""A lifted version of `Event`."""
|
|
73
|
-
|
|
74
|
-
event: Event
|
|
75
|
-
|
|
76
|
-
def name(self) -> str:
|
|
77
|
-
return self.event.name() # pragma: no cover
|
|
78
|
-
|
|
79
|
-
def done(self) -> bool:
|
|
80
|
-
return self.event.done() # pragma: no cover
|
|
81
|
-
|
|
82
|
-
def set_done(self) -> None:
|
|
83
|
-
self.event.set_done() # pragma: no cover
|
|
84
|
-
|
|
85
|
-
def value(self) -> Any:
|
|
86
|
-
return self.event.value() # pragma: no cover
|
|
87
|
-
|
|
88
|
-
def connect[F2: Callable](
|
|
89
|
-
self,
|
|
90
|
-
listener: F,
|
|
91
|
-
/,
|
|
92
|
-
*,
|
|
93
|
-
error: Callable[[Event, BaseException], MaybeCoro[None]] | None = None,
|
|
94
|
-
ignore: TypeLike[BaseException] | None = None,
|
|
95
|
-
logger: LoggerOrName | None = None,
|
|
96
|
-
decorators: MaybeIterable[Callable[[F2], F2]] | None = None,
|
|
97
|
-
done: Callable[..., MaybeCoro[None]] | None = None,
|
|
98
|
-
keep_ref: bool = False,
|
|
99
|
-
) -> Event:
|
|
100
|
-
return add_listener(
|
|
101
|
-
self.event,
|
|
102
|
-
listener,
|
|
103
|
-
error=error,
|
|
104
|
-
ignore=ignore,
|
|
105
|
-
logger=logger,
|
|
106
|
-
decorators=decorators,
|
|
107
|
-
done=done,
|
|
108
|
-
keep_ref=keep_ref,
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
def disconnect(
|
|
112
|
-
self, listener: Any, /, *, error: Any = None, done: Any = None
|
|
113
|
-
) -> Any:
|
|
114
|
-
return self.event.disconnect( # pragma: no cover
|
|
115
|
-
listener, error=error, done=done
|
|
116
|
-
)
|
|
117
|
-
|
|
118
|
-
def disconnect_obj(self, obj: Any, /) -> None:
|
|
119
|
-
self.event.disconnect_obj(obj) # pragma: no cover
|
|
120
|
-
|
|
121
|
-
def emit(self, *args: Any) -> None:
|
|
122
|
-
self.event.emit(*args) # pragma: no cover
|
|
123
|
-
|
|
124
|
-
def emit_threadsafe(self, *args: Any) -> None:
|
|
125
|
-
self.event.emit_threadsafe(*args) # pragma: no cover
|
|
126
|
-
|
|
127
|
-
def clear(self) -> None:
|
|
128
|
-
self.event.clear() # pragma: no cover
|
|
129
|
-
|
|
130
|
-
def run(self) -> list[Any]:
|
|
131
|
-
return self.event.run() # pragma: no cover
|
|
132
|
-
|
|
133
|
-
def pipe(self, *targets: Event) -> Event:
|
|
134
|
-
return self.event.pipe(*targets) # pragma: no cover
|
|
135
|
-
|
|
136
|
-
def fork(self, *targets: Event) -> Fork:
|
|
137
|
-
return self.event.fork(*targets) # pragma: no cover
|
|
138
|
-
|
|
139
|
-
def set_source(self, source: Any, /) -> None:
|
|
140
|
-
self.event.set_source(source) # pragma: no cover
|
|
141
|
-
|
|
142
|
-
def _onFinalize(self, ref: Any) -> None: # noqa: N802
|
|
143
|
-
self.event._onFinalize(ref) # noqa: SLF001 # pragma: no cover
|
|
144
|
-
|
|
145
|
-
async def aiter(self, *, skip_to_last: bool = False, tuples: bool = False) -> Any:
|
|
146
|
-
async for i in self.event.aiter( # pragma: no cover
|
|
147
|
-
skip_to_last=skip_to_last, tuples=tuples
|
|
148
|
-
):
|
|
149
|
-
yield i
|
|
150
|
-
|
|
151
|
-
__iadd__ = connect
|
|
152
|
-
__isub__ = disconnect
|
|
153
|
-
__call__ = emit
|
|
154
|
-
__or__ = pipe
|
|
155
|
-
|
|
156
|
-
@override
|
|
157
|
-
def __repr__(self) -> str:
|
|
158
|
-
return self.event.__repr__() # pragma: no cover
|
|
159
|
-
|
|
160
|
-
def __len__(self) -> int:
|
|
161
|
-
return self.event.__len__() # pragma: no cover
|
|
162
|
-
|
|
163
|
-
def __bool__(self) -> bool:
|
|
164
|
-
return self.event.__bool__() # pragma: no cover
|
|
165
|
-
|
|
166
|
-
def __getitem__(self, fork_targets: Any, /) -> Fork:
|
|
167
|
-
return self.event.__getitem__(fork_targets) # pragma: no cover
|
|
168
|
-
|
|
169
|
-
def __await__(self) -> Any:
|
|
170
|
-
return self.event.__await__() # pragma: no cover
|
|
171
|
-
|
|
172
|
-
__aiter__ = aiter
|
|
173
|
-
|
|
174
|
-
def __contains__(self, c: Any, /) -> bool:
|
|
175
|
-
return self.event.__contains__(c) # pragma: no cover
|
|
176
|
-
|
|
177
|
-
@override
|
|
178
|
-
def __reduce__(self) -> Any:
|
|
179
|
-
return self.event.__reduce__() # pragma: no cover
|
|
180
|
-
|
|
181
|
-
def filter(self, *, predicate: Any = bool) -> Filter:
|
|
182
|
-
return self.event.filter(predicate=predicate) # pragma: no cover
|
|
183
|
-
|
|
184
|
-
def skip(self, *, count: int = 1) -> Skip:
|
|
185
|
-
return self.event.skip(count=count) # pragma: no cover
|
|
186
|
-
|
|
187
|
-
def take(self, *, count: int = 1) -> Take:
|
|
188
|
-
return self.event.take(count=count) # pragma: no cover
|
|
189
|
-
|
|
190
|
-
def takewhile(self, *, predicate: Any = bool) -> TakeWhile:
|
|
191
|
-
return self.event.takewhile(predicate=predicate) # pragma: no cover
|
|
192
|
-
|
|
193
|
-
def dropwhile(self, *, predicate: Any = lambda x: not x) -> DropWhile: # pyright: ignore[reportUnknownLambdaType]
|
|
194
|
-
return self.event.dropwhile(predicate=predicate) # pragma: no cover
|
|
195
|
-
|
|
196
|
-
def takeuntil(self, notifier: Event, /) -> TakeUntil:
|
|
197
|
-
return self.event.takeuntil(notifier) # pragma: no cover
|
|
198
|
-
|
|
199
|
-
def constant(self, constant: Any, /) -> Constant:
|
|
200
|
-
return self.event.constant(constant) # pragma: no cover
|
|
201
|
-
|
|
202
|
-
def iterate(self, it: Any, /) -> Iterate:
|
|
203
|
-
return self.event.iterate(it) # pragma: no cover
|
|
204
|
-
|
|
205
|
-
def count(self, *, start: int = 0, step: int = 1) -> Count:
|
|
206
|
-
return self.event.count(start=start, step=step) # pragma: no cover
|
|
207
|
-
|
|
208
|
-
def enumerate(self, *, start: int = 0, step: int = 1) -> Enumerate:
|
|
209
|
-
return self.event.enumerate(start=start, step=step) # pragma: no cover
|
|
210
|
-
|
|
211
|
-
def timestamp(self) -> Timestamp:
|
|
212
|
-
return self.event.timestamp() # pragma: no cover
|
|
213
|
-
|
|
214
|
-
def partial(self, *left_args: Any) -> Partial:
|
|
215
|
-
return self.event.partial(*left_args) # pragma: no cover
|
|
216
|
-
|
|
217
|
-
def partial_right(self, *right_args: Any) -> PartialRight:
|
|
218
|
-
return self.event.partial_right(*right_args) # pragma: no cover
|
|
219
|
-
|
|
220
|
-
def star(self) -> Star:
|
|
221
|
-
return self.event.star() # pragma: no cover
|
|
222
|
-
|
|
223
|
-
def pack(self) -> Pack:
|
|
224
|
-
return self.event.pack() # pragma: no cover
|
|
225
|
-
|
|
226
|
-
def pluck(self, *selections: int | str) -> Pluck:
|
|
227
|
-
return self.event.pluck(*selections) # pragma: no cover
|
|
228
|
-
|
|
229
|
-
def map(
|
|
230
|
-
self,
|
|
231
|
-
func: Any,
|
|
232
|
-
/,
|
|
233
|
-
*,
|
|
234
|
-
timeout: float | None = None,
|
|
235
|
-
ordered: bool = True,
|
|
236
|
-
task_limit: int | None = None,
|
|
237
|
-
) -> Map:
|
|
238
|
-
return self.event.map( # pragma: no cover
|
|
239
|
-
func, timeout=timeout, ordered=ordered, task_limit=task_limit
|
|
240
|
-
)
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
##
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
class TypedEvent[F: Callable[..., MaybeCoro[None]]](Event):
|
|
247
|
-
"""A typed version of `Event`."""
|
|
248
|
-
|
|
249
|
-
@override
|
|
250
|
-
def connect[F2: Callable](
|
|
251
|
-
self,
|
|
252
|
-
listener: F,
|
|
253
|
-
error: Callable[[Self, BaseException], MaybeCoro[None]] | None = None,
|
|
254
|
-
done: Callable[[Self], MaybeCoro[None]] | None = None,
|
|
255
|
-
keep_ref: bool = False,
|
|
256
|
-
*,
|
|
257
|
-
ignore: TypeLike[BaseException] | None = None,
|
|
258
|
-
logger: LoggerOrName | None = None,
|
|
259
|
-
decorators: MaybeIterable[Callable[[F2], F2]] | None = None,
|
|
260
|
-
) -> Self:
|
|
261
|
-
lifted = lift_listener(
|
|
262
|
-
listener,
|
|
263
|
-
self,
|
|
264
|
-
error=cast(
|
|
265
|
-
"Callable[[Event, BaseException], MaybeCoro[None]] | None", error
|
|
266
|
-
),
|
|
267
|
-
ignore=ignore,
|
|
268
|
-
logger=logger,
|
|
269
|
-
decorators=decorators,
|
|
270
|
-
)
|
|
271
|
-
return cast(
|
|
272
|
-
"Self", super().connect(lifted, error=error, done=done, keep_ref=keep_ref)
|
|
273
|
-
)
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
##
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
def lift_listener[F1: Callable[..., MaybeCoro[None]], F2: Callable](
|
|
280
|
-
listener: F1,
|
|
281
|
-
event: Event,
|
|
282
|
-
/,
|
|
283
|
-
*,
|
|
284
|
-
error: Callable[[Event, BaseException], MaybeCoro[None]] | None = None,
|
|
285
|
-
ignore: TypeLike[BaseException] | None = None,
|
|
286
|
-
logger: LoggerOrName | None = None,
|
|
287
|
-
decorators: MaybeIterable[Callable[[F2], F2]] | None = None,
|
|
288
|
-
) -> F1:
|
|
289
|
-
match error, bool(iscoroutinefunction(listener)):
|
|
290
|
-
case None, False:
|
|
291
|
-
listener_typed = cast("Callable[..., None]", listener)
|
|
292
|
-
|
|
293
|
-
@wraps(listener)
|
|
294
|
-
def listener_no_error_sync(*args: Any, **kwargs: Any) -> None:
|
|
295
|
-
try:
|
|
296
|
-
listener_typed(*args, **kwargs)
|
|
297
|
-
except Exception as exc: # noqa: BLE001
|
|
298
|
-
if (ignore is not None) and isinstance(exc, ignore):
|
|
299
|
-
return
|
|
300
|
-
get_logger(logger=logger).exception("")
|
|
301
|
-
|
|
302
|
-
lifted = listener_no_error_sync
|
|
303
|
-
|
|
304
|
-
case None, True:
|
|
305
|
-
listener_typed = cast("Callable[..., Coro[None]]", listener)
|
|
306
|
-
|
|
307
|
-
@wraps(listener)
|
|
308
|
-
async def listener_no_error_async(*args: Any, **kwargs: Any) -> None:
|
|
309
|
-
try:
|
|
310
|
-
await listener_typed(*args, **kwargs)
|
|
311
|
-
except Exception as exc: # noqa: BLE001
|
|
312
|
-
if (ignore is not None) and isinstance(exc, ignore):
|
|
313
|
-
return
|
|
314
|
-
get_logger(logger=logger).exception("")
|
|
315
|
-
|
|
316
|
-
lifted = listener_no_error_async
|
|
317
|
-
case _, _:
|
|
318
|
-
match bool(iscoroutinefunction(listener)), bool(iscoroutinefunction(error)):
|
|
319
|
-
case False, False:
|
|
320
|
-
listener_typed = cast("Callable[..., None]", listener)
|
|
321
|
-
error_typed = cast("Callable[[Event, Exception], None]", error)
|
|
322
|
-
|
|
323
|
-
@wraps(listener)
|
|
324
|
-
def listener_have_error_sync(*args: Any, **kwargs: Any) -> None:
|
|
325
|
-
try:
|
|
326
|
-
listener_typed(*args, **kwargs)
|
|
327
|
-
except Exception as exc: # noqa: BLE001
|
|
328
|
-
if (ignore is not None) and isinstance(exc, ignore):
|
|
329
|
-
return
|
|
330
|
-
error_typed(event, exc)
|
|
331
|
-
|
|
332
|
-
lifted = listener_have_error_sync
|
|
333
|
-
case False, True:
|
|
334
|
-
listener_typed = cast("Callable[..., None]", listener)
|
|
335
|
-
error_typed = cast(
|
|
336
|
-
"Callable[[Event, Exception], Coro[None]]", error
|
|
337
|
-
)
|
|
338
|
-
raise LiftListenerError(listener=listener_typed, error=error_typed)
|
|
339
|
-
case True, _:
|
|
340
|
-
listener_typed = cast("Callable[..., Coro[None]]", listener)
|
|
341
|
-
|
|
342
|
-
@wraps(listener)
|
|
343
|
-
async def listener_have_error_async(
|
|
344
|
-
*args: Any, **kwargs: Any
|
|
345
|
-
) -> None:
|
|
346
|
-
try:
|
|
347
|
-
await listener_typed(*args, **kwargs)
|
|
348
|
-
except Exception as exc: # noqa: BLE001
|
|
349
|
-
if (ignore is not None) and isinstance(exc, ignore):
|
|
350
|
-
return None
|
|
351
|
-
if iscoroutinefunction(error):
|
|
352
|
-
error_typed = cast(
|
|
353
|
-
"Callable[[Event, Exception], Coro[None]]", error
|
|
354
|
-
)
|
|
355
|
-
return await error_typed(event, exc)
|
|
356
|
-
error_typed = cast(
|
|
357
|
-
"Callable[[Event, Exception], None]", error
|
|
358
|
-
)
|
|
359
|
-
error_typed(event, exc)
|
|
360
|
-
|
|
361
|
-
lifted = listener_have_error_async
|
|
362
|
-
case _ as never:
|
|
363
|
-
assert_never(never)
|
|
364
|
-
case _ as never:
|
|
365
|
-
assert_never(never)
|
|
366
|
-
|
|
367
|
-
if decorators is not None:
|
|
368
|
-
lifted = apply_decorators(lifted, *always_iterable(decorators))
|
|
369
|
-
return cast("F1", lifted)
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
@dataclass(kw_only=True, slots=True)
|
|
373
|
-
class LiftListenerError(Exception):
|
|
374
|
-
listener: Callable[..., None]
|
|
375
|
-
error: Callable[[Event, Exception], Coro[None]]
|
|
376
|
-
|
|
377
|
-
@override
|
|
378
|
-
def __str__(self) -> str:
|
|
379
|
-
return f"Synchronous listener {self.listener} cannot be paired with an asynchronous error handler {self.error}"
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
__all__ = [
|
|
383
|
-
"LiftListenerError",
|
|
384
|
-
"LiftedEvent",
|
|
385
|
-
"TypedEvent",
|
|
386
|
-
"add_listener",
|
|
387
|
-
"lift_listener",
|
|
388
|
-
]
|
utilities/luigi.py
DELETED
|
@@ -1,183 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from abc import ABC, abstractmethod
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
from typing import TYPE_CHECKING, Any, Literal, assert_never, cast, overload, override
|
|
6
|
-
|
|
7
|
-
import luigi
|
|
8
|
-
from luigi import Parameter, PathParameter, Target, Task
|
|
9
|
-
from luigi import build as _build
|
|
10
|
-
from luigi.parameter import ParameterVisibility, _no_value
|
|
11
|
-
from whenever import ZonedDateTime
|
|
12
|
-
|
|
13
|
-
if TYPE_CHECKING:
|
|
14
|
-
from collections.abc import Callable, Iterable
|
|
15
|
-
|
|
16
|
-
from luigi.execution_summary import LuigiRunResult
|
|
17
|
-
|
|
18
|
-
from utilities.types import DateTimeRoundUnit, LogLevel, PathLike, ZonedDateTimeLike
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
# parameters
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
class ZonedDateTimeParameter(Parameter):
|
|
25
|
-
"""A parameter which takes the value of a zoned datetime."""
|
|
26
|
-
|
|
27
|
-
_unit: DateTimeRoundUnit
|
|
28
|
-
_increment: int
|
|
29
|
-
|
|
30
|
-
@override
|
|
31
|
-
def __init__[T](
|
|
32
|
-
self,
|
|
33
|
-
default: Any = _no_value,
|
|
34
|
-
is_global: bool = False,
|
|
35
|
-
significant: bool = True,
|
|
36
|
-
description: str | None = None,
|
|
37
|
-
config_path: None = None,
|
|
38
|
-
positional: bool = True,
|
|
39
|
-
always_in_help: bool = False,
|
|
40
|
-
batch_method: Callable[[Iterable[T]], T] | None = None,
|
|
41
|
-
visibility: ParameterVisibility = ParameterVisibility.PUBLIC,
|
|
42
|
-
*,
|
|
43
|
-
unit: DateTimeRoundUnit = "second",
|
|
44
|
-
increment: int = 1,
|
|
45
|
-
) -> None:
|
|
46
|
-
super().__init__(
|
|
47
|
-
default,
|
|
48
|
-
is_global,
|
|
49
|
-
significant,
|
|
50
|
-
description,
|
|
51
|
-
config_path,
|
|
52
|
-
positional,
|
|
53
|
-
always_in_help,
|
|
54
|
-
batch_method,
|
|
55
|
-
visibility,
|
|
56
|
-
)
|
|
57
|
-
self._unit = unit
|
|
58
|
-
self._increment = increment
|
|
59
|
-
|
|
60
|
-
@override
|
|
61
|
-
def normalize(self, x: ZonedDateTimeLike) -> ZonedDateTime:
|
|
62
|
-
match x:
|
|
63
|
-
case ZonedDateTime() as date_time:
|
|
64
|
-
...
|
|
65
|
-
case str() as text:
|
|
66
|
-
date_time = ZonedDateTime.parse_common_iso(text)
|
|
67
|
-
case _ as never:
|
|
68
|
-
assert_never(never)
|
|
69
|
-
return date_time.round(self._unit, increment=self._increment, mode="floor")
|
|
70
|
-
|
|
71
|
-
@override
|
|
72
|
-
def parse(self, x: str) -> ZonedDateTime:
|
|
73
|
-
return ZonedDateTime.parse_common_iso(x)
|
|
74
|
-
|
|
75
|
-
@override
|
|
76
|
-
def serialize(self, x: ZonedDateTime) -> str:
|
|
77
|
-
return x.format_common_iso()
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
# targets
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
class PathTarget(Target):
|
|
84
|
-
"""A local target whose `path` attribute is a Pathlib instance."""
|
|
85
|
-
|
|
86
|
-
def __init__(self, path: PathLike, /) -> None:
|
|
87
|
-
super().__init__()
|
|
88
|
-
self.path = Path(path)
|
|
89
|
-
|
|
90
|
-
@override
|
|
91
|
-
def exists(self) -> bool: # pyright: ignore[reportIncompatibleMethodOverride]
|
|
92
|
-
"""Check if the target exists."""
|
|
93
|
-
return self.path.exists()
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
# tasks
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
class ExternalTask(ABC, luigi.ExternalTask):
|
|
100
|
-
"""An external task with `exists()` defined here."""
|
|
101
|
-
|
|
102
|
-
@abstractmethod
|
|
103
|
-
def exists(self) -> bool:
|
|
104
|
-
"""Predicate on which the external task is deemed to exist."""
|
|
105
|
-
msg = f"{self=}" # pragma: no cover
|
|
106
|
-
raise NotImplementedError(msg) # pragma: no cover
|
|
107
|
-
|
|
108
|
-
@override
|
|
109
|
-
def output(self) -> _ExternalTaskDummyTarget: # pyright: ignore[reportIncompatibleMethodOverride]
|
|
110
|
-
return _ExternalTaskDummyTarget(self)
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
class _ExternalTaskDummyTarget(Target):
|
|
114
|
-
"""Dummy target for `ExternalTask`."""
|
|
115
|
-
|
|
116
|
-
def __init__(self, task: ExternalTask, /) -> None:
|
|
117
|
-
super().__init__()
|
|
118
|
-
self._task = task
|
|
119
|
-
|
|
120
|
-
@override
|
|
121
|
-
def exists(self) -> bool: # pyright: ignore[reportIncompatibleMethodOverride]
|
|
122
|
-
return self._task.exists()
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
class ExternalFile(ExternalTask):
|
|
126
|
-
"""Await an external file on the local disk."""
|
|
127
|
-
|
|
128
|
-
path: Path = cast("Any", PathParameter())
|
|
129
|
-
|
|
130
|
-
@override
|
|
131
|
-
def exists(self) -> bool:
|
|
132
|
-
return self.path.exists()
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
# functions
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
@overload
|
|
139
|
-
def build(
|
|
140
|
-
task: Iterable[Task],
|
|
141
|
-
/,
|
|
142
|
-
*,
|
|
143
|
-
detailed_summary: Literal[False] = False,
|
|
144
|
-
local_scheduler: bool = False,
|
|
145
|
-
log_level: LogLevel | None = None,
|
|
146
|
-
workers: int | None = None,
|
|
147
|
-
) -> bool: ...
|
|
148
|
-
@overload
|
|
149
|
-
def build(
|
|
150
|
-
task: Iterable[Task],
|
|
151
|
-
/,
|
|
152
|
-
*,
|
|
153
|
-
detailed_summary: Literal[True],
|
|
154
|
-
local_scheduler: bool = False,
|
|
155
|
-
log_level: LogLevel | None = None,
|
|
156
|
-
workers: int | None = None,
|
|
157
|
-
) -> LuigiRunResult: ...
|
|
158
|
-
def build(
|
|
159
|
-
task: Iterable[Task],
|
|
160
|
-
/,
|
|
161
|
-
*,
|
|
162
|
-
detailed_summary: bool = False,
|
|
163
|
-
local_scheduler: bool = False,
|
|
164
|
-
log_level: LogLevel | None = None,
|
|
165
|
-
workers: int | None = None,
|
|
166
|
-
) -> bool | LuigiRunResult:
|
|
167
|
-
"""Build a set of tasks."""
|
|
168
|
-
return _build(
|
|
169
|
-
task,
|
|
170
|
-
detailed_summary=detailed_summary,
|
|
171
|
-
local_scheduler=local_scheduler,
|
|
172
|
-
**({} if log_level is None else {"log_level": log_level}),
|
|
173
|
-
**({} if workers is None else {"workers": workers}),
|
|
174
|
-
)
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
__all__ = [
|
|
178
|
-
"ExternalFile",
|
|
179
|
-
"ExternalTask",
|
|
180
|
-
"PathTarget",
|
|
181
|
-
"ZonedDateTimeParameter",
|
|
182
|
-
"build",
|
|
183
|
-
]
|