ddeutil-workflow 0.0.61__tar.gz → 0.0.62__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/PKG-INFO +1 -1
- ddeutil_workflow-0.0.62/src/ddeutil/workflow/__about__.py +1 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/__cron.py +24 -25
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/event.py +11 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/reusables.py +67 -15
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/workflow.py +6 -6
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil_workflow.egg-info/PKG-INFO +1 -1
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test__cron.py +36 -28
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_reusables_call_tag.py +2 -1
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_reusables_template.py +76 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_reusables_template_filter.py +8 -2
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_workflow_exec.py +4 -2
- ddeutil_workflow-0.0.61/src/ddeutil/workflow/__about__.py +0 -1
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/LICENSE +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/README.md +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/pyproject.toml +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/setup.cfg +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/__init__.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/__main__.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/__types.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/api/__init__.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/api/logs.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/api/routes/__init__.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/api/routes/job.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/api/routes/logs.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/api/routes/schedules.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/api/routes/workflows.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/api/utils.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/conf.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/exceptions.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/job.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/logs.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/params.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/result.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/scheduler.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/stages.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/utils.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil_workflow.egg-info/SOURCES.txt +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil_workflow.egg-info/dependency_links.txt +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil_workflow.egg-info/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil_workflow.egg-info/requires.txt +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil_workflow.egg-info/top_level.txt +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test__regex.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_conf.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_event.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_job.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_job_exec.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_job_exec_strategy.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_logs_audit.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_logs_trace.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_params.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_release.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_release_queue.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_result.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_schedule.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_schedule_pending.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_schedule_tasks.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_schedule_workflow.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_scheduler_control.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_stage.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_stage_handler_exec.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_strategy.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_utils.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_workflow.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_workflow_exec_job.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_workflow_poke.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_workflow_release.py +0 -0
- {ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/tests/test_workflow_task.py +0 -0
@@ -0,0 +1 @@
|
|
1
|
+
__version__: str = "0.0.62"
|
@@ -18,7 +18,7 @@ from ddeutil.core import (
|
|
18
18
|
isinstance_check,
|
19
19
|
must_split,
|
20
20
|
)
|
21
|
-
from ddeutil.core.dtutils import next_date, replace_date
|
21
|
+
from ddeutil.core.dtutils import DatetimeMode, next_date, replace_date
|
22
22
|
|
23
23
|
WEEKDAYS: dict[str, int] = {
|
24
24
|
"Sun": 0,
|
@@ -31,7 +31,8 @@ WEEKDAYS: dict[str, int] = {
|
|
31
31
|
}
|
32
32
|
|
33
33
|
|
34
|
-
class
|
34
|
+
class YearReachLimit(Exception):
|
35
|
+
""""""
|
35
36
|
|
36
37
|
|
37
38
|
def str2cron(value: str) -> str: # pragma: no cov
|
@@ -178,7 +179,7 @@ class CronPart:
|
|
178
179
|
def __init__(
|
179
180
|
self,
|
180
181
|
unit: Unit,
|
181
|
-
values: str
|
182
|
+
values: Union[str, list[int]],
|
182
183
|
options: Options,
|
183
184
|
) -> None:
|
184
185
|
self.unit: Unit = unit
|
@@ -229,19 +230,21 @@ class CronPart:
|
|
229
230
|
f"(unit={self.unit}, values={self.__str__()!r})"
|
230
231
|
)
|
231
232
|
|
232
|
-
def __lt__(self, other) -> bool:
|
233
|
+
def __lt__(self, other: Union[CronPart, list]) -> bool:
|
233
234
|
"""Override __lt__ method."""
|
234
235
|
if isinstance(other, CronPart):
|
235
236
|
return self.values < other.values
|
236
237
|
elif isinstance(other, list):
|
237
238
|
return self.values < other
|
239
|
+
return NotImplemented
|
238
240
|
|
239
|
-
def __eq__(self, other) -> bool:
|
241
|
+
def __eq__(self, other: Union[CronPart, list]) -> bool:
|
240
242
|
"""Override __eq__ method."""
|
241
243
|
if isinstance(other, CronPart):
|
242
244
|
return self.values == other.values
|
243
245
|
elif isinstance(other, list):
|
244
246
|
return self.values == other
|
247
|
+
return NotImplemented
|
245
248
|
|
246
249
|
@property
|
247
250
|
def min(self) -> int:
|
@@ -271,6 +274,7 @@ class CronPart:
|
|
271
274
|
and (step := self.values[1] - self.values[0]) > 1
|
272
275
|
):
|
273
276
|
return step
|
277
|
+
return None
|
274
278
|
|
275
279
|
@property
|
276
280
|
def is_full(self) -> bool:
|
@@ -355,6 +359,8 @@ class CronPart:
|
|
355
359
|
f"Invalid interval step value {value_step!r} for "
|
356
360
|
f"{self.unit.name!r}"
|
357
361
|
)
|
362
|
+
elif value_step:
|
363
|
+
value_step: int = int(value_step)
|
358
364
|
|
359
365
|
# NOTE: Generate interval that has step
|
360
366
|
interval_list.append(self._interval(value_range_list, value_step))
|
@@ -375,7 +381,9 @@ class CronPart:
|
|
375
381
|
value: str = value.replace(alt, str(self.unit.min + i))
|
376
382
|
return value
|
377
383
|
|
378
|
-
def replace_weekday(
|
384
|
+
def replace_weekday(
|
385
|
+
self, values: Union[list[int], Iterator[int]]
|
386
|
+
) -> list[int]:
|
379
387
|
"""Replaces all 7 with 0 as Sunday can be represented by both.
|
380
388
|
|
381
389
|
:param values: list or iter of int that want to mode by 7
|
@@ -433,12 +441,12 @@ class CronPart:
|
|
433
441
|
def _interval(
|
434
442
|
self,
|
435
443
|
values: list[int],
|
436
|
-
step: int
|
444
|
+
step: Optional[int] = None,
|
437
445
|
) -> list[int]:
|
438
446
|
"""Applies an interval step to a collection of values.
|
439
447
|
|
440
448
|
:param values:
|
441
|
-
:param step:
|
449
|
+
:param step: (int) A step
|
442
450
|
|
443
451
|
:rtype: list[int]
|
444
452
|
"""
|
@@ -515,7 +523,7 @@ class CronPart:
|
|
515
523
|
start_number: Optional[int] = value
|
516
524
|
return multi_dim_values
|
517
525
|
|
518
|
-
def filler(self, value: int) -> int
|
526
|
+
def filler(self, value: int) -> Union[int, str]:
|
519
527
|
"""Formats weekday and month names as string when the relevant options
|
520
528
|
are set.
|
521
529
|
|
@@ -765,12 +773,12 @@ class CronRunner:
|
|
765
773
|
|
766
774
|
def __init__(
|
767
775
|
self,
|
768
|
-
cron: CronJob
|
776
|
+
cron: Union[CronJob, CronJobYear],
|
769
777
|
date: Optional[datetime] = None,
|
770
778
|
*,
|
771
|
-
tz: str
|
779
|
+
tz: Optional[Union[str, ZoneInfo]] = None,
|
772
780
|
) -> None:
|
773
|
-
self.tz: ZoneInfo
|
781
|
+
self.tz: Optional[ZoneInfo] = None
|
774
782
|
if tz:
|
775
783
|
if isinstance(tz, ZoneInfo):
|
776
784
|
self.tz = tz
|
@@ -810,7 +818,7 @@ class CronRunner:
|
|
810
818
|
)
|
811
819
|
|
812
820
|
self.__start_date: datetime = self.date
|
813
|
-
self.cron: CronJob
|
821
|
+
self.cron: Union[CronJob, CronJobYear] = cron
|
814
822
|
self.is_year: bool = isinstance(cron, CronJobYear)
|
815
823
|
self.reset_flag: bool = True
|
816
824
|
|
@@ -863,7 +871,7 @@ class CronRunner:
|
|
863
871
|
|
864
872
|
raise RecursionError("Unable to find execution time for schedule")
|
865
873
|
|
866
|
-
def __shift_date(self, mode:
|
874
|
+
def __shift_date(self, mode: DatetimeMode, reverse: bool = False) -> bool:
|
867
875
|
"""Increments the mode of date value ("month", "day", "hour", "minute")
|
868
876
|
until matches with the schedule.
|
869
877
|
|
@@ -897,8 +905,8 @@ class CronRunner:
|
|
897
905
|
getattr(self.date, mode)
|
898
906
|
> (max_year := max(self.cron.year.values))
|
899
907
|
):
|
900
|
-
raise
|
901
|
-
f"The year is
|
908
|
+
raise YearReachLimit(
|
909
|
+
f"The year is reach the limit with this crontab setting: "
|
902
910
|
f"{max_year}."
|
903
911
|
)
|
904
912
|
|
@@ -917,12 +925,3 @@ class CronRunner:
|
|
917
925
|
|
918
926
|
# NOTE: Return False if the date that match with condition.
|
919
927
|
return False
|
920
|
-
|
921
|
-
|
922
|
-
__all__ = (
|
923
|
-
"CronJob",
|
924
|
-
"CronJobYear",
|
925
|
-
"CronRunner",
|
926
|
-
"Options",
|
927
|
-
"WEEKDAYS",
|
928
|
-
)
|
@@ -316,6 +316,17 @@ class CrontabYear(Crontab):
|
|
316
316
|
)
|
317
317
|
|
318
318
|
|
319
|
+
class ReleaseEvent(BaseModel): # pragma: no cov
|
320
|
+
"""Release trigger event."""
|
321
|
+
|
322
|
+
release: list[str] = Field(
|
323
|
+
description=(
|
324
|
+
"A list of workflow name that want to receive event from release"
|
325
|
+
"trigger."
|
326
|
+
)
|
327
|
+
)
|
328
|
+
|
329
|
+
|
319
330
|
Event = Annotated[
|
320
331
|
Union[
|
321
332
|
CronJobYear,
|
@@ -4,7 +4,7 @@
|
|
4
4
|
# license information.
|
5
5
|
# ------------------------------------------------------------------------------
|
6
6
|
# [x] Use dynamic config
|
7
|
-
"""Reusables module that keep any
|
7
|
+
"""Reusables module that keep any template and template filter functions."""
|
8
8
|
from __future__ import annotations
|
9
9
|
|
10
10
|
import copy
|
@@ -14,7 +14,7 @@ from ast import Call, Constant, Expr, Module, Name, parse
|
|
14
14
|
from datetime import datetime
|
15
15
|
from functools import wraps
|
16
16
|
from importlib import import_module
|
17
|
-
from typing import Any, Callable, Optional, Protocol, TypeVar, Union
|
17
|
+
from typing import Any, Callable, Literal, Optional, Protocol, TypeVar, Union
|
18
18
|
|
19
19
|
try:
|
20
20
|
from typing import ParamSpec
|
@@ -32,7 +32,7 @@ from .exceptions import UtilException
|
|
32
32
|
T = TypeVar("T")
|
33
33
|
P = ParamSpec("P")
|
34
34
|
|
35
|
-
|
35
|
+
# NOTE: Adjust logging level of the `asyncio` to INFO level.
|
36
36
|
logging.getLogger("asyncio").setLevel(logging.INFO)
|
37
37
|
|
38
38
|
|
@@ -40,12 +40,14 @@ FILTERS: dict[str, Callable] = { # pragma: no cov
|
|
40
40
|
"abs": abs,
|
41
41
|
"str": str,
|
42
42
|
"int": int,
|
43
|
+
"list": list,
|
44
|
+
"dict": dict,
|
43
45
|
"title": lambda x: x.title(),
|
44
46
|
"upper": lambda x: x.upper(),
|
45
47
|
"lower": lambda x: x.lower(),
|
46
48
|
"rstr": [str, repr],
|
47
|
-
"keys": lambda x:
|
48
|
-
"values": lambda x:
|
49
|
+
"keys": lambda x: x.keys(),
|
50
|
+
"values": lambda x: x.values(),
|
49
51
|
}
|
50
52
|
|
51
53
|
|
@@ -55,6 +57,7 @@ class FilterFunc(Protocol):
|
|
55
57
|
"""
|
56
58
|
|
57
59
|
filter: str
|
60
|
+
mark: Literal["filter"] = "filter"
|
58
61
|
|
59
62
|
def __call__(self, *args, **kwargs): ... # pragma: no cov
|
60
63
|
|
@@ -73,6 +76,7 @@ def custom_filter(name: str) -> Callable[P, FilterFunc]:
|
|
73
76
|
|
74
77
|
def func_internal(func: Callable[[...], Any]) -> FilterFunc:
|
75
78
|
func.filter = name
|
79
|
+
func.mark = "filter"
|
76
80
|
|
77
81
|
@wraps(func)
|
78
82
|
def wrapped(*args, **kwargs):
|
@@ -104,7 +108,10 @@ def make_filter_registry(
|
|
104
108
|
for fstr, func in inspect.getmembers(importer, inspect.isfunction):
|
105
109
|
# NOTE: check function attribute that already set tag by
|
106
110
|
# ``utils.tag`` decorator.
|
107
|
-
if not
|
111
|
+
if not (
|
112
|
+
hasattr(func, "filter")
|
113
|
+
and str(getattr(func, "mark", "NOT SET")) == "filter"
|
114
|
+
):
|
108
115
|
continue
|
109
116
|
|
110
117
|
func: FilterFunc
|
@@ -209,11 +216,9 @@ def map_post_filter(
|
|
209
216
|
value: T = func(value)
|
210
217
|
else:
|
211
218
|
value: T = f_func(value, *args, **kwargs)
|
212
|
-
except UtilException
|
213
|
-
logger.warning(str(err))
|
219
|
+
except UtilException:
|
214
220
|
raise
|
215
|
-
except Exception
|
216
|
-
logger.warning(str(err))
|
221
|
+
except Exception:
|
217
222
|
raise UtilException(
|
218
223
|
f"The post-filter: {func_name!r} does not fit with {value!r} "
|
219
224
|
f"(type: {type(value).__name__})."
|
@@ -303,7 +308,7 @@ def str2template(
|
|
303
308
|
getter: Any = getdot(caller, params)
|
304
309
|
except ValueError as err:
|
305
310
|
raise UtilException(
|
306
|
-
f"
|
311
|
+
f"Parameters does not get dot with caller: {caller!r}."
|
307
312
|
) from err
|
308
313
|
|
309
314
|
# NOTE:
|
@@ -332,7 +337,7 @@ def param2template(
|
|
332
337
|
filters: Optional[dict[str, FilterRegistry]] = None,
|
333
338
|
*,
|
334
339
|
extras: Optional[DictData] = None,
|
335
|
-
) ->
|
340
|
+
) -> Any:
|
336
341
|
"""Pass param to template string that can search by ``RE_CALLER`` regular
|
337
342
|
expression.
|
338
343
|
|
@@ -342,7 +347,7 @@ def param2template(
|
|
342
347
|
:param filters: A filter mapping for mapping with `map_post_filter` func.
|
343
348
|
:param extras: (Optional[list[str]]) An Override extras.
|
344
349
|
|
345
|
-
:rtype:
|
350
|
+
:rtype: Any
|
346
351
|
:returns: An any getter value from the params input.
|
347
352
|
"""
|
348
353
|
registers: Optional[list[str]] = (
|
@@ -369,6 +374,11 @@ def param2template(
|
|
369
374
|
def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
|
370
375
|
"""Format datetime object to string with the format.
|
371
376
|
|
377
|
+
Examples:
|
378
|
+
|
379
|
+
> ${{ start-date | fmt('%Y%m%d') }}
|
380
|
+
> ${{ start-date | fmt }}
|
381
|
+
|
372
382
|
:param value: (datetime) A datetime value that want to format to string
|
373
383
|
value.
|
374
384
|
:param fmt: (str) A format string pattern that passing to the `dt.strftime`
|
@@ -385,15 +395,54 @@ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
|
|
385
395
|
|
386
396
|
@custom_filter("coalesce") # pragma: no cov
|
387
397
|
def coalesce(value: Optional[T], default: Any) -> T:
|
388
|
-
"""Coalesce with default value if the main value is None.
|
398
|
+
"""Coalesce with default value if the main value is None.
|
399
|
+
|
400
|
+
Examples:
|
401
|
+
|
402
|
+
> ${{ value | coalesce("foo") }}
|
403
|
+
|
404
|
+
:param value: A value that want to check nullable.
|
405
|
+
:param default: A default value that use to returned value if an input
|
406
|
+
value was null.
|
407
|
+
"""
|
389
408
|
return default if value is None else value
|
390
409
|
|
391
410
|
|
411
|
+
@custom_filter("getitem") # pragma: no cov
|
412
|
+
def get_item(
|
413
|
+
value: DictData, key: Union[str, int], default: Optional[Any] = None
|
414
|
+
) -> Any:
|
415
|
+
"""Get a value with an input specific key."""
|
416
|
+
if not isinstance(value, dict):
|
417
|
+
raise UtilException(
|
418
|
+
f"The value that pass to `getitem` filter should be `dict` not "
|
419
|
+
f"`{type(value)}`."
|
420
|
+
)
|
421
|
+
return value.get(key, default)
|
422
|
+
|
423
|
+
|
424
|
+
@custom_filter("getindex") # pragma: no cov
|
425
|
+
def get_index(value: list[Any], index: int):
|
426
|
+
if not isinstance(value, list):
|
427
|
+
raise UtilException(
|
428
|
+
f"The value that pass to `getindex` filter should be `list` not "
|
429
|
+
f"`{type(value)}`."
|
430
|
+
)
|
431
|
+
try:
|
432
|
+
return value[index]
|
433
|
+
except IndexError as e:
|
434
|
+
raise UtilException(
|
435
|
+
f"Index: {index} is out of range of value (The maximum range is "
|
436
|
+
f"{len(value)})."
|
437
|
+
) from e
|
438
|
+
|
439
|
+
|
392
440
|
class TagFunc(Protocol):
|
393
441
|
"""Tag Function Protocol"""
|
394
442
|
|
395
443
|
name: str
|
396
444
|
tag: str
|
445
|
+
mark: Literal["tag"] = "tag"
|
397
446
|
|
398
447
|
def __call__(self, *args, **kwargs): ... # pragma: no cov
|
399
448
|
|
@@ -421,6 +470,7 @@ def tag(
|
|
421
470
|
def func_internal(func: Callable[[...], Any]) -> ReturnTagFunc:
|
422
471
|
func.tag = name or "latest"
|
423
472
|
func.name = alias or func.__name__.replace("_", "-")
|
473
|
+
func.mark = "tag"
|
424
474
|
|
425
475
|
@wraps(func)
|
426
476
|
def wrapped(*args: P.args, **kwargs: P.kwargs) -> TagFunc:
|
@@ -468,7 +518,9 @@ def make_registry(
|
|
468
518
|
# NOTE: check function attribute that already set tag by
|
469
519
|
# ``utils.tag`` decorator.
|
470
520
|
if not (
|
471
|
-
hasattr(func, "tag")
|
521
|
+
hasattr(func, "tag")
|
522
|
+
and hasattr(func, "name")
|
523
|
+
and str(getattr(func, "mark", "NOT SET")) == "tag"
|
472
524
|
): # pragma: no cov
|
473
525
|
continue
|
474
526
|
|
@@ -774,7 +774,7 @@ class Workflow(BaseModel):
|
|
774
774
|
parent_run_id: Optional[str] = None,
|
775
775
|
result: Optional[Result] = None,
|
776
776
|
event: Optional[Event] = None,
|
777
|
-
timeout:
|
777
|
+
timeout: float = 3600,
|
778
778
|
max_job_parallel: int = 2,
|
779
779
|
) -> Result:
|
780
780
|
"""Execute workflow with passing a dynamic parameters to all jobs that
|
@@ -805,10 +805,10 @@ class Workflow(BaseModel):
|
|
805
805
|
:param result: (Result) A Result instance for return context and status.
|
806
806
|
:param event: (Event) An Event manager instance that use to cancel this
|
807
807
|
execution if it forces stopped by parent execution.
|
808
|
-
:param timeout: (
|
809
|
-
use for limit time of execution and waiting job dependency.
|
810
|
-
value does not force stop the task that still running more than
|
811
|
-
limit time. (Default: 60 * 60 seconds)
|
808
|
+
:param timeout: (float) A workflow execution time out in second unit
|
809
|
+
that use for limit time of execution and waiting job dependency.
|
810
|
+
This value does not force stop the task that still running more than
|
811
|
+
this limit time. (Default: 60 * 60 seconds)
|
812
812
|
:param max_job_parallel: (int) The maximum workers that use for job
|
813
813
|
execution in `PoolThreadExecutor` object. (Default: 2 workers)
|
814
814
|
|
@@ -840,7 +840,7 @@ class Workflow(BaseModel):
|
|
840
840
|
job_queue.put(job_id)
|
841
841
|
|
842
842
|
not_timeout_flag: bool = True
|
843
|
-
timeout:
|
843
|
+
timeout: float = dynamic(
|
844
844
|
"max_job_exec_timeout", f=timeout, extras=self.extras
|
845
845
|
)
|
846
846
|
|
@@ -2,15 +2,23 @@ from datetime import datetime
|
|
2
2
|
from functools import partial
|
3
3
|
from zoneinfo import ZoneInfo
|
4
4
|
|
5
|
-
import ddeutil.workflow.__cron as cron
|
6
5
|
import pytest
|
6
|
+
from ddeutil.workflow.__cron import (
|
7
|
+
CRON_UNITS,
|
8
|
+
CronJob,
|
9
|
+
CronJobYear,
|
10
|
+
CronPart,
|
11
|
+
Options,
|
12
|
+
Unit,
|
13
|
+
YearReachLimit,
|
14
|
+
)
|
7
15
|
|
8
16
|
from tests.utils import str2dt
|
9
17
|
|
10
18
|
|
11
19
|
def test_cron_cron_part():
|
12
|
-
cron_part =
|
13
|
-
unit=
|
20
|
+
cron_part = CronPart(
|
21
|
+
unit=Unit(
|
14
22
|
name="month",
|
15
23
|
range=partial(range, 1, 13),
|
16
24
|
min=1,
|
@@ -31,7 +39,7 @@ def test_cron_cron_part():
|
|
31
39
|
],
|
32
40
|
),
|
33
41
|
values="3,5-8",
|
34
|
-
options=
|
42
|
+
options=Options(),
|
35
43
|
)
|
36
44
|
assert [3, 5, 6, 7, 8] == cron_part.values
|
37
45
|
assert repr(cron_part) == (
|
@@ -41,45 +49,45 @@ def test_cron_cron_part():
|
|
41
49
|
"'OCT', 'NOV', 'DEC']), values='3,5-8')"
|
42
50
|
)
|
43
51
|
|
44
|
-
cron_part =
|
52
|
+
cron_part = CronPart(CRON_UNITS[1], [1, 12], Options())
|
45
53
|
assert [1, 12] == cron_part.values
|
46
54
|
assert cron_part > [1]
|
47
55
|
assert cron_part == [1, 12]
|
48
56
|
|
49
57
|
with pytest.raises(ValueError):
|
50
|
-
|
58
|
+
CronPart(CRON_UNITS[1], [45], Options())
|
51
59
|
|
52
60
|
with pytest.raises(TypeError):
|
53
|
-
|
61
|
+
CronPart(CRON_UNITS[1], 45, Options())
|
54
62
|
|
55
63
|
|
56
64
|
def test_cron_cronjob():
|
57
|
-
cr1 =
|
58
|
-
cr2 =
|
65
|
+
cr1 = CronJob("*/5 * * * *")
|
66
|
+
cr2 = CronJob("*/5,3,6 9-17/2 * 1-3 1-5")
|
59
67
|
|
60
68
|
assert str(cr1) == "*/5 * * * *"
|
61
69
|
assert str(cr2) == "0,3,5-6,10,15,20,25,30,35,40,45,50,55 9-17/2 * 1-3 1-5"
|
62
70
|
assert cr1 != cr2
|
63
71
|
assert cr1 < cr2
|
64
72
|
|
65
|
-
cr =
|
73
|
+
cr = CronJob("0 */12 1 ? 0")
|
66
74
|
assert str(cr) == "0 0,12 1 ? 0"
|
67
75
|
|
68
|
-
cr =
|
76
|
+
cr = CronJob("*/4 0 1 * 1")
|
69
77
|
assert str(cr) == "*/4 0 1 * 1"
|
70
78
|
|
71
|
-
cr =
|
79
|
+
cr = CronJob("*/4 */3 1 * 1")
|
72
80
|
assert str(cr) == "*/4 */3 1 * 1"
|
73
81
|
|
74
82
|
with pytest.raises(ValueError):
|
75
|
-
|
83
|
+
CronJob("*/4 */3 1 *")
|
76
84
|
|
77
85
|
|
78
86
|
def test_cron_cronjob_to_list():
|
79
|
-
cr =
|
87
|
+
cr = CronJob("0 */12 1 1 0")
|
80
88
|
assert cr.to_list() == [[0], [0, 12], [1], [1], [0]]
|
81
89
|
|
82
|
-
cr =
|
90
|
+
cr = CronJob("*/4 */3 1 * 1")
|
83
91
|
assert cr.to_list() == [
|
84
92
|
[0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56],
|
85
93
|
[0, 3, 6, 9, 12, 15, 18, 21],
|
@@ -88,7 +96,7 @@ def test_cron_cronjob_to_list():
|
|
88
96
|
[1],
|
89
97
|
]
|
90
98
|
|
91
|
-
cr =
|
99
|
+
cr = CronJob("*/30 */12 23 */3 *")
|
92
100
|
assert cr.to_list() == [
|
93
101
|
[0, 30],
|
94
102
|
[0, 12],
|
@@ -99,7 +107,7 @@ def test_cron_cronjob_to_list():
|
|
99
107
|
|
100
108
|
|
101
109
|
def test_cron_option():
|
102
|
-
cr =
|
110
|
+
cr = CronJob(
|
103
111
|
"*/5,3,6 9-17/2 * 1-3 1-5",
|
104
112
|
option={
|
105
113
|
"output_hashes": True,
|
@@ -108,7 +116,7 @@ def test_cron_option():
|
|
108
116
|
assert (
|
109
117
|
str(cr) == "0,3,5-6,10,15,20,25,30,35,40,45,50,55 H(9-17)/2 H 1-3 1-5"
|
110
118
|
)
|
111
|
-
cr =
|
119
|
+
cr = CronJob(
|
112
120
|
"*/5 9-17/2 * 1-3,5 1-5",
|
113
121
|
option={
|
114
122
|
"output_weekday_names": True,
|
@@ -119,7 +127,7 @@ def test_cron_option():
|
|
119
127
|
|
120
128
|
|
121
129
|
def test_cron_runner_next_previous():
|
122
|
-
sch =
|
130
|
+
sch = CronJob("*/30 */12 23 */3 *").schedule(
|
123
131
|
date=datetime(2024, 1, 1, 12, tzinfo=ZoneInfo("Asia/Bangkok")),
|
124
132
|
)
|
125
133
|
t = sch.next
|
@@ -149,12 +157,12 @@ def test_cron_runner_next_previous():
|
|
149
157
|
|
150
158
|
def test_cron_runner_tz():
|
151
159
|
with pytest.raises(TypeError):
|
152
|
-
|
160
|
+
CronJob("*/5 * * * *").schedule(tz=1)
|
153
161
|
|
154
162
|
with pytest.raises(ValueError):
|
155
|
-
|
163
|
+
CronJob("*/5 * * * *").schedule(tz="UUID")
|
156
164
|
|
157
|
-
sch =
|
165
|
+
sch = CronJob("*/5 * * * *").schedule(
|
158
166
|
date=datetime(2024, 1, 1, 12, tzinfo=ZoneInfo("Asia/Bangkok")),
|
159
167
|
tz="UTC",
|
160
168
|
)
|
@@ -167,14 +175,14 @@ def test_cron_runner_tz():
|
|
167
175
|
assert sch.prev == str2dt("2024-01-01 11:55:00", "UTC")
|
168
176
|
assert sch.prev == str2dt("2024-01-01 11:50:00", "UTC")
|
169
177
|
|
170
|
-
sch =
|
178
|
+
sch = CronJob("*/5 * * * *").schedule(date=datetime(2024, 1, 1, 12))
|
171
179
|
assert sch.date == datetime(2024, 1, 1, 12)
|
172
180
|
assert sch.next == datetime(2024, 1, 1, 12)
|
173
181
|
assert sch.next == datetime(2024, 1, 1, 12, 5)
|
174
182
|
|
175
183
|
|
176
184
|
def test_cron_cronjob_year():
|
177
|
-
cr =
|
185
|
+
cr = CronJobYear("*/5 * * * * */8,1999")
|
178
186
|
assert str(cr) == (
|
179
187
|
"*/5 * * * * 1990,1998-1999,2006,2014,2022,2030,2038,2046,2054,2062,"
|
180
188
|
"2070,2078,2086,2094"
|
@@ -182,7 +190,7 @@ def test_cron_cronjob_year():
|
|
182
190
|
|
183
191
|
|
184
192
|
def test_cron_next_year():
|
185
|
-
sch =
|
193
|
+
sch = CronJob("0 0 1 * *").schedule(
|
186
194
|
date=datetime(2024, 10, 1, 12, tzinfo=ZoneInfo("Asia/Bangkok")),
|
187
195
|
)
|
188
196
|
assert sch.next == str2dt("2024-11-01 00:00:00")
|
@@ -191,7 +199,7 @@ def test_cron_next_year():
|
|
191
199
|
|
192
200
|
|
193
201
|
def test_cron_year_next_year():
|
194
|
-
sch =
|
202
|
+
sch = CronJobYear("0 0 1 * * *").schedule(
|
195
203
|
date=datetime(2024, 10, 1, 12, tzinfo=ZoneInfo("Asia/Bangkok")),
|
196
204
|
)
|
197
205
|
assert sch.next == str2dt("2024-11-01 00:00:00")
|
@@ -200,8 +208,8 @@ def test_cron_year_next_year():
|
|
200
208
|
|
201
209
|
|
202
210
|
def test_cron_year_next_year_raise():
|
203
|
-
sch =
|
211
|
+
sch = CronJobYear("0 0 1 * * 2023").schedule(
|
204
212
|
date=datetime(2024, 10, 1, 12, tzinfo=ZoneInfo("Asia/Bangkok")),
|
205
213
|
)
|
206
|
-
with pytest.raises(
|
214
|
+
with pytest.raises(YearReachLimit):
|
207
215
|
_ = sch.next
|
@@ -121,8 +121,9 @@ def test_make_registry_raise(call_function_dup):
|
|
121
121
|
make_registry("new_tasks_dup")
|
122
122
|
|
123
123
|
|
124
|
-
def
|
124
|
+
def test_extract_caller():
|
125
125
|
func = extract_call("tasks/el-csv-to-parquet@polars-dir")
|
126
126
|
call_func = func()
|
127
127
|
assert call_func.name == "el-csv-to-parquet"
|
128
128
|
assert call_func.tag == "polars-dir"
|
129
|
+
assert call_func.mark == "tag"
|
@@ -76,6 +76,70 @@ def test_param2template_with_filter():
|
|
76
76
|
)
|
77
77
|
assert 5 == value
|
78
78
|
|
79
|
+
assert (
|
80
|
+
param2template(
|
81
|
+
value="${{ params.start-dt | fmt('%Y%m%d') }}",
|
82
|
+
params={"params": {"start-dt": datetime(2024, 6, 12)}},
|
83
|
+
)
|
84
|
+
== "20240612"
|
85
|
+
)
|
86
|
+
|
87
|
+
assert (
|
88
|
+
param2template(
|
89
|
+
value="${{ params.start-dt | fmt }}",
|
90
|
+
params={"params": {"start-dt": datetime(2024, 6, 12)}},
|
91
|
+
)
|
92
|
+
== "2024-06-12 00:00:00"
|
93
|
+
)
|
94
|
+
|
95
|
+
assert (
|
96
|
+
param2template(
|
97
|
+
value="${{ params.value | coalesce('foo') }}",
|
98
|
+
params={"params": {"value": None}},
|
99
|
+
)
|
100
|
+
== "foo"
|
101
|
+
)
|
102
|
+
|
103
|
+
assert (
|
104
|
+
param2template(
|
105
|
+
value="${{ params.value | coalesce('foo') }}",
|
106
|
+
params={"params": {"value": "bar"}},
|
107
|
+
)
|
108
|
+
== "bar"
|
109
|
+
)
|
110
|
+
|
111
|
+
assert (
|
112
|
+
param2template(
|
113
|
+
value="${{ params.data | getitem('key') }}",
|
114
|
+
params={"params": {"data": {"key": "value"}}},
|
115
|
+
)
|
116
|
+
== "value"
|
117
|
+
)
|
118
|
+
|
119
|
+
assert (
|
120
|
+
param2template(
|
121
|
+
value="${{ params.data | getitem('foo', 'bar') }}",
|
122
|
+
params={"params": {"data": {"key": "value"}}},
|
123
|
+
)
|
124
|
+
== "bar"
|
125
|
+
)
|
126
|
+
|
127
|
+
assert (
|
128
|
+
param2template(
|
129
|
+
value="${{ params.data | getitem(1, 'bar') }}",
|
130
|
+
params={"params": {"data": {1: "value"}}},
|
131
|
+
)
|
132
|
+
== "value"
|
133
|
+
)
|
134
|
+
|
135
|
+
assert (
|
136
|
+
param2template(
|
137
|
+
value="${{ params.range | getindex(0) }}",
|
138
|
+
params={"params": {"range": [1, 2, 3]}},
|
139
|
+
)
|
140
|
+
== 1
|
141
|
+
)
|
142
|
+
|
79
143
|
with pytest.raises(UtilException):
|
80
144
|
param2template(
|
81
145
|
value="${{ params.value | abs12 }}",
|
@@ -96,6 +160,18 @@ def test_param2template_with_filter():
|
|
96
160
|
},
|
97
161
|
)
|
98
162
|
|
163
|
+
with pytest.raises(UtilException):
|
164
|
+
param2template(
|
165
|
+
value="${{ params.data | getitem(1, 'bar') }}",
|
166
|
+
params={"params": {"data": 1}},
|
167
|
+
)
|
168
|
+
|
169
|
+
with pytest.raises(UtilException):
|
170
|
+
param2template(
|
171
|
+
value="${{ params.range | getindex(4) }}",
|
172
|
+
params={"params": {"range": [1, 2, 3]}},
|
173
|
+
)
|
174
|
+
|
99
175
|
|
100
176
|
def test_not_in_template():
|
101
177
|
assert not not_in_template(
|
@@ -40,6 +40,10 @@ def test_make_registry_raise():
|
|
40
40
|
assert isfunction(make_filter_registry()["foo"])
|
41
41
|
assert "bar" == make_filter_registry()["foo"]("")
|
42
42
|
|
43
|
+
filter_func = make_filter_registry()["foo"]
|
44
|
+
assert filter_func.filter == "foo"
|
45
|
+
assert filter_func.mark == "filter"
|
46
|
+
|
43
47
|
|
44
48
|
def test_get_args_const():
|
45
49
|
name, args, kwargs = get_args_const('fmt(fmt="str")')
|
@@ -80,9 +84,11 @@ def test_map_post_filter():
|
|
80
84
|
assert "'bar'" == map_post_filter("bar", ["rstr"], registry)
|
81
85
|
assert 1 == map_post_filter("1", ["int"], registry)
|
82
86
|
assert ["foo", "bar"] == map_post_filter(
|
83
|
-
{"foo": 1, "bar": 2}, ["keys"], registry
|
87
|
+
{"foo": 1, "bar": 2}, ["keys", "list"], registry
|
88
|
+
)
|
89
|
+
assert [1, 2] == map_post_filter(
|
90
|
+
{"foo": 1, "bar": 2}, ["values", "list"], registry
|
84
91
|
)
|
85
|
-
assert [1, 2] == map_post_filter({"foo": 1, "bar": 2}, ["values"], registry)
|
86
92
|
|
87
93
|
with pytest.raises(UtilException):
|
88
94
|
map_post_filter("demo", ['rstr(fmt="foo")'], registry)
|
@@ -859,7 +859,8 @@ def test_workflow_exec_raise_param(test_path):
|
|
859
859
|
"errors": {
|
860
860
|
"name": "UtilException",
|
861
861
|
"message": (
|
862
|
-
"
|
862
|
+
"Parameters does not get dot with caller: "
|
863
|
+
"'params.name'."
|
863
864
|
),
|
864
865
|
},
|
865
866
|
}
|
@@ -921,7 +922,8 @@ def test_workflow_exec_raise_job_trigger(test_path):
|
|
921
922
|
"errors": {
|
922
923
|
"name": "UtilException",
|
923
924
|
"message": (
|
924
|
-
"
|
925
|
+
"Parameters does not get dot with caller: "
|
926
|
+
"'params.name'."
|
925
927
|
),
|
926
928
|
},
|
927
929
|
},
|
@@ -1 +0,0 @@
|
|
1
|
-
__version__: str = "0.0.61"
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/api/routes/__init__.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
{ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/api/routes/schedules.py
RENAMED
File without changes
|
{ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil/workflow/api/routes/workflows.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil_workflow.egg-info/SOURCES.txt
RENAMED
File without changes
|
File without changes
|
{ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil_workflow.egg-info/entry_points.txt
RENAMED
File without changes
|
{ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil_workflow.egg-info/requires.txt
RENAMED
File without changes
|
{ddeutil_workflow-0.0.61 → ddeutil_workflow-0.0.62}/src/ddeutil_workflow.egg-info/top_level.txt
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|