ddeutil-workflow 0.0.64__py3-none-any.whl → 0.0.65__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +1 -1
- ddeutil/workflow/api/routes/job.py +2 -2
- ddeutil/workflow/conf.py +0 -4
- ddeutil/workflow/{exceptions.py → errors.py} +49 -11
- ddeutil/workflow/job.py +249 -118
- ddeutil/workflow/params.py +11 -11
- ddeutil/workflow/result.py +86 -10
- ddeutil/workflow/reusables.py +15 -17
- ddeutil/workflow/stages.py +676 -450
- ddeutil/workflow/utils.py +33 -0
- ddeutil/workflow/workflow.py +163 -664
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.65.dist-info}/METADATA +14 -12
- ddeutil_workflow-0.0.65.dist-info/RECORD +28 -0
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.65.dist-info}/WHEEL +1 -1
- ddeutil_workflow-0.0.64.dist-info/RECORD +0 -28
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.65.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.65.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.64.dist-info → ddeutil_workflow-0.0.65.dist-info}/top_level.txt +0 -0
ddeutil/workflow/params.py
CHANGED
@@ -21,7 +21,7 @@ from ddeutil.core import str2dict, str2list
|
|
21
21
|
from pydantic import BaseModel, Field
|
22
22
|
|
23
23
|
from .__types import StrOrInt
|
24
|
-
from .
|
24
|
+
from .errors import ParamError
|
25
25
|
from .utils import get_d_now, get_dt_now
|
26
26
|
|
27
27
|
T = TypeVar("T")
|
@@ -101,14 +101,14 @@ class DateParam(DefaultParam): # pragma: no cov
|
|
101
101
|
elif isinstance(value, date):
|
102
102
|
return value
|
103
103
|
elif not isinstance(value, str):
|
104
|
-
raise
|
104
|
+
raise ParamError(
|
105
105
|
f"Value that want to convert to date does not support for "
|
106
106
|
f"type: {type(value)}"
|
107
107
|
)
|
108
108
|
try:
|
109
109
|
return date.fromisoformat(value)
|
110
110
|
except ValueError:
|
111
|
-
raise
|
111
|
+
raise ParamError(
|
112
112
|
f"Invalid the ISO format string for date: {value!r}"
|
113
113
|
) from None
|
114
114
|
|
@@ -143,14 +143,14 @@ class DatetimeParam(DefaultParam):
|
|
143
143
|
elif isinstance(value, date):
|
144
144
|
return datetime(value.year, value.month, value.day)
|
145
145
|
elif not isinstance(value, str):
|
146
|
-
raise
|
146
|
+
raise ParamError(
|
147
147
|
f"Value that want to convert to datetime does not support for "
|
148
148
|
f"type: {type(value)}"
|
149
149
|
)
|
150
150
|
try:
|
151
151
|
return datetime.fromisoformat(value)
|
152
152
|
except ValueError:
|
153
|
-
raise
|
153
|
+
raise ParamError(
|
154
154
|
f"Invalid the ISO format string for datetime: {value!r}"
|
155
155
|
) from None
|
156
156
|
|
@@ -189,7 +189,7 @@ class IntParam(DefaultParam):
|
|
189
189
|
try:
|
190
190
|
return int(str(value))
|
191
191
|
except ValueError as err:
|
192
|
-
raise
|
192
|
+
raise ParamError(
|
193
193
|
f"Value can not convert to int, {value}, with base 10"
|
194
194
|
) from err
|
195
195
|
return value
|
@@ -299,7 +299,7 @@ class ChoiceParam(BaseParam):
|
|
299
299
|
if value is None:
|
300
300
|
return self.options[0]
|
301
301
|
if value not in self.options:
|
302
|
-
raise
|
302
|
+
raise ParamError(
|
303
303
|
f"{value!r} does not match any value in choice options."
|
304
304
|
)
|
305
305
|
return value
|
@@ -331,12 +331,12 @@ class MapParam(DefaultParam):
|
|
331
331
|
try:
|
332
332
|
value: dict[Any, Any] = str2dict(value)
|
333
333
|
except ValueError as e:
|
334
|
-
raise
|
334
|
+
raise ParamError(
|
335
335
|
f"Value that want to convert to map does not support for "
|
336
336
|
f"type: {type(value)}"
|
337
337
|
) from e
|
338
338
|
elif not isinstance(value, dict):
|
339
|
-
raise
|
339
|
+
raise ParamError(
|
340
340
|
f"Value of map param support only string-dict or dict type, "
|
341
341
|
f"not {type(value)}"
|
342
342
|
)
|
@@ -366,14 +366,14 @@ class ArrayParam(DefaultParam):
|
|
366
366
|
try:
|
367
367
|
value: list[T] = str2list(value)
|
368
368
|
except ValueError as e:
|
369
|
-
raise
|
369
|
+
raise ParamError(
|
370
370
|
f"Value that want to convert to array does not support for "
|
371
371
|
f"type: {type(value)}"
|
372
372
|
) from e
|
373
373
|
elif isinstance(value, (tuple, set)):
|
374
374
|
return list(value)
|
375
375
|
elif not isinstance(value, list):
|
376
|
-
raise
|
376
|
+
raise ParamError(
|
377
377
|
f"Value of map param support only string-list or list type, "
|
378
378
|
f"not {type(value)}"
|
379
379
|
)
|
ddeutil/workflow/result.py
CHANGED
@@ -11,7 +11,7 @@ from __future__ import annotations
|
|
11
11
|
|
12
12
|
from dataclasses import field
|
13
13
|
from datetime import datetime
|
14
|
-
from enum import IntEnum
|
14
|
+
from enum import IntEnum, auto
|
15
15
|
from typing import Optional, Union
|
16
16
|
|
17
17
|
from pydantic import ConfigDict
|
@@ -19,9 +19,20 @@ from pydantic.dataclasses import dataclass
|
|
19
19
|
from pydantic.functional_validators import model_validator
|
20
20
|
from typing_extensions import Self
|
21
21
|
|
22
|
+
from . import (
|
23
|
+
JobCancelError,
|
24
|
+
JobError,
|
25
|
+
JobSkipError,
|
26
|
+
StageCancelError,
|
27
|
+
StageError,
|
28
|
+
StageSkipError,
|
29
|
+
WorkflowCancelError,
|
30
|
+
WorkflowError,
|
31
|
+
WorkflowSkipError,
|
32
|
+
)
|
22
33
|
from .__types import DictData
|
23
34
|
from .conf import dynamic
|
24
|
-
from .
|
35
|
+
from .errors import ResultError
|
25
36
|
from .logs import TraceModel, get_dt_tznow, get_trace
|
26
37
|
from .utils import default_gen_id, gen_id, get_dt_now
|
27
38
|
|
@@ -31,11 +42,11 @@ class Status(IntEnum):
|
|
31
42
|
Result dataclass object.
|
32
43
|
"""
|
33
44
|
|
34
|
-
SUCCESS =
|
35
|
-
FAILED =
|
36
|
-
WAIT =
|
37
|
-
SKIP =
|
38
|
-
CANCEL =
|
45
|
+
SUCCESS = auto()
|
46
|
+
FAILED = auto()
|
47
|
+
WAIT = auto()
|
48
|
+
SKIP = auto()
|
49
|
+
CANCEL = auto()
|
39
50
|
|
40
51
|
@property
|
41
52
|
def emoji(self) -> str: # pragma: no cov
|
@@ -43,7 +54,19 @@ class Status(IntEnum):
|
|
43
54
|
|
44
55
|
:rtype: str
|
45
56
|
"""
|
46
|
-
return {
|
57
|
+
return {
|
58
|
+
"SUCCESS": "✅",
|
59
|
+
"FAILED": "❌",
|
60
|
+
"WAIT": "🟡",
|
61
|
+
"SKIP": "⏩",
|
62
|
+
"CANCEL": "🚫",
|
63
|
+
}[self.name]
|
64
|
+
|
65
|
+
def __repr__(self) -> str:
|
66
|
+
return self.name
|
67
|
+
|
68
|
+
def __str__(self) -> str:
|
69
|
+
return self.name
|
47
70
|
|
48
71
|
|
49
72
|
SUCCESS = Status.SUCCESS
|
@@ -53,6 +76,55 @@ SKIP = Status.SKIP
|
|
53
76
|
CANCEL = Status.CANCEL
|
54
77
|
|
55
78
|
|
79
|
+
def validate_statuses(statuses: list[Status]) -> Status:
|
80
|
+
"""Validate the final status from list of Status object.
|
81
|
+
|
82
|
+
:param statuses: (list[Status]) A list of status that want to validate the
|
83
|
+
final status.
|
84
|
+
|
85
|
+
:rtype: Status
|
86
|
+
"""
|
87
|
+
if any(s == CANCEL for s in statuses):
|
88
|
+
return CANCEL
|
89
|
+
elif any(s == FAILED for s in statuses):
|
90
|
+
return FAILED
|
91
|
+
elif any(s == WAIT for s in statuses):
|
92
|
+
return WAIT
|
93
|
+
for status in (SUCCESS, SKIP):
|
94
|
+
if all(s == status for s in statuses):
|
95
|
+
return status
|
96
|
+
return FAILED if FAILED in statuses else SUCCESS
|
97
|
+
|
98
|
+
|
99
|
+
def get_status_from_error(
|
100
|
+
error: Union[
|
101
|
+
StageError,
|
102
|
+
StageCancelError,
|
103
|
+
StageSkipError,
|
104
|
+
JobError,
|
105
|
+
JobCancelError,
|
106
|
+
JobSkipError,
|
107
|
+
WorkflowError,
|
108
|
+
WorkflowCancelError,
|
109
|
+
WorkflowSkipError,
|
110
|
+
Exception,
|
111
|
+
BaseException,
|
112
|
+
]
|
113
|
+
) -> Status:
|
114
|
+
"""Get the Status from the error object."""
|
115
|
+
if isinstance(error, (StageSkipError, JobSkipError, WorkflowSkipError)):
|
116
|
+
return SKIP
|
117
|
+
elif isinstance(
|
118
|
+
error, (StageCancelError, JobCancelError, WorkflowCancelError)
|
119
|
+
):
|
120
|
+
return CANCEL
|
121
|
+
return FAILED
|
122
|
+
|
123
|
+
|
124
|
+
def default_context() -> DictData:
|
125
|
+
return {"status": WAIT}
|
126
|
+
|
127
|
+
|
56
128
|
@dataclass(
|
57
129
|
config=ConfigDict(arbitrary_types_allowed=True, use_enum_values=True),
|
58
130
|
)
|
@@ -70,7 +142,7 @@ class Result:
|
|
70
142
|
"""
|
71
143
|
|
72
144
|
status: Status = field(default=WAIT)
|
73
|
-
context: DictData = field(default_factory=
|
145
|
+
context: DictData = field(default_factory=default_context)
|
74
146
|
run_id: Optional[str] = field(default_factory=default_gen_id)
|
75
147
|
parent_run_id: Optional[str] = field(default=None, compare=False)
|
76
148
|
ts: datetime = field(default_factory=get_dt_tznow, compare=False)
|
@@ -160,12 +232,16 @@ class Result:
|
|
160
232
|
Status(status) if isinstance(status, int) else status
|
161
233
|
)
|
162
234
|
self.__dict__["context"].update(context or {})
|
235
|
+
self.__dict__["context"]["status"] = self.status
|
163
236
|
if kwargs:
|
164
237
|
for k in kwargs:
|
165
238
|
if k in self.__dict__["context"]:
|
166
239
|
self.__dict__["context"][k].update(kwargs[k])
|
240
|
+
# NOTE: Exclude the `info` key for update information data.
|
241
|
+
elif k == "info":
|
242
|
+
self.__dict__["context"][k].update(kwargs[k])
|
167
243
|
else:
|
168
|
-
raise
|
244
|
+
raise ResultError(
|
169
245
|
f"The key {k!r} does not exists on context data."
|
170
246
|
)
|
171
247
|
return self
|
ddeutil/workflow/reusables.py
CHANGED
@@ -39,7 +39,7 @@ from pydantic.dataclasses import dataclass
|
|
39
39
|
|
40
40
|
from .__types import DictData, Re
|
41
41
|
from .conf import dynamic
|
42
|
-
from .
|
42
|
+
from .errors import UtilError
|
43
43
|
|
44
44
|
T = TypeVar("T")
|
45
45
|
P = ParamSpec("P")
|
@@ -146,13 +146,13 @@ def get_args_const(
|
|
146
146
|
try:
|
147
147
|
mod: Module = parse(expr)
|
148
148
|
except SyntaxError:
|
149
|
-
raise
|
149
|
+
raise UtilError(
|
150
150
|
f"Post-filter: {expr} does not valid because it raise syntax error."
|
151
151
|
) from None
|
152
152
|
|
153
153
|
body: list[Expr] = mod.body
|
154
154
|
if len(body) > 1:
|
155
|
-
raise
|
155
|
+
raise UtilError(
|
156
156
|
"Post-filter function should be only one calling per workflow."
|
157
157
|
)
|
158
158
|
|
@@ -160,7 +160,7 @@ def get_args_const(
|
|
160
160
|
if isinstance((caller := body[0].value), Name):
|
161
161
|
return caller.id, [], {}
|
162
162
|
elif not isinstance(caller, Call):
|
163
|
-
raise
|
163
|
+
raise UtilError(
|
164
164
|
f"Get arguments does not support for caller type: {type(caller)}"
|
165
165
|
)
|
166
166
|
|
@@ -169,10 +169,10 @@ def get_args_const(
|
|
169
169
|
keywords: dict[str, Constant] = {k.arg: k.value for k in caller.keywords}
|
170
170
|
|
171
171
|
if any(not isinstance(i, Constant) for i in args):
|
172
|
-
raise
|
172
|
+
raise UtilError(f"Argument of {expr} should be constant.")
|
173
173
|
|
174
174
|
if any(not isinstance(i, Constant) for i in keywords.values()):
|
175
|
-
raise
|
175
|
+
raise UtilError(f"Keyword argument of {expr} should be constant.")
|
176
176
|
|
177
177
|
return name.id, args, keywords
|
178
178
|
|
@@ -194,12 +194,10 @@ def get_args_from_filter(
|
|
194
194
|
kwargs: dict[Any, Any] = {k: v.value for k, v in _kwargs.items()}
|
195
195
|
|
196
196
|
if func_name not in filters:
|
197
|
-
raise
|
198
|
-
f"The post-filter: {func_name!r} does not support yet."
|
199
|
-
)
|
197
|
+
raise UtilError(f"The post-filter: {func_name!r} does not support yet.")
|
200
198
|
|
201
199
|
if isinstance((f_func := filters[func_name]), list) and (args or kwargs):
|
202
|
-
raise
|
200
|
+
raise UtilError(
|
203
201
|
"Chain filter function does not support for passing arguments."
|
204
202
|
)
|
205
203
|
|
@@ -228,10 +226,10 @@ def map_post_filter(
|
|
228
226
|
value: T = func(value)
|
229
227
|
else:
|
230
228
|
value: T = f_func(value, *args, **kwargs)
|
231
|
-
except
|
229
|
+
except UtilError:
|
232
230
|
raise
|
233
231
|
except Exception:
|
234
|
-
raise
|
232
|
+
raise UtilError(
|
235
233
|
f"The post-filter: {func_name!r} does not fit with {value!r} "
|
236
234
|
f"(type: {type(value).__name__})."
|
237
235
|
) from None
|
@@ -322,7 +320,7 @@ def str2template(
|
|
322
320
|
try:
|
323
321
|
getter: Any = getdot(caller, params)
|
324
322
|
except ValueError:
|
325
|
-
raise
|
323
|
+
raise UtilError(
|
326
324
|
f"Parameters does not get dot with caller: {caller!r}."
|
327
325
|
) from None
|
328
326
|
|
@@ -404,7 +402,7 @@ def datetime_format(value: datetime, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
|
|
404
402
|
"""
|
405
403
|
if isinstance(value, datetime):
|
406
404
|
return value.strftime(fmt)
|
407
|
-
raise
|
405
|
+
raise UtilError(
|
408
406
|
"This custom function should pass input value with datetime type."
|
409
407
|
)
|
410
408
|
|
@@ -437,7 +435,7 @@ def get_item(
|
|
437
435
|
|
438
436
|
"""
|
439
437
|
if not isinstance(value, dict):
|
440
|
-
raise
|
438
|
+
raise UtilError(
|
441
439
|
f"The value that pass to `getitem` filter should be `dict` not "
|
442
440
|
f"`{type(value)}`."
|
443
441
|
)
|
@@ -454,14 +452,14 @@ def get_index(value: list[Any], index: int) -> Any:
|
|
454
452
|
|
455
453
|
"""
|
456
454
|
if not isinstance(value, list):
|
457
|
-
raise
|
455
|
+
raise UtilError(
|
458
456
|
f"The value that pass to `getindex` filter should be `list` not "
|
459
457
|
f"`{type(value)}`."
|
460
458
|
)
|
461
459
|
try:
|
462
460
|
return value[index]
|
463
461
|
except IndexError as e:
|
464
|
-
raise
|
462
|
+
raise UtilError(
|
465
463
|
f"Index: {index} is out of range of value (The maximum range is "
|
466
464
|
f"{len(value)})."
|
467
465
|
) from e
|