ddeutil-workflow 0.0.50__py3-none-any.whl → 0.0.51__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +4 -26
- ddeutil/workflow/conf.py +2 -2
- ddeutil/workflow/cron.py +46 -20
- ddeutil/workflow/job.py +166 -93
- ddeutil/workflow/logs.py +22 -18
- ddeutil/workflow/params.py +56 -16
- ddeutil/workflow/reusables.py +4 -2
- ddeutil/workflow/scheduler.py +5 -1
- ddeutil/workflow/stages.py +309 -146
- ddeutil/workflow/workflow.py +76 -72
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.51.dist-info}/METADATA +69 -13
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.51.dist-info}/RECORD +16 -16
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.51.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.51.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.51.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -56,9 +56,10 @@ from typing_extensions import Self
|
|
56
56
|
from .__types import DictData, DictStr, TupleStr
|
57
57
|
from .conf import dynamic
|
58
58
|
from .exceptions import StageException, UtilException, to_dict
|
59
|
-
from .result import FAILED, SUCCESS, WAIT, Result, Status
|
59
|
+
from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
|
60
60
|
from .reusables import TagFunc, extract_call, not_in_template, param2template
|
61
61
|
from .utils import (
|
62
|
+
delay,
|
62
63
|
filter_func,
|
63
64
|
gen_id,
|
64
65
|
make_exec,
|
@@ -72,7 +73,8 @@ class BaseStage(BaseModel, ABC):
|
|
72
73
|
metadata. If you want to implement any custom stage, you can use this class
|
73
74
|
to parent and implement ``self.execute()`` method only.
|
74
75
|
|
75
|
-
This class is the abstraction class for any stage
|
76
|
+
This class is the abstraction class for any stage model that want to
|
77
|
+
implement to workflow model.
|
76
78
|
"""
|
77
79
|
|
78
80
|
extras: DictData = Field(
|
@@ -226,7 +228,7 @@ class BaseStage(BaseModel, ABC):
|
|
226
228
|
and want to set on the `to` like;
|
227
229
|
|
228
230
|
... (i) output: {'foo': bar}
|
229
|
-
... (ii) to: {}
|
231
|
+
... (ii) to: {'stages': {}}
|
230
232
|
|
231
233
|
The result of the `to` argument will be;
|
232
234
|
|
@@ -234,11 +236,15 @@ class BaseStage(BaseModel, ABC):
|
|
234
236
|
'stages': {
|
235
237
|
'<stage-id>': {
|
236
238
|
'outputs': {'foo': 'bar'},
|
237
|
-
'skipped': False
|
239
|
+
'skipped': False,
|
238
240
|
}
|
239
241
|
}
|
240
242
|
}
|
241
243
|
|
244
|
+
Important:
|
245
|
+
This method is use for reconstruct the result context and transfer
|
246
|
+
to the `to` argument.
|
247
|
+
|
242
248
|
:param output: (DictData) An output data that want to extract to an
|
243
249
|
output key.
|
244
250
|
:param to: (DictData) A context data that want to add output result.
|
@@ -269,7 +275,11 @@ class BaseStage(BaseModel, ABC):
|
|
269
275
|
if "skipped" in output
|
270
276
|
else {}
|
271
277
|
)
|
272
|
-
to["stages"][_id] = {
|
278
|
+
to["stages"][_id] = {
|
279
|
+
"outputs": copy.deepcopy(output),
|
280
|
+
**skipping,
|
281
|
+
**errors,
|
282
|
+
}
|
273
283
|
return to
|
274
284
|
|
275
285
|
def get_outputs(self, outputs: DictData) -> DictData:
|
@@ -330,6 +340,7 @@ class BaseStage(BaseModel, ABC):
|
|
330
340
|
|
331
341
|
|
332
342
|
class BaseAsyncStage(BaseStage):
|
343
|
+
"""Base Async Stage model."""
|
333
344
|
|
334
345
|
@abstractmethod
|
335
346
|
def execute(
|
@@ -432,12 +443,13 @@ class EmptyStage(BaseAsyncStage):
|
|
432
443
|
|
433
444
|
echo: Optional[str] = Field(
|
434
445
|
default=None,
|
435
|
-
description="A string
|
446
|
+
description="A string message that want to show on the stdout.",
|
436
447
|
)
|
437
448
|
sleep: float = Field(
|
438
449
|
default=0,
|
439
|
-
description="A second value to sleep before start execution",
|
450
|
+
description="A second value to sleep before start execution.",
|
440
451
|
ge=0,
|
452
|
+
lt=1800,
|
441
453
|
)
|
442
454
|
|
443
455
|
def execute(
|
@@ -464,12 +476,23 @@ class EmptyStage(BaseAsyncStage):
|
|
464
476
|
:rtype: Result
|
465
477
|
"""
|
466
478
|
result: Result = result or Result(
|
467
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
479
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
480
|
+
extras=self.extras,
|
468
481
|
)
|
469
482
|
|
483
|
+
if not self.echo:
|
484
|
+
message: str = "..."
|
485
|
+
else:
|
486
|
+
message: str = param2template(
|
487
|
+
dedent(self.echo), params, extras=self.extras
|
488
|
+
)
|
489
|
+
if "\n" in self.echo:
|
490
|
+
message: str = "\n\t" + message.replace("\n", "\n\t").strip(
|
491
|
+
"\n"
|
492
|
+
)
|
493
|
+
|
470
494
|
result.trace.info(
|
471
|
-
f"[STAGE]: Empty-Execute: {self.name!r}: "
|
472
|
-
f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
|
495
|
+
f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
|
473
496
|
)
|
474
497
|
if self.sleep > 0:
|
475
498
|
if self.sleep > 5:
|
@@ -497,9 +520,10 @@ class EmptyStage(BaseAsyncStage):
|
|
497
520
|
|
498
521
|
:rtype: Result
|
499
522
|
"""
|
500
|
-
if result is None:
|
523
|
+
if result is None:
|
501
524
|
result: Result = Result(
|
502
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
525
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
526
|
+
extras=self.extras,
|
503
527
|
)
|
504
528
|
|
505
529
|
await result.trace.ainfo(
|
@@ -540,8 +564,8 @@ class BashStage(BaseStage):
|
|
540
564
|
env: DictStr = Field(
|
541
565
|
default_factory=dict,
|
542
566
|
description=(
|
543
|
-
"An environment
|
544
|
-
"
|
567
|
+
"An environment variables that set before start execute by adding "
|
568
|
+
"on the header of the `.sh` file."
|
545
569
|
),
|
546
570
|
)
|
547
571
|
|
@@ -603,7 +627,8 @@ class BashStage(BaseStage):
|
|
603
627
|
"""
|
604
628
|
if result is None: # pragma: no cov
|
605
629
|
result: Result = Result(
|
606
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
630
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
631
|
+
extras=self.extras,
|
607
632
|
)
|
608
633
|
|
609
634
|
bash: str = param2template(
|
@@ -616,7 +641,7 @@ class BashStage(BaseStage):
|
|
616
641
|
env=param2template(self.env, params, extras=self.extras),
|
617
642
|
run_id=result.run_id,
|
618
643
|
) as sh:
|
619
|
-
result.trace.debug(f"...
|
644
|
+
result.trace.debug(f"... Create `{sh[1]}` file.")
|
620
645
|
rs: CompletedProcess = subprocess.run(
|
621
646
|
sh, shell=False, capture_output=True, text=True
|
622
647
|
)
|
@@ -660,18 +685,20 @@ class PyStage(BaseStage):
|
|
660
685
|
"""
|
661
686
|
|
662
687
|
run: str = Field(
|
663
|
-
description="A Python string statement that want to run with exec
|
688
|
+
description="A Python string statement that want to run with `exec`.",
|
664
689
|
)
|
665
690
|
vars: DictData = Field(
|
666
691
|
default_factory=dict,
|
667
692
|
description=(
|
668
|
-
"A mapping
|
693
|
+
"A variable mapping that want to pass to globals parameter in the "
|
694
|
+
"`exec` func."
|
669
695
|
),
|
670
696
|
)
|
671
697
|
|
672
698
|
@staticmethod
|
673
699
|
def filter_locals(values: DictData) -> Iterator[str]:
|
674
|
-
"""Filter a locals
|
700
|
+
"""Filter a locals mapping values that be module, class, or
|
701
|
+
__annotations__.
|
675
702
|
|
676
703
|
:param values: (DictData) A locals values that want to filter.
|
677
704
|
|
@@ -728,7 +755,8 @@ class PyStage(BaseStage):
|
|
728
755
|
"""
|
729
756
|
if result is None: # pragma: no cov
|
730
757
|
result: Result = Result(
|
731
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
758
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
759
|
+
extras=self.extras,
|
732
760
|
)
|
733
761
|
|
734
762
|
lc: DictData = {}
|
@@ -741,11 +769,11 @@ class PyStage(BaseStage):
|
|
741
769
|
|
742
770
|
# NOTE: Start exec the run statement.
|
743
771
|
result.trace.info(f"[STAGE]: Py-Execute: {self.name}")
|
744
|
-
result.trace.warning(
|
745
|
-
|
746
|
-
|
747
|
-
)
|
748
|
-
|
772
|
+
# result.trace.warning(
|
773
|
+
# "[STAGE]: This stage allow use `eval` function, so, please "
|
774
|
+
# "check your statement be safe before execute."
|
775
|
+
# )
|
776
|
+
#
|
749
777
|
# WARNING: The exec build-in function is very dangerous. So, it
|
750
778
|
# should use the re module to validate exec-string before running.
|
751
779
|
exec(
|
@@ -811,11 +839,17 @@ class CallStage(BaseStage):
|
|
811
839
|
:param event: (Event) An event manager that use to track parent execute
|
812
840
|
was not force stopped.
|
813
841
|
|
842
|
+
:raise ValueError: If necessary arguments does not pass from the `args`
|
843
|
+
field.
|
844
|
+
:raise TypeError: If the result from the caller function does not by
|
845
|
+
a `dict` type.
|
846
|
+
|
814
847
|
:rtype: Result
|
815
848
|
"""
|
816
849
|
if result is None: # pragma: no cov
|
817
850
|
result: Result = Result(
|
818
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
851
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
852
|
+
extras=self.extras,
|
819
853
|
)
|
820
854
|
|
821
855
|
has_keyword: bool = False
|
@@ -890,9 +924,12 @@ class CallStage(BaseStage):
|
|
890
924
|
"""Parse Pydantic model from any dict data before parsing to target
|
891
925
|
caller function.
|
892
926
|
|
893
|
-
:param func:
|
894
|
-
:param args:
|
895
|
-
:param result: (Result)
|
927
|
+
:param func: A tag function that want to get typing.
|
928
|
+
:param args: An arguments before passing to this tag function.
|
929
|
+
:param result: (Result) A result object for keeping context and status
|
930
|
+
data.
|
931
|
+
|
932
|
+
:rtype: DictData
|
896
933
|
"""
|
897
934
|
try:
|
898
935
|
type_hints: dict[str, Any] = get_type_hints(func)
|
@@ -965,22 +1002,29 @@ class TriggerStage(BaseStage):
|
|
965
1002
|
|
966
1003
|
:rtype: Result
|
967
1004
|
"""
|
1005
|
+
from .exceptions import WorkflowException
|
968
1006
|
from .workflow import Workflow
|
969
1007
|
|
970
1008
|
if result is None:
|
971
1009
|
result: Result = Result(
|
972
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
1010
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1011
|
+
extras=self.extras,
|
973
1012
|
)
|
974
1013
|
|
975
1014
|
_trigger: str = param2template(self.trigger, params, extras=self.extras)
|
976
1015
|
result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
|
977
|
-
|
978
|
-
|
979
|
-
|
980
|
-
|
981
|
-
|
982
|
-
|
983
|
-
|
1016
|
+
try:
|
1017
|
+
rs: Result = Workflow.from_conf(
|
1018
|
+
name=_trigger,
|
1019
|
+
extras=self.extras | {"stage_raise_error": True},
|
1020
|
+
).execute(
|
1021
|
+
params=param2template(self.params, params, extras=self.extras),
|
1022
|
+
parent_run_id=result.run_id,
|
1023
|
+
event=event,
|
1024
|
+
)
|
1025
|
+
except WorkflowException as e:
|
1026
|
+
raise StageException("Trigger workflow stage was failed") from e
|
1027
|
+
|
984
1028
|
if rs.status == FAILED:
|
985
1029
|
err_msg: str | None = (
|
986
1030
|
f" with:\n{msg}"
|
@@ -1020,17 +1064,25 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1020
1064
|
"""
|
1021
1065
|
|
1022
1066
|
parallel: dict[str, list[Stage]] = Field(
|
1023
|
-
description="A mapping of parallel branch
|
1067
|
+
description="A mapping of parallel branch name and stages.",
|
1068
|
+
)
|
1069
|
+
max_workers: int = Field(
|
1070
|
+
default=2,
|
1071
|
+
ge=1,
|
1072
|
+
lt=20,
|
1073
|
+
description=(
|
1074
|
+
"The maximum thread pool worker size for execution parallel."
|
1075
|
+
),
|
1076
|
+
alias="max-workers",
|
1024
1077
|
)
|
1025
|
-
max_parallel_core: int = Field(default=2)
|
1026
1078
|
|
1027
|
-
|
1028
|
-
|
1079
|
+
def execute_task(
|
1080
|
+
self,
|
1029
1081
|
branch: str,
|
1030
1082
|
params: DictData,
|
1031
1083
|
result: Result,
|
1032
|
-
stages: list[Stage],
|
1033
1084
|
*,
|
1085
|
+
event: Event | None = None,
|
1034
1086
|
extras: DictData | None = None,
|
1035
1087
|
) -> DictData:
|
1036
1088
|
"""Task execution method for passing a branch to each thread.
|
@@ -1039,32 +1091,82 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1039
1091
|
:param params: A parameter data that want to use in this execution.
|
1040
1092
|
:param result: (Result) A result object for keeping context and status
|
1041
1093
|
data.
|
1042
|
-
:param
|
1043
|
-
|
1094
|
+
:param event: (Event) An event manager that use to track parent execute
|
1095
|
+
was not force stopped.
|
1096
|
+
:param extras: (DictData) An extra parameters that want to override
|
1097
|
+
config values.
|
1044
1098
|
|
1045
1099
|
:rtype: DictData
|
1046
1100
|
"""
|
1047
|
-
|
1048
|
-
|
1049
|
-
|
1101
|
+
result.trace.debug(f"... Execute branch: {branch!r}")
|
1102
|
+
context: DictData = copy.deepcopy(params)
|
1103
|
+
context.update({"branch": branch, "stages": {}})
|
1104
|
+
for stage in self.parallel[branch]:
|
1105
|
+
|
1050
1106
|
if extras:
|
1051
1107
|
stage.extras = extras
|
1052
1108
|
|
1109
|
+
if stage.is_skipped(params=context):
|
1110
|
+
result.trace.info(f"... Skip stage: {stage.iden!r}")
|
1111
|
+
stage.set_outputs(output={"skipped": True}, to=context)
|
1112
|
+
continue
|
1113
|
+
|
1114
|
+
if event and event.is_set():
|
1115
|
+
error_msg: str = (
|
1116
|
+
"Branch-Stage was canceled from event that had set before "
|
1117
|
+
"stage item execution."
|
1118
|
+
)
|
1119
|
+
return result.catch(
|
1120
|
+
status=CANCEL,
|
1121
|
+
parallel={
|
1122
|
+
branch: {
|
1123
|
+
"branch": branch,
|
1124
|
+
"stages": filter_func(context.pop("stages", {})),
|
1125
|
+
"errors": StageException(error_msg).to_dict(),
|
1126
|
+
}
|
1127
|
+
},
|
1128
|
+
)
|
1129
|
+
|
1053
1130
|
try:
|
1054
|
-
stage.
|
1055
|
-
|
1056
|
-
|
1057
|
-
|
1058
|
-
|
1059
|
-
|
1060
|
-
to=context,
|
1131
|
+
rs: Result = stage.handler_execute(
|
1132
|
+
params=context,
|
1133
|
+
run_id=result.run_id,
|
1134
|
+
parent_run_id=result.parent_run_id,
|
1135
|
+
raise_error=True,
|
1136
|
+
event=event,
|
1061
1137
|
)
|
1138
|
+
stage.set_outputs(rs.context, to=context)
|
1062
1139
|
except StageException as e: # pragma: no cov
|
1063
|
-
result.trace.error(
|
1064
|
-
|
1140
|
+
result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
|
1141
|
+
raise StageException(
|
1142
|
+
f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
|
1143
|
+
) from None
|
1144
|
+
|
1145
|
+
if rs.status == FAILED:
|
1146
|
+
error_msg: str = (
|
1147
|
+
f"Item-Stage was break because it has a sub stage, "
|
1148
|
+
f"{stage.iden}, failed without raise error."
|
1149
|
+
)
|
1150
|
+
return result.catch(
|
1151
|
+
status=FAILED,
|
1152
|
+
parallel={
|
1153
|
+
branch: {
|
1154
|
+
"branch": branch,
|
1155
|
+
"stages": filter_func(context.pop("stages", {})),
|
1156
|
+
"errors": StageException(error_msg).to_dict(),
|
1157
|
+
},
|
1158
|
+
},
|
1065
1159
|
)
|
1066
|
-
|
1067
|
-
return
|
1160
|
+
|
1161
|
+
return result.catch(
|
1162
|
+
status=SUCCESS,
|
1163
|
+
parallel={
|
1164
|
+
branch: {
|
1165
|
+
"branch": branch,
|
1166
|
+
"stages": filter_func(context.pop("stages", {})),
|
1167
|
+
},
|
1168
|
+
},
|
1169
|
+
)
|
1068
1170
|
|
1069
1171
|
def execute(
|
1070
1172
|
self,
|
@@ -1086,41 +1188,46 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1086
1188
|
"""
|
1087
1189
|
if result is None: # pragma: no cov
|
1088
1190
|
result: Result = Result(
|
1089
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
1191
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1192
|
+
extras=self.extras,
|
1090
1193
|
)
|
1091
|
-
|
1194
|
+
event: Event = Event() if event is None else event
|
1092
1195
|
result.trace.info(
|
1093
|
-
f"[STAGE]: Parallel-Execute
|
1196
|
+
f"[STAGE]: Parallel-Execute: {self.max_workers} workers."
|
1094
1197
|
)
|
1095
|
-
|
1096
|
-
status = SUCCESS
|
1198
|
+
result.catch(status=WAIT, context={"parallel": {}})
|
1097
1199
|
with ThreadPoolExecutor(
|
1098
|
-
max_workers=self.
|
1200
|
+
max_workers=self.max_workers,
|
1099
1201
|
thread_name_prefix="parallel_stage_exec_",
|
1100
1202
|
) as executor:
|
1101
1203
|
|
1102
|
-
|
1103
|
-
|
1104
|
-
|
1105
|
-
|
1106
|
-
|
1107
|
-
|
1108
|
-
|
1109
|
-
|
1110
|
-
|
1111
|
-
|
1112
|
-
|
1204
|
+
context: DictData = {}
|
1205
|
+
status: Status = SUCCESS
|
1206
|
+
|
1207
|
+
futures: list[Future] = (
|
1208
|
+
executor.submit(
|
1209
|
+
self.execute_task,
|
1210
|
+
branch=branch,
|
1211
|
+
params=params,
|
1212
|
+
result=result,
|
1213
|
+
event=event,
|
1214
|
+
extras=self.extras,
|
1113
1215
|
)
|
1216
|
+
for branch in self.parallel
|
1217
|
+
)
|
1114
1218
|
|
1115
1219
|
done = as_completed(futures, timeout=1800)
|
1116
1220
|
for future in done:
|
1117
|
-
|
1118
|
-
|
1119
|
-
|
1120
|
-
if "errors" in context:
|
1221
|
+
try:
|
1222
|
+
future.result()
|
1223
|
+
except StageException as e:
|
1121
1224
|
status = FAILED
|
1225
|
+
result.trace.error(
|
1226
|
+
f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
|
1227
|
+
)
|
1228
|
+
context.update({"errors": e.to_dict()})
|
1122
1229
|
|
1123
|
-
return result.catch(status=status, context=
|
1230
|
+
return result.catch(status=status, context=context)
|
1124
1231
|
|
1125
1232
|
|
1126
1233
|
class ForEachStage(BaseStage):
|
@@ -1182,9 +1289,11 @@ class ForEachStage(BaseStage):
|
|
1182
1289
|
:param event: (Event) An event manager that use to track parent execute
|
1183
1290
|
was not force stopped.
|
1184
1291
|
|
1292
|
+
:raise StageException: If the stage execution raise errors.
|
1293
|
+
|
1185
1294
|
:rtype: Result
|
1186
1295
|
"""
|
1187
|
-
result.trace.debug(f"
|
1296
|
+
result.trace.debug(f"... Execute item: {item!r}")
|
1188
1297
|
context: DictData = copy.deepcopy(params)
|
1189
1298
|
context.update({"item": item, "stages": {}})
|
1190
1299
|
for stage in self.stages:
|
@@ -1193,17 +1302,17 @@ class ForEachStage(BaseStage):
|
|
1193
1302
|
stage.extras = self.extras
|
1194
1303
|
|
1195
1304
|
if stage.is_skipped(params=context):
|
1196
|
-
result.trace.info(f"
|
1305
|
+
result.trace.info(f"... Skip stage: {stage.iden!r}")
|
1197
1306
|
stage.set_outputs(output={"skipped": True}, to=context)
|
1198
1307
|
continue
|
1199
1308
|
|
1200
|
-
if event and event.is_set():
|
1309
|
+
if event and event.is_set(): # pragma: no cov
|
1201
1310
|
error_msg: str = (
|
1202
1311
|
"Item-Stage was canceled from event that had set before "
|
1203
1312
|
"stage item execution."
|
1204
1313
|
)
|
1205
1314
|
return result.catch(
|
1206
|
-
status=
|
1315
|
+
status=CANCEL,
|
1207
1316
|
foreach={
|
1208
1317
|
item: {
|
1209
1318
|
"item": item,
|
@@ -1219,31 +1328,31 @@ class ForEachStage(BaseStage):
|
|
1219
1328
|
run_id=result.run_id,
|
1220
1329
|
parent_run_id=result.parent_run_id,
|
1221
1330
|
raise_error=True,
|
1331
|
+
event=event,
|
1222
1332
|
)
|
1223
1333
|
stage.set_outputs(rs.context, to=context)
|
1224
|
-
if rs.status == FAILED:
|
1225
|
-
error_msg: str = (
|
1226
|
-
f"Item-Stage was break because it has a sub stage, "
|
1227
|
-
f"{stage.iden}, failed without raise error."
|
1228
|
-
)
|
1229
|
-
return result.catch(
|
1230
|
-
status=FAILED,
|
1231
|
-
foreach={
|
1232
|
-
item: {
|
1233
|
-
"item": item,
|
1234
|
-
"stages": filter_func(
|
1235
|
-
context.pop("stages", {})
|
1236
|
-
),
|
1237
|
-
"errors": StageException(error_msg).to_dict(),
|
1238
|
-
},
|
1239
|
-
},
|
1240
|
-
)
|
1241
1334
|
except (StageException, UtilException) as e:
|
1242
1335
|
result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
|
1243
1336
|
raise StageException(
|
1244
1337
|
f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
|
1245
1338
|
) from None
|
1246
1339
|
|
1340
|
+
if rs.status == FAILED:
|
1341
|
+
error_msg: str = (
|
1342
|
+
f"Item-Stage was break because it has a sub stage, "
|
1343
|
+
f"{stage.iden}, failed without raise error."
|
1344
|
+
)
|
1345
|
+
return result.catch(
|
1346
|
+
status=FAILED,
|
1347
|
+
foreach={
|
1348
|
+
item: {
|
1349
|
+
"item": item,
|
1350
|
+
"stages": filter_func(context.pop("stages", {})),
|
1351
|
+
"errors": StageException(error_msg).to_dict(),
|
1352
|
+
},
|
1353
|
+
},
|
1354
|
+
)
|
1355
|
+
|
1247
1356
|
return result.catch(
|
1248
1357
|
status=SUCCESS,
|
1249
1358
|
foreach={
|
@@ -1291,7 +1400,7 @@ class ForEachStage(BaseStage):
|
|
1291
1400
|
result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
|
1292
1401
|
if event and event.is_set(): # pragma: no cov
|
1293
1402
|
return result.catch(
|
1294
|
-
status=
|
1403
|
+
status=CANCEL,
|
1295
1404
|
context={
|
1296
1405
|
"errors": StageException(
|
1297
1406
|
"Stage was canceled from event that had set "
|
@@ -1310,6 +1419,7 @@ class ForEachStage(BaseStage):
|
|
1310
1419
|
item=item,
|
1311
1420
|
params=params,
|
1312
1421
|
result=result,
|
1422
|
+
event=event,
|
1313
1423
|
)
|
1314
1424
|
for item in foreach
|
1315
1425
|
]
|
@@ -1334,7 +1444,7 @@ class ForEachStage(BaseStage):
|
|
1334
1444
|
except StageException as e:
|
1335
1445
|
status = FAILED
|
1336
1446
|
result.trace.error(
|
1337
|
-
f"[STAGE]:
|
1447
|
+
f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
|
1338
1448
|
)
|
1339
1449
|
context.update({"errors": e.to_dict()})
|
1340
1450
|
|
@@ -1358,7 +1468,12 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1358
1468
|
... }
|
1359
1469
|
"""
|
1360
1470
|
|
1361
|
-
item: Union[str, int, bool] = Field(
|
1471
|
+
item: Union[str, int, bool] = Field(
|
1472
|
+
default=0,
|
1473
|
+
description=(
|
1474
|
+
"An initial value that can be any value in str, int, or bool type."
|
1475
|
+
),
|
1476
|
+
)
|
1362
1477
|
until: str = Field(description="A until condition.")
|
1363
1478
|
stages: list[Stage] = Field(
|
1364
1479
|
default_factory=list,
|
@@ -1367,11 +1482,12 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1367
1482
|
"correct."
|
1368
1483
|
),
|
1369
1484
|
)
|
1370
|
-
|
1485
|
+
max_loop: int = Field(
|
1371
1486
|
default=10,
|
1372
1487
|
ge=1,
|
1373
1488
|
lt=100,
|
1374
1489
|
description="The maximum value of loop for this until stage.",
|
1490
|
+
alias="max-loop",
|
1375
1491
|
)
|
1376
1492
|
|
1377
1493
|
def execute_item(
|
@@ -1396,7 +1512,7 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1396
1512
|
|
1397
1513
|
:rtype: tuple[Result, T]
|
1398
1514
|
"""
|
1399
|
-
result.trace.debug(f"
|
1515
|
+
result.trace.debug(f"... Execute until item: {item!r}")
|
1400
1516
|
context: DictData = copy.deepcopy(params)
|
1401
1517
|
context.update({"loop": loop, "item": item, "stages": {}})
|
1402
1518
|
next_item: T = None
|
@@ -1406,7 +1522,7 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1406
1522
|
stage.extras = self.extras
|
1407
1523
|
|
1408
1524
|
if stage.is_skipped(params=context):
|
1409
|
-
result.trace.info(f"
|
1525
|
+
result.trace.info(f"... Skip stage: {stage.iden!r}")
|
1410
1526
|
stage.set_outputs(output={"skipped": True}, to=context)
|
1411
1527
|
continue
|
1412
1528
|
|
@@ -1417,7 +1533,7 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1417
1533
|
)
|
1418
1534
|
return (
|
1419
1535
|
result.catch(
|
1420
|
-
status=
|
1536
|
+
status=CANCEL,
|
1421
1537
|
until={
|
1422
1538
|
loop: {
|
1423
1539
|
"loop": loop,
|
@@ -1438,6 +1554,7 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1438
1554
|
run_id=result.run_id,
|
1439
1555
|
parent_run_id=result.parent_run_id,
|
1440
1556
|
raise_error=True,
|
1557
|
+
event=event,
|
1441
1558
|
)
|
1442
1559
|
stage.set_outputs(rs.context, to=context)
|
1443
1560
|
if "item" in (
|
@@ -1486,6 +1603,7 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1486
1603
|
result: Result = Result(
|
1487
1604
|
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
1488
1605
|
)
|
1606
|
+
|
1489
1607
|
result.trace.info(f"[STAGE]: Until-Execution: {self.until}")
|
1490
1608
|
item: Union[str, int, bool] = param2template(
|
1491
1609
|
self.item, params, extras=self.extras
|
@@ -1494,11 +1612,11 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1494
1612
|
track: bool = True
|
1495
1613
|
exceed_loop: bool = False
|
1496
1614
|
result.catch(status=WAIT, context={"until": {}})
|
1497
|
-
while track and not (exceed_loop := loop >= self.
|
1615
|
+
while track and not (exceed_loop := loop >= self.max_loop):
|
1498
1616
|
|
1499
1617
|
if event and event.is_set():
|
1500
1618
|
return result.catch(
|
1501
|
-
status=
|
1619
|
+
status=CANCEL,
|
1502
1620
|
context={
|
1503
1621
|
"errors": StageException(
|
1504
1622
|
"Stage was canceled from event that had set "
|
@@ -1512,6 +1630,7 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1512
1630
|
loop=loop,
|
1513
1631
|
params=params,
|
1514
1632
|
result=result,
|
1633
|
+
event=event,
|
1515
1634
|
)
|
1516
1635
|
|
1517
1636
|
loop += 1
|
@@ -1536,11 +1655,12 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1536
1655
|
"Return type of until condition does not be boolean, it"
|
1537
1656
|
f"return: {next_track!r}"
|
1538
1657
|
)
|
1539
|
-
track = not next_track
|
1658
|
+
track: bool = not next_track
|
1659
|
+
delay(0.025)
|
1540
1660
|
|
1541
1661
|
if exceed_loop:
|
1542
1662
|
raise StageException(
|
1543
|
-
f"The until loop was exceed {self.
|
1663
|
+
f"The until loop was exceed {self.max_loop} loops"
|
1544
1664
|
)
|
1545
1665
|
return result.catch(status=SUCCESS)
|
1546
1666
|
|
@@ -1552,7 +1672,7 @@ class Match(BaseModel):
|
|
1552
1672
|
stage: Stage = Field(description="A stage to execution for this case.")
|
1553
1673
|
|
1554
1674
|
|
1555
|
-
class CaseStage(BaseStage):
|
1675
|
+
class CaseStage(BaseStage):
|
1556
1676
|
"""Case execution stage.
|
1557
1677
|
|
1558
1678
|
Data Validate:
|
@@ -1590,6 +1710,14 @@ class CaseStage(BaseStage): # pragma: no cov
|
|
1590
1710
|
match: list[Match] = Field(
|
1591
1711
|
description="A list of Match model that should not be an empty list.",
|
1592
1712
|
)
|
1713
|
+
skip_not_match: bool = Field(
|
1714
|
+
default=False,
|
1715
|
+
description=(
|
1716
|
+
"A flag for making skip if it does not match and else condition "
|
1717
|
+
"does not set too."
|
1718
|
+
),
|
1719
|
+
alias="skip-not-match",
|
1720
|
+
)
|
1593
1721
|
|
1594
1722
|
def execute(
|
1595
1723
|
self,
|
@@ -1610,53 +1738,73 @@ class CaseStage(BaseStage): # pragma: no cov
|
|
1610
1738
|
"""
|
1611
1739
|
if result is None: # pragma: no cov
|
1612
1740
|
result: Result = Result(
|
1613
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
1741
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1742
|
+
extras=self.extras,
|
1614
1743
|
)
|
1615
1744
|
|
1616
|
-
_case = param2template(
|
1745
|
+
_case: Optional[str] = param2template(
|
1746
|
+
self.case, params, extras=self.extras
|
1747
|
+
)
|
1617
1748
|
|
1618
1749
|
result.trace.info(f"[STAGE]: Case-Execute: {_case!r}.")
|
1619
|
-
_else = None
|
1750
|
+
_else: Optional[Match] = None
|
1620
1751
|
stage: Optional[Stage] = None
|
1621
|
-
context = {}
|
1622
|
-
status = SUCCESS
|
1623
1752
|
for match in self.match:
|
1624
|
-
if (c := match.case)
|
1625
|
-
|
1626
|
-
else:
|
1627
|
-
_else = match
|
1753
|
+
if (c := match.case) == "_":
|
1754
|
+
_else: Match = match
|
1628
1755
|
continue
|
1629
1756
|
|
1757
|
+
_condition: str = param2template(c, params, extras=self.extras)
|
1630
1758
|
if stage is None and _case == _condition:
|
1631
1759
|
stage: Stage = match.stage
|
1632
1760
|
|
1633
1761
|
if stage is None:
|
1634
1762
|
if _else is None:
|
1635
|
-
|
1636
|
-
|
1637
|
-
|
1763
|
+
if not self.skip_not_match:
|
1764
|
+
raise StageException(
|
1765
|
+
"This stage does not set else for support not match "
|
1766
|
+
"any case."
|
1767
|
+
)
|
1768
|
+
result.trace.info(
|
1769
|
+
"... Skip this stage because it does not match."
|
1770
|
+
)
|
1771
|
+
error_msg: str = (
|
1772
|
+
"Case-Stage was canceled because it does not match any "
|
1773
|
+
"case and else condition does not set too."
|
1774
|
+
)
|
1775
|
+
return result.catch(
|
1776
|
+
status=CANCEL,
|
1777
|
+
context={"errors": StageException(error_msg).to_dict()},
|
1638
1778
|
)
|
1639
|
-
|
1640
1779
|
stage: Stage = _else.stage
|
1641
1780
|
|
1642
1781
|
if self.extras:
|
1643
1782
|
stage.extras = self.extras
|
1644
1783
|
|
1784
|
+
if event and event.is_set(): # pragma: no cov
|
1785
|
+
return result.catch(
|
1786
|
+
status=CANCEL,
|
1787
|
+
context={
|
1788
|
+
"errors": StageException(
|
1789
|
+
"Stage was canceled from event that had set before "
|
1790
|
+
"case-stage execution."
|
1791
|
+
).to_dict()
|
1792
|
+
},
|
1793
|
+
)
|
1794
|
+
|
1645
1795
|
try:
|
1646
|
-
|
1647
|
-
|
1796
|
+
return result.catch(
|
1797
|
+
status=SUCCESS,
|
1798
|
+
context=stage.handler_execute(
|
1648
1799
|
params=params,
|
1649
1800
|
run_id=result.run_id,
|
1650
1801
|
parent_run_id=result.parent_run_id,
|
1651
|
-
|
1802
|
+
event=event,
|
1803
|
+
).context,
|
1652
1804
|
)
|
1653
1805
|
except StageException as e: # pragma: no cov
|
1654
|
-
|
1655
|
-
result.
|
1656
|
-
f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
|
1657
|
-
)
|
1658
|
-
context.update({"errors": e.to_dict()})
|
1659
|
-
return result.catch(status=status, context=context)
|
1806
|
+
result.trace.error(f"[STAGE]: {e.__class__.__name__}:" f"\n\t{e}")
|
1807
|
+
return result.catch(status=FAILED, context={"errors": e.to_dict()})
|
1660
1808
|
|
1661
1809
|
|
1662
1810
|
class RaiseStage(BaseStage): # pragma: no cov
|
@@ -1672,7 +1820,9 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1672
1820
|
"""
|
1673
1821
|
|
1674
1822
|
message: str = Field(
|
1675
|
-
description=
|
1823
|
+
description=(
|
1824
|
+
"An error message that want to raise with StageException class"
|
1825
|
+
),
|
1676
1826
|
alias="raise",
|
1677
1827
|
)
|
1678
1828
|
|
@@ -1693,7 +1843,8 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1693
1843
|
"""
|
1694
1844
|
if result is None: # pragma: no cov
|
1695
1845
|
result: Result = Result(
|
1696
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
1846
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1847
|
+
extras=self.extras,
|
1697
1848
|
)
|
1698
1849
|
message: str = param2template(self.message, params, extras=self.extras)
|
1699
1850
|
result.trace.info(f"[STAGE]: Raise-Execute: {message!r}.")
|
@@ -1728,10 +1879,15 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1728
1879
|
... "image": "image-name.pkg.com",
|
1729
1880
|
... "env": {
|
1730
1881
|
... "ENV": "dev",
|
1882
|
+
... "DEBUG": "true",
|
1731
1883
|
... },
|
1732
1884
|
... "volume": {
|
1733
1885
|
... "secrets": "/secrets",
|
1734
1886
|
... },
|
1887
|
+
... "auth": {
|
1888
|
+
... "username": "__json_key",
|
1889
|
+
... "password": "${GOOGLE_CREDENTIAL_JSON_STRING}",
|
1890
|
+
... },
|
1735
1891
|
... }
|
1736
1892
|
"""
|
1737
1893
|
|
@@ -1750,6 +1906,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1750
1906
|
|
1751
1907
|
def execute_task(
|
1752
1908
|
self,
|
1909
|
+
params: DictData,
|
1753
1910
|
result: Result,
|
1754
1911
|
):
|
1755
1912
|
from docker import DockerClient
|
@@ -1762,26 +1919,32 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1762
1919
|
resp = client.api.pull(
|
1763
1920
|
repository=f"{self.image}",
|
1764
1921
|
tag=self.tag,
|
1765
|
-
|
1766
|
-
# {
|
1767
|
-
# "username": "_json_key",
|
1768
|
-
# "password": credential-json-string,
|
1769
|
-
# }
|
1770
|
-
auth_config=self.auth,
|
1922
|
+
auth_config=param2template(self.auth, params, extras=self.extras),
|
1771
1923
|
stream=True,
|
1772
1924
|
decode=True,
|
1773
1925
|
)
|
1774
1926
|
for line in resp:
|
1775
1927
|
result.trace.info(f"... {line}")
|
1776
1928
|
|
1777
|
-
unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S}"
|
1929
|
+
unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
|
1778
1930
|
container = client.containers.run(
|
1779
1931
|
image=f"{self.image}:{self.tag}",
|
1780
1932
|
name=unique_image_name,
|
1781
1933
|
environment=self.env,
|
1782
1934
|
volumes=(
|
1783
|
-
{
|
1784
|
-
|
1935
|
+
{
|
1936
|
+
Path.cwd()
|
1937
|
+
/ f".docker.{result.run_id}.logs": {
|
1938
|
+
"bind": "/logs",
|
1939
|
+
"mode": "rw",
|
1940
|
+
},
|
1941
|
+
}
|
1942
|
+
| {
|
1943
|
+
Path.cwd() / source: {"bind": target, "mode": "rw"}
|
1944
|
+
for source, target in (
|
1945
|
+
volume.split(":", maxsplit=1) for volume in self.volume
|
1946
|
+
)
|
1947
|
+
}
|
1785
1948
|
),
|
1786
1949
|
detach=True,
|
1787
1950
|
)
|