ddeutil-workflow 0.0.50__py3-none-any.whl → 0.0.52__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +7 -56
- ddeutil/workflow/conf.py +2 -2
- ddeutil/workflow/cron.py +46 -20
- ddeutil/workflow/job.py +181 -100
- ddeutil/workflow/logs.py +22 -18
- ddeutil/workflow/params.py +56 -16
- ddeutil/workflow/reusables.py +4 -2
- ddeutil/workflow/scheduler.py +5 -1
- ddeutil/workflow/stages.py +320 -154
- ddeutil/workflow/workflow.py +83 -74
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.52.dist-info}/METADATA +69 -13
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.52.dist-info}/RECORD +16 -16
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.52.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.52.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.50.dist-info → ddeutil_workflow-0.0.52.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -56,9 +56,10 @@ from typing_extensions import Self
|
|
56
56
|
from .__types import DictData, DictStr, TupleStr
|
57
57
|
from .conf import dynamic
|
58
58
|
from .exceptions import StageException, UtilException, to_dict
|
59
|
-
from .result import FAILED, SUCCESS, WAIT, Result, Status
|
59
|
+
from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
|
60
60
|
from .reusables import TagFunc, extract_call, not_in_template, param2template
|
61
61
|
from .utils import (
|
62
|
+
delay,
|
62
63
|
filter_func,
|
63
64
|
gen_id,
|
64
65
|
make_exec,
|
@@ -72,7 +73,8 @@ class BaseStage(BaseModel, ABC):
|
|
72
73
|
metadata. If you want to implement any custom stage, you can use this class
|
73
74
|
to parent and implement ``self.execute()`` method only.
|
74
75
|
|
75
|
-
This class is the abstraction class for any stage
|
76
|
+
This class is the abstraction class for any stage model that want to
|
77
|
+
implement to workflow model.
|
76
78
|
"""
|
77
79
|
|
78
80
|
extras: DictData = Field(
|
@@ -226,7 +228,7 @@ class BaseStage(BaseModel, ABC):
|
|
226
228
|
and want to set on the `to` like;
|
227
229
|
|
228
230
|
... (i) output: {'foo': bar}
|
229
|
-
... (ii) to: {}
|
231
|
+
... (ii) to: {'stages': {}}
|
230
232
|
|
231
233
|
The result of the `to` argument will be;
|
232
234
|
|
@@ -234,11 +236,15 @@ class BaseStage(BaseModel, ABC):
|
|
234
236
|
'stages': {
|
235
237
|
'<stage-id>': {
|
236
238
|
'outputs': {'foo': 'bar'},
|
237
|
-
'skipped': False
|
239
|
+
'skipped': False,
|
238
240
|
}
|
239
241
|
}
|
240
242
|
}
|
241
243
|
|
244
|
+
Important:
|
245
|
+
This method is use for reconstruct the result context and transfer
|
246
|
+
to the `to` argument.
|
247
|
+
|
242
248
|
:param output: (DictData) An output data that want to extract to an
|
243
249
|
output key.
|
244
250
|
:param to: (DictData) A context data that want to add output result.
|
@@ -269,7 +275,11 @@ class BaseStage(BaseModel, ABC):
|
|
269
275
|
if "skipped" in output
|
270
276
|
else {}
|
271
277
|
)
|
272
|
-
to["stages"][_id] = {
|
278
|
+
to["stages"][_id] = {
|
279
|
+
"outputs": copy.deepcopy(output),
|
280
|
+
**skipping,
|
281
|
+
**errors,
|
282
|
+
}
|
273
283
|
return to
|
274
284
|
|
275
285
|
def get_outputs(self, outputs: DictData) -> DictData:
|
@@ -330,6 +340,7 @@ class BaseStage(BaseModel, ABC):
|
|
330
340
|
|
331
341
|
|
332
342
|
class BaseAsyncStage(BaseStage):
|
343
|
+
"""Base Async Stage model."""
|
333
344
|
|
334
345
|
@abstractmethod
|
335
346
|
def execute(
|
@@ -432,12 +443,13 @@ class EmptyStage(BaseAsyncStage):
|
|
432
443
|
|
433
444
|
echo: Optional[str] = Field(
|
434
445
|
default=None,
|
435
|
-
description="A string
|
446
|
+
description="A string message that want to show on the stdout.",
|
436
447
|
)
|
437
448
|
sleep: float = Field(
|
438
449
|
default=0,
|
439
|
-
description="A second value to sleep before start execution",
|
450
|
+
description="A second value to sleep before start execution.",
|
440
451
|
ge=0,
|
452
|
+
lt=1800,
|
441
453
|
)
|
442
454
|
|
443
455
|
def execute(
|
@@ -464,12 +476,23 @@ class EmptyStage(BaseAsyncStage):
|
|
464
476
|
:rtype: Result
|
465
477
|
"""
|
466
478
|
result: Result = result or Result(
|
467
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
479
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
480
|
+
extras=self.extras,
|
468
481
|
)
|
469
482
|
|
483
|
+
if not self.echo:
|
484
|
+
message: str = "..."
|
485
|
+
else:
|
486
|
+
message: str = param2template(
|
487
|
+
dedent(self.echo), params, extras=self.extras
|
488
|
+
)
|
489
|
+
if "\n" in self.echo:
|
490
|
+
message: str = "\n\t" + message.replace("\n", "\n\t").strip(
|
491
|
+
"\n"
|
492
|
+
)
|
493
|
+
|
470
494
|
result.trace.info(
|
471
|
-
f"[STAGE]: Empty-Execute: {self.name!r}: "
|
472
|
-
f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
|
495
|
+
f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
|
473
496
|
)
|
474
497
|
if self.sleep > 0:
|
475
498
|
if self.sleep > 5:
|
@@ -497,9 +520,10 @@ class EmptyStage(BaseAsyncStage):
|
|
497
520
|
|
498
521
|
:rtype: Result
|
499
522
|
"""
|
500
|
-
if result is None:
|
523
|
+
if result is None:
|
501
524
|
result: Result = Result(
|
502
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
525
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
526
|
+
extras=self.extras,
|
503
527
|
)
|
504
528
|
|
505
529
|
await result.trace.ainfo(
|
@@ -540,8 +564,8 @@ class BashStage(BaseStage):
|
|
540
564
|
env: DictStr = Field(
|
541
565
|
default_factory=dict,
|
542
566
|
description=(
|
543
|
-
"An environment
|
544
|
-
"
|
567
|
+
"An environment variables that set before start execute by adding "
|
568
|
+
"on the header of the `.sh` file."
|
545
569
|
),
|
546
570
|
)
|
547
571
|
|
@@ -603,7 +627,8 @@ class BashStage(BaseStage):
|
|
603
627
|
"""
|
604
628
|
if result is None: # pragma: no cov
|
605
629
|
result: Result = Result(
|
606
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
630
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
631
|
+
extras=self.extras,
|
607
632
|
)
|
608
633
|
|
609
634
|
bash: str = param2template(
|
@@ -616,7 +641,7 @@ class BashStage(BaseStage):
|
|
616
641
|
env=param2template(self.env, params, extras=self.extras),
|
617
642
|
run_id=result.run_id,
|
618
643
|
) as sh:
|
619
|
-
result.trace.debug(f"...
|
644
|
+
result.trace.debug(f"... Create `{sh[1]}` file.")
|
620
645
|
rs: CompletedProcess = subprocess.run(
|
621
646
|
sh, shell=False, capture_output=True, text=True
|
622
647
|
)
|
@@ -660,18 +685,20 @@ class PyStage(BaseStage):
|
|
660
685
|
"""
|
661
686
|
|
662
687
|
run: str = Field(
|
663
|
-
description="A Python string statement that want to run with exec
|
688
|
+
description="A Python string statement that want to run with `exec`.",
|
664
689
|
)
|
665
690
|
vars: DictData = Field(
|
666
691
|
default_factory=dict,
|
667
692
|
description=(
|
668
|
-
"A mapping
|
693
|
+
"A variable mapping that want to pass to globals parameter in the "
|
694
|
+
"`exec` func."
|
669
695
|
),
|
670
696
|
)
|
671
697
|
|
672
698
|
@staticmethod
|
673
699
|
def filter_locals(values: DictData) -> Iterator[str]:
|
674
|
-
"""Filter a locals
|
700
|
+
"""Filter a locals mapping values that be module, class, or
|
701
|
+
__annotations__.
|
675
702
|
|
676
703
|
:param values: (DictData) A locals values that want to filter.
|
677
704
|
|
@@ -728,7 +755,8 @@ class PyStage(BaseStage):
|
|
728
755
|
"""
|
729
756
|
if result is None: # pragma: no cov
|
730
757
|
result: Result = Result(
|
731
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
758
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
759
|
+
extras=self.extras,
|
732
760
|
)
|
733
761
|
|
734
762
|
lc: DictData = {}
|
@@ -741,11 +769,11 @@ class PyStage(BaseStage):
|
|
741
769
|
|
742
770
|
# NOTE: Start exec the run statement.
|
743
771
|
result.trace.info(f"[STAGE]: Py-Execute: {self.name}")
|
744
|
-
result.trace.warning(
|
745
|
-
|
746
|
-
|
747
|
-
)
|
748
|
-
|
772
|
+
# result.trace.warning(
|
773
|
+
# "[STAGE]: This stage allow use `eval` function, so, please "
|
774
|
+
# "check your statement be safe before execute."
|
775
|
+
# )
|
776
|
+
#
|
749
777
|
# WARNING: The exec build-in function is very dangerous. So, it
|
750
778
|
# should use the re module to validate exec-string before running.
|
751
779
|
exec(
|
@@ -811,11 +839,17 @@ class CallStage(BaseStage):
|
|
811
839
|
:param event: (Event) An event manager that use to track parent execute
|
812
840
|
was not force stopped.
|
813
841
|
|
842
|
+
:raise ValueError: If necessary arguments does not pass from the `args`
|
843
|
+
field.
|
844
|
+
:raise TypeError: If the result from the caller function does not by
|
845
|
+
a `dict` type.
|
846
|
+
|
814
847
|
:rtype: Result
|
815
848
|
"""
|
816
849
|
if result is None: # pragma: no cov
|
817
850
|
result: Result = Result(
|
818
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
851
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
852
|
+
extras=self.extras,
|
819
853
|
)
|
820
854
|
|
821
855
|
has_keyword: bool = False
|
@@ -890,9 +924,12 @@ class CallStage(BaseStage):
|
|
890
924
|
"""Parse Pydantic model from any dict data before parsing to target
|
891
925
|
caller function.
|
892
926
|
|
893
|
-
:param func:
|
894
|
-
:param args:
|
895
|
-
:param result: (Result)
|
927
|
+
:param func: A tag function that want to get typing.
|
928
|
+
:param args: An arguments before passing to this tag function.
|
929
|
+
:param result: (Result) A result object for keeping context and status
|
930
|
+
data.
|
931
|
+
|
932
|
+
:rtype: DictData
|
896
933
|
"""
|
897
934
|
try:
|
898
935
|
type_hints: dict[str, Any] = get_type_hints(func)
|
@@ -965,22 +1002,29 @@ class TriggerStage(BaseStage):
|
|
965
1002
|
|
966
1003
|
:rtype: Result
|
967
1004
|
"""
|
1005
|
+
from .exceptions import WorkflowException
|
968
1006
|
from .workflow import Workflow
|
969
1007
|
|
970
1008
|
if result is None:
|
971
1009
|
result: Result = Result(
|
972
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
1010
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1011
|
+
extras=self.extras,
|
973
1012
|
)
|
974
1013
|
|
975
1014
|
_trigger: str = param2template(self.trigger, params, extras=self.extras)
|
976
1015
|
result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
|
977
|
-
|
978
|
-
|
979
|
-
|
980
|
-
|
981
|
-
|
982
|
-
|
983
|
-
|
1016
|
+
try:
|
1017
|
+
rs: Result = Workflow.from_conf(
|
1018
|
+
name=_trigger,
|
1019
|
+
extras=self.extras | {"stage_raise_error": True},
|
1020
|
+
).execute(
|
1021
|
+
params=param2template(self.params, params, extras=self.extras),
|
1022
|
+
parent_run_id=result.run_id,
|
1023
|
+
event=event,
|
1024
|
+
)
|
1025
|
+
except WorkflowException as e:
|
1026
|
+
raise StageException("Trigger workflow stage was failed") from e
|
1027
|
+
|
984
1028
|
if rs.status == FAILED:
|
985
1029
|
err_msg: str | None = (
|
986
1030
|
f" with:\n{msg}"
|
@@ -1020,17 +1064,25 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1020
1064
|
"""
|
1021
1065
|
|
1022
1066
|
parallel: dict[str, list[Stage]] = Field(
|
1023
|
-
description="A mapping of parallel branch
|
1067
|
+
description="A mapping of parallel branch name and stages.",
|
1068
|
+
)
|
1069
|
+
max_workers: int = Field(
|
1070
|
+
default=2,
|
1071
|
+
ge=1,
|
1072
|
+
lt=20,
|
1073
|
+
description=(
|
1074
|
+
"The maximum thread pool worker size for execution parallel."
|
1075
|
+
),
|
1076
|
+
alias="max-workers",
|
1024
1077
|
)
|
1025
|
-
max_parallel_core: int = Field(default=2)
|
1026
1078
|
|
1027
|
-
|
1028
|
-
|
1079
|
+
def execute_task(
|
1080
|
+
self,
|
1029
1081
|
branch: str,
|
1030
1082
|
params: DictData,
|
1031
1083
|
result: Result,
|
1032
|
-
stages: list[Stage],
|
1033
1084
|
*,
|
1085
|
+
event: Event | None = None,
|
1034
1086
|
extras: DictData | None = None,
|
1035
1087
|
) -> DictData:
|
1036
1088
|
"""Task execution method for passing a branch to each thread.
|
@@ -1039,32 +1091,83 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1039
1091
|
:param params: A parameter data that want to use in this execution.
|
1040
1092
|
:param result: (Result) A result object for keeping context and status
|
1041
1093
|
data.
|
1042
|
-
:param
|
1043
|
-
|
1094
|
+
:param event: (Event) An event manager that use to track parent execute
|
1095
|
+
was not force stopped.
|
1096
|
+
:param extras: (DictData) An extra parameters that want to override
|
1097
|
+
config values.
|
1044
1098
|
|
1045
1099
|
:rtype: DictData
|
1046
1100
|
"""
|
1047
|
-
|
1048
|
-
|
1049
|
-
|
1101
|
+
result.trace.debug(f"... Execute branch: {branch!r}")
|
1102
|
+
_params: DictData = copy.deepcopy(params)
|
1103
|
+
_params.update({"branch": branch})
|
1104
|
+
context: DictData = {"branch": branch, "stages": {}}
|
1105
|
+
for stage in self.parallel[branch]:
|
1106
|
+
|
1050
1107
|
if extras:
|
1051
1108
|
stage.extras = extras
|
1052
1109
|
|
1110
|
+
if stage.is_skipped(params=_params):
|
1111
|
+
result.trace.info(f"... Skip stage: {stage.iden!r}")
|
1112
|
+
stage.set_outputs(output={"skipped": True}, to=context)
|
1113
|
+
continue
|
1114
|
+
|
1115
|
+
if event and event.is_set():
|
1116
|
+
error_msg: str = (
|
1117
|
+
"Branch-Stage was canceled from event that had set before "
|
1118
|
+
"stage item execution."
|
1119
|
+
)
|
1120
|
+
return result.catch(
|
1121
|
+
status=CANCEL,
|
1122
|
+
parallel={
|
1123
|
+
branch: {
|
1124
|
+
"branch": branch,
|
1125
|
+
"stages": filter_func(context.pop("stages", {})),
|
1126
|
+
"errors": StageException(error_msg).to_dict(),
|
1127
|
+
}
|
1128
|
+
},
|
1129
|
+
)
|
1130
|
+
|
1053
1131
|
try:
|
1054
|
-
stage.
|
1055
|
-
|
1056
|
-
|
1057
|
-
|
1058
|
-
|
1059
|
-
|
1060
|
-
to=context,
|
1132
|
+
rs: Result = stage.handler_execute(
|
1133
|
+
params=_params,
|
1134
|
+
run_id=result.run_id,
|
1135
|
+
parent_run_id=result.parent_run_id,
|
1136
|
+
raise_error=True,
|
1137
|
+
event=event,
|
1061
1138
|
)
|
1139
|
+
stage.set_outputs(rs.context, to=context)
|
1062
1140
|
except StageException as e: # pragma: no cov
|
1063
|
-
result.trace.error(
|
1064
|
-
|
1141
|
+
result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
|
1142
|
+
raise StageException(
|
1143
|
+
f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
|
1144
|
+
) from None
|
1145
|
+
|
1146
|
+
if rs.status == FAILED:
|
1147
|
+
error_msg: str = (
|
1148
|
+
f"Item-Stage was break because it has a sub stage, "
|
1149
|
+
f"{stage.iden}, failed without raise error."
|
1065
1150
|
)
|
1066
|
-
|
1067
|
-
|
1151
|
+
return result.catch(
|
1152
|
+
status=FAILED,
|
1153
|
+
parallel={
|
1154
|
+
branch: {
|
1155
|
+
"branch": branch,
|
1156
|
+
"stages": filter_func(context.pop("stages", {})),
|
1157
|
+
"errors": StageException(error_msg).to_dict(),
|
1158
|
+
},
|
1159
|
+
},
|
1160
|
+
)
|
1161
|
+
|
1162
|
+
return result.catch(
|
1163
|
+
status=SUCCESS,
|
1164
|
+
parallel={
|
1165
|
+
branch: {
|
1166
|
+
"branch": branch,
|
1167
|
+
"stages": filter_func(context.pop("stages", {})),
|
1168
|
+
},
|
1169
|
+
},
|
1170
|
+
)
|
1068
1171
|
|
1069
1172
|
def execute(
|
1070
1173
|
self,
|
@@ -1086,41 +1189,46 @@ class ParallelStage(BaseStage): # pragma: no cov
|
|
1086
1189
|
"""
|
1087
1190
|
if result is None: # pragma: no cov
|
1088
1191
|
result: Result = Result(
|
1089
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
1192
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1193
|
+
extras=self.extras,
|
1090
1194
|
)
|
1091
|
-
|
1195
|
+
event: Event = Event() if event is None else event
|
1092
1196
|
result.trace.info(
|
1093
|
-
f"[STAGE]: Parallel-Execute
|
1197
|
+
f"[STAGE]: Parallel-Execute: {self.max_workers} workers."
|
1094
1198
|
)
|
1095
|
-
|
1096
|
-
status = SUCCESS
|
1199
|
+
result.catch(status=WAIT, context={"parallel": {}})
|
1097
1200
|
with ThreadPoolExecutor(
|
1098
|
-
max_workers=self.
|
1201
|
+
max_workers=self.max_workers,
|
1099
1202
|
thread_name_prefix="parallel_stage_exec_",
|
1100
1203
|
) as executor:
|
1101
1204
|
|
1102
|
-
|
1103
|
-
|
1104
|
-
|
1105
|
-
|
1106
|
-
|
1107
|
-
|
1108
|
-
|
1109
|
-
|
1110
|
-
|
1111
|
-
|
1112
|
-
|
1205
|
+
context: DictData = {}
|
1206
|
+
status: Status = SUCCESS
|
1207
|
+
|
1208
|
+
futures: list[Future] = (
|
1209
|
+
executor.submit(
|
1210
|
+
self.execute_task,
|
1211
|
+
branch=branch,
|
1212
|
+
params=params,
|
1213
|
+
result=result,
|
1214
|
+
event=event,
|
1215
|
+
extras=self.extras,
|
1113
1216
|
)
|
1217
|
+
for branch in self.parallel
|
1218
|
+
)
|
1114
1219
|
|
1115
1220
|
done = as_completed(futures, timeout=1800)
|
1116
1221
|
for future in done:
|
1117
|
-
|
1118
|
-
|
1119
|
-
|
1120
|
-
if "errors" in context:
|
1222
|
+
try:
|
1223
|
+
future.result()
|
1224
|
+
except StageException as e:
|
1121
1225
|
status = FAILED
|
1226
|
+
result.trace.error(
|
1227
|
+
f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
|
1228
|
+
)
|
1229
|
+
context.update({"errors": e.to_dict()})
|
1122
1230
|
|
1123
|
-
return result.catch(status=status, context=
|
1231
|
+
return result.catch(status=status, context=context)
|
1124
1232
|
|
1125
1233
|
|
1126
1234
|
class ForEachStage(BaseStage):
|
@@ -1182,28 +1290,31 @@ class ForEachStage(BaseStage):
|
|
1182
1290
|
:param event: (Event) An event manager that use to track parent execute
|
1183
1291
|
was not force stopped.
|
1184
1292
|
|
1293
|
+
:raise StageException: If the stage execution raise errors.
|
1294
|
+
|
1185
1295
|
:rtype: Result
|
1186
1296
|
"""
|
1187
|
-
result.trace.debug(f"
|
1188
|
-
|
1189
|
-
|
1297
|
+
result.trace.debug(f"... Execute item: {item!r}")
|
1298
|
+
_params: DictData = copy.deepcopy(params)
|
1299
|
+
_params.update({"item": item})
|
1300
|
+
context: DictData = {"item": item, "stages": {}}
|
1190
1301
|
for stage in self.stages:
|
1191
1302
|
|
1192
1303
|
if self.extras:
|
1193
1304
|
stage.extras = self.extras
|
1194
1305
|
|
1195
|
-
if stage.is_skipped(params=
|
1196
|
-
result.trace.info(f"
|
1306
|
+
if stage.is_skipped(params=_params):
|
1307
|
+
result.trace.info(f"... Skip stage: {stage.iden!r}")
|
1197
1308
|
stage.set_outputs(output={"skipped": True}, to=context)
|
1198
1309
|
continue
|
1199
1310
|
|
1200
|
-
if event and event.is_set():
|
1311
|
+
if event and event.is_set(): # pragma: no cov
|
1201
1312
|
error_msg: str = (
|
1202
1313
|
"Item-Stage was canceled from event that had set before "
|
1203
1314
|
"stage item execution."
|
1204
1315
|
)
|
1205
1316
|
return result.catch(
|
1206
|
-
status=
|
1317
|
+
status=CANCEL,
|
1207
1318
|
foreach={
|
1208
1319
|
item: {
|
1209
1320
|
"item": item,
|
@@ -1215,35 +1326,35 @@ class ForEachStage(BaseStage):
|
|
1215
1326
|
|
1216
1327
|
try:
|
1217
1328
|
rs: Result = stage.handler_execute(
|
1218
|
-
params=
|
1329
|
+
params=_params,
|
1219
1330
|
run_id=result.run_id,
|
1220
1331
|
parent_run_id=result.parent_run_id,
|
1221
1332
|
raise_error=True,
|
1333
|
+
event=event,
|
1222
1334
|
)
|
1223
1335
|
stage.set_outputs(rs.context, to=context)
|
1224
|
-
if rs.status == FAILED:
|
1225
|
-
error_msg: str = (
|
1226
|
-
f"Item-Stage was break because it has a sub stage, "
|
1227
|
-
f"{stage.iden}, failed without raise error."
|
1228
|
-
)
|
1229
|
-
return result.catch(
|
1230
|
-
status=FAILED,
|
1231
|
-
foreach={
|
1232
|
-
item: {
|
1233
|
-
"item": item,
|
1234
|
-
"stages": filter_func(
|
1235
|
-
context.pop("stages", {})
|
1236
|
-
),
|
1237
|
-
"errors": StageException(error_msg).to_dict(),
|
1238
|
-
},
|
1239
|
-
},
|
1240
|
-
)
|
1241
1336
|
except (StageException, UtilException) as e:
|
1242
1337
|
result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
|
1243
1338
|
raise StageException(
|
1244
1339
|
f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
|
1245
1340
|
) from None
|
1246
1341
|
|
1342
|
+
if rs.status == FAILED:
|
1343
|
+
error_msg: str = (
|
1344
|
+
f"Item-Stage was break because it has a sub stage, "
|
1345
|
+
f"{stage.iden}, failed without raise error."
|
1346
|
+
)
|
1347
|
+
return result.catch(
|
1348
|
+
status=FAILED,
|
1349
|
+
foreach={
|
1350
|
+
item: {
|
1351
|
+
"item": item,
|
1352
|
+
"stages": filter_func(context.pop("stages", {})),
|
1353
|
+
"errors": StageException(error_msg).to_dict(),
|
1354
|
+
},
|
1355
|
+
},
|
1356
|
+
)
|
1357
|
+
|
1247
1358
|
return result.catch(
|
1248
1359
|
status=SUCCESS,
|
1249
1360
|
foreach={
|
@@ -1291,7 +1402,7 @@ class ForEachStage(BaseStage):
|
|
1291
1402
|
result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
|
1292
1403
|
if event and event.is_set(): # pragma: no cov
|
1293
1404
|
return result.catch(
|
1294
|
-
status=
|
1405
|
+
status=CANCEL,
|
1295
1406
|
context={
|
1296
1407
|
"errors": StageException(
|
1297
1408
|
"Stage was canceled from event that had set "
|
@@ -1310,6 +1421,7 @@ class ForEachStage(BaseStage):
|
|
1310
1421
|
item=item,
|
1311
1422
|
params=params,
|
1312
1423
|
result=result,
|
1424
|
+
event=event,
|
1313
1425
|
)
|
1314
1426
|
for item in foreach
|
1315
1427
|
]
|
@@ -1334,7 +1446,7 @@ class ForEachStage(BaseStage):
|
|
1334
1446
|
except StageException as e:
|
1335
1447
|
status = FAILED
|
1336
1448
|
result.trace.error(
|
1337
|
-
f"[STAGE]:
|
1449
|
+
f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
|
1338
1450
|
)
|
1339
1451
|
context.update({"errors": e.to_dict()})
|
1340
1452
|
|
@@ -1358,7 +1470,12 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1358
1470
|
... }
|
1359
1471
|
"""
|
1360
1472
|
|
1361
|
-
item: Union[str, int, bool] = Field(
|
1473
|
+
item: Union[str, int, bool] = Field(
|
1474
|
+
default=0,
|
1475
|
+
description=(
|
1476
|
+
"An initial value that can be any value in str, int, or bool type."
|
1477
|
+
),
|
1478
|
+
)
|
1362
1479
|
until: str = Field(description="A until condition.")
|
1363
1480
|
stages: list[Stage] = Field(
|
1364
1481
|
default_factory=list,
|
@@ -1367,11 +1484,12 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1367
1484
|
"correct."
|
1368
1485
|
),
|
1369
1486
|
)
|
1370
|
-
|
1487
|
+
max_loop: int = Field(
|
1371
1488
|
default=10,
|
1372
1489
|
ge=1,
|
1373
1490
|
lt=100,
|
1374
1491
|
description="The maximum value of loop for this until stage.",
|
1492
|
+
alias="max-loop",
|
1375
1493
|
)
|
1376
1494
|
|
1377
1495
|
def execute_item(
|
@@ -1396,17 +1514,18 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1396
1514
|
|
1397
1515
|
:rtype: tuple[Result, T]
|
1398
1516
|
"""
|
1399
|
-
result.trace.debug(f"
|
1400
|
-
|
1401
|
-
|
1517
|
+
result.trace.debug(f"... Execute until item: {item!r}")
|
1518
|
+
_params: DictData = copy.deepcopy(params)
|
1519
|
+
_params.update({"item": item})
|
1520
|
+
context: DictData = {"loop": loop, "item": item, "stages": {}}
|
1402
1521
|
next_item: T = None
|
1403
1522
|
for stage in self.stages:
|
1404
1523
|
|
1405
1524
|
if self.extras:
|
1406
1525
|
stage.extras = self.extras
|
1407
1526
|
|
1408
|
-
if stage.is_skipped(params=
|
1409
|
-
result.trace.info(f"
|
1527
|
+
if stage.is_skipped(params=_params):
|
1528
|
+
result.trace.info(f"... Skip stage: {stage.iden!r}")
|
1410
1529
|
stage.set_outputs(output={"skipped": True}, to=context)
|
1411
1530
|
continue
|
1412
1531
|
|
@@ -1417,7 +1536,7 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1417
1536
|
)
|
1418
1537
|
return (
|
1419
1538
|
result.catch(
|
1420
|
-
status=
|
1539
|
+
status=CANCEL,
|
1421
1540
|
until={
|
1422
1541
|
loop: {
|
1423
1542
|
"loop": loop,
|
@@ -1434,10 +1553,11 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1434
1553
|
|
1435
1554
|
try:
|
1436
1555
|
rs: Result = stage.handler_execute(
|
1437
|
-
params=
|
1556
|
+
params=_params,
|
1438
1557
|
run_id=result.run_id,
|
1439
1558
|
parent_run_id=result.parent_run_id,
|
1440
1559
|
raise_error=True,
|
1560
|
+
event=event,
|
1441
1561
|
)
|
1442
1562
|
stage.set_outputs(rs.context, to=context)
|
1443
1563
|
if "item" in (
|
@@ -1486,6 +1606,7 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1486
1606
|
result: Result = Result(
|
1487
1607
|
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
1488
1608
|
)
|
1609
|
+
|
1489
1610
|
result.trace.info(f"[STAGE]: Until-Execution: {self.until}")
|
1490
1611
|
item: Union[str, int, bool] = param2template(
|
1491
1612
|
self.item, params, extras=self.extras
|
@@ -1494,11 +1615,11 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1494
1615
|
track: bool = True
|
1495
1616
|
exceed_loop: bool = False
|
1496
1617
|
result.catch(status=WAIT, context={"until": {}})
|
1497
|
-
while track and not (exceed_loop := loop >= self.
|
1618
|
+
while track and not (exceed_loop := loop >= self.max_loop):
|
1498
1619
|
|
1499
1620
|
if event and event.is_set():
|
1500
1621
|
return result.catch(
|
1501
|
-
status=
|
1622
|
+
status=CANCEL,
|
1502
1623
|
context={
|
1503
1624
|
"errors": StageException(
|
1504
1625
|
"Stage was canceled from event that had set "
|
@@ -1512,6 +1633,7 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1512
1633
|
loop=loop,
|
1513
1634
|
params=params,
|
1514
1635
|
result=result,
|
1636
|
+
event=event,
|
1515
1637
|
)
|
1516
1638
|
|
1517
1639
|
loop += 1
|
@@ -1536,11 +1658,12 @@ class UntilStage(BaseStage): # pragma: no cov
|
|
1536
1658
|
"Return type of until condition does not be boolean, it"
|
1537
1659
|
f"return: {next_track!r}"
|
1538
1660
|
)
|
1539
|
-
track = not next_track
|
1661
|
+
track: bool = not next_track
|
1662
|
+
delay(0.025)
|
1540
1663
|
|
1541
1664
|
if exceed_loop:
|
1542
1665
|
raise StageException(
|
1543
|
-
f"The until loop was exceed {self.
|
1666
|
+
f"The until loop was exceed {self.max_loop} loops"
|
1544
1667
|
)
|
1545
1668
|
return result.catch(status=SUCCESS)
|
1546
1669
|
|
@@ -1552,7 +1675,7 @@ class Match(BaseModel):
|
|
1552
1675
|
stage: Stage = Field(description="A stage to execution for this case.")
|
1553
1676
|
|
1554
1677
|
|
1555
|
-
class CaseStage(BaseStage):
|
1678
|
+
class CaseStage(BaseStage):
|
1556
1679
|
"""Case execution stage.
|
1557
1680
|
|
1558
1681
|
Data Validate:
|
@@ -1590,6 +1713,14 @@ class CaseStage(BaseStage): # pragma: no cov
|
|
1590
1713
|
match: list[Match] = Field(
|
1591
1714
|
description="A list of Match model that should not be an empty list.",
|
1592
1715
|
)
|
1716
|
+
skip_not_match: bool = Field(
|
1717
|
+
default=False,
|
1718
|
+
description=(
|
1719
|
+
"A flag for making skip if it does not match and else condition "
|
1720
|
+
"does not set too."
|
1721
|
+
),
|
1722
|
+
alias="skip-not-match",
|
1723
|
+
)
|
1593
1724
|
|
1594
1725
|
def execute(
|
1595
1726
|
self,
|
@@ -1610,53 +1741,73 @@ class CaseStage(BaseStage): # pragma: no cov
|
|
1610
1741
|
"""
|
1611
1742
|
if result is None: # pragma: no cov
|
1612
1743
|
result: Result = Result(
|
1613
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
1744
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1745
|
+
extras=self.extras,
|
1614
1746
|
)
|
1615
1747
|
|
1616
|
-
_case = param2template(
|
1748
|
+
_case: Optional[str] = param2template(
|
1749
|
+
self.case, params, extras=self.extras
|
1750
|
+
)
|
1617
1751
|
|
1618
1752
|
result.trace.info(f"[STAGE]: Case-Execute: {_case!r}.")
|
1619
|
-
_else = None
|
1753
|
+
_else: Optional[Match] = None
|
1620
1754
|
stage: Optional[Stage] = None
|
1621
|
-
context = {}
|
1622
|
-
status = SUCCESS
|
1623
1755
|
for match in self.match:
|
1624
|
-
if (c := match.case)
|
1625
|
-
|
1626
|
-
else:
|
1627
|
-
_else = match
|
1756
|
+
if (c := match.case) == "_":
|
1757
|
+
_else: Match = match
|
1628
1758
|
continue
|
1629
1759
|
|
1760
|
+
_condition: str = param2template(c, params, extras=self.extras)
|
1630
1761
|
if stage is None and _case == _condition:
|
1631
1762
|
stage: Stage = match.stage
|
1632
1763
|
|
1633
1764
|
if stage is None:
|
1634
1765
|
if _else is None:
|
1635
|
-
|
1636
|
-
|
1637
|
-
|
1766
|
+
if not self.skip_not_match:
|
1767
|
+
raise StageException(
|
1768
|
+
"This stage does not set else for support not match "
|
1769
|
+
"any case."
|
1770
|
+
)
|
1771
|
+
result.trace.info(
|
1772
|
+
"... Skip this stage because it does not match."
|
1773
|
+
)
|
1774
|
+
error_msg: str = (
|
1775
|
+
"Case-Stage was canceled because it does not match any "
|
1776
|
+
"case and else condition does not set too."
|
1777
|
+
)
|
1778
|
+
return result.catch(
|
1779
|
+
status=CANCEL,
|
1780
|
+
context={"errors": StageException(error_msg).to_dict()},
|
1638
1781
|
)
|
1639
|
-
|
1640
1782
|
stage: Stage = _else.stage
|
1641
1783
|
|
1642
1784
|
if self.extras:
|
1643
1785
|
stage.extras = self.extras
|
1644
1786
|
|
1787
|
+
if event and event.is_set(): # pragma: no cov
|
1788
|
+
return result.catch(
|
1789
|
+
status=CANCEL,
|
1790
|
+
context={
|
1791
|
+
"errors": StageException(
|
1792
|
+
"Stage was canceled from event that had set before "
|
1793
|
+
"case-stage execution."
|
1794
|
+
).to_dict()
|
1795
|
+
},
|
1796
|
+
)
|
1797
|
+
|
1645
1798
|
try:
|
1646
|
-
|
1647
|
-
|
1799
|
+
return result.catch(
|
1800
|
+
status=SUCCESS,
|
1801
|
+
context=stage.handler_execute(
|
1648
1802
|
params=params,
|
1649
1803
|
run_id=result.run_id,
|
1650
1804
|
parent_run_id=result.parent_run_id,
|
1651
|
-
|
1805
|
+
event=event,
|
1806
|
+
).context,
|
1652
1807
|
)
|
1653
1808
|
except StageException as e: # pragma: no cov
|
1654
|
-
|
1655
|
-
result.
|
1656
|
-
f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
|
1657
|
-
)
|
1658
|
-
context.update({"errors": e.to_dict()})
|
1659
|
-
return result.catch(status=status, context=context)
|
1809
|
+
result.trace.error(f"[STAGE]: {e.__class__.__name__}:" f"\n\t{e}")
|
1810
|
+
return result.catch(status=FAILED, context={"errors": e.to_dict()})
|
1660
1811
|
|
1661
1812
|
|
1662
1813
|
class RaiseStage(BaseStage): # pragma: no cov
|
@@ -1672,7 +1823,9 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1672
1823
|
"""
|
1673
1824
|
|
1674
1825
|
message: str = Field(
|
1675
|
-
description=
|
1826
|
+
description=(
|
1827
|
+
"An error message that want to raise with StageException class"
|
1828
|
+
),
|
1676
1829
|
alias="raise",
|
1677
1830
|
)
|
1678
1831
|
|
@@ -1693,7 +1846,8 @@ class RaiseStage(BaseStage): # pragma: no cov
|
|
1693
1846
|
"""
|
1694
1847
|
if result is None: # pragma: no cov
|
1695
1848
|
result: Result = Result(
|
1696
|
-
run_id=gen_id(self.name + (self.id or ""), unique=True)
|
1849
|
+
run_id=gen_id(self.name + (self.id or ""), unique=True),
|
1850
|
+
extras=self.extras,
|
1697
1851
|
)
|
1698
1852
|
message: str = param2template(self.message, params, extras=self.extras)
|
1699
1853
|
result.trace.info(f"[STAGE]: Raise-Execute: {message!r}.")
|
@@ -1728,10 +1882,15 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1728
1882
|
... "image": "image-name.pkg.com",
|
1729
1883
|
... "env": {
|
1730
1884
|
... "ENV": "dev",
|
1885
|
+
... "DEBUG": "true",
|
1731
1886
|
... },
|
1732
1887
|
... "volume": {
|
1733
1888
|
... "secrets": "/secrets",
|
1734
1889
|
... },
|
1890
|
+
... "auth": {
|
1891
|
+
... "username": "__json_key",
|
1892
|
+
... "password": "${GOOGLE_CREDENTIAL_JSON_STRING}",
|
1893
|
+
... },
|
1735
1894
|
... }
|
1736
1895
|
"""
|
1737
1896
|
|
@@ -1750,6 +1909,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1750
1909
|
|
1751
1910
|
def execute_task(
|
1752
1911
|
self,
|
1912
|
+
params: DictData,
|
1753
1913
|
result: Result,
|
1754
1914
|
):
|
1755
1915
|
from docker import DockerClient
|
@@ -1762,26 +1922,32 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
1762
1922
|
resp = client.api.pull(
|
1763
1923
|
repository=f"{self.image}",
|
1764
1924
|
tag=self.tag,
|
1765
|
-
|
1766
|
-
# {
|
1767
|
-
# "username": "_json_key",
|
1768
|
-
# "password": credential-json-string,
|
1769
|
-
# }
|
1770
|
-
auth_config=self.auth,
|
1925
|
+
auth_config=param2template(self.auth, params, extras=self.extras),
|
1771
1926
|
stream=True,
|
1772
1927
|
decode=True,
|
1773
1928
|
)
|
1774
1929
|
for line in resp:
|
1775
1930
|
result.trace.info(f"... {line}")
|
1776
1931
|
|
1777
|
-
unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S}"
|
1932
|
+
unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
|
1778
1933
|
container = client.containers.run(
|
1779
1934
|
image=f"{self.image}:{self.tag}",
|
1780
1935
|
name=unique_image_name,
|
1781
1936
|
environment=self.env,
|
1782
1937
|
volumes=(
|
1783
|
-
{
|
1784
|
-
|
1938
|
+
{
|
1939
|
+
Path.cwd()
|
1940
|
+
/ f".docker.{result.run_id}.logs": {
|
1941
|
+
"bind": "/logs",
|
1942
|
+
"mode": "rw",
|
1943
|
+
},
|
1944
|
+
}
|
1945
|
+
| {
|
1946
|
+
Path.cwd() / source: {"bind": target, "mode": "rw"}
|
1947
|
+
for source, target in (
|
1948
|
+
volume.split(":", maxsplit=1) for volume in self.volume
|
1949
|
+
)
|
1950
|
+
}
|
1785
1951
|
),
|
1786
1952
|
detach=True,
|
1787
1953
|
)
|