ddeutil-workflow 0.0.83__py3-none-any.whl → 0.0.85__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ddeutil/workflow/__about__.py +1 -1
- ddeutil/workflow/__init__.py +4 -4
- ddeutil/workflow/audits.py +8 -6
- ddeutil/workflow/conf.py +4 -17
- ddeutil/workflow/errors.py +31 -19
- ddeutil/workflow/job.py +276 -156
- ddeutil/workflow/plugins/providers/az.py +2 -2
- ddeutil/workflow/stages.py +700 -330
- ddeutil/workflow/traces.py +125 -185
- ddeutil/workflow/utils.py +14 -3
- ddeutil/workflow/workflow.py +49 -37
- {ddeutil_workflow-0.0.83.dist-info → ddeutil_workflow-0.0.85.dist-info}/METADATA +13 -16
- {ddeutil_workflow-0.0.83.dist-info → ddeutil_workflow-0.0.85.dist-info}/RECORD +17 -17
- {ddeutil_workflow-0.0.83.dist-info → ddeutil_workflow-0.0.85.dist-info}/WHEEL +0 -0
- {ddeutil_workflow-0.0.83.dist-info → ddeutil_workflow-0.0.85.dist-info}/entry_points.txt +0 -0
- {ddeutil_workflow-0.0.83.dist-info → ddeutil_workflow-0.0.85.dist-info}/licenses/LICENSE +0 -0
- {ddeutil_workflow-0.0.83.dist-info → ddeutil_workflow-0.0.85.dist-info}/top_level.txt +0 -0
ddeutil/workflow/stages.py
CHANGED
@@ -138,10 +138,10 @@ class BaseStage(BaseModel, ABC):
|
|
138
138
|
implement, ensuring consistent behavior across different stage types.
|
139
139
|
|
140
140
|
This abstract class handles core stage functionality including:
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
141
|
+
- Stage identification and naming
|
142
|
+
- Conditional execution logic
|
143
|
+
- Output management and templating
|
144
|
+
- Execution lifecycle management
|
145
145
|
|
146
146
|
Custom stages should inherit from this class and implement the abstract
|
147
147
|
`process()` method to define their specific execution behavior.
|
@@ -162,7 +162,6 @@ class BaseStage(BaseModel, ABC):
|
|
162
162
|
...
|
163
163
|
... def process(self, params: DictData, **kwargs) -> Result:
|
164
164
|
... return Result(status=SUCCESS)
|
165
|
-
```
|
166
165
|
"""
|
167
166
|
|
168
167
|
action_stage: ClassVar[bool] = False
|
@@ -206,13 +205,13 @@ class BaseStage(BaseModel, ABC):
|
|
206
205
|
return self.id or self.name
|
207
206
|
|
208
207
|
@field_validator("desc", mode="after")
|
209
|
-
def ___prepare_desc__(cls, value: str) -> str:
|
208
|
+
def ___prepare_desc__(cls, value: Optional[str]) -> Optional[str]:
|
210
209
|
"""Prepare description string that was created on a template.
|
211
210
|
|
212
211
|
Returns:
|
213
212
|
str: A dedent and left strip newline of description string.
|
214
213
|
"""
|
215
|
-
return dedent(value.lstrip("\n"))
|
214
|
+
return value if value is None else dedent(value.lstrip("\n"))
|
216
215
|
|
217
216
|
@model_validator(mode="after")
|
218
217
|
def __prepare_running_id(self) -> Self:
|
@@ -240,7 +239,8 @@ class BaseStage(BaseModel, ABC):
|
|
240
239
|
|
241
240
|
Args:
|
242
241
|
value (Any): An any value.
|
243
|
-
params (DictData):
|
242
|
+
params (DictData): A parameter data that want to use in this
|
243
|
+
execution.
|
244
244
|
|
245
245
|
Returns:
|
246
246
|
Any: A templated value.
|
@@ -264,10 +264,11 @@ class BaseStage(BaseModel, ABC):
|
|
264
264
|
params (DictData): A parameter data that want to use in this
|
265
265
|
execution.
|
266
266
|
run_id (str): A running stage ID.
|
267
|
-
context (DictData): A context data
|
268
|
-
|
269
|
-
|
270
|
-
|
267
|
+
context (DictData): A context data that was passed from handler
|
268
|
+
method.
|
269
|
+
parent_run_id (str, default None): A parent running ID.
|
270
|
+
event (Event, default None): An event manager that use to track
|
271
|
+
parent process was not force stopped.
|
271
272
|
|
272
273
|
Returns:
|
273
274
|
Result: The execution result with status and context data.
|
@@ -308,10 +309,10 @@ class BaseStage(BaseModel, ABC):
|
|
308
309
|
object from the current stage ID before release the final result.
|
309
310
|
|
310
311
|
Args:
|
311
|
-
params: A parameter data.
|
312
|
-
run_id: A running
|
313
|
-
event: An event manager that pass to the stage
|
314
|
-
|
312
|
+
params (DictData): A parameter data.
|
313
|
+
run_id (str, default None): A running ID.
|
314
|
+
event (Event, default None): An event manager that pass to the stage
|
315
|
+
execution.
|
315
316
|
|
316
317
|
Returns:
|
317
318
|
Result: The execution result with updated status and context.
|
@@ -370,28 +371,26 @@ class BaseStage(BaseModel, ABC):
|
|
370
371
|
StageNestedError,
|
371
372
|
StageError,
|
372
373
|
) as e: # pragma: no cov
|
374
|
+
updated: Optional[DictData] = {"errors": e.to_dict()}
|
373
375
|
if isinstance(e, StageNestedError):
|
374
|
-
trace.
|
376
|
+
trace.error(f"[STAGE]: Nested: {e}")
|
375
377
|
elif isinstance(e, (StageSkipError, StageNestedSkipError)):
|
376
|
-
trace.
|
377
|
-
|
378
|
-
|
378
|
+
trace.error(f"[STAGE]: ⏭️ Skip: {e}")
|
379
|
+
updated = None
|
380
|
+
elif e.allow_traceback:
|
381
|
+
trace.error(
|
379
382
|
f"[STAGE]: Stage Failed:||🚨 {traceback.format_exc()}||"
|
380
383
|
)
|
384
|
+
else:
|
385
|
+
trace.error(
|
386
|
+
f"[STAGE]: 🤫 Stage Failed with disable traceback:||{e}"
|
387
|
+
)
|
381
388
|
st: Status = get_status_from_error(e)
|
382
389
|
return Result(
|
383
390
|
run_id=run_id,
|
384
391
|
parent_run_id=parent_run_id,
|
385
392
|
status=st,
|
386
|
-
context=catch(
|
387
|
-
context,
|
388
|
-
status=st,
|
389
|
-
updated=(
|
390
|
-
None
|
391
|
-
if isinstance(e, (StageSkipError, StageNestedSkipError))
|
392
|
-
else {"errors": e.to_dict()}
|
393
|
-
),
|
394
|
-
),
|
393
|
+
context=catch(context, status=st, updated=updated),
|
395
394
|
info={"execution_time": time.monotonic() - ts},
|
396
395
|
extras=self.extras,
|
397
396
|
)
|
@@ -409,6 +408,8 @@ class BaseStage(BaseModel, ABC):
|
|
409
408
|
info={"execution_time": time.monotonic() - ts},
|
410
409
|
extras=self.extras,
|
411
410
|
)
|
411
|
+
finally:
|
412
|
+
trace.debug("[STAGE]: End Handler stage execution.")
|
412
413
|
|
413
414
|
def _execute(
|
414
415
|
self,
|
@@ -421,8 +422,10 @@ class BaseStage(BaseModel, ABC):
|
|
421
422
|
"""Wrapped the process method before returning to handler execution.
|
422
423
|
|
423
424
|
Args:
|
424
|
-
params: A parameter data that want to use in this
|
425
|
-
|
425
|
+
params: A parameter data that want to use in this execution.
|
426
|
+
run_id (str):
|
427
|
+
context:
|
428
|
+
parent_run_id:
|
426
429
|
event: An event manager that use to track parent process
|
427
430
|
was not force stopped.
|
428
431
|
|
@@ -552,7 +555,7 @@ class BaseStage(BaseModel, ABC):
|
|
552
555
|
# should use the `re` module to validate eval-string before
|
553
556
|
# running.
|
554
557
|
rs: bool = eval(
|
555
|
-
|
558
|
+
self.pass_template(self.condition, params),
|
556
559
|
globals() | params,
|
557
560
|
{},
|
558
561
|
)
|
@@ -583,7 +586,8 @@ class BaseStage(BaseModel, ABC):
|
|
583
586
|
def is_nested(self) -> bool:
|
584
587
|
"""Return true if this stage is nested stage.
|
585
588
|
|
586
|
-
:
|
589
|
+
Returns:
|
590
|
+
bool: True if this stage is nested stage.
|
587
591
|
"""
|
588
592
|
return False
|
589
593
|
|
@@ -593,14 +597,46 @@ class BaseStage(BaseModel, ABC):
|
|
593
597
|
Returns:
|
594
598
|
DictData: A dict that was dumped from this model with alias mode.
|
595
599
|
"""
|
596
|
-
return self.model_dump(
|
600
|
+
return self.model_dump(
|
601
|
+
by_alias=True,
|
602
|
+
exclude_defaults=True,
|
603
|
+
exclude={"extras", "id", "name", "desc"},
|
604
|
+
)
|
597
605
|
|
598
|
-
def md(self) -> str: # pragma: no cov
|
606
|
+
def md(self, level: int = 1) -> str: # pragma: no cov
|
599
607
|
"""Return generated document that will be the interface of this stage.
|
600
608
|
|
601
|
-
:
|
609
|
+
Args:
|
610
|
+
level (int, default 0): A header level that want to generate
|
611
|
+
markdown content.
|
612
|
+
|
613
|
+
Returns:
|
614
|
+
str
|
602
615
|
"""
|
603
|
-
|
616
|
+
assert level >= 1, "Header level should gather than 0"
|
617
|
+
|
618
|
+
def align_newline(value: Optional[str]) -> str:
|
619
|
+
space: str = " " * 16
|
620
|
+
if value is None:
|
621
|
+
return ""
|
622
|
+
return value.rstrip("\n").replace("\n", f"\n{space}")
|
623
|
+
|
624
|
+
header: str = "#" * level
|
625
|
+
return dedent(
|
626
|
+
f"""
|
627
|
+
{header} Stage: {self.iden}\n
|
628
|
+
{align_newline(self.desc)}\n
|
629
|
+
#{header} Parameters\n
|
630
|
+
| name | type | default | description |
|
631
|
+
| --- | --- | --- | : --- : |\n\n
|
632
|
+
#{header} Details\n
|
633
|
+
```json
|
634
|
+
{self.detail()}
|
635
|
+
```
|
636
|
+
""".lstrip(
|
637
|
+
"\n"
|
638
|
+
)
|
639
|
+
)
|
604
640
|
|
605
641
|
def dryrun(
|
606
642
|
self,
|
@@ -610,26 +646,75 @@ class BaseStage(BaseModel, ABC):
|
|
610
646
|
*,
|
611
647
|
parent_run_id: Optional[str] = None,
|
612
648
|
event: Optional[Event] = None,
|
613
|
-
) ->
|
649
|
+
) -> Result:
|
614
650
|
"""Pre-process method that will use to run with dry-run mode, and it
|
615
|
-
should be used
|
651
|
+
should be used replace of process method when workflow release set with
|
652
|
+
DRYRUN mode.
|
653
|
+
|
654
|
+
By default, this method will set logic to convert this stage model
|
655
|
+
to am EmptyStage if it is action stage before use process method
|
656
|
+
instead process itself.
|
657
|
+
|
658
|
+
Args:
|
659
|
+
params (DictData): A parameter data that want to use in this
|
660
|
+
execution.
|
661
|
+
run_id (str): A running stage ID.
|
662
|
+
context (DictData): A context data.
|
663
|
+
parent_run_id (str, default None): A parent running ID.
|
664
|
+
event (Event, default None): An event manager that use to track
|
665
|
+
parent process was not force stopped.
|
666
|
+
|
667
|
+
Returns:
|
668
|
+
Result: The execution result with status and context data.
|
616
669
|
"""
|
670
|
+
trace: Trace = get_trace(
|
671
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
672
|
+
)
|
673
|
+
trace.debug("[STAGE]: Start Dryrun ...")
|
674
|
+
if self.action_stage:
|
675
|
+
return self.to_empty().process(
|
676
|
+
params,
|
677
|
+
run_id,
|
678
|
+
context,
|
679
|
+
parent_run_id=parent_run_id,
|
680
|
+
event=event,
|
681
|
+
)
|
682
|
+
return self.process(
|
683
|
+
params, run_id, context, parent_run_id=parent_run_id, event=event
|
684
|
+
)
|
617
685
|
|
618
|
-
def to_empty(
|
686
|
+
def to_empty(
|
687
|
+
self,
|
688
|
+
sleep: int = 0.35,
|
689
|
+
*,
|
690
|
+
message: Optional[str] = None,
|
691
|
+
) -> EmptyStage:
|
619
692
|
"""Convert the current Stage model to the EmptyStage model for dry-run
|
620
693
|
mode if the `action_stage` class attribute has set.
|
621
694
|
|
695
|
+
Some use-case for this method is use for deactivate.
|
696
|
+
|
697
|
+
Args:
|
698
|
+
sleep (int, default 0.35): An adjustment sleep time.
|
699
|
+
message (str, default None): A message that want to override default
|
700
|
+
message on EmptyStage model.
|
701
|
+
|
622
702
|
Returns:
|
623
703
|
EmptyStage: An EmptyStage model that passing itself model data to
|
624
704
|
message.
|
625
705
|
"""
|
706
|
+
if isinstance(self, EmptyStage):
|
707
|
+
return self.model_copy(update={"sleep": sleep})
|
626
708
|
return EmptyStage.model_validate(
|
627
709
|
{
|
628
710
|
"name": self.name,
|
629
711
|
"id": self.id,
|
630
712
|
"desc": self.desc,
|
631
713
|
"if": self.condition,
|
632
|
-
"echo":
|
714
|
+
"echo": (
|
715
|
+
message
|
716
|
+
or f"Convert from {self.__class__.__name__} to EmptyStage"
|
717
|
+
),
|
633
718
|
"sleep": sleep,
|
634
719
|
}
|
635
720
|
)
|
@@ -783,6 +868,8 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
783
868
|
info={"execution_time": time.monotonic() - ts},
|
784
869
|
extras=self.extras,
|
785
870
|
)
|
871
|
+
finally:
|
872
|
+
trace.debug("[STAGE]: End Handler stage process.")
|
786
873
|
|
787
874
|
async def _axecute(
|
788
875
|
self,
|
@@ -794,12 +881,14 @@ class BaseAsyncStage(BaseStage, ABC):
|
|
794
881
|
) -> Result:
|
795
882
|
"""Wrapped the axecute method before returning to handler axecute.
|
796
883
|
|
797
|
-
:
|
798
|
-
|
799
|
-
|
800
|
-
|
884
|
+
Args:
|
885
|
+
params: (DictData) A parameter data that want to use in this
|
886
|
+
execution.
|
887
|
+
event: (Event) An event manager that use to track parent execute
|
888
|
+
was not force stopped.
|
801
889
|
|
802
|
-
:
|
890
|
+
Returns:
|
891
|
+
Result: A Result object.
|
803
892
|
"""
|
804
893
|
catch(context, status=WAIT)
|
805
894
|
return await self.async_process(
|
@@ -820,7 +909,10 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
820
909
|
default=0,
|
821
910
|
ge=0,
|
822
911
|
lt=20,
|
823
|
-
description=
|
912
|
+
description=(
|
913
|
+
"A retry number if stage process got the error exclude skip and "
|
914
|
+
"cancel exception class."
|
915
|
+
),
|
824
916
|
)
|
825
917
|
|
826
918
|
def _execute(
|
@@ -834,12 +926,14 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
834
926
|
"""Wrapped the execute method with retry strategy before returning to
|
835
927
|
handler execute.
|
836
928
|
|
837
|
-
:
|
838
|
-
|
839
|
-
|
840
|
-
|
929
|
+
Args:
|
930
|
+
params: (DictData) A parameter data that want to use in this
|
931
|
+
execution.
|
932
|
+
event: (Event) An event manager that use to track parent execute
|
933
|
+
was not force stopped.
|
841
934
|
|
842
|
-
:
|
935
|
+
Returns:
|
936
|
+
Result: A Result object.
|
843
937
|
"""
|
844
938
|
current_retry: int = 0
|
845
939
|
exception: Exception
|
@@ -847,9 +941,19 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
847
941
|
trace: Trace = get_trace(
|
848
942
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
849
943
|
)
|
850
|
-
|
851
944
|
# NOTE: First execution for not pass to retry step if it passes.
|
852
945
|
try:
|
946
|
+
if (
|
947
|
+
self.extras.get("__sys_release_dryrun_mode", False)
|
948
|
+
and self.action_stage
|
949
|
+
):
|
950
|
+
return self.dryrun(
|
951
|
+
params | {"retry": current_retry},
|
952
|
+
run_id=run_id,
|
953
|
+
context=context,
|
954
|
+
parent_run_id=parent_run_id,
|
955
|
+
event=event,
|
956
|
+
)
|
853
957
|
return self.process(
|
854
958
|
params | {"retry": current_retry},
|
855
959
|
run_id=run_id,
|
@@ -857,9 +961,19 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
857
961
|
parent_run_id=parent_run_id,
|
858
962
|
event=event,
|
859
963
|
)
|
964
|
+
except (
|
965
|
+
StageNestedSkipError,
|
966
|
+
StageNestedCancelError,
|
967
|
+
StageSkipError,
|
968
|
+
StageCancelError,
|
969
|
+
):
|
970
|
+
trace.debug("[STAGE]: process raise skip or cancel error.")
|
971
|
+
raise
|
860
972
|
except Exception as e:
|
861
973
|
current_retry += 1
|
862
974
|
exception = e
|
975
|
+
finally:
|
976
|
+
trace.debug("[STAGE]: Failed at the first execution.")
|
863
977
|
|
864
978
|
if self.retry == 0:
|
865
979
|
raise exception
|
@@ -876,6 +990,17 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
876
990
|
status=WAIT,
|
877
991
|
updated={"retry": current_retry},
|
878
992
|
)
|
993
|
+
if (
|
994
|
+
self.extras.get("__sys_release_dryrun_mode", False)
|
995
|
+
and self.action_stage
|
996
|
+
):
|
997
|
+
return self.dryrun(
|
998
|
+
params | {"retry": current_retry},
|
999
|
+
run_id=run_id,
|
1000
|
+
context=context,
|
1001
|
+
parent_run_id=parent_run_id,
|
1002
|
+
event=event,
|
1003
|
+
)
|
879
1004
|
return self.process(
|
880
1005
|
params | {"retry": current_retry},
|
881
1006
|
run_id=run_id,
|
@@ -884,10 +1009,10 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
884
1009
|
event=event,
|
885
1010
|
)
|
886
1011
|
except (
|
887
|
-
StageSkipError,
|
888
1012
|
StageNestedSkipError,
|
889
|
-
StageCancelError,
|
890
1013
|
StageNestedCancelError,
|
1014
|
+
StageSkipError,
|
1015
|
+
StageCancelError,
|
891
1016
|
):
|
892
1017
|
trace.debug("[STAGE]: process raise skip or cancel error.")
|
893
1018
|
raise
|
@@ -916,12 +1041,14 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
916
1041
|
"""Wrapped the axecute method with retry strategy before returning to
|
917
1042
|
handler axecute.
|
918
1043
|
|
919
|
-
:
|
920
|
-
|
921
|
-
|
922
|
-
|
1044
|
+
Args:
|
1045
|
+
params: (DictData) A parameter data that want to use in this
|
1046
|
+
execution.
|
1047
|
+
event: (Event) An event manager that use to track parent execute
|
1048
|
+
was not force stopped.
|
923
1049
|
|
924
|
-
:
|
1050
|
+
Returns:
|
1051
|
+
Result: A Result object.
|
925
1052
|
"""
|
926
1053
|
current_retry: int = 0
|
927
1054
|
exception: Exception
|
@@ -932,6 +1059,17 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
932
1059
|
|
933
1060
|
# NOTE: First execution for not pass to retry step if it passes.
|
934
1061
|
try:
|
1062
|
+
if (
|
1063
|
+
self.extras.get("__sys_release_dryrun_mode", False)
|
1064
|
+
and self.action_stage
|
1065
|
+
):
|
1066
|
+
return self.dryrun(
|
1067
|
+
params | {"retry": current_retry},
|
1068
|
+
run_id=run_id,
|
1069
|
+
context=context,
|
1070
|
+
parent_run_id=parent_run_id,
|
1071
|
+
event=event,
|
1072
|
+
)
|
935
1073
|
return await self.async_process(
|
936
1074
|
params | {"retry": current_retry},
|
937
1075
|
run_id=run_id,
|
@@ -939,9 +1077,19 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
939
1077
|
parent_run_id=parent_run_id,
|
940
1078
|
event=event,
|
941
1079
|
)
|
1080
|
+
except (
|
1081
|
+
StageNestedSkipError,
|
1082
|
+
StageNestedCancelError,
|
1083
|
+
StageSkipError,
|
1084
|
+
StageCancelError,
|
1085
|
+
):
|
1086
|
+
await trace.adebug("[STAGE]: process raise skip or cancel error.")
|
1087
|
+
raise
|
942
1088
|
except Exception as e:
|
943
1089
|
current_retry += 1
|
944
1090
|
exception = e
|
1091
|
+
finally:
|
1092
|
+
await trace.adebug("[STAGE]: Failed at the first execution.")
|
945
1093
|
|
946
1094
|
if self.retry == 0:
|
947
1095
|
raise exception
|
@@ -958,6 +1106,17 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
958
1106
|
status=WAIT,
|
959
1107
|
updated={"retry": current_retry},
|
960
1108
|
)
|
1109
|
+
if (
|
1110
|
+
self.extras.get("__sys_release_dryrun_mode", False)
|
1111
|
+
and self.action_stage
|
1112
|
+
):
|
1113
|
+
return self.dryrun(
|
1114
|
+
params | {"retry": current_retry},
|
1115
|
+
run_id=run_id,
|
1116
|
+
context=context,
|
1117
|
+
parent_run_id=parent_run_id,
|
1118
|
+
event=event,
|
1119
|
+
)
|
961
1120
|
return await self.async_process(
|
962
1121
|
params | {"retry": current_retry},
|
963
1122
|
run_id=run_id,
|
@@ -966,10 +1125,10 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
|
|
966
1125
|
event=event,
|
967
1126
|
)
|
968
1127
|
except (
|
969
|
-
StageSkipError,
|
970
1128
|
StageNestedSkipError,
|
971
|
-
StageCancelError,
|
972
1129
|
StageNestedCancelError,
|
1130
|
+
StageSkipError,
|
1131
|
+
StageCancelError,
|
973
1132
|
):
|
974
1133
|
await trace.adebug(
|
975
1134
|
"[STAGE]: process raise skip or cancel error."
|
@@ -1004,22 +1163,13 @@ class EmptyStage(BaseAsyncStage):
|
|
1004
1163
|
for a specified duration, making it useful for workflow timing control
|
1005
1164
|
and debugging scenarios.
|
1006
1165
|
|
1007
|
-
|
1008
|
-
|
1009
|
-
|
1010
|
-
|
1011
|
-
|
1012
|
-
|
1013
|
-
|
1014
|
-
- name: "Debug Parameters"
|
1015
|
-
echo: "Processing file: ${{ params.filename }}"
|
1016
|
-
```
|
1017
|
-
|
1018
|
-
>>> stage = EmptyStage(
|
1019
|
-
... name="Status Update",
|
1020
|
-
... echo="Processing completed successfully",
|
1021
|
-
... sleep=1.0
|
1022
|
-
... )
|
1166
|
+
Examples:
|
1167
|
+
>>> stage = EmptyStage.model_validate({
|
1168
|
+
... "id": "empty-stage",
|
1169
|
+
... "name": "Status Update",
|
1170
|
+
... "echo": "Processing completed successfully",
|
1171
|
+
... "sleep": 1.0,
|
1172
|
+
... })
|
1023
1173
|
"""
|
1024
1174
|
|
1025
1175
|
echo: StrOrNone = Field(
|
@@ -1063,6 +1213,9 @@ class EmptyStage(BaseAsyncStage):
|
|
1063
1213
|
event: An event manager that use to track parent process
|
1064
1214
|
was not force stopped.
|
1065
1215
|
|
1216
|
+
Raises:
|
1217
|
+
StageCancelError: If event was set before start process.
|
1218
|
+
|
1066
1219
|
Returns:
|
1067
1220
|
Result: The execution result with status and context data.
|
1068
1221
|
"""
|
@@ -1114,6 +1267,9 @@ class EmptyStage(BaseAsyncStage):
|
|
1114
1267
|
event: An event manager that use to track parent process
|
1115
1268
|
was not force stopped.
|
1116
1269
|
|
1270
|
+
Raises:
|
1271
|
+
StageCancelError: If event was set before start process.
|
1272
|
+
|
1117
1273
|
Returns:
|
1118
1274
|
Result: The execution result with status and context data.
|
1119
1275
|
"""
|
@@ -1155,16 +1311,18 @@ class BashStage(BaseRetryStage):
|
|
1155
1311
|
statement. Thus, it will write the `.sh` file before start running bash
|
1156
1312
|
command for fix this issue.
|
1157
1313
|
|
1158
|
-
|
1159
|
-
>>> stage = {
|
1314
|
+
Examples:
|
1315
|
+
>>> stage = BaseStage.model_validate({
|
1316
|
+
... "id": "bash-stage",
|
1160
1317
|
... "name": "The Shell stage execution",
|
1161
1318
|
... "bash": 'echo "Hello $FOO"',
|
1162
1319
|
... "env": {
|
1163
1320
|
... "FOO": "BAR",
|
1164
1321
|
... },
|
1165
|
-
... }
|
1322
|
+
... })
|
1166
1323
|
"""
|
1167
1324
|
|
1325
|
+
action_stage: ClassVar[bool] = True
|
1168
1326
|
bash: str = Field(
|
1169
1327
|
description=(
|
1170
1328
|
"A bash statement that want to execute via Python subprocess."
|
@@ -1179,17 +1337,20 @@ class BashStage(BaseRetryStage):
|
|
1179
1337
|
)
|
1180
1338
|
|
1181
1339
|
@contextlib.asynccontextmanager
|
1182
|
-
async def
|
1340
|
+
async def async_make_sh_file(
|
1183
1341
|
self, bash: str, env: DictStr, run_id: StrOrNone = None
|
1184
1342
|
) -> AsyncIterator[TupleStr]:
|
1185
1343
|
"""Async create and write `.sh` file with the `aiofiles` package.
|
1186
1344
|
|
1187
|
-
:
|
1188
|
-
|
1189
|
-
|
1190
|
-
|
1345
|
+
Args:
|
1346
|
+
bash (str): A bash statement.
|
1347
|
+
env (DictStr): An environment variable that set before run bash.
|
1348
|
+
run_id (StrOrNone, default None): A running stage ID that use for
|
1349
|
+
writing `.sh` file instead generate by UUID4.
|
1191
1350
|
|
1192
|
-
:
|
1351
|
+
Returns:
|
1352
|
+
AsyncIterator[TupleStr]: Return context of prepared bash statement
|
1353
|
+
that want to execute.
|
1193
1354
|
"""
|
1194
1355
|
import aiofiles
|
1195
1356
|
|
@@ -1215,19 +1376,21 @@ class BashStage(BaseRetryStage):
|
|
1215
1376
|
Path(f"./{f_name}").unlink()
|
1216
1377
|
|
1217
1378
|
@contextlib.contextmanager
|
1218
|
-
def
|
1379
|
+
def make_sh_file(
|
1219
1380
|
self, bash: str, env: DictStr, run_id: StrOrNone = None
|
1220
1381
|
) -> Iterator[TupleStr]:
|
1221
1382
|
"""Create and write the `.sh` file before giving this file name to
|
1222
1383
|
context. After that, it will auto delete this file automatic.
|
1223
1384
|
|
1224
|
-
:
|
1225
|
-
|
1226
|
-
|
1227
|
-
|
1385
|
+
Args:
|
1386
|
+
bash (str): A bash statement.
|
1387
|
+
env (DictStr): An environment variable that set before run bash.
|
1388
|
+
run_id (StrOrNone, default None): A running stage ID that use for
|
1389
|
+
writing `.sh` file instead generate by UUID4.
|
1228
1390
|
|
1229
|
-
:
|
1230
|
-
|
1391
|
+
Returns:
|
1392
|
+
Iterator[TupleStr]: Return context of prepared bash statement that
|
1393
|
+
want to execute.
|
1231
1394
|
"""
|
1232
1395
|
f_name: str = f"{run_id or uuid.uuid4()}.sh"
|
1233
1396
|
f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
|
@@ -1269,13 +1432,19 @@ class BashStage(BaseRetryStage):
|
|
1269
1432
|
`return_code`, `stdout`, and `stderr`.
|
1270
1433
|
|
1271
1434
|
Args:
|
1272
|
-
params: A parameter data that want to use in this
|
1435
|
+
params (DictData): A parameter data that want to use in this
|
1273
1436
|
execution.
|
1274
|
-
run_id: A running stage ID.
|
1275
|
-
context: A context data
|
1276
|
-
|
1277
|
-
|
1278
|
-
|
1437
|
+
run_id (str): A running stage ID.
|
1438
|
+
context (DictData): A context data that was passed from handler
|
1439
|
+
method.
|
1440
|
+
parent_run_id (str, default None): A parent running ID.
|
1441
|
+
event (Event, default None): An event manager that use to track
|
1442
|
+
parent process was not force stopped.
|
1443
|
+
|
1444
|
+
Raises:
|
1445
|
+
StageCancelError: If event was set before start process.
|
1446
|
+
StageError: If the return code form subprocess run function gather
|
1447
|
+
than 0.
|
1279
1448
|
|
1280
1449
|
Returns:
|
1281
1450
|
Result: The execution result with status and context data.
|
@@ -1286,12 +1455,16 @@ class BashStage(BaseRetryStage):
|
|
1286
1455
|
bash: str = param2template(
|
1287
1456
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
1288
1457
|
)
|
1289
|
-
with self.
|
1458
|
+
with self.make_sh_file(
|
1290
1459
|
bash=bash,
|
1291
1460
|
env=param2template(self.env, params, extras=self.extras),
|
1292
1461
|
run_id=run_id,
|
1293
1462
|
) as sh:
|
1294
|
-
|
1463
|
+
|
1464
|
+
if event and event.is_set():
|
1465
|
+
raise StageCancelError("Cancel before start bash process.")
|
1466
|
+
|
1467
|
+
trace.debug(f"[STAGE]: Create `{sh[1]}` file.", module="stage")
|
1295
1468
|
rs: CompletedProcess = subprocess.run(
|
1296
1469
|
sh,
|
1297
1470
|
shell=False,
|
@@ -1333,13 +1506,19 @@ class BashStage(BaseRetryStage):
|
|
1333
1506
|
stdout.
|
1334
1507
|
|
1335
1508
|
Args:
|
1336
|
-
params: A parameter data that want to use in this
|
1509
|
+
params (DictData): A parameter data that want to use in this
|
1337
1510
|
execution.
|
1338
|
-
run_id: A running stage ID.
|
1339
|
-
context: A context data
|
1340
|
-
|
1341
|
-
|
1342
|
-
|
1511
|
+
run_id (str): A running stage ID.
|
1512
|
+
context (DictData): A context data that was passed from handler
|
1513
|
+
method.
|
1514
|
+
parent_run_id (str, default None): A parent running ID.
|
1515
|
+
event (Event, default None): An event manager that use to track
|
1516
|
+
parent process was not force stopped.
|
1517
|
+
|
1518
|
+
Raises:
|
1519
|
+
StageCancelError: If event was set before start process.
|
1520
|
+
StageError: If the return code form subprocess run function gather
|
1521
|
+
than 0.
|
1343
1522
|
|
1344
1523
|
Returns:
|
1345
1524
|
Result: The execution result with status and context data.
|
@@ -1350,11 +1529,15 @@ class BashStage(BaseRetryStage):
|
|
1350
1529
|
bash: str = param2template(
|
1351
1530
|
dedent(self.bash.strip("\n")), params, extras=self.extras
|
1352
1531
|
)
|
1353
|
-
async with self.
|
1532
|
+
async with self.async_make_sh_file(
|
1354
1533
|
bash=bash,
|
1355
1534
|
env=param2template(self.env, params, extras=self.extras),
|
1356
1535
|
run_id=run_id,
|
1357
1536
|
) as sh:
|
1537
|
+
|
1538
|
+
if event and event.is_set():
|
1539
|
+
raise StageCancelError("Cancel before start bash process.")
|
1540
|
+
|
1358
1541
|
await trace.adebug(f"[STAGE]: Create `{sh[1]}` file.")
|
1359
1542
|
rs: CompletedProcess = subprocess.run(
|
1360
1543
|
sh,
|
@@ -1364,7 +1547,6 @@ class BashStage(BaseRetryStage):
|
|
1364
1547
|
text=True,
|
1365
1548
|
encoding="utf-8",
|
1366
1549
|
)
|
1367
|
-
|
1368
1550
|
if rs.returncode > 0:
|
1369
1551
|
e: str = rs.stderr.removesuffix("\n")
|
1370
1552
|
e_bash: str = bash.replace("\n", "\n\t")
|
@@ -1399,16 +1581,18 @@ class PyStage(BaseRetryStage):
|
|
1399
1581
|
module to validate exec-string before running or exclude the `os` package
|
1400
1582
|
from the current globals variable.
|
1401
1583
|
|
1402
|
-
|
1403
|
-
>>> stage = {
|
1584
|
+
Examples:
|
1585
|
+
>>> stage = PyStage.model_validate({
|
1586
|
+
... "id": "py-stage",
|
1404
1587
|
... "name": "Python stage execution",
|
1405
1588
|
... "run": 'print(f"Hello {VARIABLE}")',
|
1406
1589
|
... "vars": {
|
1407
1590
|
... "VARIABLE": "WORLD",
|
1408
1591
|
... },
|
1409
|
-
... }
|
1592
|
+
... })
|
1410
1593
|
"""
|
1411
1594
|
|
1595
|
+
action_stage: ClassVar[bool] = True
|
1412
1596
|
run: str = Field(
|
1413
1597
|
description="A Python string statement that want to run with `exec`.",
|
1414
1598
|
)
|
@@ -1423,11 +1607,13 @@ class PyStage(BaseRetryStage):
|
|
1423
1607
|
@staticmethod
|
1424
1608
|
def filter_locals(values: DictData) -> Iterator[str]:
|
1425
1609
|
"""Filter a locals mapping values that be module, class, or
|
1426
|
-
__annotations__
|
1610
|
+
`__annotations__`.
|
1427
1611
|
|
1428
|
-
:
|
1612
|
+
Args:
|
1613
|
+
values: (DictData) A locals values that want to filter.
|
1429
1614
|
|
1430
|
-
:
|
1615
|
+
Returns:
|
1616
|
+
Iterator[str]: Iter string value.
|
1431
1617
|
"""
|
1432
1618
|
for value in values:
|
1433
1619
|
|
@@ -1447,12 +1633,14 @@ class PyStage(BaseRetryStage):
|
|
1447
1633
|
"""Override set an outputs method for the Python execution process that
|
1448
1634
|
extract output from all the locals values.
|
1449
1635
|
|
1450
|
-
:
|
1451
|
-
output
|
1452
|
-
|
1453
|
-
|
1636
|
+
Args:
|
1637
|
+
output (DictData): An output data that want to extract to an
|
1638
|
+
output key.
|
1639
|
+
to (DictData): A context data that want to add output result.
|
1640
|
+
info (DictData):
|
1454
1641
|
|
1455
|
-
:
|
1642
|
+
Returns:
|
1643
|
+
DictData: A context data that have merged with the output data.
|
1456
1644
|
"""
|
1457
1645
|
output: DictData = output.copy()
|
1458
1646
|
lc: DictData = output.pop("locals", {})
|
@@ -1504,18 +1692,13 @@ class PyStage(BaseRetryStage):
|
|
1504
1692
|
}
|
1505
1693
|
)
|
1506
1694
|
|
1695
|
+
if event and event.is_set():
|
1696
|
+
raise StageCancelError("Cancel before start exec process.")
|
1697
|
+
|
1507
1698
|
# WARNING: The exec build-in function is very dangerous. So, it
|
1508
1699
|
# should use the re module to validate exec-string before running.
|
1509
|
-
exec(
|
1510
|
-
|
1511
|
-
param2template(dedent(self.run), params, extras=self.extras)
|
1512
|
-
),
|
1513
|
-
gb,
|
1514
|
-
lc,
|
1515
|
-
)
|
1516
|
-
return Result(
|
1517
|
-
run_id=run_id,
|
1518
|
-
parent_run_id=parent_run_id,
|
1700
|
+
exec(self.pass_template(dedent(self.run), params), gb, lc)
|
1701
|
+
return Result.from_trace(trace).catch(
|
1519
1702
|
status=SUCCESS,
|
1520
1703
|
context=catch(
|
1521
1704
|
context=context,
|
@@ -1536,7 +1719,6 @@ class PyStage(BaseRetryStage):
|
|
1536
1719
|
},
|
1537
1720
|
},
|
1538
1721
|
),
|
1539
|
-
extras=self.extras,
|
1540
1722
|
)
|
1541
1723
|
|
1542
1724
|
async def async_process(
|
@@ -1555,13 +1737,17 @@ class PyStage(BaseRetryStage):
|
|
1555
1737
|
- https://stackoverflow.com/questions/44859165/async-exec-in-python
|
1556
1738
|
|
1557
1739
|
Args:
|
1558
|
-
params: A parameter data that want to use in this
|
1740
|
+
params (DictData): A parameter data that want to use in this
|
1559
1741
|
execution.
|
1560
|
-
run_id: A running stage ID.
|
1561
|
-
context: A context data
|
1562
|
-
|
1563
|
-
|
1564
|
-
|
1742
|
+
run_id (str): A running stage ID.
|
1743
|
+
context (DictData): A context data that was passed from handler
|
1744
|
+
method.
|
1745
|
+
parent_run_id (str, default None): A parent running ID.
|
1746
|
+
event (Event, default None): An event manager that use to track
|
1747
|
+
parent process was not force stopped.
|
1748
|
+
|
1749
|
+
Raises:
|
1750
|
+
StageCancelError: If event was set before start process.
|
1565
1751
|
|
1566
1752
|
Returns:
|
1567
1753
|
Result: The execution result with status and context data.
|
@@ -1584,16 +1770,14 @@ class PyStage(BaseRetryStage):
|
|
1584
1770
|
)
|
1585
1771
|
}
|
1586
1772
|
)
|
1773
|
+
|
1774
|
+
if event and event.is_set():
|
1775
|
+
raise StageCancelError("Cancel before start exec process.")
|
1776
|
+
|
1587
1777
|
# WARNING: The exec build-in function is very dangerous. So, it
|
1588
1778
|
# should use the re module to validate exec-string before running.
|
1589
|
-
exec(
|
1590
|
-
|
1591
|
-
gb,
|
1592
|
-
lc,
|
1593
|
-
)
|
1594
|
-
return Result(
|
1595
|
-
run_id=run_id,
|
1596
|
-
parent_run_id=parent_run_id,
|
1779
|
+
exec(self.pass_template(dedent(self.run), params), gb, lc)
|
1780
|
+
return Result.from_trace(trace).catch(
|
1597
1781
|
status=SUCCESS,
|
1598
1782
|
context=catch(
|
1599
1783
|
context=context,
|
@@ -1614,7 +1798,6 @@ class PyStage(BaseRetryStage):
|
|
1614
1798
|
},
|
1615
1799
|
},
|
1616
1800
|
),
|
1617
|
-
extras=self.extras,
|
1618
1801
|
)
|
1619
1802
|
|
1620
1803
|
|
@@ -1639,14 +1822,16 @@ class CallStage(BaseRetryStage):
|
|
1639
1822
|
The caller registry to get a caller function should importable by the
|
1640
1823
|
current Python execution pointer.
|
1641
1824
|
|
1642
|
-
|
1643
|
-
>>> stage = {
|
1825
|
+
Examples:
|
1826
|
+
>>> stage = CallStage.model_validate({
|
1827
|
+
... "id": "call-stage",
|
1644
1828
|
... "name": "Task stage execution",
|
1645
1829
|
... "uses": "tasks/function-name@tag-name",
|
1646
1830
|
... "args": {"arg01": "BAR", "kwarg01": 10},
|
1647
|
-
... }
|
1831
|
+
... })
|
1648
1832
|
"""
|
1649
1833
|
|
1834
|
+
action_stage: ClassVar[bool] = True
|
1650
1835
|
uses: str = Field(
|
1651
1836
|
description=(
|
1652
1837
|
"A caller function with registry importer syntax that use to load "
|
@@ -1663,26 +1848,36 @@ class CallStage(BaseRetryStage):
|
|
1663
1848
|
)
|
1664
1849
|
|
1665
1850
|
@field_validator("args", mode="before")
|
1666
|
-
def __validate_args_key(cls,
|
1851
|
+
def __validate_args_key(cls, data: Any) -> Any:
|
1667
1852
|
"""Validate argument keys on the ``args`` field should not include the
|
1668
1853
|
special keys.
|
1669
1854
|
|
1670
|
-
:
|
1855
|
+
Args:
|
1856
|
+
data (Any): A data that want to check the special keys.
|
1671
1857
|
|
1672
|
-
:
|
1858
|
+
Returns:
|
1859
|
+
Any: An any data.
|
1673
1860
|
"""
|
1674
|
-
if isinstance(
|
1675
|
-
k in
|
1861
|
+
if isinstance(data, dict) and any(
|
1862
|
+
k in data for k in ("result", "extras")
|
1676
1863
|
):
|
1677
1864
|
raise ValueError(
|
1678
1865
|
"The argument on workflow template for the caller stage "
|
1679
1866
|
"should not pass `result` and `extras`. They are special "
|
1680
1867
|
"arguments."
|
1681
1868
|
)
|
1682
|
-
return
|
1869
|
+
return data
|
1683
1870
|
|
1684
1871
|
def get_caller(self, params: DictData) -> Callable[[], TagFunc]:
|
1685
|
-
"""Get the lazy TagFuc object from registry.
|
1872
|
+
"""Get the lazy TagFuc object from registry.
|
1873
|
+
|
1874
|
+
Args:
|
1875
|
+
params (DictData): A parameters.
|
1876
|
+
|
1877
|
+
Returns:
|
1878
|
+
Callable[[], TagFunc]: A lazy partial function that return the
|
1879
|
+
TagFunc object.
|
1880
|
+
"""
|
1686
1881
|
return extract_call(
|
1687
1882
|
param2template(self.uses, params, extras=self.extras),
|
1688
1883
|
registries=self.extras.get("registry_caller"),
|
@@ -1700,13 +1895,19 @@ class CallStage(BaseRetryStage):
|
|
1700
1895
|
"""Execute this caller function with its argument parameter.
|
1701
1896
|
|
1702
1897
|
Args:
|
1703
|
-
params: A parameter data that want to use in this
|
1898
|
+
params (DictData): A parameter data that want to use in this
|
1704
1899
|
execution.
|
1705
|
-
run_id: A running stage ID.
|
1706
|
-
context: A context data
|
1707
|
-
|
1708
|
-
|
1709
|
-
|
1900
|
+
run_id (str): A running stage ID.
|
1901
|
+
context (DictData): A context data that was passed from handler
|
1902
|
+
method.
|
1903
|
+
parent_run_id (str, default None): A parent running ID.
|
1904
|
+
event (Event, default None): An event manager that use to track
|
1905
|
+
parent process was not force stopped.
|
1906
|
+
|
1907
|
+
Raises:
|
1908
|
+
ValueError: If the necessary parameters do not exist in args field.
|
1909
|
+
TypeError: If the returning type of caller function does not match
|
1910
|
+
with dict type.
|
1710
1911
|
|
1711
1912
|
Returns:
|
1712
1913
|
Result: The execution result with status and context data.
|
@@ -1728,8 +1929,10 @@ class CallStage(BaseRetryStage):
|
|
1728
1929
|
extras=self.extras,
|
1729
1930
|
),
|
1730
1931
|
"extras": self.extras,
|
1731
|
-
} |
|
1732
|
-
|
1932
|
+
} | self.pass_template(self.args, params)
|
1933
|
+
|
1934
|
+
# NOTE: Catch the necessary parameters.
|
1935
|
+
sig: inspect.Signature = inspect.signature(call_func)
|
1733
1936
|
necessary_params: list[str] = []
|
1734
1937
|
has_keyword: bool = False
|
1735
1938
|
for k in sig.parameters:
|
@@ -1743,15 +1946,14 @@ class CallStage(BaseRetryStage):
|
|
1743
1946
|
elif v.kind == Parameter.VAR_KEYWORD:
|
1744
1947
|
has_keyword = True
|
1745
1948
|
|
1949
|
+
# NOTE: Validate private parameter should exist in the args field.
|
1746
1950
|
if any(
|
1747
1951
|
(k.removeprefix("_") not in args and k not in args)
|
1748
1952
|
for k in necessary_params
|
1749
1953
|
):
|
1750
|
-
|
1751
|
-
necessary_params
|
1752
|
-
|
1753
|
-
if "extras" in necessary_params:
|
1754
|
-
necessary_params.remove("extras")
|
1954
|
+
for k in ("result", "extras"):
|
1955
|
+
if k in necessary_params:
|
1956
|
+
necessary_params.remove(k)
|
1755
1957
|
|
1756
1958
|
args.pop("result")
|
1757
1959
|
args.pop("extras")
|
@@ -1760,18 +1962,16 @@ class CallStage(BaseRetryStage):
|
|
1760
1962
|
f"does not set to args. It already set {list(args.keys())}."
|
1761
1963
|
)
|
1762
1964
|
|
1763
|
-
if
|
1764
|
-
|
1965
|
+
if not has_keyword:
|
1966
|
+
for k in ("result", "extras"):
|
1967
|
+
if k not in sig.parameters:
|
1968
|
+
args.pop(k)
|
1765
1969
|
|
1766
|
-
|
1767
|
-
args.pop("extras")
|
1970
|
+
args: DictData = self.validate_model_args(call_func, args)
|
1768
1971
|
|
1769
1972
|
if event and event.is_set():
|
1770
1973
|
raise StageCancelError("Cancel before start call process.")
|
1771
1974
|
|
1772
|
-
args: DictData = self.validate_model_args(
|
1773
|
-
call_func, args, run_id, parent_run_id, extras=self.extras
|
1774
|
-
)
|
1775
1975
|
if inspect.iscoroutinefunction(call_func):
|
1776
1976
|
loop = asyncio.get_event_loop()
|
1777
1977
|
rs: DictData = loop.run_until_complete(
|
@@ -1825,6 +2025,11 @@ class CallStage(BaseRetryStage):
|
|
1825
2025
|
event: An event manager that use to track parent process
|
1826
2026
|
was not force stopped.
|
1827
2027
|
|
2028
|
+
Raises:
|
2029
|
+
ValueError: If the necessary parameters do not exist in args field.
|
2030
|
+
TypeError: If the returning type of caller function does not match
|
2031
|
+
with dict type.
|
2032
|
+
|
1828
2033
|
Returns:
|
1829
2034
|
Result: The execution result with status and context data.
|
1830
2035
|
"""
|
@@ -1847,8 +2052,10 @@ class CallStage(BaseRetryStage):
|
|
1847
2052
|
extras=self.extras,
|
1848
2053
|
),
|
1849
2054
|
"extras": self.extras,
|
1850
|
-
} |
|
1851
|
-
|
2055
|
+
} | self.pass_template(self.args, params)
|
2056
|
+
|
2057
|
+
# NOTE: Catch the necessary parameters.
|
2058
|
+
sig: inspect.Signature = inspect.signature(call_func)
|
1852
2059
|
necessary_params: list[str] = []
|
1853
2060
|
has_keyword: bool = False
|
1854
2061
|
for k in sig.parameters:
|
@@ -1866,11 +2073,9 @@ class CallStage(BaseRetryStage):
|
|
1866
2073
|
(k.removeprefix("_") not in args and k not in args)
|
1867
2074
|
for k in necessary_params
|
1868
2075
|
):
|
1869
|
-
|
1870
|
-
necessary_params
|
1871
|
-
|
1872
|
-
if "extras" in necessary_params:
|
1873
|
-
necessary_params.remove("extras")
|
2076
|
+
for k in ("result", "extras"):
|
2077
|
+
if k in necessary_params:
|
2078
|
+
necessary_params.remove(k)
|
1874
2079
|
|
1875
2080
|
args.pop("result")
|
1876
2081
|
args.pop("extras")
|
@@ -1878,18 +2083,17 @@ class CallStage(BaseRetryStage):
|
|
1878
2083
|
f"Necessary params, ({', '.join(necessary_params)}, ), "
|
1879
2084
|
f"does not set to args. It already set {list(args.keys())}."
|
1880
2085
|
)
|
1881
|
-
if "result" not in sig.parameters and not has_keyword:
|
1882
|
-
args.pop("result")
|
1883
2086
|
|
1884
|
-
if
|
1885
|
-
|
2087
|
+
if not has_keyword:
|
2088
|
+
for k in ("result", "extras"):
|
2089
|
+
if k not in sig.parameters:
|
2090
|
+
args.pop(k)
|
2091
|
+
|
2092
|
+
args: DictData = self.validate_model_args(call_func, args)
|
1886
2093
|
|
1887
2094
|
if event and event.is_set():
|
1888
2095
|
raise StageCancelError("Cancel before start call process.")
|
1889
2096
|
|
1890
|
-
args: DictData = self.validate_model_args(
|
1891
|
-
call_func, args, run_id, parent_run_id, extras=self.extras
|
1892
|
-
)
|
1893
2097
|
if inspect.iscoroutinefunction(call_func):
|
1894
2098
|
rs: DictOrModel = await call_func(
|
1895
2099
|
**param2template(args, params, extras=self.extras)
|
@@ -1922,21 +2126,18 @@ class CallStage(BaseRetryStage):
|
|
1922
2126
|
)
|
1923
2127
|
|
1924
2128
|
@staticmethod
|
1925
|
-
def validate_model_args(
|
1926
|
-
func: TagFunc,
|
1927
|
-
args: DictData,
|
1928
|
-
run_id: str,
|
1929
|
-
parent_run_id: Optional[str] = None,
|
1930
|
-
extras: Optional[DictData] = None,
|
1931
|
-
) -> DictData:
|
2129
|
+
def validate_model_args(func: TagFunc, args: DictData) -> DictData:
|
1932
2130
|
"""Validate an input arguments before passing to the caller function.
|
1933
2131
|
|
1934
2132
|
Args:
|
1935
|
-
func
|
1936
|
-
args
|
1937
|
-
run_id: A running stage ID.
|
2133
|
+
func (TagFunc): A tag function object that want to get typing.
|
2134
|
+
args (DictData): An arguments before passing to this tag func.
|
1938
2135
|
|
1939
|
-
:
|
2136
|
+
Raises:
|
2137
|
+
StageError: If model validation was raised the ValidationError.
|
2138
|
+
|
2139
|
+
Returns:
|
2140
|
+
DictData: A prepared args parameter that validate with model args.
|
1940
2141
|
"""
|
1941
2142
|
try:
|
1942
2143
|
override: DictData = dict(
|
@@ -1959,55 +2160,8 @@ class CallStage(BaseRetryStage):
|
|
1959
2160
|
raise StageError(
|
1960
2161
|
"Validate argument from the caller function raise invalid type."
|
1961
2162
|
) from e
|
1962
|
-
except TypeError as e:
|
1963
|
-
trace: Trace = get_trace(
|
1964
|
-
run_id, parent_run_id=parent_run_id, extras=extras
|
1965
|
-
)
|
1966
|
-
trace.warning(
|
1967
|
-
f"[STAGE]: Get type hint raise TypeError: {e}, so, it skip "
|
1968
|
-
f"parsing model args process."
|
1969
|
-
)
|
1970
|
-
return args
|
1971
|
-
|
1972
|
-
|
1973
|
-
class BaseNestedStage(BaseRetryStage, ABC):
|
1974
|
-
"""Base Nested Stage model. This model is use for checking the child stage
|
1975
|
-
is the nested stage or not.
|
1976
|
-
"""
|
1977
|
-
|
1978
|
-
def set_outputs(
|
1979
|
-
self, output: DictData, to: DictData, info: Optional[DictData] = None
|
1980
|
-
) -> DictData:
|
1981
|
-
"""Override the set outputs method that support for nested-stage."""
|
1982
|
-
return super().set_outputs(output, to=to)
|
1983
|
-
|
1984
|
-
def get_outputs(self, output: DictData) -> DictData:
|
1985
|
-
"""Override the get outputs method that support for nested-stage"""
|
1986
|
-
return super().get_outputs(output)
|
1987
|
-
|
1988
|
-
@property
|
1989
|
-
def is_nested(self) -> bool:
|
1990
|
-
"""Check if this stage is a nested stage or not.
|
1991
2163
|
|
1992
|
-
|
1993
|
-
"""
|
1994
|
-
return True
|
1995
|
-
|
1996
|
-
@staticmethod
|
1997
|
-
def mark_errors(context: DictData, error: StageError) -> None:
|
1998
|
-
"""Make the errors context result with the refs value depends on the nested
|
1999
|
-
execute func.
|
2000
|
-
|
2001
|
-
Args:
|
2002
|
-
context: (DictData) A context data.
|
2003
|
-
error: (StageError) A stage exception object.
|
2004
|
-
"""
|
2005
|
-
if "errors" in context:
|
2006
|
-
context["errors"][error.refs] = error.to_dict()
|
2007
|
-
else:
|
2008
|
-
context["errors"] = error.to_dict(with_refs=True)
|
2009
|
-
|
2010
|
-
async def async_process(
|
2164
|
+
def dryrun(
|
2011
2165
|
self,
|
2012
2166
|
params: DictData,
|
2013
2167
|
run_id: str,
|
@@ -2015,27 +2169,90 @@ class BaseNestedStage(BaseRetryStage, ABC):
|
|
2015
2169
|
*,
|
2016
2170
|
parent_run_id: Optional[str] = None,
|
2017
2171
|
event: Optional[Event] = None,
|
2018
|
-
) -> Result:
|
2019
|
-
"""
|
2172
|
+
) -> Result: # pragma: no cov
|
2173
|
+
"""Override the dryrun method for this CallStage.
|
2174
|
+
|
2175
|
+
Steps:
|
2176
|
+
- Pre-hook caller function that exist.
|
2177
|
+
- Show function parameters
|
2020
2178
|
|
2021
2179
|
Args:
|
2022
|
-
params: A parameter data that want to use in this
|
2180
|
+
params (DictData): A parameter data that want to use in this
|
2023
2181
|
execution.
|
2024
|
-
run_id: A running stage ID.
|
2025
|
-
context: A context data
|
2026
|
-
|
2027
|
-
|
2028
|
-
|
2182
|
+
run_id (str): A running stage ID.
|
2183
|
+
context (DictData): A context data that was passed from handler
|
2184
|
+
method.
|
2185
|
+
parent_run_id (str, default None): A parent running ID.
|
2186
|
+
event (Event, default None): An event manager that use to track
|
2187
|
+
parent process was not force stopped.
|
2029
2188
|
|
2030
|
-
Returns:
|
2031
|
-
Result: The execution result with status and context data.
|
2032
2189
|
"""
|
2033
|
-
|
2034
|
-
|
2190
|
+
trace: Trace = get_trace(
|
2191
|
+
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2192
|
+
)
|
2193
|
+
call_func: TagFunc = self.get_caller(params=params)()
|
2194
|
+
trace.info(f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'")
|
2195
|
+
|
2196
|
+
args: DictData = {
|
2197
|
+
"result": Result(
|
2198
|
+
run_id=run_id,
|
2199
|
+
parent_run_id=parent_run_id,
|
2200
|
+
status=WAIT,
|
2201
|
+
context=context,
|
2202
|
+
extras=self.extras,
|
2203
|
+
),
|
2204
|
+
"extras": self.extras,
|
2205
|
+
} | self.pass_template(self.args, params)
|
2206
|
+
|
2207
|
+
# NOTE: Catch the necessary parameters.
|
2208
|
+
sig: inspect.Signature = inspect.signature(call_func)
|
2209
|
+
trace.debug(f"[STAGE]: {sig.parameters}")
|
2210
|
+
necessary_params: list[str] = []
|
2211
|
+
has_keyword: bool = False
|
2212
|
+
for k in sig.parameters:
|
2213
|
+
if (
|
2214
|
+
v := sig.parameters[k]
|
2215
|
+
).default == Parameter.empty and v.kind not in (
|
2216
|
+
Parameter.VAR_KEYWORD,
|
2217
|
+
Parameter.VAR_POSITIONAL,
|
2218
|
+
):
|
2219
|
+
necessary_params.append(k)
|
2220
|
+
elif v.kind == Parameter.VAR_KEYWORD:
|
2221
|
+
has_keyword = True
|
2222
|
+
|
2223
|
+
func_typed: dict[str, Any] = get_type_hints(call_func)
|
2224
|
+
map_type: str = "||".join(
|
2225
|
+
f"\t{p}: {func_typed[p]}"
|
2226
|
+
for p in necessary_params
|
2227
|
+
if p in func_typed
|
2228
|
+
)
|
2229
|
+
map_type_args: str = "||".join(f"\t{a}: {type(a)}" for a in args)
|
2230
|
+
|
2231
|
+
if not has_keyword:
|
2232
|
+
for k in ("result", "extras"):
|
2233
|
+
if k not in sig.parameters:
|
2234
|
+
args.pop(k)
|
2235
|
+
|
2236
|
+
trace.info(
|
2237
|
+
f"[STAGE]: Details"
|
2238
|
+
f"||Necessary Params:"
|
2239
|
+
f"||{map_type}"
|
2240
|
+
f"||Supported Keyword Params: {has_keyword}"
|
2241
|
+
f"||Return Type: {func_typed['return']}"
|
2242
|
+
f"||Argument Params:"
|
2243
|
+
f"||{map_type_args}"
|
2244
|
+
f"||"
|
2245
|
+
)
|
2246
|
+
return Result(
|
2247
|
+
run_id=run_id,
|
2248
|
+
parent_run_id=parent_run_id,
|
2249
|
+
status=SUCCESS,
|
2250
|
+
context=catch(context=context, status=SUCCESS),
|
2251
|
+
extras=self.extras,
|
2035
2252
|
)
|
2036
2253
|
|
2037
2254
|
|
2038
|
-
class TriggerStage(
|
2255
|
+
class TriggerStage(BaseRetryStage):
|
2039
2256
|
"""Trigger workflow executor stage that run an input trigger Workflow
|
2040
2257
|
execute method. This is the stage that allow you to create the reusable
|
2041
2258
|
Workflow template with dynamic parameters.
|
@@ -2043,12 +2260,13 @@ class TriggerStage(BaseNestedStage):
|
|
2043
2260
|
This stage does not allow to pass the workflow model directly to the
|
2044
2261
|
trigger field. A trigger workflow name should exist on the config path only.
|
2045
2262
|
|
2046
|
-
|
2047
|
-
>>> stage = {
|
2263
|
+
Examples:
|
2264
|
+
>>> stage = TriggerStage.model_validate({
|
2265
|
+
... "id": "trigger-stage",
|
2048
2266
|
... "name": "Trigger workflow stage execution",
|
2049
2267
|
... "trigger": 'workflow-name-for-loader',
|
2050
2268
|
... "params": {"run-date": "2024-08-01", "source": "src"},
|
2051
|
-
... }
|
2269
|
+
... })
|
2052
2270
|
"""
|
2053
2271
|
|
2054
2272
|
trigger: str = Field(
|
@@ -2093,55 +2311,135 @@ class TriggerStage(BaseNestedStage):
|
|
2093
2311
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
2094
2312
|
)
|
2095
2313
|
_trigger: str = param2template(self.trigger, params, extras=self.extras)
|
2096
|
-
if _trigger == self.extras.get("
|
2314
|
+
if _trigger == self.extras.get("__sys_exec_break_circle", "NOTSET"):
|
2097
2315
|
raise StageError("Circle execute via trigger itself workflow name.")
|
2316
|
+
|
2098
2317
|
trace.info(f"[NESTED]: Load Workflow Config: {_trigger!r}")
|
2099
|
-
|
2318
|
+
workflow: Workflow = Workflow.from_conf(
|
2100
2319
|
name=pass_env(_trigger),
|
2101
2320
|
extras=self.extras,
|
2102
|
-
)
|
2103
|
-
|
2321
|
+
)
|
2322
|
+
|
2323
|
+
if event and event.is_set():
|
2324
|
+
raise StageCancelError("Cancel before start trigger process.")
|
2325
|
+
|
2326
|
+
# IMPORTANT: Should not use the `pass_env` function on this `params`
|
2327
|
+
# parameter.
|
2328
|
+
result: Result = workflow.execute(
|
2104
2329
|
params=param2template(self.params, params, extras=self.extras),
|
2105
2330
|
run_id=parent_run_id,
|
2106
2331
|
event=event,
|
2107
2332
|
)
|
2333
|
+
catch(context, status=result.status, updated=result.context)
|
2108
2334
|
if result.status == FAILED:
|
2109
2335
|
err_msg: str = (
|
2110
2336
|
f" with:\n{msg}"
|
2111
2337
|
if (msg := result.context.get("errors", {}).get("message"))
|
2112
2338
|
else "."
|
2113
2339
|
)
|
2114
|
-
|
2115
|
-
|
2116
|
-
context={
|
2117
|
-
"status": FAILED,
|
2118
|
-
"errors": StageError(
|
2119
|
-
f"Trigger workflow was failed{err_msg}"
|
2120
|
-
).to_dict(),
|
2121
|
-
},
|
2340
|
+
err = StageError(
|
2341
|
+
f"Trigger workflow was failed{err_msg}", allow_traceback=False
|
2122
2342
|
)
|
2343
|
+
raise err
|
2123
2344
|
elif result.status == CANCEL:
|
2124
|
-
|
2125
|
-
status=CANCEL,
|
2126
|
-
context={
|
2127
|
-
"status": CANCEL,
|
2128
|
-
"errors": StageCancelError(
|
2129
|
-
"Trigger workflow was cancel."
|
2130
|
-
).to_dict(),
|
2131
|
-
},
|
2132
|
-
)
|
2345
|
+
raise StageCancelError("Trigger workflow was cancel.")
|
2133
2346
|
elif result.status == SKIP:
|
2134
|
-
|
2135
|
-
status=SKIP,
|
2136
|
-
context={
|
2137
|
-
"status": SKIP,
|
2138
|
-
"errors": StageSkipError(
|
2139
|
-
"Trigger workflow was skipped."
|
2140
|
-
).to_dict(),
|
2141
|
-
},
|
2142
|
-
)
|
2347
|
+
raise StageSkipError("Trigger workflow was skipped.")
|
2143
2348
|
return result
|
2144
2349
|
|
2350
|
+
async def async_process(
|
2351
|
+
self,
|
2352
|
+
params: DictData,
|
2353
|
+
run_id: str,
|
2354
|
+
context: DictData,
|
2355
|
+
*,
|
2356
|
+
parent_run_id: Optional[str] = None,
|
2357
|
+
event: Optional[Event] = None,
|
2358
|
+
) -> Result: # pragma: no cov
|
2359
|
+
"""Async process for trigger-stage do not implement yet.
|
2360
|
+
|
2361
|
+
Args:
|
2362
|
+
params: A parameter data that want to use in this
|
2363
|
+
execution.
|
2364
|
+
run_id: A running stage ID.
|
2365
|
+
context: A context data.
|
2366
|
+
parent_run_id: A parent running ID. (Default is None)
|
2367
|
+
event: An event manager that use to track parent process
|
2368
|
+
was not force stopped.
|
2369
|
+
|
2370
|
+
Returns:
|
2371
|
+
Result: The execution result with status and context data.
|
2372
|
+
"""
|
2373
|
+
raise NotImplementedError(
|
2374
|
+
"The Trigger stage does not implement the `axecute` method yet."
|
2375
|
+
)
|
2376
|
+
|
2377
|
+
|
2378
|
+
class BaseNestedStage(BaseAsyncStage, ABC):
|
2379
|
+
"""Base Nested Stage model. This model is use for checking the child stage
|
2380
|
+
is the nested stage or not.
|
2381
|
+
"""
|
2382
|
+
|
2383
|
+
def set_outputs(
|
2384
|
+
self, output: DictData, to: DictData, info: Optional[DictData] = None
|
2385
|
+
) -> DictData:
|
2386
|
+
"""Override the set outputs method that support for nested-stage."""
|
2387
|
+
return super().set_outputs(output, to=to)
|
2388
|
+
|
2389
|
+
def get_outputs(self, output: DictData) -> DictData:
|
2390
|
+
"""Override the get outputs method that support for nested-stage"""
|
2391
|
+
return super().get_outputs(output)
|
2392
|
+
|
2393
|
+
@property
|
2394
|
+
def is_nested(self) -> bool:
|
2395
|
+
"""Check if this stage is a nested stage or not.
|
2396
|
+
|
2397
|
+
Returns:
|
2398
|
+
bool: True only.
|
2399
|
+
"""
|
2400
|
+
return True
|
2401
|
+
|
2402
|
+
@staticmethod
|
2403
|
+
def mark_errors(context: DictData, error: StageError) -> None:
|
2404
|
+
"""Make the errors context result with the refs value depends on the nested
|
2405
|
+
execute func.
|
2406
|
+
|
2407
|
+
Args:
|
2408
|
+
context: (DictData) A context data.
|
2409
|
+
error: (StageError) A stage exception object.
|
2410
|
+
"""
|
2411
|
+
if "errors" in context:
|
2412
|
+
context["errors"][error.refs] = error.to_dict()
|
2413
|
+
else:
|
2414
|
+
context["errors"] = error.to_dict(with_refs=True)
|
2415
|
+
|
2416
|
+
async def async_process(
|
2417
|
+
self,
|
2418
|
+
params: DictData,
|
2419
|
+
run_id: str,
|
2420
|
+
context: DictData,
|
2421
|
+
*,
|
2422
|
+
parent_run_id: Optional[str] = None,
|
2423
|
+
event: Optional[Event] = None,
|
2424
|
+
) -> Result:
|
2425
|
+
"""Async process for nested-stage do not implement yet.
|
2426
|
+
|
2427
|
+
Args:
|
2428
|
+
params: A parameter data that want to use in this
|
2429
|
+
execution.
|
2430
|
+
run_id: A running stage ID.
|
2431
|
+
context: A context data.
|
2432
|
+
parent_run_id: A parent running ID. (Default is None)
|
2433
|
+
event: An event manager that use to track parent process
|
2434
|
+
was not force stopped.
|
2435
|
+
|
2436
|
+
Returns:
|
2437
|
+
Result: The execution result with status and context data.
|
2438
|
+
"""
|
2439
|
+
raise NotImplementedError(
|
2440
|
+
"The nested-stage does not implement the `axecute` method yet."
|
2441
|
+
)
|
2442
|
+
|
2145
2443
|
|
2146
2444
|
class ParallelContext(TypedDict):
|
2147
2445
|
branch: str
|
@@ -2156,8 +2454,9 @@ class ParallelStage(BaseNestedStage):
|
|
2156
2454
|
This stage is not the low-level stage model because it runs multi-stages
|
2157
2455
|
in this stage execution.
|
2158
2456
|
|
2159
|
-
|
2160
|
-
>>> stage = {
|
2457
|
+
Examples:
|
2458
|
+
>>> stage = ParallelStage.model_validate({
|
2459
|
+
... "id": "parallel-stage",
|
2161
2460
|
... "name": "Parallel stage execution.",
|
2162
2461
|
... "parallel": {
|
2163
2462
|
... "branch01": [
|
@@ -2179,7 +2478,7 @@ class ParallelStage(BaseNestedStage):
|
|
2179
2478
|
... },
|
2180
2479
|
... ],
|
2181
2480
|
... }
|
2182
|
-
... }
|
2481
|
+
... })
|
2183
2482
|
"""
|
2184
2483
|
|
2185
2484
|
parallel: dict[str, list[Stage]] = Field(
|
@@ -2429,8 +2728,9 @@ class ForEachStage(BaseNestedStage):
|
|
2429
2728
|
This stage is not the low-level stage model because it runs
|
2430
2729
|
multi-stages in this stage execution.
|
2431
2730
|
|
2432
|
-
|
2433
|
-
>>> stage = {
|
2731
|
+
Examples:
|
2732
|
+
>>> stage = ForEachStage.model_validate({
|
2733
|
+
... "id": "foreach-stage",
|
2434
2734
|
... "name": "For-each stage execution",
|
2435
2735
|
... "foreach": [1, 2, 3]
|
2436
2736
|
... "stages": [
|
@@ -2439,7 +2739,7 @@ class ForEachStage(BaseNestedStage):
|
|
2439
2739
|
... "echo": "Start run with item ${{ item }}"
|
2440
2740
|
... },
|
2441
2741
|
... ],
|
2442
|
-
... }
|
2742
|
+
... })
|
2443
2743
|
"""
|
2444
2744
|
|
2445
2745
|
foreach: EachType = Field(
|
@@ -2614,15 +2914,18 @@ class ForEachStage(BaseNestedStage):
|
|
2614
2914
|
"""Validate foreach value that already passed to this model.
|
2615
2915
|
|
2616
2916
|
Args:
|
2617
|
-
value:
|
2917
|
+
value (Any): An any foreach value.
|
2618
2918
|
|
2619
2919
|
Raises:
|
2620
2920
|
TypeError: If value can not try-convert to list type.
|
2621
|
-
ValueError:
|
2921
|
+
ValueError: If the foreach value is dict type.
|
2922
|
+
ValueError: If the foreach value contain duplication item without
|
2923
|
+
enable using index as key flag.
|
2622
2924
|
|
2623
2925
|
Returns:
|
2624
2926
|
list[Any]: list of item.
|
2625
2927
|
"""
|
2928
|
+
# NOTE: Try to cast a foreach with string type to list of items.
|
2626
2929
|
if isinstance(value, str):
|
2627
2930
|
try:
|
2628
2931
|
value: list[Any] = str2list(value)
|
@@ -2631,6 +2934,7 @@ class ForEachStage(BaseNestedStage):
|
|
2631
2934
|
f"Does not support string foreach: {value!r} that can "
|
2632
2935
|
f"not convert to list."
|
2633
2936
|
) from e
|
2937
|
+
|
2634
2938
|
# [VALIDATE]: Type of the foreach should be `list` type.
|
2635
2939
|
elif isinstance(value, dict):
|
2636
2940
|
raise TypeError(
|
@@ -2752,8 +3056,9 @@ class UntilStage(BaseNestedStage):
|
|
2752
3056
|
This stage is not the low-level stage model because it runs
|
2753
3057
|
multi-stages in this stage execution.
|
2754
3058
|
|
2755
|
-
|
2756
|
-
>>> stage = {
|
3059
|
+
Examples:
|
3060
|
+
>>> stage = UntilStage.model_validate({
|
3061
|
+
... "id": "until-stage",
|
2757
3062
|
... "name": "Until stage execution",
|
2758
3063
|
... "item": 1,
|
2759
3064
|
... "until": "${{ item }} > 3"
|
@@ -2766,7 +3071,7 @@ class UntilStage(BaseNestedStage):
|
|
2766
3071
|
... )
|
2767
3072
|
... },
|
2768
3073
|
... ],
|
2769
|
-
... }
|
3074
|
+
... })
|
2770
3075
|
"""
|
2771
3076
|
|
2772
3077
|
item: Union[str, int, bool] = Field(
|
@@ -2776,7 +3081,7 @@ class UntilStage(BaseNestedStage):
|
|
2776
3081
|
),
|
2777
3082
|
)
|
2778
3083
|
until: str = Field(description="A until condition for stop the while loop.")
|
2779
|
-
stages: list[
|
3084
|
+
stages: list[SubStage] = Field(
|
2780
3085
|
default_factory=list,
|
2781
3086
|
description=(
|
2782
3087
|
"A list of stage that will run with each item in until loop."
|
@@ -3037,6 +3342,8 @@ class Match(BaseModel):
|
|
3037
3342
|
|
3038
3343
|
|
3039
3344
|
class Else(BaseModel):
|
3345
|
+
"""Else model for the Case Stage."""
|
3346
|
+
|
3040
3347
|
other: list[Stage] = Field(
|
3041
3348
|
description="A list of stage that does not match any case.",
|
3042
3349
|
alias="else",
|
@@ -3046,8 +3353,9 @@ class Else(BaseModel):
|
|
3046
3353
|
class CaseStage(BaseNestedStage):
|
3047
3354
|
"""Case stage executor that execute all stages if the condition was matched.
|
3048
3355
|
|
3049
|
-
|
3050
|
-
>>> stage = {
|
3356
|
+
Examples:
|
3357
|
+
>>> stage = CaseStage.model_validate({
|
3358
|
+
... "id": "case-stage",
|
3051
3359
|
... "name": "If stage execution.",
|
3052
3360
|
... "case": "${{ param.test }}",
|
3053
3361
|
... "match": [
|
@@ -3070,9 +3378,10 @@ class CaseStage(BaseNestedStage):
|
|
3070
3378
|
... ],
|
3071
3379
|
... },
|
3072
3380
|
... ],
|
3073
|
-
... }
|
3381
|
+
... })
|
3074
3382
|
|
3075
|
-
>>> stage = {
|
3383
|
+
>>> stage = CaseStage.model_validate({
|
3384
|
+
... "id": "case-stage",
|
3076
3385
|
... "name": "If stage execution.",
|
3077
3386
|
... "case": "${{ param.test }}",
|
3078
3387
|
... "match": [
|
@@ -3094,7 +3403,7 @@ class CaseStage(BaseNestedStage):
|
|
3094
3403
|
... ],
|
3095
3404
|
... },
|
3096
3405
|
... ],
|
3097
|
-
... }
|
3406
|
+
... })
|
3098
3407
|
|
3099
3408
|
"""
|
3100
3409
|
|
@@ -3113,9 +3422,16 @@ class CaseStage(BaseNestedStage):
|
|
3113
3422
|
|
3114
3423
|
@field_validator("match", mode="after")
|
3115
3424
|
def __validate_match(
|
3116
|
-
cls,
|
3425
|
+
cls,
|
3426
|
+
match: list[Union[Match, Else]],
|
3117
3427
|
) -> list[Union[Match, Else]]:
|
3118
|
-
"""Validate the match field should contain only one Else model.
|
3428
|
+
"""Validate the match field should contain only one Else model.
|
3429
|
+
|
3430
|
+
Raises:
|
3431
|
+
ValueError: If match field contain Else more than 1 model.
|
3432
|
+
ValueError: If match field contain Match with '_' case (it represent
|
3433
|
+
the else case) more than 1 model.
|
3434
|
+
"""
|
3119
3435
|
c_else_case: int = 0
|
3120
3436
|
c_else_model: int = 0
|
3121
3437
|
for m in match:
|
@@ -3314,8 +3630,10 @@ class CaseStage(BaseNestedStage):
|
|
3314
3630
|
case: StrOrNone = param2template(self.case, params, extras=self.extras)
|
3315
3631
|
trace.info(f"[NESTED]: Get Case: {case!r}.")
|
3316
3632
|
case, stages = self.extract_stages_from_case(case, params=params)
|
3633
|
+
|
3317
3634
|
if event and event.is_set():
|
3318
3635
|
raise StageCancelError("Cancel before start case process.")
|
3636
|
+
|
3319
3637
|
status, context = self._process_nested(
|
3320
3638
|
case=case,
|
3321
3639
|
stages=stages,
|
@@ -3337,11 +3655,12 @@ class RaiseStage(BaseAsyncStage):
|
|
3337
3655
|
"""Raise error stage executor that raise `StageError` that use a message
|
3338
3656
|
field for making error message before raise.
|
3339
3657
|
|
3340
|
-
|
3341
|
-
>>> stage = {
|
3658
|
+
Examples:
|
3659
|
+
>>> stage = RaiseStage.model_validate({
|
3660
|
+
... "id": "raise-stage",
|
3342
3661
|
... "name": "Raise stage",
|
3343
3662
|
... "raise": "raise this stage",
|
3344
|
-
... }
|
3663
|
+
... })
|
3345
3664
|
|
3346
3665
|
"""
|
3347
3666
|
|
@@ -3414,7 +3733,7 @@ class RaiseStage(BaseAsyncStage):
|
|
3414
3733
|
raise StageError(message)
|
3415
3734
|
|
3416
3735
|
|
3417
|
-
class DockerStage(
|
3736
|
+
class DockerStage(BaseRetryStage): # pragma: no cov
|
3418
3737
|
"""Docker container stage execution that will pull the specific Docker image
|
3419
3738
|
with custom authentication and run this image by passing environment
|
3420
3739
|
variables and mounting local volume to this Docker container.
|
@@ -3437,6 +3756,7 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
3437
3756
|
... }
|
3438
3757
|
"""
|
3439
3758
|
|
3759
|
+
action_stage: ClassVar[bool] = True
|
3440
3760
|
image: str = Field(
|
3441
3761
|
description="A Docker image url with tag that want to run.",
|
3442
3762
|
)
|
@@ -3593,6 +3913,33 @@ class DockerStage(BaseStage): # pragma: no cov
|
|
3593
3913
|
trace.info(f"[STAGE]: Docker: {self.image}:{self.tag}")
|
3594
3914
|
raise NotImplementedError("Docker Stage does not implement yet.")
|
3595
3915
|
|
3916
|
+
async def async_process(
|
3917
|
+
self,
|
3918
|
+
params: DictData,
|
3919
|
+
run_id: str,
|
3920
|
+
context: DictData,
|
3921
|
+
*,
|
3922
|
+
parent_run_id: Optional[str] = None,
|
3923
|
+
event: Optional[Event] = None,
|
3924
|
+
) -> Result: # pragma: no cov
|
3925
|
+
"""Async process for nested-stage do not implement yet.
|
3926
|
+
|
3927
|
+
Args:
|
3928
|
+
params: A parameter data that want to use in this
|
3929
|
+
execution.
|
3930
|
+
run_id: A running stage ID.
|
3931
|
+
context: A context data.
|
3932
|
+
parent_run_id: A parent running ID. (Default is None)
|
3933
|
+
event: An event manager that use to track parent process
|
3934
|
+
was not force stopped.
|
3935
|
+
|
3936
|
+
Returns:
|
3937
|
+
Result: The execution result with status and context data.
|
3938
|
+
"""
|
3939
|
+
raise NotImplementedError(
|
3940
|
+
"The Docker stage does not implement the `axecute` method yet."
|
3941
|
+
)
|
3942
|
+
|
3596
3943
|
|
3597
3944
|
class VirtualPyStage(PyStage): # pragma: no cov
|
3598
3945
|
"""Virtual Python stage executor that run Python statement on the dependent
|
@@ -3614,7 +3961,7 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3614
3961
|
)
|
3615
3962
|
|
3616
3963
|
@contextlib.contextmanager
|
3617
|
-
def
|
3964
|
+
def make_py_file(
|
3618
3965
|
self,
|
3619
3966
|
py: str,
|
3620
3967
|
values: DictData,
|
@@ -3629,7 +3976,7 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3629
3976
|
|
3630
3977
|
Args:
|
3631
3978
|
py: A Python string statement.
|
3632
|
-
values: A variable that want to set before running
|
3979
|
+
values: A variable that want to set before running these
|
3633
3980
|
deps: An additional Python dependencies that want install before
|
3634
3981
|
run this python stage.
|
3635
3982
|
run_id: (StrOrNone) A running ID of this stage execution.
|
@@ -3690,13 +4037,14 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3690
4037
|
- Execution python file with `uv run` via Python subprocess module.
|
3691
4038
|
|
3692
4039
|
Args:
|
3693
|
-
params: A parameter data that want to use in this
|
4040
|
+
params (DictData): A parameter data that want to use in this
|
3694
4041
|
execution.
|
3695
|
-
run_id: A running stage ID.
|
3696
|
-
context: A context data
|
3697
|
-
|
3698
|
-
|
3699
|
-
|
4042
|
+
run_id (str): A running stage ID.
|
4043
|
+
context (DictData): A context data that was passed from handler
|
4044
|
+
method.
|
4045
|
+
parent_run_id (str, default None): A parent running ID.
|
4046
|
+
event (Event, default None): An event manager that use to track
|
4047
|
+
parent process was not force stopped.
|
3700
4048
|
|
3701
4049
|
Returns:
|
3702
4050
|
Result: The execution result with status and context data.
|
@@ -3705,12 +4053,18 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3705
4053
|
run_id, parent_run_id=parent_run_id, extras=self.extras
|
3706
4054
|
)
|
3707
4055
|
run: str = param2template(dedent(self.run), params, extras=self.extras)
|
3708
|
-
with self.
|
4056
|
+
with self.make_py_file(
|
3709
4057
|
py=run,
|
3710
4058
|
values=param2template(self.vars, params, extras=self.extras),
|
3711
4059
|
deps=param2template(self.deps, params, extras=self.extras),
|
3712
4060
|
run_id=run_id,
|
3713
4061
|
) as py:
|
4062
|
+
|
4063
|
+
if event and event.is_set():
|
4064
|
+
raise StageCancelError(
|
4065
|
+
"Cancel before start virtual python process."
|
4066
|
+
)
|
4067
|
+
|
3714
4068
|
trace.debug(f"[STAGE]: Create `{py}` file.")
|
3715
4069
|
rs: CompletedProcess = subprocess.run(
|
3716
4070
|
["python", "-m", "uv", "run", py, "--no-cache"],
|
@@ -3756,12 +4110,24 @@ class VirtualPyStage(PyStage): # pragma: no cov
|
|
3756
4110
|
parent_run_id: Optional[str] = None,
|
3757
4111
|
event: Optional[Event] = None,
|
3758
4112
|
) -> Result:
|
4113
|
+
"""Async execution method for this Virtual Python stage.
|
4114
|
+
|
4115
|
+
Args:
|
4116
|
+
params (DictData): A parameter data that want to use in this
|
4117
|
+
execution.
|
4118
|
+
run_id (str): A running stage ID.
|
4119
|
+
context (DictData): A context data that was passed from handler
|
4120
|
+
method.
|
4121
|
+
parent_run_id (str, default None): A parent running ID.
|
4122
|
+
event (Event, default None): An event manager that use to track
|
4123
|
+
parent process was not force stopped.
|
4124
|
+
"""
|
3759
4125
|
raise NotImplementedError(
|
3760
4126
|
"Async process of Virtual Python stage does not implement yet."
|
3761
4127
|
)
|
3762
4128
|
|
3763
4129
|
|
3764
|
-
|
4130
|
+
SubStage = Annotated[
|
3765
4131
|
Union[
|
3766
4132
|
BashStage,
|
3767
4133
|
CallStage,
|
@@ -3777,7 +4143,10 @@ NestedStage = Annotated[
|
|
3777
4143
|
],
|
3778
4144
|
Field(
|
3779
4145
|
union_mode="smart",
|
3780
|
-
description=
|
4146
|
+
description=(
|
4147
|
+
"A nested-stage allow list that able to use on the NestedStage "
|
4148
|
+
"model."
|
4149
|
+
),
|
3781
4150
|
),
|
3782
4151
|
] # pragma: no cov
|
3783
4152
|
|
@@ -3790,6 +4159,7 @@ ActionStage = Annotated[
|
|
3790
4159
|
PyStage,
|
3791
4160
|
RaiseStage,
|
3792
4161
|
DockerStage,
|
4162
|
+
TriggerStage,
|
3793
4163
|
EmptyStage,
|
3794
4164
|
],
|
3795
4165
|
Field(
|