ddeutil-workflow 0.0.83__py3-none-any.whl → 0.0.85__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -138,10 +138,10 @@ class BaseStage(BaseModel, ABC):
138
138
  implement, ensuring consistent behavior across different stage types.
139
139
 
140
140
  This abstract class handles core stage functionality including:
141
- - Stage identification and naming
142
- - Conditional execution logic
143
- - Output management and templating
144
- - Execution lifecycle management
141
+ - Stage identification and naming
142
+ - Conditional execution logic
143
+ - Output management and templating
144
+ - Execution lifecycle management
145
145
 
146
146
  Custom stages should inherit from this class and implement the abstract
147
147
  `process()` method to define their specific execution behavior.
@@ -162,7 +162,6 @@ class BaseStage(BaseModel, ABC):
162
162
  ...
163
163
  ... def process(self, params: DictData, **kwargs) -> Result:
164
164
  ... return Result(status=SUCCESS)
165
- ```
166
165
  """
167
166
 
168
167
  action_stage: ClassVar[bool] = False
@@ -206,13 +205,13 @@ class BaseStage(BaseModel, ABC):
206
205
  return self.id or self.name
207
206
 
208
207
  @field_validator("desc", mode="after")
209
- def ___prepare_desc__(cls, value: str) -> str:
208
+ def ___prepare_desc__(cls, value: Optional[str]) -> Optional[str]:
210
209
  """Prepare description string that was created on a template.
211
210
 
212
211
  Returns:
213
212
  str: A dedent and left strip newline of description string.
214
213
  """
215
- return dedent(value.lstrip("\n"))
214
+ return value if value is None else dedent(value.lstrip("\n"))
216
215
 
217
216
  @model_validator(mode="after")
218
217
  def __prepare_running_id(self) -> Self:
@@ -240,7 +239,8 @@ class BaseStage(BaseModel, ABC):
240
239
 
241
240
  Args:
242
241
  value (Any): An any value.
243
- params (DictData):
242
+ params (DictData): A parameter data that want to use in this
243
+ execution.
244
244
 
245
245
  Returns:
246
246
  Any: A templated value.
@@ -264,10 +264,11 @@ class BaseStage(BaseModel, ABC):
264
264
  params (DictData): A parameter data that want to use in this
265
265
  execution.
266
266
  run_id (str): A running stage ID.
267
- context (DictData): A context data.
268
- parent_run_id: A parent running ID. (Default is None)
269
- event: An event manager that use to track parent process
270
- was not force stopped.
267
+ context (DictData): A context data that was passed from handler
268
+ method.
269
+ parent_run_id (str, default None): A parent running ID.
270
+ event (Event, default None): An event manager that use to track
271
+ parent process was not force stopped.
271
272
 
272
273
  Returns:
273
274
  Result: The execution result with status and context data.
@@ -308,10 +309,10 @@ class BaseStage(BaseModel, ABC):
308
309
  object from the current stage ID before release the final result.
309
310
 
310
311
  Args:
311
- params: A parameter data.
312
- run_id: A running stage ID. (Default is None)
313
- event: An event manager that pass to the stage execution.
314
- (Default is None)
312
+ params (DictData): A parameter data.
313
+ run_id (str, default None): A running ID.
314
+ event (Event, default None): An event manager that pass to the stage
315
+ execution.
315
316
 
316
317
  Returns:
317
318
  Result: The execution result with updated status and context.
@@ -370,28 +371,26 @@ class BaseStage(BaseModel, ABC):
370
371
  StageNestedError,
371
372
  StageError,
372
373
  ) as e: # pragma: no cov
374
+ updated: Optional[DictData] = {"errors": e.to_dict()}
373
375
  if isinstance(e, StageNestedError):
374
- trace.info(f"[STAGE]: Nested: {e}")
376
+ trace.error(f"[STAGE]: Nested: {e}")
375
377
  elif isinstance(e, (StageSkipError, StageNestedSkipError)):
376
- trace.info(f"[STAGE]: ⏭️ Skip: {e}")
377
- else:
378
- trace.info(
378
+ trace.error(f"[STAGE]: ⏭️ Skip: {e}")
379
+ updated = None
380
+ elif e.allow_traceback:
381
+ trace.error(
379
382
  f"[STAGE]: Stage Failed:||🚨 {traceback.format_exc()}||"
380
383
  )
384
+ else:
385
+ trace.error(
386
+ f"[STAGE]: 🤫 Stage Failed with disable traceback:||{e}"
387
+ )
381
388
  st: Status = get_status_from_error(e)
382
389
  return Result(
383
390
  run_id=run_id,
384
391
  parent_run_id=parent_run_id,
385
392
  status=st,
386
- context=catch(
387
- context,
388
- status=st,
389
- updated=(
390
- None
391
- if isinstance(e, (StageSkipError, StageNestedSkipError))
392
- else {"errors": e.to_dict()}
393
- ),
394
- ),
393
+ context=catch(context, status=st, updated=updated),
395
394
  info={"execution_time": time.monotonic() - ts},
396
395
  extras=self.extras,
397
396
  )
@@ -409,6 +408,8 @@ class BaseStage(BaseModel, ABC):
409
408
  info={"execution_time": time.monotonic() - ts},
410
409
  extras=self.extras,
411
410
  )
411
+ finally:
412
+ trace.debug("[STAGE]: End Handler stage execution.")
412
413
 
413
414
  def _execute(
414
415
  self,
@@ -421,8 +422,10 @@ class BaseStage(BaseModel, ABC):
421
422
  """Wrapped the process method before returning to handler execution.
422
423
 
423
424
  Args:
424
- params: A parameter data that want to use in this
425
- execution.
425
+ params: A parameter data that want to use in this execution.
426
+ run_id (str):
427
+ context:
428
+ parent_run_id:
426
429
  event: An event manager that use to track parent process
427
430
  was not force stopped.
428
431
 
@@ -552,7 +555,7 @@ class BaseStage(BaseModel, ABC):
552
555
  # should use the `re` module to validate eval-string before
553
556
  # running.
554
557
  rs: bool = eval(
555
- param2template(self.condition, params, extras=self.extras),
558
+ self.pass_template(self.condition, params),
556
559
  globals() | params,
557
560
  {},
558
561
  )
@@ -583,7 +586,8 @@ class BaseStage(BaseModel, ABC):
583
586
  def is_nested(self) -> bool:
584
587
  """Return true if this stage is nested stage.
585
588
 
586
- :rtype: bool
589
+ Returns:
590
+ bool: True if this stage is nested stage.
587
591
  """
588
592
  return False
589
593
 
@@ -593,14 +597,46 @@ class BaseStage(BaseModel, ABC):
593
597
  Returns:
594
598
  DictData: A dict that was dumped from this model with alias mode.
595
599
  """
596
- return self.model_dump(by_alias=True)
600
+ return self.model_dump(
601
+ by_alias=True,
602
+ exclude_defaults=True,
603
+ exclude={"extras", "id", "name", "desc"},
604
+ )
597
605
 
598
- def md(self) -> str: # pragma: no cov
606
+ def md(self, level: int = 1) -> str: # pragma: no cov
599
607
  """Return generated document that will be the interface of this stage.
600
608
 
601
- :rtype: str
609
+ Args:
610
+ level (int, default 0): A header level that want to generate
611
+ markdown content.
612
+
613
+ Returns:
614
+ str
602
615
  """
603
- return self.desc
616
+ assert level >= 1, "Header level should gather than 0"
617
+
618
+ def align_newline(value: Optional[str]) -> str:
619
+ space: str = " " * 16
620
+ if value is None:
621
+ return ""
622
+ return value.rstrip("\n").replace("\n", f"\n{space}")
623
+
624
+ header: str = "#" * level
625
+ return dedent(
626
+ f"""
627
+ {header} Stage: {self.iden}\n
628
+ {align_newline(self.desc)}\n
629
+ #{header} Parameters\n
630
+ | name | type | default | description |
631
+ | --- | --- | --- | : --- : |\n\n
632
+ #{header} Details\n
633
+ ```json
634
+ {self.detail()}
635
+ ```
636
+ """.lstrip(
637
+ "\n"
638
+ )
639
+ )
604
640
 
605
641
  def dryrun(
606
642
  self,
@@ -610,26 +646,75 @@ class BaseStage(BaseModel, ABC):
610
646
  *,
611
647
  parent_run_id: Optional[str] = None,
612
648
  event: Optional[Event] = None,
613
- ) -> Optional[Result]: # pragma: no cov
649
+ ) -> Result:
614
650
  """Pre-process method that will use to run with dry-run mode, and it
615
- should be used before process method.
651
+ should be used replace of process method when workflow release set with
652
+ DRYRUN mode.
653
+
654
+ By default, this method will set logic to convert this stage model
655
+ to am EmptyStage if it is action stage before use process method
656
+ instead process itself.
657
+
658
+ Args:
659
+ params (DictData): A parameter data that want to use in this
660
+ execution.
661
+ run_id (str): A running stage ID.
662
+ context (DictData): A context data.
663
+ parent_run_id (str, default None): A parent running ID.
664
+ event (Event, default None): An event manager that use to track
665
+ parent process was not force stopped.
666
+
667
+ Returns:
668
+ Result: The execution result with status and context data.
616
669
  """
670
+ trace: Trace = get_trace(
671
+ run_id, parent_run_id=parent_run_id, extras=self.extras
672
+ )
673
+ trace.debug("[STAGE]: Start Dryrun ...")
674
+ if self.action_stage:
675
+ return self.to_empty().process(
676
+ params,
677
+ run_id,
678
+ context,
679
+ parent_run_id=parent_run_id,
680
+ event=event,
681
+ )
682
+ return self.process(
683
+ params, run_id, context, parent_run_id=parent_run_id, event=event
684
+ )
617
685
 
618
- def to_empty(self, sleep: int = 0.35) -> EmptyStage: # pragma: no cov
686
+ def to_empty(
687
+ self,
688
+ sleep: int = 0.35,
689
+ *,
690
+ message: Optional[str] = None,
691
+ ) -> EmptyStage:
619
692
  """Convert the current Stage model to the EmptyStage model for dry-run
620
693
  mode if the `action_stage` class attribute has set.
621
694
 
695
+ Some use-case for this method is use for deactivate.
696
+
697
+ Args:
698
+ sleep (int, default 0.35): An adjustment sleep time.
699
+ message (str, default None): A message that want to override default
700
+ message on EmptyStage model.
701
+
622
702
  Returns:
623
703
  EmptyStage: An EmptyStage model that passing itself model data to
624
704
  message.
625
705
  """
706
+ if isinstance(self, EmptyStage):
707
+ return self.model_copy(update={"sleep": sleep})
626
708
  return EmptyStage.model_validate(
627
709
  {
628
710
  "name": self.name,
629
711
  "id": self.id,
630
712
  "desc": self.desc,
631
713
  "if": self.condition,
632
- "echo": f"Convert from {self.__class__.__name__}",
714
+ "echo": (
715
+ message
716
+ or f"Convert from {self.__class__.__name__} to EmptyStage"
717
+ ),
633
718
  "sleep": sleep,
634
719
  }
635
720
  )
@@ -783,6 +868,8 @@ class BaseAsyncStage(BaseStage, ABC):
783
868
  info={"execution_time": time.monotonic() - ts},
784
869
  extras=self.extras,
785
870
  )
871
+ finally:
872
+ trace.debug("[STAGE]: End Handler stage process.")
786
873
 
787
874
  async def _axecute(
788
875
  self,
@@ -794,12 +881,14 @@ class BaseAsyncStage(BaseStage, ABC):
794
881
  ) -> Result:
795
882
  """Wrapped the axecute method before returning to handler axecute.
796
883
 
797
- :param params: (DictData) A parameter data that want to use in this
798
- execution.
799
- :param event: (Event) An event manager that use to track parent execute
800
- was not force stopped.
884
+ Args:
885
+ params: (DictData) A parameter data that want to use in this
886
+ execution.
887
+ event: (Event) An event manager that use to track parent execute
888
+ was not force stopped.
801
889
 
802
- :rtype: Result
890
+ Returns:
891
+ Result: A Result object.
803
892
  """
804
893
  catch(context, status=WAIT)
805
894
  return await self.async_process(
@@ -820,7 +909,10 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
820
909
  default=0,
821
910
  ge=0,
822
911
  lt=20,
823
- description="A retry number if stage execution get the error.",
912
+ description=(
913
+ "A retry number if stage process got the error exclude skip and "
914
+ "cancel exception class."
915
+ ),
824
916
  )
825
917
 
826
918
  def _execute(
@@ -834,12 +926,14 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
834
926
  """Wrapped the execute method with retry strategy before returning to
835
927
  handler execute.
836
928
 
837
- :param params: (DictData) A parameter data that want to use in this
838
- execution.
839
- :param event: (Event) An event manager that use to track parent execute
840
- was not force stopped.
929
+ Args:
930
+ params: (DictData) A parameter data that want to use in this
931
+ execution.
932
+ event: (Event) An event manager that use to track parent execute
933
+ was not force stopped.
841
934
 
842
- :rtype: Result
935
+ Returns:
936
+ Result: A Result object.
843
937
  """
844
938
  current_retry: int = 0
845
939
  exception: Exception
@@ -847,9 +941,19 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
847
941
  trace: Trace = get_trace(
848
942
  run_id, parent_run_id=parent_run_id, extras=self.extras
849
943
  )
850
-
851
944
  # NOTE: First execution for not pass to retry step if it passes.
852
945
  try:
946
+ if (
947
+ self.extras.get("__sys_release_dryrun_mode", False)
948
+ and self.action_stage
949
+ ):
950
+ return self.dryrun(
951
+ params | {"retry": current_retry},
952
+ run_id=run_id,
953
+ context=context,
954
+ parent_run_id=parent_run_id,
955
+ event=event,
956
+ )
853
957
  return self.process(
854
958
  params | {"retry": current_retry},
855
959
  run_id=run_id,
@@ -857,9 +961,19 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
857
961
  parent_run_id=parent_run_id,
858
962
  event=event,
859
963
  )
964
+ except (
965
+ StageNestedSkipError,
966
+ StageNestedCancelError,
967
+ StageSkipError,
968
+ StageCancelError,
969
+ ):
970
+ trace.debug("[STAGE]: process raise skip or cancel error.")
971
+ raise
860
972
  except Exception as e:
861
973
  current_retry += 1
862
974
  exception = e
975
+ finally:
976
+ trace.debug("[STAGE]: Failed at the first execution.")
863
977
 
864
978
  if self.retry == 0:
865
979
  raise exception
@@ -876,6 +990,17 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
876
990
  status=WAIT,
877
991
  updated={"retry": current_retry},
878
992
  )
993
+ if (
994
+ self.extras.get("__sys_release_dryrun_mode", False)
995
+ and self.action_stage
996
+ ):
997
+ return self.dryrun(
998
+ params | {"retry": current_retry},
999
+ run_id=run_id,
1000
+ context=context,
1001
+ parent_run_id=parent_run_id,
1002
+ event=event,
1003
+ )
879
1004
  return self.process(
880
1005
  params | {"retry": current_retry},
881
1006
  run_id=run_id,
@@ -884,10 +1009,10 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
884
1009
  event=event,
885
1010
  )
886
1011
  except (
887
- StageSkipError,
888
1012
  StageNestedSkipError,
889
- StageCancelError,
890
1013
  StageNestedCancelError,
1014
+ StageSkipError,
1015
+ StageCancelError,
891
1016
  ):
892
1017
  trace.debug("[STAGE]: process raise skip or cancel error.")
893
1018
  raise
@@ -916,12 +1041,14 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
916
1041
  """Wrapped the axecute method with retry strategy before returning to
917
1042
  handler axecute.
918
1043
 
919
- :param params: (DictData) A parameter data that want to use in this
920
- execution.
921
- :param event: (Event) An event manager that use to track parent execute
922
- was not force stopped.
1044
+ Args:
1045
+ params: (DictData) A parameter data that want to use in this
1046
+ execution.
1047
+ event: (Event) An event manager that use to track parent execute
1048
+ was not force stopped.
923
1049
 
924
- :rtype: Result
1050
+ Returns:
1051
+ Result: A Result object.
925
1052
  """
926
1053
  current_retry: int = 0
927
1054
  exception: Exception
@@ -932,6 +1059,17 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
932
1059
 
933
1060
  # NOTE: First execution for not pass to retry step if it passes.
934
1061
  try:
1062
+ if (
1063
+ self.extras.get("__sys_release_dryrun_mode", False)
1064
+ and self.action_stage
1065
+ ):
1066
+ return self.dryrun(
1067
+ params | {"retry": current_retry},
1068
+ run_id=run_id,
1069
+ context=context,
1070
+ parent_run_id=parent_run_id,
1071
+ event=event,
1072
+ )
935
1073
  return await self.async_process(
936
1074
  params | {"retry": current_retry},
937
1075
  run_id=run_id,
@@ -939,9 +1077,19 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
939
1077
  parent_run_id=parent_run_id,
940
1078
  event=event,
941
1079
  )
1080
+ except (
1081
+ StageNestedSkipError,
1082
+ StageNestedCancelError,
1083
+ StageSkipError,
1084
+ StageCancelError,
1085
+ ):
1086
+ await trace.adebug("[STAGE]: process raise skip or cancel error.")
1087
+ raise
942
1088
  except Exception as e:
943
1089
  current_retry += 1
944
1090
  exception = e
1091
+ finally:
1092
+ await trace.adebug("[STAGE]: Failed at the first execution.")
945
1093
 
946
1094
  if self.retry == 0:
947
1095
  raise exception
@@ -958,6 +1106,17 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
958
1106
  status=WAIT,
959
1107
  updated={"retry": current_retry},
960
1108
  )
1109
+ if (
1110
+ self.extras.get("__sys_release_dryrun_mode", False)
1111
+ and self.action_stage
1112
+ ):
1113
+ return self.dryrun(
1114
+ params | {"retry": current_retry},
1115
+ run_id=run_id,
1116
+ context=context,
1117
+ parent_run_id=parent_run_id,
1118
+ event=event,
1119
+ )
961
1120
  return await self.async_process(
962
1121
  params | {"retry": current_retry},
963
1122
  run_id=run_id,
@@ -966,10 +1125,10 @@ class BaseRetryStage(BaseAsyncStage, ABC): # pragma: no cov
966
1125
  event=event,
967
1126
  )
968
1127
  except (
969
- StageSkipError,
970
1128
  StageNestedSkipError,
971
- StageCancelError,
972
1129
  StageNestedCancelError,
1130
+ StageSkipError,
1131
+ StageCancelError,
973
1132
  ):
974
1133
  await trace.adebug(
975
1134
  "[STAGE]: process raise skip or cancel error."
@@ -1004,22 +1163,13 @@ class EmptyStage(BaseAsyncStage):
1004
1163
  for a specified duration, making it useful for workflow timing control
1005
1164
  and debugging scenarios.
1006
1165
 
1007
- Example:
1008
- ```yaml
1009
- stages:
1010
- - name: "Workflow Started"
1011
- echo: "Beginning data processing workflow"
1012
- sleep: 2
1013
-
1014
- - name: "Debug Parameters"
1015
- echo: "Processing file: ${{ params.filename }}"
1016
- ```
1017
-
1018
- >>> stage = EmptyStage(
1019
- ... name="Status Update",
1020
- ... echo="Processing completed successfully",
1021
- ... sleep=1.0
1022
- ... )
1166
+ Examples:
1167
+ >>> stage = EmptyStage.model_validate({
1168
+ ... "id": "empty-stage",
1169
+ ... "name": "Status Update",
1170
+ ... "echo": "Processing completed successfully",
1171
+ ... "sleep": 1.0,
1172
+ ... })
1023
1173
  """
1024
1174
 
1025
1175
  echo: StrOrNone = Field(
@@ -1063,6 +1213,9 @@ class EmptyStage(BaseAsyncStage):
1063
1213
  event: An event manager that use to track parent process
1064
1214
  was not force stopped.
1065
1215
 
1216
+ Raises:
1217
+ StageCancelError: If event was set before start process.
1218
+
1066
1219
  Returns:
1067
1220
  Result: The execution result with status and context data.
1068
1221
  """
@@ -1114,6 +1267,9 @@ class EmptyStage(BaseAsyncStage):
1114
1267
  event: An event manager that use to track parent process
1115
1268
  was not force stopped.
1116
1269
 
1270
+ Raises:
1271
+ StageCancelError: If event was set before start process.
1272
+
1117
1273
  Returns:
1118
1274
  Result: The execution result with status and context data.
1119
1275
  """
@@ -1155,16 +1311,18 @@ class BashStage(BaseRetryStage):
1155
1311
  statement. Thus, it will write the `.sh` file before start running bash
1156
1312
  command for fix this issue.
1157
1313
 
1158
- Data Validate:
1159
- >>> stage = {
1314
+ Examples:
1315
+ >>> stage = BaseStage.model_validate({
1316
+ ... "id": "bash-stage",
1160
1317
  ... "name": "The Shell stage execution",
1161
1318
  ... "bash": 'echo "Hello $FOO"',
1162
1319
  ... "env": {
1163
1320
  ... "FOO": "BAR",
1164
1321
  ... },
1165
- ... }
1322
+ ... })
1166
1323
  """
1167
1324
 
1325
+ action_stage: ClassVar[bool] = True
1168
1326
  bash: str = Field(
1169
1327
  description=(
1170
1328
  "A bash statement that want to execute via Python subprocess."
@@ -1179,17 +1337,20 @@ class BashStage(BaseRetryStage):
1179
1337
  )
1180
1338
 
1181
1339
  @contextlib.asynccontextmanager
1182
- async def async_create_sh_file(
1340
+ async def async_make_sh_file(
1183
1341
  self, bash: str, env: DictStr, run_id: StrOrNone = None
1184
1342
  ) -> AsyncIterator[TupleStr]:
1185
1343
  """Async create and write `.sh` file with the `aiofiles` package.
1186
1344
 
1187
- :param bash: (str) A bash statement.
1188
- :param env: (DictStr) An environment variable that set before run bash.
1189
- :param run_id: (StrOrNone) A running stage ID that use for writing sh
1190
- file instead generate by UUID4.
1345
+ Args:
1346
+ bash (str): A bash statement.
1347
+ env (DictStr): An environment variable that set before run bash.
1348
+ run_id (StrOrNone, default None): A running stage ID that use for
1349
+ writing `.sh` file instead generate by UUID4.
1191
1350
 
1192
- :rtype: AsyncIterator[TupleStr]
1351
+ Returns:
1352
+ AsyncIterator[TupleStr]: Return context of prepared bash statement
1353
+ that want to execute.
1193
1354
  """
1194
1355
  import aiofiles
1195
1356
 
@@ -1215,19 +1376,21 @@ class BashStage(BaseRetryStage):
1215
1376
  Path(f"./{f_name}").unlink()
1216
1377
 
1217
1378
  @contextlib.contextmanager
1218
- def create_sh_file(
1379
+ def make_sh_file(
1219
1380
  self, bash: str, env: DictStr, run_id: StrOrNone = None
1220
1381
  ) -> Iterator[TupleStr]:
1221
1382
  """Create and write the `.sh` file before giving this file name to
1222
1383
  context. After that, it will auto delete this file automatic.
1223
1384
 
1224
- :param bash: (str) A bash statement.
1225
- :param env: (DictStr) An environment variable that set before run bash.
1226
- :param run_id: (StrOrNone) A running stage ID that use for writing sh
1227
- file instead generate by UUID4.
1385
+ Args:
1386
+ bash (str): A bash statement.
1387
+ env (DictStr): An environment variable that set before run bash.
1388
+ run_id (StrOrNone, default None): A running stage ID that use for
1389
+ writing `.sh` file instead generate by UUID4.
1228
1390
 
1229
- :rtype: Iterator[TupleStr]
1230
- :return: Return context of prepared bash statement that want to execute.
1391
+ Returns:
1392
+ Iterator[TupleStr]: Return context of prepared bash statement that
1393
+ want to execute.
1231
1394
  """
1232
1395
  f_name: str = f"{run_id or uuid.uuid4()}.sh"
1233
1396
  f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
@@ -1269,13 +1432,19 @@ class BashStage(BaseRetryStage):
1269
1432
  `return_code`, `stdout`, and `stderr`.
1270
1433
 
1271
1434
  Args:
1272
- params: A parameter data that want to use in this
1435
+ params (DictData): A parameter data that want to use in this
1273
1436
  execution.
1274
- run_id: A running stage ID.
1275
- context: A context data.
1276
- parent_run_id: A parent running ID. (Default is None)
1277
- event: An event manager that use to track parent process
1278
- was not force stopped.
1437
+ run_id (str): A running stage ID.
1438
+ context (DictData): A context data that was passed from handler
1439
+ method.
1440
+ parent_run_id (str, default None): A parent running ID.
1441
+ event (Event, default None): An event manager that use to track
1442
+ parent process was not force stopped.
1443
+
1444
+ Raises:
1445
+ StageCancelError: If event was set before start process.
1446
+ StageError: If the return code form subprocess run function gather
1447
+ than 0.
1279
1448
 
1280
1449
  Returns:
1281
1450
  Result: The execution result with status and context data.
@@ -1286,12 +1455,16 @@ class BashStage(BaseRetryStage):
1286
1455
  bash: str = param2template(
1287
1456
  dedent(self.bash.strip("\n")), params, extras=self.extras
1288
1457
  )
1289
- with self.create_sh_file(
1458
+ with self.make_sh_file(
1290
1459
  bash=bash,
1291
1460
  env=param2template(self.env, params, extras=self.extras),
1292
1461
  run_id=run_id,
1293
1462
  ) as sh:
1294
- trace.debug(f"[STAGE]: Create `{sh[1]}` file.")
1463
+
1464
+ if event and event.is_set():
1465
+ raise StageCancelError("Cancel before start bash process.")
1466
+
1467
+ trace.debug(f"[STAGE]: Create `{sh[1]}` file.", module="stage")
1295
1468
  rs: CompletedProcess = subprocess.run(
1296
1469
  sh,
1297
1470
  shell=False,
@@ -1333,13 +1506,19 @@ class BashStage(BaseRetryStage):
1333
1506
  stdout.
1334
1507
 
1335
1508
  Args:
1336
- params: A parameter data that want to use in this
1509
+ params (DictData): A parameter data that want to use in this
1337
1510
  execution.
1338
- run_id: A running stage ID.
1339
- context: A context data.
1340
- parent_run_id: A parent running ID. (Default is None)
1341
- event: An event manager that use to track parent process
1342
- was not force stopped.
1511
+ run_id (str): A running stage ID.
1512
+ context (DictData): A context data that was passed from handler
1513
+ method.
1514
+ parent_run_id (str, default None): A parent running ID.
1515
+ event (Event, default None): An event manager that use to track
1516
+ parent process was not force stopped.
1517
+
1518
+ Raises:
1519
+ StageCancelError: If event was set before start process.
1520
+ StageError: If the return code form subprocess run function gather
1521
+ than 0.
1343
1522
 
1344
1523
  Returns:
1345
1524
  Result: The execution result with status and context data.
@@ -1350,11 +1529,15 @@ class BashStage(BaseRetryStage):
1350
1529
  bash: str = param2template(
1351
1530
  dedent(self.bash.strip("\n")), params, extras=self.extras
1352
1531
  )
1353
- async with self.async_create_sh_file(
1532
+ async with self.async_make_sh_file(
1354
1533
  bash=bash,
1355
1534
  env=param2template(self.env, params, extras=self.extras),
1356
1535
  run_id=run_id,
1357
1536
  ) as sh:
1537
+
1538
+ if event and event.is_set():
1539
+ raise StageCancelError("Cancel before start bash process.")
1540
+
1358
1541
  await trace.adebug(f"[STAGE]: Create `{sh[1]}` file.")
1359
1542
  rs: CompletedProcess = subprocess.run(
1360
1543
  sh,
@@ -1364,7 +1547,6 @@ class BashStage(BaseRetryStage):
1364
1547
  text=True,
1365
1548
  encoding="utf-8",
1366
1549
  )
1367
-
1368
1550
  if rs.returncode > 0:
1369
1551
  e: str = rs.stderr.removesuffix("\n")
1370
1552
  e_bash: str = bash.replace("\n", "\n\t")
@@ -1399,16 +1581,18 @@ class PyStage(BaseRetryStage):
1399
1581
  module to validate exec-string before running or exclude the `os` package
1400
1582
  from the current globals variable.
1401
1583
 
1402
- Data Validate:
1403
- >>> stage = {
1584
+ Examples:
1585
+ >>> stage = PyStage.model_validate({
1586
+ ... "id": "py-stage",
1404
1587
  ... "name": "Python stage execution",
1405
1588
  ... "run": 'print(f"Hello {VARIABLE}")',
1406
1589
  ... "vars": {
1407
1590
  ... "VARIABLE": "WORLD",
1408
1591
  ... },
1409
- ... }
1592
+ ... })
1410
1593
  """
1411
1594
 
1595
+ action_stage: ClassVar[bool] = True
1412
1596
  run: str = Field(
1413
1597
  description="A Python string statement that want to run with `exec`.",
1414
1598
  )
@@ -1423,11 +1607,13 @@ class PyStage(BaseRetryStage):
1423
1607
  @staticmethod
1424
1608
  def filter_locals(values: DictData) -> Iterator[str]:
1425
1609
  """Filter a locals mapping values that be module, class, or
1426
- __annotations__.
1610
+ `__annotations__`.
1427
1611
 
1428
- :param values: (DictData) A locals values that want to filter.
1612
+ Args:
1613
+ values: (DictData) A locals values that want to filter.
1429
1614
 
1430
- :rtype: Iterator[str]
1615
+ Returns:
1616
+ Iterator[str]: Iter string value.
1431
1617
  """
1432
1618
  for value in values:
1433
1619
 
@@ -1447,12 +1633,14 @@ class PyStage(BaseRetryStage):
1447
1633
  """Override set an outputs method for the Python execution process that
1448
1634
  extract output from all the locals values.
1449
1635
 
1450
- :param output: (DictData) An output data that want to extract to an
1451
- output key.
1452
- :param to: (DictData) A context data that want to add output result.
1453
- :param info: (DictData)
1636
+ Args:
1637
+ output (DictData): An output data that want to extract to an
1638
+ output key.
1639
+ to (DictData): A context data that want to add output result.
1640
+ info (DictData):
1454
1641
 
1455
- :rtype: DictData
1642
+ Returns:
1643
+ DictData: A context data that have merged with the output data.
1456
1644
  """
1457
1645
  output: DictData = output.copy()
1458
1646
  lc: DictData = output.pop("locals", {})
@@ -1504,18 +1692,13 @@ class PyStage(BaseRetryStage):
1504
1692
  }
1505
1693
  )
1506
1694
 
1695
+ if event and event.is_set():
1696
+ raise StageCancelError("Cancel before start exec process.")
1697
+
1507
1698
  # WARNING: The exec build-in function is very dangerous. So, it
1508
1699
  # should use the re module to validate exec-string before running.
1509
- exec(
1510
- pass_env(
1511
- param2template(dedent(self.run), params, extras=self.extras)
1512
- ),
1513
- gb,
1514
- lc,
1515
- )
1516
- return Result(
1517
- run_id=run_id,
1518
- parent_run_id=parent_run_id,
1700
+ exec(self.pass_template(dedent(self.run), params), gb, lc)
1701
+ return Result.from_trace(trace).catch(
1519
1702
  status=SUCCESS,
1520
1703
  context=catch(
1521
1704
  context=context,
@@ -1536,7 +1719,6 @@ class PyStage(BaseRetryStage):
1536
1719
  },
1537
1720
  },
1538
1721
  ),
1539
- extras=self.extras,
1540
1722
  )
1541
1723
 
1542
1724
  async def async_process(
@@ -1555,13 +1737,17 @@ class PyStage(BaseRetryStage):
1555
1737
  - https://stackoverflow.com/questions/44859165/async-exec-in-python
1556
1738
 
1557
1739
  Args:
1558
- params: A parameter data that want to use in this
1740
+ params (DictData): A parameter data that want to use in this
1559
1741
  execution.
1560
- run_id: A running stage ID.
1561
- context: A context data.
1562
- parent_run_id: A parent running ID. (Default is None)
1563
- event: An event manager that use to track parent process
1564
- was not force stopped.
1742
+ run_id (str): A running stage ID.
1743
+ context (DictData): A context data that was passed from handler
1744
+ method.
1745
+ parent_run_id (str, default None): A parent running ID.
1746
+ event (Event, default None): An event manager that use to track
1747
+ parent process was not force stopped.
1748
+
1749
+ Raises:
1750
+ StageCancelError: If event was set before start process.
1565
1751
 
1566
1752
  Returns:
1567
1753
  Result: The execution result with status and context data.
@@ -1584,16 +1770,14 @@ class PyStage(BaseRetryStage):
1584
1770
  )
1585
1771
  }
1586
1772
  )
1773
+
1774
+ if event and event.is_set():
1775
+ raise StageCancelError("Cancel before start exec process.")
1776
+
1587
1777
  # WARNING: The exec build-in function is very dangerous. So, it
1588
1778
  # should use the re module to validate exec-string before running.
1589
- exec(
1590
- param2template(dedent(self.run), params, extras=self.extras),
1591
- gb,
1592
- lc,
1593
- )
1594
- return Result(
1595
- run_id=run_id,
1596
- parent_run_id=parent_run_id,
1779
+ exec(self.pass_template(dedent(self.run), params), gb, lc)
1780
+ return Result.from_trace(trace).catch(
1597
1781
  status=SUCCESS,
1598
1782
  context=catch(
1599
1783
  context=context,
@@ -1614,7 +1798,6 @@ class PyStage(BaseRetryStage):
1614
1798
  },
1615
1799
  },
1616
1800
  ),
1617
- extras=self.extras,
1618
1801
  )
1619
1802
 
1620
1803
 
@@ -1639,14 +1822,16 @@ class CallStage(BaseRetryStage):
1639
1822
  The caller registry to get a caller function should importable by the
1640
1823
  current Python execution pointer.
1641
1824
 
1642
- Data Validate:
1643
- >>> stage = {
1825
+ Examples:
1826
+ >>> stage = CallStage.model_validate({
1827
+ ... "id": "call-stage",
1644
1828
  ... "name": "Task stage execution",
1645
1829
  ... "uses": "tasks/function-name@tag-name",
1646
1830
  ... "args": {"arg01": "BAR", "kwarg01": 10},
1647
- ... }
1831
+ ... })
1648
1832
  """
1649
1833
 
1834
+ action_stage: ClassVar[bool] = True
1650
1835
  uses: str = Field(
1651
1836
  description=(
1652
1837
  "A caller function with registry importer syntax that use to load "
@@ -1663,26 +1848,36 @@ class CallStage(BaseRetryStage):
1663
1848
  )
1664
1849
 
1665
1850
  @field_validator("args", mode="before")
1666
- def __validate_args_key(cls, value: Any) -> Any:
1851
+ def __validate_args_key(cls, data: Any) -> Any:
1667
1852
  """Validate argument keys on the ``args`` field should not include the
1668
1853
  special keys.
1669
1854
 
1670
- :param value: (Any) A value that want to check the special keys.
1855
+ Args:
1856
+ data (Any): A data that want to check the special keys.
1671
1857
 
1672
- :rtype: Any
1858
+ Returns:
1859
+ Any: An any data.
1673
1860
  """
1674
- if isinstance(value, dict) and any(
1675
- k in value for k in ("result", "extras")
1861
+ if isinstance(data, dict) and any(
1862
+ k in data for k in ("result", "extras")
1676
1863
  ):
1677
1864
  raise ValueError(
1678
1865
  "The argument on workflow template for the caller stage "
1679
1866
  "should not pass `result` and `extras`. They are special "
1680
1867
  "arguments."
1681
1868
  )
1682
- return value
1869
+ return data
1683
1870
 
1684
1871
  def get_caller(self, params: DictData) -> Callable[[], TagFunc]:
1685
- """Get the lazy TagFuc object from registry."""
1872
+ """Get the lazy TagFuc object from registry.
1873
+
1874
+ Args:
1875
+ params (DictData): A parameters.
1876
+
1877
+ Returns:
1878
+ Callable[[], TagFunc]: A lazy partial function that return the
1879
+ TagFunc object.
1880
+ """
1686
1881
  return extract_call(
1687
1882
  param2template(self.uses, params, extras=self.extras),
1688
1883
  registries=self.extras.get("registry_caller"),
@@ -1700,13 +1895,19 @@ class CallStage(BaseRetryStage):
1700
1895
  """Execute this caller function with its argument parameter.
1701
1896
 
1702
1897
  Args:
1703
- params: A parameter data that want to use in this
1898
+ params (DictData): A parameter data that want to use in this
1704
1899
  execution.
1705
- run_id: A running stage ID.
1706
- context: A context data.
1707
- parent_run_id: A parent running ID. (Default is None)
1708
- event: An event manager that use to track parent process
1709
- was not force stopped.
1900
+ run_id (str): A running stage ID.
1901
+ context (DictData): A context data that was passed from handler
1902
+ method.
1903
+ parent_run_id (str, default None): A parent running ID.
1904
+ event (Event, default None): An event manager that use to track
1905
+ parent process was not force stopped.
1906
+
1907
+ Raises:
1908
+ ValueError: If the necessary parameters do not exist in args field.
1909
+ TypeError: If the returning type of caller function does not match
1910
+ with dict type.
1710
1911
 
1711
1912
  Returns:
1712
1913
  Result: The execution result with status and context data.
@@ -1728,8 +1929,10 @@ class CallStage(BaseRetryStage):
1728
1929
  extras=self.extras,
1729
1930
  ),
1730
1931
  "extras": self.extras,
1731
- } | param2template(self.args, params, extras=self.extras)
1732
- sig = inspect.signature(call_func)
1932
+ } | self.pass_template(self.args, params)
1933
+
1934
+ # NOTE: Catch the necessary parameters.
1935
+ sig: inspect.Signature = inspect.signature(call_func)
1733
1936
  necessary_params: list[str] = []
1734
1937
  has_keyword: bool = False
1735
1938
  for k in sig.parameters:
@@ -1743,15 +1946,14 @@ class CallStage(BaseRetryStage):
1743
1946
  elif v.kind == Parameter.VAR_KEYWORD:
1744
1947
  has_keyword = True
1745
1948
 
1949
+ # NOTE: Validate private parameter should exist in the args field.
1746
1950
  if any(
1747
1951
  (k.removeprefix("_") not in args and k not in args)
1748
1952
  for k in necessary_params
1749
1953
  ):
1750
- if "result" in necessary_params:
1751
- necessary_params.remove("result")
1752
-
1753
- if "extras" in necessary_params:
1754
- necessary_params.remove("extras")
1954
+ for k in ("result", "extras"):
1955
+ if k in necessary_params:
1956
+ necessary_params.remove(k)
1755
1957
 
1756
1958
  args.pop("result")
1757
1959
  args.pop("extras")
@@ -1760,18 +1962,16 @@ class CallStage(BaseRetryStage):
1760
1962
  f"does not set to args. It already set {list(args.keys())}."
1761
1963
  )
1762
1964
 
1763
- if "result" not in sig.parameters and not has_keyword:
1764
- args.pop("result")
1965
+ if not has_keyword:
1966
+ for k in ("result", "extras"):
1967
+ if k not in sig.parameters:
1968
+ args.pop(k)
1765
1969
 
1766
- if "extras" not in sig.parameters and not has_keyword:
1767
- args.pop("extras")
1970
+ args: DictData = self.validate_model_args(call_func, args)
1768
1971
 
1769
1972
  if event and event.is_set():
1770
1973
  raise StageCancelError("Cancel before start call process.")
1771
1974
 
1772
- args: DictData = self.validate_model_args(
1773
- call_func, args, run_id, parent_run_id, extras=self.extras
1774
- )
1775
1975
  if inspect.iscoroutinefunction(call_func):
1776
1976
  loop = asyncio.get_event_loop()
1777
1977
  rs: DictData = loop.run_until_complete(
@@ -1825,6 +2025,11 @@ class CallStage(BaseRetryStage):
1825
2025
  event: An event manager that use to track parent process
1826
2026
  was not force stopped.
1827
2027
 
2028
+ Raises:
2029
+ ValueError: If the necessary parameters do not exist in args field.
2030
+ TypeError: If the returning type of caller function does not match
2031
+ with dict type.
2032
+
1828
2033
  Returns:
1829
2034
  Result: The execution result with status and context data.
1830
2035
  """
@@ -1847,8 +2052,10 @@ class CallStage(BaseRetryStage):
1847
2052
  extras=self.extras,
1848
2053
  ),
1849
2054
  "extras": self.extras,
1850
- } | param2template(self.args, params, extras=self.extras)
1851
- sig = inspect.signature(call_func)
2055
+ } | self.pass_template(self.args, params)
2056
+
2057
+ # NOTE: Catch the necessary parameters.
2058
+ sig: inspect.Signature = inspect.signature(call_func)
1852
2059
  necessary_params: list[str] = []
1853
2060
  has_keyword: bool = False
1854
2061
  for k in sig.parameters:
@@ -1866,11 +2073,9 @@ class CallStage(BaseRetryStage):
1866
2073
  (k.removeprefix("_") not in args and k not in args)
1867
2074
  for k in necessary_params
1868
2075
  ):
1869
- if "result" in necessary_params:
1870
- necessary_params.remove("result")
1871
-
1872
- if "extras" in necessary_params:
1873
- necessary_params.remove("extras")
2076
+ for k in ("result", "extras"):
2077
+ if k in necessary_params:
2078
+ necessary_params.remove(k)
1874
2079
 
1875
2080
  args.pop("result")
1876
2081
  args.pop("extras")
@@ -1878,18 +2083,17 @@ class CallStage(BaseRetryStage):
1878
2083
  f"Necessary params, ({', '.join(necessary_params)}, ), "
1879
2084
  f"does not set to args. It already set {list(args.keys())}."
1880
2085
  )
1881
- if "result" not in sig.parameters and not has_keyword:
1882
- args.pop("result")
1883
2086
 
1884
- if "extras" not in sig.parameters and not has_keyword:
1885
- args.pop("extras")
2087
+ if not has_keyword:
2088
+ for k in ("result", "extras"):
2089
+ if k not in sig.parameters:
2090
+ args.pop(k)
2091
+
2092
+ args: DictData = self.validate_model_args(call_func, args)
1886
2093
 
1887
2094
  if event and event.is_set():
1888
2095
  raise StageCancelError("Cancel before start call process.")
1889
2096
 
1890
- args: DictData = self.validate_model_args(
1891
- call_func, args, run_id, parent_run_id, extras=self.extras
1892
- )
1893
2097
  if inspect.iscoroutinefunction(call_func):
1894
2098
  rs: DictOrModel = await call_func(
1895
2099
  **param2template(args, params, extras=self.extras)
@@ -1922,21 +2126,18 @@ class CallStage(BaseRetryStage):
1922
2126
  )
1923
2127
 
1924
2128
  @staticmethod
1925
- def validate_model_args(
1926
- func: TagFunc,
1927
- args: DictData,
1928
- run_id: str,
1929
- parent_run_id: Optional[str] = None,
1930
- extras: Optional[DictData] = None,
1931
- ) -> DictData:
2129
+ def validate_model_args(func: TagFunc, args: DictData) -> DictData:
1932
2130
  """Validate an input arguments before passing to the caller function.
1933
2131
 
1934
2132
  Args:
1935
- func: (TagFunc) A tag function that want to get typing.
1936
- args: (DictData) An arguments before passing to this tag func.
1937
- run_id: A running stage ID.
2133
+ func (TagFunc): A tag function object that want to get typing.
2134
+ args (DictData): An arguments before passing to this tag func.
1938
2135
 
1939
- :rtype: DictData
2136
+ Raises:
2137
+ StageError: If model validation was raised the ValidationError.
2138
+
2139
+ Returns:
2140
+ DictData: A prepared args parameter that validate with model args.
1940
2141
  """
1941
2142
  try:
1942
2143
  override: DictData = dict(
@@ -1959,55 +2160,8 @@ class CallStage(BaseRetryStage):
1959
2160
  raise StageError(
1960
2161
  "Validate argument from the caller function raise invalid type."
1961
2162
  ) from e
1962
- except TypeError as e:
1963
- trace: Trace = get_trace(
1964
- run_id, parent_run_id=parent_run_id, extras=extras
1965
- )
1966
- trace.warning(
1967
- f"[STAGE]: Get type hint raise TypeError: {e}, so, it skip "
1968
- f"parsing model args process."
1969
- )
1970
- return args
1971
-
1972
-
1973
- class BaseNestedStage(BaseRetryStage, ABC):
1974
- """Base Nested Stage model. This model is use for checking the child stage
1975
- is the nested stage or not.
1976
- """
1977
-
1978
- def set_outputs(
1979
- self, output: DictData, to: DictData, info: Optional[DictData] = None
1980
- ) -> DictData:
1981
- """Override the set outputs method that support for nested-stage."""
1982
- return super().set_outputs(output, to=to)
1983
-
1984
- def get_outputs(self, output: DictData) -> DictData:
1985
- """Override the get outputs method that support for nested-stage"""
1986
- return super().get_outputs(output)
1987
-
1988
- @property
1989
- def is_nested(self) -> bool:
1990
- """Check if this stage is a nested stage or not.
1991
2163
 
1992
- :rtype: bool
1993
- """
1994
- return True
1995
-
1996
- @staticmethod
1997
- def mark_errors(context: DictData, error: StageError) -> None:
1998
- """Make the errors context result with the refs value depends on the nested
1999
- execute func.
2000
-
2001
- Args:
2002
- context: (DictData) A context data.
2003
- error: (StageError) A stage exception object.
2004
- """
2005
- if "errors" in context:
2006
- context["errors"][error.refs] = error.to_dict()
2007
- else:
2008
- context["errors"] = error.to_dict(with_refs=True)
2009
-
2010
- async def async_process(
2164
+ def dryrun(
2011
2165
  self,
2012
2166
  params: DictData,
2013
2167
  run_id: str,
@@ -2015,27 +2169,90 @@ class BaseNestedStage(BaseRetryStage, ABC):
2015
2169
  *,
2016
2170
  parent_run_id: Optional[str] = None,
2017
2171
  event: Optional[Event] = None,
2018
- ) -> Result:
2019
- """Async process for nested-stage do not implement yet.
2172
+ ) -> Result: # pragma: no cov
2173
+ """Override the dryrun method for this CallStage.
2174
+
2175
+ Steps:
2176
+ - Pre-hook caller function that exist.
2177
+ - Show function parameters
2020
2178
 
2021
2179
  Args:
2022
- params: A parameter data that want to use in this
2180
+ params (DictData): A parameter data that want to use in this
2023
2181
  execution.
2024
- run_id: A running stage ID.
2025
- context: A context data.
2026
- parent_run_id: A parent running ID. (Default is None)
2027
- event: An event manager that use to track parent process
2028
- was not force stopped.
2182
+ run_id (str): A running stage ID.
2183
+ context (DictData): A context data that was passed from handler
2184
+ method.
2185
+ parent_run_id (str, default None): A parent running ID.
2186
+ event (Event, default None): An event manager that use to track
2187
+ parent process was not force stopped.
2029
2188
 
2030
- Returns:
2031
- Result: The execution result with status and context data.
2032
2189
  """
2033
- raise NotImplementedError(
2034
- "The nested-stage does not implement the `axecute` method yet."
2190
+ trace: Trace = get_trace(
2191
+ run_id, parent_run_id=parent_run_id, extras=self.extras
2192
+ )
2193
+ call_func: TagFunc = self.get_caller(params=params)()
2194
+ trace.info(f"[STAGE]: Caller Func: '{call_func.name}@{call_func.tag}'")
2195
+
2196
+ args: DictData = {
2197
+ "result": Result(
2198
+ run_id=run_id,
2199
+ parent_run_id=parent_run_id,
2200
+ status=WAIT,
2201
+ context=context,
2202
+ extras=self.extras,
2203
+ ),
2204
+ "extras": self.extras,
2205
+ } | self.pass_template(self.args, params)
2206
+
2207
+ # NOTE: Catch the necessary parameters.
2208
+ sig: inspect.Signature = inspect.signature(call_func)
2209
+ trace.debug(f"[STAGE]: {sig.parameters}")
2210
+ necessary_params: list[str] = []
2211
+ has_keyword: bool = False
2212
+ for k in sig.parameters:
2213
+ if (
2214
+ v := sig.parameters[k]
2215
+ ).default == Parameter.empty and v.kind not in (
2216
+ Parameter.VAR_KEYWORD,
2217
+ Parameter.VAR_POSITIONAL,
2218
+ ):
2219
+ necessary_params.append(k)
2220
+ elif v.kind == Parameter.VAR_KEYWORD:
2221
+ has_keyword = True
2222
+
2223
+ func_typed: dict[str, Any] = get_type_hints(call_func)
2224
+ map_type: str = "||".join(
2225
+ f"\t{p}: {func_typed[p]}"
2226
+ for p in necessary_params
2227
+ if p in func_typed
2228
+ )
2229
+ map_type_args: str = "||".join(f"\t{a}: {type(a)}" for a in args)
2230
+
2231
+ if not has_keyword:
2232
+ for k in ("result", "extras"):
2233
+ if k not in sig.parameters:
2234
+ args.pop(k)
2235
+
2236
+ trace.info(
2237
+ f"[STAGE]: Details"
2238
+ f"||Necessary Params:"
2239
+ f"||{map_type}"
2240
+ f"||Supported Keyword Params: {has_keyword}"
2241
+ f"||Return Type: {func_typed['return']}"
2242
+ f"||Argument Params:"
2243
+ f"||{map_type_args}"
2244
+ f"||"
2245
+ )
2246
+ return Result(
2247
+ run_id=run_id,
2248
+ parent_run_id=parent_run_id,
2249
+ status=SUCCESS,
2250
+ context=catch(context=context, status=SUCCESS),
2251
+ extras=self.extras,
2035
2252
  )
2036
2253
 
2037
2254
 
2038
- class TriggerStage(BaseNestedStage):
2255
+ class TriggerStage(BaseRetryStage):
2039
2256
  """Trigger workflow executor stage that run an input trigger Workflow
2040
2257
  execute method. This is the stage that allow you to create the reusable
2041
2258
  Workflow template with dynamic parameters.
@@ -2043,12 +2260,13 @@ class TriggerStage(BaseNestedStage):
2043
2260
  This stage does not allow to pass the workflow model directly to the
2044
2261
  trigger field. A trigger workflow name should exist on the config path only.
2045
2262
 
2046
- Data Validate:
2047
- >>> stage = {
2263
+ Examples:
2264
+ >>> stage = TriggerStage.model_validate({
2265
+ ... "id": "trigger-stage",
2048
2266
  ... "name": "Trigger workflow stage execution",
2049
2267
  ... "trigger": 'workflow-name-for-loader',
2050
2268
  ... "params": {"run-date": "2024-08-01", "source": "src"},
2051
- ... }
2269
+ ... })
2052
2270
  """
2053
2271
 
2054
2272
  trigger: str = Field(
@@ -2093,55 +2311,135 @@ class TriggerStage(BaseNestedStage):
2093
2311
  run_id, parent_run_id=parent_run_id, extras=self.extras
2094
2312
  )
2095
2313
  _trigger: str = param2template(self.trigger, params, extras=self.extras)
2096
- if _trigger == self.extras.get("__sys_break_circle_exec", "NOTSET"):
2314
+ if _trigger == self.extras.get("__sys_exec_break_circle", "NOTSET"):
2097
2315
  raise StageError("Circle execute via trigger itself workflow name.")
2316
+
2098
2317
  trace.info(f"[NESTED]: Load Workflow Config: {_trigger!r}")
2099
- result: Result = Workflow.from_conf(
2318
+ workflow: Workflow = Workflow.from_conf(
2100
2319
  name=pass_env(_trigger),
2101
2320
  extras=self.extras,
2102
- ).execute(
2103
- # NOTE: Should not use the `pass_env` function on this params parameter.
2321
+ )
2322
+
2323
+ if event and event.is_set():
2324
+ raise StageCancelError("Cancel before start trigger process.")
2325
+
2326
+ # IMPORTANT: Should not use the `pass_env` function on this `params`
2327
+ # parameter.
2328
+ result: Result = workflow.execute(
2104
2329
  params=param2template(self.params, params, extras=self.extras),
2105
2330
  run_id=parent_run_id,
2106
2331
  event=event,
2107
2332
  )
2333
+ catch(context, status=result.status, updated=result.context)
2108
2334
  if result.status == FAILED:
2109
2335
  err_msg: str = (
2110
2336
  f" with:\n{msg}"
2111
2337
  if (msg := result.context.get("errors", {}).get("message"))
2112
2338
  else "."
2113
2339
  )
2114
- return result.catch(
2115
- status=FAILED,
2116
- context={
2117
- "status": FAILED,
2118
- "errors": StageError(
2119
- f"Trigger workflow was failed{err_msg}"
2120
- ).to_dict(),
2121
- },
2340
+ err = StageError(
2341
+ f"Trigger workflow was failed{err_msg}", allow_traceback=False
2122
2342
  )
2343
+ raise err
2123
2344
  elif result.status == CANCEL:
2124
- return result.catch(
2125
- status=CANCEL,
2126
- context={
2127
- "status": CANCEL,
2128
- "errors": StageCancelError(
2129
- "Trigger workflow was cancel."
2130
- ).to_dict(),
2131
- },
2132
- )
2345
+ raise StageCancelError("Trigger workflow was cancel.")
2133
2346
  elif result.status == SKIP:
2134
- return result.catch(
2135
- status=SKIP,
2136
- context={
2137
- "status": SKIP,
2138
- "errors": StageSkipError(
2139
- "Trigger workflow was skipped."
2140
- ).to_dict(),
2141
- },
2142
- )
2347
+ raise StageSkipError("Trigger workflow was skipped.")
2143
2348
  return result
2144
2349
 
2350
+ async def async_process(
2351
+ self,
2352
+ params: DictData,
2353
+ run_id: str,
2354
+ context: DictData,
2355
+ *,
2356
+ parent_run_id: Optional[str] = None,
2357
+ event: Optional[Event] = None,
2358
+ ) -> Result: # pragma: no cov
2359
+ """Async process for trigger-stage do not implement yet.
2360
+
2361
+ Args:
2362
+ params: A parameter data that want to use in this
2363
+ execution.
2364
+ run_id: A running stage ID.
2365
+ context: A context data.
2366
+ parent_run_id: A parent running ID. (Default is None)
2367
+ event: An event manager that use to track parent process
2368
+ was not force stopped.
2369
+
2370
+ Returns:
2371
+ Result: The execution result with status and context data.
2372
+ """
2373
+ raise NotImplementedError(
2374
+ "The Trigger stage does not implement the `axecute` method yet."
2375
+ )
2376
+
2377
+
2378
+ class BaseNestedStage(BaseAsyncStage, ABC):
2379
+ """Base Nested Stage model. This model is use for checking the child stage
2380
+ is the nested stage or not.
2381
+ """
2382
+
2383
+ def set_outputs(
2384
+ self, output: DictData, to: DictData, info: Optional[DictData] = None
2385
+ ) -> DictData:
2386
+ """Override the set outputs method that support for nested-stage."""
2387
+ return super().set_outputs(output, to=to)
2388
+
2389
+ def get_outputs(self, output: DictData) -> DictData:
2390
+ """Override the get outputs method that support for nested-stage"""
2391
+ return super().get_outputs(output)
2392
+
2393
+ @property
2394
+ def is_nested(self) -> bool:
2395
+ """Check if this stage is a nested stage or not.
2396
+
2397
+ Returns:
2398
+ bool: True only.
2399
+ """
2400
+ return True
2401
+
2402
+ @staticmethod
2403
+ def mark_errors(context: DictData, error: StageError) -> None:
2404
+ """Make the errors context result with the refs value depends on the nested
2405
+ execute func.
2406
+
2407
+ Args:
2408
+ context: (DictData) A context data.
2409
+ error: (StageError) A stage exception object.
2410
+ """
2411
+ if "errors" in context:
2412
+ context["errors"][error.refs] = error.to_dict()
2413
+ else:
2414
+ context["errors"] = error.to_dict(with_refs=True)
2415
+
2416
+ async def async_process(
2417
+ self,
2418
+ params: DictData,
2419
+ run_id: str,
2420
+ context: DictData,
2421
+ *,
2422
+ parent_run_id: Optional[str] = None,
2423
+ event: Optional[Event] = None,
2424
+ ) -> Result:
2425
+ """Async process for nested-stage do not implement yet.
2426
+
2427
+ Args:
2428
+ params: A parameter data that want to use in this
2429
+ execution.
2430
+ run_id: A running stage ID.
2431
+ context: A context data.
2432
+ parent_run_id: A parent running ID. (Default is None)
2433
+ event: An event manager that use to track parent process
2434
+ was not force stopped.
2435
+
2436
+ Returns:
2437
+ Result: The execution result with status and context data.
2438
+ """
2439
+ raise NotImplementedError(
2440
+ "The nested-stage does not implement the `axecute` method yet."
2441
+ )
2442
+
2145
2443
 
2146
2444
  class ParallelContext(TypedDict):
2147
2445
  branch: str
@@ -2156,8 +2454,9 @@ class ParallelStage(BaseNestedStage):
2156
2454
  This stage is not the low-level stage model because it runs multi-stages
2157
2455
  in this stage execution.
2158
2456
 
2159
- Data Validate:
2160
- >>> stage = {
2457
+ Examples:
2458
+ >>> stage = ParallelStage.model_validate({
2459
+ ... "id": "parallel-stage",
2161
2460
  ... "name": "Parallel stage execution.",
2162
2461
  ... "parallel": {
2163
2462
  ... "branch01": [
@@ -2179,7 +2478,7 @@ class ParallelStage(BaseNestedStage):
2179
2478
  ... },
2180
2479
  ... ],
2181
2480
  ... }
2182
- ... }
2481
+ ... })
2183
2482
  """
2184
2483
 
2185
2484
  parallel: dict[str, list[Stage]] = Field(
@@ -2429,8 +2728,9 @@ class ForEachStage(BaseNestedStage):
2429
2728
  This stage is not the low-level stage model because it runs
2430
2729
  multi-stages in this stage execution.
2431
2730
 
2432
- Data Validate:
2433
- >>> stage = {
2731
+ Examples:
2732
+ >>> stage = ForEachStage.model_validate({
2733
+ ... "id": "foreach-stage",
2434
2734
  ... "name": "For-each stage execution",
2435
2735
  ... "foreach": [1, 2, 3]
2436
2736
  ... "stages": [
@@ -2439,7 +2739,7 @@ class ForEachStage(BaseNestedStage):
2439
2739
  ... "echo": "Start run with item ${{ item }}"
2440
2740
  ... },
2441
2741
  ... ],
2442
- ... }
2742
+ ... })
2443
2743
  """
2444
2744
 
2445
2745
  foreach: EachType = Field(
@@ -2614,15 +2914,18 @@ class ForEachStage(BaseNestedStage):
2614
2914
  """Validate foreach value that already passed to this model.
2615
2915
 
2616
2916
  Args:
2617
- value:
2917
+ value (Any): An any foreach value.
2618
2918
 
2619
2919
  Raises:
2620
2920
  TypeError: If value can not try-convert to list type.
2621
- ValueError:
2921
+ ValueError: If the foreach value is dict type.
2922
+ ValueError: If the foreach value contain duplication item without
2923
+ enable using index as key flag.
2622
2924
 
2623
2925
  Returns:
2624
2926
  list[Any]: list of item.
2625
2927
  """
2928
+ # NOTE: Try to cast a foreach with string type to list of items.
2626
2929
  if isinstance(value, str):
2627
2930
  try:
2628
2931
  value: list[Any] = str2list(value)
@@ -2631,6 +2934,7 @@ class ForEachStage(BaseNestedStage):
2631
2934
  f"Does not support string foreach: {value!r} that can "
2632
2935
  f"not convert to list."
2633
2936
  ) from e
2937
+
2634
2938
  # [VALIDATE]: Type of the foreach should be `list` type.
2635
2939
  elif isinstance(value, dict):
2636
2940
  raise TypeError(
@@ -2752,8 +3056,9 @@ class UntilStage(BaseNestedStage):
2752
3056
  This stage is not the low-level stage model because it runs
2753
3057
  multi-stages in this stage execution.
2754
3058
 
2755
- Data Validate:
2756
- >>> stage = {
3059
+ Examples:
3060
+ >>> stage = UntilStage.model_validate({
3061
+ ... "id": "until-stage",
2757
3062
  ... "name": "Until stage execution",
2758
3063
  ... "item": 1,
2759
3064
  ... "until": "${{ item }} > 3"
@@ -2766,7 +3071,7 @@ class UntilStage(BaseNestedStage):
2766
3071
  ... )
2767
3072
  ... },
2768
3073
  ... ],
2769
- ... }
3074
+ ... })
2770
3075
  """
2771
3076
 
2772
3077
  item: Union[str, int, bool] = Field(
@@ -2776,7 +3081,7 @@ class UntilStage(BaseNestedStage):
2776
3081
  ),
2777
3082
  )
2778
3083
  until: str = Field(description="A until condition for stop the while loop.")
2779
- stages: list[NestedStage] = Field(
3084
+ stages: list[SubStage] = Field(
2780
3085
  default_factory=list,
2781
3086
  description=(
2782
3087
  "A list of stage that will run with each item in until loop."
@@ -3037,6 +3342,8 @@ class Match(BaseModel):
3037
3342
 
3038
3343
 
3039
3344
  class Else(BaseModel):
3345
+ """Else model for the Case Stage."""
3346
+
3040
3347
  other: list[Stage] = Field(
3041
3348
  description="A list of stage that does not match any case.",
3042
3349
  alias="else",
@@ -3046,8 +3353,9 @@ class Else(BaseModel):
3046
3353
  class CaseStage(BaseNestedStage):
3047
3354
  """Case stage executor that execute all stages if the condition was matched.
3048
3355
 
3049
- Data Validate:
3050
- >>> stage = {
3356
+ Examples:
3357
+ >>> stage = CaseStage.model_validate({
3358
+ ... "id": "case-stage",
3051
3359
  ... "name": "If stage execution.",
3052
3360
  ... "case": "${{ param.test }}",
3053
3361
  ... "match": [
@@ -3070,9 +3378,10 @@ class CaseStage(BaseNestedStage):
3070
3378
  ... ],
3071
3379
  ... },
3072
3380
  ... ],
3073
- ... }
3381
+ ... })
3074
3382
 
3075
- >>> stage = {
3383
+ >>> stage = CaseStage.model_validate({
3384
+ ... "id": "case-stage",
3076
3385
  ... "name": "If stage execution.",
3077
3386
  ... "case": "${{ param.test }}",
3078
3387
  ... "match": [
@@ -3094,7 +3403,7 @@ class CaseStage(BaseNestedStage):
3094
3403
  ... ],
3095
3404
  ... },
3096
3405
  ... ],
3097
- ... }
3406
+ ... })
3098
3407
 
3099
3408
  """
3100
3409
 
@@ -3113,9 +3422,16 @@ class CaseStage(BaseNestedStage):
3113
3422
 
3114
3423
  @field_validator("match", mode="after")
3115
3424
  def __validate_match(
3116
- cls, match: list[Union[Match, Else]]
3425
+ cls,
3426
+ match: list[Union[Match, Else]],
3117
3427
  ) -> list[Union[Match, Else]]:
3118
- """Validate the match field should contain only one Else model."""
3428
+ """Validate the match field should contain only one Else model.
3429
+
3430
+ Raises:
3431
+ ValueError: If match field contain Else more than 1 model.
3432
+ ValueError: If match field contain Match with '_' case (it represent
3433
+ the else case) more than 1 model.
3434
+ """
3119
3435
  c_else_case: int = 0
3120
3436
  c_else_model: int = 0
3121
3437
  for m in match:
@@ -3314,8 +3630,10 @@ class CaseStage(BaseNestedStage):
3314
3630
  case: StrOrNone = param2template(self.case, params, extras=self.extras)
3315
3631
  trace.info(f"[NESTED]: Get Case: {case!r}.")
3316
3632
  case, stages = self.extract_stages_from_case(case, params=params)
3633
+
3317
3634
  if event and event.is_set():
3318
3635
  raise StageCancelError("Cancel before start case process.")
3636
+
3319
3637
  status, context = self._process_nested(
3320
3638
  case=case,
3321
3639
  stages=stages,
@@ -3337,11 +3655,12 @@ class RaiseStage(BaseAsyncStage):
3337
3655
  """Raise error stage executor that raise `StageError` that use a message
3338
3656
  field for making error message before raise.
3339
3657
 
3340
- Data Validate:
3341
- >>> stage = {
3658
+ Examples:
3659
+ >>> stage = RaiseStage.model_validate({
3660
+ ... "id": "raise-stage",
3342
3661
  ... "name": "Raise stage",
3343
3662
  ... "raise": "raise this stage",
3344
- ... }
3663
+ ... })
3345
3664
 
3346
3665
  """
3347
3666
 
@@ -3414,7 +3733,7 @@ class RaiseStage(BaseAsyncStage):
3414
3733
  raise StageError(message)
3415
3734
 
3416
3735
 
3417
- class DockerStage(BaseStage): # pragma: no cov
3736
+ class DockerStage(BaseRetryStage): # pragma: no cov
3418
3737
  """Docker container stage execution that will pull the specific Docker image
3419
3738
  with custom authentication and run this image by passing environment
3420
3739
  variables and mounting local volume to this Docker container.
@@ -3437,6 +3756,7 @@ class DockerStage(BaseStage): # pragma: no cov
3437
3756
  ... }
3438
3757
  """
3439
3758
 
3759
+ action_stage: ClassVar[bool] = True
3440
3760
  image: str = Field(
3441
3761
  description="A Docker image url with tag that want to run.",
3442
3762
  )
@@ -3593,6 +3913,33 @@ class DockerStage(BaseStage): # pragma: no cov
3593
3913
  trace.info(f"[STAGE]: Docker: {self.image}:{self.tag}")
3594
3914
  raise NotImplementedError("Docker Stage does not implement yet.")
3595
3915
 
3916
+ async def async_process(
3917
+ self,
3918
+ params: DictData,
3919
+ run_id: str,
3920
+ context: DictData,
3921
+ *,
3922
+ parent_run_id: Optional[str] = None,
3923
+ event: Optional[Event] = None,
3924
+ ) -> Result: # pragma: no cov
3925
+ """Async process for nested-stage do not implement yet.
3926
+
3927
+ Args:
3928
+ params: A parameter data that want to use in this
3929
+ execution.
3930
+ run_id: A running stage ID.
3931
+ context: A context data.
3932
+ parent_run_id: A parent running ID. (Default is None)
3933
+ event: An event manager that use to track parent process
3934
+ was not force stopped.
3935
+
3936
+ Returns:
3937
+ Result: The execution result with status and context data.
3938
+ """
3939
+ raise NotImplementedError(
3940
+ "The Docker stage does not implement the `axecute` method yet."
3941
+ )
3942
+
3596
3943
 
3597
3944
  class VirtualPyStage(PyStage): # pragma: no cov
3598
3945
  """Virtual Python stage executor that run Python statement on the dependent
@@ -3614,7 +3961,7 @@ class VirtualPyStage(PyStage): # pragma: no cov
3614
3961
  )
3615
3962
 
3616
3963
  @contextlib.contextmanager
3617
- def create_py_file(
3964
+ def make_py_file(
3618
3965
  self,
3619
3966
  py: str,
3620
3967
  values: DictData,
@@ -3629,7 +3976,7 @@ class VirtualPyStage(PyStage): # pragma: no cov
3629
3976
 
3630
3977
  Args:
3631
3978
  py: A Python string statement.
3632
- values: A variable that want to set before running this
3979
+ values: A variable that want to set before running these
3633
3980
  deps: An additional Python dependencies that want install before
3634
3981
  run this python stage.
3635
3982
  run_id: (StrOrNone) A running ID of this stage execution.
@@ -3690,13 +4037,14 @@ class VirtualPyStage(PyStage): # pragma: no cov
3690
4037
  - Execution python file with `uv run` via Python subprocess module.
3691
4038
 
3692
4039
  Args:
3693
- params: A parameter data that want to use in this
4040
+ params (DictData): A parameter data that want to use in this
3694
4041
  execution.
3695
- run_id: A running stage ID.
3696
- context: A context data.
3697
- parent_run_id: A parent running ID. (Default is None)
3698
- event: An event manager that use to track parent process
3699
- was not force stopped.
4042
+ run_id (str): A running stage ID.
4043
+ context (DictData): A context data that was passed from handler
4044
+ method.
4045
+ parent_run_id (str, default None): A parent running ID.
4046
+ event (Event, default None): An event manager that use to track
4047
+ parent process was not force stopped.
3700
4048
 
3701
4049
  Returns:
3702
4050
  Result: The execution result with status and context data.
@@ -3705,12 +4053,18 @@ class VirtualPyStage(PyStage): # pragma: no cov
3705
4053
  run_id, parent_run_id=parent_run_id, extras=self.extras
3706
4054
  )
3707
4055
  run: str = param2template(dedent(self.run), params, extras=self.extras)
3708
- with self.create_py_file(
4056
+ with self.make_py_file(
3709
4057
  py=run,
3710
4058
  values=param2template(self.vars, params, extras=self.extras),
3711
4059
  deps=param2template(self.deps, params, extras=self.extras),
3712
4060
  run_id=run_id,
3713
4061
  ) as py:
4062
+
4063
+ if event and event.is_set():
4064
+ raise StageCancelError(
4065
+ "Cancel before start virtual python process."
4066
+ )
4067
+
3714
4068
  trace.debug(f"[STAGE]: Create `{py}` file.")
3715
4069
  rs: CompletedProcess = subprocess.run(
3716
4070
  ["python", "-m", "uv", "run", py, "--no-cache"],
@@ -3756,12 +4110,24 @@ class VirtualPyStage(PyStage): # pragma: no cov
3756
4110
  parent_run_id: Optional[str] = None,
3757
4111
  event: Optional[Event] = None,
3758
4112
  ) -> Result:
4113
+ """Async execution method for this Virtual Python stage.
4114
+
4115
+ Args:
4116
+ params (DictData): A parameter data that want to use in this
4117
+ execution.
4118
+ run_id (str): A running stage ID.
4119
+ context (DictData): A context data that was passed from handler
4120
+ method.
4121
+ parent_run_id (str, default None): A parent running ID.
4122
+ event (Event, default None): An event manager that use to track
4123
+ parent process was not force stopped.
4124
+ """
3759
4125
  raise NotImplementedError(
3760
4126
  "Async process of Virtual Python stage does not implement yet."
3761
4127
  )
3762
4128
 
3763
4129
 
3764
- NestedStage = Annotated[
4130
+ SubStage = Annotated[
3765
4131
  Union[
3766
4132
  BashStage,
3767
4133
  CallStage,
@@ -3777,7 +4143,10 @@ NestedStage = Annotated[
3777
4143
  ],
3778
4144
  Field(
3779
4145
  union_mode="smart",
3780
- description="A nested-stage allow list",
4146
+ description=(
4147
+ "A nested-stage allow list that able to use on the NestedStage "
4148
+ "model."
4149
+ ),
3781
4150
  ),
3782
4151
  ] # pragma: no cov
3783
4152
 
@@ -3790,6 +4159,7 @@ ActionStage = Annotated[
3790
4159
  PyStage,
3791
4160
  RaiseStage,
3792
4161
  DockerStage,
4162
+ TriggerStage,
3793
4163
  EmptyStage,
3794
4164
  ],
3795
4165
  Field(