ddeutil-workflow 0.0.50__py3-none-any.whl → 0.0.51__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -56,9 +56,10 @@ from typing_extensions import Self
56
56
  from .__types import DictData, DictStr, TupleStr
57
57
  from .conf import dynamic
58
58
  from .exceptions import StageException, UtilException, to_dict
59
- from .result import FAILED, SUCCESS, WAIT, Result, Status
59
+ from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
60
60
  from .reusables import TagFunc, extract_call, not_in_template, param2template
61
61
  from .utils import (
62
+ delay,
62
63
  filter_func,
63
64
  gen_id,
64
65
  make_exec,
@@ -72,7 +73,8 @@ class BaseStage(BaseModel, ABC):
72
73
  metadata. If you want to implement any custom stage, you can use this class
73
74
  to parent and implement ``self.execute()`` method only.
74
75
 
75
- This class is the abstraction class for any stage class.
76
+ This class is the abstraction class for any stage model that want to
77
+ implement to workflow model.
76
78
  """
77
79
 
78
80
  extras: DictData = Field(
@@ -226,7 +228,7 @@ class BaseStage(BaseModel, ABC):
226
228
  and want to set on the `to` like;
227
229
 
228
230
  ... (i) output: {'foo': bar}
229
- ... (ii) to: {}
231
+ ... (ii) to: {'stages': {}}
230
232
 
231
233
  The result of the `to` argument will be;
232
234
 
@@ -234,11 +236,15 @@ class BaseStage(BaseModel, ABC):
234
236
  'stages': {
235
237
  '<stage-id>': {
236
238
  'outputs': {'foo': 'bar'},
237
- 'skipped': False
239
+ 'skipped': False,
238
240
  }
239
241
  }
240
242
  }
241
243
 
244
+ Important:
245
+ This method is use for reconstruct the result context and transfer
246
+ to the `to` argument.
247
+
242
248
  :param output: (DictData) An output data that want to extract to an
243
249
  output key.
244
250
  :param to: (DictData) A context data that want to add output result.
@@ -269,7 +275,11 @@ class BaseStage(BaseModel, ABC):
269
275
  if "skipped" in output
270
276
  else {}
271
277
  )
272
- to["stages"][_id] = {"outputs": output, **skipping, **errors}
278
+ to["stages"][_id] = {
279
+ "outputs": copy.deepcopy(output),
280
+ **skipping,
281
+ **errors,
282
+ }
273
283
  return to
274
284
 
275
285
  def get_outputs(self, outputs: DictData) -> DictData:
@@ -330,6 +340,7 @@ class BaseStage(BaseModel, ABC):
330
340
 
331
341
 
332
342
  class BaseAsyncStage(BaseStage):
343
+ """Base Async Stage model."""
333
344
 
334
345
  @abstractmethod
335
346
  def execute(
@@ -432,12 +443,13 @@ class EmptyStage(BaseAsyncStage):
432
443
 
433
444
  echo: Optional[str] = Field(
434
445
  default=None,
435
- description="A string statement that want to logging",
446
+ description="A string message that want to show on the stdout.",
436
447
  )
437
448
  sleep: float = Field(
438
449
  default=0,
439
- description="A second value to sleep before start execution",
450
+ description="A second value to sleep before start execution.",
440
451
  ge=0,
452
+ lt=1800,
441
453
  )
442
454
 
443
455
  def execute(
@@ -464,12 +476,23 @@ class EmptyStage(BaseAsyncStage):
464
476
  :rtype: Result
465
477
  """
466
478
  result: Result = result or Result(
467
- run_id=gen_id(self.name + (self.id or ""), unique=True)
479
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
480
+ extras=self.extras,
468
481
  )
469
482
 
483
+ if not self.echo:
484
+ message: str = "..."
485
+ else:
486
+ message: str = param2template(
487
+ dedent(self.echo), params, extras=self.extras
488
+ )
489
+ if "\n" in self.echo:
490
+ message: str = "\n\t" + message.replace("\n", "\n\t").strip(
491
+ "\n"
492
+ )
493
+
470
494
  result.trace.info(
471
- f"[STAGE]: Empty-Execute: {self.name!r}: "
472
- f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
495
+ f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
473
496
  )
474
497
  if self.sleep > 0:
475
498
  if self.sleep > 5:
@@ -497,9 +520,10 @@ class EmptyStage(BaseAsyncStage):
497
520
 
498
521
  :rtype: Result
499
522
  """
500
- if result is None: # pragma: no cov
523
+ if result is None:
501
524
  result: Result = Result(
502
- run_id=gen_id(self.name + (self.id or ""), unique=True)
525
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
526
+ extras=self.extras,
503
527
  )
504
528
 
505
529
  await result.trace.ainfo(
@@ -540,8 +564,8 @@ class BashStage(BaseStage):
540
564
  env: DictStr = Field(
541
565
  default_factory=dict,
542
566
  description=(
543
- "An environment variable mapping that want to set before execute "
544
- "this shell statement."
567
+ "An environment variables that set before start execute by adding "
568
+ "on the header of the `.sh` file."
545
569
  ),
546
570
  )
547
571
 
@@ -603,7 +627,8 @@ class BashStage(BaseStage):
603
627
  """
604
628
  if result is None: # pragma: no cov
605
629
  result: Result = Result(
606
- run_id=gen_id(self.name + (self.id or ""), unique=True)
630
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
631
+ extras=self.extras,
607
632
  )
608
633
 
609
634
  bash: str = param2template(
@@ -616,7 +641,7 @@ class BashStage(BaseStage):
616
641
  env=param2template(self.env, params, extras=self.extras),
617
642
  run_id=result.run_id,
618
643
  ) as sh:
619
- result.trace.debug(f"... Start create `{sh[1]}` file.")
644
+ result.trace.debug(f"... Create `{sh[1]}` file.")
620
645
  rs: CompletedProcess = subprocess.run(
621
646
  sh, shell=False, capture_output=True, text=True
622
647
  )
@@ -660,18 +685,20 @@ class PyStage(BaseStage):
660
685
  """
661
686
 
662
687
  run: str = Field(
663
- description="A Python string statement that want to run with exec.",
688
+ description="A Python string statement that want to run with `exec`.",
664
689
  )
665
690
  vars: DictData = Field(
666
691
  default_factory=dict,
667
692
  description=(
668
- "A mapping to variable that want to pass to globals in exec."
693
+ "A variable mapping that want to pass to globals parameter in the "
694
+ "`exec` func."
669
695
  ),
670
696
  )
671
697
 
672
698
  @staticmethod
673
699
  def filter_locals(values: DictData) -> Iterator[str]:
674
- """Filter a locals input values.
700
+ """Filter a locals mapping values that be module, class, or
701
+ __annotations__.
675
702
 
676
703
  :param values: (DictData) A locals values that want to filter.
677
704
 
@@ -728,7 +755,8 @@ class PyStage(BaseStage):
728
755
  """
729
756
  if result is None: # pragma: no cov
730
757
  result: Result = Result(
731
- run_id=gen_id(self.name + (self.id or ""), unique=True)
758
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
759
+ extras=self.extras,
732
760
  )
733
761
 
734
762
  lc: DictData = {}
@@ -741,11 +769,11 @@ class PyStage(BaseStage):
741
769
 
742
770
  # NOTE: Start exec the run statement.
743
771
  result.trace.info(f"[STAGE]: Py-Execute: {self.name}")
744
- result.trace.warning(
745
- "[STAGE]: This stage allow use `eval` function, so, please "
746
- "check your statement be safe before execute."
747
- )
748
-
772
+ # result.trace.warning(
773
+ # "[STAGE]: This stage allow use `eval` function, so, please "
774
+ # "check your statement be safe before execute."
775
+ # )
776
+ #
749
777
  # WARNING: The exec build-in function is very dangerous. So, it
750
778
  # should use the re module to validate exec-string before running.
751
779
  exec(
@@ -811,11 +839,17 @@ class CallStage(BaseStage):
811
839
  :param event: (Event) An event manager that use to track parent execute
812
840
  was not force stopped.
813
841
 
842
+ :raise ValueError: If necessary arguments does not pass from the `args`
843
+ field.
844
+ :raise TypeError: If the result from the caller function does not by
845
+ a `dict` type.
846
+
814
847
  :rtype: Result
815
848
  """
816
849
  if result is None: # pragma: no cov
817
850
  result: Result = Result(
818
- run_id=gen_id(self.name + (self.id or ""), unique=True)
851
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
852
+ extras=self.extras,
819
853
  )
820
854
 
821
855
  has_keyword: bool = False
@@ -890,9 +924,12 @@ class CallStage(BaseStage):
890
924
  """Parse Pydantic model from any dict data before parsing to target
891
925
  caller function.
892
926
 
893
- :param func:
894
- :param args:
895
- :param result: (Result)
927
+ :param func: A tag function that want to get typing.
928
+ :param args: An arguments before passing to this tag function.
929
+ :param result: (Result) A result object for keeping context and status
930
+ data.
931
+
932
+ :rtype: DictData
896
933
  """
897
934
  try:
898
935
  type_hints: dict[str, Any] = get_type_hints(func)
@@ -965,22 +1002,29 @@ class TriggerStage(BaseStage):
965
1002
 
966
1003
  :rtype: Result
967
1004
  """
1005
+ from .exceptions import WorkflowException
968
1006
  from .workflow import Workflow
969
1007
 
970
1008
  if result is None:
971
1009
  result: Result = Result(
972
- run_id=gen_id(self.name + (self.id or ""), unique=True)
1010
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1011
+ extras=self.extras,
973
1012
  )
974
1013
 
975
1014
  _trigger: str = param2template(self.trigger, params, extras=self.extras)
976
1015
  result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
977
- rs: Result = Workflow.from_conf(
978
- _trigger, extras=self.extras | {"stage_raise_error": True}
979
- ).execute(
980
- params=param2template(self.params, params, extras=self.extras),
981
- result=result,
982
- event=event,
983
- )
1016
+ try:
1017
+ rs: Result = Workflow.from_conf(
1018
+ name=_trigger,
1019
+ extras=self.extras | {"stage_raise_error": True},
1020
+ ).execute(
1021
+ params=param2template(self.params, params, extras=self.extras),
1022
+ parent_run_id=result.run_id,
1023
+ event=event,
1024
+ )
1025
+ except WorkflowException as e:
1026
+ raise StageException("Trigger workflow stage was failed") from e
1027
+
984
1028
  if rs.status == FAILED:
985
1029
  err_msg: str | None = (
986
1030
  f" with:\n{msg}"
@@ -1020,17 +1064,25 @@ class ParallelStage(BaseStage): # pragma: no cov
1020
1064
  """
1021
1065
 
1022
1066
  parallel: dict[str, list[Stage]] = Field(
1023
- description="A mapping of parallel branch ID.",
1067
+ description="A mapping of parallel branch name and stages.",
1068
+ )
1069
+ max_workers: int = Field(
1070
+ default=2,
1071
+ ge=1,
1072
+ lt=20,
1073
+ description=(
1074
+ "The maximum thread pool worker size for execution parallel."
1075
+ ),
1076
+ alias="max-workers",
1024
1077
  )
1025
- max_parallel_core: int = Field(default=2)
1026
1078
 
1027
- @staticmethod
1028
- def task(
1079
+ def execute_task(
1080
+ self,
1029
1081
  branch: str,
1030
1082
  params: DictData,
1031
1083
  result: Result,
1032
- stages: list[Stage],
1033
1084
  *,
1085
+ event: Event | None = None,
1034
1086
  extras: DictData | None = None,
1035
1087
  ) -> DictData:
1036
1088
  """Task execution method for passing a branch to each thread.
@@ -1039,32 +1091,82 @@ class ParallelStage(BaseStage): # pragma: no cov
1039
1091
  :param params: A parameter data that want to use in this execution.
1040
1092
  :param result: (Result) A result object for keeping context and status
1041
1093
  data.
1042
- :param stages:
1043
- :param extras
1094
+ :param event: (Event) An event manager that use to track parent execute
1095
+ was not force stopped.
1096
+ :param extras: (DictData) An extra parameters that want to override
1097
+ config values.
1044
1098
 
1045
1099
  :rtype: DictData
1046
1100
  """
1047
- context = {"branch": branch, "stages": {}}
1048
- result.trace.debug(f"[STAGE]: Execute parallel branch: {branch!r}")
1049
- for stage in stages:
1101
+ result.trace.debug(f"... Execute branch: {branch!r}")
1102
+ context: DictData = copy.deepcopy(params)
1103
+ context.update({"branch": branch, "stages": {}})
1104
+ for stage in self.parallel[branch]:
1105
+
1050
1106
  if extras:
1051
1107
  stage.extras = extras
1052
1108
 
1109
+ if stage.is_skipped(params=context):
1110
+ result.trace.info(f"... Skip stage: {stage.iden!r}")
1111
+ stage.set_outputs(output={"skipped": True}, to=context)
1112
+ continue
1113
+
1114
+ if event and event.is_set():
1115
+ error_msg: str = (
1116
+ "Branch-Stage was canceled from event that had set before "
1117
+ "stage item execution."
1118
+ )
1119
+ return result.catch(
1120
+ status=CANCEL,
1121
+ parallel={
1122
+ branch: {
1123
+ "branch": branch,
1124
+ "stages": filter_func(context.pop("stages", {})),
1125
+ "errors": StageException(error_msg).to_dict(),
1126
+ }
1127
+ },
1128
+ )
1129
+
1053
1130
  try:
1054
- stage.set_outputs(
1055
- stage.handler_execute(
1056
- params=params,
1057
- run_id=result.run_id,
1058
- parent_run_id=result.parent_run_id,
1059
- ).context,
1060
- to=context,
1131
+ rs: Result = stage.handler_execute(
1132
+ params=context,
1133
+ run_id=result.run_id,
1134
+ parent_run_id=result.parent_run_id,
1135
+ raise_error=True,
1136
+ event=event,
1061
1137
  )
1138
+ stage.set_outputs(rs.context, to=context)
1062
1139
  except StageException as e: # pragma: no cov
1063
- result.trace.error(
1064
- f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
1140
+ result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1141
+ raise StageException(
1142
+ f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
1143
+ ) from None
1144
+
1145
+ if rs.status == FAILED:
1146
+ error_msg: str = (
1147
+ f"Item-Stage was break because it has a sub stage, "
1148
+ f"{stage.iden}, failed without raise error."
1149
+ )
1150
+ return result.catch(
1151
+ status=FAILED,
1152
+ parallel={
1153
+ branch: {
1154
+ "branch": branch,
1155
+ "stages": filter_func(context.pop("stages", {})),
1156
+ "errors": StageException(error_msg).to_dict(),
1157
+ },
1158
+ },
1065
1159
  )
1066
- context.update({"errors": e.to_dict()})
1067
- return context
1160
+
1161
+ return result.catch(
1162
+ status=SUCCESS,
1163
+ parallel={
1164
+ branch: {
1165
+ "branch": branch,
1166
+ "stages": filter_func(context.pop("stages", {})),
1167
+ },
1168
+ },
1169
+ )
1068
1170
 
1069
1171
  def execute(
1070
1172
  self,
@@ -1086,41 +1188,46 @@ class ParallelStage(BaseStage): # pragma: no cov
1086
1188
  """
1087
1189
  if result is None: # pragma: no cov
1088
1190
  result: Result = Result(
1089
- run_id=gen_id(self.name + (self.id or ""), unique=True)
1191
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1192
+ extras=self.extras,
1090
1193
  )
1091
-
1194
+ event: Event = Event() if event is None else event
1092
1195
  result.trace.info(
1093
- f"[STAGE]: Parallel-Execute with {self.max_parallel_core} cores."
1196
+ f"[STAGE]: Parallel-Execute: {self.max_workers} workers."
1094
1197
  )
1095
- rs: DictData = {"parallel": {}}
1096
- status = SUCCESS
1198
+ result.catch(status=WAIT, context={"parallel": {}})
1097
1199
  with ThreadPoolExecutor(
1098
- max_workers=self.max_parallel_core,
1200
+ max_workers=self.max_workers,
1099
1201
  thread_name_prefix="parallel_stage_exec_",
1100
1202
  ) as executor:
1101
1203
 
1102
- futures: list[Future] = []
1103
- for branch in self.parallel:
1104
- futures.append(
1105
- executor.submit(
1106
- self.task,
1107
- branch=branch,
1108
- params=params,
1109
- result=result,
1110
- stages=self.parallel[branch],
1111
- extras=self.extras,
1112
- )
1204
+ context: DictData = {}
1205
+ status: Status = SUCCESS
1206
+
1207
+ futures: list[Future] = (
1208
+ executor.submit(
1209
+ self.execute_task,
1210
+ branch=branch,
1211
+ params=params,
1212
+ result=result,
1213
+ event=event,
1214
+ extras=self.extras,
1113
1215
  )
1216
+ for branch in self.parallel
1217
+ )
1114
1218
 
1115
1219
  done = as_completed(futures, timeout=1800)
1116
1220
  for future in done:
1117
- context: DictData = future.result()
1118
- rs["parallel"][context.pop("branch")] = context
1119
-
1120
- if "errors" in context:
1221
+ try:
1222
+ future.result()
1223
+ except StageException as e:
1121
1224
  status = FAILED
1225
+ result.trace.error(
1226
+ f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
1227
+ )
1228
+ context.update({"errors": e.to_dict()})
1122
1229
 
1123
- return result.catch(status=status, context=rs)
1230
+ return result.catch(status=status, context=context)
1124
1231
 
1125
1232
 
1126
1233
  class ForEachStage(BaseStage):
@@ -1182,9 +1289,11 @@ class ForEachStage(BaseStage):
1182
1289
  :param event: (Event) An event manager that use to track parent execute
1183
1290
  was not force stopped.
1184
1291
 
1292
+ :raise StageException: If the stage execution raise errors.
1293
+
1185
1294
  :rtype: Result
1186
1295
  """
1187
- result.trace.debug(f"[STAGE]: Execute item: {item!r}")
1296
+ result.trace.debug(f"... Execute item: {item!r}")
1188
1297
  context: DictData = copy.deepcopy(params)
1189
1298
  context.update({"item": item, "stages": {}})
1190
1299
  for stage in self.stages:
@@ -1193,17 +1302,17 @@ class ForEachStage(BaseStage):
1193
1302
  stage.extras = self.extras
1194
1303
 
1195
1304
  if stage.is_skipped(params=context):
1196
- result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
1305
+ result.trace.info(f"... Skip stage: {stage.iden!r}")
1197
1306
  stage.set_outputs(output={"skipped": True}, to=context)
1198
1307
  continue
1199
1308
 
1200
- if event and event.is_set():
1309
+ if event and event.is_set(): # pragma: no cov
1201
1310
  error_msg: str = (
1202
1311
  "Item-Stage was canceled from event that had set before "
1203
1312
  "stage item execution."
1204
1313
  )
1205
1314
  return result.catch(
1206
- status=FAILED,
1315
+ status=CANCEL,
1207
1316
  foreach={
1208
1317
  item: {
1209
1318
  "item": item,
@@ -1219,31 +1328,31 @@ class ForEachStage(BaseStage):
1219
1328
  run_id=result.run_id,
1220
1329
  parent_run_id=result.parent_run_id,
1221
1330
  raise_error=True,
1331
+ event=event,
1222
1332
  )
1223
1333
  stage.set_outputs(rs.context, to=context)
1224
- if rs.status == FAILED:
1225
- error_msg: str = (
1226
- f"Item-Stage was break because it has a sub stage, "
1227
- f"{stage.iden}, failed without raise error."
1228
- )
1229
- return result.catch(
1230
- status=FAILED,
1231
- foreach={
1232
- item: {
1233
- "item": item,
1234
- "stages": filter_func(
1235
- context.pop("stages", {})
1236
- ),
1237
- "errors": StageException(error_msg).to_dict(),
1238
- },
1239
- },
1240
- )
1241
1334
  except (StageException, UtilException) as e:
1242
1335
  result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1243
1336
  raise StageException(
1244
1337
  f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
1245
1338
  ) from None
1246
1339
 
1340
+ if rs.status == FAILED:
1341
+ error_msg: str = (
1342
+ f"Item-Stage was break because it has a sub stage, "
1343
+ f"{stage.iden}, failed without raise error."
1344
+ )
1345
+ return result.catch(
1346
+ status=FAILED,
1347
+ foreach={
1348
+ item: {
1349
+ "item": item,
1350
+ "stages": filter_func(context.pop("stages", {})),
1351
+ "errors": StageException(error_msg).to_dict(),
1352
+ },
1353
+ },
1354
+ )
1355
+
1247
1356
  return result.catch(
1248
1357
  status=SUCCESS,
1249
1358
  foreach={
@@ -1291,7 +1400,7 @@ class ForEachStage(BaseStage):
1291
1400
  result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
1292
1401
  if event and event.is_set(): # pragma: no cov
1293
1402
  return result.catch(
1294
- status=FAILED,
1403
+ status=CANCEL,
1295
1404
  context={
1296
1405
  "errors": StageException(
1297
1406
  "Stage was canceled from event that had set "
@@ -1310,6 +1419,7 @@ class ForEachStage(BaseStage):
1310
1419
  item=item,
1311
1420
  params=params,
1312
1421
  result=result,
1422
+ event=event,
1313
1423
  )
1314
1424
  for item in foreach
1315
1425
  ]
@@ -1334,7 +1444,7 @@ class ForEachStage(BaseStage):
1334
1444
  except StageException as e:
1335
1445
  status = FAILED
1336
1446
  result.trace.error(
1337
- f"[STAGE]: Catch:\n\t{e.__class__.__name__}:\n\t{e}"
1447
+ f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
1338
1448
  )
1339
1449
  context.update({"errors": e.to_dict()})
1340
1450
 
@@ -1358,7 +1468,12 @@ class UntilStage(BaseStage): # pragma: no cov
1358
1468
  ... }
1359
1469
  """
1360
1470
 
1361
- item: Union[str, int, bool] = Field(description="An initial value.")
1471
+ item: Union[str, int, bool] = Field(
1472
+ default=0,
1473
+ description=(
1474
+ "An initial value that can be any value in str, int, or bool type."
1475
+ ),
1476
+ )
1362
1477
  until: str = Field(description="A until condition.")
1363
1478
  stages: list[Stage] = Field(
1364
1479
  default_factory=list,
@@ -1367,11 +1482,12 @@ class UntilStage(BaseStage): # pragma: no cov
1367
1482
  "correct."
1368
1483
  ),
1369
1484
  )
1370
- max_until_loop: int = Field(
1485
+ max_loop: int = Field(
1371
1486
  default=10,
1372
1487
  ge=1,
1373
1488
  lt=100,
1374
1489
  description="The maximum value of loop for this until stage.",
1490
+ alias="max-loop",
1375
1491
  )
1376
1492
 
1377
1493
  def execute_item(
@@ -1396,7 +1512,7 @@ class UntilStage(BaseStage): # pragma: no cov
1396
1512
 
1397
1513
  :rtype: tuple[Result, T]
1398
1514
  """
1399
- result.trace.debug(f"[STAGE]: Execute until item: {item!r}")
1515
+ result.trace.debug(f"... Execute until item: {item!r}")
1400
1516
  context: DictData = copy.deepcopy(params)
1401
1517
  context.update({"loop": loop, "item": item, "stages": {}})
1402
1518
  next_item: T = None
@@ -1406,7 +1522,7 @@ class UntilStage(BaseStage): # pragma: no cov
1406
1522
  stage.extras = self.extras
1407
1523
 
1408
1524
  if stage.is_skipped(params=context):
1409
- result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
1525
+ result.trace.info(f"... Skip stage: {stage.iden!r}")
1410
1526
  stage.set_outputs(output={"skipped": True}, to=context)
1411
1527
  continue
1412
1528
 
@@ -1417,7 +1533,7 @@ class UntilStage(BaseStage): # pragma: no cov
1417
1533
  )
1418
1534
  return (
1419
1535
  result.catch(
1420
- status=FAILED,
1536
+ status=CANCEL,
1421
1537
  until={
1422
1538
  loop: {
1423
1539
  "loop": loop,
@@ -1438,6 +1554,7 @@ class UntilStage(BaseStage): # pragma: no cov
1438
1554
  run_id=result.run_id,
1439
1555
  parent_run_id=result.parent_run_id,
1440
1556
  raise_error=True,
1557
+ event=event,
1441
1558
  )
1442
1559
  stage.set_outputs(rs.context, to=context)
1443
1560
  if "item" in (
@@ -1486,6 +1603,7 @@ class UntilStage(BaseStage): # pragma: no cov
1486
1603
  result: Result = Result(
1487
1604
  run_id=gen_id(self.name + (self.id or ""), unique=True)
1488
1605
  )
1606
+
1489
1607
  result.trace.info(f"[STAGE]: Until-Execution: {self.until}")
1490
1608
  item: Union[str, int, bool] = param2template(
1491
1609
  self.item, params, extras=self.extras
@@ -1494,11 +1612,11 @@ class UntilStage(BaseStage): # pragma: no cov
1494
1612
  track: bool = True
1495
1613
  exceed_loop: bool = False
1496
1614
  result.catch(status=WAIT, context={"until": {}})
1497
- while track and not (exceed_loop := loop >= self.max_until_loop):
1615
+ while track and not (exceed_loop := loop >= self.max_loop):
1498
1616
 
1499
1617
  if event and event.is_set():
1500
1618
  return result.catch(
1501
- status=FAILED,
1619
+ status=CANCEL,
1502
1620
  context={
1503
1621
  "errors": StageException(
1504
1622
  "Stage was canceled from event that had set "
@@ -1512,6 +1630,7 @@ class UntilStage(BaseStage): # pragma: no cov
1512
1630
  loop=loop,
1513
1631
  params=params,
1514
1632
  result=result,
1633
+ event=event,
1515
1634
  )
1516
1635
 
1517
1636
  loop += 1
@@ -1536,11 +1655,12 @@ class UntilStage(BaseStage): # pragma: no cov
1536
1655
  "Return type of until condition does not be boolean, it"
1537
1656
  f"return: {next_track!r}"
1538
1657
  )
1539
- track = not next_track
1658
+ track: bool = not next_track
1659
+ delay(0.025)
1540
1660
 
1541
1661
  if exceed_loop:
1542
1662
  raise StageException(
1543
- f"The until loop was exceed {self.max_until_loop} loops"
1663
+ f"The until loop was exceed {self.max_loop} loops"
1544
1664
  )
1545
1665
  return result.catch(status=SUCCESS)
1546
1666
 
@@ -1552,7 +1672,7 @@ class Match(BaseModel):
1552
1672
  stage: Stage = Field(description="A stage to execution for this case.")
1553
1673
 
1554
1674
 
1555
- class CaseStage(BaseStage): # pragma: no cov
1675
+ class CaseStage(BaseStage):
1556
1676
  """Case execution stage.
1557
1677
 
1558
1678
  Data Validate:
@@ -1590,6 +1710,14 @@ class CaseStage(BaseStage): # pragma: no cov
1590
1710
  match: list[Match] = Field(
1591
1711
  description="A list of Match model that should not be an empty list.",
1592
1712
  )
1713
+ skip_not_match: bool = Field(
1714
+ default=False,
1715
+ description=(
1716
+ "A flag for making skip if it does not match and else condition "
1717
+ "does not set too."
1718
+ ),
1719
+ alias="skip-not-match",
1720
+ )
1593
1721
 
1594
1722
  def execute(
1595
1723
  self,
@@ -1610,53 +1738,73 @@ class CaseStage(BaseStage): # pragma: no cov
1610
1738
  """
1611
1739
  if result is None: # pragma: no cov
1612
1740
  result: Result = Result(
1613
- run_id=gen_id(self.name + (self.id or ""), unique=True)
1741
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1742
+ extras=self.extras,
1614
1743
  )
1615
1744
 
1616
- _case = param2template(self.case, params, extras=self.extras)
1745
+ _case: Optional[str] = param2template(
1746
+ self.case, params, extras=self.extras
1747
+ )
1617
1748
 
1618
1749
  result.trace.info(f"[STAGE]: Case-Execute: {_case!r}.")
1619
- _else = None
1750
+ _else: Optional[Match] = None
1620
1751
  stage: Optional[Stage] = None
1621
- context = {}
1622
- status = SUCCESS
1623
1752
  for match in self.match:
1624
- if (c := match.case) != "_":
1625
- _condition = param2template(c, params, extras=self.extras)
1626
- else:
1627
- _else = match
1753
+ if (c := match.case) == "_":
1754
+ _else: Match = match
1628
1755
  continue
1629
1756
 
1757
+ _condition: str = param2template(c, params, extras=self.extras)
1630
1758
  if stage is None and _case == _condition:
1631
1759
  stage: Stage = match.stage
1632
1760
 
1633
1761
  if stage is None:
1634
1762
  if _else is None:
1635
- raise StageException(
1636
- "This stage does not set else for support not match "
1637
- "any case."
1763
+ if not self.skip_not_match:
1764
+ raise StageException(
1765
+ "This stage does not set else for support not match "
1766
+ "any case."
1767
+ )
1768
+ result.trace.info(
1769
+ "... Skip this stage because it does not match."
1770
+ )
1771
+ error_msg: str = (
1772
+ "Case-Stage was canceled because it does not match any "
1773
+ "case and else condition does not set too."
1774
+ )
1775
+ return result.catch(
1776
+ status=CANCEL,
1777
+ context={"errors": StageException(error_msg).to_dict()},
1638
1778
  )
1639
-
1640
1779
  stage: Stage = _else.stage
1641
1780
 
1642
1781
  if self.extras:
1643
1782
  stage.extras = self.extras
1644
1783
 
1784
+ if event and event.is_set(): # pragma: no cov
1785
+ return result.catch(
1786
+ status=CANCEL,
1787
+ context={
1788
+ "errors": StageException(
1789
+ "Stage was canceled from event that had set before "
1790
+ "case-stage execution."
1791
+ ).to_dict()
1792
+ },
1793
+ )
1794
+
1645
1795
  try:
1646
- context.update(
1647
- stage.handler_execute(
1796
+ return result.catch(
1797
+ status=SUCCESS,
1798
+ context=stage.handler_execute(
1648
1799
  params=params,
1649
1800
  run_id=result.run_id,
1650
1801
  parent_run_id=result.parent_run_id,
1651
- ).context
1802
+ event=event,
1803
+ ).context,
1652
1804
  )
1653
1805
  except StageException as e: # pragma: no cov
1654
- status = FAILED
1655
- result.trace.error(
1656
- f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
1657
- )
1658
- context.update({"errors": e.to_dict()})
1659
- return result.catch(status=status, context=context)
1806
+ result.trace.error(f"[STAGE]: {e.__class__.__name__}:" f"\n\t{e}")
1807
+ return result.catch(status=FAILED, context={"errors": e.to_dict()})
1660
1808
 
1661
1809
 
1662
1810
  class RaiseStage(BaseStage): # pragma: no cov
@@ -1672,7 +1820,9 @@ class RaiseStage(BaseStage): # pragma: no cov
1672
1820
  """
1673
1821
 
1674
1822
  message: str = Field(
1675
- description="An error message that want to raise",
1823
+ description=(
1824
+ "An error message that want to raise with StageException class"
1825
+ ),
1676
1826
  alias="raise",
1677
1827
  )
1678
1828
 
@@ -1693,7 +1843,8 @@ class RaiseStage(BaseStage): # pragma: no cov
1693
1843
  """
1694
1844
  if result is None: # pragma: no cov
1695
1845
  result: Result = Result(
1696
- run_id=gen_id(self.name + (self.id or ""), unique=True)
1846
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1847
+ extras=self.extras,
1697
1848
  )
1698
1849
  message: str = param2template(self.message, params, extras=self.extras)
1699
1850
  result.trace.info(f"[STAGE]: Raise-Execute: {message!r}.")
@@ -1728,10 +1879,15 @@ class DockerStage(BaseStage): # pragma: no cov
1728
1879
  ... "image": "image-name.pkg.com",
1729
1880
  ... "env": {
1730
1881
  ... "ENV": "dev",
1882
+ ... "DEBUG": "true",
1731
1883
  ... },
1732
1884
  ... "volume": {
1733
1885
  ... "secrets": "/secrets",
1734
1886
  ... },
1887
+ ... "auth": {
1888
+ ... "username": "__json_key",
1889
+ ... "password": "${GOOGLE_CREDENTIAL_JSON_STRING}",
1890
+ ... },
1735
1891
  ... }
1736
1892
  """
1737
1893
 
@@ -1750,6 +1906,7 @@ class DockerStage(BaseStage): # pragma: no cov
1750
1906
 
1751
1907
  def execute_task(
1752
1908
  self,
1909
+ params: DictData,
1753
1910
  result: Result,
1754
1911
  ):
1755
1912
  from docker import DockerClient
@@ -1762,26 +1919,32 @@ class DockerStage(BaseStage): # pragma: no cov
1762
1919
  resp = client.api.pull(
1763
1920
  repository=f"{self.image}",
1764
1921
  tag=self.tag,
1765
- # NOTE: For pulling from GCS.
1766
- # {
1767
- # "username": "_json_key",
1768
- # "password": credential-json-string,
1769
- # }
1770
- auth_config=self.auth,
1922
+ auth_config=param2template(self.auth, params, extras=self.extras),
1771
1923
  stream=True,
1772
1924
  decode=True,
1773
1925
  )
1774
1926
  for line in resp:
1775
1927
  result.trace.info(f"... {line}")
1776
1928
 
1777
- unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S}"
1929
+ unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
1778
1930
  container = client.containers.run(
1779
1931
  image=f"{self.image}:{self.tag}",
1780
1932
  name=unique_image_name,
1781
1933
  environment=self.env,
1782
1934
  volumes=(
1783
- {Path.cwd() / ".docker.logs": {"bind": "/logs", "mode": "rw"}}
1784
- | self.volume
1935
+ {
1936
+ Path.cwd()
1937
+ / f".docker.{result.run_id}.logs": {
1938
+ "bind": "/logs",
1939
+ "mode": "rw",
1940
+ },
1941
+ }
1942
+ | {
1943
+ Path.cwd() / source: {"bind": target, "mode": "rw"}
1944
+ for source, target in (
1945
+ volume.split(":", maxsplit=1) for volume in self.volume
1946
+ )
1947
+ }
1785
1948
  ),
1786
1949
  detach=True,
1787
1950
  )