ddeutil-workflow 0.0.50__py3-none-any.whl → 0.0.52__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -56,9 +56,10 @@ from typing_extensions import Self
56
56
  from .__types import DictData, DictStr, TupleStr
57
57
  from .conf import dynamic
58
58
  from .exceptions import StageException, UtilException, to_dict
59
- from .result import FAILED, SUCCESS, WAIT, Result, Status
59
+ from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
60
60
  from .reusables import TagFunc, extract_call, not_in_template, param2template
61
61
  from .utils import (
62
+ delay,
62
63
  filter_func,
63
64
  gen_id,
64
65
  make_exec,
@@ -72,7 +73,8 @@ class BaseStage(BaseModel, ABC):
72
73
  metadata. If you want to implement any custom stage, you can use this class
73
74
  to parent and implement ``self.execute()`` method only.
74
75
 
75
- This class is the abstraction class for any stage class.
76
+ This class is the abstraction class for any stage model that want to
77
+ implement to workflow model.
76
78
  """
77
79
 
78
80
  extras: DictData = Field(
@@ -226,7 +228,7 @@ class BaseStage(BaseModel, ABC):
226
228
  and want to set on the `to` like;
227
229
 
228
230
  ... (i) output: {'foo': bar}
229
- ... (ii) to: {}
231
+ ... (ii) to: {'stages': {}}
230
232
 
231
233
  The result of the `to` argument will be;
232
234
 
@@ -234,11 +236,15 @@ class BaseStage(BaseModel, ABC):
234
236
  'stages': {
235
237
  '<stage-id>': {
236
238
  'outputs': {'foo': 'bar'},
237
- 'skipped': False
239
+ 'skipped': False,
238
240
  }
239
241
  }
240
242
  }
241
243
 
244
+ Important:
245
+ This method is use for reconstruct the result context and transfer
246
+ to the `to` argument.
247
+
242
248
  :param output: (DictData) An output data that want to extract to an
243
249
  output key.
244
250
  :param to: (DictData) A context data that want to add output result.
@@ -269,7 +275,11 @@ class BaseStage(BaseModel, ABC):
269
275
  if "skipped" in output
270
276
  else {}
271
277
  )
272
- to["stages"][_id] = {"outputs": output, **skipping, **errors}
278
+ to["stages"][_id] = {
279
+ "outputs": copy.deepcopy(output),
280
+ **skipping,
281
+ **errors,
282
+ }
273
283
  return to
274
284
 
275
285
  def get_outputs(self, outputs: DictData) -> DictData:
@@ -330,6 +340,7 @@ class BaseStage(BaseModel, ABC):
330
340
 
331
341
 
332
342
  class BaseAsyncStage(BaseStage):
343
+ """Base Async Stage model."""
333
344
 
334
345
  @abstractmethod
335
346
  def execute(
@@ -432,12 +443,13 @@ class EmptyStage(BaseAsyncStage):
432
443
 
433
444
  echo: Optional[str] = Field(
434
445
  default=None,
435
- description="A string statement that want to logging",
446
+ description="A string message that want to show on the stdout.",
436
447
  )
437
448
  sleep: float = Field(
438
449
  default=0,
439
- description="A second value to sleep before start execution",
450
+ description="A second value to sleep before start execution.",
440
451
  ge=0,
452
+ lt=1800,
441
453
  )
442
454
 
443
455
  def execute(
@@ -464,12 +476,23 @@ class EmptyStage(BaseAsyncStage):
464
476
  :rtype: Result
465
477
  """
466
478
  result: Result = result or Result(
467
- run_id=gen_id(self.name + (self.id or ""), unique=True)
479
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
480
+ extras=self.extras,
468
481
  )
469
482
 
483
+ if not self.echo:
484
+ message: str = "..."
485
+ else:
486
+ message: str = param2template(
487
+ dedent(self.echo), params, extras=self.extras
488
+ )
489
+ if "\n" in self.echo:
490
+ message: str = "\n\t" + message.replace("\n", "\n\t").strip(
491
+ "\n"
492
+ )
493
+
470
494
  result.trace.info(
471
- f"[STAGE]: Empty-Execute: {self.name!r}: "
472
- f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
495
+ f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
473
496
  )
474
497
  if self.sleep > 0:
475
498
  if self.sleep > 5:
@@ -497,9 +520,10 @@ class EmptyStage(BaseAsyncStage):
497
520
 
498
521
  :rtype: Result
499
522
  """
500
- if result is None: # pragma: no cov
523
+ if result is None:
501
524
  result: Result = Result(
502
- run_id=gen_id(self.name + (self.id or ""), unique=True)
525
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
526
+ extras=self.extras,
503
527
  )
504
528
 
505
529
  await result.trace.ainfo(
@@ -540,8 +564,8 @@ class BashStage(BaseStage):
540
564
  env: DictStr = Field(
541
565
  default_factory=dict,
542
566
  description=(
543
- "An environment variable mapping that want to set before execute "
544
- "this shell statement."
567
+ "An environment variables that set before start execute by adding "
568
+ "on the header of the `.sh` file."
545
569
  ),
546
570
  )
547
571
 
@@ -603,7 +627,8 @@ class BashStage(BaseStage):
603
627
  """
604
628
  if result is None: # pragma: no cov
605
629
  result: Result = Result(
606
- run_id=gen_id(self.name + (self.id or ""), unique=True)
630
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
631
+ extras=self.extras,
607
632
  )
608
633
 
609
634
  bash: str = param2template(
@@ -616,7 +641,7 @@ class BashStage(BaseStage):
616
641
  env=param2template(self.env, params, extras=self.extras),
617
642
  run_id=result.run_id,
618
643
  ) as sh:
619
- result.trace.debug(f"... Start create `{sh[1]}` file.")
644
+ result.trace.debug(f"... Create `{sh[1]}` file.")
620
645
  rs: CompletedProcess = subprocess.run(
621
646
  sh, shell=False, capture_output=True, text=True
622
647
  )
@@ -660,18 +685,20 @@ class PyStage(BaseStage):
660
685
  """
661
686
 
662
687
  run: str = Field(
663
- description="A Python string statement that want to run with exec.",
688
+ description="A Python string statement that want to run with `exec`.",
664
689
  )
665
690
  vars: DictData = Field(
666
691
  default_factory=dict,
667
692
  description=(
668
- "A mapping to variable that want to pass to globals in exec."
693
+ "A variable mapping that want to pass to globals parameter in the "
694
+ "`exec` func."
669
695
  ),
670
696
  )
671
697
 
672
698
  @staticmethod
673
699
  def filter_locals(values: DictData) -> Iterator[str]:
674
- """Filter a locals input values.
700
+ """Filter a locals mapping values that be module, class, or
701
+ __annotations__.
675
702
 
676
703
  :param values: (DictData) A locals values that want to filter.
677
704
 
@@ -728,7 +755,8 @@ class PyStage(BaseStage):
728
755
  """
729
756
  if result is None: # pragma: no cov
730
757
  result: Result = Result(
731
- run_id=gen_id(self.name + (self.id or ""), unique=True)
758
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
759
+ extras=self.extras,
732
760
  )
733
761
 
734
762
  lc: DictData = {}
@@ -741,11 +769,11 @@ class PyStage(BaseStage):
741
769
 
742
770
  # NOTE: Start exec the run statement.
743
771
  result.trace.info(f"[STAGE]: Py-Execute: {self.name}")
744
- result.trace.warning(
745
- "[STAGE]: This stage allow use `eval` function, so, please "
746
- "check your statement be safe before execute."
747
- )
748
-
772
+ # result.trace.warning(
773
+ # "[STAGE]: This stage allow use `eval` function, so, please "
774
+ # "check your statement be safe before execute."
775
+ # )
776
+ #
749
777
  # WARNING: The exec build-in function is very dangerous. So, it
750
778
  # should use the re module to validate exec-string before running.
751
779
  exec(
@@ -811,11 +839,17 @@ class CallStage(BaseStage):
811
839
  :param event: (Event) An event manager that use to track parent execute
812
840
  was not force stopped.
813
841
 
842
+ :raise ValueError: If necessary arguments does not pass from the `args`
843
+ field.
844
+ :raise TypeError: If the result from the caller function does not by
845
+ a `dict` type.
846
+
814
847
  :rtype: Result
815
848
  """
816
849
  if result is None: # pragma: no cov
817
850
  result: Result = Result(
818
- run_id=gen_id(self.name + (self.id or ""), unique=True)
851
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
852
+ extras=self.extras,
819
853
  )
820
854
 
821
855
  has_keyword: bool = False
@@ -890,9 +924,12 @@ class CallStage(BaseStage):
890
924
  """Parse Pydantic model from any dict data before parsing to target
891
925
  caller function.
892
926
 
893
- :param func:
894
- :param args:
895
- :param result: (Result)
927
+ :param func: A tag function that want to get typing.
928
+ :param args: An arguments before passing to this tag function.
929
+ :param result: (Result) A result object for keeping context and status
930
+ data.
931
+
932
+ :rtype: DictData
896
933
  """
897
934
  try:
898
935
  type_hints: dict[str, Any] = get_type_hints(func)
@@ -965,22 +1002,29 @@ class TriggerStage(BaseStage):
965
1002
 
966
1003
  :rtype: Result
967
1004
  """
1005
+ from .exceptions import WorkflowException
968
1006
  from .workflow import Workflow
969
1007
 
970
1008
  if result is None:
971
1009
  result: Result = Result(
972
- run_id=gen_id(self.name + (self.id or ""), unique=True)
1010
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1011
+ extras=self.extras,
973
1012
  )
974
1013
 
975
1014
  _trigger: str = param2template(self.trigger, params, extras=self.extras)
976
1015
  result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
977
- rs: Result = Workflow.from_conf(
978
- _trigger, extras=self.extras | {"stage_raise_error": True}
979
- ).execute(
980
- params=param2template(self.params, params, extras=self.extras),
981
- result=result,
982
- event=event,
983
- )
1016
+ try:
1017
+ rs: Result = Workflow.from_conf(
1018
+ name=_trigger,
1019
+ extras=self.extras | {"stage_raise_error": True},
1020
+ ).execute(
1021
+ params=param2template(self.params, params, extras=self.extras),
1022
+ parent_run_id=result.run_id,
1023
+ event=event,
1024
+ )
1025
+ except WorkflowException as e:
1026
+ raise StageException("Trigger workflow stage was failed") from e
1027
+
984
1028
  if rs.status == FAILED:
985
1029
  err_msg: str | None = (
986
1030
  f" with:\n{msg}"
@@ -1020,17 +1064,25 @@ class ParallelStage(BaseStage): # pragma: no cov
1020
1064
  """
1021
1065
 
1022
1066
  parallel: dict[str, list[Stage]] = Field(
1023
- description="A mapping of parallel branch ID.",
1067
+ description="A mapping of parallel branch name and stages.",
1068
+ )
1069
+ max_workers: int = Field(
1070
+ default=2,
1071
+ ge=1,
1072
+ lt=20,
1073
+ description=(
1074
+ "The maximum thread pool worker size for execution parallel."
1075
+ ),
1076
+ alias="max-workers",
1024
1077
  )
1025
- max_parallel_core: int = Field(default=2)
1026
1078
 
1027
- @staticmethod
1028
- def task(
1079
+ def execute_task(
1080
+ self,
1029
1081
  branch: str,
1030
1082
  params: DictData,
1031
1083
  result: Result,
1032
- stages: list[Stage],
1033
1084
  *,
1085
+ event: Event | None = None,
1034
1086
  extras: DictData | None = None,
1035
1087
  ) -> DictData:
1036
1088
  """Task execution method for passing a branch to each thread.
@@ -1039,32 +1091,83 @@ class ParallelStage(BaseStage): # pragma: no cov
1039
1091
  :param params: A parameter data that want to use in this execution.
1040
1092
  :param result: (Result) A result object for keeping context and status
1041
1093
  data.
1042
- :param stages:
1043
- :param extras
1094
+ :param event: (Event) An event manager that use to track parent execute
1095
+ was not force stopped.
1096
+ :param extras: (DictData) An extra parameters that want to override
1097
+ config values.
1044
1098
 
1045
1099
  :rtype: DictData
1046
1100
  """
1047
- context = {"branch": branch, "stages": {}}
1048
- result.trace.debug(f"[STAGE]: Execute parallel branch: {branch!r}")
1049
- for stage in stages:
1101
+ result.trace.debug(f"... Execute branch: {branch!r}")
1102
+ _params: DictData = copy.deepcopy(params)
1103
+ _params.update({"branch": branch})
1104
+ context: DictData = {"branch": branch, "stages": {}}
1105
+ for stage in self.parallel[branch]:
1106
+
1050
1107
  if extras:
1051
1108
  stage.extras = extras
1052
1109
 
1110
+ if stage.is_skipped(params=_params):
1111
+ result.trace.info(f"... Skip stage: {stage.iden!r}")
1112
+ stage.set_outputs(output={"skipped": True}, to=context)
1113
+ continue
1114
+
1115
+ if event and event.is_set():
1116
+ error_msg: str = (
1117
+ "Branch-Stage was canceled from event that had set before "
1118
+ "stage item execution."
1119
+ )
1120
+ return result.catch(
1121
+ status=CANCEL,
1122
+ parallel={
1123
+ branch: {
1124
+ "branch": branch,
1125
+ "stages": filter_func(context.pop("stages", {})),
1126
+ "errors": StageException(error_msg).to_dict(),
1127
+ }
1128
+ },
1129
+ )
1130
+
1053
1131
  try:
1054
- stage.set_outputs(
1055
- stage.handler_execute(
1056
- params=params,
1057
- run_id=result.run_id,
1058
- parent_run_id=result.parent_run_id,
1059
- ).context,
1060
- to=context,
1132
+ rs: Result = stage.handler_execute(
1133
+ params=_params,
1134
+ run_id=result.run_id,
1135
+ parent_run_id=result.parent_run_id,
1136
+ raise_error=True,
1137
+ event=event,
1061
1138
  )
1139
+ stage.set_outputs(rs.context, to=context)
1062
1140
  except StageException as e: # pragma: no cov
1063
- result.trace.error(
1064
- f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
1141
+ result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1142
+ raise StageException(
1143
+ f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
1144
+ ) from None
1145
+
1146
+ if rs.status == FAILED:
1147
+ error_msg: str = (
1148
+ f"Item-Stage was break because it has a sub stage, "
1149
+ f"{stage.iden}, failed without raise error."
1065
1150
  )
1066
- context.update({"errors": e.to_dict()})
1067
- return context
1151
+ return result.catch(
1152
+ status=FAILED,
1153
+ parallel={
1154
+ branch: {
1155
+ "branch": branch,
1156
+ "stages": filter_func(context.pop("stages", {})),
1157
+ "errors": StageException(error_msg).to_dict(),
1158
+ },
1159
+ },
1160
+ )
1161
+
1162
+ return result.catch(
1163
+ status=SUCCESS,
1164
+ parallel={
1165
+ branch: {
1166
+ "branch": branch,
1167
+ "stages": filter_func(context.pop("stages", {})),
1168
+ },
1169
+ },
1170
+ )
1068
1171
 
1069
1172
  def execute(
1070
1173
  self,
@@ -1086,41 +1189,46 @@ class ParallelStage(BaseStage): # pragma: no cov
1086
1189
  """
1087
1190
  if result is None: # pragma: no cov
1088
1191
  result: Result = Result(
1089
- run_id=gen_id(self.name + (self.id or ""), unique=True)
1192
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1193
+ extras=self.extras,
1090
1194
  )
1091
-
1195
+ event: Event = Event() if event is None else event
1092
1196
  result.trace.info(
1093
- f"[STAGE]: Parallel-Execute with {self.max_parallel_core} cores."
1197
+ f"[STAGE]: Parallel-Execute: {self.max_workers} workers."
1094
1198
  )
1095
- rs: DictData = {"parallel": {}}
1096
- status = SUCCESS
1199
+ result.catch(status=WAIT, context={"parallel": {}})
1097
1200
  with ThreadPoolExecutor(
1098
- max_workers=self.max_parallel_core,
1201
+ max_workers=self.max_workers,
1099
1202
  thread_name_prefix="parallel_stage_exec_",
1100
1203
  ) as executor:
1101
1204
 
1102
- futures: list[Future] = []
1103
- for branch in self.parallel:
1104
- futures.append(
1105
- executor.submit(
1106
- self.task,
1107
- branch=branch,
1108
- params=params,
1109
- result=result,
1110
- stages=self.parallel[branch],
1111
- extras=self.extras,
1112
- )
1205
+ context: DictData = {}
1206
+ status: Status = SUCCESS
1207
+
1208
+ futures: list[Future] = (
1209
+ executor.submit(
1210
+ self.execute_task,
1211
+ branch=branch,
1212
+ params=params,
1213
+ result=result,
1214
+ event=event,
1215
+ extras=self.extras,
1113
1216
  )
1217
+ for branch in self.parallel
1218
+ )
1114
1219
 
1115
1220
  done = as_completed(futures, timeout=1800)
1116
1221
  for future in done:
1117
- context: DictData = future.result()
1118
- rs["parallel"][context.pop("branch")] = context
1119
-
1120
- if "errors" in context:
1222
+ try:
1223
+ future.result()
1224
+ except StageException as e:
1121
1225
  status = FAILED
1226
+ result.trace.error(
1227
+ f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
1228
+ )
1229
+ context.update({"errors": e.to_dict()})
1122
1230
 
1123
- return result.catch(status=status, context=rs)
1231
+ return result.catch(status=status, context=context)
1124
1232
 
1125
1233
 
1126
1234
  class ForEachStage(BaseStage):
@@ -1182,28 +1290,31 @@ class ForEachStage(BaseStage):
1182
1290
  :param event: (Event) An event manager that use to track parent execute
1183
1291
  was not force stopped.
1184
1292
 
1293
+ :raise StageException: If the stage execution raise errors.
1294
+
1185
1295
  :rtype: Result
1186
1296
  """
1187
- result.trace.debug(f"[STAGE]: Execute item: {item!r}")
1188
- context: DictData = copy.deepcopy(params)
1189
- context.update({"item": item, "stages": {}})
1297
+ result.trace.debug(f"... Execute item: {item!r}")
1298
+ _params: DictData = copy.deepcopy(params)
1299
+ _params.update({"item": item})
1300
+ context: DictData = {"item": item, "stages": {}}
1190
1301
  for stage in self.stages:
1191
1302
 
1192
1303
  if self.extras:
1193
1304
  stage.extras = self.extras
1194
1305
 
1195
- if stage.is_skipped(params=context):
1196
- result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
1306
+ if stage.is_skipped(params=_params):
1307
+ result.trace.info(f"... Skip stage: {stage.iden!r}")
1197
1308
  stage.set_outputs(output={"skipped": True}, to=context)
1198
1309
  continue
1199
1310
 
1200
- if event and event.is_set():
1311
+ if event and event.is_set(): # pragma: no cov
1201
1312
  error_msg: str = (
1202
1313
  "Item-Stage was canceled from event that had set before "
1203
1314
  "stage item execution."
1204
1315
  )
1205
1316
  return result.catch(
1206
- status=FAILED,
1317
+ status=CANCEL,
1207
1318
  foreach={
1208
1319
  item: {
1209
1320
  "item": item,
@@ -1215,35 +1326,35 @@ class ForEachStage(BaseStage):
1215
1326
 
1216
1327
  try:
1217
1328
  rs: Result = stage.handler_execute(
1218
- params=context,
1329
+ params=_params,
1219
1330
  run_id=result.run_id,
1220
1331
  parent_run_id=result.parent_run_id,
1221
1332
  raise_error=True,
1333
+ event=event,
1222
1334
  )
1223
1335
  stage.set_outputs(rs.context, to=context)
1224
- if rs.status == FAILED:
1225
- error_msg: str = (
1226
- f"Item-Stage was break because it has a sub stage, "
1227
- f"{stage.iden}, failed without raise error."
1228
- )
1229
- return result.catch(
1230
- status=FAILED,
1231
- foreach={
1232
- item: {
1233
- "item": item,
1234
- "stages": filter_func(
1235
- context.pop("stages", {})
1236
- ),
1237
- "errors": StageException(error_msg).to_dict(),
1238
- },
1239
- },
1240
- )
1241
1336
  except (StageException, UtilException) as e:
1242
1337
  result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1243
1338
  raise StageException(
1244
1339
  f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
1245
1340
  ) from None
1246
1341
 
1342
+ if rs.status == FAILED:
1343
+ error_msg: str = (
1344
+ f"Item-Stage was break because it has a sub stage, "
1345
+ f"{stage.iden}, failed without raise error."
1346
+ )
1347
+ return result.catch(
1348
+ status=FAILED,
1349
+ foreach={
1350
+ item: {
1351
+ "item": item,
1352
+ "stages": filter_func(context.pop("stages", {})),
1353
+ "errors": StageException(error_msg).to_dict(),
1354
+ },
1355
+ },
1356
+ )
1357
+
1247
1358
  return result.catch(
1248
1359
  status=SUCCESS,
1249
1360
  foreach={
@@ -1291,7 +1402,7 @@ class ForEachStage(BaseStage):
1291
1402
  result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
1292
1403
  if event and event.is_set(): # pragma: no cov
1293
1404
  return result.catch(
1294
- status=FAILED,
1405
+ status=CANCEL,
1295
1406
  context={
1296
1407
  "errors": StageException(
1297
1408
  "Stage was canceled from event that had set "
@@ -1310,6 +1421,7 @@ class ForEachStage(BaseStage):
1310
1421
  item=item,
1311
1422
  params=params,
1312
1423
  result=result,
1424
+ event=event,
1313
1425
  )
1314
1426
  for item in foreach
1315
1427
  ]
@@ -1334,7 +1446,7 @@ class ForEachStage(BaseStage):
1334
1446
  except StageException as e:
1335
1447
  status = FAILED
1336
1448
  result.trace.error(
1337
- f"[STAGE]: Catch:\n\t{e.__class__.__name__}:\n\t{e}"
1449
+ f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
1338
1450
  )
1339
1451
  context.update({"errors": e.to_dict()})
1340
1452
 
@@ -1358,7 +1470,12 @@ class UntilStage(BaseStage): # pragma: no cov
1358
1470
  ... }
1359
1471
  """
1360
1472
 
1361
- item: Union[str, int, bool] = Field(description="An initial value.")
1473
+ item: Union[str, int, bool] = Field(
1474
+ default=0,
1475
+ description=(
1476
+ "An initial value that can be any value in str, int, or bool type."
1477
+ ),
1478
+ )
1362
1479
  until: str = Field(description="A until condition.")
1363
1480
  stages: list[Stage] = Field(
1364
1481
  default_factory=list,
@@ -1367,11 +1484,12 @@ class UntilStage(BaseStage): # pragma: no cov
1367
1484
  "correct."
1368
1485
  ),
1369
1486
  )
1370
- max_until_loop: int = Field(
1487
+ max_loop: int = Field(
1371
1488
  default=10,
1372
1489
  ge=1,
1373
1490
  lt=100,
1374
1491
  description="The maximum value of loop for this until stage.",
1492
+ alias="max-loop",
1375
1493
  )
1376
1494
 
1377
1495
  def execute_item(
@@ -1396,17 +1514,18 @@ class UntilStage(BaseStage): # pragma: no cov
1396
1514
 
1397
1515
  :rtype: tuple[Result, T]
1398
1516
  """
1399
- result.trace.debug(f"[STAGE]: Execute until item: {item!r}")
1400
- context: DictData = copy.deepcopy(params)
1401
- context.update({"loop": loop, "item": item, "stages": {}})
1517
+ result.trace.debug(f"... Execute until item: {item!r}")
1518
+ _params: DictData = copy.deepcopy(params)
1519
+ _params.update({"item": item})
1520
+ context: DictData = {"loop": loop, "item": item, "stages": {}}
1402
1521
  next_item: T = None
1403
1522
  for stage in self.stages:
1404
1523
 
1405
1524
  if self.extras:
1406
1525
  stage.extras = self.extras
1407
1526
 
1408
- if stage.is_skipped(params=context):
1409
- result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
1527
+ if stage.is_skipped(params=_params):
1528
+ result.trace.info(f"... Skip stage: {stage.iden!r}")
1410
1529
  stage.set_outputs(output={"skipped": True}, to=context)
1411
1530
  continue
1412
1531
 
@@ -1417,7 +1536,7 @@ class UntilStage(BaseStage): # pragma: no cov
1417
1536
  )
1418
1537
  return (
1419
1538
  result.catch(
1420
- status=FAILED,
1539
+ status=CANCEL,
1421
1540
  until={
1422
1541
  loop: {
1423
1542
  "loop": loop,
@@ -1434,10 +1553,11 @@ class UntilStage(BaseStage): # pragma: no cov
1434
1553
 
1435
1554
  try:
1436
1555
  rs: Result = stage.handler_execute(
1437
- params=context,
1556
+ params=_params,
1438
1557
  run_id=result.run_id,
1439
1558
  parent_run_id=result.parent_run_id,
1440
1559
  raise_error=True,
1560
+ event=event,
1441
1561
  )
1442
1562
  stage.set_outputs(rs.context, to=context)
1443
1563
  if "item" in (
@@ -1486,6 +1606,7 @@ class UntilStage(BaseStage): # pragma: no cov
1486
1606
  result: Result = Result(
1487
1607
  run_id=gen_id(self.name + (self.id or ""), unique=True)
1488
1608
  )
1609
+
1489
1610
  result.trace.info(f"[STAGE]: Until-Execution: {self.until}")
1490
1611
  item: Union[str, int, bool] = param2template(
1491
1612
  self.item, params, extras=self.extras
@@ -1494,11 +1615,11 @@ class UntilStage(BaseStage): # pragma: no cov
1494
1615
  track: bool = True
1495
1616
  exceed_loop: bool = False
1496
1617
  result.catch(status=WAIT, context={"until": {}})
1497
- while track and not (exceed_loop := loop >= self.max_until_loop):
1618
+ while track and not (exceed_loop := loop >= self.max_loop):
1498
1619
 
1499
1620
  if event and event.is_set():
1500
1621
  return result.catch(
1501
- status=FAILED,
1622
+ status=CANCEL,
1502
1623
  context={
1503
1624
  "errors": StageException(
1504
1625
  "Stage was canceled from event that had set "
@@ -1512,6 +1633,7 @@ class UntilStage(BaseStage): # pragma: no cov
1512
1633
  loop=loop,
1513
1634
  params=params,
1514
1635
  result=result,
1636
+ event=event,
1515
1637
  )
1516
1638
 
1517
1639
  loop += 1
@@ -1536,11 +1658,12 @@ class UntilStage(BaseStage): # pragma: no cov
1536
1658
  "Return type of until condition does not be boolean, it"
1537
1659
  f"return: {next_track!r}"
1538
1660
  )
1539
- track = not next_track
1661
+ track: bool = not next_track
1662
+ delay(0.025)
1540
1663
 
1541
1664
  if exceed_loop:
1542
1665
  raise StageException(
1543
- f"The until loop was exceed {self.max_until_loop} loops"
1666
+ f"The until loop was exceed {self.max_loop} loops"
1544
1667
  )
1545
1668
  return result.catch(status=SUCCESS)
1546
1669
 
@@ -1552,7 +1675,7 @@ class Match(BaseModel):
1552
1675
  stage: Stage = Field(description="A stage to execution for this case.")
1553
1676
 
1554
1677
 
1555
- class CaseStage(BaseStage): # pragma: no cov
1678
+ class CaseStage(BaseStage):
1556
1679
  """Case execution stage.
1557
1680
 
1558
1681
  Data Validate:
@@ -1590,6 +1713,14 @@ class CaseStage(BaseStage): # pragma: no cov
1590
1713
  match: list[Match] = Field(
1591
1714
  description="A list of Match model that should not be an empty list.",
1592
1715
  )
1716
+ skip_not_match: bool = Field(
1717
+ default=False,
1718
+ description=(
1719
+ "A flag for making skip if it does not match and else condition "
1720
+ "does not set too."
1721
+ ),
1722
+ alias="skip-not-match",
1723
+ )
1593
1724
 
1594
1725
  def execute(
1595
1726
  self,
@@ -1610,53 +1741,73 @@ class CaseStage(BaseStage): # pragma: no cov
1610
1741
  """
1611
1742
  if result is None: # pragma: no cov
1612
1743
  result: Result = Result(
1613
- run_id=gen_id(self.name + (self.id or ""), unique=True)
1744
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1745
+ extras=self.extras,
1614
1746
  )
1615
1747
 
1616
- _case = param2template(self.case, params, extras=self.extras)
1748
+ _case: Optional[str] = param2template(
1749
+ self.case, params, extras=self.extras
1750
+ )
1617
1751
 
1618
1752
  result.trace.info(f"[STAGE]: Case-Execute: {_case!r}.")
1619
- _else = None
1753
+ _else: Optional[Match] = None
1620
1754
  stage: Optional[Stage] = None
1621
- context = {}
1622
- status = SUCCESS
1623
1755
  for match in self.match:
1624
- if (c := match.case) != "_":
1625
- _condition = param2template(c, params, extras=self.extras)
1626
- else:
1627
- _else = match
1756
+ if (c := match.case) == "_":
1757
+ _else: Match = match
1628
1758
  continue
1629
1759
 
1760
+ _condition: str = param2template(c, params, extras=self.extras)
1630
1761
  if stage is None and _case == _condition:
1631
1762
  stage: Stage = match.stage
1632
1763
 
1633
1764
  if stage is None:
1634
1765
  if _else is None:
1635
- raise StageException(
1636
- "This stage does not set else for support not match "
1637
- "any case."
1766
+ if not self.skip_not_match:
1767
+ raise StageException(
1768
+ "This stage does not set else for support not match "
1769
+ "any case."
1770
+ )
1771
+ result.trace.info(
1772
+ "... Skip this stage because it does not match."
1773
+ )
1774
+ error_msg: str = (
1775
+ "Case-Stage was canceled because it does not match any "
1776
+ "case and else condition does not set too."
1777
+ )
1778
+ return result.catch(
1779
+ status=CANCEL,
1780
+ context={"errors": StageException(error_msg).to_dict()},
1638
1781
  )
1639
-
1640
1782
  stage: Stage = _else.stage
1641
1783
 
1642
1784
  if self.extras:
1643
1785
  stage.extras = self.extras
1644
1786
 
1787
+ if event and event.is_set(): # pragma: no cov
1788
+ return result.catch(
1789
+ status=CANCEL,
1790
+ context={
1791
+ "errors": StageException(
1792
+ "Stage was canceled from event that had set before "
1793
+ "case-stage execution."
1794
+ ).to_dict()
1795
+ },
1796
+ )
1797
+
1645
1798
  try:
1646
- context.update(
1647
- stage.handler_execute(
1799
+ return result.catch(
1800
+ status=SUCCESS,
1801
+ context=stage.handler_execute(
1648
1802
  params=params,
1649
1803
  run_id=result.run_id,
1650
1804
  parent_run_id=result.parent_run_id,
1651
- ).context
1805
+ event=event,
1806
+ ).context,
1652
1807
  )
1653
1808
  except StageException as e: # pragma: no cov
1654
- status = FAILED
1655
- result.trace.error(
1656
- f"[STAGE]: Catch:\n\t{e.__class__.__name__}:" f"\n\t{e}"
1657
- )
1658
- context.update({"errors": e.to_dict()})
1659
- return result.catch(status=status, context=context)
1809
+ result.trace.error(f"[STAGE]: {e.__class__.__name__}:" f"\n\t{e}")
1810
+ return result.catch(status=FAILED, context={"errors": e.to_dict()})
1660
1811
 
1661
1812
 
1662
1813
  class RaiseStage(BaseStage): # pragma: no cov
@@ -1672,7 +1823,9 @@ class RaiseStage(BaseStage): # pragma: no cov
1672
1823
  """
1673
1824
 
1674
1825
  message: str = Field(
1675
- description="An error message that want to raise",
1826
+ description=(
1827
+ "An error message that want to raise with StageException class"
1828
+ ),
1676
1829
  alias="raise",
1677
1830
  )
1678
1831
 
@@ -1693,7 +1846,8 @@ class RaiseStage(BaseStage): # pragma: no cov
1693
1846
  """
1694
1847
  if result is None: # pragma: no cov
1695
1848
  result: Result = Result(
1696
- run_id=gen_id(self.name + (self.id or ""), unique=True)
1849
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1850
+ extras=self.extras,
1697
1851
  )
1698
1852
  message: str = param2template(self.message, params, extras=self.extras)
1699
1853
  result.trace.info(f"[STAGE]: Raise-Execute: {message!r}.")
@@ -1728,10 +1882,15 @@ class DockerStage(BaseStage): # pragma: no cov
1728
1882
  ... "image": "image-name.pkg.com",
1729
1883
  ... "env": {
1730
1884
  ... "ENV": "dev",
1885
+ ... "DEBUG": "true",
1731
1886
  ... },
1732
1887
  ... "volume": {
1733
1888
  ... "secrets": "/secrets",
1734
1889
  ... },
1890
+ ... "auth": {
1891
+ ... "username": "__json_key",
1892
+ ... "password": "${GOOGLE_CREDENTIAL_JSON_STRING}",
1893
+ ... },
1735
1894
  ... }
1736
1895
  """
1737
1896
 
@@ -1750,6 +1909,7 @@ class DockerStage(BaseStage): # pragma: no cov
1750
1909
 
1751
1910
  def execute_task(
1752
1911
  self,
1912
+ params: DictData,
1753
1913
  result: Result,
1754
1914
  ):
1755
1915
  from docker import DockerClient
@@ -1762,26 +1922,32 @@ class DockerStage(BaseStage): # pragma: no cov
1762
1922
  resp = client.api.pull(
1763
1923
  repository=f"{self.image}",
1764
1924
  tag=self.tag,
1765
- # NOTE: For pulling from GCS.
1766
- # {
1767
- # "username": "_json_key",
1768
- # "password": credential-json-string,
1769
- # }
1770
- auth_config=self.auth,
1925
+ auth_config=param2template(self.auth, params, extras=self.extras),
1771
1926
  stream=True,
1772
1927
  decode=True,
1773
1928
  )
1774
1929
  for line in resp:
1775
1930
  result.trace.info(f"... {line}")
1776
1931
 
1777
- unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S}"
1932
+ unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
1778
1933
  container = client.containers.run(
1779
1934
  image=f"{self.image}:{self.tag}",
1780
1935
  name=unique_image_name,
1781
1936
  environment=self.env,
1782
1937
  volumes=(
1783
- {Path.cwd() / ".docker.logs": {"bind": "/logs", "mode": "rw"}}
1784
- | self.volume
1938
+ {
1939
+ Path.cwd()
1940
+ / f".docker.{result.run_id}.logs": {
1941
+ "bind": "/logs",
1942
+ "mode": "rw",
1943
+ },
1944
+ }
1945
+ | {
1946
+ Path.cwd() / source: {"bind": target, "mode": "rw"}
1947
+ for source, target in (
1948
+ volume.split(":", maxsplit=1) for volume in self.volume
1949
+ )
1950
+ }
1785
1951
  ),
1786
1952
  detach=True,
1787
1953
  )