ddeutil-workflow 0.0.53__py3-none-any.whl → 0.0.54__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,23 +4,27 @@
4
4
  # license information.
5
5
  # ------------------------------------------------------------------------------
6
6
  # [x] Use dynamic config
7
- """Stage model. It stores all stage model that use for getting stage data template
8
- from the Job Model. The stage handle the minimize task that run in some thread
9
- (same thread at its job owner) that mean it is the lowest executor of a workflow
10
- that can tracking logs.
7
+ """Stages module include all stage model that use be the minimum execution layer
8
+ of this workflow engine. The stage handle the minimize task that run in some
9
+ thread (same thread at its job owner) that mean it is the lowest executor that
10
+ you can track logs.
11
11
 
12
- The output of stage execution only return 0 status because I do not want to
13
- handle stage error on this stage model. I think stage model should have a lot of
14
- use-case, and it does not worry when I want to create a new one.
12
+ The output of stage execution only return SUCCESS or CANCEL status because
13
+ I do not want to handle stage error on this stage execution. I think stage model
14
+ have a lot of use-case, and it should does not worry about it error output.
15
15
 
16
- Execution --> Ok --> Result with SUCCESS
16
+ So, I will create `handler_execute` for any exception class that raise from
17
+ the stage execution method.
17
18
 
18
- --> Error ┬-> Result with FAILED (if env var was set)
19
- ╰-> Raise StageException(...)
19
+ Execution --> Ok ---( handler )--> Result with `SUCCESS` or `CANCEL`
20
+
21
+ --> Error ┬--( handler )-> Result with `FAILED` (Set `raise_error` flag)
22
+ |
23
+ ╰--( handler )-> Raise StageException(...)
20
24
 
21
25
  On the context I/O that pass to a stage object at execute process. The
22
- execute method receives a `params={"params": {...}}` value for mapping to
23
- template searching.
26
+ execute method receives a `params={"params": {...}}` value for passing template
27
+ searching.
24
28
  """
25
29
  from __future__ import annotations
26
30
 
@@ -28,12 +32,13 @@ import asyncio
28
32
  import contextlib
29
33
  import copy
30
34
  import inspect
35
+ import json
31
36
  import subprocess
32
37
  import sys
33
38
  import time
34
39
  import uuid
35
40
  from abc import ABC, abstractmethod
36
- from collections.abc import Iterator
41
+ from collections.abc import AsyncIterator, Iterator
37
42
  from concurrent.futures import (
38
43
  FIRST_EXCEPTION,
39
44
  Future,
@@ -59,6 +64,7 @@ from .exceptions import StageException, UtilException, to_dict
59
64
  from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
60
65
  from .reusables import TagFunc, extract_call, not_in_template, param2template
61
66
  from .utils import (
67
+ NEWLINE,
62
68
  delay,
63
69
  filter_func,
64
70
  gen_id,
@@ -66,20 +72,22 @@ from .utils import (
66
72
  )
67
73
 
68
74
  T = TypeVar("T")
75
+ StrOrInt = Union[str, int]
69
76
 
70
77
 
71
78
  class BaseStage(BaseModel, ABC):
72
- """Base Stage Model that keep only id and name fields for the stage
73
- metadata. If you want to implement any custom stage, you can use this class
74
- to parent and implement ``self.execute()`` method only.
79
+ """Base Stage Model that keep only necessary fields like `id`, `name` or
80
+ `condition` for the stage metadata. If you want to implement any custom
81
+ stage, you can inherit this class and implement `self.execute()` method
82
+ only.
75
83
 
76
- This class is the abstraction class for any stage model that want to
77
- implement to workflow model.
84
+ This class is the abstraction class for any inherit stage model that
85
+ want to implement on this workflow package.
78
86
  """
79
87
 
80
88
  extras: DictData = Field(
81
89
  default_factory=dict,
82
- description="An extra override config values.",
90
+ description="An extra parameter that override core config values.",
83
91
  )
84
92
  id: Optional[str] = Field(
85
93
  default=None,
@@ -93,14 +101,17 @@ class BaseStage(BaseModel, ABC):
93
101
  )
94
102
  condition: Optional[str] = Field(
95
103
  default=None,
96
- description="A stage condition statement to allow stage executable.",
104
+ description=(
105
+ "A stage condition statement to allow stage executable. This field "
106
+ "alise with `if` field."
107
+ ),
97
108
  alias="if",
98
109
  )
99
110
 
100
111
  @property
101
112
  def iden(self) -> str:
102
- """Return identity of this stage object that return the id field first.
103
- If the id does not set, it will use name field instead.
113
+ """Return this stage identity that return the `id` field first and if
114
+ this `id` field does not set, it will use the `name` field instead.
104
115
 
105
116
  :rtype: str
106
117
  """
@@ -156,8 +167,8 @@ class BaseStage(BaseModel, ABC):
156
167
  run_id: str | None = None,
157
168
  parent_run_id: str | None = None,
158
169
  result: Result | None = None,
159
- raise_error: bool | None = None,
160
170
  event: Event | None = None,
171
+ raise_error: bool | None = None,
161
172
  ) -> Result | DictData:
162
173
  """Handler stage execution result from the stage `execute` method.
163
174
 
@@ -170,7 +181,7 @@ class BaseStage(BaseModel, ABC):
170
181
  ╰-context:
171
182
  ╰-outputs: ...
172
183
 
173
- --> Error --> Result (if env var was set)
184
+ --> Error --> Result (if `raise_error` was set)
174
185
  |-status: FAILED
175
186
  ╰-errors:
176
187
  |-class: ...
@@ -179,18 +190,17 @@ class BaseStage(BaseModel, ABC):
179
190
 
180
191
  --> Error --> Raise StageException(...)
181
192
 
182
- On the last step, it will set the running ID on a return result object
183
- from current stage ID before release the final result.
193
+ On the last step, it will set the running ID on a return result
194
+ object from the current stage ID before release the final result.
184
195
 
185
- :param params: (DictData) A parameterize value data that use in this
186
- stage execution.
196
+ :param params: (DictData) A parameter data.
187
197
  :param run_id: (str) A running stage ID for this execution.
188
198
  :param parent_run_id: (str) A parent workflow running ID for this
189
199
  execution.
190
200
  :param result: (Result) A result object for keeping context and status
191
201
  data before execution.
192
- :param raise_error: (bool) A flag that all this method raise error
193
202
  :param event: (Event) An event manager that pass to the stage execution.
203
+ :param raise_error: (bool) A flag that all this method raise error
194
204
 
195
205
  :rtype: Result
196
206
  """
@@ -206,8 +216,9 @@ class BaseStage(BaseModel, ABC):
206
216
  rs: Result = self.execute(params, result=result, event=event)
207
217
  return rs
208
218
  except Exception as e:
209
- result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
210
-
219
+ result.trace.error(
220
+ f"[STAGE]: Handler:{NEWLINE}{e.__class__.__name__}: {e}"
221
+ )
211
222
  if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
212
223
  if isinstance(e, StageException):
213
224
  raise
@@ -217,37 +228,39 @@ class BaseStage(BaseModel, ABC):
217
228
  f"{e.__class__.__name__}: {e}"
218
229
  ) from e
219
230
 
220
- errors: DictData = {"errors": to_dict(e)}
221
- return result.catch(status=FAILED, context=errors)
231
+ return result.catch(status=FAILED, context={"errors": to_dict(e)})
222
232
 
223
233
  def set_outputs(self, output: DictData, to: DictData) -> DictData:
224
- """Set an outputs from execution process to the received context. The
225
- result from execution will pass to value of `outputs` key.
234
+ """Set an outputs from execution result context to the received context
235
+ with a `to` input parameter. The result context from stage execution
236
+ will be set with `outputs` key in this stage ID key.
226
237
 
227
238
  For example of setting output method, If you receive execute output
228
239
  and want to set on the `to` like;
229
240
 
230
- ... (i) output: {'foo': bar}
241
+ ... (i) output: {'foo': 'bar', 'skipped': True}
231
242
  ... (ii) to: {'stages': {}}
232
243
 
233
- The result of the `to` argument will be;
244
+ The received context in the `to` argument will be;
234
245
 
235
246
  ... (iii) to: {
236
247
  'stages': {
237
248
  '<stage-id>': {
238
249
  'outputs': {'foo': 'bar'},
239
- 'skipped': False,
250
+ 'skipped': True,
240
251
  }
241
252
  }
242
253
  }
243
254
 
244
255
  Important:
256
+
245
257
  This method is use for reconstruct the result context and transfer
246
- to the `to` argument.
258
+ to the `to` argument. The result context was soft copied before set
259
+ output step.
247
260
 
248
- :param output: (DictData) An output data that want to extract to an
249
- output key.
250
- :param to: (DictData) A context data that want to add output result.
261
+ :param output: (DictData) A result data context that want to extract
262
+ and transfer to the `outputs` key in receive context.
263
+ :param to: (DictData) A received context data.
251
264
 
252
265
  :rtype: DictData
253
266
  """
@@ -282,11 +295,12 @@ class BaseStage(BaseModel, ABC):
282
295
  }
283
296
  return to
284
297
 
285
- def get_outputs(self, outputs: DictData) -> DictData:
298
+ def get_outputs(self, output: DictData) -> DictData:
286
299
  """Get the outputs from stages data. It will get this stage ID from
287
300
  the stage outputs mapping.
288
301
 
289
- :param outputs: (DictData) A stage outputs that want to get by stage ID.
302
+ :param output: (DictData) A stage output context that want to get this
303
+ stage ID `outputs` key.
290
304
 
291
305
  :rtype: DictData
292
306
  """
@@ -296,33 +310,31 @@ class BaseStage(BaseModel, ABC):
296
310
  return {}
297
311
 
298
312
  _id: str = (
299
- param2template(self.id, params=outputs, extras=self.extras)
313
+ param2template(self.id, params=output, extras=self.extras)
300
314
  if self.id
301
315
  else gen_id(
302
- param2template(self.name, params=outputs, extras=self.extras)
316
+ param2template(self.name, params=output, extras=self.extras)
303
317
  )
304
318
  )
305
- return outputs.get("stages", {}).get(_id, {}).get("outputs", {})
319
+ return output.get("stages", {}).get(_id, {}).get("outputs", {})
306
320
 
307
- def is_skipped(self, params: DictData | None = None) -> bool:
321
+ def is_skipped(self, params: DictData) -> bool:
308
322
  """Return true if condition of this stage do not correct. This process
309
323
  use build-in eval function to execute the if-condition.
310
324
 
325
+ :param params: (DictData) A parameters that want to pass to condition
326
+ template.
327
+
311
328
  :raise StageException: When it has any error raise from the eval
312
329
  condition statement.
313
330
  :raise StageException: When return type of the eval condition statement
314
331
  does not return with boolean type.
315
332
 
316
- :param params: (DictData) A parameters that want to pass to condition
317
- template.
318
-
319
333
  :rtype: bool
320
334
  """
321
335
  if self.condition is None:
322
336
  return False
323
337
 
324
- params: DictData = {} if params is None else params
325
-
326
338
  try:
327
339
  # WARNING: The eval build-in function is very dangerous. So, it
328
340
  # should use the `re` module to validate eval-string before
@@ -340,7 +352,14 @@ class BaseStage(BaseModel, ABC):
340
352
 
341
353
 
342
354
  class BaseAsyncStage(BaseStage):
343
- """Base Async Stage model."""
355
+ """Base Async Stage model to make any stage model allow async execution for
356
+ optimize CPU and Memory on the current node. If you want to implement any
357
+ custom async stage, you can inherit this class and implement
358
+ `self.axecute()` (async + execute = axecute) method only.
359
+
360
+ This class is the abstraction class for any inherit asyncable stage
361
+ model.
362
+ """
344
363
 
345
364
  @abstractmethod
346
365
  def execute(
@@ -385,20 +404,18 @@ class BaseAsyncStage(BaseStage):
385
404
  run_id: str | None = None,
386
405
  parent_run_id: str | None = None,
387
406
  result: Result | None = None,
388
- raise_error: bool | None = None,
389
407
  event: Event | None = None,
408
+ raise_error: bool | None = None,
390
409
  ) -> Result:
391
410
  """Async Handler stage execution result from the stage `execute` method.
392
411
 
393
- :param params: (DictData) A parameterize value data that use in this
394
- stage execution.
395
- :param run_id: (str) A running stage ID for this execution.
396
- :param parent_run_id: (str) A parent workflow running ID for this
397
- execution.
398
- :param result: (Result) A result object for keeping context and status
399
- data before execution.
412
+ :param params: (DictData) A parameter data.
413
+ :param run_id: (str) A stage running ID.
414
+ :param parent_run_id: (str) A parent job running ID.
415
+ :param result: (Result) A Result instance for return context and status.
416
+ :param event: (Event) An Event manager instance that use to cancel this
417
+ execution if it forces stopped by parent execution.
400
418
  :param raise_error: (bool) A flag that all this method raise error
401
- :param event: (Event) An event manager that pass to the stage execution.
402
419
 
403
420
  :rtype: Result
404
421
  """
@@ -414,7 +431,9 @@ class BaseAsyncStage(BaseStage):
414
431
  rs: Result = await self.axecute(params, result=result, event=event)
415
432
  return rs
416
433
  except Exception as e: # pragma: no cov
417
- await result.trace.aerror(f"[STAGE]: {e.__class__.__name__}: {e}")
434
+ await result.trace.aerror(
435
+ f"[STAGE]: Handler {e.__class__.__name__}: {e}"
436
+ )
418
437
 
419
438
  if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
420
439
  if isinstance(e, StageException):
@@ -425,13 +444,16 @@ class BaseAsyncStage(BaseStage):
425
444
  f"{e.__class__.__name__}: {e}"
426
445
  ) from None
427
446
 
428
- errors: DictData = {"errors": to_dict(e)}
429
- return result.catch(status=FAILED, context=errors)
447
+ return result.catch(status=FAILED, context={"errors": to_dict(e)})
430
448
 
431
449
 
432
450
  class EmptyStage(BaseAsyncStage):
433
- """Empty stage that do nothing (context equal empty stage) and logging the
434
- name of stage only to stdout.
451
+ """Empty stage executor that do nothing and log the `message` field to
452
+ stdout only. It can use for tracking a template parameter on the workflow or
453
+ debug step.
454
+
455
+ You can pass a sleep value in second unit to this stage for waiting
456
+ after log message.
435
457
 
436
458
  Data Validate:
437
459
  >>> stage = {
@@ -443,11 +465,14 @@ class EmptyStage(BaseAsyncStage):
443
465
 
444
466
  echo: Optional[str] = Field(
445
467
  default=None,
446
- description="A string message that want to show on the stdout.",
468
+ description="A message that want to show on the stdout.",
447
469
  )
448
470
  sleep: float = Field(
449
471
  default=0,
450
- description="A second value to sleep before start execution.",
472
+ description=(
473
+ "A second value to sleep before start execution. This value should "
474
+ "gather or equal 0, and less than 1800 seconds."
475
+ ),
451
476
  ge=0,
452
477
  lt=1800,
453
478
  )
@@ -460,18 +485,15 @@ class EmptyStage(BaseAsyncStage):
460
485
  event: Event | None = None,
461
486
  ) -> Result:
462
487
  """Execution method for the Empty stage that do only logging out to
463
- stdout. This method does not use the `handler_result` decorator because
464
- it does not get any error from logging function.
488
+ stdout.
465
489
 
466
490
  The result context should be empty and do not process anything
467
491
  without calling logging function.
468
492
 
469
- :param params: (DictData) A context data that want to add output result.
470
- But this stage does not pass any output.
471
- :param result: (Result) A result object for keeping context and status
472
- data.
473
- :param event: (Event) An event manager that use to track parent execute
474
- was not force stopped.
493
+ :param params: (DictData) A parameter data.
494
+ :param result: (Result) A Result instance for return context and status.
495
+ :param event: (Event) An Event manager instance that use to cancel this
496
+ execution if it forces stopped by parent execution.
475
497
 
476
498
  :rtype: Result
477
499
  """
@@ -484,21 +506,18 @@ class EmptyStage(BaseAsyncStage):
484
506
  message: str = "..."
485
507
  else:
486
508
  message: str = param2template(
487
- dedent(self.echo), params, extras=self.extras
509
+ dedent(self.echo.strip("\n")), params, extras=self.extras
488
510
  )
489
- if "\n" in self.echo:
490
- message: str = "\n\t" + message.replace("\n", "\n\t").strip(
491
- "\n"
492
- )
511
+ if "\n" in message:
512
+ message: str = NEWLINE + message.replace("\n", NEWLINE)
493
513
 
494
514
  result.trace.info(
495
515
  f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
496
516
  )
497
517
  if self.sleep > 0:
498
518
  if self.sleep > 5:
499
- result.trace.info(f"[STAGE]: ... sleep ({self.sleep} seconds)")
519
+ result.trace.info(f"[STAGE]: ... sleep ({self.sleep} sec)")
500
520
  time.sleep(self.sleep)
501
-
502
521
  return result.catch(status=SUCCESS)
503
522
 
504
523
  async def axecute(
@@ -511,12 +530,10 @@ class EmptyStage(BaseAsyncStage):
511
530
  """Async execution method for this Empty stage that only logging out to
512
531
  stdout.
513
532
 
514
- :param params: (DictData) A context data that want to add output result.
515
- But this stage does not pass any output.
516
- :param result: (Result) A result object for keeping context and status
517
- data.
518
- :param event: (Event) An event manager that use to track parent execute
519
- was not force stopped.
533
+ :param params: (DictData) A parameter data.
534
+ :param result: (Result) A Result instance for return context and status.
535
+ :param event: (Event) An Event manager instance that use to cancel this
536
+ execution if it forces stopped by parent execution.
520
537
 
521
538
  :rtype: Result
522
539
  """
@@ -526,29 +543,36 @@ class EmptyStage(BaseAsyncStage):
526
543
  extras=self.extras,
527
544
  )
528
545
 
529
- await result.trace.ainfo(
530
- f"[STAGE]: Empty-Execute: {self.name!r}: "
531
- f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
532
- )
546
+ if not self.echo:
547
+ message: str = "..."
548
+ else:
549
+ message: str = param2template(
550
+ dedent(self.echo.strip("\n")), params, extras=self.extras
551
+ )
552
+ if "\n" in message:
553
+ message: str = NEWLINE + message.replace("\n", NEWLINE)
533
554
 
555
+ result.trace.info(
556
+ f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
557
+ )
534
558
  if self.sleep > 0:
535
559
  if self.sleep > 5:
536
560
  await result.trace.ainfo(
537
- f"[STAGE]: ... sleep ({self.sleep} seconds)"
561
+ f"[STAGE]: ... sleep ({self.sleep} sec)"
538
562
  )
539
563
  await asyncio.sleep(self.sleep)
540
-
541
564
  return result.catch(status=SUCCESS)
542
565
 
543
566
 
544
567
  class BashStage(BaseStage):
545
- """Bash execution stage that execute bash script on the current OS.
546
- If your current OS is Windows, it will run on the bash in the WSL.
568
+ """Bash stage executor that execute bash script on the current OS.
569
+ If your current OS is Windows, it will run on the bash from the current WSL.
570
+ It will use `bash` for Windows OS and use `sh` for Linux OS.
547
571
 
548
- I get some limitation when I run shell statement with the built-in
549
- subprocess package. It does not good enough to use multiline statement.
550
- Thus, I add writing ``.sh`` file before execution process for fix this
551
- issue.
572
+ This stage has some limitation when it runs shell statement with the
573
+ built-in subprocess package. It does not good enough to use multiline
574
+ statement. Thus, it will write the `.sh` file before start running bash
575
+ command for fix this issue.
552
576
 
553
577
  Data Validate:
554
578
  >>> stage = {
@@ -560,15 +584,46 @@ class BashStage(BaseStage):
560
584
  ... }
561
585
  """
562
586
 
563
- bash: str = Field(description="A bash statement that want to execute.")
587
+ bash: str = Field(
588
+ description=(
589
+ "A bash statement that want to execute via Python subprocess."
590
+ )
591
+ )
564
592
  env: DictStr = Field(
565
593
  default_factory=dict,
566
594
  description=(
567
- "An environment variables that set before start execute by adding "
568
- "on the header of the `.sh` file."
595
+ "An environment variables that set before run bash command. It "
596
+ "will add on the header of the `.sh` file."
569
597
  ),
570
598
  )
571
599
 
600
+ @contextlib.asynccontextmanager
601
+ async def acreate_sh_file(
602
+ self, bash: str, env: DictStr, run_id: str | None = None
603
+ ) -> AsyncIterator: # pragma no cov
604
+ import aiofiles
605
+
606
+ f_name: str = f"{run_id or uuid.uuid4()}.sh"
607
+ f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
608
+
609
+ async with aiofiles.open(f"./{f_name}", mode="w", newline="\n") as f:
610
+ # NOTE: write header of `.sh` file
611
+ await f.write(f"#!/bin/{f_shebang}\n\n")
612
+
613
+ # NOTE: add setting environment variable before bash skip statement.
614
+ await f.writelines([f"{k}='{env[k]}';\n" for k in env])
615
+
616
+ # NOTE: make sure that shell script file does not have `\r` char.
617
+ await f.write("\n" + bash.replace("\r\n", "\n"))
618
+
619
+ # NOTE: Make this .sh file able to executable.
620
+ make_exec(f"./{f_name}")
621
+
622
+ yield [f_shebang, f_name]
623
+
624
+ # Note: Remove .sh file that use to run bash.
625
+ Path(f"./{f_name}").unlink()
626
+
572
627
  @contextlib.contextmanager
573
628
  def create_sh_file(
574
629
  self, bash: str, env: DictStr, run_id: str | None = None
@@ -577,16 +632,14 @@ class BashStage(BaseStage):
577
632
  step will write the `.sh` file before giving this file name to context.
578
633
  After that, it will auto delete this file automatic.
579
634
 
580
- :param bash: (str) A bash statement that want to execute.
581
- :param env: (DictStr) An environment variable that use on this bash
582
- statement.
635
+ :param bash: (str) A bash statement.
636
+ :param env: (DictStr) An environment variable that set before run bash.
583
637
  :param run_id: (str | None) A running stage ID that use for writing sh
584
638
  file instead generate by UUID4.
585
639
 
586
640
  :rtype: Iterator[TupleStr]
587
641
  """
588
- run_id: str = run_id or uuid.uuid4()
589
- f_name: str = f"{run_id}.sh"
642
+ f_name: str = f"{run_id or uuid.uuid4()}.sh"
590
643
  f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
591
644
 
592
645
  with open(f"./{f_name}", mode="w", newline="\n") as f:
@@ -614,14 +667,14 @@ class BashStage(BaseStage):
614
667
  result: Result | None = None,
615
668
  event: Event | None = None,
616
669
  ) -> Result:
617
- """Execute the Bash statement with the Python build-in ``subprocess``
618
- package.
670
+ """Execute bash statement with the Python build-in `subprocess` package.
671
+ It will catch result from the `subprocess.run` returning output like
672
+ `return_code`, `stdout`, and `stderr`.
619
673
 
620
- :param params: A parameter data that want to use in this execution.
621
- :param result: (Result) A result object for keeping context and status
622
- data.
623
- :param event: (Event) An event manager that use to track parent execute
624
- was not force stopped.
674
+ :param params: (DictData) A parameter data.
675
+ :param result: (Result) A Result instance for return context and status.
676
+ :param event: (Event) An Event manager instance that use to cancel this
677
+ execution if it forces stopped by parent execution.
625
678
 
626
679
  :rtype: Result
627
680
  """
@@ -631,11 +684,12 @@ class BashStage(BaseStage):
631
684
  extras=self.extras,
632
685
  )
633
686
 
687
+ result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
688
+
634
689
  bash: str = param2template(
635
- dedent(self.bash), params, extras=self.extras
690
+ dedent(self.bash.strip("\n")), params, extras=self.extras
636
691
  )
637
692
 
638
- result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
639
693
  with self.create_sh_file(
640
694
  bash=bash,
641
695
  env=param2template(self.env, params, extras=self.extras),
@@ -654,7 +708,7 @@ class BashStage(BaseStage):
654
708
  else rs.stderr
655
709
  ).removesuffix("\n")
656
710
  raise StageException(
657
- f"Subprocess: {e}\nRunning Statement:\n---\n"
711
+ f"Subprocess: {e}\n---( statement )---\n"
658
712
  f"```bash\n{bash}\n```"
659
713
  )
660
714
  return result.catch(
@@ -668,18 +722,24 @@ class BashStage(BaseStage):
668
722
 
669
723
 
670
724
  class PyStage(BaseStage):
671
- """Python executor stage that running the Python statement with receiving
672
- globals and additional variables.
725
+ """Python stage that running the Python statement with the current globals
726
+ and passing an input additional variables via `exec` built-in function.
673
727
 
674
728
  This stage allow you to use any Python object that exists on the globals
675
729
  such as import your installed package.
676
730
 
731
+ Warning:
732
+
733
+ The exec build-in function is very dangerous. So, it should use the `re`
734
+ module to validate exec-string before running or exclude the `os` package
735
+ from the current globals variable.
736
+
677
737
  Data Validate:
678
738
  >>> stage = {
679
739
  ... "name": "Python stage execution",
680
- ... "run": 'print("Hello {x}")',
740
+ ... "run": 'print(f"Hello {VARIABLE}")',
681
741
  ... "vars": {
682
- ... "x": "BAR",
742
+ ... "VARIABLE": "WORLD",
683
743
  ... },
684
744
  ... }
685
745
  """
@@ -741,9 +801,9 @@ class PyStage(BaseStage):
741
801
  event: Event | None = None,
742
802
  ) -> Result:
743
803
  """Execute the Python statement that pass all globals and input params
744
- to globals argument on ``exec`` build-in function.
804
+ to globals argument on `exec` build-in function.
745
805
 
746
- :param params: A parameter that want to pass before run any statement.
806
+ :param params: (DictData) A parameter data.
747
807
  :param result: (Result) A result object for keeping context and status
748
808
  data.
749
809
  :param event: (Event) An event manager that use to track parent execute
@@ -792,20 +852,36 @@ class PyStage(BaseStage):
792
852
  },
793
853
  )
794
854
 
855
+ async def axecute(
856
+ self,
857
+ ):
858
+ """Async execution method.
859
+
860
+ References:
861
+ - https://stackoverflow.com/questions/44859165/async-exec-in-python
862
+ """
863
+
795
864
 
796
865
  class CallStage(BaseStage):
797
- """Call executor that call the Python function from registry with tag
798
- decorator function in ``utils`` module and run it with input arguments.
866
+ """Call stage executor that call the Python function from registry with tag
867
+ decorator function in `reusables` module and run it with input arguments.
868
+
869
+ This stage is different with PyStage because the PyStage is just run
870
+ a Python statement with the `exec` function and pass the current locals and
871
+ globals before exec that statement. This stage will import the caller
872
+ function can call it with an input arguments. So, you can create your
873
+ function complexly that you can for your objective to invoked by this stage
874
+ object.
799
875
 
800
- This stage is different with PyStage because the PyStage is just calling
801
- a Python statement with the ``eval`` and pass that locale before eval that
802
- statement. So, you can create your function complexly that you can for your
803
- objective to invoked by this stage object.
876
+ This stage is the most powerfull stage of this package for run every
877
+ use-case by a custom requirement that you want by creating the Python
878
+ function and adding it to the caller registry value by importer syntax like
879
+ `module.caller.registry` not path style like `module/caller/registry`.
804
880
 
805
- This stage is the usefull stage for run every job by a custom requirement
806
- that you want by creating the Python function and adding it to the task
807
- registry by importer syntax like `module.tasks.registry` not path style like
808
- `module/tasks/registry`.
881
+ Warning:
882
+
883
+ The caller registry to get a caller function should importable by the
884
+ current Python execution pointer.
809
885
 
810
886
  Data Validate:
811
887
  >>> stage = {
@@ -817,12 +893,16 @@ class CallStage(BaseStage):
817
893
 
818
894
  uses: str = Field(
819
895
  description=(
820
- "A pointer that want to load function from the call registry."
896
+ "A caller function with registry importer syntax that use to load "
897
+ "function before execute step. The caller registry syntax should "
898
+ "be `<import.part>/<func-name>@<tag-name>`."
821
899
  ),
822
900
  )
823
901
  args: DictData = Field(
824
902
  default_factory=dict,
825
- description="An arguments that want to pass to the call function.",
903
+ description=(
904
+ "An argument parameter that will pass to this caller function."
905
+ ),
826
906
  alias="with",
827
907
  )
828
908
 
@@ -833,19 +913,12 @@ class CallStage(BaseStage):
833
913
  result: Result | None = None,
834
914
  event: Event | None = None,
835
915
  ) -> Result:
836
- """Execute the Call function that already in the call registry.
916
+ """Execute this caller function with its argument parameter.
837
917
 
838
- :raise ValueError: When the necessary arguments of call function do not
839
- set from the input params argument.
840
- :raise TypeError: When the return type of call function does not be
841
- dict type.
842
-
843
- :param params: (DictData) A parameter that want to pass before run any
844
- statement.
845
- :param result: (Result) A result object for keeping context and status
846
- data.
847
- :param event: (Event) An event manager that use to track parent execute
848
- was not force stopped.
918
+ :param params: (DictData) A parameter data.
919
+ :param result: (Result) A Result instance for return context and status.
920
+ :param event: (Event) An Event manager instance that use to cancel this
921
+ execution if it forces stopped by parent execution.
849
922
 
850
923
  :raise ValueError: If necessary arguments does not pass from the `args`
851
924
  field.
@@ -860,12 +933,15 @@ class CallStage(BaseStage):
860
933
  extras=self.extras,
861
934
  )
862
935
 
863
- has_keyword: bool = False
864
936
  call_func: TagFunc = extract_call(
865
937
  param2template(self.uses, params, extras=self.extras),
866
938
  registries=self.extras.get("registry_caller"),
867
939
  )()
868
940
 
941
+ result.trace.info(
942
+ f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
943
+ )
944
+
869
945
  # VALIDATE: check input task caller parameters that exists before
870
946
  # calling.
871
947
  args: DictData = {"result": result} | param2template(
@@ -873,6 +949,7 @@ class CallStage(BaseStage):
873
949
  )
874
950
  ips = inspect.signature(call_func)
875
951
  necessary_params: list[str] = []
952
+ has_keyword: bool = False
876
953
  for k in ips.parameters:
877
954
  if (
878
955
  v := ips.parameters[k]
@@ -896,10 +973,6 @@ class CallStage(BaseStage):
896
973
  if "result" not in ips.parameters and not has_keyword:
897
974
  args.pop("result")
898
975
 
899
- result.trace.info(
900
- f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
901
- )
902
-
903
976
  args = self.parse_model_args(call_func, args, result)
904
977
 
905
978
  if inspect.iscoroutinefunction(call_func):
@@ -970,9 +1043,9 @@ class CallStage(BaseStage):
970
1043
 
971
1044
 
972
1045
  class TriggerStage(BaseStage):
973
- """Trigger Workflow execution stage that execute another workflow. This
974
- the core stage that allow you to create the reusable workflow object or
975
- dynamic parameters workflow for common usecase.
1046
+ """Trigger workflow executor stage that run an input trigger Workflow
1047
+ execute method. This is the stage that allow you to create the reusable
1048
+ Workflow template with dynamic parameters.
976
1049
 
977
1050
  Data Validate:
978
1051
  >>> stage = {
@@ -984,12 +1057,13 @@ class TriggerStage(BaseStage):
984
1057
 
985
1058
  trigger: str = Field(
986
1059
  description=(
987
- "A trigger workflow name that should exist on the config path."
1060
+ "A trigger workflow name. This workflow name should exist on the "
1061
+ "config path because it will load by the `load_conf` method."
988
1062
  ),
989
1063
  )
990
1064
  params: DictData = Field(
991
1065
  default_factory=dict,
992
- description="A parameter that want to pass to workflow execution.",
1066
+ description="A parameter that will pass to workflow execution method.",
993
1067
  )
994
1068
 
995
1069
  def execute(
@@ -1002,7 +1076,7 @@ class TriggerStage(BaseStage):
1002
1076
  """Trigger another workflow execution. It will wait the trigger
1003
1077
  workflow running complete before catching its result.
1004
1078
 
1005
- :param params: A parameter data that want to use in this execution.
1079
+ :param params: (DictData) A parameter data.
1006
1080
  :param result: (Result) A result object for keeping context and status
1007
1081
  data.
1008
1082
  :param event: (Event) An event manager that use to track parent execute
@@ -1037,14 +1111,18 @@ class TriggerStage(BaseStage):
1037
1111
  err_msg: str | None = (
1038
1112
  f" with:\n{msg}"
1039
1113
  if (msg := rs.context.get("errors", {}).get("message"))
1040
- else ""
1114
+ else "."
1115
+ )
1116
+ raise StageException(
1117
+ f"Trigger workflow return failed status{err_msg}"
1041
1118
  )
1042
- raise StageException(f"Trigger workflow was failed{err_msg}.")
1043
1119
  return rs
1044
1120
 
1045
1121
 
1046
1122
  class ParallelStage(BaseStage): # pragma: no cov
1047
- """Parallel execution stage that execute child stages with parallel.
1123
+ """Parallel stage executor that execute branch stages with multithreading.
1124
+ This stage let you set the fix branches for running substage inside it on
1125
+ multithread pool.
1048
1126
 
1049
1127
  This stage is not the low-level stage model because it runs muti-stages
1050
1128
  in this stage execution.
@@ -1072,48 +1150,46 @@ class ParallelStage(BaseStage): # pragma: no cov
1072
1150
  """
1073
1151
 
1074
1152
  parallel: dict[str, list[Stage]] = Field(
1075
- description="A mapping of parallel branch name and stages.",
1153
+ description="A mapping of branch name and its stages.",
1076
1154
  )
1077
1155
  max_workers: int = Field(
1078
1156
  default=2,
1079
1157
  ge=1,
1080
1158
  lt=20,
1081
1159
  description=(
1082
- "The maximum thread pool worker size for execution parallel."
1160
+ "The maximum multi-thread pool worker size for execution parallel. "
1161
+ "This value should be gather or equal than 1, and less than 20."
1083
1162
  ),
1084
1163
  alias="max-workers",
1085
1164
  )
1086
1165
 
1087
- def execute_task(
1166
+ def execute_branch(
1088
1167
  self,
1089
1168
  branch: str,
1090
1169
  params: DictData,
1091
1170
  result: Result,
1092
1171
  *,
1093
1172
  event: Event | None = None,
1094
- extras: DictData | None = None,
1095
1173
  ) -> DictData:
1096
- """Task execution method for passing a branch to each thread.
1174
+ """Branch execution method for execute all stages of a specific branch
1175
+ ID.
1097
1176
 
1098
- :param branch: A branch ID.
1099
- :param params: A parameter data that want to use in this execution.
1100
- :param result: (Result) A result object for keeping context and status
1101
- data.
1102
- :param event: (Event) An event manager that use to track parent execute
1103
- was not force stopped.
1104
- :param extras: (DictData) An extra parameters that want to override
1105
- config values.
1177
+ :param branch: (str) A branch ID.
1178
+ :param params: (DictData) A parameter data.
1179
+ :param result: (Result) A Result instance for return context and status.
1180
+ :param event: (Event) An Event manager instance that use to cancel this
1181
+ execution if it forces stopped by parent execution.
1106
1182
 
1107
1183
  :rtype: DictData
1108
1184
  """
1109
- result.trace.debug(f"... Execute branch: {branch!r}")
1185
+ result.trace.debug(f"[STAGE]: Execute Branch: {branch!r}")
1110
1186
  context: DictData = copy.deepcopy(params)
1111
1187
  context.update({"branch": branch})
1112
1188
  output: DictData = {"branch": branch, "stages": {}}
1113
1189
  for stage in self.parallel[branch]:
1114
1190
 
1115
- if extras:
1116
- stage.extras = extras
1191
+ if self.extras:
1192
+ stage.extras = self.extras
1117
1193
 
1118
1194
  if stage.is_skipped(params=context):
1119
1195
  result.trace.info(f"... Skip stage: {stage.iden!r}")
@@ -1154,7 +1230,7 @@ class ParallelStage(BaseStage): # pragma: no cov
1154
1230
 
1155
1231
  if rs.status == FAILED:
1156
1232
  error_msg: str = (
1157
- f"Item-Stage was break because it has a sub stage, "
1233
+ f"Branch-Stage was break because it has a sub stage, "
1158
1234
  f"{stage.iden}, failed without raise error."
1159
1235
  )
1160
1236
  return result.catch(
@@ -1185,14 +1261,12 @@ class ParallelStage(BaseStage): # pragma: no cov
1185
1261
  result: Result | None = None,
1186
1262
  event: Event | None = None,
1187
1263
  ) -> Result:
1188
- """Execute the stages that parallel each branch via multi-threading mode
1189
- or async mode by changing `async_mode` flag.
1264
+ """Execute parallel each branch via multi-threading pool.
1190
1265
 
1191
- :param params: A parameter that want to pass before run any statement.
1192
- :param result: (Result) A result object for keeping context and status
1193
- data.
1194
- :param event: (Event) An event manager that use to track parent execute
1195
- was not force stopped.
1266
+ :param params: (DictData) A parameter data.
1267
+ :param result: (Result) A Result instance for return context and status.
1268
+ :param event: (Event) An Event manager instance that use to cancel this
1269
+ execution if it forces stopped by parent execution.
1196
1270
 
1197
1271
  :rtype: Result
1198
1272
  """
@@ -1216,12 +1290,11 @@ class ParallelStage(BaseStage): # pragma: no cov
1216
1290
 
1217
1291
  futures: list[Future] = (
1218
1292
  executor.submit(
1219
- self.execute_task,
1293
+ self.execute_branch,
1220
1294
  branch=branch,
1221
1295
  params=params,
1222
1296
  result=result,
1223
1297
  event=event,
1224
- extras=self.extras,
1225
1298
  )
1226
1299
  for branch in self.parallel
1227
1300
  )
@@ -1241,11 +1314,11 @@ class ParallelStage(BaseStage): # pragma: no cov
1241
1314
 
1242
1315
 
1243
1316
  class ForEachStage(BaseStage):
1244
- """For-Each execution stage that execute child stages with an item in list
1245
- of item values. This stage is not the low-level stage model because it runs
1246
- muti-stages in this stage execution.
1317
+ """For-Each stage executor that execute all stages with each item in the
1318
+ foreach list.
1247
1319
 
1248
- The concept of this stage use the same logic of the Job execution.
1320
+ This stage is not the low-level stage model because it runs
1321
+ muti-stages in this stage execution.
1249
1322
 
1250
1323
  Data Validate:
1251
1324
  >>> stage = {
@@ -1254,7 +1327,7 @@ class ForEachStage(BaseStage):
1254
1327
  ... "stages": [
1255
1328
  ... {
1256
1329
  ... "name": "Echo stage",
1257
- ... "echo": "Start run with item {{ item }}"
1330
+ ... "echo": "Start run with item ${{ item }}"
1258
1331
  ... },
1259
1332
  ... ],
1260
1333
  ... }
@@ -1262,13 +1335,14 @@ class ForEachStage(BaseStage):
1262
1335
 
1263
1336
  foreach: Union[list[str], list[int], str] = Field(
1264
1337
  description=(
1265
- "A items for passing to each stages via ${{ item }} template."
1338
+ "A items for passing to stages via ${{ item }} template parameter."
1266
1339
  ),
1267
1340
  )
1268
1341
  stages: list[Stage] = Field(
1269
1342
  default_factory=list,
1270
1343
  description=(
1271
- "A list of stage that will run with each item in the foreach field."
1344
+ "A list of stage that will run with each item in the `foreach` "
1345
+ "field."
1272
1346
  ),
1273
1347
  )
1274
1348
  concurrent: int = Field(
@@ -1283,7 +1357,7 @@ class ForEachStage(BaseStage):
1283
1357
 
1284
1358
  def execute_item(
1285
1359
  self,
1286
- item: Union[str, int],
1360
+ item: StrOrInt,
1287
1361
  params: DictData,
1288
1362
  result: Result,
1289
1363
  *,
@@ -1292,18 +1366,16 @@ class ForEachStage(BaseStage):
1292
1366
  """Execute foreach item from list of item.
1293
1367
 
1294
1368
  :param item: (str | int) An item that want to execution.
1295
- :param params: (DictData) A parameter that want to pass to stage
1296
- execution.
1297
- :param result: (Result) A result object for keeping context and status
1298
- data.
1299
- :param event: (Event) An event manager that use to track parent execute
1300
- was not force stopped.
1369
+ :param params: (DictData) A parameter data.
1370
+ :param result: (Result) A Result instance for return context and status.
1371
+ :param event: (Event) An Event manager instance that use to cancel this
1372
+ execution if it forces stopped by parent execution.
1301
1373
 
1302
1374
  :raise StageException: If the stage execution raise errors.
1303
1375
 
1304
1376
  :rtype: Result
1305
1377
  """
1306
- result.trace.debug(f"[STAGE]: Execute item: {item!r}")
1378
+ result.trace.debug(f"[STAGE]: Execute Item: {item!r}")
1307
1379
  context: DictData = copy.deepcopy(params)
1308
1380
  context.update({"item": item})
1309
1381
  output: DictData = {"item": item, "stages": {}}
@@ -1345,8 +1417,18 @@ class ForEachStage(BaseStage):
1345
1417
  stage.set_outputs(stage.get_outputs(output), to=context)
1346
1418
  except (StageException, UtilException) as e:
1347
1419
  result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1420
+ result.catch(
1421
+ status=FAILED,
1422
+ foreach={
1423
+ item: {
1424
+ "item": item,
1425
+ "stages": filter_func(output.pop("stages", {})),
1426
+ "errors": e.to_dict(),
1427
+ },
1428
+ },
1429
+ )
1348
1430
  raise StageException(
1349
- f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
1431
+ f"Sub-Stage raise: {e.__class__.__name__}: {e}"
1350
1432
  ) from None
1351
1433
 
1352
1434
  if rs.status == FAILED:
@@ -1383,11 +1465,10 @@ class ForEachStage(BaseStage):
1383
1465
  ) -> Result:
1384
1466
  """Execute the stages that pass each item form the foreach field.
1385
1467
 
1386
- :param params: A parameter that want to pass before run any statement.
1387
- :param result: (Result) A result object for keeping context and status
1388
- data.
1389
- :param event: (Event) An event manager that use to track parent execute
1390
- was not force stopped.
1468
+ :param params: (DictData) A parameter data.
1469
+ :param result: (Result) A Result instance for return context and status.
1470
+ :param event: (Event) An Event manager instance that use to cancel this
1471
+ execution if it forces stopped by parent execution.
1391
1472
 
1392
1473
  :rtype: Result
1393
1474
  """
@@ -1403,9 +1484,7 @@ class ForEachStage(BaseStage):
1403
1484
  else self.foreach
1404
1485
  )
1405
1486
  if not isinstance(foreach, list):
1406
- raise StageException(
1407
- f"Foreach does not support foreach value: {foreach!r}"
1408
- )
1487
+ raise StageException(f"Does not support foreach: {foreach!r}")
1409
1488
 
1410
1489
  result.trace.info(f"[STAGE]: Foreach-Execute: {foreach!r}.")
1411
1490
  result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
@@ -1437,33 +1516,33 @@ class ForEachStage(BaseStage):
1437
1516
  context: DictData = {}
1438
1517
  status: Status = SUCCESS
1439
1518
 
1440
- done, not_done = wait(
1441
- futures, timeout=1800, return_when=FIRST_EXCEPTION
1442
- )
1443
-
1519
+ done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
1444
1520
  if len(done) != len(futures):
1445
1521
  result.trace.warning(
1446
- "[STAGE]: Set the event for stop running stage."
1522
+ "[STAGE]: Set event for stop pending stage future."
1447
1523
  )
1448
1524
  event.set()
1449
1525
  for future in not_done:
1450
1526
  future.cancel()
1451
1527
 
1528
+ nd: str = f", item not run: {not_done}" if not_done else ""
1529
+ result.trace.debug(f"... Foreach set Fail-Fast{nd}")
1530
+
1452
1531
  for future in done:
1453
1532
  try:
1454
1533
  future.result()
1455
- except StageException as e:
1534
+ except (StageException, UtilException) as e:
1456
1535
  status = FAILED
1457
1536
  result.trace.error(
1458
- f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
1537
+ f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
1459
1538
  )
1460
1539
  context.update({"errors": e.to_dict()})
1461
-
1462
1540
  return result.catch(status=status, context=context)
1463
1541
 
1464
1542
 
1465
- class UntilStage(BaseStage): # pragma: no cov
1466
- """Until execution stage.
1543
+ class UntilStage(BaseStage):
1544
+ """Until stage executor that will run stages in each loop until it valid
1545
+ with stop loop condition.
1467
1546
 
1468
1547
  Data Validate:
1469
1548
  >>> stage = {
@@ -1485,19 +1564,21 @@ class UntilStage(BaseStage): # pragma: no cov
1485
1564
  "An initial value that can be any value in str, int, or bool type."
1486
1565
  ),
1487
1566
  )
1488
- until: str = Field(description="A until condition.")
1567
+ until: str = Field(description="A until condition for stop the while loop.")
1489
1568
  stages: list[Stage] = Field(
1490
1569
  default_factory=list,
1491
1570
  description=(
1492
- "A list of stage that will run with each item until condition "
1493
- "correct."
1571
+ "A list of stage that will run with each item in until loop."
1494
1572
  ),
1495
1573
  )
1496
1574
  max_loop: int = Field(
1497
1575
  default=10,
1498
1576
  ge=1,
1499
1577
  lt=100,
1500
- description="The maximum value of loop for this until stage.",
1578
+ description=(
1579
+ "The maximum value of loop for this until stage. This value should "
1580
+ "be gather or equal than 1, and less than 100."
1581
+ ),
1501
1582
  alias="max-loop",
1502
1583
  )
1503
1584
 
@@ -1509,17 +1590,15 @@ class UntilStage(BaseStage): # pragma: no cov
1509
1590
  result: Result,
1510
1591
  event: Event | None = None,
1511
1592
  ) -> tuple[Result, T]:
1512
- """Execute until item set item by some stage or by default loop
1593
+ """Execute loop item that was set from some stage or set by default loop
1513
1594
  variable.
1514
1595
 
1515
1596
  :param item: (T) An item that want to execution.
1516
1597
  :param loop: (int) A number of loop.
1517
- :param params: (DictData) A parameter that want to pass to stage
1518
- execution.
1519
- :param result: (Result) A result object for keeping context and status
1520
- data.
1521
- :param event: (Event) An event manager that use to track parent execute
1522
- was not force stopped.
1598
+ :param params: (DictData) A parameter data.
1599
+ :param result: (Result) A Result instance for return context and status.
1600
+ :param event: (Event) An Event manager instance that use to cancel this
1601
+ execution if it forces stopped by parent execution.
1523
1602
 
1524
1603
  :rtype: tuple[Result, T]
1525
1604
  """
@@ -1602,11 +1681,10 @@ class UntilStage(BaseStage): # pragma: no cov
1602
1681
  """Execute the stages that pass item from until condition field and
1603
1682
  setter step.
1604
1683
 
1605
- :param params: A parameter that want to pass before run any statement.
1606
- :param result: (Result) A result object for keeping context and status
1607
- data.
1608
- :param event: (Event) An event manager that use to track parent execute
1609
- was not force stopped.
1684
+ :param params: (DictData) A parameter data.
1685
+ :param result: (Result) A Result instance for return context and status.
1686
+ :param event: (Event) An Event manager instance that use to cancel this
1687
+ execution if it forces stopped by parent execution.
1610
1688
 
1611
1689
  :rtype: Result
1612
1690
  """
@@ -1679,14 +1757,14 @@ class UntilStage(BaseStage): # pragma: no cov
1679
1757
  class Match(BaseModel):
1680
1758
  """Match model for the Case Stage."""
1681
1759
 
1682
- case: Union[str, int] = Field(description="A match case.")
1760
+ case: StrOrInt = Field(description="A match case.")
1683
1761
  stages: list[Stage] = Field(
1684
1762
  description="A list of stage to execution for this case."
1685
1763
  )
1686
1764
 
1687
1765
 
1688
1766
  class CaseStage(BaseStage):
1689
- """Case execution stage.
1767
+ """Case stage executor that execute all stages if the condition was matched.
1690
1768
 
1691
1769
  Data Validate:
1692
1770
  >>> stage = {
@@ -1742,12 +1820,10 @@ class CaseStage(BaseStage):
1742
1820
 
1743
1821
  :param case: (str) A case that want to execution.
1744
1822
  :param stages: (list[Stage]) A list of stage.
1745
- :param params: (DictData) A parameter that want to pass to stage
1746
- execution.
1747
- :param result: (Result) A result object for keeping context and status
1748
- data.
1749
- :param event: (Event) An event manager that use to track parent execute
1750
- was not force stopped.
1823
+ :param params: (DictData) A parameter data.
1824
+ :param result: (Result) A Result instance for return context and status.
1825
+ :param event: (Event) An Event manager instance that use to cancel this
1826
+ execution if it forces stopped by parent execution.
1751
1827
 
1752
1828
  :rtype: Result
1753
1829
  """
@@ -1830,11 +1906,10 @@ class CaseStage(BaseStage):
1830
1906
  ) -> Result:
1831
1907
  """Execute case-match condition that pass to the case field.
1832
1908
 
1833
- :param params: A parameter that want to pass before run any statement.
1834
- :param result: (Result) A result object for keeping context and status
1835
- data.
1836
- :param event: (Event) An event manager that use to track parent execute
1837
- was not force stopped.
1909
+ :param params: (DictData) A parameter data.
1910
+ :param result: (Result) A Result instance for return context and status.
1911
+ :param event: (Event) An Event manager instance that use to cancel this
1912
+ execution if it forces stopped by parent execution.
1838
1913
 
1839
1914
  :rtype: Result
1840
1915
  """
@@ -1911,7 +1986,7 @@ class RaiseStage(BaseStage): # pragma: no cov
1911
1986
 
1912
1987
  message: str = Field(
1913
1988
  description=(
1914
- "An error message that want to raise with StageException class"
1989
+ "An error message that want to raise with `StageException` class"
1915
1990
  ),
1916
1991
  alias="raise",
1917
1992
  )
@@ -1925,11 +2000,10 @@ class RaiseStage(BaseStage): # pragma: no cov
1925
2000
  ) -> Result:
1926
2001
  """Raise the StageException object with the message field execution.
1927
2002
 
1928
- :param params: A parameter that want to pass before run any statement.
1929
- :param result: (Result) A result object for keeping context and status
1930
- data.
1931
- :param event: (Event) An event manager that use to track parent execute
1932
- was not force stopped.
2003
+ :param params: (DictData) A parameter data.
2004
+ :param result: (Result) A Result instance for return context and status.
2005
+ :param event: (Event) An Event manager instance that use to cancel this
2006
+ execution if it forces stopped by parent execution.
1933
2007
  """
1934
2008
  if result is None: # pragma: no cov
1935
2009
  result: Result = Result(
@@ -1941,27 +2015,13 @@ class RaiseStage(BaseStage): # pragma: no cov
1941
2015
  raise StageException(message)
1942
2016
 
1943
2017
 
1944
- # TODO: Not implement this stages yet
1945
- class HookStage(BaseStage): # pragma: no cov
1946
- """Hook stage execution."""
1947
-
1948
- hook: str
1949
- args: DictData = Field(default_factory=dict)
1950
- callback: str
1951
-
1952
- def execute(
1953
- self,
1954
- params: DictData,
1955
- *,
1956
- result: Result | None = None,
1957
- event: Event | None = None,
1958
- ) -> Result:
1959
- raise NotImplementedError("Hook Stage does not implement yet.")
1960
-
1961
-
1962
- # TODO: Not implement this stages yet
1963
2018
  class DockerStage(BaseStage): # pragma: no cov
1964
- """Docker container stage execution.
2019
+ """Docker container stage execution that will pull the specific Docker image
2020
+ with custom authentication and run this image by passing environment
2021
+ variables and mounting local volume to this Docker container.
2022
+
2023
+ The volume path that mount to this Docker container will limit. That is
2024
+ this stage does not allow you to mount any path to this container.
1965
2025
 
1966
2026
  Data Validate:
1967
2027
  >>> stage = {
@@ -1969,10 +2029,7 @@ class DockerStage(BaseStage): # pragma: no cov
1969
2029
  ... "image": "image-name.pkg.com",
1970
2030
  ... "env": {
1971
2031
  ... "ENV": "dev",
1972
- ... "DEBUG": "true",
1973
- ... },
1974
- ... "volume": {
1975
- ... "secrets": "/secrets",
2032
+ ... "SECRET": "${SPECIFIC_SECRET}",
1976
2033
  ... },
1977
2034
  ... "auth": {
1978
2035
  ... "username": "__json_key",
@@ -1985,8 +2042,16 @@ class DockerStage(BaseStage): # pragma: no cov
1985
2042
  description="A Docker image url with tag that want to run.",
1986
2043
  )
1987
2044
  tag: str = Field(default="latest", description="An Docker image tag.")
1988
- env: DictData = Field(default_factory=dict)
1989
- volume: DictData = Field(default_factory=dict)
2045
+ env: DictData = Field(
2046
+ default_factory=dict,
2047
+ description=(
2048
+ "An environment variable that want pass to Docker container.",
2049
+ ),
2050
+ )
2051
+ volume: DictData = Field(
2052
+ default_factory=dict,
2053
+ description="A mapping of local and target mounting path.",
2054
+ )
1990
2055
  auth: DictData = Field(
1991
2056
  default_factory=dict,
1992
2057
  description=(
@@ -1998,7 +2063,17 @@ class DockerStage(BaseStage): # pragma: no cov
1998
2063
  self,
1999
2064
  params: DictData,
2000
2065
  result: Result,
2001
- ):
2066
+ event: Event | None = None,
2067
+ ) -> Result:
2068
+ """Execute Docker container task.
2069
+
2070
+ :param params: (DictData) A parameter data.
2071
+ :param result: (Result) A Result instance for return context and status.
2072
+ :param event: (Event) An Event manager instance that use to cancel this
2073
+ execution if it forces stopped by parent execution.
2074
+
2075
+ :rtype: Result
2076
+ """
2002
2077
  from docker import DockerClient
2003
2078
  from docker.errors import ContainerError
2004
2079
 
@@ -2016,6 +2091,16 @@ class DockerStage(BaseStage): # pragma: no cov
2016
2091
  for line in resp:
2017
2092
  result.trace.info(f"... {line}")
2018
2093
 
2094
+ if event and event.is_set():
2095
+ error_msg: str = (
2096
+ "Docker-Stage was canceled from event that had set before "
2097
+ "run the Docker container."
2098
+ )
2099
+ return result.catch(
2100
+ status=CANCEL,
2101
+ context={"errors": StageException(error_msg).to_dict()},
2102
+ )
2103
+
2019
2104
  unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
2020
2105
  container = client.containers.run(
2021
2106
  image=f"{self.image}:{self.tag}",
@@ -2054,6 +2139,13 @@ class DockerStage(BaseStage): # pragma: no cov
2054
2139
  f"{self.image}:{self.tag}",
2055
2140
  out,
2056
2141
  )
2142
+ output_file: Path = Path(f".docker.{result.run_id}.logs/outputs.json")
2143
+ if not output_file.exists():
2144
+ return result.catch(status=SUCCESS)
2145
+
2146
+ with output_file.open(mode="rt") as f:
2147
+ data = json.load(f)
2148
+ return result.catch(status=SUCCESS, context=data)
2057
2149
 
2058
2150
  def execute(
2059
2151
  self,
@@ -2062,13 +2154,34 @@ class DockerStage(BaseStage): # pragma: no cov
2062
2154
  result: Result | None = None,
2063
2155
  event: Event | None = None,
2064
2156
  ) -> Result:
2157
+ """Execute the Docker image via Python API.
2158
+
2159
+ :param params: (DictData) A parameter data.
2160
+ :param result: (Result) A Result instance for return context and status.
2161
+ :param event: (Event) An Event manager instance that use to cancel this
2162
+ execution if it forces stopped by parent execution.
2163
+
2164
+ :rtype: Result
2165
+ """
2166
+ if result is None: # pragma: no cov
2167
+ result: Result = Result(
2168
+ run_id=gen_id(self.name + (self.id or ""), unique=True)
2169
+ )
2170
+
2171
+ result.trace.info(f"[STAGE]: Docker-Execute: {self.image}:{self.tag}")
2172
+
2065
2173
  raise NotImplementedError("Docker Stage does not implement yet.")
2066
2174
 
2067
2175
 
2068
- # TODO: Not implement this stages yet
2069
2176
  class VirtualPyStage(PyStage): # pragma: no cov
2070
- """Python Virtual Environment stage execution."""
2177
+ """Virtual Python stage executor that run Python statement on the dependent
2178
+ Python virtual environment via the `uv` package.
2179
+ """
2071
2180
 
2181
+ version: str = Field(
2182
+ default="3.9",
2183
+ description="A Python version that want to run.",
2184
+ )
2072
2185
  deps: list[str] = Field(
2073
2186
  description=(
2074
2187
  "list of Python dependency that want to install before execution "
@@ -2076,7 +2189,54 @@ class VirtualPyStage(PyStage): # pragma: no cov
2076
2189
  ),
2077
2190
  )
2078
2191
 
2079
- def create_py_file(self, py: str, run_id: str | None): ...
2192
+ @contextlib.contextmanager
2193
+ def create_py_file(
2194
+ self,
2195
+ py: str,
2196
+ values: DictData,
2197
+ deps: list[str],
2198
+ run_id: str | None = None,
2199
+ ) -> Iterator[str]:
2200
+ """Create the .py file with an input Python string statement.
2201
+
2202
+ :param py: A Python string statement.
2203
+ :param values: A variable that want to set before running this
2204
+ :param deps: An additional Python dependencies that want install before
2205
+ run this python stage.
2206
+ :param run_id: (str | None) A running ID of this stage execution.
2207
+ """
2208
+ run_id: str = run_id or uuid.uuid4()
2209
+ f_name: str = f"{run_id}.py"
2210
+ with open(f"./{f_name}", mode="w", newline="\n") as f:
2211
+ # NOTE: Create variable mapping that write before running statement.
2212
+ vars_str: str = "\n ".join(
2213
+ f"{var} = {value!r}" for var, value in values.items()
2214
+ )
2215
+
2216
+ # NOTE: uv supports PEP 723 — inline TOML metadata.
2217
+ f.write(
2218
+ dedent(
2219
+ f"""
2220
+ # /// script
2221
+ # dependencies = [{', '.join(f'"{dep}"' for dep in deps)}]
2222
+ # ///
2223
+ {vars_str}
2224
+ """.strip(
2225
+ "\n"
2226
+ )
2227
+ )
2228
+ )
2229
+
2230
+ # NOTE: make sure that py script file does not have `\r` char.
2231
+ f.write("\n" + py.replace("\r\n", "\n"))
2232
+
2233
+ # NOTE: Make this .py file able to executable.
2234
+ make_exec(f"./{f_name}")
2235
+
2236
+ yield f_name
2237
+
2238
+ # Note: Remove .py file that use to run Python.
2239
+ Path(f"./{f_name}").unlink()
2080
2240
 
2081
2241
  def execute(
2082
2242
  self,
@@ -2088,15 +2248,13 @@ class VirtualPyStage(PyStage): # pragma: no cov
2088
2248
  """Execute the Python statement via Python virtual environment.
2089
2249
 
2090
2250
  Steps:
2091
- - Create python file.
2092
- - Create `.venv` and install necessary Python deps.
2093
- - Execution python file with uv and specific `.venv`.
2251
+ - Create python file with the `uv` syntax.
2252
+ - Execution python file with `uv run` via Python subprocess module.
2094
2253
 
2095
- :param params: A parameter that want to pass before run any statement.
2096
- :param result: (Result) A result object for keeping context and status
2097
- data.
2098
- :param event: (Event) An event manager that use to track parent execute
2099
- was not force stopped.
2254
+ :param params: (DictData) A parameter data.
2255
+ :param result: (Result) A Result instance for return context and status.
2256
+ :param event: (Event) An Event manager instance that use to cancel this
2257
+ execution if it forces stopped by parent execution.
2100
2258
 
2101
2259
  :rtype: Result
2102
2260
  """
@@ -2106,13 +2264,45 @@ class VirtualPyStage(PyStage): # pragma: no cov
2106
2264
  )
2107
2265
 
2108
2266
  result.trace.info(f"[STAGE]: Py-Virtual-Execute: {self.name}")
2109
- raise NotImplementedError(
2110
- "Python Virtual Stage does not implement yet."
2267
+ run: str = param2template(dedent(self.run), params, extras=self.extras)
2268
+ with self.create_py_file(
2269
+ py=run,
2270
+ values=param2template(self.vars, params, extras=self.extras),
2271
+ deps=param2template(self.deps, params, extras=self.extras),
2272
+ run_id=result.run_id,
2273
+ ) as py:
2274
+ result.trace.debug(f"... Create `{py}` file.")
2275
+ rs: CompletedProcess = subprocess.run(
2276
+ ["uv", "run", py, "--no-cache"],
2277
+ # ["uv", "run", "--python", "3.9", py],
2278
+ shell=False,
2279
+ capture_output=True,
2280
+ text=True,
2281
+ )
2282
+
2283
+ if rs.returncode > 0:
2284
+ # NOTE: Prepare stderr message that returning from subprocess.
2285
+ e: str = (
2286
+ rs.stderr.encode("utf-8").decode("utf-16")
2287
+ if "\\x00" in rs.stderr
2288
+ else rs.stderr
2289
+ ).removesuffix("\n")
2290
+ raise StageException(
2291
+ f"Subprocess: {e}\nRunning Statement:\n---\n"
2292
+ f"```python\n{run}\n```"
2293
+ )
2294
+ return result.catch(
2295
+ status=SUCCESS,
2296
+ context={
2297
+ "return_code": rs.returncode,
2298
+ "stdout": None if (out := rs.stdout.strip("\n")) == "" else out,
2299
+ "stderr": None if (out := rs.stderr.strip("\n")) == "" else out,
2300
+ },
2111
2301
  )
2112
2302
 
2113
2303
 
2114
2304
  # NOTE:
2115
- # An order of parsing stage model on the Job model with ``stages`` field.
2305
+ # An order of parsing stage model on the Job model with `stages` field.
2116
2306
  # From the current build-in stages, they do not have stage that have the same
2117
2307
  # fields that because of parsing on the Job's stages key.
2118
2308
  #
@@ -2121,7 +2311,6 @@ Stage = Annotated[
2121
2311
  DockerStage,
2122
2312
  BashStage,
2123
2313
  CallStage,
2124
- HookStage,
2125
2314
  TriggerStage,
2126
2315
  ForEachStage,
2127
2316
  UntilStage,