ddeutil-workflow 0.0.53__py3-none-any.whl → 0.0.55__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,23 +4,27 @@
4
4
  # license information.
5
5
  # ------------------------------------------------------------------------------
6
6
  # [x] Use dynamic config
7
- """Stage model. It stores all stage model that use for getting stage data template
8
- from the Job Model. The stage handle the minimize task that run in some thread
9
- (same thread at its job owner) that mean it is the lowest executor of a workflow
10
- that can tracking logs.
7
+ """Stages module include all stage model that use be the minimum execution layer
8
+ of this workflow engine. The stage handle the minimize task that run in some
9
+ thread (same thread at its job owner) that mean it is the lowest executor that
10
+ you can track logs.
11
11
 
12
- The output of stage execution only return 0 status because I do not want to
13
- handle stage error on this stage model. I think stage model should have a lot of
14
- use-case, and it does not worry when I want to create a new one.
12
+ The output of stage execution only return SUCCESS or CANCEL status because
13
+ I do not want to handle stage error on this stage execution. I think stage model
14
+ have a lot of use-case, and it should does not worry about it error output.
15
15
 
16
- Execution --> Ok --> Result with SUCCESS
16
+ So, I will create `handler_execute` for any exception class that raise from
17
+ the stage execution method.
17
18
 
18
- --> Error ┬-> Result with FAILED (if env var was set)
19
- ╰-> Raise StageException(...)
19
+ Execution --> Ok ---( handler )--> Result with `SUCCESS` or `CANCEL`
20
+
21
+ --> Error ┬--( handler )-> Result with `FAILED` (Set `raise_error` flag)
22
+ |
23
+ ╰--( handler )-> Raise StageException(...)
20
24
 
21
25
  On the context I/O that pass to a stage object at execute process. The
22
- execute method receives a `params={"params": {...}}` value for mapping to
23
- template searching.
26
+ execute method receives a `params={"params": {...}}` value for passing template
27
+ searching.
24
28
  """
25
29
  from __future__ import annotations
26
30
 
@@ -28,12 +32,13 @@ import asyncio
28
32
  import contextlib
29
33
  import copy
30
34
  import inspect
35
+ import json
31
36
  import subprocess
32
37
  import sys
33
38
  import time
34
39
  import uuid
35
40
  from abc import ABC, abstractmethod
36
- from collections.abc import Iterator
41
+ from collections.abc import AsyncIterator, Iterator
37
42
  from concurrent.futures import (
38
43
  FIRST_EXCEPTION,
39
44
  Future,
@@ -59,6 +64,7 @@ from .exceptions import StageException, UtilException, to_dict
59
64
  from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
60
65
  from .reusables import TagFunc, extract_call, not_in_template, param2template
61
66
  from .utils import (
67
+ NEWLINE,
62
68
  delay,
63
69
  filter_func,
64
70
  gen_id,
@@ -66,20 +72,22 @@ from .utils import (
66
72
  )
67
73
 
68
74
  T = TypeVar("T")
75
+ StrOrInt = Union[str, int]
69
76
 
70
77
 
71
78
  class BaseStage(BaseModel, ABC):
72
- """Base Stage Model that keep only id and name fields for the stage
73
- metadata. If you want to implement any custom stage, you can use this class
74
- to parent and implement ``self.execute()`` method only.
79
+ """Base Stage Model that keep only necessary fields like `id`, `name` or
80
+ `condition` for the stage metadata. If you want to implement any custom
81
+ stage, you can inherit this class and implement `self.execute()` method
82
+ only.
75
83
 
76
- This class is the abstraction class for any stage model that want to
77
- implement to workflow model.
84
+ This class is the abstraction class for any inherit stage model that
85
+ want to implement on this workflow package.
78
86
  """
79
87
 
80
88
  extras: DictData = Field(
81
89
  default_factory=dict,
82
- description="An extra override config values.",
90
+ description="An extra parameter that override core config values.",
83
91
  )
84
92
  id: Optional[str] = Field(
85
93
  default=None,
@@ -93,14 +101,17 @@ class BaseStage(BaseModel, ABC):
93
101
  )
94
102
  condition: Optional[str] = Field(
95
103
  default=None,
96
- description="A stage condition statement to allow stage executable.",
104
+ description=(
105
+ "A stage condition statement to allow stage executable. This field "
106
+ "alise with `if` field."
107
+ ),
97
108
  alias="if",
98
109
  )
99
110
 
100
111
  @property
101
112
  def iden(self) -> str:
102
- """Return identity of this stage object that return the id field first.
103
- If the id does not set, it will use name field instead.
113
+ """Return this stage identity that return the `id` field first and if
114
+ this `id` field does not set, it will use the `name` field instead.
104
115
 
105
116
  :rtype: str
106
117
  """
@@ -156,8 +167,8 @@ class BaseStage(BaseModel, ABC):
156
167
  run_id: str | None = None,
157
168
  parent_run_id: str | None = None,
158
169
  result: Result | None = None,
159
- raise_error: bool | None = None,
160
170
  event: Event | None = None,
171
+ raise_error: bool | None = None,
161
172
  ) -> Result | DictData:
162
173
  """Handler stage execution result from the stage `execute` method.
163
174
 
@@ -170,7 +181,7 @@ class BaseStage(BaseModel, ABC):
170
181
  ╰-context:
171
182
  ╰-outputs: ...
172
183
 
173
- --> Error --> Result (if env var was set)
184
+ --> Error --> Result (if `raise_error` was set)
174
185
  |-status: FAILED
175
186
  ╰-errors:
176
187
  |-class: ...
@@ -179,18 +190,16 @@ class BaseStage(BaseModel, ABC):
179
190
 
180
191
  --> Error --> Raise StageException(...)
181
192
 
182
- On the last step, it will set the running ID on a return result object
183
- from current stage ID before release the final result.
193
+ On the last step, it will set the running ID on a return result
194
+ object from the current stage ID before release the final result.
184
195
 
185
- :param params: (DictData) A parameterize value data that use in this
186
- stage execution.
187
- :param run_id: (str) A running stage ID for this execution.
188
- :param parent_run_id: (str) A parent workflow running ID for this
189
- execution.
196
+ :param params: (DictData) A parameter data.
197
+ :param run_id: (str) A running stage ID.
198
+ :param parent_run_id: (str) A parent running ID.
190
199
  :param result: (Result) A result object for keeping context and status
191
200
  data before execution.
192
- :param raise_error: (bool) A flag that all this method raise error
193
201
  :param event: (Event) An event manager that pass to the stage execution.
202
+ :param raise_error: (bool) A flag that all this method raise error
194
203
 
195
204
  :rtype: Result
196
205
  """
@@ -203,51 +212,56 @@ class BaseStage(BaseModel, ABC):
203
212
  )
204
213
 
205
214
  try:
206
- rs: Result = self.execute(params, result=result, event=event)
207
- return rs
215
+ return self.execute(params, result=result, event=event)
208
216
  except Exception as e:
209
- result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
210
-
217
+ e_name: str = e.__class__.__name__
218
+ result.trace.error(f"[STAGE]: Handler:{NEWLINE}{e_name}: {e}")
211
219
  if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
212
220
  if isinstance(e, StageException):
213
221
  raise
214
222
 
215
223
  raise StageException(
216
- f"{self.__class__.__name__}: \n\t"
217
- f"{e.__class__.__name__}: {e}"
224
+ f"{self.__class__.__name__}: {NEWLINE}{e_name}: {e}"
218
225
  ) from e
219
226
 
220
- errors: DictData = {"errors": to_dict(e)}
221
- return result.catch(status=FAILED, context=errors)
227
+ return result.catch(status=FAILED, context={"errors": to_dict(e)})
222
228
 
223
229
  def set_outputs(self, output: DictData, to: DictData) -> DictData:
224
- """Set an outputs from execution process to the received context. The
225
- result from execution will pass to value of `outputs` key.
230
+ """Set an outputs from execution result context to the received context
231
+ with a `to` input parameter. The result context from stage execution
232
+ will be set with `outputs` key in this stage ID key.
226
233
 
227
234
  For example of setting output method, If you receive execute output
228
235
  and want to set on the `to` like;
229
236
 
230
- ... (i) output: {'foo': bar}
237
+ ... (i) output: {'foo': 'bar', 'skipped': True}
231
238
  ... (ii) to: {'stages': {}}
232
239
 
233
- The result of the `to` argument will be;
240
+ The received context in the `to` argument will be;
234
241
 
235
242
  ... (iii) to: {
236
243
  'stages': {
237
244
  '<stage-id>': {
238
245
  'outputs': {'foo': 'bar'},
239
- 'skipped': False,
246
+ 'skipped': True,
240
247
  }
241
248
  }
242
249
  }
243
250
 
251
+ The keys that will set to the received context is `outputs`,
252
+ `errors`, and `skipped` keys. The `errors` and `skipped` keys will
253
+ extract from the result context if it exists. If it does not found, it
254
+ will not set on the received context.
255
+
244
256
  Important:
257
+
245
258
  This method is use for reconstruct the result context and transfer
246
- to the `to` argument.
259
+ to the `to` argument. The result context was soft copied before set
260
+ output step.
247
261
 
248
- :param output: (DictData) An output data that want to extract to an
249
- output key.
250
- :param to: (DictData) A context data that want to add output result.
262
+ :param output: (DictData) A result data context that want to extract
263
+ and transfer to the `outputs` key in receive context.
264
+ :param to: (DictData) A received context data.
251
265
 
252
266
  :rtype: DictData
253
267
  """
@@ -282,11 +296,12 @@ class BaseStage(BaseModel, ABC):
282
296
  }
283
297
  return to
284
298
 
285
- def get_outputs(self, outputs: DictData) -> DictData:
299
+ def get_outputs(self, output: DictData) -> DictData:
286
300
  """Get the outputs from stages data. It will get this stage ID from
287
301
  the stage outputs mapping.
288
302
 
289
- :param outputs: (DictData) A stage outputs that want to get by stage ID.
303
+ :param output: (DictData) A stage output context that want to get this
304
+ stage ID `outputs` key.
290
305
 
291
306
  :rtype: DictData
292
307
  """
@@ -296,33 +311,31 @@ class BaseStage(BaseModel, ABC):
296
311
  return {}
297
312
 
298
313
  _id: str = (
299
- param2template(self.id, params=outputs, extras=self.extras)
314
+ param2template(self.id, params=output, extras=self.extras)
300
315
  if self.id
301
316
  else gen_id(
302
- param2template(self.name, params=outputs, extras=self.extras)
317
+ param2template(self.name, params=output, extras=self.extras)
303
318
  )
304
319
  )
305
- return outputs.get("stages", {}).get(_id, {}).get("outputs", {})
320
+ return output.get("stages", {}).get(_id, {}).get("outputs", {})
306
321
 
307
- def is_skipped(self, params: DictData | None = None) -> bool:
322
+ def is_skipped(self, params: DictData) -> bool:
308
323
  """Return true if condition of this stage do not correct. This process
309
324
  use build-in eval function to execute the if-condition.
310
325
 
326
+ :param params: (DictData) A parameters that want to pass to condition
327
+ template.
328
+
311
329
  :raise StageException: When it has any error raise from the eval
312
330
  condition statement.
313
331
  :raise StageException: When return type of the eval condition statement
314
332
  does not return with boolean type.
315
333
 
316
- :param params: (DictData) A parameters that want to pass to condition
317
- template.
318
-
319
334
  :rtype: bool
320
335
  """
321
336
  if self.condition is None:
322
337
  return False
323
338
 
324
- params: DictData = {} if params is None else params
325
-
326
339
  try:
327
340
  # WARNING: The eval build-in function is very dangerous. So, it
328
341
  # should use the `re` module to validate eval-string before
@@ -340,7 +353,14 @@ class BaseStage(BaseModel, ABC):
340
353
 
341
354
 
342
355
  class BaseAsyncStage(BaseStage):
343
- """Base Async Stage model."""
356
+ """Base Async Stage model to make any stage model allow async execution for
357
+ optimize CPU and Memory on the current node. If you want to implement any
358
+ custom async stage, you can inherit this class and implement
359
+ `self.axecute()` (async + execute = axecute) method only.
360
+
361
+ This class is the abstraction class for any inherit asyncable stage
362
+ model.
363
+ """
344
364
 
345
365
  @abstractmethod
346
366
  def execute(
@@ -385,20 +405,18 @@ class BaseAsyncStage(BaseStage):
385
405
  run_id: str | None = None,
386
406
  parent_run_id: str | None = None,
387
407
  result: Result | None = None,
388
- raise_error: bool | None = None,
389
408
  event: Event | None = None,
409
+ raise_error: bool | None = None,
390
410
  ) -> Result:
391
411
  """Async Handler stage execution result from the stage `execute` method.
392
412
 
393
- :param params: (DictData) A parameterize value data that use in this
394
- stage execution.
395
- :param run_id: (str) A running stage ID for this execution.
396
- :param parent_run_id: (str) A parent workflow running ID for this
397
- execution.
398
- :param result: (Result) A result object for keeping context and status
399
- data before execution.
413
+ :param params: (DictData) A parameter data.
414
+ :param run_id: (str) A stage running ID.
415
+ :param parent_run_id: (str) A parent job running ID.
416
+ :param result: (Result) A Result instance for return context and status.
417
+ :param event: (Event) An Event manager instance that use to cancel this
418
+ execution if it forces stopped by parent execution.
400
419
  :param raise_error: (bool) A flag that all this method raise error
401
- :param event: (Event) An event manager that pass to the stage execution.
402
420
 
403
421
  :rtype: Result
404
422
  """
@@ -414,24 +432,26 @@ class BaseAsyncStage(BaseStage):
414
432
  rs: Result = await self.axecute(params, result=result, event=event)
415
433
  return rs
416
434
  except Exception as e: # pragma: no cov
417
- await result.trace.aerror(f"[STAGE]: {e.__class__.__name__}: {e}")
418
-
435
+ e_name: str = e.__class__.__name__
436
+ await result.trace.aerror(f"[STAGE]: Handler {e_name}: {e}")
419
437
  if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
420
438
  if isinstance(e, StageException):
421
439
  raise
422
440
 
423
441
  raise StageException(
424
- f"{self.__class__.__name__}: \n\t"
425
- f"{e.__class__.__name__}: {e}"
442
+ f"{self.__class__.__name__}: {NEWLINE}{e_name}: {e}"
426
443
  ) from None
427
444
 
428
- errors: DictData = {"errors": to_dict(e)}
429
- return result.catch(status=FAILED, context=errors)
445
+ return result.catch(status=FAILED, context={"errors": to_dict(e)})
430
446
 
431
447
 
432
448
  class EmptyStage(BaseAsyncStage):
433
- """Empty stage that do nothing (context equal empty stage) and logging the
434
- name of stage only to stdout.
449
+ """Empty stage executor that do nothing and log the `message` field to
450
+ stdout only. It can use for tracking a template parameter on the workflow or
451
+ debug step.
452
+
453
+ You can pass a sleep value in second unit to this stage for waiting
454
+ after log message.
435
455
 
436
456
  Data Validate:
437
457
  >>> stage = {
@@ -443,11 +463,14 @@ class EmptyStage(BaseAsyncStage):
443
463
 
444
464
  echo: Optional[str] = Field(
445
465
  default=None,
446
- description="A string message that want to show on the stdout.",
466
+ description="A message that want to show on the stdout.",
447
467
  )
448
468
  sleep: float = Field(
449
469
  default=0,
450
- description="A second value to sleep before start execution.",
470
+ description=(
471
+ "A second value to sleep before start execution. This value should "
472
+ "gather or equal 0, and less than 1800 seconds."
473
+ ),
451
474
  ge=0,
452
475
  lt=1800,
453
476
  )
@@ -460,18 +483,15 @@ class EmptyStage(BaseAsyncStage):
460
483
  event: Event | None = None,
461
484
  ) -> Result:
462
485
  """Execution method for the Empty stage that do only logging out to
463
- stdout. This method does not use the `handler_result` decorator because
464
- it does not get any error from logging function.
486
+ stdout.
465
487
 
466
488
  The result context should be empty and do not process anything
467
489
  without calling logging function.
468
490
 
469
- :param params: (DictData) A context data that want to add output result.
470
- But this stage does not pass any output.
471
- :param result: (Result) A result object for keeping context and status
472
- data.
473
- :param event: (Event) An event manager that use to track parent execute
474
- was not force stopped.
491
+ :param params: (DictData) A parameter data.
492
+ :param result: (Result) A Result instance for return context and status.
493
+ :param event: (Event) An Event manager instance that use to cancel this
494
+ execution if it forces stopped by parent execution.
475
495
 
476
496
  :rtype: Result
477
497
  """
@@ -484,21 +504,18 @@ class EmptyStage(BaseAsyncStage):
484
504
  message: str = "..."
485
505
  else:
486
506
  message: str = param2template(
487
- dedent(self.echo), params, extras=self.extras
507
+ dedent(self.echo.strip("\n")), params, extras=self.extras
488
508
  )
489
- if "\n" in self.echo:
490
- message: str = "\n\t" + message.replace("\n", "\n\t").strip(
491
- "\n"
492
- )
509
+ if "\n" in message:
510
+ message: str = NEWLINE + message.replace("\n", NEWLINE)
493
511
 
494
512
  result.trace.info(
495
513
  f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
496
514
  )
497
515
  if self.sleep > 0:
498
516
  if self.sleep > 5:
499
- result.trace.info(f"[STAGE]: ... sleep ({self.sleep} seconds)")
517
+ result.trace.info(f"[STAGE]: ... sleep ({self.sleep} sec)")
500
518
  time.sleep(self.sleep)
501
-
502
519
  return result.catch(status=SUCCESS)
503
520
 
504
521
  async def axecute(
@@ -511,44 +528,48 @@ class EmptyStage(BaseAsyncStage):
511
528
  """Async execution method for this Empty stage that only logging out to
512
529
  stdout.
513
530
 
514
- :param params: (DictData) A context data that want to add output result.
515
- But this stage does not pass any output.
516
- :param result: (Result) A result object for keeping context and status
517
- data.
518
- :param event: (Event) An event manager that use to track parent execute
519
- was not force stopped.
531
+ :param params: (DictData) A parameter data.
532
+ :param result: (Result) A Result instance for return context and status.
533
+ :param event: (Event) An Event manager instance that use to cancel this
534
+ execution if it forces stopped by parent execution.
520
535
 
521
536
  :rtype: Result
522
537
  """
523
- if result is None:
524
- result: Result = Result(
525
- run_id=gen_id(self.name + (self.id or ""), unique=True),
526
- extras=self.extras,
538
+ result: Result = result or Result(
539
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
540
+ extras=self.extras,
541
+ )
542
+
543
+ if not self.echo:
544
+ message: str = "..."
545
+ else:
546
+ message: str = param2template(
547
+ dedent(self.echo.strip("\n")), params, extras=self.extras
527
548
  )
549
+ if "\n" in message:
550
+ message: str = NEWLINE + message.replace("\n", NEWLINE)
528
551
 
529
- await result.trace.ainfo(
530
- f"[STAGE]: Empty-Execute: {self.name!r}: "
531
- f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
552
+ result.trace.info(
553
+ f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
532
554
  )
533
-
534
555
  if self.sleep > 0:
535
556
  if self.sleep > 5:
536
557
  await result.trace.ainfo(
537
- f"[STAGE]: ... sleep ({self.sleep} seconds)"
558
+ f"[STAGE]: ... sleep ({self.sleep} sec)"
538
559
  )
539
560
  await asyncio.sleep(self.sleep)
540
-
541
561
  return result.catch(status=SUCCESS)
542
562
 
543
563
 
544
564
  class BashStage(BaseStage):
545
- """Bash execution stage that execute bash script on the current OS.
546
- If your current OS is Windows, it will run on the bash in the WSL.
565
+ """Bash stage executor that execute bash script on the current OS.
566
+ If your current OS is Windows, it will run on the bash from the current WSL.
567
+ It will use `bash` for Windows OS and use `sh` for Linux OS.
547
568
 
548
- I get some limitation when I run shell statement with the built-in
549
- subprocess package. It does not good enough to use multiline statement.
550
- Thus, I add writing ``.sh`` file before execution process for fix this
551
- issue.
569
+ This stage has some limitation when it runs shell statement with the
570
+ built-in subprocess package. It does not good enough to use multiline
571
+ statement. Thus, it will write the `.sh` file before start running bash
572
+ command for fix this issue.
552
573
 
553
574
  Data Validate:
554
575
  >>> stage = {
@@ -560,15 +581,46 @@ class BashStage(BaseStage):
560
581
  ... }
561
582
  """
562
583
 
563
- bash: str = Field(description="A bash statement that want to execute.")
584
+ bash: str = Field(
585
+ description=(
586
+ "A bash statement that want to execute via Python subprocess."
587
+ )
588
+ )
564
589
  env: DictStr = Field(
565
590
  default_factory=dict,
566
591
  description=(
567
- "An environment variables that set before start execute by adding "
568
- "on the header of the `.sh` file."
592
+ "An environment variables that set before run bash command. It "
593
+ "will add on the header of the `.sh` file."
569
594
  ),
570
595
  )
571
596
 
597
+ @contextlib.asynccontextmanager
598
+ async def acreate_sh_file(
599
+ self, bash: str, env: DictStr, run_id: str | None = None
600
+ ) -> AsyncIterator: # pragma no cov
601
+ import aiofiles
602
+
603
+ f_name: str = f"{run_id or uuid.uuid4()}.sh"
604
+ f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
605
+
606
+ async with aiofiles.open(f"./{f_name}", mode="w", newline="\n") as f:
607
+ # NOTE: write header of `.sh` file
608
+ await f.write(f"#!/bin/{f_shebang}\n\n")
609
+
610
+ # NOTE: add setting environment variable before bash skip statement.
611
+ await f.writelines([f"{k}='{env[k]}';\n" for k in env])
612
+
613
+ # NOTE: make sure that shell script file does not have `\r` char.
614
+ await f.write("\n" + bash.replace("\r\n", "\n"))
615
+
616
+ # NOTE: Make this .sh file able to executable.
617
+ make_exec(f"./{f_name}")
618
+
619
+ yield [f_shebang, f_name]
620
+
621
+ # Note: Remove .sh file that use to run bash.
622
+ Path(f"./{f_name}").unlink()
623
+
572
624
  @contextlib.contextmanager
573
625
  def create_sh_file(
574
626
  self, bash: str, env: DictStr, run_id: str | None = None
@@ -577,16 +629,14 @@ class BashStage(BaseStage):
577
629
  step will write the `.sh` file before giving this file name to context.
578
630
  After that, it will auto delete this file automatic.
579
631
 
580
- :param bash: (str) A bash statement that want to execute.
581
- :param env: (DictStr) An environment variable that use on this bash
582
- statement.
632
+ :param bash: (str) A bash statement.
633
+ :param env: (DictStr) An environment variable that set before run bash.
583
634
  :param run_id: (str | None) A running stage ID that use for writing sh
584
635
  file instead generate by UUID4.
585
636
 
586
637
  :rtype: Iterator[TupleStr]
587
638
  """
588
- run_id: str = run_id or uuid.uuid4()
589
- f_name: str = f"{run_id}.sh"
639
+ f_name: str = f"{run_id or uuid.uuid4()}.sh"
590
640
  f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
591
641
 
592
642
  with open(f"./{f_name}", mode="w", newline="\n") as f:
@@ -614,28 +664,28 @@ class BashStage(BaseStage):
614
664
  result: Result | None = None,
615
665
  event: Event | None = None,
616
666
  ) -> Result:
617
- """Execute the Bash statement with the Python build-in ``subprocess``
618
- package.
667
+ """Execute bash statement with the Python build-in `subprocess` package.
668
+ It will catch result from the `subprocess.run` returning output like
669
+ `return_code`, `stdout`, and `stderr`.
619
670
 
620
- :param params: A parameter data that want to use in this execution.
621
- :param result: (Result) A result object for keeping context and status
622
- data.
623
- :param event: (Event) An event manager that use to track parent execute
624
- was not force stopped.
671
+ :param params: (DictData) A parameter data.
672
+ :param result: (Result) A Result instance for return context and status.
673
+ :param event: (Event) An Event manager instance that use to cancel this
674
+ execution if it forces stopped by parent execution.
625
675
 
626
676
  :rtype: Result
627
677
  """
628
- if result is None: # pragma: no cov
629
- result: Result = Result(
630
- run_id=gen_id(self.name + (self.id or ""), unique=True),
631
- extras=self.extras,
632
- )
678
+ result: Result = result or Result(
679
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
680
+ extras=self.extras,
681
+ )
682
+
683
+ result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
633
684
 
634
685
  bash: str = param2template(
635
- dedent(self.bash), params, extras=self.extras
686
+ dedent(self.bash.strip("\n")), params, extras=self.extras
636
687
  )
637
688
 
638
- result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
639
689
  with self.create_sh_file(
640
690
  bash=bash,
641
691
  env=param2template(self.env, params, extras=self.extras),
@@ -654,7 +704,7 @@ class BashStage(BaseStage):
654
704
  else rs.stderr
655
705
  ).removesuffix("\n")
656
706
  raise StageException(
657
- f"Subprocess: {e}\nRunning Statement:\n---\n"
707
+ f"Subprocess: {e}\n---( statement )---\n"
658
708
  f"```bash\n{bash}\n```"
659
709
  )
660
710
  return result.catch(
@@ -668,18 +718,24 @@ class BashStage(BaseStage):
668
718
 
669
719
 
670
720
  class PyStage(BaseStage):
671
- """Python executor stage that running the Python statement with receiving
672
- globals and additional variables.
721
+ """Python stage that running the Python statement with the current globals
722
+ and passing an input additional variables via `exec` built-in function.
673
723
 
674
724
  This stage allow you to use any Python object that exists on the globals
675
725
  such as import your installed package.
676
726
 
727
+ Warning:
728
+
729
+ The exec build-in function is very dangerous. So, it should use the `re`
730
+ module to validate exec-string before running or exclude the `os` package
731
+ from the current globals variable.
732
+
677
733
  Data Validate:
678
734
  >>> stage = {
679
735
  ... "name": "Python stage execution",
680
- ... "run": 'print("Hello {x}")',
736
+ ... "run": 'print(f"Hello {VARIABLE}")',
681
737
  ... "vars": {
682
- ... "x": "BAR",
738
+ ... "VARIABLE": "WORLD",
683
739
  ... },
684
740
  ... }
685
741
  """
@@ -741,9 +797,9 @@ class PyStage(BaseStage):
741
797
  event: Event | None = None,
742
798
  ) -> Result:
743
799
  """Execute the Python statement that pass all globals and input params
744
- to globals argument on ``exec`` build-in function.
800
+ to globals argument on `exec` build-in function.
745
801
 
746
- :param params: A parameter that want to pass before run any statement.
802
+ :param params: (DictData) A parameter data.
747
803
  :param result: (Result) A result object for keeping context and status
748
804
  data.
749
805
  :param event: (Event) An event manager that use to track parent execute
@@ -751,11 +807,10 @@ class PyStage(BaseStage):
751
807
 
752
808
  :rtype: Result
753
809
  """
754
- if result is None: # pragma: no cov
755
- result: Result = Result(
756
- run_id=gen_id(self.name + (self.id or ""), unique=True),
757
- extras=self.extras,
758
- )
810
+ result: Result = result or Result(
811
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
812
+ extras=self.extras,
813
+ )
759
814
 
760
815
  lc: DictData = {}
761
816
  gb: DictData = (
@@ -792,20 +847,36 @@ class PyStage(BaseStage):
792
847
  },
793
848
  )
794
849
 
850
+ async def axecute(
851
+ self,
852
+ ):
853
+ """Async execution method.
854
+
855
+ References:
856
+ - https://stackoverflow.com/questions/44859165/async-exec-in-python
857
+ """
858
+
795
859
 
796
860
  class CallStage(BaseStage):
797
- """Call executor that call the Python function from registry with tag
798
- decorator function in ``utils`` module and run it with input arguments.
861
+ """Call stage executor that call the Python function from registry with tag
862
+ decorator function in `reusables` module and run it with input arguments.
863
+
864
+ This stage is different with PyStage because the PyStage is just run
865
+ a Python statement with the `exec` function and pass the current locals and
866
+ globals before exec that statement. This stage will import the caller
867
+ function can call it with an input arguments. So, you can create your
868
+ function complexly that you can for your objective to invoked by this stage
869
+ object.
799
870
 
800
- This stage is different with PyStage because the PyStage is just calling
801
- a Python statement with the ``eval`` and pass that locale before eval that
802
- statement. So, you can create your function complexly that you can for your
803
- objective to invoked by this stage object.
871
+ This stage is the most powerfull stage of this package for run every
872
+ use-case by a custom requirement that you want by creating the Python
873
+ function and adding it to the caller registry value by importer syntax like
874
+ `module.caller.registry` not path style like `module/caller/registry`.
804
875
 
805
- This stage is the usefull stage for run every job by a custom requirement
806
- that you want by creating the Python function and adding it to the task
807
- registry by importer syntax like `module.tasks.registry` not path style like
808
- `module/tasks/registry`.
876
+ Warning:
877
+
878
+ The caller registry to get a caller function should importable by the
879
+ current Python execution pointer.
809
880
 
810
881
  Data Validate:
811
882
  >>> stage = {
@@ -817,12 +888,16 @@ class CallStage(BaseStage):
817
888
 
818
889
  uses: str = Field(
819
890
  description=(
820
- "A pointer that want to load function from the call registry."
891
+ "A caller function with registry importer syntax that use to load "
892
+ "function before execute step. The caller registry syntax should "
893
+ "be `<import.part>/<func-name>@<tag-name>`."
821
894
  ),
822
895
  )
823
896
  args: DictData = Field(
824
897
  default_factory=dict,
825
- description="An arguments that want to pass to the call function.",
898
+ description=(
899
+ "An argument parameter that will pass to this caller function."
900
+ ),
826
901
  alias="with",
827
902
  )
828
903
 
@@ -833,19 +908,12 @@ class CallStage(BaseStage):
833
908
  result: Result | None = None,
834
909
  event: Event | None = None,
835
910
  ) -> Result:
836
- """Execute the Call function that already in the call registry.
911
+ """Execute this caller function with its argument parameter.
837
912
 
838
- :raise ValueError: When the necessary arguments of call function do not
839
- set from the input params argument.
840
- :raise TypeError: When the return type of call function does not be
841
- dict type.
842
-
843
- :param params: (DictData) A parameter that want to pass before run any
844
- statement.
845
- :param result: (Result) A result object for keeping context and status
846
- data.
847
- :param event: (Event) An event manager that use to track parent execute
848
- was not force stopped.
913
+ :param params: (DictData) A parameter data.
914
+ :param result: (Result) A Result instance for return context and status.
915
+ :param event: (Event) An Event manager instance that use to cancel this
916
+ execution if it forces stopped by parent execution.
849
917
 
850
918
  :raise ValueError: If necessary arguments does not pass from the `args`
851
919
  field.
@@ -854,18 +922,20 @@ class CallStage(BaseStage):
854
922
 
855
923
  :rtype: Result
856
924
  """
857
- if result is None: # pragma: no cov
858
- result: Result = Result(
859
- run_id=gen_id(self.name + (self.id or ""), unique=True),
860
- extras=self.extras,
861
- )
925
+ result: Result = result or Result(
926
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
927
+ extras=self.extras,
928
+ )
862
929
 
863
- has_keyword: bool = False
864
930
  call_func: TagFunc = extract_call(
865
931
  param2template(self.uses, params, extras=self.extras),
866
932
  registries=self.extras.get("registry_caller"),
867
933
  )()
868
934
 
935
+ result.trace.info(
936
+ f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
937
+ )
938
+
869
939
  # VALIDATE: check input task caller parameters that exists before
870
940
  # calling.
871
941
  args: DictData = {"result": result} | param2template(
@@ -873,6 +943,7 @@ class CallStage(BaseStage):
873
943
  )
874
944
  ips = inspect.signature(call_func)
875
945
  necessary_params: list[str] = []
946
+ has_keyword: bool = False
876
947
  for k in ips.parameters:
877
948
  if (
878
949
  v := ips.parameters[k]
@@ -896,10 +967,6 @@ class CallStage(BaseStage):
896
967
  if "result" not in ips.parameters and not has_keyword:
897
968
  args.pop("result")
898
969
 
899
- result.trace.info(
900
- f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
901
- )
902
-
903
970
  args = self.parse_model_args(call_func, args, result)
904
971
 
905
972
  if inspect.iscoroutinefunction(call_func):
@@ -970,9 +1037,9 @@ class CallStage(BaseStage):
970
1037
 
971
1038
 
972
1039
  class TriggerStage(BaseStage):
973
- """Trigger Workflow execution stage that execute another workflow. This
974
- the core stage that allow you to create the reusable workflow object or
975
- dynamic parameters workflow for common usecase.
1040
+ """Trigger workflow executor stage that run an input trigger Workflow
1041
+ execute method. This is the stage that allow you to create the reusable
1042
+ Workflow template with dynamic parameters.
976
1043
 
977
1044
  Data Validate:
978
1045
  >>> stage = {
@@ -984,12 +1051,13 @@ class TriggerStage(BaseStage):
984
1051
 
985
1052
  trigger: str = Field(
986
1053
  description=(
987
- "A trigger workflow name that should exist on the config path."
1054
+ "A trigger workflow name. This workflow name should exist on the "
1055
+ "config path because it will load by the `load_conf` method."
988
1056
  ),
989
1057
  )
990
1058
  params: DictData = Field(
991
1059
  default_factory=dict,
992
- description="A parameter that want to pass to workflow execution.",
1060
+ description="A parameter that will pass to workflow execution method.",
993
1061
  )
994
1062
 
995
1063
  def execute(
@@ -1002,7 +1070,7 @@ class TriggerStage(BaseStage):
1002
1070
  """Trigger another workflow execution. It will wait the trigger
1003
1071
  workflow running complete before catching its result.
1004
1072
 
1005
- :param params: A parameter data that want to use in this execution.
1073
+ :param params: (DictData) A parameter data.
1006
1074
  :param result: (Result) A result object for keeping context and status
1007
1075
  data.
1008
1076
  :param event: (Event) An event manager that use to track parent execute
@@ -1013,11 +1081,10 @@ class TriggerStage(BaseStage):
1013
1081
  from .exceptions import WorkflowException
1014
1082
  from .workflow import Workflow
1015
1083
 
1016
- if result is None:
1017
- result: Result = Result(
1018
- run_id=gen_id(self.name + (self.id or ""), unique=True),
1019
- extras=self.extras,
1020
- )
1084
+ result: Result = result or Result(
1085
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1086
+ extras=self.extras,
1087
+ )
1021
1088
 
1022
1089
  _trigger: str = param2template(self.trigger, params, extras=self.extras)
1023
1090
  result.trace.info(f"[STAGE]: Trigger-Execute: {_trigger!r}")
@@ -1037,16 +1104,20 @@ class TriggerStage(BaseStage):
1037
1104
  err_msg: str | None = (
1038
1105
  f" with:\n{msg}"
1039
1106
  if (msg := rs.context.get("errors", {}).get("message"))
1040
- else ""
1107
+ else "."
1108
+ )
1109
+ raise StageException(
1110
+ f"Trigger workflow return failed status{err_msg}"
1041
1111
  )
1042
- raise StageException(f"Trigger workflow was failed{err_msg}.")
1043
1112
  return rs
1044
1113
 
1045
1114
 
1046
1115
  class ParallelStage(BaseStage): # pragma: no cov
1047
- """Parallel execution stage that execute child stages with parallel.
1116
+ """Parallel stage executor that execute branch stages with multithreading.
1117
+ This stage let you set the fix branches for running child stage inside it on
1118
+ multithread pool.
1048
1119
 
1049
- This stage is not the low-level stage model because it runs muti-stages
1120
+ This stage is not the low-level stage model because it runs multi-stages
1050
1121
  in this stage execution.
1051
1122
 
1052
1123
  Data Validate:
@@ -1072,58 +1143,55 @@ class ParallelStage(BaseStage): # pragma: no cov
1072
1143
  """
1073
1144
 
1074
1145
  parallel: dict[str, list[Stage]] = Field(
1075
- description="A mapping of parallel branch name and stages.",
1146
+ description="A mapping of branch name and its stages.",
1076
1147
  )
1077
1148
  max_workers: int = Field(
1078
1149
  default=2,
1079
1150
  ge=1,
1080
1151
  lt=20,
1081
1152
  description=(
1082
- "The maximum thread pool worker size for execution parallel."
1153
+ "The maximum multi-thread pool worker size for execution parallel. "
1154
+ "This value should be gather or equal than 1, and less than 20."
1083
1155
  ),
1084
1156
  alias="max-workers",
1085
1157
  )
1086
1158
 
1087
- def execute_task(
1159
+ def execute_branch(
1088
1160
  self,
1089
1161
  branch: str,
1090
1162
  params: DictData,
1091
1163
  result: Result,
1092
1164
  *,
1093
1165
  event: Event | None = None,
1094
- extras: DictData | None = None,
1095
- ) -> DictData:
1096
- """Task execution method for passing a branch to each thread.
1166
+ ) -> Result:
1167
+ """Execute all stage with specific branch ID.
1097
1168
 
1098
- :param branch: A branch ID.
1099
- :param params: A parameter data that want to use in this execution.
1100
- :param result: (Result) A result object for keeping context and status
1101
- data.
1102
- :param event: (Event) An event manager that use to track parent execute
1103
- was not force stopped.
1104
- :param extras: (DictData) An extra parameters that want to override
1105
- config values.
1169
+ :param branch: (str) A branch ID.
1170
+ :param params: (DictData) A parameter data.
1171
+ :param result: (Result) A Result instance for return context and status.
1172
+ :param event: (Event) An Event manager instance that use to cancel this
1173
+ execution if it forces stopped by parent execution.
1106
1174
 
1107
- :rtype: DictData
1175
+ :rtype: Result
1108
1176
  """
1109
- result.trace.debug(f"... Execute branch: {branch!r}")
1177
+ result.trace.debug(f"[STAGE]: Execute Branch: {branch!r}")
1110
1178
  context: DictData = copy.deepcopy(params)
1111
1179
  context.update({"branch": branch})
1112
1180
  output: DictData = {"branch": branch, "stages": {}}
1113
1181
  for stage in self.parallel[branch]:
1114
1182
 
1115
- if extras:
1116
- stage.extras = extras
1183
+ if self.extras:
1184
+ stage.extras = self.extras
1117
1185
 
1118
1186
  if stage.is_skipped(params=context):
1119
- result.trace.info(f"... Skip stage: {stage.iden!r}")
1187
+ result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
1120
1188
  stage.set_outputs(output={"skipped": True}, to=output)
1121
1189
  continue
1122
1190
 
1123
1191
  if event and event.is_set():
1124
1192
  error_msg: str = (
1125
1193
  "Branch-Stage was canceled from event that had set before "
1126
- "stage item execution."
1194
+ "stage branch execution."
1127
1195
  )
1128
1196
  return result.catch(
1129
1197
  status=CANCEL,
@@ -1147,17 +1215,29 @@ class ParallelStage(BaseStage): # pragma: no cov
1147
1215
  stage.set_outputs(rs.context, to=output)
1148
1216
  stage.set_outputs(stage.get_outputs(output), to=context)
1149
1217
  except (StageException, UtilException) as e: # pragma: no cov
1150
- result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1218
+ result.trace.error(
1219
+ f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
1220
+ )
1221
+ result.catch(
1222
+ status=FAILED,
1223
+ parallel={
1224
+ branch: {
1225
+ "branch": branch,
1226
+ "stages": filter_func(output.pop("stages", {})),
1227
+ "errors": e.to_dict(),
1228
+ },
1229
+ },
1230
+ )
1151
1231
  raise StageException(
1152
- f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
1232
+ f"Sub-Stage raise: {e.__class__.__name__}: {e}"
1153
1233
  ) from None
1154
1234
 
1155
1235
  if rs.status == FAILED:
1156
1236
  error_msg: str = (
1157
- f"Item-Stage was break because it has a sub stage, "
1237
+ f"Branch-Stage was break because it has a sub stage, "
1158
1238
  f"{stage.iden}, failed without raise error."
1159
1239
  )
1160
- return result.catch(
1240
+ result.catch(
1161
1241
  status=FAILED,
1162
1242
  parallel={
1163
1243
  branch: {
@@ -1167,6 +1247,7 @@ class ParallelStage(BaseStage): # pragma: no cov
1167
1247
  },
1168
1248
  },
1169
1249
  )
1250
+ raise StageException(error_msg)
1170
1251
 
1171
1252
  return result.catch(
1172
1253
  status=SUCCESS,
@@ -1185,30 +1266,37 @@ class ParallelStage(BaseStage): # pragma: no cov
1185
1266
  result: Result | None = None,
1186
1267
  event: Event | None = None,
1187
1268
  ) -> Result:
1188
- """Execute the stages that parallel each branch via multi-threading mode
1189
- or async mode by changing `async_mode` flag.
1269
+ """Execute parallel each branch via multi-threading pool.
1190
1270
 
1191
- :param params: A parameter that want to pass before run any statement.
1192
- :param result: (Result) A result object for keeping context and status
1193
- data.
1194
- :param event: (Event) An event manager that use to track parent execute
1195
- was not force stopped.
1271
+ :param params: (DictData) A parameter data.
1272
+ :param result: (Result) A Result instance for return context and status.
1273
+ :param event: (Event) An Event manager instance that use to cancel this
1274
+ execution if it forces stopped by parent execution.
1196
1275
 
1197
1276
  :rtype: Result
1198
1277
  """
1199
- if result is None: # pragma: no cov
1200
- result: Result = Result(
1201
- run_id=gen_id(self.name + (self.id or ""), unique=True),
1202
- extras=self.extras,
1203
- )
1278
+ result: Result = result or Result(
1279
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1280
+ extras=self.extras,
1281
+ )
1204
1282
  event: Event = Event() if event is None else event
1205
1283
  result.trace.info(
1206
1284
  f"[STAGE]: Parallel-Execute: {self.max_workers} workers."
1207
1285
  )
1208
1286
  result.catch(status=WAIT, context={"parallel": {}})
1287
+ if event and event.is_set(): # pragma: no cov
1288
+ return result.catch(
1289
+ status=CANCEL,
1290
+ context={
1291
+ "errors": StageException(
1292
+ "Stage was canceled from event that had set "
1293
+ "before stage parallel execution."
1294
+ ).to_dict()
1295
+ },
1296
+ )
1297
+
1209
1298
  with ThreadPoolExecutor(
1210
- max_workers=self.max_workers,
1211
- thread_name_prefix="parallel_stage_exec_",
1299
+ max_workers=self.max_workers, thread_name_prefix="stage_parallel_"
1212
1300
  ) as executor:
1213
1301
 
1214
1302
  context: DictData = {}
@@ -1216,36 +1304,36 @@ class ParallelStage(BaseStage): # pragma: no cov
1216
1304
 
1217
1305
  futures: list[Future] = (
1218
1306
  executor.submit(
1219
- self.execute_task,
1307
+ self.execute_branch,
1220
1308
  branch=branch,
1221
1309
  params=params,
1222
1310
  result=result,
1223
1311
  event=event,
1224
- extras=self.extras,
1225
1312
  )
1226
1313
  for branch in self.parallel
1227
1314
  )
1228
1315
 
1229
- done = as_completed(futures, timeout=1800)
1230
- for future in done:
1316
+ for future in as_completed(futures):
1231
1317
  try:
1232
1318
  future.result()
1233
1319
  except StageException as e:
1234
1320
  status = FAILED
1235
1321
  result.trace.error(
1236
- f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
1322
+ f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
1237
1323
  )
1238
- context.update({"errors": e.to_dict()})
1239
-
1324
+ if "errors" in context:
1325
+ context["errors"].append(e.to_dict())
1326
+ else:
1327
+ context["errors"] = [e.to_dict()]
1240
1328
  return result.catch(status=status, context=context)
1241
1329
 
1242
1330
 
1243
1331
  class ForEachStage(BaseStage):
1244
- """For-Each execution stage that execute child stages with an item in list
1245
- of item values. This stage is not the low-level stage model because it runs
1246
- muti-stages in this stage execution.
1332
+ """For-Each stage executor that execute all stages with each item in the
1333
+ foreach list.
1247
1334
 
1248
- The concept of this stage use the same logic of the Job execution.
1335
+ This stage is not the low-level stage model because it runs
1336
+ multi-stages in this stage execution.
1249
1337
 
1250
1338
  Data Validate:
1251
1339
  >>> stage = {
@@ -1254,7 +1342,7 @@ class ForEachStage(BaseStage):
1254
1342
  ... "stages": [
1255
1343
  ... {
1256
1344
  ... "name": "Echo stage",
1257
- ... "echo": "Start run with item {{ item }}"
1345
+ ... "echo": "Start run with item ${{ item }}"
1258
1346
  ... },
1259
1347
  ... ],
1260
1348
  ... }
@@ -1262,13 +1350,14 @@ class ForEachStage(BaseStage):
1262
1350
 
1263
1351
  foreach: Union[list[str], list[int], str] = Field(
1264
1352
  description=(
1265
- "A items for passing to each stages via ${{ item }} template."
1353
+ "A items for passing to stages via ${{ item }} template parameter."
1266
1354
  ),
1267
1355
  )
1268
1356
  stages: list[Stage] = Field(
1269
1357
  default_factory=list,
1270
1358
  description=(
1271
- "A list of stage that will run with each item in the foreach field."
1359
+ "A list of stage that will run with each item in the `foreach` "
1360
+ "field."
1272
1361
  ),
1273
1362
  )
1274
1363
  concurrent: int = Field(
@@ -1283,27 +1372,25 @@ class ForEachStage(BaseStage):
1283
1372
 
1284
1373
  def execute_item(
1285
1374
  self,
1286
- item: Union[str, int],
1375
+ item: StrOrInt,
1287
1376
  params: DictData,
1288
1377
  result: Result,
1289
1378
  *,
1290
1379
  event: Event | None = None,
1291
1380
  ) -> Result:
1292
- """Execute foreach item from list of item.
1381
+ """Execute all stage with specific foreach item.
1293
1382
 
1294
1383
  :param item: (str | int) An item that want to execution.
1295
- :param params: (DictData) A parameter that want to pass to stage
1296
- execution.
1297
- :param result: (Result) A result object for keeping context and status
1298
- data.
1299
- :param event: (Event) An event manager that use to track parent execute
1300
- was not force stopped.
1384
+ :param params: (DictData) A parameter data.
1385
+ :param result: (Result) A Result instance for return context and status.
1386
+ :param event: (Event) An Event manager instance that use to cancel this
1387
+ execution if it forces stopped by parent execution.
1301
1388
 
1302
1389
  :raise StageException: If the stage execution raise errors.
1303
1390
 
1304
1391
  :rtype: Result
1305
1392
  """
1306
- result.trace.debug(f"[STAGE]: Execute item: {item!r}")
1393
+ result.trace.debug(f"[STAGE]: Execute Item: {item!r}")
1307
1394
  context: DictData = copy.deepcopy(params)
1308
1395
  context.update({"item": item})
1309
1396
  output: DictData = {"item": item, "stages": {}}
@@ -1313,7 +1400,7 @@ class ForEachStage(BaseStage):
1313
1400
  stage.extras = self.extras
1314
1401
 
1315
1402
  if stage.is_skipped(params=context):
1316
- result.trace.info(f"... Skip stage: {stage.iden!r}")
1403
+ result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
1317
1404
  stage.set_outputs(output={"skipped": True}, to=output)
1318
1405
  continue
1319
1406
 
@@ -1344,9 +1431,21 @@ class ForEachStage(BaseStage):
1344
1431
  stage.set_outputs(rs.context, to=output)
1345
1432
  stage.set_outputs(stage.get_outputs(output), to=context)
1346
1433
  except (StageException, UtilException) as e:
1347
- result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1434
+ result.trace.error(
1435
+ f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
1436
+ )
1437
+ result.catch(
1438
+ status=FAILED,
1439
+ foreach={
1440
+ item: {
1441
+ "item": item,
1442
+ "stages": filter_func(output.pop("stages", {})),
1443
+ "errors": e.to_dict(),
1444
+ },
1445
+ },
1446
+ )
1348
1447
  raise StageException(
1349
- f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
1448
+ f"Sub-Stage raise: {e.__class__.__name__}: {e}"
1350
1449
  ) from None
1351
1450
 
1352
1451
  if rs.status == FAILED:
@@ -1354,7 +1453,8 @@ class ForEachStage(BaseStage):
1354
1453
  f"Item-Stage was break because it has a sub stage, "
1355
1454
  f"{stage.iden}, failed without raise error."
1356
1455
  )
1357
- return result.catch(
1456
+ result.trace.warning(f"[STAGE]: {error_msg}")
1457
+ result.catch(
1358
1458
  status=FAILED,
1359
1459
  foreach={
1360
1460
  item: {
@@ -1364,6 +1464,8 @@ class ForEachStage(BaseStage):
1364
1464
  },
1365
1465
  },
1366
1466
  )
1467
+ raise StageException(error_msg)
1468
+
1367
1469
  return result.catch(
1368
1470
  status=SUCCESS,
1369
1471
  foreach={
@@ -1383,29 +1485,29 @@ class ForEachStage(BaseStage):
1383
1485
  ) -> Result:
1384
1486
  """Execute the stages that pass each item form the foreach field.
1385
1487
 
1386
- :param params: A parameter that want to pass before run any statement.
1387
- :param result: (Result) A result object for keeping context and status
1388
- data.
1389
- :param event: (Event) An event manager that use to track parent execute
1390
- was not force stopped.
1488
+ :param params: (DictData) A parameter data.
1489
+ :param result: (Result) A Result instance for return context and status.
1490
+ :param event: (Event) An Event manager instance that use to cancel this
1491
+ execution if it forces stopped by parent execution.
1492
+
1493
+ :raise TypeError: If the foreach does not match with type list.
1391
1494
 
1392
1495
  :rtype: Result
1393
1496
  """
1394
- if result is None: # pragma: no cov
1395
- result: Result = Result(
1396
- run_id=gen_id(self.name + (self.id or ""), unique=True),
1397
- extras=self.extras,
1398
- )
1497
+ result: Result = result or Result(
1498
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1499
+ extras=self.extras,
1500
+ )
1399
1501
  event: Event = Event() if event is None else event
1400
1502
  foreach: Union[list[str], list[int]] = (
1401
1503
  param2template(self.foreach, params, extras=self.extras)
1402
1504
  if isinstance(self.foreach, str)
1403
1505
  else self.foreach
1404
1506
  )
1507
+
1508
+ # [VALIDATE]: Type of the foreach should be `list` type.
1405
1509
  if not isinstance(foreach, list):
1406
- raise StageException(
1407
- f"Foreach does not support foreach value: {foreach!r}"
1408
- )
1510
+ raise TypeError(f"Does not support foreach: {foreach!r}")
1409
1511
 
1410
1512
  result.trace.info(f"[STAGE]: Foreach-Execute: {foreach!r}.")
1411
1513
  result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
@@ -1437,33 +1539,36 @@ class ForEachStage(BaseStage):
1437
1539
  context: DictData = {}
1438
1540
  status: Status = SUCCESS
1439
1541
 
1440
- done, not_done = wait(
1441
- futures, timeout=1800, return_when=FIRST_EXCEPTION
1442
- )
1443
-
1542
+ done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
1444
1543
  if len(done) != len(futures):
1445
1544
  result.trace.warning(
1446
- "[STAGE]: Set the event for stop running stage."
1545
+ "[STAGE]: Set event for stop pending stage future."
1447
1546
  )
1448
1547
  event.set()
1449
1548
  for future in not_done:
1450
1549
  future.cancel()
1451
1550
 
1551
+ nd: str = f", item not run: {not_done}" if not_done else ""
1552
+ result.trace.debug(f"... Foreach set Fail-Fast{nd}")
1553
+
1452
1554
  for future in done:
1453
1555
  try:
1454
1556
  future.result()
1455
1557
  except StageException as e:
1456
1558
  status = FAILED
1457
1559
  result.trace.error(
1458
- f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
1560
+ f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
1459
1561
  )
1460
1562
  context.update({"errors": e.to_dict()})
1461
-
1462
1563
  return result.catch(status=status, context=context)
1463
1564
 
1464
1565
 
1465
- class UntilStage(BaseStage): # pragma: no cov
1466
- """Until execution stage.
1566
+ class UntilStage(BaseStage):
1567
+ """Until stage executor that will run stages in each loop until it valid
1568
+ with stop loop condition.
1569
+
1570
+ This stage is not the low-level stage model because it runs
1571
+ multi-stages in this stage execution.
1467
1572
 
1468
1573
  Data Validate:
1469
1574
  >>> stage = {
@@ -1485,23 +1590,25 @@ class UntilStage(BaseStage): # pragma: no cov
1485
1590
  "An initial value that can be any value in str, int, or bool type."
1486
1591
  ),
1487
1592
  )
1488
- until: str = Field(description="A until condition.")
1593
+ until: str = Field(description="A until condition for stop the while loop.")
1489
1594
  stages: list[Stage] = Field(
1490
1595
  default_factory=list,
1491
1596
  description=(
1492
- "A list of stage that will run with each item until condition "
1493
- "correct."
1597
+ "A list of stage that will run with each item in until loop."
1494
1598
  ),
1495
1599
  )
1496
1600
  max_loop: int = Field(
1497
1601
  default=10,
1498
1602
  ge=1,
1499
1603
  lt=100,
1500
- description="The maximum value of loop for this until stage.",
1604
+ description=(
1605
+ "The maximum value of loop for this until stage. This value should "
1606
+ "be gather or equal than 1, and less than 100."
1607
+ ),
1501
1608
  alias="max-loop",
1502
1609
  )
1503
1610
 
1504
- def execute_item(
1611
+ def execute_loop(
1505
1612
  self,
1506
1613
  item: T,
1507
1614
  loop: int,
@@ -1509,19 +1616,17 @@ class UntilStage(BaseStage): # pragma: no cov
1509
1616
  result: Result,
1510
1617
  event: Event | None = None,
1511
1618
  ) -> tuple[Result, T]:
1512
- """Execute until item set item by some stage or by default loop
1513
- variable.
1619
+ """Execute all stage with specific loop and item.
1514
1620
 
1515
1621
  :param item: (T) An item that want to execution.
1516
1622
  :param loop: (int) A number of loop.
1517
- :param params: (DictData) A parameter that want to pass to stage
1518
- execution.
1519
- :param result: (Result) A result object for keeping context and status
1520
- data.
1521
- :param event: (Event) An event manager that use to track parent execute
1522
- was not force stopped.
1623
+ :param params: (DictData) A parameter data.
1624
+ :param result: (Result) A Result instance for return context and status.
1625
+ :param event: (Event) An Event manager instance that use to cancel this
1626
+ execution if it forces stopped by parent execution.
1523
1627
 
1524
1628
  :rtype: tuple[Result, T]
1629
+ :return: Return a pair of Result and changed item.
1525
1630
  """
1526
1631
  result.trace.debug(f"... Execute until item: {item!r}")
1527
1632
  context: DictData = copy.deepcopy(params)
@@ -1534,14 +1639,14 @@ class UntilStage(BaseStage): # pragma: no cov
1534
1639
  stage.extras = self.extras
1535
1640
 
1536
1641
  if stage.is_skipped(params=context):
1537
- result.trace.info(f"... Skip stage: {stage.iden!r}")
1642
+ result.trace.info(f"[STAGE]: Skip stage: {stage.iden!r}")
1538
1643
  stage.set_outputs(output={"skipped": True}, to=output)
1539
1644
  continue
1540
1645
 
1541
1646
  if event and event.is_set():
1542
1647
  error_msg: str = (
1543
- "Item-Stage was canceled from event that had set before "
1544
- "stage item execution."
1648
+ "Loop-Stage was canceled from event that had set before "
1649
+ "stage loop execution."
1545
1650
  )
1546
1651
  return (
1547
1652
  result.catch(
@@ -1573,11 +1678,42 @@ class UntilStage(BaseStage): # pragma: no cov
1573
1678
 
1574
1679
  stage.set_outputs(_output, to=context)
1575
1680
  except (StageException, UtilException) as e:
1576
- result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1681
+ result.trace.error(
1682
+ f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
1683
+ )
1684
+ result.catch(
1685
+ status=FAILED,
1686
+ until={
1687
+ loop: {
1688
+ "loop": loop,
1689
+ "item": item,
1690
+ "stages": filter_func(output.pop("stages", {})),
1691
+ "errors": e.to_dict(),
1692
+ }
1693
+ },
1694
+ )
1577
1695
  raise StageException(
1578
1696
  f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
1579
1697
  ) from None
1580
1698
 
1699
+ if rs.status == FAILED:
1700
+ error_msg: str = (
1701
+ f"Loop-Stage was break because it has a sub stage, "
1702
+ f"{stage.iden}, failed without raise error."
1703
+ )
1704
+ result.catch(
1705
+ status=FAILED,
1706
+ until={
1707
+ loop: {
1708
+ "loop": loop,
1709
+ "item": item,
1710
+ "stages": filter_func(output.pop("stages", {})),
1711
+ "errors": StageException(error_msg).to_dict(),
1712
+ }
1713
+ },
1714
+ )
1715
+ raise StageException(error_msg)
1716
+
1581
1717
  return (
1582
1718
  result.catch(
1583
1719
  status=SUCCESS,
@@ -1599,21 +1735,19 @@ class UntilStage(BaseStage): # pragma: no cov
1599
1735
  result: Result | None = None,
1600
1736
  event: Event | None = None,
1601
1737
  ) -> Result:
1602
- """Execute the stages that pass item from until condition field and
1603
- setter step.
1738
+ """Execute until loop with checking until condition.
1604
1739
 
1605
- :param params: A parameter that want to pass before run any statement.
1606
- :param result: (Result) A result object for keeping context and status
1607
- data.
1608
- :param event: (Event) An event manager that use to track parent execute
1609
- was not force stopped.
1740
+ :param params: (DictData) A parameter data.
1741
+ :param result: (Result) A Result instance for return context and status.
1742
+ :param event: (Event) An Event manager instance that use to cancel this
1743
+ execution if it forces stopped by parent execution.
1610
1744
 
1611
1745
  :rtype: Result
1612
1746
  """
1613
- if result is None: # pragma: no cov
1614
- result: Result = Result(
1615
- run_id=gen_id(self.name + (self.id or ""), unique=True)
1616
- )
1747
+ result: Result = result or Result(
1748
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1749
+ extras=self.extras,
1750
+ )
1617
1751
 
1618
1752
  result.trace.info(f"[STAGE]: Until-Execution: {self.until}")
1619
1753
  item: Union[str, int, bool] = param2template(
@@ -1631,12 +1765,12 @@ class UntilStage(BaseStage): # pragma: no cov
1631
1765
  context={
1632
1766
  "errors": StageException(
1633
1767
  "Stage was canceled from event that had set "
1634
- "before stage until execution."
1768
+ "before stage loop execution."
1635
1769
  ).to_dict()
1636
1770
  },
1637
1771
  )
1638
1772
 
1639
- result, item = self.execute_item(
1773
+ result, item = self.execute_loop(
1640
1774
  item=item,
1641
1775
  loop=loop,
1642
1776
  params=params,
@@ -1647,10 +1781,10 @@ class UntilStage(BaseStage): # pragma: no cov
1647
1781
  loop += 1
1648
1782
  if item is None:
1649
1783
  result.trace.warning(
1650
- "... Does not have set item stage. It will use loop by "
1651
- "default."
1784
+ f"... Loop-Execute not set item. It use loop: {loop} by "
1785
+ f"default."
1652
1786
  )
1653
- item = loop
1787
+ item: int = loop
1654
1788
 
1655
1789
  next_track: bool = eval(
1656
1790
  param2template(
@@ -1663,8 +1797,8 @@ class UntilStage(BaseStage): # pragma: no cov
1663
1797
  )
1664
1798
  if not isinstance(next_track, bool):
1665
1799
  raise StageException(
1666
- "Return type of until condition does not be boolean, it"
1667
- f"return: {next_track!r}"
1800
+ "Return type of until condition not be `boolean`, getting"
1801
+ f": {next_track!r}"
1668
1802
  )
1669
1803
  track: bool = not next_track
1670
1804
  delay(0.025)
@@ -1679,14 +1813,14 @@ class UntilStage(BaseStage): # pragma: no cov
1679
1813
  class Match(BaseModel):
1680
1814
  """Match model for the Case Stage."""
1681
1815
 
1682
- case: Union[str, int] = Field(description="A match case.")
1816
+ case: StrOrInt = Field(description="A match case.")
1683
1817
  stages: list[Stage] = Field(
1684
1818
  description="A list of stage to execution for this case."
1685
1819
  )
1686
1820
 
1687
1821
 
1688
1822
  class CaseStage(BaseStage):
1689
- """Case execution stage.
1823
+ """Case stage executor that execute all stages if the condition was matched.
1690
1824
 
1691
1825
  Data Validate:
1692
1826
  >>> stage = {
@@ -1742,19 +1876,16 @@ class CaseStage(BaseStage):
1742
1876
 
1743
1877
  :param case: (str) A case that want to execution.
1744
1878
  :param stages: (list[Stage]) A list of stage.
1745
- :param params: (DictData) A parameter that want to pass to stage
1746
- execution.
1747
- :param result: (Result) A result object for keeping context and status
1748
- data.
1749
- :param event: (Event) An event manager that use to track parent execute
1750
- was not force stopped.
1879
+ :param params: (DictData) A parameter data.
1880
+ :param result: (Result) A Result instance for return context and status.
1881
+ :param event: (Event) An Event manager instance that use to cancel this
1882
+ execution if it forces stopped by parent execution.
1751
1883
 
1752
1884
  :rtype: Result
1753
1885
  """
1754
1886
  context: DictData = copy.deepcopy(params)
1755
1887
  context.update({"case": case})
1756
1888
  output: DictData = {"case": case, "stages": {}}
1757
-
1758
1889
  for stage in stages:
1759
1890
 
1760
1891
  if self.extras:
@@ -1830,19 +1961,17 @@ class CaseStage(BaseStage):
1830
1961
  ) -> Result:
1831
1962
  """Execute case-match condition that pass to the case field.
1832
1963
 
1833
- :param params: A parameter that want to pass before run any statement.
1834
- :param result: (Result) A result object for keeping context and status
1835
- data.
1836
- :param event: (Event) An event manager that use to track parent execute
1837
- was not force stopped.
1964
+ :param params: (DictData) A parameter data.
1965
+ :param result: (Result) A Result instance for return context and status.
1966
+ :param event: (Event) An Event manager instance that use to cancel this
1967
+ execution if it forces stopped by parent execution.
1838
1968
 
1839
1969
  :rtype: Result
1840
1970
  """
1841
- if result is None: # pragma: no cov
1842
- result: Result = Result(
1843
- run_id=gen_id(self.name + (self.id or ""), unique=True),
1844
- extras=self.extras,
1845
- )
1971
+ result: Result = result or Result(
1972
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
1973
+ extras=self.extras,
1974
+ )
1846
1975
 
1847
1976
  _case: Optional[str] = param2template(
1848
1977
  self.case, params, extras=self.extras
@@ -1898,7 +2027,7 @@ class CaseStage(BaseStage):
1898
2027
 
1899
2028
 
1900
2029
  class RaiseStage(BaseStage): # pragma: no cov
1901
- """Raise error stage execution that raise StageException that use a message
2030
+ """Raise error stage executor that raise `StageException` that use a message
1902
2031
  field for making error message before raise.
1903
2032
 
1904
2033
  Data Validate:
@@ -1911,7 +2040,7 @@ class RaiseStage(BaseStage): # pragma: no cov
1911
2040
 
1912
2041
  message: str = Field(
1913
2042
  description=(
1914
- "An error message that want to raise with StageException class"
2043
+ "An error message that want to raise with `StageException` class"
1915
2044
  ),
1916
2045
  alias="raise",
1917
2046
  )
@@ -1925,43 +2054,27 @@ class RaiseStage(BaseStage): # pragma: no cov
1925
2054
  ) -> Result:
1926
2055
  """Raise the StageException object with the message field execution.
1927
2056
 
1928
- :param params: A parameter that want to pass before run any statement.
1929
- :param result: (Result) A result object for keeping context and status
1930
- data.
1931
- :param event: (Event) An event manager that use to track parent execute
1932
- was not force stopped.
2057
+ :param params: (DictData) A parameter data.
2058
+ :param result: (Result) A Result instance for return context and status.
2059
+ :param event: (Event) An Event manager instance that use to cancel this
2060
+ execution if it forces stopped by parent execution.
1933
2061
  """
1934
- if result is None: # pragma: no cov
1935
- result: Result = Result(
1936
- run_id=gen_id(self.name + (self.id or ""), unique=True),
1937
- extras=self.extras,
1938
- )
2062
+ result: Result = result or Result(
2063
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
2064
+ extras=self.extras,
2065
+ )
1939
2066
  message: str = param2template(self.message, params, extras=self.extras)
1940
2067
  result.trace.info(f"[STAGE]: Raise-Execute: {message!r}.")
1941
2068
  raise StageException(message)
1942
2069
 
1943
2070
 
1944
- # TODO: Not implement this stages yet
1945
- class HookStage(BaseStage): # pragma: no cov
1946
- """Hook stage execution."""
1947
-
1948
- hook: str
1949
- args: DictData = Field(default_factory=dict)
1950
- callback: str
1951
-
1952
- def execute(
1953
- self,
1954
- params: DictData,
1955
- *,
1956
- result: Result | None = None,
1957
- event: Event | None = None,
1958
- ) -> Result:
1959
- raise NotImplementedError("Hook Stage does not implement yet.")
1960
-
1961
-
1962
- # TODO: Not implement this stages yet
1963
2071
  class DockerStage(BaseStage): # pragma: no cov
1964
- """Docker container stage execution.
2072
+ """Docker container stage execution that will pull the specific Docker image
2073
+ with custom authentication and run this image by passing environment
2074
+ variables and mounting local volume to this Docker container.
2075
+
2076
+ The volume path that mount to this Docker container will limit. That is
2077
+ this stage does not allow you to mount any path to this container.
1965
2078
 
1966
2079
  Data Validate:
1967
2080
  >>> stage = {
@@ -1969,10 +2082,7 @@ class DockerStage(BaseStage): # pragma: no cov
1969
2082
  ... "image": "image-name.pkg.com",
1970
2083
  ... "env": {
1971
2084
  ... "ENV": "dev",
1972
- ... "DEBUG": "true",
1973
- ... },
1974
- ... "volume": {
1975
- ... "secrets": "/secrets",
2085
+ ... "SECRET": "${SPECIFIC_SECRET}",
1976
2086
  ... },
1977
2087
  ... "auth": {
1978
2088
  ... "username": "__json_key",
@@ -1985,8 +2095,16 @@ class DockerStage(BaseStage): # pragma: no cov
1985
2095
  description="A Docker image url with tag that want to run.",
1986
2096
  )
1987
2097
  tag: str = Field(default="latest", description="An Docker image tag.")
1988
- env: DictData = Field(default_factory=dict)
1989
- volume: DictData = Field(default_factory=dict)
2098
+ env: DictData = Field(
2099
+ default_factory=dict,
2100
+ description=(
2101
+ "An environment variable that want pass to Docker container.",
2102
+ ),
2103
+ )
2104
+ volume: DictData = Field(
2105
+ default_factory=dict,
2106
+ description="A mapping of local and target mounting path.",
2107
+ )
1990
2108
  auth: DictData = Field(
1991
2109
  default_factory=dict,
1992
2110
  description=(
@@ -1998,7 +2116,17 @@ class DockerStage(BaseStage): # pragma: no cov
1998
2116
  self,
1999
2117
  params: DictData,
2000
2118
  result: Result,
2001
- ):
2119
+ event: Event | None = None,
2120
+ ) -> Result:
2121
+ """Execute Docker container task.
2122
+
2123
+ :param params: (DictData) A parameter data.
2124
+ :param result: (Result) A Result instance for return context and status.
2125
+ :param event: (Event) An Event manager instance that use to cancel this
2126
+ execution if it forces stopped by parent execution.
2127
+
2128
+ :rtype: Result
2129
+ """
2002
2130
  from docker import DockerClient
2003
2131
  from docker.errors import ContainerError
2004
2132
 
@@ -2016,6 +2144,16 @@ class DockerStage(BaseStage): # pragma: no cov
2016
2144
  for line in resp:
2017
2145
  result.trace.info(f"... {line}")
2018
2146
 
2147
+ if event and event.is_set():
2148
+ error_msg: str = (
2149
+ "Docker-Stage was canceled from event that had set before "
2150
+ "run the Docker container."
2151
+ )
2152
+ return result.catch(
2153
+ status=CANCEL,
2154
+ context={"errors": StageException(error_msg).to_dict()},
2155
+ )
2156
+
2019
2157
  unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
2020
2158
  container = client.containers.run(
2021
2159
  image=f"{self.image}:{self.tag}",
@@ -2054,6 +2192,13 @@ class DockerStage(BaseStage): # pragma: no cov
2054
2192
  f"{self.image}:{self.tag}",
2055
2193
  out,
2056
2194
  )
2195
+ output_file: Path = Path(f".docker.{result.run_id}.logs/outputs.json")
2196
+ if not output_file.exists():
2197
+ return result.catch(status=SUCCESS)
2198
+
2199
+ with output_file.open(mode="rt") as f:
2200
+ data = json.load(f)
2201
+ return result.catch(status=SUCCESS, context=data)
2057
2202
 
2058
2203
  def execute(
2059
2204
  self,
@@ -2062,13 +2207,34 @@ class DockerStage(BaseStage): # pragma: no cov
2062
2207
  result: Result | None = None,
2063
2208
  event: Event | None = None,
2064
2209
  ) -> Result:
2210
+ """Execute the Docker image via Python API.
2211
+
2212
+ :param params: (DictData) A parameter data.
2213
+ :param result: (Result) A Result instance for return context and status.
2214
+ :param event: (Event) An Event manager instance that use to cancel this
2215
+ execution if it forces stopped by parent execution.
2216
+
2217
+ :rtype: Result
2218
+ """
2219
+ result: Result = result or Result(
2220
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
2221
+ extras=self.extras,
2222
+ )
2223
+
2224
+ result.trace.info(f"[STAGE]: Docker-Execute: {self.image}:{self.tag}")
2225
+
2065
2226
  raise NotImplementedError("Docker Stage does not implement yet.")
2066
2227
 
2067
2228
 
2068
- # TODO: Not implement this stages yet
2069
2229
  class VirtualPyStage(PyStage): # pragma: no cov
2070
- """Python Virtual Environment stage execution."""
2230
+ """Virtual Python stage executor that run Python statement on the dependent
2231
+ Python virtual environment via the `uv` package.
2232
+ """
2071
2233
 
2234
+ version: str = Field(
2235
+ default="3.9",
2236
+ description="A Python version that want to run.",
2237
+ )
2072
2238
  deps: list[str] = Field(
2073
2239
  description=(
2074
2240
  "list of Python dependency that want to install before execution "
@@ -2076,7 +2242,54 @@ class VirtualPyStage(PyStage): # pragma: no cov
2076
2242
  ),
2077
2243
  )
2078
2244
 
2079
- def create_py_file(self, py: str, run_id: str | None): ...
2245
+ @contextlib.contextmanager
2246
+ def create_py_file(
2247
+ self,
2248
+ py: str,
2249
+ values: DictData,
2250
+ deps: list[str],
2251
+ run_id: str | None = None,
2252
+ ) -> Iterator[str]:
2253
+ """Create the .py file with an input Python string statement.
2254
+
2255
+ :param py: A Python string statement.
2256
+ :param values: A variable that want to set before running this
2257
+ :param deps: An additional Python dependencies that want install before
2258
+ run this python stage.
2259
+ :param run_id: (str | None) A running ID of this stage execution.
2260
+ """
2261
+ run_id: str = run_id or uuid.uuid4()
2262
+ f_name: str = f"{run_id}.py"
2263
+ with open(f"./{f_name}", mode="w", newline="\n") as f:
2264
+ # NOTE: Create variable mapping that write before running statement.
2265
+ vars_str: str = "\n ".join(
2266
+ f"{var} = {value!r}" for var, value in values.items()
2267
+ )
2268
+
2269
+ # NOTE: uv supports PEP 723 — inline TOML metadata.
2270
+ f.write(
2271
+ dedent(
2272
+ f"""
2273
+ # /// script
2274
+ # dependencies = [{', '.join(f'"{dep}"' for dep in deps)}]
2275
+ # ///
2276
+ {vars_str}
2277
+ """.strip(
2278
+ "\n"
2279
+ )
2280
+ )
2281
+ )
2282
+
2283
+ # NOTE: make sure that py script file does not have `\r` char.
2284
+ f.write("\n" + py.replace("\r\n", "\n"))
2285
+
2286
+ # NOTE: Make this .py file able to executable.
2287
+ make_exec(f"./{f_name}")
2288
+
2289
+ yield f_name
2290
+
2291
+ # Note: Remove .py file that use to run Python.
2292
+ Path(f"./{f_name}").unlink()
2080
2293
 
2081
2294
  def execute(
2082
2295
  self,
@@ -2088,31 +2301,61 @@ class VirtualPyStage(PyStage): # pragma: no cov
2088
2301
  """Execute the Python statement via Python virtual environment.
2089
2302
 
2090
2303
  Steps:
2091
- - Create python file.
2092
- - Create `.venv` and install necessary Python deps.
2093
- - Execution python file with uv and specific `.venv`.
2304
+ - Create python file with the `uv` syntax.
2305
+ - Execution python file with `uv run` via Python subprocess module.
2094
2306
 
2095
- :param params: A parameter that want to pass before run any statement.
2096
- :param result: (Result) A result object for keeping context and status
2097
- data.
2098
- :param event: (Event) An event manager that use to track parent execute
2099
- was not force stopped.
2307
+ :param params: (DictData) A parameter data.
2308
+ :param result: (Result) A Result instance for return context and status.
2309
+ :param event: (Event) An Event manager instance that use to cancel this
2310
+ execution if it forces stopped by parent execution.
2100
2311
 
2101
2312
  :rtype: Result
2102
2313
  """
2103
- if result is None: # pragma: no cov
2104
- result: Result = Result(
2105
- run_id=gen_id(self.name + (self.id or ""), unique=True)
2106
- )
2314
+ result: Result = result or Result(
2315
+ run_id=gen_id(self.name + (self.id or ""), unique=True),
2316
+ extras=self.extras,
2317
+ )
2107
2318
 
2108
2319
  result.trace.info(f"[STAGE]: Py-Virtual-Execute: {self.name}")
2109
- raise NotImplementedError(
2110
- "Python Virtual Stage does not implement yet."
2320
+ run: str = param2template(dedent(self.run), params, extras=self.extras)
2321
+ with self.create_py_file(
2322
+ py=run,
2323
+ values=param2template(self.vars, params, extras=self.extras),
2324
+ deps=param2template(self.deps, params, extras=self.extras),
2325
+ run_id=result.run_id,
2326
+ ) as py:
2327
+ result.trace.debug(f"... Create `{py}` file.")
2328
+ rs: CompletedProcess = subprocess.run(
2329
+ ["uv", "run", py, "--no-cache"],
2330
+ # ["uv", "run", "--python", "3.9", py],
2331
+ shell=False,
2332
+ capture_output=True,
2333
+ text=True,
2334
+ )
2335
+
2336
+ if rs.returncode > 0:
2337
+ # NOTE: Prepare stderr message that returning from subprocess.
2338
+ e: str = (
2339
+ rs.stderr.encode("utf-8").decode("utf-16")
2340
+ if "\\x00" in rs.stderr
2341
+ else rs.stderr
2342
+ ).removesuffix("\n")
2343
+ raise StageException(
2344
+ f"Subprocess: {e}\nRunning Statement:\n---\n"
2345
+ f"```python\n{run}\n```"
2346
+ )
2347
+ return result.catch(
2348
+ status=SUCCESS,
2349
+ context={
2350
+ "return_code": rs.returncode,
2351
+ "stdout": None if (out := rs.stdout.strip("\n")) == "" else out,
2352
+ "stderr": None if (out := rs.stderr.strip("\n")) == "" else out,
2353
+ },
2111
2354
  )
2112
2355
 
2113
2356
 
2114
2357
  # NOTE:
2115
- # An order of parsing stage model on the Job model with ``stages`` field.
2358
+ # An order of parsing stage model on the Job model with `stages` field.
2116
2359
  # From the current build-in stages, they do not have stage that have the same
2117
2360
  # fields that because of parsing on the Job's stages key.
2118
2361
  #
@@ -2121,7 +2364,6 @@ Stage = Annotated[
2121
2364
  DockerStage,
2122
2365
  BashStage,
2123
2366
  CallStage,
2124
- HookStage,
2125
2367
  TriggerStage,
2126
2368
  ForEachStage,
2127
2369
  UntilStage,