ddeutil-workflow 0.0.52__py3-none-any.whl → 0.0.54__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,23 +4,27 @@
4
4
  # license information.
5
5
  # ------------------------------------------------------------------------------
6
6
  # [x] Use dynamic config
7
- """Stage model. It stores all stage model that use for getting stage data template
8
- from the Job Model. The stage handle the minimize task that run in some thread
9
- (same thread at its job owner) that mean it is the lowest executor of a workflow
10
- that can tracking logs.
7
+ """Stages module include all stage model that use be the minimum execution layer
8
+ of this workflow engine. The stage handle the minimize task that run in some
9
+ thread (same thread at its job owner) that mean it is the lowest executor that
10
+ you can track logs.
11
11
 
12
- The output of stage execution only return 0 status because I do not want to
13
- handle stage error on this stage model. I think stage model should have a lot of
14
- use-case, and it does not worry when I want to create a new one.
12
+ The output of stage execution only return SUCCESS or CANCEL status because
13
+ I do not want to handle stage error on this stage execution. I think stage model
14
+ have a lot of use-case, and it should does not worry about it error output.
15
15
 
16
- Execution --> Ok --> Result with SUCCESS
16
+ So, I will create `handler_execute` for any exception class that raise from
17
+ the stage execution method.
17
18
 
18
- --> Error ┬-> Result with FAILED (if env var was set)
19
- ╰-> Raise StageException(...)
19
+ Execution --> Ok ---( handler )--> Result with `SUCCESS` or `CANCEL`
20
+
21
+ --> Error ┬--( handler )-> Result with `FAILED` (Set `raise_error` flag)
22
+ |
23
+ ╰--( handler )-> Raise StageException(...)
20
24
 
21
25
  On the context I/O that pass to a stage object at execute process. The
22
- execute method receives a `params={"params": {...}}` value for mapping to
23
- template searching.
26
+ execute method receives a `params={"params": {...}}` value for passing template
27
+ searching.
24
28
  """
25
29
  from __future__ import annotations
26
30
 
@@ -28,12 +32,13 @@ import asyncio
28
32
  import contextlib
29
33
  import copy
30
34
  import inspect
35
+ import json
31
36
  import subprocess
32
37
  import sys
33
38
  import time
34
39
  import uuid
35
40
  from abc import ABC, abstractmethod
36
- from collections.abc import Iterator
41
+ from collections.abc import AsyncIterator, Iterator
37
42
  from concurrent.futures import (
38
43
  FIRST_EXCEPTION,
39
44
  Future,
@@ -42,7 +47,7 @@ from concurrent.futures import (
42
47
  wait,
43
48
  )
44
49
  from datetime import datetime
45
- from inspect import Parameter
50
+ from inspect import Parameter, isclass, isfunction, ismodule
46
51
  from pathlib import Path
47
52
  from subprocess import CompletedProcess
48
53
  from textwrap import dedent
@@ -59,6 +64,7 @@ from .exceptions import StageException, UtilException, to_dict
59
64
  from .result import CANCEL, FAILED, SUCCESS, WAIT, Result, Status
60
65
  from .reusables import TagFunc, extract_call, not_in_template, param2template
61
66
  from .utils import (
67
+ NEWLINE,
62
68
  delay,
63
69
  filter_func,
64
70
  gen_id,
@@ -66,20 +72,22 @@ from .utils import (
66
72
  )
67
73
 
68
74
  T = TypeVar("T")
75
+ StrOrInt = Union[str, int]
69
76
 
70
77
 
71
78
  class BaseStage(BaseModel, ABC):
72
- """Base Stage Model that keep only id and name fields for the stage
73
- metadata. If you want to implement any custom stage, you can use this class
74
- to parent and implement ``self.execute()`` method only.
79
+ """Base Stage Model that keep only necessary fields like `id`, `name` or
80
+ `condition` for the stage metadata. If you want to implement any custom
81
+ stage, you can inherit this class and implement `self.execute()` method
82
+ only.
75
83
 
76
- This class is the abstraction class for any stage model that want to
77
- implement to workflow model.
84
+ This class is the abstraction class for any inherit stage model that
85
+ want to implement on this workflow package.
78
86
  """
79
87
 
80
88
  extras: DictData = Field(
81
89
  default_factory=dict,
82
- description="An extra override config values.",
90
+ description="An extra parameter that override core config values.",
83
91
  )
84
92
  id: Optional[str] = Field(
85
93
  default=None,
@@ -93,14 +101,17 @@ class BaseStage(BaseModel, ABC):
93
101
  )
94
102
  condition: Optional[str] = Field(
95
103
  default=None,
96
- description="A stage condition statement to allow stage executable.",
104
+ description=(
105
+ "A stage condition statement to allow stage executable. This field "
106
+ "alise with `if` field."
107
+ ),
97
108
  alias="if",
98
109
  )
99
110
 
100
111
  @property
101
112
  def iden(self) -> str:
102
- """Return identity of this stage object that return the id field first.
103
- If the id does not set, it will use name field instead.
113
+ """Return this stage identity that return the `id` field first and if
114
+ this `id` field does not set, it will use the `name` field instead.
104
115
 
105
116
  :rtype: str
106
117
  """
@@ -156,8 +167,8 @@ class BaseStage(BaseModel, ABC):
156
167
  run_id: str | None = None,
157
168
  parent_run_id: str | None = None,
158
169
  result: Result | None = None,
159
- raise_error: bool | None = None,
160
170
  event: Event | None = None,
171
+ raise_error: bool | None = None,
161
172
  ) -> Result | DictData:
162
173
  """Handler stage execution result from the stage `execute` method.
163
174
 
@@ -170,7 +181,7 @@ class BaseStage(BaseModel, ABC):
170
181
  ╰-context:
171
182
  ╰-outputs: ...
172
183
 
173
- --> Error --> Result (if env var was set)
184
+ --> Error --> Result (if `raise_error` was set)
174
185
  |-status: FAILED
175
186
  ╰-errors:
176
187
  |-class: ...
@@ -179,18 +190,17 @@ class BaseStage(BaseModel, ABC):
179
190
 
180
191
  --> Error --> Raise StageException(...)
181
192
 
182
- On the last step, it will set the running ID on a return result object
183
- from current stage ID before release the final result.
193
+ On the last step, it will set the running ID on a return result
194
+ object from the current stage ID before release the final result.
184
195
 
185
- :param params: (DictData) A parameterize value data that use in this
186
- stage execution.
196
+ :param params: (DictData) A parameter data.
187
197
  :param run_id: (str) A running stage ID for this execution.
188
198
  :param parent_run_id: (str) A parent workflow running ID for this
189
199
  execution.
190
200
  :param result: (Result) A result object for keeping context and status
191
201
  data before execution.
192
- :param raise_error: (bool) A flag that all this method raise error
193
202
  :param event: (Event) An event manager that pass to the stage execution.
203
+ :param raise_error: (bool) A flag that all this method raise error
194
204
 
195
205
  :rtype: Result
196
206
  """
@@ -206,8 +216,9 @@ class BaseStage(BaseModel, ABC):
206
216
  rs: Result = self.execute(params, result=result, event=event)
207
217
  return rs
208
218
  except Exception as e:
209
- result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
210
-
219
+ result.trace.error(
220
+ f"[STAGE]: Handler:{NEWLINE}{e.__class__.__name__}: {e}"
221
+ )
211
222
  if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
212
223
  if isinstance(e, StageException):
213
224
  raise
@@ -217,37 +228,39 @@ class BaseStage(BaseModel, ABC):
217
228
  f"{e.__class__.__name__}: {e}"
218
229
  ) from e
219
230
 
220
- errors: DictData = {"errors": to_dict(e)}
221
- return result.catch(status=FAILED, context=errors)
231
+ return result.catch(status=FAILED, context={"errors": to_dict(e)})
222
232
 
223
233
  def set_outputs(self, output: DictData, to: DictData) -> DictData:
224
- """Set an outputs from execution process to the received context. The
225
- result from execution will pass to value of `outputs` key.
234
+ """Set an outputs from execution result context to the received context
235
+ with a `to` input parameter. The result context from stage execution
236
+ will be set with `outputs` key in this stage ID key.
226
237
 
227
238
  For example of setting output method, If you receive execute output
228
239
  and want to set on the `to` like;
229
240
 
230
- ... (i) output: {'foo': bar}
241
+ ... (i) output: {'foo': 'bar', 'skipped': True}
231
242
  ... (ii) to: {'stages': {}}
232
243
 
233
- The result of the `to` argument will be;
244
+ The received context in the `to` argument will be;
234
245
 
235
246
  ... (iii) to: {
236
247
  'stages': {
237
248
  '<stage-id>': {
238
249
  'outputs': {'foo': 'bar'},
239
- 'skipped': False,
250
+ 'skipped': True,
240
251
  }
241
252
  }
242
253
  }
243
254
 
244
255
  Important:
256
+
245
257
  This method is use for reconstruct the result context and transfer
246
- to the `to` argument.
258
+ to the `to` argument. The result context was soft copied before set
259
+ output step.
247
260
 
248
- :param output: (DictData) An output data that want to extract to an
249
- output key.
250
- :param to: (DictData) A context data that want to add output result.
261
+ :param output: (DictData) A result data context that want to extract
262
+ and transfer to the `outputs` key in receive context.
263
+ :param to: (DictData) A received context data.
251
264
 
252
265
  :rtype: DictData
253
266
  """
@@ -266,7 +279,7 @@ class BaseStage(BaseModel, ABC):
266
279
  param2template(self.name, params=to, extras=self.extras)
267
280
  )
268
281
  )
269
-
282
+ output: DictData = output.copy()
270
283
  errors: DictData = (
271
284
  {"errors": output.pop("errors", {})} if "errors" in output else {}
272
285
  )
@@ -282,11 +295,12 @@ class BaseStage(BaseModel, ABC):
282
295
  }
283
296
  return to
284
297
 
285
- def get_outputs(self, outputs: DictData) -> DictData:
298
+ def get_outputs(self, output: DictData) -> DictData:
286
299
  """Get the outputs from stages data. It will get this stage ID from
287
300
  the stage outputs mapping.
288
301
 
289
- :param outputs: (DictData) A stage outputs that want to get by stage ID.
302
+ :param output: (DictData) A stage output context that want to get this
303
+ stage ID `outputs` key.
290
304
 
291
305
  :rtype: DictData
292
306
  """
@@ -296,33 +310,31 @@ class BaseStage(BaseModel, ABC):
296
310
  return {}
297
311
 
298
312
  _id: str = (
299
- param2template(self.id, params=outputs, extras=self.extras)
313
+ param2template(self.id, params=output, extras=self.extras)
300
314
  if self.id
301
315
  else gen_id(
302
- param2template(self.name, params=outputs, extras=self.extras)
316
+ param2template(self.name, params=output, extras=self.extras)
303
317
  )
304
318
  )
305
- return outputs.get("stages", {}).get(_id, {})
319
+ return output.get("stages", {}).get(_id, {}).get("outputs", {})
306
320
 
307
- def is_skipped(self, params: DictData | None = None) -> bool:
321
+ def is_skipped(self, params: DictData) -> bool:
308
322
  """Return true if condition of this stage do not correct. This process
309
323
  use build-in eval function to execute the if-condition.
310
324
 
325
+ :param params: (DictData) A parameters that want to pass to condition
326
+ template.
327
+
311
328
  :raise StageException: When it has any error raise from the eval
312
329
  condition statement.
313
330
  :raise StageException: When return type of the eval condition statement
314
331
  does not return with boolean type.
315
332
 
316
- :param params: (DictData) A parameters that want to pass to condition
317
- template.
318
-
319
333
  :rtype: bool
320
334
  """
321
335
  if self.condition is None:
322
336
  return False
323
337
 
324
- params: DictData = {} if params is None else params
325
-
326
338
  try:
327
339
  # WARNING: The eval build-in function is very dangerous. So, it
328
340
  # should use the `re` module to validate eval-string before
@@ -340,7 +352,14 @@ class BaseStage(BaseModel, ABC):
340
352
 
341
353
 
342
354
  class BaseAsyncStage(BaseStage):
343
- """Base Async Stage model."""
355
+ """Base Async Stage model to make any stage model allow async execution for
356
+ optimize CPU and Memory on the current node. If you want to implement any
357
+ custom async stage, you can inherit this class and implement
358
+ `self.axecute()` (async + execute = axecute) method only.
359
+
360
+ This class is the abstraction class for any inherit asyncable stage
361
+ model.
362
+ """
344
363
 
345
364
  @abstractmethod
346
365
  def execute(
@@ -385,20 +404,18 @@ class BaseAsyncStage(BaseStage):
385
404
  run_id: str | None = None,
386
405
  parent_run_id: str | None = None,
387
406
  result: Result | None = None,
388
- raise_error: bool | None = None,
389
407
  event: Event | None = None,
408
+ raise_error: bool | None = None,
390
409
  ) -> Result:
391
410
  """Async Handler stage execution result from the stage `execute` method.
392
411
 
393
- :param params: (DictData) A parameterize value data that use in this
394
- stage execution.
395
- :param run_id: (str) A running stage ID for this execution.
396
- :param parent_run_id: (str) A parent workflow running ID for this
397
- execution.
398
- :param result: (Result) A result object for keeping context and status
399
- data before execution.
412
+ :param params: (DictData) A parameter data.
413
+ :param run_id: (str) A stage running ID.
414
+ :param parent_run_id: (str) A parent job running ID.
415
+ :param result: (Result) A Result instance for return context and status.
416
+ :param event: (Event) An Event manager instance that use to cancel this
417
+ execution if it forces stopped by parent execution.
400
418
  :param raise_error: (bool) A flag that all this method raise error
401
- :param event: (Event) An event manager that pass to the stage execution.
402
419
 
403
420
  :rtype: Result
404
421
  """
@@ -414,7 +431,9 @@ class BaseAsyncStage(BaseStage):
414
431
  rs: Result = await self.axecute(params, result=result, event=event)
415
432
  return rs
416
433
  except Exception as e: # pragma: no cov
417
- await result.trace.aerror(f"[STAGE]: {e.__class__.__name__}: {e}")
434
+ await result.trace.aerror(
435
+ f"[STAGE]: Handler {e.__class__.__name__}: {e}"
436
+ )
418
437
 
419
438
  if dynamic("stage_raise_error", f=raise_error, extras=self.extras):
420
439
  if isinstance(e, StageException):
@@ -425,13 +444,16 @@ class BaseAsyncStage(BaseStage):
425
444
  f"{e.__class__.__name__}: {e}"
426
445
  ) from None
427
446
 
428
- errors: DictData = {"errors": to_dict(e)}
429
- return result.catch(status=FAILED, context=errors)
447
+ return result.catch(status=FAILED, context={"errors": to_dict(e)})
430
448
 
431
449
 
432
450
  class EmptyStage(BaseAsyncStage):
433
- """Empty stage that do nothing (context equal empty stage) and logging the
434
- name of stage only to stdout.
451
+ """Empty stage executor that do nothing and log the `message` field to
452
+ stdout only. It can use for tracking a template parameter on the workflow or
453
+ debug step.
454
+
455
+ You can pass a sleep value in second unit to this stage for waiting
456
+ after log message.
435
457
 
436
458
  Data Validate:
437
459
  >>> stage = {
@@ -443,11 +465,14 @@ class EmptyStage(BaseAsyncStage):
443
465
 
444
466
  echo: Optional[str] = Field(
445
467
  default=None,
446
- description="A string message that want to show on the stdout.",
468
+ description="A message that want to show on the stdout.",
447
469
  )
448
470
  sleep: float = Field(
449
471
  default=0,
450
- description="A second value to sleep before start execution.",
472
+ description=(
473
+ "A second value to sleep before start execution. This value should "
474
+ "gather or equal 0, and less than 1800 seconds."
475
+ ),
451
476
  ge=0,
452
477
  lt=1800,
453
478
  )
@@ -460,18 +485,15 @@ class EmptyStage(BaseAsyncStage):
460
485
  event: Event | None = None,
461
486
  ) -> Result:
462
487
  """Execution method for the Empty stage that do only logging out to
463
- stdout. This method does not use the `handler_result` decorator because
464
- it does not get any error from logging function.
488
+ stdout.
465
489
 
466
490
  The result context should be empty and do not process anything
467
491
  without calling logging function.
468
492
 
469
- :param params: (DictData) A context data that want to add output result.
470
- But this stage does not pass any output.
471
- :param result: (Result) A result object for keeping context and status
472
- data.
473
- :param event: (Event) An event manager that use to track parent execute
474
- was not force stopped.
493
+ :param params: (DictData) A parameter data.
494
+ :param result: (Result) A Result instance for return context and status.
495
+ :param event: (Event) An Event manager instance that use to cancel this
496
+ execution if it forces stopped by parent execution.
475
497
 
476
498
  :rtype: Result
477
499
  """
@@ -484,21 +506,18 @@ class EmptyStage(BaseAsyncStage):
484
506
  message: str = "..."
485
507
  else:
486
508
  message: str = param2template(
487
- dedent(self.echo), params, extras=self.extras
509
+ dedent(self.echo.strip("\n")), params, extras=self.extras
488
510
  )
489
- if "\n" in self.echo:
490
- message: str = "\n\t" + message.replace("\n", "\n\t").strip(
491
- "\n"
492
- )
511
+ if "\n" in message:
512
+ message: str = NEWLINE + message.replace("\n", NEWLINE)
493
513
 
494
514
  result.trace.info(
495
515
  f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
496
516
  )
497
517
  if self.sleep > 0:
498
518
  if self.sleep > 5:
499
- result.trace.info(f"[STAGE]: ... sleep ({self.sleep} seconds)")
519
+ result.trace.info(f"[STAGE]: ... sleep ({self.sleep} sec)")
500
520
  time.sleep(self.sleep)
501
-
502
521
  return result.catch(status=SUCCESS)
503
522
 
504
523
  async def axecute(
@@ -511,12 +530,10 @@ class EmptyStage(BaseAsyncStage):
511
530
  """Async execution method for this Empty stage that only logging out to
512
531
  stdout.
513
532
 
514
- :param params: (DictData) A context data that want to add output result.
515
- But this stage does not pass any output.
516
- :param result: (Result) A result object for keeping context and status
517
- data.
518
- :param event: (Event) An event manager that use to track parent execute
519
- was not force stopped.
533
+ :param params: (DictData) A parameter data.
534
+ :param result: (Result) A Result instance for return context and status.
535
+ :param event: (Event) An Event manager instance that use to cancel this
536
+ execution if it forces stopped by parent execution.
520
537
 
521
538
  :rtype: Result
522
539
  """
@@ -526,29 +543,36 @@ class EmptyStage(BaseAsyncStage):
526
543
  extras=self.extras,
527
544
  )
528
545
 
529
- await result.trace.ainfo(
530
- f"[STAGE]: Empty-Execute: {self.name!r}: "
531
- f"( {param2template(self.echo, params, extras=self.extras) or '...'} )"
532
- )
546
+ if not self.echo:
547
+ message: str = "..."
548
+ else:
549
+ message: str = param2template(
550
+ dedent(self.echo.strip("\n")), params, extras=self.extras
551
+ )
552
+ if "\n" in message:
553
+ message: str = NEWLINE + message.replace("\n", NEWLINE)
533
554
 
555
+ result.trace.info(
556
+ f"[STAGE]: Empty-Execute: {self.name!r}: ( {message} )"
557
+ )
534
558
  if self.sleep > 0:
535
559
  if self.sleep > 5:
536
560
  await result.trace.ainfo(
537
- f"[STAGE]: ... sleep ({self.sleep} seconds)"
561
+ f"[STAGE]: ... sleep ({self.sleep} sec)"
538
562
  )
539
563
  await asyncio.sleep(self.sleep)
540
-
541
564
  return result.catch(status=SUCCESS)
542
565
 
543
566
 
544
567
  class BashStage(BaseStage):
545
- """Bash execution stage that execute bash script on the current OS.
546
- If your current OS is Windows, it will run on the bash in the WSL.
568
+ """Bash stage executor that execute bash script on the current OS.
569
+ If your current OS is Windows, it will run on the bash from the current WSL.
570
+ It will use `bash` for Windows OS and use `sh` for Linux OS.
547
571
 
548
- I get some limitation when I run shell statement with the built-in
549
- subprocess package. It does not good enough to use multiline statement.
550
- Thus, I add writing ``.sh`` file before execution process for fix this
551
- issue.
572
+ This stage has some limitation when it runs shell statement with the
573
+ built-in subprocess package. It does not good enough to use multiline
574
+ statement. Thus, it will write the `.sh` file before start running bash
575
+ command for fix this issue.
552
576
 
553
577
  Data Validate:
554
578
  >>> stage = {
@@ -560,15 +584,46 @@ class BashStage(BaseStage):
560
584
  ... }
561
585
  """
562
586
 
563
- bash: str = Field(description="A bash statement that want to execute.")
587
+ bash: str = Field(
588
+ description=(
589
+ "A bash statement that want to execute via Python subprocess."
590
+ )
591
+ )
564
592
  env: DictStr = Field(
565
593
  default_factory=dict,
566
594
  description=(
567
- "An environment variables that set before start execute by adding "
568
- "on the header of the `.sh` file."
595
+ "An environment variables that set before run bash command. It "
596
+ "will add on the header of the `.sh` file."
569
597
  ),
570
598
  )
571
599
 
600
+ @contextlib.asynccontextmanager
601
+ async def acreate_sh_file(
602
+ self, bash: str, env: DictStr, run_id: str | None = None
603
+ ) -> AsyncIterator: # pragma no cov
604
+ import aiofiles
605
+
606
+ f_name: str = f"{run_id or uuid.uuid4()}.sh"
607
+ f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
608
+
609
+ async with aiofiles.open(f"./{f_name}", mode="w", newline="\n") as f:
610
+ # NOTE: write header of `.sh` file
611
+ await f.write(f"#!/bin/{f_shebang}\n\n")
612
+
613
+ # NOTE: add setting environment variable before bash skip statement.
614
+ await f.writelines([f"{k}='{env[k]}';\n" for k in env])
615
+
616
+ # NOTE: make sure that shell script file does not have `\r` char.
617
+ await f.write("\n" + bash.replace("\r\n", "\n"))
618
+
619
+ # NOTE: Make this .sh file able to executable.
620
+ make_exec(f"./{f_name}")
621
+
622
+ yield [f_shebang, f_name]
623
+
624
+ # Note: Remove .sh file that use to run bash.
625
+ Path(f"./{f_name}").unlink()
626
+
572
627
  @contextlib.contextmanager
573
628
  def create_sh_file(
574
629
  self, bash: str, env: DictStr, run_id: str | None = None
@@ -577,16 +632,14 @@ class BashStage(BaseStage):
577
632
  step will write the `.sh` file before giving this file name to context.
578
633
  After that, it will auto delete this file automatic.
579
634
 
580
- :param bash: (str) A bash statement that want to execute.
581
- :param env: (DictStr) An environment variable that use on this bash
582
- statement.
635
+ :param bash: (str) A bash statement.
636
+ :param env: (DictStr) An environment variable that set before run bash.
583
637
  :param run_id: (str | None) A running stage ID that use for writing sh
584
638
  file instead generate by UUID4.
585
639
 
586
640
  :rtype: Iterator[TupleStr]
587
641
  """
588
- run_id: str = run_id or uuid.uuid4()
589
- f_name: str = f"{run_id}.sh"
642
+ f_name: str = f"{run_id or uuid.uuid4()}.sh"
590
643
  f_shebang: str = "bash" if sys.platform.startswith("win") else "sh"
591
644
 
592
645
  with open(f"./{f_name}", mode="w", newline="\n") as f:
@@ -614,14 +667,14 @@ class BashStage(BaseStage):
614
667
  result: Result | None = None,
615
668
  event: Event | None = None,
616
669
  ) -> Result:
617
- """Execute the Bash statement with the Python build-in ``subprocess``
618
- package.
670
+ """Execute bash statement with the Python build-in `subprocess` package.
671
+ It will catch result from the `subprocess.run` returning output like
672
+ `return_code`, `stdout`, and `stderr`.
619
673
 
620
- :param params: A parameter data that want to use in this execution.
621
- :param result: (Result) A result object for keeping context and status
622
- data.
623
- :param event: (Event) An event manager that use to track parent execute
624
- was not force stopped.
674
+ :param params: (DictData) A parameter data.
675
+ :param result: (Result) A Result instance for return context and status.
676
+ :param event: (Event) An Event manager instance that use to cancel this
677
+ execution if it forces stopped by parent execution.
625
678
 
626
679
  :rtype: Result
627
680
  """
@@ -631,11 +684,12 @@ class BashStage(BaseStage):
631
684
  extras=self.extras,
632
685
  )
633
686
 
687
+ result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
688
+
634
689
  bash: str = param2template(
635
- dedent(self.bash), params, extras=self.extras
690
+ dedent(self.bash.strip("\n")), params, extras=self.extras
636
691
  )
637
692
 
638
- result.trace.info(f"[STAGE]: Shell-Execute: {self.name}")
639
693
  with self.create_sh_file(
640
694
  bash=bash,
641
695
  env=param2template(self.env, params, extras=self.extras),
@@ -654,7 +708,7 @@ class BashStage(BaseStage):
654
708
  else rs.stderr
655
709
  ).removesuffix("\n")
656
710
  raise StageException(
657
- f"Subprocess: {e}\nRunning Statement:\n---\n"
711
+ f"Subprocess: {e}\n---( statement )---\n"
658
712
  f"```bash\n{bash}\n```"
659
713
  )
660
714
  return result.catch(
@@ -668,18 +722,24 @@ class BashStage(BaseStage):
668
722
 
669
723
 
670
724
  class PyStage(BaseStage):
671
- """Python executor stage that running the Python statement with receiving
672
- globals and additional variables.
725
+ """Python stage that running the Python statement with the current globals
726
+ and passing an input additional variables via `exec` built-in function.
673
727
 
674
728
  This stage allow you to use any Python object that exists on the globals
675
729
  such as import your installed package.
676
730
 
731
+ Warning:
732
+
733
+ The exec build-in function is very dangerous. So, it should use the `re`
734
+ module to validate exec-string before running or exclude the `os` package
735
+ from the current globals variable.
736
+
677
737
  Data Validate:
678
738
  >>> stage = {
679
739
  ... "name": "Python stage execution",
680
- ... "run": 'print("Hello {x}")',
740
+ ... "run": 'print(f"Hello {VARIABLE}")',
681
741
  ... "vars": {
682
- ... "x": "BAR",
742
+ ... "VARIABLE": "WORLD",
683
743
  ... },
684
744
  ... }
685
745
  """
@@ -704,12 +764,11 @@ class PyStage(BaseStage):
704
764
 
705
765
  :rtype: Iterator[str]
706
766
  """
707
- from inspect import isclass, ismodule
708
-
709
767
  for value in values:
710
768
 
711
769
  if (
712
770
  value == "__annotations__"
771
+ or (value.startswith("__") and value.endswith("__"))
713
772
  or ismodule(values[value])
714
773
  or isclass(values[value])
715
774
  ):
@@ -727,11 +786,10 @@ class PyStage(BaseStage):
727
786
 
728
787
  :rtype: DictData
729
788
  """
789
+ output: DictData = output.copy()
730
790
  lc: DictData = output.pop("locals", {})
731
791
  gb: DictData = output.pop("globals", {})
732
- super().set_outputs(
733
- {k: lc[k] for k in self.filter_locals(lc)} | output, to=to
734
- )
792
+ super().set_outputs(lc | output, to=to)
735
793
  to.update({k: gb[k] for k in to if k in gb})
736
794
  return to
737
795
 
@@ -743,9 +801,9 @@ class PyStage(BaseStage):
743
801
  event: Event | None = None,
744
802
  ) -> Result:
745
803
  """Execute the Python statement that pass all globals and input params
746
- to globals argument on ``exec`` build-in function.
804
+ to globals argument on `exec` build-in function.
747
805
 
748
- :param params: A parameter that want to pass before run any statement.
806
+ :param params: (DictData) A parameter data.
749
807
  :param result: (Result) A result object for keeping context and status
750
808
  data.
751
809
  :param event: (Event) An event manager that use to track parent execute
@@ -762,42 +820,68 @@ class PyStage(BaseStage):
762
820
  lc: DictData = {}
763
821
  gb: DictData = (
764
822
  globals()
765
- | params
766
823
  | param2template(self.vars, params, extras=self.extras)
767
824
  | {"result": result}
768
825
  )
769
826
 
770
- # NOTE: Start exec the run statement.
771
827
  result.trace.info(f"[STAGE]: Py-Execute: {self.name}")
772
- # result.trace.warning(
773
- # "[STAGE]: This stage allow use `eval` function, so, please "
774
- # "check your statement be safe before execute."
775
- # )
776
- #
828
+
777
829
  # WARNING: The exec build-in function is very dangerous. So, it
778
830
  # should use the re module to validate exec-string before running.
779
831
  exec(
780
- param2template(dedent(self.run), params, extras=self.extras), gb, lc
832
+ param2template(dedent(self.run), params, extras=self.extras),
833
+ gb,
834
+ lc,
781
835
  )
782
836
 
783
837
  return result.catch(
784
- status=SUCCESS, context={"locals": lc, "globals": gb}
838
+ status=SUCCESS,
839
+ context={
840
+ "locals": {k: lc[k] for k in self.filter_locals(lc)},
841
+ "globals": {
842
+ k: gb[k]
843
+ for k in gb
844
+ if (
845
+ not k.startswith("__")
846
+ and k != "annotations"
847
+ and not ismodule(gb[k])
848
+ and not isclass(gb[k])
849
+ and not isfunction(gb[k])
850
+ )
851
+ },
852
+ },
785
853
  )
786
854
 
855
+ async def axecute(
856
+ self,
857
+ ):
858
+ """Async execution method.
859
+
860
+ References:
861
+ - https://stackoverflow.com/questions/44859165/async-exec-in-python
862
+ """
863
+
787
864
 
788
865
  class CallStage(BaseStage):
789
- """Call executor that call the Python function from registry with tag
790
- decorator function in ``utils`` module and run it with input arguments.
866
+ """Call stage executor that call the Python function from registry with tag
867
+ decorator function in `reusables` module and run it with input arguments.
868
+
869
+ This stage is different with PyStage because the PyStage is just run
870
+ a Python statement with the `exec` function and pass the current locals and
871
+ globals before exec that statement. This stage will import the caller
872
+ function can call it with an input arguments. So, you can create your
873
+ function complexly that you can for your objective to invoked by this stage
874
+ object.
875
+
876
+ This stage is the most powerfull stage of this package for run every
877
+ use-case by a custom requirement that you want by creating the Python
878
+ function and adding it to the caller registry value by importer syntax like
879
+ `module.caller.registry` not path style like `module/caller/registry`.
791
880
 
792
- This stage is different with PyStage because the PyStage is just calling
793
- a Python statement with the ``eval`` and pass that locale before eval that
794
- statement. So, you can create your function complexly that you can for your
795
- objective to invoked by this stage object.
881
+ Warning:
796
882
 
797
- This stage is the usefull stage for run every job by a custom requirement
798
- that you want by creating the Python function and adding it to the task
799
- registry by importer syntax like `module.tasks.registry` not path style like
800
- `module/tasks/registry`.
883
+ The caller registry to get a caller function should importable by the
884
+ current Python execution pointer.
801
885
 
802
886
  Data Validate:
803
887
  >>> stage = {
@@ -809,12 +893,16 @@ class CallStage(BaseStage):
809
893
 
810
894
  uses: str = Field(
811
895
  description=(
812
- "A pointer that want to load function from the call registry."
896
+ "A caller function with registry importer syntax that use to load "
897
+ "function before execute step. The caller registry syntax should "
898
+ "be `<import.part>/<func-name>@<tag-name>`."
813
899
  ),
814
900
  )
815
901
  args: DictData = Field(
816
902
  default_factory=dict,
817
- description="An arguments that want to pass to the call function.",
903
+ description=(
904
+ "An argument parameter that will pass to this caller function."
905
+ ),
818
906
  alias="with",
819
907
  )
820
908
 
@@ -825,24 +913,17 @@ class CallStage(BaseStage):
825
913
  result: Result | None = None,
826
914
  event: Event | None = None,
827
915
  ) -> Result:
828
- """Execute the Call function that already in the call registry.
916
+ """Execute this caller function with its argument parameter.
829
917
 
830
- :raise ValueError: When the necessary arguments of call function do not
831
- set from the input params argument.
832
- :raise TypeError: When the return type of call function does not be
833
- dict type.
834
-
835
- :param params: (DictData) A parameter that want to pass before run any
836
- statement.
837
- :param result: (Result) A result object for keeping context and status
838
- data.
839
- :param event: (Event) An event manager that use to track parent execute
840
- was not force stopped.
918
+ :param params: (DictData) A parameter data.
919
+ :param result: (Result) A Result instance for return context and status.
920
+ :param event: (Event) An Event manager instance that use to cancel this
921
+ execution if it forces stopped by parent execution.
841
922
 
842
923
  :raise ValueError: If necessary arguments does not pass from the `args`
843
924
  field.
844
- :raise TypeError: If the result from the caller function does not by
845
- a `dict` type.
925
+ :raise TypeError: If the result from the caller function does not match
926
+ with a `dict` type.
846
927
 
847
928
  :rtype: Result
848
929
  """
@@ -852,12 +933,15 @@ class CallStage(BaseStage):
852
933
  extras=self.extras,
853
934
  )
854
935
 
855
- has_keyword: bool = False
856
936
  call_func: TagFunc = extract_call(
857
937
  param2template(self.uses, params, extras=self.extras),
858
938
  registries=self.extras.get("registry_caller"),
859
939
  )()
860
940
 
941
+ result.trace.info(
942
+ f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
943
+ )
944
+
861
945
  # VALIDATE: check input task caller parameters that exists before
862
946
  # calling.
863
947
  args: DictData = {"result": result} | param2template(
@@ -865,6 +949,7 @@ class CallStage(BaseStage):
865
949
  )
866
950
  ips = inspect.signature(call_func)
867
951
  necessary_params: list[str] = []
952
+ has_keyword: bool = False
868
953
  for k in ips.parameters:
869
954
  if (
870
955
  v := ips.parameters[k]
@@ -888,10 +973,6 @@ class CallStage(BaseStage):
888
973
  if "result" not in ips.parameters and not has_keyword:
889
974
  args.pop("result")
890
975
 
891
- result.trace.info(
892
- f"[STAGE]: Call-Execute: {call_func.name}@{call_func.tag}"
893
- )
894
-
895
976
  args = self.parse_model_args(call_func, args, result)
896
977
 
897
978
  if inspect.iscoroutinefunction(call_func):
@@ -962,9 +1043,9 @@ class CallStage(BaseStage):
962
1043
 
963
1044
 
964
1045
  class TriggerStage(BaseStage):
965
- """Trigger Workflow execution stage that execute another workflow. This
966
- the core stage that allow you to create the reusable workflow object or
967
- dynamic parameters workflow for common usecase.
1046
+ """Trigger workflow executor stage that run an input trigger Workflow
1047
+ execute method. This is the stage that allow you to create the reusable
1048
+ Workflow template with dynamic parameters.
968
1049
 
969
1050
  Data Validate:
970
1051
  >>> stage = {
@@ -976,12 +1057,13 @@ class TriggerStage(BaseStage):
976
1057
 
977
1058
  trigger: str = Field(
978
1059
  description=(
979
- "A trigger workflow name that should exist on the config path."
1060
+ "A trigger workflow name. This workflow name should exist on the "
1061
+ "config path because it will load by the `load_conf` method."
980
1062
  ),
981
1063
  )
982
1064
  params: DictData = Field(
983
1065
  default_factory=dict,
984
- description="A parameter that want to pass to workflow execution.",
1066
+ description="A parameter that will pass to workflow execution method.",
985
1067
  )
986
1068
 
987
1069
  def execute(
@@ -994,7 +1076,7 @@ class TriggerStage(BaseStage):
994
1076
  """Trigger another workflow execution. It will wait the trigger
995
1077
  workflow running complete before catching its result.
996
1078
 
997
- :param params: A parameter data that want to use in this execution.
1079
+ :param params: (DictData) A parameter data.
998
1080
  :param result: (Result) A result object for keeping context and status
999
1081
  data.
1000
1082
  :param event: (Event) An event manager that use to track parent execute
@@ -1029,14 +1111,18 @@ class TriggerStage(BaseStage):
1029
1111
  err_msg: str | None = (
1030
1112
  f" with:\n{msg}"
1031
1113
  if (msg := rs.context.get("errors", {}).get("message"))
1032
- else ""
1114
+ else "."
1115
+ )
1116
+ raise StageException(
1117
+ f"Trigger workflow return failed status{err_msg}"
1033
1118
  )
1034
- raise StageException(f"Trigger workflow was failed{err_msg}.")
1035
1119
  return rs
1036
1120
 
1037
1121
 
1038
1122
  class ParallelStage(BaseStage): # pragma: no cov
1039
- """Parallel execution stage that execute child stages with parallel.
1123
+ """Parallel stage executor that execute branch stages with multithreading.
1124
+ This stage let you set the fix branches for running substage inside it on
1125
+ multithread pool.
1040
1126
 
1041
1127
  This stage is not the low-level stage model because it runs muti-stages
1042
1128
  in this stage execution.
@@ -1064,52 +1150,50 @@ class ParallelStage(BaseStage): # pragma: no cov
1064
1150
  """
1065
1151
 
1066
1152
  parallel: dict[str, list[Stage]] = Field(
1067
- description="A mapping of parallel branch name and stages.",
1153
+ description="A mapping of branch name and its stages.",
1068
1154
  )
1069
1155
  max_workers: int = Field(
1070
1156
  default=2,
1071
1157
  ge=1,
1072
1158
  lt=20,
1073
1159
  description=(
1074
- "The maximum thread pool worker size for execution parallel."
1160
+ "The maximum multi-thread pool worker size for execution parallel. "
1161
+ "This value should be gather or equal than 1, and less than 20."
1075
1162
  ),
1076
1163
  alias="max-workers",
1077
1164
  )
1078
1165
 
1079
- def execute_task(
1166
+ def execute_branch(
1080
1167
  self,
1081
1168
  branch: str,
1082
1169
  params: DictData,
1083
1170
  result: Result,
1084
1171
  *,
1085
1172
  event: Event | None = None,
1086
- extras: DictData | None = None,
1087
1173
  ) -> DictData:
1088
- """Task execution method for passing a branch to each thread.
1174
+ """Branch execution method for execute all stages of a specific branch
1175
+ ID.
1089
1176
 
1090
- :param branch: A branch ID.
1091
- :param params: A parameter data that want to use in this execution.
1092
- :param result: (Result) A result object for keeping context and status
1093
- data.
1094
- :param event: (Event) An event manager that use to track parent execute
1095
- was not force stopped.
1096
- :param extras: (DictData) An extra parameters that want to override
1097
- config values.
1177
+ :param branch: (str) A branch ID.
1178
+ :param params: (DictData) A parameter data.
1179
+ :param result: (Result) A Result instance for return context and status.
1180
+ :param event: (Event) An Event manager instance that use to cancel this
1181
+ execution if it forces stopped by parent execution.
1098
1182
 
1099
1183
  :rtype: DictData
1100
1184
  """
1101
- result.trace.debug(f"... Execute branch: {branch!r}")
1102
- _params: DictData = copy.deepcopy(params)
1103
- _params.update({"branch": branch})
1104
- context: DictData = {"branch": branch, "stages": {}}
1185
+ result.trace.debug(f"[STAGE]: Execute Branch: {branch!r}")
1186
+ context: DictData = copy.deepcopy(params)
1187
+ context.update({"branch": branch})
1188
+ output: DictData = {"branch": branch, "stages": {}}
1105
1189
  for stage in self.parallel[branch]:
1106
1190
 
1107
- if extras:
1108
- stage.extras = extras
1191
+ if self.extras:
1192
+ stage.extras = self.extras
1109
1193
 
1110
- if stage.is_skipped(params=_params):
1194
+ if stage.is_skipped(params=context):
1111
1195
  result.trace.info(f"... Skip stage: {stage.iden!r}")
1112
- stage.set_outputs(output={"skipped": True}, to=context)
1196
+ stage.set_outputs(output={"skipped": True}, to=output)
1113
1197
  continue
1114
1198
 
1115
1199
  if event and event.is_set():
@@ -1122,7 +1206,7 @@ class ParallelStage(BaseStage): # pragma: no cov
1122
1206
  parallel={
1123
1207
  branch: {
1124
1208
  "branch": branch,
1125
- "stages": filter_func(context.pop("stages", {})),
1209
+ "stages": filter_func(output.pop("stages", {})),
1126
1210
  "errors": StageException(error_msg).to_dict(),
1127
1211
  }
1128
1212
  },
@@ -1130,14 +1214,15 @@ class ParallelStage(BaseStage): # pragma: no cov
1130
1214
 
1131
1215
  try:
1132
1216
  rs: Result = stage.handler_execute(
1133
- params=_params,
1217
+ params=context,
1134
1218
  run_id=result.run_id,
1135
1219
  parent_run_id=result.parent_run_id,
1136
1220
  raise_error=True,
1137
1221
  event=event,
1138
1222
  )
1139
- stage.set_outputs(rs.context, to=context)
1140
- except StageException as e: # pragma: no cov
1223
+ stage.set_outputs(rs.context, to=output)
1224
+ stage.set_outputs(stage.get_outputs(output), to=context)
1225
+ except (StageException, UtilException) as e: # pragma: no cov
1141
1226
  result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1142
1227
  raise StageException(
1143
1228
  f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
@@ -1145,7 +1230,7 @@ class ParallelStage(BaseStage): # pragma: no cov
1145
1230
 
1146
1231
  if rs.status == FAILED:
1147
1232
  error_msg: str = (
1148
- f"Item-Stage was break because it has a sub stage, "
1233
+ f"Branch-Stage was break because it has a sub stage, "
1149
1234
  f"{stage.iden}, failed without raise error."
1150
1235
  )
1151
1236
  return result.catch(
@@ -1153,7 +1238,7 @@ class ParallelStage(BaseStage): # pragma: no cov
1153
1238
  parallel={
1154
1239
  branch: {
1155
1240
  "branch": branch,
1156
- "stages": filter_func(context.pop("stages", {})),
1241
+ "stages": filter_func(output.pop("stages", {})),
1157
1242
  "errors": StageException(error_msg).to_dict(),
1158
1243
  },
1159
1244
  },
@@ -1164,7 +1249,7 @@ class ParallelStage(BaseStage): # pragma: no cov
1164
1249
  parallel={
1165
1250
  branch: {
1166
1251
  "branch": branch,
1167
- "stages": filter_func(context.pop("stages", {})),
1252
+ "stages": filter_func(output.pop("stages", {})),
1168
1253
  },
1169
1254
  },
1170
1255
  )
@@ -1176,14 +1261,12 @@ class ParallelStage(BaseStage): # pragma: no cov
1176
1261
  result: Result | None = None,
1177
1262
  event: Event | None = None,
1178
1263
  ) -> Result:
1179
- """Execute the stages that parallel each branch via multi-threading mode
1180
- or async mode by changing `async_mode` flag.
1264
+ """Execute parallel each branch via multi-threading pool.
1181
1265
 
1182
- :param params: A parameter that want to pass before run any statement.
1183
- :param result: (Result) A result object for keeping context and status
1184
- data.
1185
- :param event: (Event) An event manager that use to track parent execute
1186
- was not force stopped.
1266
+ :param params: (DictData) A parameter data.
1267
+ :param result: (Result) A Result instance for return context and status.
1268
+ :param event: (Event) An Event manager instance that use to cancel this
1269
+ execution if it forces stopped by parent execution.
1187
1270
 
1188
1271
  :rtype: Result
1189
1272
  """
@@ -1207,12 +1290,11 @@ class ParallelStage(BaseStage): # pragma: no cov
1207
1290
 
1208
1291
  futures: list[Future] = (
1209
1292
  executor.submit(
1210
- self.execute_task,
1293
+ self.execute_branch,
1211
1294
  branch=branch,
1212
1295
  params=params,
1213
1296
  result=result,
1214
1297
  event=event,
1215
- extras=self.extras,
1216
1298
  )
1217
1299
  for branch in self.parallel
1218
1300
  )
@@ -1232,11 +1314,11 @@ class ParallelStage(BaseStage): # pragma: no cov
1232
1314
 
1233
1315
 
1234
1316
  class ForEachStage(BaseStage):
1235
- """For-Each execution stage that execute child stages with an item in list
1236
- of item values. This stage is not the low-level stage model because it runs
1237
- muti-stages in this stage execution.
1317
+ """For-Each stage executor that execute all stages with each item in the
1318
+ foreach list.
1238
1319
 
1239
- The concept of this stage use the same logic of the Job execution.
1320
+ This stage is not the low-level stage model because it runs
1321
+ muti-stages in this stage execution.
1240
1322
 
1241
1323
  Data Validate:
1242
1324
  >>> stage = {
@@ -1245,7 +1327,7 @@ class ForEachStage(BaseStage):
1245
1327
  ... "stages": [
1246
1328
  ... {
1247
1329
  ... "name": "Echo stage",
1248
- ... "echo": "Start run with item {{ item }}"
1330
+ ... "echo": "Start run with item ${{ item }}"
1249
1331
  ... },
1250
1332
  ... ],
1251
1333
  ... }
@@ -1253,13 +1335,14 @@ class ForEachStage(BaseStage):
1253
1335
 
1254
1336
  foreach: Union[list[str], list[int], str] = Field(
1255
1337
  description=(
1256
- "A items for passing to each stages via ${{ item }} template."
1338
+ "A items for passing to stages via ${{ item }} template parameter."
1257
1339
  ),
1258
1340
  )
1259
1341
  stages: list[Stage] = Field(
1260
1342
  default_factory=list,
1261
1343
  description=(
1262
- "A list of stage that will run with each item in the foreach field."
1344
+ "A list of stage that will run with each item in the `foreach` "
1345
+ "field."
1263
1346
  ),
1264
1347
  )
1265
1348
  concurrent: int = Field(
@@ -1274,7 +1357,7 @@ class ForEachStage(BaseStage):
1274
1357
 
1275
1358
  def execute_item(
1276
1359
  self,
1277
- item: Union[str, int],
1360
+ item: StrOrInt,
1278
1361
  params: DictData,
1279
1362
  result: Result,
1280
1363
  *,
@@ -1283,29 +1366,27 @@ class ForEachStage(BaseStage):
1283
1366
  """Execute foreach item from list of item.
1284
1367
 
1285
1368
  :param item: (str | int) An item that want to execution.
1286
- :param params: (DictData) A parameter that want to pass to stage
1287
- execution.
1288
- :param result: (Result) A result object for keeping context and status
1289
- data.
1290
- :param event: (Event) An event manager that use to track parent execute
1291
- was not force stopped.
1369
+ :param params: (DictData) A parameter data.
1370
+ :param result: (Result) A Result instance for return context and status.
1371
+ :param event: (Event) An Event manager instance that use to cancel this
1372
+ execution if it forces stopped by parent execution.
1292
1373
 
1293
1374
  :raise StageException: If the stage execution raise errors.
1294
1375
 
1295
1376
  :rtype: Result
1296
1377
  """
1297
- result.trace.debug(f"... Execute item: {item!r}")
1298
- _params: DictData = copy.deepcopy(params)
1299
- _params.update({"item": item})
1300
- context: DictData = {"item": item, "stages": {}}
1378
+ result.trace.debug(f"[STAGE]: Execute Item: {item!r}")
1379
+ context: DictData = copy.deepcopy(params)
1380
+ context.update({"item": item})
1381
+ output: DictData = {"item": item, "stages": {}}
1301
1382
  for stage in self.stages:
1302
1383
 
1303
1384
  if self.extras:
1304
1385
  stage.extras = self.extras
1305
1386
 
1306
- if stage.is_skipped(params=_params):
1387
+ if stage.is_skipped(params=context):
1307
1388
  result.trace.info(f"... Skip stage: {stage.iden!r}")
1308
- stage.set_outputs(output={"skipped": True}, to=context)
1389
+ stage.set_outputs(output={"skipped": True}, to=output)
1309
1390
  continue
1310
1391
 
1311
1392
  if event and event.is_set(): # pragma: no cov
@@ -1318,7 +1399,7 @@ class ForEachStage(BaseStage):
1318
1399
  foreach={
1319
1400
  item: {
1320
1401
  "item": item,
1321
- "stages": filter_func(context.pop("stages", {})),
1402
+ "stages": filter_func(output.pop("stages", {})),
1322
1403
  "errors": StageException(error_msg).to_dict(),
1323
1404
  }
1324
1405
  },
@@ -1326,17 +1407,28 @@ class ForEachStage(BaseStage):
1326
1407
 
1327
1408
  try:
1328
1409
  rs: Result = stage.handler_execute(
1329
- params=_params,
1410
+ params=context,
1330
1411
  run_id=result.run_id,
1331
1412
  parent_run_id=result.parent_run_id,
1332
1413
  raise_error=True,
1333
1414
  event=event,
1334
1415
  )
1335
- stage.set_outputs(rs.context, to=context)
1416
+ stage.set_outputs(rs.context, to=output)
1417
+ stage.set_outputs(stage.get_outputs(output), to=context)
1336
1418
  except (StageException, UtilException) as e:
1337
1419
  result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1420
+ result.catch(
1421
+ status=FAILED,
1422
+ foreach={
1423
+ item: {
1424
+ "item": item,
1425
+ "stages": filter_func(output.pop("stages", {})),
1426
+ "errors": e.to_dict(),
1427
+ },
1428
+ },
1429
+ )
1338
1430
  raise StageException(
1339
- f"Sub-Stage execution error: {e.__class__.__name__}: {e}"
1431
+ f"Sub-Stage raise: {e.__class__.__name__}: {e}"
1340
1432
  ) from None
1341
1433
 
1342
1434
  if rs.status == FAILED:
@@ -1349,18 +1441,17 @@ class ForEachStage(BaseStage):
1349
1441
  foreach={
1350
1442
  item: {
1351
1443
  "item": item,
1352
- "stages": filter_func(context.pop("stages", {})),
1444
+ "stages": filter_func(output.pop("stages", {})),
1353
1445
  "errors": StageException(error_msg).to_dict(),
1354
1446
  },
1355
1447
  },
1356
1448
  )
1357
-
1358
1449
  return result.catch(
1359
1450
  status=SUCCESS,
1360
1451
  foreach={
1361
1452
  item: {
1362
1453
  "item": item,
1363
- "stages": filter_func(context.pop("stages", {})),
1454
+ "stages": filter_func(output.pop("stages", {})),
1364
1455
  },
1365
1456
  },
1366
1457
  )
@@ -1374,11 +1465,10 @@ class ForEachStage(BaseStage):
1374
1465
  ) -> Result:
1375
1466
  """Execute the stages that pass each item form the foreach field.
1376
1467
 
1377
- :param params: A parameter that want to pass before run any statement.
1378
- :param result: (Result) A result object for keeping context and status
1379
- data.
1380
- :param event: (Event) An event manager that use to track parent execute
1381
- was not force stopped.
1468
+ :param params: (DictData) A parameter data.
1469
+ :param result: (Result) A Result instance for return context and status.
1470
+ :param event: (Event) An Event manager instance that use to cancel this
1471
+ execution if it forces stopped by parent execution.
1382
1472
 
1383
1473
  :rtype: Result
1384
1474
  """
@@ -1394,9 +1484,7 @@ class ForEachStage(BaseStage):
1394
1484
  else self.foreach
1395
1485
  )
1396
1486
  if not isinstance(foreach, list):
1397
- raise StageException(
1398
- f"Foreach does not support foreach value: {foreach!r}"
1399
- )
1487
+ raise StageException(f"Does not support foreach: {foreach!r}")
1400
1488
 
1401
1489
  result.trace.info(f"[STAGE]: Foreach-Execute: {foreach!r}.")
1402
1490
  result.catch(status=WAIT, context={"items": foreach, "foreach": {}})
@@ -1428,33 +1516,33 @@ class ForEachStage(BaseStage):
1428
1516
  context: DictData = {}
1429
1517
  status: Status = SUCCESS
1430
1518
 
1431
- done, not_done = wait(
1432
- futures, timeout=1800, return_when=FIRST_EXCEPTION
1433
- )
1434
-
1519
+ done, not_done = wait(futures, return_when=FIRST_EXCEPTION)
1435
1520
  if len(done) != len(futures):
1436
1521
  result.trace.warning(
1437
- "[STAGE]: Set the event for stop running stage."
1522
+ "[STAGE]: Set event for stop pending stage future."
1438
1523
  )
1439
1524
  event.set()
1440
1525
  for future in not_done:
1441
1526
  future.cancel()
1442
1527
 
1528
+ nd: str = f", item not run: {not_done}" if not_done else ""
1529
+ result.trace.debug(f"... Foreach set Fail-Fast{nd}")
1530
+
1443
1531
  for future in done:
1444
1532
  try:
1445
1533
  future.result()
1446
- except StageException as e:
1534
+ except (StageException, UtilException) as e:
1447
1535
  status = FAILED
1448
1536
  result.trace.error(
1449
- f"[STAGE]: {e.__class__.__name__}:\n\t{e}"
1537
+ f"[STAGE]: {e.__class__.__name__}:{NEWLINE}{e}"
1450
1538
  )
1451
1539
  context.update({"errors": e.to_dict()})
1452
-
1453
1540
  return result.catch(status=status, context=context)
1454
1541
 
1455
1542
 
1456
- class UntilStage(BaseStage): # pragma: no cov
1457
- """Until execution stage.
1543
+ class UntilStage(BaseStage):
1544
+ """Until stage executor that will run stages in each loop until it valid
1545
+ with stop loop condition.
1458
1546
 
1459
1547
  Data Validate:
1460
1548
  >>> stage = {
@@ -1476,19 +1564,21 @@ class UntilStage(BaseStage): # pragma: no cov
1476
1564
  "An initial value that can be any value in str, int, or bool type."
1477
1565
  ),
1478
1566
  )
1479
- until: str = Field(description="A until condition.")
1567
+ until: str = Field(description="A until condition for stop the while loop.")
1480
1568
  stages: list[Stage] = Field(
1481
1569
  default_factory=list,
1482
1570
  description=(
1483
- "A list of stage that will run with each item until condition "
1484
- "correct."
1571
+ "A list of stage that will run with each item in until loop."
1485
1572
  ),
1486
1573
  )
1487
1574
  max_loop: int = Field(
1488
1575
  default=10,
1489
1576
  ge=1,
1490
1577
  lt=100,
1491
- description="The maximum value of loop for this until stage.",
1578
+ description=(
1579
+ "The maximum value of loop for this until stage. This value should "
1580
+ "be gather or equal than 1, and less than 100."
1581
+ ),
1492
1582
  alias="max-loop",
1493
1583
  )
1494
1584
 
@@ -1500,33 +1590,31 @@ class UntilStage(BaseStage): # pragma: no cov
1500
1590
  result: Result,
1501
1591
  event: Event | None = None,
1502
1592
  ) -> tuple[Result, T]:
1503
- """Execute until item set item by some stage or by default loop
1593
+ """Execute loop item that was set from some stage or set by default loop
1504
1594
  variable.
1505
1595
 
1506
1596
  :param item: (T) An item that want to execution.
1507
1597
  :param loop: (int) A number of loop.
1508
- :param params: (DictData) A parameter that want to pass to stage
1509
- execution.
1510
- :param result: (Result) A result object for keeping context and status
1511
- data.
1512
- :param event: (Event) An event manager that use to track parent execute
1513
- was not force stopped.
1598
+ :param params: (DictData) A parameter data.
1599
+ :param result: (Result) A Result instance for return context and status.
1600
+ :param event: (Event) An Event manager instance that use to cancel this
1601
+ execution if it forces stopped by parent execution.
1514
1602
 
1515
1603
  :rtype: tuple[Result, T]
1516
1604
  """
1517
1605
  result.trace.debug(f"... Execute until item: {item!r}")
1518
- _params: DictData = copy.deepcopy(params)
1519
- _params.update({"item": item})
1520
- context: DictData = {"loop": loop, "item": item, "stages": {}}
1606
+ context: DictData = copy.deepcopy(params)
1607
+ context.update({"item": item})
1608
+ output: DictData = {"loop": loop, "item": item, "stages": {}}
1521
1609
  next_item: T = None
1522
1610
  for stage in self.stages:
1523
1611
 
1524
1612
  if self.extras:
1525
1613
  stage.extras = self.extras
1526
1614
 
1527
- if stage.is_skipped(params=_params):
1615
+ if stage.is_skipped(params=context):
1528
1616
  result.trace.info(f"... Skip stage: {stage.iden!r}")
1529
- stage.set_outputs(output={"skipped": True}, to=context)
1617
+ stage.set_outputs(output={"skipped": True}, to=output)
1530
1618
  continue
1531
1619
 
1532
1620
  if event and event.is_set():
@@ -1541,9 +1629,7 @@ class UntilStage(BaseStage): # pragma: no cov
1541
1629
  loop: {
1542
1630
  "loop": loop,
1543
1631
  "item": item,
1544
- "stages": filter_func(
1545
- context.pop("stages", {})
1546
- ),
1632
+ "stages": filter_func(output.pop("stages", {})),
1547
1633
  "errors": StageException(error_msg).to_dict(),
1548
1634
  }
1549
1635
  },
@@ -1553,17 +1639,18 @@ class UntilStage(BaseStage): # pragma: no cov
1553
1639
 
1554
1640
  try:
1555
1641
  rs: Result = stage.handler_execute(
1556
- params=_params,
1642
+ params=context,
1557
1643
  run_id=result.run_id,
1558
1644
  parent_run_id=result.parent_run_id,
1559
1645
  raise_error=True,
1560
1646
  event=event,
1561
1647
  )
1562
- stage.set_outputs(rs.context, to=context)
1563
- if "item" in (
1564
- outputs := stage.get_outputs(context).get("outputs", {})
1565
- ):
1566
- next_item = outputs["item"]
1648
+ stage.set_outputs(rs.context, to=output)
1649
+
1650
+ if "item" in (_output := stage.get_outputs(output)):
1651
+ next_item = _output["item"]
1652
+
1653
+ stage.set_outputs(_output, to=context)
1567
1654
  except (StageException, UtilException) as e:
1568
1655
  result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1569
1656
  raise StageException(
@@ -1577,7 +1664,7 @@ class UntilStage(BaseStage): # pragma: no cov
1577
1664
  loop: {
1578
1665
  "loop": loop,
1579
1666
  "item": item,
1580
- "stages": filter_func(context.pop("stages", {})),
1667
+ "stages": filter_func(output.pop("stages", {})),
1581
1668
  }
1582
1669
  },
1583
1670
  ),
@@ -1594,11 +1681,10 @@ class UntilStage(BaseStage): # pragma: no cov
1594
1681
  """Execute the stages that pass item from until condition field and
1595
1682
  setter step.
1596
1683
 
1597
- :param params: A parameter that want to pass before run any statement.
1598
- :param result: (Result) A result object for keeping context and status
1599
- data.
1600
- :param event: (Event) An event manager that use to track parent execute
1601
- was not force stopped.
1684
+ :param params: (DictData) A parameter data.
1685
+ :param result: (Result) A Result instance for return context and status.
1686
+ :param event: (Event) An Event manager instance that use to cancel this
1687
+ execution if it forces stopped by parent execution.
1602
1688
 
1603
1689
  :rtype: Result
1604
1690
  """
@@ -1671,12 +1757,14 @@ class UntilStage(BaseStage): # pragma: no cov
1671
1757
  class Match(BaseModel):
1672
1758
  """Match model for the Case Stage."""
1673
1759
 
1674
- case: Union[str, int] = Field(description="A match case.")
1675
- stage: Stage = Field(description="A stage to execution for this case.")
1760
+ case: StrOrInt = Field(description="A match case.")
1761
+ stages: list[Stage] = Field(
1762
+ description="A list of stage to execution for this case."
1763
+ )
1676
1764
 
1677
1765
 
1678
1766
  class CaseStage(BaseStage):
1679
- """Case execution stage.
1767
+ """Case stage executor that execute all stages if the condition was matched.
1680
1768
 
1681
1769
  Data Validate:
1682
1770
  >>> stage = {
@@ -1685,24 +1773,21 @@ class CaseStage(BaseStage):
1685
1773
  ... "match": [
1686
1774
  ... {
1687
1775
  ... "case": "1",
1688
- ... "stage": {
1689
- ... "name": "Stage case 1",
1690
- ... "eche": "Hello case 1",
1691
- ... },
1692
- ... },
1693
- ... {
1694
- ... "case": "2",
1695
- ... "stage": {
1696
- ... "name": "Stage case 2",
1697
- ... "eche": "Hello case 2",
1698
- ... },
1776
+ ... "stages": [
1777
+ ... {
1778
+ ... "name": "Stage case 1",
1779
+ ... "eche": "Hello case 1",
1780
+ ... },
1781
+ ... ],
1699
1782
  ... },
1700
1783
  ... {
1701
1784
  ... "case": "_",
1702
- ... "stage": {
1703
- ... "name": "Stage else",
1704
- ... "eche": "Hello case else",
1705
- ... },
1785
+ ... "stages": [
1786
+ ... {
1787
+ ... "name": "Stage else",
1788
+ ... "eche": "Hello case else",
1789
+ ... },
1790
+ ... ],
1706
1791
  ... },
1707
1792
  ... ],
1708
1793
  ... }
@@ -1722,6 +1807,96 @@ class CaseStage(BaseStage):
1722
1807
  alias="skip-not-match",
1723
1808
  )
1724
1809
 
1810
+ def execute_case(
1811
+ self,
1812
+ case: str,
1813
+ stages: list[Stage],
1814
+ params: DictData,
1815
+ result: Result,
1816
+ *,
1817
+ event: Event | None = None,
1818
+ ) -> Result:
1819
+ """Execute case.
1820
+
1821
+ :param case: (str) A case that want to execution.
1822
+ :param stages: (list[Stage]) A list of stage.
1823
+ :param params: (DictData) A parameter data.
1824
+ :param result: (Result) A Result instance for return context and status.
1825
+ :param event: (Event) An Event manager instance that use to cancel this
1826
+ execution if it forces stopped by parent execution.
1827
+
1828
+ :rtype: Result
1829
+ """
1830
+ context: DictData = copy.deepcopy(params)
1831
+ context.update({"case": case})
1832
+ output: DictData = {"case": case, "stages": {}}
1833
+
1834
+ for stage in stages:
1835
+
1836
+ if self.extras:
1837
+ stage.extras = self.extras
1838
+
1839
+ if stage.is_skipped(params=context):
1840
+ result.trace.info(f"... Skip stage: {stage.iden!r}")
1841
+ stage.set_outputs(output={"skipped": True}, to=output)
1842
+ continue
1843
+
1844
+ if event and event.is_set(): # pragma: no cov
1845
+ error_msg: str = (
1846
+ "Case-Stage was canceled from event that had set before "
1847
+ "stage case execution."
1848
+ )
1849
+ return result.catch(
1850
+ status=CANCEL,
1851
+ context={
1852
+ "case": case,
1853
+ "stages": filter_func(output.pop("stages", {})),
1854
+ "errors": StageException(error_msg).to_dict(),
1855
+ },
1856
+ )
1857
+
1858
+ try:
1859
+ rs: Result = stage.handler_execute(
1860
+ params=context,
1861
+ run_id=result.run_id,
1862
+ parent_run_id=result.parent_run_id,
1863
+ raise_error=True,
1864
+ event=event,
1865
+ )
1866
+ stage.set_outputs(rs.context, to=output)
1867
+ stage.set_outputs(stage.get_outputs(output), to=context)
1868
+ except (StageException, UtilException) as e: # pragma: no cov
1869
+ result.trace.error(f"[STAGE]: {e.__class__.__name__}: {e}")
1870
+ return result.catch(
1871
+ status=FAILED,
1872
+ context={
1873
+ "case": case,
1874
+ "stages": filter_func(output.pop("stages", {})),
1875
+ "errors": e.to_dict(),
1876
+ },
1877
+ )
1878
+
1879
+ if rs.status == FAILED:
1880
+ error_msg: str = (
1881
+ f"Case-Stage was break because it has a sub stage, "
1882
+ f"{stage.iden}, failed without raise error."
1883
+ )
1884
+ return result.catch(
1885
+ status=FAILED,
1886
+ context={
1887
+ "case": case,
1888
+ "stages": filter_func(output.pop("stages", {})),
1889
+ "errors": StageException(error_msg).to_dict(),
1890
+ },
1891
+ )
1892
+ return result.catch(
1893
+ status=SUCCESS,
1894
+ context={
1895
+ "case": case,
1896
+ "stages": filter_func(output.pop("stages", {})),
1897
+ },
1898
+ )
1899
+
1725
1900
  def execute(
1726
1901
  self,
1727
1902
  params: DictData,
@@ -1731,11 +1906,10 @@ class CaseStage(BaseStage):
1731
1906
  ) -> Result:
1732
1907
  """Execute case-match condition that pass to the case field.
1733
1908
 
1734
- :param params: A parameter that want to pass before run any statement.
1735
- :param result: (Result) A result object for keeping context and status
1736
- data.
1737
- :param event: (Event) An event manager that use to track parent execute
1738
- was not force stopped.
1909
+ :param params: (DictData) A parameter data.
1910
+ :param result: (Result) A Result instance for return context and status.
1911
+ :param event: (Event) An Event manager instance that use to cancel this
1912
+ execution if it forces stopped by parent execution.
1739
1913
 
1740
1914
  :rtype: Result
1741
1915
  """
@@ -1751,17 +1925,17 @@ class CaseStage(BaseStage):
1751
1925
 
1752
1926
  result.trace.info(f"[STAGE]: Case-Execute: {_case!r}.")
1753
1927
  _else: Optional[Match] = None
1754
- stage: Optional[Stage] = None
1928
+ stages: Optional[list[Stage]] = None
1755
1929
  for match in self.match:
1756
1930
  if (c := match.case) == "_":
1757
1931
  _else: Match = match
1758
1932
  continue
1759
1933
 
1760
1934
  _condition: str = param2template(c, params, extras=self.extras)
1761
- if stage is None and _case == _condition:
1762
- stage: Stage = match.stage
1935
+ if stages is None and _case == _condition:
1936
+ stages: list[Stage] = match.stages
1763
1937
 
1764
- if stage is None:
1938
+ if stages is None:
1765
1939
  if _else is None:
1766
1940
  if not self.skip_not_match:
1767
1941
  raise StageException(
@@ -1779,10 +1953,8 @@ class CaseStage(BaseStage):
1779
1953
  status=CANCEL,
1780
1954
  context={"errors": StageException(error_msg).to_dict()},
1781
1955
  )
1782
- stage: Stage = _else.stage
1783
-
1784
- if self.extras:
1785
- stage.extras = self.extras
1956
+ _case: str = "_"
1957
+ stages: list[Stage] = _else.stages
1786
1958
 
1787
1959
  if event and event.is_set(): # pragma: no cov
1788
1960
  return result.catch(
@@ -1795,19 +1967,9 @@ class CaseStage(BaseStage):
1795
1967
  },
1796
1968
  )
1797
1969
 
1798
- try:
1799
- return result.catch(
1800
- status=SUCCESS,
1801
- context=stage.handler_execute(
1802
- params=params,
1803
- run_id=result.run_id,
1804
- parent_run_id=result.parent_run_id,
1805
- event=event,
1806
- ).context,
1807
- )
1808
- except StageException as e: # pragma: no cov
1809
- result.trace.error(f"[STAGE]: {e.__class__.__name__}:" f"\n\t{e}")
1810
- return result.catch(status=FAILED, context={"errors": e.to_dict()})
1970
+ return self.execute_case(
1971
+ case=_case, stages=stages, params=params, result=result, event=event
1972
+ )
1811
1973
 
1812
1974
 
1813
1975
  class RaiseStage(BaseStage): # pragma: no cov
@@ -1824,7 +1986,7 @@ class RaiseStage(BaseStage): # pragma: no cov
1824
1986
 
1825
1987
  message: str = Field(
1826
1988
  description=(
1827
- "An error message that want to raise with StageException class"
1989
+ "An error message that want to raise with `StageException` class"
1828
1990
  ),
1829
1991
  alias="raise",
1830
1992
  )
@@ -1838,11 +2000,10 @@ class RaiseStage(BaseStage): # pragma: no cov
1838
2000
  ) -> Result:
1839
2001
  """Raise the StageException object with the message field execution.
1840
2002
 
1841
- :param params: A parameter that want to pass before run any statement.
1842
- :param result: (Result) A result object for keeping context and status
1843
- data.
1844
- :param event: (Event) An event manager that use to track parent execute
1845
- was not force stopped.
2003
+ :param params: (DictData) A parameter data.
2004
+ :param result: (Result) A Result instance for return context and status.
2005
+ :param event: (Event) An Event manager instance that use to cancel this
2006
+ execution if it forces stopped by parent execution.
1846
2007
  """
1847
2008
  if result is None: # pragma: no cov
1848
2009
  result: Result = Result(
@@ -1854,27 +2015,13 @@ class RaiseStage(BaseStage): # pragma: no cov
1854
2015
  raise StageException(message)
1855
2016
 
1856
2017
 
1857
- # TODO: Not implement this stages yet
1858
- class HookStage(BaseStage): # pragma: no cov
1859
- """Hook stage execution."""
1860
-
1861
- hook: str
1862
- args: DictData = Field(default_factory=dict)
1863
- callback: str
1864
-
1865
- def execute(
1866
- self,
1867
- params: DictData,
1868
- *,
1869
- result: Result | None = None,
1870
- event: Event | None = None,
1871
- ) -> Result:
1872
- raise NotImplementedError("Hook Stage does not implement yet.")
1873
-
1874
-
1875
- # TODO: Not implement this stages yet
1876
2018
  class DockerStage(BaseStage): # pragma: no cov
1877
- """Docker container stage execution.
2019
+ """Docker container stage execution that will pull the specific Docker image
2020
+ with custom authentication and run this image by passing environment
2021
+ variables and mounting local volume to this Docker container.
2022
+
2023
+ The volume path that mount to this Docker container will limit. That is
2024
+ this stage does not allow you to mount any path to this container.
1878
2025
 
1879
2026
  Data Validate:
1880
2027
  >>> stage = {
@@ -1882,10 +2029,7 @@ class DockerStage(BaseStage): # pragma: no cov
1882
2029
  ... "image": "image-name.pkg.com",
1883
2030
  ... "env": {
1884
2031
  ... "ENV": "dev",
1885
- ... "DEBUG": "true",
1886
- ... },
1887
- ... "volume": {
1888
- ... "secrets": "/secrets",
2032
+ ... "SECRET": "${SPECIFIC_SECRET}",
1889
2033
  ... },
1890
2034
  ... "auth": {
1891
2035
  ... "username": "__json_key",
@@ -1898,8 +2042,16 @@ class DockerStage(BaseStage): # pragma: no cov
1898
2042
  description="A Docker image url with tag that want to run.",
1899
2043
  )
1900
2044
  tag: str = Field(default="latest", description="An Docker image tag.")
1901
- env: DictData = Field(default_factory=dict)
1902
- volume: DictData = Field(default_factory=dict)
2045
+ env: DictData = Field(
2046
+ default_factory=dict,
2047
+ description=(
2048
+ "An environment variable that want pass to Docker container.",
2049
+ ),
2050
+ )
2051
+ volume: DictData = Field(
2052
+ default_factory=dict,
2053
+ description="A mapping of local and target mounting path.",
2054
+ )
1903
2055
  auth: DictData = Field(
1904
2056
  default_factory=dict,
1905
2057
  description=(
@@ -1911,7 +2063,17 @@ class DockerStage(BaseStage): # pragma: no cov
1911
2063
  self,
1912
2064
  params: DictData,
1913
2065
  result: Result,
1914
- ):
2066
+ event: Event | None = None,
2067
+ ) -> Result:
2068
+ """Execute Docker container task.
2069
+
2070
+ :param params: (DictData) A parameter data.
2071
+ :param result: (Result) A Result instance for return context and status.
2072
+ :param event: (Event) An Event manager instance that use to cancel this
2073
+ execution if it forces stopped by parent execution.
2074
+
2075
+ :rtype: Result
2076
+ """
1915
2077
  from docker import DockerClient
1916
2078
  from docker.errors import ContainerError
1917
2079
 
@@ -1929,6 +2091,16 @@ class DockerStage(BaseStage): # pragma: no cov
1929
2091
  for line in resp:
1930
2092
  result.trace.info(f"... {line}")
1931
2093
 
2094
+ if event and event.is_set():
2095
+ error_msg: str = (
2096
+ "Docker-Stage was canceled from event that had set before "
2097
+ "run the Docker container."
2098
+ )
2099
+ return result.catch(
2100
+ status=CANCEL,
2101
+ context={"errors": StageException(error_msg).to_dict()},
2102
+ )
2103
+
1932
2104
  unique_image_name: str = f"{self.image}_{datetime.now():%Y%m%d%H%M%S%f}"
1933
2105
  container = client.containers.run(
1934
2106
  image=f"{self.image}:{self.tag}",
@@ -1967,6 +2139,13 @@ class DockerStage(BaseStage): # pragma: no cov
1967
2139
  f"{self.image}:{self.tag}",
1968
2140
  out,
1969
2141
  )
2142
+ output_file: Path = Path(f".docker.{result.run_id}.logs/outputs.json")
2143
+ if not output_file.exists():
2144
+ return result.catch(status=SUCCESS)
2145
+
2146
+ with output_file.open(mode="rt") as f:
2147
+ data = json.load(f)
2148
+ return result.catch(status=SUCCESS, context=data)
1970
2149
 
1971
2150
  def execute(
1972
2151
  self,
@@ -1975,13 +2154,34 @@ class DockerStage(BaseStage): # pragma: no cov
1975
2154
  result: Result | None = None,
1976
2155
  event: Event | None = None,
1977
2156
  ) -> Result:
2157
+ """Execute the Docker image via Python API.
2158
+
2159
+ :param params: (DictData) A parameter data.
2160
+ :param result: (Result) A Result instance for return context and status.
2161
+ :param event: (Event) An Event manager instance that use to cancel this
2162
+ execution if it forces stopped by parent execution.
2163
+
2164
+ :rtype: Result
2165
+ """
2166
+ if result is None: # pragma: no cov
2167
+ result: Result = Result(
2168
+ run_id=gen_id(self.name + (self.id or ""), unique=True)
2169
+ )
2170
+
2171
+ result.trace.info(f"[STAGE]: Docker-Execute: {self.image}:{self.tag}")
2172
+
1978
2173
  raise NotImplementedError("Docker Stage does not implement yet.")
1979
2174
 
1980
2175
 
1981
- # TODO: Not implement this stages yet
1982
2176
  class VirtualPyStage(PyStage): # pragma: no cov
1983
- """Python Virtual Environment stage execution."""
2177
+ """Virtual Python stage executor that run Python statement on the dependent
2178
+ Python virtual environment via the `uv` package.
2179
+ """
1984
2180
 
2181
+ version: str = Field(
2182
+ default="3.9",
2183
+ description="A Python version that want to run.",
2184
+ )
1985
2185
  deps: list[str] = Field(
1986
2186
  description=(
1987
2187
  "list of Python dependency that want to install before execution "
@@ -1989,7 +2189,54 @@ class VirtualPyStage(PyStage): # pragma: no cov
1989
2189
  ),
1990
2190
  )
1991
2191
 
1992
- def create_py_file(self, py: str, run_id: str | None): ...
2192
+ @contextlib.contextmanager
2193
+ def create_py_file(
2194
+ self,
2195
+ py: str,
2196
+ values: DictData,
2197
+ deps: list[str],
2198
+ run_id: str | None = None,
2199
+ ) -> Iterator[str]:
2200
+ """Create the .py file with an input Python string statement.
2201
+
2202
+ :param py: A Python string statement.
2203
+ :param values: A variable that want to set before running this
2204
+ :param deps: An additional Python dependencies that want install before
2205
+ run this python stage.
2206
+ :param run_id: (str | None) A running ID of this stage execution.
2207
+ """
2208
+ run_id: str = run_id or uuid.uuid4()
2209
+ f_name: str = f"{run_id}.py"
2210
+ with open(f"./{f_name}", mode="w", newline="\n") as f:
2211
+ # NOTE: Create variable mapping that write before running statement.
2212
+ vars_str: str = "\n ".join(
2213
+ f"{var} = {value!r}" for var, value in values.items()
2214
+ )
2215
+
2216
+ # NOTE: uv supports PEP 723 — inline TOML metadata.
2217
+ f.write(
2218
+ dedent(
2219
+ f"""
2220
+ # /// script
2221
+ # dependencies = [{', '.join(f'"{dep}"' for dep in deps)}]
2222
+ # ///
2223
+ {vars_str}
2224
+ """.strip(
2225
+ "\n"
2226
+ )
2227
+ )
2228
+ )
2229
+
2230
+ # NOTE: make sure that py script file does not have `\r` char.
2231
+ f.write("\n" + py.replace("\r\n", "\n"))
2232
+
2233
+ # NOTE: Make this .py file able to executable.
2234
+ make_exec(f"./{f_name}")
2235
+
2236
+ yield f_name
2237
+
2238
+ # Note: Remove .py file that use to run Python.
2239
+ Path(f"./{f_name}").unlink()
1993
2240
 
1994
2241
  def execute(
1995
2242
  self,
@@ -2001,15 +2248,13 @@ class VirtualPyStage(PyStage): # pragma: no cov
2001
2248
  """Execute the Python statement via Python virtual environment.
2002
2249
 
2003
2250
  Steps:
2004
- - Create python file.
2005
- - Create `.venv` and install necessary Python deps.
2006
- - Execution python file with uv and specific `.venv`.
2251
+ - Create python file with the `uv` syntax.
2252
+ - Execution python file with `uv run` via Python subprocess module.
2007
2253
 
2008
- :param params: A parameter that want to pass before run any statement.
2009
- :param result: (Result) A result object for keeping context and status
2010
- data.
2011
- :param event: (Event) An event manager that use to track parent execute
2012
- was not force stopped.
2254
+ :param params: (DictData) A parameter data.
2255
+ :param result: (Result) A Result instance for return context and status.
2256
+ :param event: (Event) An Event manager instance that use to cancel this
2257
+ execution if it forces stopped by parent execution.
2013
2258
 
2014
2259
  :rtype: Result
2015
2260
  """
@@ -2019,13 +2264,45 @@ class VirtualPyStage(PyStage): # pragma: no cov
2019
2264
  )
2020
2265
 
2021
2266
  result.trace.info(f"[STAGE]: Py-Virtual-Execute: {self.name}")
2022
- raise NotImplementedError(
2023
- "Python Virtual Stage does not implement yet."
2267
+ run: str = param2template(dedent(self.run), params, extras=self.extras)
2268
+ with self.create_py_file(
2269
+ py=run,
2270
+ values=param2template(self.vars, params, extras=self.extras),
2271
+ deps=param2template(self.deps, params, extras=self.extras),
2272
+ run_id=result.run_id,
2273
+ ) as py:
2274
+ result.trace.debug(f"... Create `{py}` file.")
2275
+ rs: CompletedProcess = subprocess.run(
2276
+ ["uv", "run", py, "--no-cache"],
2277
+ # ["uv", "run", "--python", "3.9", py],
2278
+ shell=False,
2279
+ capture_output=True,
2280
+ text=True,
2281
+ )
2282
+
2283
+ if rs.returncode > 0:
2284
+ # NOTE: Prepare stderr message that returning from subprocess.
2285
+ e: str = (
2286
+ rs.stderr.encode("utf-8").decode("utf-16")
2287
+ if "\\x00" in rs.stderr
2288
+ else rs.stderr
2289
+ ).removesuffix("\n")
2290
+ raise StageException(
2291
+ f"Subprocess: {e}\nRunning Statement:\n---\n"
2292
+ f"```python\n{run}\n```"
2293
+ )
2294
+ return result.catch(
2295
+ status=SUCCESS,
2296
+ context={
2297
+ "return_code": rs.returncode,
2298
+ "stdout": None if (out := rs.stdout.strip("\n")) == "" else out,
2299
+ "stderr": None if (out := rs.stderr.strip("\n")) == "" else out,
2300
+ },
2024
2301
  )
2025
2302
 
2026
2303
 
2027
2304
  # NOTE:
2028
- # An order of parsing stage model on the Job model with ``stages`` field.
2305
+ # An order of parsing stage model on the Job model with `stages` field.
2029
2306
  # From the current build-in stages, they do not have stage that have the same
2030
2307
  # fields that because of parsing on the Job's stages key.
2031
2308
  #
@@ -2034,7 +2311,6 @@ Stage = Annotated[
2034
2311
  DockerStage,
2035
2312
  BashStage,
2036
2313
  CallStage,
2037
- HookStage,
2038
2314
  TriggerStage,
2039
2315
  ForEachStage,
2040
2316
  UntilStage,