ddeutil-workflow 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

ddeutil/workflow/stage.py

@@ -12,16 +12,19 @@ can tracking logs.
 handle stage error on this stage model. I think stage model should have a lot of
 usecase and it does not worry when I want to create a new one.
 
-    Execution --> Ok    --> Result with 0
-              --> Error --> Raise StageException
+    Execution   --> Ok    --> Result with 0
+                --> Error --> Raise StageException
+
+    On the context I/O that pass to stage object at execute process. The execute
+method receive `{"params": {...}}` for mapping to template.
 """
 from __future__ import annotations
 
 import contextlib
 import inspect
-import os
 import subprocess
 import sys
+import time
 import uuid
 from abc import ABC, abstractmethod
 from collections.abc import Iterator
@@ -38,12 +41,12 @@ try:
 except ImportError:
     from typing_extensions import ParamSpec
 
-from ddeutil.core import str2bool
 from pydantic import BaseModel, Field
 from pydantic.functional_validators import model_validator
 from typing_extensions import Self
 
 from .__types import DictData, DictStr, Re, TupleStr
+from .conf import config
 from .exceptions import StageException
 from .log import get_logger
 from .utils import (
@@ -58,6 +61,8 @@ from .utils import (
 )
 
 P = ParamSpec("P")
+ReturnResult = Callable[P, Result]
+DecoratorResult = Callable[[ReturnResult], ReturnResult]
 logger = get_logger("ddeutil.workflow")
 
 
@@ -69,13 +74,13 @@ __all__: TupleStr = (
     "HookStage",
     "TriggerStage",
     "Stage",
-    "HookSearch",
+    "HookSearchData",
     "extract_hook",
     "handler_result",
 )
 
 
-def handler_result(message: str | None = None) -> Callable[P, Result]:
+def handler_result(message: str | None = None) -> DecoratorResult:
     """Decorator function for handler result from the stage execution. This
     function should to use with execution method only.
 
@@ -87,11 +92,21 @@ def handler_result(message: str | None = None) -> Callable[P, Result]:
                     --> Error   --> Raise StageException
                                 --> Result with 1 (if env var was set)
 
+        On the last step, it will set the running ID on a return result object
+    from current stage ID before release the final result.
+
     :param message: A message that want to add at prefix of exception statement.
+    :rtype: Callable[P, Result]
     """
+    # NOTE: The prefix message string that want to add on the first exception
+    #   message dialog.
+    #
+    #       ... ValueError: {message}
+    #       ...     raise value error from the stage execution process.
+    #
     message: str = message or ""
 
-    def decorator(func: Callable[P, Result]) -> Callable[P, Result]:
+    def decorator(func: ReturnResult) -> ReturnResult:
 
         @wraps(func)
         def wrapped(self: Stage, *args, **kwargs):
@@ -103,9 +118,7 @@ def handler_result(message: str | None = None) -> Callable[P, Result]:
                 logger.error(
                     f"({self.run_id}) [STAGE]: {err.__class__.__name__}: {err}"
                 )
-                if str2bool(
-                    os.getenv("WORKFLOW_CORE_STAGE_RAISE_ERROR", "true")
-                ):
+                if config.stage_raise_error:
                     # NOTE: If error that raise from stage execution course by
                     #   itself, it will return that error with previous
                     #   dependency.
@@ -119,14 +132,14 @@ def handler_result(message: str | None = None) -> Callable[P, Result]:
                     ) from None
 
                 # NOTE: Catching exception error object to result with
-                #   error_message key.
-                rs: Result = Result(
+                #   error_message and error keys.
+                return Result(
                     status=1,
                     context={
+                        "error": err,
                         "error_message": f"{err.__class__.__name__}: {err}",
                     },
-                )
-                return rs.set_run_id(self.run_id)
+                ).set_run_id(self.run_id)
 
         return wrapped
 
@@ -162,10 +175,12 @@ class BaseStage(BaseModel, ABC):
     )
 
     @model_validator(mode="after")
-    def __prepare_running_id(self):
+    def __prepare_running_id(self) -> Self:
         """Prepare stage running ID that use default value of field and this
         method will validate name and id fields should not contain any template
         parameter (exclude matrix template).
+
+        :rtype: Self
         """
         if self.run_id is None:
             self.run_id = gen_id(self.name + (self.id or ""), unique=True)
@@ -199,16 +214,28 @@ class BaseStage(BaseModel, ABC):
         raise NotImplementedError("Stage should implement ``execute`` method.")
 
     def set_outputs(self, output: DictData, to: DictData) -> DictData:
-        """Set an outputs from execution process to an input params.
+        """Set an outputs from execution process to the receive context. The
+        result from execution will pass to value of ``outputs`` key.
+
+            For example of setting output method, If you receive execute output
+        and want to set on the `to` like;
+
+            ... (i)   output: {'foo': bar}
+            ... (ii)  to: {}
+
+        The result of the `to` variable will be;
+
+            ... (iii) to: {
+                            'stages': {
+                                '<stage-id>': {'outputs': {'foo': 'bar'}}
+                            }
+                        }
 
         :param output: A output data that want to extract to an output key.
         :param to: A context data that want to add output result.
         :rtype: DictData
         """
-        if not (
-            self.id
-            or str2bool(os.getenv("WORKFLOW_CORE_STAGE_DEFAULT_ID", "false"))
-        ):
+        if not (self.id or config.stage_default_id):
             logger.debug(
                 f"({self.run_id}) [STAGE]: Output does not set because this "
                 f"stage does not set ID or default stage ID config flag not be "
@@ -220,16 +247,15 @@ class BaseStage(BaseModel, ABC):
         if "stages" not in to:
             to["stages"] = {}
 
-        if self.id:
-            _id: str = param2template(self.id, params=to)
-        else:
-            # NOTE: If the stage ID did not set, it will use its name instead.
-            _id: str = gen_id(param2template(self.name, params=to))
-
-        # NOTE: Set the output to that stage generated ID.
-        logger.debug(
-            f"({self.run_id}) [STAGE]: Set output complete with stage ID: {_id}"
+        # NOTE: If the stage ID did not set, it will use its name instead.
+        _id: str = (
+            param2template(self.id, params=to)
+            if self.id
+            else gen_id(param2template(self.name, params=to))
         )
+
+        # NOTE: Set the output to that stage generated ID with ``outputs`` key.
+        logger.debug(f"({self.run_id}) [STAGE]: Set outputs on: {_id}")
         to["stages"][_id] = {"outputs": output}
         return to
 
@@ -240,10 +266,10 @@ class BaseStage(BaseModel, ABC):
         :param params: A parameters that want to pass to condition template.
         :rtype: bool
         """
-        params: DictData = params or {}
         if self.condition is None:
             return False
 
+        params: DictData = {} if params is None else params
         _g: DictData = globals() | params
         try:
             rs: bool = eval(param2template(self.condition, params), _g, {})
@@ -270,6 +296,10 @@ class EmptyStage(BaseStage):
         default=None,
         description="A string statement that want to logging",
     )
+    sleep: float = Field(
+        default=0,
+        description="A second value to sleep before finish execution",
+    )
 
     def execute(self, params: DictData) -> Result:
         """Execution method for the Empty stage that do only logging out to
@@ -287,6 +317,8 @@ class EmptyStage(BaseStage):
             f"({self.run_id}) [STAGE]: Empty-Execute: {self.name!r}: "
             f"( {param2template(self.echo, params=params) or '...'} )"
         )
+        if self.sleep > 0:
+            time.sleep(self.sleep)
         return Result(status=0, context={})
 
 
@@ -462,12 +494,13 @@ class PyStage(BaseStage):
         exec(run, _globals, _locals)
 
         return Result(
-            status=0, context={"locals": _locals, "globals": _globals}
+            status=0,
+            context={"locals": _locals, "globals": _globals},
         )
 
 
 @dataclass(frozen=True)
-class HookSearch:
+class HookSearchData:
     """Hook Search dataclass that use for receive regular expression grouping
     dict from searching hook string value.
     """
@@ -490,7 +523,7 @@ def extract_hook(hook: str) -> Callable[[], TagFunc]:
         )
 
     # NOTE: Pass the searching hook string to `path`, `func`, and `tag`.
-    hook: HookSearch = HookSearch(**found.groupdict())
+    hook: HookSearchData = HookSearchData(**found.groupdict())
 
     # NOTE: Registry object should implement on this package only.
     rgt: dict[str, Registry] = make_registry(f"{hook.path}")
@@ -596,7 +629,11 @@ class TriggerStage(BaseStage):
         ... }
     """
 
-    trigger: str = Field(description="A trigger workflow name.")
+    trigger: str = Field(
+        description=(
+            "A trigger workflow name that should already exist on the config."
+        ),
+    )
     params: DictData = Field(
         default_factory=dict,
         description="A parameter that want to pass to workflow execution.",
@@ -610,6 +647,7 @@ class TriggerStage(BaseStage):
         :param params: A parameter data that want to use in this execution.
         :rtype: Result
         """
+        # NOTE: Lazy import this workflow object.
         from . import Workflow
 
         # NOTE: Loading workflow object from trigger name.
@@ -624,7 +662,11 @@ class TriggerStage(BaseStage):
         return wf.execute(params=param2template(self.params, params))
 
 
-# NOTE: An order of parsing stage model.
+# NOTE:
+#   An order of parsing stage model on the Job model with ``stages`` field.
+#   From the current build-in stages, they do not have stage that have the same
+#   fields that be cause of parsing on the Job's stages key.
+#
 Stage = Union[
     PyStage,
     BashStage,